fix:Fixed the issue where the rich text processing in the DingTalk AP…

libs/dify_service_api/v1/client.py

@@ -5,6 +5,8 @@
 import json
 
 from .errors import DifyAPIError
+from pathlib import Path
+import os
 
 
 class AsyncDifyServiceClient:
@@ -109,7 +111,23 @@ async def upload_file(
         user: str,
         timeout: float = 30.0,
     ) -> str:
-        """上传文件"""
+        # 处理 Path 对象
+        if isinstance(file, Path):
+            if not file.exists():
+                raise ValueError(f'File not found: {file}')
+            with open(file, 'rb') as f:
+                file = f.read()
+
+        # 处理文件路径字符串
+        elif isinstance(file, str):
+            if not os.path.isfile(file):
+                raise ValueError(f'File not found: {file}')
+            with open(file, 'rb') as f:
+                file = f.read()
+
+        # 处理文件对象
+        elif hasattr(file, 'read'):
+            file = file.read()
         async with httpx.AsyncClient(
             base_url=self.base_url,
             trust_env=True,
@@ -121,6 +139,8 @@ async def upload_file(
                 headers={'Authorization': f'Bearer {self.api_key}'},
                 files={
                     'file': file,
+                },
+                data={
                     'user': (None, user),
                 },
             )

libs/dingtalk_api/api.py

@@ -188,12 +188,80 @@ async def get_message(self, incoming_message: dingtalk_stream.chatbot.ChatbotMes
 
             if incoming_message.message_type == 'richText':
                 data = incoming_message.rich_text_content.to_dict()
+
+                # 使用统一的结构化数据格式，保持顺序
+                rich_content = {
+                    'Type': 'richText',
+                    'Elements': [],  # 按顺序存储所有元素
+                    'SimpleContent': '',  # 兼容字段：纯文本内容
+                    'SimplePicture': ''  # 兼容字段：第一张图片
+                }
+
+                # 先收集所有文本和图片占位符
+                text_elements = []
+                image_placeholders = []
+
+                # 解析富文本内容，保持原始顺序
                 for item in data['richText']:
-                    if 'text' in item:
-                        message_data['Content'] = item['text']
-                    if incoming_message.get_image_list()[0]:
-                        message_data['Picture'] = await self.download_image(incoming_message.get_image_list()[0])
-                message_data['Type'] = 'text'
+
+                    # 处理文本内容
+                    if 'text' in item and item['text'] != "\n":
+                        element = {
+                            'Type': 'text',
+                            'Content': item['text']
+                        }
+                        rich_content['Elements'].append(element)
+                        text_elements.append(item['text'])
+
+                    # 检查是否是图片元素 - 根据钉钉API的实际结构调整
+                    # 钉钉富文本中的图片通常有特定标识，可能需要根据实际返回调整
+                    elif item.get("type") == "picture":
+                        # 创建图片占位符
+                        element = {
+                            'Type': 'image_placeholder',
+                        }
+                        rich_content['Elements'].append(element)
+
+                # 获取并下载所有图片
+                image_list = incoming_message.get_image_list()
+                if image_list:
+                    new_elements = []
+                    image_index = 0
+
+                    for element in rich_content['Elements']:
+                        if element['Type'] == 'image_placeholder':
+                            if image_index < len(image_list) and image_list[image_index]:
+                                image_url = await self.download_image(image_list[image_index])
+                                new_elements.append({
+                                    'Type': 'image',
+                                    'Picture': image_url
+                                })
+                                image_index += 1
+                            else:
+                                # 如果没有对应的图片，保留占位符或跳过
+                                continue
+                        else:
+                            new_elements.append(element)
+
+                    rich_content['Elements'] = new_elements
+
+
+                # 设置兼容字段
+                all_texts = [elem['Content'] for elem in rich_content['Elements'] if elem.get('Type') == 'text']
+                rich_content['SimpleContent'] = '\n'.join(all_texts) if all_texts else ''
+
+                all_images = [elem['Picture'] for elem in rich_content['Elements'] if elem.get('Type') == 'image']
+                if all_images:
+                    rich_content['SimplePicture'] = all_images[0]
+                    rich_content['AllImages'] = all_images  # 所有图片的列表
+
+                # 设置原始的 content 和 picture 字段以保持兼容
+                message_data['Content'] = rich_content['SimpleContent']
+                message_data['Rich_Content'] = rich_content
+                if all_images:
+                    message_data['Picture'] = all_images[0]
+
+
 
             elif incoming_message.message_type == 'text':
                 message_data['Content'] = incoming_message.get_text_list()[0]

libs/dingtalk_api/dingtalkevent.py

@@ -15,6 +15,10 @@ def from_payload(payload: Dict[str, Any]) -> Optional['DingTalkEvent']:
     def content(self):
         return self.get('Content', '')
 
+    @property
+    def rich_content(self):
+        return self.get('Rich_Content', '')
+
     @property
     def incoming_message(self) -> Optional['dingtalk_stream.chatbot.ChatbotMessage']:
         return self.get('IncomingMessage')

pkg/persistence/migrations/dbm011_dify_base_prompt_config.py

@@ -0,0 +1,40 @@
+from .. import migration
+
+import sqlalchemy
+
+from ...entity.persistence import pipeline as persistence_pipeline
+
+
+@migration.migration_class(11)
+class DBMigrateDifyApiConfig(migration.DBMigration):
+    """Langflow API config"""
+
+    async def upgrade(self):
+        """Upgrade"""
+        # read all pipelines
+        pipelines = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_pipeline.LegacyPipeline))
+
+        for pipeline in pipelines:
+            serialized_pipeline = self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
+
+            config = serialized_pipeline['config']
+
+            if 'base-prompt' not in config['ai']['dify-service-api']:
+                config['ai']['dify-service-api']['base-prompt'] = (
+                    'When the file content is readable, please read the content of this file. When the file is an image, describe the content of this image.',
+                )
+
+            await self.ap.persistence_mgr.execute_async(
+                sqlalchemy.update(persistence_pipeline.LegacyPipeline)
+                .where(persistence_pipeline.LegacyPipeline.uuid == serialized_pipeline['uuid'])
+                .values(
+                    {
+                        'config': config,
+                        'for_version': self.ap.ver_mgr.get_current_version(),
+                    }
+                )
+            )
+
+    async def downgrade(self):
+        """Downgrade"""
+        pass

pkg/platform/sources/dingtalk.py

@@ -1,3 +1,4 @@
+
 import traceback
 import typing
 from libs.dingtalk_api.dingtalkevent import DingTalkEvent
@@ -36,16 +37,31 @@ async def target2yiri(event: DingTalkEvent, bot_name: str):
             if atUser.dingtalk_id == event.incoming_message.chatbot_user_id:
                 yiri_msg_list.append(platform_message.At(target=bot_name))
 
-        if event.content:
-            text_content = event.content.replace('@' + bot_name, '')
-            yiri_msg_list.append(platform_message.Plain(text=text_content))
-        if event.picture:
-            yiri_msg_list.append(platform_message.Image(base64=event.picture))
+        if event.rich_content:
+            elements = event.rich_content.get("Elements")
+            for element in elements:
+                if element.get('Type') == 'text':
+                    text = element.get('Content', '').replace('@' + bot_name, '')
+                    if text.strip():
+                        yiri_msg_list.append(platform_message.Plain(text=text))
+                elif element.get('Type') == 'image' and element.get('Picture'):
+                    yiri_msg_list.append(platform_message.Image(base64=element['Picture']))
+        else:
+            # 回退到原有简单逻辑
+            if event.content:
+                text_content = event.content.replace('@' + bot_name, '')
+                yiri_msg_list.append(platform_message.Plain(text=text_content))
+            if event.picture:
+                yiri_msg_list.append(platform_message.Image(base64=event.picture))
+
+            # 处理其他类型消息（文件、音频等）
         if event.file:
             yiri_msg_list.append(platform_message.File(url=event.file, name=event.name))
         if event.audio:
             yiri_msg_list.append(platform_message.Voice(base64=event.audio))
 
+
+
         chain = platform_message.MessageChain(yiri_msg_list)
 
         return chain

pkg/provider/runners/difysvapi.py

@@ -14,6 +14,7 @@
 from libs.dify_service_api.v1 import client, errors
 
 
+
 @runner.runner_class('dify-service-api')
 class DifyServiceAPIRunner(runner.RequestRunner):
     """Dify Service API 对话请求器"""
@@ -77,7 +78,7 @@ async def _preprocess_user_message(self, query: pipeline_query.Query) -> tuple[s
             tuple[str, list[str]]: 纯文本和图片的 Dify 服务图片 ID
         """
         plain_text = ''
-        image_ids = []
+        file_ids = []
 
         if isinstance(query.user_message.content, list):
             for ce in query.user_message.content:
@@ -92,11 +93,24 @@ async def _preprocess_user_message(self, query: pipeline_query.Query) -> tuple[s
                         f'{query.session.launcher_type.value}_{query.session.launcher_id}',
                     )
                     image_id = file_upload_resp['id']
-                    image_ids.append(image_id)
+                    file_ids.append(image_id)
+                # elif ce.type == "file_url":
+                #     file_bytes = base64.b64decode(ce.file_url)
+                #     file_upload_resp = await self.dify_client.upload_file(
+                #         file_bytes,
+                #         f'{query.session.launcher_type.value}_{query.session.launcher_id}',
+                #     )
+                #     file_id = file_upload_resp['id']
+                #     file_ids.append(file_id)
         elif isinstance(query.user_message.content, str):
             plain_text = query.user_message.content
+        # plain_text = "When the file content is readable, please read the content of this file. When the file is an image, describe the content of this image." if file_ids and not plain_text else plain_text
+        # plain_text = "The user message type cannot be parsed." if not file_ids and not plain_text else plain_text
+        # plain_text = plain_text if plain_text else "When the file content is readable, please read the content of this file. When the file is an image, describe the content of this image."
+        # print(self.pipeline_config['ai'])
+        plain_text = plain_text if plain_text else self.pipeline_config['ai']['dify-service-api']['base-prompt']
 
-        return plain_text, image_ids
+        return plain_text, file_ids
 
     async def _chat_messages(
         self, query: pipeline_query.Query
@@ -110,7 +124,6 @@ async def _chat_messages(
         files = [
             {
                 'type': 'image',
-                'transfer_method': 'local_file',
                 'upload_file_id': image_id,
             }
             for image_id in image_ids

pkg/utils/constants.py

@@ -1,6 +1,6 @@
 semantic_version = 'v4.4.1'
 
-required_database_version = 10
+required_database_version = 11
 """Tag the version of the database schema, used to check if the database needs to be migrated"""
 
 debug_mode = False

templates/metadata/pipeline/ai.yaml

@@ -124,6 +124,16 @@ stages:
           zh_Hans: 基础 URL
         type: string
         required: true
+      - name: base-prompt
+        label:
+          en_US: Base PROMPT
+          zh_Hans: 基础提示词
+        description:
+          en_US: When Dify receives a message with empty input (only images), it will pass this default prompt into it.
+          zh_Hans: 当 Dify 接收到输入文字为空（仅图片）的消息时，传入该默认提示词
+        type: string
+        required: true
+        default: "When the file content is readable, please read the content of this file. When the file is an image, describe the content of this image."
       - name: app-type
         label:
           en_US: App Type