From 56c73daf133dff4431441e3ed97425124f1ecc9d Mon Sep 17 00:00:00 2001
From: Vitaly <bikevit2008@gmail.com>
Date: Sat, 25 Jan 2025 02:11:16 +0300
Subject: [PATCH 1/6] Relative WS path for comfyui
By default now we can use links like abc.com/ but abc.com/foo/bar/comfy is not valid
---
api/core/tools/provider/builtin/comfyui/comfyui.py | 3 ++-
.../tools/provider/builtin/comfyui/tools/comfyui_client.py | 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/api/core/tools/provider/builtin/comfyui/comfyui.py b/api/core/tools/provider/builtin/comfyui/comfyui.py
index a8127dd23f1553..d2bb12d9a9fdd9 100644
--- a/api/core/tools/provider/builtin/comfyui/comfyui.py
+++ b/api/core/tools/provider/builtin/comfyui/comfyui.py
@@ -14,7 +14,8 @@ def _validate_credentials(self, credentials: dict[str, Any]) -> None:
ws_protocol = "ws"
if base_url.scheme == "https":
ws_protocol = "wss"
- ws_address = f"{ws_protocol}://{base_url.authority}/ws?clientId=test123"
+ base_path = str(base_url.path).rstrip('/')
+ ws_address = f"{ws_protocol}://{base_url.authority}{base_path}/ws?clientId=test123"
try:
ws.connect(ws_address)
diff --git a/api/core/tools/provider/builtin/comfyui/tools/comfyui_client.py b/api/core/tools/provider/builtin/comfyui/tools/comfyui_client.py
index 2bf10ce8ff2632..65dc1737798c2e 100644
--- a/api/core/tools/provider/builtin/comfyui/tools/comfyui_client.py
+++ b/api/core/tools/provider/builtin/comfyui/tools/comfyui_client.py
@@ -43,7 +43,8 @@ def open_websocket_connection(self) -> tuple[WebSocket, str]:
ws_protocol = "ws"
if self.base_url.scheme == "https":
ws_protocol = "wss"
- ws_address = f"{ws_protocol}://{self.base_url.authority}/ws?clientId={client_id}"
+ ws_url = self.base_url.with_scheme(ws_protocol).join(URL('ws'))
+ ws_address = str(ws_url.with_query({'clientId': client_id}))
ws.connect(ws_address)
return ws, client_id
From f3915b695f74379c7f2e5d19ea1a9f4c0a6beace Mon Sep 17 00:00:00 2001
From: Vitaly <bikevit2008@gmail.com>
Date: Sat, 25 Jan 2025 03:16:48 +0300
Subject: [PATCH 2/6] build from source for api in docker compose
---
docker/docker-compose.yaml | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
index a11ec261f3ac0d..37b74d16073a83 100644
--- a/docker/docker-compose.yaml
+++ b/docker/docker-compose.yaml
@@ -393,7 +393,9 @@ x-shared-env: &shared-api-worker-env
services:
# API service
api:
- image: langgenius/dify-api:0.15.2
+ build:
+ dockerfile: ../api/Dockerfile
+ context: ../api
restart: always
environment:
# Use the shared environment variables.
@@ -416,7 +418,9 @@ services:
# worker service
# The Celery worker for processing the queue.
worker:
- image: langgenius/dify-api:0.15.2
+ build:
+ dockerfile: ../api/Dockerfile
+ context: ../api
restart: always
environment:
# Use the shared environment variables.
From f37c8720391b85c7640d7423f338a6e2e44f311e Mon Sep 17 00:00:00 2001
From: Vitaly <bikevit2008@gmail.com>
Date: Tue, 28 Jan 2025 05:04:01 +0300
Subject: [PATCH 3/6] Linted
---
api/core/tools/provider/builtin/comfyui/comfyui.py | 2 +-
.../tools/provider/builtin/comfyui/tools/comfyui_client.py | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/api/core/tools/provider/builtin/comfyui/comfyui.py b/api/core/tools/provider/builtin/comfyui/comfyui.py
index d2bb12d9a9fdd9..fac17db28afe4e 100644
--- a/api/core/tools/provider/builtin/comfyui/comfyui.py
+++ b/api/core/tools/provider/builtin/comfyui/comfyui.py
@@ -14,7 +14,7 @@ def _validate_credentials(self, credentials: dict[str, Any]) -> None:
ws_protocol = "ws"
if base_url.scheme == "https":
ws_protocol = "wss"
- base_path = str(base_url.path).rstrip('/')
+ base_path = str(base_url.path).rstrip("/")
ws_address = f"{ws_protocol}://{base_url.authority}{base_path}/ws?clientId=test123"
try:
diff --git a/api/core/tools/provider/builtin/comfyui/tools/comfyui_client.py b/api/core/tools/provider/builtin/comfyui/tools/comfyui_client.py
index 65dc1737798c2e..6cf9110d1f71b9 100644
--- a/api/core/tools/provider/builtin/comfyui/tools/comfyui_client.py
+++ b/api/core/tools/provider/builtin/comfyui/tools/comfyui_client.py
@@ -43,8 +43,8 @@ def open_websocket_connection(self) -> tuple[WebSocket, str]:
ws_protocol = "ws"
if self.base_url.scheme == "https":
ws_protocol = "wss"
- ws_url = self.base_url.with_scheme(ws_protocol).join(URL('ws'))
- ws_address = str(ws_url.with_query({'clientId': client_id}))
+ ws_url = self.base_url.with_scheme(ws_protocol).join(URL("ws"))
+ ws_address = str(ws_url.with_query({"clientId": client_id}))
ws.connect(ws_address)
return ws, client_id
From f96d84cf5b83d918c5685338045d117bf6ca220e Mon Sep 17 00:00:00 2001
From: Vitaly <bikevit2008@gmail.com>
Date: Tue, 28 Jan 2025 16:51:16 +0300
Subject: [PATCH 4/6] Base64 encode tool
---
api/constants/__init__.py | 33 +++------
.../tools/provider/builtin/_positions.yaml | 6 ++
.../builtin/url_to_base64/__init__.py | 3 +
.../builtin/url_to_base64/_assets/icon.svg | 9 +++
.../tools/url_to_base64_converter.py | 73 +++++++++++++++++++
.../tools/url_to_base64_converter.yaml | 66 +++++++++++++++++
.../builtin/url_to_base64/url_to_base64.py | 22 ++++++
.../builtin/url_to_base64/url_to_base64.yaml | 16 ++++
.../url_to_base64/url_to_base64_tool.py | 45 ++++++++++++
9 files changed, 252 insertions(+), 21 deletions(-)
create mode 100644 api/core/tools/provider/builtin/_positions.yaml
create mode 100644 api/core/tools/provider/builtin/url_to_base64/__init__.py
create mode 100644 api/core/tools/provider/builtin/url_to_base64/_assets/icon.svg
create mode 100644 api/core/tools/provider/builtin/url_to_base64/tools/url_to_base64_converter.py
create mode 100644 api/core/tools/provider/builtin/url_to_base64/tools/url_to_base64_converter.yaml
create mode 100644 api/core/tools/provider/builtin/url_to_base64/url_to_base64.py
create mode 100644 api/core/tools/provider/builtin/url_to_base64/url_to_base64.yaml
create mode 100644 api/core/tools/provider/builtin/url_to_base64/url_to_base64_tool.py
diff --git a/api/constants/__init__.py b/api/constants/__init__.py
index 4500ef4306fc2a..19bd11e03f695d 100644
--- a/api/constants/__init__.py
+++ b/api/constants/__init__.py
@@ -1,24 +1,15 @@
-from configs import dify_config
+# File types and extensions
+AUDIO_EXTENSIONS = ['mp3', 'wav', 'ogg', 'm4a', 'aac', "ogg"]
+DOCUMENT_EXTENSIONS = ['txt', 'pdf', 'doc', 'docx', 'csv', 'xls', 'xlsx']
+IMAGE_EXTENSIONS = ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp']
+VIDEO_EXTENSIONS = ['mp4', 'avi', 'mov', 'wmv', 'flv', 'mkv']
-HIDDEN_VALUE = "[__HIDDEN__]"
-UUID_NIL = "00000000-0000-0000-0000-000000000000"
-
-IMAGE_EXTENSIONS = ["jpg", "jpeg", "png", "webp", "gif", "svg"]
-IMAGE_EXTENSIONS.extend([ext.upper() for ext in IMAGE_EXTENSIONS])
-
-VIDEO_EXTENSIONS = ["mp4", "mov", "mpeg", "mpga"]
-VIDEO_EXTENSIONS.extend([ext.upper() for ext in VIDEO_EXTENSIONS])
+# File identifiers
+DIFY_FILE_IDENTIFIER = '__dify__file__'
-AUDIO_EXTENSIONS = ["mp3", "m4a", "wav", "webm", "amr"]
-AUDIO_EXTENSIONS.extend([ext.upper() for ext in AUDIO_EXTENSIONS])
+# File transfer methods
+LOCAL_FILE_TRANSFER = 'local_file'
+REMOTE_URL_TRANSFER = 'remote_url'
-
-if dify_config.ETL_TYPE == "Unstructured":
- DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "mdx", "pdf", "html", "htm", "xlsx", "xls"]
- DOCUMENT_EXTENSIONS.extend(("docx", "csv", "eml", "msg", "pptx", "xml", "epub"))
- if dify_config.UNSTRUCTURED_API_URL:
- DOCUMENT_EXTENSIONS.append("ppt")
- DOCUMENT_EXTENSIONS.extend([ext.upper() for ext in DOCUMENT_EXTENSIONS])
-else:
- DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "mdx", "pdf", "html", "htm", "xlsx", "xls", "docx", "csv"]
- DOCUMENT_EXTENSIONS.extend([ext.upper() for ext in DOCUMENT_EXTENSIONS])
+HIDDEN_VALUE = "[__HIDDEN__]"
+UUID_NIL = "00000000-0000-0000-0000-000000000000"
diff --git a/api/core/tools/provider/builtin/_positions.yaml b/api/core/tools/provider/builtin/_positions.yaml
new file mode 100644
index 00000000000000..23cab8b6731b38
--- /dev/null
+++ b/api/core/tools/provider/builtin/_positions.yaml
@@ -0,0 +1,6 @@
+google: 1
+dalle: 2
+stable_diffusion: 3
+serpapi: 4
+wolfram: 5
+url_to_base64: 6
diff --git a/api/core/tools/provider/builtin/url_to_base64/__init__.py b/api/core/tools/provider/builtin/url_to_base64/__init__.py
new file mode 100644
index 00000000000000..9fb37b71eb75f2
--- /dev/null
+++ b/api/core/tools/provider/builtin/url_to_base64/__init__.py
@@ -0,0 +1,3 @@
+from .url_to_base64 import URLToBase64Provider
+
+__all__ = ['URLToBase64Provider']
diff --git a/api/core/tools/provider/builtin/url_to_base64/_assets/icon.svg b/api/core/tools/provider/builtin/url_to_base64/_assets/icon.svg
new file mode 100644
index 00000000000000..3901c1c7f0119e
--- /dev/null
+++ b/api/core/tools/provider/builtin/url_to_base64/_assets/icon.svg
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="32px" height="32px" viewBox="0 0 32 32" version="1.1" xmlns="http://www.w3.org/2000/svg">
+ <g stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+ <rect fill="#4A90E2" x="0" y="0" width="32" height="32" rx="6"/>
+ <text font-family="Arial-BoldMT, Arial" font-size="12" font-weight="bold" fill="#FFFFFF">
+ <tspan x="4" y="20">B64</tspan>
+ </text>
+ </g>
+</svg>
diff --git a/api/core/tools/provider/builtin/url_to_base64/tools/url_to_base64_converter.py b/api/core/tools/provider/builtin/url_to_base64/tools/url_to_base64_converter.py
new file mode 100644
index 00000000000000..f8095ac6a1c087
--- /dev/null
+++ b/api/core/tools/provider/builtin/url_to_base64/tools/url_to_base64_converter.py
@@ -0,0 +1,73 @@
+from typing import Any, Dict, List, Union
+import base64
+import requests
+import json
+import logging
+
+from core.tools.entities.tool_entities import ToolInvokeMessage
+from core.tools.tool.builtin_tool import BuiltinTool
+from core.tools.tool_file_manager import ToolFileManager
+from extensions.ext_storage import storage
+from core.file.models import File
+from core.file.file_manager import download
+
+logger = logging.getLogger(__name__)
+
+class URLToBase64Converter(BuiltinTool):
+ def _invoke(self,
+ user_id: str,
+ tool_parameters: Dict[str, Any],
+ ) -> ToolInvokeMessage:
+ """
+ Конвертирует файл в base64 строку.
+ Поддерживает как внешние URL, так и локальные файлы Dify.
+ """
+ logger.info(f"Received parameters: {tool_parameters}")
+
+ # Получаем источник файла
+ file_source = tool_parameters.get('file_source')
+ if not file_source:
+ return self.create_text_message('Please specify file source (url or local)')
+
+ try:
+ # Обработка внешнего URL
+ if file_source == 'url':
+ url = tool_parameters.get('url')
+ if not url:
+ return self.create_text_message('Please provide a URL')
+
+ response = requests.get(url)
+ response.raise_for_status()
+ content = response.content
+
+ # Обработка локального файла Dify
+ elif file_source == 'local':
+ file = tool_parameters.get('file')
+ logger.info(f"File data: {file}")
+
+ if not file:
+ return self.create_text_message('Please provide a file')
+
+ if not isinstance(file, File):
+ logger.error(f"Invalid file type: {type(file)}")
+ return self.create_text_message('Invalid file type. Expected Dify File object')
+
+ # Загружаем содержимое файла
+ content = download(file)
+ if not content:
+ logger.error("Failed to download file content")
+ return self.create_text_message('Failed to download file content')
+
+ logger.info(f"Successfully loaded file: {len(content)} bytes")
+
+ else:
+ return self.create_text_message('Invalid file source. Use "url" or "local"')
+
+ # Конвертируем содержимое в base64
+ base64_content = base64.b64encode(content).decode('utf-8')
+ logger.info(f"Successfully converted file to base64: {len(base64_content)} characters")
+ return self.create_text_message(base64_content)
+
+ except Exception as e:
+ logger.exception("Error processing file")
+ return self.create_text_message(f'Error processing file: {str(e)}')
diff --git a/api/core/tools/provider/builtin/url_to_base64/tools/url_to_base64_converter.yaml b/api/core/tools/provider/builtin/url_to_base64/tools/url_to_base64_converter.yaml
new file mode 100644
index 00000000000000..09c3b37781184c
--- /dev/null
+++ b/api/core/tools/provider/builtin/url_to_base64/tools/url_to_base64_converter.yaml
@@ -0,0 +1,66 @@
+identity:
+ name: url_to_base64
+ author: vitalijsatilov
+ label:
+ en_US: URL to Base64
+ zh_Hans: URL转Base64
+ ru_RU: URL в Base64
+
+description:
+ human:
+ en_US: Convert URL or local file to base64 string
+ zh_Hans: 将URL或本地文件转换为base64字符串
+ ru_RU: Конвертировать URL или локальный файл в строку base64
+ llm: Convert a file from URL or local storage to base64 string format
+
+parameters:
+ - name: file_source
+ type: string
+ required: true
+ label:
+ en_US: File Source
+ zh_Hans: 文件来源
+ ru_RU: Источник файла
+ human_description:
+ en_US: Source of the file (url or local)
+ zh_Hans: 文件来源（url或本地）
+ ru_RU: Источник файла (url или локальный)
+ llm_description: Specify the source of the file. Use "url" for external URLs or "local" for files from Dify storage.
+ options:
+ - value: url
+ label:
+ en_US: URL
+ zh_Hans: URL
+ ru_RU: URL
+ - value: local
+ label:
+ en_US: Local File
+ zh_Hans: 本地文件
+ ru_RU: Локальный файл
+ form: llm
+ - name: url
+ type: string
+ required: false
+ label:
+ en_US: URL
+ zh_Hans: URL
+ ru_RU: URL
+ human_description:
+ en_US: URL of the file to convert (required if file_source is "url")
+ zh_Hans: 要转换的文件的URL（当file_source为"url"时必填）
+ ru_RU: URL файла для конвертации (обязательно если file_source - "url")
+ llm_description: URL of the file to convert to base64. Only required when file_source is "url".
+ form: llm
+ - name: file
+ type: file
+ required: false
+ label:
+ en_US: File
+ zh_Hans: 文件
+ ru_RU: Файл
+ human_description:
+ en_US: Local file to convert (required if file_source is "local")
+ zh_Hans: 要转换的本地文件（当file_source为"local"时必填）
+ ru_RU: Локальный файл для конвертации (обязательно если file_source - "local")
+ llm_description: Local file to convert to base64. Only required when file_source is "local".
+ form: llm
diff --git a/api/core/tools/provider/builtin/url_to_base64/url_to_base64.py b/api/core/tools/provider/builtin/url_to_base64/url_to_base64.py
new file mode 100644
index 00000000000000..d048f2cb225574
--- /dev/null
+++ b/api/core/tools/provider/builtin/url_to_base64/url_to_base64.py
@@ -0,0 +1,22 @@
+from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController
+from core.tools.provider.builtin.url_to_base64.tools.url_to_base64_converter import URLToBase64Converter
+
+class URLToBase64Provider(BuiltinToolProviderController):
+ @property
+ def need_credentials(self) -> bool:
+ """
+ Whether the provider needs credentials
+ """
+ return False
+
+ def _tools(self) -> list:
+ return [
+ URLToBase64Converter(),
+ ]
+
+ def _validate_credentials(self, credentials: dict) -> bool:
+ """
+ Validate the credentials.
+ Our tool doesn't require any credentials, so we always return True.
+ """
+ return True
diff --git a/api/core/tools/provider/builtin/url_to_base64/url_to_base64.yaml b/api/core/tools/provider/builtin/url_to_base64/url_to_base64.yaml
new file mode 100644
index 00000000000000..aa7bf17c7b745b
--- /dev/null
+++ b/api/core/tools/provider/builtin/url_to_base64/url_to_base64.yaml
@@ -0,0 +1,16 @@
+identity:
+ author: bikevit2008
+ name: url_to_base64
+ label:
+ en_US: URL to Base64
+ ru_RU: URL в Base64
+ zh_Hans: URL 转 Base64
+ description:
+ en_US: A tool for converting files from URL to base64 string format
+ ru_RU: Инструмент для конвертации файлов из URL в формат base64
+ zh_Hans: 将 URL 文件转换为 base64 字符串的工具
+ icon: icon.svg
+ tags:
+ - utilities
+
+credentials_for_provider: {}
diff --git a/api/core/tools/provider/builtin/url_to_base64/url_to_base64_tool.py b/api/core/tools/provider/builtin/url_to_base64/url_to_base64_tool.py
new file mode 100644
index 00000000000000..ecb2b251c7d9db
--- /dev/null
+++ b/api/core/tools/provider/builtin/url_to_base64/url_to_base64_tool.py
@@ -0,0 +1,45 @@
+from typing import Any, Dict, List, Union
+import base64
+import requests
+
+from core.tools.entities.tool_entities import ToolInvokeMessage
+from core.tools.tool.builtin_tool import BuiltinTool
+
+class URLToBase64Tool(BuiltinTool):
+ def _invoke(self,
+ user_id: str,
+ tool_parameters: Dict[str, Any],
+ ) -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
+ """
+ Конвертирует файл из URL в base64 строку
+ """
+ # Получаем URL из параметров
+ url = tool_parameters.get('url', '')
+ if not url:
+ return self.create_text_message('Пожалуйста, укажите URL')
+
+ try:
+ # Загружаем файл по URL
+ response = requests.get(url)
+ response.raise_for_status() # Проверяем на ошибки
+
+ # Конвертируем содержимое в base64
+ base64_content = base64.b64encode(response.content).decode('utf-8')
+
+ # Возвращаем результат
+ return self.create_text_message(base64_content)
+
+ except Exception as e:
+ return self.create_text_message(f'Ошибка при обработке файла: {str(e)}')
+
+ def get_runtime_parameters(self) -> List[Dict]:
+ """
+ Определяем параметры инструмента
+ """
+ return [{
+ "name": "url",
+ "type": "string",
+ "required": True,
+ "label": "URL файла",
+ "description": "URL файла, который нужно конвертировать в base64"
+ }]
From 7b1d1126d954f655da337e8407bb1a46f6828624 Mon Sep 17 00:00:00 2001
From: Vitaly <bikevit2008@gmail.com>
Date: Wed, 29 Jan 2025 01:35:24 +0300
Subject: [PATCH 5/6] Fixed mime type -> extension file by HTTP request
---
api/constants/__init__.py | 30 +++++++++++++++++++++++++----
api/core/tools/tool_file_manager.py | 27 ++++++++++++++++++++++++--
2 files changed, 51 insertions(+), 6 deletions(-)
diff --git a/api/constants/__init__.py b/api/constants/__init__.py
index 19bd11e03f695d..229c7400fb4444 100644
--- a/api/constants/__init__.py
+++ b/api/constants/__init__.py
@@ -1,9 +1,21 @@
+from configs import dify_config
+
# File types and extensions
-AUDIO_EXTENSIONS = ['mp3', 'wav', 'ogg', 'm4a', 'aac', "ogg"]
-DOCUMENT_EXTENSIONS = ['txt', 'pdf', 'doc', 'docx', 'csv', 'xls', 'xlsx']
-IMAGE_EXTENSIONS = ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp']
-VIDEO_EXTENSIONS = ['mp4', 'avi', 'mov', 'wmv', 'flv', 'mkv']
+AUDIO_EXTENSIONS = ["mp3", "m4a", "wav", "webm", "amr", "ogg"]
+AUDIO_EXTENSIONS.extend([ext.upper() for ext in AUDIO_EXTENSIONS])
+
+IMAGE_EXTENSIONS = ["jpg", "jpeg", "png", "webp", "gif", "svg"]
+IMAGE_EXTENSIONS.extend([ext.upper() for ext in IMAGE_EXTENSIONS])
+VIDEO_EXTENSIONS = ["mp4", "mov", "mpeg", "mpga"]
+VIDEO_EXTENSIONS.extend([ext.upper() for ext in VIDEO_EXTENSIONS])
+
+MIME_TO_EXTENSION = {
+ 'audio/wav': '.wav',
+ 'audio/x-wav': '.wav',
+ 'audio/wave': '.wav',
+ 'audio/x-pn-wav': '.wav',
+}
# File identifiers
DIFY_FILE_IDENTIFIER = '__dify__file__'
@@ -11,5 +23,15 @@
LOCAL_FILE_TRANSFER = 'local_file'
REMOTE_URL_TRANSFER = 'remote_url'
+if dify_config.ETL_TYPE == "Unstructured":
+ DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "mdx", "pdf", "html", "htm", "xlsx", "xls"]
+ DOCUMENT_EXTENSIONS.extend(("docx", "csv", "eml", "msg", "pptx", "xml", "epub"))
+ if dify_config.UNSTRUCTURED_API_URL:
+ DOCUMENT_EXTENSIONS.append("ppt")
+ DOCUMENT_EXTENSIONS.extend([ext.upper() for ext in DOCUMENT_EXTENSIONS])
+else:
+ DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "mdx", "pdf", "html", "htm", "xlsx", "xls", "docx", "csv"]
+ DOCUMENT_EXTENSIONS.extend([ext.upper() for ext in DOCUMENT_EXTENSIONS])
+
HIDDEN_VALUE = "[__HIDDEN__]"
UUID_NIL = "00000000-0000-0000-0000-000000000000"
diff --git a/api/core/tools/tool_file_manager.py b/api/core/tools/tool_file_manager.py
index 2aaca6d82e36b1..92a28c6bfe6600 100644
--- a/api/core/tools/tool_file_manager.py
+++ b/api/core/tools/tool_file_manager.py
@@ -16,6 +16,7 @@
from extensions.ext_storage import storage
from models.model import MessageFile
from models.tools import ToolFile
+from constants import MIME_TO_EXTENSION
logger = logging.getLogger(__name__)
@@ -64,7 +65,18 @@ def create_file_by_raw(
file_binary: bytes,
mimetype: str,
) -> ToolFile:
- extension = guess_extension(mimetype) or ".bin"
+ logger.info(f"Creating file with mimetype: {mimetype}")
+
+ # Проверяем наше сопоставление MIME-типов
+ extension = MIME_TO_EXTENSION.get(mimetype)
+ if not extension:
+ # Если нет в нашем сопоставлении, пробуем стандартный способ
+ extension = guess_extension(mimetype)
+ if not extension:
+ logger.warning(f"Could not determine extension for mimetype {mimetype}, using .bin")
+ extension = ".bin"
+
+ logger.info(f"Using extension: {extension}")
unique_name = uuid4().hex
filename = f"{unique_name}{extension}"
filepath = f"tools/{tenant_id}/{filename}"
@@ -102,7 +114,18 @@ def create_file_by_url(
raise ValueError(f"timeout when downloading file from {file_url}")
mimetype = guess_type(file_url)[0] or "octet/stream"
- extension = guess_extension(mimetype) or ".bin"
+ logger.info(f"Creating file from URL with mimetype: {mimetype}")
+
+ # Проверяем наше сопоставление MIME-типов
+ extension = MIME_TO_EXTENSION.get(mimetype)
+ if not extension:
+ # Если нет в нашем сопоставлении, пробуем стандартный способ
+ extension = guess_extension(mimetype)
+ if not extension:
+ logger.warning(f"Could not determine extension for mimetype {mimetype}, using .bin")
+ extension = ".bin"
+
+ logger.info(f"Using extension: {extension}")
unique_name = uuid4().hex
filename = f"{unique_name}{extension}"
filepath = f"tools/{tenant_id}/{filename}"
From f88d7f658148199e9e7d0ec891db21b286e78c5f Mon Sep 17 00:00:00 2001
From: Vitaly <bikevit2008@gmail.com>
Date: Wed, 29 Jan 2025 01:41:44 +0300
Subject: [PATCH 6/6] Linted success
---
api/constants/__init__.py | 14 ++--
.../builtin/url_to_base64/__init__.py | 2 +-
.../tools/url_to_base64_converter.py | 66 +++++++++----------
.../builtin/url_to_base64/url_to_base64.py | 5 +-
.../url_to_base64/url_to_base64_tool.py | 46 +++++++------
api/core/tools/tool_file_manager.py | 10 +--
6 files changed, 74 insertions(+), 69 deletions(-)
diff --git a/api/constants/__init__.py b/api/constants/__init__.py
index 229c7400fb4444..2872d00de92ac9 100644
--- a/api/constants/__init__.py
+++ b/api/constants/__init__.py
@@ -11,17 +11,17 @@
VIDEO_EXTENSIONS.extend([ext.upper() for ext in VIDEO_EXTENSIONS])
MIME_TO_EXTENSION = {
- 'audio/wav': '.wav',
- 'audio/x-wav': '.wav',
- 'audio/wave': '.wav',
- 'audio/x-pn-wav': '.wav',
+ "audio/wav": ".wav",
+ "audio/x-wav": ".wav",
+ "audio/wave": ".wav",
+ "audio/x-pn-wav": ".wav",
}
# File identifiers
-DIFY_FILE_IDENTIFIER = '__dify__file__'
+DIFY_FILE_IDENTIFIER = "__dify__file__"
# File transfer methods
-LOCAL_FILE_TRANSFER = 'local_file'
-REMOTE_URL_TRANSFER = 'remote_url'
+LOCAL_FILE_TRANSFER = "local_file"
+REMOTE_URL_TRANSFER = "remote_url"
if dify_config.ETL_TYPE == "Unstructured":
DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "mdx", "pdf", "html", "htm", "xlsx", "xls"]
diff --git a/api/core/tools/provider/builtin/url_to_base64/__init__.py b/api/core/tools/provider/builtin/url_to_base64/__init__.py
index 9fb37b71eb75f2..6a8f5685e91dc4 100644
--- a/api/core/tools/provider/builtin/url_to_base64/__init__.py
+++ b/api/core/tools/provider/builtin/url_to_base64/__init__.py
@@ -1,3 +1,3 @@
from .url_to_base64 import URLToBase64Provider
-__all__ = ['URLToBase64Provider']
+__all__ = ["URLToBase64Provider"]
diff --git a/api/core/tools/provider/builtin/url_to_base64/tools/url_to_base64_converter.py b/api/core/tools/provider/builtin/url_to_base64/tools/url_to_base64_converter.py
index f8095ac6a1c087..df6abd2c24b145 100644
--- a/api/core/tools/provider/builtin/url_to_base64/tools/url_to_base64_converter.py
+++ b/api/core/tools/provider/builtin/url_to_base64/tools/url_to_base64_converter.py
@@ -1,73 +1,73 @@
-from typing import Any, Dict, List, Union
import base64
-import requests
-import json
import logging
+from typing import Any
+
+import requests
+from core.file.file_manager import download
+from core.file.models import File
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool.builtin_tool import BuiltinTool
-from core.tools.tool_file_manager import ToolFileManager
-from extensions.ext_storage import storage
-from core.file.models import File
-from core.file.file_manager import download
logger = logging.getLogger(__name__)
+
class URLToBase64Converter(BuiltinTool):
- def _invoke(self,
- user_id: str,
- tool_parameters: Dict[str, Any],
- ) -> ToolInvokeMessage:
+ def _invoke(
+ self,
+ user_id: str,
+ tool_parameters: dict[str, Any],
+ ) -> ToolInvokeMessage:
"""
Конвертирует файл в base64 строку.
Поддерживает как внешние URL, так и локальные файлы Dify.
"""
logger.info(f"Received parameters: {tool_parameters}")
-
+
# Получаем источник файла
- file_source = tool_parameters.get('file_source')
+ file_source = tool_parameters.get("file_source")
if not file_source:
- return self.create_text_message('Please specify file source (url or local)')
+ return self.create_text_message("Please specify file source (url or local)")
try:
# Обработка внешнего URL
- if file_source == 'url':
- url = tool_parameters.get('url')
+ if file_source == "url":
+ url = tool_parameters.get("url")
if not url:
- return self.create_text_message('Please provide a URL')
-
+ return self.create_text_message("Please provide a URL")
+
response = requests.get(url)
response.raise_for_status()
content = response.content
-
+
# Обработка локального файла Dify
- elif file_source == 'local':
- file = tool_parameters.get('file')
+ elif file_source == "local":
+ file = tool_parameters.get("file")
logger.info(f"File data: {file}")
-
+
if not file:
- return self.create_text_message('Please provide a file')
-
+ return self.create_text_message("Please provide a file")
+
if not isinstance(file, File):
logger.error(f"Invalid file type: {type(file)}")
- return self.create_text_message('Invalid file type. Expected Dify File object')
-
+ return self.create_text_message("Invalid file type. Expected Dify File object")
+
# Загружаем содержимое файла
content = download(file)
if not content:
logger.error("Failed to download file content")
- return self.create_text_message('Failed to download file content')
-
+ return self.create_text_message("Failed to download file content")
+
logger.info(f"Successfully loaded file: {len(content)} bytes")
-
+
else:
return self.create_text_message('Invalid file source. Use "url" or "local"')
-
+
# Конвертируем содержимое в base64
- base64_content = base64.b64encode(content).decode('utf-8')
+ base64_content = base64.b64encode(content).decode("utf-8")
logger.info(f"Successfully converted file to base64: {len(base64_content)} characters")
return self.create_text_message(base64_content)
-
+
except Exception as e:
logger.exception("Error processing file")
- return self.create_text_message(f'Error processing file: {str(e)}')
+ return self.create_text_message(f"Error processing file: {str(e)}")
diff --git a/api/core/tools/provider/builtin/url_to_base64/url_to_base64.py b/api/core/tools/provider/builtin/url_to_base64/url_to_base64.py
index d048f2cb225574..bc4d5ece873c26 100644
--- a/api/core/tools/provider/builtin/url_to_base64/url_to_base64.py
+++ b/api/core/tools/provider/builtin/url_to_base64/url_to_base64.py
@@ -1,5 +1,6 @@
-from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController
from core.tools.provider.builtin.url_to_base64.tools.url_to_base64_converter import URLToBase64Converter
+from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController
+
class URLToBase64Provider(BuiltinToolProviderController):
@property
@@ -13,7 +14,7 @@ def _tools(self) -> list:
return [
URLToBase64Converter(),
]
-
+
def _validate_credentials(self, credentials: dict) -> bool:
"""
Validate the credentials.
diff --git a/api/core/tools/provider/builtin/url_to_base64/url_to_base64_tool.py b/api/core/tools/provider/builtin/url_to_base64/url_to_base64_tool.py
index ecb2b251c7d9db..9bfc359593185d 100644
--- a/api/core/tools/provider/builtin/url_to_base64/url_to_base64_tool.py
+++ b/api/core/tools/provider/builtin/url_to_base64/url_to_base64_tool.py
@@ -1,45 +1,49 @@
-from typing import Any, Dict, List, Union
import base64
+from typing import Any, Union
+
import requests
from core.tools.entities.tool_entities import ToolInvokeMessage
from core.tools.tool.builtin_tool import BuiltinTool
+
class URLToBase64Tool(BuiltinTool):
- def _invoke(self,
- user_id: str,
- tool_parameters: Dict[str, Any],
- ) -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]:
+ def _invoke(
+ self,
+ user_id: str,
+ tool_parameters: dict[str, Any],
+ ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
"""
Конвертирует файл из URL в base64 строку
"""
- # Получаем URL из параметров
- url = tool_parameters.get('url', '')
+ url = tool_parameters.get("url", "")
if not url:
- return self.create_text_message('Пожалуйста, укажите URL')
+ return self.create_text_message("Пожалуйста, укажите URL")
try:
# Загружаем файл по URL
response = requests.get(url)
response.raise_for_status() # Проверяем на ошибки
-
+
# Конвертируем содержимое в base64
- base64_content = base64.b64encode(response.content).decode('utf-8')
-
+ base64_content = base64.b64encode(response.content).decode("utf-8")
+
# Возвращаем результат
return self.create_text_message(base64_content)
-
+
except Exception as e:
- return self.create_text_message(f'Ошибка при обработке файла: {str(e)}')
+ return self.create_text_message(f"Ошибка при обработке файла: {str(e)}")
- def get_runtime_parameters(self) -> List[Dict]:
+ def get_runtime_parameters(self) -> list[dict]:
"""
Определяем параметры инструмента
"""
- return [{
- "name": "url",
- "type": "string",
- "required": True,
- "label": "URL файла",
- "description": "URL файла, который нужно конвертировать в base64"
- }]
+ return [
+ {
+ "name": "url",
+ "type": "string",
+ "required": True,
+ "label": "URL файла",
+ "description": "URL файла, который нужно конвертировать в base64",
+ }
+ ]
diff --git a/api/core/tools/tool_file_manager.py b/api/core/tools/tool_file_manager.py
index 92a28c6bfe6600..3c649d8f8c5722 100644
--- a/api/core/tools/tool_file_manager.py
+++ b/api/core/tools/tool_file_manager.py
@@ -11,12 +11,12 @@
import httpx
from configs import dify_config
+from constants import MIME_TO_EXTENSION
from core.helper import ssrf_proxy
from extensions.ext_database import db
from extensions.ext_storage import storage
from models.model import MessageFile
from models.tools import ToolFile
-from constants import MIME_TO_EXTENSION
logger = logging.getLogger(__name__)
@@ -66,7 +66,7 @@ def create_file_by_raw(
mimetype: str,
) -> ToolFile:
logger.info(f"Creating file with mimetype: {mimetype}")
-
+
# Проверяем наше сопоставление MIME-типов
extension = MIME_TO_EXTENSION.get(mimetype)
if not extension:
@@ -75,7 +75,7 @@ def create_file_by_raw(
if not extension:
logger.warning(f"Could not determine extension for mimetype {mimetype}, using .bin")
extension = ".bin"
-
+
logger.info(f"Using extension: {extension}")
unique_name = uuid4().hex
filename = f"{unique_name}{extension}"
@@ -115,7 +115,7 @@ def create_file_by_url(
mimetype = guess_type(file_url)[0] or "octet/stream"
logger.info(f"Creating file from URL with mimetype: {mimetype}")
-
+
# Проверяем наше сопоставление MIME-типов
extension = MIME_TO_EXTENSION.get(mimetype)
if not extension:
@@ -124,7 +124,7 @@ def create_file_by_url(
if not extension:
logger.warning(f"Could not determine extension for mimetype {mimetype}, using .bin")
extension = ".bin"
-
+
logger.info(f"Using extension: {extension}")
unique_name = uuid4().hex
filename = f"{unique_name}{extension}"