Introduce Plugins (#13836)

Signed-off-by: yihong0618 <[email protected]>
Signed-off-by: -LAN- <[email protected]>
Signed-off-by: xhe <[email protected]>
Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: takatost <[email protected]>
Co-authored-by: kurokobo <[email protected]>
Co-authored-by: Novice Lee <[email protected]>
Co-authored-by: zxhlyh <[email protected]>
Co-authored-by: AkaraChen <[email protected]>
Co-authored-by: Yi <[email protected]>
Co-authored-by: Joel <[email protected]>
Co-authored-by: JzoNg <[email protected]>
Co-authored-by: twwu <[email protected]>
Co-authored-by: Hiroshi Fujita <[email protected]>
Co-authored-by: AkaraChen <[email protected]>
Co-authored-by: NFish <[email protected]>
Co-authored-by: Wu Tianwei <[email protected]>
Co-authored-by: 非法操作 <[email protected]>
Co-authored-by: Novice <[email protected]>
Co-authored-by: Hiroki Nagai <[email protected]>
Co-authored-by: Gen Sato <[email protected]>
Co-authored-by: eux <[email protected]>
Co-authored-by: huangzhuo1949 <[email protected]>
Co-authored-by: huangzhuo <[email protected]>
Co-authored-by: lotsik <[email protected]>
Co-authored-by: crazywoola <[email protected]>
Co-authored-by: nite-knite <[email protected]>
Co-authored-by: Jyong <[email protected]>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: gakkiyomi <[email protected]>
Co-authored-by: CN-P5 <[email protected]>
Co-authored-by: CN-P5 <[email protected]>
Co-authored-by: Chuehnone <[email protected]>
Co-authored-by: yihong <[email protected]>
Co-authored-by: Kevin9703 <[email protected]>
Co-authored-by: -LAN- <[email protected]>
Co-authored-by: Boris Feld <[email protected]>
Co-authored-by: mbo <[email protected]>
Co-authored-by: mabo <[email protected]>
Co-authored-by: Warren Chen <[email protected]>
Co-authored-by: JzoNgKVO <[email protected]>
Co-authored-by: jiandanfeng <[email protected]>
Co-authored-by: zhu-an <[email protected]>
Co-authored-by: zhaoqingyu.1075 <[email protected]>
Co-authored-by: 海狸大師 <[email protected]>
Co-authored-by: Xu Song <[email protected]>
Co-authored-by: rayshaw001 <[email protected]>
Co-authored-by: Ding Jiatong <[email protected]>
Co-authored-by: Bowen Liang <[email protected]>
Co-authored-by: JasonVV <[email protected]>
Co-authored-by: le0zh <[email protected]>
Co-authored-by: zhuxinliang <[email protected]>
Co-authored-by: k-zaku <[email protected]>
Co-authored-by: luckylhb90 <[email protected]>
Co-authored-by: hobo.l <[email protected]>
Co-authored-by: jiangbo721 <[email protected]>
Co-authored-by: 刘江波 <[email protected]>
Co-authored-by: Shun Miyazawa <[email protected]>
Co-authored-by: EricPan <[email protected]>
Co-authored-by: crazywoola <[email protected]>
Co-authored-by: sino <[email protected]>
Co-authored-by: Jhvcc <[email protected]>
Co-authored-by: lowell <[email protected]>
Co-authored-by: Boris Polonsky <[email protected]>
Co-authored-by: Ademílson Tonato <[email protected]>
Co-authored-by: Ademílson Tonato <[email protected]>
Co-authored-by: IWAI, Masaharu <[email protected]>
Co-authored-by: Yueh-Po Peng (Yabi) <[email protected]>
Co-authored-by: Jason <[email protected]>
Co-authored-by: Xin Zhang <[email protected]>
Co-authored-by: yjc980121 <[email protected]>
Co-authored-by: heyszt <[email protected]>
Co-authored-by: Abdullah AlOsaimi <[email protected]>
Co-authored-by: Abdullah AlOsaimi <[email protected]>
Co-authored-by: Yingchun Lai <[email protected]>
Co-authored-by: Hash Brown <[email protected]>
Co-authored-by: zuodongxu <[email protected]>
Co-authored-by: Masashi Tomooka <[email protected]>
Co-authored-by: aplio <[email protected]>
Co-authored-by: Obada Khalili <[email protected]>
Co-authored-by: Nam Vu <[email protected]>
Co-authored-by: Kei YAMAZAKI <[email protected]>
Co-authored-by: TechnoHouse <[email protected]>
Co-authored-by: Riddhimaan-Senapati <[email protected]>
Co-authored-by: MaFee921 <[email protected]>
Co-authored-by: te-chan <[email protected]>
Co-authored-by: HQidea <[email protected]>
Co-authored-by: Joshbly <[email protected]>
Co-authored-by: xhe <[email protected]>
Co-authored-by: weiwenyan-dev <[email protected]>
Co-authored-by: ex_wenyan.wei <[email protected]>
Co-authored-by: engchina <[email protected]>
Co-authored-by: engchina <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: 呆萌闷油瓶 <[email protected]>
Co-authored-by: Kemal <[email protected]>
Co-authored-by: Lazy_Frog <[email protected]>
Co-authored-by: Yi Xiao <[email protected]>
Co-authored-by: Steven sun <[email protected]>
Co-authored-by: steven <[email protected]>
Co-authored-by: Kalo Chin <[email protected]>
Co-authored-by: Katy Tao <[email protected]>
Co-authored-by: depy <[email protected]>
Co-authored-by: 胡春东 <[email protected]>
Co-authored-by: Junjie.M <[email protected]>
Co-authored-by: MuYu <[email protected]>
Co-authored-by: Naoki Takashima <[email protected]>
Co-authored-by: Summer-Gu <[email protected]>
Co-authored-by: Fei He <[email protected]>
Co-authored-by: ybalbert001 <[email protected]>
Co-authored-by: Yuanbo Li <[email protected]>
Co-authored-by: douxc <[email protected]>
Co-authored-by: liuzhenghua <[email protected]>
Co-authored-by: Wu Jiayang <[email protected]>
Co-authored-by: Your Name <[email protected]>
Co-authored-by: kimjion <[email protected]>
Co-authored-by: AugNSo <[email protected]>
Co-authored-by: llinvokerl <[email protected]>
Co-authored-by: liusurong.lsr <[email protected]>
Co-authored-by: Vasu Negi <[email protected]>
Co-authored-by: Hundredwz <[email protected]>
Co-authored-by: Xiyuan Chen <[email protected]>

.devcontainer/post_create_command.sh

@@ -1,11 +1,12 @@
 #!/bin/bash
 
-cd web && npm install
+npm add -g [email protected]
+cd web && pnpm install
 pipx install poetry
 
 echo 'alias start-api="cd /workspaces/dify/api && poetry run python -m flask run --host 0.0.0.0 --port=5001 --debug"' >> ~/.bashrc
 echo 'alias start-worker="cd /workspaces/dify/api && poetry run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion"' >> ~/.bashrc
-echo 'alias start-web="cd /workspaces/dify/web && npm run dev"' >> ~/.bashrc
+echo 'alias start-web="cd /workspaces/dify/web && pnpm dev"' >> ~/.bashrc
 echo 'alias start-containers="cd /workspaces/dify/docker && docker-compose -f docker-compose.middleware.yaml -p dify up -d"' >> ~/.bashrc
 echo 'alias stop-containers="cd /workspaces/dify/docker && docker-compose -f docker-compose.middleware.yaml -p dify down"' >> ~/.bashrc
 

.github/workflows/api-tests.yml

@@ -50,15 +50,9 @@ jobs:
       - name: Run Unit tests
         run: poetry run -P api bash dev/pytest/pytest_unit_tests.sh
 
-      - name: Run ModelRuntime
-        run: poetry run -P api bash dev/pytest/pytest_model_runtime.sh
-
       - name: Run dify config tests
         run: poetry run -P api python dev/pytest/pytest_config_tests.py
 
-      - name: Run Tool
-        run: poetry run -P api bash dev/pytest/pytest_tools.sh
-
       - name: Run mypy
         run: |
           poetry run -C api python -m mypy --install-types --non-interactive .

.github/workflows/db-migration-test.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - main
+      - plugins/beta
     paths:
       - api/migrations/**
       - .github/workflows/db-migration-test.yml

.github/workflows/style.yml

@@ -72,17 +72,23 @@ jobs:
         with:
           files: web/**
 
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10
+          run_install: false
+
       - name: Setup NodeJS
         uses: actions/setup-node@v4
         if: steps.changed-files.outputs.any_changed == 'true'
         with:
           node-version: 20
-          cache: yarn
+          cache: pnpm
           cache-dependency-path: ./web/package.json
 
       - name: Web dependencies
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: yarn install --frozen-lockfile
+        run: pnpm install --frozen-lockfile
 
       - name: Web style check
         if: steps.changed-files.outputs.any_changed == 'true'

.github/workflows/tool-test-sdks.yaml

@@ -35,10 +35,10 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
           cache: ''
-          cache-dependency-path: 'yarn.lock'
+          cache-dependency-path: 'pnpm-lock.yaml'
 
       - name: Install Dependencies
-        run: yarn install
+        run: pnpm install
 
       - name: Test
-        run: yarn test
+        run: pnpm test

.github/workflows/translate-i18n-base-on-english.yml

@@ -39,11 +39,11 @@ jobs:
 
       - name: Install dependencies
         if: env.FILES_CHANGED == 'true'
-        run: yarn install --frozen-lockfile
+        run: pnpm install --frozen-lockfile
 
       - name: Run npm script
         if: env.FILES_CHANGED == 'true'
-        run: npm run auto-gen-i18n
+        run: pnpm run auto-gen-i18n
 
       - name: Create Pull Request
         if: env.FILES_CHANGED == 'true'

.github/workflows/web-tests.yml

@@ -37,13 +37,13 @@ jobs:
         if: steps.changed-files.outputs.any_changed == 'true'
         with:
           node-version: 20
-          cache: yarn
+          cache: pnpm
           cache-dependency-path: ./web/package.json
 
       - name: Install dependencies
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: yarn install --frozen-lockfile
+        run: pnpm install --frozen-lockfile
 
       - name: Run tests
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: yarn test
+        run: pnpm test

.gitignore

@@ -176,6 +176,7 @@ docker/volumes/pgvector/data/*
 docker/volumes/pgvecto_rs/data/*
 docker/volumes/couchbase/*
 docker/volumes/oceanbase/*
+docker/volumes/plugin_daemon/*
 !docker/volumes/oceanbase/init.d
 
 docker/nginx/conf.d/default.conf
@@ -194,3 +195,9 @@ api/.vscode
 
 .idea/
 .vscode
+
+# pnpm
+/.pnpm-store
+
+# plugin migrate
+plugins.jsonl

api/.dockerignore

@@ -1,14 +1,19 @@
 .env
 *.env.*
 
+storage/generate_files/*
 storage/privkeys/*
+storage/tools/*
+storage/upload_files/*
 
 # Logs
 logs
 *.log*
 
 # jetbrains
 .idea
+.mypy_cache
+.ruff_cache
 
 # venv
 .venv

api/.env.example

@@ -409,7 +409,6 @@ MAX_VARIABLE_SIZE=204800
 APP_MAX_EXECUTION_TIME=1200
 APP_MAX_ACTIVE_REQUESTS=0
 
-
 # Celery beat configuration
 CELERY_BEAT_SCHEDULER_TIME=1
 
@@ -422,6 +421,22 @@ POSITION_PROVIDER_PINS=
 POSITION_PROVIDER_INCLUDES=
 POSITION_PROVIDER_EXCLUDES=
 
+# Plugin configuration
+PLUGIN_DAEMON_KEY=lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi
+PLUGIN_DAEMON_URL=http://127.0.0.1:5002
+PLUGIN_REMOTE_INSTALL_PORT=5003
+PLUGIN_REMOTE_INSTALL_HOST=localhost
+PLUGIN_MAX_PACKAGE_SIZE=15728640
+INNER_API_KEY=QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1
+INNER_API_KEY_FOR_PLUGIN=QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1
+
+# Marketplace configuration
+MARKETPLACE_ENABLED=true
+MARKETPLACE_API_URL=https://marketplace.dify.ai
+
+# Endpoint configuration
+ENDPOINT_URL_TEMPLATE=http://localhost:5002/e/{hook_id}
+
 # Reset password token expiry minutes
 RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5
 

api/Dockerfile

@@ -73,6 +73,10 @@ ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 # Download nltk data
 RUN python -c "import nltk; nltk.download('punkt'); nltk.download('averaged_perceptron_tagger')"
 
+ENV TIKTOKEN_CACHE_DIR=/app/api/.tiktoken_cache
+
+RUN python -c "import tiktoken; tiktoken.encoding_for_model('gpt2')"
+
 # Copy source code
 COPY . /app/api/
 

api/commands.py

@@ -25,6 +25,8 @@
 from models.model import Account, App, AppAnnotationSetting, AppMode, Conversation, MessageAnnotation
 from models.provider import Provider, ProviderModel
 from services.account_service import RegisterService, TenantService
+from services.plugin.data_migration import PluginDataMigration
+from services.plugin.plugin_migration import PluginMigration
 
 
 @click.command("reset-password", help="Reset the account password.")
@@ -524,7 +526,7 @@ def add_qdrant_doc_id_index(field: str):
                         )
                     )
 
-    except Exception as e:
+    except Exception:
         click.echo(click.style("Failed to create Qdrant client.", fg="red"))
 
     click.echo(click.style(f"Index creation complete. Created {create_count} collection indexes.", fg="green"))
@@ -593,7 +595,7 @@ def upgrade_db():
 
             click.echo(click.style("Database migration successful!", fg="green"))
 
-        except Exception as e:
+        except Exception:
             logging.exception("Failed to execute database migration")
         finally:
             lock.release()
@@ -639,7 +641,7 @@ def fix_app_site_missing():
                         account = accounts[0]
                         print("Fixing missing site for app {}".format(app.id))
                         app_was_created.send(app, account=account)
-                except Exception as e:
+                except Exception:
                     failed_app_ids.append(app_id)
                     click.echo(click.style("Failed to fix missing site for app {}".format(app_id), fg="red"))
                     logging.exception(f"Failed to fix app related site missing issue, app_id: {app_id}")
@@ -649,3 +651,68 @@ def fix_app_site_missing():
                 break
 
     click.echo(click.style("Fix for missing app-related sites completed successfully!", fg="green"))
+
+
+@click.command("migrate-data-for-plugin", help="Migrate data for plugin.")
+def migrate_data_for_plugin():
+    """
+    Migrate data for plugin.
+    """
+    click.echo(click.style("Starting migrate data for plugin.", fg="white"))
+
+    PluginDataMigration.migrate()
+
+    click.echo(click.style("Migrate data for plugin completed.", fg="green"))
+
+
+@click.command("extract-plugins", help="Extract plugins.")
+@click.option("--output_file", prompt=True, help="The file to store the extracted plugins.", default="plugins.jsonl")
+@click.option("--workers", prompt=True, help="The number of workers to extract plugins.", default=10)
+def extract_plugins(output_file: str, workers: int):
+    """
+    Extract plugins.
+    """
+    click.echo(click.style("Starting extract plugins.", fg="white"))
+
+    PluginMigration.extract_plugins(output_file, workers)
+
+    click.echo(click.style("Extract plugins completed.", fg="green"))
+
+
+@click.command("extract-unique-identifiers", help="Extract unique identifiers.")
+@click.option(
+    "--output_file",
+    prompt=True,
+    help="The file to store the extracted unique identifiers.",
+    default="unique_identifiers.json",
+)
+@click.option(
+    "--input_file", prompt=True, help="The file to store the extracted unique identifiers.", default="plugins.jsonl"
+)
+def extract_unique_plugins(output_file: str, input_file: str):
+    """
+    Extract unique plugins.
+    """
+    click.echo(click.style("Starting extract unique plugins.", fg="white"))
+
+    PluginMigration.extract_unique_plugins_to_file(input_file, output_file)
+
+    click.echo(click.style("Extract unique plugins completed.", fg="green"))
+
+
+@click.command("install-plugins", help="Install plugins.")
+@click.option(
+    "--input_file", prompt=True, help="The file to store the extracted unique identifiers.", default="plugins.jsonl"
+)
+@click.option(
+    "--output_file", prompt=True, help="The file to store the installed plugins.", default="installed_plugins.jsonl"
+)
+def install_plugins(input_file: str, output_file: str):
+    """
+    Install plugins.
+    """
+    click.echo(click.style("Starting install plugins.", fg="white"))
+
+    PluginMigration.install_plugins(input_file, output_file)
+
+    click.echo(click.style("Install plugins completed.", fg="green"))

api/configs/feature/__init__.py

@@ -134,6 +134,60 @@ class CodeExecutionSandboxConfig(BaseSettings):
     )
 
 
+class PluginConfig(BaseSettings):
+    """
+    Plugin configs
+    """
+
+    PLUGIN_DAEMON_URL: HttpUrl = Field(
+        description="Plugin API URL",
+        default="http://localhost:5002",
+    )
+
+    PLUGIN_DAEMON_KEY: str = Field(
+        description="Plugin API key",
+        default="plugin-api-key",
+    )
+
+    INNER_API_KEY_FOR_PLUGIN: str = Field(description="Inner api key for plugin", default="inner-api-key")
+
+    PLUGIN_REMOTE_INSTALL_HOST: str = Field(
+        description="Plugin Remote Install Host",
+        default="localhost",
+    )
+
+    PLUGIN_REMOTE_INSTALL_PORT: PositiveInt = Field(
+        description="Plugin Remote Install Port",
+        default=5003,
+    )
+
+    PLUGIN_MAX_PACKAGE_SIZE: PositiveInt = Field(
+        description="Maximum allowed size for plugin packages in bytes",
+        default=15728640,
+    )
+
+    PLUGIN_MAX_BUNDLE_SIZE: PositiveInt = Field(
+        description="Maximum allowed size for plugin bundles in bytes",
+        default=15728640 * 12,
+    )
+
+
+class MarketplaceConfig(BaseSettings):
+    """
+    Configuration for marketplace
+    """
+
+    MARKETPLACE_ENABLED: bool = Field(
+        description="Enable or disable marketplace",
+        default=True,
+    )
+
+    MARKETPLACE_API_URL: HttpUrl = Field(
+        description="Marketplace API URL",
+        default="https://marketplace.dify.ai",
+    )
+
+
 class EndpointConfig(BaseSettings):
     """
     Configuration for various application endpoints and URLs
@@ -160,6 +214,10 @@ class EndpointConfig(BaseSettings):
         default="",
     )
 
+    ENDPOINT_URL_TEMPLATE: str = Field(
+        description="Template url for endpoint plugin", default="http://localhost:5002/e/{hook_id}"
+    )
+
 
 class FileAccessConfig(BaseSettings):
     """
@@ -793,6 +851,8 @@ class FeatureConfig(
     AuthConfig,  # Changed from OAuthConfig to AuthConfig
     BillingConfig,
     CodeExecutionSandboxConfig,
+    PluginConfig,
+    MarketplaceConfig,
     DataSetConfig,
     EndpointConfig,
     FileAccessConfig,

api/configs/packaging/__init__.py

@@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings):
 
     CURRENT_VERSION: str = Field(
         description="Dify version",
-        default="0.15.3",
+        default="1.0.0",
     )
 
     COMMIT_SHA: str = Field(