diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 7062c29a..3e6be37d 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -1,3 +1,7 @@ +--- +applyTo: '**' +--- + # REPO SPECIFIC INSTRUCTIONS --- diff --git a/.github/instructions/python-lang.instructions.md b/.github/instructions/python-lang.instructions.md new file mode 100644 index 00000000..c37b99c7 --- /dev/null +++ b/.github/instructions/python-lang.instructions.md @@ -0,0 +1,13 @@ +--- +applyTo: '**' +--- + +# Python Language Guide + +- Files should start with a comment of the file name. Ex: `# functions_personal_agents.py` + +- Imports should be grouped at the top of the document after the module docstring, unless otherwise indicated by the user or for performance reasons in which case the import should be as close as possible to the usage with a documented note as to why the import is not at the top of the file. + +- Use 4 spaces per indentation level. No tabs. + +- Code and definitions should occur after the imports block. \ No newline at end of file diff --git a/.github/workflows/docker_image_publish_nadoyle.yml b/.github/workflows/docker_image_publish_nadoyle.yml index c39f99e4..4edc6dbf 100644 --- a/.github/workflows/docker_image_publish_nadoyle.yml +++ b/.github/workflows/docker_image_publish_nadoyle.yml @@ -5,6 +5,7 @@ on: push: branches: - nadoyle + - keyvaultForSecrets workflow_dispatch: @@ -19,11 +20,11 @@ jobs: uses: Azure/docker-login@v2 with: # Container registry username - username: ${{ secrets.ACR_USERNAME }} + username: ${{ secrets.ACR_USERNAME_NADOYLE }} # Container registry password - password: ${{ secrets.ACR_PASSWORD }} + password: ${{ secrets.ACR_PASSWORD_NADOYLE }} # Container registry server url - login-server: ${{ secrets.ACR_LOGIN_SERVER }} + login-server: ${{ secrets.ACR_LOGIN_SERVER_NADOYLE }} - uses: actions/checkout@v3 - name: Set up Node.js @@ -36,7 +37,7 @@ jobs: run: node scripts/generate-validators.mjs - name: Build the Docker image run: - docker build . --file application/single_app/Dockerfile --tag ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat-dev:$(date +'%Y-%m-%d')_$GITHUB_RUN_NUMBER; - docker tag ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat-dev:$(date +'%Y-%m-%d')_$GITHUB_RUN_NUMBER ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat-dev:latest; - docker push ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat-dev:$(date +'%Y-%m-%d')_$GITHUB_RUN_NUMBER; - docker push ${{ secrets.ACR_LOGIN_SERVER }}/simple-chat-dev:latest; + docker build . --file application/single_app/Dockerfile --tag ${{ secrets.ACR_LOGIN_SERVER_NADOYLE }}/simple-chat-dev:$(date +'%Y-%m-%d')_$GITHUB_RUN_NUMBER; + docker tag ${{ secrets.ACR_LOGIN_SERVER_NADOYLE }}/simple-chat-dev:$(date +'%Y-%m-%d')_$GITHUB_RUN_NUMBER ${{ secrets.ACR_LOGIN_SERVER_NADOYLE }}/simple-chat-dev:latest; + docker push ${{ secrets.ACR_LOGIN_SERVER_NADOYLE }}/simple-chat-dev:$(date +'%Y-%m-%d')_$GITHUB_RUN_NUMBER; + docker push ${{ secrets.ACR_LOGIN_SERVER_NADOYLE }}/simple-chat-dev:latest; diff --git a/application/single_app/agent_logging_chat_completion.py b/application/single_app/agent_logging_chat_completion.py index 1e1ae3ce..e4173ef2 100644 --- a/application/single_app/agent_logging_chat_completion.py +++ b/application/single_app/agent_logging_chat_completion.py @@ -144,13 +144,6 @@ async def invoke(self, *args, **kwargs): } ) - log_event("[Logging Agent Request] Agent invoke started", - extra={ - "agent": self.name, - "prompt_preview": [m.content[:30] for m in args[0]] if args else None - }, - level=logging.DEBUG) - # Store user question context for better tool detection if args and args[0] and hasattr(args[0][-1], 'content'): self._user_question = args[0][-1].content @@ -163,12 +156,14 @@ async def invoke(self, *args, **kwargs): initial_message_count = len(args[0]) if args and args[0] else 0 result = super().invoke(*args, **kwargs) - log_event("[Logging Agent Request] Result received", - extra={ - "agent": self.name, - "result_type": type(result).__name__ - }, - level=logging.DEBUG) + log_event( + "[Logging Agent Request] Result received", + extra={ + "agent": self.name, + "result_type": type(result).__name__ + }, + level=logging.DEBUG + ) if hasattr(result, "__aiter__"): # Streaming/async generator response @@ -180,13 +175,15 @@ async def invoke(self, *args, **kwargs): # Regular coroutine response response = await result - log_event("[Logging Agent Request] Response received", - extra={ - "agent": self.name, - "response_type": type(response).__name__, - "response_preview": str(response)[:100] if response else None - }, - level=logging.DEBUG) + log_event( + "[Logging Agent Request] Response received", + extra={ + "agent": self.name, + "response_type": type(response).__name__, + "response_preview": str(response)[:100] if response else None + }, + level=logging.DEBUG + ) # Store the response for analysis self._last_response = response diff --git a/application/single_app/app.py b/application/single_app/app.py index 0dccf70a..528908c5 100644 --- a/application/single_app/app.py +++ b/application/single_app/app.py @@ -105,6 +105,7 @@ from route_external_health import * +#TODO: Remove this after speaking with Paul configure_azure_monitor() # =================== Session Configuration =================== diff --git a/application/single_app/config.py b/application/single_app/config.py index 5c740776..a35e3075 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -158,9 +158,6 @@ else: AUTHORITY = f"https://login.microsoftonline.com/{TENANT_ID}" -# Commercial Azure Video Indexer Endpoint -video_indexer_endpoint = "https://api.videoindexer.ai" - WORD_CHUNK_SIZE = 400 if AZURE_ENVIRONMENT == "usgovernment": @@ -171,6 +168,7 @@ cognitive_services_scope = "https://cognitiveservices.azure.us/.default" video_indexer_endpoint = "https://api.videoindexer.ai.azure.us" search_resource_manager = "https://search.azure.us" + KEY_VAULT_DOMAIN = ".vault.usgovcloudapi.net" elif AZURE_ENVIRONMENT == "custom": resource_manager = CUSTOM_RESOURCE_MANAGER_URL_VALUE @@ -178,6 +176,7 @@ credential_scopes=[resource_manager + "/.default"] cognitive_services_scope = CUSTOM_COGNITIVE_SERVICES_URL_VALUE search_resource_manager = CUSTOM_SEARCH_RESOURCE_MANAGER_URL_VALUE + KEY_VAULT_DOMAIN = os.getenv("KEY_VAULT_DOMAIN", ".vault.azure.net") else: OIDC_METADATA_URL = f"https://login.microsoftonline.com/{TENANT_ID}/v2.0/.well-known/openid-configuration" resource_manager = "https://management.azure.com" @@ -185,6 +184,7 @@ credential_scopes=[resource_manager + "/.default"] cognitive_services_scope = "https://cognitiveservices.azure.com/.default" video_indexer_endpoint = "https://api.videoindexer.ai" + KEY_VAULT_DOMAIN = ".vault.azure.net" def get_redis_cache_infrastructure_endpoint(redis_hostname: str) -> str: """ @@ -205,6 +205,7 @@ def get_redis_cache_infrastructure_endpoint(redis_hostname: str) -> str: else: # Default to Azure Public Cloud return f"https://{redis_hostname}.cacheinfra.windows.net:10225/appid" + storage_account_user_documents_container_name = "user-documents" storage_account_group_documents_container_name = "group-documents" diff --git a/application/single_app/functions_appinsights.py b/application/single_app/functions_appinsights.py index 320f8c5f..05c656e4 100644 --- a/application/single_app/functions_appinsights.py +++ b/application/single_app/functions_appinsights.py @@ -44,29 +44,36 @@ def log_event( exceptionTraceback (Any, optional): If set to True, includes exception traceback. """ try: + # Limit message to 32767 characters + if message and isinstance(message, str) and len(message) > 32767: + message = message[:32767] + # Get logger - use Azure Monitor logger if configured, otherwise standard logger logger = get_appinsights_logger() if not logger: + print(f"[Log] {message} -- {extra}") logger = logging.getLogger('standard') if not logger.handlers: logger.addHandler(logging.StreamHandler()) logger.setLevel(logging.INFO) - + # Enhanced exception handling for Application Insights # When exceptionTraceback=True, ensure we capture full exception context exc_info_to_use = exceptionTraceback - + # For ERROR level logs with exceptionTraceback=True, always log as exception if level >= logging.ERROR and exceptionTraceback: if logger and hasattr(logger, 'exception'): # Use logger.exception() for better exception capture in Application Insights - logger.exception(message, extra=extra, stacklevel=stacklevel) + logger.exception(message, extra=extra, stacklevel=stacklevel, stack_info=includeStack, exc_info=True) return else: # Fallback to standard logging with exc_info exc_info_to_use = True - + # Format message with extra properties for structured logging + + print(f"[Log] {message} -- {extra}") # Debug print to console if extra: # For modern Azure Monitor, extra properties are automatically captured logger.log( @@ -85,12 +92,12 @@ def log_event( stack_info=includeStack, exc_info=exc_info_to_use ) - + # For Azure Monitor, ensure exception-level logs are properly categorized if level >= logging.ERROR and _azure_monitor_configured: # Add a debug print to verify exception logging is working print(f"[Azure Monitor] Exception logged: {message[:100]}...") - + except Exception as e: # Fallback to basic logging if anything fails try: @@ -98,11 +105,11 @@ def log_event( if not fallback_logger.handlers: fallback_logger.addHandler(logging.StreamHandler()) fallback_logger.setLevel(logging.INFO) - + fallback_message = f"{message} | Original error: {str(e)}" if extra: fallback_message += f" | Extra: {extra}" - + fallback_logger.log(level, fallback_message) except: # If even basic logging fails, print to console diff --git a/application/single_app/functions_global_actions.py b/application/single_app/functions_global_actions.py index 07ce3a19..91f0d9f9 100644 --- a/application/single_app/functions_global_actions.py +++ b/application/single_app/functions_global_actions.py @@ -10,10 +10,10 @@ import json import traceback from datetime import datetime - from config import cosmos_global_actions_container +from functions_keyvault import keyvault_plugin_save_helper, keyvault_plugin_get_helper, keyvault_plugin_delete_helper, SecretReturnType -def get_global_actions(): +def get_global_actions(return_type=SecretReturnType.TRIGGER): """ Get all global actions. @@ -25,7 +25,8 @@ def get_global_actions(): query="SELECT * FROM c", enable_cross_partition_query=True )) - + # Resolve Key Vault references for each action + actions = [keyvault_plugin_get_helper(a, scope_value=a.get('id'), scope="global", return_type=return_type) for a in actions] return actions except Exception as e: @@ -34,7 +35,7 @@ def get_global_actions(): return [] -def get_global_action(action_id): +def get_global_action(action_id, return_type=SecretReturnType.TRIGGER): """ Get a specific global action by ID. @@ -45,13 +46,12 @@ def get_global_action(action_id): dict: Action data or None if not found """ try: - from config import cosmos_global_actions_container - action = cosmos_global_actions_container.read_item( item=action_id, partition_key=action_id ) - + # Resolve Key Vault references + action = keyvault_plugin_get_helper(action, scope_value=action_id, scope="global", return_type=return_type) print(f"✅ Found global action: {action_id}") return action @@ -71,21 +71,17 @@ def save_global_action(action_data): dict: Saved action data or None if failed """ try: - from config import cosmos_global_actions_container - # Ensure required fields if 'id' not in action_data: action_data['id'] = str(uuid.uuid4()) - # Add metadata action_data['is_global'] = True action_data['created_at'] = datetime.utcnow().isoformat() action_data['updated_at'] = datetime.utcnow().isoformat() - print(f"💾 Saving global action: {action_data.get('name', 'Unknown')}") - + # Store secrets in Key Vault before upsert + action_data = keyvault_plugin_save_helper(action_data, scope_value=action_data.get('id'), scope="global") result = cosmos_global_actions_container.upsert_item(body=action_data) - print(f"✅ Global action saved successfully: {result['id']}") return result @@ -106,15 +102,15 @@ def delete_global_action(action_id): bool: True if successful, False otherwise """ try: - from config import cosmos_global_actions_container - print(f"🗑ī¸ Deleting global action: {action_id}") - + # Delete secrets from Key Vault before deleting the action + action = get_global_action(action_id) + if action: + keyvault_plugin_delete_helper(action, scope_value=action_id, scope="global") cosmos_global_actions_container.delete_item( item=action_id, partition_key=action_id ) - print(f"✅ Global action deleted successfully: {action_id}") return True diff --git a/application/single_app/functions_global_agents.py b/application/single_app/functions_global_agents.py index 9d4e934a..b7d907fb 100644 --- a/application/single_app/functions_global_agents.py +++ b/application/single_app/functions_global_agents.py @@ -14,6 +14,8 @@ from functions_authentication import get_current_user_id from datetime import datetime from config import cosmos_global_agents_container +from functions_keyvault import keyvault_agent_save_helper, store_secret_in_key_vault, keyvault_agent_get_helper, keyvault_agent_delete_helper +from functions_settings import * def ensure_default_global_agent_exists(): @@ -54,13 +56,27 @@ def ensure_default_global_agent_exists(): "agent_name": default_agent["name"] }, ) - print("✅ Default global agent created.") + print("Default global agent created.") else: log_event( "At least one global agent already exists.", extra={"existing_agents_count": len(agents)}, ) - print("ℹī¸ At least one global agent already exists.") + print("At least one global agent already exists.") + + settings = get_settings() + needs_default = False + global_selected = settings.get("global_selected_agent") if settings else None + if not isinstance(global_selected, dict): + needs_default = True + elif global_selected.get("name", "") == "": + needs_default = True + if settings and needs_default: + settings["global_selected_agent"] = { + "name": default_agent["name"], + "is_global": True + } + save_settings(settings) except Exception as e: log_event( f"Error ensuring default global agent exists: {e}", @@ -68,7 +84,7 @@ def ensure_default_global_agent_exists(): level=logging.ERROR, exceptionTraceback=True ) - print(f"❌ Error ensuring default global agent exists: {e}") + print(f"Error ensuring default global agent exists: {e}") traceback.print_exc() def get_global_agents(): @@ -83,6 +99,8 @@ def get_global_agents(): query="SELECT * FROM c", enable_cross_partition_query=True )) + # Mask or replace sensitive keys for UI display + agents = [keyvault_agent_get_helper(agent, agent.get('id', ''), scope="global") for agent in agents] return agents except Exception as e: log_event( @@ -90,7 +108,7 @@ def get_global_agents(): extra={"exception": str(e)}, exceptionTraceback=True ) - print(f"❌ Error getting global agents: {str(e)}") + print(f"Error getting global agents: {str(e)}") traceback.print_exc() return [] @@ -110,7 +128,8 @@ def get_global_agent(agent_id): item=agent_id, partition_key=agent_id ) - print(f"✅ Found global agent: {agent_id}") + agent = keyvault_agent_get_helper(agent, agent_id, scope="global") + print(f"Found global agent: {agent_id}") return agent except Exception as e: log_event( @@ -119,7 +138,7 @@ def get_global_agent(agent_id): level=logging.ERROR, exceptionTraceback=True ) - print(f"❌ Error getting global agent {agent_id}: {str(e)}") + print(f"Error getting global agent {agent_id}: {str(e)}") return None @@ -146,13 +165,17 @@ def save_global_agent(agent_data): "Saving global agent.", extra={"agent_name": agent_data.get('name', 'Unknown')}, ) - print(f"💾 Saving global agent: {agent_data.get('name', 'Unknown')}") + print(f"Saving global agent: {agent_data.get('name', 'Unknown')}") + + # Use the new helper to store sensitive agent keys in Key Vault + agent_data = keyvault_agent_save_helper(agent_data, agent_data['id'], scope="global") + result = cosmos_global_agents_container.upsert_item(body=agent_data) log_event( "Global agent saved successfully.", extra={"agent_id": result['id'], "user_id": user_id}, ) - print(f"✅ Global agent saved successfully: {result['id']}") + print(f"Global agent saved successfully: {result['id']}") return result except Exception as e: log_event( @@ -161,7 +184,7 @@ def save_global_agent(agent_data): level=logging.ERROR, exceptionTraceback=True ) - print(f"❌ Error saving global agent: {str(e)}") + print(f"Error saving global agent: {str(e)}") traceback.print_exc() return None @@ -178,7 +201,9 @@ def delete_global_agent(agent_id): """ try: user_id = get_current_user_id() - print(f"🗑ī¸ Deleting global agent: {agent_id}") + print(f"Deleting global agent: {agent_id}") + agent_dict = get_global_agent(agent_id) + keyvault_agent_delete_helper(agent_dict, agent_id, scope="global") cosmos_global_agents_container.delete_item( item=agent_id, partition_key=agent_id @@ -187,7 +212,7 @@ def delete_global_agent(agent_id): "Global agent deleted successfully.", extra={"agent_id": agent_id, "user_id": user_id}, ) - print(f"✅ Global agent deleted successfully: {agent_id}") + print(f"Global agent deleted successfully: {agent_id}") return True except Exception as e: log_event( @@ -196,6 +221,6 @@ def delete_global_agent(agent_id): level=logging.ERROR, exceptionTraceback=True ) - print(f"❌ Error deleting global agent {agent_id}: {str(e)}") + print(f"Error deleting global agent {agent_id}: {str(e)}") traceback.print_exc() return False diff --git a/application/single_app/functions_keyvault.py b/application/single_app/functions_keyvault.py new file mode 100644 index 00000000..11a6f0bd --- /dev/null +++ b/application/single_app/functions_keyvault.py @@ -0,0 +1,551 @@ +# functions_keyvault.py + +import re +import logging +from functions_appinsights import log_event +from config import * +from functions_authentication import * +from functions_settings import * +from enum import Enum + +try: + from azure.identity import DefaultAzureCredential + from azure.keyvault.secrets import SecretClient +except ImportError as e: + raise ImportError("Required Azure SDK packages are not installed. Please install azure-identity and azure-keyvault-secrets.") from e + +""" +KEY_VAULT_DOMAIN # ENV VAR from config.py +enable_key_vault_secret_storage # setting from functions_settings.py +key_vault_name # setting from functions_settings.py +key_vault_identity # setting from functions_settings.py +""" + +supported_sources = [ + 'action', + 'action-addset', + 'agent', + 'other' +] + +supported_scopes = [ + 'global', + 'user', + 'group' +] + +supported_action_auth_types = [ + 'key', + 'servicePrincipal', + 'basic', + 'username_password', + 'connection_string' +] + +ui_trigger_word = "Stored_In_KeyVault" + +class SecretReturnType(Enum): + VALUE = "value" + TRIGGER = "trigger" + NAME = "name" + +def retrieve_secret_from_key_vault(secret_name, scope_value, scope="global", source="global"): + """ + Retrieve a secret from Key Vault using a dynamic name based on source, scope, and scope_value. + + Args: + secret_name (str): The base name of the secret. + scope_value (str): The value for the scope (e.g., user id). + scope (str): The scope (e.g., 'user', 'global'). + source (str): The source (e.g., 'agent', 'plugin'). + + Returns: + str: The value of the retrieved secret. + Raises: + Exception: If retrieval fails or configuration is invalid. + """ + if source not in supported_sources: + logging.error(f"Source '{source}' is not supported. Supported sources: {supported_sources}") + raise ValueError(f"Source '{source}' is not supported. Supported sources: {supported_sources}") + if scope not in supported_scopes: + logging.error(f"Scope '{scope}' is not supported. Supported scopes: {supported_scopes}") + raise ValueError(f"Scope '{scope}' is not supported. Supported scopes: {supported_scopes}") + + full_secret_name = build_full_secret_name(secret_name, scope_value, source, scope) + return retrieve_secret_from_key_vault_by_full_name(full_secret_name) + +def retrieve_secret_from_key_vault_by_full_name(full_secret_name): + """ + Retrieve a secret from Key Vault using a preformatted full secret name. + + Args: + full_secret_name (str): The full secret name (already formatted). + + Returns: + str: The value of the retrieved secret. + Raises: + Exception: If retrieval fails or configuration is invalid. + """ + settings = get_settings() + enable_key_vault_secret_storage = settings.get("enable_key_vault_secret_storage", False) + if not enable_key_vault_secret_storage: + return full_secret_name + + key_vault_name = settings.get("key_vault_name", None) + if not key_vault_name: + return full_secret_name + + if not validate_secret_name_dynamic(full_secret_name): + return full_secret_name + + try: + key_vault_url = f"https://{key_vault_name}{KEY_VAULT_DOMAIN}" + secret_client = SecretClient(vault_url=key_vault_url, credential=get_keyvault_credential()) + + retrieved_secret = secret_client.get_secret(full_secret_name) + print(f"Secret '{full_secret_name}' retrieved successfully from Key Vault.") + return retrieved_secret.value + except Exception as e: + logging.error(f"Failed to retrieve secret '{full_secret_name}' from Key Vault: {str(e)}") + return full_secret_name + + +def store_secret_in_key_vault(secret_name, secret_value, scope_value, source="global", scope="global"): + """ + Store a secret in Key Vault using a dynamic name based on source, scope, and scope_value. + + Args: + secret_name (str): The base name of the secret. + secret_value (str): The value to store in Key Vault. + scope_value (str): The value for the scope (e.g., user id). + source (str): The source (e.g., 'agent', 'plugin'). + scope (str): The scope (e.g., 'user', 'global'). + + Returns: + str: The full secret name used in Key Vault. + Raises: + Exception: If storing fails or configuration is invalid. + """ + settings = get_settings() + enable_key_vault_secret_storage = settings.get("enable_key_vault_secret_storage", False) + if not enable_key_vault_secret_storage: + logging.warn(f"Key Vault secret storage is not enabled.") + return secret_value + + key_vault_name = settings.get("key_vault_name", None) + if not key_vault_name: + logging.warn(f"Key Vault name is not configured.") + return secret_value + + if source not in supported_sources: + logging.error(f"Source '{source}' is not supported. Supported sources: {supported_sources}") + raise ValueError(f"Source '{source}' is not supported. Supported sources: {supported_sources}") + if scope not in supported_scopes: + logging.error(f"Scope '{scope}' is not supported. Supported scopes: {supported_scopes}") + raise ValueError(f"Scope '{scope}' is not supported. Supported scopes: {supported_scopes}") + + + full_secret_name = build_full_secret_name(secret_name, scope_value, source, scope) + + try: + key_vault_url = f"https://{key_vault_name}{KEY_VAULT_DOMAIN}" + secret_client = SecretClient(vault_url=key_vault_url, credential=get_keyvault_credential()) + secret_client.set_secret(full_secret_name, secret_value) + print(f"Secret '{full_secret_name}' stored successfully in Key Vault.") + return full_secret_name + except Exception as e: + logging.error(f"Failed to store secret '{full_secret_name}' in Key Vault: {str(e)}") + return secret_value + +def build_full_secret_name(secret_name, scope_value, source, scope): + """ + Build the full secret name for Key Vault and check its length. + + Args: + secret_name (str): The base name of the secret. + scope_value (str): The value for the scope (e.g., user id). + source (str): The source (e.g., 'agent', 'plugin'). + scope (str): The scope (e.g., 'user', 'global'). + + Returns: + str: The constructed full secret name. + Raises: + ValueError: If the name exceeds 127 characters. + """ + full_secret_name = f"{clean_name_for_keyvault(scope_value)}--{source}--{scope}--{clean_name_for_keyvault(secret_name)}" + if not validate_secret_name_dynamic(full_secret_name): + logging.error(f"The full secret name '{full_secret_name}' is invalid.") + raise ValueError(f"The full secret name '{full_secret_name}' is invalid.") + return full_secret_name + +def validate_secret_name_dynamic(secret_name): + """ + Validate a Key Vault secret name using a dynamically built regex based on supported scopes and sources. + The secret_name and scope_value can be wildcards, but scope and source must match supported lists. + + Args: + secret_name (str): The full secret name to validate. + + Returns: + bool: True if valid, False otherwise. + """ + # Build regex pattern dynamically + scopes_pattern = '|'.join(re.escape(scope) for scope in supported_scopes) + sources_pattern = '|'.join(re.escape(source) for source in supported_sources) + # Wildcards for secret_name and scope_value + pattern = rf"^(.+)--({sources_pattern})--({scopes_pattern})--(.+)$" + match = re.match(pattern, secret_name) + if not match: + return False + # Optionally, check length + if len(secret_name) > 127: + return False + return True + +def keyvault_agent_save_helper(agent_dict, scope_value, scope="global"): + """ + For agent dicts, store sensitive keys in Key Vault and replace their values with the Key Vault secret name. + Only processes 'azure_agent_apim_gpt_subscription_key' and 'azure_openai_gpt_key'. + + Args: + agent_dict (dict): The agent dictionary to process. + scope_value (str): The value for the scope (e.g., agent id). + scope (str): The scope (e.g., 'user', 'global'). + + Returns: + dict: A new agent dict with sensitive values replaced by Key Vault references. + Raises: + Exception: If storing a key in Key Vault fails. + """ + settings = get_settings() + enable_key_vault_secret_storage = settings.get("enable_key_vault_secret_storage", False) + key_vault_name = settings.get("key_vault_name", None) + if not enable_key_vault_secret_storage or not key_vault_name: + return agent_dict + source = "agent" + updated = dict(agent_dict) + agent_name = updated.get('name', 'agent') + use_apim = updated.get('enable_agent_gpt_apim', False) + key = 'azure_agent_apim_gpt_subscription_key' if use_apim else 'azure_openai_gpt_key' + if key in updated and updated[key]: + value = updated[key] + secret_name = agent_name + if value == ui_trigger_word: + updated[key] = build_full_secret_name(secret_name, scope_value, source, scope) + elif validate_secret_name_dynamic(value): + updated[key] = build_full_secret_name(secret_name, scope_value, source, scope) + else: + try: + full_secret_name = store_secret_in_key_vault(secret_name, value, scope_value, source=source, scope=scope) + updated[key] = full_secret_name + except Exception as e: + logging.error(f"Failed to store agent key '{key}' in Key Vault: {e}") + raise Exception(f"Failed to store agent key '{key}' in Key Vault: {e}") + else: + log_event(f"Agent key '{key}' not found while APIM is '{use_apim}' or empty in agent '{agent_name}'. No action taken.", level="INFO") + return updated + +def keyvault_agent_get_helper(agent_dict, scope_value, scope="global", return_type=SecretReturnType.TRIGGER): + """ + For agent dicts, retrieve sensitive keys from Key Vault if they are stored as Key Vault references. + Only processes 'azure_agent_apim_gpt_subscription_key' and 'azure_openai_gpt_key'. + + Args: + agent_dict (dict): The agent dictionary to process. + scope_value (str): The value for the scope (e.g., agent id). + scope (str): The scope (e.g., 'user', 'global'). + return_actual_key (bool): If True, retrieves the actual secret value from Key Vault. If False, replaces with ui_trigger_word. + + Returns: + dict: A new agent dict with sensitive values replaced by Key Vault references. + Raises: + Exception: If retrieving a key from Key Vault fails. + """ + settings = get_settings() + enable_key_vault_secret_storage = settings.get("enable_key_vault_secret_storage", False) + key_vault_name = settings.get("key_vault_name", None) + if not enable_key_vault_secret_storage or not key_vault_name: + return agent_dict + source = "agent" + updated = dict(agent_dict) + agent_name = updated.get('name', 'agent') + use_apim = updated.get('enable_agent_gpt_apim', False) + key = 'azure_agent_apim_gpt_subscription_key' if use_apim else 'azure_openai_gpt_key' + if key in updated and updated[key]: + value = updated[key] + if validate_secret_name_dynamic(value): + try: + if return_type == SecretReturnType.VALUE: + actual_key = retrieve_secret_from_key_vault_by_full_name(value) + updated[key] = actual_key + elif return_type == SecretReturnType.NAME: + updated[key] = value + else: + updated[key] = ui_trigger_word + except Exception as e: + logging.error(f"Failed to retrieve agent key '{key}' from Key Vault: {e}") + return updated + return updated + +def keyvault_plugin_save_helper(plugin_dict, scope_value, scope="global"): + """ + For plugin dicts, store the auth.key in Key Vault if auth.type is 'key', 'servicePrincipal', 'basic', or 'connection_string', + and replace its value with the Key Vault secret name. Also supports dynamic secret storage for any additionalFields key ending with '__Secret'. + + Args: + plugin_dict (dict): The plugin dictionary to process. + scope_value (str): The value for the scope (e.g., plugin id). + scope (str): The scope (e.g., 'user', 'global'). + + Returns: + dict: A new plugin dict with sensitive values replaced by Key Vault references. + Raises: + Exception: If storing a key in Key Vault fails. + + Feature: + Any key in additionalFields ending with '__Secret' will be stored in Key Vault and replaced with a Key Vault reference. + This allows plugin writers to dynamically store secrets without custom code. + """ + if scope not in supported_scopes: + logging.error(f"Scope '{scope}' is not supported. Supported scopes: {supported_scopes}") + raise ValueError(f"Scope '{scope}' is not supported. Supported scopes: {supported_scopes}") + source = "action" # Use 'action' for plugins per app convention + updated = dict(plugin_dict) + plugin_name = updated.get('name', 'plugin') + auth = updated.get('auth', {}) + if isinstance(auth, dict): + auth_type = auth.get('type', None) + if auth_type in supported_action_auth_types and 'key' in auth and auth['key']: + value = auth['key'] + if value == ui_trigger_word: + auth['key'] = build_full_secret_name(plugin_name, scope_value, source, scope) + updated['auth'] = auth + elif validate_secret_name_dynamic(value): + auth['key'] = build_full_secret_name(plugin_name, scope_value, source, scope) + updated['auth'] = auth + else: + try: + full_secret_name = store_secret_in_key_vault(plugin_name, value, scope_value, source=source, scope=scope) + new_auth = dict(auth) + new_auth['key'] = full_secret_name + updated['auth'] = new_auth + except Exception as e: + logging.error(f"Failed to store plugin key in Key Vault: {e}") + raise Exception(f"Failed to store plugin key in Key Vault: {e}") + else: + print(f"Auth type '{auth_type}' does not require Key Vault storage. Does not match ") + + # Handle additionalFields dynamic secrets + additional_fields = updated.get('additionalFields', {}) + if isinstance(additional_fields, dict): + new_additional_fields = dict(additional_fields) + for k, v in additional_fields.items(): + if k.endswith('__Secret') and v: + addset_source = 'action-addset' + base_field = k[:-8] # Remove '__Secret' + akv_key = f"{plugin_name}-{base_field}".replace('__', '-') + full_secret_name = build_full_secret_name(akv_key, scope_value, addset_source, scope) + if v == ui_trigger_word: + new_additional_fields[k] = full_secret_name + continue + elif validate_secret_name_dynamic(v): + new_additional_fields[k] = full_secret_name + continue + else: + try: + full_secret_name = store_secret_in_key_vault(akv_key, v, scope_value, source=addset_source, scope=scope) + new_additional_fields[k] = full_secret_name + except Exception as e: + logging.error(f"Failed to store plugin additionalField secret '{k}' in Key Vault: {e}") + raise Exception(f"Failed to store plugin additionalField secret '{k}' in Key Vault: {e}") + updated['additionalFields'] = new_additional_fields + return updated +# Helper to retrieve plugin secrets from Key Vault +def keyvault_plugin_get_helper(plugin_dict, scope_value, scope="global", return_type=SecretReturnType.TRIGGER): + """ + For plugin dicts, retrieve secrets from Key Vault for auth.key and any additionalFields key ending with '__Secret'. + If the value is a Key Vault reference, retrieve the actual secret and replace with ui_trigger_word. + + Args: + plugin_dict (dict): The plugin dictionary to process. + scope_value (str): The value for the scope (e.g., plugin id). + scope (str): The scope (e.g., 'user', 'global'). + + Returns: + dict: A new plugin dict with sensitive values replaced by ui_trigger_word if stored in Key Vault. + """ + if scope not in supported_scopes: + logging.error(f"Scope '{scope}' is not supported. Supported scopes: {supported_scopes}") + raise ValueError(f"Scope '{scope}' is not supported. Supported scopes: {supported_scopes}") + source = "action" + updated = dict(plugin_dict) + plugin_name = updated.get('name', 'plugin') + auth = updated.get('auth', {}) + if isinstance(auth, dict): + if 'key' in auth and auth['key']: + value = auth['key'] + if validate_secret_name_dynamic(value): + try: + if return_type == SecretReturnType.VALUE: + actual_key = retrieve_secret_from_key_vault_by_full_name(value) + new_auth = dict(auth) + new_auth['key'] = actual_key + updated['auth'] = new_auth + elif return_type == SecretReturnType.NAME: + new_auth = dict(auth) + new_auth['key'] = value + updated['auth'] = new_auth + else: + new_auth = dict(auth) + new_auth['key'] = ui_trigger_word + updated['auth'] = new_auth + except Exception as e: + logging.error(f"Failed to retrieve action key from Key Vault: {e}") + raise Exception(f"Failed to retrieve action key from Key Vault: {e}") + + additional_fields = updated.get('additionalFields', {}) + if isinstance(additional_fields, dict): + new_additional_fields = dict(additional_fields) + for k, v in additional_fields.items(): + if k.endswith('__Secret') and v and validate_secret_name_dynamic(v): + addset_source = 'action-addset' + base_field = k[:-8] # Remove '__Secret' + akv_key = f"{plugin_name}-{base_field}".replace('__', '-') + try: + if return_type == SecretReturnType.VALUE: + actual_secret = retrieve_secret_from_key_vault(f"{akv_key}", scope_value, scope, addset_source) + new_additional_fields[k] = actual_secret + elif return_type == SecretReturnType.NAME: + new_additional_fields[k] = v + else: + new_additional_fields[k] = ui_trigger_word + except Exception as e: + logging.error(f"Failed to retrieve action additionalField secret '{k}' from Key Vault: {e}") + raise Exception(f"Failed to retrieve action additionalField secret '{k}' from Key Vault: {e}") + updated['additionalFields'] = new_additional_fields + return updated +# Helper to delete plugin secrets from Key Vault +def keyvault_plugin_delete_helper(plugin_dict, scope_value, scope="global"): + """ + For plugin dicts, delete secrets from Key Vault for auth.key and any additionalFields key ending with '__Secret'. + Only deletes if the value is a Key Vault reference. + + Args: + plugin_dict (dict): The plugin dictionary to process. + scope_value (str): The value for the scope (e.g., plugin id). + scope (str): The scope (e.g., 'user', 'global'). + + Returns: + plugin_dict (dict): The original plugin dict. + Raises: + """ + if scope not in supported_scopes: + log_event(f"Scope '{scope}' is not supported. Supported scopes: {supported_scopes}", level="WARNING") + raise ValueError(f"Scope '{scope}' is not supported. Supported scopes: {supported_scopes}") + settings = get_settings() + enable_key_vault_secret_storage = settings.get("enable_key_vault_secret_storage", False) + key_vault_name = settings.get("key_vault_name", None) + if not enable_key_vault_secret_storage or not key_vault_name: + log_event(f"Key Vault secret storage is not enabled or key vault name is missing.", level="WARNING") + return plugin_dict + source = "action" + plugin_name = plugin_dict.get('name', 'plugin') + auth = plugin_dict.get('auth', {}) + if isinstance(auth, dict): + if 'key' in auth and auth['key']: + secret_name = auth['key'] + if validate_secret_name_dynamic(secret_name): + try: + key_vault_url = f"https://{key_vault_name}{KEY_VAULT_DOMAIN}" + log_event(f"Deleting action secret '{secret_name}' for action '{plugin_name}' for '{scope}' '{scope_value}'", level="INFO") + client = SecretClient(vault_url=key_vault_url, credential=get_keyvault_credential()) + client.begin_delete_secret(secret_name) + except Exception as e: + logging.error(f"Error deleting action secret '{secret_name}' for action '{plugin_name}': {e}") + raise Exception(f"Error deleting action secret '{secret_name}' for action '{plugin_name}': {e}") + + additional_fields = plugin_dict.get('additionalFields', {}) + if isinstance(additional_fields, dict): + for k, v in additional_fields.items(): + if k.endswith('__Secret') and v and validate_secret_name_dynamic(v): + addset_source = 'action-addset' + base_field = k[:-8] # Remove '__Secret' + akv_key = f"{plugin_name}-{base_field}".replace('__', '-') + try: + keyvault_secret_name = build_full_secret_name(akv_key, scope_value, addset_source, scope) + key_vault_url = f"https://{key_vault_name}{KEY_VAULT_DOMAIN}" + log_event(f"Deleting action additionalField secret '{k}' for action '{plugin_name}' for '{scope}' '{scope_value}'", level="INFO") + client = SecretClient(vault_url=key_vault_url, credential=get_keyvault_credential()) + client.begin_delete_secret(keyvault_secret_name) + except Exception as e: + logging.error(f"Error deleting action additionalField secret '{k}' for action '{plugin_name}': {e}") + raise Exception(f"Error deleting action additionalField secret '{k}' for action '{plugin_name}': {e}") + return plugin_dict + +# Helper to delete agent secrets from Key Vault +def keyvault_agent_delete_helper(agent_dict, scope_value, scope="global"): + """ + For agent dicts, delete sensitive keys from Key Vault if they are stored as Key Vault references. + Only processes 'azure_agent_apim_gpt_subscription_key' and 'azure_openai_gpt_key'. + + Args: + agent_dict (dict): The agent dictionary to process. + scope_value (str): The value for the scope (e.g., agent id). + scope (str): The scope (e.g., 'user', 'global'). + + Returns: + agent_dict (dict): The original agent dict. + """ + settings = get_settings() + enable_key_vault_secret_storage = settings.get("enable_key_vault_secret_storage", False) + key_vault_name = settings.get("key_vault_name", None) + if not enable_key_vault_secret_storage or not key_vault_name: + return agent_dict + source = "agent" + updated = dict(agent_dict) + agent_name = updated.get('name', 'agent') + use_apim = updated.get('enable_agent_gpt_apim', False) + keys = ['azure_agent_apim_gpt_subscription_key'] if use_apim else ['azure_openai_gpt_key'] + for key in keys: + if key in updated and updated[key]: + secret_name = updated[key] + if validate_secret_name_dynamic(secret_name): + try: + key_vault_url = f"https://{key_vault_name}{KEY_VAULT_DOMAIN}" + log_event(f"Deleting agent secret '{secret_name}' for agent '{agent_name}' for '{scope}' '{scope_value}'", level="INFO") + client = SecretClient(vault_url=key_vault_url, credential=get_keyvault_credential()) + client.begin_delete_secret(secret_name) + except Exception as e: + logging.error(f"Error deleting secret '{secret_name}' for agent '{agent_name}': {e}") + raise Exception(f"Error deleting secret '{secret_name}' for agent '{agent_name}': {e}") + return agent_dict + +def get_keyvault_credential(): + """ + Get the Key Vault credential using DefaultAzureCredential, optionally with a managed identity client ID. + + Returns: + DefaultAzureCredential: The credential object for Key Vault access. + """ + settings = get_settings() + key_vault_identity = settings.get("key_vault_identity", None) + if key_vault_identity is not None: + credential = DefaultAzureCredential(managed_identity_client_id=key_vault_identity) + else: + credential = DefaultAzureCredential() + return credential + +def clean_name_for_keyvault(name): + """ + Clean a name to be used as a Key Vault secret name by removing invalid characters and truncating to 127 characters. + + Args: + name (str): The name to clean. + + Returns: + str: The cleaned name. + """ + # Remove invalid characters + cleaned_name = re.sub(r"[^a-zA-Z0-9-]", "-", name) + # Truncate to 127 characters + return cleaned_name[:127] \ No newline at end of file diff --git a/application/single_app/functions_personal_actions.py b/application/single_app/functions_personal_actions.py index b9cbaa64..108d3151 100644 --- a/application/single_app/functions_personal_actions.py +++ b/application/single_app/functions_personal_actions.py @@ -11,9 +11,12 @@ from datetime import datetime from azure.cosmos import exceptions from flask import current_app +from functions_keyvault import keyvault_plugin_save_helper, keyvault_plugin_get_helper, keyvault_plugin_delete_helper, SecretReturnType +from functions_settings import get_user_settings, update_user_settings +from config import cosmos_personal_actions_container import logging -def get_personal_actions(user_id): +def get_personal_actions(user_id, return_type=SecretReturnType.TRIGGER): """ Fetch all personal actions/plugins for a user. @@ -24,8 +27,6 @@ def get_personal_actions(user_id): list: List of action/plugin dictionaries """ try: - from config import cosmos_personal_actions_container - query = "SELECT * FROM c WHERE c.user_id = @user_id" parameters = [{"name": "@user_id", "value": user_id}] @@ -35,12 +36,12 @@ def get_personal_actions(user_id): partition_key=user_id )) - # Remove Cosmos metadata for cleaner response + # Remove Cosmos metadata for cleaner response and resolve Key Vault references cleaned_actions = [] for action in actions: cleaned_action = {k: v for k, v in action.items() if not k.startswith('_')} + cleaned_action = keyvault_plugin_get_helper(cleaned_action, scope_value=user_id, scope="user", return_type=return_type) cleaned_actions.append(cleaned_action) - return cleaned_actions except exceptions.CosmosResourceNotFoundError: @@ -49,7 +50,7 @@ def get_personal_actions(user_id): current_app.logger.error(f"Error fetching personal actions for user {user_id}: {e}") return [] -def get_personal_action(user_id, action_id): +def get_personal_action(user_id, action_id, return_type=SecretReturnType.TRIGGER): """ Fetch a specific personal action/plugin. @@ -61,9 +62,6 @@ def get_personal_action(user_id, action_id): dict: Action dictionary or None if not found """ try: - from config import cosmos_personal_actions_container - - # Try to find by ID first try: action = cosmos_personal_actions_container.read_item( item=action_id, @@ -87,8 +85,9 @@ def get_personal_action(user_id, action_id): return None action = actions[0] - # Remove Cosmos metadata + # Remove Cosmos metadata and resolve Key Vault references cleaned_action = {k: v for k, v in action.items() if not k.startswith('_')} + cleaned_action = keyvault_plugin_get_helper(cleaned_action, scope_value=user_id, scope="user", return_type=return_type) return cleaned_action except Exception as e: @@ -107,8 +106,6 @@ def save_personal_action(user_id, action_data): dict: Saved action data with ID """ try: - from config import cosmos_personal_actions_container - # Check if an action with this name already exists existing_action = None if 'name' in action_data and action_data['name']: @@ -146,8 +143,9 @@ def save_personal_action(user_id, action_data): elif 'type' not in action_data['auth']: action_data['auth']['type'] = 'identity' + # Store secrets in Key Vault before upsert + action_data = keyvault_plugin_save_helper(action_data, scope_value=user_id, scope="user") result = cosmos_personal_actions_container.upsert_item(body=action_data) - # Remove Cosmos metadata from response cleaned_result = {k: v for k, v in result.items() if not k.startswith('_')} return cleaned_result @@ -168,13 +166,13 @@ def delete_personal_action(user_id, action_id): bool: True if deleted, False if not found """ try: - from config import cosmos_personal_actions_container - # Try to find the action first to get the correct ID action = get_personal_action(user_id, action_id) if not action: return False + # Delete secrets from Key Vault before deleting the action + keyvault_plugin_delete_helper(action, scope_value=user_id, scope="user") cosmos_personal_actions_container.delete_item( item=action['id'], partition_key=user_id @@ -199,8 +197,6 @@ def ensure_migration_complete(user_id): int: Number of actions migrated (0 if already migrated) """ try: - from functions_settings import get_user_settings, update_user_settings - user_settings = get_user_settings(user_id) plugins = user_settings.get('settings', {}).get('plugins', []) @@ -237,8 +233,6 @@ def migrate_actions_from_user_settings(user_id): int: Number of actions migrated """ try: - from functions_settings import get_user_settings, update_user_settings - user_settings = get_user_settings(user_id) plugins = user_settings.get('settings', {}).get('plugins', []) @@ -253,14 +247,13 @@ def migrate_actions_from_user_settings(user_id): if plugin.get('name') in existing_action_names: current_app.logger.info(f"Skipping migration of plugin '{plugin.get('name')}' - already exists") continue - # Ensure plugin has an ID (generate GUID if missing) if 'id' not in plugin or not plugin['id']: plugin['id'] = str(uuid.uuid4()) - + # Store secrets in Key Vault before migration + plugin = keyvault_plugin_save_helper(plugin, scope_value=user_id, scope="user") save_personal_action(user_id, plugin) migrated_count += 1 - except Exception as e: current_app.logger.error(f"Error migrating plugin {plugin.get('name', 'unknown')} for user {user_id}: {e}") @@ -276,7 +269,7 @@ def migrate_actions_from_user_settings(user_id): current_app.logger.error(f"Error during action migration for user {user_id}: {e}") return 0 -def get_actions_by_names(user_id, action_names): +def get_actions_by_names(user_id, action_names, return_type=SecretReturnType.TRIGGER): """ Get multiple actions by their names. @@ -288,8 +281,6 @@ def get_actions_by_names(user_id, action_names): list: List of action dictionaries """ try: - from config import cosmos_personal_actions_container - if not action_names: return [] @@ -311,6 +302,7 @@ def get_actions_by_names(user_id, action_names): cleaned_actions = [] for action in actions: cleaned_action = {k: v for k, v in action.items() if not k.startswith('_')} + cleaned_action = keyvault_plugin_get_helper(cleaned_action, scope_value=user_id, scope="user", return_type=return_type) cleaned_actions.append(cleaned_action) return cleaned_actions @@ -319,7 +311,7 @@ def get_actions_by_names(user_id, action_names): current_app.logger.error(f"Error fetching actions by names for user {user_id}: {e}") return [] -def get_actions_by_type(user_id, action_type): +def get_actions_by_type(user_id, action_type, return_type=SecretReturnType.TRIGGER): """ Get all actions of a specific type for a user. @@ -331,8 +323,6 @@ def get_actions_by_type(user_id, action_type): list: List of action dictionaries """ try: - from config import cosmos_personal_actions_container - query = "SELECT * FROM c WHERE c.user_id = @user_id AND c.type = @type" parameters = [ {"name": "@user_id", "value": user_id}, @@ -349,6 +339,7 @@ def get_actions_by_type(user_id, action_type): cleaned_actions = [] for action in actions: cleaned_action = {k: v for k, v in action.items() if not k.startswith('_')} + cleaned_action = keyvault_plugin_get_helper(cleaned_action, scope_value=user_id, scope="user", return_type=return_type) cleaned_actions.append(cleaned_action) return cleaned_actions diff --git a/application/single_app/functions_personal_agents.py b/application/single_app/functions_personal_agents.py index 6cca4aa1..4e04e624 100644 --- a/application/single_app/functions_personal_agents.py +++ b/application/single_app/functions_personal_agents.py @@ -1,17 +1,23 @@ + # functions_personal_agents.py """ Personal Agents Management -This module handles all operations related to personal agents stored in the +This module handles all operations related to personal agents stored in the personal_agents container with user_id partitioning. """ + +# Imports (grouped after docstring) import uuid from datetime import datetime from azure.cosmos import exceptions from flask import current_app import logging +from config import cosmos_personal_agents_container +from functions_settings import get_settings, get_user_settings, update_user_settings +from functions_keyvault import keyvault_agent_save_helper, keyvault_agent_get_helper, keyvault_agent_delete_helper def get_personal_agents(user_id): """ @@ -24,8 +30,6 @@ def get_personal_agents(user_id): list: List of agent dictionaries """ try: - from config import cosmos_personal_agents_container - query = "SELECT * FROM c WHERE c.user_id = @user_id" parameters = [{"name": "@user_id", "value": user_id}] @@ -35,12 +39,12 @@ def get_personal_agents(user_id): partition_key=user_id )) - # Remove Cosmos metadata for cleaner response + # Remove Cosmos metadata for cleaner response and retrieve secrets from Key Vault cleaned_agents = [] for agent in agents: cleaned_agent = {k: v for k, v in agent.items() if not k.startswith('_')} + cleaned_agent = keyvault_agent_get_helper(cleaned_agent, cleaned_agent.get('id', ''), scope="user") cleaned_agents.append(cleaned_agent) - return cleaned_agents except exceptions.CosmosResourceNotFoundError: @@ -61,18 +65,17 @@ def get_personal_agent(user_id, agent_id): dict: Agent dictionary or None if not found """ try: - from config import cosmos_personal_agents_container - agent = cosmos_personal_agents_container.read_item( item=agent_id, partition_key=user_id ) - # Remove Cosmos metadata + # Remove Cosmos metadata and retrieve secrets from Key Vault cleaned_agent = {k: v for k, v in agent.items() if not k.startswith('_')} + cleaned_agent = keyvault_agent_get_helper(cleaned_agent, cleaned_agent.get('id', agent_id), scope="user") return cleaned_agent - except exceptions.CosmosResourceNotFoundError: + current_app.logger.warning(f"Agent {agent_id} not found for user {user_id}") return None except Exception as e: current_app.logger.error(f"Error fetching agent {agent_id} for user {user_id}: {e}") @@ -90,8 +93,6 @@ def save_personal_agent(user_id, agent_data): dict: Saved agent data with ID """ try: - from config import cosmos_personal_agents_container - # Ensure required fields if 'id' not in agent_data: agent_data['id'] = str(f"{user_id}_{agent_data.get('name', 'default')}") @@ -115,8 +116,9 @@ def save_personal_agent(user_id, agent_data): agent_data.setdefault('other_settings', {}) agent_data.setdefault('is_global', False) + # Store sensitive keys in Key Vault if enabled + agent_data = keyvault_agent_save_helper(agent_data, agent_data.get('id', ''), scope="user") result = cosmos_personal_agents_container.upsert_item(body=agent_data) - # Remove Cosmos metadata from response cleaned_result = {k: v for k, v in result.items() if not k.startswith('_')} return cleaned_result @@ -137,8 +139,6 @@ def delete_personal_agent(user_id, agent_id): bool: True if deleted, False if not found """ try: - from config import cosmos_personal_agents_container - # Try to find the agent first to get the correct ID # Check if agent_id is actually a name and we need to find the real ID agent = get_personal_agent(user_id, agent_id) @@ -146,17 +146,17 @@ def delete_personal_agent(user_id, agent_id): # Try to find by name if direct ID lookup failed agents = get_personal_agents(user_id) agent = next((a for a in agents if a['name'] == agent_id), None) - if not agent: return False - + # Delete secrets from Key Vault if present + keyvault_agent_delete_helper(agent, agent.get('id', agent_id), scope="user") cosmos_personal_agents_container.delete_item( item=agent['id'], partition_key=user_id ) return True - except exceptions.CosmosResourceNotFoundError: + current_app.logger.warning(f"Agent {agent_id} not found for user {user_id}") return False except Exception as e: current_app.logger.error(f"Error deleting agent {agent_id} for user {user_id}: {e}") @@ -174,8 +174,6 @@ def ensure_migration_complete(user_id): int: Number of agents migrated (0 if already migrated) """ try: - from functions_settings import get_user_settings, update_user_settings - user_settings = get_user_settings(user_id) agents = user_settings.get('settings', {}).get('agents', []) @@ -212,15 +210,11 @@ def migrate_agents_from_user_settings(user_id): int: Number of agents migrated """ try: - from functions_settings import get_user_settings, update_user_settings - user_settings = get_user_settings(user_id) agents = user_settings.get('settings', {}).get('agents', []) - # Get existing personal agents to avoid duplicates existing_personal_agents = get_personal_agents(user_id) existing_agent_names = {agent['name'] for agent in existing_personal_agents} - migrated_count = 0 for agent in agents: try: @@ -228,77 +222,20 @@ def migrate_agents_from_user_settings(user_id): if agent.get('name') in existing_agent_names: current_app.logger.info(f"Skipping migration of agent '{agent.get('name')}' - already exists") continue - # Ensure agent has an ID if 'id' not in agent: agent['id'] = str(uuid.uuid4()) - save_personal_agent(user_id, agent) migrated_count += 1 - except Exception as e: current_app.logger.error(f"Error migrating agent {agent.get('name', 'unknown')} for user {user_id}: {e}") - # Always remove agents from user settings after processing (even if no new ones migrated) settings_to_update = user_settings.get('settings', {}) settings_to_update['agents'] = [] # Set to empty array instead of removing update_user_settings(user_id, settings_to_update) - current_app.logger.info(f"Migrated {migrated_count} new agents for user {user_id}, cleaned up legacy data") return migrated_count - except Exception as e: current_app.logger.error(f"Error during agent migration for user {user_id}: {e}") return 0 -def get_selected_agent(user_id): - """ - Get the user's selected agent preference. - - Args: - user_id (str): The user's unique identifier - - Returns: - dict: Selected agent info or None - """ - try: - from functions_settings import get_user_settings - - user_settings = get_user_settings(user_id) - selected_agent = user_settings.get('settings', {}).get('selected_agent') - - return selected_agent - - except Exception as e: - current_app.logger.error(f"Error getting selected agent for user {user_id}: {e}") - return None - -def set_selected_agent(user_id, agent_name, is_global=False): - """ - Set the user's selected agent preference. - - Args: - user_id (str): The user's unique identifier - agent_name (str): Name of the selected agent - is_global (bool): Whether the agent is global or personal - - Returns: - bool: True if successful - """ - try: - from functions_settings import get_user_settings, update_user_settings - - user_settings = get_user_settings(user_id) - settings_to_update = user_settings.get('settings', {}) - - settings_to_update['selected_agent'] = { - 'name': agent_name, - 'is_global': is_global - } - - update_user_settings(user_id, settings_to_update) - return True - - except Exception as e: - current_app.logger.error(f"Error setting selected agent for user {user_id}: {e}") - return False diff --git a/application/single_app/functions_personal_agents_plugins.py b/application/single_app/functions_personal_agents_plugins.py deleted file mode 100644 index e69de29b..00000000 diff --git a/application/single_app/functions_settings.py b/application/single_app/functions_settings.py index 712a8d1c..d1bd4752 100644 --- a/application/single_app/functions_settings.py +++ b/application/single_app/functions_settings.py @@ -226,7 +226,12 @@ def get_settings(): "speech_service_endpoint": '', "speech_service_location": '', "speech_service_locale": "en-US", - "speech_service_key": "" + "speech_service_key": "", + + #key vault settings + 'enable_key_vault_secret_storage': False, + 'key_vault_name': '', + 'key_vault_identity': '', } try: diff --git a/application/single_app/json_schema_validation.py b/application/single_app/json_schema_validation.py index 4cda4da2..c7c58a3c 100644 --- a/application/single_app/json_schema_validation.py +++ b/application/single_app/json_schema_validation.py @@ -43,7 +43,7 @@ def validate_plugin(plugin): validator = Draft7Validator(schema['definitions']['Plugin']) errors = sorted(validator.iter_errors(plugin_copy), key=lambda e: e.path) if errors: - return '; '.join([e.message for e in errors]) + return '; '.join([f"{plugin.get('name', '')}: {e.message}" for e in errors]) # Additional business logic validation # For non-SQL plugins, endpoint must not be empty diff --git a/application/single_app/requirements.txt b/application/single_app/requirements.txt index 4acd2326..09bd3e8f 100644 --- a/application/single_app/requirements.txt +++ b/application/single_app/requirements.txt @@ -30,6 +30,7 @@ azure-identity==1.23.0 azure-ai-contentsafety==1.0.0 azure-storage-blob==12.24.1 azure-storage-queue==12.12.0 +azure-keyvault-secrets==4.10.0 pypdf==6.0.0 python-docx==1.1.2 flask-executor==1.0.0 diff --git a/application/single_app/route_backend_agents.py b/application/single_app/route_backend_agents.py index 2af1d8df..c5229bb1 100644 --- a/application/single_app/route_backend_agents.py +++ b/application/single_app/route_backend_agents.py @@ -8,6 +8,7 @@ from semantic_kernel_loader import get_agent_orchestration_types from functions_settings import get_settings, update_settings, get_user_settings, update_user_settings from functions_global_agents import get_global_agents, save_global_agent, delete_global_agent +from functions_personal_agents import get_personal_agents, ensure_migration_complete, save_personal_agent, delete_personal_agent from functions_authentication import * from functions_appinsights import log_event from json_schema_validation import validate_agent @@ -33,10 +34,6 @@ def generate_agent_id(): @login_required def get_user_agents(): user_id = get_current_user_id() - - # Import the new personal agents functions - from functions_personal_agents import get_personal_agents, ensure_migration_complete - # Ensure migration is complete (will migrate any remaining legacy data) ensure_migration_complete(user_id) @@ -53,7 +50,6 @@ def get_user_agents(): merge_global = settings.get('merge_global_semantic_kernel_with_workspace', False) if per_user and merge_global: # Import and get global agents from container - from functions_global_agents import get_global_agents global_agents = get_global_agents() # Mark global agents for agent in global_agents: @@ -87,10 +83,6 @@ def set_user_agents(): user_id = get_current_user_id() agents = request.json if isinstance(request.json, list) else [] settings = get_settings() - - # Import the new personal agents functions - from functions_personal_agents import save_personal_agent, delete_personal_agent, get_personal_agents - # If custom endpoints are not allowed, strip deployment settings for endpoint, key, and api-revision if not settings.get('allow_user_custom_agent_endpoints', False): for agent in agents: @@ -151,10 +143,6 @@ def set_user_agents(): @login_required def delete_user_agent(agent_name): user_id = get_current_user_id() - - # Import the new personal agents functions - from functions_personal_agents import get_personal_agents, delete_personal_agent - # Get current agents from personal_agents container agents = get_personal_agents(user_id) agent_to_delete = next((a for a in agents if a['name'] == agent_name), None) @@ -236,7 +224,6 @@ def set_selected_agent(): return jsonify({'error': 'Agent name is required.'}), 400 # Import and get global agents from container - from functions_global_agents import get_global_agents agents = get_global_agents() # Check that the agent exists @@ -266,8 +253,6 @@ def set_selected_agent(): def list_agents(): try: # Use new global agents container - from functions_global_agents import get_global_agents - agents = get_global_agents() # Ensure each agent has an actions_to_load field @@ -400,8 +385,6 @@ def update_agent_setting(setting_name): @admin_required def edit_agent(agent_name): try: - from functions_global_agents import get_global_agents, save_global_agent - agents = get_global_agents() updated_agent = request.json.copy() if hasattr(request.json, 'copy') else dict(request.json) updated_agent['is_global'] = True @@ -465,8 +448,6 @@ def edit_agent(agent_name): @admin_required def delete_agent(agent_name): try: - from functions_global_agents import get_global_agents, delete_global_agent - agents = get_global_agents() # Find the agent to delete @@ -550,9 +531,7 @@ def orchestration_settings(): log_event(f"Error updating orchestration settings: {e}", level=logging.ERROR, exceptionTraceback=True) return jsonify({'error': 'Failed to update orchestration settings.'}), 500 -def get_global_agent_settings(include_admin_extras=False): - from functions_global_agents import get_global_agents - +def get_global_agent_settings(include_admin_extras=False): settings = get_settings() agents = get_global_agents() diff --git a/application/single_app/route_backend_chats.py b/application/single_app/route_backend_chats.py index 8e6aa196..fb1d14a5 100644 --- a/application/single_app/route_backend_chats.py +++ b/application/single_app/route_backend_chats.py @@ -1328,6 +1328,7 @@ async def run_sk_call(callable_obj, *args, **kwargs): per_user_semantic_kernel = settings.get('per_user_semantic_kernel', False) enable_semantic_kernel = settings.get('enable_semantic_kernel', False) user_enable_agents = user_settings.get('enable_agents', True) # Default to True for backward compatibility + enable_key_vault_secret_storage = settings.get('enable_key_vault_secret_storage', False) redis_client = None # --- Semantic Kernel state management (per-user mode) --- if enable_semantic_kernel and per_user_semantic_kernel: diff --git a/application/single_app/route_backend_plugins.py b/application/single_app/route_backend_plugins.py index 3ece6a8c..5edcba1b 100644 --- a/application/single_app/route_backend_plugins.py +++ b/application/single_app/route_backend_plugins.py @@ -18,8 +18,9 @@ from functions_global_actions import * from functions_personal_actions import * +#from functions_personal_actions import delete_personal_action - +from functions_debug import debug_print from json_schema_validation import validate_plugin def discover_plugin_types(): @@ -109,6 +110,7 @@ def get_plugin_types(): safe_manifest = {} # Only add minimal required fields based on plugin type + #TODO: This can be improved by ensuring we have additional fields from the schemas we have not created if needed. if 'databricks' in module_name.lower(): safe_manifest = { 'endpoint': 'https://example.databricks.com', @@ -151,12 +153,15 @@ def get_plugin_types(): try: plugin_instance = obj(safe_manifest) except (TypeError, ValueError, KeyError) as e: + debug_print(f"[RBEP] Failed to instantiate {attr} with safe manifest: {e}") try: plugin_instance = obj({}) except (TypeError, ValueError) as e2: + debug_print(f"[RBEP] Failed to instantiate {attr} with empty manifest: {e2}") try: plugin_instance = obj() except Exception as e3: + debug_print(f"[RBEP] Failed to instantiate {attr} with no args: {e3}") instantiation_error = e3 except Exception as e: instantiation_error = e @@ -288,6 +293,7 @@ def set_user_plugins(): plugin.setdefault('endpoint', f'sql://{plugin_type}') elif plugin_type == 'msgraph': # MS Graph plugin does not require an endpoint, but schema validation requires one + #TODO: Update to support different clouds plugin.setdefault('endpoint', 'https://graph.microsoft.com') else: # For other plugin types, require a real endpoint @@ -338,9 +344,6 @@ def set_user_plugins(): def delete_user_plugin(plugin_name): user_id = get_current_user_id() - # Import the new personal actions functions - from functions_personal_actions import delete_personal_action - # Try to delete from personal_actions container deleted = delete_personal_action(user_id, plugin_name) diff --git a/application/single_app/route_backend_settings.py b/application/single_app/route_backend_settings.py index bf1aec86..68e9ccaa 100644 --- a/application/single_app/route_backend_settings.py +++ b/application/single_app/route_backend_settings.py @@ -4,6 +4,9 @@ from functions_documents import * from functions_authentication import * from functions_settings import * +from functions_appinsights import log_event +from azure.identity import DefaultAzureCredential +from azure.keyvault.secrets import SecretClient from swagger_wrapper import swagger_route, get_auth_security import redis @@ -279,6 +282,9 @@ def test_connection(): elif test_type == 'chunking_api': # If you have a chunking API test, implement it here. return jsonify({'message': 'Chunking API connection successful'}), 200 + + elif test_type == 'key_vault': + return _test_key_vault_connection(data) else: return jsonify({'error': f'Unknown test_type: {test_type}'}), 400 @@ -693,3 +699,35 @@ def _test_azure_doc_intelligence_connection(payload): return jsonify({'message': 'Azure document intelligence connection successful'}), 200 else: return jsonify({'error': f"Document Intelligence error: {status}"}), 500 + +def _test_key_vault_connection(payload): + """Attempt to connect to Azure Key Vault using ephemeral settings.""" + vault_name = payload.get('vault_name', '').strip() + client_id = payload.get('client_id', '').strip() + + if not vault_name: + return jsonify({'error': 'Key Vault name is required'}), 400 + + try: + vault_url = f"https://{vault_name}{KEY_VAULT_DOMAIN}" + + if client_id: + credential = DefaultAzureCredential(managed_identity_client_id=client_id) + else: + credential = DefaultAzureCredential() + + if AZURE_ENVIRONMENT == "custom": + #TODO: Needs to be tested with a custom environment + kv_client = SecretClient(vault_url=vault_url, credential=credential) + else: + kv_client = SecretClient(vault_url=vault_url, credential=credential) + + # Perform a simple list operation to verify connectivity + secrets = kv_client.list_properties_of_secrets() + _ = next(secrets, None) # Attempt to get the first secret (if any) + + return jsonify({'message': 'Key Vault connection successful'}), 200 + + except Exception as e: + log_event(f"[AKV_TEST] Key Vault connection error: {str(e)}", level="error") + return jsonify({'error': f'Key Vault connection error. Check Application Insights using "[AKV_TEST]" for details.'}), 500 \ No newline at end of file diff --git a/application/single_app/route_frontend_admin_settings.py b/application/single_app/route_frontend_admin_settings.py index 17e46e0b..46db34a7 100644 --- a/application/single_app/route_frontend_admin_settings.py +++ b/application/single_app/route_frontend_admin_settings.py @@ -191,6 +191,14 @@ def admin_settings(): if 'classification_banner_color' not in settings: settings['classification_banner_color'] = '#ffc107' # Bootstrap warning color + # --- Add defaults for key vault + if 'enable_key_vault_secret_storage' not in settings: + settings['enable_key_vault_secret_storage'] = False + if 'key_vault_name' not in settings: + settings['key_vault_name'] = '' + if 'key_vault_identity' not in settings: + settings['key_vault_identity'] = '' + # --- Add defaults for left nav --- if 'enable_left_nav_default' not in settings: settings['enable_left_nav_default'] = True @@ -760,6 +768,10 @@ def is_valid_url(url): 'azure_apim_document_intelligence_endpoint': form_data.get('azure_apim_document_intelligence_endpoint', '').strip(), 'azure_apim_document_intelligence_subscription_key': form_data.get('azure_apim_document_intelligence_subscription_key', '').strip(), + 'enable_key_vault_secret_storage': form_data.get('enable_key_vault_secret_storage') == 'on', + 'key_vault_name': form_data.get('key_vault_name', '').strip(), + 'key_vault_identity': form_data.get('key_vault_identity', ''), + # Authentication & Redirect Settings 'enable_front_door': enable_front_door, 'front_door_url': front_door_url, diff --git a/application/single_app/route_frontend_authentication.py b/application/single_app/route_frontend_authentication.py index 395e757a..a208168e 100644 --- a/application/single_app/route_frontend_authentication.py +++ b/application/single_app/route_frontend_authentication.py @@ -123,7 +123,6 @@ def authorized(): # Store user identity info (claims from ID token) debug_print(f" [claims] User {result.get('id_token_claims', {}).get('name', 'Unknown')} logged in.") debug_print(f" [claims] User claims: {result.get('id_token_claims', {})}") - debug_print(f" [claims] User token: {result.get('access_token', 'Unknown')}") session["user"] = result.get("id_token_claims") diff --git a/application/single_app/semantic_kernel_loader.py b/application/single_app/semantic_kernel_loader.py index daadc587..d248c5e6 100644 --- a/application/single_app/semantic_kernel_loader.py +++ b/application/single_app/semantic_kernel_loader.py @@ -5,6 +5,12 @@ - Registers plugins with the Semantic Kernel instance """ +import logging +import importlib +import os +import importlib.util +import inspect +import builtins from agent_orchestrator_groupchat import OrchestratorAgent, SCGroupChatManager from semantic_kernel import Kernel from semantic_kernel.agents import Agent @@ -22,14 +28,18 @@ from semantic_kernel_plugins.plugin_health_checker import PluginHealthChecker, PluginErrorRecovery from semantic_kernel_plugins.logged_plugin_loader import create_logged_plugin_loader from semantic_kernel_plugins.plugin_invocation_logger import get_plugin_logger +from semantic_kernel_plugins.smart_http_plugin import SmartHttpPlugin from functions_debug import debug_print from flask import g -import logging -import importlib -import os -import importlib.util -import inspect -import builtins +from functions_keyvault import validate_secret_name_dynamic, retrieve_secret_from_key_vault, retrieve_secret_from_key_vault_by_full_name, SecretReturnType +from functions_global_actions import get_global_actions +from functions_global_agents import get_global_agents +from functions_personal_actions import get_personal_actions, ensure_migration_complete as ensure_actions_migration_complete +from functions_personal_agents import get_personal_agents, ensure_migration_complete as ensure_agents_migration_complete +from semantic_kernel_plugins.plugin_loader import discover_plugins +from semantic_kernel_plugins.openapi_plugin_factory import OpenApiPluginFactory + + # Agent and Azure OpenAI chat service imports log_event("[SK Loader] Starting loader imports") @@ -107,6 +117,11 @@ def resolve_agent_config(agent, settings): debug_print(f"[SK Loader] user_apim_enabled: {user_apim_enabled}, global_apim_enabled: {global_apim_enabled}, per_user_enabled: {per_user_enabled}") + def resolve_secret_value_if_needed(value, scope_value, source, scope): + if validate_secret_name_dynamic(value): + return retrieve_secret_from_key_vault(value, scope_value, scope, source) + return value + def any_filled(*fields): return any(bool(f) for f in fields) @@ -114,36 +129,88 @@ def all_filled(*fields): return all(bool(f) for f in fields) def get_user_apim(): - return ( - agent.get("azure_apim_gpt_endpoint"), - agent.get("azure_apim_gpt_subscription_key"), - agent.get("azure_apim_gpt_deployment"), - agent.get("azure_apim_gpt_api_version") - ) + endpoint = agent.get("azure_apim_gpt_endpoint") + key = agent.get("azure_apim_gpt_subscription_key") + deployment = agent.get("azure_apim_gpt_deployment") + api_version = agent.get("azure_apim_gpt_api_version") + + # Check if key vault secret storage is enabled in settings + if settings.get("enable_key_vault_secret_storage", False) and settings.get("key_vault_name") and key: + try: + if validate_secret_name_dynamic(key): + # Try to retrieve the secret from Key Vault + resolved_key = retrieve_secret_from_key_vault_by_full_name(key) + if resolved_key: + # Update the agent dict with the resolved key for this session + agent["azure_apim_gpt_subscription_key"] = resolved_key + key = resolved_key + except Exception as e: + log_event(f"[SK Loader] Failed to resolve Key Vault secret for agent '{agent.get('name')}' in get_user_apim: {e}", level=logging.ERROR, exceptionTraceback=True) + # Fallback to using the value as-is + return (endpoint, key, deployment, api_version) def get_global_apim(): - return ( - settings.get("azure_apim_gpt_endpoint"), - settings.get("azure_apim_gpt_subscription_key"), - first_if_comma(settings.get("azure_apim_gpt_deployment")), - settings.get("azure_apim_gpt_api_version") - ) + endpoint = settings.get("azure_apim_gpt_endpoint") + key = settings.get("azure_apim_gpt_subscription_key") + deployment = first_if_comma(settings.get("azure_apim_gpt_deployment")) + api_version = settings.get("azure_apim_gpt_api_version") + + # Check if key vault secret storage is enabled in settings + if settings.get("enable_key_vault_secret_storage", False) and settings.get("key_vault_name") and key: + try: + if validate_secret_name_dynamic(key): + # Try to retrieve the secret from Key Vault + resolved_key = retrieve_secret_from_key_vault_by_full_name(key) + if resolved_key: + # Update the settings dict with the resolved key for this session + settings["azure_apim_gpt_subscription_key"] = resolved_key + key = resolved_key + except Exception as e: + log_event(f"[SK Loader] Failed to resolve Key Vault secret in get_global_apim: {e}", level=logging.ERROR, exceptionTraceback=True) + # Fallback to using the value as-is + return (endpoint, key, deployment, api_version) def get_user_gpt(): - return ( - agent.get("azure_openai_gpt_endpoint"), - agent.get("azure_openai_gpt_key"), - agent.get("azure_openai_gpt_deployment"), - agent.get("azure_openai_gpt_api_version") - ) + endpoint = agent.get("azure_openai_gpt_endpoint") + key = agent.get("azure_openai_gpt_key") + deployment = agent.get("azure_openai_gpt_deployment") + api_version = agent.get("azure_openai_gpt_api_version") + + # Check if key vault secret storage is enabled in settings + if settings.get("enable_key_vault_secret_storage", False) and settings.get("key_vault_name") and key: + try: + if validate_secret_name_dynamic(key): + # Try to retrieve the secret from Key Vault + resolved_key = retrieve_secret_from_key_vault_by_full_name(key) + if resolved_key: + # Update the agent dict with the resolved key for this session + agent["azure_openai_gpt_key"] = resolved_key + key = resolved_key + except Exception as e: + log_event(f"[SK Loader] Failed to resolve Key Vault secret for agent '{agent.get('name')}' in get_user_gpt: {e}", level=logging.ERROR, exceptionTraceback=True) + # Fallback to using the value as-is + return (endpoint, key, deployment, api_version) def get_global_gpt(): - return ( - settings.get("azure_openai_gpt_endpoint") or selected_model.get("endpoint"), - settings.get("azure_openai_gpt_key") or selected_model.get("key"), - settings.get("azure_openai_gpt_deployment") or selected_model.get("deploymentName"), - settings.get("azure_openai_gpt_api_version") or selected_model.get("api_version") - ) + endpoint = settings.get("azure_openai_gpt_endpoint") or selected_model.get("endpoint") + key = settings.get("azure_openai_gpt_key") or selected_model.get("key") + deployment = settings.get("azure_openai_gpt_deployment") or selected_model.get("deploymentName") + api_version = settings.get("azure_openai_gpt_api_version") or selected_model.get("api_version") + + # Check if key vault secret storage is enabled in settings + if settings.get("enable_key_vault_secret_storage", False) and settings.get("key_vault_name") and key: + try: + if validate_secret_name_dynamic(key): + # Try to retrieve the secret from Key Vault + resolved_key = retrieve_secret_from_key_vault_by_full_name(key) + if resolved_key: + # Update the settings dict with the resolved key for this session + settings["azure_openai_gpt_key"] = resolved_key + key = resolved_key + except Exception as e: + log_event(f"[SK Loader] Failed to resolve Key Vault secret in get_global_gpt: {e}", level=logging.ERROR, exceptionTraceback=True) + # Fallback to using the value as-is + return (endpoint, key, deployment, api_version) def merge_fields(primary, fallback): return tuple(p if p not in [None, ""] else f for p, f in zip(primary, fallback)) @@ -236,9 +303,7 @@ def load_time_plugin(kernel: Kernel): ) def load_http_plugin(kernel: Kernel): - # Import the smart HTTP plugin for better content size management try: - from semantic_kernel_plugins.smart_http_plugin import SmartHttpPlugin # Use smart HTTP plugin with 75k character limit (≈50k tokens) smart_plugin = SmartHttpPlugin(max_content_size=75000, extract_text_only=True) kernel.add_plugin( @@ -384,7 +449,7 @@ def initialize_semantic_kernel(user_id: str=None, redis_client=None): ) debug_print(f"[SK Loader] Semantic Kernel Agent and Plugins loading completed.") -def load_agent_specific_plugins(kernel, plugin_names, mode_label="global", user_id=None): +def load_agent_specific_plugins(kernel, plugin_names, settings, mode_label="global", user_id=None): """ Load specific plugins by name for an agent with enhanced logging. @@ -395,27 +460,30 @@ def load_agent_specific_plugins(kernel, plugin_names, mode_label="global", user_ user_id: User ID for per-user mode """ if not plugin_names: + debug_print(f"[SK Loader] No plugin names provided to load_agent_specific_plugins") return print(f"[SK Loader] Loading {len(plugin_names)} agent-specific plugins: {plugin_names}") try: + merge_global = settings.get('merge_global_semantic_kernel_with_workspace', False) # Create logged plugin loader for enhanced logging logged_loader = create_logged_plugin_loader(kernel) - # Get plugin manifests based on mode if mode_label == "per-user": - from functions_personal_actions import get_personal_actions if user_id: - all_plugin_manifests = get_personal_actions(user_id) - print(f"[SK Loader] Retrieved {len(all_plugin_manifests)} personal plugin manifests for user {user_id}") + all_plugin_manifests = get_personal_actions(user_id, return_type=SecretReturnType.NAME) + if merge_global: + global_plugins = get_global_actions(return_type=SecretReturnType.NAME) + for g in global_plugins: + all_plugin_manifests.append(g) + debug_print(f"[SK Loader] Retrieved {len(all_plugin_manifests)} personal plugin manifests for user {user_id}") else: - print(f"[SK Loader] Warning: No user_id provided for per-user plugin loading") + debug_print(f"[SK Loader] Warning: No user_id provided for per-user plugin loading") all_plugin_manifests = [] else: # Global mode - get from global actions container - from functions_global_actions import get_global_actions - all_plugin_manifests = get_global_actions() + all_plugin_manifests = get_global_plugins(return_type=SecretReturnType.NAME) print(f"[SK Loader] Retrieved {len(all_plugin_manifests)} global plugin manifests") # Filter manifests to only include requested plugins @@ -424,6 +492,18 @@ def load_agent_specific_plugins(kernel, plugin_names, mode_label="global", user_ p for p in all_plugin_manifests if p.get('name') in plugin_names or p.get('id') in plugin_names ] + + debug_print(f"[SK Loader] Filtered to {len(plugin_manifests)} plugin manifests after matching names/IDs") + debug_print(f"[SK Loader] Plugin manifests to load: {plugin_manifests}") + + if settings.get("enable_key_vault_secret_storage", False) and settings.get("key_vault_name"): + debug_print(f"[SK Loader] Resolving Key Vault secrets in plugin manifests if needed") + try: + plugin_manifests = [resolve_key_vault_secrets_in_plugins(p, settings) for p in plugin_manifests] + debug_print(f"[SK Loader] Resolved Key Vault secrets in plugin manifests {plugin_manifests}") + except Exception as e: + log_event(f"[SK Loader] Failed to resolve Key Vault secrets in plugin manifests: {e}", level=logging.ERROR, exceptionTraceback=True) + print(f"[SK Loader] Failed to resolve Key Vault secrets in plugin manifests: {e}") if not plugin_manifests: print(f"[SK Loader] Warning: No plugin manifests found for names/IDs: {plugin_names}") @@ -468,35 +548,38 @@ def load_agent_specific_plugins(kernel, plugin_names, mode_label="global", user_ except Exception as e: log_event( - f"[SK Loader] Error in agent-specific plugin loading: {e}", + f"[SK Loader][Error] Error in agent-specific plugin loading: {e}", extra={"error": str(e), "mode": mode_label, "user_id": user_id, "plugin_names": plugin_names}, level=logging.ERROR, exceptionTraceback=True ) + print(f"[SK Loader][Error] Error in agent-specific plugin loading: {e}") # Fallback to original method - log_event("[SK Loader] Falling back to original plugin loading method due to error", level=logging.WARNING) try: # Get plugin manifests again for fallback if mode_label == "per-user": - from functions_personal_actions import get_personal_actions if user_id: - all_plugin_manifests = get_personal_actions(user_id) + all_plugin_manifests = get_personal_actions(user_id, return_type=SecretReturnType.NAME) + if merge_global: + global_plugins = get_global_actions(return_type=SecretReturnType.NAME) + for g in global_plugins: + all_plugin_manifests.append(g) else: all_plugin_manifests = [] else: - from functions_global_actions import get_global_actions - all_plugin_manifests = get_global_actions() - + all_plugin_manifests = get_global_actions(return_type=SecretReturnType.NAME) + plugin_manifests = [p for p in all_plugin_manifests if p.get('name') in plugin_names] _load_agent_plugins_original_method(kernel, plugin_manifests, mode_label) except Exception as fallback_error: log_event( - f"[SK Loader] Fallback plugin loading also failed: {fallback_error}", + f"[SK Loader][Error] Fallback plugin loading also failed: {fallback_error}", extra={"error": str(fallback_error), "mode": mode_label, "user_id": user_id}, level=logging.ERROR, exceptionTraceback=True ) + print(f"[SK Loader][Error] Fallback plugin loading also failed: {fallback_error}") def _load_agent_plugins_original_method(kernel, plugin_manifests, mode_label="global"): @@ -505,7 +588,6 @@ def _load_agent_plugins_original_method(kernel, plugin_manifests, mode_label="gl """ try: # Load the filtered plugins using original method - from semantic_kernel_plugins.plugin_loader import discover_plugins discovered_plugins = discover_plugins() for manifest in plugin_manifests: @@ -529,12 +611,11 @@ def normalize(s): try: # Special handling for OpenAPI plugins if normalized_type == normalize('openapi') or 'openapi' in normalized_type: - from semantic_kernel_plugins.openapi_plugin_factory import OpenApiPluginFactory plugin = OpenApiPluginFactory.create_from_config(manifest) print(f"[SK Loader] Created OpenAPI plugin: {name}") else: # Standard plugin instantiation - from semantic_kernel_plugins.plugin_health_checker import PluginHealthChecker, PluginErrorRecovery + plugin_instance, instantiation_errors = PluginHealthChecker.create_plugin_safely( matched_class, manifest, name ) @@ -547,9 +628,6 @@ def normalize(s): plugin = plugin_instance - # Add plugin to kernel - from semantic_kernel.functions.kernel_plugin import KernelPlugin - # Special handling for OpenAPI plugins with dynamic functions if hasattr(plugin, 'get_kernel_plugin'): print(f"[SK Loader] Using custom kernel plugin method for: {name}") @@ -681,7 +759,7 @@ def load_single_agent_for_kernel(kernel, agent_cfg, settings, context_obj, redis plugin_mode = "global" if agent_is_global else mode_label user_id = get_current_user_id() if not agent_is_global else None print(f"[SK Loader] Agent is_global: {agent_is_global}, using plugin_mode: {plugin_mode}") - load_agent_specific_plugins(kernel, agent_config["actions_to_load"], plugin_mode, user_id=user_id) + load_agent_specific_plugins(kernel, agent_config["actions_to_load"], settings, plugin_mode, user_id=user_id) try: kwargs = { @@ -737,10 +815,48 @@ def load_single_agent_for_kernel(kernel, agent_cfg, settings, context_obj, redis log_event(f"[SK Loader] load_single_agent_for_kernel completed - returning {len(agent_objs)} agents: {list(agent_objs.keys())}", level=logging.INFO) return kernel, agent_objs +def resolve_key_vault_secrets_in_plugins(plugin_manifest, settings): + """ + Resolve any Key Vault secrets in a plugin manifest. + """ + if not isinstance(plugin_manifest, dict): + raise ValueError("Plugin manifest must be a dictionary") + + kv_name = settings.get("key_vault_name") + if not kv_name: + raise ValueError("Key Vault name not configured in settings") + + def resolve_value(value): + if isinstance(value, str) and validate_secret_name_dynamic(value): + resolved = retrieve_secret_from_key_vault_by_full_name(value) + if resolved: + return resolved + else: + raise ValueError(f"Failed to retrieve secret '{value}' from Key Vault '{kv_name}'") + return value + + resolved_manifest = {} + for k, v in plugin_manifest.items(): + print(f"[SK Loader] Resolving plugin manifest key: {k} with value type: {type(v)}") + if isinstance(v, str): + resolved_manifest[k] = resolve_value(v) + elif isinstance(v, list): + resolved_manifest[k] = [resolve_value(item) for item in v] + elif isinstance(v, dict): + resolved_manifest[k] = {sub_k: resolve_value(sub_v) for sub_k, sub_v in v.items()} + else: + resolved_manifest[k] = v # Leave other types unchanged + return resolved_manifest + def load_plugins_for_kernel(kernel, plugin_manifests, settings, mode_label="global"): """ DRY helper to load plugins from a manifest list (user or global). """ + if settings.get("enable_key_vault_secret_storage", False) and settings.get("key_vault_name"): + try: + plugin_manifests = [resolve_key_vault_secrets_in_plugins(p, settings) for p in plugin_manifests] + except Exception as e: + log_event(f"[SK Loader] Failed to resolve Key Vault secrets in plugin manifests: {e}", level=logging.ERROR, exceptionTraceback=True) # Create logged plugin loader for enhanced logging logged_loader = create_logged_plugin_loader(kernel) @@ -854,7 +970,6 @@ def _load_plugins_original_method(kernel, plugin_manifests, settings, mode_label Original plugin loading method as fallback. """ try: - from semantic_kernel_plugins.plugin_loader import discover_plugins discovered_plugins = discover_plugins() for manifest in plugin_manifests: plugin_type = manifest.get('type') @@ -874,7 +989,6 @@ def normalize(s): try: # Special handling for OpenAPI plugins if normalized_type == normalize('openapi') or 'openapi' in normalized_type: - from semantic_kernel_plugins.openapi_plugin_factory import OpenApiPluginFactory # Use the factory to create OpenAPI plugins from configuration plugin = OpenApiPluginFactory.create_from_config(manifest) else: @@ -946,13 +1060,8 @@ def load_user_semantic_kernel(kernel: Kernel, settings, user_id: str, redis_clie load_core_plugins_only(kernel, settings) return kernel, None - # Redis is now optional for per-user mode. If not present, state will not persist. - - # Load agents from personal_agents container - from functions_personal_agents import get_personal_agents, ensure_migration_complete - # Ensure migration is complete (will migrate any remaining legacy data) - ensure_migration_complete(user_id) + ensure_agents_migration_complete(user_id) agents_cfg = get_personal_agents(user_id) print(f"[SK Loader] User settings found {len(agents_cfg)} agents for user '{user_id}'") @@ -965,7 +1074,6 @@ def load_user_semantic_kernel(kernel: Kernel, settings, user_id: str, redis_clie merge_global = settings.get('merge_global_semantic_kernel_with_workspace', False) print(f"[SK Loader] merge_global_semantic_kernel_with_workspace: {merge_global}") if merge_global: - from functions_global_agents import get_global_agents global_agents = get_global_agents() print(f"[SK Loader] Found {len(global_agents)} global agents to merge") # Mark global agents @@ -998,18 +1106,13 @@ def load_user_semantic_kernel(kernel: Kernel, settings, user_id: str, redis_clie "agents": agents_cfg }, level=logging.INFO) - - # Load plugins from personal_actions container - from functions_personal_actions import get_personal_actions, ensure_migration_complete - # Ensure migration is complete (will migrate any remaining legacy data) - ensure_migration_complete(user_id) - plugin_manifests = get_personal_actions(user_id) + ensure_actions_migration_complete(user_id) + plugin_manifests = get_personal_actions(user_id, return_type=SecretReturnType.NAME) # PATCH: Merge global plugins if enabled if merge_global: - from functions_global_actions import get_global_actions - global_plugins = get_global_actions() + global_plugins = get_global_actions(return_type=SecretReturnType.NAME) # User plugins take precedence all_plugins = {p.get('name'): p for p in plugin_manifests} all_plugins.update({p.get('name'): p for p in global_plugins}) @@ -1021,30 +1124,37 @@ def load_user_semantic_kernel(kernel: Kernel, settings, user_id: str, redis_clie # Only load core Semantic Kernel plugins here if settings.get('enable_time_plugin', True): load_time_plugin(kernel) + print(f"[SK Loader] Loaded Time plugin.") log_event("[SK Loader] Loaded Time plugin.", level=logging.INFO) if settings.get('enable_fact_memory_plugin', True): load_fact_memory_plugin(kernel) + print(f"[SK Loader] Loaded Fact Memory plugin.") log_event("[SK Loader] Loaded Fact Memory plugin.", level=logging.INFO) if settings.get('enable_math_plugin', True): load_math_plugin(kernel) + print(f"[SK Loader] Loaded Math plugin.") log_event("[SK Loader] Loaded Math plugin.", level=logging.INFO) if settings.get('enable_text_plugin', True): load_text_plugin(kernel) + print(f"[SK Loader] Loaded Text plugin.") log_event("[SK Loader] Loaded Text plugin.", level=logging.INFO) if settings.get('enable_http_plugin', True): load_http_plugin(kernel) + print(f"[SK Loader] Loaded HTTP plugin.") log_event("[SK Loader] Loaded HTTP plugin.", level=logging.INFO) if settings.get('enable_wait_plugin', True): load_wait_plugin(kernel) + print(f"[SK Loader] Loaded Wait plugin.") log_event("[SK Loader] Loaded Wait plugin.", level=logging.INFO) if settings.get('enable_default_embedding_model_plugin', True): load_embedding_model_plugin(kernel, settings) + print(f"[SK Loader] Loaded Default Embedding Model plugin.") log_event("[SK Loader] Loaded Default Embedding Model plugin.", level=logging.INFO) # Get selected agent from user settings (this still needs to be in user settings for UI state) @@ -1116,6 +1226,7 @@ def load_user_semantic_kernel(kernel: Kernel, settings, user_id: str, redis_clie f"[SK Loader] User {user_id} No agent found matching global selected agent: {global_selected_agent_name}", level=logging.WARNING ) + # If still not found, DON'T use first agent - only load when explicitly selected if agent_cfg is None and agents_cfg: debug_print(f"[SK Loader] User {user_id} Agent selection final status: agent_cfg is None") @@ -1148,8 +1259,8 @@ def load_semantic_kernel(kernel: Kernel, settings): log_event("[SK Loader] Global Semantic Kernel mode enabled. Loading global plugins and agents.", level=logging.INFO) # Conditionally load core plugins based on settings - from functions_global_actions import get_global_actions - plugin_manifests = get_global_actions() + + plugin_manifests = get_global_actions(return_type=SecretReturnType.NAME) log_event(f"[SK Loader] Found {len(plugin_manifests)} plugin manifests", level=logging.INFO) # --- Dynamic Plugin Type Loading (semantic_kernel_plugins) --- @@ -1157,7 +1268,7 @@ def load_semantic_kernel(kernel: Kernel, settings): # --- Agent and Service Loading --- # region Multi-agent Orchestration - from functions_global_agents import get_global_agents + agents_cfg = get_global_agents() enable_multi_agent_orchestration = settings.get('enable_multi_agent_orchestration', False) merge_global = settings.get('merge_global_semantic_kernel_with_workspace', False) diff --git a/application/single_app/semantic_kernel_plugins/PLUGIN_DYNAMIC_SECRET_STORAGE.md b/application/single_app/semantic_kernel_plugins/PLUGIN_DYNAMIC_SECRET_STORAGE.md new file mode 100644 index 00000000..6518e569 --- /dev/null +++ b/application/single_app/semantic_kernel_plugins/PLUGIN_DYNAMIC_SECRET_STORAGE.md @@ -0,0 +1,69 @@ +# PLUGIN_DYNAMIC_SECRET_STORAGE.md + +## Feature: Dynamic Secret Storage for Plugins/Actions + +**Implemented in version:** (add your current config.py version here) + +### Overview +This feature allows plugin writers to store secrets in Azure Key Vault dynamically by simply naming any key in the plugin's `additionalFields` dictionary with the suffix `__Secret`. The application will automatically detect these keys, store their values in Key Vault, and replace the value with a Key Vault reference. This works in addition to the standard `auth.key` secret handling. + + +### How It Works +- When saving a plugin, any key in `additionalFields` ending with `__Secret` (two underscores and a capital S) will be stored in Key Vault. +- The Key Vault secret name for these fields is constructed as `{pluginName-additionalsettingnamewithout__Secret}` (e.g., `loganal-alpharoemo` for plugin `loganal` and field `alpharoemo__Secret`). +- The value in the plugin dict will be replaced with the Key Vault reference (the full secret name). +- When retrieving a plugin, any Key Vault reference in `auth.key` or `additionalFields` ending with `__Secret` will be replaced with a UI trigger word (or optionally, the actual secret value). +- When deleting a plugin, any Key Vault reference in `auth.key` or `additionalFields` ending with `__Secret` will be deleted from Key Vault. + + +### Example +```json +{ + "name": "loganal", + "auth": { + "type": "key", + "key": "my-actual-secret-value" + }, + "additionalFields": { + "alpharoemo__Secret": "supersecretvalue", + "otherSetting__Secret": "anothersecret" + } +} +``` +After saving, the plugin dict will look like: +```json +{ + "name": "loganal", + "auth": { + "type": "key", + "key": "loganal--action--global--loganal" // Key Vault reference + }, + "additionalFields": { + "alpharoemo__Secret": "loganal--action-addset--global--loganal-alpharoemo", // Key Vault reference + "otherSetting__Secret": "loganal--action-addset--global--loganal-otherSetting" // Key Vault reference + } +} +``` +**Note:** The Key Vault secret name for each additional setting is constructed as `{pluginName}-{additionalsettingname}` (with __Secret removed). + + +### Benefits +- No custom code required for plugin writers to leverage Key Vault for secrets. +- Supports any number of dynamic secrets per plugin. +- Consistent with existing agent secret handling. +- Secret names are AKV-compliant and descriptive, making management and debugging easier. + + +### Usage +- To store a secret, add a key to `additionalFields` ending with `__Secret` and set its value to the secret. +- The application will handle storing, retrieving, and deleting the secret in Key Vault automatically. +- Secret names for additional settings will follow the `{pluginName-additionalsettingname}` pattern. + +### Related Files +- `functions_keyvault.py` (helpers for save, get, delete) +- `plugin.schema.json` (schema supports arbitrary additionalFields) + +### Version History +- Feature added in version: (add your current config.py version here) + +--- diff --git a/application/single_app/semantic_kernel_plugins/azure_function_plugin.py b/application/single_app/semantic_kernel_plugins/azure_function_plugin.py index e24fd6e5..2e0928c6 100644 --- a/application/single_app/semantic_kernel_plugins/azure_function_plugin.py +++ b/application/single_app/semantic_kernel_plugins/azure_function_plugin.py @@ -7,7 +7,7 @@ class AzureFunctionPlugin(BasePlugin): def __init__(self, manifest: Dict[str, Any]): - self.manifest = manifest + super().__init__(manifest) self.endpoint = manifest.get('endpoint') self.key = manifest.get('auth', {}).get('key') self.auth_type = manifest.get('auth', {}).get('type', 'key') diff --git a/application/single_app/semantic_kernel_plugins/base_plugin.py b/application/single_app/semantic_kernel_plugins/base_plugin.py index 4aba46dc..e56e97ca 100644 --- a/application/single_app/semantic_kernel_plugins/base_plugin.py +++ b/application/single_app/semantic_kernel_plugins/base_plugin.py @@ -1,5 +1,6 @@ from abc import ABC, abstractmethod from typing import Dict, Any, List, Optional +import re class BasePlugin(ABC): @property @@ -36,8 +37,6 @@ def display_name(self) -> str: # Remove 'Plugin' suffix and format nicely name = class_name.replace('Plugin', '') - # Handle common acronyms by keeping them together - import re # Split on word boundaries while preserving acronyms parts = re.findall(r'[A-Z]+(?=[A-Z][a-z]|$)|[A-Z][a-z]*', name) @@ -45,6 +44,11 @@ def display_name(self) -> str: formatted = ' '.join(parts).replace('_', ' ').strip() return formatted if formatted else name + """ + This class provides common functionality and enforces a standard interface. + All plugins should inherit from this base class. + All plugins should call super().__init__(manifest) in their init constructor. + """ @abstractmethod def __init__(self, manifest: Optional[Dict[str, Any]] = None): self.manifest = manifest or {} diff --git a/application/single_app/semantic_kernel_plugins/databricks_table_example.json b/application/single_app/semantic_kernel_plugins/databricks_table_example.json deleted file mode 100644 index 32cfecdd..00000000 --- a/application/single_app/semantic_kernel_plugins/databricks_table_example.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "name": "users_table", - "type": "databricks_table", - "description": "Query the users table in Databricks.", - "endpoint": "https:///api/2.0/sql/statements", - "auth": { - "type": "key", // Authentication type, can be 'key' or 'identity', etc. - "key": "", - "managedIdentity": "" // Optional, if using identity-based auth - }, - "metadata": {}, - "additionalFields": { - "table": "users", - "warehouse_id": "", - "columns": [ - { - "name": "id", - "type": "int", - "description": "User ID" - }, - { - "name": "name", - "type": "string", - "description": "User's full name" - }, - { - "name": "email", - "type": "string", - "description": "User's email address" - } - ] - } -} \ No newline at end of file diff --git a/application/single_app/semantic_kernel_plugins/logged_plugin_loader.py b/application/single_app/semantic_kernel_plugins/logged_plugin_loader.py index 5b2c7193..c7cc7d57 100644 --- a/application/single_app/semantic_kernel_plugins/logged_plugin_loader.py +++ b/application/single_app/semantic_kernel_plugins/logged_plugin_loader.py @@ -13,13 +13,12 @@ from semantic_kernel.functions import kernel_function from semantic_kernel.functions.kernel_plugin import KernelPlugin from semantic_kernel_plugins.base_plugin import BasePlugin -from semantic_kernel_plugins.plugin_invocation_logger import ( - get_plugin_logger, - plugin_function_logger, - auto_wrap_plugin_functions -) +from semantic_kernel_plugins.plugin_invocation_logger import get_plugin_logger, plugin_function_logger, auto_wrap_plugin_functions +from semantic_kernel_plugins.plugin_loader import discover_plugins from functions_appinsights import log_event - +from semantic_kernel_plugins.openapi_plugin_factory import OpenApiPluginFactory +from semantic_kernel_plugins.sql_schema_plugin import SQLSchemaPlugin +from semantic_kernel_plugins.sql_query_plugin import SQLQueryPlugin class LoggedPluginLoader: """Enhanced plugin loader that automatically adds invocation logging.""" @@ -48,17 +47,21 @@ def load_plugin_from_manifest(self, manifest: Dict[str, Any], log_event(f"[Logged Plugin Loader] Starting to load plugin: {plugin_name} (type: {plugin_type})") if not plugin_name: - self.logger.error("Plugin manifest missing required 'name' field") + log_event(f"[Logged Plugin Loader] Plugin manifest missing required 'name' field", level=logging.ERROR) return False try: # Load the plugin instance + debug_print(f"[Logged Plugin Loader] Creating plugin instance for {plugin_name} of type {plugin_type}") plugin_instance = self._create_plugin_instance(manifest) + debug_print(f"[Logged Plugin Loader] Created plugin instance: {plugin_instance}") if not plugin_instance: + debug_print(f"[Logged Plugin Loader] Failed to create plugin instance for {plugin_name} of type {plugin_type}") return False # Enable logging if the plugin supports it if hasattr(plugin_instance, 'enable_invocation_logging'): + debug_print(f"[Logged Plugin Loader] Enabling invocation logging for {plugin_name}") plugin_instance.enable_invocation_logging(True) # Auto-wrap plugin functions with logging @@ -76,7 +79,7 @@ def load_plugin_from_manifest(self, manifest: Dict[str, Any], self._register_plugin_with_kernel(plugin_instance, plugin_name) log_event( - f"[Plugin Loader] Successfully loaded plugin: {plugin_name}", + f"[Logged Plugin Loader] Successfully loaded plugin: {plugin_name}", extra={ "plugin_name": plugin_name, "plugin_type": plugin_type, @@ -90,7 +93,7 @@ def load_plugin_from_manifest(self, manifest: Dict[str, Any], except Exception as e: log_event( - f"[Plugin Loader] Failed to load plugin: {plugin_name}", + f"[Logged Plugin Loader] Failed to load plugin: {plugin_name}", extra={ "plugin_name": plugin_name, "plugin_type": plugin_type, @@ -112,13 +115,40 @@ def _create_plugin_instance(self, manifest: Dict[str, Any]): return self._create_openapi_plugin(manifest) elif plugin_type == 'python': return self._create_python_plugin(manifest) - elif plugin_type == 'custom': - return self._create_custom_plugin(manifest) - elif plugin_type in ['sql_schema', 'sql_query']: - return self._create_sql_plugin(manifest) + #elif plugin_type in ['sql_schema', 'sql_query']: + # return self._create_sql_plugin(manifest) else: - self.logger.warning(f"Unknown plugin type: {plugin_type} for plugin: {plugin_name}") - return None + try: + debug_print("[Logged Plugin Loader] Attempting to discover plugin type:", plugin_type) + discovered_plugins = discover_plugins() + plugin_type = manifest.get('type') + name = manifest.get('name') + description = manifest.get('description', '') + # Normalize for matching + def normalize(s): + return s.replace('_', '').replace('-', '').replace('plugin', '').lower() if s else '' + debug_print("[Logged Plugin Loader] Normalizing plugin type for matching:", plugin_type) + normalized_type = normalize(plugin_type) + debug_print(f"[Logged Plugin Loader] Normalized plugin type: {normalized_type}") + matched_class = None + for class_name, cls in discovered_plugins.items(): + normalized_class = normalize(class_name) + print("[Logged Plugin Loader] Checking plugin class:", class_name, "normalized:", normalized_class) + if normalized_type == normalized_class or normalized_type in normalized_class: + matched_class = cls + break + debug_print(f"[Logged Plugin Loader] Matched class for plugin '{name}' of type '{plugin_type}': {matched_class}") + if matched_class: + try: + plugin = matched_class(manifest) if 'manifest' in matched_class.__init__.__code__.co_varnames else matched_class() + log_event(f"[Logged Plugin Loader] Instanced plugin: {name} (type: {plugin_type})", {"plugin_name": name, "plugin_type": plugin_type}, level=logging.INFO) + return plugin + except Exception as e: + log_event(f"[Logged Plugin Loader] Failed to instantiate plugin: {name}: {e}", {"plugin_name": name, "plugin_type": plugin_type, "error": str(e)}, level=logging.ERROR, exceptionTraceback=True) + else: + log_event(f"[Logged Plugin Loader] Unknown plugin type: {plugin_type} for plugin '{name}'", {"plugin_name": name, "plugin_type": plugin_type}, level=logging.WARNING) + except Exception as e: + log_event(f"[Logged Plugin Loader] Error discovering plugin types: {e}", {"error": str(e)}, level=logging.ERROR, exceptionTraceback=True) def _create_openapi_plugin(self, manifest: Dict[str, Any]): """Create an OpenAPI plugin instance.""" @@ -126,9 +156,6 @@ def _create_openapi_plugin(self, manifest: Dict[str, Any]): log_event(f"[Logged Plugin Loader] Attempting to create OpenAPI plugin: {plugin_name}", level=logging.DEBUG) try: - from semantic_kernel_plugins.openapi_plugin_factory import OpenApiPluginFactory - log_event(f"[Logged Plugin Loader] Successfully imported OpenApiPluginFactory", level=logging.DEBUG) - log_event(f"[Logged Plugin Loader] Creating OpenAPI plugin using factory", extra={"plugin_name": plugin_name, "manifest": manifest}, level=logging.DEBUG) @@ -176,22 +203,14 @@ def _create_python_plugin(self, manifest: Dict[str, Any]): self.logger.error(f"Failed to create Python plugin {class_name} from {module_name}: {e}") return None - def _create_custom_plugin(self, manifest: Dict[str, Any]): - """Create a custom plugin instance.""" - # This is where you'd handle custom plugin types specific to your application - self.logger.warning(f"Custom plugin type not yet implemented: {manifest}") - return None - def _create_sql_plugin(self, manifest: Dict[str, Any]): """Create a SQL plugin instance.""" plugin_type = manifest.get('type') try: if plugin_type == 'sql_schema': - from semantic_kernel_plugins.sql_schema_plugin import SQLSchemaPlugin return SQLSchemaPlugin(manifest) elif plugin_type == 'sql_query': - from semantic_kernel_plugins.sql_query_plugin import SQLQueryPlugin return SQLQueryPlugin(manifest) else: self.logger.error(f"Unknown SQL plugin type: {plugin_type}") @@ -336,7 +355,7 @@ def load_multiple_plugins(self, manifests: List[Dict[str, Any]], total_count = len(results) log_event( - f"[Plugin Loader] Loaded {successful_count}/{total_count} plugins", + f"[Logged Plugin Loader] Loaded {successful_count}/{total_count} plugins", extra={ "successful_plugins": [name for name, success in results.items() if success], "failed_plugins": [name for name, success in results.items() if not success], diff --git a/application/single_app/semantic_kernel_plugins/queue_storage_plugin.py b/application/single_app/semantic_kernel_plugins/queue_storage_plugin.py index f3ca9aad..58e918bc 100644 --- a/application/single_app/semantic_kernel_plugins/queue_storage_plugin.py +++ b/application/single_app/semantic_kernel_plugins/queue_storage_plugin.py @@ -10,7 +10,7 @@ def __init__(self, manifest: Dict[str, Any]): super().__init__(manifest) self.manifest = manifest self.endpoint = manifest.get('endpoint') - self.queue_name = manifest.get('queue_name') + self.queue_name = manifest.get('additional_settings', {}).get('queue_name') self.key = manifest.get('auth', {}).get('key') self.auth_type = manifest.get('auth', {}).get('type', 'key') self._metadata = manifest.get('metadata', {}) diff --git a/application/single_app/semantic_kernel_plugins/ui_test_plugin.py b/application/single_app/semantic_kernel_plugins/ui_test_plugin.py new file mode 100644 index 00000000..44068d43 --- /dev/null +++ b/application/single_app/semantic_kernel_plugins/ui_test_plugin.py @@ -0,0 +1,80 @@ +""" +UI Test Plugin for Semantic Kernel +- Provides demonstration methods for UI testing (greeting, farewell, manifest retrieval) +- Useful for testing plugin integration and UI workflows +- Does not interact with external systems or databases +""" + +import json +import logging +from typing import Dict, Any, List, Optional, Union +from semantic_kernel_plugins.base_plugin import BasePlugin +from semantic_kernel.functions import kernel_function +from functions_appinsights import log_event +from semantic_kernel_plugins.plugin_invocation_logger import plugin_function_logger +from functions_debug import debug_print + +# Helper class to wrap results with metadata +class ResultWithMetadata: + def __init__(self, data, metadata): + self.data = data + self.metadata = metadata + def __str__(self): + return str(self.data) + def __repr__(self): + return f"ResultWithMetadata(data={self.data!r}, metadata={self.metadata!r})" + +class UITestPlugin(BasePlugin): + def __init__(self, manifest: Dict[str, Any]): + super().__init__(manifest) + + @property + def display_name(self) -> str: + return "UI Test Plugin" + + @property + def metadata(self) -> Dict[str, Any]: + return { + "name": "ui_test_plugin", + "type": "ui_test", + "description": "A plugin for UI testing and demonstration purposes.", + "methods": [ + { + "name": "greet_user", + "description": "Returns a greeting message.", + "parameters": [ + {"name": "name", "type": "str", "description": "Name to greet.", "required": True} + ], + "returns": {"type": "str", "description": "Greeting message."} + }, + { + "name": "farewell_user", + "description": "Returns a farewell message.", + "parameters": [ + {"name": "name", "type": "str", "description": "Name to bid farewell.", "required": True} + ], + "returns": {"type": "str", "description": "Farewell message."} + }, + { + "name": "get_manifest", + "description": "Returns the plugin manifest.", + "parameters": [], + "returns": {"type": "str", "description": "Manifest as JSON string."} + } + ] + } + + @kernel_function(description="A function that returns a greeting message.") + @plugin_function_logger("UITestPlugin") + def greet_user(self, name: str) -> str: + return f"Hello, {name}!" + + @kernel_function(description="A function that returns a farewell message.") + @plugin_function_logger("UITestPlugin") + def farewell_user(self, name: str) -> str: + return f"Goodbye, {name}!" + + @kernel_function(description="A function that returns the plugin manifest") + @plugin_function_logger("UITestPlugin") + def get_manifest(self) -> str: + return json.dumps(self.manifest, indent=2) \ No newline at end of file diff --git a/application/single_app/static/css/chats.css b/application/single_app/static/css/chats.css index 61ac309a..147fa696 100644 --- a/application/single_app/static/css/chats.css +++ b/application/single_app/static/css/chats.css @@ -947,6 +947,11 @@ a.citation-link:hover { text-decoration: underline; } +[data-bs-theme="dark"] .message-text a { + color: #ffeb3b; + text-decoration: underline; +} + .message-text a:hover { color: #0a58ca; text-decoration: none; diff --git a/application/single_app/static/js/admin/admin_plugins.js b/application/single_app/static/js/admin/admin_plugins.js index 682d329d..ad497f62 100644 --- a/application/single_app/static/js/admin/admin_plugins.js +++ b/application/single_app/static/js/admin/admin_plugins.js @@ -55,7 +55,11 @@ function setupSaveHandler(plugin, modal) { saveBtn.onclick = async (event) => { event.preventDefault(); - + const errorDiv = document.getElementById('plugin-modal-error'); + if (errorDiv) { + errorDiv.classList.add('d-none'); + errorDiv.textContent = ''; + } try { // Get form data from the stepper const formData = window.pluginModalStepper.getFormData(); @@ -67,8 +71,19 @@ function setupSaveHandler(plugin, modal) { return; } + const originalText = saveBtn.innerHTML; + saveBtn.innerHTML = `Saving...`; + saveBtn.disabled = true; // Save the action - await savePlugin(formData, plugin); + try { + await savePlugin(formData, plugin); + } catch (error) { + window.pluginModalStepper.showError(error.message); + return; + } finally { + saveBtn.innerHTML = originalText; + saveBtn.disabled = false; + } // Close modal and refresh if (modal && typeof modal.hide === 'function') { diff --git a/application/single_app/static/js/admin/admin_settings.js b/application/single_app/static/js/admin/admin_settings.js index ba620b4c..b2b75b64 100644 --- a/application/single_app/static/js/admin/admin_settings.js +++ b/application/single_app/static/js/admin/admin_settings.js @@ -2267,6 +2267,34 @@ function setupTestButtons() { } }); } + + const testKeyVaultBtn = document.getElementById('test_key_vault_button'); + if (testKeyVaultBtn) { + testKeyVaultBtn.addEventListener('click', async () => { + const resultDiv = document.getElementById('test_key_vault_result'); + resultDiv.innerHTML = 'Testing Key Vault...'; + const payload = { + test_type: 'key_vault', + vault_name: document.getElementById('key_vault_name').value, + client_id: document.getElementById('key_vault_identity').value, + }; + try { + const resp = await fetch('/api/admin/settings/test_connection', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload) + }); + const data = await resp.json(); + if (resp.ok) { + resultDiv.innerHTML = `${data.message}`; + } else { + resultDiv.innerHTML = `${data.error || 'Error testing Key Vault'}`; + } + } catch (err) { + resultDiv.innerHTML = `Error: ${err.message}`; + } + }); + } } function toggleEnhancedCitation(isEnabled) { diff --git a/application/single_app/static/js/agent_modal_stepper.js b/application/single_app/static/js/agent_modal_stepper.js index 28a22a64..41111c26 100644 --- a/application/single_app/static/js/agent_modal_stepper.js +++ b/application/single_app/static/js/agent_modal_stepper.js @@ -1165,12 +1165,22 @@ export class AgentModalStepper { } // Use appropriate endpoint and save method based on context - if (this.isAdmin) { - // Admin context - save to global agents - await this.saveGlobalAgent(agentData); - } else { - // User context - save to personal agents - await this.savePersonalAgent(agentData); + let saveBtn = document.getElementById('agent-modal-save-btn'); + const originalText = saveBtn.innerHTML; + saveBtn.innerHTML = `Saving...`; + saveBtn.disabled = true; + try { + if (this.isAdmin) { + // Admin context - save to global agents + await this.saveGlobalAgent(agentData); + } else { + // User context - save to personal agents + await this.savePersonalAgent(agentData); + } + //No catch to allow outer catch to handle errors + } finally { + saveBtn.innerHTML = originalText; + saveBtn.disabled = false; } } catch (error) { diff --git a/application/single_app/static/js/plugin_modal_stepper.js b/application/single_app/static/js/plugin_modal_stepper.js index 955af056..84df9a5f 100644 --- a/application/single_app/static/js/plugin_modal_stepper.js +++ b/application/single_app/static/js/plugin_modal_stepper.js @@ -3,6 +3,8 @@ import { showToast } from "./chat/chat-toast.js"; export class PluginModalStepper { + + constructor() { this.currentStep = 1; this.maxSteps = 5; @@ -13,10 +15,62 @@ export class PluginModalStepper { this.itemsPerPage = 12; this.filteredTypes = []; this.originalPlugin = null; // Store original state for change tracking - + this.pluginSchemaCache = null; // Will hold plugin.schema.json + this.additionalSettingsSchemaCache = {}; // Cache for additional settings schemas + this.lastAdditionalFieldsType = null; // Track last type to avoid unnecessary redraws + this.defaultAuthTypes = ["key", "identity", "user", "servicePrincipal", "connection_string", "basic", "username_password"]; + + this._loadPluginSchema().then(() => { // Load schema on initialization + this._populateGenericAuthTypeDropdown(); // Dynamically populate generic auth type dropdown after schema loads (will be called again after schema loads) + }); this.bindEvents(); } + async _loadPluginSchema() { + try { + const res = await fetch('/static/json/schemas/plugin.schema.json'); + if (!res.ok) throw new Error('Failed to load plugin.schema.json'); + this.pluginSchemaCache = await res.json(); + } catch (err) { + console.error('Error loading plugin.schema.json:', err); + this.pluginSchemaCache = null; + } + } + + _populateGenericAuthTypeDropdown() { + // Only run if dropdown exists + const dropdown = document.getElementById('plugin-auth-type-generic'); + if (!dropdown) return; + // If schema not loaded, fallback to static options + if (!this.pluginSchemaCache) { + dropdown.innerHTML = ''; + this.defaultAuthTypes.forEach(type => { + const option = document.createElement('option'); + option.value = type; + option.textContent = this.formatAuthType(type); + dropdown.appendChild(option); + }); + return; + } + // Find the enum for generic auth type in the schema + let authTypeEnum = []; + if (this.pluginSchemaCache.properties && this.pluginSchemaCache.properties.authTypeGeneric) { + authTypeEnum = this.pluginSchemaCache.properties.authTypeGeneric.enum || []; + } + // Fallback: if not found, use a default + if (!authTypeEnum.length) { + authTypeEnum = this.defaultAuthTypes; + } + // Clear existing options + dropdown.innerHTML = ''; + authTypeEnum.forEach(type => { + const option = document.createElement('option'); + option.value = type; + option.textContent = this.formatAuthType(type); + dropdown.appendChild(option); + }); + } + bindEvents() { // Step navigation buttons document.getElementById('plugin-modal-next').addEventListener('click', () => this.nextStep()); @@ -368,12 +422,12 @@ export class PluginModalStepper { goToStep(stepNumber) { if (stepNumber < 1 || stepNumber > this.maxSteps) return; - + this.currentStep = stepNumber; this.showStep(stepNumber); this.updateStepIndicator(); this.updateNavigationButtons(); - + // Handle step-specific logic if (stepNumber === 3) { this.showConfigSectionForType(); @@ -426,6 +480,10 @@ export class PluginModalStepper { if (currentStepEl) { currentStepEl.classList.remove('d-none'); } + + if (stepNumber === 2) { + + } // Update step 3 title based on plugin type if (stepNumber === 3) { @@ -449,7 +507,45 @@ export class PluginModalStepper { } if (stepNumber === 4) { - // Only run for new plugins (not editing) + // Load additional settings schema for selected type + let options = {forceReload: true}; + this.getAdditionalSettingsSchema(this.selectedType, options); + const additionalFieldsDiv = document.getElementById('plugin-additional-fields-div'); + if (additionalFieldsDiv) { + // Only clear and rebuild if type changes + if (this.selectedType !== this.lastAdditionalFieldsType) { + additionalFieldsDiv.innerHTML = ''; + additionalFieldsDiv.classList.remove('d-none'); + if (this.selectedType) { + this.getAdditionalSettingsSchema(this.selectedType) + .then(schema => { + if (schema) { + this.buildAdditionalFieldsUI(schema, additionalFieldsDiv); + try { + if (this.isEditMode && this.originalPlugin && this.originalPlugin.additionalFields) { + this.populateDynamicAdditionalFields(this.originalPlugin.additionalFields); + } + } catch (error) { + console.error('Error populating dynamic additional fields:', error); + } + } else { + console.log('No additional settings schema found'); + additionalFieldsDiv.classList.add('d-none'); + } + }) + .catch(error => { + console.error(`Error fetching additional settings schema for type: ${this.selectedType} -- ${error}`); + additionalFieldsDiv.classList.add('d-none'); + }); + } else { + console.warn('No plugin type selected'); + additionalFieldsDiv.classList.add('d-none'); + } + this.lastAdditionalFieldsType = this.selectedType; + } + // Otherwise, preserve user data and do not redraw + } + if (!this.isEditMode) { const typeField = document.getElementById('plugin-type'); const selectedType = typeField && typeField.value ? typeField.value : null; @@ -458,13 +554,13 @@ export class PluginModalStepper { import('./plugin_common.js').then(module => { module.fetchAndMergePluginSettings(selectedType, {}).then(merged => { document.getElementById('plugin-metadata').value = merged.metadata ? JSON.stringify(merged.metadata, null, 2) : '{}'; - document.getElementById('plugin-additional-fields').value = merged.additionalFields ? JSON.stringify(merged.additionalFields, null, 2) : '{}'; + //document.getElementById('plugin-additional-fields').value = merged.additionalFields ? JSON.stringify(merged.additionalFields, null, 2) : '{}'; }); }); } else { // Fallback to empty objects if no type selected document.getElementById('plugin-metadata').value = '{}'; - document.getElementById('plugin-additional-fields').value = '{}'; + //document.getElementById('plugin-additional-fields').value = '{}'; } } } @@ -695,7 +791,7 @@ export class PluginModalStepper { case 4: // Validate JSON fields if (!this.validateJSONField('plugin-metadata', 'Metadata')) return false; - if (!this.validateJSONField('plugin-additional-fields', 'Additional Fields')) return false; + //if (!this.validateJSONField('plugin-additional-fields', 'Additional Fields')) return false; break; } @@ -885,33 +981,63 @@ export class PluginModalStepper { } toggleGenericAuthFields() { - const authType = document.getElementById('plugin-auth-type-generic').value; - const identityGroup = document.getElementById('auth-identity-group'); - const keyGroup = document.getElementById('auth-key-group'); - const tenantIdGroup = document.getElementById('auth-tenantid-group'); - - // Hide all groups first - [identityGroup, keyGroup, tenantIdGroup].forEach(group => { - if (group) group.style.display = 'none'; - }); - - // Show relevant groups based on auth type - switch (authType) { - case 'key': - if (keyGroup) keyGroup.style.display = 'flex'; - break; - case 'identity': - if (identityGroup) identityGroup.style.display = 'flex'; - break; - case 'servicePrincipal': - if (identityGroup) identityGroup.style.display = 'flex'; - if (keyGroup) keyGroup.style.display = 'flex'; - if (tenantIdGroup) tenantIdGroup.style.display = 'flex'; - break; - case 'user': - // No additional fields needed - break; + const dropdown = document.getElementById('plugin-auth-type-generic'); + if (!dropdown) return; + const authType = dropdown.value; + + // Get required fields for selected auth type from schema + let requiredFields = []; + // Defensive: find the correct schema location + const pluginDef = this.pluginSchemaCache?.definitions?.Plugin; + const authSchema = pluginDef?.properties?.auth; + if (authSchema && Array.isArray(authSchema.allOf)) { + for (const cond of authSchema.allOf) { + // Check if this allOf block matches the selected type + if (cond.if && cond.if.properties && cond.if.properties.type && cond.if.properties.type.const === authType) { + // Use the required array from then + if (cond.then && Array.isArray(cond.then.required)) { + requiredFields = cond.then.required.filter(f => f !== 'type'); + } + break; + } + } } + + // Map field keys to DOM groups + const fieldMap = { + identity: document.getElementById('auth-identity-group'), + key: document.getElementById('auth-key-group'), + tenantId: document.getElementById('auth-tenantid-group') + }; + + // Hide all groups first using d-none + Object.values(fieldMap).forEach(group => { if (group) group.classList.add('d-none'); }); + + // Show only required fields for selected auth type using d-none + requiredFields.forEach(field => { + if (fieldMap[field]) { + fieldMap[field].classList.remove('d-none'); + // Update label using mapping or schema description + const label = fieldMap[field].querySelector('span.input-group-text'); + console.log('Updating label for field:', field, 'Auth type:', authType, 'label:', label); + if (label) { + if (authType === 'username_password') { + if (field === 'key') label.textContent = 'Password'; + else if (field === 'identity') label.textContent = 'Username'; + } else if (authType === 'connection_string') { + if (field === 'key') label.textContent = 'Connection String'; + } else if (authType === 'servicePrincipal') { + if (field === 'key') label.textContent = 'Client Secret'; + else if (field === 'identity') label.textContent = 'Client ID'; + else if (field === 'tenantId') label.textContent = 'Tenant ID'; + } else { + if (field === 'key') label.textContent = 'Key'; + else if (field === 'identity') label.textContent = 'Identity'; + else if (field === 'tenantId') label.textContent = 'Tenant ID'; + } + } + } + }); } // SQL Plugin Configuration Methods @@ -1354,7 +1480,11 @@ export class PluginModalStepper { JSON.stringify(plugin.additionalFields, null, 2) : '{}'; document.getElementById('plugin-metadata').value = metadata; - document.getElementById('plugin-additional-fields').value = additionalFields; + try { + document.getElementById('plugin-additional-fields').value = additionalFields; + } catch (e) { + console.warn('Legacy additional fields accessed:', e); + } } getFormData() { @@ -1556,13 +1686,11 @@ export class PluginModalStepper { } } - // Parse existing additional fields and merge + // Collect additional fields from the dynamic UI try { - const additionalFieldsValue = document.getElementById('plugin-additional-fields').value.trim(); - const existingAdditionalFields = additionalFieldsValue ? JSON.parse(additionalFieldsValue) : {}; - additionalFields = { ...existingAdditionalFields, ...additionalFields }; + additionalFields = this.collectAdditionalFields(); } catch (e) { - throw new Error('Invalid additional fields JSON'); + throw new Error('Invalid additional fields input'); } let metadata = {}; @@ -1703,7 +1831,14 @@ export class PluginModalStepper { 'basic': 'Basic Authentication', 'oauth2': 'OAuth2', 'windows': 'Windows Authentication', - 'sql': 'SQL Authentication' + 'sql': 'SQL Authentication', + 'username_password': 'Username/Password', + 'key': 'Key', + 'identity': 'Identity', + 'user': 'User', + 'servicePrincipal': 'Service Principal', + 'connection_string': 'Connection String', + 'basic': 'Basic' }; return authTypeMap[authType] || authType; } @@ -1939,7 +2074,7 @@ export class PluginModalStepper { // Check if there's any metadata or additional fields const metadata = document.getElementById('plugin-metadata').value.trim(); - const additionalFields = document.getElementById('plugin-additional-fields').value.trim(); + //const additionalFields = document.getElementById('plugin-additional-fields').value.trim(); // Check if metadata/additional fields actually contain meaningful data (not just empty objects) let hasMetadata = false; @@ -1953,13 +2088,9 @@ export class PluginModalStepper { hasMetadata = metadata.length > 0 && metadata !== '{}'; } - try { - const additionalFieldsObj = JSON.parse(additionalFields || '{}'); - hasAdditionalFields = Object.keys(additionalFieldsObj).length > 0; - } catch (e) { - // If it's not valid JSON, consider it as having content if it's not empty - hasAdditionalFields = additionalFields.length > 0 && additionalFields !== '{}'; - } + // DRY: Use private helper to collect additional fields + let additionalFieldsObj = this.collectAdditionalFields(); + hasAdditionalFields = Object.keys(additionalFieldsObj).length > 0; // Update has metadata/additional fields indicators document.getElementById('summary-has-metadata').textContent = hasMetadata ? 'Yes' : 'No'; @@ -1977,7 +2108,13 @@ export class PluginModalStepper { // Show/hide additional fields preview const additionalFieldsPreview = document.getElementById('summary-additional-fields-preview'); if (hasAdditionalFields) { - document.getElementById('summary-additional-fields-content').textContent = additionalFields; + let previewContent = ''; + if (typeof additionalFieldsObj === 'object' && additionalFieldsObj !== null) { + previewContent = JSON.stringify(additionalFieldsObj, null, 2); + } else { + previewContent = ''; + } + document.getElementById('summary-additional-fields-content').textContent = previewContent; additionalFieldsPreview.style.display = ''; } else { additionalFieldsPreview.style.display = 'none'; @@ -2148,6 +2285,434 @@ export class PluginModalStepper { div.textContent = str; return div.innerHTML; } + + formatLabel(str) { + // Convert snake_case, camelCase, PascalCase to spaced words + return str + .replace(/([a-z])([A-Z])/g, '$1 $2') // camelCase, PascalCase + .replace(/_/g, ' ') // snake_case + .replace(/\b([A-Z]+)\b/g, match => match.charAt(0) + match.slice(1).toLowerCase()) // ALLCAPS to Capitalized + .replace(/^\w/, c => c.toUpperCase()); + } + + // Build the additional fields UI from a JSON schema + buildAdditionalFieldsUI(schema, parentDiv) { + // Utility to create a labeled field + const self = this; + // Render title and description + const title = document.createElement('h6'); + title.textContent = schema.title || 'Additional Settings'; + parentDiv.appendChild(title); + if (schema.description) { + const desc = document.createElement('p'); + desc.className = 'text-muted'; + desc.textContent = schema.description; + parentDiv.appendChild(desc); + } + // Render all top-level properties + if (schema.properties) { + Object.entries(schema.properties).forEach(([key, prop]) => { + if (prop.type === 'array') { + this.addArrayFieldUI(prop, key, parentDiv, prop.default || []); + } else if (prop.type === 'object') { + const wrapper = document.createElement('div'); + wrapper.className = 'additional-field-object'; + // Create a fieldset for the object + const fieldset = document.createElement('fieldset'); + fieldset.dataset.schemaKey = key; + // Optionally add a legend for the object + const legend = document.createElement('legend'); + legend.textContent = this.formatLabel(key); + fieldset.appendChild(legend); + // Render all sub-properties inside the fieldset + if (prop.properties) { + Object.entries(prop.properties).forEach(([subKey, subProp]) => { + this.createField(subKey, subProp, fieldset); + }); + } + wrapper.appendChild(fieldset); + parentDiv.appendChild(wrapper); + } else { + const wrapper = document.createElement('div'); + wrapper.className = 'additional-field-primitive'; + this.createField(key, prop, wrapper); + parentDiv.appendChild(wrapper); + } + }); + } + } + + // Recursively populate dynamic additional fields UI + populateDynamicAdditionalFields(fields, parentKey = '') { + if (!fields || typeof fields !== 'object') return; + if (this.additionalSettingsSchemaCache && this.selectedType && !this.additionalSettingsSchemaCache[this.getSafeType(this.selectedType)]) { + this.getAdditionalSettingsSchema(this.selectedType); + } + const schema = this.additionalSettingsSchemaCache && this.selectedType ? this.additionalSettingsSchemaCache[this.getSafeType(this.selectedType)] : null; + Object.entries(fields).forEach(([key, value]) => { + console.log('Processing field:', key, 'with value:', value, 'under parentKey:', parentKey); + let fieldName = key; + if (Array.isArray(value)) { + // Find array wrapper, add items if needed + let arrayWrapper = document.querySelector(`#plugin-additional-fields-div [data-schema-key="${fieldName}"]`); + if (!arrayWrapper) { + // Try to find schema for this array (assume you have access to schema) + if (this.additionalSettingsSchemaCache && this.selectedType) { + if (schema && schema.properties && schema.properties[fieldName] && schema.properties[fieldName].type === 'array') { + this.addArrayFieldUI(schema.properties[fieldName], fieldName, document.getElementById('plugin-additional-fields-div'), value); + arrayWrapper = document.querySelector(`#plugin-additional-fields-div [data-schema-key="${fieldName}"]`); + } + } + } + // Now populate each item + if (arrayWrapper) { + const itemsContainer = arrayWrapper.querySelector('.array-group'); + // Remove existing items + while (itemsContainer && itemsContainer.firstChild) itemsContainer.removeChild(itemsContainer.firstChild); + value.forEach(item => { + this.addArrayItemUI( + (schema && schema.properties && schema.properties[fieldName] && schema.properties[fieldName].items) || {}, + fieldName, + itemsContainer, + item + ); + }); + } + } else if (value && typeof value === 'object') { + this.populateDynamicAdditionalFields(value, fieldName); + } else { + let query = parentKey ? `#plugin-additional-fields-div [data-schema-key="${parentKey}"] [name="${fieldName}"]` : `#plugin-additional-fields-div [name="${fieldName}"]`; + console.log('Querying elements with:', query); + const elements = document.querySelectorAll(query); + console.log('Found elements for field', fieldName, ':', elements); + elements.forEach(el => { + console.log('Setting field:', fieldName, 'with value:', value, 'on element:', el); + if (el.type === 'checkbox') { + el.checked = !!value; + } else if (el.type === 'radio') { + el.checked = el.value == value; + } else if (el.tagName === 'SELECT') { + el.value = value; + } else if (el.tagName === 'TEXTAREA') { + el.value = value; + } else if (el.type === 'number') { + el.value = value !== undefined && value !== null ? Number(value) : ''; + } else { + el.value = value; + } + }); + } + }); + } + + // Private deep merge utility + deepMerge(target, source) { + for (const key in source) { + if (source[key] && typeof source[key] === 'object' && + !Array.isArray(source[key]) && target[key] && typeof target[key] === 'object' && + !Array.isArray(target[key]) + ) { + target[key] = this.deepMerge(target[key], source[key]); + } else { + target[key] = source[key]; + } + } + return target; + } + + // Private method to collect additional fields from both legacy textarea and dynamic UI + collectAdditionalFields() { + // 1. Get from textarea (legacy) + const additionalFieldsValue = document.getElementById('plugin-additional-fields')?.value?.trim() || ''; + let legacyFields = {}; + if (additionalFieldsValue && additionalFieldsValue !== '{}') { + try { + legacyFields = JSON.parse(additionalFieldsValue); + } catch { + // If not valid JSON, skip + } + } + + // 2. Get from dynamic UI + let uiFields = {}; + const additionalFieldsDiv = document.getElementById('plugin-additional-fields-div'); + if (additionalFieldsDiv) { + // Arrays + const arrayWrappers = additionalFieldsDiv.querySelectorAll('.additional-field-array'); + arrayWrappers.forEach(wrapper => { + const arrayGroup = wrapper.querySelector('.array-group'); + if (arrayGroup) { + const arrayKey = arrayGroup.dataset.schemaKey; + const items = []; + // Loop over each .array-item inside .array-group + const arrayItems = arrayGroup.querySelectorAll('.array-item'); + arrayItems.forEach(itemDiv => { + // Check for array of objects (fieldset present) + const fieldset = itemDiv.querySelector('fieldset'); + if (fieldset) { + let obj = {}; + const subInputs = fieldset.querySelectorAll('input, select, textarea'); + subInputs.forEach(subEl => { + let subKey = subEl.name || subEl.id; + if (!subKey) return; + let subValue = subEl.type === 'checkbox' ? subEl.checked : (subEl.type === 'number' ? (subEl.value !== '' ? Number(subEl.value) : '') : subEl.value); + obj[subKey] = subValue; + }); + items.push(obj); + } else { + // Primitive array: find first input/select/textarea directly inside .array-item (not in fieldset or button) + const possibleInputs = Array.from(itemDiv.querySelectorAll('input, select, textarea')); + // Exclude those inside a fieldset or button + const input = possibleInputs.find(el => { + // Not inside a fieldset or button + return !el.closest('fieldset') && !el.closest('button'); + }); + if (input) { + let subValue = input.type === 'checkbox' ? input.checked : (input.type === 'number' ? (input.value !== '' ? Number(input.value) : '') : input.value); + items.push(subValue); + } + } + }); + if (arrayKey) { + uiFields[arrayKey] = items; + } + } + }); + // Objects + const objectWrappers = additionalFieldsDiv.querySelectorAll('.additional-field-object'); + objectWrappers.forEach(wrapper => { + const objFieldset = wrapper.querySelector('fieldset'); + const objKey = objFieldset.dataset.schemaKey; + let obj = {}; + const subInputs = objFieldset.querySelectorAll('input, select, textarea'); + subInputs.forEach(subEl => { + let subKey = subEl.name || subEl.id; + if (!subKey) return; + let subValue = subEl.type === 'checkbox' ? subEl.checked : (subEl.type === 'number' ? (subEl.value !== '' ? Number(subEl.value) : '') : subEl.value); + obj[subKey] = subValue; + }); + if (objKey) { + uiFields[objKey] = obj; + } + }); + // Primitives + const primitiveWrappers = additionalFieldsDiv.querySelectorAll('.additional-field-primitive'); + primitiveWrappers.forEach(wrapper => { + const inputs = wrapper.querySelectorAll('input, select, textarea'); + inputs.forEach(input => { + let key = input.name || input.id; + let value = input.type === 'checkbox' ? input.checked : (input.type === 'number' ? (input.value !== '' ? Number(input.value) : '') : input.value); + uiFields[key] = value; + }); + }); + } + + // 3. Deep merge (UI fields take precedence) + return this.deepMerge(legacyFields, uiFields); + } + + getSafeType(type) { + return type ? type.replace(/[^a-zA-Z0-9_]/g, '_').toLowerCase() : null; + } + + async getAdditionalSettingsSchema(type, options = {}) { + if (!type) return null; + const { useLegacyPattern = false, forceReload = false } = options; + // Normalize type for filename + const safeType = this.getSafeType(type); + // Choose filename pattern + const schemaFile = `${safeType}_plugin.additional_settings.schema.json` + + const schemaPath = `/static/json/schemas/${schemaFile}`; + + // Use cache unless forceReload + if (!forceReload && this.additionalSettingsSchemaCache[safeType]) { + return this.additionalSettingsSchemaCache[safeType]; + } + try { + console.log(`Fetching additional settings schema for type: ${safeType} (pattern: ${safeType})`); + const res = await fetch(schemaPath); + if (res.status === 404) { + console.log(`No additional settings schema found for type: ${type} (404)`); + this.additionalSettingsSchemaCache[safeType] = null; + return null; + } + if (!res.ok) throw new Error(`Failed to load additional settings schema for type: ${type}`); + const schema = await res.json(); + this.additionalSettingsSchemaCache[safeType] = schema; + return schema; + } catch (err) { + console.error(`Error loading additional settings schema for type ${type}:`, err); + this.additionalSettingsSchemaCache[safeType] = null; + return null; + } + } + + // Utility to create a labeled field (refactored from buildAdditionalFieldsUI) + createField(key, prop, parent, prefix = '') { + // If prefix is a number, treat as array index for uniqueness + let fieldId; + if (typeof prefix === 'number') { + fieldId = `${key}_${prefix}`; + } else { + fieldId = `${prefix}${key}`; + } + const wrapper = document.createElement('div'); + wrapper.className = 'mb-3'; + // Label with tooltip if description exists + const label = document.createElement('label'); + label.className = 'form-label'; + label.htmlFor = fieldId; + label.textContent = this.formatLabel(key); + if (prop.description) { + label.title = prop.description; + // Add help icon + const helpIcon = document.createElement('span'); + helpIcon.className = 'ms-1 bi bi-question-circle-fill text-info'; + helpIcon.setAttribute('tabindex', '0'); + helpIcon.setAttribute('data-bs-toggle', 'tooltip'); + helpIcon.setAttribute('title', prop.description); + label.appendChild(helpIcon); + } + wrapper.appendChild(label); + + let input; + if (prop.enum) { + input = document.createElement('select'); + input.className = 'form-select'; + input.id = fieldId; + input.name = key; + prop.enum.forEach(opt => { + const option = document.createElement('option'); + option.value = opt; + option.textContent = this.formatLabel(opt); + option.title = opt; + input.appendChild(option); + }); + if (prop.default) input.value = prop.default; + } else if (prop.type === 'boolean') { + input = document.createElement('input'); + input.type = 'checkbox'; + input.className = 'form-check-input'; + input.id = fieldId; + input.name = key; + input.checked = !!prop.default; + wrapper.className += ' form-check'; + } else if (prop.type === 'number' || prop.type === 'integer') { + input = document.createElement('input'); + input.type = 'number'; + input.className = 'form-control'; + input.id = fieldId; + input.name = key; + if (prop.minimum !== undefined) input.min = prop.minimum; + if (prop.maximum !== undefined) input.max = prop.maximum; + if (prop.default !== undefined) input.value = prop.default; + if (prop.pattern) input.pattern = prop.pattern; + } else if (prop.type === 'string' && prop.format === 'email') { + input = document.createElement('input'); + input.type = 'email'; + input.className = 'form-control'; + input.id = fieldId; + input.name = key; + if (prop.default) input.value = prop.default; + } else if (prop.type === 'string') { + input = document.createElement('input'); + input.type = 'text'; + input.className = 'form-control'; + input.id = fieldId; + input.name = key; + if (prop.minLength !== undefined) input.minLength = prop.minLength; + if (prop.maxLength !== undefined) input.maxLength = prop.maxLength; + if (prop.default) input.value = prop.default; + if (prop.pattern) input.pattern = prop.pattern; + } + if (input) wrapper.appendChild(input); + parent.appendChild(wrapper); + } + + // New: Array field builder for both initial render and dynamic population + addArrayFieldUI(arraySchema, arrayKey, parentDiv, initialValues = []) { + // Create array wrapper + const wrapper = document.createElement('div'); + wrapper.className = 'additional-field-array'; + wrapper.dataset.schemaKey = arrayKey; + + // Title + const label = document.createElement('label'); + label.className = 'form-label'; + label.textContent = this.formatLabel(arrayKey); + wrapper.appendChild(label); + + // Items container + const itemsContainer = document.createElement('div'); + itemsContainer.className = 'array-group'; + itemsContainer.dataset.schemaKey = arrayKey; + wrapper.appendChild(itemsContainer); + + // Add button + const addBtn = document.createElement('button'); + addBtn.type = 'button'; + addBtn.className = 'btn btn-sm btn-outline-primary mb-2'; + addBtn.textContent = 'Add Item'; + addBtn.onclick = () => { + this.addArrayItemUI(arraySchema.items, arrayKey, itemsContainer); + }; + wrapper.appendChild(addBtn); + + // Initial values + if (Array.isArray(initialValues)) { + initialValues.forEach(val => { + this.addArrayItemUI(arraySchema.items, arrayKey, itemsContainer, val); + }); + } + + parentDiv.appendChild(wrapper); + return wrapper; + } + + // Helper to add a single array item + addArrayItemUI(itemSchema, arrayKey, itemsContainer, initialValue = undefined) { + const itemDiv = document.createElement('div'); + itemDiv.className = 'array-item mb-2 p-2 border rounded'; + // Remove button + const removeBtn = document.createElement('button'); + removeBtn.type = 'button'; + removeBtn.className = 'btn btn-sm btn-outline-danger float-end'; + removeBtn.textContent = 'Remove'; + removeBtn.onclick = () => { + itemsContainer.removeChild(itemDiv); + }; + itemDiv.appendChild(removeBtn); + // Determine index for uniqueness + let index = itemsContainer.childNodes.length; + // Render item fields + if (itemSchema.type === 'object' && itemSchema.properties) { + // Create a fieldset for the object item + const fieldset = document.createElement('fieldset'); + fieldset.dataset.schemaKey = arrayKey; + // Optionally add a legend for the object item + const legend = document.createElement('legend'); + legend.textContent = this.formatLabel(arrayKey); + fieldset.appendChild(legend); + Object.entries(itemSchema.properties).forEach(([subKey, subProp]) => { + this.createField(subKey, subProp, fieldset, index); + // Set initial value if provided + if (initialValue && initialValue[subKey] !== undefined) { + const input = fieldset.querySelector(`[name="${subKey}"]`); + if (input) input.value = initialValue[subKey]; + } + }); + itemDiv.appendChild(fieldset); + } else { + // Primitive array + this.createField(arrayKey, itemSchema, itemDiv, index); + if (initialValue !== undefined) { + const input = itemDiv.querySelector(`[name="${arrayKey}"]`); + if (input) input.value = initialValue; + } + } + itemsContainer.appendChild(itemDiv); + } } // Create global instance diff --git a/application/single_app/static/js/validatePlugin.mjs b/application/single_app/static/js/validatePlugin.mjs index f43176b0..5e54f76d 100644 --- a/application/single_app/static/js/validatePlugin.mjs +++ b/application/single_app/static/js/validatePlugin.mjs @@ -1 +1 @@ -"use strict";export const validate = validate11;export default validate11;const schema13 = {"$schema":"http://json-schema.org/draft-07/schema#","$ref":"#/definitions/Plugin","definitions":{"Plugin":{"type":"object","additionalProperties":false,"properties":{"id":{"type":"string","description":"Plugin unique identifier (UUID)"},"user_id":{"type":"string","description":"User ID that owns this personal plugin"},"last_updated":{"type":"string","description":"ISO timestamp of last update"},"name":{"type":"string","pattern":"^[A-Za-z0-9_-]+$","description":"Alphanumeric, underscore, and dash only"},"displayName":{"type":"string","description":"Human-readable display name for the plugin"},"type":{"type":"string"},"description":{"type":"string"},"endpoint":{"type":"string"},"auth":{"type":"object","properties":{"type":{"type":"string","enum":["key","identity","user","servicePrincipal"],"description":"Auth type must be 'key', 'user', 'identity', or 'servicePrincipal'"},"key":{"type":"string"},"identity":{"type":"string"},"tenantId":{"type":"string"}},"required":["type"],"additionalProperties":false},"metadata":{"type":"object","description":"Arbitrary metadata","additionalProperties":true},"additionalFields":{"type":"object","description":"Arbitrary additional fields","additionalProperties":true}},"required":["name","type","description","endpoint","auth","metadata","additionalFields"],"title":"Plugin"}}};const schema14 = {"type":"object","additionalProperties":false,"properties":{"id":{"type":"string","description":"Plugin unique identifier (UUID)"},"user_id":{"type":"string","description":"User ID that owns this personal plugin"},"last_updated":{"type":"string","description":"ISO timestamp of last update"},"name":{"type":"string","pattern":"^[A-Za-z0-9_-]+$","description":"Alphanumeric, underscore, and dash only"},"displayName":{"type":"string","description":"Human-readable display name for the plugin"},"type":{"type":"string"},"description":{"type":"string"},"endpoint":{"type":"string"},"auth":{"type":"object","properties":{"type":{"type":"string","enum":["key","identity","user","servicePrincipal"],"description":"Auth type must be 'key', 'user', 'identity', or 'servicePrincipal'"},"key":{"type":"string"},"identity":{"type":"string"},"tenantId":{"type":"string"}},"required":["type"],"additionalProperties":false},"metadata":{"type":"object","description":"Arbitrary metadata","additionalProperties":true},"additionalFields":{"type":"object","description":"Arbitrary additional fields","additionalProperties":true}},"required":["name","type","description","endpoint","auth","metadata","additionalFields"],"title":"Plugin"};const func2 = Object.prototype.hasOwnProperty;const pattern1 = new RegExp("^[A-Za-z0-9_-]+$", "u");function validate11(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){let vErrors = null;let errors = 0;const _errs0 = errors;if(errors === _errs0){if(data && typeof data == "object" && !Array.isArray(data)){let missing0;if((((((((data.name === undefined) && (missing0 = "name")) || ((data.type === undefined) && (missing0 = "type"))) || ((data.description === undefined) && (missing0 = "description"))) || ((data.endpoint === undefined) && (missing0 = "endpoint"))) || ((data.auth === undefined) && (missing0 = "auth"))) || ((data.metadata === undefined) && (missing0 = "metadata"))) || ((data.additionalFields === undefined) && (missing0 = "additionalFields"))){validate11.errors = [{instancePath,schemaPath:"#/definitions/Plugin/required",keyword:"required",params:{missingProperty: missing0},message:"must have required property '"+missing0+"'"}];return false;}else {const _errs2 = errors;for(const key0 in data){if(!(func2.call(schema14.properties, key0))){validate11.errors = [{instancePath,schemaPath:"#/definitions/Plugin/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key0},message:"must NOT have additional properties"}];return false;break;}}if(_errs2 === errors){if(data.id !== undefined){const _errs3 = errors;if(typeof data.id !== "string"){validate11.errors = [{instancePath:instancePath+"/id",schemaPath:"#/definitions/Plugin/properties/id/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}var valid1 = _errs3 === errors;}else {var valid1 = true;}if(valid1){if(data.user_id !== undefined){const _errs5 = errors;if(typeof data.user_id !== "string"){validate11.errors = [{instancePath:instancePath+"/user_id",schemaPath:"#/definitions/Plugin/properties/user_id/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}var valid1 = _errs5 === errors;}else {var valid1 = true;}if(valid1){if(data.last_updated !== undefined){const _errs7 = errors;if(typeof data.last_updated !== "string"){validate11.errors = [{instancePath:instancePath+"/last_updated",schemaPath:"#/definitions/Plugin/properties/last_updated/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}var valid1 = _errs7 === errors;}else {var valid1 = true;}if(valid1){if(data.name !== undefined){let data3 = data.name;const _errs9 = errors;if(errors === _errs9){if(typeof data3 === "string"){if(!pattern1.test(data3)){validate11.errors = [{instancePath:instancePath+"/name",schemaPath:"#/definitions/Plugin/properties/name/pattern",keyword:"pattern",params:{pattern: "^[A-Za-z0-9_-]+$"},message:"must match pattern \""+"^[A-Za-z0-9_-]+$"+"\""}];return false;}}else {validate11.errors = [{instancePath:instancePath+"/name",schemaPath:"#/definitions/Plugin/properties/name/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}}var valid1 = _errs9 === errors;}else {var valid1 = true;}if(valid1){if(data.displayName !== undefined){const _errs11 = errors;if(typeof data.displayName !== "string"){validate11.errors = [{instancePath:instancePath+"/displayName",schemaPath:"#/definitions/Plugin/properties/displayName/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}var valid1 = _errs11 === errors;}else {var valid1 = true;}if(valid1){if(data.type !== undefined){const _errs13 = errors;if(typeof data.type !== "string"){validate11.errors = [{instancePath:instancePath+"/type",schemaPath:"#/definitions/Plugin/properties/type/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}var valid1 = _errs13 === errors;}else {var valid1 = true;}if(valid1){if(data.description !== undefined){const _errs15 = errors;if(typeof data.description !== "string"){validate11.errors = [{instancePath:instancePath+"/description",schemaPath:"#/definitions/Plugin/properties/description/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}var valid1 = _errs15 === errors;}else {var valid1 = true;}if(valid1){if(data.endpoint !== undefined){const _errs17 = errors;if(typeof data.endpoint !== "string"){validate11.errors = [{instancePath:instancePath+"/endpoint",schemaPath:"#/definitions/Plugin/properties/endpoint/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}var valid1 = _errs17 === errors;}else {var valid1 = true;}if(valid1){if(data.auth !== undefined){let data8 = data.auth;const _errs19 = errors;if(errors === _errs19){if(data8 && typeof data8 == "object" && !Array.isArray(data8)){let missing1;if((data8.type === undefined) && (missing1 = "type")){validate11.errors = [{instancePath:instancePath+"/auth",schemaPath:"#/definitions/Plugin/properties/auth/required",keyword:"required",params:{missingProperty: missing1},message:"must have required property '"+missing1+"'"}];return false;}else {const _errs21 = errors;for(const key1 in data8){if(!((((key1 === "type") || (key1 === "key")) || (key1 === "identity")) || (key1 === "tenantId"))){validate11.errors = [{instancePath:instancePath+"/auth",schemaPath:"#/definitions/Plugin/properties/auth/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key1},message:"must NOT have additional properties"}];return false;break;}}if(_errs21 === errors){if(data8.type !== undefined){let data9 = data8.type;const _errs22 = errors;if(typeof data9 !== "string"){validate11.errors = [{instancePath:instancePath+"/auth/type",schemaPath:"#/definitions/Plugin/properties/auth/properties/type/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}if(!((((data9 === "key") || (data9 === "identity")) || (data9 === "user")) || (data9 === "servicePrincipal"))){validate11.errors = [{instancePath:instancePath+"/auth/type",schemaPath:"#/definitions/Plugin/properties/auth/properties/type/enum",keyword:"enum",params:{allowedValues: schema14.properties.auth.properties.type.enum},message:"must be equal to one of the allowed values"}];return false;}var valid2 = _errs22 === errors;}else {var valid2 = true;}if(valid2){if(data8.key !== undefined){const _errs24 = errors;if(typeof data8.key !== "string"){validate11.errors = [{instancePath:instancePath+"/auth/key",schemaPath:"#/definitions/Plugin/properties/auth/properties/key/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}var valid2 = _errs24 === errors;}else {var valid2 = true;}if(valid2){if(data8.identity !== undefined){const _errs26 = errors;if(typeof data8.identity !== "string"){validate11.errors = [{instancePath:instancePath+"/auth/identity",schemaPath:"#/definitions/Plugin/properties/auth/properties/identity/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}var valid2 = _errs26 === errors;}else {var valid2 = true;}if(valid2){if(data8.tenantId !== undefined){const _errs28 = errors;if(typeof data8.tenantId !== "string"){validate11.errors = [{instancePath:instancePath+"/auth/tenantId",schemaPath:"#/definitions/Plugin/properties/auth/properties/tenantId/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}var valid2 = _errs28 === errors;}else {var valid2 = true;}}}}}}}else {validate11.errors = [{instancePath:instancePath+"/auth",schemaPath:"#/definitions/Plugin/properties/auth/type",keyword:"type",params:{type: "object"},message:"must be object"}];return false;}}var valid1 = _errs19 === errors;}else {var valid1 = true;}if(valid1){if(data.metadata !== undefined){let data13 = data.metadata;const _errs30 = errors;if(errors === _errs30){if(data13 && typeof data13 == "object" && !Array.isArray(data13)){}else {validate11.errors = [{instancePath:instancePath+"/metadata",schemaPath:"#/definitions/Plugin/properties/metadata/type",keyword:"type",params:{type: "object"},message:"must be object"}];return false;}}var valid1 = _errs30 === errors;}else {var valid1 = true;}if(valid1){if(data.additionalFields !== undefined){let data14 = data.additionalFields;const _errs33 = errors;if(errors === _errs33){if(data14 && typeof data14 == "object" && !Array.isArray(data14)){}else {validate11.errors = [{instancePath:instancePath+"/additionalFields",schemaPath:"#/definitions/Plugin/properties/additionalFields/type",keyword:"type",params:{type: "object"},message:"must be object"}];return false;}}var valid1 = _errs33 === errors;}else {var valid1 = true;}}}}}}}}}}}}}}else {validate11.errors = [{instancePath,schemaPath:"#/definitions/Plugin/type",keyword:"type",params:{type: "object"},message:"must be object"}];return false;}}validate11.errors = vErrors;return errors === 0;} \ No newline at end of file +"use strict";export const validate = validate11;export default validate11;const schema13 = {"$schema":"http://json-schema.org/draft-07/schema#","$ref":"#/definitions/Plugin","definitions":{"Plugin":{"type":"object","additionalProperties":false,"properties":{"id":{"type":"string","description":"Plugin unique identifier (UUID)"},"user_id":{"type":"string","description":"User ID that owns this personal plugin"},"last_updated":{"type":"string","description":"ISO timestamp of last update"},"name":{"type":"string","pattern":"^[A-Za-z0-9_-]+$","description":"Alphanumeric, underscore, and dash only"},"displayName":{"type":"string","description":"Human-readable display name for the plugin"},"type":{"type":"string"},"description":{"type":"string"},"endpoint":{"type":"string"},"auth":{"type":"object","properties":{"type":{"type":"string","enum":["key","identity","user","servicePrincipal","connection_string","basic","username_password"],"description":"Auth type must be 'key', 'user', 'identity', 'servicePrincipal', 'connection_string', 'basic', or 'username_password'"},"key":{"type":"string","description":"The secret value for the plugin should be stored here, such as a SQL connection string, a password for a service principal or username/password combination"},"identity":{"type":"string","description":"This could be the Id of an (managed) identity, a user name, or similar to pair with the key, in most situations"},"tenantId":{"type":"string","description":"The Azure AD tenant ID used with Service Principal authentication"}},"additionalProperties":false,"allOf":[{"if":{"properties":{"type":{"const":"key"}}},"then":{"required":["type","key"]}},{"if":{"properties":{"type":{"const":"identity"}}},"then":{"required":["type","identity"]}},{"if":{"properties":{"type":{"const":"user"}}},"then":{"required":["type"]}},{"if":{"properties":{"type":{"const":"servicePrincipal"}}},"then":{"required":["type","tenantId","identity","key"]}},{"if":{"properties":{"type":{"const":"connection_string"}}},"then":{"required":["type","key"]}},{"if":{"properties":{"type":{"const":"basic"}}},"then":{"required":["type","key","identity"]}},{"if":{"properties":{"type":{"const":"username_password"}}},"then":{"required":["type","key","identity"]}},{"required":["type"]}]},"metadata":{"type":"object","description":"Arbitrary metadata","additionalProperties":true},"additionalFields":{"type":"object","description":"Additional fields for plugin configuration based on plugin type. See plugin documentation for details. Any fields named __Secret (double underscore) will be stored in key vault if the feature is enabled.","additionalProperties":true}},"required":["name","type","description","endpoint","auth","metadata","additionalFields"],"title":"Plugin"}}};const schema14 = {"type":"object","additionalProperties":false,"properties":{"id":{"type":"string","description":"Plugin unique identifier (UUID)"},"user_id":{"type":"string","description":"User ID that owns this personal plugin"},"last_updated":{"type":"string","description":"ISO timestamp of last update"},"name":{"type":"string","pattern":"^[A-Za-z0-9_-]+$","description":"Alphanumeric, underscore, and dash only"},"displayName":{"type":"string","description":"Human-readable display name for the plugin"},"type":{"type":"string"},"description":{"type":"string"},"endpoint":{"type":"string"},"auth":{"type":"object","properties":{"type":{"type":"string","enum":["key","identity","user","servicePrincipal","connection_string","basic","username_password"],"description":"Auth type must be 'key', 'user', 'identity', 'servicePrincipal', 'connection_string', 'basic', or 'username_password'"},"key":{"type":"string","description":"The secret value for the plugin should be stored here, such as a SQL connection string, a password for a service principal or username/password combination"},"identity":{"type":"string","description":"This could be the Id of an (managed) identity, a user name, or similar to pair with the key, in most situations"},"tenantId":{"type":"string","description":"The Azure AD tenant ID used with Service Principal authentication"}},"additionalProperties":false,"allOf":[{"if":{"properties":{"type":{"const":"key"}}},"then":{"required":["type","key"]}},{"if":{"properties":{"type":{"const":"identity"}}},"then":{"required":["type","identity"]}},{"if":{"properties":{"type":{"const":"user"}}},"then":{"required":["type"]}},{"if":{"properties":{"type":{"const":"servicePrincipal"}}},"then":{"required":["type","tenantId","identity","key"]}},{"if":{"properties":{"type":{"const":"connection_string"}}},"then":{"required":["type","key"]}},{"if":{"properties":{"type":{"const":"basic"}}},"then":{"required":["type","key","identity"]}},{"if":{"properties":{"type":{"const":"username_password"}}},"then":{"required":["type","key","identity"]}},{"required":["type"]}]},"metadata":{"type":"object","description":"Arbitrary metadata","additionalProperties":true},"additionalFields":{"type":"object","description":"Additional fields for plugin configuration based on plugin type. See plugin documentation for details. Any fields named __Secret (double underscore) will be stored in key vault if the feature is enabled.","additionalProperties":true}},"required":["name","type","description","endpoint","auth","metadata","additionalFields"],"title":"Plugin"};const func2 = Object.prototype.hasOwnProperty;const pattern1 = new RegExp("^[A-Za-z0-9_-]+$", "u");function validate11(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){let vErrors = null;let errors = 0;const _errs0 = errors;if(errors === _errs0){if(data && typeof data == "object" && !Array.isArray(data)){let missing0;if((((((((data.name === undefined) && (missing0 = "name")) || ((data.type === undefined) && (missing0 = "type"))) || ((data.description === undefined) && (missing0 = "description"))) || ((data.endpoint === undefined) && (missing0 = "endpoint"))) || ((data.auth === undefined) && (missing0 = "auth"))) || ((data.metadata === undefined) && (missing0 = "metadata"))) || ((data.additionalFields === undefined) && (missing0 = "additionalFields"))){validate11.errors = [{instancePath,schemaPath:"#/definitions/Plugin/required",keyword:"required",params:{missingProperty: missing0},message:"must have required property '"+missing0+"'"}];return false;}else {const _errs2 = errors;for(const key0 in data){if(!(func2.call(schema14.properties, key0))){validate11.errors = [{instancePath,schemaPath:"#/definitions/Plugin/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key0},message:"must NOT have additional properties"}];return false;break;}}if(_errs2 === errors){if(data.id !== undefined){const _errs3 = errors;if(typeof data.id !== "string"){validate11.errors = [{instancePath:instancePath+"/id",schemaPath:"#/definitions/Plugin/properties/id/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}var valid1 = _errs3 === errors;}else {var valid1 = true;}if(valid1){if(data.user_id !== undefined){const _errs5 = errors;if(typeof data.user_id !== "string"){validate11.errors = [{instancePath:instancePath+"/user_id",schemaPath:"#/definitions/Plugin/properties/user_id/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}var valid1 = _errs5 === errors;}else {var valid1 = true;}if(valid1){if(data.last_updated !== undefined){const _errs7 = errors;if(typeof data.last_updated !== "string"){validate11.errors = [{instancePath:instancePath+"/last_updated",schemaPath:"#/definitions/Plugin/properties/last_updated/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}var valid1 = _errs7 === errors;}else {var valid1 = true;}if(valid1){if(data.name !== undefined){let data3 = data.name;const _errs9 = errors;if(errors === _errs9){if(typeof data3 === "string"){if(!pattern1.test(data3)){validate11.errors = [{instancePath:instancePath+"/name",schemaPath:"#/definitions/Plugin/properties/name/pattern",keyword:"pattern",params:{pattern: "^[A-Za-z0-9_-]+$"},message:"must match pattern \""+"^[A-Za-z0-9_-]+$"+"\""}];return false;}}else {validate11.errors = [{instancePath:instancePath+"/name",schemaPath:"#/definitions/Plugin/properties/name/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}}var valid1 = _errs9 === errors;}else {var valid1 = true;}if(valid1){if(data.displayName !== undefined){const _errs11 = errors;if(typeof data.displayName !== "string"){validate11.errors = [{instancePath:instancePath+"/displayName",schemaPath:"#/definitions/Plugin/properties/displayName/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}var valid1 = _errs11 === errors;}else {var valid1 = true;}if(valid1){if(data.type !== undefined){const _errs13 = errors;if(typeof data.type !== "string"){validate11.errors = [{instancePath:instancePath+"/type",schemaPath:"#/definitions/Plugin/properties/type/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}var valid1 = _errs13 === errors;}else {var valid1 = true;}if(valid1){if(data.description !== undefined){const _errs15 = errors;if(typeof data.description !== "string"){validate11.errors = [{instancePath:instancePath+"/description",schemaPath:"#/definitions/Plugin/properties/description/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}var valid1 = _errs15 === errors;}else {var valid1 = true;}if(valid1){if(data.endpoint !== undefined){const _errs17 = errors;if(typeof data.endpoint !== "string"){validate11.errors = [{instancePath:instancePath+"/endpoint",schemaPath:"#/definitions/Plugin/properties/endpoint/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}var valid1 = _errs17 === errors;}else {var valid1 = true;}if(valid1){if(data.auth !== undefined){let data8 = data.auth;const _errs19 = errors;const _errs21 = errors;const _errs22 = errors;let valid3 = true;const _errs23 = errors;if(data8 && typeof data8 == "object" && !Array.isArray(data8)){if(data8.type !== undefined){if("key" !== data8.type){const err0 = {};if(vErrors === null){vErrors = [err0];}else {vErrors.push(err0);}errors++;}}}var _valid0 = _errs23 === errors;errors = _errs22;if(vErrors !== null){if(_errs22){vErrors.length = _errs22;}else {vErrors = null;}}if(_valid0){const _errs25 = errors;if(data8 && typeof data8 == "object" && !Array.isArray(data8)){let missing1;if(((data8.type === undefined) && (missing1 = "type")) || ((data8.key === undefined) && (missing1 = "key"))){validate11.errors = [{instancePath:instancePath+"/auth",schemaPath:"#/definitions/Plugin/properties/auth/allOf/0/then/required",keyword:"required",params:{missingProperty: missing1},message:"must have required property '"+missing1+"'"}];return false;}}var _valid0 = _errs25 === errors;valid3 = _valid0;}if(!valid3){const err1 = {instancePath:instancePath+"/auth",schemaPath:"#/definitions/Plugin/properties/auth/allOf/0/if",keyword:"if",params:{failingKeyword: "then"},message:"must match \"then\" schema"};if(vErrors === null){vErrors = [err1];}else {vErrors.push(err1);}errors++;validate11.errors = vErrors;return false;}var valid2 = _errs21 === errors;if(valid2){const _errs26 = errors;const _errs27 = errors;let valid5 = true;const _errs28 = errors;if(data8 && typeof data8 == "object" && !Array.isArray(data8)){if(data8.type !== undefined){if("identity" !== data8.type){const err2 = {};if(vErrors === null){vErrors = [err2];}else {vErrors.push(err2);}errors++;}}}var _valid1 = _errs28 === errors;errors = _errs27;if(vErrors !== null){if(_errs27){vErrors.length = _errs27;}else {vErrors = null;}}if(_valid1){const _errs30 = errors;if(data8 && typeof data8 == "object" && !Array.isArray(data8)){let missing2;if(((data8.type === undefined) && (missing2 = "type")) || ((data8.identity === undefined) && (missing2 = "identity"))){validate11.errors = [{instancePath:instancePath+"/auth",schemaPath:"#/definitions/Plugin/properties/auth/allOf/1/then/required",keyword:"required",params:{missingProperty: missing2},message:"must have required property '"+missing2+"'"}];return false;}}var _valid1 = _errs30 === errors;valid5 = _valid1;}if(!valid5){const err3 = {instancePath:instancePath+"/auth",schemaPath:"#/definitions/Plugin/properties/auth/allOf/1/if",keyword:"if",params:{failingKeyword: "then"},message:"must match \"then\" schema"};if(vErrors === null){vErrors = [err3];}else {vErrors.push(err3);}errors++;validate11.errors = vErrors;return false;}var valid2 = _errs26 === errors;if(valid2){const _errs31 = errors;const _errs32 = errors;let valid7 = true;const _errs33 = errors;if(data8 && typeof data8 == "object" && !Array.isArray(data8)){if(data8.type !== undefined){if("user" !== data8.type){const err4 = {};if(vErrors === null){vErrors = [err4];}else {vErrors.push(err4);}errors++;}}}var _valid2 = _errs33 === errors;errors = _errs32;if(vErrors !== null){if(_errs32){vErrors.length = _errs32;}else {vErrors = null;}}if(_valid2){const _errs35 = errors;if(data8 && typeof data8 == "object" && !Array.isArray(data8)){let missing3;if((data8.type === undefined) && (missing3 = "type")){validate11.errors = [{instancePath:instancePath+"/auth",schemaPath:"#/definitions/Plugin/properties/auth/allOf/2/then/required",keyword:"required",params:{missingProperty: missing3},message:"must have required property '"+missing3+"'"}];return false;}}var _valid2 = _errs35 === errors;valid7 = _valid2;}if(!valid7){const err5 = {instancePath:instancePath+"/auth",schemaPath:"#/definitions/Plugin/properties/auth/allOf/2/if",keyword:"if",params:{failingKeyword: "then"},message:"must match \"then\" schema"};if(vErrors === null){vErrors = [err5];}else {vErrors.push(err5);}errors++;validate11.errors = vErrors;return false;}var valid2 = _errs31 === errors;if(valid2){const _errs36 = errors;const _errs37 = errors;let valid9 = true;const _errs38 = errors;if(data8 && typeof data8 == "object" && !Array.isArray(data8)){if(data8.type !== undefined){if("servicePrincipal" !== data8.type){const err6 = {};if(vErrors === null){vErrors = [err6];}else {vErrors.push(err6);}errors++;}}}var _valid3 = _errs38 === errors;errors = _errs37;if(vErrors !== null){if(_errs37){vErrors.length = _errs37;}else {vErrors = null;}}if(_valid3){const _errs40 = errors;if(data8 && typeof data8 == "object" && !Array.isArray(data8)){let missing4;if(((((data8.type === undefined) && (missing4 = "type")) || ((data8.tenantId === undefined) && (missing4 = "tenantId"))) || ((data8.identity === undefined) && (missing4 = "identity"))) || ((data8.key === undefined) && (missing4 = "key"))){validate11.errors = [{instancePath:instancePath+"/auth",schemaPath:"#/definitions/Plugin/properties/auth/allOf/3/then/required",keyword:"required",params:{missingProperty: missing4},message:"must have required property '"+missing4+"'"}];return false;}}var _valid3 = _errs40 === errors;valid9 = _valid3;}if(!valid9){const err7 = {instancePath:instancePath+"/auth",schemaPath:"#/definitions/Plugin/properties/auth/allOf/3/if",keyword:"if",params:{failingKeyword: "then"},message:"must match \"then\" schema"};if(vErrors === null){vErrors = [err7];}else {vErrors.push(err7);}errors++;validate11.errors = vErrors;return false;}var valid2 = _errs36 === errors;if(valid2){const _errs41 = errors;const _errs42 = errors;let valid11 = true;const _errs43 = errors;if(data8 && typeof data8 == "object" && !Array.isArray(data8)){if(data8.type !== undefined){if("connection_string" !== data8.type){const err8 = {};if(vErrors === null){vErrors = [err8];}else {vErrors.push(err8);}errors++;}}}var _valid4 = _errs43 === errors;errors = _errs42;if(vErrors !== null){if(_errs42){vErrors.length = _errs42;}else {vErrors = null;}}if(_valid4){const _errs45 = errors;if(data8 && typeof data8 == "object" && !Array.isArray(data8)){let missing5;if(((data8.type === undefined) && (missing5 = "type")) || ((data8.key === undefined) && (missing5 = "key"))){validate11.errors = [{instancePath:instancePath+"/auth",schemaPath:"#/definitions/Plugin/properties/auth/allOf/4/then/required",keyword:"required",params:{missingProperty: missing5},message:"must have required property '"+missing5+"'"}];return false;}}var _valid4 = _errs45 === errors;valid11 = _valid4;}if(!valid11){const err9 = {instancePath:instancePath+"/auth",schemaPath:"#/definitions/Plugin/properties/auth/allOf/4/if",keyword:"if",params:{failingKeyword: "then"},message:"must match \"then\" schema"};if(vErrors === null){vErrors = [err9];}else {vErrors.push(err9);}errors++;validate11.errors = vErrors;return false;}var valid2 = _errs41 === errors;if(valid2){const _errs46 = errors;const _errs47 = errors;let valid13 = true;const _errs48 = errors;if(data8 && typeof data8 == "object" && !Array.isArray(data8)){if(data8.type !== undefined){if("basic" !== data8.type){const err10 = {};if(vErrors === null){vErrors = [err10];}else {vErrors.push(err10);}errors++;}}}var _valid5 = _errs48 === errors;errors = _errs47;if(vErrors !== null){if(_errs47){vErrors.length = _errs47;}else {vErrors = null;}}if(_valid5){const _errs50 = errors;if(data8 && typeof data8 == "object" && !Array.isArray(data8)){let missing6;if((((data8.type === undefined) && (missing6 = "type")) || ((data8.key === undefined) && (missing6 = "key"))) || ((data8.identity === undefined) && (missing6 = "identity"))){validate11.errors = [{instancePath:instancePath+"/auth",schemaPath:"#/definitions/Plugin/properties/auth/allOf/5/then/required",keyword:"required",params:{missingProperty: missing6},message:"must have required property '"+missing6+"'"}];return false;}}var _valid5 = _errs50 === errors;valid13 = _valid5;}if(!valid13){const err11 = {instancePath:instancePath+"/auth",schemaPath:"#/definitions/Plugin/properties/auth/allOf/5/if",keyword:"if",params:{failingKeyword: "then"},message:"must match \"then\" schema"};if(vErrors === null){vErrors = [err11];}else {vErrors.push(err11);}errors++;validate11.errors = vErrors;return false;}var valid2 = _errs46 === errors;if(valid2){const _errs51 = errors;const _errs52 = errors;let valid15 = true;const _errs53 = errors;if(data8 && typeof data8 == "object" && !Array.isArray(data8)){if(data8.type !== undefined){if("username_password" !== data8.type){const err12 = {};if(vErrors === null){vErrors = [err12];}else {vErrors.push(err12);}errors++;}}}var _valid6 = _errs53 === errors;errors = _errs52;if(vErrors !== null){if(_errs52){vErrors.length = _errs52;}else {vErrors = null;}}if(_valid6){const _errs55 = errors;if(data8 && typeof data8 == "object" && !Array.isArray(data8)){let missing7;if((((data8.type === undefined) && (missing7 = "type")) || ((data8.key === undefined) && (missing7 = "key"))) || ((data8.identity === undefined) && (missing7 = "identity"))){validate11.errors = [{instancePath:instancePath+"/auth",schemaPath:"#/definitions/Plugin/properties/auth/allOf/6/then/required",keyword:"required",params:{missingProperty: missing7},message:"must have required property '"+missing7+"'"}];return false;}}var _valid6 = _errs55 === errors;valid15 = _valid6;}if(!valid15){const err13 = {instancePath:instancePath+"/auth",schemaPath:"#/definitions/Plugin/properties/auth/allOf/6/if",keyword:"if",params:{failingKeyword: "then"},message:"must match \"then\" schema"};if(vErrors === null){vErrors = [err13];}else {vErrors.push(err13);}errors++;validate11.errors = vErrors;return false;}var valid2 = _errs51 === errors;if(valid2){const _errs56 = errors;if(data8 && typeof data8 == "object" && !Array.isArray(data8)){let missing8;if((data8.type === undefined) && (missing8 = "type")){validate11.errors = [{instancePath:instancePath+"/auth",schemaPath:"#/definitions/Plugin/properties/auth/allOf/7/required",keyword:"required",params:{missingProperty: missing8},message:"must have required property '"+missing8+"'"}];return false;}}var valid2 = _errs56 === errors;}}}}}}}if(errors === _errs19){if(data8 && typeof data8 == "object" && !Array.isArray(data8)){const _errs57 = errors;for(const key1 in data8){if(!((((key1 === "type") || (key1 === "key")) || (key1 === "identity")) || (key1 === "tenantId"))){validate11.errors = [{instancePath:instancePath+"/auth",schemaPath:"#/definitions/Plugin/properties/auth/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key1},message:"must NOT have additional properties"}];return false;break;}}if(_errs57 === errors){if(data8.type !== undefined){let data16 = data8.type;const _errs58 = errors;if(typeof data16 !== "string"){validate11.errors = [{instancePath:instancePath+"/auth/type",schemaPath:"#/definitions/Plugin/properties/auth/properties/type/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}if(!(((((((data16 === "key") || (data16 === "identity")) || (data16 === "user")) || (data16 === "servicePrincipal")) || (data16 === "connection_string")) || (data16 === "basic")) || (data16 === "username_password"))){validate11.errors = [{instancePath:instancePath+"/auth/type",schemaPath:"#/definitions/Plugin/properties/auth/properties/type/enum",keyword:"enum",params:{allowedValues: schema14.properties.auth.properties.type.enum},message:"must be equal to one of the allowed values"}];return false;}var valid17 = _errs58 === errors;}else {var valid17 = true;}if(valid17){if(data8.key !== undefined){const _errs60 = errors;if(typeof data8.key !== "string"){validate11.errors = [{instancePath:instancePath+"/auth/key",schemaPath:"#/definitions/Plugin/properties/auth/properties/key/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}var valid17 = _errs60 === errors;}else {var valid17 = true;}if(valid17){if(data8.identity !== undefined){const _errs62 = errors;if(typeof data8.identity !== "string"){validate11.errors = [{instancePath:instancePath+"/auth/identity",schemaPath:"#/definitions/Plugin/properties/auth/properties/identity/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}var valid17 = _errs62 === errors;}else {var valid17 = true;}if(valid17){if(data8.tenantId !== undefined){const _errs64 = errors;if(typeof data8.tenantId !== "string"){validate11.errors = [{instancePath:instancePath+"/auth/tenantId",schemaPath:"#/definitions/Plugin/properties/auth/properties/tenantId/type",keyword:"type",params:{type: "string"},message:"must be string"}];return false;}var valid17 = _errs64 === errors;}else {var valid17 = true;}}}}}}else {validate11.errors = [{instancePath:instancePath+"/auth",schemaPath:"#/definitions/Plugin/properties/auth/type",keyword:"type",params:{type: "object"},message:"must be object"}];return false;}}var valid1 = _errs19 === errors;}else {var valid1 = true;}if(valid1){if(data.metadata !== undefined){let data20 = data.metadata;const _errs66 = errors;if(errors === _errs66){if(data20 && typeof data20 == "object" && !Array.isArray(data20)){}else {validate11.errors = [{instancePath:instancePath+"/metadata",schemaPath:"#/definitions/Plugin/properties/metadata/type",keyword:"type",params:{type: "object"},message:"must be object"}];return false;}}var valid1 = _errs66 === errors;}else {var valid1 = true;}if(valid1){if(data.additionalFields !== undefined){let data21 = data.additionalFields;const _errs69 = errors;if(errors === _errs69){if(data21 && typeof data21 == "object" && !Array.isArray(data21)){}else {validate11.errors = [{instancePath:instancePath+"/additionalFields",schemaPath:"#/definitions/Plugin/properties/additionalFields/type",keyword:"type",params:{type: "object"},message:"must be object"}];return false;}}var valid1 = _errs69 === errors;}else {var valid1 = true;}}}}}}}}}}}}}}else {validate11.errors = [{instancePath,schemaPath:"#/definitions/Plugin/type",keyword:"type",params:{type: "object"},message:"must be object"}];return false;}}validate11.errors = vErrors;return errors === 0;} \ No newline at end of file diff --git a/application/single_app/static/js/workspace/workspace_plugins.js b/application/single_app/static/js/workspace/workspace_plugins.js index 61ce9a2f..30fef0d5 100644 --- a/application/single_app/static/js/workspace/workspace_plugins.js +++ b/application/single_app/static/js/workspace/workspace_plugins.js @@ -88,7 +88,11 @@ function setupSaveHandler(plugin, modal) { saveBtn.onclick = async (event) => { event.preventDefault(); - + const errorDiv = document.getElementById('plugin-modal-error'); + if (errorDiv) { + errorDiv.classList.add('d-none'); + errorDiv.textContent = ''; + } try { // Get form data from the stepper const formData = window.pluginModalStepper.getFormData(); @@ -100,8 +104,19 @@ function setupSaveHandler(plugin, modal) { return; } + const originalText = saveBtn.innerHTML; + saveBtn.innerHTML = `Saving...`; + saveBtn.disabled = true; // Save the action - await savePlugin(formData, plugin); + try { + await savePlugin(formData, plugin); + } catch (error) { + window.pluginModalStepper.showError(error.message); + return; + } finally { + saveBtn.innerHTML = originalText; + saveBtn.disabled = false; + } // Close modal and refresh if (modal && typeof modal.hide === 'function') { @@ -124,6 +139,7 @@ function setupSaveHandler(plugin, modal) { async function savePlugin(pluginData, existingPlugin = null) { // Get all plugins first const res = await fetch('/api/user/plugins'); + if (!res.ok) throw new Error('Failed to load existing actions'); let plugins = await res.json(); diff --git a/application/single_app/static/json/schemas/PLUGIN_SCHEMAS.md b/application/single_app/static/json/schemas/PLUGIN_SCHEMAS.md new file mode 100644 index 00000000..3fca9371 --- /dev/null +++ b/application/single_app/static/json/schemas/PLUGIN_SCHEMAS.md @@ -0,0 +1,18 @@ +# Plugin Schemas + +This document provides information on how plugin schemas are structured and how to define them for your plugins. + +## Overview + +### .plugin.schema.json files + +These files define the main configuration schema for each plugin. They are written in JSON Schema [DRAFT7](https://json-schema.org/draft-07) format and provide a way to validate the configuration options available for each plugin, as well as instantiate the plugin with the correct settings in both the UI and the application code. Having accurate schemas ensures that users can configure plugins correctly and that the application can handle these configurations without errors. + +Your schema SHOULD declare which of the auth types your plugin supports. +Your schema MAY declare which patterns that need to be matched for other fields, default values, etc. It should inherit from the base schema located at [`application/single_app/static/json/schemas/plugin.schema.json`](/application/single_app/static/json/schemas/plugin.schema.json). + +### .additional_settings.schema.json files + +These files define the additional settings required for specific plugins. They are also written in JSON Schema [DRAFT7](https://json-schema.org/draft-07) format and provide a way to validate the additional configuration options available for each plugin, as well as instantiate the plugin with the correct settings in both the UI and the application code. Having accurate schemas ensures that users can configure plugins correctly and that the application can handle these configurations without errors. + +Any additional settings schema properties that end with `__Secret` (double underscore) will be treated as sensitive information and will be stored in key vault if the option is enabled. \ No newline at end of file diff --git a/application/single_app/static/json/schemas/plugin.schema.json b/application/single_app/static/json/schemas/plugin.schema.json index a990484b..c1226d7c 100644 --- a/application/single_app/static/json/schemas/plugin.schema.json +++ b/application/single_app/static/json/schemas/plugin.schema.json @@ -41,21 +41,33 @@ "properties": { "type": { "type": "string", - "enum": ["key", "identity", "user", "servicePrincipal"], - "description": "Auth type must be 'key', 'user', 'identity', or 'servicePrincipal'" + "enum": ["key", "identity", "user", "servicePrincipal", "connection_string", "basic", "username_password"], + "description": "Auth type must be 'key', 'user', 'identity', 'servicePrincipal', 'connection_string', 'basic', or 'username_password'" }, "key": { - "type": "string" + "type": "string", + "description": "The secret value for the plugin should be stored here, such as a SQL connection string, a password for a service principal or username/password combination" }, "identity": { - "type": "string" + "type": "string", + "description": "This could be the Id of an (managed) identity, a user name, or similar to pair with the key, in most situations" }, "tenantId": { - "type": "string" + "type": "string", + "description": "The Azure AD tenant ID used with Service Principal authentication" } }, - "required": ["type"], - "additionalProperties": false + "additionalProperties": false, + "allOf": [ + { "if": { "properties": { "type": { "const": "key" } } }, "then": { "required": ["type", "key"] } }, + { "if": { "properties": { "type": { "const": "identity" } } }, "then": { "required": ["type", "identity"] } }, + { "if": { "properties": { "type": { "const": "user" } } }, "then": { "required": ["type"] } }, + { "if": { "properties": { "type": { "const": "servicePrincipal" } } }, "then": { "required": ["type", "tenantId", "identity", "key"] } }, + { "if": { "properties": { "type": { "const": "connection_string" } } }, "then": { "required": ["type", "key"] } }, + { "if": { "properties": { "type": { "const": "basic" } } }, "then": { "required": ["type", "key", "identity"] } }, + { "if": { "properties": { "type": { "const": "username_password" } } }, "then": { "required": ["type", "key", "identity"] } }, + { "required": ["type"] } + ] }, "metadata": { "type": "object", @@ -64,7 +76,7 @@ }, "additionalFields": { "type": "object", - "description": "Arbitrary additional fields", + "description": "Additional fields for plugin configuration based on plugin type. See plugin documentation for details. Any fields named __Secret (double underscore) will be stored in key vault if the feature is enabled.", "additionalProperties": true } }, diff --git a/application/single_app/static/json/schemas/queue_storage_plugin.additional_settings.schema.json b/application/single_app/static/json/schemas/queue_storage_plugin.additional_settings.schema.json new file mode 100644 index 00000000..f9ee4b64 --- /dev/null +++ b/application/single_app/static/json/schemas/queue_storage_plugin.additional_settings.schema.json @@ -0,0 +1,14 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Queue Storage Additional Settings", + "description": "Additional settings required for Azure Queue Storage plugin.", + "type": "object", + "properties": { + "queue_name": { + "type": "string", + "title": "Queue Name", + "description": "The name of the Azure Storage Queue to use." + } + }, + "required": ["queue_name"] +} diff --git a/application/single_app/static/json/schemas/queue_storage_plugin.schema.json b/application/single_app/static/json/schemas/queue_storage_plugin.schema.json new file mode 100644 index 00000000..5a0886fd --- /dev/null +++ b/application/single_app/static/json/schemas/queue_storage_plugin.schema.json @@ -0,0 +1,28 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Queue Storage Plugin", + "description": "Schema for Azure Queue Storage plugin configuration.", + "allOf": [ + { "$ref": "plugin.schema.json" }, + { + "type": "object", + "properties": { + "endpoint": { + "type": "string", + "pattern": "^https://.*\\.queue\\.core\\.windows\\.net/?$", + "description": "Must be a valid Azure Queue Storage endpoint." + }, + "auth": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["key", "identity"], + "description": "Only 'key' or 'identity' are allowed for queue storage." + } + } + } + } + } + ] +} \ No newline at end of file diff --git a/application/single_app/static/json/schemas/sql_query_plugin.additional_settings.schema.json b/application/single_app/static/json/schemas/sql_query_plugin.additional_settings.schema.json new file mode 100644 index 00000000..9e4f6d34 --- /dev/null +++ b/application/single_app/static/json/schemas/sql_query_plugin.additional_settings.schema.json @@ -0,0 +1,55 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "SQL Query Plugin Additional Settings", + "type": "object", + "properties": { + "connection_string__Secret": { + "type": "string", + "description": "Database connection string. Required if server/database not provided." + }, + "database_type": { + "type": "string", + "enum": ["sqlserver", "postgresql", "mysql", "sqlite", "azure_sql", "azuresql"], + "description": "Type of database engine." + }, + "server": { + "type": "string", + "description": "Database server hostname or IP." + }, + "database": { + "type": "string", + "description": "Database name or path." + }, + "username": { + "type": "string", + "description": "Username for authentication." + }, + "password__Secret": { + "type": "string", + "description": "Password for authentication." + }, + "driver": { + "type": "string", + "description": "ODBC or DB driver name." + }, + "read_only": { + "type": "boolean", + "default": true, + "description": "If true, restricts queries to read-only operations." + }, + "max_rows": { + "type": "integer", + "default": 1000, + "minimum": 1, + "description": "Maximum number of rows returned by a query." + }, + "timeout": { + "type": "integer", + "default": 30, + "minimum": 1, + "description": "Query timeout in seconds." + } + }, + "required": ["database_type", "database"], + "additionalProperties": false +} diff --git a/application/single_app/static/json/schemas/sql_schema_plugin.additional_settings.schema.json b/application/single_app/static/json/schemas/sql_schema_plugin.additional_settings.schema.json new file mode 100644 index 00000000..e97c7b4b --- /dev/null +++ b/application/single_app/static/json/schemas/sql_schema_plugin.additional_settings.schema.json @@ -0,0 +1,38 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "SQL Schema Plugin Additional Settings", + "type": "object", + "properties": { + "connection_string__Secret": { + "type": "string", + "description": "Database connection string. Required if server/database not provided." + }, + "database_type": { + "type": "string", + "enum": ["sqlserver", "postgresql", "mysql", "sqlite", "azure_sql", "azuresql"], + "description": "Type of database engine." + }, + "server": { + "type": "string", + "description": "Database server hostname or IP." + }, + "database": { + "type": "string", + "description": "Database name or path." + }, + "username": { + "type": "string", + "description": "Username for authentication." + }, + "password__Secret": { + "type": "string", + "description": "Password for authentication." + }, + "driver": { + "type": "string", + "description": "ODBC or DB driver name." + } + }, + "required": ["database_type", "database"], + "additionalProperties": false +} diff --git a/application/single_app/static/json/schemas/ui_test_plugin.additional_settings.schema.json b/application/single_app/static/json/schemas/ui_test_plugin.additional_settings.schema.json new file mode 100644 index 00000000..d611c034 --- /dev/null +++ b/application/single_app/static/json/schemas/ui_test_plugin.additional_settings.schema.json @@ -0,0 +1,113 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "UI Test Plugin Additional Settings", + "type": "object", + "properties": { + "string": { + "type": "string", + "description": "A string value", + "minLength": 1, + "maxLength": 100 + }, + "string__Secret": { + "type": "string", + "description": "A string value", + "minLength": 1, + "maxLength": 100 + }, + "email": { + "type": "string", + "description": "An email address", + "format": "email", + "pattern": "^[a-zA-Z0-9._%+\\-]+@[a-zA-Z0-9.\\-]+\\.[a-zA-Z]{2,}$", + "default": "user@example.com" + }, + "enum": { + "type": "string", + "enum": ["", "bob", "alice", "eve"], + "description": "An enumeration of string values" + }, + "number": { + "type": "number", + "description": "A numeric value", + "minimum": 0, + "maximum": 100, + "default": 50 + }, + "integer": { + "type": "integer", + "description": "An integer value", + "minimum": 0, + "maximum": 10, + "default": 5 + }, + "boolean": { + "type": "boolean", + "description": "A boolean value", + "default": false + }, + "object": { + "type": "object", + "description": "An object with string and number properties", + "properties": { + "object_string": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "object_number": { + "type": "number", + "minimum": 0, + "maximum": 100 + } + }, + "required": ["object_string"] + }, + "array": { + "type": "array", + "description": "An array of objects with string and number properties", + "items": { + "type": "object", + "properties": { + "array_string": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "array_number": { + "type": "number", + "minimum": 0, + "maximum": 100 + } + }, + "required": ["array_string"] + } + }, + "string_array": { + "type": "array", + "description": "An array of strings", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 100 + } + } + }, + "required": ["string", "enum", "string__Secret"], + "allOf": [ + { + "if": { + "properties": { "boolean": { "const": true } } + }, + "then": { + "required": ["number", "integer"] + }, + "else": { + "not": { + "required": ["number", "integer"] + } + } + } + ], + "additionalProperties": false +} diff --git a/application/single_app/static/json/schemas/ui_test_plugin.plugin.schema.json b/application/single_app/static/json/schemas/ui_test_plugin.plugin.schema.json new file mode 100644 index 00000000..59b2f051 --- /dev/null +++ b/application/single_app/static/json/schemas/ui_test_plugin.plugin.schema.json @@ -0,0 +1,23 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "UI Test Plugin", + "description": "NYI-Schema for UI Test Plugin configuration, restricting auth.type to user, key, connection_string, and identity.", + "allOf": [ + { "$ref": "plugin.schema.json" }, + { + "type": "object", + "properties": { + "auth": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["user", "key", "connection_string", "identity"], + "description": "Allowed values for UI Test Plugin: user, key, connection_string, identity." + } + } + } + } + } + ] +} \ No newline at end of file diff --git a/application/single_app/templates/_plugin_modal.html b/application/single_app/templates/_plugin_modal.html index 563a4981..3af18019 100644 --- a/application/single_app/templates/_plugin_modal.html +++ b/application/single_app/templates/_plugin_modal.html @@ -174,15 +174,15 @@
API Information
- +
+

+ Configure Security Settings. +

+
+
Key Vault
+

+ Configure Key Vault settings. +

+
+ + + +
+
+ ⚠ī¸ Warning: Once you enable Key Vault, you should NOT disable it. Disabling Key Vault after enabling WILL cause loss of access to secrets and break application functionality. +
+
+ + + +
+
+ + + +
+ +
+
+
+
+ diff --git a/deployers/New-CosmosContainerDynamicRUs.ps1 b/deployers/New-CosmosContainerDynamicRUs.ps1 new file mode 100644 index 00000000..2d64b25e --- /dev/null +++ b/deployers/New-CosmosContainerDynamicRUs.ps1 @@ -0,0 +1,99 @@ +#requires -Module Az.CosmosDB +#requires -Module Az.Accounts +param( + [Parameter(Mandatory=$true)] + [string]$ResourceGroup, + [Parameter(Mandatory=$true)] + [string]$AccountName, + [string]$DatabaseName = "SimpleChat", + [ValidateRange(1000, 1000000)] + [int]$NewMaxRU = 1000, + [Parameter(Mandatory=$false, HelpMessage="Azure Cloud Environment: AzureCloud, AzureUSGovernment, Custom")] + [ValidateSet("AzureCloud", "AzureUSGovernment", "Custom")] + [string]$AzureCloudEnvironment = "AzureCloud" +) +$CloudEndpoint = "" +if ($AzureCloudEnvironment -eq "Custom") +{ + $CloudEndpoint = Read-Host "Enter the custom Azure Cloud Environment endpoint (e.g. https://management.azurecustom.you/)" + if ([string]::IsNullOrEmpty($CloudEndpoint)) + { + throw "Custom environment selected but no endpoint provided." + } + Write-Host "Using custom Azure Cloud Environment endpoint: $CloudEndpoint" + $AzureCloudEnvironment = New-AzEnvironment -Name "Custom" -ActiveDirectoryAuthority "https://login.microsoftonline.com/" -ResourceManagerEndpoint $CloudEndpoint -GraphEndpoint "https://graph.windows.net/" -GalleryEndpoint "https://gallery.azure.com/" -ManagementEndpoint $CloudEndpoint -StorageEndpointSuffix "core.windows.net" -SqlDatabaseDnsSuffix "database.windows.net" -TrafficManagerDnsSuffix "trafficmanager.net" -KeyVaultDnsSuffix "vault.azure.net" -ServiceManagementUrl $CloudEndpoint +} + +if ($(Get-AzContext)?.Account) +{ + Write-Host "Logged in as $((Get-AzContext).Account.Name)" +} +else +{ + Login-AzAccount -Environment $AzureCloudEnvironment -UseDeviceAuthentication +} + +$subscriptionName = $(Get-AzContext)?.Subscription?.Name +while ($subChoice -notin ("Y","y","N","n")) +{ + $subChoice = Read-Host "Use subscription '$subscriptionName'? (Y/N)" + if ($subChoice -eq "N") + { + $subscriptions = Get-AzSubscription + $subscriptions | ForEach-Object { Write-Host "$($_.SubscriptionId): $($_.Name)" } + $subId = Read-Host "Enter SubscriptionId to use" + Set-AzContext -SubscriptionId $subId + $subscriptionName = $(Get-AzContext)?.Subscription?.Name + Write-Host "Using subscription '$subscriptionName'" + } + elseif ($subChoice -ne "Y") + { + Write-Host "Please enter Y or N." + } +} + +# Get all containers in the database +$containers = Get-AzCosmosDBSqlContainer -ResourceGroupName $ResourceGroup -AccountName $AccountName -DatabaseName $DatabaseName + +foreach ($container in $containers) { + $containerName = $container.Name + Write-Host "Processing container: $containerName..." + + # Get current throughput settings + $throughput = Get-AzCosmosDBSqlContainerThroughput -ResourceGroupName $ResourceGroup -AccountName $AccountName -DatabaseName $DatabaseName -Name $containerName -ErrorAction SilentlyContinue + + Write-Host " Current Throughput Type: $($throughput.Throughput)" + + if ($null -eq $throughput) { + Write-Warning "No throughput found for $containerName. Skipping." + continue + } + + if ($throughput.AutoscaleSettings.MaxThroughput -eq 0) { + Write-Host " Migrating $containerName from Manual to Autoscale (max $NewMaxRU RU/s)..." + Invoke-AzCosmosDBSqlContainerThroughputMigration ` + -ResourceGroupName $ResourceGroup ` + -AccountName $AccountName ` + -DatabaseName $DatabaseName ` + -Name $containerName ` + -ThroughputType "Autoscale" + Write-Host "Updating $containerName to $NewMaxRU RU/s" + Update-AzCosmosDBSqlContainerThroughput ` + -ResourceGroupName $ResourceGroup ` + -AccountName $AccountName ` + -DatabaseName $DatabaseName ` + -Name $containerName ` + -AutoscaleMaxThroughput $NewMaxRU + Write-Host "Updated $containerName to $NewMaxRU RU/s" + } else { + Write-Host " $containerName already Autoscale. Updating max RU/s to $NewMaxRU..." + Update-AzCosmosDBSqlContainerThroughput ` + -ResourceGroupName $ResourceGroup ` + -AccountName $AccountName ` + -DatabaseName $DatabaseName ` + -Name $containerName ` + -AutoscaleMaxThroughput $NewMaxRU + } +} + +Write-Host "All containers processed for autoscale ($($NewMaxRU*.10)-$NewMaxRU RU/s)." \ No newline at end of file