feat: functioncall demo with NexaAI/OmniNeural-4B

demos/README.md

@@ -11,6 +11,12 @@ Lightweight on-device AI assistant with function calling (web search) using Gran
 - [Python-Binding-Example](./Agent-Granite/Python-Binding-Example)
 - [Serve-Example](./Agent-Granite/Serve-Example)
 
+### 🔧 Function-Calling
+
+Function calling capabilities with NexaAI VLM model, integrated with Google Calendar via MCP protocol. Supports multi-modal input (text, image, audio) with Web UI and CLI interfaces.
+
+- [Demo](./function-calling)
+
 ### 📚 RAG-LLM
 
 End-to-end Retrieval-Augmented Generation pipeline with embeddings, reranking, and generation models. Query your own documents (PDFs, Word, text) locally on device.

demos/function-calling/.gitignore

@@ -0,0 +1,3 @@
+__pycache__/
+*.json
+app/uploads/

demos/function-calling/README.md

@@ -0,0 +1,70 @@
+# NexaAI VLM Function Call Demo with Google Calendar MCP
+
+Demonstrates function calling capabilities of NexaAI/OmniNeural-4B model, integrated with Google Calendar via MCP protocol.
+
+## Features
+
+- Function calling with NexaAI VLM model
+- Google Calendar integration via MCP
+- Automatic function call parsing and execution
+- Multi-modal input support (text, image, audio)
+- Web UI and command-line interfaces
+
+## Prerequisites
+
+- **Python 3** (arm64 architecture recommended)
+  - See [bindings/python/notebook/windows(arm64).ipynb](../../bindings/python/notebook/windows(arm64).ipynb) for setup
+- **Node.js and npm** (for MCP server)
+  ```powershell
+  winget install OpenJS.NodeJS.LTS
+  ```
+  Restart terminal/IDE after installation.
+
+## Installation
+
+```bash
+pip install -r requirements.txt
+```
+
+## Google Calendar Setup
+
+1. Go to [Google Cloud Console](https://console.cloud.google.com)
+2. Create/select a project and enable [Google Calendar API](https://console.cloud.google.com/apis/library/calendar-json.googleapis.com)
+3. Go to [OAuth consent screen](https://console.cloud.google.com/apis/credentials/consent) follow the instructions to configure your consent screen.
+3. Create OAuth 2.0 credentials:
+   - Go to [Credentials](https://console.cloud.google.com/apis/credentials)
+   - Create "Create Credentials" > "OAuth client ID" > Select "Desktop app" > click "Create"
+   - Click "Download JSON" and save as `gcp-oauth.keys.json` on the same directory as this README.md
+4. Add your email as a test user in [Audience](https://console.cloud.google.com/auth/audience)
+   - Click "Add User" > enter your email address > click "Save"
+   - Note: Test mode tokens expire after 1 week
+
+5. Authenication (only need to do once)
+```powershell
+$env:GOOGLE_OAUTH_CREDENTIALS="gcp-oauth.keys.json"
+npx @cocal/google-calendar-mcp auth
+```
+follow the instructions to authorize the application to access your Google Calendar.
+
+**Tip**: Ensure the OAuth client ID is enabled for Calendar API at [Credentials](https://console.cloud.google.com/apis/api/calendar-json.googleapis.com/credentials)
+For detailed setup, see: https://github.com/nspady/google-calendar-mcp?tab=readme-ov-file#google-cloud-setup
+
+## Usage
+
+### Command Line
+
+```bash
+# Text only
+python main.py --text "what is the time now?"
+
+# Image with text
+python main.py --image image.png --text "help me add this event to my calendar"
+
+# Audio with text
+python main.py --audio audio.mp3 --text "transcribe and add to calendar"
+```
+### Web UI
+
+```powershell
+python .\app\flask_ui.py
+```

demos/function-calling/app/flask_ui.py

@@ -0,0 +1,231 @@
+from flask import Flask, render_template, request, jsonify, send_from_directory
+from pathlib import Path
+from datetime import datetime
+from image_utils import image_to_base64
+import uuid
+
+import sys
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from main import call_agent_wrapper, FunctionCallAgentResult
+
+app = Flask(__name__)
+app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024  # 16MB max file size
+app.config['UPLOAD_FOLDER'] = Path(__file__).resolve().parent / 'uploads'
+app.config['UPLOAD_FOLDER'].mkdir(parents=True, exist_ok=True)
+
+chat_history = []
+processing_tasks = {}
+
+def allowed_file(filename):
+    """check allowed file types"""
+    allowed_extensions = {'png', 'jpg', 'jpeg', 'gif', 'webp', 'bmp'}
+    return '.' in filename and filename.rsplit('.', 1)[1].lower() in allowed_extensions
+
+def process_message(text_content=None, image_path=None):
+    """handle user message"""
+    message = {
+        'type': 'user',
+        'timestamp': datetime.now().isoformat(),
+        'text': text_content if text_content else None,
+        'image': None
+    }
+
+    if image_path:
+        try:
+            message['image'] = image_to_base64(image_path)
+        except Exception as e:
+            return None, f"Image handle error: {str(e)}"
+
+    chat_history.append(message)
+    return message, None
+
+def add_bot_response(response_type='text', content=None):
+    bot_message = {
+        'type': 'bot',
+        'timestamp': datetime.now().isoformat(),
+        'response_type': response_type,  # 'text', 'event'
+        'content': content
+    }
+    chat_history.append(bot_message)
+    return bot_message
+
+@app.route('/')
+def index():
+    """chat page"""
+    return render_template('chat.html')
+
+@app.route('/api/send-message', methods=['POST'])
+def send_message():
+    """
+    handle user message
+    inputs: text and/or image
+    return: task_id
+    """
+    try:
+        text_content = request.form.get('message', '').strip()
+        image_file = request.files.get('image')
+
+        if not text_content and not image_file:
+            return jsonify({'error': 'Please provide text or image'}), 400
+
+        image_path = None
+        if image_file and image_file.filename:
+            if not allowed_file(image_file.filename):
+                return jsonify({'error': 'not allowed file'}), 400
+
+            # save uploaded image
+            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S_')
+            filename = timestamp + image_file.filename
+            image_path = app.config['UPLOAD_FOLDER'] / filename
+            image_file.save(image_path)
+
+        # handle user message
+        user_message, error = process_message(text_content, image_path)
+        if error:
+            return jsonify({'error': error}), 400
+
+        # create task id
+        task_id = str(uuid.uuid4())
+        # register task so client can poll /api/get-response/<task_id>
+        processing_tasks[task_id] = {
+            'status': 'pending',
+            'user_text': text_content,
+            'image_path': str(image_path) if image_path else None
+        }
+
+        return jsonify({
+            'user_message': user_message,
+            'task_id': task_id,
+            'success': True
+        })
+
+    except Exception as e:
+        return jsonify({'error': f'Serve error: {str(e)}'}), 500
+
+@app.route('/api/get-response/<task_id>', methods=['GET'])
+async def get_response(task_id):
+    if task_id in processing_tasks:
+        task = processing_tasks[task_id]
+        if task['status'] == 'done':
+            # clear task
+            del processing_tasks[task_id]
+            return jsonify(task['result'])
+        elif task['status'] == 'processing':
+            return jsonify({'status': 'processing'})
+        elif task['status'] == 'pending':
+            # process now (synchronous handling for simplicity)
+            processing_tasks[task_id]['status'] = 'processing'
+            try:
+                response = await call_agent_wrapper(
+                    text=task['user_text'],
+                    image=task['image_path']
+                )
+
+                if not response:
+                    bot_response = add_bot_response(
+                        response_type='text',
+                        content="Sorry, I couldn't process your request."
+                    )
+                    result = {'status': 'done', 'bot_response': bot_response}
+                    processing_tasks[task_id]['status'] = 'done'
+                    processing_tasks[task_id]['result'] = result
+                    return jsonify(result)
+
+                if response.func_name == "create-event" and response.func_result is not None:
+                    import json
+                    data = json.loads(response.func_result)
+                    is_error = bool(data.get("isError"))
+                    content = data.get("content") or []
+                    bot_response = None
+                    if is_error:
+                        bot_response = add_bot_response(
+                            response_type='text',
+                            content=content[0]["text"]
+                        )
+                    else:
+                        text = json.loads(content[0]["text"])
+                        event = text["event"]
+                        print(f"[info] Event detected: {event}")
+                        summary=event.get("summary", "No Title")
+                        date = "N/A"
+                        if event.get("start"):
+                            start_time = event["start"].get("dateTime", "N/A")
+                            date = start_time.split("T")[0]
+                            start_time = start_time.split("T")[1] if "T" in start_time else "N/A"
+                        else:
+                            start_time = "N/A"
+                        if event.get("end"):
+                            end_time = event["end"].get("dateTime", "N/A")
+                            end_time = end_time.split("T")[1] if "T" in end_time else "N/A"
+                        else:
+                            end_time = "N/A"
+                        venue = event.get("location", "N/A")
+                        description = event.get("description", summary)
+                        address = event.get("address", "N/A")
+                        htmlLink = event.get("htmlLink", "")
+                        bot_response = add_bot_response(
+                            response_type='event',
+                            content={
+                                'event_name': summary,
+                                'date': date,
+                                'start_time': start_time,
+                                'end_time': end_time,
+                                'venue': venue,
+                                'address': address,
+                                'description': description,
+                                'htmlLink': htmlLink
+                            }
+                        )
+                    result = {'status': 'done', 'bot_response': bot_response}
+                    processing_tasks[task_id]['status'] = 'done'
+                    processing_tasks[task_id]['result'] = result
+                    # return and let client clear on next poll
+                    return jsonify(result)
+                elif response.response_text is not None:
+                    bot_response = add_bot_response(
+                        response_type='text',
+                        content=response.response_text
+                    )
+                    result = {'status': 'done', 'bot_response': bot_response}
+                    processing_tasks[task_id]['status'] = 'done'
+                    processing_tasks[task_id]['result'] = result
+                    return jsonify(result)
+                else:
+                    bot_response = add_bot_response(
+                        response_type='text',
+                        content="Sorry, I couldn't process your request."
+                    )
+                    result = {'status': 'done', 'bot_response': bot_response}
+                    processing_tasks[task_id]['status'] = 'done'
+                    processing_tasks[task_id]['result'] = result
+                    return jsonify(result)
+            except Exception as e:
+                bot_response = add_bot_response(
+                    response_type='text',
+                    content=str(e)
+                )
+                result = {'status': 'done', 'bot_response': bot_response}
+                processing_tasks[task_id]['status'] = 'done'
+                processing_tasks[task_id]['result'] = result
+                return jsonify(result)
+
+    return jsonify({
+        'status': 'done'
+    })
+
+@app.route('/api/chat-history', methods=['GET'])
+def get_chat_history():
+    return jsonify(chat_history)
+
+@app.route('/api/clear-history', methods=['POST'])
+def clear_history():
+    global chat_history
+    chat_history = []
+    return jsonify({'success': True})
+
+@app.route('/uploads/<path:filename>')
+def serve_upload(filename):
+    return send_from_directory(app.config['UPLOAD_FOLDER'], filename)
+
+if __name__ == '__main__':
+    app.run(debug=True, host='0.0.0.0', port=3000)

demos/function-calling/app/image_utils.py

@@ -0,0 +1,29 @@
+
+import base64
+import os
+from pathlib import Path
+from typing import Union
+
+
+def image_to_base64(image_path: Union[str, Path]) -> str:
+    image_path = Path(image_path)
+    if not image_path.exists():
+        raise FileNotFoundError(f"Image not exists: {image_path}")
+    suffix = image_path.suffix.lower()
+    mime_types = {
+        '.jpg': 'image/jpeg',
+        '.jpeg': 'image/jpeg',
+        '.png': 'image/png',
+        '.gif': 'image/gif',
+        '.webp': 'image/webp',
+        '.bmp': 'image/bmp',
+    }
+
+    if suffix not in mime_types:
+        raise ValueError(f"Not support: {suffix}")
+
+    mime_type = mime_types[suffix]
+    with open(image_path, 'rb') as f:
+        image_data = f.read()
+    base64_data = base64.b64encode(image_data).decode('utf-8')
+    return f"data:{mime_type};base64,{base64_data}"