From adb9843c7f2adaad96457d4fb7c4c83d8122bd08 Mon Sep 17 00:00:00 2001 From: Dimitri Glazkov Date: Thu, 9 May 2024 16:05:33 -0500 Subject: [PATCH] Add super-worker based tool-calling. (#334) --- .../bgl/insta/chat-and-research.bgl.json | 178 ++++++++++++++++++ .../bgl/insta/specialist-tools.bgl.json | 145 ++++++++++++++ .../insta/tool-page-screenshot-content.json | 104 ++-------- .../bgl/insta/tool-page-screenshot.json | 33 ++-- 4 files changed, 346 insertions(+), 114 deletions(-) create mode 100644 seeds/team-experiments/public/bgl/insta/chat-and-research.bgl.json create mode 100644 seeds/team-experiments/public/bgl/insta/specialist-tools.bgl.json diff --git a/seeds/team-experiments/public/bgl/insta/chat-and-research.bgl.json b/seeds/team-experiments/public/bgl/insta/chat-and-research.bgl.json new file mode 100644 index 00000000..ac42d000 --- /dev/null +++ b/seeds/team-experiments/public/bgl/insta/chat-and-research.bgl.json @@ -0,0 +1,178 @@ +{ + "title": "Blank board", + "description": "A blank board. Use it as a starting point for your creations.", + "version": "0.0.1", + "nodes": [ + { + "id": "superWorker-997913c1", + "type": "superWorker", + "metadata": { + "visual": { + "x": -66, + "y": -84 + }, + "title": "Interviewer", + "logLevel": "debug" + }, + "configuration": { + "persona": { + "role": "user", + "parts": [ + { + "text": "You are a friendly chat for who helps small businesses build better online presence. Your job is to collect the name, the location, and the website of the customer's business. You typically start conversation with a warm greeting, and then get to work.\n\nWhen you have the business name, its location, and the website, reply with a brief summary of the information you've collected in a neat bulleted list, then conclude the conversation by saying \"OK, hold on one moment while I look that up. I'll be with you in just a couple of minutes. Stand by. ##DONE##\"" + } + ] + } + } + }, + { + "id": "human-b009493b", + "type": "human", + "metadata": { + "visual": { + "x": 194, + "y": -110 + }, + "title": "User", + "logLevel": "debug" + } + }, + { + "id": "looper-f3060a3f", + "type": "looper", + "metadata": { + "visual": { + "x": 20, + "y": -315 + }, + "title": "Chat Loop", + "logLevel": "debug" + }, + "configuration": { + "task": { + "role": "user", + "parts": [ + { + "text": "Chat until \"##DONE##\"." + } + ] + } + } + }, + { + "id": "output-c3f84eca", + "type": "output", + "metadata": { + "visual": { + "x": 561, + "y": -322 + }, + "title": "Results", + "logLevel": "debug" + }, + "configuration": { + "schema": { + "properties": { + "property-1": { + "type": "object", + "title": "Context Out", + "examples": [], + "behavior": [ + "llm-content" + ] + } + }, + "type": "object", + "required": [] + } + } + }, + { + "id": "input-880b671c", + "type": "input", + "configuration": { + "schema": { + "properties": { + "property-1": { + "type": "object", + "title": "Context", + "examples": [], + "behavior": [ + "llm-content" + ] + } + }, + "type": "object", + "required": [] + } + }, + "metadata": { + "visual": { + "x": -219, + "y": -288 + }, + "title": "Start", + "logLevel": "debug" + } + }, + { + "id": "toolWorker-7d26ed05", + "type": "toolWorker", + "metadata": { + "visual": { + "x": 289, + "y": -417 + }, + "title": "Researcher", + "logLevel": "debug" + }, + "configuration": { + "instruction": "You are a super-helpful researcher who helps small businesses have a better online presence. With these tools, you can search the web and find relevant information, go to specific pages on the web and get their contents, and even get summaries of the screenshots of the pages. \n\nReview the conversation and call the necessary tool to gather information about the business.", + "tools": [ + "https://raw.githubusercontent.com/google/labs-prototypes/main/seeds/team-experiments/public/bgl/insta/tool-get-web-page.bgl.json", + "https://raw.githubusercontent.com/google/labs-prototypes/main/seeds/team-experiments/public/bgl/insta/tool-search.bgl.json", + "https://raw.githubusercontent.com/google/labs-prototypes/main/seeds/team-experiments/public/bgl/insta/tool-page-screenshot.json" + ], + "instruction2": "Using the tool output, provide a detailed summary of the information you've gathered, organized in these sections:\n\n- Overview (brief summary of the information found)\n- Product information (what is the product or service that the business provides or sells.\n- Target audience (use bullet points and detailed descriptions for each potential audience segment)\n- Use cases (how would the audience typically use this product or service)\n- Benefits (what are the benefits that this product or service offers to the target audiences)\n- Differentiating capabilities (what stands this product or service apart from others?)\n- Key messages (what is important to communicate about this business, what sets it apart from others)\n" + } + } + ], + "edges": [ + { + "from": "superWorker-997913c1", + "to": "human-b009493b", + "out": "out", + "in": "context" + }, + { + "from": "looper-f3060a3f", + "to": "superWorker-997913c1", + "out": "loop", + "in": "in" + }, + { + "from": "human-b009493b", + "to": "looper-f3060a3f", + "out": "context", + "in": "context" + }, + { + "from": "input-880b671c", + "to": "looper-f3060a3f", + "out": "property-1", + "in": "context" + }, + { + "from": "toolWorker-7d26ed05", + "to": "output-c3f84eca", + "out": "context", + "in": "property-1" + }, + { + "from": "looper-f3060a3f", + "to": "toolWorker-7d26ed05", + "out": "done", + "in": "context" + } + ] +} \ No newline at end of file diff --git a/seeds/team-experiments/public/bgl/insta/specialist-tools.bgl.json b/seeds/team-experiments/public/bgl/insta/specialist-tools.bgl.json new file mode 100644 index 00000000..3d6115a1 --- /dev/null +++ b/seeds/team-experiments/public/bgl/insta/specialist-tools.bgl.json @@ -0,0 +1,145 @@ +{ + "title": "Blank board", + "description": "A blank board. Use it as a starting point for your creations.", + "version": "0.0.1", + "nodes": [ + { + "type": "input", + "id": "input", + "configuration": { + "schema": { + "type": "object", + "properties": { + "text": { + "type": "object", + "title": "Context", + "examples": [], + "behavior": [ + "llm-content" + ], + "default": "{\"role\":\"user\",\"parts\":[{\"text\":\"Business name: June Care\\nWebsite: junecare.co\"}]}" + } + }, + "required": [] + } + }, + "metadata": { + "visual": { + "x": -348, + "y": -42 + } + } + }, + { + "type": "output", + "id": "output", + "configuration": { + "schema": { + "type": "object", + "properties": { + "text": { + "type": "object", + "title": "text", + "examples": [], + "behavior": [ + "llm-content" + ] + } + }, + "required": [] + } + }, + "metadata": { + "visual": { + "x": 634, + "y": -47 + } + } + }, + { + "id": "superWorker-0ce54463", + "type": "superWorker", + "metadata": { + "visual": { + "x": -84, + "y": -50 + }, + "title": "Researcher", + "logLevel": "debug" + }, + "configuration": { + "persona": { + "role": "user", + "parts": [ + { + "text": "You are a super-helpful researcher who helps small businesses have a better online presence. With these tools, you can search the web and find relevant information, go to specific pages on the web and get their contents, and even get summaries of the screenshots of the pages. \n" + } + ] + }, + "task": { + "role": "user", + "parts": [ + { + "text": "Review the conversation and call the necessary tool to gather information about the business." + } + ] + }, + "tools": [ + "https://raw.githubusercontent.com/google/labs-prototypes/main/seeds/team-experiments/public/bgl/insta/tool-get-web-page.bgl.json", + "https://raw.githubusercontent.com/google/labs-prototypes/main/seeds/team-experiments/public/bgl/insta/tool-search.bgl.json", + "https://raw.githubusercontent.com/google/labs-prototypes/main/seeds/team-experiments/public/bgl/insta/tool-page-screenshot.json" + ] + } + }, + { + "id": "superWorker-ef2f062d", + "type": "superWorker", + "metadata": { + "visual": { + "x": 281, + "y": -45 + }, + "title": "Summarizer", + "logLevel": "debug" + }, + "configuration": { + "persona": { + "role": "user", + "parts": [ + { + "text": "You are a super-helpful researcher who helps small businesses have a better online presence. You provide a detailed summary of the information gathered so far, organized in these sections:\n\n- Overview (brief summary of the information found)\n- Product information (what is the product or service that the business provides or sells.\n- Target audience (use bullet points and detailed descriptions for each potential audience segment)\n- Use cases (how would the audience typically use this product or service)\n- Benefits (what are the benefits that this product or service offers to the target audiences)\n- Differentiating capabilities (what stands this product or service apart from others?)\n- Key messages (what is important to communicate about this business, what sets it apart from others)\n" + } + ] + }, + "task": { + "role": "user", + "parts": [ + { + "text": "Provide a detailed summary." + } + ] + } + } + } + ], + "edges": [ + { + "from": "input", + "to": "superWorker-0ce54463", + "out": "text", + "in": "in" + }, + { + "from": "superWorker-0ce54463", + "to": "superWorker-ef2f062d", + "out": "out", + "in": "in" + }, + { + "from": "superWorker-ef2f062d", + "to": "output", + "out": "out", + "in": "text" + } + ] +} \ No newline at end of file diff --git a/seeds/team-experiments/public/bgl/insta/tool-page-screenshot-content.json b/seeds/team-experiments/public/bgl/insta/tool-page-screenshot-content.json index bf518487..8b8315ec 100644 --- a/seeds/team-experiments/public/bgl/insta/tool-page-screenshot-content.json +++ b/seeds/team-experiments/public/bgl/insta/tool-page-screenshot-content.json @@ -1,6 +1,6 @@ { - "title": "Page Screenshot Analyzer", - "description": "Takes a screenshot of a page at at a given URL, then follows the provided instruction to analyze it and returns the analysis.", + "title": "Page Screenshot Taker", + "description": "Takes a screenshot of a page at at a given URL and returns it.", "version": "0.0.1", "edges": [ { @@ -29,30 +29,6 @@ }, { "from": "jsonata-15bf2c54", - "to": "superWorker-14a88d05", - "out": "result", - "in": "in" - }, - { - "from": "superWorker-14a88d05", - "to": "jsonata-d676e36a", - "out": "out", - "in": "json" - }, - { - "from": "input-2ccb0f3b", - "to": "jsonata-dfbb4886", - "out": "instruction", - "in": "json" - }, - { - "from": "jsonata-dfbb4886", - "to": "superWorker-14a88d05", - "out": "result", - "in": "task" - }, - { - "from": "jsonata-d676e36a", "to": "output-0a357b6f", "out": "result", "in": "property-1" @@ -64,8 +40,8 @@ "type": "output", "metadata": { "visual": { - "x": 543, - "y": -461 + "x": 303, + "y": -437 }, "logLevel": "debug", "title": "Output" @@ -75,14 +51,15 @@ "properties": { "property-1": { "type": "string", - "title": "Analysis", + "title": "Content", "examples": [], "default": "null" } }, "type": "object", "required": [] - } + }, + "property-1": "null" } }, { @@ -114,7 +91,7 @@ "logLevel": "debug", "visual": { "x": 6, - "y": -813 + "y": -812.9999999999999 } } }, @@ -127,7 +104,7 @@ "logLevel": "debug", "visual": { "x": -343, - "y": -634 + "y": -633.9999999999999 } }, "configuration": { @@ -138,13 +115,6 @@ "title": "url", "examples": [], "description": "The URL of the page whose screenshot will be taken" - }, - "instruction": { - "type": "string", - "title": "task", - "examples": [], - "format": "multiline", - "description": "Instructions on what information to get from the screenshot" } }, "type": "object", @@ -170,65 +140,13 @@ "metadata": { "visual": { "x": -68, - "y": -469 + "y": -468.99999999999994 }, "title": "Format as LLM Content", "logLevel": "debug" }, "configuration": { - "expression": "{ \"parts\": [{ \"inlineData\": { \"mimeType\": \"image/png\", \"data\": screenshot }}], \"role\": \"user\" } " - } - }, - { - "id": "superWorker-14a88d05", - "type": "superWorker", - "metadata": { - "visual": { - "x": -48, - "y": -66 - }, - "title": "Analyze Screenshot", - "logLevel": "debug" - }, - "configuration": { - "persona": { - "role": "user", - "parts": [ - { - "text": "You are an expert analyzer of web page screenshots." - } - ] - } - } - }, - { - "id": "jsonata-dfbb4886", - "type": "jsonata", - "configuration": { - "expression": "{ \"parts\": [{ \"text\": $}], \"role\": \"user\" }" - }, - "metadata": { - "visual": { - "x": -324, - "y": -129 - }, - "title": "Format Task", - "logLevel": "debug" - } - }, - { - "id": "jsonata-d676e36a", - "type": "jsonata", - "metadata": { - "visual": { - "x": 279, - "y": -281 - }, - "title": "Format Output", - "logLevel": "debug" - }, - "configuration": { - "expression": "parts[-1].text" + "expression": "{ \"content\": { \"parts\": [{ \"inlineData\": { \"mimeType\": \"image/png\", \"data\": screenshot }}], \"role\": \"user\" } }" } } ], diff --git a/seeds/team-experiments/public/bgl/insta/tool-page-screenshot.json b/seeds/team-experiments/public/bgl/insta/tool-page-screenshot.json index bf518487..5ddbb322 100644 --- a/seeds/team-experiments/public/bgl/insta/tool-page-screenshot.json +++ b/seeds/team-experiments/public/bgl/insta/tool-page-screenshot.json @@ -65,7 +65,7 @@ "metadata": { "visual": { "x": 543, - "y": -461 + "y": -461.0000000000001 }, "logLevel": "debug", "title": "Output" @@ -91,16 +91,14 @@ "metadata": { "visual": { "x": -299, - "y": -807 + "y": -807.0000000000001 }, "title": "Get API Key", "description": "Ask for API Key", "logLevel": "debug" }, "configuration": { - "keys": [ - "SCRAPING_BEE_KEY" - ] + "keys": ["SCRAPING_BEE_KEY"] } }, { @@ -114,7 +112,7 @@ "logLevel": "debug", "visual": { "x": 6, - "y": -813 + "y": -813.0000000000001 } } }, @@ -127,7 +125,7 @@ "logLevel": "debug", "visual": { "x": -343, - "y": -634 + "y": -634.0000000000001 } }, "configuration": { @@ -138,13 +136,6 @@ "title": "url", "examples": [], "description": "The URL of the page whose screenshot will be taken" - }, - "instruction": { - "type": "string", - "title": "task", - "examples": [], - "format": "multiline", - "description": "Instructions on what information to get from the screenshot" } }, "type": "object", @@ -157,8 +148,8 @@ "type": "fetch", "metadata": { "visual": { - "x": 294, - "y": -821 + "x": 294.00000000000006, + "y": -821.0000000000001 }, "title": "Call Scraping Bee", "logLevel": "debug" @@ -170,7 +161,7 @@ "metadata": { "visual": { "x": -68, - "y": -469 + "y": -469.00000000000006 }, "title": "Format as LLM Content", "logLevel": "debug" @@ -184,8 +175,8 @@ "type": "superWorker", "metadata": { "visual": { - "x": -48, - "y": -66 + "x": -47.99999999999994, + "y": -65.99999999999997 }, "title": "Analyze Screenshot", "logLevel": "debug" @@ -221,7 +212,7 @@ "type": "jsonata", "metadata": { "visual": { - "x": 279, + "x": 279.00000000000006, "y": -281 }, "title": "Format Output", @@ -335,4 +326,4 @@ "url": "idb://default/blank.json#58849527-9663-4146-9531-91c1f0c986ab" } } -} \ No newline at end of file +}