From aa730c478781c1758e9d93d2603122484ac6be0d Mon Sep 17 00:00:00 2001
From: Jo Booth <jo@humansignal.com>
Date: Tue, 21 Oct 2025 00:16:43 -0400
Subject: [PATCH] Add section on refining prompt with SDK for workshop

---
 .../SDK_notebook.ipynb                        | 315 ++++++++++++++----
 1 file changed, 248 insertions(+), 67 deletions(-)

diff --git a/programmatic_actions_workshop/SDK_notebook.ipynb b/programmatic_actions_workshop/SDK_notebook.ipynb
index 96bc64b..350d8ea 100644
--- a/programmatic_actions_workshop/SDK_notebook.ipynb
+++ b/programmatic_actions_workshop/SDK_notebook.ipynb
@@ -20,14 +20,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 205,
+   "execution_count": null,
    "id": "c066e663-a7ae-4077-a1d0-55a0439bb3d1",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# credentials and setup \n",
-    "label_studio_url = \"http://app.heartex.com\" \n",
-    "label_studio_api_key = \"Your Label Studio API Key\""
+    "# credentials and setup\n",
+    "label_studio_url = \"your_label_studio_url_here\"\n",
+    "label_studio_api_key = \"your_api_key_here\""
    ]
   },
   {
@@ -40,7 +40,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 206,
+   "execution_count": null,
    "id": "6678f533-b167-460f-954e-5e5a031db3d4",
    "metadata": {},
    "outputs": [],
@@ -64,7 +64,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 207,
+   "execution_count": null,
    "id": "e10ab73f-c7ba-4e3f-aeda-57a2eb1d2533",
    "metadata": {},
    "outputs": [],
@@ -73,7 +73,7 @@
     "\n",
     "# the Label Studio Client is the main entry point for the SDK.\n",
     "ls = LabelStudio(\n",
-    "    base_url=label_studio_url,  \n",
+    "    base_url=label_studio_url,\n",
     "    api_key=label_studio_api_key,\n",
     ")"
    ]
@@ -96,22 +96,22 @@
     "from label_studio_sdk.label_interface import LabelInterface\n",
     "from label_studio_sdk.label_interface.create import labels, choices\n",
     "\n",
-    "# We can define the labeling config in one of two ways: \n",
+    "# We can define the labeling config in one of two ways:\n",
     "# 1. a string\n",
     "label_config = \"\"\"\n",
-    "<View>    \n",
+    "<View>\n",
     "   <Image name=\"image\" value=\"$image\" zoom=\"true\" zoomControl=\"false\"\n",
     "         rotateControl=\"true\" width=\"100%\" height=\"100%\"\n",
     "         maxHeight=\"auto\" maxWidth=\"auto\"/>\n",
-    "   \n",
+    "\n",
     "   <RectangleLabels name=\"bbox\" toName=\"image\" strokeWidth=\"1\" smart=\"true\">\n",
     "      <Label value=\"Heading\" background=\"green\"/>\n",
     "      <Label value=\"Paragraph\" background=\"blue\"/>\n",
     "   </RectangleLabels>\n",
     "\n",
-    "   <TextArea name=\"transcription\" toName=\"image\" \n",
-    "   editable=\"true\" perRegion=\"true\" required=\"false\" \n",
-    "   maxSubmissions=\"1\" rows=\"5\" placeholder=\"Recognized Text\" \n",
+    "   <TextArea name=\"transcription\" toName=\"image\"\n",
+    "   editable=\"true\" perRegion=\"true\" required=\"false\"\n",
+    "   maxSubmissions=\"1\" rows=\"5\" placeholder=\"Recognized Text\"\n",
     "   displayMode=\"region-list\"/>\n",
     "</View>\n",
     "\"\"\"\n",
@@ -124,11 +124,11 @@
     "\n",
     "print(label_config2)\n",
     "\n",
-    "# Projects need a title and a labeling config, at a minimum. \n",
+    "# Projects need a title and a labeling config, at a minimum.\n",
     "proj = ls.projects.create(\n",
-    "    title=\"OCR for RAG PDF\",\n",
-    "    label_config=label_config, \n",
-    "    workspace=94519, \n",
+    "    title=\"OCR for RAG PDF (Acronyms)\",\n",
+    "    label_config=label_config,\n",
+    "    workspace=110870,\n",
     "    color=\"#00FFFF\",\n",
     ")\n",
     "pid = proj.id\n",
@@ -152,12 +152,12 @@
    "source": [
     "# There are a few ways to upload your data to Label Studio\n",
     "\n",
-    "# 1. As a CSV: \n",
+    "# 1. As a CSV:\n",
     "# task_ids = project.import_tasks('/full/path/to/your_tasks.csv')\n",
     "# print('Imported task IDs:', task_ids)\n",
     "\n",
     "\n",
-    "# 2 As a JSON: \n",
+    "# 2 As a JSON:\n",
     "ls.projects.import_tasks(\n",
     "    id=pid,\n",
     "    request=data\n",
@@ -182,7 +182,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 167,
+   "execution_count": null,
    "id": "ae19abb7-78dd-49e5-a9af-681a2fd8387d",
    "metadata": {},
    "outputs": [],
@@ -193,19 +193,18 @@
     "ner_label_config = \"\"\"\n",
     "<View>\n",
     "  \t<Text name=\"text\" value=\"$text\" />\n",
-    "    <Labels name=\"NER\" toName=\"text\">\n",
-    "        <Label value=\"PER\" />\n",
-    "        <Label value=\"ORG\" />\n",
+    "    <Labels name=\"ACRONYM_RECOGNITION\" toName=\"text\">\n",
+    "        <Label value=\"ACRONYM\" />\n",
     "  </Labels>\n",
     "</View>\n",
     "\"\"\"\n",
     "\n",
-    "# Projects need a title and a labeling config, at a minimum. \n",
+    "# Projects need a title and a labeling config, at a minimum.\n",
     "proj2 = ls.projects.create(\n",
-    "    title=\"NER on OCR\",\n",
+    "    title=\"NER on OCR: Acronyms\",\n",
     "    description=\"\",\n",
-    "    label_config=ner_label_config, \n",
-    "    workspace=93723, \n",
+    "    label_config=ner_label_config,\n",
+    "    workspace=110870,\n",
     "    color=\"#800080\",\n",
     ")\n",
     "p2id = proj2.id"
@@ -219,12 +218,12 @@
    "outputs": [],
    "source": [
     "# Now, we can export data from our first project and upload it to our second project\n",
-    "\n",
+    "import json\n",
     "tasks = ls.tasks.list(project=proj.id)\n",
     "\n",
     "#Let's see what a task annotation looks like\n",
     "\n",
-    "#IF TASK HAS ANNOTATIONS \n",
+    "#IF TASK HAS ANNOTATIONS\n",
     "for i, task in enumerate(tasks):\n",
     "    text = \"\"\n",
     "    # You can access annotations in Label Studio JSON format\n",
@@ -232,9 +231,9 @@
     "        # print just the first task for investigation\n",
     "        print(f\"TASK {i}\")\n",
     "        print(json.dumps(task.annotations[0], indent=4))\n",
-    "    for a in task.annotations: \n",
+    "    for a in task.annotations:\n",
     "        results = a[\"result\"]\n",
-    "        for r in results: \n",
+    "        for r in results:\n",
     "            if \"text\" in r[\"value\"].keys():\n",
     "                text = text + r[\"value\"][\"text\"][0]\n",
     "    if text:\n",
@@ -242,7 +241,6 @@
     "            project=p2id,\n",
     "            data={'text': text}\n",
     "        )"
-
    ]
   },
   {
@@ -250,62 +248,246 @@
    "id": "88671f2a-c6b0-4c7a-8fff-896b420826ce",
    "metadata": {},
    "source": [
-    "## Step 4: Adding Preannotations"
+    "## Step 5: Refining Prompts with the Label Studio SDK"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "5315459b-4ec4-4dba-873f-9a42f52cfeb1",
+   "id": "f744f057",
    "metadata": {},
    "outputs": [],
    "source": [
-    "import json\n",
+    "# First, create a prompt in the Label Studio UI and copy the ID\n",
+    "prompt_id = -1  # Change this to your prompt ID\n",
+    "\n",
+    "# List all versions of the prompt\n",
+    "versions = ls.prompts.versions.list(\n",
+    "    prompt_id=prompt_id,\n",
+    ")\n",
+    "\n",
+    "for v in versions:\n",
+    "    print(\"Version ID:\", v.id)\n",
+    "    print(\"Title:\", v.title)\n",
+    "    print(\"Provider:\", v.provider)\n",
+    "    print(\"Provider Model ID:\", v.provider_model_id)\n",
+    "    print(\"Created at:\", v.created_at)\n",
+    "    print(\"Updated at:\", v.updated_at)\n",
+    "    print(\"Prompt:\", v.prompt[:100])\n",
+    "    print(\"\\n\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bad4e6f0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# List model provider connections and find VertexAI connection ID\n",
+    "# List all model provider connections\n",
+    "model_providers = ls.model_providers.list(\n",
+    ")\n",
+    "\n",
+    "print(f\"Found {len(model_providers)} model provider connection(s)\")\n",
+    "print()\n",
     "\n",
-    "print(p2id)\n",
-    "project = ls.projects.get(id=p2id)\n",
-    "li = project.get_label_interface()\n",
+    "# Filter for VertexAI provider\n",
+    "vertexai_connection = None\n",
+    "vertexai_connection_id = None\n",
     "\n",
-    "sample_pred = li.generate_sample_prediction()\n",
-    "sample_pred = json.dumps(sample_pred, indent=4)\n",
-    "print(sample_pred)\n"
+    "for provider in model_providers:\n",
+    "    print(f\"Provider: {provider.provider} | ID: {provider.id}\")\n",
+    "\n",
+    "    if provider.provider == \"VertexAI\":\n",
+    "        vertexai_connection = provider\n",
+    "        vertexai_connection_id = provider.id\n",
+    "\n",
+    "print()\n",
+    "\n",
+    "if vertexai_connection:\n",
+    "    print(\"✓ Found VertexAI connection!\")\n",
+    "    print(f\"Connection ID: {vertexai_connection_id}\")\n",
+    "else:\n",
+    "    print(\"⚠ No VertexAI provider connection found\")\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 175,
-   "id": "dcb55d06-1a67-409f-bec0-c05c3a910ae3",
+   "execution_count": null,
+   "id": "f6caf984",
    "metadata": {},
    "outputs": [],
    "source": [
-    "import random\n",
-    "from label_studio_sdk.label_interface.objects import PredictionValue\n",
-    "\n",
-    "tasks = ls.tasks.list(project=p2id)\n",
-    "for task in tasks:\n",
-    "    predicted_slogan = {\"from_name\": \"NER\",\n",
-    "                        \"to_name\": \"text\",\n",
-    "                        \"type\": \"labels\",\n",
-    "                        \"value\": {\n",
-    "                            \"start\": 10,\n",
-    "                            \"end\": 20,\n",
-    "                            \"labels\": [\n",
-    "                                \"PER\"\n",
-    "                            ]\n",
-    "                        }\n",
-    "                       }\n",
-    "    prediction = PredictionValue(\n",
-    "        model_version='random-ai-prediction',\n",
-    "        score=0.99,\n",
-    "        result=[predicted_label, predicted_slogan]\n",
+    "# Refine a specific prompt version using the VertexAI connection\n",
+    "# Specify the version ID you want to refine\n",
+    "version_id = -1  # Change this to the version ID you want to refine\n",
+    "project_id = -1  # Change this to your project ID\n",
+    "teacher_model_name = \"gemini-2.5-pro\"  # The model to use for refinement\n",
+    "teacher_model_provider_connection_id = vertexai_connection_id  # Using VertexAI connection ID from previous cell\n",
+    "\n",
+    "# Refine the prompt version\n",
+    "refined_result = ls.prompts.versions.refine_prompt(\n",
+    "    prompt_id=prompt_id,\n",
+    "    version_id=version_id,\n",
+    "    project_id=project_id,\n",
+    "    teacher_model_name=teacher_model_name,\n",
+    "    teacher_model_provider_connection_id=teacher_model_provider_connection_id,\n",
+    "    # Optional: Set to True to run refinement asynchronously\n",
+    "    # async_=True\n",
+    ")\n",
+    "\n",
+    "print(\"Prompt refinement completed!\")\n",
+    "print()\n",
+    "print(f\"Title: {refined_result.title}\")\n",
+    "print(f\"Refined Prompt: {refined_result.prompt}\")\n",
+    "print()\n",
+    "print(f\"Refinement Status: {refined_result.refinement_status}\")\n",
+    "print(f\"Refinement Job ID: {refined_result.refinement_job_id}\")\n",
+    "print(f\"Total Cost: ${refined_result.total_cost}\")\n",
+    "print()\n",
+    "print(\"Reasoning:\")\n",
+    "print(refined_result.reasoning)\n",
+    "print()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0a5384bf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Save the refined prompt as a new version\n",
+    "\n",
+    "# Create a new version using the refined prompt\n",
+    "new_version = ls.prompts.versions.create(\n",
+    "    prompt_id=prompt_id,\n",
+    "    prompt=refined_result.prompt,  # Using the refined prompt from previous cell\n",
+    "    title=refined_result.title or \"Refined Version\",  # Use refined title or default\n",
+    "    provider_model_id=\"gemini-2.5-pro\",  # Model ID to use\n",
+    "    provider=\"VertexAI\",  # Provider type\n",
+    "    model_provider_connection=vertexai_connection_id,  # Connection ID\n",
+    ")\n",
+    "\n",
+    "print(\"✓ New prompt version created successfully!\")\n",
+    "print()\n",
+    "print(f\"Version ID: {new_version.id}\")\n",
+    "print(f\"Title: {new_version.title}\")\n",
+    "print(f\"Provider: {new_version.provider}\")\n",
+    "print(f\"Model ID: {new_version.provider_model_id}\")\n",
+    "print(f\"Created at: {new_version.created_at}\")\n",
+    "print()\n",
+    "print(\"Prompt text:\")\n",
+    "print(new_version.prompt)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "96209f58",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Run inference on the new version using HasGT subset\n",
+    "import time\n",
+    "\n",
+    "# Start the inference run\n",
+    "print(\"Starting inference run on HasGT subset...\")\n",
+    "inference_run = ls.prompts.runs.create(\n",
+    "    prompt_id=prompt_id,\n",
+    "    version_id=new_version.id,  # Using the newly created version from previous cell\n",
+    "    project=project_id,\n",
+    "    project_subset=\"HasGT\",  # Only run on tasks with ground truth annotations\n",
+    ")\n",
+    "\n",
+    "print(f\"✓ Inference run started!\")\n",
+    "print(f\"Run ID: {inference_run.id}\")\n",
+    "print(f\"Status: {inference_run.status}\")\n",
+    "print(f\"Project: {inference_run.project}\")\n",
+    "print(f\"Subset: {inference_run.project_subset}\")\n",
+    "print()\n",
+    "\n",
+    "# Poll for completion\n",
+    "print(\"Polling for completion...\")\n",
+    "max_attempts = 60  # Maximum number of polling attempts\n",
+    "poll_interval = 5  # Seconds between polls\n",
+    "\n",
+    "for attempt in range(max_attempts):\n",
+    "    # Get the current status of all inference runs for this version\n",
+    "    runs = ls.prompts.runs.list(\n",
+    "        prompt_id=prompt_id,\n",
+    "        version_id=new_version.id,\n",
     "    )\n",
-    "    ls.predictions.create(task=task.id, **prediction.model_dump())\n"
+    "\n",
+    "    # Find our specific run\n",
+    "    current_run = None\n",
+    "    for run in runs:\n",
+    "        if run.id == inference_run.id:\n",
+    "            current_run = run\n",
+    "            break\n",
+    "\n",
+    "    if current_run:\n",
+    "        status = current_run.status\n",
+    "        print(f\"[{attempt + 1}/{max_attempts}] Status: {status}\", end=\"\")\n",
+    "\n",
+    "        if status == \"Completed\":\n",
+    "            print()\n",
+    "            print()\n",
+    "            print(\"✓ Inference run completed successfully!\")\n",
+    "            print(f\"Total tasks: {current_run.total_tasks}\")\n",
+    "            print(f\"Total predictions: {current_run.total_predictions}\")\n",
+    "            print(f\"Total correct predictions: {current_run.total_correct_predictions}\")\n",
+    "            print(f\"Completed at: {current_run.completed_at}\")\n",
+    "            break\n",
+    "        elif status == \"Failed\":\n",
+    "            print()\n",
+    "            print(\"✗ Inference run failed!\")\n",
+    "            break\n",
+    "        elif status == \"Canceled\":\n",
+    "            print()\n",
+    "            print(\"⚠ Inference run was canceled!\")\n",
+    "            break\n",
+    "        else:\n",
+    "            # Still in progress\n",
+    "            print(f\" - waiting {poll_interval}s...\")\n",
+    "            time.sleep(poll_interval)\n",
+    "    else:\n",
+    "        print(f\"Could not find run {inference_run.id}\")\n",
+    "        break\n",
+    "else:\n",
+    "    print()\n",
+    "    print(f\"⚠ Inference run did not complete within {max_attempts * poll_interval} seconds\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "aed95ec2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Get the micro_avg_prf1 indicator for the inference run\n",
+    "indicator_key = \"micro_avg_prf1\"\n",
+    "\n",
+    "# Get the specific indicator\n",
+    "indicator = ls.prompts.indicators.get(\n",
+    "    id=current_run.id,  # Using the inference run ID from previous cell\n",
+    "    indicator_key=indicator_key\n",
+    ")\n",
+    "\n",
+    "print(f\"✓ Retrieved indicator: {indicator_key}\")\n",
+    "print()\n",
+    "print(f\"Title: {indicator.title}\")\n",
+    "print()\n",
+    "print(\"Values:\")\n",
+    "print(indicator.values)\n"
    ]
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "lsworkshop",
    "language": "python",
    "name": "python3"
   },
@@ -319,8 +501,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.4"
-
+   "version": "3.13.7"
   }
  },
  "nbformat": 4,