Merge pull request #237 from roboflow/upload-more

Expand upload capabilities from the command line

Author: tonylampada

.dockerignore

@@ -0,0 +1 @@
+/tests/manual/data

.github/workflows/test.yml

@@ -31,8 +31,4 @@ jobs:
         run: |
           make check_code_quality
       - name: 🧪 Test
-        env:
-          ROBOFLOW_API_KEY: ${{ secrets.ROBOFLOW_API_KEY }}
-          PROJECT_NAME: ${{ secrets.PROJECT_NAME }}
-          PROJECT_VERSION: ${{ secrets.PROJECT_VERSION }}
         run: "python -m unittest"

CLI-COMMANDS.md

@@ -78,7 +78,7 @@ options:
   -w WORKSPACE    specify a workspace url or id (will use default workspace if not specified)
   -p PROJECT      project will be created if it does not exist
   -c CONCURRENCY  how many image uploads to perform concurrently (default: 10)
-  -f FORMAT       dataset format. Valid options are [voc, yolov8, yolov5, auto] (use auto for autodetect)
+  -n BATCH_NAME   name of batch to upload to within project
 ```
 
 ## Example: download dataset
@@ -105,6 +105,26 @@ drwxr-xr-x@ 1214 tony  staff    38K Jan  5 10:32 train
 drwxr-xr-x@  118 tony  staff   3.7K Jan  5 10:32 valid
 ```
 
+## Example: import a dataset
+
+Upload a dataset from a folder to a project in your workspace
+
+```bash
+roboflow import -w my-workspace -p my-chess ~/tmp/chess
+```
+
+```
+loading Roboflow workspace...
+loading Roboflow project...
+Uploading to existing project my-workspace/my-chess
+[UPLOADED] /home/jonny/tmp/chess/102_jpg.rf.205e2a0cb0fabbbf32b4a936e2d6f1e4.jpg (sFpTfnyLpLA8QcqPwdvf) / annotations = OK
+[UPLOADED] /home/jonny/tmp/chess/2_jpg.rf.c1a4ed4e0c3947743b22ede09f7e1212.jpg (wDA2yxnLJWY5YwYwO7dP) / annotations = OK
+[UPLOADED] /home/jonny/tmp/chess/221_jpg.rf.e841c9bbb31a135b8f6274643f522686.jpg (UCv7MeuvEqo7PYElatEn) / annotations = OK
+[UPLOADED] /home/jonny/tmp/chess/10_jpg.rf.841f3ccdfc4b93ee68566e602025c03f.jpg (HnkCpUcYzxStvQF49VQW) / annotations = OK
+[UPLOADED] /home/jonny/tmp/chess/130_jpg.rf.29f756d510d2e488eb5e12769c7707ff.jpg (WxrFIhfaJ9H1JvaXMgfF) / annotations = OK
+[UPLOADED] /home/jonny/tmp/chess/112_jpg.rf.1a6e7b87410fa3f787f10e82bd02b54e.jpg (7tWtAn573cKrefeg5pIO) / annotations = OK
+```
+
 ## Example: list workspaces
 List the workspaces you have access to
 

pyproject.toml

@@ -30,8 +30,7 @@ tests = ["B201", "B301"]
 
 [tool.autoflake]
 check = true
-imports = ["cv2", "roboflow", "supervision"]
-
+imports = ["cv2", "roboflow"]
 
 [tool.ruff]
 target-version = "py38"

requirements.txt

@@ -11,7 +11,6 @@ python-dateutil
 python-dotenv
 requests
 six
-supervision
 urllib3>=1.26.6
 tqdm>=4.41.0
 PyYAML>=5.3.1

roboflow/__init__.py

@@ -14,7 +14,7 @@
 from roboflow.models import CLIPModel, GazeModel  # noqa: F401
 from roboflow.util.general import write_line
 
-__version__ = "1.1.21"
+__version__ = "1.1.22"
 
 
 def check_key(api_key, model, notebook, num_retries=0):

roboflow/adapters/rfapi.py

@@ -5,7 +5,7 @@
 import requests
 from requests_toolbelt.multipart.encoder import MultipartEncoder
 
-from roboflow.config import API_URL, DEFAULT_BATCH_NAME
+from roboflow.config import API_URL, DEFAULT_BATCH_NAME, DEFAULT_JOB_NAME
 from roboflow.util import image_utils
 
 
@@ -58,7 +58,6 @@ def upload_image(
 
     # If image is not a hosted image
     if not hosted_image:
-        batch_name = batch_name or DEFAULT_BATCH_NAME
         image_name = os.path.basename(image_path)
         imgjpeg = image_utils.file2jpeg(image_path)
 
@@ -103,6 +102,7 @@ def save_annotation(
     annotation_name: str,
     annotation_string: str,
     image_id: str,
+    job_name: str = DEFAULT_JOB_NAME,
     is_prediction: bool = False,
     annotation_labelmap=None,
     overwrite: bool = False,
@@ -115,7 +115,9 @@ def save_annotation(
         image_id (str): image id you'd like to upload that has annotations for it.
     """
 
-    upload_url = _save_annotation_url(api_key, project_url, annotation_name, image_id, is_prediction, overwrite)
+    upload_url = _save_annotation_url(
+        api_key, project_url, annotation_name, image_id, job_name, is_prediction, overwrite
+    )
 
     response = requests.post(
         upload_url,
@@ -143,8 +145,10 @@ def save_annotation(
     return responsejson
 
 
-def _save_annotation_url(api_key, project_url, name, image_id, is_prediction, overwrite=False):
+def _save_annotation_url(api_key, project_url, name, image_id, job_name, is_prediction, overwrite=False):
     url = f"{API_URL}/dataset/{project_url}/annotate/{image_id}?api_key={api_key}" f"&name={name}"
+    if job_name:
+        url += f"&jobName={job_name}"
     if is_prediction:
         url += "&prediction=true"
     if overwrite:

roboflow/config.py

@@ -71,13 +71,14 @@ def get_conditional_configuration_variable(key, default):
 TYPE_SEMANTIC_SEGMENTATION = "semantic-segmentation"
 
 DEFAULT_BATCH_NAME = "Pip Package Upload"
+DEFAULT_JOB_NAME = "Annotated via API"
 
 RF_WORKSPACES = get_conditional_configuration_variable("workspaces", default={})
 
 
 def load_roboflow_api_key(workspace_url=None):
-    if os.getenv("RF_API_KEY") is not None:
-        return os["RF_API_KEY"]
+    if os.getenv("ROBOFLOW_API_KEY") is not None:
+        return os.getenv("ROBOFLOW_API_KEY")
     RF_WORKSPACES = get_conditional_configuration_variable("workspaces", default={})
     workspaces_by_url = {w["url"]: w for w in RF_WORKSPACES.values()}
     default_workspace_url = get_conditional_configuration_variable("RF_WORKSPACE", default=None)

roboflow/core/project.py

@@ -2,13 +2,14 @@
 import json
 import os
 import sys
+import time
 import warnings
 
 import requests
 from PIL import Image, UnidentifiedImageError
 
 from roboflow.adapters import rfapi
-from roboflow.config import API_URL, DEFAULT_BATCH_NAME, DEMO_KEYS
+from roboflow.config import API_URL, DEMO_KEYS
 from roboflow.core.version import Version
 from roboflow.util.general import retry
 from roboflow.util.image_utils import load_labelmap
@@ -361,7 +362,7 @@ def upload(
         image_id: str = None,
         split: str = "train",
         num_retry_uploads: int = 0,
-        batch_name: str = DEFAULT_BATCH_NAME,
+        batch_name: str = None,
         tag_names: list = [],
         is_prediction: bool = False,
         **kwargs,
@@ -454,7 +455,7 @@ def single_upload(
         image_id=None,
         split="train",
         num_retry_uploads=0,
-        batch_name=DEFAULT_BATCH_NAME,
+        batch_name=None,
         tag_names=[],
         is_prediction: bool = False,
         annotation_overwrite=False,
@@ -470,44 +471,64 @@ def single_upload(
         if isinstance(annotation_labelmap, str):
             annotation_labelmap = load_labelmap(annotation_labelmap)
         uploaded_image, uploaded_annotation = None, None
+        upload_time = None
         if image_path:
-            uploaded_image = retry(
-                num_retry_uploads,
-                Exception,
-                rfapi.upload_image,
-                self.__api_key,
-                project_url,
-                image_path,
-                hosted_image=hosted_image,
-                split=split,
-                batch_name=batch_name,
-                tag_names=tag_names,
-                sequence_number=sequence_number,
-                sequence_size=sequence_size,
-                **kwargs,
-            )
-            image_id = uploaded_image["id"]
+            t0 = time.time()
+            try:
+                uploaded_image = retry(
+                    num_retry_uploads,
+                    Exception,
+                    rfapi.upload_image,
+                    self.__api_key,
+                    project_url,
+                    image_path,
+                    hosted_image=hosted_image,
+                    split=split,
+                    batch_name=batch_name,
+                    tag_names=tag_names,
+                    sequence_number=sequence_number,
+                    sequence_size=sequence_size,
+                    **kwargs,
+                )
+                image_id = uploaded_image["id"]
+            except BaseException as e:
+                uploaded_image = {"error": e}
+            finally:
+                upload_time = time.time() - t0
 
-        if annotation_path:
+        annotation_time = None
+        if annotation_path and image_id:
             annotation_name, annotation_str = self._annotation_params(annotation_path)
             try:
+                t0 = time.time()
                 uploaded_annotation = rfapi.save_annotation(
                     self.__api_key,
                     project_url,
                     annotation_name,
                     annotation_str,
                     image_id,
+                    job_name=batch_name,
                     is_prediction=is_prediction,
                     annotation_labelmap=annotation_labelmap,
                     overwrite=annotation_overwrite,
                 )
             except BaseException as e:
                 uploaded_annotation = {"error": e}
-        return {"image": uploaded_image, "annotation": uploaded_annotation}
+            finally:
+                annotation_time = time.time() - t0
+        return {
+            "image": uploaded_image,
+            "annotation": uploaded_annotation,
+            "upload_time": upload_time,
+            "annotation_time": annotation_time,
+        }
 
     def _annotation_params(self, annotation_path):
         annotation_name, annotation_string = None, None
-        if os.path.exists(annotation_path):
+        if isinstance(annotation_path, dict):
+            annotation_name = annotation_path["name"]
+            annotation_string = json.dumps(annotation_path["parsed"])
+        elif os.path.exists(annotation_path):
             with open(annotation_path, "r"):
                 annotation_string = open(annotation_path, "r").read()
             annotation_name = os.path.basename(annotation_path)