Add pyright CI

.github/workflows/ci.yml

@@ -0,0 +1,26 @@
+name: pyright
+
+on:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  pyright:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Set up Python
+        run: uv python install
+
+      - name: Install the project
+        run: uv sync --all-extras --dev
+
+      - name: Run pyright
+        run: pyright

README.md

@@ -136,16 +136,19 @@ For a complete example of a ComputerInterface implementation, you can refer to t
 ### Best Practices
 
 1. **Resource Management**
+
    - Implement proper cleanup in your interface
    - Handle container/VM lifecycle appropriately
    - Clean up temporary files
 
 2. **Security**
+
    - Implement proper isolation between tasks
    - Handle sensitive data appropriately
    - Control network access
 
 3. **Scalability**
+
    - Consider implementing a pool of compute resources
    - Handle concurrent task execution
    - Implement proper resource limits
@@ -157,4 +160,4 @@ For a complete example of a ComputerInterface implementation, you can refer to t
 
 ## Contributing
 
-For questions or contributions, please open an issue or pull request.
+For questions or contributions, please open an issue or pull request.

project/alcatraz/alcatraz/clusters/local.py

@@ -1355,7 +1355,7 @@ async def commit_and_push(
                 max_pool_size=1000,
                 timeout=self.limits["docker_client_timeout_seconds"],
             )
-        res = docker_client_push.images.push(repository=repository, tag=tag)
+        res = docker_client_push.images.push(repository=repository, tag=tag) # type: ignore
         logger.info(f"pushed {repository}:{tag}")
         final_res = json.loads(res.strip().split("\n")[-1])
         if "error" in final_res:

project/nanoeval/nanoeval/solvers/computer_tasks/solver.py

@@ -348,7 +348,7 @@ def _get_convo_len_stats(
         # compute percentiles
         convo_lens = np.array(convo_lens)
         percentiles = np.percentile(convo_lens, [25, 50, 75])
-        summary_dict["convo_len_percentiles"] = percentiles.tolist()
+        summary_dict["convo_len_percentiles"] = percentiles.tolist() # type: ignore
 
         return {
             "frac_correct": frac_correct,

project/nanoeval/pyproject.toml

@@ -27,6 +27,7 @@ dependencies = [
     "ptpython",
     "ipython",
     "structlog",
+    "pandas-stubs>=2.2.3.241126",
 ]
 
 [tool.setuptools.packages.find]

pyproject.toml

@@ -4,9 +4,10 @@ version = "0.1.0"
 dependencies = [
     "alcatraz",
     "nanoeval",
-    "nanoeval-alcatraz",
+    "nanoeval_alcatraz",
     "python-dotenv>=1.0.1",
     "tiktoken>=0.9.0",
+    "pyright",
 ]
 requires-python = ">=3.11"
 
@@ -16,4 +17,4 @@ packages = ["swelancer"]
 [tool.uv.sources]
 nanoeval = { path = "project/nanoeval", editable = true }
 alcatraz = { path = "project/alcatraz", editable = true }
-nanoeval-alcatraz = { path = "project/nanoeval_alcatraz", editable = true }
+nanoeval_alcatraz = { path = "project/nanoeval_alcatraz", editable = true }

swelancer.py

@@ -331,7 +331,7 @@ async def get_tasks(self) -> list[SWELancerTask]:
 
             del task['price_limit']
             del task['canary']
-            swelancer_tasks.append(SWELancerTask(**task, attempt_id=str(i), environment=SWEFL_ENV, grade_every_step=False, docker_image=docker_image, instance=SwelancerInstance(repo="expensify")))
+            swelancer_tasks.append(SWELancerTask(**task, attempt_id=str(i), environment=SWEFL_ENV, grade_every_step=False, docker_image=docker_image, instance=SwelancerInstance(repo="expensify"))) # type: ignore
             i += 1
         return swelancer_tasks
 

swelancer_agent.py

@@ -69,10 +69,10 @@ def get_model_response(messages: list[dict[str, Any]]) -> str:
     messages = trim_messages(messages, 110000)
 
     chat_completion = client.chat.completions.create(
-        messages=messages,
+        messages=messages, # type: ignore
         model="gpt-4o",
     )
-    return chat_completion.choices[0].message.content
+    return chat_completion.choices[0].message.content # type: ignore
 
 
 @chz.chz
@@ -105,7 +105,7 @@ async def run(self, task: ComputerTask) -> AsyncGenerator[Step | FinalResult, No
                 for prompt_message in task.prompt:
                     messages.append({
                         "role": "user",
-                        "content": str(prompt_message["content"])
+                        "content": str(prompt_message["content"]) # type: ignore
                     })
                 messages.append({"role": "user", "content": """The repository is cloned in your CWD. You must send Python code in backticks in each response to me, and I will execute the code and send you back the result, for example: