increment project query count for manager features POC (#78)

* increment project query count for manager features POC

* bump version and add release notes

* fix tests

* more fixes

* reformat

* reformat again

Author: kelsey-wong

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.0.14] 2025-04-23
+- Update `codex-sdk` dependency to `0.1.0-alpha.17`.
+- Capture data for the number of times the validator API is called on a Codex project.
+
 ## [1.0.13] - 2025-04-22
 
 - Update `cleanlab-tlm` dependancy to `~=1.1`.
@@ -67,7 +71,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Initial release of the `cleanlab-codex` client library.
 
-[Unreleased]: https://github.com/cleanlab/cleanlab-codex/compare/v1.0.13...HEAD
+[Unreleased]: https://github.com/cleanlab/cleanlab-codex/compare/v1.0.14...HEAD
+[1.0.14]: https://github.com/cleanlab/cleanlab-codex/compare/v1.0.13...v1.0.14
 [1.0.13]: https://github.com/cleanlab/cleanlab-codex/compare/v1.0.12...v1.0.13
 [1.0.12]: https://github.com/cleanlab/cleanlab-codex/compare/v1.0.11...v1.0.12
 [1.0.11]: https://github.com/cleanlab/cleanlab-codex/compare/v1.0.10...v1.0.11

pyproject.toml

@@ -26,7 +26,7 @@ classifiers = [
 ]
 dependencies = [
   "cleanlab-tlm~=1.1",
-  "codex-sdk==0.1.0-alpha.14",
+  "codex-sdk==0.1.0-alpha.17",
   "pydantic>=2.0.0, <3",
 ]
 

src/cleanlab_codex/__about__.py

@@ -1,2 +1,2 @@
 # SPDX-License-Identifier: MIT
-__version__ = "1.0.13"
+__version__ = "1.0.14"

src/cleanlab_codex/project.py

@@ -221,3 +221,6 @@ def _query_project(
             return query_res.answer, entry
 
         return fallback_answer, entry
+
+    def increment_queries(self) -> None:
+        self._sdk_client.projects.increment_queries(self._id)

src/cleanlab_codex/validator.py

@@ -140,6 +140,8 @@ def validate(
                 final_metadata.update(processed_metadata)
             expert_answer = self._remediate(query=query, metadata=final_metadata)
 
+        self._project.increment_queries()
+
         return {
             "expert_answer": expert_answer,
             "is_bad_response": is_bad_response,
@@ -183,6 +185,8 @@ async def validate_async(
         if is_bad_response:
             expert_answer = self._remediate(query=query, metadata=final_metadata)
 
+        self._project.increment_queries()
+
         return {
             "expert_answer": expert_answer,
             "is_bad_response": is_bad_response,

tests/test_project.py

@@ -7,8 +7,14 @@
 from codex.types.projects.access_key_retrieve_project_id_response import (
     AccessKeyRetrieveProjectIDResponse,
 )
-from codex.types.projects.entry_query_response import Entry as SDKEntry
-from codex.types.projects.entry_query_response import EntryQueryResponse
+from codex.types.projects.entry_query_response import (
+    Entry as SDKEntry,
+)
+from codex.types.projects.entry_query_response import (
+    EntryManagedMetadata,
+    EntryManagedMetadataTrustworthiness,
+    EntryQueryResponse,
+)
 
 from cleanlab_codex.project import MissingProjectError, Project
 from cleanlab_codex.types.entry import EntryCreate
@@ -143,7 +149,12 @@ def test_init_nonexistent_project_id(mock_client_from_access_key: MagicMock) ->
 def test_query_question_found_fallback_answer(
     mock_client_from_access_key: MagicMock,
 ) -> None:
-    unanswered_entry = SDKEntry(id=str(uuid.uuid4()), question="What is the capital of France?", answer=None)
+    unanswered_entry = SDKEntry(
+        id=str(uuid.uuid4()),
+        question="What is the capital of France?",
+        answer=None,
+        managed_metadata=EntryManagedMetadata(trustworthiness=EntryManagedMetadataTrustworthiness(scores=[0.95])),
+    )
 
     mock_client_from_access_key.projects.entries.query.return_value = EntryQueryResponse(
         entry=unanswered_entry, answer=None
@@ -158,7 +169,12 @@ def test_query_question_found_fallback_answer(
 def test_query_question_not_found_fallback_answer(
     mock_client_from_access_key: MagicMock,
 ) -> None:
-    mock_entry = SDKEntry(id="fake-id", question="What is the capital of France?", answer=None)
+    mock_entry = SDKEntry(
+        id="fake-id",
+        question="What is the capital of France?",
+        answer=None,
+        managed_metadata=EntryManagedMetadata(trustworthiness=EntryManagedMetadataTrustworthiness(scores=[0.95])),
+    )
     mock_client_from_access_key.projects.entries.query.return_value = EntryQueryResponse(entry=mock_entry, answer=None)
 
     project = Project(mock_client_from_access_key, FAKE_PROJECT_ID)
@@ -173,6 +189,7 @@ def test_query_answer_found(mock_client_from_access_key: MagicMock) -> None:
         id=str(uuid.uuid4()),
         question="What is the capital of France?",
         answer="Paris",
+        managed_metadata=EntryManagedMetadata(trustworthiness=EntryManagedMetadataTrustworthiness(scores=[0.95])),
     )
     mock_client_from_access_key.projects.entries.query.return_value = EntryQueryResponse(
         answer="Paris", entry=answered_entry
@@ -190,6 +207,7 @@ def test_query_answer_found_with_metadata(mock_client_from_access_key: MagicMock
         question="What is the capital of France?",
         answer="Paris",
         client_query_metadata=[{"trustworthiness_score": 0.95}],
+        managed_metadata=EntryManagedMetadata(trustworthiness=EntryManagedMetadataTrustworthiness(scores=[0.95])),
     )
     mock_client_from_access_key.projects.entries.query.return_value = EntryQueryResponse(
         answer="Paris", entry=answered_entry