Use timestamp in place of version for query table for expired tables

caveclient/materializationengine.py

@@ -53,7 +53,7 @@
             )
     else:
         raise ValueError(
-            f'Unknown response type: {response.headers.get("Content-Type")}'
+            f"Unknown response type: {response.headers.get('Content-Type')}"
         )
 
 
@@ -228,6 +228,12 @@
         self._tables = None
         self._views = None
 
+    @property
+    @cached(cache=TTLCache(maxsize=1, ttl=60 * 60 * 1))
+    def available_versions(self) -> list[int]:
+        """Get the available versions for this materialization client."""
+        return sorted(self.get_versions(expired=False))
+
     @property
     def datastack_name(self):
         """The name of the datastack."""
@@ -361,6 +367,10 @@
             datastack_name = self.datastack_name
         if version is None:
             version = self.version
+        if version not in self.available_versions:
+            raise ValueError(
+                f"Annotation count must use a materialized version ({self.available_versions})."
+            )
         endpoint_mapping = self.default_url_mapping
         endpoint_mapping["datastack_name"] = datastack_name
         endpoint_mapping["table_name"] = table_name
@@ -406,6 +416,7 @@
         d["expires_on"] = convert_timestamp(d["expires_on"])
         return d
 
+    @cached(cache=TTLCache(maxsize=50, ttl=60 * 60 * 24))
     def get_timestamp(
         self, version: Optional[int] = None, datastack_name: str = None
     ) -> datetime:
@@ -741,6 +752,14 @@
 
         if desired_resolution is None:
             desired_resolution = self.desired_resolution
+        if (
+            materialization_version not in self.available_versions
+            and materialization_version is not None
+        ):
+            timestamp = self.get_timestamp(
+                version=materialization_version, datastack_name=datastack_name
+            )
+            materialization_version = None
         if timestamp is not None:
             if materialization_version is not None:
                 raise ValueError("cannot specify timestamp and materialization version")
@@ -975,6 +994,10 @@
 
         if materialization_version is None:
             materialization_version = self.version
+        if materialization_version not in self.available_versions:
+            raise ValueError(
+                f"Cannot use `join_query` for a non-materialized version. Please use a materialized version ({self.available_versions}) or live_live_query."
+            )
         if datastack_name is None:
             datastack_name = self.datastack_name
         if desired_resolution is None:
@@ -1178,7 +1201,7 @@
                     all_svid_lengths.append(n_svids)
                     logger.info(f"{sv_col} has {n_svids} to update")
                     all_svids = np.append(all_svids, svids[~is_latest_root])
-        logger.info(f"num zero svids: {np.sum(all_svids==0)}")
+        logger.info(f"num zero svids: {np.sum(all_svids == 0)}")
         logger.info(f"all_svids dtype {all_svids.dtype}")
         logger.info(f"all_svid_lengths {all_svid_lengths}")
         with MyTimeIt("get_roots"):
@@ -2225,6 +2248,11 @@
             datastack_name = self.datastack_name
         if version is None:
             version = self.version
+        if version not in self.available_versions:
+            raise ValueError(
+                f"Materialization version must not be expired for views. "
+                f"Available versions: {self.available_versions}"
+            )
         endpoint_mapping = self.default_url_mapping
         endpoint_mapping["datastack_name"] = datastack_name
         endpoint_mapping["version"] = version
@@ -2262,6 +2290,14 @@
             datastack_name = self.datastack_name
         if materialization_version is None:
             materialization_version = self.version
+        if (
+            materialization_version not in self.available_versions
+            and materialization_version is not None
+        ):
+            raise ValueError(
+                f"Materialization version must not be expired for view metadata query. "
+                f"Available versions: {self.available_versions}"
+            )
 
         endpoint_mapping = self.default_url_mapping
         endpoint_mapping["view_name"] = view_name
@@ -2338,6 +2374,14 @@
             datastack_name = self.datastack_name
         if materialization_version is None:
             materialization_version = self.version
+        if (
+            materialization_version not in self.available_versions
+            and materialization_version is not None
+        ):
+            raise ValueError(
+                f"Materialization version must not be expired for view schema query. "
+                f"Available versions: {self.available_versions}"
+            )
 
         endpoint_mapping = self.default_url_mapping
         endpoint_mapping["datastack_name"] = datastack_name
@@ -2452,6 +2496,14 @@
             materialization_version = self.version
         if datastack_name is None:
             datastack_name = self.datastack_name
+        if (
+            materialization_version not in self.available_versions
+            and materialization_version is not None
+        ):
+            raise ValueError(
+                f"Materialization version must not be expired for view query. "
+                f"Available versions: {self.available_versions}"
+            )
 
         url, data, query_args, encoding = self._format_query_components(
             datastack_name,

caveclient/tools/testing.py

@@ -483,7 +483,6 @@ def mockedCAVEclient():
                 status=200,
                 match=[query_param_matcher({"expired": True})],
             )
-
             if set_version is not None:
                 mat_mapping["version"] = set_version
                 version_metadata_url = mat_endpoints["version_metadata"].format_map(
@@ -533,6 +532,14 @@ def mockedCAVEclient():
             client.chunkedgraph
         if materialization:
             client.materialize
+            responses.add(
+                responses.GET,
+                mat_version_list_url,
+                json=available_materialization_versions,
+                status=200,
+                match=[query_param_matcher({"expired": False})],
+            )
+            client.materialize.available_versions
         if json_service:
             client.state
         if skeleton_service: