WIP: allow proofreading without agglomerate file (identity mapping)

frontend/javascripts/admin/admin_rest_api.ts

@@ -1939,19 +1939,25 @@ export async function getAgglomeratesForSegmentsFromDatastore<T extends number |
   segmentIds: Array<T>,
 ): Promise<Mapping> {
   const segmentIdBuffer = serializeProtoListOfLong<T>(segmentIds);
-  const listArrayBuffer: ArrayBuffer = await doWithToken((token) => {
-    const params = new URLSearchParams({ token });
-    return Request.receiveArraybuffer(
-      `${dataStoreUrl}/data/datasets/${dataSourceId.owningOrganization}/${dataSourceId.directoryName}/layers/${layerName}/agglomerates/${mappingId}/agglomeratesForSegments?${params}`,
-      {
-        method: "POST",
-        body: segmentIdBuffer,
-        headers: {
-          "Content-Type": "application/octet-stream",
+  let listArrayBuffer: ArrayBuffer;
+  if (mappingId === "") {
+    // Identity mapping, every segment is its own agglomerate
+    listArrayBuffer = segmentIdBuffer;
+  } else {
+    listArrayBuffer = await doWithToken((token) => {
+      const params = new URLSearchParams({ token });
+      return Request.receiveArraybuffer(
+        `${dataStoreUrl}/data/datasets/${dataSourceId.owningOrganization}/${dataSourceId.directoryName}/layers/${layerName}/agglomerates/${mappingId}/agglomeratesForSegments?${params}`,
+        {
+          method: "POST",
+          body: segmentIdBuffer,
+          headers: {
+            "Content-Type": "application/octet-stream",
+          },
         },
-      },
-    );
-  });
+      );
+    });
+  }
   // Ensure that the values are bigint if the keys are bigint
   const adaptToType = Utils.isBigInt(segmentIds[0])
     ? (el: NumberLike) => BigInt(el)

webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala

@@ -17,7 +17,13 @@ import com.scalableminds.webknossos.datastore.helpers.{
   SegmentStatisticsParameters
 }
 import com.scalableminds.webknossos.datastore.models.datasource.inbox.InboxDataSource
-import com.scalableminds.webknossos.datastore.models.datasource.{DataLayer, DataSource, DataSourceId, GenericDataSource}
+import com.scalableminds.webknossos.datastore.models.datasource.{
+  DataLayer,
+  DataSource,
+  DataSourceId,
+  GenericDataSource,
+  SegmentationLayer
+}
 import com.scalableminds.webknossos.datastore.services._
 import com.scalableminds.webknossos.datastore.services.uploading._
 import com.scalableminds.webknossos.datastore.storage.{AgglomerateFileKey, DataVaultService}
@@ -338,6 +344,26 @@ class DataSourceController @Inject()(
     }
   }
 
+  def largestSegmentId(
+      organizationId: String,
+      datasetDirectoryName: String,
+      dataLayerName: String,
+  ): Action[AnyContent] = Action.async { implicit request =>
+    accessTokenService.validateAccessFromTokenContext(
+      UserAccessRequest.readDataSources(DataSourceId(datasetDirectoryName, organizationId))) {
+
+      for {
+        (_, layer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId,
+                                                                     datasetDirectoryName,
+                                                                     dataLayerName)
+        largestSegmentId <- layer match {
+          case l: SegmentationLayer => Fox.successful(l.largestSegmentId)
+          case _                    => Fox.failure("tried looking up largesetAgglomerateId on layer that is not a segmentation layer")
+        }
+      } yield Ok(Json.toJson(largestSegmentId))
+    }
+  }
+
   def agglomerateIdsForSegmentIds(
       organizationId: String,
       datasetDirectoryName: String,

webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/MappingProvider.scala

@@ -25,9 +25,9 @@ class MappingProvider(layer: SegmentationLayer) {
 
 object MappingProvider {
 
-  val mappingsDir = "mappings"
+  private val mappingsDir = "mappings"
 
-  val mappingFileExtension = "json"
+  private val mappingFileExtension = "json"
 
   def exploreMappings(layerDir: Path): Option[Set[String]] = {
     val mappingSet = PathUtils

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala

@@ -69,7 +69,8 @@ class BinaryDataService(val dataBaseDir: Path,
           data <- handleDataRequest(request)
           mappedDataFox <- agglomerateServiceOpt.map { agglomerateService =>
             convertIfNecessary(
-              request.settings.appliedAgglomerate.isDefined && request.dataLayer.category == Category.segmentation && request.cuboid.mag.maxDim <= MaxMagForAgglomerateMapping,
+              request.settings.appliedAgglomerate.isDefined && request.dataLayer.category == Category.segmentation && request.cuboid.mag.maxDim <= MaxMagForAgglomerateMapping && !request.settings.appliedAgglomerate
+                .contains(""),
               data,
               agglomerateService.applyAgglomerate(request),
               request

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/MeshMappingHelper.scala

@@ -52,7 +52,7 @@ trait MeshMappingHelper {
             case _                => if (omitMissing) Fox.successful(List.empty) else segmentIdsBox.toFox
           }
         } yield segmentIds
-      case (Some(mappingName), Some(tracingId)) =>
+      case (mappingNameOpt, Some(tracingId)) =>
         // An editable mapping tracing id is supplied. Ask the tracingstore for the segment ids. If it doesn’t know,
         // use the mappingName (here the editable mapping’s base mapping) to look it up from file.
         for {
@@ -65,17 +65,21 @@ trait MeshMappingHelper {
           else // the agglomerate id is not present in the editable mapping. Fetch its info from the base mapping.
             for {
               agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox
-              localSegmentIds <- agglomerateService.segmentIdsForAgglomerateId(
-                AgglomerateFileKey(
-                  organizationId,
-                  datasetDirectoryName,
-                  dataLayerName,
-                  mappingName
-                ),
-                agglomerateId
-              )
+              localSegmentIds <- mappingNameOpt match {
+                case Some(mappingName) =>
+                  agglomerateService.segmentIdsForAgglomerateId(
+                    AgglomerateFileKey(
+                      organizationId,
+                      datasetDirectoryName,
+                      dataLayerName,
+                      mappingName
+                    ),
+                    agglomerateId
+                  )
+                case None =>
+                  Full(List(agglomerateId)) // Proofreading with no base mapping. Segment id is mapped to self.
+              }
             } yield localSegmentIds
         } yield segmentIds
-      case _ => Fox.failure("Cannot determine segment ids for editable mapping without base mapping")
     }
 }

webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes

@@ -77,6 +77,7 @@ GET           /datasets/:organizationId/:datasetDirectoryName/layers/:dataLayerN
 POST          /datasets/:organizationId/:datasetDirectoryName/layers/:dataLayerName/agglomerates/:mappingName/agglomeratesForSegments                 @com.scalableminds.webknossos.datastore.controllers.DataSourceController.agglomerateIdsForSegmentIds(organizationId: String, datasetDirectoryName: String, dataLayerName: String, mappingName: String)
 GET           /datasets/:organizationId/:datasetDirectoryName/layers/:dataLayerName/agglomerates/:mappingName/agglomeratesForAllSegments              @com.scalableminds.webknossos.datastore.controllers.DataSourceController.agglomerateIdsForAllSegmentIds(organizationId: String, datasetDirectoryName: String, dataLayerName: String, mappingName: String)
 GET           /datasets/:organizationId/:datasetDirectoryName/layers/:dataLayerName/agglomerates/:mappingName/positionForSegment                      @com.scalableminds.webknossos.datastore.controllers.DataSourceController.positionForSegmentViaAgglomerateFile(organizationId: String, datasetDirectoryName: String, dataLayerName: String, mappingName: String, segmentId: Long)
+GET           /datasets/:organizationId/:datasetDirectoryName/layers/:dataLayerName/largestSegmentId                                                  @com.scalableminds.webknossos.datastore.controllers.DataSourceController.largestSegmentId(organizationId: String, datasetDirectoryName: String, dataLayerName: String)
 
 # Mesh files
 GET           /datasets/:organizationId/:datasetDirectoryName/layers/:dataLayerName/meshes                                                            @com.scalableminds.webknossos.datastore.controllers.DSMeshController.listMeshFiles(organizationId: String, datasetDirectoryName: String, dataLayerName: String)

webknossos-datastore/proto/EditableMappingInfo.proto

@@ -3,7 +3,7 @@ syntax = "proto2";
 package com.scalableminds.webknossos.datastore;
 
 message EditableMappingInfo {
-  required string baseMappingName = 1;
+  optional string baseMappingName = 1;
   required int64 createdTimestamp = 2;
   required int64 largestAgglomerateId = 3;
 }

webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala

@@ -11,6 +11,7 @@ import com.scalableminds.webknossos.datastore.helpers.{
   GetMultipleSegmentIndexParameters,
   GetSegmentIndexParameters,
   MissingBucketHeaders,
+  ProtoGeometryImplicits,
   SegmentIndexData
 }
 import com.scalableminds.webknossos.datastore.models.datasource.DataLayer
@@ -32,16 +33,19 @@ class TSRemoteDatastoreClient @Inject()(
     val lifecycle: ApplicationLifecycle
 )(implicit ec: ExecutionContext)
     extends LazyLogging
+    with ProtoGeometryImplicits
     with MissingBucketHeaders {
 
   private lazy val dataStoreUriCache: AlfuCache[(String, String), String] = AlfuCache()
   private lazy val voxelSizeCache: AlfuCache[String, VoxelSize] = AlfuCache(timeToLive = 10 minutes)
-  private lazy val largestAgglomerateIdCache: AlfuCache[(RemoteFallbackLayer, String, Option[String]), Long] =
+  private lazy val largestAgglomerateIdCache: AlfuCache[(RemoteFallbackLayer, Option[String], Option[String]), Long] =
     AlfuCache(timeToLive = 10 minutes)
 
-  def getAgglomerateSkeleton(remoteFallbackLayer: RemoteFallbackLayer, mappingName: String, agglomerateId: Long)(
-      implicit tc: TokenContext): Fox[Array[Byte]] =
+  def getAgglomerateSkeleton(remoteFallbackLayer: RemoteFallbackLayer,
+                             mappingNameOpt: Option[String],
+                             agglomerateId: Long)(implicit tc: TokenContext): Fox[Array[Byte]] =
     for {
+      mappingName <- mappingNameOpt.toFox ?~> "cannot get agglomerate skeleton without hdf5 base mapping"
       remoteLayerUri <- getRemoteLayerUri(remoteFallbackLayer)
       result <- rpc(s"$remoteLayerUri/agglomerates/$mappingName/skeleton/$agglomerateId").withTokenFromContext.getWithBytesResponse
     } yield result
@@ -73,35 +77,51 @@ class TSRemoteDatastoreClient @Inject()(
     } yield result
 
   def getAgglomerateIdsForSegmentIds(remoteFallbackLayer: RemoteFallbackLayer,
-                                     mappingName: String,
+                                     mappingNameOpt: Option[String],
                                      segmentIdsOrdered: List[Long])(implicit tc: TokenContext): Fox[List[Long]] =
-    for {
-      remoteLayerUri <- getRemoteLayerUri(remoteFallbackLayer)
-      segmentIdsOrderedProto = ListOfLong(items = segmentIdsOrdered)
-      result <- rpc(s"$remoteLayerUri/agglomerates/$mappingName/agglomeratesForSegments").withTokenFromContext.silent
-        .postProtoWithProtoResponse[ListOfLong, ListOfLong](segmentIdsOrderedProto)(ListOfLong)
-    } yield result.items.toList
-
-  def getAgglomerateGraph(remoteFallbackLayer: RemoteFallbackLayer, baseMappingName: String, agglomerateId: Long)(
-      implicit tc: TokenContext): Fox[AgglomerateGraph] =
-    for {
-      remoteLayerUri <- getRemoteLayerUri(remoteFallbackLayer)
-      result <- rpc(s"$remoteLayerUri/agglomerates/$baseMappingName/agglomerateGraph/$agglomerateId").silent.withTokenFromContext.silent
-        .getWithProtoResponse[AgglomerateGraph](AgglomerateGraph)
-    } yield result
+    mappingNameOpt.map { mappingName =>
+      for {
+        remoteLayerUri <- getRemoteLayerUri(remoteFallbackLayer)
+        segmentIdsOrderedProto = ListOfLong(items = segmentIdsOrdered)
+        result <- rpc(s"$remoteLayerUri/agglomerates/$mappingName/agglomeratesForSegments").withTokenFromContext.silent
+          .postProtoWithProtoResponse[ListOfLong, ListOfLong](segmentIdsOrderedProto)(ListOfLong)
+      } yield result.items.toList
+    }.getOrElse(Fox.successful(segmentIdsOrdered))
+
+  def getAgglomerateGraph(remoteFallbackLayer: RemoteFallbackLayer,
+                          mappingNameOpt: Option[String],
+                          agglomerateId: Long,
+                          segmentPosition: Option[Vec3Int])(implicit tc: TokenContext): Fox[AgglomerateGraph] =
+    mappingNameOpt match {
+      case None =>
+        // Identity mapping: graph with just the agglomerate id as mapping.
+        // passed segmentPosition is used if supplied. Should only be None where it is not later needed in the graph.
+        Fox.successful(
+          AgglomerateGraph(List(agglomerateId),
+                           List.empty,
+                           List(vec3IntToProto(segmentPosition.getOrElse(Vec3Int.zeros))),
+                           List.empty))
+      case Some(mappingName) =>
+        for {
+          remoteLayerUri <- getRemoteLayerUri(remoteFallbackLayer)
+          result <- rpc(s"$remoteLayerUri/agglomerates/$mappingName/agglomerateGraph/$agglomerateId").silent.withTokenFromContext.silent
+            .getWithProtoResponse[AgglomerateGraph](AgglomerateGraph)
+        } yield result
+    }
 
-  def getLargestAgglomerateId(remoteFallbackLayer: RemoteFallbackLayer, mappingName: String)(
+  def getLargestAgglomerateId(remoteFallbackLayer: RemoteFallbackLayer, mappingNameOpt: Option[String])(
       implicit tc: TokenContext): Fox[Long] = {
-    val cacheKey = (remoteFallbackLayer, mappingName, tc.userTokenOpt)
+    val cacheKey = (remoteFallbackLayer, mappingNameOpt, tc.userTokenOpt)
     largestAgglomerateIdCache.getOrLoad(
       cacheKey,
       k =>
         for {
           remoteLayerUri <- getRemoteLayerUri(k._1)
-          result <- rpc(s"$remoteLayerUri/agglomerates/${k._2}/largestAgglomerateId")
-            .addQueryStringOptional("token", k._3)
-            .silent
-            .getWithJsonResponse[Long]
+          uri = k._2 match {
+            case Some(mappingName) => s"$remoteLayerUri/agglomerates/$mappingName/largestAgglomerateId"
+            case None              => s"$remoteLayerUri/largestSegmentId"
+          }
+          result <- rpc(uri).addQueryStringOptional("token", k._3).silent.getWithJsonResponse[Long]
         } yield result
     )
   }

webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala

@@ -271,9 +271,8 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss
     for {
       volumeTracing <- annotationWithTracings.getVolume(action.actionTracingId).toFox
       _ <- assertMappingIsNotLocked(volumeTracing)
-      baseMappingName <- volumeTracing.mappingName.toFox ?~> "makeEditable.failed.noBaseMapping"
       _ <- bool2Fox(volumeTracingService.volumeBucketsAreEmpty(action.actionTracingId)) ?~> "annotation.volumeBucketsNotEmpty"
-      editableMappingInfo = editableMappingService.create(baseMappingName)
+      editableMappingInfo = editableMappingService.create(volumeTracing.mappingName)
       updater <- editableMappingUpdaterFor(annotationId,
                                            action.actionTracingId,
                                            volumeTracing,
@@ -619,7 +618,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss
     if (tracing.getHasEditableMapping)
       for {
         editableMappingInfo <- findEditableMappingInfo(annotationId, tracingId)
-      } yield Some(editableMappingInfo.baseMappingName)
+      } yield editableMappingInfo.baseMappingName
     else Fox.successful(tracing.mappingName)
 
   def findVolumeRaw(tracingId: String, version: Option[Long] = None): Fox[VersionedKeyValuePair[VolumeTracing]] =

webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala

@@ -122,7 +122,7 @@ class EditableMappingService @Inject()(
       "createdTimestamp" -> editableMappingInfo.createdTimestamp
     )
 
-  def create(baseMappingName: String): EditableMappingInfo =
+  def create(baseMappingName: Option[String]): EditableMappingInfo =
     EditableMappingInfo(
       baseMappingName = baseMappingName,
       createdTimestamp = Instant.now.epochMillis,
@@ -324,7 +324,7 @@ class EditableMappingService @Inject()(
   }
 
   def getBaseSegmentToAgglomerate(
-      baseMappingName: String,
+      baseMappingName: Option[String],
       segmentIds: Set[Long],
       remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[Map[Long, Long]] = {
     val segmentIdsOrdered = segmentIds.toList
@@ -402,13 +402,17 @@ class EditableMappingService @Inject()(
       tracingId: String,
       version: Long,
       agglomerateId: Long,
+      segmentPosition: Option[Vec3Int],
       remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[AgglomerateGraph] =
     for {
       agglomerateGraphBox <- getAgglomerateGraphForId(tracingId, version, agglomerateId).futureBox
       agglomerateGraph <- agglomerateGraphBox match {
         case Full(agglomerateGraph) => Fox.successful(agglomerateGraph)
         case Empty =>
-          remoteDatastoreClient.getAgglomerateGraph(remoteFallbackLayer, mapping.baseMappingName, agglomerateId)
+          remoteDatastoreClient.getAgglomerateGraph(remoteFallbackLayer,
+                                                    mapping.baseMappingName,
+                                                    agglomerateId,
+                                                    segmentPosition)
         case f: Failure => f.toFox
       }
     } yield agglomerateGraph
@@ -425,6 +429,7 @@ class EditableMappingService @Inject()(
                                                                tracingId,
                                                                version,
                                                                parameters.agglomerateId,
+                                                               None,
                                                                remoteFallbackLayer) ?~> "getAgglomerateGraph.failed"
       edgesToCut <- minCut(agglomerateGraph, parameters.segmentId1, parameters.segmentId2) ?~> "Could not calculate min-cut on agglomerate graph."
       edgesWithPositions = annotateEdgesWithPositions(edgesToCut, agglomerateGraph)
@@ -493,6 +498,7 @@ class EditableMappingService @Inject()(
                                                                tracingId,
                                                                version,
                                                                parameters.agglomerateId,
+                                                               None,
                                                                remoteFallbackLayer)
       neighborNodes = neighbors(agglomerateGraph, parameters.segmentId)
       nodesWithPositions = annotateNodesWithPositions(neighborNodes, agglomerateGraph)