Human Annotation

.github/workflows/build-docker-internal.yml

@@ -3,7 +3,7 @@ on:
   workflow_dispatch: {}
   schedule:
     # 4:05AM On Thursdays for vuln assessments.
-    - cron: '5 4 * * 4'
+    - cron: "5 4 * * 4"
 jobs:
   build-app:
     uses: ./.github/workflows/build.yml
@@ -12,8 +12,8 @@ jobs:
     runs-on: ubuntu-latest
     needs: [build-app]
     permissions:
-      contents: 'read'
-      id-token: 'write'
+      contents: "read"
+      id-token: "write"
     strategy:
       matrix:
         python:

app/package.json

@@ -54,6 +54,7 @@
     "packageManager": "[email protected]+sha512.f95ce356460e05be48d66401c1ae64ef84d163dd689964962c6888a9810865e39097a5e9de748876c2e0bf89b232d583c33982773e9903ae7a76257270986538",
     "dependencies": {
         "colormap": "^2.3.2",
+        "pixi-viewport": "^6.0.3",
         "react-player": "^2.16.0",
         "react-plotly.js": "^2.6.0"
     },

app/packages/app/index.html

@@ -15,6 +15,7 @@
     <div id="modal"></div>
     <div id="colorModal"></div>
     <div id="queryPerformance"></div>
+    <div id="schemaManager"></div>
     <script type="module" src="/src/index.tsx"></script>
   </body>
 </html>

app/packages/components/package.json

@@ -33,6 +33,7 @@
         "jotai": "*",
         "re-resizable": "*",
         "react": "*",
+        "react-markdown": "*",
         "react-relay": "*",
         "recoil": "*",
         "styled-components": "*"
@@ -48,8 +49,10 @@
         "classnames": "^2.3.1",
         "framer-motion": "^6.2.7",
         "material-icons": "^1.13.12",
+        "re-resizable": "^6.9.17",
         "react-input-autosize": "^3.0.0",
         "react-laag": "^2.0.3",
+        "react-markdown": "^8.0.7",
         "react-syntax-highlighter": "^15.5.0",
         "react-use": "^17.5.1"
     }

app/packages/components/src/components/CodeBlock/index.tsx

@@ -14,6 +14,8 @@ export default function CodeBlock({ text, ...props }: CodeBlockProps) {
   return (
     <Box
       sx={{
+        width: "100%",
+        height: "100%",
         position: "relative",
         fontFamily: "Roboto Mono, monospace",
         fontSize: props?.fontSize || 14,

app/packages/components/src/components/Loading/index.ts

@@ -1,2 +1,3 @@
 export { default } from "./Loading";
 export { default as LoadingDots } from "./LoadingDots";
+export { default as LoadingSpinner } from "./LoadingSpinner";

app/packages/components/src/components/index.ts

@@ -8,6 +8,7 @@ export { default as CodeTabs } from "./CodeTabs";
 export { default as ColoredDot } from "./ColoredDot";
 export { default as CopyButton } from "./CopyButton";
 export { default as Dialog } from "./Dialog";
+export { default as EditableLabel } from "./EditableLabel";
 export { default as ErrorBoundary, ErrorDisplayMarkup } from "./ErrorBoundary";
 export { default as ExternalLink, useExternalLink } from "./ExternalLink";
 export { default as FilterAndSelectionIndicator } from "./FilterAndSelectionIndicator";
@@ -19,10 +20,11 @@ export * from "./Icons";
 export { default as JSONPanel } from "./JSONPanel";
 export { default as JSONViewer } from "./JSONViewer";
 export { default as Link } from "./Link";
-export { default as Loading, LoadingDots } from "./Loading";
+export { default as Loading, LoadingDots, LoadingSpinner } from "./Loading";
 export { default as Markdown } from "./Markdown";
 export { default as MuiButton } from "./MuiButton";
 export { default as MuiIconFont } from "./MuiIconFont";
+export { default as PanelCTA } from "./PanelCTA";
 export { default as Pending } from "./Pending";
 export { default as PillButton } from "./PillButton";
 export { default as Popout, PopoutDiv } from "./Popout";
@@ -32,15 +34,13 @@ export { default as Resizable } from "./Resizable";
 export * from "./Selection";
 export { default as Selection } from "./Selection";
 export { default as Selector, SelectorValidationError } from "./Selector";
-export { default as StatusButton } from "./StatusButton";
 export type { UseSearch } from "./Selector";
+export { default as StatusButton } from "./StatusButton";
 export { default as TabOption } from "./TabOption";
 export { default as TextField } from "./TextField";
 export { default as ThemeProvider, useFont, useTheme } from "./ThemeProvider";
-export { default as Tooltip } from "./Tooltip";
 export { default as Toast } from "./Toast";
-export { default as PanelCTA } from "./PanelCTA";
-export { default as EditableLabel } from "./EditableLabel";
+export { default as Tooltip } from "./Tooltip";
 export { default as TooltipProvider } from "./TooltipProvider";
 
 export * from "./types";

app/packages/core/package.json

@@ -26,8 +26,10 @@
         "classnames": "^2.2.6",
         "color": "^4.2.3",
         "color-string": "^1.9.1",
+        "fast-json-patch": "^3.1.1",
         "framer-motion": "^6.2.8",
         "history": "^5.3.0",
+        "json-edit-react": "^1.28.2",
         "lodash": "^4.17.21",
         "lru-cache": "^11.0.1",
         "luxon": "^3.6.1",
@@ -61,7 +63,7 @@
         "@types/lodash": "^4.14.182",
         "@types/luxon": "^3.6.2",
         "@types/mime": "^2.0.3",
-        "@types/react": "^18.0.9",
+        "@types/react": "^18.2.0",
         "@types/react-color": "^3.0.6",
         "@types/react-dom": "^18.0.3",
         "@types/react-grid-layout": "^1.3.5",

app/packages/core/src/client/annotationClient.ts

@@ -0,0 +1,114 @@
+/**
+ * Copyright 2017-2025, Voxel51, Inc.
+ */
+
+import { encodeURIPath, parseETag } from "./util";
+import { Sample } from "@fiftyone/looker";
+import {
+  FetchFunctionConfig,
+  FetchFunctionResult,
+  getFetchFunctionExtended,
+  MalformedRequestError,
+  NotFoundError,
+} from "@fiftyone/utilities";
+import * as jsonpatch from "fast-json-patch";
+
+/**
+ * List of JSON-patch operation deltas between two versions of a json object.
+ */
+export type JSONDeltas = jsonpatch.Operation[];
+
+export type PatchSampleRequest = {
+  datasetId: string;
+  sampleId: string;
+  deltas: JSONDeltas;
+  versionToken: string;
+  path?: string;
+  labelId?: string;
+};
+
+export type ErrorResponse = {
+  errors: string[];
+};
+
+export type PatchSampleResponse = {
+  sample: Sample;
+  versionToken: string;
+};
+
+/**
+ * Error resulting from a failed update operation.
+ */
+export class PatchApplicationError extends Error {
+  constructor(message?: string) {
+    super(message);
+    this.name = "Patch Application Error";
+  }
+}
+
+const handleErrorResponse = async (response: Response) => {
+  if (response.status === 400) {
+    // either a malformed request, or a list of errors from applying the patch
+    let errorResponse: ErrorResponse | undefined;
+    try {
+      // expected error response: '["error 1","error 2"]'
+      const body = await response.text();
+      const errorList = JSON.parse(body);
+      if (Array.isArray(errorList)) {
+        errorResponse = { errors: errorList };
+      }
+    } catch (err) {
+      // doesn't look like a list of errors
+    }
+    if (errorResponse?.errors) {
+      throw new PatchApplicationError(errorResponse.errors.join(", "));
+    }
+
+    throw new MalformedRequestError();
+  } else if (response.status === 404) {
+    throw new NotFoundError({ path: "sample" });
+  }
+};
+
+/**
+ * `fetch` with headers, error handling, etc.
+ *
+ * @param config fetch configuration
+ */
+const doFetch = <A, R>(
+  config: FetchFunctionConfig<A>
+): Promise<FetchFunctionResult<R>> => {
+  return getFetchFunctionExtended()({
+    errorHandler: handleErrorResponse,
+    ...config,
+  });
+};
+
+/**
+ * Patch a sample, applying the specified updates to its fields.
+ *
+ * @param request Patch sample request
+ */
+export const patchSample = async (
+  request: PatchSampleRequest
+): Promise<PatchSampleResponse> => {
+  const pathParts = ["dataset", request.datasetId, "sample", request.sampleId];
+  if (request.path && request.labelId) {
+    pathParts.push(request.path, request.labelId);
+  }
+
+  const response = await doFetch<JSONDeltas, Sample>({
+    path: encodeURIPath(pathParts),
+    method: "PATCH",
+    body: request.deltas,
+    headers: {
+      "Content-Type": "application/json-patch+json",
+      "If-Match": `"${request.versionToken}"`,
+    },
+  });
+
+  return {
+    sample: response.response,
+    versionToken: parseETag(response.headers.get("ETag")),
+  };
+};

app/packages/core/src/client/index.ts

@@ -0,0 +1 @@
+export * from "./annotationClient";

app/packages/core/src/client/transformer.ts

@@ -0,0 +1,95 @@
+/**
+ * Copyright 2017-2025, Voxel51, Inc.
+ */
+
+import * as _ from "lodash";
+import { isObject } from "./util";
+
+/**
+ * DataTransformer implementations offer conversion from one record type to another.
+ */
+export type DataTransformer = {
+  transform: (data: Record<string, unknown>) => Record<string, unknown>;
+};
+
+/**
+ * FieldTransformer implementations offer conversion for a specific field type.
+ */
+export type FieldTransformer = {
+  canTransform: (data: unknown) => boolean;
+  transform: (data: unknown) => unknown;
+};
+
+/**
+ * Convert serialized python datetime.datetime data to a format consistent with
+ * graphql.
+ */
+const DateTimeTransformer: FieldTransformer = {
+  canTransform: (data: unknown): boolean => {
+    return isObject(data) && "$date" in (data as object);
+  },
+  transform: (data: unknown): { _cls: "DateTime"; datetime: number } => {
+    const date = new Date((data as { $date: string }).$date);
+    return { _cls: "DateTime", datetime: date.getTime() };
+  },
+};
+
+/**
+ * Convert serialized bson.ObjectId data to a format consistent with graphql.
+ */
+const ObjectIdTransformer: FieldTransformer = {
+  canTransform: (data: unknown): boolean => {
+    return isObject(data) && "$oid" in (data as object);
+  },
+  transform: (data: unknown): string => {
+    return (data as { $oid: string }).$oid;
+  },
+};
+
+const fieldTransformers = [DateTimeTransformer, ObjectIdTransformer];
+
+/**
+ * Transformer which converts `fo.Sample.to_dict()` serialization to a format
+ * consistent with graphql.
+ */
+const SampleTransformer: DataTransformer = {
+  transform: (data: Record<string, unknown>): Record<string, unknown> => {
+    // recursive helper function
+    const transformInner = (innerData: unknown): unknown => {
+      // try to use a registered transformer
+      for (const transformer of fieldTransformers) {
+        if (transformer.canTransform(innerData)) {
+          return transformer.transform(innerData);
+        }
+      }
+
+      // otherwise recursively transform
+      if (Array.isArray(innerData)) {
+        return innerData.map((e) => transformInner(e));
+      } else if (isObject(innerData)) {
+        for (const key of Object.keys(innerData)) {
+          innerData[key] = transformInner(innerData[key]);
+        }
+      }
+
+      return innerData;
+    };
+
+    // transformation happens in-place, so create a new copy of the data first
+    const result = _.cloneDeep(data);
+    transformInner(result);
+    return result;
+  },
+};
+
+/**
+ * Transform sample data returned by `fo.Sample.to_dict()` to a format
+ * compatible with graphql query results.
+ *
+ * @param sample Serialized sample data
+ */
+export const transformSampleData = (
+  sample: Record<string, unknown>
+): Record<string, unknown> => {
+  return SampleTransformer.transform(sample);
+};