doc ingestion patch + fix worker for sanction check

checkmarble · Feb 18, 2025 · ba0bfc9 · ba0bfc9
1 parent 6e40bb8
commit ba0bfc9
Show file tree

Hide file tree

Showing 3 changed files with 162 additions and 70 deletions.
diff --git a/cmd/worker.go b/cmd/worker.go
@@ -46,6 +46,16 @@ func RunTaskQueue() error {
 		ProjectID: utils.GetEnv("CONVOY_PROJECT_ID", ""),
 		RateLimit: utils.GetEnv("CONVOY_RATE_LIMIT", 50),
 	}
+	openSanctionsConfig := infra.InitializeOpenSanctions(
+		http.DefaultClient,
+		utils.GetEnv("OPENSANCTIONS_API_HOST", ""),
+		utils.GetEnv("OPENSANCTIONS_AUTH_METHOD", ""),
+		utils.GetEnv("OPENSANCTIONS_API_KEY", ""),
+	)
+	if apiUrl := utils.GetEnv("NAME_RECOGNITION_API_URL", ""); apiUrl != "" {
+		openSanctionsConfig.WithNameRecognition(apiUrl)
+	}
+
 	licenseConfig := models.LicenseConfiguration{
 		LicenseKey:             utils.GetEnv("LICENSE_KEY", ""),
 		KillIfReadLicenseError: utils.GetEnv("KILL_IF_READ_LICENSE_ERROR", false),
@@ -117,6 +127,7 @@ func RunTaskQueue() error {
 		),
 		repositories.WithClientDbConfig(clientDbConfig),
 		repositories.WithTracerProvider(telemetryRessources.TracerProvider),
+		repositories.WithOpenSanctions(openSanctionsConfig),
 	)
 
 	// Start the task queue workers

diff --git a/dto/openapi.go b/dto/openapi.go
@@ -64,7 +64,8 @@ type MethodObject struct {
 }
 
 type PathObject struct {
-	Post MethodObject `json:"post"`
+	Post  *MethodObject `json:"post"`
+	Patch *MethodObject `json:"patch,omitempty"`
 }
 
 type ComponentsSchema struct {
@@ -436,97 +437,112 @@ func OpenAPIFromDataModel(dataModel models.DataModel) Reference {
 			Type:       "object",
 			Properties: properties,
 		}
+		ref.Components.Schemas[fmt.Sprintf("%s-partial", table.Name)] = ComponentsSchema{
+			Required:   []string{"object_id", "updated_at"},
+			Type:       "object",
+			Properties: properties,
+		}
 
-		object := PathObject{
-			Post: MethodObject{
-				Security: []map[string][]string{
-					{
-						"api_key": []string{},
+		methodObj := MethodObject{
+			Security: []map[string][]string{
+				{
+					"api_key": []string{},
+				},
+			},
+			Tags:        []string{"Ingestion"},
+			Description: table.Description,
+			RequestBody: RequestBody{
+				Content: Content{
+					ApplicationJSON: ApplicationJSON{
+						Schema: Schema{
+							Ref: fmt.Sprintf("#/components/schemas/%s", table.Name),
+						},
 					},
 				},
-				Tags:        []string{"Ingestion"},
-				Description: table.Description,
-				RequestBody: RequestBody{
-					Content: Content{
+				Required: true,
+			},
+			Responses: map[string]Response{
+				"200": {
+					Description: "Data was successfully treated but no new object version was created",
+				},
+				"201": {
+					Description: "Data was successfully ingested",
+				},
+				"400": {
+					Description: "One at least of the objects does not match the data model",
+					Content: &Content{
 						ApplicationJSON: ApplicationJSON{
 							Schema: Schema{
-								Ref: fmt.Sprintf("#/components/schemas/%s", table.Name),
+								Ref: "#/components/schemas/ErrorDto",
 							},
 						},
 					},
-					Required: true,
 				},
-				Responses: map[string]Response{
-					"200": {
-						Description: "Data was successfully treated but no new object version was created",
-					},
-					"201": {
-						Description: "Data was successfully ingested",
-					},
-					"400": {
-						Description: "One at least of the objects does not match the data model",
-						Content: &Content{
-							ApplicationJSON: ApplicationJSON{
-								Schema: Schema{
-									Ref: "#/components/schemas/ErrorDto",
-								},
-							},
-						},
-					},
-					"500": {
-						Description: "An error happened while ingesting data",
-					},
+				"500": {
+					Description: "An error happened while ingesting data",
 				},
 			},
 		}
+		methodObjPatch := methodObj
+		methodObjPatch.RequestBody.Content.ApplicationJSON.Schema.Ref =
+			fmt.Sprintf("#/components/schemas/%s-partial", table.Name)
+		object := PathObject{
+			Post:  &methodObj,
+			Patch: &methodObjPatch,
+		}
 		ref.Paths[fmt.Sprintf("/ingestion/%s", table.Name)] = object
 
-		object = PathObject{
-			Post: MethodObject{
-				Security: []map[string][]string{
-					{
-						"api_key": []string{},
-					},
+		methodObj = MethodObject{
+			Security: []map[string][]string{
+				{
+					"api_key": []string{},
 				},
-				Tags:        []string{"Ingestion"},
-				Description: table.Description,
-				RequestBody: RequestBody{
-					Content: Content{
-						ApplicationJSON: ApplicationJSON{
-							Schema: Schema{
-								Type: "array",
-								Items: &Schema{
-									Ref: fmt.Sprintf("#/components/schemas/%s", table.Name),
-								},
-								MaxItems: 100,
+			},
+			Tags:        []string{"Ingestion"},
+			Description: table.Description,
+			RequestBody: RequestBody{
+				Content: Content{
+					ApplicationJSON: ApplicationJSON{
+						Schema: Schema{
+							Type: "array",
+							Items: &Schema{
+								Ref: fmt.Sprintf("#/components/schemas/%s", table.Name),
 							},
+							MaxItems: 100,
 						},
 					},
-					Required: true,
 				},
-				Responses: map[string]Response{
-					"200": {
-						Description: "Data was successfully treated but no new object version was created",
-					},
-					"201": {
-						Description: "Data was successfully ingested",
-					},
-					"400": {
-						Description: "The array of objects is too long, or one at least of the objects does not match the data model",
-						Content: &Content{
-							ApplicationJSON: ApplicationJSON{
-								Schema: Schema{
-									Ref: "#/components/schemas/ErrorDto",
-								},
+				Required: true,
+			},
+			Responses: map[string]Response{
+				"200": {
+					Description: "Data was successfully treated but no new object version was created",
+				},
+				"201": {
+					Description: "Data was successfully ingested",
+				},
+				"400": {
+					Description: "The array of objects is too long, or one at least of the objects does not match the data model",
+					Content: &Content{
+						ApplicationJSON: ApplicationJSON{
+							Schema: Schema{
+								Ref: "#/components/schemas/ErrorDto",
 							},
 						},
 					},
-					"500": {
-						Description: "An error happened while ingesting data",
-					},
+				},
+				"500": {
+					Description: "An error happened while ingesting data",
 				},
 			},
 		}
+		methodObjPatch = methodObj
+		methodObjPatch.RequestBody.Content.ApplicationJSON.Schema.Ref =
+			fmt.Sprintf("#/components/schemas/%s-partial", table.Name)
+		object = PathObject{
+			Post:  &methodObj,
+			Patch: &methodObjPatch,
+		}
 		ref.Paths[fmt.Sprintf("/ingestion/%s/multiple", table.Name)] = object
 	}
 
@@ -558,7 +574,7 @@ func OpenAPIFromDataModel(dataModel models.DataModel) Reference {
 	}
 
 	ref.Paths["/decisions"] = PathObject{
-		Post: MethodObject{
+		Post: &MethodObject{
 			Security: []map[string][]string{
 				{
 					"api_key": []string{},
@@ -605,7 +621,7 @@ func OpenAPIFromDataModel(dataModel models.DataModel) Reference {
 	}
 
 	ref.Paths["/decisions/all"] = PathObject{
-		Post: MethodObject{
+		Post: &MethodObject{
 			Security: []map[string][]string{
 				{
 					"api_key": []string{},

diff --git a/specs/public_api.yaml b/specs/public_api.yaml
@@ -49,14 +49,79 @@ paths:
                 $ref: '#/components/schemas/ErrorDto'
         500:
           description: An error happened while ingesting the object.
+    patch:
+      summary: Ingest a new object, and allow a partial payload (only the fields to update)
+      tags:
+        - Ingestion
+      security:
+        - ApiKeyAuth: []
+      description: Ingest (upsert) a new version of an object from the data model
+      parameters:
+        - in: path
+          name: object_type
+          schema:
+            type: string
+            example: transactions
+          required: true
+          description: name of the object type used for ingestion.
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/data_model_object"
+      responses:
+        200:
+          description: The object was successfully ingested.
+        400:
+          description: The provided object is invalid.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErrorDto'
+        500:
+          description: An error happened while ingesting the object.
   /ingestion/{object_type}/multiple:
     post:
       summary: Ingest new objects by batch
       tags:
         - Ingestion
       security:
         - ApiKeyAuth: []
-      description: Ingest a new object from the data model
+      description: Ingest an array of objects from the data model
+      parameters:
+        - in: path
+          name: object_type
+          schema:
+            type: string
+            example: transactions
+          required: true
+          description: name of the object type used for ingestion.
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: array
+              items:
+                $ref: "#/components/schemas/data_model_object"
+              maxItems: 100
+      responses:
+        200:
+          description: The object was successfully ingested.
+        400:
+          description: One of the provided object is invalid (with respect to the data model), or too many objects have been sent.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ErrorDto'
+        500:
+          description: An error happened while ingesting the object.
+    patch:
+      summary: Ingest new objects by batch, and allow a partial payload (only the fields to update)
+      tags:
+        - Ingestion
+      security:
+        - ApiKeyAuth: []
+      description: Ingest (upsert) an array of new versions of objects from the data model
       parameters:
         - in: path
           name: object_type