From cb77aa31699bcde8ece046d3006320dfb4e9e488 Mon Sep 17 00:00:00 2001 From: Antoine Popineau Date: Wed, 26 Feb 2025 11:37:45 +0100 Subject: [PATCH] Store enrichment status and prevent enriching twice. --- dto/sanction_check_dto.go | 2 ++ models/sanction_check.go | 1 + repositories/dbmodels/db_sanction_check_match.go | 2 ++ ...0250226112700_add_sanction_check_match_enriched.sql | 9 +++++++++ repositories/sanction_check_repository.go | 1 + usecases/sanction_check_usecase.go | 5 +++++ usecases/sanction_check_usecase_test.go | 10 +++++----- 7 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 repositories/migrations/20250226112700_add_sanction_check_match_enriched.sql diff --git a/dto/sanction_check_dto.go b/dto/sanction_check_dto.go index 8cc6416bc..cc0823656 100644 --- a/dto/sanction_check_dto.go +++ b/dto/sanction_check_dto.go @@ -87,6 +87,7 @@ type SanctionCheckMatchDto struct { Datasets []string `json:"datasets"` UniqueCounterpartyIdentifier *string `json:"unique_counterparty_identifier"` Payload json.RawMessage `json:"payload"` + Enriched bool `json:"enriched"` Comments []SanctionCheckMatchCommentDto `json:"comments"` } @@ -99,6 +100,7 @@ func AdaptSanctionCheckMatchDto(m models.SanctionCheckMatch) SanctionCheckMatchD QueryIds: m.QueryIds, Datasets: make([]string, 0), Payload: m.Payload, + Enriched: m.Enriched, UniqueCounterpartyIdentifier: m.UniqueCounterpartyIdentifier, Comments: pure_utils.Map(m.Comments, AdaptSanctionCheckMatchCommentDto), } diff --git a/models/sanction_check.go b/models/sanction_check.go index 1e0e594b1..db1e49abc 100644 --- a/models/sanction_check.go +++ b/models/sanction_check.go @@ -165,6 +165,7 @@ type SanctionCheckMatch struct { QueryIds []string UniqueCounterpartyIdentifier *string Payload []byte + Enriched bool ReviewedBy *string Comments []SanctionCheckMatchComment } diff --git a/repositories/dbmodels/db_sanction_check_match.go b/repositories/dbmodels/db_sanction_check_match.go index 7f3f47da6..7043e983e 100644 --- a/repositories/dbmodels/db_sanction_check_match.go +++ b/repositories/dbmodels/db_sanction_check_match.go @@ -20,6 +20,7 @@ type DBSanctionCheckMatch struct { QueryIds []string `db:"query_ids"` CounterpartyId *string `db:"counterparty_id"` Payload json.RawMessage `db:"payload"` + Enriched bool `db:"enriched"` ReviewedBy *string `db:"reviewed_by"` CreatedAt time.Time `db:"created_at"` UpdatedAt time.Time `db:"updated_at"` @@ -37,6 +38,7 @@ func AdaptSanctionCheckMatch(dto DBSanctionCheckMatch) (models.SanctionCheckMatc QueryIds: dto.QueryIds, UniqueCounterpartyIdentifier: dto.CounterpartyId, Payload: dto.Payload, + Enriched: dto.Enriched, } return match, nil diff --git a/repositories/migrations/20250226112700_add_sanction_check_match_enriched.sql b/repositories/migrations/20250226112700_add_sanction_check_match_enriched.sql new file mode 100644 index 000000000..d086f1fd4 --- /dev/null +++ b/repositories/migrations/20250226112700_add_sanction_check_match_enriched.sql @@ -0,0 +1,9 @@ +-- +goose Up + +alter table sanction_check_matches + add column enriched bool default false; + +-- +goose Down + +alter table sanction_check_matches + drop column enriched; diff --git a/repositories/sanction_check_repository.go b/repositories/sanction_check_repository.go index 9fb14b9f9..c045e6a15 100644 --- a/repositories/sanction_check_repository.go +++ b/repositories/sanction_check_repository.go @@ -126,6 +126,7 @@ func (*MarbleDbRepository) UpdateSanctionCheckMatchPayload(ctx context.Context, sql := NewQueryBuilder(). Update(dbmodels.TABLE_SANCTION_CHECK_MATCHES). Set("payload", newPayload). + Set("enriched", true). Set("updated_at", "NOW()"). Where(squirrel.Eq{"id": match.Id}).Suffix(fmt.Sprintf("RETURNING %s", strings.Join(dbmodels.SelectSanctionCheckMatchesColumn, ","))) diff --git a/usecases/sanction_check_usecase.go b/usecases/sanction_check_usecase.go index 2b07ed6eb..e0affcd0e 100644 --- a/usecases/sanction_check_usecase.go +++ b/usecases/sanction_check_usecase.go @@ -491,6 +491,11 @@ func (uc SanctionCheckUsecase) EnrichMatch(ctx context.Context, matchId string) return models.SanctionCheckMatch{}, err } + if match.Enriched { + return models.SanctionCheckMatch{}, errors.Wrap(models.ConflictError, + "this sanction check match was already enriched") + } + newPayload, err := uc.openSanctionsProvider.EnrichMatch(ctx, match) if err != nil { return models.SanctionCheckMatch{}, err diff --git a/usecases/sanction_check_usecase_test.go b/usecases/sanction_check_usecase_test.go index 8c792d6a4..200780363 100644 --- a/usecases/sanction_check_usecase_test.go +++ b/usecases/sanction_check_usecase_test.go @@ -67,7 +67,7 @@ func TestListSanctionChecksOnDecision(t *testing.T) { exec.Mock.ExpectQuery(escapeSql(` SELECT sc.id, sc.decision_id, sc.status, sc.search_input, sc.search_datasets, sc.match_threshold, sc.match_limit, sc.is_manual, sc.requested_by, sc.is_partial, sc.is_archived, sc.initial_has_matches, sc.whitelisted_entities, sc.error_codes, sc.created_at, sc.updated_at, - ARRAY_AGG(ROW(scm.id,scm.sanction_check_id,scm.opensanction_entity_id,scm.status,scm.query_ids,scm.counterparty_id,scm.payload,scm.reviewed_by,scm.created_at,scm.updated_at) ORDER BY array_position(.+, scm.status), scm.payload->>'score' DESC) FILTER (WHERE scm.id IS NOT NULL) AS matches + ARRAY_AGG(ROW(scm.id,scm.sanction_check_id,scm.opensanction_entity_id,scm.status,scm.query_ids,scm.counterparty_id,scm.payload,scm.enriched,scm.reviewed_by,scm.created_at,scm.updated_at) ORDER BY array_position(.+, scm.status), scm.payload->>'score' DESC) FILTER (WHERE scm.id IS NOT NULL) AS matches FROM sanction_checks AS sc LEFT JOIN sanction_check_matches AS scm ON sc.id = scm.sanction_check_id WHERE sc.decision_id = $1 AND sc.is_archived = $2 @@ -146,7 +146,7 @@ func TestUpdateMatchStatus(t *testing.T) { })) exec.Mock. - ExpectQuery(`SELECT id, sanction_check_id, opensanction_entity_id, status, query_ids, counterparty_id, payload, reviewed_by, created_at, updated_at FROM sanction_check_matches WHERE id = \$1`). + ExpectQuery(`SELECT id, sanction_check_id, opensanction_entity_id, status, query_ids, counterparty_id, payload, enriched, reviewed_by, created_at, updated_at FROM sanction_check_matches WHERE id = \$1`). WithArgs("matchid"). WillReturnRows(pgxmock.NewRows(dbmodels.SelectSanctionCheckMatchesColumn). AddRow(mockScmRow...), @@ -157,20 +157,20 @@ func TestUpdateMatchStatus(t *testing.T) { WillReturnRows(pgxmock.NewRows(dbmodels.SelectSanctionChecksColumn). AddRow(mockScRow...), ) - exec.Mock.ExpectQuery(`SELECT id, sanction_check_id, opensanction_entity_id, status, query_ids, counterparty_id, payload, reviewed_by, created_at, updated_at FROM sanction_check_matches WHERE sanction_check_id = \$1`). + exec.Mock.ExpectQuery(`SELECT id, sanction_check_id, opensanction_entity_id, status, query_ids, counterparty_id, payload, enriched, reviewed_by, created_at, updated_at FROM sanction_check_matches WHERE sanction_check_id = \$1`). WithArgs("sanction_check_id"). WillReturnRows(pgxmock.NewRows(dbmodels.SelectSanctionCheckMatchesColumn). AddRow(mockScmRow...). AddRows(mockOtherScmRows...), ) - exec.Mock.ExpectQuery(`UPDATE sanction_check_matches SET reviewed_by = \$1, status = \$2, updated_at = \$3 WHERE id = \$4 RETURNING id,sanction_check_id,opensanction_entity_id,status,query_ids,counterparty_id,payload,reviewed_by,created_at,updated_at`). + exec.Mock.ExpectQuery(`UPDATE sanction_check_matches SET reviewed_by = \$1, status = \$2, updated_at = \$3 WHERE id = \$4 RETURNING id,sanction_check_id,opensanction_entity_id,status,query_ids,counterparty_id,payload,enriched,reviewed_by,created_at,updated_at`). WithArgs(models.UserId(""), models.SanctionMatchStatusConfirmedHit, "NOW()", "matchid"). WillReturnRows(pgxmock.NewRows(dbmodels.SelectSanctionCheckMatchesColumn). AddRow(mockScmRow...), ) for i := range 3 { - exec.Mock.ExpectQuery(`UPDATE sanction_check_matches SET reviewed_by = \$1, status = \$2, updated_at = \$3 WHERE id = \$4 RETURNING id,sanction_check_id,opensanction_entity_id,status,query_ids,counterparty_id,payload,reviewed_by,created_at,updated_at`). + exec.Mock.ExpectQuery(`UPDATE sanction_check_matches SET reviewed_by = \$1, status = \$2, updated_at = \$3 WHERE id = \$4 RETURNING id,sanction_check_id,opensanction_entity_id,status,query_ids,counterparty_id,payload,enriched,reviewed_by,created_at,updated_at`). WithArgs(models.UserId(""), models.SanctionMatchStatusSkipped, "NOW()", mockOtherScms[i].Id). WillReturnRows(pgxmock.NewRows(dbmodels.SelectSanctionCheckMatchesColumn). AddRow(mockOtherScmRows[i]...),