diff --git a/.env.example b/.env.example index c4d57618c..0bdf87273 100644 --- a/.env.example +++ b/.env.example @@ -77,6 +77,7 @@ CONVOY_API_URL= CONVOY_API_KEY= CONVOY_PROJECT_ID= +<<<<<<< HEAD # # TUNING # diff --git a/cmd/server.go b/cmd/server.go index 68fe4c0d1..f2951ea6d 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -64,6 +64,10 @@ func RunServer() error { ProjectID: utils.GetEnv("CONVOY_PROJECT_ID", ""), RateLimit: utils.GetEnv("CONVOY_RATE_LIMIT", 50), } + openSanctionsConfig := infra.InitializeOpenSanctions( + utils.GetEnv("OPENSANCTIONS_API_HOST", ""), + utils.GetEnv("OPENSANCTIONS_API_KEY", ""), + ) seedOrgConfig := models.SeedOrgConfiguration{ CreateGlobalAdminEmail: utils.GetEnv("CREATE_GLOBAL_ADMIN_EMAIL", ""), @@ -133,6 +137,7 @@ func RunServer() error { infra.InitializeConvoyRessources(convoyConfiguration), convoyConfiguration.RateLimit, ), + repositories.WithOpenSanctions(openSanctionsConfig), repositories.WithClientDbConfig(clientDbConfig), repositories.WithTracerProvider(telemetryRessources.TracerProvider), ) diff --git a/infra/opensanctions.go b/infra/opensanctions.go new file mode 100644 index 000000000..126311b34 --- /dev/null +++ b/infra/opensanctions.go @@ -0,0 +1,35 @@ +package infra + +const ( + OPEN_SANCTIONS_API_HOST = "https://api.opensanctions.org" +) + +type OpenSanctions struct { + host string + // TODO: this is only for SaaS OpenSanctions API, we may need to abstract + // over authentication to at least offer Basic and Bearer for self-hosted. + apiKey string +} + +func InitializeOpenSanctions(host, apiKey string) OpenSanctions { + return OpenSanctions{ + host: host, + apiKey: apiKey, + } +} + +func (os OpenSanctions) IsSelfHosted() bool { + return len(os.host) > 0 +} + +func (os OpenSanctions) Host() string { + if os.IsSelfHosted() { + return os.host + } + + return OPEN_SANCTIONS_API_HOST +} + +func (os OpenSanctions) ApiKey() string { + return os.apiKey +} diff --git a/repositories/httpmodels/http_opensanctions_result.go b/repositories/httpmodels/http_opensanctions_result.go index 4b87adb92..713a3ba27 100644 --- a/repositories/httpmodels/http_opensanctions_result.go +++ b/repositories/httpmodels/http_opensanctions_result.go @@ -12,7 +12,6 @@ type HTTPOpenSanctionsResult struct { Total struct { Value int `json:"value"` } `json:"total"` - Limit int `json:"limit"` Results []struct { Id string `json:"id"` Schema string `json:"schema"` @@ -30,7 +29,7 @@ func AdaptOpenSanctionsResult(result HTTPOpenSanctionsResult) (models.SanctionCh matches := make(map[string]models.SanctionCheckResultMatch) for _, resp := range result.Responses { - if resp.Total.Value != resp.Limit { + if resp.Total.Value > len(resp.Results) { partial = true } diff --git a/repositories/opensanctions_repository.go b/repositories/opensanctions_repository.go index b22708a8f..e8698da8d 100644 --- a/repositories/opensanctions_repository.go +++ b/repositories/opensanctions_repository.go @@ -6,7 +6,9 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" + "github.com/checkmarble/marble-backend/infra" "github.com/checkmarble/marble-backend/models" "github.com/checkmarble/marble-backend/repositories/httpmodels" "github.com/checkmarble/marble-backend/utils" @@ -14,12 +16,9 @@ import ( "github.com/google/uuid" ) -const ( - // TODO: Pull this as server configuration - DEV_YENTE_URL = "http://app.yente.orb.local" -) - -type OpenSanctionsRepository struct{} +type OpenSanctionsRepository struct { + opensanctions infra.OpenSanctions +} type openSanctionsRequest struct { Queries map[string]openSanctionsRequestQuery `json:"queries"` @@ -62,7 +61,7 @@ func (repo OpenSanctionsRepository) Search(ctx context.Context, cfg models.Sanct return httpmodels.AdaptOpenSanctionsResult(matches) } -func (OpenSanctionsRepository) searchRequest(ctx context.Context, query models.OpenSanctionsQuery) (*http.Request, error) { +func (repo OpenSanctionsRepository) searchRequest(ctx context.Context, query models.OpenSanctionsQuery) (*http.Request, error) { q := openSanctionsRequest{ Queries: make(map[string]openSanctionsRequestQuery, len(query.Queries)), } @@ -80,8 +79,16 @@ func (OpenSanctionsRepository) searchRequest(ctx context.Context, query models.O return nil, errors.Wrap(err, "could not parse OpenSanctions response") } - url := fmt.Sprintf("%s/match/sanctions", DEV_YENTE_URL) - req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, &body) + requestUrl := fmt.Sprintf("%s/match/sanctions", repo.opensanctions.Host()) + + if len(repo.opensanctions.ApiKey()) > 0 { + qs := url.Values{} + qs.Set("api_key", repo.opensanctions.ApiKey()) + + requestUrl = fmt.Sprintf("%s?%s", requestUrl, qs.Encode()) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, requestUrl, &body) return req, err } diff --git a/repositories/repositories.go b/repositories/repositories.go index ea0aa558c..6520cc88c 100644 --- a/repositories/repositories.go +++ b/repositories/repositories.go @@ -17,6 +17,7 @@ type options struct { clientDbConfig map[string]infra.ClientDbConfig convoyClientProvider ConvoyClientProvider convoyRateLimit int + openSanctions infra.OpenSanctions riverClient *river.Client[pgx.Tx] tp trace.TracerProvider } @@ -50,6 +51,12 @@ func WithConvoyClientProvider(convoyResources ConvoyClientProvider, convoyRateLi } } +func WithOpenSanctions(openSanctionsConfig infra.OpenSanctions) Option { + return func(o *options) { + o.openSanctions = openSanctionsConfig + } +} + func WithRiverClient(client *river.Client[pgx.Tx]) Option { return func(o *options) { o.riverClient = client @@ -84,6 +91,7 @@ type Repositories struct { CustomListRepository CustomListRepository UploadLogRepository UploadLogRepository MarbleAnalyticsRepository MarbleAnalyticsRepository + OpenSanctionsRepository OpenSanctionsRepository TransferCheckEnrichmentRepository *TransferCheckEnrichmentRepository TaskQueueRepository TaskQueueRepository ScenarioTestrunRepository ScenarioTestRunRepository @@ -131,6 +139,9 @@ func NewRepositories( MarbleAnalyticsRepository: MarbleAnalyticsRepository{ metabase: options.metabase, }, + OpenSanctionsRepository: OpenSanctionsRepository{ + opensanctions: options.openSanctions, + }, TransferCheckEnrichmentRepository: NewTransferCheckEnrichmentRepository( blobRepository, options.transfercheckEnrichmentBucket, diff --git a/usecases/evaluate_scenario/evaluate_scenario.go b/usecases/evaluate_scenario/evaluate_scenario.go index 2a912b2f8..a91a4097c 100644 --- a/usecases/evaluate_scenario/evaluate_scenario.go +++ b/usecases/evaluate_scenario/evaluate_scenario.go @@ -129,6 +129,7 @@ func processScenarioIteration(ctx context.Context, params ScenarioEvaluationPara if iteration.SanctionCheckConfig != nil { query := models.OpenSanctionsQuery{Queries: models.OpenSanctionCheckFilter{ + // TODO: take this from the context and the scenario configuration "name": []string{"obama"}, }} @@ -137,7 +138,7 @@ func processScenarioIteration(ctx context.Context, params ScenarioEvaluationPara return models.ScenarioExecution{}, errors.Wrap(err, "could not perform sanction check") } - logger.Debug("SANCTION CHECK: found", "matches", result.Count) + logger.Debug("SANCTION CHECK: found", "matches", result.Count, "partial", result.Partial) } // Compute outcome from score diff --git a/usecases/usecases_with_creds.go b/usecases/usecases_with_creds.go index d9ac72b5f..46515f5e9 100644 --- a/usecases/usecases_with_creds.go +++ b/usecases/usecases_with_creds.go @@ -2,7 +2,6 @@ package usecases import ( "github.com/checkmarble/marble-backend/models" - "github.com/checkmarble/marble-backend/repositories" "github.com/checkmarble/marble-backend/usecases/decision_phantom" "github.com/checkmarble/marble-backend/usecases/decision_workflows" "github.com/checkmarble/marble-backend/usecases/inboxes" @@ -120,7 +119,7 @@ func (usecases *UsecasesWithCreds) NewDecisionUsecase() DecisionUsecase { func (usecases *UsecasesWithCreds) NewSanctionCheckUsecase() SanctionCheckUsecase { return SanctionCheckUsecase{ - openSanctionsProvider: repositories.OpenSanctionsRepository{}, + openSanctionsProvider: usecases.Repositories.OpenSanctionsRepository, repository: &usecases.Repositories.MarbleDbRepository, } }