Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement query AST, with a new function, StringConcat. #808

Merged
merged 9 commits into from
Jan 31, 2025
Merged

Implement query AST, with a new function, StringConcat. #808

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
fdd5667
Implement query AST, with a new function, StringConcat.
apognu Jan 29, 2025
f1895f1
Added a few unit tests for the evaluate sanction check function.
apognu Jan 30, 2025
d314156
Add on delete cascade between sanction check configs and scenario ite…
apognu Jan 30, 2025
4039974
Mark the trigger rule of a sanction check as non-null, only its updat…
apognu Jan 30, 2025
ed784a0
Edit scenario validation to include sanction check config.
apognu Jan 30, 2025
a8e9f1d
StringConcat node: skip nil values.
apognu Jan 30, 2025
88d5ccb
StringConcat node: error out when all values are nil.
apognu Jan 30, 2025
1938ad8
Bump migration version.
apognu Jan 31, 2025
520c4f9
Fixed tests from feature branch changes.
apognu Jan 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 54 additions & 6 deletions dto/scenario_iterations.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,42 @@ type ScenarioIterationBodyDto struct {
}

type SanctionCheckConfig struct {
Enabled *bool `json:"enabled"`
Datasets []string `json:"datasets,omitempty"`
ForceOutcome *string `json:"force_outcome,omitempty"`
ScoreModifier *int `json:"score_modifier,omitempty"`
TriggerRule *NodeDto `json:"trigger_rule"`
Enabled *bool `json:"enabled"`
Datasets []string `json:"datasets,omitempty"`
ForceOutcome *string `json:"force_outcome,omitempty"`
ScoreModifier *int `json:"score_modifier,omitempty"`
TriggerRule *NodeDto `json:"trigger_rule"`
Query *SanctionCheckConfigQuery `json:"query"`
}

type SanctionCheckConfigQuery struct {
Name NodeDto `json:"name"`
}

func AdaptSanctionCheckConfigQuery(model models.SanctionCheckConfigQuery) (SanctionCheckConfigQuery, error) {
nameAst, err := AdaptNodeDto(model.Name)
if err != nil {
return SanctionCheckConfigQuery{}, err
}

dto := SanctionCheckConfigQuery{
Name: nameAst,
}

return dto, nil
}

func AdaptSanctionCheckConfigQueryDto(dto SanctionCheckConfigQuery) (models.SanctionCheckConfigQuery, error) {
nameAst, err := AdaptASTNode(dto.Name)
if err != nil {
return models.SanctionCheckConfigQuery{}, err
}

model := models.SanctionCheckConfigQuery{
Name: nameAst,
}

return model, nil
}

func AdaptScenarioIterationWithBodyDto(si models.ScenarioIteration) (ScenarioIterationWithBodyDto, error) {
Expand All @@ -62,7 +93,12 @@ func AdaptScenarioIterationWithBodyDto(si models.ScenarioIteration) (ScenarioIte
body.Rules[i] = apiRule
}
if si.SanctionCheckConfig != nil {
nodeDto, err := AdaptNodeDto(*si.SanctionCheckConfig.TriggerRule)
nodeDto, err := AdaptNodeDto(si.SanctionCheckConfig.TriggerRule)
if err != nil {
return ScenarioIterationWithBodyDto{},
errors.Wrap(err, "could not parse the sanction check trigger rule")
}
queryDto, err := AdaptSanctionCheckConfigQuery(si.SanctionCheckConfig.Query)
if err != nil {
return ScenarioIterationWithBodyDto{},
errors.Wrap(err, "could not parse the sanction check trigger rule")
Expand All @@ -74,6 +110,7 @@ func AdaptScenarioIterationWithBodyDto(si models.ScenarioIteration) (ScenarioIte
ForceOutcome: nil,
ScoreModifier: &si.SanctionCheckConfig.Outcome.ScoreModifier,
TriggerRule: &nodeDto,
Query: &queryDto,
}

if si.SanctionCheckConfig.Outcome.ForceOutcome != models.Approve {
Expand Down Expand Up @@ -150,6 +187,17 @@ func AdaptUpdateScenarioIterationInput(input UpdateScenarioIterationBody, iterat

updateScenarioIterationInput.Body.SanctionCheckConfig.TriggerRule = &astNode
}
if input.Body.SanctionCheckConfig.Query != nil {
query, err := AdaptSanctionCheckConfigQueryDto(*input.Body.SanctionCheckConfig.Query)
if err != nil {
return models.UpdateScenarioIterationInput{}, errors.Wrap(
models.BadParameterError,
"invalid trigger",
)
}

updateScenarioIterationInput.Body.SanctionCheckConfig.Query = &query
}
if input.Body.SanctionCheckConfig.ForceOutcome != nil {
updateScenarioIterationInput.Body.SanctionCheckConfig.Outcome.ForceOutcome = utils.Ptr(models.ForcedOutcomeFrom(
*input.Body.SanctionCheckConfig.ForceOutcome))
Expand Down
22 changes: 19 additions & 3 deletions dto/scenario_validation_dto.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,16 @@ type decisionValidationDto struct {
Errors []ScenarioValidationErrorDto `json:"errors"`
}

type sanctionCheckConfigValidationDto struct {
Trigger triggerValidationDto `json:"trigger"`
NameFilter ruleValidationDto `json:"name_filter"`
}

type ScenarioValidationDto struct {
Trigger triggerValidationDto `json:"trigger"`
Rules rulesValidationDto `json:"rules"`
Decision decisionValidationDto `json:"decision"`
Trigger triggerValidationDto `json:"trigger"`
Rules rulesValidationDto `json:"rules"`
SanctionCheckConfig sanctionCheckConfigValidationDto `json:"sanction_check_config"`
Decision decisionValidationDto `json:"decision"`
}

func AdaptScenarioValidationDto(s models.ScenarioValidation) ScenarioValidationDto {
Expand All @@ -58,6 +64,16 @@ func AdaptScenarioValidationDto(s models.ScenarioValidation) ScenarioValidationD
}
}),
},
SanctionCheckConfig: sanctionCheckConfigValidationDto{
Trigger: triggerValidationDto{
Errors: pure_utils.Map(s.SanctionCheck.TriggerRule.Errors, AdaptScenarioValidationErrorDto),
TriggerEvaluation: ast.AdaptNodeEvaluationDto(s.SanctionCheck.TriggerRule.TriggerEvaluation),
},
NameFilter: ruleValidationDto{
Errors: pure_utils.Map(s.SanctionCheck.NameFilter.Errors, AdaptScenarioValidationErrorDto),
RuleEvaluation: ast.AdaptNodeEvaluationDto(s.SanctionCheck.NameFilter.RuleEvaluation),
},
},
Decision: decisionValidationDto{
Errors: pure_utils.Map(s.Decision.Errors, AdaptScenarioValidationErrorDto),
},
Expand Down
5 changes: 5 additions & 0 deletions models/ast/ast_function.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ const (
FUNC_STRING_ENDS_WITH
FUNC_IS_MULTIPLE_OF
FUNC_STRING_TEMPLATE
FUNC_STRING_CONCAT
FUNC_UNDEFINED Function = -1
FUNC_UNKNOWN Function = -2
)
Expand Down Expand Up @@ -226,6 +227,10 @@ var FuncAttributesMap = map[Function]FuncAttributes{
DebugName: "FUNC_STRING_TEMPLATE",
AstName: "StringTemplate",
},
FUNC_STRING_CONCAT: {
DebugName: "FUNC_STRING_CONCAT",
AstName: "StringConcat",
},
FUNC_FILTER: FuncFilterAttributes,
}

Expand Down
8 changes: 7 additions & 1 deletion models/scenario_iterations.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,8 @@ type UpdateScenarioIterationBody struct {
type SanctionCheckConfig struct {
Enabled bool
Datasets []string
TriggerRule *ast.Node
TriggerRule ast.Node
Query SanctionCheckConfigQuery
Outcome SanctionCheckOutcome
}

Expand All @@ -70,9 +71,14 @@ type UpdateSanctionCheckConfigInput struct {
Enabled *bool
Datasets []string
TriggerRule *ast.Node
Query *SanctionCheckConfigQuery
Outcome UpdateSanctionCheckOutcomeInput
}

type SanctionCheckConfigQuery struct {
Name ast.Node
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if I understand correctly this is the "name" on which we filter on yente, would it make sense for future compatibility to also store the entity (here, default to "thing") on which we filter ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, but it doesn't map directly to OpenSanctions queries, those are managed separatly in a model specific to the provider. This is purely our own internal representation of the query set up from the UI.

This is where we would put Label and a generic Attributes if we decide to keep them in the UI.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok, got it. So it's "name" as in the design where we filter on the "name" (and optionally other things)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. It translates, on execution, to an OpenSanctions query which is stored in the sanction_checks table.

}

type UpdateSanctionCheckOutcomeInput struct {
ForceOutcome *Outcome
ScoreModifier *int
Expand Down
22 changes: 18 additions & 4 deletions models/scenario_validation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ const (
RuleFormulaRequired
// Ast output
FormulaMustReturnBoolean
FormulaMustReturnString
// Decision
ScoreThresholdMissing
ScoreThresholdsMismatch
Expand All @@ -32,6 +33,8 @@ func (e ScenarioValidationErrorCode) String() string {
return "RULE_FORMULA_REQUIRED"
case FormulaMustReturnBoolean:
return "FORMULA_MUST_RETURN_BOOLEAN"
case FormulaMustReturnString:
return "FORMULA_MUST_RETURN_STRING"
case ScoreThresholdMissing:
return "SCORE_THRESHOLD_MISSING"
case ScoreThresholdsMismatch:
Expand Down Expand Up @@ -71,11 +74,17 @@ type decisionValidation struct {
Errors []ScenarioValidationError
}

type sanctionCheckConfigValidation struct {
TriggerRule triggerValidation
NameFilter RuleValidation
}

type ScenarioValidation struct {
Errors []ScenarioValidationError
Trigger triggerValidation
Rules rulesValidation
Decision decisionValidation
Errors []ScenarioValidationError
Trigger triggerValidation
Rules rulesValidation
SanctionCheck sanctionCheckConfigValidation
Decision decisionValidation
}

func NewScenarioValidation() ScenarioValidation {
Expand All @@ -88,6 +97,11 @@ func NewScenarioValidation() ScenarioValidation {
Errors: make([]ScenarioValidationError, 0),
Rules: make(map[string]RuleValidation),
},
SanctionCheck: sanctionCheckConfigValidation{
TriggerRule: triggerValidation{
Errors: make([]ScenarioValidationError, 0),
},
},
Decision: decisionValidation{
Errors: make([]ScenarioValidationError, 0),
},
Expand Down
49 changes: 40 additions & 9 deletions repositories/dbmodels/db_sanction_check_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,24 +1,36 @@
package dbmodels

import (
"encoding/json"
"fmt"
"time"

"github.com/checkmarble/marble-backend/dto"
"github.com/checkmarble/marble-backend/models"
"github.com/checkmarble/marble-backend/utils"
"github.com/pkg/errors"
)

const TABLE_SANCTION_CHECK_CONFIGS = "sanction_check_configs"

type DBSanctionCheckConfigs struct {
Id string `db:"id"`
ScenarioIterationId string `db:"scenario_iteration_id"`
Enabled bool `db:"enabled"`
Datasets []string `db:"datasets"`
TriggerRule []byte `db:"trigger_rule"`
ForcedOutcome *string `db:"forced_outcome"`
ScoreModifier int `db:"score_modifier"`
UpdatedAt time.Time `db:"updated_at"`
Id string `db:"id"`
ScenarioIterationId string `db:"scenario_iteration_id"`
Enabled bool `db:"enabled"`
Datasets []string `db:"datasets"`
TriggerRule []byte `db:"trigger_rule"`
Query DBSanctionCheckConfigQuery `db:"query"`
ForcedOutcome *string `db:"forced_outcome"`
ScoreModifier int `db:"score_modifier"`
UpdatedAt time.Time `db:"updated_at"`
}

type DBSanctionCheckConfigQuery struct {
Name json.RawMessage `json:"name"`
}

type DBSanctionCheckConfigQueryInput struct {
Name dto.NodeDto `json:"name"`
}

var SanctionCheckConfigColumnList = utils.ColumnList[DBSanctionCheckConfigs]()
Expand All @@ -30,6 +42,11 @@ func AdaptSanctionCheckConfig(db DBSanctionCheckConfigs) (models.SanctionCheckCo
"unable to unmarshal formula ast expression: %w", err)
}

query, err := AdaptSanctionCheckConfigQuery(db.Query)
if err != nil {
return models.SanctionCheckConfig{}, errors.Wrap(err, "unable to unmarshal query formula")
}

var forcedOutcome models.Outcome = models.UnsetForcedOutcome

if db.ForcedOutcome != nil {
Expand All @@ -39,7 +56,8 @@ func AdaptSanctionCheckConfig(db DBSanctionCheckConfigs) (models.SanctionCheckCo
scc := models.SanctionCheckConfig{
Enabled: db.Enabled,
Datasets: db.Datasets,
TriggerRule: triggerRuleAst,
TriggerRule: *triggerRuleAst,
Query: query,
Outcome: models.SanctionCheckOutcome{
ForceOutcome: forcedOutcome,
ScoreModifier: db.ScoreModifier,
Expand All @@ -48,3 +66,16 @@ func AdaptSanctionCheckConfig(db DBSanctionCheckConfigs) (models.SanctionCheckCo

return scc, nil
}

func AdaptSanctionCheckConfigQuery(db DBSanctionCheckConfigQuery) (models.SanctionCheckConfigQuery, error) {
nameAst, err := AdaptSerializedAstExpression(db.Name)
if err != nil {
return models.SanctionCheckConfigQuery{}, err
}

model := models.SanctionCheckConfigQuery{
Name: *nameAst,
}

return model, nil
}
14 changes: 14 additions & 0 deletions repositories/migrations/20250130150000_add_sanctions_query_builder.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
-- +goose Up

alter table sanction_check_configs add column query jsonb not null default '{}';

alter table sanction_check_configs
drop constraint fk_scenario_iteration,
add constraint fk_scenario_iteration
foreign key (scenario_iteration_id)
references scenario_iterations (id)
on delete cascade;

-- +goose Down

alter table sanction_check_configs drop column query;
17 changes: 16 additions & 1 deletion repositories/sanction_check_config_repository.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"strings"

"github.com/Masterminds/squirrel"
"github.com/checkmarble/marble-backend/dto"
"github.com/checkmarble/marble-backend/models"
"github.com/checkmarble/marble-backend/repositories/dbmodels"
"github.com/checkmarble/marble-backend/utils"
Expand Down Expand Up @@ -44,16 +45,27 @@ func (repo *MarbleDbRepository) UpdateSanctionCheckConfig(ctx context.Context, e
triggerRule = astJson
}

var query dbmodels.DBSanctionCheckConfigQueryInput

if cfg.Query != nil {
ser, err := dto.AdaptNodeDto(cfg.Query.Name)
if err != nil {
return models.SanctionCheckConfig{}, err
}
query.Name = ser
}

sql := NewQueryBuilder().
Insert(dbmodels.TABLE_SANCTION_CHECK_CONFIGS).
Columns("scenario_iteration_id", "enabled", "datasets", "forced_outcome", "score_modifier", "trigger_rule").
Columns("scenario_iteration_id", "enabled", "datasets", "forced_outcome", "score_modifier", "trigger_rule", "query").
Values(
scenarioIterationId,
utils.Or(cfg.Enabled, true),
cfg.Datasets,
cfg.Outcome.ForceOutcome.MaybeString(),
utils.Or(cfg.Outcome.ScoreModifier, 0),
utils.Or(triggerRule, []byte(``)),
query,
)

updateFields := make([]string, 0, 4)
Expand All @@ -67,6 +79,9 @@ func (repo *MarbleDbRepository) UpdateSanctionCheckConfig(ctx context.Context, e
if cfg.TriggerRule != nil {
updateFields = append(updateFields, "trigger_rule = EXCLUDED.trigger_rule")
}
if cfg.Query != nil {
updateFields = append(updateFields, "query = EXCLUDED.query")
}
if cfg.Outcome.ForceOutcome != nil {
switch *cfg.Outcome.ForceOutcome {
case models.UnsetForcedOutcome:
Expand Down
Loading