Merge pull request #801 from checkmarble/christopher/add-aggregate-op…

…erators

feat: add is empty, is not empty, starts with, ends with filters operators

models/aggregate_filter.go

@@ -0,0 +1,9 @@
+package models
+
+import "github.com/checkmarble/marble-backend/models/ast"
+
+// Define where to put that
+type FilterWithType struct {
+	Filter    ast.Filter
+	FieldType DataType
+}

models/ast/ast_filter.go

@@ -1,5 +1,7 @@
 package ast
 
+import "slices"
+
 type FilterOperator string
 
 const (
@@ -11,9 +13,17 @@ const (
 	FILTER_LESSER_OR_EQUAL   FilterOperator = "<="
 	FILTER_IS_IN_LIST        FilterOperator = "IsInList"
 	FILTER_IS_NOT_IN_LIST    FilterOperator = "IsNotInList"
+	FILTER_IS_EMPTY          FilterOperator = "IsEmpty"
+	FILTER_IS_NOT_EMPTY      FilterOperator = "IsNotEmpty"
+	FILTER_STARTS_WITH       FilterOperator = "StringStartsWith"
+	FILTER_ENDS_WITH         FilterOperator = "StringEndsWith"
 	FILTER_UNKNOWN_OPERATION FilterOperator = "FILTER_UNKNOWN_OPERATION"
 )
 
+func (op FilterOperator) IsUnary() bool {
+	return slices.Contains([]FilterOperator{FILTER_IS_EMPTY, FILTER_IS_NOT_EMPTY}, op)
+}
+
 type Filter struct {
 	TableName string
 	FieldName string

repositories/ingested_data_read_repository.go

@@ -33,7 +33,7 @@ type IngestedDataReadRepository interface {
 		fieldName string,
 		fieldType models.DataType,
 		aggregator ast.Aggregator,
-		filters []ast.Filter,
+		filters []models.FilterWithType,
 	) (any, error)
 }
 
@@ -314,7 +314,7 @@ func createQueryAggregated(
 	fieldName string,
 	fieldType models.DataType,
 	aggregator ast.Aggregator,
-	filters []ast.Filter,
+	filters []models.FilterWithType,
 ) (squirrel.SelectBuilder, error) {
 	var selectExpression string
 	if aggregator == ast.AGGREGATOR_COUNT_DISTINCT {
@@ -339,7 +339,8 @@ func createQueryAggregated(
 
 	var err error
 	for _, filter := range filters {
-		query, err = addConditionForOperator(query, qualifiedTableName, filter.FieldName, filter.Operator, filter.Value)
+		query, err = addConditionForOperator(query, qualifiedTableName,
+			filter.Filter.FieldName, filter.FieldType, filter.Filter.Operator, filter.Filter.Value)
 		if err != nil {
 			return squirrel.SelectBuilder{}, err
 		}
@@ -354,7 +355,7 @@ func (repo *IngestedDataReadRepositoryImpl) QueryAggregatedValue(
 	fieldName string,
 	fieldType models.DataType,
 	aggregator ast.Aggregator,
-	filters []ast.Filter,
+	filters []models.FilterWithType,
 ) (any, error) {
 	if err := validateClientDbExecutor(exec); err != nil {
 		return nil, err
@@ -377,7 +378,7 @@ func (repo *IngestedDataReadRepositoryImpl) QueryAggregatedValue(
 	return result, nil
 }
 
-func addConditionForOperator(query squirrel.SelectBuilder, tableName string, fieldName string,
+func addConditionForOperator(query squirrel.SelectBuilder, tableName string, fieldName string, fieldType models.DataType,
 	operator ast.FilterOperator, value any,
 ) (squirrel.SelectBuilder, error) {
 	switch operator {
@@ -393,6 +394,30 @@ func addConditionForOperator(query squirrel.SelectBuilder, tableName string, fie
 		return query.Where(squirrel.Lt{fmt.Sprintf("%s.%s", tableName, fieldName): value}), nil
 	case ast.FILTER_LESSER_OR_EQUAL:
 		return query.Where(squirrel.LtOrEq{fmt.Sprintf("%s.%s", tableName, fieldName): value}), nil
+	case ast.FILTER_IS_EMPTY:
+		orCondition := squirrel.Or{
+			squirrel.Eq{fmt.Sprintf("%s.%s", tableName, fieldName): nil},
+		}
+		if fieldType == models.String {
+			orCondition = append(orCondition, squirrel.Eq{
+				fmt.Sprintf("%s.%s", tableName, fieldName): "",
+			})
+		}
+		return query.Where(orCondition), nil
+	case ast.FILTER_IS_NOT_EMPTY:
+		andCondition := squirrel.And{
+			squirrel.NotEq{fmt.Sprintf("%s.%s", tableName, fieldName): nil},
+		}
+		if fieldType == models.String {
+			andCondition = append(andCondition,
+				squirrel.NotEq{fmt.Sprintf("%s.%s", tableName, fieldName): ""},
+			)
+		}
+		return query.Where(andCondition), nil
+	case ast.FILTER_STARTS_WITH:
+		return query.Where(squirrel.Like{fmt.Sprintf("%s.%s", tableName, fieldName): fmt.Sprintf("%s%%", value)}), nil
+	case ast.FILTER_ENDS_WITH:
+		return query.Where(squirrel.Like{fmt.Sprintf("%s.%s", tableName, fieldName): fmt.Sprintf("%%%s", value)}), nil
 	default:
 		return query, fmt.Errorf("unknown operator %s: %w", operator, models.BadParameterError)
 	}

repositories/ingested_data_read_repository_test.go

@@ -105,7 +105,7 @@ func TestIngestedDataQueryAggregatedValueWithoutFilter(t *testing.T) {
 		utils.DummyFieldNameForInt,
 		models.Int,
 		ast.AGGREGATOR_AVG,
-		[]ast.Filter{},
+		[]models.FilterWithType{},
 	)
 	assert.Empty(t, err)
 	sql, args, err := query.ToSql()
@@ -123,7 +123,7 @@ func TestIngestedDataQueryCountWithoutFilter(t *testing.T) {
 		utils.DummyFieldNameForInt,
 		models.Int,
 		ast.AGGREGATOR_COUNT,
-		[]ast.Filter{})
+		[]models.FilterWithType{})
 	assert.Empty(t, err)
 	sql, args, err := query.ToSql()
 	assert.Empty(t, err)
@@ -134,18 +134,24 @@ func TestIngestedDataQueryCountWithoutFilter(t *testing.T) {
 }
 
 func TestIngestedDataQueryAggregatedValueWithFilter(t *testing.T) {
-	filters := []ast.Filter{
+	filters := []models.FilterWithType{
 		{
-			TableName: utils.DummyTableNameFirst,
-			FieldName: utils.DummyFieldNameForInt,
-			Operator:  ast.FILTER_EQUAL,
-			Value:     1,
+			Filter: ast.Filter{
+				TableName: utils.DummyTableNameFirst,
+				FieldName: utils.DummyFieldNameForInt,
+				Operator:  ast.FILTER_EQUAL,
+				Value:     1,
+			},
+			FieldType: models.Int,
 		},
 		{
-			TableName: utils.DummyTableNameFirst,
-			FieldName: utils.DummyFieldNameForBool,
-			Operator:  ast.FILTER_NOT_EQUAL,
-			Value:     true,
+			Filter: ast.Filter{
+				TableName: utils.DummyTableNameFirst,
+				FieldName: utils.DummyFieldNameForBool,
+				Operator:  ast.FILTER_NOT_EQUAL,
+				Value:     true,
+			},
+			FieldType: models.Bool,
 		},
 	}
 

usecases/ast_eval/evaluate/evaluate_aggregator.go

@@ -73,6 +73,7 @@ func (a AggregatorEvaluator) Evaluate(ctx context.Context, arguments ast.Argumen
 	}
 
 	// Filters validation
+	var filtersWithType []models.FilterWithType
 	if len(filters) > 0 {
 		for _, filter := range filters {
 			if filter.TableName != tableName {
@@ -82,14 +83,28 @@ func (a AggregatorEvaluator) Evaluate(ctx context.Context, arguments ast.Argumen
 					ast.NewNamedArgumentError("filters"),
 				))
 			}
-			// At the first nil filter value found, stop and just return the default value for the aggregator
-			if filter.Value == nil {
+
+			// At the first nil filter value found if we're not on an unary operator, stop and just return the default value for the aggregator
+			if filter.Value == nil && !filter.Operator.IsUnary() {
 				return a.defaultValueForAggregator(aggregator)
 			}
+
+			filterFieldType, err := getFieldType(a.DataModel, filter.TableName, filter.FieldName)
+			if err != nil {
+				return MakeEvaluateError(errors.Join(
+					errors.Wrap(err, fmt.Sprintf("field type for %s.%s not found in data model in Evaluate aggregator", filter.TableName, filter.FieldName)),
+					ast.NewNamedArgumentError("fieldName"),
+				))
+			}
+
+			filtersWithType = append(filtersWithType, models.FilterWithType{
+				Filter:    filter,
+				FieldType: filterFieldType,
+			})
 		}
 	}
 
-	result, err := a.runQueryInRepository(ctx, tableName, fieldName, fieldType, aggregator, filters)
+	result, err := a.runQueryInRepository(ctx, tableName, fieldName, fieldType, aggregator, filtersWithType)
 	if err != nil {
 		return MakeEvaluateError(errors.Wrap(err, "Error running aggregation query in repository"))
 	}
@@ -107,7 +122,7 @@ func (a AggregatorEvaluator) runQueryInRepository(
 	fieldName string,
 	fieldType models.DataType,
 	aggregator ast.Aggregator,
-	filters []ast.Filter,
+	filters []models.FilterWithType,
 ) (any, error) {
 	if a.ReturnFakeValue {
 		return DryRunQueryAggregatedValue(a.DataModel, tableName, fieldName, aggregator)
@@ -117,7 +132,8 @@ func (a AggregatorEvaluator) runQueryInRepository(
 	if err != nil {
 		return nil, err
 	}
-	return a.IngestedDataReadRepository.QueryAggregatedValue(ctx, db, tableName, fieldName, fieldType, aggregator, filters)
+	return a.IngestedDataReadRepository.QueryAggregatedValue(ctx, db, tableName,
+		fieldName, fieldType, aggregator, filters)
 }
 
 func (a AggregatorEvaluator) defaultValueForAggregator(aggregator ast.Aggregator) (any, []error) {

usecases/ast_eval/evaluate/evaluate_filter.go

@@ -25,6 +25,10 @@ var validTypeForFilterOperators = map[ast.FilterOperator][]models.DataType{
 	ast.FILTER_LESSER_OR_EQUAL:  {models.Int, models.Float, models.String, models.Timestamp},
 	ast.FILTER_IS_IN_LIST:       {models.String},
 	ast.FILTER_IS_NOT_IN_LIST:   {models.String},
+	ast.FILTER_IS_EMPTY:         {models.Bool, models.Int, models.Float, models.String, models.Timestamp},
+	ast.FILTER_IS_NOT_EMPTY:     {models.Bool, models.Int, models.Float, models.String, models.Timestamp},
+	ast.FILTER_STARTS_WITH:      {models.String},
+	ast.FILTER_ENDS_WITH:        {models.String},
 }
 
 func (f FilterEvaluator) Evaluate(ctx context.Context, arguments ast.Arguments) (any, []error) {