Skip to content

Commit fc69bb5

Browse files
keenkeystrokeskeenkeystrokes
authored
DataQuality - Use eval for aggregate val func composition & PCT Tests for helper functions (#3445)
1 parent fbc4d6e commit fc69bb5

File tree

7 files changed

+142
-131
lines changed

7 files changed

+142
-131
lines changed

‎legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/DataQualityCompilerExtension.java

+1-9
Original file line numberDiff line numberDiff line change
@@ -379,15 +379,7 @@ public List<Function<Handlers, List<FunctionExpressionBuilderRegistrationInfo>>>
379379
new FunctionExpressionBuilderRegistrationInfo(null, handlers.m(handlers.h("meta::external::dataquality::rowCountGreaterThanEqual_Relation_1__Number_1__Boolean_1_", false, ps -> handlers.res("Boolean", "one"), ps -> ps.size() == 2 && handlers.typeOne(ps.get(0), "Relation") && handlers.typeOne(ps.get(1), Sets.immutable.with("Number", "Integer", "Float", "Decimal"))))),
380380
new FunctionExpressionBuilderRegistrationInfo(null, handlers.m(handlers.h("meta::external::dataquality::rowCountLowerThan_Relation_1__Number_1__Boolean_1_", false, ps -> handlers.res("Boolean", "one"), ps -> ps.size() == 2 && handlers.typeOne(ps.get(0), "Relation") && handlers.typeOne(ps.get(1), Sets.immutable.with("Number", "Integer", "Float", "Decimal"))))),
381381
new FunctionExpressionBuilderRegistrationInfo(null, handlers.m(handlers.h("meta::external::dataquality::rowCountLowerThanEqual_Relation_1__Number_1__Boolean_1_", false, ps -> handlers.res("Boolean", "one"), ps -> ps.size() == 2 && handlers.typeOne(ps.get(0), "Relation") && handlers.typeOne(ps.get(1), Sets.immutable.with("Number", "Integer", "Float", "Decimal"))))),
382-
new FunctionExpressionBuilderRegistrationInfo(null, handlers.m(handlers.h("meta::external::dataquality::rowCountEqual_Relation_1__Number_1__Boolean_1_", false, ps -> handlers.res("Boolean", "one"), ps -> ps.size() == 2 && handlers.typeOne(ps.get(0), "Relation") && handlers.typeOne(ps.get(1), Sets.immutable.with("Number", "Integer", "Float", "Decimal"))))),
383-
384-
// column value
385-
new FunctionExpressionBuilderRegistrationInfo(null, handlers.m(handlers.h("meta::external::dataquality::columnValueGreaterThan_Relation_1__ColSpec_1__Number_1__Boolean_1_", false, ps -> handlers.res("Boolean", "one"), ps -> ps.size() == 3 && handlers.typeOne(ps.get(0), "Relation") && handlers.typeOne(ps.get(1), "ColSpec") && handlers.typeOne(ps.get(1), Sets.immutable.with("Number", "Integer", "Float", "Decimal"))))),
386-
new FunctionExpressionBuilderRegistrationInfo(null, handlers.m(handlers.h("meta::external::dataquality::columnValueGreaterThanEqual_Relation_1__ColSpec_1__Number_1__Boolean_1_", false, ps -> handlers.res("Boolean", "one"), ps -> ps.size() == 3 && handlers.typeOne(ps.get(0), "Relation") && handlers.typeOne(ps.get(1), "ColSpec") && handlers.typeOne(ps.get(1), Sets.immutable.with("Number", "Integer", "Float", "Decimal"))))),
387-
new FunctionExpressionBuilderRegistrationInfo(null, handlers.m(handlers.h("meta::external::dataquality::columnValueLessThan_Relation_1__ColSpec_1__Number_1__Boolean_1_", false, ps -> handlers.res("Boolean", "one"), ps -> ps.size() == 3 && handlers.typeOne(ps.get(0), "Relation") && handlers.typeOne(ps.get(1), "ColSpec") && handlers.typeOne(ps.get(1), Sets.immutable.with("Number", "Integer", "Float", "Decimal"))))),
388-
new FunctionExpressionBuilderRegistrationInfo(null, handlers.m(handlers.h("meta::external::dataquality::columnValueLessThanEqual_Relation_1__ColSpec_1__Number_1__Boolean_1_", false, ps -> handlers.res("Boolean", "one"), ps -> ps.size() == 3 && handlers.typeOne(ps.get(0), "Relation") && handlers.typeOne(ps.get(1), "ColSpec") && handlers.typeOne(ps.get(1), Sets.immutable.with("Number", "Integer", "Float", "Decimal"))))),
389-
new FunctionExpressionBuilderRegistrationInfo(null, handlers.m(handlers.h("meta::external::dataquality::columnValueUnique_Relation_1__ColSpecArray_1__Boolean_1_", false, ps -> handlers.res("Boolean", "one"), ps -> ps.size() == 2 && handlers.typeOne(ps.get(0), "Relation") && handlers.typeOne(ps.get(1), "ColSpec")))),
390-
new FunctionExpressionBuilderRegistrationInfo(null, handlers.m(handlers.h("meta::external::dataquality::columnValueUnique_Relation_1__Boolean_1_", false, ps -> handlers.res("Boolean", "one"), ps -> ps.size() == 1)))
382+
new FunctionExpressionBuilderRegistrationInfo(null, handlers.m(handlers.h("meta::external::dataquality::rowCountEqual_Relation_1__Number_1__Boolean_1_", false, ps -> handlers.res("Boolean", "one"), ps -> ps.size() == 2 && handlers.typeOne(ps.get(0), "Relation") && handlers.typeOne(ps.get(1), Sets.immutable.with("Number", "Integer", "Float", "Decimal")))))
391383
));
392384
}
393385

‎legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/test/java/org/finos/legend/engine/generation/dataquality/TestDataQualityLambdaGenerator.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -114,8 +114,8 @@ public void testLambdaGeneration_relationValidation_aggregate()
114114
" }\n" +
115115
" ];\n" +
116116
"}";
117-
118-
assertLambda(validation, "{\"_type\":\"lambda\",\"body\":[{\"fControl\":\"from_T_m__Runtime_1__T_m_\",\"function\":\"from\",\"_type\":\"func\",\"parameters\":[{\"fControl\":\"map_T_m__Function_1__V_m_\",\"function\":\"map\",\"_type\":\"func\",\"parameters\":[{\"fControl\":\"extend_Relation_1__FuncColSpec_1__Relation_1_\",\"function\":\"extend\",\"_type\":\"func\",\"parameters\":[{\"fControl\":\"select_Relation_1__ColSpecArray_1__Relation_1_\",\"function\":\"select\",\"_type\":\"func\",\"parameters\":[{\"_type\":\"classInstance\",\"type\":\">\",\"value\":{}},{\"_type\":\"classInstance\",\"type\":\"colSpecArray\",\"value\":{\"colSpecs\":[{\"name\":\"FIRSTNAME\"},{\"name\":\"LASTNAME\"}]}}]},{\"_type\":\"classInstance\",\"type\":\"colSpec\",\"value\":{\"function1\":{\"_type\":\"lambda\",\"body\":[{\"fControl\":\"joinStrings_String_MANY__String_1_\",\"function\":\"joinStrings\",\"_type\":\"func\",\"parameters\":[{\"multiplicity\":{\"upperBound\":2,\"lowerBound\":2},\"values\":[{\"fControl\":\"toOne_T_MANY__T_1_\",\"function\":\"toOne\",\"_type\":\"func\",\"parameters\":[{\"_type\":\"property\",\"property\":\"FIRSTNAME\",\"parameters\":[{\"_type\":\"var\",\"name\":\"x\"}]}]},{\"fControl\":\"toOne_T_MANY__T_1_\",\"function\":\"toOne\",\"_type\":\"func\",\"parameters\":[{\"_type\":\"property\",\"property\":\"LASTNAME\",\"parameters\":[{\"_type\":\"var\",\"name\":\"x\"}]}]}],\"_type\":\"collection\"}]}],\"parameters\":[{\"multiplicity\":{\"upperBound\":1,\"lowerBound\":1},\"_type\":\"var\",\"name\":\"x\",\"genericType\":{\"rawType\":{\"columns\":[{\"multiplicity\":{\"upperBound\":1,\"lowerBound\":0},\"name\":\"FIRSTNAME\",\"genericType\":{\"rawType\":{\"fullPath\":\"String\",\"_type\":\"packageableType\"}}},{\"multiplicity\":{\"upperBound\":1,\"lowerBound\":0},\"name\":\"LASTNAME\",\"genericType\":{\"rawType\":{\"fullPath\":\"String\",\"_type\":\"packageableType\"}}}],\"_type\":\"relationType\"}}}]},\"name\":\"fullname\"}}]},{\"_type\":\"lambda\",\"body\":[{\"fControl\":\"relationNotEmpty_Relation_1__Boolean_1_\",\"function\":\"relationNotEmpty\",\"_type\":\"func\",\"parameters\":[{\"_type\":\"var\",\"name\":\"rel\"}]}],\"parameters\":[{\"multiplicity\":{\"upperBound\":1,\"lowerBound\":1},\"_type\":\"var\",\"name\":\"rel\",\"genericType\":{\"typeArguments\":[{\"rawType\":{\"columns\":[{\"multiplicity\":{\"upperBound\":1,\"lowerBound\":0},\"name\":\"FIRSTNAME\",\"genericType\":{\"rawType\":{\"fullPath\":\"String\",\"_type\":\"packageableType\"}}},{\"multiplicity\":{\"upperBound\":1,\"lowerBound\":0},\"name\":\"LASTNAME\",\"genericType\":{\"rawType\":{\"fullPath\":\"String\",\"_type\":\"packageableType\"}}},{\"multiplicity\":{\"upperBound\":1,\"lowerBound\":1},\"name\":\"fullname\",\"genericType\":{\"rawType\":{\"fullPath\":\"String\",\"_type\":\"packageableType\"}}}],\"_type\":\"relationType\"}}],\"rawType\":{\"fullPath\":\"meta::pure::metamodel::relation::Relation\",\"_type\":\"packageableType\"}}}]}]},{\"_type\":\"classInstance\",\"type\":\"runtimeInstance\",\"value\":{\"runtime\":{\"_type\":\"legacyRuntime\",\"connections\":[{\"_type\":\"RelationalDatabaseConnection\",\"authenticationStrategy\":{\"_type\":\"h2Default\"},\"type\":\"H2\",\"datasourceSpecification\":{\"_type\":\"h2Local\"},\"element\":\"meta::external::dataquality::tests::domain::db\"}]}}}]}]}", "nonEmptyDataset");
117+
118+
assertLambda(validation, "{\"_type\":\"lambda\",\"body\":[{\"fControl\":\"from_T_m__Runtime_1__T_m_\",\"function\":\"from\",\"_type\":\"func\",\"parameters\":[{\"fControl\":\"eval_Function_1__T_n__V_m_\",\"function\":\"eval\",\"_type\":\"func\",\"parameters\":[{\"_type\":\"lambda\",\"body\":[{\"fControl\":\"relationNotEmpty_Relation_1__Boolean_1_\",\"function\":\"relationNotEmpty\",\"_type\":\"func\",\"parameters\":[{\"_type\":\"var\",\"name\":\"rel\"}]}],\"parameters\":[{\"multiplicity\":{\"upperBound\":1,\"lowerBound\":1},\"_type\":\"var\",\"name\":\"rel\",\"genericType\":{\"typeArguments\":[{\"rawType\":{\"columns\":[{\"multiplicity\":{\"upperBound\":1,\"lowerBound\":0},\"name\":\"FIRSTNAME\",\"genericType\":{\"rawType\":{\"fullPath\":\"String\",\"_type\":\"packageableType\"}}},{\"multiplicity\":{\"upperBound\":1,\"lowerBound\":0},\"name\":\"LASTNAME\",\"genericType\":{\"rawType\":{\"fullPath\":\"String\",\"_type\":\"packageableType\"}}},{\"multiplicity\":{\"upperBound\":1,\"lowerBound\":1},\"name\":\"fullname\",\"genericType\":{\"rawType\":{\"fullPath\":\"String\",\"_type\":\"packageableType\"}}}],\"_type\":\"relationType\"}}],\"rawType\":{\"fullPath\":\"meta::pure::metamodel::relation::Relation\",\"_type\":\"packageableType\"}}}]},{\"fControl\":\"extend_Relation_1__FuncColSpec_1__Relation_1_\",\"function\":\"extend\",\"_type\":\"func\",\"parameters\":[{\"fControl\":\"select_Relation_1__ColSpecArray_1__Relation_1_\",\"function\":\"select\",\"_type\":\"func\",\"parameters\":[{\"_type\":\"classInstance\",\"type\":\">\",\"value\":{}},{\"_type\":\"classInstance\",\"type\":\"colSpecArray\",\"value\":{\"colSpecs\":[{\"name\":\"FIRSTNAME\"},{\"name\":\"LASTNAME\"}]}}]},{\"_type\":\"classInstance\",\"type\":\"colSpec\",\"value\":{\"function1\":{\"_type\":\"lambda\",\"body\":[{\"fControl\":\"joinStrings_String_MANY__String_1_\",\"function\":\"joinStrings\",\"_type\":\"func\",\"parameters\":[{\"multiplicity\":{\"upperBound\":2,\"lowerBound\":2},\"values\":[{\"fControl\":\"toOne_T_MANY__T_1_\",\"function\":\"toOne\",\"_type\":\"func\",\"parameters\":[{\"_type\":\"property\",\"property\":\"FIRSTNAME\",\"parameters\":[{\"_type\":\"var\",\"name\":\"x\"}]}]},{\"fControl\":\"toOne_T_MANY__T_1_\",\"function\":\"toOne\",\"_type\":\"func\",\"parameters\":[{\"_type\":\"property\",\"property\":\"LASTNAME\",\"parameters\":[{\"_type\":\"var\",\"name\":\"x\"}]}]}],\"_type\":\"collection\"}]}],\"parameters\":[{\"multiplicity\":{\"upperBound\":1,\"lowerBound\":1},\"_type\":\"var\",\"name\":\"x\",\"genericType\":{\"rawType\":{\"columns\":[{\"multiplicity\":{\"upperBound\":1,\"lowerBound\":0},\"name\":\"FIRSTNAME\",\"genericType\":{\"rawType\":{\"fullPath\":\"String\",\"_type\":\"packageableType\"}}},{\"multiplicity\":{\"upperBound\":1,\"lowerBound\":0},\"name\":\"LASTNAME\",\"genericType\":{\"rawType\":{\"fullPath\":\"String\",\"_type\":\"packageableType\"}}}],\"_type\":\"relationType\"}}}]},\"name\":\"fullname\"}}]}]},{\"_type\":\"classInstance\",\"type\":\"runtimeInstance\",\"value\":{\"runtime\":{\"_type\":\"legacyRuntime\",\"connections\":[{\"_type\":\"RelationalDatabaseConnection\",\"authenticationStrategy\":{\"_type\":\"h2Default\"},\"type\":\"H2\",\"datasourceSpecification\":{\"_type\":\"h2Local\"},\"element\":\"meta::external::dataquality::tests::domain::db\"}]}}}]}]}", "nonEmptyDataset");
119119
}
120120

121121
private void assertLambda(String modelString, String expected)

‎legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_relation_helper_test.pure

-92
Original file line numberDiff line numberDiff line change
@@ -86,95 +86,3 @@ function <<test.Test>> meta::external::dataquality::tests::rowCountEqual():Boole
8686
$rel->meta::external::dataquality::rowCountEqual(3)->assert();
8787
}
8888

89-
// column value -
90-
function <<test.Test>> meta::external::dataquality::tests::columnValueGreaterThan():Boolean[1]
91-
{
92-
let rel = #TDS
93-
val, str
94-
1, a
95-
3, ewe
96-
4, aa
97-
#;
98-
99-
$rel->meta::external::dataquality::columnValueGreaterThan(~val, 0)->assert();
100-
}
101-
102-
function <<test.Test>> meta::external::dataquality::tests::columnValueGreaterThanEqual():Boolean[1]
103-
{
104-
let rel = #TDS
105-
val, str
106-
1, a
107-
3, ewe
108-
4, aa
109-
#;
110-
111-
$rel->meta::external::dataquality::columnValueGreaterThanEqual(~val, 1)->assert();
112-
}
113-
114-
function <<test.Test>> meta::external::dataquality::tests::columnValueLessThan():Boolean[1]
115-
{
116-
let rel = #TDS
117-
val, str
118-
1, a
119-
3, ewe
120-
4, aa
121-
#;
122-
123-
$rel->meta::external::dataquality::columnValueLessThan(~val, 5)->assert();
124-
}
125-
126-
function <<test.Test>> meta::external::dataquality::tests::columnValueLessThanEqual():Boolean[1]
127-
{
128-
let rel = #TDS
129-
val, str
130-
1, a
131-
3, ewe
132-
4, aa
133-
#;
134-
135-
$rel->meta::external::dataquality::columnValueLessThanEqual(~val, 4)->assert();
136-
}
137-
138-
function <<test.Test>> meta::external::dataquality::tests::columnValueUnique():Boolean[1]
139-
{
140-
let rel = #TDS
141-
val, str
142-
1, a
143-
3, ewe
144-
4, aa
145-
#;
146-
147-
$rel->meta::external::dataquality::columnValueUnique()->assert();
148-
149-
let rel_with_duplicates = #TDS
150-
val, str
151-
1, a
152-
3, ewe
153-
4, aa
154-
4, aa
155-
#;
156-
157-
$rel_with_duplicates->meta::external::dataquality::columnValueUnique()->assertFalse();
158-
}
159-
160-
function <<test.Test>> meta::external::dataquality::tests::columnValueUnique_columns():Boolean[1]
161-
{
162-
let rel = #TDS
163-
val, str
164-
1, a
165-
3, ewe
166-
4, aa
167-
#;
168-
169-
$rel->meta::external::dataquality::columnValueUnique(~[val])->assert();
170-
171-
let rel_val_duplicates = #TDS
172-
val, str
173-
1, a
174-
3, ewe
175-
4, aa
176-
4, aab
177-
#;
178-
179-
$rel_val_duplicates->meta::external::dataquality::columnValueUnique(~[val])->assertFalse();
180-
}

‎legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test.pure

+1-1
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,6 @@ function <<test.Test>> meta::external::dataquality::tests::testLambdaGeneration_
246246
' ];' +
247247
' }',
248248
'nonEmptyDataset',
249-
{|#>{meta::external::dataquality::tests::domain::db.personTable}#->select(~[FIRSTNAME, LASTNAME])->meta::pure::functions::collection::map(rel| $rel->meta::external::dataquality::relationNotEmpty())}
249+
{|{rel|$rel->meta::external::dataquality::relationNotEmpty();}->meta::pure::functions::lang::eval(#>{meta::external::dataquality::tests::domain::db.personTable}#->meta::pure::functions::relation::select(~[FIRSTNAME,LASTNAME]));}
250250
)
251251
}

‎legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/pom.xml

+14
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,11 @@
8585
<artifactId>legend-engine-pure-functions-relation-pure</artifactId>
8686
<version>${project.version}</version>
8787
</dependency>
88+
<dependency>
89+
<groupId>org.finos.legend.pure</groupId>
90+
<artifactId>legend-pure-m2-dsl-tds-pure</artifactId>
91+
<version>${legend.pure.version}</version>
92+
</dependency>
8893
</dependencies>
8994
</plugin>
9095
<plugin>
@@ -158,6 +163,11 @@
158163
<artifactId>legend-engine-pure-functions-relation-pure</artifactId>
159164
<version>${project.version}</version>
160165
</dependency>
166+
<dependency>
167+
<groupId>org.finos.legend.pure</groupId>
168+
<artifactId>legend-pure-m2-dsl-tds-pure</artifactId>
169+
<version>${legend.pure.version}</version>
170+
</dependency>
161171
</dependencies>
162172
</plugin>
163173
</plugins>
@@ -232,6 +242,10 @@
232242
<groupId>org.finos.legend.engine</groupId>
233243
<artifactId>legend-engine-pure-platform-java</artifactId>
234244
</dependency>
245+
<dependency>
246+
<groupId>org.finos.legend.pure</groupId>
247+
<artifactId>legend-pure-m2-dsl-tds-pure</artifactId>
248+
</dependency>
235249
<dependency>
236250
<groupId>org.finos.legend.engine</groupId>
237251
<artifactId>legend-engine-pure-functions-relation-pure</artifactId>

‎legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/generation/dataquality.pure

+7-7
Original file line numberDiff line numberDiff line change
@@ -552,18 +552,18 @@ function meta::external::dataquality::buildRelationExpressionForAggregateLevelVa
552552

553553
^SimpleFunctionExpression
554554
(
555-
func = map_T_m__Function_1__V_m_,
555+
func = eval_Function_1__T_n__V_m_,
556556
multiplicity = PureOne,
557-
genericType = ^GenericType(rawType = Boolean),
557+
genericType = ^GenericType(rawType=Boolean),
558558
importGroup = system::imports::coreImport,
559559
parametersValues =
560-
[ $dqRelationValidation.query->evaluateAndDeactivate()->toOne().expressionSequence->toOne(),
561-
^InstanceValue (
562-
genericType = ^GenericType(rawType=LambdaFunction, typeArguments=^GenericType(rawType=^FunctionType(parameters=^VariableExpression(name='validationFilter', genericType=$dqRelationValidation.query->evaluateAndDeactivate()->functionReturnType(), multiplicity=PureOne), returnMultiplicity=PureOne, returnType=^GenericType(rawType=Boolean)))),
560+
[ ^InstanceValue (
561+
genericType = ^GenericType(rawType=LambdaFunction, typeArguments=^GenericType(rawType=^FunctionType(parameters=^VariableExpression(name='aggfilter', genericType=$dqRelationValidation.query->evaluateAndDeactivate()->functionReturnType(), multiplicity=PureOne), returnMultiplicity=PureOne, returnType=^GenericType(rawType=Boolean)))),
563562
multiplicity = PureOne,
564-
values = meta::external::dataquality::lambda(^FunctionType(returnMultiplicity = PureOne, returnType = ^GenericType(rawType = Boolean), parameters = [^VariableExpression(multiplicity=PureOne,genericType=$dqRelationValidation.query->evaluateAndDeactivate()->functionReturnType(),name=$assertion->meta::pure::metamodel::serialization::grammar::printLambdaParameters())]),
563+
values = meta::external::dataquality::lambda(^FunctionType(returnMultiplicity = PureOne, returnType = ^GenericType(rawType=Boolean), parameters = [^VariableExpression(multiplicity=PureOne,genericType=$dqRelationValidation.query->evaluateAndDeactivate()->functionReturnType(),name=$assertion->meta::pure::metamodel::serialization::grammar::printLambdaParameters())]),
565564
$assertion.expressionSequence->evaluateAndDeactivate()->toOne())
566-
)
565+
),
566+
$dqRelationValidation.query->evaluateAndDeactivate()->toOne().expressionSequence->toOne()
567567
]
568568
)->evaluateAndDeactivate();
569569
}

0 commit comments

Comments
 (0)