diff --git a/java/src/main/java/io/cucumber/query/OrderableMessage.java b/java/src/main/java/io/cucumber/query/OrderableMessage.java new file mode 100644 index 00000000..7187060d --- /dev/null +++ b/java/src/main/java/io/cucumber/query/OrderableMessage.java @@ -0,0 +1,37 @@ +package io.cucumber.query; + +import java.util.Comparator; + +import static java.util.Comparator.naturalOrder; +import static java.util.Comparator.nullsFirst; +import static java.util.Objects.requireNonNull; + +final class OrderableMessage implements Comparable> { + private final T message; + private final String uri; + private final Long line; + + OrderableMessage(T message) { + this(message, null,null); + } + + OrderableMessage(T message, String uri, Long line) { + this.message = requireNonNull(message); + this.uri = uri; + this.line = line; + } + + private final Comparator> comparator = Comparator + .comparing((OrderableMessage ord) -> ord.uri, nullsFirst(naturalOrder())) + .thenComparing(ord -> ord.line, nullsFirst(naturalOrder())); + + + @Override + public int compareTo(OrderableMessage o) { + return comparator.compare(this, o); + } + + T getMessage() { + return message; + } +} diff --git a/java/src/main/java/io/cucumber/query/Query.java b/java/src/main/java/io/cucumber/query/Query.java index b25acd76..894453a7 100644 --- a/java/src/main/java/io/cucumber/query/Query.java +++ b/java/src/main/java/io/cucumber/query/Query.java @@ -43,6 +43,7 @@ import java.util.Map.Entry; import java.util.Optional; import java.util.Objects; +import java.util.function.Function; import static java.util.Collections.emptyList; import static java.util.Comparator.comparing; @@ -81,6 +82,7 @@ public Map countMostSevereTestStepResultStatus() { .collect(groupingBy(identity(), LinkedHashMap::new, counting()))); return results; } + public int countTestCasesStarted() { return findAllTestCaseStarted().size(); } @@ -101,12 +103,38 @@ public List findAllTestCaseStarted() { .collect(toList()); } + public List findAllTestCaseStartedInCanonicalOrder() { + return findAllTestCaseStarted().stream() + .map(createOrderableMessage(this::findPickleBy)) + .map(OrderableMessage::getMessage) + .collect(toList()); + } + public List findAllTestCaseFinished() { return repository.testCaseFinishedByTestCaseStartedId.values().stream() .filter(testCaseFinished -> !testCaseFinished.getWillBeRetried()) .collect(toList()); } + public List findAllTestCaseFinishedInCanonicalOrder() { + return findAllTestCaseFinished().stream() + .map(createOrderableMessage(this::findPickleBy)) + .map(OrderableMessage::getMessage) + .collect(toList()); + } + + private Function> createOrderableMessage(Function> findPickleBy) { + return message -> findPickleBy.apply(message) + .map(pickle -> { + String uri = pickle.getUri(); + Long location = findLocationOf(pickle) + .map(Location::getLine) + .orElse(null); + return new OrderableMessage<>(message, uri, location); + }) + .orElseGet(() -> new OrderableMessage<>(message)); + } + public List findAllTestSteps() { return new ArrayList<>(repository.testStepById.values()); } diff --git a/java/src/test/java/io/cucumber/query/MessageOrderer.java b/java/src/test/java/io/cucumber/query/MessageOrderer.java new file mode 100644 index 00000000..d3dd4521 --- /dev/null +++ b/java/src/test/java/io/cucumber/query/MessageOrderer.java @@ -0,0 +1,99 @@ +package io.cucumber.query; + +import io.cucumber.messages.types.Envelope; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Random; +import java.util.function.Consumer; + +class MessageOrderer { + + /** + * Make test cases deterministically reproducible. + */ + private static final Random random = new Random(202509171705L); + + static Consumer> originalOrder() { + return envelopes -> { + }; + } + + /** + * Simulates parallel execution of testcases by interleaving the + * execution of different test cases. + */ + static Consumer> simulateParallelExecution() { + return messages -> { + List testCaseMessagesInSerial = testCasesSubList(messages); + List> testCaseMessagesGrouped = groupTestCasesIntoBuckets(testCaseMessagesInSerial); + List testCaseMessagesInterleaved = interleafMessagesFromTestCases(testCaseMessagesGrouped); + replaceAll(testCaseMessagesInSerial, testCaseMessagesInterleaved); + }; + } + + private static void replaceAll(List testCaseMessagesInSerial, List envelopes) { + for (int i = 0; i < testCaseMessagesInSerial.size(); i++) { + testCaseMessagesInSerial.set(i, envelopes.get(i)); + } + } + + private static List interleafMessagesFromTestCases(List> messagesGroupedByTestCase) { + List serial = new ArrayList<>(); + while (!messagesGroupedByTestCase.isEmpty()) { + Collections.shuffle(messagesGroupedByTestCase, random); + Iterator> bucketIterator = messagesGroupedByTestCase.iterator(); + while (bucketIterator.hasNext()) { + List bucket = bucketIterator.next(); + if (bucket.isEmpty()) { + bucketIterator.remove(); + } else { + serial.add(bucket.remove(0)); + } + } + } + return serial; + } + + private static List> groupTestCasesIntoBuckets(List testCaseMessages) { + List> buckets = new ArrayList<>(); + List currentBucket = new ArrayList<>(); + for (Envelope middleMessage : testCaseMessages) { + if (middleMessage.getTestCaseStarted().isPresent()) { + buckets.add(currentBucket); + currentBucket = new ArrayList<>(); + } + currentBucket.add(middleMessage); + } + buckets.add(currentBucket); + return buckets; + } + + private static List testCasesSubList(List messages) { + int testRunStartedIndex = findTestRunStartedIndex(messages); + int testRunFinishedIndex = findTestRunFinishedIndex(messages); + return messages.subList(testRunStartedIndex + 1, testRunFinishedIndex - 1); + } + + private static int findTestRunFinishedIndex(List messages) { + int testRunFinishedIndex = messages.size() - 1; + for (; testRunFinishedIndex >= 0; testRunFinishedIndex--) { + if (messages.get(testRunFinishedIndex).getTestRunFinished().isPresent()) { + break; + } + } + return testRunFinishedIndex; + } + + private static int findTestRunStartedIndex(List messages) { + int testRunStartedIndex = 0; + for (; testRunStartedIndex < messages.size(); testRunStartedIndex++) { + if (messages.get(testRunStartedIndex).getTestRunStarted().isPresent()) { + break; + } + } + return testRunStartedIndex; + } +} diff --git a/java/src/test/java/io/cucumber/query/QueryAcceptanceTest.java b/java/src/test/java/io/cucumber/query/QueryAcceptanceTest.java index ae87a845..52ac388b 100644 --- a/java/src/test/java/io/cucumber/query/QueryAcceptanceTest.java +++ b/java/src/test/java/io/cucumber/query/QueryAcceptanceTest.java @@ -37,15 +37,19 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Consumer; import java.util.function.Function; import static com.fasterxml.jackson.core.util.DefaultIndenter.SYSTEM_LINEFEED_INSTANCE; import static io.cucumber.query.Jackson.OBJECT_MAPPER; +import static io.cucumber.query.MessageOrderer.originalOrder; +import static io.cucumber.query.MessageOrderer.simulateParallelExecution; import static io.cucumber.query.Repository.RepositoryFeature.INCLUDE_ATTACHMENTS; import static io.cucumber.query.Repository.RepositoryFeature.INCLUDE_GHERKIN_DOCUMENTS; import static io.cucumber.query.Repository.RepositoryFeature.INCLUDE_HOOKS; import static io.cucumber.query.Repository.RepositoryFeature.INCLUDE_STEP_DEFINITIONS; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.nio.file.Files.readAllBytes; import static java.util.Arrays.asList; import static java.util.stream.Collectors.toList; import static org.assertj.core.api.Assertions.assertThat; @@ -82,7 +86,16 @@ private static List getSources() { @ParameterizedTest @MethodSource("acceptance") void test(QueryTestCase testCase) throws IOException { - ByteArrayOutputStream bytes = writeQueryResults(testCase, new ByteArrayOutputStream()); + ByteArrayOutputStream bytes = writeQueryResults(testCase, new ByteArrayOutputStream(), originalOrder()); + String expected = new String(Files.readAllBytes(testCase.expected), UTF_8); + String actual = new String(bytes.toByteArray(), UTF_8); + assertThat(actual).isEqualTo(expected); + } + + @ParameterizedTest + @MethodSource("acceptance") + void testWithSimulatedParallelExecution(QueryTestCase testCase) throws IOException { + ByteArrayOutputStream bytes = writeQueryResults(testCase, new ByteArrayOutputStream(), originalOrder()); String expected = new String(Files.readAllBytes(testCase.expected), UTF_8); String actual = new String(bytes.toByteArray(), UTF_8); assertThat(actual).isEqualTo(expected); @@ -93,24 +106,28 @@ void test(QueryTestCase testCase) throws IOException { @Disabled void updateExpectedQueryResultFiles(QueryTestCase testCase) throws IOException { try (OutputStream out = Files.newOutputStream(testCase.expected)) { - writeQueryResults(testCase, out); + writeQueryResults(testCase, out, originalOrder()); } } - private static T writeQueryResults(QueryTestCase testCase, T out) throws IOException { + private static T writeQueryResults(QueryTestCase testCase, T out, Consumer> orderer) throws IOException { + List messages = new ArrayList<>(); try (InputStream in = Files.newInputStream(testCase.source)) { try (NdjsonToMessageIterable envelopes = new NdjsonToMessageIterable(in, deserializer)) { - Repository repository = createRepository(); - for (Envelope envelope : envelopes) { - repository.update(envelope); - } - Query query = new Query(repository); - Object queryResults = testCase.query.apply(query); - DefaultPrettyPrinter prettyPrinter = new DefaultPrettyPrinter() - .withArrayIndenter(SYSTEM_LINEFEED_INSTANCE); - OBJECT_MAPPER.writer(prettyPrinter).writeValue(out, queryResults); + envelopes.forEach(messages::add); } } + orderer.accept(messages); + + Repository repository = createRepository(); + for (Envelope envelope : messages) { + repository.update(envelope); + } + Query query = new Query(repository); + Object queryResults = testCase.query.apply(query); + DefaultPrettyPrinter prettyPrinter = new DefaultPrettyPrinter() + .withArrayIndenter(SYSTEM_LINEFEED_INSTANCE); + OBJECT_MAPPER.writer(prettyPrinter).writeValue(out, queryResults); return out; } @@ -132,7 +149,15 @@ static Map> createQueries() { queries.put("findAllPickles", (query) -> query.findAllPickles().size()); queries.put("findAllPickleSteps", (query) -> query.findAllPickleSteps().size()); queries.put("findAllTestCaseStarted", (query) -> query.findAllTestCaseStarted().size()); + queries.put("findAllTestCaseStartedInCanonicalOrder", (query) -> query.findAllTestCaseStartedInCanonicalOrder() + .stream() + .map(TestCaseStarted::getId) + .collect(toList())); queries.put("findAllTestCaseFinished", (query) -> query.findAllTestCaseFinished().size()); + queries.put("findAllTestCaseFinishedInCanonicalOrder", (query) -> query.findAllTestCaseFinishedInCanonicalOrder() + .stream() + .map(TestCaseFinished::getTestCaseStartedId) + .collect(toList())); queries.put("findAllTestRunHookStarted", (query) -> query.findAllTestRunHookStarted().size()); queries.put("findAllTestRunHookFinished", (query) -> query.findAllTestRunHookFinished().size()); queries.put("findAllTestSteps", (query) -> query.findAllTestSteps().size()); diff --git a/testdata/src/attachments.findAllTestCaseFinishedInCanonicalOrder.results.json b/testdata/src/attachments.findAllTestCaseFinishedInCanonicalOrder.results.json new file mode 100644 index 00000000..59706629 --- /dev/null +++ b/testdata/src/attachments.findAllTestCaseFinishedInCanonicalOrder.results.json @@ -0,0 +1,9 @@ +[ + "50", + "51", + "52", + "53", + "54", + "55", + "56" +] \ No newline at end of file diff --git a/testdata/src/attachments.findAllTestCaseStartedInCanonicalOrder.results.json b/testdata/src/attachments.findAllTestCaseStartedInCanonicalOrder.results.json new file mode 100644 index 00000000..59706629 --- /dev/null +++ b/testdata/src/attachments.findAllTestCaseStartedInCanonicalOrder.results.json @@ -0,0 +1,9 @@ +[ + "50", + "51", + "52", + "53", + "54", + "55", + "56" +] \ No newline at end of file diff --git a/testdata/src/empty.findAllTestCaseFinishedInCanonicalOrder.results.json b/testdata/src/empty.findAllTestCaseFinishedInCanonicalOrder.results.json new file mode 100644 index 00000000..a01887c5 --- /dev/null +++ b/testdata/src/empty.findAllTestCaseFinishedInCanonicalOrder.results.json @@ -0,0 +1,3 @@ +[ + "4" +] \ No newline at end of file diff --git a/testdata/src/empty.findAllTestCaseStartedInCanonicalOrder.results.json b/testdata/src/empty.findAllTestCaseStartedInCanonicalOrder.results.json new file mode 100644 index 00000000..a01887c5 --- /dev/null +++ b/testdata/src/empty.findAllTestCaseStartedInCanonicalOrder.results.json @@ -0,0 +1,3 @@ +[ + "4" +] \ No newline at end of file diff --git a/testdata/src/examples-tables.findAllTestCaseFinishedInCanonicalOrder.results.json b/testdata/src/examples-tables.findAllTestCaseFinishedInCanonicalOrder.results.json new file mode 100644 index 00000000..ed923faf --- /dev/null +++ b/testdata/src/examples-tables.findAllTestCaseFinishedInCanonicalOrder.results.json @@ -0,0 +1,11 @@ +[ + "106", + "107", + "108", + "109", + "110", + "111", + "112", + "113", + "114" +] \ No newline at end of file diff --git a/testdata/src/examples-tables.findAllTestCaseStartedInCanonicalOrder.results.json b/testdata/src/examples-tables.findAllTestCaseStartedInCanonicalOrder.results.json new file mode 100644 index 00000000..ed923faf --- /dev/null +++ b/testdata/src/examples-tables.findAllTestCaseStartedInCanonicalOrder.results.json @@ -0,0 +1,11 @@ +[ + "106", + "107", + "108", + "109", + "110", + "111", + "112", + "113", + "114" +] \ No newline at end of file diff --git a/testdata/src/global-hooks-attachments.findAllTestCaseFinishedInCanonicalOrder.results.json b/testdata/src/global-hooks-attachments.findAllTestCaseFinishedInCanonicalOrder.results.json new file mode 100644 index 00000000..575b77f4 --- /dev/null +++ b/testdata/src/global-hooks-attachments.findAllTestCaseFinishedInCanonicalOrder.results.json @@ -0,0 +1,3 @@ +[ + "11" +] \ No newline at end of file diff --git a/testdata/src/global-hooks-attachments.findAllTestCaseStartedInCanonicalOrder.results.json b/testdata/src/global-hooks-attachments.findAllTestCaseStartedInCanonicalOrder.results.json new file mode 100644 index 00000000..575b77f4 --- /dev/null +++ b/testdata/src/global-hooks-attachments.findAllTestCaseStartedInCanonicalOrder.results.json @@ -0,0 +1,3 @@ +[ + "11" +] \ No newline at end of file diff --git a/testdata/src/global-hooks.findAllTestCaseFinishedInCanonicalOrder.results.json b/testdata/src/global-hooks.findAllTestCaseFinishedInCanonicalOrder.results.json new file mode 100644 index 00000000..1d36096d --- /dev/null +++ b/testdata/src/global-hooks.findAllTestCaseFinishedInCanonicalOrder.results.json @@ -0,0 +1,4 @@ +[ + "21", + "22" +] \ No newline at end of file diff --git a/testdata/src/global-hooks.findAllTestCaseStartedInCanonicalOrder.results.json b/testdata/src/global-hooks.findAllTestCaseStartedInCanonicalOrder.results.json new file mode 100644 index 00000000..1d36096d --- /dev/null +++ b/testdata/src/global-hooks.findAllTestCaseStartedInCanonicalOrder.results.json @@ -0,0 +1,4 @@ +[ + "21", + "22" +] \ No newline at end of file diff --git a/testdata/src/hooks.findAllTestCaseFinishedInCanonicalOrder.results.json b/testdata/src/hooks.findAllTestCaseFinishedInCanonicalOrder.results.json new file mode 100644 index 00000000..abeeda9f --- /dev/null +++ b/testdata/src/hooks.findAllTestCaseFinishedInCanonicalOrder.results.json @@ -0,0 +1,5 @@ +[ + "29", + "30", + "31" +] \ No newline at end of file diff --git a/testdata/src/hooks.findAllTestCaseStartedInCanonicalOrder.results.json b/testdata/src/hooks.findAllTestCaseStartedInCanonicalOrder.results.json new file mode 100644 index 00000000..abeeda9f --- /dev/null +++ b/testdata/src/hooks.findAllTestCaseStartedInCanonicalOrder.results.json @@ -0,0 +1,5 @@ +[ + "29", + "30", + "31" +] \ No newline at end of file diff --git a/testdata/src/minimal.findAllTestCaseFinishedInCanonicalOrder.results.json b/testdata/src/minimal.findAllTestCaseFinishedInCanonicalOrder.results.json new file mode 100644 index 00000000..4aebf7e7 --- /dev/null +++ b/testdata/src/minimal.findAllTestCaseFinishedInCanonicalOrder.results.json @@ -0,0 +1,3 @@ +[ + "8" +] \ No newline at end of file diff --git a/testdata/src/minimal.findAllTestCaseStartedInCanonicalOrder.results.json b/testdata/src/minimal.findAllTestCaseStartedInCanonicalOrder.results.json new file mode 100644 index 00000000..4aebf7e7 --- /dev/null +++ b/testdata/src/minimal.findAllTestCaseStartedInCanonicalOrder.results.json @@ -0,0 +1,3 @@ +[ + "8" +] \ No newline at end of file diff --git a/testdata/src/rules.findAllTestCaseFinishedInCanonicalOrder.results.json b/testdata/src/rules.findAllTestCaseFinishedInCanonicalOrder.results.json new file mode 100644 index 00000000..ec34e4ac --- /dev/null +++ b/testdata/src/rules.findAllTestCaseFinishedInCanonicalOrder.results.json @@ -0,0 +1,5 @@ +[ + "55", + "56", + "57" +] \ No newline at end of file diff --git a/testdata/src/rules.findAllTestCaseStartedInCanonicalOrder.results.json b/testdata/src/rules.findAllTestCaseStartedInCanonicalOrder.results.json new file mode 100644 index 00000000..ec34e4ac --- /dev/null +++ b/testdata/src/rules.findAllTestCaseStartedInCanonicalOrder.results.json @@ -0,0 +1,5 @@ +[ + "55", + "56", + "57" +] \ No newline at end of file