From 2cae7713249a6d48539d6394c46ae2cc36c134b3 Mon Sep 17 00:00:00 2001
From: Nenad <nenad@better.giving>
Date: Wed, 17 Jul 2024 10:14:39 +0200
Subject: [PATCH 01/23] Update readme

---
 .github/workflows/ci.yml   |  2 +-
 README.md                  | 23 ++---------------------
 bin/run-in-docker.sh       |  4 ++--
 bin/run-tests-in-docker.sh |  4 ++--
 4 files changed, 7 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8214057..acb9a3e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -27,7 +27,7 @@ jobs:
           context: .
           push: false
           load: true
-          tags: exercism/test-runner
+          tags: exercism/cairo-test-runner
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
diff --git a/README.md b/README.md
index 72dbb03..ba4f968 100644
--- a/README.md
+++ b/README.md
@@ -1,25 +1,6 @@
-# Exercism Test Runner Template
+# Exercism Cairo Test Runner
 
-This repository is a [template repository](https://help.github.com/en/github/creating-cloning-and-archiving-repositories/creating-a-template-repository) for creating [test runners][test-runners] for [Exercism][exercism] tracks.
-
-## Using the Test Runner Template
-
-1. Ensure that your track has not already implemented a test runner. If there is, there will be a `https://github.com/exercism/<track>-test-runner` repository (i.e. if your track's slug is `python`, the test runner repo would be `https://github.com/exercism/python-test-runner`)
-2. Follow [GitHub's documentation](https://help.github.com/en/github/creating-cloning-and-archiving-repositories/creating-a-repository-from-a-template) for creating a repository from a template repository
-   - Name your new repository based on your language track's slug (i.e. if your track is for Python, your test runner repo name is `python-test-runner`)
-3. Remove this [Exercism Test Runner Template](#exercism-test-runner-template) section from the `README.md` file
-4. Replace `TRACK_NAME_HERE` with your track's name in the `README.md` file
-5. Replace any occurances of `exercism/test-runner` with `exercism/<track>-test-runner` (e.g. `exercism/python-test-runner`)
-6. Build the test runner, conforming to the [Test Runner interface specification](https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md).
-   - Update the files to match your track's needs. At the very least, you'll need to update `bin/run.sh`, `Dockerfile` and the test solutions in the `tests` directory
-   - Tip: look for `TODO:` comments to point you towards code that need updating
-   - Tip: look for `OPTIONAL:` comments to point you towards code that _could_ be useful
-
-Once you're happy with your test runner, [open an issue on the exercism/exercism](https://github.com/exercism/exercism/issues/new?assignees=&labels=&template=new-test-runner.md&title=%5BNew+Test+Runner%5D+) to request an official test runner repository for your track.
-
-# Exercism TRACK_NAME_HERE Test Runner
-
-The Docker image to automatically run tests on TRACK_NAME_HERE solutions submitted to [Exercism].
+The Docker image to automatically run tests on Cairo solutions submitted to [Exercism].
 
 ## Run the test runner
 
diff --git a/bin/run-in-docker.sh b/bin/run-in-docker.sh
index 79a3ece..9554593 100755
--- a/bin/run-in-docker.sh
+++ b/bin/run-in-docker.sh
@@ -33,7 +33,7 @@ output_dir=$(realpath "${3%/}")
 mkdir -p "${output_dir}"
 
 # Build the Docker image
-docker build --rm -t exercism/test-runner .
+docker build --rm -t exercism/cairo-test-runner .
 
 # Run the Docker image using the settings mimicking the production environment
 docker run \
@@ -43,4 +43,4 @@ docker run \
     --mount type=bind,src="${solution_dir}",dst=/solution \
     --mount type=bind,src="${output_dir}",dst=/output \
     --mount type=tmpfs,dst=/tmp \
-    exercism/test-runner "${slug}" /solution /output 
+    exercism/cairo-test-runner "${slug}" /solution /output 
diff --git a/bin/run-tests-in-docker.sh b/bin/run-tests-in-docker.sh
index 8078a67..f590e8b 100755
--- a/bin/run-tests-in-docker.sh
+++ b/bin/run-tests-in-docker.sh
@@ -16,7 +16,7 @@
 set -e
 
 # Build the Docker image
-docker build --rm -t exercism/test-runner .
+docker build --rm -t exercism/cairo-test-runner .
 
 # Run the Docker image using the settings mimicking the production environment
 docker run \
@@ -28,4 +28,4 @@ docker run \
     --volume "${PWD}/bin/run-tests.sh:/opt/test-runner/bin/run-tests.sh" \
     --workdir /opt/test-runner \
     --entrypoint /opt/test-runner/bin/run-tests.sh \
-    exercism/test-runner
+    exercism/cairo-test-runner

From ab21759bdfe75015cbc67e32ab251ea0e26dfd4f Mon Sep 17 00:00:00 2001
From: Nenad <nenad@better.giving>
Date: Wed, 17 Jul 2024 19:34:40 +0200
Subject: [PATCH 02/23] Implement run(-test).sh files

---
 bin/run-tests.sh | 26 ++++++++++++++++----------
 bin/run.sh       | 43 ++++++++++++++++++++++++-------------------
 2 files changed, 40 insertions(+), 29 deletions(-)

diff --git a/bin/run-tests.sh b/bin/run-tests.sh
index 2bb47b8..ed9ae9f 100755
--- a/bin/run-tests.sh
+++ b/bin/run-tests.sh
@@ -12,26 +12,32 @@
 # ./bin/run-tests.sh
 
 exit_code=0
+# Copy the tests dir to a temp dir, because in the container the user lacks
+# permissions to write to the tests dir.
+tmp_dir='/tmp/exercism-cairo-test-runner'
+rm -rf "${tmp_dir}"
+mkdir -p "${tmp_dir}"
+cp -r tests/* "${tmp_dir}"
 
 # Iterate over all test directories
-for test_dir in tests/*; do
+for test_dir in "${tmp_dir}"/*; do
     test_dir_name=$(basename "${test_dir}")
     test_dir_path=$(realpath "${test_dir}")
+    results_file_path="${test_dir_path}/results.json"
+    expected_results_file_path="${test_dir_path}/expected_results.json"
 
     bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}"
 
-    # OPTIONAL: Normalize the results file
-    # If the results.json file contains information that changes between 
-    # different test runs (e.g. timing information or paths), you should normalize
-    # the results file to allow the diff comparison below to work as expected
+    for file in "$results_file_path" "$expected_results_file_path"; do
+       jq 'if (.tests != null) then .tests |= sort_by(.name) else . end' "${file}" > "${file}.tmp"
+    done
 
-    file="results.json"
-    expected_file="expected_${file}"
-    echo "${test_dir_name}: comparing ${file} to ${expected_file}"
+    echo "${test_dir_name}: comparing results.json to expected_results.json"
 
-    if ! diff "${test_dir_path}/${file}" "${test_dir_path}/${expected_file}"; then
+    if ! diff "${results_file_path}.tmp" "${expected_results_file_path}.tmp"; then
         exit_code=1
     fi
 done
 
-exit ${exit_code}
+rm -rf "${tmp_dir}"
+exit ${exit_code}
\ No newline at end of file
diff --git a/bin/run.sh b/bin/run.sh
index f1e8b78..d1a6643 100755
--- a/bin/run.sh
+++ b/bin/run.sh
@@ -31,30 +31,35 @@ mkdir -p "${output_dir}"
 
 echo "${slug}: testing..."
 
+start_dir="$(pwd)"
+cd "${solution_dir}" || exit 1
+
 # Run the tests for the provided implementation file and redirect stdout and
 # stderr to capture it
-test_output=$(false)
-# TODO: substitute "false" with the actual command to run the test:
-# test_output=$(command_to_run_tests 2>&1)
+test_output=$(scarb cairo-test --include-ignored 2>&1)
+exit_code=$?
+
+cd "${start_dir}" || exit 1
 
 # Write the results.json file based on the exit code of the command that was 
 # just executed that tested the implementation file
-if [ $? -eq 0 ]; then
-    jq -n '{version: 1, status: "pass"}' > ${results_file}
+if [ ${exit_code} -eq 0 ]; then
+    jq -n '{version: 1, status: "pass"}' > "${results_file}"
 else
-    # OPTIONAL: Sanitize the output
-    # In some cases, the test output might be overly verbose, in which case stripping
-    # the unneeded information can be very helpful to the student
-    # sanitized_test_output=$(printf "${test_output}" | sed -n '/Test results:/,$p')
-
-    # OPTIONAL: Manually add colors to the output to help scanning the output for errors
-    # If the test output does not contain colors to help identify failing (or passing)
-    # tests, it can be helpful to manually add colors to the output
-    # colorized_test_output=$(echo "${test_output}" \
-    #      | GREP_COLOR='01;31' grep --color=always -E -e '^(ERROR:.*|.*failed)$|$' \
-    #      | GREP_COLOR='01;32' grep --color=always -E -e '^.*passed$|$')
-
-    jq -n --arg output "${test_output}" '{version: 1, status: "fail", message: $output}' > ${results_file}
+    # Sanitize the output
+    # TODO: enable if needed to escape quotes: test_output_inline=$(printf '%s' "${test_output}" | sed -r 's/\"/\\"/g')
+    test_output_inline=$(printf '%s' "${test_output}")
+
+    # Try to distinguish between failing tests and errors
+    if echo "${test_output_inline}" | grep "error:"; then
+        status="error"
+        sanitized_test_output=$(echo "$test_output_inline" | awk '/Compiling/{y=1;next}y' | sed -n -e '/error: could not compile/q;p')
+    else
+        status="fail"
+        sanitized_test_output=$(echo "$test_output_inline" | awk '/failures:/{y=1;next}y' | sed -n -e '/Error: test result/q;p' | sed -r 's/   //g')
+    fi    
+
+    jq -n --arg output "${sanitized_test_output}" --arg status "${status}" '{version: 1, status: $status, message: $output}' > "${results_file}"
 fi
 
-echo "${slug}: done"
+echo "${slug}: done"
\ No newline at end of file

From e5489635d67b03c1cfef9101dfe2ec0719b529fa Mon Sep 17 00:00:00 2001
From: Nenad <nenad@better.giving>
Date: Wed, 17 Jul 2024 19:48:05 +0200
Subject: [PATCH 03/23] Add Dockerfile

---
 Dockerfile | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index bd3578c..169eb48 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,8 +1,17 @@
 FROM alpine:3.18
 
 # install packages required to run the tests
-RUN apk add --no-cache jq coreutils
+RUN apk add --no-cache jq curl git
+
+# install asdf
+RUN git clone https://github.com/asdf-vm/asdf.git ~/.asdf --branch v0.14.0
+
+RUN . "$HOME/.asdf/asdf.sh"
+
+RUN asdf plugin add scarb && asdf install scarb 2.6.5 && asdf global scarb 2.6.5
+
+COPY . /opt/test-runner
 
 WORKDIR /opt/test-runner
-COPY . .
+
 ENTRYPOINT ["/opt/test-runner/bin/run.sh"]

From 66e0defaf6c3cbbcbc56f0c95b9101c3af646a29 Mon Sep 17 00:00:00 2001
From: Nenad <nenad.misic@nethermind.io>
Date: Thu, 18 Jul 2024 10:16:54 +0200
Subject: [PATCH 04/23] Remove asdf installation

---
 Dockerfile | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 169eb48..0b3e479 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,12 +3,8 @@ FROM alpine:3.18
 # install packages required to run the tests
 RUN apk add --no-cache jq curl git
 
-# install asdf
-RUN git clone https://github.com/asdf-vm/asdf.git ~/.asdf --branch v0.14.0
-
-RUN . "$HOME/.asdf/asdf.sh"
-
-RUN asdf plugin add scarb && asdf install scarb 2.6.5 && asdf global scarb 2.6.5
+# install Scarb
+RUN curl --proto '=https' --tlsv1.2 -sSf https://docs.swmansion.com/scarb/install.sh | sh -s -- -v 2.6.5
 
 COPY . /opt/test-runner
 

From 0537c677a79549b6dbfb227ebee68c6830f6b53e Mon Sep 17 00:00:00 2001
From: Nenad <nenad.misic@nethermind.io>
Date: Thu, 18 Jul 2024 10:52:21 +0200
Subject: [PATCH 05/23] Sort test results in run.sh

---
 bin/run.sh | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/bin/run.sh b/bin/run.sh
index d1a6643..5c19f69 100755
--- a/bin/run.sh
+++ b/bin/run.sh
@@ -41,10 +41,10 @@ exit_code=$?
 
 cd "${start_dir}" || exit 1
 
-# Write the results.json file based on the exit code of the command that was 
+# Write the results.json file based on the exit code of the command that was
 # just executed that tested the implementation file
 if [ ${exit_code} -eq 0 ]; then
-    jq -n '{version: 1, status: "pass"}' > "${results_file}"
+    jq -n '{version: 1, status: "pass"}' >"${results_file}"
 else
     # Sanitize the output
     # TODO: enable if needed to escape quotes: test_output_inline=$(printf '%s' "${test_output}" | sed -r 's/\"/\\"/g')
@@ -57,9 +57,15 @@ else
     else
         status="fail"
         sanitized_test_output=$(echo "$test_output_inline" | awk '/failures:/{y=1;next}y' | sed -n -e '/Error: test result/q;p' | sed -r 's/   //g')
-    fi    
+    fi
 
-    jq -n --arg output "${sanitized_test_output}" --arg status "${status}" '{version: 1, status: $status, message: $output}' > "${results_file}"
+    tmp_file=$(mktemp -p .)
+    trap 'rm $tmp_file' EXIT INT TERM
+
+    printf "$sanitized_test_output" >"$tmp_file"
+    sorted_output=$(sort "$tmp_file")
+
+    jq -n --arg output "${sorted_output}" --arg status "${status}" '{version: 1, status: $status, message: $output}' >"${results_file}"
 fi
 
-echo "${slug}: done"
\ No newline at end of file
+echo "${slug}: done"

From 2647013d313fda564d4e40df3c578cb06d157342 Mon Sep 17 00:00:00 2001
From: Nenad <nenad.misic@nethermind.io>
Date: Thu, 18 Jul 2024 10:55:54 +0200
Subject: [PATCH 06/23] remove the need to sort result contents in run-tests.sh

---
 bin/run-tests.sh | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/bin/run-tests.sh b/bin/run-tests.sh
index ed9ae9f..b2df9c6 100755
--- a/bin/run-tests.sh
+++ b/bin/run-tests.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env sh
 
 # Synopsis:
-# Test the test runner by running it against a predefined set of solutions 
+# Test the test runner by running it against a predefined set of solutions
 # with an expected output.
 
 # Output:
@@ -28,10 +28,6 @@ for test_dir in "${tmp_dir}"/*; do
 
     bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}"
 
-    for file in "$results_file_path" "$expected_results_file_path"; do
-       jq 'if (.tests != null) then .tests |= sort_by(.name) else . end' "${file}" > "${file}.tmp"
-    done
-
     echo "${test_dir_name}: comparing results.json to expected_results.json"
 
     if ! diff "${results_file_path}.tmp" "${expected_results_file_path}.tmp"; then
@@ -40,4 +36,4 @@ for test_dir in "${tmp_dir}"/*; do
 done
 
 rm -rf "${tmp_dir}"
-exit ${exit_code}
\ No newline at end of file
+exit ${exit_code}

From dfd3dfbbb9d64766bcf1401794c9b4084545301f Mon Sep 17 00:00:00 2001
From: Nenad <nenad.misic@nethermind.io>
Date: Thu, 18 Jul 2024 11:30:13 +0200
Subject: [PATCH 07/23] Add tests/**/Scarb.lock & target to gitignore

---
 .gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitignore b/.gitignore
index c6f4076..3f4403b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,3 @@
 /tests/**/*/results.json
+/tests/**/Scarb.lock
+/tests/**/target

From 5e473669ba7ee25cb6ea593fcb99d4e2a04afa18 Mon Sep 17 00:00:00 2001
From: Nenad <nenad.misic@nethermind.io>
Date: Thu, 18 Jul 2024 11:32:41 +0200
Subject: [PATCH 08/23] Add all-fail tests

---
 tests/all-fail/Scarb.toml            |  4 +++
 tests/all-fail/expected_results.json |  2 +-
 tests/all-fail/src/lib.cairo         |  6 ++++
 tests/all-fail/src/tests.cairo       | 54 ++++++++++++++++++++++++++++
 4 files changed, 65 insertions(+), 1 deletion(-)
 create mode 100644 tests/all-fail/Scarb.toml
 create mode 100644 tests/all-fail/src/lib.cairo
 create mode 100644 tests/all-fail/src/tests.cairo

diff --git a/tests/all-fail/Scarb.toml b/tests/all-fail/Scarb.toml
new file mode 100644
index 0000000..0f2f16b
--- /dev/null
+++ b/tests/all-fail/Scarb.toml
@@ -0,0 +1,4 @@
+[package]
+name = "leap"
+version = "0.1.0"
+edition = "2023_11"
diff --git a/tests/all-fail/expected_results.json b/tests/all-fail/expected_results.json
index 9ef8b6f..39bcf2c 100644
--- a/tests/all-fail/expected_results.json
+++ b/tests/all-fail/expected_results.json
@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "fail",
-  "message": "TODO: replace with correct output"
+  "message": "leap::tests::year_divisible_by_100_but_not_by_3_is_still_not_a_leap_year - Panicked with \"assertion failed: `!is_leap_year(1900)`.\".\nleap::tests::year_divisible_by_100_not_divisible_by_400_in_common_year - Panicked with \"assertion failed: `!is_leap_year(2100)`.\".\nleap::tests::year_divisible_by_200_not_divisible_by_400_in_common_year - Panicked with \"assertion failed: `!is_leap_year(1800)`.\".\nleap::tests::year_divisible_by_2_not_divisible_by_4_in_common_year - Panicked with \"assertion failed: `!is_leap_year(1970)`.\".\nleap::tests::year_divisible_by_400_but_not_by_125_is_still_a_leap_year - Panicked with \"assertion failed: `is_leap_year(2400)`.\".\nleap::tests::year_divisible_by_400_is_leap_year - Panicked with \"assertion failed: `is_leap_year(2000)`.\".\nleap::tests::year_divisible_by_4_and_5_is_still_a_leap_year - Panicked with \"assertion failed: `is_leap_year(1960)`.\".\nleap::tests::year_divisible_by_4_not_divisible_by_100_in_leap_year - Panicked with \"assertion failed: `is_leap_year(1996)`.\".\nleap::tests::year_not_divisible_by_4_in_common_year - Panicked with \"assertion failed: `!is_leap_year(2015)`.\"."
 }
diff --git a/tests/all-fail/src/lib.cairo b/tests/all-fail/src/lib.cairo
new file mode 100644
index 0000000..72df0c9
--- /dev/null
+++ b/tests/all-fail/src/lib.cairo
@@ -0,0 +1,6 @@
+fn is_leap_year(year: u64) -> bool {
+    !(year % 4 == 0 && (year % 100 != 0 || year % 400 == 0))
+}
+
+#[cfg(test)]
+mod tests;
diff --git a/tests/all-fail/src/tests.cairo b/tests/all-fail/src/tests.cairo
new file mode 100644
index 0000000..30be29d
--- /dev/null
+++ b/tests/all-fail/src/tests.cairo
@@ -0,0 +1,54 @@
+use leap::is_leap_year;
+
+#[test]
+fn year_not_divisible_by_4_in_common_year() {
+    assert!(!is_leap_year(2015));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_2_not_divisible_by_4_in_common_year() {
+    assert!(!is_leap_year(1970));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_4_not_divisible_by_100_in_leap_year() {
+    assert!(is_leap_year(1996));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_4_and_5_is_still_a_leap_year() {
+    assert!(is_leap_year(1960));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_100_not_divisible_by_400_in_common_year() {
+    assert!(!is_leap_year(2100));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_100_but_not_by_3_is_still_not_a_leap_year() {
+    assert!(!is_leap_year(1900));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_400_is_leap_year() {
+    assert!(is_leap_year(2000));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_400_but_not_by_125_is_still_a_leap_year() {
+    assert!(is_leap_year(2400));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_200_not_divisible_by_400_in_common_year() {
+    assert!(!is_leap_year(1800));
+}

From fcaeeb092da028eb58c7069474d3a03525492694 Mon Sep 17 00:00:00 2001
From: Nenad <nenad.misic@nethermind.io>
Date: Thu, 18 Jul 2024 14:58:52 +0200
Subject: [PATCH 09/23] Update all-file + empty-file not yet working

---
 bin/run.sh                                    |  2 +-
 tests/all-fail/src/lib.cairo                  |  5 +-
 .../{src/tests.cairo => tests/leap.cairo}     |  0
 tests/empty-file/Scarb.toml                   |  4 ++
 tests/empty-file/expected_results.json        |  2 +-
 tests/empty-file/src/lib.cairo                |  1 +
 tests/empty-file/tests/leap.cairo             | 54 +++++++++++++++++++
 7 files changed, 62 insertions(+), 6 deletions(-)
 rename tests/all-fail/{src/tests.cairo => tests/leap.cairo} (100%)
 create mode 100644 tests/empty-file/Scarb.toml
 create mode 100644 tests/empty-file/src/lib.cairo
 create mode 100644 tests/empty-file/tests/leap.cairo

diff --git a/bin/run.sh b/bin/run.sh
index 5c19f69..b4071bc 100755
--- a/bin/run.sh
+++ b/bin/run.sh
@@ -62,7 +62,7 @@ else
     tmp_file=$(mktemp -p .)
     trap 'rm $tmp_file' EXIT INT TERM
 
-    printf "$sanitized_test_output" >"$tmp_file"
+    printf '%s' "$sanitized_test_output" >"$tmp_file"
     sorted_output=$(sort "$tmp_file")
 
     jq -n --arg output "${sorted_output}" --arg status "${status}" '{version: 1, status: $status, message: $output}' >"${results_file}"
diff --git a/tests/all-fail/src/lib.cairo b/tests/all-fail/src/lib.cairo
index 72df0c9..27be966 100644
--- a/tests/all-fail/src/lib.cairo
+++ b/tests/all-fail/src/lib.cairo
@@ -1,6 +1,3 @@
-fn is_leap_year(year: u64) -> bool {
+pub fn is_leap_year(year: u64) -> bool {
     !(year % 4 == 0 && (year % 100 != 0 || year % 400 == 0))
 }
-
-#[cfg(test)]
-mod tests;
diff --git a/tests/all-fail/src/tests.cairo b/tests/all-fail/tests/leap.cairo
similarity index 100%
rename from tests/all-fail/src/tests.cairo
rename to tests/all-fail/tests/leap.cairo
diff --git a/tests/empty-file/Scarb.toml b/tests/empty-file/Scarb.toml
new file mode 100644
index 0000000..0f2f16b
--- /dev/null
+++ b/tests/empty-file/Scarb.toml
@@ -0,0 +1,4 @@
+[package]
+name = "leap"
+version = "0.1.0"
+edition = "2023_11"
diff --git a/tests/empty-file/expected_results.json b/tests/empty-file/expected_results.json
index 9ef8b6f..39bcf2c 100644
--- a/tests/empty-file/expected_results.json
+++ b/tests/empty-file/expected_results.json
@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "fail",
-  "message": "TODO: replace with correct output"
+  "message": "leap::tests::year_divisible_by_100_but_not_by_3_is_still_not_a_leap_year - Panicked with \"assertion failed: `!is_leap_year(1900)`.\".\nleap::tests::year_divisible_by_100_not_divisible_by_400_in_common_year - Panicked with \"assertion failed: `!is_leap_year(2100)`.\".\nleap::tests::year_divisible_by_200_not_divisible_by_400_in_common_year - Panicked with \"assertion failed: `!is_leap_year(1800)`.\".\nleap::tests::year_divisible_by_2_not_divisible_by_4_in_common_year - Panicked with \"assertion failed: `!is_leap_year(1970)`.\".\nleap::tests::year_divisible_by_400_but_not_by_125_is_still_a_leap_year - Panicked with \"assertion failed: `is_leap_year(2400)`.\".\nleap::tests::year_divisible_by_400_is_leap_year - Panicked with \"assertion failed: `is_leap_year(2000)`.\".\nleap::tests::year_divisible_by_4_and_5_is_still_a_leap_year - Panicked with \"assertion failed: `is_leap_year(1960)`.\".\nleap::tests::year_divisible_by_4_not_divisible_by_100_in_leap_year - Panicked with \"assertion failed: `is_leap_year(1996)`.\".\nleap::tests::year_not_divisible_by_4_in_common_year - Panicked with \"assertion failed: `!is_leap_year(2015)`.\"."
 }
diff --git a/tests/empty-file/src/lib.cairo b/tests/empty-file/src/lib.cairo
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/tests/empty-file/src/lib.cairo
@@ -0,0 +1 @@
+
diff --git a/tests/empty-file/tests/leap.cairo b/tests/empty-file/tests/leap.cairo
new file mode 100644
index 0000000..30be29d
--- /dev/null
+++ b/tests/empty-file/tests/leap.cairo
@@ -0,0 +1,54 @@
+use leap::is_leap_year;
+
+#[test]
+fn year_not_divisible_by_4_in_common_year() {
+    assert!(!is_leap_year(2015));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_2_not_divisible_by_4_in_common_year() {
+    assert!(!is_leap_year(1970));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_4_not_divisible_by_100_in_leap_year() {
+    assert!(is_leap_year(1996));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_4_and_5_is_still_a_leap_year() {
+    assert!(is_leap_year(1960));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_100_not_divisible_by_400_in_common_year() {
+    assert!(!is_leap_year(2100));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_100_but_not_by_3_is_still_not_a_leap_year() {
+    assert!(!is_leap_year(1900));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_400_is_leap_year() {
+    assert!(is_leap_year(2000));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_400_but_not_by_125_is_still_a_leap_year() {
+    assert!(is_leap_year(2400));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_200_not_divisible_by_400_in_common_year() {
+    assert!(!is_leap_year(1800));
+}

From 3472a667aeab675ccece578a3ba0a8b7dae8f1ec Mon Sep 17 00:00:00 2001
From: Nenad <nenad@better.giving>
Date: Thu, 18 Jul 2024 17:26:13 +0200
Subject: [PATCH 10/23] Fix error case in run.sh

---
 bin/run.sh | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/bin/run.sh b/bin/run.sh
index b4071bc..fce81e8 100755
--- a/bin/run.sh
+++ b/bin/run.sh
@@ -47,13 +47,12 @@ if [ ${exit_code} -eq 0 ]; then
     jq -n '{version: 1, status: "pass"}' >"${results_file}"
 else
     # Sanitize the output
-    # TODO: enable if needed to escape quotes: test_output_inline=$(printf '%s' "${test_output}" | sed -r 's/\"/\\"/g')
-    test_output_inline=$(printf '%s' "${test_output}")
+    test_output_inline=$(printf '%s' "$test_output")
 
     # Try to distinguish between failing tests and errors
-    if echo "${test_output_inline}" | grep "error:"; then
+    if echo "$test_output_inline" | grep "error:"; then
         status="error"
-        sanitized_test_output=$(echo "$test_output_inline" | awk '/Compiling/{y=1;next}y' | sed -n -e '/error: could not compile/q;p')
+        sanitized_test_output=$(echo "$test_output_inline" | sed '/Compiling\|-->.*$/d' | sed -n -e '/error: could not compile/q;p')
     else
         status="fail"
         sanitized_test_output=$(echo "$test_output_inline" | awk '/failures:/{y=1;next}y' | sed -n -e '/Error: test result/q;p' | sed -r 's/   //g')

From 98efd8db2ef0ad940fce49707f9d4434f4551358 Mon Sep 17 00:00:00 2001
From: Nenad <nenad@better.giving>
Date: Thu, 18 Jul 2024 18:08:21 +0200
Subject: [PATCH 11/23] Move sorting to run-tests.sh

---
 bin/run-tests.sh                       | 7 +++++++
 bin/run.sh                             | 8 +-------
 tests/empty-file/expected_results.json | 4 ++--
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/bin/run-tests.sh b/bin/run-tests.sh
index b2df9c6..fcef9e7 100755
--- a/bin/run-tests.sh
+++ b/bin/run-tests.sh
@@ -28,6 +28,13 @@ for test_dir in "${tmp_dir}"/*; do
 
     bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}"
 
+    for file in "$results_file_path" "$expected_results_file_path"; do
+        # We sort both the '.message' values in results.json and expected_results.json files
+        tmp_file=$(mktemp -p .)
+        sorted_message=$(cat $file | jq -r '.message' >"$tmp_file" && sort "$tmp_file")
+        jq --arg msg "$sorted_message" '.message = $msg' "$file" >"$tmp_file" && mv "$tmp_file" "$file"
+    done
+
     echo "${test_dir_name}: comparing results.json to expected_results.json"
 
     if ! diff "${results_file_path}.tmp" "${expected_results_file_path}.tmp"; then
diff --git a/bin/run.sh b/bin/run.sh
index fce81e8..076b1af 100755
--- a/bin/run.sh
+++ b/bin/run.sh
@@ -58,13 +58,7 @@ else
         sanitized_test_output=$(echo "$test_output_inline" | awk '/failures:/{y=1;next}y' | sed -n -e '/Error: test result/q;p' | sed -r 's/   //g')
     fi
 
-    tmp_file=$(mktemp -p .)
-    trap 'rm $tmp_file' EXIT INT TERM
-
-    printf '%s' "$sanitized_test_output" >"$tmp_file"
-    sorted_output=$(sort "$tmp_file")
-
-    jq -n --arg output "${sorted_output}" --arg status "${status}" '{version: 1, status: $status, message: $output}' >"${results_file}"
+    jq -n --arg output "${sanitized_test_output}" --arg status "${status}" '{version: 1, status: $status, message: $output}' >"${results_file}"
 fi
 
 echo "${slug}: done"
diff --git a/tests/empty-file/expected_results.json b/tests/empty-file/expected_results.json
index 39bcf2c..22263a2 100644
--- a/tests/empty-file/expected_results.json
+++ b/tests/empty-file/expected_results.json
@@ -1,5 +1,5 @@
 {
   "version": 1,
-  "status": "fail",
-  "message": "leap::tests::year_divisible_by_100_but_not_by_3_is_still_not_a_leap_year - Panicked with \"assertion failed: `!is_leap_year(1900)`.\".\nleap::tests::year_divisible_by_100_not_divisible_by_400_in_common_year - Panicked with \"assertion failed: `!is_leap_year(2100)`.\".\nleap::tests::year_divisible_by_200_not_divisible_by_400_in_common_year - Panicked with \"assertion failed: `!is_leap_year(1800)`.\".\nleap::tests::year_divisible_by_2_not_divisible_by_4_in_common_year - Panicked with \"assertion failed: `!is_leap_year(1970)`.\".\nleap::tests::year_divisible_by_400_but_not_by_125_is_still_a_leap_year - Panicked with \"assertion failed: `is_leap_year(2400)`.\".\nleap::tests::year_divisible_by_400_is_leap_year - Panicked with \"assertion failed: `is_leap_year(2000)`.\".\nleap::tests::year_divisible_by_4_and_5_is_still_a_leap_year - Panicked with \"assertion failed: `is_leap_year(1960)`.\".\nleap::tests::year_divisible_by_4_not_divisible_by_100_in_leap_year - Panicked with \"assertion failed: `is_leap_year(1996)`.\".\nleap::tests::year_not_divisible_by_4_in_common_year - Panicked with \"assertion failed: `!is_leap_year(2015)`.\"."
+  "status": "error",
+  "message": "error: The name `is_leap_year` is defined multiple times.\npub fn is_leap_year(v: u128) -> bool {}\n       ^**********^\n\nerror: Unexpected return type. Expected: \"core::bool\", found: \"()\".\npub fn is_leap_year(v: u128) -> bool {}\n                                     ^^\n\nerror: Unexpected return type. Expected: \"core::bool\", found: \"()\".\npub fn is_leap_year(v: u128) -> bool {}\n                                     ^^"
 }

From bb3e170a50254a08a7f679a3308d0b1e38715b2e Mon Sep 17 00:00:00 2001
From: Nenad <nenad@better.giving>
Date: Thu, 18 Jul 2024 18:18:04 +0200
Subject: [PATCH 12/23] Fix comparison in run-tests.sh

---
 bin/run-tests.sh                     | 7 ++++++-
 tests/all-fail/expected_results.json | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/bin/run-tests.sh b/bin/run-tests.sh
index fcef9e7..9d22b5b 100755
--- a/bin/run-tests.sh
+++ b/bin/run-tests.sh
@@ -37,10 +37,15 @@ for test_dir in "${tmp_dir}"/*; do
 
     echo "${test_dir_name}: comparing results.json to expected_results.json"
 
-    if ! diff "${results_file_path}.tmp" "${expected_results_file_path}.tmp"; then
+    if ! diff "$results_file_path" "$expected_results_file_path"; then
         exit_code=1
     fi
 done
 
 rm -rf "${tmp_dir}"
+
+if [ ${exit_code} -eq 0 ]; then
+    echo "${test_dir_name}: done"
+fi
+
 exit ${exit_code}
diff --git a/tests/all-fail/expected_results.json b/tests/all-fail/expected_results.json
index 39bcf2c..ccb4e22 100644
--- a/tests/all-fail/expected_results.json
+++ b/tests/all-fail/expected_results.json
@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "fail",
-  "message": "leap::tests::year_divisible_by_100_but_not_by_3_is_still_not_a_leap_year - Panicked with \"assertion failed: `!is_leap_year(1900)`.\".\nleap::tests::year_divisible_by_100_not_divisible_by_400_in_common_year - Panicked with \"assertion failed: `!is_leap_year(2100)`.\".\nleap::tests::year_divisible_by_200_not_divisible_by_400_in_common_year - Panicked with \"assertion failed: `!is_leap_year(1800)`.\".\nleap::tests::year_divisible_by_2_not_divisible_by_4_in_common_year - Panicked with \"assertion failed: `!is_leap_year(1970)`.\".\nleap::tests::year_divisible_by_400_but_not_by_125_is_still_a_leap_year - Panicked with \"assertion failed: `is_leap_year(2400)`.\".\nleap::tests::year_divisible_by_400_is_leap_year - Panicked with \"assertion failed: `is_leap_year(2000)`.\".\nleap::tests::year_divisible_by_4_and_5_is_still_a_leap_year - Panicked with \"assertion failed: `is_leap_year(1960)`.\".\nleap::tests::year_divisible_by_4_not_divisible_by_100_in_leap_year - Panicked with \"assertion failed: `is_leap_year(1996)`.\".\nleap::tests::year_not_divisible_by_4_in_common_year - Panicked with \"assertion failed: `!is_leap_year(2015)`.\"."
+  "message": "leap_leap::leap::year_not_divisible_by_4_in_common_year - Panicked with \"assertion failed: `!is_leap_year(2015)`.\".\nleap_leap::leap::year_divisible_by_2_not_divisible_by_4_in_common_year - Panicked with \"assertion failed: `!is_leap_year(1970)`.\".\nleap_leap::leap::year_divisible_by_4_not_divisible_by_100_in_leap_year - Panicked with \"assertion failed: `is_leap_year(1996)`.\".\nleap_leap::leap::year_divisible_by_100_not_divisible_by_400_in_common_year - Panicked with \"assertion failed: `!is_leap_year(2100)`.\".\nleap_leap::leap::year_divisible_by_200_not_divisible_by_400_in_common_year - Panicked with \"assertion failed: `!is_leap_year(1800)`.\".\nleap_leap::leap::year_divisible_by_400_but_not_by_125_is_still_a_leap_year - Panicked with \"assertion failed: `is_leap_year(2400)`.\".\nleap_leap::leap::year_divisible_by_400_is_leap_year - Panicked with \"assertion failed: `is_leap_year(2000)`.\".\nleap_leap::leap::year_divisible_by_4_and_5_is_still_a_leap_year - Panicked with \"assertion failed: `is_leap_year(1960)`.\".\nleap_leap::leap::year_divisible_by_100_but_not_by_3_is_still_not_a_leap_year - Panicked with \"assertion failed: `!is_leap_year(1900)`.\"."
 }

From b4534eee07912ea39af12d06d4c17003da7c4898 Mon Sep 17 00:00:00 2001
From: Nenad <nenad@better.giving>
Date: Thu, 18 Jul 2024 18:18:52 +0200
Subject: [PATCH 13/23] Regenerate empty-file's expected_results.json

---
 tests/empty-file/expected_results.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/empty-file/expected_results.json b/tests/empty-file/expected_results.json
index 22263a2..00e05fe 100644
--- a/tests/empty-file/expected_results.json
+++ b/tests/empty-file/expected_results.json
@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "error",
-  "message": "error: The name `is_leap_year` is defined multiple times.\npub fn is_leap_year(v: u128) -> bool {}\n       ^**********^\n\nerror: Unexpected return type. Expected: \"core::bool\", found: \"()\".\npub fn is_leap_year(v: u128) -> bool {}\n                                     ^^\n\nerror: Unexpected return type. Expected: \"core::bool\", found: \"()\".\npub fn is_leap_year(v: u128) -> bool {}\n                                     ^^"
+  "message": "error: Identifier not found.\nuse leap::is_leap_year;\n          ^**********^"
 }

From 82c46784b46feae95ee22ec36bc8a75f2a943586 Mon Sep 17 00:00:00 2001
From: Nenad <nenad@better.giving>
Date: Thu, 18 Jul 2024 18:21:20 +0200
Subject: [PATCH 14/23] generate partial-fail's expected_results.json

---
 tests/partial-fail/Scarb.toml            |  4 ++
 tests/partial-fail/expected_results.json |  2 +-
 tests/partial-fail/src/lib.cairo         |  3 ++
 tests/partial-fail/tests/leap.cairo      | 54 ++++++++++++++++++++++++
 4 files changed, 62 insertions(+), 1 deletion(-)
 create mode 100644 tests/partial-fail/Scarb.toml
 create mode 100644 tests/partial-fail/src/lib.cairo
 create mode 100644 tests/partial-fail/tests/leap.cairo

diff --git a/tests/partial-fail/Scarb.toml b/tests/partial-fail/Scarb.toml
new file mode 100644
index 0000000..0f2f16b
--- /dev/null
+++ b/tests/partial-fail/Scarb.toml
@@ -0,0 +1,4 @@
+[package]
+name = "leap"
+version = "0.1.0"
+edition = "2023_11"
diff --git a/tests/partial-fail/expected_results.json b/tests/partial-fail/expected_results.json
index 9ef8b6f..93ef5aa 100644
--- a/tests/partial-fail/expected_results.json
+++ b/tests/partial-fail/expected_results.json
@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "fail",
-  "message": "TODO: replace with correct output"
+  "message": "leap_leap::leap::year_divisible_by_100_but_not_by_3_is_still_not_a_leap_year - Panicked with \"assertion failed: `!is_leap_year(1900)`.\".\nleap_leap::leap::year_divisible_by_200_not_divisible_by_400_in_common_year - Panicked with \"assertion failed: `!is_leap_year(1800)`.\".\nleap_leap::leap::year_divisible_by_100_not_divisible_by_400_in_common_year - Panicked with \"assertion failed: `!is_leap_year(2100)`.\"."
 }
diff --git a/tests/partial-fail/src/lib.cairo b/tests/partial-fail/src/lib.cairo
new file mode 100644
index 0000000..d791d9e
--- /dev/null
+++ b/tests/partial-fail/src/lib.cairo
@@ -0,0 +1,3 @@
+pub fn is_leap_year(year: u64) -> bool {
+    year % 4 == 0 && (year % 101 != 0 || year % 400 == 0)
+}
diff --git a/tests/partial-fail/tests/leap.cairo b/tests/partial-fail/tests/leap.cairo
new file mode 100644
index 0000000..30be29d
--- /dev/null
+++ b/tests/partial-fail/tests/leap.cairo
@@ -0,0 +1,54 @@
+use leap::is_leap_year;
+
+#[test]
+fn year_not_divisible_by_4_in_common_year() {
+    assert!(!is_leap_year(2015));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_2_not_divisible_by_4_in_common_year() {
+    assert!(!is_leap_year(1970));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_4_not_divisible_by_100_in_leap_year() {
+    assert!(is_leap_year(1996));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_4_and_5_is_still_a_leap_year() {
+    assert!(is_leap_year(1960));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_100_not_divisible_by_400_in_common_year() {
+    assert!(!is_leap_year(2100));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_100_but_not_by_3_is_still_not_a_leap_year() {
+    assert!(!is_leap_year(1900));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_400_is_leap_year() {
+    assert!(is_leap_year(2000));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_400_but_not_by_125_is_still_a_leap_year() {
+    assert!(is_leap_year(2400));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_200_not_divisible_by_400_in_common_year() {
+    assert!(!is_leap_year(1800));
+}

From 9ce7ade8ca6e8f02b7f4cfd4b362508db1e0efca Mon Sep 17 00:00:00 2001
From: Nenad <nenad@better.giving>
Date: Thu, 18 Jul 2024 18:27:26 +0200
Subject: [PATCH 15/23] generate success's & syntax-error's
 expected_results.json

---
 tests/success/Scarb.toml                 |  4 ++
 tests/success/src/lib.cairo              |  3 ++
 tests/success/tests/leap.cairo           | 54 ++++++++++++++++++++++++
 tests/syntax-error/Scarb.toml            |  4 ++
 tests/syntax-error/expected_results.json |  2 +-
 tests/syntax-error/src/lib.cairo         |  1 +
 tests/syntax-error/tests/leap.cairo      | 54 ++++++++++++++++++++++++
 7 files changed, 121 insertions(+), 1 deletion(-)
 create mode 100644 tests/success/Scarb.toml
 create mode 100644 tests/success/src/lib.cairo
 create mode 100644 tests/success/tests/leap.cairo
 create mode 100644 tests/syntax-error/Scarb.toml
 create mode 100644 tests/syntax-error/src/lib.cairo
 create mode 100644 tests/syntax-error/tests/leap.cairo

diff --git a/tests/success/Scarb.toml b/tests/success/Scarb.toml
new file mode 100644
index 0000000..0f2f16b
--- /dev/null
+++ b/tests/success/Scarb.toml
@@ -0,0 +1,4 @@
+[package]
+name = "leap"
+version = "0.1.0"
+edition = "2023_11"
diff --git a/tests/success/src/lib.cairo b/tests/success/src/lib.cairo
new file mode 100644
index 0000000..a50ac06
--- /dev/null
+++ b/tests/success/src/lib.cairo
@@ -0,0 +1,3 @@
+pub fn is_leap_year(year: u64) -> bool {
+    year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)
+}
diff --git a/tests/success/tests/leap.cairo b/tests/success/tests/leap.cairo
new file mode 100644
index 0000000..30be29d
--- /dev/null
+++ b/tests/success/tests/leap.cairo
@@ -0,0 +1,54 @@
+use leap::is_leap_year;
+
+#[test]
+fn year_not_divisible_by_4_in_common_year() {
+    assert!(!is_leap_year(2015));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_2_not_divisible_by_4_in_common_year() {
+    assert!(!is_leap_year(1970));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_4_not_divisible_by_100_in_leap_year() {
+    assert!(is_leap_year(1996));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_4_and_5_is_still_a_leap_year() {
+    assert!(is_leap_year(1960));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_100_not_divisible_by_400_in_common_year() {
+    assert!(!is_leap_year(2100));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_100_but_not_by_3_is_still_not_a_leap_year() {
+    assert!(!is_leap_year(1900));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_400_is_leap_year() {
+    assert!(is_leap_year(2000));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_400_but_not_by_125_is_still_a_leap_year() {
+    assert!(is_leap_year(2400));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_200_not_divisible_by_400_in_common_year() {
+    assert!(!is_leap_year(1800));
+}
diff --git a/tests/syntax-error/Scarb.toml b/tests/syntax-error/Scarb.toml
new file mode 100644
index 0000000..0f2f16b
--- /dev/null
+++ b/tests/syntax-error/Scarb.toml
@@ -0,0 +1,4 @@
+[package]
+name = "leap"
+version = "0.1.0"
+edition = "2023_11"
diff --git a/tests/syntax-error/expected_results.json b/tests/syntax-error/expected_results.json
index afc4d4e..f566987 100644
--- a/tests/syntax-error/expected_results.json
+++ b/tests/syntax-error/expected_results.json
@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "error",
-  "message": "TODO: replace with correct output"
+  "message": "error: Missing token TerminalLParen.\npub fn leap1236^&gv13n\n               ^\n\nerror: Skipped tokens. Expected: parameter.\npub fn leap1236^&gv13n\n               ^^\n\nerror: Unexpected token, expected ':' followed by a type.\npub fn leap1236^&gv13n\n                      ^\n\nerror: Missing token TerminalRParen.\npub fn leap1236^&gv13n\n                      ^\n\nerror: Missing token TerminalLBrace.\npub fn leap1236^&gv13n\n                      ^\n\nerror: Unknown type.\npub fn leap1236^&gv13n\n                      ^"
 }
diff --git a/tests/syntax-error/src/lib.cairo b/tests/syntax-error/src/lib.cairo
new file mode 100644
index 0000000..d60a27a
--- /dev/null
+++ b/tests/syntax-error/src/lib.cairo
@@ -0,0 +1 @@
+pub fn leap1236^&gv13n
\ No newline at end of file
diff --git a/tests/syntax-error/tests/leap.cairo b/tests/syntax-error/tests/leap.cairo
new file mode 100644
index 0000000..30be29d
--- /dev/null
+++ b/tests/syntax-error/tests/leap.cairo
@@ -0,0 +1,54 @@
+use leap::is_leap_year;
+
+#[test]
+fn year_not_divisible_by_4_in_common_year() {
+    assert!(!is_leap_year(2015));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_2_not_divisible_by_4_in_common_year() {
+    assert!(!is_leap_year(1970));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_4_not_divisible_by_100_in_leap_year() {
+    assert!(is_leap_year(1996));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_4_and_5_is_still_a_leap_year() {
+    assert!(is_leap_year(1960));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_100_not_divisible_by_400_in_common_year() {
+    assert!(!is_leap_year(2100));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_100_but_not_by_3_is_still_not_a_leap_year() {
+    assert!(!is_leap_year(1900));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_400_is_leap_year() {
+    assert!(is_leap_year(2000));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_400_but_not_by_125_is_still_a_leap_year() {
+    assert!(is_leap_year(2400));
+}
+
+#[test]
+#[ignore]
+fn year_divisible_by_200_not_divisible_by_400_in_common_year() {
+    assert!(!is_leap_year(1800));
+}

From 3c14cdc18bfa54b5bcd64418ef7e53713208d40d Mon Sep 17 00:00:00 2001
From: Nenad <nenad@better.giving>
Date: Thu, 18 Jul 2024 19:55:59 +0200
Subject: [PATCH 16/23] Include relative file path in the error message

---
 bin/run-tests.sh                         | 2 +-
 bin/run.sh                               | 4 ++--
 tests/all-fail/expected_results.json     | 2 +-
 tests/empty-file/expected_results.json   | 2 +-
 tests/partial-fail/expected_results.json | 2 +-
 tests/syntax-error/expected_results.json | 2 +-
 6 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/bin/run-tests.sh b/bin/run-tests.sh
index 9d22b5b..77f9b21 100755
--- a/bin/run-tests.sh
+++ b/bin/run-tests.sh
@@ -30,7 +30,7 @@ for test_dir in "${tmp_dir}"/*; do
 
     for file in "$results_file_path" "$expected_results_file_path"; do
         # We sort both the '.message' values in results.json and expected_results.json files
-        tmp_file=$(mktemp -p .)
+        tmp_file=$(mktemp -p "$test_dir/")
         sorted_message=$(cat $file | jq -r '.message' >"$tmp_file" && sort "$tmp_file")
         jq --arg msg "$sorted_message" '.message = $msg' "$file" >"$tmp_file" && mv "$tmp_file" "$file"
     done
diff --git a/bin/run.sh b/bin/run.sh
index 076b1af..e41ec39 100755
--- a/bin/run.sh
+++ b/bin/run.sh
@@ -50,9 +50,9 @@ else
     test_output_inline=$(printf '%s' "$test_output")
 
     # Try to distinguish between failing tests and errors
-    if echo "$test_output_inline" | grep "error:"; then
+    if echo "$test_output_inline" | grep -q "error:"; then
         status="error"
-        sanitized_test_output=$(echo "$test_output_inline" | sed '/Compiling\|-->.*$/d' | sed -n -e '/error: could not compile/q;p')
+        sanitized_test_output=$(echo "$test_output_inline" | sed '/Compiling.*$/d' | perl -pe "s@(?<= --> ).*$solution_dir@@g" | sed -n -e '/error: could not compile/q;p')
     else
         status="fail"
         sanitized_test_output=$(echo "$test_output_inline" | awk '/failures:/{y=1;next}y' | sed -n -e '/Error: test result/q;p' | sed -r 's/   //g')
diff --git a/tests/all-fail/expected_results.json b/tests/all-fail/expected_results.json
index ccb4e22..c74fcd5 100644
--- a/tests/all-fail/expected_results.json
+++ b/tests/all-fail/expected_results.json
@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "fail",
-  "message": "leap_leap::leap::year_not_divisible_by_4_in_common_year - Panicked with \"assertion failed: `!is_leap_year(2015)`.\".\nleap_leap::leap::year_divisible_by_2_not_divisible_by_4_in_common_year - Panicked with \"assertion failed: `!is_leap_year(1970)`.\".\nleap_leap::leap::year_divisible_by_4_not_divisible_by_100_in_leap_year - Panicked with \"assertion failed: `is_leap_year(1996)`.\".\nleap_leap::leap::year_divisible_by_100_not_divisible_by_400_in_common_year - Panicked with \"assertion failed: `!is_leap_year(2100)`.\".\nleap_leap::leap::year_divisible_by_200_not_divisible_by_400_in_common_year - Panicked with \"assertion failed: `!is_leap_year(1800)`.\".\nleap_leap::leap::year_divisible_by_400_but_not_by_125_is_still_a_leap_year - Panicked with \"assertion failed: `is_leap_year(2400)`.\".\nleap_leap::leap::year_divisible_by_400_is_leap_year - Panicked with \"assertion failed: `is_leap_year(2000)`.\".\nleap_leap::leap::year_divisible_by_4_and_5_is_still_a_leap_year - Panicked with \"assertion failed: `is_leap_year(1960)`.\".\nleap_leap::leap::year_divisible_by_100_but_not_by_3_is_still_not_a_leap_year - Panicked with \"assertion failed: `!is_leap_year(1900)`.\"."
+  "message": "leap_leap::leap::year_divisible_by_400_is_leap_year - Panicked with \"assertion failed: `is_leap_year(2000)`.\".\nleap_leap::leap::year_divisible_by_4_and_5_is_still_a_leap_year - Panicked with \"assertion failed: `is_leap_year(1960)`.\".\nleap_leap::leap::year_divisible_by_4_not_divisible_by_100_in_leap_year - Panicked with \"assertion failed: `is_leap_year(1996)`.\".\nleap_leap::leap::year_divisible_by_2_not_divisible_by_4_in_common_year - Panicked with \"assertion failed: `!is_leap_year(1970)`.\".\nleap_leap::leap::year_divisible_by_400_but_not_by_125_is_still_a_leap_year - Panicked with \"assertion failed: `is_leap_year(2400)`.\".\nleap_leap::leap::year_not_divisible_by_4_in_common_year - Panicked with \"assertion failed: `!is_leap_year(2015)`.\".\nleap_leap::leap::year_divisible_by_200_not_divisible_by_400_in_common_year - Panicked with \"assertion failed: `!is_leap_year(1800)`.\".\nleap_leap::leap::year_divisible_by_100_not_divisible_by_400_in_common_year - Panicked with \"assertion failed: `!is_leap_year(2100)`.\".\nleap_leap::leap::year_divisible_by_100_but_not_by_3_is_still_not_a_leap_year - Panicked with \"assertion failed: `!is_leap_year(1900)`.\"."
 }
diff --git a/tests/empty-file/expected_results.json b/tests/empty-file/expected_results.json
index 00e05fe..946edef 100644
--- a/tests/empty-file/expected_results.json
+++ b/tests/empty-file/expected_results.json
@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "error",
-  "message": "error: Identifier not found.\nuse leap::is_leap_year;\n          ^**********^"
+  "message": "error: Identifier not found.\n --> /tests/leap.cairo:1:11\nuse leap::is_leap_year;\n          ^**********^"
 }
diff --git a/tests/partial-fail/expected_results.json b/tests/partial-fail/expected_results.json
index 93ef5aa..2d41d68 100644
--- a/tests/partial-fail/expected_results.json
+++ b/tests/partial-fail/expected_results.json
@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "fail",
-  "message": "leap_leap::leap::year_divisible_by_100_but_not_by_3_is_still_not_a_leap_year - Panicked with \"assertion failed: `!is_leap_year(1900)`.\".\nleap_leap::leap::year_divisible_by_200_not_divisible_by_400_in_common_year - Panicked with \"assertion failed: `!is_leap_year(1800)`.\".\nleap_leap::leap::year_divisible_by_100_not_divisible_by_400_in_common_year - Panicked with \"assertion failed: `!is_leap_year(2100)`.\"."
+  "message": "leap_leap::leap::year_divisible_by_100_but_not_by_3_is_still_not_a_leap_year - Panicked with \"assertion failed: `!is_leap_year(1900)`.\".\nleap_leap::leap::year_divisible_by_100_not_divisible_by_400_in_common_year - Panicked with \"assertion failed: `!is_leap_year(2100)`.\".\nleap_leap::leap::year_divisible_by_200_not_divisible_by_400_in_common_year - Panicked with \"assertion failed: `!is_leap_year(1800)`.\"."
 }
diff --git a/tests/syntax-error/expected_results.json b/tests/syntax-error/expected_results.json
index f566987..e4fffde 100644
--- a/tests/syntax-error/expected_results.json
+++ b/tests/syntax-error/expected_results.json
@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "error",
-  "message": "error: Missing token TerminalLParen.\npub fn leap1236^&gv13n\n               ^\n\nerror: Skipped tokens. Expected: parameter.\npub fn leap1236^&gv13n\n               ^^\n\nerror: Unexpected token, expected ':' followed by a type.\npub fn leap1236^&gv13n\n                      ^\n\nerror: Missing token TerminalRParen.\npub fn leap1236^&gv13n\n                      ^\n\nerror: Missing token TerminalLBrace.\npub fn leap1236^&gv13n\n                      ^\n\nerror: Unknown type.\npub fn leap1236^&gv13n\n                      ^"
+  "message": "error: Missing token TerminalLParen.\n --> /src/lib.cairo:1:16\npub fn leap1236^&gv13n\n               ^\n\nerror: Skipped tokens. Expected: parameter.\n --> /src/lib.cairo:1:16\npub fn leap1236^&gv13n\n               ^^\n\nerror: Unexpected token, expected ':' followed by a type.\n --> /src/lib.cairo:1:23\npub fn leap1236^&gv13n\n                      ^\n\nerror: Missing token TerminalRParen.\n --> /src/lib.cairo:1:23\npub fn leap1236^&gv13n\n                      ^\n\nerror: Missing token TerminalLBrace.\n --> /src/lib.cairo:1:23\npub fn leap1236^&gv13n\n                      ^\n\nerror: Unknown type.\n --> /src/lib.cairo:1:23\npub fn leap1236^&gv13n\n                      ^"
 }

From ce62962ec7b31af403f33f9431f14b4a70fd058e Mon Sep 17 00:00:00 2001
From: Nenad <nenad@better.giving>
Date: Fri, 19 Jul 2024 19:15:52 +0200
Subject: [PATCH 17/23] UPdate Dockerfile to make run-in-docker work

---
 Dockerfile | 33 +++++++++++++++++++++++++--------
 bin/run.sh |  6 +++---
 2 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 0b3e479..602cd3b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,13 +1,30 @@
-FROM alpine:3.18
+ARG REPO=ubuntu
+ARG IMAGE=22.04
+FROM ${REPO}:${IMAGE} AS builder
 
-# install packages required to run the tests
-RUN apk add --no-cache jq curl git
+ARG VERSION=v2.6.5
+ARG RELEASE=scarb-${VERSION}-x86_64-unknown-linux-gnu
 
-# install Scarb
-RUN curl --proto '=https' --tlsv1.2 -sSf https://docs.swmansion.com/scarb/install.sh | sh -s -- -v 2.6.5
+RUN apt-get update && apt-get install -y curl
 
-COPY . /opt/test-runner
+RUN mkdir opt/test-runner
+RUN mkdir opt/test-runner/bin
+WORKDIR /tmp
+ADD https://github.com/software-mansion/scarb/releases/download/${VERSION}/${RELEASE}.tar.gz .
+RUN tar -xf ${RELEASE}.tar.gz \
+    && rm -rf /tmp/${RELEASE}/doc \
+    && mv /tmp/${RELEASE} /opt/test-runner/bin/scarb
 
-WORKDIR /opt/test-runner
+FROM ${REPO}:${IMAGE} AS runner
+
+# install packages required to run the tests
+# hadolint ignore=DL3018
+RUN apt-get update && apt-get install -y jq
+
+COPY --from=builder /opt/test-runner/bin/scarb /opt/test-runner/bin/scarb
+ENV PATH=$PATH:/opt/test-runner/bin/scarb/bin
 
-ENTRYPOINT ["/opt/test-runner/bin/run.sh"]
+WORKDIR /opt/test-runner
+COPY . .
+# Initialize a scarb cache
+ENTRYPOINT ["/opt/test-runner/bin/run.sh"]
\ No newline at end of file
diff --git a/bin/run.sh b/bin/run.sh
index e41ec39..3fc8e3e 100755
--- a/bin/run.sh
+++ b/bin/run.sh
@@ -34,9 +34,9 @@ echo "${slug}: testing..."
 start_dir="$(pwd)"
 cd "${solution_dir}" || exit 1
 
-# Run the tests for the provided implementation file and redirect stdout and
-# stderr to capture it
-test_output=$(scarb cairo-test --include-ignored 2>&1)
+# Run the tests for the provided implementation file and redirect stdout and stderr to capture it.
+# We also redirect the global cache from some default global directory to the output directory.
+test_output=$(scarb --global-cache-dir "$output_dir/.cache" cairo-test --include-ignored 2>&1)
 exit_code=$?
 
 cd "${start_dir}" || exit 1

From 1eea511a8794ce7d0e134c8c4d6762a0e8001640 Mon Sep 17 00:00:00 2001
From: Nenad <nenad@better.giving>
Date: Fri, 19 Jul 2024 19:26:53 +0200
Subject: [PATCH 18/23] Improve 'done' msgs in run-x.sh scripts

---
 bin/run-tests.sh | 2 +-
 bin/run.sh       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/run-tests.sh b/bin/run-tests.sh
index 77f9b21..328f1a2 100755
--- a/bin/run-tests.sh
+++ b/bin/run-tests.sh
@@ -45,7 +45,7 @@ done
 rm -rf "${tmp_dir}"
 
 if [ ${exit_code} -eq 0 ]; then
-    echo "${test_dir_name}: done"
+    echo "run-tests: done"
 fi
 
 exit ${exit_code}
diff --git a/bin/run.sh b/bin/run.sh
index 3fc8e3e..37d7499 100755
--- a/bin/run.sh
+++ b/bin/run.sh
@@ -61,4 +61,4 @@ else
     jq -n --arg output "${sanitized_test_output}" --arg status "${status}" '{version: 1, status: $status, message: $output}' >"${results_file}"
 fi
 
-echo "${slug}: done"
+echo "${slug}: generated $results_file"

From 69c3cf756bd8f6b3f57d6f3d9e6c86acf803fb4f Mon Sep 17 00:00:00 2001
From: Nenad <nenad@better.giving>
Date: Sat, 20 Jul 2024 11:33:12 +0200
Subject: [PATCH 19/23] Switch Docker (and scarb install) to alpine

---
 Dockerfile | 10 +++++-----
 bin/run.sh |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 602cd3b..43e8979 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,11 +1,11 @@
-ARG REPO=ubuntu
-ARG IMAGE=22.04
+ARG REPO=alpine
+ARG IMAGE=3.18
 FROM ${REPO}:${IMAGE} AS builder
 
 ARG VERSION=v2.6.5
-ARG RELEASE=scarb-${VERSION}-x86_64-unknown-linux-gnu
+ARG RELEASE=scarb-${VERSION}-x86_64-unknown-linux-musl
 
-RUN apt-get update && apt-get install -y curl
+RUN apk add --no-cache curl
 
 RUN mkdir opt/test-runner
 RUN mkdir opt/test-runner/bin
@@ -19,7 +19,7 @@ FROM ${REPO}:${IMAGE} AS runner
 
 # install packages required to run the tests
 # hadolint ignore=DL3018
-RUN apt-get update && apt-get install -y jq
+RUN apk add --no-cache jq
 
 COPY --from=builder /opt/test-runner/bin/scarb /opt/test-runner/bin/scarb
 ENV PATH=$PATH:/opt/test-runner/bin/scarb/bin
diff --git a/bin/run.sh b/bin/run.sh
index 37d7499..ebed94b 100755
--- a/bin/run.sh
+++ b/bin/run.sh
@@ -52,7 +52,7 @@ else
     # Try to distinguish between failing tests and errors
     if echo "$test_output_inline" | grep -q "error:"; then
         status="error"
-        sanitized_test_output=$(echo "$test_output_inline" | sed '/Compiling.*$/d' | perl -pe "s@(?<= --> ).*$solution_dir@@g" | sed -n -e '/error: could not compile/q;p')
+        sanitized_test_output=$(echo "$test_output_inline" | sed '/Compiling.*$/d' | sed "s@$solution_dir@@g" | sed -n -e '/error: could not compile/q;p')
     else
         status="fail"
         sanitized_test_output=$(echo "$test_output_inline" | awk '/failures:/{y=1;next}y' | sed -n -e '/Error: test result/q;p' | sed -r 's/   //g')

From ed419024506e1be79c08a335b3fc1f393fcb5dfd Mon Sep 17 00:00:00 2001
From: Nenad <nenad@better.giving>
Date: Sat, 20 Jul 2024 11:35:01 +0200
Subject: [PATCH 20/23] Remove solution_dir prefix after reading errors up to
 'could not compile'

---
 bin/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/run.sh b/bin/run.sh
index ebed94b..ffca9fa 100755
--- a/bin/run.sh
+++ b/bin/run.sh
@@ -52,7 +52,7 @@ else
     # Try to distinguish between failing tests and errors
     if echo "$test_output_inline" | grep -q "error:"; then
         status="error"
-        sanitized_test_output=$(echo "$test_output_inline" | sed '/Compiling.*$/d' | sed "s@$solution_dir@@g" | sed -n -e '/error: could not compile/q;p')
+        sanitized_test_output=$(echo "$test_output_inline" | sed '/Compiling.*$/d' | sed -n -e '/error: could not compile/q;p' | sed "s@$solution_dir@@g")
     else
         status="fail"
         sanitized_test_output=$(echo "$test_output_inline" | awk '/failures:/{y=1;next}y' | sed -n -e '/Error: test result/q;p' | sed -r 's/   //g')

From bea0e9ddf97c1f546897395fdb531f0a669e50b1 Mon Sep 17 00:00:00 2001
From: Nenad <nenad@better.giving>
Date: Sat, 20 Jul 2024 11:37:31 +0200
Subject: [PATCH 21/23] Add msg if results match

---
 bin/run-tests.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/run-tests.sh b/bin/run-tests.sh
index 328f1a2..b82aad7 100755
--- a/bin/run-tests.sh
+++ b/bin/run-tests.sh
@@ -39,6 +39,8 @@ for test_dir in "${tmp_dir}"/*; do
 
     if ! diff "$results_file_path" "$expected_results_file_path"; then
         exit_code=1
+    else
+        echo "$test_dir_name: results match"
     fi
 done
 

From 9f380660f7e42f16da82123d6485fb9477ac2bfd Mon Sep 17 00:00:00 2001
From: Nenad <nenad@better.giving>
Date: Sat, 20 Jul 2024 11:38:02 +0200
Subject: [PATCH 22/23] Remove echo of run-tests done

---
 bin/run-tests.sh | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/bin/run-tests.sh b/bin/run-tests.sh
index b82aad7..39519b7 100755
--- a/bin/run-tests.sh
+++ b/bin/run-tests.sh
@@ -45,9 +45,4 @@ for test_dir in "${tmp_dir}"/*; do
 done
 
 rm -rf "${tmp_dir}"
-
-if [ ${exit_code} -eq 0 ]; then
-    echo "run-tests: done"
-fi
-
 exit ${exit_code}

From 2deac8ca5701d2abf52bffe89312b7ae7d8fb457 Mon Sep 17 00:00:00 2001
From: Nenad <nenad@better.giving>
Date: Sat, 20 Jul 2024 11:45:29 +0200
Subject: [PATCH 23/23] Refactor

---
 Dockerfile       | 2 +-
 bin/run-tests.sh | 2 +-
 bin/run.sh       | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 43e8979..ec8c704 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -27,4 +27,4 @@ ENV PATH=$PATH:/opt/test-runner/bin/scarb/bin
 WORKDIR /opt/test-runner
 COPY . .
 # Initialize a scarb cache
-ENTRYPOINT ["/opt/test-runner/bin/run.sh"]
\ No newline at end of file
+ENTRYPOINT ["/opt/test-runner/bin/run.sh"]
diff --git a/bin/run-tests.sh b/bin/run-tests.sh
index 39519b7..7c1a70a 100755
--- a/bin/run-tests.sh
+++ b/bin/run-tests.sh
@@ -35,7 +35,7 @@ for test_dir in "${tmp_dir}"/*; do
         jq --arg msg "$sorted_message" '.message = $msg' "$file" >"$tmp_file" && mv "$tmp_file" "$file"
     done
 
-    echo "${test_dir_name}: comparing results.json to expected_results.json"
+    echo "$test_dir_name: comparing $(basename "${results_file_path}") to $(basename "${expected_results_file_path}")"
 
     if ! diff "$results_file_path" "$expected_results_file_path"; then
         exit_code=1
diff --git a/bin/run.sh b/bin/run.sh
index ffca9fa..7061ac4 100755
--- a/bin/run.sh
+++ b/bin/run.sh
@@ -61,4 +61,4 @@ else
     jq -n --arg output "${sanitized_test_output}" --arg status "${status}" '{version: 1, status: $status, message: $output}' >"${results_file}"
 fi
 
-echo "${slug}: generated $results_file"
+echo "$slug: generated $results_file"