Snapshot testing

Author: jiripudil

src/Framework/Environment.php

@@ -27,6 +27,9 @@ class Environment
 	/** Thread number when run tests in multi threads */
 	public const THREAD = 'NETTE_TESTER_THREAD';
 
+	/** Should Tester update snapshots? */
+	public const UPDATE_SNAPSHOTS = 'NETTE_TESTER_UPDATE_SNAPSHOTS';
+
 	/** @var bool  used for debugging Tester itself */
 	public static $debugMode = true;
 
@@ -108,6 +111,11 @@ public static function setupErrors(): void
 						self::removeOutputBuffers();
 						echo "\nFatal error: $error[message] in $error[file] on line $error[line]\n";
 					}
+				} elseif (getenv(self::UPDATE_SNAPSHOTS) && Snapshot::$updatedSnapshots) {
+					self::removeOutputBuffers();
+					echo "\nThe following snapshots were updated, please make sure they are correct:\n"
+						. implode("\n", Snapshot::$updatedSnapshots) . "\n";
+					exit(Runner\Job::CODE_FAIL);
 				} elseif (self::$checkAssertions && !Assert::$counter) {
 					self::removeOutputBuffers();
 					echo "\nError: This test forgets to execute an assertion.\n";

src/Framework/Snapshot.php

@@ -0,0 +1,91 @@
+<?php
+
+/**
+ * This file is part of the Nette Tester.
+ * Copyright (c) 2009 David Grudl (https://davidgrudl.com)
+ */
+
+declare(strict_types=1);
+
+namespace Tester;
+
+
+/**
+ * Snapshot testing helper.
+ */
+class Snapshot
+{
+	public static $snapshotDir = 'snapshots';
+
+	public static $updatedSnapshots = [];
+
+
+	/**
+	 * Compares value with a previously created snapshot.
+	 */
+	public static function match($value, string $snapshotName): void
+	{
+		$updateSnapshots = (bool) getenv(Environment::UPDATE_SNAPSHOTS);
+
+		$testFile = $_SERVER['argv'][0];
+		$snapshotFile = self::getSnapshotFile($testFile, $snapshotName);
+
+		if (!file_exists($snapshotFile)) {
+			if (!$updateSnapshots) {
+				Assert::fail("Missing snapshot file '$snapshotFile', use --update-snapshots option to generate it.");
+			}
+
+			self::write($snapshotFile, $value);
+		}
+
+		$snapshot = self::read($snapshotFile);
+
+		try {
+			Assert::equal($snapshot, $value, "Snapshot $snapshotName");
+
+		} catch (AssertException $e) {
+			if (!$updateSnapshots) {
+				throw $e;
+			}
+
+			self::write($snapshotFile, $value);
+		}
+	}
+
+
+	private static function getSnapshotFile(string $testFile, string $snapshotName): string
+	{
+		$path = self::$snapshotDir . DIRECTORY_SEPARATOR . pathinfo($testFile, PATHINFO_FILENAME) . '.' . $snapshotName . '.phps';
+		if (!preg_match('#/|\w:#A', self::$snapshotDir)) {
+			$path = dirname($testFile) . DIRECTORY_SEPARATOR . $path;
+		}
+		return $path;
+	}
+
+
+	private static function read(string $snapshotFile)
+	{
+		$snapshotContents = @file_get_contents($snapshotFile);
+		if ($snapshotContents === false) {
+			throw new \Exception("Unable to read snapshot file '$snapshotFile'.");
+		}
+
+		return eval(substr($snapshotContents, strlen('<?php ')));
+	}
+
+
+	private static function write(string $snapshotFile, $value): void
+	{
+		$snapshotDirectory = dirname($snapshotFile);
+		if (!is_dir($snapshotDirectory) && !mkdir($snapshotDirectory)) {
+			throw new \Exception("Unable to create snapshot directory '$snapshotDirectory'.");
+		}
+
+		$snapshotContents = '<?php return ' . var_export($value, true) . ';' . PHP_EOL;
+		if (file_put_contents($snapshotFile, $snapshotContents) === false) {
+			throw new \Exception("Unable to write snapshot file '$snapshotFile'.");
+		}
+
+		self::$updatedSnapshots[] = $snapshotFile;
+	}
+}

src/Runner/CliTester.php

@@ -65,6 +65,9 @@ public function run(): ?int
 		if (isset($coverageFile)) {
 			$runner->setEnvironmentVariable(Environment::COVERAGE, $coverageFile);
 		}
+		if ($this->options['--update-snapshots']) {
+			$runner->setEnvironmentVariable(Environment::UPDATE_SNAPSHOTS, '1');
+		}
 
 		if ($this->options['-o'] !== null) {
 			ob_clean();
@@ -117,6 +120,7 @@ private function loadOptions(): CommandLine
     --colors [1|0]               Enable or disable colors.
     --coverage <path>            Generate code coverage report to file.
     --coverage-src <path>        Path to source code.
+    --update-snapshots           Create or update snapshot files. Tests with snapshot changes will be marked as skipped.
     -h | --help                  This help.
 
 XX

src/bootstrap.php

@@ -16,6 +16,7 @@
 require __DIR__ . '/Framework/TestCase.php';
 require __DIR__ . '/Framework/DomQuery.php';
 require __DIR__ . '/Framework/FileMutator.php';
+require __DIR__ . '/Framework/Snapshot.php';
 require __DIR__ . '/CodeCoverage/Collector.php';
 require __DIR__ . '/Runner/Job.php';
 

tests/Framework/.gitignore

@@ -0,0 +1 @@
+snapshots/

tests/Framework/Snapshot.match.phpt

@@ -0,0 +1,21 @@
+<?php
+
+declare(strict_types=1);
+
+use Tester\Assert;
+use Tester\AssertException;
+use Tester\Snapshot;
+
+require __DIR__ . '/../bootstrap.php';
+
+Snapshot::$snapshotDir = __DIR__ . '/fixtures';
+
+Snapshot::match(['answer' => 42], 'existingSnapshot');
+
+Assert::exception(function () {
+	Snapshot::match(['answer' => 43], 'existingSnapshot');
+}, AssertException::class, "Snapshot existingSnapshot: %a% should be equal to %a%");
+
+Assert::exception(function () {
+	Snapshot::match('value', 'nonExistingSnapshot');
+}, AssertException::class, "Missing snapshot file '%A%', use --update-snapshots option to generate it.");

tests/Framework/Snapshot.update.phpt

@@ -0,0 +1,43 @@
+<?php
+
+declare(strict_types=1);
+
+use Tester\Assert;
+use Tester\Environment;
+use Tester\Helpers;
+use Tester\Snapshot;
+
+require __DIR__ . '/../bootstrap.php';
+
+putenv(Environment::UPDATE_SNAPSHOTS . '=1');
+Snapshot::$snapshotDir = __DIR__ . '/snapshots';
+Helpers::purge(Snapshot::$snapshotDir);
+
+// newly created
+
+Assert::false(file_exists(Snapshot::$snapshotDir . '/Snapshot.update.newSnapshot.phps'));
+Snapshot::match(['answer' => 42], 'newSnapshot');
+Assert::true(file_exists(Snapshot::$snapshotDir . '/Snapshot.update.newSnapshot.phps'));
+Assert::contains('42', file_get_contents(Snapshot::$snapshotDir . '/Snapshot.update.newSnapshot.phps'));
+
+// existing
+
+file_put_contents(
+	Snapshot::$snapshotDir . '/Snapshot.update.updatedSnapshot.phps',
+	'<?php return array(\'answer\' => 43);' . PHP_EOL
+);
+
+Assert::true(file_exists(Snapshot::$snapshotDir . '/Snapshot.update.updatedSnapshot.phps'));
+Snapshot::match(['answer' => 42], 'updatedSnapshot');
+Assert::true(file_exists(Snapshot::$snapshotDir . '/Snapshot.update.updatedSnapshot.phps'));
+Assert::contains('42', file_get_contents(Snapshot::$snapshotDir . '/Snapshot.update.updatedSnapshot.phps'));
+
+// Snapshot::$updatedSnapshots
+
+Assert::equal([
+	Snapshot::$snapshotDir . '/Snapshot.update.newSnapshot.phps',
+	Snapshot::$snapshotDir . '/Snapshot.update.updatedSnapshot.phps',
+], Snapshot::$updatedSnapshots);
+
+// reset the env variable so that the test does not fail due to updated snapshots
+putenv(Environment::UPDATE_SNAPSHOTS . '=0');

tests/Framework/fixtures/Snapshot.match.existingSnapshot.phps

@@ -0,0 +1 @@
+<?php return ['answer' => 42];

tests/Runner/Runner.snapshots.phpt

@@ -0,0 +1,80 @@
+<?php
+
+declare(strict_types=1);
+
+use Tester\Assert;
+use Tester\Dumper;
+use Tester\Runner\Test;
+
+require __DIR__ . '/../bootstrap.php';
+require __DIR__ . '/../../src/Runner/OutputHandler.php';
+require __DIR__ . '/../../src/Runner/Test.php';
+require __DIR__ . '/../../src/Runner/TestHandler.php';
+require __DIR__ . '/../../src/Runner/Runner.php';
+
+
+class Logger implements Tester\Runner\OutputHandler
+{
+	public $results = [];
+
+
+	public function prepare(Test $test): void
+	{
+	}
+
+
+	public function finish(Test $test): void
+	{
+		$this->results[basename($test->getFile())] = [$test->getResult(), $test->message];
+	}
+
+
+	public function begin(): void
+	{
+	}
+
+
+	public function end(): void
+	{
+	}
+}
+
+
+Tester\Helpers::purge(__DIR__ . '/snapshots/snapshots');
+
+
+// first run, without update -> fail
+
+$runner = new Tester\Runner\Runner(createInterpreter());
+$runner->paths[] = __DIR__ . '/snapshots/*.phptx';
+$runner->outputHandlers[] = $logger = new Logger;
+$runner->run();
+
+Assert::same(Test::FAILED, $logger->results['update-snapshots.phptx'][0]);
+Assert::match(
+	"Failed: Missing snapshot file '%a%', use --update-snapshots option to generate it.\n%A%",
+	trim(Dumper::removeColors($logger->results['update-snapshots.phptx'][1]))
+);
+
+// second run, with update -> skipped
+
+$runner = new Tester\Runner\Runner(createInterpreter());
+$runner->paths[] = __DIR__ . '/snapshots/*.phptx';
+$runner->outputHandlers[] = $logger = new Logger;
+$runner->setEnvironmentVariable(Tester\Environment::UPDATE_SNAPSHOTS, '1');
+$runner->run();
+
+Assert::same(Test::FAILED, $logger->results['update-snapshots.phptx'][0]);
+Assert::match(
+	"The following snapshots were updated, please make sure they are correct:\n%a%.snapshot.phps",
+	trim(Dumper::removeColors($logger->results['update-snapshots.phptx'][1]))
+);
+
+// third run, without update -> pass
+
+$runner = new Tester\Runner\Runner(createInterpreter());
+$runner->paths[] = __DIR__ . '/snapshots/*.phptx';
+$runner->outputHandlers[] = $logger = new Logger;
+$runner->run();
+
+Assert::same(Test::PASSED, $logger->results['update-snapshots.phptx'][0]);

tests/Runner/snapshots/.gitignore

@@ -0,0 +1 @@
+snapshots/

tests/Runner/snapshots/update-snapshots.phptx

@@ -0,0 +1,8 @@
+<?php
+
+use Tester\Snapshot;
+
+require __DIR__ . '/../../bootstrap.php';
+
+Snapshot::$snapshotDir = __DIR__ . '/snapshots';
+Snapshot::match('snapshot value', 'snapshot');