ASAN related workaround in hook_test

Ref: pmem#7

While executing ASAN builds, some unexpected syscalls are observed. These
can not be predicted, so this patch attempts to filter them out.
Also, the return value of the function named hook in test/hook_test_preload.c
is changed to 7 from 99. This also helps with ASAN builds: ASAN reads
the buffers passed write syscalls, after they return. Due to this, an ASAN
error was triggered every time it tried to read dummy_data buffer's contents,
which has fewer than 99 bytes.

test/hook_test.c

@@ -43,6 +43,7 @@
 #include <stdlib.h>
 #include <unistd.h>
 #include <syscall.h>
+#include <string.h>
 
 #include "magic_syscalls.h"
 
@@ -55,11 +56,13 @@ main(int argc, char *argv[])
 		return EXIT_FAILURE;
 
 	magic_syscall_start_log(argv[2], "1");
+	assert(syscall(test_magic_syscall) == test_magic_syscall_result);
 
-	assert(write(1, dummy_data, sizeof(dummy_data)) == 99);
+	assert(write(1, dummy_data, sizeof(dummy_data)) == 7);
 	assert(write(1, "thing", 4) == 4);
-	assert(write(1, dummy_data, sizeof(dummy_data)) == 99);
+	assert(write(1, dummy_data, sizeof(dummy_data)) == 7);
 
+	assert(syscall(test_magic_syscall) == test_magic_syscall_result);
 	magic_syscall_stop_log();
 
 	return EXIT_SUCCESS;

test/hook_test_data.h

@@ -34,5 +34,7 @@
 #define INTERCEPT_HOOK_TEST_DATA_H
 
 static const char dummy_data[] = "dummy_data";
+static const long test_magic_syscall = 9999;
+static const long test_magic_syscall_result = 7777;
 
 #endif

test/hook_test_preload.c

@@ -43,13 +43,16 @@
 #include <unistd.h>
 #include <syscall.h>
 
+#include <stdio.h>
+
 #include "libsyscall_intercept_hook_point.h"
 
 #include "hook_test_data.h"
 
 static int hook_counter;
 static bool in_hook;
 static bool deinit_called;
+static bool during_test;
 
 static int
 hook(long syscall_number, long arg0, long arg1, long arg2, long *result)
@@ -62,7 +65,7 @@ hook(long syscall_number, long arg0, long arg1, long arg2, long *result)
 			assert(arg0 == 1);
 			assert(strcmp((void *)(intptr_t)arg1, dummy_data) == 0);
 			assert(arg2 == (long)sizeof(dummy_data));
-			*result = 99;
+			*result = 7;
 			return 0;
 
 		case 1:
@@ -76,6 +79,31 @@ hook(long syscall_number, long arg0, long arg1, long arg2, long *result)
 	}
 }
 
+/*
+ * is_likely_asan_initiated
+ * Filters some syscalls that are normally not happening, but are made by
+ * ASAN code when ASAN does instrumenting.
+ */
+static bool
+is_likely_asan_initiated(long syscall_number, long arg0)
+{
+	switch (syscall_number) {
+		case SYS_mmap:
+		case SYS_munmap:
+		case SYS_ioctl:
+		case SYS_getpid:
+		case SYS_futex:
+		case SYS_exit_group:
+			return true;
+		case SYS_write:
+			if (arg0 == 2)
+				return true;
+			/* fallthrough */
+		default:
+			return false;
+	}
+}
+
 static int
 hook_wrapper(long syscall_number,
 	long arg0, long arg1,
@@ -90,6 +118,18 @@ hook_wrapper(long syscall_number,
 	if (in_hook || deinit_called)
 		return 1;
 
+	if (is_likely_asan_initiated(syscall_number, arg0))
+		return 1;
+
+	if (syscall_number == test_magic_syscall) {
+		during_test = !during_test;
+		*result = test_magic_syscall_result;
+		return 0;
+	}
+
+	if (!during_test)
+		return 1;
+
 	in_hook = true;
 
 	int ret = hook(syscall_number, arg0, arg1, arg2, result);

test/libcintercept1.log.match

@@ -1,6 +1,10 @@
+$(S) $(XX) -- syscall(9999, $(XX), $(XX), $(XX), $(XX), $(XX), $(XX)) = ?
+$(S) $(XX) -- syscall(9999, $(XX), $(XX), $(XX), $(XX), $(XX), $(XX)) = $(N)
 $(S) $(XX) -- write(1, "dummy_data\0", 11) = ?
-$(S) $(XX) -- write(1, "dummy_data\0", 11) = 99
+$(S) $(XX) -- write(1, "dummy_data\0", 11) = 7
 $(S) $(XX) -- write(1, "thin", 4) = ?
 $(S) $(XX) -- write(1, "thin", 4) = 4
 $(S) $(XX) -- write(1, "dummy_data\0", 11) = ?
-$(S) $(XX) -- write(1, "dummy_data\0", 11) = 99
+$(S) $(XX) -- write(1, "dummy_data\0", 11) = 7
+$(S) $(XX) -- syscall(9999, $(XX), $(XX), $(XX), $(XX), $(XX), $(XX)) = ?
+$(S) $(XX) -- syscall(9999, $(XX), $(XX), $(XX), $(XX), $(XX), $(XX)) = $(N)