added module param to optionally perform a thorough scrub

Author: alek-p

man/man4/zfs.4

@@ -4,6 +4,7 @@
 .\" Copyright (c) 2019, 2021 by Delphix. All rights reserved.
 .\" Copyright (c) 2019 Datto Inc.
 .\" Copyright (c) 2023, 2024, 2025, Klara, Inc.
+.\" Copyright (c) 2026 ConnectWise, Inc.
 .\" The contents of this file are subject to the terms of the Common Development
 .\" and Distribution License (the "License").  You may not use this file except
 .\" in compliance with the License. You can obtain a copy of the license at
@@ -17,7 +18,7 @@
 .\" own identifying information:
 .\" Portions Copyright [yyyy] [name of copyright owner]
 .\"
-.Dd May 29, 2025
+.Dd Jan 23, 2026
 .Dt ZFS 4
 .Os
 .
@@ -2087,7 +2088,15 @@ working on a scrub between TXG flushes.
 .It Sy zfs_scrub_error_blocks_per_txg Ns = Ns Sy 4096 Pq uint
 Error blocks to be scrubbed in one txg.
 .
-.It Sy zfs_scan_checkpoint_intval Ns = Ns Sy 7200 Ns s Po 2 hour Pc Pq uint
+.It Sy zfs_scrub_thorough Ns = Ns Sy 0 Ns | Ns 1 Pq uint
+When set will cause scrub to decrypt and decompress blocks it reads so that it
+will catch the rare type of corruption where the checksum matches the data, but
+decryption and/or decompression fails. For encrypted datasets keys have to be
+have to be loaded in order to perform thorough scrub. If keys are not loaded
+or unloaded during a thorough scrub the scrub will revert to doing a normal
+scrub for the encrypted dataset.
+.
+.It Sy zfs_scan_checkpoint_intval Ns = Ns Sy 7200 Ns s Po 2 hours Pc Pq uint
 To preserve progress across reboots, the sequential scan algorithm periodically
 needs to stop metadata scanning and issue all the verification I/O to disk.
 The frequency of this flushing is determined by this tunable.

module/zfs/dsl_scan.c

@@ -26,13 +26,15 @@
  * Copyright (c) 2017, 2019, Datto Inc. All rights reserved.
  * Copyright (c) 2015, Nexenta Systems, Inc. All rights reserved.
  * Copyright 2019 Joyent, Inc.
+ * Copyright 2025 ConnectWise, Inc.
  */
 
 #include <sys/dsl_scan.h>
 #include <sys/dsl_pool.h>
 #include <sys/dsl_dataset.h>
 #include <sys/dsl_prop.h>
 #include <sys/dsl_dir.h>
+#include <sys/dsl_crypt.h>
 #include <sys/dsl_synctask.h>
 #include <sys/dnode.h>
 #include <sys/dmu_tx.h>
@@ -56,6 +58,7 @@
 #include <sys/abd.h>
 #include <sys/range_tree.h>
 #include <sys/dbuf.h>
+#include <sys/fm/fs/zfs.h>
 #ifdef _KERNEL
 #include <sys/zfs_vfsops.h>
 #endif
@@ -246,6 +249,16 @@ static int zfs_free_bpobj_enabled = 1;
 /* Error blocks to be scrubbed in one txg. */
 static uint_t zfs_scrub_error_blocks_per_txg = 1 << 12;
 
+/*
+ * When set will cause scrub to decrypt and decompress blocks it reads so that
+ * it will catch the rare type of corruption where the checksum matches the
+ * data, but decryption and/or decompression fails. For encrypted datasets keys
+ * have to be loaded in order to perform thorough scrub. If keys are not loaded
+ * or unloaded during a thorough scrub the scrub will revert to doing a normal
+ * scrub for the encrypted dataset.
+*/
+static uint_t zfs_scrub_thorough = 0;
+
 /* the order has to match pool_scan_type */
 static scan_cb_t *scan_funcs[POOL_SCAN_FUNCS] = {
 	NULL,
@@ -287,6 +300,10 @@ typedef struct scan_io {
 	uint64_t		sio_birth;
 	zio_cksum_t		sio_cksum;
 	uint32_t		sio_nr_dvas;
+	uint64_t		sio_salt;
+	uint64_t		sio_iv1;
+	uint64_t		sio_iv2;
+
 
 	/* fields from zio_t */
 	uint32_t		sio_flags;
@@ -442,6 +459,11 @@ sio2bp(const scan_io_t *sio, blkptr_t *bp)
 	BP_SET_PHYSICAL_BIRTH(bp, sio->sio_phys_birth);
 	BP_SET_LOGICAL_BIRTH(bp, sio->sio_birth);
 	bp->blk_fill = 1;	/* we always only work with data pointers */
+	if (BP_IS_ENCRYPTED(bp)) {
+		bp->blk_dva[2].dva_word[0] = sio->sio_salt;
+		bp->blk_dva[2].dva_word[1] = sio->sio_iv1;
+		BP_SET_IV2(bp, sio->sio_iv2);
+	}
 	bp->blk_cksum = sio->sio_cksum;
 
 	ASSERT3U(sio->sio_nr_dvas, >, 0);
@@ -458,6 +480,11 @@ bp2sio(const blkptr_t *bp, scan_io_t *sio, int dva_i)
 	sio->sio_birth = BP_GET_LOGICAL_BIRTH(bp);
 	sio->sio_cksum = bp->blk_cksum;
 	sio->sio_nr_dvas = BP_GET_NDVAS(bp);
+	if (BP_IS_ENCRYPTED(bp)) {
+		sio->sio_salt = bp->blk_dva[2].dva_word[0];
+		sio->sio_iv1 = bp->blk_dva[2].dva_word[1];
+		sio->sio_iv2 = BP_GET_IV2(bp);
+	}
 
 	/*
 	 * Copy the DVAs to the sio. We need all copies of the block so
@@ -4013,8 +4040,11 @@ read_by_block_level(dsl_scan_t *scn, zbookmark_phys_t zb)
 		return;
 	}
 
-	int zio_flags = ZIO_FLAG_SCAN_THREAD | ZIO_FLAG_RAW |
-	    ZIO_FLAG_CANFAIL | ZIO_FLAG_SCRUB;
+	int zio_flags = ZIO_FLAG_SCAN_THREAD | ZIO_FLAG_CANFAIL |
+	    ZIO_FLAG_SCRUB;
+
+	if (zfs_scrub_thorough == 0)
+		zio_flags |= ZIO_FLAG_RAW;
 
 	/* If it's an intent log block, failure is expected. */
 	if (zb.zb_level == ZB_ZIL_LEVEL)
@@ -4814,7 +4844,24 @@ dsl_scan_scrub_cb(dsl_pool_t *dp,
 	uint64_t phys_birth = BP_GET_PHYSICAL_BIRTH(bp);
 	size_t psize = BP_GET_PSIZE(bp);
 	boolean_t needs_io = B_FALSE;
-	int zio_flags = ZIO_FLAG_SCAN_THREAD | ZIO_FLAG_RAW | ZIO_FLAG_CANFAIL;
+	int zio_flags = ZIO_FLAG_SCAN_THREAD | ZIO_FLAG_CANFAIL;
+
+	if (zfs_scrub_thorough == 0) {
+		zio_flags |= ZIO_FLAG_RAW;
+	} else if (BP_IS_ENCRYPTED(bp)) {
+		/*
+		 * If we are doing a thorough scrub, the key may be unloaded
+		 * at any time. So we fall back to a raw scrub if we don't have
+		 * the key loaded.
+		 */
+		dsl_crypto_key_t *dck = NULL;
+		int err = spa_keystore_lookup_key(spa, zb->zb_objset, FTAG, &dck);
+		if (err == 0) {
+			spa_keystore_dsl_key_rele(spa, dck, FTAG);
+		} else {
+			zio_flags |= ZIO_FLAG_RAW;
+		}
+	}
 
 	count_block(dp->dp_blkstats, bp);
 	if (phys_birth <= scn->scn_phys.scn_min_txg ||
@@ -4891,7 +4938,9 @@ dsl_scan_scrub_done(zio_t *zio)
 	}
 
 	if (zio->io_error && (zio->io_error != ECKSUM ||
-	    !(zio->io_flags & ZIO_FLAG_SPECULATIVE))) {
+	    !(zio->io_flags & ZIO_FLAG_SPECULATIVE)) &&
+	    !(zio->io_error == EACCES && (zio->io_flags & ZIO_FLAG_SCRUB) &&
+	    zfs_scrub_thorough != 0)) {
 		if (dsl_errorscrubbing(spa->spa_dsl_pool) &&
 		    !dsl_errorscrub_is_paused(spa->spa_dsl_pool->dp_scan)) {
 			atomic_inc_64(&spa->spa_dsl_pool->dp_scan
@@ -4916,7 +4965,25 @@ scan_exec_io(dsl_pool_t *dp, const blkptr_t *bp, int zio_flags,
 {
 	spa_t *spa = dp->dp_spa;
 	dsl_scan_t *scn = dp->dp_scan;
-	size_t size = BP_GET_PSIZE(bp);
+
+	/*
+	 * If we are trying to do a thorough scrub, the key may have
+	 * been unloaded since we queued the IO. Check if we have the
+	 * key and if not, fall back to a raw scrub.
+	 */
+	if ((zio_flags & ZIO_FLAG_RAW) == 0 && BP_IS_ENCRYPTED(bp)) {
+		dsl_crypto_key_t *dck = NULL;
+		int err = spa_keystore_lookup_key(spa, zb->zb_objset,
+		    FTAG, &dck);
+		if (err == 0) {
+			spa_keystore_dsl_key_rele(spa, dck, FTAG);
+		} else {
+			zio_flags |= ZIO_FLAG_RAW;
+		}
+	}
+
+	size_t size = (zio_flags & ZIO_FLAG_RAW) ?
+	    BP_GET_PSIZE(bp) : BP_GET_LSIZE(bp);
 	abd_t *data = abd_alloc_for_io(size, B_FALSE);
 	zio_t *pio;
 
@@ -5362,3 +5429,6 @@ ZFS_MODULE_PARAM(zfs, zfs_, resilver_defer_percent, UINT, ZMOD_RW,
 
 ZFS_MODULE_PARAM(zfs, zfs_, scrub_error_blocks_per_txg, UINT, ZMOD_RW,
 	"Error blocks to be scrubbed in one txg");
+
+ZFS_MODULE_PARAM(zfs, zfs_, scrub_thorough, UINT, ZMOD_RW,
+	"Scrub will decrypt and decompress blocks");

tests/runfiles/common.run

@@ -546,7 +546,7 @@ tests = ['zpool_scrub_001_neg', 'zpool_scrub_002_pos', 'zpool_scrub_003_pos',
     'zpool_scrub_multiple_pools',
     'zpool_error_scrub_001_pos', 'zpool_error_scrub_002_pos',
     'zpool_error_scrub_003_pos', 'zpool_error_scrub_004_pos',
-    'zpool_scrub_date_range_001']
+    'zpool_scrub_date_range_001', 'zpool_scrub_thorough']
 tags = ['functional', 'cli_root', 'zpool_scrub']
 
 [tests/functional/cli_root/zpool_set]

tests/zfs-tests/include/tunables.cfg

@@ -81,6 +81,7 @@ SCAN_LEGACY			scan_legacy			zfs_scan_legacy
 SCAN_SUSPEND_PROGRESS		scan_suspend_progress		zfs_scan_suspend_progress
 SCAN_VDEV_LIMIT			scan_vdev_limit			zfs_scan_vdev_limit
 SCRUB_AFTER_EXPAND		scrub_after_expand		zfs_scrub_after_expand
+SCRUB_THOROUGH			scrub_thorough			zfs_scrub_thorough
 SEND_HOLES_WITHOUT_BIRTH_TIME	send_holes_without_birth_time	send_holes_without_birth_time
 SLOW_IO_EVENTS_PER_SECOND	slow_io_events_per_second	zfs_slow_io_events_per_second
 SPA_ASIZE_INFLATION		spa.asize_inflation		spa_asize_inflation

tests/zfs-tests/tests/Makefile.am

@@ -1250,6 +1250,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/cli_root/zpool_scrub/zpool_error_scrub_002_pos.ksh \
 	functional/cli_root/zpool_scrub/zpool_error_scrub_003_pos.ksh \
 	functional/cli_root/zpool_scrub/zpool_error_scrub_004_pos.ksh \
+	functional/cli_root/zpool_scrub/zpool_scrub_thorough.ksh \
 	functional/cli_root/zpool_set/cleanup.ksh \
 	functional/cli_root/zpool_set/setup.ksh \
 	functional/cli_root/zpool/setup.ksh \

tests/zfs-tests/tests/functional/cli_root/zpool_scrub/zpool_scrub_thorough.ksh

@@ -0,0 +1,98 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2025 ConnectWise Inc. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zpool_scrub/zpool_scrub.cfg
+
+#
+# DESCRIPTION:
+# 'zpool scrub' with thorough scrub enabled works with various configurations:
+# 1) Compressed dataset
+# 2) Compressed and encrypted dataset
+# 3) Encrypted dataset
+# 4) Compressed and encrypted dataset with keys unloaded
+#
+# STRATEGY:
+# 1. Create datasets for the 4 scenarios.
+# 2. Write data to them.
+# 3. Enable thorough scrub tunable.
+# 4. Start a scrub and wait for it to finish.
+# 5. Verify 0 errors.
+# 6. Repeat with thorough scrub disabled.
+#
+
+function cleanup
+{
+	set_tunable32 SCRUB_THOROUGH 0
+	datasetexists $TESTPOOL/comp && destroy_dataset $TESTPOOL/comp
+	datasetexists $TESTPOOL/comp_enc && destroy_dataset $TESTPOOL/comp_enc
+	datasetexists $TESTPOOL/enc && destroy_dataset $TESTPOOL/enc
+	datasetexists $TESTPOOL/comp_enc_unloaded && destroy_dataset $TESTPOOL/comp_enc_unloaded
+}
+
+verify_runnable "global"
+
+log_onexit cleanup
+
+log_assert "Thorough scrub works with various dataset configurations."
+
+# 1) Compressed dataset
+log_must zfs create $TESTPOOL/comp
+log_must zfs set compression=on $TESTPOOL/comp
+typeset file_comp="/$TESTPOOL/comp/$TESTFILE0"
+log_must dd if=/dev/urandom of=$file_comp bs=1024 count=1024 oflag=sync
+# Make sure data is compressible
+log_must eval "echo 'aaaaaaaa' >> $file_comp"
+
+# 2) Compressed and encrypted dataset
+log_must eval "echo 'password' | zfs create -o encryption=on -o keyformat=passphrase $TESTPOOL/comp_enc"
+log_must zfs set compression=on $TESTPOOL/comp_enc
+typeset file_comp_enc="/$TESTPOOL/comp_enc/$TESTFILE0"
+log_must dd if=/dev/urandom of=$file_comp_enc bs=1024 count=1024 oflag=sync
+log_must eval "echo 'aaaaaaaa' >> $file_comp_enc"
+
+# 3) Encrypted dataset
+log_must eval "echo 'password' | zfs create -o encryption=on -o keyformat=passphrase $TESTPOOL/enc"
+log_must zfs set compression=off $TESTPOOL/enc
+typeset file_enc="/$TESTPOOL/enc/$TESTFILE0"
+log_must dd if=/dev/urandom of=$file_enc bs=1024 count=1024 oflag=sync
+
+# 4) Compressed and encrypted dataset with keys unloaded
+log_must eval "echo 'password' | zfs create -o encryption=on -o keyformat=passphrase $TESTPOOL/comp_enc_unloaded"
+log_must zfs set compression=on $TESTPOOL/comp_enc_unloaded
+typeset file_comp_enc_unloaded="/$TESTPOOL/comp_enc_unloaded/$TESTFILE0"
+log_must dd if=/dev/urandom of=$file_comp_enc_unloaded bs=1024 count=1024 oflag=sync
+log_must eval "echo 'aaaaaaaa' >> $file_comp_enc_unloaded"
+
+log_must zfs unmount $TESTPOOL/comp_enc_unloaded
+log_must zfs unload-key $TESTPOOL/comp_enc_unloaded
+
+# Enable thorough scrub
+log_must set_tunable32 SCRUB_THOROUGH 1
+# Run and wait for scrub
+log_must zpool scrub -w $TESTPOOL
+
+log_must check_pool_status $TESTPOOL "scan" "with 0 errors"
+
+# Disable thorough scrub
+log_must set_tunable32 SCRUB_THOROUGH 0
+# Run and wait for scrub
+log_must zpool scrub -w $TESTPOOL
+
+log_must check_pool_status $TESTPOOL "scan" "with 0 errors"
+
+log_pass "Thorough scrub works with various dataset configurations."