zfs: merge openzfs/zfs@a382e2119

Notable upstream pull request merges: #15693 a382e21 Add Gotify notification support to ZED #15732 e78aca3 Fix livelist assertions for dedup and cloning #15733 7ecaa07 make zdb_decompress_block check decompression reliably #15735 255741f Improve block sizes checks during cloning Obtained from: OpenZFS OpenZFS commit: a382e21
rcbchan · Jan 10, 2024 · f552d7a · f552d7a
2 parents 1372013 + a382e21
commit f552d7a
Show file tree

Hide file tree

Showing 12 changed files with 233 additions and 39 deletions.
diff --git a/sys/contrib/openzfs/cmd/zdb/zdb.c b/sys/contrib/openzfs/cmd/zdb/zdb.c
@@ -8533,11 +8533,14 @@ zdb_decompress_block(abd_t *pabd, void *buf, void *lbuf, uint64_t lsize,
 			}
 
 			/*
-			 * We randomize lbuf2, and decompress to both
-			 * lbuf and lbuf2. This way, we will know if
-			 * decompression fill exactly to lsize.
+			 * We set lbuf to all zeros and lbuf2 to all
+			 * ones, then decompress to both buffers and
+			 * compare their contents. This way we can
+			 * know if decompression filled exactly to
+			 * lsize or if it left some bytes unwritten.
 			 */
-			VERIFY0(random_get_pseudo_bytes(lbuf2, lsize));
+			memset(lbuf, 0x00, lsize);
+			memset(lbuf2, 0xff, lsize);
 
 			if (zio_decompress_data(*cfuncp, pabd,
 			    lbuf, psize, lsize, NULL) == 0 &&

diff --git a/sys/contrib/openzfs/cmd/zed/zed.d/zed-functions.sh b/sys/contrib/openzfs/cmd/zed/zed.d/zed-functions.sh
@@ -209,6 +209,10 @@ zed_notify()
     [ "${rv}" -eq 0 ] && num_success=$((num_success + 1))
     [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1))
 
+    zed_notify_gotify "${subject}" "${pathname}"; rv=$?
+    [ "${rv}" -eq 0 ] && num_success=$((num_success + 1))
+    [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1))
+
     [ "${num_success}" -gt 0 ] && return 0
     [ "${num_failure}" -gt 0 ] && return 1
     return 2
@@ -624,6 +628,97 @@ zed_notify_ntfy()
 }
 
 
+# zed_notify_gotify (subject, pathname)
+#
+# Send a notification via Gotify <https://gotify.net/>.
+# The Gotify URL (ZED_GOTIFY_URL) defines a self-hosted Gotify location.
+# The Gotify application token (ZED_GOTIFY_APPTOKEN) defines a
+# Gotify application token which is associated with a message.
+# The optional Gotify priority value (ZED_GOTIFY_PRIORITY) overrides the
+# default or configured priority at the Gotify server for the application.
+#
+# Requires curl and sed executables to be installed in the standard PATH.
+#
+# References
+#   https://gotify.net/docs/index
+#
+# Arguments
+#   subject: notification subject
+#   pathname: pathname containing the notification message (OPTIONAL)
+#
+# Globals
+#   ZED_GOTIFY_URL
+#   ZED_GOTIFY_APPTOKEN
+#   ZED_GOTIFY_PRIORITY
+#
+# Return
+#   0: notification sent
+#   1: notification failed
+#   2: not configured
+#
+zed_notify_gotify()
+{
+    local subject="$1"
+    local pathname="${2:-"/dev/null"}"
+    local msg_body
+    local msg_out
+    local msg_err
+
+    [ -n "${ZED_GOTIFY_URL}" ] && [ -n "${ZED_GOTIFY_APPTOKEN}" ] || return 2
+    local url="${ZED_GOTIFY_URL}/message?token=${ZED_GOTIFY_APPTOKEN}"
+
+    if [ ! -r "${pathname}" ]; then
+        zed_log_err "gotify cannot read \"${pathname}\""
+        return 1
+    fi
+
+    zed_check_cmd "curl" "sed" || return 1
+
+    # Read the message body in.
+    #
+    msg_body="$(cat "${pathname}")"
+
+    if [ -z "${msg_body}" ]
+    then
+        msg_body=$subject
+        subject=""
+    fi
+
+    # Send the POST request and check for errors.
+    #
+    if [ -n "${ZED_GOTIFY_PRIORITY}" ]; then
+        msg_out="$( \
+        curl \
+        --form-string "title=${subject}" \
+        --form-string "message=${msg_body}" \
+        --form-string "priority=${ZED_GOTIFY_PRIORITY}" \
+        "${url}" \
+        2>/dev/null \
+        )"; rv=$?
+    else
+        msg_out="$( \
+        curl \
+        --form-string "title=${subject}" \
+        --form-string "message=${msg_body}" \
+        "${url}" \
+        2>/dev/null \
+        )"; rv=$?
+    fi
+
+    if [ "${rv}" -ne 0 ]; then
+        zed_log_err "curl exit=${rv}"
+        return 1
+    fi
+    msg_err="$(echo "${msg_out}" \
+        | sed -n -e 's/.*"errors" *:.*\[\(.*\)\].*/\1/p')"
+    if [ -n "${msg_err}" ]; then
+        zed_log_err "gotify \"${msg_err}"\"
+        return 1
+    fi
+    return 0
+}
+
+
 
 # zed_rate_limit (tag, [interval])
 #

diff --git a/sys/contrib/openzfs/cmd/zed/zed.d/zed.rc b/sys/contrib/openzfs/cmd/zed/zed.d/zed.rc
@@ -169,3 +169,24 @@ ZED_SYSLOG_SUBCLASS_EXCLUDE="history_event"
 #  <https://docs.ntfy.sh/install/>
 # https://ntfy.sh by default; uncomment to enable an alternative service url.
 #ZED_NTFY_URL="https://ntfy.sh"
+
+##
+# Gotify server URL
+# This defines a URL that the Gotify call will be directed toward.
+# <https://gotify.net/docs/index>
+# Disabled by default; uncomment to enable.
+#ZED_GOTIFY_URL=""
+
+##
+# Gotify application token
+# This defines a Gotify application token which a message is associated with.
+# This token is generated when an application is created on the Gotify server.
+# Disabled by default; uncomment to enable.
+#ZED_GOTIFY_APPTOKEN=""
+
+##
+# Gotify priority (optional)
+# If defined, this overrides the default priority of the
+# Gotify application associated with ZED_GOTIFY_APPTOKEN.
+# Value is an integer 0 and up.
+#ZED_GOTIFY_PRIORITY=""
diff --git a/sys/contrib/openzfs/config/kernel-fpu.m4 b/sys/contrib/openzfs/config/kernel-fpu.m4
@@ -79,6 +79,12 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_FPU], [
 		__kernel_fpu_end();
 	], [], [ZFS_META_LICENSE])
 
+	ZFS_LINUX_TEST_SRC([kernel_neon], [
+		#include <asm/neon.h>
+	], [
+		kernel_neon_begin();
+		kernel_neon_end();
+	], [], [ZFS_META_LICENSE])
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_FPU], [
@@ -105,9 +111,20 @@ AC_DEFUN([ZFS_AC_KERNEL_FPU], [
 			AC_DEFINE(KERNEL_EXPORTS_X86_FPU, 1,
 			    [kernel exports FPU functions])
 		],[
-			AC_MSG_RESULT(internal)
-			AC_DEFINE(HAVE_KERNEL_FPU_INTERNAL, 1,
-			    [kernel fpu internal])
+			dnl #
+			dnl # ARM neon symbols (only on arm and arm64)
+			dnl # could be GPL-only on arm64 after Linux 6.2
+			dnl #
+			ZFS_LINUX_TEST_RESULT([kernel_neon_license],[
+				AC_MSG_RESULT(kernel_neon_*)
+				AC_DEFINE(HAVE_KERNEL_NEON, 1,
+				    [kernel has kernel_neon_* functions])
+			],[
+				# catch-all
+				AC_MSG_RESULT(internal)
+				AC_DEFINE(HAVE_KERNEL_FPU_INTERNAL, 1,
+				    [kernel fpu internal])
+			])
 		])
 	])
 ])
diff --git a/sys/contrib/openzfs/include/os/linux/kernel/linux/simd_aarch64.h b/sys/contrib/openzfs/include/os/linux/kernel/linux/simd_aarch64.h
@@ -71,9 +71,15 @@
 #define	ID_AA64PFR0_EL1		sys_reg(3, 0, 0, 1, 0)
 #define	ID_AA64ISAR0_EL1	sys_reg(3, 0, 0, 6, 0)
 
+#if (defined(HAVE_KERNEL_NEON) && defined(CONFIG_KERNEL_MODE_NEON))
 #define	kfpu_allowed()		1
 #define	kfpu_begin()		kernel_neon_begin()
 #define	kfpu_end()		kernel_neon_end()
+#else
+#define	kfpu_allowed()		0
+#define	kfpu_begin()		do {} while (0)
+#define	kfpu_end()		do {} while (0)
+#endif
 #define	kfpu_init()		(0)
 #define	kfpu_fini()		do {} while (0)
 

diff --git a/sys/contrib/openzfs/include/os/linux/kernel/linux/simd_arm.h b/sys/contrib/openzfs/include/os/linux/kernel/linux/simd_arm.h
@@ -53,9 +53,15 @@
 #include <asm/elf.h>
 #include <asm/hwcap.h>
 
+#if (defined(HAVE_KERNEL_NEON) && defined(CONFIG_KERNEL_MODE_NEON))
 #define	kfpu_allowed()		1
 #define	kfpu_begin()		kernel_neon_begin()
 #define	kfpu_end()		kernel_neon_end()
+#else
+#define	kfpu_allowed()		0
+#define	kfpu_begin()		do {} while (0)
+#define	kfpu_end()		do {} while (0)
+#endif
 #define	kfpu_init()		(0)
 #define	kfpu_fini()		do {} while (0)
 

diff --git a/sys/contrib/openzfs/include/os/linux/zfs/sys/trace_zil.h b/sys/contrib/openzfs/include/os/linux/zfs/sys/trace_zil.h
@@ -51,7 +51,9 @@
 		__field(uint64_t,	zl_parse_lr_seq)		    \
 		__field(uint64_t,	zl_parse_blk_count)		    \
 		__field(uint64_t,	zl_parse_lr_count)		    \
-		__field(uint64_t,	zl_cur_used)			    \
+		__field(uint64_t,	zl_cur_size)			    \
+		__field(uint64_t,	zl_cur_left)			    \
+		__field(uint64_t,	zl_cur_max)			    \
 		__field(clock_t,	zl_replay_time)			    \
 		__field(uint64_t,	zl_replay_blks)
 
@@ -72,7 +74,9 @@
 		__entry->zl_parse_lr_seq	= zilog->zl_parse_lr_seq;   \
 		__entry->zl_parse_blk_count	= zilog->zl_parse_blk_count;\
 		__entry->zl_parse_lr_count	= zilog->zl_parse_lr_count; \
-		__entry->zl_cur_used	= zilog->zl_cur_used;		    \
+		__entry->zl_cur_size	= zilog->zl_cur_size;		    \
+		__entry->zl_cur_left	= zilog->zl_cur_left;		    \
+		__entry->zl_cur_max	= zilog->zl_cur_max;		    \
 		__entry->zl_replay_time	= zilog->zl_replay_time;	    \
 		__entry->zl_replay_blks	= zilog->zl_replay_blks;
 
@@ -82,7 +86,8 @@
 	"replay %u stop_sync %u logbias %u sync %u "			    \
 	"parse_error %u parse_blk_seq %llu parse_lr_seq %llu "		    \
 	"parse_blk_count %llu parse_lr_count %llu "			    \
-	"cur_used %llu replay_time %lu replay_blks %llu }"
+	"cur_size %llu cur_left %llu cur_max %llu replay_time %lu "	    \
+	"replay_blks %llu }"
 
 #define	ZILOG_TP_PRINTK_ARGS						    \
 	    __entry->zl_lr_seq, __entry->zl_commit_lr_seq,		    \
@@ -92,7 +97,8 @@
 	    __entry->zl_stop_sync, __entry->zl_logbias, __entry->zl_sync,   \
 	    __entry->zl_parse_error, __entry->zl_parse_blk_seq,		    \
 	    __entry->zl_parse_lr_seq, __entry->zl_parse_blk_count,	    \
-	    __entry->zl_parse_lr_count, __entry->zl_cur_used,		    \
+	    __entry->zl_parse_lr_count, __entry->zl_cur_size,		    \
+	    __entry->zl_cur_left, __entry->zl_cur_max,			    \
 	    __entry->zl_replay_time, __entry->zl_replay_blks
 
 #define	ITX_TP_STRUCT_ENTRY						    \

diff --git a/sys/contrib/openzfs/man/man7/zpoolprops.7 b/sys/contrib/openzfs/man/man7/zpoolprops.7
@@ -28,7 +28,7 @@
 .\" Copyright (c) 2021, Colm Buckley <[email protected]>
 .\" Copyright (c) 2023, Klara Inc.
 .\"
-.Dd April 18, 2023
+.Dd January 2, 2024
 .Dt ZPOOLPROPS 7
 .Os
 .
@@ -331,7 +331,7 @@ Specifies that the pool maintain compatibility with specific feature sets.
 When set to
 .Sy off
 (or unset) compatibility is disabled (all features may be enabled); when set to
-.Sy legacy Ns
+.Sy legacy
 no features may be enabled.
 When set to a comma-separated list of filenames
 (each filename may either be an absolute path, or relative to

diff --git a/sys/contrib/openzfs/module/zfs/dsl_deadlist.c b/sys/contrib/openzfs/module/zfs/dsl_deadlist.c
@@ -1000,8 +1000,6 @@ livelist_compare(const void *larg, const void *rarg)
 	/* if vdevs are equal, sort by offsets. */
 	uint64_t l_dva0_offset = DVA_GET_OFFSET(&l->blk_dva[0]);
 	uint64_t r_dva0_offset = DVA_GET_OFFSET(&r->blk_dva[0]);
-	if (l_dva0_offset == r_dva0_offset)
-		ASSERT3U(l->blk_birth, ==, r->blk_birth);
 	return (TREE_CMP(l_dva0_offset, r_dva0_offset));
 }
 
@@ -1016,9 +1014,9 @@ struct livelist_iter_arg {
  * and used to match up ALLOC/FREE pairs. ALLOC'd blkptrs without a
  * corresponding FREE are stored in the supplied bplist.
  *
- * Note that multiple FREE and ALLOC entries for the same blkptr may
- * be encountered when dedup is involved. For this reason we keep a
- * refcount for all the FREE entries of each blkptr and ensure that
+ * Note that multiple FREE and ALLOC entries for the same blkptr may be
+ * encountered when dedup or block cloning is involved.  For this reason we
+ * keep a refcount for all the FREE entries of each blkptr and ensure that
  * each of those FREE entries has a corresponding ALLOC preceding it.
  */
 static int
@@ -1037,6 +1035,13 @@ dsl_livelist_iterate(void *arg, const blkptr_t *bp, boolean_t bp_freed,
 	livelist_entry_t node;
 	node.le_bp = *bp;
 	livelist_entry_t *found = avl_find(avl, &node, NULL);
+	if (found) {
+		ASSERT3U(BP_GET_PSIZE(bp), ==, BP_GET_PSIZE(&found->le_bp));
+		ASSERT3U(BP_GET_CHECKSUM(bp), ==,
+		    BP_GET_CHECKSUM(&found->le_bp));
+		ASSERT3U(BP_PHYSICAL_BIRTH(bp), ==,
+		    BP_PHYSICAL_BIRTH(&found->le_bp));
+	}
 	if (bp_freed) {
 		if (found == NULL) {
 			/* first free entry for this blkptr */
@@ -1046,10 +1051,10 @@ dsl_livelist_iterate(void *arg, const blkptr_t *bp, boolean_t bp_freed,
 			e->le_refcnt = 1;
 			avl_add(avl, e);
 		} else {
-			/* dedup block free */
-			ASSERT(BP_GET_DEDUP(bp));
-			ASSERT3U(BP_GET_CHECKSUM(bp), ==,
-			    BP_GET_CHECKSUM(&found->le_bp));
+			/*
+			 * Deduped or cloned block free.  We could assert D bit
+			 * for dedup, but there is no such one for cloning.
+			 */
 			ASSERT3U(found->le_refcnt + 1, >, found->le_refcnt);
 			found->le_refcnt++;
 		}
@@ -1065,14 +1070,6 @@ dsl_livelist_iterate(void *arg, const blkptr_t *bp, boolean_t bp_freed,
 				/* all tracked free pairs have been matched */
 				avl_remove(avl, found);
 				kmem_free(found, sizeof (livelist_entry_t));
-			} else {
-				/*
-				 * This is definitely a deduped blkptr so
-				 * let's validate it.
-				 */
-				ASSERT(BP_GET_DEDUP(bp));
-				ASSERT3U(BP_GET_CHECKSUM(bp), ==,
-				    BP_GET_CHECKSUM(&found->le_bp));
 			}
 		}
 	}

diff --git a/sys/contrib/openzfs/module/zfs/zfs_vnops.c b/sys/contrib/openzfs/module/zfs/zfs_vnops.c
@@ -1186,11 +1186,18 @@ zfs_clone_range(znode_t *inzp, uint64_t *inoffp, znode_t *outzp,
 	inblksz = inzp->z_blksz;
 
 	/*
-	 * We cannot clone into files with different block size if we can't
-	 * grow it (block size is already bigger or more than one block).
+	 * We cannot clone into a file with different block size if we can't
+	 * grow it (block size is already bigger, has more than one block, or
+	 * not locked for growth).  There are other possible reasons for the
+	 * grow to fail, but we cover what we can before opening transaction
+	 * and the rest detect after we try to do it.
 	 */
+	if (inblksz < outzp->z_blksz) {
+		error = SET_ERROR(EINVAL);
+		goto unlock;
+	}
 	if (inblksz != outzp->z_blksz && (outzp->z_size > outzp->z_blksz ||
-	    outzp->z_size > inblksz)) {
+	    outlr->lr_length != UINT64_MAX)) {
 		error = SET_ERROR(EINVAL);
 		goto unlock;
 	}
@@ -1309,12 +1316,24 @@ zfs_clone_range(znode_t *inzp, uint64_t *inoffp, znode_t *outzp,
 		}
 
 		/*
-		 * Copy source znode's block size. This only happens on the
-		 * first iteration since zfs_rangelock_reduce() will shrink down
-		 * lr_len to the appropriate size.
+		 * Copy source znode's block size. This is done only if the
+		 * whole znode is locked (see zfs_rangelock_cb()) and only
+		 * on the first iteration since zfs_rangelock_reduce() will
+		 * shrink down lr_length to the appropriate size.
 		 */
 		if (outlr->lr_length == UINT64_MAX) {
 			zfs_grow_blocksize(outzp, inblksz, tx);
+
+			/*
+			 * Block growth may fail for many reasons we can not
+			 * predict here.  If it happen the cloning is doomed.
+			 */
+			if (inblksz != outzp->z_blksz) {
+				error = SET_ERROR(EINVAL);
+				dmu_tx_abort(tx);
+				break;
+			}
+
 			/*
 			 * Round range lock up to the block boundary, so we
 			 * prevent appends until we are done.