loader: zfs should support bootonce an nextboot

bootonce feature is temporary, one time boot, activated by
"bectl activate -t BE", "bectl activate -T BE" will reset the bootonce flag.

By default, the bootonce setting is reset on attempt to boot and the next
boot will use previously active BE.

By setting zfs_bootonce_activate="YES" in rc.conf, the bootonce BE will
be set permanently active.

bootonce dataset name is recorded in boot pool labels, bootenv area.

in case of nextboot, the nextboot_enable boolean variable is recorded in
freebsd:nvstore nvlist, also stored in boot pool label bootenv area.
On boot, the loader will process /boot/nextboot.conf if nextboot_enable
is "YES", and will set nextboot_enable to "NO", preventing /boot/nextboot.conf
processing on next boot.

bootonce and nextboot features are usable in both UEFI and BIOS boot.

To use bootonce/nextboot features, the boot loader needs to be updated on disk;
if loader.efi is stored on ESP, then ESP needs to be updated and
for BIOS boot, stage2 (zfsboot or gptzfsboot) needs to be updated
(gpart or other tools).

At this time, only lua loader is updated.

Sponsored by:	Netflix, Klara Inc.
Differential Revision:	https://reviews.freebsd.org/D25512

Makefile.inc1

@@ -2835,7 +2835,7 @@ _prebuild_libs=	${_kerberos5_lib_libasn1} \
 		${_cddl_lib_libtpool} \
 		${_cddl_lib_libzfs_core} ${_cddl_lib_libzfs} \
 		${_cddl_lib_libzutil} \
-		${_cddl_lib_libctf} \
+		${_cddl_lib_libctf} ${_cddl_lib_libzfsbootenv} \
 		lib/libufs \
 		lib/libutil lib/libpjdlog ${_lib_libypclnt} lib/libz lib/msun \
 		${_secure_lib_libcrypto} ${_secure_lib_libssl} \
@@ -2915,6 +2915,7 @@ _cddl_lib_libtpool= cddl/lib/libtpool
 _cddl_lib_libzutil= cddl/lib/libzutil
 _cddl_lib_libzfs_core= cddl/lib/libzfs_core
 _cddl_lib_libzfs= cddl/lib/libzfs
+_cddl_lib_libzfsbootenv= cddl/lib/libzfsbootenv
 
 cddl/lib/libtpool__L: cddl/lib/libspl__L
 
@@ -2928,7 +2929,8 @@ cddl/lib/libzfs__L: cddl/lib/libuutil__L cddl/lib/libavl__L lib/libgeom__L
 cddl/lib/libzfs__L: cddl/lib/libnvpair__L cddl/lib/libzutil__L
 cddl/lib/libzfs__L: secure/lib/libcrypto__L
 
-lib/libbe__L: cddl/lib/libzfs__L
+cddl/lib/libzfsbootenv__L: cddl/lib/libzfs__L
+lib/libbe__L: cddl/lib/libzfs__L cddl/lib/libzfsbootenv__L
 .endif
 _cddl_lib_libctf= cddl/lib/libctf
 _cddl_lib= cddl/lib

cddl/lib/Makefile

@@ -15,6 +15,7 @@ SUBDIR=	drti \
 	libuutil \
 	${_libzfs_core} \
 	${_libzfs} \
+	${_libzfsbootenv} \
 	${_libzpool} \
 	${_libzutil}
 
@@ -26,6 +27,7 @@ _libicp=	libicp
 _libicp_rescue=	libicp_rescue
 _libzfs=	libzfs
 _libzutil=	libzutil
+_libzfsbootenv=	libzfsbootenv
 .if ${MK_LIBTHR} != "no"
 _libzpool=	libzpool
 _libtpool=	libtpool
@@ -40,6 +42,7 @@ SUBDIR_DEPEND_libzfs_core=	libnvpair
 SUBDIR_DEPEND_libzfs=	libavl libnvpair libumem libuutil libzfs_core libzutil
 SUBDIR_DEPEND_libzpool=	libavl libnvpair libumem libicp
 SUBDIR_DEPEND_libzutil=	libavl libtpool
+SUBDIR_DEPEND_libzfsbootenv= libzfs libnvpair
 
 SUBDIR_PARALLEL=
 

cddl/lib/libzfsbootenv/Makefile

@@ -0,0 +1,33 @@
+# $FreeBSD$
+
+.PATH: ${SRCTOP}/sys/contrib/openzfs/lib/libzfsbootenv
+.PATH: ${SRCTOP}/sys/contrib/openzfs/include
+
+PACKAGE=	runtime
+LIB=		zfsbootenv
+SHLIB_MAJOR=	1
+
+LIBADD=		zfs
+LIBADD+=	nvpair
+
+INCS=	libzfsbootenv.h
+USER_C=	\
+	lzbe_device.c \
+	lzbe_util.c \
+	lzbe_pair.c
+
+SRCS= $(USER_C)
+
+CSTD=	c99
+CFLAGS+= -DIN_BASE
+CFLAGS+= -I${SRCTOP}/sys/contrib/openzfs/include
+CFLAGS+= -I${SRCTOP}/sys/contrib/openzfs/lib/libspl/include/
+CFLAGS+= -I${SRCTOP}/sys/contrib/openzfs/lib/libspl/include/os/freebsd
+CFLAGS+= -I${SRCTOP}/cddl/compat/opensolaris/include
+CFLAGS+= -I${SRCTOP}/sys/contrib/openzfs/module/icp/include
+CFLAGS+= -include ${SRCTOP}/sys/contrib/openzfs/include/os/freebsd/spl/sys/ccompile.h
+CFLAGS+= -DHAVE_ISSETUGID
+CFLAGS+= -include ${SRCTOP}/sys/modules/zfs/zfs_config.h
+CFLAGS+= -I${SRCTOP}/sys/contrib/openzfs/include/os/freebsd/zfs
+
+.include <bsd.lib.mk>

cddl/lib/libzpool/Makefile

@@ -228,6 +228,7 @@ CFLAGS+= \
 	-I${ZFSTOP}/lib/libspl/include \
 	-I${ZFSTOP}/lib/libspl/include/os/freebsd \
 	-I${SRCTOP}/sys \
+	-I${ZFSTOP}/include/os/freebsd/zfs \
 	-I${SRCTOP}/cddl/compat/opensolaris/include \
 	-I${ZFSTOP}/module/icp/include \
 	-include ${ZFSTOP}/include/os/freebsd/spl/sys/ccompile.h \

lib/libbe/Makefile

@@ -13,7 +13,9 @@ INCS=		be.h
 MAN=		libbe.3
 
 LIBADD+= zfs
-LIBADD+= nvpair spl
+LIBADD+= nvpair
+LIBADD+= spl
+LIBADD+= zfsbootenv
 
 CFLAGS+= -DIN_BASE -DHAVE_RPC_TYPES
 CFLAGS+= -I${SRCTOP}/sys/contrib/openzfs/include

lib/libbe/be.c

@@ -46,6 +46,7 @@ __FBSDID("$FreeBSD$");
 #include <stdlib.h>
 #include <time.h>
 #include <unistd.h>
+#include <libzfsbootenv.h>
 
 #include "be.h"
 #include "be_impl.h"
@@ -1221,43 +1222,20 @@ be_add_child(libbe_handle_t *lbh, const char *child_path, bool cp_if_exists)
 }
 #endif	/* SOON */
 
-static int
-be_set_nextboot(libbe_handle_t *lbh, nvlist_t *config, uint64_t pool_guid,
-    const char *zfsdev)
-{
-	nvlist_t **child;
-	uint64_t vdev_guid;
-	int c, children;
-
-	if (nvlist_lookup_nvlist_array(config, ZPOOL_CONFIG_CHILDREN, &child,
-	    &children) == 0) {
-		for (c = 0; c < children; ++c)
-			if (be_set_nextboot(lbh, child[c], pool_guid, zfsdev) != 0)
-				return (1);
-		return (0);
-	}
-
-	if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_GUID,
-	    &vdev_guid) != 0) {
-		return (1);
-	}
-
-	if (zpool_nextboot(lbh->lzh, pool_guid, vdev_guid, zfsdev) != 0) {
-		perror("ZFS_IOC_NEXTBOOT failed");
-		return (1);
-	}
-
-	return (0);
-}
-
 /*
- * Deactivate old BE dataset; currently just sets canmount=noauto
+ * Deactivate old BE dataset; currently just sets canmount=noauto or
+ * resets boot once configuration.
  */
-static int
-be_deactivate(libbe_handle_t *lbh, const char *ds)
+int
+be_deactivate(libbe_handle_t *lbh, const char *ds, bool temporary)
 {
 	zfs_handle_t *zfs;
 
+	if (temporary) {
+		return (lzbe_set_boot_device(
+		    zpool_get_name(lbh->active_phandle), lzbe_add, NULL));
+	}
+
 	if ((zfs = zfs_open(lbh->lzh, ds, ZFS_TYPE_DATASET)) == NULL)
 		return (1);
 	if (zfs_prop_set(zfs, "canmount", "noauto") != 0)
@@ -1270,10 +1248,8 @@ int
 be_activate(libbe_handle_t *lbh, const char *bootenv, bool temporary)
 {
 	char be_path[BE_MAXPATHLEN];
-	char buf[BE_MAXPATHLEN];
-	nvlist_t *config, *dsprops, *vdevs;
+	nvlist_t *dsprops;
 	char *origin;
-	uint64_t pool_guid;
 	zfs_handle_t *zhp;
 	int err;
 
@@ -1284,27 +1260,10 @@ be_activate(libbe_handle_t *lbh, const char *bootenv, bool temporary)
 		return (set_error(lbh, err));
 
 	if (temporary) {
-		config = zpool_get_config(lbh->active_phandle, NULL);
-		if (config == NULL)
-			/* config should be fetchable... */
-			return (set_error(lbh, BE_ERR_UNKNOWN));
-
-		if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
-		    &pool_guid) != 0)
-			/* Similarly, it shouldn't be possible */
-			return (set_error(lbh, BE_ERR_UNKNOWN));
-
-		/* Expected format according to zfsbootcfg(8) man */
-		snprintf(buf, sizeof(buf), "zfs:%s:", be_path);
-
-		/* We have no config tree */
-		if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
-		    &vdevs) != 0)
-			return (set_error(lbh, BE_ERR_NOPOOL));
-
-		return (be_set_nextboot(lbh, vdevs, pool_guid, buf));
+		return (lzbe_set_boot_device(
+		    zpool_get_name(lbh->active_phandle), lzbe_add, be_path));
 	} else {
-		if (be_deactivate(lbh, lbh->bootfs) != 0)
+		if (be_deactivate(lbh, lbh->bootfs, false) != 0)
 			return (-1);
 
 		/* Obtain bootenv zpool */

lib/libbe/be.h

@@ -81,6 +81,7 @@ int be_prop_list_alloc(nvlist_t **be_list);
 void be_prop_list_free(nvlist_t *be_list);
 
 int be_activate(libbe_handle_t *, const char *, bool);
+int be_deactivate(libbe_handle_t *, const char *, bool);
 
 bool be_is_auto_snapshot_name(libbe_handle_t *, const char *);
 

lib/libbe/be_impl.h

@@ -63,6 +63,7 @@ typedef struct prop_data {
 	nvlist_t *list;
 	libbe_handle_t *lbh;
 	bool single_object;	/* list will contain props directly */
+	char *bootonce;
 } prop_data_t;
 
 int prop_list_builder_cb(zfs_handle_t *, void *);

lib/libbe/be_info.c

@@ -30,6 +30,7 @@
 __FBSDID("$FreeBSD$");
 
 #include <sys/zfs_context.h>
+#include <libzfsbootenv.h>
 
 #include "be.h"
 #include "be_impl.h"
@@ -108,6 +109,7 @@ be_get_bootenv_props(libbe_handle_t *lbh, nvlist_t *dsnvl)
 	data.lbh = lbh;
 	data.list = dsnvl;
 	data.single_object = false;
+	data.bootonce = NULL;
 	return (be_proplist_update(&data));
 }
 
@@ -121,6 +123,7 @@ be_get_dataset_props(libbe_handle_t *lbh, const char *name, nvlist_t *props)
 	data.lbh = lbh;
 	data.list = props;
 	data.single_object = true;
+	data.bootonce = NULL;
 	if ((snap_hdl = zfs_open(lbh->lzh, name,
 	    ZFS_TYPE_FILESYSTEM | ZFS_TYPE_SNAPSHOT)) == NULL)
 		return (BE_ERR_ZFSOPEN);
@@ -140,6 +143,7 @@ be_get_dataset_snapshots(libbe_handle_t *lbh, const char *name, nvlist_t *props)
 	data.lbh = lbh;
 	data.list = props;
 	data.single_object = false;
+	data.bootonce = NULL;
 	if ((ds_hdl = zfs_open(lbh->lzh, name,
 	    ZFS_TYPE_FILESYSTEM)) == NULL)
 		return (BE_ERR_ZFSOPEN);
@@ -179,6 +183,10 @@ prop_list_builder_cb(zfs_handle_t *zfs_hdl, void *data_p)
 	dataset = zfs_get_name(zfs_hdl);
 	nvlist_add_string(props, "dataset", dataset);
 
+	if (data->bootonce != NULL &&
+	    strcmp(dataset, data->bootonce) == 0)
+		nvlist_add_boolean_value(props, "bootonce", true);
+
 	name = strrchr(dataset, '/') + 1;
 	nvlist_add_string(props, "name", name);
 
@@ -246,6 +254,9 @@ be_proplist_update(prop_data_t *data)
 	    ZFS_TYPE_FILESYSTEM)) == NULL)
 		return (BE_ERR_ZFSOPEN);
 
+	(void) lzbe_get_boot_device(zpool_get_name(data->lbh->active_phandle),
+	    &data->bootonce);
+
 	/* XXX TODO: some error checking here */
 	zfs_iter_filesystems(root_hdl, prop_list_builder_cb, data);
 

lib/libbe/libbe.3

@@ -27,7 +27,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd October 16, 2019
+.Dd July 22, 2020
 .Dt LIBBE 3
 .Os
 .Sh NAME
@@ -78,6 +78,10 @@
 .Pp
 .Ft int
 .Fn be_activate "libbe_handle_t *hdl" "const char *be_name" "bool temporary"
+.Pp
+.Ft int
+.Fn be_deactivate "libbe_handle_t *hdl" "const char *be_name" "bool temporary"
+.Pp
 .Ft int
 .Fn be_destroy "libbe_handle_t *hdl" "const char *be_name" "int options"
 .Pp
@@ -270,8 +274,24 @@ If the
 .Fa temporary
 flag is set, then it will be active for the next boot only, as done by
 .Xr zfsbootcfg 8 .
-Next boot functionality is currently only available when booting in x86 BIOS
-mode.
+.Pp
+The
+.Fn be_deactivate
+function deactivates a boot environment.
+If the
+.Fa temporary
+flag is set, then it will cause removal of boot once configuration, set by
+.Fn be_activate
+function or by
+.Xr zfsbootcfg 8 .
+If the
+.Fa temporary
+flag is not set,
+.Fn be_deactivate
+function will set zfs
+.Dv canmount
+property to
+.Dv noauto .
 .Pp
 The
 .Fn be_destroy

libexec/rc/rc.conf

@@ -64,6 +64,7 @@ rc_conf_files="/etc/rc.conf /etc/rc.conf.local"
 
 # ZFS support
 zfs_enable="NO"		# Set to YES to automatically mount ZFS file systems
+zfs_bootonce_activate="NO" # Set YES to make successful bootonce BE permanent
 
 # ZFSD support
 zfsd_enable="NO"	# Set to YES to automatically start the ZFS fault

libexec/rc/rc.d/zfsbe

@@ -48,6 +48,21 @@ mount_subordinate()
 	done
 }
 
+activate_bootonce()
+{
+	local _dev
+	local _bootonce
+	local _be
+
+	_dev=$1
+	_be=${_dev##*/}
+
+	_bootonce="`kenv -q zfs-bootonce`"
+	if [ "$_bootonce" = "zfs:${_dev}:" ] ; then
+		bectl activate $_be
+	fi
+}
+
 be_start()
 {
 	if [ `$SYSCTL_N security.jail.jailed` -eq 1 ]; then
@@ -57,6 +72,9 @@ be_start()
 			[ $_mp  = "/" ] || continue
 			if [ $_type = "zfs" ] ; then
 				mount_subordinate $_dev
+				if checkyesno zfs_bootonce_activate; then
+					activate_bootonce $_dev
+				fi
 			fi
 			break
 		done

rescue/rescue/Makefile

@@ -129,7 +129,7 @@ CRUNCH_PROGS_usr.sbin+= zdb
 CRUNCH_LIBS+= -l80211 -lalias -lcam -lncursesw -ldevstat -lipsec -llzma
 .if ${MK_ZFS} != "no"
 CRUNCH_LIBS+= -lavl -lzpool -lzfs_core -lzfs -lnvpair -lpthread -luutil -lumem
-CRUNCH_LIBS+= -lbe -lzutil -ltpool -lspl -licp_rescue
+CRUNCH_LIBS+= -lbe -lzfsbootenv -lzutil -ltpool -lspl -licp_rescue
 .else
 # liblzma needs pthread
 CRUNCH_LIBS+= -lpthread

sbin/bectl/Makefile

@@ -12,6 +12,7 @@ LIBADD+= be \
 	nvpair \
 	spl \
 	util \
+	zfsbootenv
 
 CFLAGS+= -DIN_BASE
 CFLAGS+= -I${SRCTOP}/sys/contrib/openzfs/include