Generalise libcompat to be a list rather than a single option

Whilst the kernel can support any number of COMPAT_FOO, world can only
build a single libfoo. Upstream this isn't such an issue, since the only
option is lib32 anyway, but downstreams, such as CheriBSD, may wish to
support multiple at the same time. Thus, adjust the top-level Makefiles
to turn _LIBCOMPAT into a _LIBCOMPATS list that gets iterated over, and
adjust bsd.compat.mk to support this use-case.

For the normal NEED_COMPAT/WANT_COMPAT case, LIBCOMPATFOO remain set and
refer to the requested compat's, preserving the current interface. For
the top-level Makefiles those variables are no longer set (since there
is no longer "the" compat) and only the per-compat ones are available.

Reviewed by:	brooks, jhb, imp
Differential Revision:	https://reviews.freebsd.org/D40571

Makefile.inc1

@@ -852,9 +852,9 @@ XCFLAGS+=	${BFLAGS}
 .endif
 
 .if ${MK_LIB32} == "yes"
-_LIBCOMPAT= 32
-.include "Makefile.libcompat"
+_LIBCOMPATS+=	32
 .endif
+.include "Makefile.libcompat"
 
 # META_MODE normally ignores host file changes since every build updates
 # timestamps (see NO_META_IGNORE_HOST in sys.mk).  There are known times
@@ -1100,9 +1100,9 @@ _cleanobj:
 	# Avoid including bsd.compiler.mk in clean and obj with _NO_INCLUDE_COMPILERMK
 	# since the restricted $PATH might not contain a valid cc binary
 	${_+_}cd ${.CURDIR}; ${WMAKE} _NO_INCLUDE_COMPILERMK=t ${CLEANDIR}
-.if defined(_LIBCOMPAT)
-	${_+_}cd ${.CURDIR}; ${LIBCOMPATWMAKE} _NO_INCLUDE_COMPILERMK=t -f Makefile.inc1 ${CLEANDIR}
-.endif
+.for LIBCOMPAT in ${_LIBCOMPATS}
+	${_+_}cd ${.CURDIR}; ${LIB${LIBCOMPAT}WMAKE} _NO_INCLUDE_COMPILERMK=t -f Makefile.inc1 ${CLEANDIR}
+.endfor
 .else
 	${_+_}cd ${.CURDIR}; ${WMAKE} _NO_INCLUDE_COMPILERMK=t _cleanobj_fast_depend_hack
 .endif	# ${MK_CLEAN} == "yes"
@@ -1179,8 +1179,10 @@ WMAKE_TGTS+=	_includes
 .if !defined(NO_LIBS)
 WMAKE_TGTS+=	_libraries
 .endif
-.if defined(_LIBCOMPAT) && empty(SUBDIR_OVERRIDE)
+.if empty(SUBDIR_OVERRIDE)
+.for libcompat in ${libcompats}
 WMAKE_TGTS+=	build${libcompat}
+.endfor
 .endif
 WMAKE_TGTS+=	everything
 
@@ -1241,7 +1243,7 @@ buildenv: .PHONY
 	INSTALL="${INSTALL_CMD} ${INSTALLFLAGS}" \
 	MTREE_CMD="${MTREE_CMD} ${MTREEFLAGS}" BUILDENV=1 ${BUILDENV_SHELL}
 
-TOOLCHAIN_TGTS=	${WMAKE_TGTS:Neverything:Nbuild${libcompat}}
+TOOLCHAIN_TGTS=	${WMAKE_TGTS:Neverything:${libcompats:@v@Nbuild${v}@:ts:}}
 toolchain: ${TOOLCHAIN_TGTS} .PHONY
 KERNEL_TOOLCHAIN_TGTS=	${TOOLCHAIN_TGTS:N_obj:N_cleanobj:N_includes:N_libraries}
 .if make(kernel-toolchain)
@@ -1362,9 +1364,9 @@ ITOOLS+=${LOCAL_ITOOLS}
 
 # Non-base distributions produced by the base system
 EXTRA_DISTRIBUTIONS=
-.if defined(_LIBCOMPAT)
+.for libcompat in ${libcompats}
 EXTRA_DISTRIBUTIONS+=	lib${libcompat}
-.endif
+.endfor
 .if ${MK_TESTS} != "no"
 EXTRA_DISTRIBUTIONS+=	tests
 .endif
@@ -1420,14 +1422,14 @@ distributeworld installworld stageworld: _installcheck_world .PHONY
 	${DESTDIR_MTREE} -f ${.CURDIR}/etc/mtree/BSD.debug.dist \
 	    -p ${DESTDIR}/${DISTDIR}/${dist}/usr/lib >/dev/null
 .endif
-.if defined(_LIBCOMPAT)
+.for libcompat in ${libcompats}
 	${DESTDIR_MTREE} -f ${.CURDIR}/etc/mtree/BSD.lib${libcompat}.dist \
 	    -p ${DESTDIR}/${DISTDIR}/${dist}/usr >/dev/null
 .if ${MK_DEBUG_FILES} != "no"
 	${DESTDIR_MTREE} -f ${.CURDIR}/etc/mtree/BSD.lib${libcompat}.dist \
 	    -p ${DESTDIR}/${DISTDIR}/${dist}/usr/lib/debug/usr >/dev/null
 .endif
-.endif
+.endfor
 .if ${MK_TESTS} != "no" && ${dist} == "tests"
 	-mkdir -p ${DESTDIR}/${DISTDIR}/${dist}${TESTSBASE}
 	${DESTDIR_MTREE} -f ${.CURDIR}/etc/mtree/BSD.tests.dist \
@@ -1447,10 +1449,10 @@ distributeworld installworld stageworld: _installcheck_world .PHONY
 .if ${TARGET_ARCH} == "amd64"
 	echo "./${dist}/usr/include/i386 type=dir uname=root gname=wheel mode=0755" >> ${METALOG}
 .endif
-.if defined(_LIBCOMPAT)
+.for libcompat in ${libcompats}
 	${IMAKEENV} ${DISTR_MTREE} -C -f ${.CURDIR}/etc/mtree/BSD.lib${libcompat}.dist | \
 	    sed -e 's#^\./#./${dist}/usr/#' >> ${METALOG}
-.endif
+.endfor
 .endif
 .endfor
 	-mkdir ${DESTDIR}/${DISTDIR}/base
@@ -1579,9 +1581,9 @@ restage reinstall: .MAKE .PHONY
 	@echo ">>> Installing everything started on `LC_ALL=C date`"
 	@echo "--------------------------------------------------------------"
 	${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 install
-.if defined(_LIBCOMPAT)
+.for libcompat in ${libcompats}
 	${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 install${libcompat}
-.endif
+.endfor
 	@echo "--------------------------------------------------------------"
 	@echo ">>> Installing everything completed on `LC_ALL=C date`"
 	@echo "--------------------------------------------------------------"
@@ -1591,10 +1593,10 @@ redistribute: .MAKE .PHONY
 	@echo ">>> Distributing everything"
 	@echo "--------------------------------------------------------------"
 	${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 distribute
-.if defined(_LIBCOMPAT)
+.for libcompat in ${libcompats}
 	${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 distribute${libcompat} \
 	    DISTRIBUTION=lib${libcompat}
-.endif
+.endfor
 
 distrib-dirs distribution: .MAKE .PHONY
 	${_+_}cd ${.CURDIR}/etc; ${CROSSENV} PATH=${TMPPATH:Q} ${MAKE} \
@@ -1678,7 +1680,7 @@ _cleankernobj_fast_depend_hack: .PHONY
 .endfor
 .endif
 
-${WMAKE_TGTS:N_worldtmp:Nbuild${libcompat}} ${.ALLTARGETS:M_*:N_worldtmp}: .MAKE .PHONY
+${WMAKE_TGTS:N_worldtmp:${libcompats:@v@Nbuild${v}@:ts:}} ${.ALLTARGETS:M_*:N_worldtmp}: .MAKE .PHONY
 
 # record kernel(s) build time in seconds
 .if make(buildkernel)
@@ -3589,10 +3591,10 @@ _xi-mtree: .PHONY
 .if ${TARGET_ARCH} == "amd64"
 	mkdir -p ${XDDESTDIR}/usr/include/i386
 .endif
-.if defined(_LIBCOMPAT)
+.for libcompat in ${libcompats}
 	${DESTDIR_MTREE} -f ${.CURDIR}/etc/mtree/BSD.lib${libcompat}.dist \
 	    -p ${XDDESTDIR}/usr >/dev/null
-.endif
+.endfor
 .if ${MK_TESTS} != "no"
 	mkdir -p ${XDDESTDIR}${TESTSBASE}
 	${DESTDIR_MTREE} -f ${.CURDIR}/etc/mtree/BSD.tests.dist \

Makefile.libcompat

@@ -6,10 +6,14 @@ __<${_this:T}>__:
 # Makefile for the compatibility libraries.
 # - 32-bit compat libraries on PowerPC, and AMD64.
 
+.if defined(_LIBCOMPATS)
+
 .include <bsd.compat.mk>
 
+.for libcompatidx _LIBCOMPAT libcompat in ${_LIBCOMPATS:range:@i@${i} ${_LIBCOMPATS:[$i]} ${libcompats:[$i]}@}
+
 # Yes, the flags are redundant.
-LIBCOMPATWMAKEENV+= \
+LIB${_LIBCOMPAT}WMAKEENV+= \
 		INSTALL="${INSTALL_CMD} -U" \
 		PATH=${TMPPATH} \
 		SYSROOT=${WORLDTMP} \
@@ -19,22 +23,23 @@ LIBCOMPATWMAKEENV+= \
 		DTRACE="${LIB$COMPATDTRACE:U${DTRACE}}"
 .if ${MK_META_MODE} != "no"
 # Don't rebuild build-tools targets during normal build.
-LIBCOMPATWMAKEENV+=	BUILD_TOOLS_META=.NOMETA
+LIB${_LIBCOMPAT}WMAKEENV+=	BUILD_TOOLS_META=.NOMETA
 .endif
-LIBCOMPATWMAKEFLAGS+= CC="${XCC} ${LIBCOMPATCFLAGS}" \
-		CXX="${XCXX} ${LIBCOMPATCXXFLAGS} ${LIBCOMPATCFLAGS}" \
-		CPP="${XCPP} ${LIBCOMPATCFLAGS}" \
+LIB${_LIBCOMPAT}WMAKEFLAGS+= CC="${XCC} ${LIB${_LIBCOMPAT}CFLAGS}" \
+		CXX="${XCXX} ${LIB${_LIBCOMPAT}CXXFLAGS} ${LIB${_LIBCOMPAT}CFLAGS}" \
+		CPP="${XCPP} ${LIB${_LIBCOMPAT}CFLAGS}" \
 		DESTDIR=${WORLDTMP} \
 		-DNO_CPU_CFLAGS \
 		MK_BOOT=no \
 		MK_CTF=no \
 		MK_TESTS=no
-LIBCOMPATWMAKE+=	${TIME_ENV} ${LIBCOMPATWMAKEENV} ${MAKE} ${LIBCOMPATWMAKEFLAGS} \
-			OBJTOP=${LIBCOMPAT_OBJTOP} \
+LIB${_LIBCOMPAT}WMAKE+=	${TIME_ENV} ${LIB${_LIBCOMPAT}WMAKEENV} ${MAKE} \
+			${LIB${_LIBCOMPAT}WMAKEFLAGS} \
+			OBJTOP=${LIB${_LIBCOMPAT}_OBJTOP} \
 			OBJROOT='$${OBJTOP}/' \
 			MAKEOBJDIRPREFIX= \
 			MK_MAN=no MK_HTML=no
-LIBCOMPATIMAKE+=	${LIBCOMPATWMAKE:NINSTALL=*:NDESTDIR=*} \
+LIB${_LIBCOMPAT}IMAKE+=	${LIB${_LIBCOMPAT}WMAKE:NINSTALL=*:NDESTDIR=*} \
 			${IMAKE_INSTALL} \
 			MK_INCLUDES=no \
 			-DLIBRARIES_ONLY
@@ -43,7 +48,7 @@ LIBCOMPATIMAKE+=	${LIBCOMPATWMAKE:NINSTALL=*:NDESTDIR=*} \
 build${libcompat}: .PHONY
 	@echo
 	@echo "--------------------------------------------------------------"
-	@echo ">>> stage 4.3: building lib${libcompat} shim libraries"
+	@echo ">>> stage 4.3.${libcompatidx}: building lib${libcompat} shim libraries"
 	@echo "--------------------------------------------------------------"
 	${WORLDTMP_MTREE} -f ${.CURDIR}/etc/mtree/BSD.lib${libcompat}.dist \
 	    -p ${WORLDTMP}/usr >/dev/null
@@ -56,7 +61,7 @@ build${libcompat}: .PHONY
 	    MAKEFLAGS="-m ${.CURDIR}/tools/build/mk ${.MAKEFLAGS}" \
 	    ${MAKE} -f Makefile.inc1 \
 	    DESTDIR= \
-	    OBJTOP=${LIBCOMPAT_OBJTOP} \
+	    OBJTOP=${LIB${_LIBCOMPAT}_OBJTOP} \
 	    OBJROOT='$${OBJTOP}/' \
 	    MAKEOBJDIRPREFIX= \
 	    -DNO_LINT \
@@ -69,11 +74,15 @@ build${libcompat}: .PHONY
 	    MK_WERROR=no \
 	    _lc_build-tools
 	${_+_}cd ${.CURDIR}; \
-	    ${LIBCOMPATWMAKE} -f Makefile.inc1 -DNO_FSCHG libraries
+	    ${LIB${_LIBCOMPAT}WMAKE} -f Makefile.inc1 -DNO_FSCHG libraries
 
 distribute${libcompat} install${libcompat}: .PHONY
 	${_+_}cd ${.CURDIR}; \
-	    ${LIBCOMPATIMAKE} -f Makefile.inc1 _lc_${.TARGET:S/${libcompat}$//}
+	    ${LIB${_LIBCOMPAT}IMAKE} -f Makefile.inc1 _lc_${.TARGET:S/${libcompat}$//}
+
+.endfor # libcompatidx _LIBCOMPAT libcompat
+
+.endif # defined(_LIBCOMPATS)
 
 # These depend on MK_*, which could vary between the native and compat
 # architectures. Thus targets using them are run within a compat sub-make

share/mk/bsd.compat.mk

@@ -3,9 +3,13 @@
 .if !targets(__<${_this:T}>__)
 __<${_this:T}>__:
 
-.if defined(_LIBCOMPAT)
+_ALL_LIBCOMPATS:=	32
+
+.if defined(_LIBCOMPATS)
 COMPAT_ARCH?=	${TARGET_ARCH}
-COMPAT_CPUTYPE?= ${CPUTYPE_${_LIBCOMPAT}}
+.for _LIBCOMPAT in ${_ALL_LIBCOMPATS}
+LIB${_LIBCOMPAT}CPUTYPE?=	${CPUTYPE_${_LIBCOMPAT}}
+.endfor
 .if (defined(WANT_COMPILER_TYPE) && ${WANT_COMPILER_TYPE} == gcc) || \
     (defined(X_COMPILER_TYPE) && ${X_COMPILER_TYPE} == gcc)
 COMPAT_COMPILER_TYPE=	gcc
@@ -14,19 +18,21 @@ COMPAT_COMPILER_TYPE=	clang
 .endif
 .else
 COMPAT_ARCH=	${MACHINE_ARCH}
-COMPAT_CPUTYPE=	${CPUTYPE}
+.for _LIBCOMPAT in ${_ALL_LIBCOMPATS}
+LIB${_LIBCOMPAT}CPUTYPE=	${CPUTYPE}
+.endfor
 .include <bsd.compiler.mk>
 COMPAT_COMPILER_TYPE=${COMPILER_TYPE}
 .endif
 
 # -------------------------------------------------------------------
 # 32 bit world
 .if ${COMPAT_ARCH} == "amd64"
-HAS_COMPAT=32
-.if empty(COMPAT_CPUTYPE)
+HAS_COMPAT+=	32
+.if empty(LIB32CPUTYPE)
 LIB32CPUFLAGS=	-march=i686 -mmmx -msse -msse2
 .else
-LIB32CPUFLAGS=	-march=${COMPAT_CPUTYPE}
+LIB32CPUFLAGS=	-march=${LIB32CPUTYPE}
 .endif
 .if ${COMPAT_COMPILER_TYPE} == gcc
 .else
@@ -40,11 +46,11 @@ LIB32WMAKEFLAGS=	\
 		LD="${XLD} -m elf_i386_fbsd"
 
 .elif ${COMPAT_ARCH} == "powerpc64"
-HAS_COMPAT=32
-.if empty(COMPAT_CPUTYPE)
+HAS_COMPAT+=	32
+.if empty(LIB32CPUTYPE)
 LIB32CPUFLAGS=	-mcpu=powerpc
 .else
-LIB32CPUFLAGS=	-mcpu=${COMPAT_CPUTYPE}
+LIB32CPUFLAGS=	-mcpu=${LIB32CPUTYPE}
 .endif
 
 .if ${COMPAT_COMPILER_TYPE} == "gcc"
@@ -89,43 +95,50 @@ WANT_COMPAT:=	${NEED_COMPAT}
 
 .if defined(HAS_COMPAT) && defined(WANT_COMPAT)
 .if ${WANT_COMPAT} == "any"
-_LIBCOMPAT:=	${HAS_COMPAT:[1]}
+USE_COMPAT:=	${HAS_COMPAT:[1]}
 .else
-_LIBCOMPAT:=	${WANT_COMPAT}
+USE_COMPAT:=	${WANT_COMPAT}
 .endif
+
+_LIBCOMPATS=	${USE_COMPAT}
 .endif
 
+libcompats=	${_LIBCOMPATS:tl}
+
 # -------------------------------------------------------------------
 # Generic code for each type.
 # Set defaults based on type.
-libcompat=	${_LIBCOMPAT:tl}
-_LIBCOMPAT_MAKEVARS=	_OBJTOP TMP CPUFLAGS CFLAGS CXXFLAGS LDFLAGS \
-			_MACHINE _MACHINE_ARCH _MACHINE_ABI \
-			WMAKEENV WMAKEFLAGS WMAKE WORLDTMP
-.for _var in ${_LIBCOMPAT_MAKEVARS}
-.if !empty(LIB${_LIBCOMPAT}${_var})
-LIBCOMPAT${_var}?=	${LIB${_LIBCOMPAT}${_var}}
-.endif
-.endfor
-
+.for _LIBCOMPAT _libcompat in ${_LIBCOMPATS:@v@${v} ${v:tl}@}
 WORLDTMP?=		${SYSROOT}
 
 # Shared flags
-LIBCOMPAT_OBJTOP?=	${OBJTOP}/obj-lib${libcompat}
+LIB${_LIBCOMPAT}_OBJTOP?=	${OBJTOP}/obj-lib${_libcompat}
 
-LIBCOMPATCFLAGS+=	${LIBCOMPATCPUFLAGS} \
+LIB${_LIBCOMPAT}CFLAGS+=	${LIB${_LIBCOMPAT}CPUFLAGS} \
 			--sysroot=${WORLDTMP} \
 			${BFLAGS}
 
-LIBCOMPATLDFLAGS+=	-L${WORLDTMP}/usr/lib${libcompat}
+LIB${_LIBCOMPAT}LDFLAGS+=	-L${WORLDTMP}/usr/lib${_libcompat}
 
-LIBCOMPATWMAKEENV+=	MACHINE=${LIBCOMPAT_MACHINE}
-LIBCOMPATWMAKEENV+=	MACHINE_ARCH=${LIBCOMPAT_MACHINE_ARCH}
+LIB${_LIBCOMPAT}WMAKEENV+=	MACHINE=${LIB${_LIBCOMPAT}_MACHINE}
+LIB${_LIBCOMPAT}WMAKEENV+=	MACHINE_ARCH=${LIB${_LIBCOMPAT}_MACHINE_ARCH}
 
 # -B is needed to find /usr/lib32/crti.o for gcc.
-LIBCOMPATCFLAGS+=	-B${WORLDTMP}/usr/lib${libcompat}
+LIB${_LIBCOMPAT}CFLAGS+=	-B${WORLDTMP}/usr/lib${_libcompat}
+.endfor
+
+.if defined(USE_COMPAT)
+libcompat=	${USE_COMPAT:tl}
+
+_LIBCOMPAT_MAKEVARS=	_OBJTOP TMP CPUFLAGS CFLAGS CXXFLAGS LDFLAGS \
+			_MACHINE _MACHINE_ARCH _MACHINE_ABI \
+			WMAKEENV WMAKEFLAGS WMAKE WORLDTMP
+.for _var in ${_LIBCOMPAT_MAKEVARS}
+.if !empty(LIB${USE_COMPAT}${_var})
+LIBCOMPAT${_var}?=	${LIB${USE_COMPAT}${_var}}
+.endif
+.endfor
 
-.if defined(WANT_COMPAT)
 LIBDIR_BASE:=	/usr/lib${libcompat}
 LIBDATADIR:=	/usr/lib${libcompat}
 _LIB_OBJTOP=	${LIBCOMPAT_OBJTOP}