From Gentoo Wiki
Jump to:navigation Jump to:search

Gentoo on ZFS with native encryption on Root, TPM2 and per-user YubiKey

This page describes my personal steps of installing Gentoo on ZFS with native encryption on root enabled, TPM2 and per-user YubiKey support.


  • Add support for unlocking home directory using the fingerprint scanner as main or/and alternative - right now that is not important.

Hardware: XMG Ultra 17

Device Model Status Bus ID Driver Notes
CPU Intel i9-9900k Works None
RAM #1 Samsung DDR4 2666MHz 32 GB Works None
RAM #2 Samsung DDR4 2666MHz 32 GB Works None
Ethernet Qualcomm Atheros Killer E2500 Works None
WiFi Intel® Wireless-AC 9260 Works None
Bluetooth Intel® Wireless-AC 9260 Works 8087:0025 None
Touchpad Synaptics Works None
Fingerprint Sensor Synaptics Validity90 Unsupported 06cb:0078 WIP
MMC Reader Realtek RTS5250 Works None
Integrated Video Card Intel UHD Graphics 630 Disconnected Lanes disconnected on PCB, thanks XMG

(no mention in their docs even)

Dedicated Video Card NVIDIA RTX 2080 Mobile Works None
NVME #1 Samsung SSD 970 EVO Plus 1TB Works None
NVME #2 Samsung SSD 970 EVO 1TB Works None
SATA #1 Samsung SSD 750 Works None
SATA #2 Samsung SSD 850 Works None


  1. Time and a little bit knowledge
  2. Bootable Live Unix-based distro with ZFS support (like nchevsky/systemrescue-zfs)
  3. YubiKey (at least 2 for backup)
  4. Device with TPM2
  5. Internet connection

Preparing live environment

Boot to the image and setup the environment as follow:

root #setkmap
root #timedatectl set-timezone Europe/Berlin
root #timedatectl set-ntp true
root #zgenhostid $(hostid)
root #echo $((2 * 1024 * 1024 * 1024)) > /sys/module/zfs/parameters/zfs_arc_min
root #echo $((8 * 1024 * 1024 * 1024)) > /sys/module/zfs/parameters/zfs_arc_max
root #mkdir -p /{etc/zfs, mnt/gentoo}

Disk layout

NVME drives will be used as ZFS pools without partition. SATA drives will contain 3 partitions: EFI (512 MiB), SWAP (32 GiB) and remaining space will be assigned for the ZFS pool.


sfdisk --wipe=always --wipe-partitions=always ${NVME_DISK1} <<EOF
label: gpt
sector-size: 512

sfdisk --wipe=always --wipe-partitions=always ${NVME_DISK2} <<EOF
label: gpt
sector-size: 512

sfdisk --wipe=always --wipe-partitions=always ${ATA_DISK1} <<EOF
label: gpt
unit: sectors
sector-size: 512

start=        2048, size=     1048576, type=EBD0A0A2-B9E5-4433-87C0-68B6B72699C7, bootable
start=     1050624, size=    67108864, type=0657FD6D-A4AB-43C4-84E5-0933C84B4F4F
start=    68159488,                    type=0FC63DAF-8483-4772-8E79-3D69D8477DE4

sfdisk --wipe=always --wipe-partitions=always ${ATA_DISK2} <<EOF
label: gpt
unit: sectors
sector-size: 512

start=        2048, size=     1048576, type=EBD0A0A2-B9E5-4433-87C0-68B6B72699C7, bootable
start=     1050624, size=    67108864, type=0657FD6D-A4AB-43C4-84E5-0933C84B4F4F
start=    68159488,                    type=0FC63DAF-8483-4772-8E79-3D69D8477DE4

EFI (512 MiB available)

First partition of SATA #1 and SATA #2 will represent the EFI partition and mirrored.

root #mdadm --create --run --verbose /dev/md127 --level=1 --raid-devices=2 --metadata=0.90 ${ATA_DISK1}-part1 ${ATA_DISK2}-part1
root #mkfs.vfat -F32 /dev/md127

ZPOOL (1.33 TiB available)

First partition of NVME #1 and NVME #2 will be in RAID 1 and the third partition of SATA #1 and SATA #2 will be also in RAID 1. All partition mentioned will be used to create a single mirrored ZPOOL.

root #zpool create -o ashift=12 -o cachefile="/etc/zfs/zpool.cache" -o autotrim=on -o autoexpand=on -O mountpoint=none -O canmount=off -O compression=zstd -O xattr=sa -O relatime=on -O atime=off -O acltype=posix -O dnodesize=auto -O normalization=formD -O devices=off -O setuid=off -O exec=off -m none -R "/mnt/gentoo" "rpool" mirror ${NVME_DISK1} ${NVME_DISK2} mirror ${ATA_DISK1}-part3 ${ATA_DISK2}-part3

SWAP (64 GiB available)

Second partition of SATA #1 and SATA #2 will be used as is to create SWAP partition.

root #mkswap ${ATA_DISK1}-part2
root #mkswap ${ATA_DISK2}-part2

ZFS Datasets

System Datasets

root #zfs create -o canmount=noauto -o mountpoint=/ -o exec=on -o encryption=on -o keyformat=passphrase -o keylocation=prompt rpool/ROOT
root #zpool set bootfs="rpool/ROOT" rpool
root #zfs create -o canmount=off -o exec=off rpool/ROOT/var
root #zfs create -o canmount=off rpool/ROOT/var/lib
/var/lib/docker - optional
root #zfs create rpool/ROOT/var/lib/docker
root #zfs create -o canmount=off rpool/ROOT/var/cache
root #zfs create rpool/ROOT/var/cache/distfiles
root #zfs create rpool/ROOT/var/cache/binpkgs
root #zfs create -o canmount=off rpool/ROOT/var/db
root #zfs create rpool/ROOT/var/db/pkg
root #zfs create rpool/ROOT/var/db/repos
root #zfs create rpool/ROOT/var/lib/selinux
root #zfs create rpool/ROOT/var/lib/sepolgen
root #zfs create rpool/ROOT/var/lib/portage
root #zfs create rpool/ROOT/var/log
root #zfs create -o sync=disabled -o exec=off rpool/ROOT/tmp
root #zfs create -o canmount=off -o exec=off rpool/ROOT/usr
root #zfs create -o exec=on rpool/ROOT/usr/src

User data datasets

root #zfs create -o canmount=off -o mountpoint=none rpool/USERDATA
root #zfs create -o canmount=noauto -o mountpoint="/home/theaifam5" -o encryption=on -o keyformat=passphrase -o keylocation=prompt -o exec=on rpool/USERDATA/theaifam5
/var/db/repos/theaifam5 - optional
root #zfs create -o mountpoint="/var/db/repos/theaifam5" -o exec=off rpool/USERDATA/theaifam5/portage

Dataset structure verification

You might want to verify the structure of created datasets:

root #zfs list -o name,used,available,mountpoint,canmount,sync,utf8only,encryption,mounted,exec,setuid,devices,com.sun:auto-snapshot

Export ZFS pool:

root #zpool export rpool

Re-import ZFS pool and load keys:

root #zpool import -R "/mnt/gentoo" rpool
root #zfs load-key rpool/ROOT
root #zfs load-key rpool/USERDATA/theaifam5

Afterwards you can mount the home directory from the userdata dataset:

root #zfs mount rpool/ROOT
root #zfs mount rpool/USERDATA/theaifam5
root #zfs mount -a

Extraction and setup

First of all, we need to extract the Gentoo onto ZFS, it might take a while:

root #tar xpf /tmp/stage3.tar.xz --xattrs-include='*.*' --numeric-owner -C /mnt/gentoo

Afterwards we can create an EFI directory and mount it:

root #mkdir -p /mnt/gentoo/boot/efi
root #mount /dev/md127 /mnt/gentoo/boot/efi

Also ZFS Cache file, resolv.conf and hostid needs to be copied:

root #mkdir -p /mnt/gentoo/etc/zfs
root #cp /etc/zfs/zpool.cache /mnt/gentoo/etc/zfs/zpool.cache
root #cp --dereference /etc/resolv.conf /mnt/gentoo/etc/resolv.conf
root #cp /etc/hostid /mnt/gentoo/etc/hostid


Everything should be prepared now, we can chroot into the environment using:

root #mount --rbind /dev /mnt/gentoo/dev
root #mount --rbind /proc /mnt/gentoo/proc
root #mount --rbind /sys /mnt/gentoo/sys
root #mount --make-rslave /mnt/gentoo/dev
root #mount --make-rslave /mnt/gentoo/proc
root #mount --make-rslave /mnt/gentoo/sys
root #test -L /dev/shm && rm /dev/shm && mkdir /dev/shm
root #mount --types tmpfs --options nosuid,nodev,noexec shm /dev/shm
root #chmod 1777 /dev/shm
root #env -i HOME=/root TERM=$TERM chroot /mnt/gentoo bash

Now, just setup the chrooted environment:

(chroot) rootsource /etc/profile
(chroot) rootenv-update
(chroot) rootexport PS1="(chroot) ${PS1}"


Base System

Select the default gentoo repository, it will fetch the state of the last 24 hours

(chroot) rootmkdir -p /etc/portage/{repos.conf,package.{use,mask,unmask,provided}}
(chroot) rootcp /usr/share/portage/config/repos.conf /etc/portage/repos.conf/gentoo.conf
(chroot) rootemerge-webrsync

You might want also to update to the newest but only if is required:

(chroot) rootemerge --sync

Put Portage and genkernel TMPDIR on tmpfs

FILE /etc/fstab
tmpfs		/var/tmp/portage		tmpfs	size=8G,uid=portage,gid=portage,mode=775,nosuid,noatime,nodev	0 0
tmpfs		/var/tmp/genkernel		tmpfs	size=4G,mode=775,nosuid,noatime,nodev	0 0

Add required use flags:

FILE /etc/portage/package.use/dev-python
>=dev-python/numpy-1.21.1 lapack
FILE /etc/portage/package.use/sys-devel
>=sys-devel/gcc-10.3.0-r2::gentoo fortran
>=sys-devel/llvm-12.0.1::gentoo gold xar xml z3
>=sys-devel/clang-12.0.1::gentoo default-compiler-rt default-libcxx default-lld llvm-libunwind xml static-analyzer
>=sys-devel/clang-runtime::gentoo libcxx
FILE /etc/portage/package.use/sci-mathematics
>=sci-mathematics/z3-4.8.11b::gentoo python

Setup locale

FILE /etc/locale.gen
en_US.UTF-8 UTF-8

Generate locale

(chroot) rootrm /etc/localtime
(chroot) rootln -s /usr/share/zoneinfo/Europe/Berlin /etc/localtime
(chroot) rootemerge --config sys-libs/timezone-data
(chroot) rootlocale-gen

Select locale

(chroot) rooteselect locale list
(chroot) rooteselect locale set <locale>

List and verify that the proper profile is selected:

(chroot) rooteselect profile list
(chroot) rooteselect profile set <profile>

Configure portage make.conf

FILE /etc/portage/make.conf
COMMON_FLAGS="-march=native -ggdb ..."
RUSTFLAGS="-C target-cpu=native"
MAKEOPTS="-j16 -l12.0"
EMERGE_DEFAULT_OPTS="-q -v --with-bdeps=y --complete-graph --jobs 16 --load-average 12.0"
FEATURES="buildpkg candy cgroup collision-protect compress-build-logs compress-index compressdebug downgrade-backup fakeroot installsources mount-sandbox parallel-install splitdebug suidctl -nostrip -binpkg-dostrip"
LINGUAS="en en_US"
L10N="en en-US"


FILE /etc/security/limits.d/50-memlock.conf
# This both raises the max and sets the default lockable memory limit of every process running under a non-system account (except for nobody) from default 64 to 256 kilobytes (in increments of page size)
1000:65533      -    memlock 256

Mount tmpfs and update packages

(chroot) rootmkdir -p /var/tmp/{portage,genkernel}
(chroot) rootmount -a
(chroot) rootemerge -uDN @world

Update OpenRC configuration

FILE /etc/rc.conf

Install tools

(chroot) rootemerge --oneshot app-portage/cpuid2cpuflags
(chroot) rootemerge dev-util/debugedit app-portage/gentoolkit sys-apps/fakeroot
(chroot) rootcpuid2cpuflags | cut -d" " -f2-
aes avx avx2 f16c fma3 mmx mmxext pclmul popcnt rdrand sse sse2 sse3 sse4_1 sse4_2 ssse3

Copy the output from cpuid2cpuflags command and configure portage make.conf

FILE /etc/portage/make.conf
CPU_FLAGS_X86="aes avx avx2 f16c fma3 mmx mmxext pclmul popcnt rdrand sse sse2 sse3 sse4_1 sse4_2 ssse3"

Configure USE flags

(chroot) rooteuse -E elogind static-libs audit threads policykit pam dbus gmp savedconfig gdbm source
(chroot) rooteuse -D systemd gnome qt5 gtk kde cjk tk

Disable PAM for busybox:

FILE /etc/portage/package.use/sys-apps
>=sys-apps/busybox-1.32.1-r1::gentoo -pam

Uninstall cpuid2cpuflags and cleanup:

(chroot) rootemerge --unmerge app-portage/cpuid2cpuflags
(chroot) rootemerge --depclean -a

Update world:

(chroot) rootemerge -uDN @world

Enable unstable branch:

FILE /etc/portage/make.conf

Follow the libxcrypt migration if needed here: Project:Toolchain/libcrypt implementation or just execute:

(chroot) rootmv /usr/include/crypt.h /usr/include/crypt.h.bak
(chroot) rootemerge --oneshot -av1 virtual/libcrypt sys-libs/libxcrypt

Update the world, remove backup of crypt.h and cleanup:

(chroot) rootemerge -uDN @world
(chroot) rootrm /usr/include/crypt.h.bak
(chroot) rootetc-update
(chroot) rootemerge --depclean -a

Automatically symlink the kernel:

FILE /etc/portage/package.use/sys-kernel
>=sys-kernel/pf-sources-5.14_p1::gentoo symlink

Install microcode:

FILE /etc/portage/package.use/sys-firmware
>=sys-firmware/intel-microcode-20210608_p20210830::gentoo split-ucode hostonly

Put it into portage make.conf:

FILE /etc/portage/make.conf

Install genkernel, pf-sources and intel-microcode:

(chroot) rootemerge sys-kernel/genkernel sys-kernel/pf-sources sys-firmware/intel-microcode

Check if kernel is selected:

(chroot) rooteselect kernel list

Configure genkernel:

FILE /etc/genkernel.conf
MAKEOPTS="$(portageq envvar MAKEOPTS)"

Create wrapper around genkernel:

FILE /usr/local/bin/genkernel
#!/bin/env bash


if eix-installed -a | grep -q sys-fs/zfs-kmod-2.1.0; then

if /usr/bin/genkernel "$@"; then
	mkdir -p "${BOOT_ENTRY_PATH%%/}"

	if [ -f /boot/kernel ]; then
		echo ">> Copy efistub to ${BOOT_ENTRY_PATH%%/}..."
		cp -L /boot/kernel "${BOOT_EFI_PATH}"

	if [ -f /boot/initramfs ]; then
		echo ">> Copy initramfs to ${BOOT_ENTRY_PATH%%/}..."
		cp -L /boot/initramfs "${BOOT_INITRAMFS_PATH}"

	if [ -f "${BOOT_EFI_PATH}" ] && [ -f "${BOOT_INITRAMFS_PATH}" ]; then
		mount /sys/firmware/efi/efivars -o rw,remount
		efiboot_num=$(efibootmgr | sed -n 's/^Boot\([[:digit:]]\+\).*Gentoo.*/\1/p')
		[ ! -z $efiboot_num ] && efibootmgr -B -b "${efiboot_num}"
		efibootmgr --create --part 1 --disk /dev/sda --label "Gentoo" --loader "${EFI_PATH%%/}/${EFI_NAME}" --unicode "dozfs root=ZFS initrd=${EFI_PATH%%/}/${INITRAMFS_NAME}"
		efibootmgr -D
		mount /sys/firmware/efi/efivars -o ro,remount

And make it executable:

(chroot) rootchmod +x /usr/local/bin/genkernel

System Packages

FILE /etc/portage/package.use/media-plugins
>=media-plugins/alsa-plugins-1.2.2::gentoo pulseaudio
FILE /etc/portage/package.use/x11-libs
>=x11-libs/libxkbcommon-1.3.0::gentoo X
>=x11-libs/cairo-1.16.0-r4::gentoo X
FILE /etc/portage/package.use/media-libs
>=media-libs/libglvnd-1.3.4::gentoo X
>=media-libs/freetype-2.11.0-r1::gentoo harfbuzz
>=media-libs/libpng-1.6.37-r2::gentoo apng
>=media-libs/libvpx-1.10.0::gentoo postproc
>=media-libs/harfbuzz-2.9.1 icu
>=media-libs/gd-2.3.2 fontconfig truetype
FILE /etc/portage/package.use/dev-libs
>=dev-libs/libpcre2-10.37-r2::gentoo pcre32
>=dev-libs/libotf-0.9.16::gentoo X
>=dev-libs/m17n-lib-1.8.0::gentoo X athena fontconfig gd libotf libxml spell xft
FILE /etc/portage/package.use/sys-apps
>=sys-apps/pciutils-3.7.0-r1::gentoo -static-libs
FILE /etc/portage/package.use/sys-libs
>=sys-libs/libomp-12.0.1::gentoo cuda hwloc ompt offload

Add kernel config

FILE /etc/kernels/kernel-config-5.14.0-pf1

Build kernel & Update config:

(chroot) rootgenkernel --no-zfs kernel

Install system packages:

(chroot) rootemerge app-portage/flaggie sys-devel/gdb sys-boot/efibootmgr app-admin/sudo x11-base/xorg-server sys-fs/dosfstools app-portage/layman app-portage/eix net-firewall/nftables sys-apps/rng-tools dev-libs/libinput x11-drivers/xf86-input-libinput app-shells/fish app-admin/syslog-ng app-admin/logrotate sys-fs/mdadm sys-auth/rtkit x11-drivers/nvidia-drivers dev-util/nvidia-cuda-toolkit dev-lua/luarocks www-client/firefox

Enable services:

(chroot) rootrc-update add mdraid boot
(chroot) rootrc-update add elogind boot
(chroot) rootrc-update add nftables default
(chroot) rootrc-update add syslog-ng default
(chroot) rootrc-update add dbus default
(chroot) root# rc-update add acpid default

Adjust eix configuration:

FILE /etc/eix-sync.conf

Enable KMS for nvidia driver:

FILE /etc/modprobe.d/nvidia.conf
options nvidia-drm modeset=1

ZFS Module & Tools

Apply compatibility patches for 5.14 kernels

(chroot) rootmkdir -p /etc/portage/patches/sys-fs/zfs-kmod-2.1.0
(chroot) rootpushd /etc/portage/patches/sys-fs/zfs-kmod-2.1.0
(chroot) rootpopd

Install ZFS 2.1.0 and set compatibility kernel version to 5.14

(chroot) rootZFS_KERNEL_COMPAT_OVERRIDE=5.14 emerge sys-fs/zfs

Configure ARC

FILE /etc/modprobe.d/zfs.conf
options zfs zfs_arc_min=2147483648 zfs_arc_max=8589934592

Check if /etc/hostid is present, otherwise execute

(chroot) rootzgenhostid $(hostid)

Enable services

(chroot) rootrc-update add zfs-import boot
(chroot) rootrc-update add zfs-mount boot
(chroot) rootrc-update add zfs-share default
(chroot) rootrc-update add zfs-zed default

Build Kernel & Initramfs & EfiStub

Generate mdamd.conf

(chroot) rootmdadm --detail --scan > /etc/mdadm.conf

Build kernel, initramfs, adjust EFI entries

(chroot) rootZFS_KERNEL_COMPAT_OVERRIDE=5.14 genkernel all

Add SWAP and boot parition to fstab

(chroot) rootSWAP_UUID=$(blkid | grep swap | sed -n '/.*/s/.*PARTUUID=\"\([^\"]*\)\".*/PARTUUID=\"\1\" none swap sw,discard 0 0/p')
(chroot) rootcat <<EOT >> /etc/fstab

/dev/md127 /boot/efi vfat noatime,discard 0 2



Change hostname

(chroot) rootecho 'send host-name "destroyer";' > /etc/dhcp/dhclient.conf
(chroot) rootsed -i "s/\(.*\)\t\{1,\}localhost/\1\tdestroyer.localdomain\tdestroyer\tlocalhost/" /etc/hosts

Setup keymap to de-latin1

FILE /etc/conf.d/keymaps

Setup networking

FILE /etc/conf.d/net
# ethernet

# wifi

# routing priority

Force --ask on portage

FILE /etc/portage/make.conf

Add user and setup password

(chroot) rootuseradd -m -G tty,wheel,audio,video,kvm,users,usb,plugdev -s /bin/fish theaifam5
(chroot) rootcp -rT /etc/skel /home/theaifam5
(chroot) rootchown -R theaifam5:theaifam5 /home/theaifam5
(chroot) rootpasswd theaifam5
(chroot) rootpasswd

Setup ZFS PAM module for home directories

FILE /etc/pam.d/zfs-home

Exit chroot and unmount the drives

root #umount /mnt/gentoo/boot/efi
root #mdadm --stop /dev/md127
root #zpool export rpool


Boot to the system, type a password for your rpool/ROOT dataset, then mount user dataset

rootzfs load-key rpool/DATASET/theaifam5
rootzfs mount rpool/DATASET/theaifam5

Configure USE

FILE /etc/portage/package.use/media-video
>=media-video/pipewire-0.3.34::gentoo bluetooth extra pipewire-alsa
FILE /etc/portage/package.use/x11-misc
>=x11-misc/picom-8.2-r2::gentoo opengl
FILE /etc/portage/package.use/app-editors
>=app-editors/emacs-27.2-r4::gentoo cairo dynamic-loading harfbuzz imagemagick jpg json libxml2 mailutils png sound svg tiff wide-int gui gzip-el gif jpeg xwidgets m17n-lib alsa xft Xaw3d athena toolkit-scroll-bars
FILE /etc/portage/package.use/app-emacs
>=app-emacs/emacs-common-1.8::gentoo gui
FILE /etc/portage/package.use/dev-java
>=dev-java/openjdk-11.0.12_p7:11::gentoo javafx alsa
>=dev-java/swt-4.10-r2::gentoo opengl cairo
FILE /etc/portage/package.use/app-text
>=app-text/ghostscript-gpl-9.54.0::gentoo cups
FILE /etc/portage/package.use/media-fonts
>=media-fonts/urw-fonts-2.4.9::gentoo X
FILE /etc/portage/package.use/sec-policy
>=sec-policy/selinux-java-2.20210203-r1::gentoo alsa
FILE /etc/portage/package.use/net-libs
>=net-libs/nodejs-16.6.2-r1::gentoo inspector
FILE /etc/portage/package.use/net-dns
>=net-dns/dnsmasq-2.85::gentoo script
FILE /etc/portage/package.use/app-text
>=app-text/docbook-sgml-utils-0.6.14-r4::gentoo jadetex
>=app-text/texlive-core-2021-r1::gentoo xetex

Add X, driver and tools

FILE /etc/portage/package.use/x11-drivers
>=x11-drivers/nvidia-drivers-470.63.01::gentoo X driver tools

Install kitty, picom, emacs, AwesomeWM and PipeWire with BlueZ

(chroot) rootemerge net-wireless/bluez media-video/pipewire x11-terms/kitty x11-misc/picom app-editors/emacs x11-wm/awesome gui-libs/display-manager-init x11-misc/lightdm-mini-greeter sys-fs/zfs-auto-snapshot dev-java/openjdk
(chroot) rootrc-update add display-manager default
(chroot) rootrc-update add cronie default

Setup pulseaudio

FILE /etc/pulse/client.conf
autospawn = no

Set hostname

FILE /etc/conf.d/hostname

Configure LightDM

FILE /etc/lightdm/lightdm.conf

Configure LightDM Mini Greeter

FILE /etc/lightdm/lightdm-mini-greeter.conf
user = theaifam5
show-image-on-all-monitors = true

Configure Xorg

FILE /etc/X11/xorg.conf.d/30-keyboard.conf

Install Discord and BetterDiscordCtl

(chroot) rootemerge net-im/discord-bin
(chroot) root(cd /usr/local/bin && curl -O && chmod +x ./betterdiscordctl)

Update world, cleanup, enable USE doc and install qemu:

(chroot) rootemerge -uDN @world
(chroot) rootemerge app-emulation/qemu
(chroot) rootemerge --depclean -a
(chroot) rootudevadm trigger -c add /dev/kvm
(chroot) rootrc-update add qemu-binfmt
(chroot) rootrc-update add libvirt-guests

Afterwards remove alsa-plugins from /etc/portage/package.use/media-plugins

remove /etc/portage/package.use/dev-tmp