dotfiles

  1## VFIO Configuration ##
  2_: {
  3  flake.modules.nixos.vfio = {
  4    config,
  5    lib,
  6    pkgs,
  7    ...
  8  }: let
  9    cfg = config.virt.vfio;
 10    inherit (config.hardware.cpu) model;
 11    shmSize = 64; # KVMFR Shared Memory size
 12  in {
 13    options.virt.vfio = {
 14      setup = lib.mkEnableOption "VFIO Setup Mode";
 15      passthrough = lib.mkOption {
 16        description = "PCI Device IDs for VFIO Passthrough";
 17        type = lib.types.listOf lib.types.str;
 18        default = [];
 19        example = [
 20          "10de:28e0" # Graphics
 21          "10de:22be" # Audio
 22        ];
 23      };
 24
 25      isolate = lib.mkOption {
 26        description = "Host CPU set to isolate for pinned guest vCPUs";
 27        type = lib.types.str;
 28        default = "";
 29        example = "2-4,10-12";
 30      };
 31
 32      hugepages = lib.mkOption {
 33        description = "Number of 1 GiB hugepages to reserve for guest";
 34        type = lib.types.ints.unsigned;
 35        default = 0;
 36        example = 16;
 37      };
 38    };
 39
 40    config = lib.mkMerge [
 41      (lib.mkIf cfg.setup {
 42        specialisation.vfio.configuration = {
 43          system.nixos.label = "special.vfio";
 44          virt.vfio.setup = lib.mkForce false;
 45        };
 46      })
 47
 48      (lib.mkIf (!cfg.setup) {
 49        # Disable GPU
 50        hardware.gpu.enable = lib.mkForce false;
 51
 52        boot = {
 53          kernelParams =
 54            [
 55              "iommu=pt"
 56              "${model}_iommu=on"
 57              "kvm.ignore_msrs=1"
 58              "kvm.report_ignored_msrs=0"
 59              ("vfio-pci.ids=" + builtins.concatStringsSep "," cfg.passthrough)
 60            ]
 61            ++ lib.optionals (cfg.isolate != "") [
 62              "isolcpus=${cfg.isolate}"
 63              "nohz_full=${cfg.isolate}"
 64              "rcu_nocbs=${cfg.isolate}"
 65            ]
 66            ++ lib.optionals (cfg.hugepages > 0) [
 67              "default_hugepagesz=1G"
 68              "hugepagesz=1G"
 69              "hugepages=${toString cfg.hugepages}"
 70            ];
 71
 72          initrd.kernelModules = [
 73            "vfio"
 74            "vfio_pci"
 75            "vfio_iommu_type1"
 76          ];
 77
 78          # KVMFR
 79          extraModulePackages = [config.boot.kernelPackages.kvmfr];
 80          kernelModules = ["kvmfr"];
 81          extraModprobeConfig = "options kvmfr static_size_mb=${toString shmSize}";
 82        };
 83
 84        # Looking Glass
 85        environment.systemPackages = [pkgs.looking-glass-client];
 86        services.udev.extraRules = ''
 87          SUBSYSTEM=="kvmfr", KERNEL=="kvmfr0", MODE="0660", GROUP="kvm", TAG+="systemd"
 88        '';
 89
 90        users.users.qemu-libvirtd.extraGroups = ["kvm"];
 91        virtualisation.libvirtd.qemu.verbatimConfig = ''
 92          namespaces = []
 93          cgroup_device_acl = [
 94            "/dev/null", "/dev/full", "/dev/zero",
 95            "/dev/random", "/dev/urandom",
 96            "/dev/ptmx", "/dev/kvm",
 97            "/dev/userfaultfd",
 98            "/dev/kvmfr0"
 99          ]
100        '';
101
102        environment.etc."looking-glass-client.ini".text = lib.generators.toINI {} {
103          app = {
104            renderer = "egl";
105            shmFile = "/dev/kvmfr0";
106          };
107          win = {
108            title = "Virtual Machine";
109            autoResize = "yes";
110            borderless = "no";
111            dontUpscale = "yes";
112            fullScreen = "no";
113            keepAspect = "yes";
114            maximize = "no";
115            noScreensaver = "yes";
116            quickSplash = "yes";
117            uiSize = 16;
118          };
119          egl = {
120            scale = 2;
121            multisample = "yes";
122            vsync = "yes";
123          };
124          input = {
125            autoCapture = "yes";
126            grabKeyboardOnFocus = "yes";
127            rawMouse = "yes";
128            releaseKeysOnFocusLoss = "yes";
129          };
130          spice = {
131            enable = "yes";
132            clipboard = "yes";
133          };
134          wayland = {
135            warpSupport = "yes";
136            fractionScale = "no";
137          };
138        };
139      })
140    ];
141  };
142}