Hi everyone.
I am trying to get PFSense working in a FreeBSD VM as per the videos by SpaceInvader One but I keep getting
2020-07-16T14:36:14.324463Z qemu-system-x86_64: vfio: Unable to power on device, stuck in D3
errors. I have tried several fixes which I found on the forums but non seem to work.
I am running:
AMD-7600 Radeon R7
Gigabyte F2A88X-D3H on F7 bios
HP 491176-001 538696-B21 NC375T PCIe 4 Port Gigabit NIC
I follow the video and if I don't try to passthrough the NIC, It starts ok but the CPU cores that are assigned go to and stay at 100% and going into the VNC Remote just shows a black screen. I also am unable to stop the VM without hitting force stop. Here is the log when I follow the SpaceInvader One video exactly. It is single core, Q35-2.11 (Tried all of them with same result), OVMF but also tried SeaBios with same result
-smp 1,sockets=1,cores=1,threads=1 \
-uuid 237132c0-e4bb-8769-64c5-b5111334c6d3 \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=34,server,nowait \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=utc,driftfix=slew \
-global kvm-pit.lost_tick_policy=delay \
-no-hpet \
-no-shutdown \
-boot strict=on \
-device pcie-root-port,port=0x10,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x2 \
-device pcie-root-port,port=0x11,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 \
-device pcie-root-port,port=0x12,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2 \
-device pcie-root-port,port=0x13,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3 \
-device ich9-usb-ehci1,id=usb,bus=pcie.0,addr=0x7.0x7 \
-device ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pcie.0,multifunction=on,addr=0x7 \
-device ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pcie.0,addr=0x7.0x1 \
-device ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pcie.0,addr=0x7.0x2 \
-device virtio-serial-pci,id=virtio-serial0,bus=pci.2,addr=0x0 \
-blockdev '{"driver":"file","filename":"/mnt/user/isos/pfSense-CE-2.4.5-RELEASE-p1-amd64.iso","node-name":"libvirt-2-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-2-format","read-only":true,"driver":"raw","file":"libvirt-2-storage"}' \
-device ide-cd,bus=ide.0,drive=libvirt-2-format,id=sata0-0-0,bootindex=2 \
-blockdev '{"driver":"file","filename":"/mnt/user/domains/PFSense2/vdisk1.img","node-name":"libvirt-1-storage","cache":{"direct":false,"no-flush":false},"auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":false,"no-flush":false},"driver":"raw","file":"libvirt-1-storage"}' \
-device ide-hd,bus=ide.2,drive=libvirt-1-format,id=sata0-0-2,bootindex=1,write-cache=on \
-chardev pty,id=charserial0 \
-device isa-serial,chardev=charserial0,id=serial0 \
-chardev socket,id=charchannel0,fd=36,server,nowait \
-device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 \
-device usb-tablet,id=input0,bus=usb.0,port=1 \
-vnc 0.0.0.0:1,websocket=5701 \
-k en-us \
-device qxl-vga,id=video0,ram_size=67108864,vram_size=67108864,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pcie.0,addr=0x1 \
-device virtio-balloon-pci,id=balloon0,bus=pci.3,addr=0x0 \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
-msg timestamp=on
2020-07-17 18:04:02.889+0000: Domain id=5 is tainted: high-privileges
2020-07-17 18:04:02.889+0000: Domain id=5 is tainted: host-cpu
char device redirected to /dev/pts/1 (label charserial0)
When I do try the NIC passthrough, this is the result
-uuid 4baf4f52-f636-5afb-fbc9-a6e05731f314 \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=33,server,nowait \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=utc,driftfix=slew \
-global kvm-pit.lost_tick_policy=delay \
-no-hpet \
-no-shutdown \
-boot strict=on \
-device ich9-usb-ehci1,id=usb,bus=pci.0,addr=0x7.0x7 \
-device ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pci.0,multifunction=on,addr=0x7 \
-device ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pci.0,addr=0x7.0x1 \
-device ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pci.0,addr=0x7.0x2 \
-device ahci,id=sata0,bus=pci.0,addr=0x3 \
-device virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x4 \
-blockdev '{"driver":"file","filename":"/mnt/user/isos/pfSense-CE-2.4.5-RELEASE-p1-amd64.iso","node-name":"libvirt-2-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-2-format","read-only":true,"driver":"raw","file":"libvirt-2-storage"}' \
-device ide-cd,bus=sata0.0,drive=libvirt-2-format,id=sata0-0-0,bootindex=2 \
-blockdev '{"driver":"file","filename":"/mnt/user/domains/FreeBSD/vdisk1.img","node-name":"libvirt-1-storage","cache":{"direct":false,"no-flush":false},"auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":false,"no-flush":false},"driver":"qcow2","file":"libvirt-1-storage","backing":null}' \
-device ide-hd,bus=sata0.2,drive=libvirt-1-format,id=sata0-0-2,bootindex=1,write-cache=on \
-chardev pty,id=charserial0 \
-device isa-serial,chardev=charserial0,id=serial0 \
-chardev socket,id=charchannel0,fd=35,server,nowait \
-device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 \
-device usb-tablet,id=input0,bus=usb.0,port=1 \
-vnc 0.0.0.0:0,websocket=5700 \
-k en-us \
-device qxl-vga,id=video0,ram_size=67108864,vram_size=67108864,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pci.0,addr=0x2 \
-device vfio-pci,host=0000:04:00.0,id=hostdev0,bus=pci.0,addr=0x5 \
-device vfio-pci,host=0000:04:00.1,id=hostdev1,bus=pci.0,addr=0x6 \
-device vfio-pci,host=0000:04:00.2,id=hostdev2,bus=pci.0,addr=0x8 \
-device vfio-pci,host=0000:04:00.3,id=hostdev3,bus=pci.0,addr=0x9 \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
-msg timestamp=on
2020-07-17 18:24:34.822+0000: Domain id=5 is tainted: high-privileges
2020-07-17 18:24:34.822+0000: Domain id=5 is tainted: host-cpu
char device redirected to /dev/pts/0 (label charserial0)
2020-07-17T18:24:36.705672Z qemu-system-x86_64: vfio: Unable to power on device, stuck in D3
I have tried emulating a different CPU with the Skylake fix, but it does not work
<cpu mode='custom' match='exact' check='full'>
<model fallback='forbid'>Skylake-Client</model>
<topology sockets='1' cores='2' threads='1'/>
<feature policy='require' name='hypervisor'/>
<feature policy='disable' name='pcid'/>
<feature policy='disable' name='hle'/>
<feature policy='disable' name='erms'/>
<feature policy='disable' name='invpcid'/>
<feature policy='disable' name='rtm'/>
<feature policy='disable' name='mpx'/>
<feature policy='disable' name='spec-ctrl'/>
</cpu>
just gives me an error that the features do not exist.
I also tried
<cpu>
<topology sockets='1' cores='2' threads='1'/>
</cpu>
but also does not work, Still suck in D3
The only way I could find to get ahead was doing a PCIe ACS override as I had a PCI Bridge with my NIC, that did start the VM and I was able to get some kind of boot but it got stuck in the PF Sense boot screen, which was more than it did before, It also separated my NIC into 4 separate IOMMU groups.
I have no idea what I can do next.