Hi
I have a clean install of Unraid (6.12.3) and have managed to get the Quadro FX1800 pass through work to an AlmLinux 9.1 guest VM. However it only works from a cold boot. In other words turn on the machine with the VM on autostart and it works fine. If I stop it though and try to restart it I get garbled video output.
The machine also has a Quadro M5000 card but I'm using that for my tests. I do have them both bound to the vfio at boot in the Tools setting though I must admit it's confusing as to whether this is need nowadays ? Give that it works the first time i'm ignoring this aspect of the problem.
In the logs for the VM I see the qemu startup log is
2023-09-20 01:33:12.838+0000: starting up libvirt version: 8.7.0, qemu version: 7.1.0, kernel: 6.1.38-Unraid, hostname: Toontown
LC_ALL=C \
PATH=/bin:/sbin:/usr/bin:/usr/sbin \
HOME=/var/lib/libvirt/qemu/domain-1-AlmaLinux \
XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-1-AlmaLinux/.local/share \
XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-1-AlmaLinux/.cache \
XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-1-AlmaLinux/.config \
/usr/local/sbin/qemu \
-name guest=AlmaLinux,debug-threads=on \
-S \
-object '{"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-1-AlmaLinux/master-key.aes"}' \
-blockdev '{"driver":"file","filename":"/usr/share/qemu/ovmf-x64/OVMF_CODE-pure-efi.fd","node-name":"libvirt-pflash0-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-pflash0-format","read-only":true,"driver":"raw","file":"libvirt-pflash0-storage"}' \
-blockdev '{"driver":"file","filename":"/etc/libvirt/qemu/nvram/ecd3a78e-809d-9e57-8985-2e980756b2f5_VARS-pure-efi.fd","node-name":"libvirt-pflash1-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-pflash1-format","read-only":false,"driver":"raw","file":"libvirt-pflash1-storage"}' \
-machine pc-q35-7.1,usb=off,dump-guest-core=off,mem-merge=off,memory-backend=pc.ram,pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-format \
-accel kvm \
-cpu host,migratable=on,host-cache-info=on,l3-cache=off \
-m 4096 \
-object '{"qom-type":"memory-backend-ram","id":"pc.ram","size":4294967296}' \
-overcommit mem-lock=off \
-smp 4,sockets=1,dies=1,cores=2,threads=2 \
-uuid ecd3a78e-809d-9e57-8985-2e980756b2f5 \
-display none \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=35,server=on,wait=off \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=utc,driftfix=slew \
-global kvm-pit.lost_tick_policy=delay \
-no-hpet \
-no-shutdown \
-boot strict=on \
-device '{"driver":"pcie-root-port","port":8,"chassis":1,"id":"pci.1","bus":"pcie.0","multifunction":true,"addr":"0x1"}' \
-device '{"driver":"pcie-root-port","port":9,"chassis":2,"id":"pci.2","bus":"pcie.0","addr":"0x1.0x1"}' \
-device '{"driver":"pcie-root-port","port":10,"chassis":3,"id":"pci.3","bus":"pcie.0","addr":"0x1.0x2"}' \
-device '{"driver":"pcie-root-port","port":11,"chassis":4,"id":"pci.4","bus":"pcie.0","addr":"0x1.0x3"}' \
-device '{"driver":"pcie-root-port","port":12,"chassis":5,"id":"pci.5","bus":"pcie.0","addr":"0x1.0x4"}' \
-device '{"driver":"pcie-root-port","port":13,"chassis":6,"id":"pci.6","bus":"pcie.0","addr":"0x1.0x5"}' \
-device '{"driver":"ich9-usb-ehci1","id":"usb","bus":"pcie.0","addr":"0x7.0x7"}' \
-device '{"driver":"ich9-usb-uhci1","masterbus":"usb.0","firstport":0,"bus":"pcie.0","multifunction":true,"addr":"0x7"}' \
-device '{"driver":"ich9-usb-uhci2","masterbus":"usb.0","firstport":2,"bus":"pcie.0","addr":"0x7.0x1"}' \
-device '{"driver":"ich9-usb-uhci3","masterbus":"usb.0","firstport":4,"bus":"pcie.0","addr":"0x7.0x2"}' \
-device '{"driver":"virtio-serial-pci","id":"virtio-serial0","bus":"pci.2","addr":"0x0"}' \
-blockdev '{"driver":"file","filename":"/mnt/user/domains/AlmaLinux/vdisk1.img","node-name":"libvirt-2-storage","cache":{"direct":false,"no-flush":false},"auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-2-format","read-only":false,"cache":{"direct":false,"no-flush":false},"driver":"raw","file":"libvirt-2-storage"}' \
-device '{"driver":"virtio-blk-pci","bus":"pci.3","addr":"0x0","drive":"libvirt-2-format","id":"virtio-disk2","bootindex":1,"write-cache":"on","serial":"vdisk1"}' \
-blockdev '{"driver":"file","filename":"/mnt/user/isos/linux/AlmaLinux-9.1-x86_64-dvd.iso","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-1-format","read-only":true,"driver":"raw","file":"libvirt-1-storage"}' \
-device '{"driver":"ide-cd","bus":"ide.0","drive":"libvirt-1-format","id":"sata0-0-0","bootindex":2}' \
-netdev tap,fd=36,id=hostnet0 \
-device '{"driver":"virtio-net","netdev":"hostnet0","id":"net0","mac":"52:54:00:83:04:ba","bus":"pci.1","addr":"0x0"}' \
-chardev pty,id=charserial0 \
-device '{"driver":"isa-serial","chardev":"charserial0","id":"serial0","index":0}' \
-chardev socket,id=charchannel0,fd=34,server=on,wait=off \
-device '{"driver":"virtserialport","bus":"virtio-serial0.0","nr":1,"chardev":"charchannel0","id":"channel0","name":"org.qemu.guest_agent.0"}' \
-audiodev '{"id":"audio1","driver":"none"}' \
-device '{"driver":"vfio-pci","host":"0000:03:00.0","id":"hostdev0","bus":"pci.4","addr":"0x0","romfile":"/mnt/user/isos/vbios/gt170.rom"}' \
-device '{"driver":"usb-host","hostdevice":"/dev/bus/usb/003/002","id":"hostdev1","bus":"usb.0","port":"1"}' \
-device '{"driver":"usb-host","hostdevice":"/dev/bus/usb/003/004","id":"hostdev2","bus":"usb.0","port":"2"}' \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
-msg timestamp=on
char device redirected to /dev/pts/0 (label charserial0)
When i stop it the logs adds these two lines to the logs
2023-09-20T01:35:29.116320Z qemu-system-x86_64: terminating on signal 15 from pid 2853 (/usr/sbin/libvirtd)
2023-09-20 01:35:30.537+0000: shutting down, reason=shutdown
I have a few questions that hopefully someone can answer
The above command using qemu - where is that defined.
Is it unique to each VM type or just in the template
When trying to run the above command in the terminal it complained about a missing `Failed to open file “/var/lib/libvirt/qemu/domain-1-AlmaLinux/master-key.aes”` file. Are there other options that I can pass to qemu to help diagnose the problem?
Is there some tool to dump the important information about the card that I can run before and after its started/stopped so I could compare and see what state it has been left in that results in it not working.
Happy to provide any further information as needed. Thanks for reading.
Cheers