I have the same problem.
MacOS VM works fine while using VNC default settings. But when I passthrough my AMD RX580 the screen looks like this:
Passthrough to my Windows and Linux VM works just fine. Not sure what else to check. My config file looks like this:
<?xml version='1.0' encoding='UTF-8'?>
<domain type='kvm' xmlns:qemu='http://libvirt.org/schemas/domain/qemu/1.0'>
<name>MacinaboxCatalina</name>
<uuid>dd5f26a3-5a0d-44d9-9808-5a2d71a25cfa</uuid>
<description>MacOS Catalina</description>
<metadata>
<vmtemplate xmlns="unraid" name="MacOS" icon="/mnt/user/domains/MacinaboxCatalina/icon/catalina.png" os="Catalina"/>
</metadata>
<memory unit='KiB'>8388608</memory>
<currentMemory unit='KiB'>8388608</currentMemory>
<memoryBacking>
<nosharepages/>
</memoryBacking>
<vcpu placement='static'>4</vcpu>
<cputune>
<vcpupin vcpu='0' cpuset='2'/>
<vcpupin vcpu='1' cpuset='6'/>
<vcpupin vcpu='2' cpuset='3'/>
<vcpupin vcpu='3' cpuset='7'/>
</cputune>
<os>
<type arch='x86_64' machine='pc-q35-3.1'>hvm</type>
<loader readonly='yes' type='pflash'>/mnt/user/domains/MacinaboxCatalina/ovmf/OVMF_CODE.fd</loader>
<nvram>/mnt/user/domains/MacinaboxCatalina/ovmf/OVMF_VARS.fd</nvram>
</os>
<features>
<acpi/>
<apic/>
</features>
<cpu mode='host-passthrough' check='none'>
<topology sockets='1' cores='2' threads='2'/>
</cpu>
<clock offset='utc'>
<timer name='rtc' tickpolicy='catchup'/>
<timer name='pit' tickpolicy='delay'/>
<timer name='hpet' present='no'/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>restart</on_crash>
<devices>
<emulator>/usr/local/sbin/qemu</emulator>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2' cache='writeback'/>
<source file='/mnt/user/domains/MacinaboxCatalina/Clover.qcow2'/>
<target dev='hdc' bus='sata'/>
<boot order='1'/>
<address type='drive' controller='0' bus='0' target='0' unit='2'/>
</disk>
<disk type='file' device='disk'>
<driver name='qemu' type='raw' cache='writeback'/>
<source file='/mnt/user/domains/MacinaboxCatalina/Catalina-install.img'/>
<target dev='hdd' bus='sata'/>
<address type='drive' controller='0' bus='0' target='0' unit='3'/>
</disk>
<disk type='file' device='disk'>
<driver name='qemu' type='raw' cache='writeback'/>
<source file='/mnt/user/domains/MacinaboxCatalina/macos_disk.img'/>
<target dev='hde' bus='sata'/>
<address type='drive' controller='0' bus='0' target='0' unit='4'/>
</disk>
<controller type='usb' index='0' model='ich9-ehci1'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x7'/>
</controller>
<controller type='usb' index='0' model='ich9-uhci1'>
<master startport='0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0' multifunction='on'/>
</controller>
<controller type='usb' index='0' model='ich9-uhci2'>
<master startport='2'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x1'/>
</controller>
<controller type='usb' index='0' model='ich9-uhci3'>
<master startport='4'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x2'/>
</controller>
<controller type='sata' index='0'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
</controller>
<controller type='pci' index='0' model='pcie-root'/>
<controller type='pci' index='1' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='1' port='0x10'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/>
</controller>
<controller type='pci' index='2' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='2' port='0x11'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x1'/>
</controller>
<controller type='pci' index='3' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='3' port='0x12'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x2'/>
</controller>
<controller type='pci' index='4' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='4' port='0x13'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x3'/>
</controller>
<controller type='virtio-serial' index='0'>
<address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
</controller>
<interface type='bridge'>
<mac address='52:54:00:66:f2:d3'/>
<source bridge='br0'/>
<model type='vmxnet3'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
</interface>
<serial type='pty'>
<target type='isa-serial' port='0'>
<model name='isa-serial'/>
</target>
</serial>
<console type='pty'>
<target type='serial' port='0'/>
</console>
<channel type='unix'>
<target type='virtio' name='org.qemu.guest_agent.0'/>
<address type='virtio-serial' controller='0' bus='0' port='1'/>
</channel>
<input type='tablet' bus='usb'>
<address type='usb' bus='0' port='1'/>
</input>
<input type='mouse' bus='ps2'/>
<input type='keyboard' bus='ps2'/>
<hostdev mode='subsystem' type='pci' managed='yes'>
<driver name='vfio'/>
<source>
<address domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
</source>
<rom file='/mnt/user/isos/vbios/Asus.RX580.8192.170301.rom'/>
<address type='pci' domain='0x0000' bus='0x03' slot='0x00' function='0x0'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<driver name='vfio'/>
<source>
<address domain='0x0000' bus='0x01' slot='0x00' function='0x1'/>
</source>
<address type='pci' domain='0x0000' bus='0x03' slot='0x00' function='0x1'/>
</hostdev>
<hostdev mode='subsystem' type='usb' managed='no'>
<source>
<vendor id='0x046d'/>
<product id='0x0a4d'/>
</source>
<address type='usb' bus='0' port='2'/>
</hostdev>
<hostdev mode='subsystem' type='usb' managed='no'>
<source>
<vendor id='0x046d'/>
<product id='0xc07d'/>
</source>
<address type='usb' bus='0' port='3'/>
</hostdev>
<hostdev mode='subsystem' type='usb' managed='no'>
<source>
<vendor id='0x046d'/>
<product id='0xc330'/>
</source>
<address type='usb' bus='0' port='4'/>
</hostdev>
<memballoon model='none'>
</memballoon>
</devices>
<qemu:commandline>
<qemu:arg value='-usb'/>
<qemu:arg value='-device'/>
<qemu:arg value='usb-kbd,bus=usb-bus.0'/>
<qemu:arg value='-device'/>
<qemu:arg value='isa-applesmc,osk=***'/>
<qemu:arg value='-smbios'/>
<qemu:arg value='type=2'/>
<qemu:arg value='-cpu'/>
<qemu:arg value='Penryn,kvm=on,vendor=GenuineIntel,+invtsc,vmware-cpuid-freq=on,+pcid,+ssse3,+sse4.2,+popcnt,+avx,+aes,+xsave,+xsaveopt,check'/>
</qemu:commandline>
</domain>
Any ideas are greatly appreciated! Thank you. @SpaceInvaderOne