Thanks for the suggestion. I ended up modifying the config.v2.json for the container and was able to get it working, this is a bit more dynamic which is good.
Here's the source incase anybody else finds it useful. I'd like to make it work with multiple VMs at once, but this is fine for the moment.
#!/bin/bash
CONTAINER_NAME="aio-miner"
CONTAINER_ID=$(docker inspect --format="{{.Id}}" $CONTAINER_NAME)
INPUT_VM=$1
INPUT_EVENT=$2
update_container() {
local devices="$(echo "$1" | cut -d, -f"2" | xargs | sed -e 's/ /,/g')"
local devices_escaped="$(echo $devices | sed 's/\-/\\-/g')"
echo "cd /Container/Config[@Name='NVIDIA_VISIBLE_DEVICES']
set $devices
cd /Container/Environment/Variable[2]/Value
set $devices
save" | xmllint --shell /boot/config/plugins/dockerMan/templates-user/my-$CONTAINER_NAME.xml
docker stop $CONTAINER_ID
sed -i "s/\"NVIDIA_VISIBLE_DEVICES=[A-Za-z0-9,\-]*\"/\"NVIDIA_VISIBLE_DEVICES=$devices_escaped\"/g" /var/lib/docker/containers/$CONTAINER_ID/config.v2.json
/etc/rc.d/rc.docker restart
}
get_vm_buses() {
local buses=""
while read -r bus; do
bus=${bus#*'"'}; bus=${bus%'"'*}
buses+="$bus|"
done <<< $(xmllint --xpath '//domain/devices/hostdev[@type="pci"]/source/address/@bus' --nowarning /etc/libvirt/qemu/$INPUT_VM.xml | sort | uniq)
echo "${buses%?}"
}
if [[ "$INPUT_EVENT" == "prepare" ]]; then
buses="$(get_vm_buses)"
if [ -z "$buses" ]; then
exit 0
fi
devices="$(nvidia-smi --query-gpu="pci.bus,uuid" --format=csv,noheader | grep -Ev "$buses")"
update_container "$devices"
fi
if [[ "$INPUT_EVENT" == "release" ]]; then
buses="$(get_vm_buses)"
if [ -z "$buses" ]; then
exit 0
fi
devices="$(nvidia-smi --query-gpu="pci.bus,uuid" --format=csv,noheader)"
update_container "$devices"
fi
(sleep 30s && nvidia-smi -pl 280) &