From 5561b08e6ab58209cb1a5febd57c355e01530f67 Mon Sep 17 00:00:00 2001 From: peter Date: Thu, 21 Jan 2021 19:48:05 -0800 Subject: add gpu passthrough support as well as dynamic OpenCore regeneration --- helm/templates/configmap.yaml | 543 ++++++++++++++++++++++------------------- helm/templates/deployment.yaml | 45 ++-- helm/templates/service.yaml | 10 +- 3 files changed, 322 insertions(+), 276 deletions(-) (limited to 'helm/templates') diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 81aeb23..b862798 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -215,6 +215,24 @@ data: Add + {{- if .Values.qemu.hardwareGpu.enabled }} + + BundlePath + mXHCD.kext + Comment + Hello There + Enabled + + ExecutablePath + Contents/MacOS/mXHCD + MaxKernel + + MinKernel + + PlistPath + Contents/Info.plist + + {{- end }} Arch x86_64 @@ -222,7 +240,11 @@ data: VoodooHDA.kext Comment Patch engine + {{- if .Values.qemu.audio.enabled }} Enabled + {{- else -}} + Disabled + {{- end }} ExecutablePath Contents/MacOS/VoodooHDA @@ -355,6 +377,28 @@ data: PlistPath Contents/Info.plist + {{- if .Values.kexts.add }} + {{- range .Values.kexts.kextsToAdd }} + + Arch + Any + BundlePath + {{ .name }} + Comment + + Enabled + + ExecutablePath + {{ .executablePath }} + MaxKernel + + MinKernel + + PlistPath + {{ .plistPath }} + + {{- end }} + {{- end }} Block @@ -375,6 +419,16 @@ data: Emulate + {{- if .Values.qemu.hardwareGpu.enabled }} + Cpuid1Data + + VwYFAAAAAAAAAAAAAAAAAA== + + Cpuid1Mask + + /////wAAAAAAAAAAAAAAAA== + + {{- else -}} Cpuid1Data VAYFAAAAAAAAAAAAAAAAAA== @@ -383,9 +437,40 @@ data: ////AAAAAAAAAAAAAAAAAA== + {{- end }} Force + {{- if .Values.qemu.hardwareGpu.enabled }} + + Base + + Comment + algrey - cpuid_set_generic_info - disable check to allow leaf7 + Count + 1 + Enabled + + Find + ADoPgg== + Identifier + kernel + Limit + 0 + Mask + + MaxKernel + 19.99.99 + MinKernel + 17.0.0 + Replace + AAAPgg== + ReplaceMask + + Skip + 0 + + {{- else -}} Arch Any @@ -406,9 +491,40 @@ data: PlistPath Contents/Info.plist + {{- end }} Patch + {{- if .Values.qemu.hardwareGpu.enabled }} + + Base + _cpu_topology_sort + Comment + algrey - cpu_topology_sort -disable _x86_validate_topology + Count + 1 + Enabled + + Find + 6AAA//8= + Identifier + kernel + Limit + 0 + Mask + /wAA//8= + MaxKernel + 19.99.99 + MinKernel + 17.0.0 + Replace + Dx9EAAA= + ReplaceMask + + Skip + 0 + + {{- else -}} Base _cpu_topology_sort @@ -444,6 +560,7 @@ data: Skip 0 + {{- end }} Base @@ -453,6 +570,7 @@ data: 1 Enabled + {{- if .Values.qemu.hardwareGpu.enabled }} Find MduAPQAAAAAGdQA= @@ -465,6 +583,20 @@ data: /////wAAAP///wA= + {{- else -}} + Find + + MduAPQAAAAAGdQA= + + Identifier + kernel + Limit + 0 + Mask + + /////wAAAP///wA= + + {{- end }} MaxKernel 20.99.99 MinKernel @@ -479,6 +611,64 @@ data: Skip 0 + {{- if .Values.qemu.hardwareGpu.enabled }} + + Base + + Comment + algrey - - skip cpuid_cores_per_package test -10.15 + Count + 0 + Enabled + + Find + gz0AAAAAAA8AAAAAAItdvA== + Identifier + kernel + Limit + 0 + Mask + //8AAAD///8AAAAA/////w== + MaxKernel + 19.99.99 + MinKernel + 19.0.0 + Replace + AAAAAAAAAQAAAAAAAAAAAA== + ReplaceMask + AAAAAAAADwAAAAAAAAAAAA== + Skip + 0 + + + Base + + Comment + algrey - - skip cpuid_cores_per_package test + Count + 0 + Enabled + + Find + gz0AAAAAAHQAi128 + Identifier + kernel + Limit + 0 + Mask + //8AAAD///8A//// + MaxKernel + 18.99.99 + MinKernel + 17.0.0 + Replace + AAAAAAAAAQAAAAAA + ReplaceMask + AAAAAAAADwAAAAAA + Skip + 0 + + {{- end }} Quirks @@ -516,6 +706,7 @@ data: Scheme + {{- if not .Values.qemu.hardwareGpu.enabled }} FuzzyMatch @@ -524,6 +715,7 @@ data: KernelCache Auto + {{- end }} Misc @@ -539,6 +731,8 @@ data: PickerAttributes 1 + PickerVariant + Modern PickerAudioAssist PickerMode @@ -550,7 +744,7 @@ data: TakeoffDelay 0 Timeout - 0 + {{ .Values.openCore.boot.timeout }} Debug @@ -657,7 +851,7 @@ data: SystemAudioVolume Rg== boot-args - -v keepsyms=1 tlbto_us=0 vti=9 + {{ .Values.configPlist.bootArgs }} run-efi-updater No csr-active-config @@ -838,7 +1032,7 @@ data: ReplaceTabWithSpace Resolution - 1920x1080@32 + {{ .Values.vnc.resolution }} SanitiseClearScreen TextRenderer @@ -905,232 +1099,37 @@ data: - macOS-libvirt-Catalina.xml: |- - - - - macOS - 2aca0dd6-cec9-4717-9ab2-0b7b13d111c3 - macOS - {{ .Values.resources.requests.memory | trimSuffix "Mi" }} - {{ .Values.resources.requests.memory | trimSuffix "Mi" }} - {{ .Values.resources.requests.cpu }} - - hvm - - /home/CHANGEME/OSX-KVM/OVMF_CODE.fd - /home/CHANGEME/OSX-KVM/OVMF_VARS-1024x768.fd - - - - - - - - - - - destroy - restart - restart - - /usr/bin/qemu-system-x86_64 - - - - - -
- - - - - - -
- - -
- - - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - -
- - -
- - - -
- - - -
- - - -
- - - - - - - - - - - - - - - - - - -
- - - -
- - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Launch_custom.sh: |- #/bin/sh - if ! [ -d "/system_image/installers" ]; then - mkdir -p /system_image/installers + # Add extra kexts to EFI/OC/kexts + {{- if .Values.kexts.add }} + {{- range .Values.kexts.kextsToAdd }} + {{- $rangeItem := . -}} + {{- with $ }} + echo 'Installing kext {{ $rangeItem.name }}..' + cp -r "{{ .Values.kexts.path }}/{{ $rangeItem.name }}" /home/{{ .Values.image.userName }}/OSX-KVM/OpenCore-Catalina/EFI/OC/Kexts/ + sudo chmod 755 /home/{{ .Values.image.userName }}/OSX-KVM/OpenCore-Catalina/EFI/OC/Kexts/{{ $rangeItem.name }} + {{- end }} + {{- end }} + {{- end }} + + # Rebuild Opencore.qcow2 after making changes to config.plist and etc.. + {{- if .Values.openCore.rebuild }} + echo 'Building new Opencore.qcow2..' + sudo apt install {{ .Values.openCore.kernel }} -y + pushd OpenCore-Catalina/ + mkdir -p EFI/OC/Resources + rm -f OpenCore.qcow2 + sudo ./opencore-image-ng.sh \ + --cfg config.plist \ + --img OpenCore.qcow2 + sudo chown {{ .Values.image.userName }}:{{ .Values.image.userName }} OpenCore.qcow2 + popd + {{- end }} + + if ! [ -d "{{ .Values.qemu.systemInstaller.path }}" ]; then + mkdir -p {{ .Values.qemu.systemInstaller.path }} fi if ! [ -d "/system_image/{{ .Values.serverName }}" ]; then @@ -1138,70 +1137,102 @@ data: fi # Download and build installer image if no system drive found.. - if ! [ -f "/system_image/installers/BaseSystem{{ .Values.qemu.systemInstaller.version }}.img" ]; then + if ! [ -f "{{ .Values.qemu.systemInstaller.path }}/BaseSystem{{ .Values.qemu.systemInstaller.version }}.img" ]; then echo "Downloading {{ .Values.qemu.systemInstaller.version }} base image.." python fetch-macOS.py --version {{ .Values.qemu.systemInstaller.version }} - echo 'Converting downloaded BaseSystem.dmg into BaseSystem.img' - qemu-img convert BaseSystem.dmg -O qcow2 -p -c /system_image/installers/BaseSystem{{ .Values.qemu.systemInstaller.version }}.img + echo 'Converting downloaded BaseSystem.dmg into BaseSystem{{ .Values.qemu.systemInstaller.version }}.img and saving in {{ .Values.qemu.systemInstaller.path }}' + qemu-img convert BaseSystem.dmg -O qcow2 -p -c {{ .Values.qemu.systemInstaller.path }}/BaseSystem{{ .Values.qemu.systemInstaller.version }}.img rm -f BaseSystem.dmg else echo 'Base Image downloaded and converted into img already..' fi if ! [ -f "/system_image/{{ .Values.serverName }}/mac_hdd_ng.img" ]; then - echo "Creating a {{ .Values.qemu.diskSize }} /system_image/{{ .Values.serverName }}/mac_hdd_ng.img for system partition.." - qemu-img create -f qcow2 /system_image/{{ .Values.serverName }}/mac_hdd_ng.img "{{ .Values.qemu.diskSize }}" + echo "Creating a {{ .Values.qemu.systemDisk.size }} /system_image/{{ .Values.serverName }}/mac_hdd_ng.img for system partition.." + qemu-img create -f qcow2 /system_image/{{ .Values.serverName }}/mac_hdd_ng.img "{{ .Values.qemu.systemDisk.size }}" echo 'Finished creating system partition!' else echo 'Image already created. Skipping creation..' fi + # # Fix permissions on usb devices.. + # {{- if .Values.qemu.usb }} + # {{- range .Values.qemu.usb }} + # echo "Updating permissions to r/w for /dev/bus/usb/$(lsusb | grep {{ .vendorId}}:{{ .productId }} | grep -o -P 'Bus.{0,4}' | tail -c 4)/$(lsusb | grep {{ .vendorId}}:{{ .productId }} | grep -o -P 'Device.{0,4}' | tail -c 4).." + # sudo chmod 666 /dev/bus/usb/$(lsusb | grep {{ .vendorId}}:{{ .productId }} | grep -o -P 'Bus.{0,4}' | tail -c 4)/$(lsusb | grep {{ .vendorId}}:{{ .productId }} | grep -o -P 'Device.{0,4}' | tail -c 4) + # {{- end }} + # {{- end }} + # Start VNC.. + {{- if .Values.vnc.enabled }} + echo 'geometry={{ .Values.vnc.resolution }} + localhost + alwaysshared' > ~/.vnc/config + sudo rm -f /tmp/.X99-lock export DISPLAY=:99 vncpasswd -f < vncpasswd_file > ${HOME}/.vnc/passwd - /usr/bin/Xvnc -geometry 1920x1080 -rfbauth "${HOME}/.vnc/passwd" :99 &\ + /usr/bin/Xvnc -geometry {{ .Values.vnc.resolution }} -rfbauth "${HOME}/.vnc/passwd" :99 &\ + sudo chmod 600 ~/.vnc/passwd + {{- end }} - {{- if .Values.qemu.gpu.enabled }} - ulimit -l $(( 8*1048576+100000 )) - user hard memlock $(( 8*1048576+100000 )) - user soft memlock $(( 8*1048576+100000 )) + sudo chown {{ .Values.image.userName }}:{{ .Values.image.userName }} /dev/kvm + {{- if .Values.qemu.hardwareGpu.enabled }} + sudo chown {{ .Values.image.userName }}:{{ .Values.image.userName }} -R /dev/vfio {{- end }} # Start QEMU.. + echo 'Starting QEMU..' set -eu sudo chown $(id -u):$(id -g) /dev/kvm 2>/dev/null || true sudo chown -R $(id -u):$(id -g) /dev/snd 2>/dev/null || true exec qemu-system-x86_64 -m {{ .Values.resources.requests.memory | trimSuffix "i" }} \ -cpu {{ .Values.qemu.cpu }} \ -machine q35,accel=kvm:tcg \ - {{- if .Values.qemu.gpu.enabled }} - -vga none \ - -device pcie-root-port,bus=pcie.0,multifunction=on,port=1,chassis=1,id=port.1 \ - -device vfio-pci,host={{ .Values.qemu.gpu.hardwareId }}.0,multifunction=on,x-vga=on,rombar=1 \ - -device vfio-pci,host={{ .Values.qemu.gpu.hardwareId }}.1,bus=port.1 \ - -display none \ - {{- else -}} - -vga {{ .Values.qemu.softwareGpu }} \ + {{- if .Values.qemu.hardwareGpu.enabled }} + -device pcie-root-port,bus=pcie.0,multifunction=on,port=1,chassis=1,id=port.1 \ + {{- if .Values.qemu.hardwareGpu.romfile }} + -device vfio-pci,host={{ .Values.qemu.hardwareGpu.hardwareId }}.0,bus=port.1,multifunction=on,romfile={{ .Values.qemu.hardwareGpu.romfile}} \ + {{- else -}} + -device vfio-pci,host={{ .Values.qemu.hardwareGpu.hardwareId }}.0,multifunction=on \ + {{- end }} + -device vfio-pci,host={{ .Values.qemu.hardwareGpu.hardwareId }}.1,bus=port.1 \ + {{- else -}} + -vga {{ .Values.qemu.softwareGpu }} \ {{- end }} -smp {{ .Values.resources.requests.cpu }},cores={{ .Values.resources.requests.cpu }} \ - -usb -device usb-kbd -device usb-tablet \ + {{- if .Values.vnc.enabled }} + -usb -device usb-kbd -device usb-tablet \ + {{- else -}} + -vga none \ + -display none \ + {{- end }} -device isa-applesmc,osk=ourhardworkbythesewordsguardedpleasedontsteal\(c\)AppleComputerInc \ - -drive if=pflash,format=raw,readonly,file=/home/arch/OSX-KVM/OVMF_CODE.fd \ - -drive if=pflash,format=raw,file=/home/arch/OSX-KVM/OVMF_VARS-1024x768.fd \ + -drive if=pflash,format=raw,readonly,file=/home/{{ .Values.image.userName }}/OSX-KVM/OVMF_CODE.fd \ + -drive if=pflash,format=raw,file=/home/{{ .Values.image.userName }}/OSX-KVM/OVMF_VARS-1024x768.fd \ -smbios type=2 \ {{- if .Values.qemu.audio.enabled }} - -audiodev {{ .Values.qemu.audo.driver }},id=hda -device ich9-intel-hda -device hda-duplex,audiodev=hda \ \ + -audiodev {{ .Values.qemu.audio.driver }},id=hda \ + -device ich9-intel-hda \ + -device hda-duplex,audiodev=hda \ {{- end }} -device ich9-ahci,id=sata \ - -drive id=OpenCoreBoot,if=none,snapshot=on,format=qcow2,file=/home/arch/OSX-KVM/OpenCore-Catalina/OpenCore.qcow2 \ + -drive id=OpenCoreBoot,if=none,snapshot=on,format=qcow2,file=/home/{{ .Values.image.userName }}/OSX-KVM/OpenCore-Catalina/OpenCore.qcow2 \ -device ide-hd,bus=sata.2,drive=OpenCoreBoot \ - -device ide-hd,bus=sata.3,drive=InstallMedia \ - -drive id=InstallMedia,if=none,file=/system_image/installers/BaseSystem{{ .Values.qemu.systemInstaller.version }}.img,format=qcow2 \ + {{- if .Values.qemu.systemInstaller.enabled }} + -device ide-hd,bus=sata.3,drive=InstallMedia \ + -drive id=InstallMedia,if=none,file={{ .Values.qemu.systemInstaller.path }}/BaseSystem{{ .Values.qemu.systemInstaller.version }}.img,format=qcow2 \ + {{- end }} -drive id=MacHDD,if=none,file=/system_image/{{ .Values.serverName }}/mac_hdd_ng.img,format=qcow2 \ -device ide-hd,bus=sata.4,drive=MacHDD \ - -netdev user,id=net0,hostfwd=tcp::${INTERNAL_SSH_PORT:-10022}-:22,hostfwd=tcp::${SCREEN_SHARE_PORT:-5900}-:5900,{{ .Values.qemu.netdev.extraArgs }} -device e1000-82545em,netdev=net0,id=net0,mac=52:54:00:09:49:17 \ - -monitor stdio \ + -netdev user,id=net0,hostfwd=tcp::${INTERNAL_SSH_PORT:-10022}-:22,hostfwd=tcp::${SCREEN_SHARE_PORT:-5900}-:5900,{{ .Values.qemu.netdev.extraPortForwarding }} \ + -device e1000-82545em,netdev=net0,id=net0,mac=52:54:00:09:49:17 \ + {{- range .Values.qemu.usb }} + -usb -device usb-host,productid=0x{{ .productId }},vendorid=0x{{ .vendorId }} \ + {{- end }} + {{- range .Values.qemu.extraArgs }} + {{ . }} \ + {{- end }} ${EXTRA:-} vncpasswd_file: |- {{ .Values.vnc.password }} @@ -1263,7 +1294,7 @@ data: #@faculty hard nproc 50 #ftp hard nproc 0 #@student - maxlogins 4 - @arch soft memlock unlimited - @arch hard memlock unlimited + @{{ .Values.image.userName }} soft memlock unlimited + @{{ .Values.image.userName }} hard memlock unlimited - # End of file \ No newline at end of file + # End of file diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index 87c33d1..4062414 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -56,30 +56,35 @@ spec: value: "{{ .Values.resources.requests.memory | trimSuffix "Mi" }}" - name: TZ value: "{{ .Values.tz }}" + - name: DISPLAY + value: ':0.0' resources: {{ toYaml .Values.resources | indent 10 }} volumeMounts: - - mountPath: /home/arch/OSX-KVM/config.plist + - mountPath: /home/{{ .Values.image.userName }}/OSX-KVM/OpenCore-Catalina/config.plist subPath: config.plist name: boot-components - - mountPath: /home/arch/OSX-KVM/macOS-libvirt-Catalina.xml - subPath: macOS-libvirt-Catalina.xml - name: boot-components - - mountPath: /home/arch/OSX-KVM/Launch_custom.sh + - mountPath: /home/{{ .Values.image.userName }}/OSX-KVM/Launch_custom.sh subPath: Launch_custom.sh name: boot-components - - mountPath: /home/arch/OSX-KVM/vncpasswd_file + - mountPath: /home/{{ .Values.image.userName }}/OSX-KVM/vncpasswd_file subPath: vncpasswd_file name: boot-components - - mountPath: /etc/security/limits.conf - subPath: limits.conf - name: boot-components - mountPath: /dev/kvm name: kvm - mountPath: /dev/net/tun name: tun - - mountPath: /dev/vfio + {{- if .Values.qemu.hardwareGpu.enabled }} + - mountPath: /etc/security/limits.conf + subPath: limits.conf + name: boot-components + - mountPath: /dev/vfio/vfio name: vfio + - mountPath: /dev/vfio/{{ .Values.qemu.hardwareGpu.vfioGroup }} + name: vfio-group + - mountPath: /lib/modules + name: lib-modules + {{- end }} - mountPath: /dev/snd name: snd - mountPath: /tmp/.X11-unix @@ -98,23 +103,31 @@ spec: items: - key: config.plist path: config.plist - - key: macOS-libvirt-Catalina.xml - path: macOS-libvirt-Catalina.xml - key: Launch_custom.sh path: Launch_custom.sh - key: vncpasswd_file path: vncpasswd_file + {{- if .Values.qemu.hardwareGpu.enabled }} - key: limits.conf path: limits.conf - - name: kvm - hostPath: - path: /dev/kvm + {{- end }} - name: tun hostPath: path: /dev/net/tun + - name: kvm + hostPath: + path: /dev/kvm + {{- if .Values.qemu.hardwareGpu.enabled }} - name: vfio hostPath: - path: /dev/vfio + path: /dev/vfio/vfio + - name: vfio-group + hostPath: + path: /dev/vfio/{{ .Values.qemu.hardwareGpu.vfioGroup }} + - name: lib-modules + hostPath: + path: /lib/modules + {{- end }} - name: snd hostPath: path: /dev/snd diff --git a/helm/templates/service.yaml b/helm/templates/service.yaml index 3161a34..a6383e3 100644 --- a/helm/templates/service.yaml +++ b/helm/templates/service.yaml @@ -18,10 +18,12 @@ spec: targetPort: 5999 protocol: TCP name: vnc - - port: 1359 - targetPort: 1359 - protocol: TCP - name: airmessage + {{- range .Values.service.extraPorts }} + - port: {{ .port }} + targetPort: {{ .targetPort }} + protocol: {{ .protocol }} + name: {{ .name }} + {{- end }} selector: app.kubernetes.io/name: {{ include "docker-osx.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} -- cgit v1.2.3