2025-03-29 Debian 12 Promox 8.3.5 IOMMU

 

# reference

https://cloud.tencent.com/developer/article/2414840

# before

root@box02:~# dmesg | grep -E "DMAR|IOMMU"

echo nothing

# after reboot: IOMMU

root@box02:~# dmesg | grep -E "DMAR|IOMMU"

[    0.000000] Warning: PCIe ACS overrides enabled; This may allow non-IOMMU protected peer-to-peer DMA

[    0.004754] ACPI: DMAR 0x0000000072312000 000088 (v01 INTEL  EDK2     00000002      01000013)

[    0.004783] ACPI: Reserving DMAR table memory at [mem 0x72312000-0x72312087]

[    0.068253] DMAR: IOMMU enabled

[    0.160656] DMAR: Host address width 39

[    0.160656] DMAR: DRHD base: 0x000000fed90000 flags: 0x0

[    0.160661] DMAR: dmar0: reg_base_addr fed90000 ver 4:0 cap 1c0000c40660462 ecap 29a00f0505e

[    0.160662] DMAR: DRHD base: 0x000000fed91000 flags: 0x1

[    0.160665] DMAR: dmar1: reg_base_addr fed91000 ver 5:0 cap d2008c40660462 ecap f050da

[    0.160666] DMAR: RMRR base: 0x0000007c000000 end: 0x000000807fffff

[    0.160668] DMAR-IR: IOAPIC id 2 under DRHD base  0xfed91000 IOMMU 1

[    0.160669] DMAR-IR: HPET id 0 under DRHD base 0xfed91000

[    0.160669] DMAR-IR: Queued invalidation will be enabled to support x2apic and Intr-remapping.

[    0.162202] DMAR-IR: Enabled IRQ remapping in x2apic mode

[    0.358227] pci 0000:00:02.0: DMAR: Skip IOMMU disabling for graphics

[    0.417403] DMAR: No ATSR found

[    0.417404] DMAR: No SATC found

[    0.417405] DMAR: IOMMU feature fl1gp_support inconsistent

[    0.417406] DMAR: IOMMU feature pgsel_inv inconsistent

[    0.417406] DMAR: IOMMU feature nwfs inconsistent

[    0.417407] DMAR: IOMMU feature dit inconsistent

[    0.417408] DMAR: IOMMU feature sc_support inconsistent

[    0.417408] DMAR: IOMMU feature dev_iotlb_support inconsistent

[    0.417409] DMAR: dmar0: Using Queued invalidation

[    0.417411] DMAR: dmar1: Using Queued invalidation

[    0.418166] DMAR: Intel(R) Virtualization Technology for Directed I/O

# verify vfio

box02:~# dmesg | grep -i vfio

[    1.783340] VFIO - User Level meta-driver version: 0.3

[    7.532731] vfio-pci 0000:01:00.0: vgaarb: VGA decodes changed: olddecodes=io+mem,decodes=io+mem:owns=none

[    7.533818] vfio-pci 0000:01:00.0: vgaarb: VGA decodes changed: olddecodes=io+mem,decodes=io+mem:owns=none

[    7.534253] vfio-pci 0000:01:00.0: vgaarb: VGA decodes changed: olddecodes=io+mem,decodes=io+mem:owns=none

# verify interrupt mapping

box02:~# dmesg | grep 'remapping'

[    0.160669] DMAR-IR: Queued invalidation will be enabled to support x2apic and Intr-remapping.

[    0.162202] DMAR-IR: Enabled IRQ remapping in x2apic mode

# grub

# update-initramfs -k all -u

update-initramfs: Generating /boot/initrd.img-6.8.12-8-pve

Running hook script 'zz-proxmox-boot'..

Re-executing '/etc/kernel/postinst.d/zz-proxmox-boot' in new private mount namespace..

No /etc/kernel/proxmox-boot-uuids found, skipping ESP sync.

Removable bootloader found at '/boot/efi/EFI/BOOT/BOOTX64.efi', but GRUB packages not set up to update it!

Run the following command:

echo 'grub-efi-amd64 grub2/force_efi_extra_removable boolean true' | debconf-set-selections -v -u

Then reinstall GRUB with 'apt install --reinstall grub-efi-amd64'

# before

$ sudo lspci

00:02.0 VGA compatible controller: Device 1234:1111 (rev 02)

# after

$ sudo lspci | grep VGA

00:02.0 VGA compatible controller: Device 1234:1111 (rev 02)

00:10.0 VGA compatible controller: NVIDIA Corporation TU106 [GeForce GTX 1650] (rev a1)

# driver

sudo apt-get remove --purge '^nvidia-.*'

sudo apt install nvidia-driver-550 nvidia-dkms-550

sudo reboot

nvidia-smi

$ nvidia-smi

Sat Mar 29 17:30:35 2025       

+-----------------------------------------------------------------------------------------+

| NVIDIA-SMI 550.120                Driver Version: 550.120        CUDA Version: 12.4     |

|-----------------------------------------+------------------------+----------------------+

| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |

| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |

|                                         |                        |               MIG M. |

|=========================================+========================+======================|

|   0  NVIDIA GeForce GTX 1650        Off |   00000000:00:10.0 Off |                  N/A |

| 26%   37C    P8              9W /  100W |       6MiB /   4096MiB |      0%      Default |

|                                         |                        |                  N/A |

+-----------------------------------------+------------------------+----------------------+

                                                                                         

+-----------------------------------------------------------------------------------------+

| Processes:                                                                              |

|  GPU   GI   CI        PID   Type   Process name                              GPU Memory |

|        ID   ID                                                               Usage      |

|=========================================================================================|

|    0   N/A  N/A      1638      G   /usr/lib/xorg/Xorg                              4MiB |

+-----------------------------------------------------------------------------------------+

# more

:~$ nvidia-smi --query-gpu=driver_version --format=csv

driver_version

550.120

# desktop manager

sudo nvidia-settings

# also

nano /etc/modprobe.d/pve-blacklist.conf

blacklist radeon

blacklist nouveau

blacklist nvidia

# something

nano /etc/modprobe.d/pve-blacklist.conf

blacklist radeon

blacklist nouveau

blacklist nvidia

touch /etc/modprobe.d/vfio.conf

nano /etc/modprobe.d/vfio.conf

options vfio-pci ids=10de:1f0a,10de:10f9

update-initramfs -u

lsmod | grep vfio

# lsmod | grep vfio

vfio_pci               16384  2

vfio_pci_core          86016  1 vfio_pci

irqbypass              12288  3 vfio_pci_core,kvm

vfio_iommu_type1       49152  1

vfio                   65536  9 vfio_pci_core,vfio_iommu_type1,vfio_pci

iommufd                94208  1 vfio

# also 2

https://www.mereith.com/post/116#%E8%99%9A%E6%8B%9F%E6%9C%BA%E8%AE%BE%E7%BD%AE

# also 3

https://upsangel.com/htpc-nas/iommu-vfio-gpo-proxmox-pve-pcie%E7%9B%B4%E9%80%9A%E5%92%8C%E6%A0%B8%E9%A1%AFhdmi%E7%9B%B4%E9%80%9A%E8%A8%AD%E7%BD%AE%E5%8F%83%E6%95%B8%E8%A7%A3%E9%87%8B/

vfio.conf 没有 disable_vga=1，有的删掉！

# also 4

https://blog.csdn.net/qq_34419607/article/details/139872906

vga=none! then win10 ok

# revisit

# 01 file

vi /etc/default/grub

# original

# GRUB_CMDLINE_LINUX_DEFAULT="quiet"

# original to updated

# GRUB_CMDLINE_LINUX_DEFAULT="quiet intel_iommu=on iommu=pt initcall_blacklist=sysfb_init pcie_acs_override=downstream video=efifb:off,vesa

fb:off vfio-pci.ids=10de:1f0a,10de:10f9"

GRUB_CMDLINE_LINUX_DEFAULT="quiet intel_iommu=on iommu=pt initcall_blacklist=sysfb_init pcie_acs_override=downstream"

# 02 file

cat /etc/modprobe.d/blacklist.conf 

# amd

blacklist radeon

blacklist amdgpu

# nvidia

blacklist nouveau

blacklist nvidia

blacklist nvidiafb

blacklist nvidia_drm

# options

options vfio_iommu_type1 allow_unsafe_interrupts=1

# 03 file

# cat /etc/modprobe.d/iommu_unsafe_interrupts.conf 

options vfio_iommu_type1 allow_unsafe_interrupts=1

# 04 execute

update-initramfs -u -k all

# 05 virtual machine

32GB

16 core

BIOS UEFI

Display none

Machine i440fx

scsi2 /dev/disk/by-id/nvme-Samsung_SSD_990

hostpci0 01:00

hostpci1 00:14

2025 Old Server Hardware

 2025 Old Server Hardware

# Chipset

Intel 7 Series X79 2011Q4 65nm PCIe2.0 8x1 4x11x4 USB 2.0 VT-d

Intel 9 Series X99 2014Q3 32nm PCIe2.0 x1 x2 x8 USB 3.0 VT-d

# bench AM4

3900x 12C 24T Single 2705 Multithread 32580 AM4 105W 2019

3700x 8C 16T Single 2658 Multithread 22493 AM4 2019

# hardware basic

DDR3 ECC 240 pin

DDR4 ECC 288 pin

HPE ProLiant Gen10 suppors Intel Optane Persistent memory

# hardware price

DDR3 32G $52

DDR4 16G $60

DDR4 32G $150

Intel Optane 128G DDR4 2666 $200

HP DL360G10 $1800

M2 250G $70

HSF $20

Case $20

PSU 400W $50

Display $25

# network card

Mellanox Technologies MT27800 Family [ConnectX-5]

# server choice

Dell R730xd $550 2U C602 24x DDR4 1866MT/s RDIMM LRDIMM 768GB H330 H730 H830 PCIe 3.0

Dell R720xd $380 2U C610 24x DDR3 2133MT/s RDIMM LRDIMM UDIMM 768GB H310 H710 H810 PCIe 3.0

Dell R630 1U C610 24x DDR4 2400MT/s (24x64GB=1.5TB)

Dell R620 1U C602

# bench X79 (Single 32GB DDR3 ECC)

E5 1650 v2 6C 12T Single 2049 Multithread 9329 FCLGA2011 130W 2013

# 14nm Gold Silver

Gold 6342 24C 48T Single 2396 Multithread 47076 FCLGA4189 230W 2021

Gold 6240 18C 36T Single 2294 Multithread 47999 FCLGA3647 150W 2019

Gold 6132 14C 28T Single 2125 Multithread 22247 FCLGA3647 140W 2018 ($100)

Silver 4314 16C 32T Single 2187 Multithread 28795 FCLGA4189 135W 2021

Platinum 8160 24C 48T Single 2077 Multithread 28825 FCLGA3647 150W 2017

# 14nm E5

E5 2696v4 22C 44T Single 2132 Multithread 24879 FCLGA2011-3 150W 2016 ($899)

E5 2686v4 18C 36T Single 1801 Multithread 20159 FCLGA2011-3 145W 2016 ($150)

E5 2680v4 14C 28T Single 1934 Multithread 17504 FCLGA2011-3 120W 2016 ($60)

HSF $55

# 22nm

E5 2698Bv3 16C 32T Single 1666 Multithread 16287 LGA2011-v3 135W 2014

E5 2690v3 12C 24T Single 1925 Multithread 16051 LGA2011-v3 135W 2014

E5 2678v3 12C 24T Single 1764 Multithread 14755 LGA2011-v3 120W 2015

E5 2676v3 12C 24T Single 1629 Multithread 13457 LGA2011-v3 120W 2015

E5 2673v3 12C 24T Single 1745 Multithread 14253 LGA2011-v3 110W 2015

E5 2666v3 10C 20T Single 1969 Multithread 14167 LGA2011-v3 135W 2015 ($70)

E5 2643v3 6C 12T Single 2079 Multithread 10434 LGA2011-v3 135W 2015

E5 2630v3 8C 16T Single 1751 Multithread 10333 LGA2011-v3 85W 2014

E5 2696v2 12C 24T Single 1695 Multithread 14048 LGA2011 120W 2013

E5 2690v2 10C 20T Single 1874 Multithread 13423 FCLGA2011 130W 2013

E5 2680v2 10C 20T Single 1790 Multithread 12697 FCLGA2011 115W 2013

E5 2670v2 10C 20T Single 1632 Multithread 11369 FCLGA2011 115W 2014

E5 2650v2 8C 16T Single 1680 Multithread 9847 FCLGA2011 95W 2013

E5 4620v3 10C 20T Single 1486 Multithread 10525 FCLGA2011-3 105W 2019

E5 4640v2 10C 20T Single 1373 Multithread 19028 LGA2011 2015

# 32nm

E5 1650 6C 12T Single 1727 Multithread 8085 LGA2011 130W 2012

E5 1620 4C 8T Single 1756 Multithread 5848 LGA2011 130W 2012

E5 2690 8C 16T Single 1660 Multithread 9755 LGA2011 135W 2012

E5 2689 8C 16T Single 1574 Multithread 9588 LGA2011 115W 2013

E5 2670 8C 16T Single 1474 Multithread 8974 LGA2011 115W 2012

E5 2665 8C 16T Single 1379 Multithread 8233 LGA2011 115W 2012

E5 2660 8C 16T Single 1393 Multithread 8096 LGA2011 95W 2012

E5 2650 8C 16T Single 1222 Multithread 7368 LGA2011 95W 2012

E5 2640 6C 12T Single 1344 Multithread 6280 LGA2011 95W 2012

E5 2620 6C 12T Single 1291 Multithread 6250 FCLGA2011 80W 2013

E5 4640 8C 16T Single 1233 Multithread 7012 LGA2011 95W 2012

E5 2450 8C 16T Single 1180 Multithread 7534 LGA1356 95W 2015

# Example - X99 Set

$265 E5 2680v4 14C 28T + X99-ZD4 (DDR4 max 128GB)

$180 E5 2676v3 12C 24T + X99 (4x DDR4)

$180 E5 2673v3 12C 24T + X99

$209 E5 2666v3 10C 20T + AngXun X99-P3 (4x DDR3 Max 128GB)

$200 E5 2666v3 10C 20T + X99 (4x DDR4)

$159 E5 2673v3 12C 24T + X99 (DDR3 DDR4)

$150 E5 4620v3 10C 20T + AngXun X99 v500 (DDR4/DDR3 Max 64GB) (choice)

# Example - X79 Set

$210 E5 2670v2 10C 20T + MSI x79 (M.2 yes) + 64GB ECC $210

$130 E5 2670 8C 16T + X79F - 2011JK2 2 slots (M.2 no) + 16GB ECC

$120 E5 1650v2 6C 12T + jginyue x79M

$138 E5 2665 8C 16T + X79 (M.2 yes) + 32GB

$70 E5 2650 8C 16T + X79

$100 E5 2689 8C 16T + X79

$138 E5 2620v2 + X79 + 16GB

$120 E5 2650v2 8C 16T + X79

$115 E5 1620 + X79 + 16GB

$90 E5 4640

$90 E5-4640 v2 + X79 + 16G

$121 E5 2680v2 10C 20T + X79-2011 (M.2) (choice)

$150 E5 2690v2 10C 20T + X79 (M.2) (choice)

$110 E5 2670v2 10C 20T + X79 (M.2) (choice)

2025 My Hardware Personal Computer Used

 2025 My Hardware Personal Computer Used

Check
Ubunut
sudo dmidecode -t 2
sudo lsblk -o NAME,FSTYPE,LABEL,MOUNTPOINT,SIZE,MODEL

2025

Intel(R) Core(TM) i9-14900 HKD$4300 @ 2025
24C 32T 65W Single 4411 Multi 46288

Samsung SSD 990 EVO 1TB PCIe 4.0 x4 HKD$500 @2025

ASUSTeK COMPUTER INC.
PRIME Z790M-PLUS
Intel® Z790 Chipset
3x M.2 PCIe 4.0 x4
4x SATA 6Gb/s
1GbE

$ lspci
00:00.0 Host bridge: Intel Corporation Device a700 (rev 01)
00:01.0 PCI bridge: Intel Corporation Device a70d (rev 01)
00:06.0 PCI bridge: Intel Corporation Device a74d (rev 01)
00:0a.0 Signal processing controller: Intel Corporation Device a77d (rev 01)
00:0e.0 RAID bus controller: Intel Corporation Device a77f
00:14.0 USB controller: Intel Corporation Device 7a60 (rev 11)
00:14.2 RAM memory: Intel Corporation Device 7a27 (rev 11)
00:15.0 Serial bus controller: Intel Corporation Device 7a4c (rev 11)
00:15.1 Serial bus controller: Intel Corporation Device 7a4d (rev 11)
0:15.2 Serial bus controller: Intel Corporation Device 7a4e (rev 11)
00:16.0 Communication controller: Intel Corporation Device 7a68 (rev 11)
00:17.0 SATA controller: Intel Corporation Device 7a62 (rev 11)
00:1a.0 PCI bridge: Intel Corporation Device 7a48 (rev 11)
00:1b.0 PCI bridge: Intel Corporation Device 7a40 (rev 11)
00:1c.0 PCI bridge: Intel Corporation Device 7a38 (rev 11)
00:1d.0 PCI bridge: Intel Corporation Device 7a30 (rev 11)
00:1d.4 PCI bridge: Intel Corporation Device 7a34 (rev 11)
00:1f.0 ISA bridge: Intel Corporation Device 7a04 (rev 11)
00:1f.3 Audio device: Intel Corporation Device 7a50 (rev 11)
00:1f.4 SMBus: Intel Corporation Device 7a23 (rev 11)
00:1f.5 Serial bus controller: Intel Corporation Device 7a24 (rev 11)
00:1f.6 Ethernet controller: Intel Corporation Device 0dc8 (rev 11)
01:00.0 VGA compatible controller: NVIDIA Corporation TU106 [GeForce GTX 1650] (rev a1)
01:00.1 Audio device: NVIDIA Corporation TU106 High Definition Audio Controller (rev a1)
02:00.0 Non-Volatile memory controller: Samsung Electronics Co Ltd Device a80d
07:00.0 Non-Volatile memory controller: Micron Technology Inc Device 5416 (rev 01)

2022-2023

AMD Ryzen 3900x
12C 24T

INTEL SSDPEKKF256G8L

B450M MORTAR MAX

2021

AMD Ryzen 7 3700x
8C 16T 65W Single 2658 Multi 22494

ADATA SX8200PNP PCIe Gen3 x4 HKD$580 @2021

ASUSTeK COMPUTER INC.
TUF GAMING B550M-PLUS
AMD B550
2x M.2 PCIe 3.0 x4
4x SATA 6Gb/s

2025 NVME Disk Speed Test

# tool
sudo apt install nvme-cli

# mytest 01 3.6 GB/s Samsung SSD 990 EVO 1TB HKD$500
native Asus onboard nvme
sudo nvme list
Samsung SSD 990 EVO 1TB
$ time dd if=/dev/zero of=testfile bs=16k count=128k
2147483648 bytes (2.1 GB, 2.0 GiB) copied, 0.599403 s, 3.6 GB/s
# if promox ubuntu pcie device passthrough, also 3.8GB/s. no loss.

harddrivebenchmark says 4,547 MBytes/Sec

# mytest 02 112 MB/s FreeNas NVME HKD$430
NVME over FreeNAS and TPLINK 1Gbe Switch
sudo nvme list
ADATA SX8200PNP
$ time dd if=/dev/zero of=/mnt/ricky/testfile bs=16k count=128k
2147483648 bytes (2.1 GB, 2.0 GiB) copied, 19.0907 s, 112 MB/s

# mytest 03 3.9 GB/s Crucial 1TB

Crucial 1TB attached to Z790M-PLUS M.2_3 PCIe 4.0
sudo nvme list
CT1000P3PSSD8
$ time dd if=/dev/zero of=testfile bs=16k count=128k
2147483648 bytes (2.1 GB, 2.0 GiB) copied, 0.545956 s, 3.9 GB/s

harddrivebenchmark says 4,214 MBytes/Sec

# mytest 04 2.0 GB/s INTEL SSDPEKKF256G8L

2147483648 bytes (2.1 GB, 2.0 GiB) copied, 0.545956 s, 3.9 GB/s

3900x Mortar Max B450
sudo nvme list
INTEL SSDPEKKF256G8L
$ time dd if=/dev/zero of=testfile bs=16k count=128k

harddrivebenchmark says 2,430 MBytes/Sec

# mytest 05 292 MB/s WDC WD30PURX 3TB SATA

2147483648 bytes (2.1 GB, 2.0 GiB) copied, 7.36371 s, 292 MB/s

78P6ZY0 89P6ZY0

# test

TEST: Samsung 990 EVO 1TB M.2 SSD
Read speed 5000 MB/s
Write speed 4200 MB/s
Interface PCIe 4.0 x4

TEST: ADATA XPG SX8200 Pro 1TB M.2 SSD
Read speed 3500 MB/s
Write speed 2800 MB/s
Interface PCIe 3.0 x4

# Network

1 Gbps = 125 MB/s
2.5 Gbps = 312.5 MB/s
10 Gbps = 1250 MB/s = 1.25 GB/s
25 Gbps = 3125 MB/s = 3.125 GB/s
40 Gbps = 5000 MB/s = 5 GB/s

# NVME

https://ssd-tester.com/m2_ssd_test.php?sort=&sort_interface=PCIe+5.0+x4

The following speed transfer rates are possible: Theoretical Practical

SATA 3 6 GBit/s (=750 MB/s) ~550 MB/s
PCIe 2.0 x2 10 GBit/s (=1.250 MB/s) ~800 MB/s
PCIe 2.0 x4 20 GBit/s (=2.500 MB/s) ~1,600 MB/s
PCIe 3.0 x2 16 GBit/s (=2.000 MB/s) ~1,600 MB/s
PCIe 3.0 x4 32 GBit/s (=4.000 MB/s) ~3,500 MB/s
PCIe 4.0 x4 64 GBit/s (=8.000 MB/s) ~7,500 MB/s
PCIe 5.0 x4 128 GBit/s (=16.000 MB/s) ~15,000 MB/s