Reference
https://blog.csdn.net/jiuhebaobao/article/details/137256664
# reference
https://cloud.tencent.com/developer/article/2414840
# before
root@box02:~# dmesg | grep -E "DMAR|IOMMU"
echo nothing
# after reboot: IOMMU
root@box02:~# dmesg | grep -E "DMAR|IOMMU"
[ 0.000000] Warning: PCIe ACS overrides enabled; This may allow non-IOMMU protected peer-to-peer DMA
[ 0.004754] ACPI: DMAR 0x0000000072312000 000088 (v01 INTEL EDK2 00000002 01000013)
[ 0.004783] ACPI: Reserving DMAR table memory at [mem 0x72312000-0x72312087]
[ 0.068253] DMAR: IOMMU enabled
[ 0.160656] DMAR: Host address width 39
[ 0.160656] DMAR: DRHD base: 0x000000fed90000 flags: 0x0
[ 0.160661] DMAR: dmar0: reg_base_addr fed90000 ver 4:0 cap 1c0000c40660462 ecap 29a00f0505e
[ 0.160662] DMAR: DRHD base: 0x000000fed91000 flags: 0x1
[ 0.160665] DMAR: dmar1: reg_base_addr fed91000 ver 5:0 cap d2008c40660462 ecap f050da
[ 0.160666] DMAR: RMRR base: 0x0000007c000000 end: 0x000000807fffff
[ 0.160668] DMAR-IR: IOAPIC id 2 under DRHD base 0xfed91000 IOMMU 1
[ 0.160669] DMAR-IR: HPET id 0 under DRHD base 0xfed91000
[ 0.160669] DMAR-IR: Queued invalidation will be enabled to support x2apic and Intr-remapping.
[ 0.162202] DMAR-IR: Enabled IRQ remapping in x2apic mode
[ 0.358227] pci 0000:00:02.0: DMAR: Skip IOMMU disabling for graphics
[ 0.417403] DMAR: No ATSR found
[ 0.417404] DMAR: No SATC found
[ 0.417405] DMAR: IOMMU feature fl1gp_support inconsistent
[ 0.417406] DMAR: IOMMU feature pgsel_inv inconsistent
[ 0.417406] DMAR: IOMMU feature nwfs inconsistent
[ 0.417407] DMAR: IOMMU feature dit inconsistent
[ 0.417408] DMAR: IOMMU feature sc_support inconsistent
[ 0.417408] DMAR: IOMMU feature dev_iotlb_support inconsistent
[ 0.417409] DMAR: dmar0: Using Queued invalidation
[ 0.417411] DMAR: dmar1: Using Queued invalidation
[ 0.418166] DMAR: Intel(R) Virtualization Technology for Directed I/O
# verify vfio
box02:~# dmesg | grep -i vfio
[ 1.783340] VFIO - User Level meta-driver version: 0.3
[ 7.532731] vfio-pci 0000:01:00.0: vgaarb: VGA decodes changed: olddecodes=io+mem,decodes=io+mem:owns=none
[ 7.533818] vfio-pci 0000:01:00.0: vgaarb: VGA decodes changed: olddecodes=io+mem,decodes=io+mem:owns=none
[ 7.534253] vfio-pci 0000:01:00.0: vgaarb: VGA decodes changed: olddecodes=io+mem,decodes=io+mem:owns=none
# verify interrupt mapping
box02:~# dmesg | grep 'remapping'
[ 0.160669] DMAR-IR: Queued invalidation will be enabled to support x2apic and Intr-remapping.
[ 0.162202] DMAR-IR: Enabled IRQ remapping in x2apic mode
# grub
# update-initramfs -k all -u
update-initramfs: Generating /boot/initrd.img-6.8.12-8-pve
Running hook script 'zz-proxmox-boot'..
Re-executing '/etc/kernel/postinst.d/zz-proxmox-boot' in new private mount namespace..
No /etc/kernel/proxmox-boot-uuids found, skipping ESP sync.
Removable bootloader found at '/boot/efi/EFI/BOOT/BOOTX64.efi', but GRUB packages not set up to update it!
Run the following command:
echo 'grub-efi-amd64 grub2/force_efi_extra_removable boolean true' | debconf-set-selections -v -u
Then reinstall GRUB with 'apt install --reinstall grub-efi-amd64'
# before
$ sudo lspci
00:02.0 VGA compatible controller: Device 1234:1111 (rev 02)
# after
$ sudo lspci | grep VGA
00:02.0 VGA compatible controller: Device 1234:1111 (rev 02)
00:10.0 VGA compatible controller: NVIDIA Corporation TU106 [GeForce GTX 1650] (rev a1)
# driver
sudo apt-get remove --purge '^nvidia-.*'
sudo apt install nvidia-driver-550 nvidia-dkms-550
sudo reboot
nvidia-smi
$ nvidia-smi
Sat Mar 29 17:30:35 2025
+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 550.120 Driver Version: 550.120 CUDA Version: 12.4 |
|-----------------------------------------+------------------------+----------------------+
| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|=========================================+========================+======================|
| 0 NVIDIA GeForce GTX 1650 Off | 00000000:00:10.0 Off | N/A |
| 26% 37C P8 9W / 100W | 6MiB / 4096MiB | 0% Default |
| | | N/A |
+-----------------------------------------+------------------------+----------------------+
+-----------------------------------------------------------------------------------------+
| Processes: |
| GPU GI CI PID Type Process name GPU Memory |
| ID ID Usage |
|=========================================================================================|
| 0 N/A N/A 1638 G /usr/lib/xorg/Xorg 4MiB |
+-----------------------------------------------------------------------------------------+
# Chipset
Intel 7 Series X79 2011Q4 65nm PCIe2.0 8x1 4x11x4 USB 2.0 VT-d
Intel 9 Series X99 2014Q3 32nm PCIe2.0 x1 x2 x8 USB 3.0 VT-d
# bench AM4
3900x 12C 24T Single 2705 Multithread 32580 AM4 105W 2019
3700x 8C 16T Single 2658 Multithread 22493 AM4 2019
# hardware basic
DDR3 ECC 240 pin
DDR4 ECC 288 pin
HPE ProLiant Gen10 suppors Intel Optane Persistent memory
# hardware price
DDR3 32G $52
DDR4 16G $60
DDR4 32G $150
Intel Optane 128G DDR4 2666 $200
HP DL360G10 $1800
M2 250G $70
HSF $20
Case $20
PSU 400W $50
Display $25
# network card
Mellanox Technologies MT27800 Family [ConnectX-5]
# server choice
Dell R730xd $550 2U C602 24x DDR4 1866MT/s RDIMM LRDIMM 768GB H330 H730 H830 PCIe 3.0
Dell R720xd $380 2U C610 24x DDR3 2133MT/s RDIMM LRDIMM UDIMM 768GB H310 H710 H810 PCIe 3.0
Dell R630 1U C610 24x DDR4 2400MT/s (24x64GB=1.5TB)
Dell R620 1U C602
# bench X79 (Single 32GB DDR3 ECC)
E5 1650 v2 6C 12T Single 2049 Multithread 9329 FCLGA2011 130W 2013
# 14nm Gold Silver
Gold 6342 24C 48T Single 2396 Multithread 47076 FCLGA4189 230W 2021
Gold 6240 18C 36T Single 2294 Multithread 47999 FCLGA3647 150W 2019
Gold 6132 14C 28T Single 2125 Multithread 22247 FCLGA3647 140W 2018 ($100)
Silver 4314 16C 32T Single 2187 Multithread 28795 FCLGA4189 135W 2021
Platinum 8160 24C 48T Single 2077 Multithread 28825 FCLGA3647 150W 2017
# 14nm E5
E5 2696v4 22C 44T Single 2132 Multithread 24879 FCLGA2011-3 150W 2016 ($899)
E5 2686v4 18C 36T Single 1801 Multithread 20159 FCLGA2011-3 145W 2016 ($150)
E5 2680v4 14C 28T Single 1934 Multithread 17504 FCLGA2011-3 120W 2016 ($60)
HSF $55
# 22nm
E5 2698Bv3 16C 32T Single 1666 Multithread 16287 LGA2011-v3 135W 2014
E5 2690v3 12C 24T Single 1925 Multithread 16051 LGA2011-v3 135W 2014
E5 2678v3 12C 24T Single 1764 Multithread 14755 LGA2011-v3 120W 2015
E5 2676v3 12C 24T Single 1629 Multithread 13457 LGA2011-v3 120W 2015
E5 2673v3 12C 24T Single 1745 Multithread 14253 LGA2011-v3 110W 2015
E5 2666v3 10C 20T Single 1969 Multithread 14167 LGA2011-v3 135W 2015 ($70)
E5 2643v3 6C 12T Single 2079 Multithread 10434 LGA2011-v3 135W 2015
E5 2630v3 8C 16T Single 1751 Multithread 10333 LGA2011-v3 85W 2014
E5 2696v2 12C 24T Single 1695 Multithread 14048 LGA2011 120W 2013
E5 2690v2 10C 20T Single 1874 Multithread 13423 FCLGA2011 130W 2013
E5 2680v2 10C 20T Single 1790 Multithread 12697 FCLGA2011 115W 2013
E5 2670v2 10C 20T Single 1632 Multithread 11369 FCLGA2011 115W 2014
E5 2650v2 8C 16T Single 1680 Multithread 9847 FCLGA2011 95W 2013
E5 4620v3 10C 20T Single 1486 Multithread 10525 FCLGA2011-3 105W 2019
E5 4640v2 10C 20T Single 1373 Multithread 19028 LGA2011 2015
# 32nm
E5 1650 6C 12T Single 1727 Multithread 8085 LGA2011 130W 2012
E5 1620 4C 8T Single 1756 Multithread 5848 LGA2011 130W 2012
E5 2690 8C 16T Single 1660 Multithread 9755 LGA2011 135W 2012
E5 2689 8C 16T Single 1574 Multithread 9588 LGA2011 115W 2013
E5 2670 8C 16T Single 1474 Multithread 8974 LGA2011 115W 2012
E5 2665 8C 16T Single 1379 Multithread 8233 LGA2011 115W 2012
E5 2660 8C 16T Single 1393 Multithread 8096 LGA2011 95W 2012
E5 2650 8C 16T Single 1222 Multithread 7368 LGA2011 95W 2012
E5 2640 6C 12T Single 1344 Multithread 6280 LGA2011 95W 2012
E5 2620 6C 12T Single 1291 Multithread 6250 FCLGA2011 80W 2013
E5 4640 8C 16T Single 1233 Multithread 7012 LGA2011 95W 2012
E5 2450 8C 16T Single 1180 Multithread 7534 LGA1356 95W 2015
# Example - X99 Set
$265 E5 2680v4 14C 28T + X99-ZD4 (DDR4 max 128GB)
$180 E5 2676v3 12C 24T + X99 (4x DDR4)
$180 E5 2673v3 12C 24T + X99
$209 E5 2666v3 10C 20T + AngXun X99-P3 (4x DDR3 Max 128GB)
$200 E5 2666v3 10C 20T + X99 (4x DDR4)
$159 E5 2673v3 12C 24T + X99 (DDR3 DDR4)
$150 E5 4620v3 10C 20T + AngXun X99 v500 (DDR4/DDR3 Max 64GB) (choice)
# Example - X79 Set
$210 E5 2670v2 10C 20T + MSI x79 (M.2 yes) + 64GB ECC $210
$130 E5 2670 8C 16T + X79F - 2011JK2 2 slots (M.2 no) + 16GB ECC
$120 E5 1650v2 6C 12T + jginyue x79M
$138 E5 2665 8C 16T + X79 (M.2 yes) + 32GB
$70 E5 2650 8C 16T + X79
$100 E5 2689 8C 16T + X79
$138 E5 2620v2 + X79 + 16GB
$120 E5 2650v2 8C 16T + X79
$115 E5 1620 + X79 + 16GB
$90 E5 4640
$90 E5-4640 v2 + X79 + 16G
$121 E5 2680v2 10C 20T + X79-2011 (M.2) (choice)
$150 E5 2690v2 10C 20T + X79 (M.2) (choice)
$110 E5 2670v2 10C 20T + X79 (M.2) (choice)
2025 My Hardware Personal Computer Used
Check
Ubunut
sudo dmidecode -t 2
sudo lsblk -o NAME,FSTYPE,LABEL,MOUNTPOINT,SIZE,MODEL
2025
Intel(R) Core(TM) i9-14900 HKD$4300 @ 2025
24C 32T 65W Single 4411 Multi 46288
Samsung SSD 990 EVO 1TB PCIe 4.0 x4 HKD$500 @2025
ASUSTeK COMPUTER INC.
PRIME Z790M-PLUS
Intel® Z790 Chipset
3x M.2 PCIe 4.0 x4
4x SATA 6Gb/s
1GbE
$ lspci
00:00.0 Host bridge: Intel Corporation Device a700 (rev 01)
00:01.0 PCI bridge: Intel Corporation Device a70d (rev 01)
00:06.0 PCI bridge: Intel Corporation Device a74d (rev 01)
00:0a.0 Signal processing controller: Intel Corporation Device a77d (rev 01)
00:0e.0 RAID bus controller: Intel Corporation Device a77f
00:14.0 USB controller: Intel Corporation Device 7a60 (rev 11)
00:14.2 RAM memory: Intel Corporation Device 7a27 (rev 11)
00:15.0 Serial bus controller: Intel Corporation Device 7a4c (rev 11)
00:15.1 Serial bus controller: Intel Corporation Device 7a4d (rev 11)
0:15.2 Serial bus controller: Intel Corporation Device 7a4e (rev 11)
00:16.0 Communication controller: Intel Corporation Device 7a68 (rev 11)
00:17.0 SATA controller: Intel Corporation Device 7a62 (rev 11)
00:1a.0 PCI bridge: Intel Corporation Device 7a48 (rev 11)
00:1b.0 PCI bridge: Intel Corporation Device 7a40 (rev 11)
00:1c.0 PCI bridge: Intel Corporation Device 7a38 (rev 11)
00:1d.0 PCI bridge: Intel Corporation Device 7a30 (rev 11)
00:1d.4 PCI bridge: Intel Corporation Device 7a34 (rev 11)
00:1f.0 ISA bridge: Intel Corporation Device 7a04 (rev 11)
00:1f.3 Audio device: Intel Corporation Device 7a50 (rev 11)
00:1f.4 SMBus: Intel Corporation Device 7a23 (rev 11)
00:1f.5 Serial bus controller: Intel Corporation Device 7a24 (rev 11)
00:1f.6 Ethernet controller: Intel Corporation Device 0dc8 (rev 11)
01:00.0 VGA compatible controller: NVIDIA Corporation TU106 [GeForce GTX 1650] (rev a1)
01:00.1 Audio device: NVIDIA Corporation TU106 High Definition Audio Controller (rev a1)
02:00.0 Non-Volatile memory controller: Samsung Electronics Co Ltd Device a80d
07:00.0 Non-Volatile memory controller: Micron Technology Inc Device 5416 (rev 01)
2022-2023
AMD Ryzen 3900x
12C 24T
INTEL SSDPEKKF256G8L
B450M MORTAR MAX
2021
AMD Ryzen 7 3700x
8C 16T 65W Single 2658 Multi 22494
ADATA SX8200PNP PCIe Gen3 x4 HKD$580 @2021
ASUSTeK COMPUTER INC.
TUF GAMING B550M-PLUS
AMD B550
2x M.2 PCIe 3.0 x4
4x SATA 6Gb/s
# tool
sudo apt install nvme-cli
# mytest 01 3.6 GB/s Samsung SSD 990 EVO 1TB HKD$500
native Asus onboard nvme
sudo nvme list
Samsung SSD 990 EVO 1TB
$ time dd if=/dev/zero of=testfile bs=16k count=128k
2147483648 bytes (2.1 GB, 2.0 GiB) copied, 0.599403 s, 3.6 GB/s
# if promox ubuntu pcie device passthrough, also 3.8GB/s. no loss.
harddrivebenchmark says 4,547 MBytes/Sec
# mytest 02 112 MB/s FreeNas NVME HKD$430
NVME over FreeNAS and TPLINK 1Gbe Switch
sudo nvme list
ADATA SX8200PNP
$ time dd if=/dev/zero of=/mnt/ricky/testfile bs=16k count=128k
2147483648 bytes (2.1 GB, 2.0 GiB) copied, 19.0907 s, 112 MB/s
# mytest 03 3.9 GB/s Crucial 1TB
Crucial 1TB attached to Z790M-PLUS M.2_3 PCIe 4.0
sudo nvme list
CT1000P3PSSD8
$ time dd if=/dev/zero of=testfile bs=16k count=128k
2147483648 bytes (2.1 GB, 2.0 GiB) copied, 0.545956 s, 3.9 GB/s
harddrivebenchmark says 4,214 MBytes/Sec
# mytest 04 2.0 GB/s INTEL SSDPEKKF256G8L
2147483648 bytes (2.1 GB, 2.0 GiB) copied, 0.545956 s, 3.9 GB/s
3900x Mortar Max B450
sudo nvme list
INTEL SSDPEKKF256G8L
$ time dd if=/dev/zero of=testfile bs=16k count=128k
harddrivebenchmark says 2,430 MBytes/Sec
# mytest 05 292 MB/s WDC WD30PURX 3TB SATA
2147483648 bytes (2.1 GB, 2.0 GiB) copied, 7.36371 s, 292 MB/s
78P6ZY0 89P6ZY0
# test
TEST: Samsung 990 EVO 1TB M.2 SSD
Read speed 5000 MB/s
Write speed 4200 MB/s
Interface PCIe 4.0 x4
TEST: ADATA XPG SX8200 Pro 1TB M.2 SSD
Read speed 3500 MB/s
Write speed 2800 MB/s
Interface PCIe 3.0 x4
# Network
1 Gbps = 125 MB/s
2.5 Gbps = 312.5 MB/s
10 Gbps = 1250 MB/s = 1.25 GB/s
25 Gbps = 3125 MB/s = 3.125 GB/s
40 Gbps = 5000 MB/s = 5 GB/s
# NVME
https://ssd-tester.com/m2_ssd_test.php?sort=&sort_interface=PCIe+5.0+x4
The following speed transfer rates are possible: Theoretical Practical
SATA 3 6 GBit/s (=750 MB/s) ~550 MB/s
PCIe 2.0 x2 10 GBit/s (=1.250 MB/s) ~800 MB/s
PCIe 2.0 x4 20 GBit/s (=2.500 MB/s) ~1,600 MB/s
PCIe 3.0 x2 16 GBit/s (=2.000 MB/s) ~1,600 MB/s
PCIe 3.0 x4 32 GBit/s (=4.000 MB/s) ~3,500 MB/s
PCIe 4.0 x4 64 GBit/s (=8.000 MB/s) ~7,500 MB/s
PCIe 5.0 x4 128 GBit/s (=16.000 MB/s) ~15,000 MB/s
202501 Debian USB LAN Card Bridge
ChatGPT Question
I have a machine running debian 12 with a LAN port using a cable to connect to my office and can obtain below ip address correctly
2: enp6s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether f0:2f:74:2e:57:ea brd ff:ff:ff:ff:ff:ff
inet 10.20.15.101/24 brd 10.20.15.255 scope global dynamic noprefixroute enp6s0
valid_lft 82884sec preferred_lft 82884sec
inet6 fe80::f22f:74ff:fe2e:57ea/64 scope link noprefixroute
valid_lft forever preferred_lft forever
Now i want to add another NIC enx8cb0e9c91446 to that machine and allow another client machine to connect to this newly added NIC so that that client machines can also obtain ip address from that NIC as if that client machine is connect to the same subnet. How?
ChatGPT Answer (does not work, do not directly use)
To allow another client machine to connect to your Debian 12 machine via the newly added NIC enx8cb0e9c91446
and obtain an IP address from the same subnet, you can set up the new NIC as a bridge interface. This will enable the client machine to be on the same subnet and communicate with other devices in the network. Here's how you can achieve this:
Identify the New NIC: First, ensure that the new NIC enx8cb0e9c91446
is properly identified and recognized by your Debian 12 system.
Install Bridge Utilities (if needed): If the bridge utilities package is not already installed on your system, you can install it by running:
sudo apt update
sudo apt install bridge-utils
Configure the Bridge Interface: Create a bridge interface that includes both enp6s0
and enx8cb0e9c91446
. You can create a bridge configuration file in the /etc/network/interfaces.d/
directory. Here's an example configuration for the bridge interface:
Add the following configuration to a new file, for example, /etc/network/interfaces.d/br0
:
auto enp6s0
iface enp6s0 inet manual
auto enx8cb0e9c91446
iface enx8cb0e9c91446 inet manual
auto br0
iface br0 inet dhcp
bridge_ports enp6s0 enx8cb0e9c91446
Restart Networking Service:
After saving the configuration, you can restart the networking service to apply the changes:
sudo systemctl restart networking
Verify the Bridge Configuration:
You can verify that the bridge interface is created and both NICs are part of it by running:
ip addr show br0
Connect the Client Machine:
Connect the client machine to the new NIC enx8cb0e9c91446
. The client machine should now be able to obtain an IP address from the same subnet as 10.20.15.101
and communicate with other devices on the network.
By setting up a bridge interface in Debian 12, you are allowing the new client machine to be on the same subnet as your original NIC enp6s0
, enabling seamless communication within the network. If you have any questions or need further assistance, feel free to ask.
Exercise
before
cat /etc/*release
PRETTY_NAME="Debian GNU/Linux 12 (bookworm)"
after
ip -br a
lo UNKNOWN 127.0.0.1/8 ::1/128
enp6s0 UP 10.20.15.101/24 fe80::f22f:74ff:fe2e:57ea/64
docker0 DOWN 172.17.0.1/16 fe80::42:fdff:feea:3e0f/64
enx8cb0e9c91446 DOWN
br0 UP 10.20.15.127/24 fe80::384e:eaff:fe87:beca/64
how to use have the original ip address 10.20.15.101 for br0?
Ask ChatGPT
my br0 now has an ip address 10.20.15.127 from dhcp. now i want to use keepalived so that it attach an virtual ip address 10.20.15.101 so that other users using 10.20.15.101:24000 will be redirect to 10.20.15.127:24000. how?
sudo apt update
sudo apt install keepalived
$ cat /etc/keepalived/keepalived.conf
global_defs {
router_id keepalived_node
}
vrrp_instance VI_1 {
state MASTER
interface br0
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass your_password
}
virtual_ipaddress {
10.20.15.101
}
}
virtual_server 10.20.15.101 24000 {
delay_loop 2
lb_algo rr
lb_kind NAT
persistence_timeout 0
protocol TCP
real_server 10.20.15.127 24000 {
weight 1
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
sudo systemctl restart keepalived
$ ip -br a
lo UNKNOWN 127.0.0.1/8 ::1/128
enp6s0 UP
docker0 DOWN 172.17.0.1/16 fe80::42:fdff:feea:3e0f/64
enx8cb0e9c91446 DOWN
br0 UP 10.20.15.127/24 10.20.15.101/32 fe80::384e:eaff:fe87:beca/64
enx00e04c534458 DOWN
Reference https://blog.csdn.net/jiuhebaobao/article/details/137256664