Linux 内核 devmem 分析

Linux 内核中 "devmem" 涵盖两个相互独立的子系统：

1. /dev/mem — 经典物理内存字符设备
2. Device Memory TCP (devmem TCP) — 新型网络零拷贝特性

一、/dev/mem — 物理内存字符设备

1.0 设计动机

/dev/mem 为什么存在？

早期 Unix/Linux 系统中，用户空间工具需要直接访问物理地址空间才能完成某些底层任务。典型场景包括：

• X11 显示服务器：早期 X server 通过 /dev/mem 直接映射显卡 MMIO 寄存器和显存来驱动硬件
• BIOS/ACPI 工具：dmidecode、acpidump 等工具需要读取低 1MB 的 BIOS 数据区（如 DMI 表、ACPI RSDP）
• dosemu：DOS 模拟器需要访问 VGA 显存（0xA0000-0xBFFFF）等传统 PC 内存映射区域
• 硬件调试：嵌入式开发中直接读写 SoC 寄存器（devmem2 工具）

这些需求在内核驱动框架尚不完善的年代是合理的。/dev/mem 提供了一个统一的"后门"，让用户空间可以直接操作物理地址空间。 1 2

1.1 概述

/dev/mem 是一个字符设备，允许用户空间程序直接读写物理内存地址空间（包括 RAM、PCI MMIO 区域、BIOS 数据区等）。它由 drivers/char/mem.c 实现，minor 号为 1：

// drivers/char/mem.c:34-35#define DEVMEM_MINOR  1#define DEVPORT_MINOR  4

1.2 设备注册

所有内存类字符设备共用 major 号 MEM_MAJOR，通过 devlist[] 数组统一管理：

// drivers/char/mem.c:688-708static const struct memdev {const char *name;const struct file_operations *fops;fmode_t fmode;umode_t mode;} devlist[] = {#ifdef CONFIG_DEVMEM  [DEVMEM_MINOR] = { "mem", &mem_fops, 0, 0 },#endif  [3] = { "null", &null_fops, FMODE_NOWAIT, 0666 },#ifdef CONFIG_DEVPORT  [4] = { "port", &port_fops, 0, 0 },#endif  [5] = { "zero", &zero_fops, FMODE_NOWAIT, 0666 },  [7] = { "full", &full_fops, 0, 0666 },  [8] = { "random", &random_fops, FMODE_NOWAIT, 0666 },  [9] = { "urandom", &urandom_fops, FMODE_NOWAIT, 0666 },#ifdef CONFIG_PRINTK  [11] = { "kmsg", &kmsg_fops, 0, 0644 },#endif};

在 chr_dev_init() 中完成注册：

// drivers/char/mem.c:749-776static int __init chr_dev_init(void){int retval;int minor;if (register_chrdev(MEM_MAJOR, "mem", &memory_fops))    printk("unable to get major %d for memory devs\n", MEM_MAJOR);  retval = class_register(&mem_class);if (retval)return retval;for (minor = 1; minor < ARRAY_SIZE(devlist); minor++) {if (!devlist[minor].name)continue;if ((minor == DEVPORT_MINOR) && !arch_has_dev_port())continue;    device_create(&mem_class, NULL, MKDEV(MEM_MAJOR, minor),NULL, devlist[minor].name);  }return tty_init();}

/dev/mem 仅在 CONFIG_DEVMEM 编译选项开启时才会被注册。

1.3 文件操作接口

// drivers/char/mem.c:634-645static const struct file_operations __maybe_unused mem_fops = {  .llseek    = memory_lseek,  .read    = read_mem,  .write    = write_mem,  .mmap_prepare  = mmap_mem_prepare,  .open    = open_mem,#ifndef CONFIG_MMU  .get_unmapped_area = get_unmapped_area_mem,  .mmap_capabilities = memory_mmap_capabilities,#endif  .fop_flags  = FOP_UNSIGNED_OFFSET,};

1.4 打开设备：权限控制

// drivers/char/mem.c:603-625static int open_port(struct inode *inode, struct file *filp){int rc;if (!capable(CAP_SYS_RAWIO))return -EPERM;  rc = security_locked_down(LOCKDOWN_DEV_MEM);if (rc)return rc;if (iminor(inode) != DEVMEM_MINOR)return 0;/*   * Use a unified address space to have a single point to manage   * revocations when drivers want to take over a /dev/mem mapped   * range.   */  filp->f_mapping = iomem_get_mapping();return 0;}

• 必须具备 CAP_SYS_RAWIO 能力
• 检查内核 lockdown 状态（LOCKDOWN_DEV_MEM）
• 使用统一的 iomem_get_mapping() 地址空间，以便驱动接管时可以撤销映射

1.5 读物理内存：read_mem()

// drivers/char/mem.c:82-167static ssize_t read_mem(struct file *file, char __user *buf,size_t count, loff_t *ppos){phys_addr_t p = *ppos;ssize_t read, sz;void *ptr;char *bounce;int err;if (p != *ppos)return 0;if (!valid_phys_addr_range(p, count))return -EFAULT;  read = 0;#ifdef __ARCH_HAS_NO_PAGE_ZERO_MAPPEDif (p < PAGE_SIZE) {    sz = size_inside_page(p, count);if (sz > 0) {if (clear_user(buf, sz))return -EFAULT;      buf += sz;      p += sz;      count -= sz;      read += sz;    }  }#endif  bounce = kmalloc(PAGE_SIZE, GFP_KERNEL);if (!bounce)return -ENOMEM;while (count > 0) {unsigned long remaining;int allowed, probe;    sz = size_inside_page(p, count);    err = -EPERM;    allowed = page_is_allowed(p >> PAGE_SHIFT);if (!allowed)goto failed;    err = -EFAULT;if (allowed == 2) {/* Show zeros for restricted memory. */      remaining = clear_user(buf, sz);    } else {      ptr = xlate_dev_mem_ptr(p);if (!ptr)goto failed;      probe = copy_from_kernel_nofault(bounce, ptr, sz);      unxlate_dev_mem_ptr(p, ptr);if (probe)goto failed;      remaining = copy_to_user(buf, bounce, sz);    }if (remaining)goto failed;    buf += sz;    p += sz;    count -= sz;    read += sz;if (should_stop_iteration())break;  }  kfree(bounce);  *ppos += read;return read;failed:  kfree(bounce);return err;}

逐页读取，使用内核 bounce buffer 避免直接操作用户指针。allowed == 2 时显示为全零（兼容低 1MB 区域）。

1.6 写物理内存：write_mem()

// drivers/char/mem.c:169-240static ssize_t write_mem(struct file *file, const char __user *buf,size_t count, loff_t *ppos){phys_addr_t p = *ppos;ssize_t written, sz;unsigned long copied;void *ptr;if (p != *ppos)return -EFBIG;if (!valid_phys_addr_range(p, count))return -EFAULT;  written = 0;while (count > 0) {int allowed;    sz = size_inside_page(p, count);    allowed = page_is_allowed(p >> PAGE_SHIFT);if (!allowed)return -EPERM;/* Skip actual writing when a page is marked as restricted. */if (allowed == 1) {      ptr = xlate_dev_mem_ptr(p);if (!ptr) {if (written)break;return -EFAULT;      }      copied = copy_from_user(ptr, buf, sz);      unxlate_dev_mem_ptr(p, ptr);if (copied) {        written += sz - copied;if (written)break;return -EFAULT;      }    }    buf += sz;    p += sz;    count -= sz;    written += sz;if (should_stop_iteration())break;  }  *ppos += written;return written;}

allowed == 2（受限区域）时跳过实际写入（静默忽略）。

1.7 内存映射：mmap_mem_prepare()

// drivers/char/mem.c:325-363static int mmap_mem_prepare(struct vm_area_desc *desc){struct file *file = desc->file;const size_t size = vma_desc_size(desc);const phys_addr_t offset = (phys_addr_t)desc->pgoff << PAGE_SHIFT;if (offset >> PAGE_SHIFT != desc->pgoff)return -EINVAL;if (offset + (phys_addr_t)size - 1 < offset)return -EINVAL;if (!valid_mmap_phys_addr_range(desc->pgoff, size))return -EINVAL;if (!private_mapping_ok(desc))return -ENOSYS;if (!range_is_allowed(desc->pgoff, size))return -EPERM;if (!phys_mem_access_prot_allowed(file, desc->pgoff, size,            &desc->page_prot))return -EINVAL;  desc->page_prot = phys_mem_access_prot(file, desc->pgoff,                 size,                 desc->page_prot);  desc->vm_ops = &mmap_mem_ops;/* Remap-pfn-range will mark the range with the I/O flag. */  mmap_action_remap_full(desc, desc->pgoff);  desc->action.error_override = -EAGAIN;return 0;}

二、访问控制：CONFIG_STRICT_DEVMEM 与 CONFIG_IO_STRICT_DEVMEM

2.0 设计动机

CONFIG_STRICT_DEVMEM 为什么引入？

/dev/mem 的无限制访问带来了严重的安全隐患：

• 内核内存泄露：拥有 root 权限的攻击者可以通过 /dev/mem 读取任意物理内存，包括内核代码、内核数据结构、加密密钥、其他进程的内存等
• 内核代码篡改：可以直接写入内核代码页，绕过所有内核安全机制（如 SELinux、AppArmor）
• 权限提升：即使在受限的 root 环境（如容器）中，/dev/mem 也可能成为逃逸手段

CONFIG_STRICT_DEVMEM 的核心思路是：只允许访问非 RAM 区域（PCI MMIO、BIOS 区域等），拒绝访问内核使用的 RAM，从而在保留硬件调试能力的同时，阻止对内核内存的直接读写。

CONFIG_IO_STRICT_DEVMEM 为什么引入？

CONFIG_STRICT_DEVMEM 仍然允许访问所有非 RAM 的 IO 内存区域。但当一个驱动已经通过 request_region() 声明占用某个 IO 资源后，/dev/mem 仍然可以访问该区域，导致：

• 用户空间与驱动竞争访问同一硬件寄存器，可能导致硬件状态混乱
• 驱动的内部状态与硬件实际状态不一致，引发难以调试的系统崩溃

CONFIG_IO_STRICT_DEVMEM 解决了这个问题：当驱动声明占用某资源时，主动撤销已有的 /dev/mem 映射，确保驱动对硬件的独占访问。 11

2.1 page_is_allowed() 包装

// drivers/char/mem.c:59-69#ifdef CONFIG_STRICT_DEVMEMstatic inline int page_is_allowed(unsigned long pfn){return devmem_is_allowed(pfn);}#elsestatic inline int page_is_allowed(unsigned long pfn){return 1;}#endif

未开启 CONFIG_STRICT_DEVMEM 时所有物理页均允许访问；开启后调用架构相关的 devmem_is_allowed(pfn)。

2.2 x86 实现

x86 的注释明确说明了兼容性考量——低 1MB 必须允许映射（但受限区域返回全零），因为 X、dosemu 等工具会映射整个低地址范围：

// arch/x86/mm/init.c:855-895/* * devmem_is_allowed() checks to see if /dev/mem access to a certain address * is valid. The argument is a physical page number. * * On x86, access has to be given to the first megabyte of RAM because that * area traditionally contains BIOS code and data regions used by X, dosemu, * and similar apps. Since they map the entire memory range, the whole range * must be allowed (for mapping), but any areas that would otherwise be * disallowed are flagged as being "zero filled" instead of rejected. * Access has to be given to non-kernel-ram areas as well, these contain the * PCI mmio resources as well as potential bios/acpi data regions. */int devmem_is_allowed(unsigned long pagenr){if (region_intersects(PFN_PHYS(pagenr), PAGE_SIZE,        IORESOURCE_SYSTEM_RAM, IORES_DESC_NONE)      != REGION_DISJOINT) {if (pagenr < 256)return 2;return 0;  }if (iomem_is_exclusive(pagenr << PAGE_SHIFT)) {if (pagenr < 256)return 1;return 0;  }return 1;}

2.3 PowerPC 实现

// arch/powerpc/mm/mem.c:354-371/* * devmem_is_allowed(): check to see if /dev/mem access to a certain address * is valid. The argument is a physical page number. * * Access has to be given to non-kernel-ram areas as well, these contain the * PCI mmio resources as well as potential bios/acpi data regions. */int devmem_is_allowed(unsigned long pfn){if (page_is_rtas_user_buf(pfn))return 1;if (iomem_is_exclusive(PFN_PHYS(pfn)))return 0;if (!page_is_ram(pfn))return 1;return 0;}

允许 RTAS 用户缓冲区，拒绝 exclusive iomem 区域，拒绝所有 RAM 页。

2.4 CONFIG_IO_STRICT_DEVMEM：驱动接管时撤销映射

当驱动声明占用某 IO 资源时，revoke_iomem() 主动撤销已有的 /dev/mem 映射：

// kernel/resource.c:1268-1302#ifdef CONFIG_IO_STRICT_DEVMEMstatic void revoke_iomem(struct resource *res){struct inode *inode = smp_load_acquire(&iomem_inode);if (!inode)return;/*   * The expectation is that the driver has successfully marked   * the resource busy by this point, so devmem_is_allowed()   * should start returning false, however for performance this   * does not iterate the entire resource range.   */if (devmem_is_allowed(PHYS_PFN(res->start)) &&      devmem_is_allowed(PHYS_PFN(res->end))) {/*     * *cringe* iomem=relaxed says "go ahead, what's the     * worst that can happen?"     */return;  }  unmap_mapping_range(inode->i_mapping, res->start, resource_size(res), 1);}#elsestatic void revoke_iomem(struct resource *res) {}#endif

resource_is_exclusive() 判断资源是否被独占（注释说明了 IORESOURCE_SYSTEM_RAM 动态增减时不受控访问的危险性）：

// kernel/resource.c:1878-1929/* * Check if an address is exclusive to the kernel and must not be mapped to * user space, for example, via /dev/mem. * * Returns true if exclusive to the kernel, otherwise returns false. */bool resource_is_exclusive(struct resource *root, u64 addr, resource_size_t size){const unsigned int exclusive_system_ram = IORESOURCE_SYSTEM_RAM |              IORESOURCE_EXCLUSIVE;  .../*   * IORESOURCE_SYSTEM_RAM resources are exclusive if   * IORESOURCE_EXCLUSIVE is set, even if they   * are not busy and even if "iomem=relaxed" is set. The   * responsible driver dynamically adds/removes system RAM within   * such an area and uncontrolled access is dangerous.   */if ((p->flags & exclusive_system_ram) == exclusive_system_ram) {    err = true;break;  }  ...}

三、Device Memory TCP（devmem TCP）

3.0 设计动机

devmem TCP 为什么引入？

随着 GPU、RDMA 网卡、NVMe SSD 等加速器的普及，设备间大量数据传输成为常见场景。传统的网络数据接收路径存在根本性的效率问题：

传统路径（三次拷贝/两次 PCIe 传输）：  NIC → 主机 DRAM（DMA）→ CPU 处理 → PCIe → GPU 显存问题：  1. 主机内存带宽被大量占用（数据在 DRAM 中中转）  2. PCIe 带宽被浪费（数据经过 Root Complex 两次）  3. CPU 参与数据搬运，浪费算力

devmem TCP 的目标是消除这个中间环节：

devmem TCP 路径（零主机内存拷贝）：  NIC header → 主机 DRAM（TCP/IP 栈处理）  NIC payload → GPU 显存（直接 DMA，绕过主机 DRAM）

具体来说，该特性解决了以下问题：

• 分布式 ML 训练：GPU 间通过网络交换梯度数据，不再需要经过主机内存中转
• 分布式块存储：大量数据从网络直接写入 NVMe 设备，不经过主机内存

3.1 概述

// Documentation/networking/devmem.rst:11-52Device memory TCP (devmem TCP) enables receiving data directly into devicememory (dmabuf). The feature is currently implemented for TCP sockets....Devmem TCP optimizes this use case by implementing socket APIs that enablethe user to receive incoming network packets directly into device memory.Packet payloads go directly from the NIC to device memory.Packet headers go to host memory and are processed by the TCP/IP stacknormally. The NIC must support header split to achieve this.Advantages:- Alleviate host memory bandwidth pressure, compared to existing  network-transfer + device-copy semantics.- Alleviate PCIe bandwidth pressure, by limiting data transfer to the lowest  level of the PCIe tree, compared to the traditional path which sends data  through the root complex.

3.2 架构图

传统路径：  NIC → 主机内存 → CPU 处理 → PCIe → 设备内存devmem TCP 路径：  NIC header  → 主机内存（TCP/IP 栈处理）  NIC payload → 设备内存（dmabuf，直接 DMA）

3.3 硬件要求

// Documentation/networking/devmem.rst:79-99Header split, flow steering, & RSS are required features for devmem TCP.Header split is used to split incoming packets into a header buffer in hostmemory, and a payload buffer in device memory.Flow steering & RSS are used to ensure that only flows targeting devmem land onan RX queue bound to devmem.Enable header split & flow steering::# enable header split  ethtool -G eth1 tcp-data-split on# enable flow steering  ethtool -K eth1 ntuple onConfigure RSS to steer all traffic away from the target RX queue (queue 15 inthis example)::  ethtool --set-rxfh-indir eth1 equal 15

3.4 RX 接口

绑定 dmabuf 到 NIC 队列（Netlink API）

// Documentation/networking/devmem.rst:102-122The user must bind a dmabuf to any number of RX queues on a given NIC usingthe netlink API::  /* Bind dmabuf to NIC RX queue 15 */  struct netdev_queue_id *queues;  queues = netdev_queue_id_alloc(1);  netdev_queue_id_set_type(&queues[0], NETDEV_QUEUE_TYPE_RX);  netdev_queue_id_set_id(&queues[0], 15);  *ys = ynl_sock_create(&ynl_netdev_family, &yerr);  req = netdev_bind_rx_req_alloc();  netdev_bind_rx_req_set_ifindex(req, 1 /* ifindex */);  netdev_bind_rx_req_set_fd(req, dmabuf_fd);  __netdev_bind_rx_req_set_queues(req, queues, 1);  rsp = netdev_bind_rx(*ys, req);  dmabuf_id = rsp->id;

Netlink 操作定义（Documentation/netlink/specs/netdev.yaml:805-818）：

name: bind-rxdoc: Bind dmabuf to netdevattribute-set: dmabufflags: [uns-admin-perm]do:request:attributes:- ifindex- fd- queuesreply:attributes:- id

接收数据

// Documentation/networking/devmem.rst:147-196The user application must signal to the kernel that it is capable of receivingdevmem data by passing the MSG_SOCK_DEVMEM flag to recvmsg::  ret = recvmsg(fd, &msg, MSG_SOCK_DEVMEM);Applications that do not specify the MSG_SOCK_DEVMEM flag will receive an EFAULTon devmem data.Devmem data is received directly into the dmabuf bound to the NIC in 'NICSetup', and the kernel signals such to the user via the SCM_DEVMEM_* cmsgs::for (cm = CMSG_FIRSTHDR(&msg); cm; cm = CMSG_NXTHDR(&msg, cm)) {if (cm->cmsg_level != SOL_SOCKET ||        (cm->cmsg_type != SCM_DEVMEM_DMABUF &&         cm->cmsg_type != SCM_DEVMEM_LINEAR))continue;      dmabuf_cmsg = (struct dmabuf_cmsg *)CMSG_DATA(cm);if (cm->cmsg_type == SCM_DEVMEM_DMABUF) {        /* Frag landed in dmabuf.         *         * dmabuf_cmsg->dmabuf_id is the dmabuf the         * frag landed on.         *         * dmabuf_cmsg->frag_offset is the offset into         * the dmabuf where the frag starts.         *         * dmabuf_cmsg->frag_size is the size of the         * frag.         *         * dmabuf_cmsg->frag_token is a token used to         * refer to this frag for later freeing.         */        struct dmabuf_token token;        token.token_start = dmabuf_cmsg->frag_token;        token.token_count = 1;continue;      }if (cm->cmsg_type == SCM_DEVMEM_LINEAR)        /* Frag landed in linear buffer. */continue;    }

Socket 选项定义：

// arch/alpha/include/uapi/asm/socket.h:143-145#define SO_DEVMEM_LINEAR  78#define SCM_DEVMEM_LINEAR  SO_DEVMEM_LINEAR#define SO_DEVMEM_DMABUF  79

释放 frag（归还 token）

// Documentation/networking/devmem.rst:215-234Frags received via SCM_DEVMEM_DMABUF are pinned by the kernel while the userprocesses the frag. The user must return the frag to the kernel viaSO_DEVMEM_DONTNEED::  ret = setsockopt(client_fd, SOL_SOCKET, SO_DEVMEM_DONTNEED, &token,       sizeof(token));The user must ensure the tokens are returned to the kernel in a timely manner.Failure to do so will exhaust the limited dmabuf that is bound to the RX queueand will lead to packet drops.The user must pass no more than 128 tokens, with no more than 1024 total fragsamong the token->token_count across all the tokens.

3.5 TX 接口

绑定 TX dmabuf（Documentation/netlink/specs/netdev.yaml:832-843）：

name: bind-txdoc: Bind dmabuf to netdev for TXattribute-set: dmabufdo:request:attributes:- ifindex- fdreply:attributes:- id

发送数据：

// Documentation/networking/devmem.rst:296-331Devmem data is sent using the SCM_DEVMEM_DMABUF cmsg.The user should create a msghdr where,* iov_base is set to the offset into the dmabuf to start sending from* iov_len is set to the number of bytes to be sent from the dmabuf       iov[0].iov_base = (void*)100;       iov[0].iov_len = 1024;       iov[1].iov_base = (void*)2000;       iov[1].iov_len = 2048;       cmsg->cmsg_level = SOL_SOCKET;       cmsg->cmsg_type = SCM_DEVMEM_DMABUF;       *((__u32 *)CMSG_DATA(cmsg)) = tx_dmabuf_id;       sendmsg(socket_fd, &msg, MSG_ZEROCOPY);

3.6 实现限制

// Documentation/networking/devmem.rst:375-390处理数据包的内核无法访问 devmem 负载。这导致 devmem skb 的负载存在以下几个特殊限制：- 环回功能不可用。环回机制依赖于负载复制，而 devmem skb 不支持此操作。- 软件校验和计算失败。- tcpdump 和 BPF 无法访问 devmem 数据包负载。

四、两种 devmem 对比

Citations

File: drivers/char/Kconfig (L259-266)

config DEVMEM  bool "/dev/mem virtual device support"  default y  help    Say Y here if you want to support the /dev/mem device.    The /dev/mem device is used to access areas of physical    memory.    When in doubt, say "Y".

File: arch/x86/mm/init.c (L855-895)

/* * devmem_is_allowed() checks to see if /dev/mem access to a certain address * is valid. The argument is a physical page number. * * On x86, access has to be given to the first megabyte of RAM because that * area traditionally contains BIOS code and data regions used by X, dosemu, * and similar apps. Since they map the entire memory range, the whole range * must be allowed (for mapping), but any areas that would otherwise be * disallowed are flagged as being "zero filled" instead of rejected. * Access has to be given to non-kernel-ram areas as well, these contain the * PCI mmio resources as well as potential bios/acpi data regions. */int devmem_is_allowed(unsigned long pagenr){if (region_intersects(PFN_PHYS(pagenr), PAGE_SIZE,        IORESOURCE_SYSTEM_RAM, IORES_DESC_NONE)      != REGION_DISJOINT) {/*     * For disallowed memory regions in the low 1MB range,     * request that the page be shown as all zeros.     */if (pagenr < 256)return 2;return 0;  }/*   * This must follow RAM test, since System RAM is considered a   * restricted resource under CONFIG_STRICT_DEVMEM.   */if (iomem_is_exclusive(pagenr << PAGE_SHIFT)) {/* Low 1MB bypasses iomem restrictions. */if (pagenr < 256)return 1;return 0;  }return 1;}

File: drivers/char/mem.c (L34-35)

#define DEVMEM_MINOR  1#define DEVPORT_MINOR  4

File: drivers/char/mem.c (L59-69)

#ifdef CONFIG_STRICT_DEVMEMstatic inline int page_is_allowed(unsigned long pfn){return devmem_is_allowed(pfn);}#elsestatic inline int page_is_allowed(unsigned long pfn){return 1;}#endif

File: drivers/char/mem.c (L82-167)

static ssize_t read_mem(struct file *file, char __user *buf,size_t count, loff_t *ppos){phys_addr_t p = *ppos;ssize_t read, sz;void *ptr;char *bounce;int err;if (p != *ppos)return 0;if (!valid_phys_addr_range(p, count))return -EFAULT;  read = 0;#ifdef __ARCH_HAS_NO_PAGE_ZERO_MAPPED/* we don't have page 0 mapped on sparc and m68k.. */if (p < PAGE_SIZE) {    sz = size_inside_page(p, count);if (sz > 0) {if (clear_user(buf, sz))return -EFAULT;      buf += sz;      p += sz;      count -= sz;      read += sz;    }  }#endif  bounce = kmalloc(PAGE_SIZE, GFP_KERNEL);if (!bounce)return -ENOMEM;while (count > 0) {unsigned long remaining;int allowed, probe;    sz = size_inside_page(p, count);    err = -EPERM;    allowed = page_is_allowed(p >> PAGE_SHIFT);if (!allowed)goto failed;    err = -EFAULT;if (allowed == 2) {/* Show zeros for restricted memory. */      remaining = clear_user(buf, sz);    } else {/*       * On ia64 if a page has been mapped somewhere as       * uncached, then it must also be accessed uncached       * by the kernel or data corruption may occur.       */      ptr = xlate_dev_mem_ptr(p);if (!ptr)goto failed;      probe = copy_from_kernel_nofault(bounce, ptr, sz);      unxlate_dev_mem_ptr(p, ptr);if (probe)goto failed;      remaining = copy_to_user(buf, bounce, sz);    }if (remaining)goto failed;    buf += sz;    p += sz;    count -= sz;    read += sz;if (should_stop_iteration())break;  }  kfree(bounce);  *ppos += read;return read;failed:  kfree(bounce);return err;}

File: drivers/char/mem.c (L169-240)

static ssize_t write_mem(struct file *file, const char __user *buf,size_t count, loff_t *ppos){phys_addr_t p = *ppos;ssize_t written, sz;unsigned long copied;void *ptr;if (p != *ppos)return -EFBIG;if (!valid_phys_addr_range(p, count))return -EFAULT;  written = 0;#ifdef __ARCH_HAS_NO_PAGE_ZERO_MAPPED/* we don't have page 0 mapped on sparc and m68k.. */if (p < PAGE_SIZE) {    sz = size_inside_page(p, count);/* Hmm. Do something? */    buf += sz;    p += sz;    count -= sz;    written += sz;  }#endifwhile (count > 0) {int allowed;    sz = size_inside_page(p, count);    allowed = page_is_allowed(p >> PAGE_SHIFT);if (!allowed)return -EPERM;/* Skip actual writing when a page is marked as restricted. */if (allowed == 1) {/*       * On ia64 if a page has been mapped somewhere as       * uncached, then it must also be accessed uncached       * by the kernel or data corruption may occur.       */      ptr = xlate_dev_mem_ptr(p);if (!ptr) {if (written)break;return -EFAULT;      }      copied = copy_from_user(ptr, buf, sz);      unxlate_dev_mem_ptr(p, ptr);if (copied) {        written += sz - copied;if (written)break;return -EFAULT;      }    }    buf += sz;    p += sz;    count -= sz;    written += sz;if (should_stop_iteration())break;  }  *ppos += written;return written;}

File: drivers/char/mem.c (L325-363)

static int mmap_mem_prepare(struct vm_area_desc *desc){struct file *file = desc->file;const size_t size = vma_desc_size(desc);const phys_addr_t offset = (phys_addr_t)desc->pgoff << PAGE_SHIFT;/* Does it even fit in phys_addr_t? */if (offset >> PAGE_SHIFT != desc->pgoff)return -EINVAL;/* It's illegal to wrap around the end of the physical address space. */if (offset + (phys_addr_t)size - 1 < offset)return -EINVAL;if (!valid_mmap_phys_addr_range(desc->pgoff, size))return -EINVAL;if (!private_mapping_ok(desc))return -ENOSYS;if (!range_is_allowed(desc->pgoff, size))return -EPERM;if (!phys_mem_access_prot_allowed(file, desc->pgoff, size,            &desc->page_prot))return -EINVAL;  desc->page_prot = phys_mem_access_prot(file, desc->pgoff,                 size,                 desc->page_prot);  desc->vm_ops = &mmap_mem_ops;/* Remap-pfn-range will mark the range with the I/O flag. */  mmap_action_remap_full(desc, desc->pgoff);  desc->action.error_override = -EAGAIN;return 0;}

File: drivers/char/mem.c (L603-625)

static int open_port(struct inode *inode, struct file *filp){int rc;if (!capable(CAP_SYS_RAWIO))return -EPERM;  rc = security_locked_down(LOCKDOWN_DEV_MEM);if (rc)return rc;if (iminor(inode) != DEVMEM_MINOR)return 0;/*   * Use a unified address space to have a single point to manage   * revocations when drivers want to take over a /dev/mem mapped   * range.   */  filp->f_mapping = iomem_get_mapping();return 0;}

File: drivers/char/mem.c (L634-645)

static const struct file_operations __maybe_unused mem_fops = {  .llseek    = memory_lseek,  .read    = read_mem,  .write    = write_mem,  .mmap_prepare  = mmap_mem_prepare,  .open    = open_mem,#ifndef CONFIG_MMU  .get_unmapped_area = get_unmapped_area_mem,  .mmap_capabilities = memory_mmap_capabilities,#endif  .fop_flags  = FOP_UNSIGNED_OFFSET,};

File: drivers/char/mem.c (L688-708)

static const struct memdev {const char *name;const struct file_operations *fops;fmode_t fmode;umode_t mode;} devlist[] = {#ifdef CONFIG_DEVMEM  [DEVMEM_MINOR] = { "mem", &mem_fops, 0, 0 },#endif  [3] = { "null", &null_fops, FMODE_NOWAIT, 0666 },#ifdef CONFIG_DEVPORT  [4] = { "port", &port_fops, 0, 0 },#endif  [5] = { "zero", &zero_fops, FMODE_NOWAIT, 0666 },  [7] = { "full", &full_fops, 0, 0666 },  [8] = { "random", &random_fops, FMODE_NOWAIT, 0666 },  [9] = { "urandom", &urandom_fops, FMODE_NOWAIT, 0666 },#ifdef CONFIG_PRINTK  [11] = { "kmsg", &kmsg_fops, 0, 0644 },#endif};

File: drivers/char/mem.c (L749-776)

static int __init chr_dev_init(void){int retval;int minor;if (register_chrdev(MEM_MAJOR, "mem", &memory_fops))    printk("unable to get major %d for memory devs\n", MEM_MAJOR);  retval = class_register(&mem_class);if (retval)return retval;for (minor = 1; minor < ARRAY_SIZE(devlist); minor++) {if (!devlist[minor].name)continue;/*     * Create /dev/port?     */if ((minor == DEVPORT_MINOR) && !arch_has_dev_port())continue;    device_create(&mem_class, NULL, MKDEV(MEM_MAJOR, minor),NULL, devlist[minor].name);  }return tty_init();}

File: kernel/resource.c (L1268-1302)

#ifdef CONFIG_IO_STRICT_DEVMEMstatic void revoke_iomem(struct resource *res){/* pairs with smp_store_release() in iomem_init_inode() */struct inode *inode = smp_load_acquire(&iomem_inode);/*   * Check that the initialization has completed. Losing the race   * is ok because it means drivers are claiming resources before   * the fs_initcall level of init and prevent iomem_get_mapping users   * from establishing mappings.   */if (!inode)return;/*   * The expectation is that the driver has successfully marked   * the resource busy by this point, so devmem_is_allowed()   * should start returning false, however for performance this   * does not iterate the entire resource range.   */if (devmem_is_allowed(PHYS_PFN(res->start)) &&      devmem_is_allowed(PHYS_PFN(res->end))) {/*     * *cringe* iomem=relaxed says "go ahead, what's the     * worst that can happen?"     */return;  }  unmap_mapping_range(inode->i_mapping, res->start, resource_size(res), 1);}#elsestatic void revoke_iomem(struct resource *res) {}#endif

File: kernel/resource.c (L1878-1929)

/* * Check if an address is exclusive to the kernel and must not be mapped to * user space, for example, via /dev/mem. * * Returns true if exclusive to the kernel, otherwise returns false. */bool resource_is_exclusive(struct resource *root, u64 addr, resource_size_t size){const unsigned int exclusive_system_ram = IORESOURCE_SYSTEM_RAM |              IORESOURCE_EXCLUSIVE;bool skip_children = false, err = false;struct resource *p;  read_lock(&resource_lock);  for_each_resource(root, p, skip_children) {if (p->start >= addr + size)break;if (p->end < addr) {      skip_children = true;continue;    }    skip_children = false;/*     * IORESOURCE_SYSTEM_RAM resources are exclusive if     * IORESOURCE_EXCLUSIVE is set, even if they     * are not busy and even if "iomem=relaxed" is set. The     * responsible driver dynamically adds/removes system RAM within     * such an area and uncontrolled access is dangerous.     */if ((p->flags & exclusive_system_ram) == exclusive_system_ram) {      err = true;break;    }/*     * A resource is exclusive if IORESOURCE_EXCLUSIVE is set     * or CONFIG_IO_STRICT_DEVMEM is enabled and the     * resource is busy.     */if (!strict_iomem_checks || !(p->flags & IORESOURCE_BUSY))continue;if (IS_ENABLED(CONFIG_IO_STRICT_DEVMEM)        || p->flags & IORESOURCE_EXCLUSIVE) {      err = true;break;    }  }  read_unlock(&resource_lock);return err;}

File: arch/powerpc/mm/mem.c (L354-371)

#ifdef CONFIG_STRICT_DEVMEM/* * devmem_is_allowed(): check to see if /dev/mem access to a certain address * is valid. The argument is a physical page number. * * Access has to be given to non-kernel-ram areas as well, these contain the * PCI mmio resources as well as potential bios/acpi data regions. */int devmem_is_allowed(unsigned long pfn){if (page_is_rtas_user_buf(pfn))return 1;if (iomem_is_exclusive(PFN_PHYS(pfn)))return 0;if (!page_is_ram(pfn))return 1;return 0;}

File: Documentation/networking/devmem.rst (L11-52)

Device memory TCP (devmem TCP) enables receiving data directly into devicememory (dmabuf). The feature is currently implemented for TCP sockets.Opportunity-----------A large number of data transfers have device memory as the source and/ordestination. Accelerators drastically increased the prevalence of suchtransfers.  Some examples include:- Distributed training, where ML accelerators, such as GPUs on different hosts,  exchange data.- Distributed raw block storage applications transfer large amounts of data with  remote SSDs. Much of this data does not require host processing.Typically the Device-to-Device data transfers in the network are implemented asthe following low-level operations: Device-to-Host copy, Host-to-Host networktransfer, and Host-to-Device copy.The flow involving host copies is suboptimal, especially for bulk data transfers,and can put significant strains on system resources such as host memorybandwidth and PCIe bandwidth.Devmem TCP optimizes this use case by implementing socket APIs that enablethe user to receive incoming network packets directly into device memory.Packet payloads go directly from the NIC to device memory.Packet headers go to host memory and are processed by the TCP/IP stacknormally. The NIC must support header split to achieve this.Advantages:- Alleviate host memory bandwidth pressure, compared to existing  network-transfer + device-copy semantics.- Alleviate PCIe bandwidth pressure, by limiting data transfer to the lowest  level of the PCIe tree, compared to the traditional path which sends data  through the root complex.

File: Documentation/networking/devmem.rst (L79-99)

Header split, flow steering, & RSS are required features for devmem TCP.Header split is used to split incoming packets into a header buffer in hostmemory, and a payload buffer in device memory.Flow steering & RSS are used to ensure that only flows targeting devmem land onan RX queue bound to devmem.Enable header split & flow steering::  # enable header split  ethtool -G eth1 tcp-data-split on  # enable flow steering  ethtool -K eth1 ntuple onConfigure RSS to steer all traffic away from the target RX queue (queue 15 inthis example)::  ethtool --set-rxfh-indir eth1 equal 15

File: Documentation/networking/devmem.rst (L102-122)

The user must bind a dmabuf to any number of RX queues on a given NIC usingthe netlink API::  /* Bind dmabuf to NIC RX queue 15 */  struct netdev_queue_id *queues;  queues = netdev_queue_id_alloc(1);  netdev_queue_id_set_type(&queues[0], NETDEV_QUEUE_TYPE_RX);  netdev_queue_id_set_id(&queues[0], 15);  *ys = ynl_sock_create(&ynl_netdev_family, &yerr);  req = netdev_bind_rx_req_alloc();  netdev_bind_rx_req_set_ifindex(req, 1 /* ifindex */);  netdev_bind_rx_req_set_fd(req, dmabuf_fd);  __netdev_bind_rx_req_set_queues(req, queues, 1);  rsp = netdev_bind_rx(*ys, req);  dmabuf_id = rsp->id;

File: Documentation/networking/devmem.rst (L147-196)

The user application must signal to the kernel that it is capable of receivingdevmem data by passing the MSG_SOCK_DEVMEM flag to recvmsg::  ret = recvmsg(fd, &msg, MSG_SOCK_DEVMEM);Applications that do not specify the MSG_SOCK_DEVMEM flag will receive an EFAULTon devmem data.Devmem data is received directly into the dmabuf bound to the NIC in 'NICSetup', and the kernel signals such to the user via the SCM_DEVMEM_* cmsgs::    for (cm = CMSG_FIRSTHDR(&msg); cm; cm = CMSG_NXTHDR(&msg, cm)) {      if (cm->cmsg_level != SOL_SOCKET ||        (cm->cmsg_type != SCM_DEVMEM_DMABUF &&         cm->cmsg_type != SCM_DEVMEM_LINEAR))        continue;      dmabuf_cmsg = (struct dmabuf_cmsg *)CMSG_DATA(cm);      if (cm->cmsg_type == SCM_DEVMEM_DMABUF) {        /* Frag landed in dmabuf.         *         * dmabuf_cmsg->dmabuf_id is the dmabuf the         * frag landed on.         *         * dmabuf_cmsg->frag_offset is the offset into         * the dmabuf where the frag starts.         *         * dmabuf_cmsg->frag_size is the size of the         * frag.         *         * dmabuf_cmsg->frag_token is a token used to         * refer to this frag for later freeing.         */        struct dmabuf_token token;        token.token_start = dmabuf_cmsg->frag_token;        token.token_count = 1;        continue;      }      if (cm->cmsg_type == SCM_DEVMEM_LINEAR)        /* Frag landed in linear buffer.         *         * dmabuf_cmsg->frag_size is the size of the         * frag.         */        continue;    }

File: Documentation/networking/devmem.rst (L215-234)

Frags received via SCM_DEVMEM_DMABUF are pinned by the kernel while the userprocesses the frag. The user must return the frag to the kernel viaSO_DEVMEM_DONTNEED::  ret = setsockopt(client_fd, SOL_SOCKET, SO_DEVMEM_DONTNEED, &token,       sizeof(token));The user must ensure the tokens are returned to the kernel in a timely manner.Failure to do so will exhaust the limited dmabuf that is bound to the RX queueand will lead to packet drops.The user must pass no more than 128 tokens, with no more than 1024 total fragsamong the token->token_count across all the tokens. If the user provides morethan 1024 frags, the kernel will free up to 1024 frags and return early.The kernel returns the number of actual frags freed. The number of frags freedcan be less than the tokens provided by the user in case of:(a) an internal kernel leak bug.(b) the user passed more than 1024 frags.

File: Documentation/networking/devmem.rst (L296-331)

Devmem data is sent using the SCM_DEVMEM_DMABUF cmsg.The user should create a msghdr where,* iov_base is set to the offset into the dmabuf to start sending from* iov_len is set to the number of bytes to be sent from the dmabufThe user passes the dma-buf id to send from as a u32 cmsg payload.The example below sends 1024 bytes from offset 100 into the dmabuf, and 2048from offset 2000 into the dmabuf. The dmabuf to send from is tx_dmabuf_id::       char ctrl_data[CMSG_SPACE(sizeof(__u32))];       struct msghdr msg = {};       struct cmsghdr *cmsg;       struct iovec iov[2];       iov[0].iov_base = (void*)100;       iov[0].iov_len = 1024;       iov[1].iov_base = (void*)2000;       iov[1].iov_len = 2048;       msg.msg_iov = iov;       msg.msg_iovlen = 2;       msg.msg_control = ctrl_data;       msg.msg_controllen = sizeof(ctrl_data);       cmsg = CMSG_FIRSTHDR(&msg);       cmsg->cmsg_level = SOL_SOCKET;       cmsg->cmsg_type = SCM_DEVMEM_DMABUF;       cmsg->cmsg_len = CMSG_LEN(sizeof(__u32));       *((__u32 *)CMSG_DATA(cmsg)) = tx_dmabuf_id;       sendmsg(socket_fd, &msg, MSG_ZEROCOPY);

File: Documentation/networking/devmem.rst (L375-390)

Implementation & Caveats========================Unreadable skbs---------------Devmem payloads are inaccessible to the kernel processing the packets. Thisresults in a few quirks for payloads of devmem skbs:- Loopback is not functional. Loopback relies on copying the payload, which is  not possible with devmem skbs.- Software checksum calculation fails.- TCP Dump and bpf can't access devmem packet payloads.

File: Documentation/netlink/specs/netdev.yaml (L805-818)

name: bind-rxdoc: Bind dmabuf to netdevattribute-set: dmabufflags: [uns-admin-perm]do:request:attributes:- ifindex- fd- queuesreply:attributes:- id-

File: Documentation/netlink/specs/netdev.yaml (L832-843)

name: bind-txdoc: Bind dmabuf to netdev for TXattribute-set: dmabufdo:request:attributes:- ifindex- fdreply:attributes:- id-

File: arch/alpha/include/uapi/asm/socket.h (L143-145)