目录

• 1.写在前面


• 2.devmem使用


• 3.应用层


• 4.内核层

1.写在前面

最近在调试时需要在用户层访问物理内存，发现应用层可以使用devmem工具访问物理地址。查看源码，实际上是对/dev/mem操作，通过mmap可以将物理地址映射到用户空间的虚拟地址上，在用户空间完成对设备寄存器的读写。藉由此原因，想深入理解下mmap的具体实现。

2.devmem使用

devmem的配置，可以在busybox的杂项中找到。

CONFIG_USER_BUSYBOX_DEVMEM:
devmem is a small program that reads and writes from physical
memory using /dev/mem.
Symbol: USER_BUSYBOX_DEVMEM [=y]
Prompt: devmem
Defined at ../user/busybox/busybox-1.23.2/miscutils/Kconfig:216
Depends on: USER_BUSYBOX_BUSYBOX
Location:
-> BusyBox (USER_BUSYBOX_BUSYBOX [=y])
-> Miscellaneous Utilities


# busybox devmem
BusyBox v1.23.2 (2018-08-02 11:08:33 CST) multi-call binary.
Usage: devmem ADDRESS [WIDTH [VALUE]]
Read/write from physical address
ADDRESS Address to act upon
WIDTH Width (8/16/...)
VALUE Data to be written


参数	详细说明
ADDRESS	需要进行读写访问的物理地址
WIDTH	访问数据类型
VALUE	如果是读操作省略；如果是写操作，表示需要写入的数据

基本测试用法

# devmem 0x44e07134 16
0xFFEF
# devmem 0x44e07134 32
0xFFFFFFEF
# devmem 0x44e07134 8
0xEF

3.应用层

接口定义如下：

#include
void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset);
int munmap(void *addr, size_t length);

详细参数如下:

参数	详细说明
addr	需要映射的虚拟内存地址；如果为NULL，系统会自动选定。映射成功后返回该地址
length	需要映射多大的数据量
prot	描述映射区域内存保护方式，包括：PROT_EXEC、PROT_READ、PROT_WRITE、PROT_NONE.
flags	描述映射区域的特性，比如是否对其他进程共享，是否建立匿名映射，是否创建私有的cow.
fd	要映射到内存中的文件描述符
offset	文件映射的偏移量

以devmem的实现为例，

如果argv[3]存在，需要映射读写权限；如果不存在，只需要映射读权限。

map_base = mmap(NULL,
mapped_size,
argv[3] ? (PROT_READ | PROT_WRITE) : PROT_READ,
MAP_SHARED,
fd,
target & ~(off_t)(page_size - 1));

4.内核层

因篇幅有限，这里不在表述glibc、系统调用的关系，直接查找系统调用的代码实现。

arch/arm/include/uapi/asm/unistd.h

#define __NR_OABI_SYSCALL_BASE 0x900000
#if defined(__thumb__) || defined(__ARM_EABI__)
#define __NR_SYSCALL_BASE 0
#else
#define __NR_SYSCALL_BASE __NR_OABI_SYSCALL_BASE
#endif
#define __NR_mmap (__NR_SYSCALL_BASE+ 90)
#define __NR_munmap (__NR_SYSCALL_BASE+ 91)
#define __NR_mmap2 (__NR_SYSCALL_BASE+192)

arch/arm/kernel/entry-common.S

/*=============================================================================
* SWI handler
*-----------------------------------------------------------------------------
*/
.align 5
ENTRY(vector_swi)
#ifdef CONFIG_CPU_V7M
v7m_exception_entry
#else
sub sp, sp, #S_FRAME_SIZE
stmia sp, {r0 - r12} @ Calling r0 - r12
ARM( add r8, sp, #S_PC )
ARM( stmdb r8, {sp, lr}^ ) @ Calling sp, lr
THUMB( mov r8, sp )
THUMB( store_user_sp_lr r8, r10, S_SP ) @ calling sp, lr
mrs r8, spsr @ called from non-FIQ mode, so ok.
str lr, [sp, #S_PC] @ Save calling PC
str r8, [sp, #S_PSR] @ Save CPSR
str r0, [sp, #S_OLD_R0] @ Save OLD_R0
#endif
zero_fp
#ifdef CONFIG_ALIGNMENT_TRAP
ldr ip, __cr_alignment
ldr ip, [ip]
mcr p15, 0, ip, c1, c0 @ update control register
#endif
enable_irq
...


/*
* Note: off_4k (r5) is always units of 4K. If we can't do the requested
* offset, we return EINVAL.
*/
sys_mmap2:
#if PAGE_SHIFT > 12
tst r5, #PGOFF_MASK
moveq r5, r5, lsr #PAGE_SHIFT - 12
streq r5, [sp, #4]
beq sys_mmap_pgoff
mov r0, #-EINVAL
mov pc, lr
#else
str r5, [sp, #4]
b sys_mmap_pgoff
#endif
ENDPROC(sys_mmap2)


arch/arm/kernel/calls.S

/* 90 */ CALL(OBSOLETE(sys_old_mmap)) /* used by libc4 */
CALL(sys_munmap)
...
/* 190 */ CALL(sys_vfork)
CALL(sys_getrlimit)
CALL(sys_mmap2)

include/linux/syscalls.h

asmlinkage long sys_mmap_pgoff(unsigned long addr, unsigned long len,
unsigned long prot, unsigned long flags,
unsigned long fd, unsigned long pgoff);


搜索mmap_pgoff函数定义，位于mm/mmap.c，省略一些我们不太关心的代码。

SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
unsigned long, prot, unsigned long, flags,
unsigned long, fd, unsigned long, pgoff)
{
struct file *file = NULL;
unsigned long retval = -EBADF;
if (!(flags & MAP_ANONYMOUS)) {
audit_mmap_fd(fd, flags);
file = fget(fd);
if (!file)
goto out;
if (is_file_hugepages(file))
len = ALIGN(len, huge_page_size(hstate_file(file)));
retval = -EINVAL;
if (unlikely(flags & MAP_HUGETLB && !is_file_hugepages(file)))
goto out_fput;
}
...

flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);
retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
out_fput:
if (file)
fput(file);
out:
return retval;
}


mm/util.c

unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr,
unsigned long len, unsigned long prot,
unsigned long flag, unsigned long pgoff)
{
unsigned long ret;
struct mm_struct *mm = current->mm;
unsigned long populate;
ret = security_mmap_file(file, prot, flag);
if (!ret) {
down_write(&mm->mmap_sem);
ret = do_mmap_pgoff(file, addr, len, prot, flag, pgoff,
&populate);
up_write(&mm->mmap_sem);
if (populate)
mm_populate(ret, populate);
}
return ret;
}

vm_area_struct结构用来描述进程的虚拟内存区域，和进程的内存描述符mm_struct关联，通过链表和红黑树进行管理。

unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
unsigned long len, unsigned long prot,
unsigned long flags, unsigned long pgoff,
unsigned long *populate)
{

struct mm_struct * mm = current->mm;
vm_flags_t vm_flags;
*populate = 0;

//搜索进程地址空间，查找一个可以使用的线性地址区间，len指定区间的长度，非空addr参数指定从哪个地址开始进行查找
addr = get_unmapped_area(file, addr, len, pgoff, flags);

vm_flags = calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags) |
mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;

//file指针不为空，建立从文件到虚拟空间的映射，根据flags标志设定访问权限。
if (file) {
struct inode *inode = file_inode(file);

switch (flags & MAP_TYPE) {
case MAP_SHARED:
vm_flags |= VM_SHARED | VM_MAYSHARE;
break;
...
} else { //file指针为空，仅创建虚拟空间，不做映射。
switch (flags & MAP_TYPE) {
case MAP_SHARED:
pgoff = 0;
vm_flags |= VM_SHARED | VM_MAYSHARE;
break;
case MAP_PRIVATE:
pgoff = addr >> PAGE_SHIFT;
break;
}

//创建虚拟空间，并进行映射。
addr = mmap_region(file, addr, len, vm_flags, pgoff);

return addr;
}

unsigned long mmap_region(struct file *file, unsigned long addr,
unsigned long len, vm_flags_t vm_flags, unsigned long pgoff)
{
...
//检查是否需要对该虚拟空间进行扩容
if (!may_expand_vm(mm, len >> PAGE_SHIFT)) {
unsigned long nr_pages;
/*
* MAP_FIXED may remove pages of mappings that intersects with
* requested mapping. Account for the pages it would unmap.
*/
if (!(vm_flags & MAP_FIXED))
return -ENOMEM;
nr_pages = count_vma_pages_range(mm, addr, addr + len);
if (!may_expand_vm(mm, (len >> PAGE_SHIFT) - nr_pages))
return -ENOMEM;
}

//扫描当前进程地址空间的vm_area_struct结构相关的红黑树，确定线性区域的位置，如果找到一个区域，说明addr所在的虚拟区间已经被使用，表示已经被映射；因此需要调用do_munmap把这个区域从进程地址空间中撤销。
munmap_back:
if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent)) {
if (do_munmap(mm, addr, len))
return -ENOMEM;
goto munmap_back;
}

vma = vma_merge(mm, prev, addr, addr + len, vm_flags, NULL, file, pgoff, NULL);
if (vma)
goto out;

//分配映射虚拟空间
vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
if (!vma) {
error = -ENOMEM;
goto unacct_error;
}
vma->vm_mm = mm;
vma->vm_start = addr;
vma->vm_end = addr + len;
vma->vm_flags = vm_flags;
vma->vm_page_prot = vm_get_page_prot(vm_flags);
vma->vm_pgoff = pgoff;
INIT_LIST_HEAD(&vma->anon_vma_chain);

if (file) {
if (vm_flags & VM_DENYWRITE) {
error = deny_write_access(file);
if (error)
goto free_vma;
}
vma->vm_file = get_file(file);
error = file->f_op->mmap(file, vma);
if (error)
goto unmap_and_free_vma;
/* Can addr have changed??
*
* Answer: Yes, several device drivers can do it in their
* f_op->mmap method. -DaveM
* Bug: If addr is changed, prev, rb_link, rb_parent should
* be updated for vma_link()
*/
WARN_ON_ONCE(addr != vma->vm_start);
addr = vma->vm_start;
vm_flags = vma->vm_flags;
} else if (vm_flags & VM_SHARED) {
error = shmem_zero_setup(vma);
if (error)
goto free_vma;
}

...
}

mmap_region函数实现中的file->f_op->mmap(file, vma)，对应mmap_mem，位于/drivers/char/mem.c，代码如下：

static const struct file_operations mem_fops = {
.llseek = memory_lseek,
.read = read_mem,
.write = write_mem,
.mmap = mmap_mem,
.open = open_mem,
.get_unmapped_area = get_unmapped_area_mem,
};
static int mmap_mem(struct file *file, struct vm_area_struct *vma)
{
size_t size = vma->vm_end - vma->vm_start;
if (!valid_mmap_phys_addr_range(vma->vm_pgoff, size))
return -EINVAL;
if (!private_mapping_ok(vma))
return -ENOSYS;
if (!range_is_allowed(vma->vm_pgoff, size))
return -EPERM;
if (!phys_mem_access_prot_allowed(file, vma->vm_pgoff, size,
&vma->vm_page_prot))
return -EINVAL;
vma->vm_page_prot = phys_mem_access_prot(file, vma->vm_pgoff,
size,
vma->vm_page_prot);
vma->vm_ops = &mmap_mem_ops;
/* Remap-pfn-range will mark the range VM_IO */
if (remap_pfn_range(vma,
vma->vm_start,
vma->vm_pgoff,
size,
vma->vm_page_prot)) {
return -EAGAIN;
}
return 0;
}


remap_pfn_range函数建立物理地址与虚拟地址页表。其中vm_pgoff代表要映射的物理地址，vm_page_prot代表该页的权限。这些参数和mmap的参数相互对应，现在就可以通过应用层访问物理地址了。