16页面迁移

    linux为页面迁移提供了一个系统调用migrate_pages&＃xff0c;最早是在linux2.6.16版本加入的&＃xff0c;它可以迁移一个进程的所有页面到指定内存节点上&＃xff0c;该系统调用在用户空间的函数接口如下&＃xff1a;

#include
long migrate_pages(int pid, unsigned long maxnode, const unsigned long *old_nodes, const unsigned long *new_nodes);

    该系统调用最早是为了在NUMA系统上提供一种能迁移进程到任意内存节点的能力。现在内核除了为NUMA系统提供页迁移能力外&＃xff0c;其他的一些模块也可以利用页迁移动能做一些事情&＃xff0c;例如内存规整和内存热插拔等。

migrate_page()函数
    页面迁移(page migration)的核心函数是migrate_pages().
[mm/migrate.c]


/** migrate_pages - migrate the pages specified in a list, to the free pages* supplied as the target for the page migration** &＃64;from: The list of pages to be migrated.* &＃64;get_new_page: The function used to allocate free pages to be used* as the target of the page migration.* &＃64;put_new_page: The function used to free target pages if migration* fails, or NULL if no special handling is necessary.* &＃64;private: Private data to be passed on to get_new_page()* &＃64;mode: The migration mode that specifies the constraints for* page migration, if any.* &＃64;reason: The reason for page migration.** The function returns after 10 attempts or if no pages are movable any more* because the list has become empty or no retryable pages exist any more.* The caller should call putback_lru_pages() to return pages to the LRU* or free list only if ret !&＃61; 0.** Returns the number of pages that were not migrated, or an error code.*/
/*参数说明&＃xff1a;
&＃64;from: 表示将要迁移的页面链表
&＃64;get_new_page: 内存函数指针
&＃64;put_new_page: 迁移失败时释放目标页面的函数指针
&＃64;private: 传递给get_new_page的参数
&＃64;mode: 迁移模式
&＃64;reason: 迁移的原因
*/
int migrate_pages(struct list_head *from, new_page_t get_new_page,free_page_t put_new_page, unsigned long private,enum migrate_mode mode, int reason)
{int retry &＃61; 1;int nr_failed &＃61; 0;int nr_succeeded &＃61; 0;int pass &＃61; 0;struct page *page;struct page *page2;int swapwrite &＃61; current->flags & PF_SWAPWRITE;int rc;if (!swapwrite)current->flags |&＃61; PF_SWAPWRITE;/*for循环表示这里会尝试10次&＃xff0c;从from链表摘取一个页面&＃xff0c;然后调用unmap_add_move()函数进行页面的迁移&＃xff0c;返回MIGRATEPAGE_SUCCESS表示迁移成功。*/for(pass &＃61; 0; pass <10 && retry; pass&＃43;&＃43;) {retry &＃61; 0;list_for_each_entry_safe(page, page2, from, lru) {cond_resched();if (PageHuge(page))rc &＃61; unmap_and_move_huge_page(get_new_page,put_new_page, private, page,pass > 2, mode);else/*下面查看unmap_and_move函数实现*/rc &＃61; unmap_and_move(get_new_page, put_new_page,private, page, pass > 2, mode);switch(rc) {case -ENOMEM:goto out;case -EAGAIN:retry&＃43;&＃43;;break;case MIGRATEPAGE_SUCCESS:nr_succeeded&＃43;&＃43;;break;default:/** Permanent failure (-EBUSY, -ENOSYS, etc.):* unlike -EAGAIN case, the failed page is* removed from migration page list and not* retried in the next outer loop.*/nr_failed&＃43;&＃43;;break;}}}rc &＃61; nr_failed &＃43; retry;
out:if (nr_succeeded)count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded);if (nr_failed)count_vm_events(PGMIGRATE_FAIL, nr_failed);trace_mm_migrate_pages(nr_succeeded, nr_failed, mode, reason);if (!swapwrite)current->flags &&＃61; ~PF_SWAPWRITE;return rc;
}

unmap_and_move()函数实现&＃xff1a;此函数进行页面迁移。

1.分配新的page

2.尝试迁移旧页面到新页面

[migrate_pages()->unmap_and_move()]

/** Obtain the lock on page, remove all ptes and migrate the page* to the newly allocated page in newpage.*/
static int unmap_and_move(new_page_t get_new_page, free_page_t put_new_page,unsigned long private, struct page *page, int force,enum migrate_mode mode)
{int rc &＃61; 0;int *result &＃61; NULL;/*调用get_new_page()分配一个新的页面newpage*/struct page *newpage &＃61; get_new_page(page, private, &result);if (!newpage)return -ENOMEM;if (page_count(page) &＃61;&＃61; 1) {/* page was freed from under us. So we are done. */goto out;}if (unlikely(PageTransHuge(page)))if (unlikely(split_huge_page(page)))goto out;/*调用__unmap_and_move()去尝试迁移页面page到新分配的页面newpage中。下面查看此函数的实现*/rc &＃61; __unmap_and_move(page, newpage, force, mode);out:/*迁移失败&＃xff0c;会把这个页面重新返回LRU链表中。会把新分配的页面释放*/if (rc !&＃61; -EAGAIN) {/** A page that has been migrated has all references* removed and will be freed. A page that has not been* migrated will have kepts its references and be* restored.*/list_del(&page->lru);dec_zone_page_state(page, NR_ISOLATED_ANON &＃43;page_is_file_cache(page));putback_lru_page(page);}/** If migration was not successful and there&＃39;s a freeing callback, use* it. Otherwise, putback_lru_page() will drop the reference grabbed* during isolation.*/if (rc !&＃61; MIGRATEPAGE_SUCCESS && put_new_page) {ClearPageSwapBacked(newpage);put_new_page(newpage, private);} else if (unlikely(__is_movable_balloon_page(newpage))) {/* drop our reference, page already in the balloon */put_page(newpage);} else /*迁移成功的话&＃xff0c;新分配的页面也会加入到LRU链表中*/putback_lru_page(newpage);if (result) {if (rc)*result &＃61; rc;else*result &＃61; page_to_nid(newpage);}return rc;
}


__unmap_and_move()函数实现:

[migrate_pages()->unmap_and_move()->__unmap_and_move()]

注意&＃xff1a;在migrate_pages()中&＃xff0c;当尝试次数大于2时&＃xff0c;会设置force等于1

static int __unmap_and_move(struct page *page, struct page *newpage,int force, enum migrate_mode mode)
{int rc &＃61; -EAGAIN;int page_was_mapped &＃61; 0;struct anon_vma *anon_vma &＃61; NULL;/*trylock_page()尝试给page加锁&＃xff0c;trylock_page()返回false&＃xff0c;表示已经有别的进程给page加锁&＃xff0c;返回true表示当前进程可以成功获取锁*/if (!trylock_page(page)) {/*如果尝试获取页面锁不成功&＃xff0c;当前不是强制迁移(force等于0)或迁移模式等于异步(mode 等于 MIGRATE_ASYNC),会直接忽略这个page&＃xff0c;因为这种情况下没有必要睡眠等待页面释放锁*/if (!force || mode &＃61;&＃61; MIGRATE_ASYNC)goto out;/** It&＃39;s not safe for direct compaction to call lock_page.* For example, during page readahead pages are added locked* to the LRU. Later, when the IO completes the pages are* marked uptodate and unlocked. However, the queueing* could be merging multiple pages for one bio (e.g.* mpage_readpages). If an allocation happens for the* second or third page, the process can end up locking* the same page twice and deadlocking. Rather than* trying to be clever about what pages can be locked,* avoid the use of lock_page for direct compaction* altogether.*//*如果当前进程设置了PF_MEMALLOC标志位&＃xff0c;表示可能是在直接内存压缩(direct compaction)的内核路径上&＃xff0c;睡眠等待页面锁是不安全的&＃xff0c;所以直接忽略page。举个例子&＃xff0c;在文件预读中&＃xff0c;预读的所有页面都会加页面锁(PG_lock)并添加到LRU链表中&＃xff0c;等到预读完成后&＃xff0c;这些页面会标记PG_uptodate并释放页面锁&＃xff0c;这个过程中块设备会把多个页面合并到一个BIO中(mpage_readpages())。如果在分配第二或第三个页面时发生内存短缺&＃xff0c;内核会运行到直接内存压缩(direct compaction)内核路径上&＃xff0c;导致一个页面已经加锁了又等待这个锁&＃xff0c;产生死锁&＃xff0c;因此在直接内存压缩(directcompaction)的内核路径会标记PF_MEMALLOC.PF_MEMALLOC标志位一般是在直接内存压缩&＃xff0c;直接内存回收和kswapd设置&＃xff0c;这些场景下也可能会有少量的内存分配行为&＃xff0c;因此设置PF_MEMALLOC标志位&＃xff0c;表示允许它们使用系统预留的内存&＃xff0c;即不用考虑Water Mark水位。可以参见__perform_reclaim()、__alloc_pages_direct_compact()和kswapd()等函数*/if (current->flags & PF_MEMALLOC)goto out;lock_page(page);}/*处理正在回写的页面即PG_writeback标志位的页面。这里只有当页面迁移的模式为MIGRATE_SYNC且设置强制迁移(force &＃61; 1)时才会去等待这个页面回写完成&＃xff0c;否则直接忽略该页面。wait_on_page_writeback()函数会等待页面回写完成。*/if (PageWriteback(page)) {/** Only in the case of a full synchronous migration is it* necessary to wait for PageWriteback. In the async case,* the retry loop is too short and in the sync-light case,* the overhead of stalling is too much*/if (mode !&＃61; MIGRATE_SYNC) {rc &＃61; -EBUSY;goto out_unlock;}if (!force)goto out_unlock;wait_on_page_writeback(page);}/** By try_to_unmap(), page->mapcount goes down to 0 here. In this case,* we cannot notice that anon_vma is freed while we migrates a page.* This get_anon_vma() delays freeing anon_vma pointer until the end* of migration. File cache pages are no problem because of page_lock()* File Caches may use write_page() or lock_page() in migration, then,* just care Anon page here.*//*处理匿名页面的anon_vma可能被释放的特殊情况&＃xff0c;因为接下来try_to_unmap()函数执行完成时&＃xff0c;page->mapcount引用计数会变成0.在页面迁移过程中&＃xff0c;我们无法知道anon_vma数据结构是否被释放了。page_get_anon_vma()会增加anon_vma->refcount引用计数防止它被其他进程释放&＃xff0c;与之对应的是put_anon_vma()减少anon_vma->refcount引用计数&＃xff0c;它们是成对出现的。*/if (PageAnon(page) && !PageKsm(page)) {/** Only page_lock_anon_vma_read() understands the subtleties of* getting a hold on an anon_vma from outside one of its mms.*/anon_vma &＃61; page_get_anon_vma(page);if (anon_vma) {/** Anon page*/} else if (PageSwapCache(page)) {/** We cannot be sure that the anon_vma of an unmapped* swapcache page is safe to use because we don&＃39;t* know in advance if the VMA that this page belonged* to still exists. If the VMA and others sharing the* data have been freed, then the anon_vma could* already be invalid.** To avoid this possibility, swapcache pages get* migrated but are not remapped when migration* completes*/} else {goto out_unlock;}}if (unlikely(isolated_balloon_page(page))) {/** A ballooned page does not need any special attention from* physical to virtual reverse mapping procedures.* Skip any attempt to unmap PTEs or to remap swap cache,* in order to avoid burning cycles at rmap level, and perform* the page migration right away (proteced by page lock).*/rc &＃61; balloon_page_migrate(newpage, page, mode);goto out_unlock;}/** Corner case handling:* 1. When a new swap-cache page is read into, it is added to the LRU* and treated as swapcache but it has no rmap yet.* Calling try_to_unmap() against a page->mapping&＃61;&＃61;NULL page will* trigger a BUG. So handle it here.* 2. An orphaned page (see truncate_complete_page) might have* fs-private metadata. The page can be picked up due to memory* offlining. Everywhere else except page reclaim, the page is* invisible to the vm, so the page can not be migrated. So try to* free the metadata, so the page can be freed.*//*这里处理一种特殊情况&＃xff0c;例如一个swap cache页面发生swap-in时&＃xff0c;在do_swap_page()中会分配一个新的页面&＃xff0c;该页面添加到LRU链表中&＃xff0c;这个页面是swapcache页面&＃xff0c;但是它 还没有建立RAMP关系&＃xff0c;因此page->mapping等于NULL&＃xff0c;接下来要进行的try_to_unmap()函数处理这种页面会触发bug。*/if (!page->mapping) {VM_BUG_ON_PAGE(PageAnon(page), page);if (page_has_private(page)) {try_to_free_buffers(page);goto out_unlock;}goto skip_unmap;}/* Establish migration ptes or remove ptes *//*对于有pte映射的页面&＃xff0c;调用try_to_unmap()解除页面所有映射的pte。try_to_unmap()定义在mm/rmap.c*/if (page_mapped(page)) {try_to_unmap(page,TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);page_was_mapped &＃61; 1;}skip_unmap:/*对于已经解除完所有映射的页面&＃xff0c;调用move_to_new_page()迁移到新的页面new_page&＃xff0c; 下面查看此函数实现*/if (!page_mapped(page))rc &＃61; move_to_new_page(newpage, page, page_was_mapped, mode);/*对于迁移页面失败&＃xff0c;调用remove_migration_ptes()把迁移失败页面的pte设置回原来的位置*/if (rc && page_was_mapped)remove_migration_ptes(page, page);/* Drop an anon_vma reference if we took one */if (anon_vma)put_anon_vma(anon_vma);out_unlock:unlock_page(page);
out:return rc;
}

move_to_new_page()函数实现&＃xff1a;

[migrate_pages()->unmap_and_move()->__unmap_and_move()->move_to_new_page()]

/** Move a page to a newly allocated page* The page is locked and all ptes have been successfully removed.** The new page will have replaced the old page if this function* is successful.** Return value:* <0 - error code* MIGRATEPAGE_SUCCESS - success*/
static int move_to_new_page(struct page *newpage, struct page *page,int page_was_mapped, enum migrate_mode mode)
{struct address_space *mapping;int rc;/** Block others from accessing the page when we get around to* establishing additional references. We are the only one* holding a reference to the new page at this point.*//*如果newpage已经被其他进程加锁&＃xff0c;那么会是一个bug&＃xff0c;调用BUG函数来处理*/if (!trylock_page(newpage))BUG();/* Prepare mapping for the new page.*//*设置newpage的index和mapping和PG_SwapBacked标志位*/newpage->index &＃61; page->index;newpage->mapping &＃61; page->mapping;if (PageSwapBacked(page))SetPageSwapBacked(newpage);/*处理页面mapping情况&＃xff0c;page_mapping()函数获取page->mapping指针&＃xff0c;查看下面对其解释*/mapping &＃61; page_mapping(page);/*下面以匿名页面为例&＃xff0c;调用migrate_page()将旧页面的相关信息迁移到新页面。对于其他有mapping的页面&＃xff0c;会调用mapping指向的migratepage()函数指针或fallback_migrate_page()函数&＃xff0c;很多文件系统都提供这样的函数接口。下面查看migrate_page()函数*/if (!mapping)rc &＃61; migrate_page(mapping, newpage, page, mode);else if (mapping->a_ops->migratepage)/** Most pages have a mapping and most filesystems provide a* migratepage callback. Anonymous pages are part of swap* space which also has its own migratepage callback. This* is the most common path for page migration.*/rc &＃61; mapping->a_ops->migratepage(mapping,newpage, page, mode);elserc &＃61; fallback_migrate_page(mapping, newpage, page, mode);if (rc !&＃61; MIGRATEPAGE_SUCCESS) {newpage->mapping &＃61; NULL;} else {mem_cgroup_migrate(page, newpage, false);if (page_was_mapped)remove_migration_ptes(page, newpage); /*迁移页面的每一个pte&＃xff0c;下面查看此函数的实现*/page->mapping &＃61; NULL;}unlock_page(newpage);return rc;
}

page_mapping()函数:获取page->mapping指针

[mm/util.c]

struct address_space *page_mapping(struct page *page)
{struct address_space *mapping &＃61; page->mapping;/* This happens if someone calls flush_dcache_page on slab page *//*如果page属于slab或是匿名页面&＃xff0c;该函数返回mapping为空*/if (unlikely(PageSlab(page)))return NULL;/*如果是PageSwapCache()&＃xff0c;则返回swap_address_space空间&＃xff0c;其余为page cache的情况&＃xff0c;直接返回page->mapping*/if (unlikely(PageSwapCache(page))) {swp_entry_t entry;entry.val &＃61; page_private(page);mapping &＃61; swap_address_space(entry);} else if ((unsigned long)mapping & PAGE_MAPPING_ANON)mapping &＃61; NULL;return mapping;
}
回到move_to_new_page()函数

migrate_page()函数实现&＃xff1a;

[migrate_pages()->unmap_and_move()->__unmap_and_move()->move_to_new_page()->migrate_page()]

/** Common logic to directly migrate a single page suitable for* pages that do not use PagePrivate/PagePrivate2.** Pages are locked upon entry and exit.*/
int migrate_page(struct address_space *mapping,struct page *newpage, struct page *page,enum migrate_mode mode)
{int rc;BUG_ON(PageWriteback(page)); /* Writeback must be complete *//*对于匿名页面来说&＃xff0c;此函数没有做任何事情*/rc &＃61; migrate_page_move_mapping(mapping, newpage, page, NULL, mode, 0);if (rc !&＃61; MIGRATEPAGE_SUCCESS)return rc;/*将旧页面的内容复制到新页面中&＃xff0c;下面查看此函数的实现*/migrate_page_copy(newpage, page);return MIGRATEPAGE_SUCCESS;
}回到move_to_new_page

migrate_page_copy()函数实现

[migrate_pages()->unmap_and_move()->__unmap_and_move()->move_to_new_page()->migrate_page()->migrate_page_copy()]

/** Copy the page to its new location*/
void migrate_page_copy(struct page *newpage, struct page *page)
{int cpupid;if (PageHuge(page) || PageTransHuge(page))copy_huge_page(newpage, page);elsecopy_highpage(newpage, page); /*复制旧页面的内容到新页面中&＃xff0c;使用kmap_atomic()函数来映射页面以便读取页面的内容。*//*下面依照旧页面中flags的比特位来设置newpage相应的标志位&＃xff0c;例如PG_error、PG_referenced、PG_uptodate、PG_active、PG_unevictable、PG_checked和PG_mappedtodisk等*/if (PageError(page))SetPageError(newpage);if (PageReferenced(page))SetPageReferenced(newpage);if (PageUptodate(page))SetPageUptodate(newpage);if (TestClearPageActive(page)) {VM_BUG_ON_PAGE(PageUnevictable(page), page);SetPageActive(newpage);} else if (TestClearPageUnevictable(page))SetPageUnevictable(newpage);if (PageChecked(page))SetPageChecked(newpage);if (PageMappedToDisk(page))SetPageMappedToDisk(newpage);/*处理旧页面是dirty的情况。如果旧页面是匿名页面(PageSwapBacked(page)),则设置PG_dirty位&＃xff1b;如果旧页面是page cache&＃xff0c;则由__set_page_dirty_nobuffers()设置radix_tree中dirty标志位。*/if (PageDirty(page)) {clear_page_dirty_for_io(page);/** Want to mark the page and the radix tree as dirty, and* redo the accounting that clear_page_dirty_for_io undid,* but we can&＃39;t use set_page_dirty because that function* is actually a signal that all of the page has become dirty.* Whereas only part of our page may be dirty.*/if (PageSwapBacked(page))SetPageDirty(newpage);else__set_page_dirty_nobuffers(newpage);}/** Copy NUMA information to the new page, to prevent over-eager* future migrations of this same page.*/cpupid &＃61; page_cpupid_xchg_last(page, -1);page_cpupid_xchg_last(newpage, cpupid);mlock_migrate_page(newpage, page);/*处理旧页面是KSM页面的情况*/ksm_migrate_page(newpage, page);/** Please do not reorder this without considering how mm/ksm.c&＃39;s* get_ksm_page() depends upon ksm_migrate_page() and PageSwapCache().*/ClearPageSwapCache(page);ClearPagePrivate(page);set_page_private(page, 0);/** If any waiters have accumulated on the new page then* wake them up.*/if (PageWriteback(newpage))end_page_writeback(newpage);
}回到migrate_page()函数

remove_migration_ptes()函数实现

[migrate_pages()->unmap_and_move()->__unmap_and_move()->move_to_new_page()->remove_migration_ptes()]

/** Get rid of all migration entries and replace them by* references to the indicated page.*/
static void remove_migration_ptes(struct page *old, struct page *new)
{struct rmap_walk_control rwc &＃61; {.rmap_one &＃61; remove_migration_pte,.arg &＃61; old,};rmap_walk(new, &rwc);
}

remove_migration_ptes()是典型地利用RMAP反向映射系统找到映射旧页面的每个PTE&＃xff0c;直接来看它的rmap_one函数指针。

/** Restore a potential migration pte to a working pte entry*/
/*remove_migration_pte函数找到其中一个映射的虚拟地址&＃xff0c;根据参数中的vma和addr。*/
static int remove_migration_pte(struct page *new, struct vm_area_struct *vma,unsigned long addr, void *old)
{struct mm_struct *mm &＃61; vma->vm_mm;swp_entry_t entry;pmd_t *pmd;pte_t *ptep, pte;spinlock_t *ptl;if (unlikely(PageHuge(new))) {ptep &＃61; huge_pte_offset(mm, addr);if (!ptep)goto out;ptl &＃61; huge_pte_lockptr(hstate_vma(vma), mm, ptep);} else {/*通过mm和虚拟地址addr找到相应的页表项pte。*/pmd &＃61; mm_find_pmd(mm, addr);if (!pmd)goto out;ptep &＃61; pte_offset_map(pmd, addr);/** Peek to check is_swap_pte() before taking ptlock? No, we* can race mremap&＃39;s move_ptes(), which skips anon_vma lock.*/ptl &＃61; pte_lockptr(mm, pmd);}/*每个进程的mm数据结构中有一个保护页表的spinlock锁(mm->page_table_lock)*/spin_lock(ptl);/*把映射的pte页表项的内容设置到新页面的pte中&＃xff0c;相当于重新建立映射关系。*/pte &＃61; *ptep;if (!is_swap_pte(pte))goto unlock;entry &＃61; pte_to_swp_entry(pte);if (!is_migration_entry(entry) ||migration_entry_to_page(entry) !&＃61; old)goto unlock;get_page(new);pte &＃61; pte_mkold(mk_pte(new, vma->vm_page_prot));if (pte_swp_soft_dirty(*ptep))pte &＃61; pte_mksoft_dirty(pte);/* Recheck VMA as permissions can change since migration started */if (is_write_migration_entry(entry))pte &＃61; maybe_mkwrite(pte, vma);#ifdef CONFIG_HUGETLB_PAGEif (PageHuge(new)) {pte &＃61; pte_mkhuge(pte);pte &＃61; arch_make_huge_pte(pte, vma, new, 0);}
#endifflush_dcache_page(new);set_pte_at(mm, addr, ptep, pte);if (PageHuge(new)) {if (PageAnon(new))hugepage_add_anon_rmap(new, vma, addr);elsepage_dup_rmap(new);/*将新的页面newpage添加到RMAP反向映射系统中。*/} else if (PageAnon(new))page_add_anon_rmap(new, vma, addr);elsepage_add_file_rmap(new);/* No need to invalidate - it was non-present before *//*调用update_mmu_cache()更新相应的cache。增加一个新的PTE&＃xff0c;或者修改PTE时需要调用该函数对cache进行管理&＃xff0c;对于ARMv6以上的CPU来说&＃xff0c;该函数是空函数&＃xff0c;cache一致性管理在set_pte_at()函数中完成。*/update_mmu_cache(vma, addr, ptep);
unlock:pte_unmap_unlock(ptep, ptl);
out:return SWAP_AGAIN;
}


内核中有多处使用到页的迁移的功能&＃xff0c;列出如下。

• 内存规整(memory compaction)

• 内存热插拔(memory hotplug)

• NUMA系统&＃xff0c;系统有一个sys_migrate_pages的系统调用。