Commit d8fe4acc authored by Linus Torvalds's avatar Linus Torvalds

Merge branch 'akpm' (patch-bomb from Andrew Morton)

Merge patches from Andrew Morton:
 - memstick fixes

 - the rest of MM

 - various misc bits that were awaiting merges from linux-next into
   mainline: seq_file, printk, rtc, completions, w1, softirqs, llist,
   kfifo, hfsplus

* emailed patches from Andrew Morton <akpm@linux-foundation.org>: (72 commits)
  cmdline-parser: fix build
  hfsplus: Fix undefined __divdi3 in hfsplus_init_header_node()
  kfifo API type safety
  kfifo: kfifo_copy_{to,from}_user: fix copied bytes calculation
  sound/core/memalloc.c: use gen_pool_dma_alloc() to allocate iram buffer
  llists-move-llist_reverse_order-from-raid5-to-llistc-fix
  llists: move llist_reverse_order from raid5 to llist.c
  kernel: fix generic_exec_single indentation
  kernel-provide-a-__smp_call_function_single-stub-for-config_smp-fix
  kernel: provide a __smp_call_function_single stub for !CONFIG_SMP
  kernel: remove CONFIG_USE_GENERIC_SMP_HELPERS
  revert "softirq: Add support for triggering softirq work on softirqs"
  drivers/w1/masters/w1-gpio.c: use dev_get_platdata()
  sched: remove INIT_COMPLETION
  tree-wide: use reinit_completion instead of INIT_COMPLETION
  sched: replace INIT_COMPLETION with reinit_completion
  drivers/rtc/rtc-hid-sensor-time.c: enable HID input processing early
  drivers/rtc/rtc-hid-sensor-time.c: use dev_get_platdata()
  vsprintf: ignore %n again
  seq_file: remove "%n" usage from seq_file users
  ...
parents 3aeb58ab 8d3ef556
Split page table lock
=====================
Originally, mm->page_table_lock spinlock protected all page tables of the
mm_struct. But this approach leads to poor page fault scalability of
multi-threaded applications due high contention on the lock. To improve
scalability, split page table lock was introduced.
With split page table lock we have separate per-table lock to serialize
access to the table. At the moment we use split lock for PTE and PMD
tables. Access to higher level tables protected by mm->page_table_lock.
There are helpers to lock/unlock a table and other accessor functions:
- pte_offset_map_lock()
maps pte and takes PTE table lock, returns pointer to the taken
lock;
- pte_unmap_unlock()
unlocks and unmaps PTE table;
- pte_alloc_map_lock()
allocates PTE table if needed and take the lock, returns pointer
to taken lock or NULL if allocation failed;
- pte_lockptr()
returns pointer to PTE table lock;
- pmd_lock()
takes PMD table lock, returns pointer to taken lock;
- pmd_lockptr()
returns pointer to PMD table lock;
Split page table lock for PTE tables is enabled compile-time if
CONFIG_SPLIT_PTLOCK_CPUS (usually 4) is less or equal to NR_CPUS.
If split lock is disabled, all tables guaded by mm->page_table_lock.
Split page table lock for PMD tables is enabled, if it's enabled for PTE
tables and the architecture supports it (see below).
Hugetlb and split page table lock
---------------------------------
Hugetlb can support several page sizes. We use split lock only for PMD
level, but not for PUD.
Hugetlb-specific helpers:
- huge_pte_lock()
takes pmd split lock for PMD_SIZE page, mm->page_table_lock
otherwise;
- huge_pte_lockptr()
returns pointer to table lock;
Support of split page table lock by an architecture
---------------------------------------------------
There's no need in special enabling of PTE split page table lock:
everything required is done by pgtable_page_ctor() and pgtable_page_dtor(),
which must be called on PTE table allocation / freeing.
Make sure the architecture doesn't use slab allocator for page table
allocation: slab uses page->slab_cache and page->first_page for its pages.
These fields share storage with page->ptl.
PMD split lock only makes sense if you have more than two page table
levels.
PMD split lock enabling requires pgtable_pmd_page_ctor() call on PMD table
allocation and pgtable_pmd_page_dtor() on freeing.
Allocation usually happens in pmd_alloc_one(), freeing in pmd_free(), but
make sure you cover all PMD table allocation / freeing paths: i.e X86_PAE
preallocate few PMDs on pgd_alloc().
With everything in place you can set CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK.
NOTE: pgtable_page_ctor() and pgtable_pmd_page_ctor() can fail -- it must
be handled properly.
page->ptl
---------
page->ptl is used to access split page table lock, where 'page' is struct
page of page containing the table. It shares storage with page->private
(and few other fields in union).
To avoid increasing size of struct page and have best performance, we use a
trick:
- if spinlock_t fits into long, we use page->ptr as spinlock, so we
can avoid indirect access and save a cache line.
- if size of spinlock_t is bigger then size of long, we use page->ptl as
pointer to spinlock_t and allocate it dynamically. This allows to use
split lock with enabled DEBUG_SPINLOCK or DEBUG_LOCK_ALLOC, but costs
one more cache line for indirect access;
The spinlock_t allocated in pgtable_page_ctor() for PTE table and in
pgtable_pmd_page_ctor() for PMD table.
Please, never access page->ptl directly -- use appropriate helper.
......@@ -207,9 +207,6 @@ config HAVE_DMA_ATTRS
config HAVE_DMA_CONTIGUOUS
bool
config USE_GENERIC_SMP_HELPERS
bool
config GENERIC_SMP_IDLE_THREAD
bool
......
......@@ -522,7 +522,6 @@ config ARCH_MAY_HAVE_PC_FDC
config SMP
bool "Symmetric multi-processing support"
depends on ALPHA_SABLE || ALPHA_LYNX || ALPHA_RAWHIDE || ALPHA_DP264 || ALPHA_WILDFIRE || ALPHA_TITAN || ALPHA_GENERIC || ALPHA_SHARK || ALPHA_MARVEL
select USE_GENERIC_SMP_HELPERS
---help---
This enables support for systems with more than one CPU. If you have
a system with only one CPU, like most personal computers, say N. If
......
......@@ -72,7 +72,10 @@ pte_alloc_one(struct mm_struct *mm, unsigned long address)
if (!pte)
return NULL;
page = virt_to_page(pte);
pgtable_page_ctor(page);
if (!pgtable_page_ctor(page)) {
__free_page(page);
return NULL;
}
return page;
}
......
......@@ -125,7 +125,6 @@ config ARC_PLAT_NEEDS_CPU_TO_DMA
config SMP
bool "Symmetric Multi-Processing (Incomplete)"
default n
select USE_GENERIC_SMP_HELPERS
help
This enables support for systems with more than one CPU. If you have
a system with only one CPU, like most personal computers, say N. If
......
......@@ -105,11 +105,16 @@ static inline pgtable_t
pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
pgtable_t pte_pg;
struct page *page;
pte_pg = __get_free_pages(GFP_KERNEL | __GFP_REPEAT, __get_order_pte());
if (pte_pg) {
memzero((void *)pte_pg, PTRS_PER_PTE * 4);
pgtable_page_ctor(virt_to_page(pte_pg));
if (!pte_pg)
return 0;
memzero((void *)pte_pg, PTRS_PER_PTE * 4);
page = virt_to_page(pte_pg);
if (!pgtable_page_ctor(page)) {
__free_page(page);
return 0;
}
return pte_pg;
......
......@@ -1432,7 +1432,6 @@ config SMP
depends on GENERIC_CLOCKEVENTS
depends on HAVE_SMP
depends on MMU || ARM_MPU
select USE_GENERIC_SMP_HELPERS
help
This enables support for systems with more than one CPU. If you have
a system with only one CPU, like most personal computers, say N. If
......
......@@ -102,12 +102,14 @@ pte_alloc_one(struct mm_struct *mm, unsigned long addr)
#else
pte = alloc_pages(PGALLOC_GFP, 0);
#endif
if (pte) {
if (!PageHighMem(pte))
clean_pte_table(page_address(pte));
pgtable_page_ctor(pte);
if (!pte)
return NULL;
if (!PageHighMem(pte))
clean_pte_table(page_address(pte));
if (!pgtable_page_ctor(pte)) {
__free_page(pte);
return NULL;
}
return pte;
}
......
......@@ -114,7 +114,7 @@ static int do_dma_transfer(unsigned long apb_add,
dma_desc->callback = apb_dma_complete;
dma_desc->callback_param = NULL;
INIT_COMPLETION(tegra_apb_wait);
reinit_completion(&tegra_apb_wait);
dmaengine_submit(dma_desc);
dma_async_issue_pending(tegra_apb_dma_chan);
......
......@@ -65,7 +65,7 @@ static int do_adjust_pte(struct vm_area_struct *vma, unsigned long address,
return ret;
}
#if USE_SPLIT_PTLOCKS
#if USE_SPLIT_PTE_PTLOCKS
/*
* If we are using split PTE locks, then we need to take the page
* lock here. Otherwise we are using shared mm->page_table_lock
......@@ -84,10 +84,10 @@ static inline void do_pte_unlock(spinlock_t *ptl)
{
spin_unlock(ptl);
}
#else /* !USE_SPLIT_PTLOCKS */
#else /* !USE_SPLIT_PTE_PTLOCKS */
static inline void do_pte_lock(spinlock_t *ptl) {}
static inline void do_pte_unlock(spinlock_t *ptl) {}
#endif /* USE_SPLIT_PTLOCKS */
#endif /* USE_SPLIT_PTE_PTLOCKS */
static int adjust_pte(struct vm_area_struct *vma, unsigned long address,
unsigned long pfn)
......
......@@ -143,7 +143,6 @@ config CPU_BIG_ENDIAN
config SMP
bool "Symmetric Multi-Processing"
select USE_GENERIC_SMP_HELPERS
help
This enables support for systems with more than one CPU. If
you say N here, the kernel will run on single and
......
......@@ -63,9 +63,12 @@ pte_alloc_one(struct mm_struct *mm, unsigned long addr)
struct page *pte;
pte = alloc_pages(PGALLOC_GFP, 0);
if (pte)
pgtable_page_ctor(pte);
if (!pte)
return NULL;
if (!pgtable_page_ctor(pte)) {
__free_page(pte);
return NULL;
}
return pte;
}
......
......@@ -68,7 +68,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
return NULL;
page = virt_to_page(pg);
pgtable_page_ctor(page);
if (!pgtable_page_ctor(page)) {
quicklist_free(QUICK_PT, NULL, pg);
return NULL;
}
return page;
}
......
......@@ -34,7 +34,6 @@ config BLACKFIN
select ARCH_WANT_IPC_PARSE_VERSION
select GENERIC_ATOMIC64
select GENERIC_IRQ_PROBE
select USE_GENERIC_SMP_HELPERS if SMP
select HAVE_NMI_WATCHDOG if NMI_WATCHDOG
select GENERIC_SMP_IDLE_THREAD
select ARCH_USES_GETTIMEOFFSET if !GENERIC_CLOCKEVENTS
......
......@@ -32,7 +32,12 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addres
{
struct page *pte;
pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
pgtable_page_ctor(pte);
if (!pte)
return NULL;
if (!pgtable_page_ctor(pte)) {
__free_page(pte);
return NULL;
}
return pte;
}
......
......@@ -37,11 +37,15 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
#else
page = alloc_pages(GFP_KERNEL|__GFP_REPEAT, 0);
#endif
if (page) {
clear_highpage(page);
pgtable_page_ctor(page);
flush_dcache_page(page);
if (!page)
return NULL;
clear_highpage(page);
if (!pgtable_page_ctor(page)) {
__free_page(page);
return NULL;
}
flush_dcache_page(page);
return page;
}
......
......@@ -4,7 +4,6 @@ comment "Linux Kernel Configuration for Hexagon"
config HEXAGON
def_bool y
select HAVE_OPROFILE
select USE_GENERIC_SMP_HELPERS if SMP
# Other pending projects/to-do items.
# select HAVE_REGS_AND_STACK_ACCESS_API
# select HAVE_HW_BREAKPOINT if PERF_EVENTS
......
......@@ -65,10 +65,12 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm,
struct page *pte;
pte = alloc_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO);
if (pte)
pgtable_page_ctor(pte);
if (!pte)
return NULL;
if (!pgtable_page_ctor(pte)) {
__free_page(pte);
return NULL;
}
return pte;
}
......
......@@ -343,7 +343,6 @@ config FORCE_MAX_ZONEORDER
config SMP
bool "Symmetric multi-processing support"
select USE_GENERIC_SMP_HELPERS
help
This enables support for systems with more than one CPU. If you have
a system with only one CPU, say N. If you have a system with more
......
......@@ -91,7 +91,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addr)
if (!pg)
return NULL;
page = virt_to_page(pg);
pgtable_page_ctor(page);
if (!pgtable_page_ctor(page)) {
quicklist_free(0, NULL, pg);
return NULL;
}
return page;
}
......
......@@ -275,7 +275,6 @@ source "kernel/Kconfig.preempt"
config SMP
bool "Symmetric multi-processing support"
select USE_GENERIC_SMP_HELPERS
---help---
This enables support for systems with more than one CPU. If you have
a system with only one CPU, like most personal computers, say N. If
......
......@@ -43,7 +43,12 @@ static __inline__ pgtable_t pte_alloc_one(struct mm_struct *mm,
{
struct page *pte = alloc_page(GFP_KERNEL|__GFP_ZERO);
pgtable_page_ctor(pte);
if (!pte)
return NULL;
if (!pgtable_page_ctor(pte)) {
__free_page(pte);
return NULL;
}
return pte;
}
......
......@@ -56,6 +56,10 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm,
if (!page)
return NULL;
if (!pgtable_page_ctor(page)) {
__free_page(page);
return NULL;
}
pte = kmap(page);
if (pte) {
......
......@@ -29,18 +29,22 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
struct page *page = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
struct page *page;
pte_t *pte;
page = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
if(!page)
return NULL;
if (!pgtable_page_ctor(page)) {
__free_page(page);
return NULL;
}
pte = kmap(page);
__flush_page_to_ram(pte);
flush_tlb_kernel_page(pte);
nocache_page(pte);
kunmap(page);
pgtable_page_ctor(page);
return page;
}
......
......@@ -59,7 +59,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
return NULL;
clear_highpage(page);
pgtable_page_ctor(page);
if (!pgtable_page_ctor(page)) {
__free_page(page);
return NULL;
}
return page;
}
......
......@@ -111,7 +111,6 @@ config METAG_META21
config SMP
bool "Symmetric multi-processing support"
depends on METAG_META21 && METAG_META21_MMU
select USE_GENERIC_SMP_HELPERS
help
This enables support for systems with more than one thread running
Linux. If you have a system with only one thread running Linux,
......
......@@ -52,8 +52,12 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
{
struct page *pte;
pte = alloc_pages(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO, 0);
if (pte)
pgtable_page_ctor(pte);
if (!pte)
return NULL;
if (!pgtable_page_ctor(pte)) {
__free_page(pte);
return NULL;
}
return pte;
}
......
......@@ -122,8 +122,13 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm,
#endif
ptepage = alloc_pages(flags, 0);
if (ptepage)
clear_highpage(ptepage);
if (!ptepage)
return NULL;
clear_highpage(ptepage);
if (!pgtable_page_ctor(ptepage)) {
__free_page(ptepage);
return NULL;
}
return ptepage;
}
......@@ -158,8 +163,9 @@ extern inline void pte_free_slow(struct page *ptepage)
__free_page(ptepage);
}
extern inline void pte_free(struct mm_struct *mm, struct page *ptepage)
static inline void pte_free(struct mm_struct *mm, struct page *ptepage)
{
pgtable_page_dtor(ptepage);
__free_page(ptepage);
}
......
......@@ -2125,7 +2125,6 @@ source "mm/Kconfig"
config SMP
bool "Multi-Processing support"
depends on SYS_SUPPORTS_SMP
select USE_GENERIC_SMP_HELPERS
help
This enables support for systems with more than one CPU. If you have
a system with only one CPU, like most personal computers, say N. If
......
......@@ -80,9 +80,12 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm,
struct page *pte;
pte = alloc_pages(GFP_KERNEL | __GFP_REPEAT, PTE_ORDER);
if (pte) {
clear_highpage(pte);
pgtable_page_ctor(pte);
if (!pte)
return NULL;
clear_highpage(pte);
if (!pgtable_page_ctor(pte)) {
__free_page(pte);
return NULL;
}
return pte;
}
......
......@@ -181,7 +181,6 @@ endmenu
config SMP
bool "Symmetric multi-processing support"
default y
select USE_GENERIC_SMP_HELPERS
depends on MN10300_PROC_MN2WS0038 || MN10300_PROC_MN2WS0050
---help---
This enables support for systems with more than one CPU. If you have
......
......@@ -46,6 +46,7 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
static inline void pte_free(struct mm_struct *mm, struct page *pte)
{
pgtable_page_dtor(pte);
__free_page(pte);
}
......
......@@ -78,8 +78,13 @@ struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
#else
pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT, 0);
#endif
if (pte)
clear_highpage(pte);
if (!pte)
return NULL;
clear_highpage(pte);
if (!pgtable_page_ctor(pte)) {
__free_page(pte);
return NULL;
}
return pte;
}
......
......@@ -78,8 +78,13 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm,
{
struct page *pte;
pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT, 0);
if (pte)
clear_page(page_address(pte));
if (!pte)
return NULL;
clear_page(page_address(pte));
if (!pgtable_page_ctor(pte)) {
__free_page(pte);
return NULL;
}
return pte;
}
......@@ -90,6 +95,7 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
static inline void pte_free(struct mm_struct *mm, struct page *pte)
{
pgtable_page_dtor(pte);
__free_page(pte);
}
......
......@@ -226,7 +226,6 @@ endchoice
config SMP
bool "Symmetric multi-processing support"
select USE_GENERIC_SMP_HELPERS
---help---
This enables support for systems with more than one CPU. If you have
a system with only one CPU, like most personal computers, say N. If
......
......@@ -121,8 +121,12 @@ static inline pgtable_t
pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
struct page *page = alloc_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO);
if (page)
pgtable_page_ctor(page);
if (!page)
return NULL;
if (!pgtable_page_ctor(page)) {
__free_page(page);
return NULL;
}
return page;
}
......
......@@ -106,7 +106,6 @@ config PPC
select HAVE_MEMBLOCK_NODE_MAP
select HAVE_DMA_ATTRS
select HAVE_DMA_API_DEBUG
select USE_GENERIC_SMP_HELPERS if SMP
select HAVE_OPROFILE
select HAVE_DEBUG_KMEMLEAK
select GENERIC_ATOMIC64 if PPC32
......
......@@ -91,7 +91,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
if (!pte)
return NULL;
page = virt_to_page(pte);
pgtable_page_ctor(page);
if (!pgtable_page_ctor(page)) {
__free_page(page);
return NULL;
}
return page;
}
......
......@@ -121,7 +121,10 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
ptepage = alloc_pages(flags, 0);
if (!ptepage)
return NULL;
pgtable_page_ctor(ptepage);
if (!pgtable_page_ctor(ptepage)) {
__free_page(ptepage);
return NULL;
}
return ptepage;
}
......
......@@ -378,6 +378,10 @@ static pte_t *__alloc_for_cache(struct mm_struct *mm, int kernel)
__GFP_REPEAT | __GFP_ZERO);
if (!page)
return NULL;
if (!kernel && !pgtable_page_ctor(page)) {
__free_page(page);
return NULL;
}
ret = page_address(page);
spin_lock(&mm->page_table_lock);
......@@ -392,9 +396,6 @@ static pte_t *__alloc_for_cache(struct mm_struct *mm, int kernel)
}
spin_unlock(&mm->page_table_lock);
if (!kernel)
pgtable_page_ctor(page);
return (pte_t *)ret;
}
......
......@@ -452,7 +452,7 @@ static int kw_i2c_xfer(struct pmac_i2c_bus *bus, u8 addrdir, int subsize,
*/
if (use_irq) {
/* Clear completion */
INIT_COMPLETION(host->complete);
reinit_completion(&host->complete);
/* Ack stale interrupts */
kw_write_reg(reg_isr, kw_read_reg(reg_isr));
/* Arm timeout */
......@@ -717,7 +717,7 @@ static int pmu_i2c_xfer(struct pmac_i2c_bus *bus, u8 addrdir, int subsize,
return -EINVAL;
}
INIT_COMPLETION(comp);
reinit_completion(&comp);
req->data[0] = PMU_I2C_CMD;
req->reply[0] = 0xff;
req->nbytes = sizeof(struct pmu_i2c_hdr) + 1;
......@@ -748,7 +748,7 @@ static int pmu_i2c_xfer(struct pmac_i2c_bus *bus, u8 addrdir, int subsize,