huge_mm.h 9.34 KB
Newer Older
1 2 3
#ifndef _LINUX_HUGE_MM_H
#define _LINUX_HUGE_MM_H

4
extern int do_huge_pmd_anonymous_page(struct vm_fault *vmf);
5 6 7
extern int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
			 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
			 struct vm_area_struct *vma);
8
extern void huge_pmd_set_accessed(struct vm_fault *vmf, pmd_t orig_pmd);
9 10 11 12 13 14 15 16 17 18 19 20
extern int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm,
			 pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
			 struct vm_area_struct *vma);

#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
extern void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud);
#else
static inline void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud)
{
}
#endif

21
extern int do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd);
22
extern struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
23 24 25
					  unsigned long addr,
					  pmd_t *pmd,
					  unsigned int flags);
26
extern bool madvise_free_huge_pmd(struct mmu_gather *tlb,
27 28
			struct vm_area_struct *vma,
			pmd_t *pmd, unsigned long addr, unsigned long next);
29 30
extern int zap_huge_pmd(struct mmu_gather *tlb,
			struct vm_area_struct *vma,
31
			pmd_t *pmd, unsigned long addr);
32 33 34
extern int zap_huge_pud(struct mmu_gather *tlb,
			struct vm_area_struct *vma,
			pud_t *pud, unsigned long addr);
35 36 37
extern int mincore_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
			unsigned long addr, unsigned long end,
			unsigned char *vec);
38
extern bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
39
			 unsigned long new_addr, unsigned long old_end,
40
			 pmd_t *old_pmd, pmd_t *new_pmd, bool *need_flush);
41
extern int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
42 43
			unsigned long addr, pgprot_t newprot,
			int prot_numa);
44 45 46 47
int vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
			pmd_t *pmd, pfn_t pfn, bool write);
int vmf_insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr,
			pud_t *pud, pfn_t pfn, bool write);
48 49 50
enum transparent_hugepage_flag {
	TRANSPARENT_HUGEPAGE_FLAG,
	TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
51 52
	TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG,
	TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG,
53
	TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG,
54
	TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
Andrea Arcangeli's avatar
Andrea Arcangeli committed
55
	TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG,
56
	TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG,
57 58 59 60 61
#ifdef CONFIG_DEBUG_VM
	TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG,
#endif
};

62 63 64 65 66 67 68 69 70 71
struct kobject;
struct kobj_attribute;

extern ssize_t single_hugepage_flag_store(struct kobject *kobj,
				 struct kobj_attribute *attr,
				 const char *buf, size_t count,
				 enum transparent_hugepage_flag flag);
extern ssize_t single_hugepage_flag_show(struct kobject *kobj,
				struct kobj_attribute *attr, char *buf,
				enum transparent_hugepage_flag flag);
72 73
extern struct kobj_attribute shmem_enabled_attr;

74 75 76
#define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT)
#define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER)

77
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
78 79 80
#define HPAGE_PMD_SHIFT PMD_SHIFT
#define HPAGE_PMD_SIZE	((1UL) << HPAGE_PMD_SHIFT)
#define HPAGE_PMD_MASK	(~(HPAGE_PMD_SIZE - 1))
81

82 83 84 85
#define HPAGE_PUD_SHIFT PUD_SHIFT
#define HPAGE_PUD_SIZE	((1UL) << HPAGE_PUD_SHIFT)
#define HPAGE_PUD_MASK	(~(HPAGE_PUD_SIZE - 1))

86 87
extern bool is_vma_temporary_stack(struct vm_area_struct *vma);

88
#define transparent_hugepage_enabled(__vma)				\
89 90 91 92 93
	((transparent_hugepage_flags &					\
	  (1<<TRANSPARENT_HUGEPAGE_FLAG) ||				\
	  (transparent_hugepage_flags &					\
	   (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG) &&			\
	   ((__vma)->vm_flags & VM_HUGEPAGE))) &&			\
94 95
	 !((__vma)->vm_flags & VM_NOHUGEPAGE) &&			\
	 !is_vma_temporary_stack(__vma))
96 97 98
#define transparent_hugepage_use_zero_page()				\
	(transparent_hugepage_flags &					\
	 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG))
99 100 101 102 103 104 105 106 107
#ifdef CONFIG_DEBUG_VM
#define transparent_hugepage_debug_cow()				\
	(transparent_hugepage_flags &					\
	 (1<<TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG))
#else /* CONFIG_DEBUG_VM */
#define transparent_hugepage_debug_cow() 0
#endif /* CONFIG_DEBUG_VM */

extern unsigned long transparent_hugepage_flags;
108

109 110 111 112
extern unsigned long thp_get_unmapped_area(struct file *filp,
		unsigned long addr, unsigned long len, unsigned long pgoff,
		unsigned long flags);

113 114 115
extern void prep_transhuge_page(struct page *page);
extern void free_transhuge_page(struct page *page);

116 117 118 119 120
int split_huge_page_to_list(struct page *page, struct list_head *list);
static inline int split_huge_page(struct page *page)
{
	return split_huge_page_to_list(page, NULL);
}
121
void deferred_split_huge_page(struct page *page);
122 123

void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
124
		unsigned long address, bool freeze, struct page *page);
125 126 127 128

#define split_huge_pmd(__vma, __pmd, __address)				\
	do {								\
		pmd_t *____pmd = (__pmd);				\
129 130
		if (pmd_trans_huge(*____pmd)				\
					|| pmd_devmap(*____pmd))	\
131
			__split_huge_pmd(__vma, __pmd, __address,	\
132
						false, NULL);		\
133
	}  while (0)
134

135

136 137
void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
		bool freeze, struct page *page);
138

139 140 141 142 143 144 145 146 147 148 149
void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
		unsigned long address);

#define split_huge_pud(__vma, __pud, __address)				\
	do {								\
		pud_t *____pud = (__pud);				\
		if (pud_trans_huge(*____pud)				\
					|| pud_devmap(*____pud))	\
			__split_huge_pud(__vma, __pud, __address);	\
	}  while (0)

150 151
extern int hugepage_madvise(struct vm_area_struct *vma,
			    unsigned long *vm_flags, int advice);
152
extern void vma_adjust_trans_huge(struct vm_area_struct *vma,
153 154 155
				    unsigned long start,
				    unsigned long end,
				    long adjust_next);
156 157
extern spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd,
		struct vm_area_struct *vma);
158 159
extern spinlock_t *__pud_trans_huge_lock(pud_t *pud,
		struct vm_area_struct *vma);
160
/* mmap_sem must be held on entry */
161 162
static inline spinlock_t *pmd_trans_huge_lock(pmd_t *pmd,
		struct vm_area_struct *vma)
163
{
164
	VM_BUG_ON_VMA(!rwsem_is_locked(&vma->vm_mm->mmap_sem), vma);
165
	if (pmd_trans_huge(*pmd) || pmd_devmap(*pmd))
166
		return __pmd_trans_huge_lock(pmd, vma);
167
	else
168
		return NULL;
169
}
170 171 172 173 174 175 176 177 178
static inline spinlock_t *pud_trans_huge_lock(pud_t *pud,
		struct vm_area_struct *vma)
{
	VM_BUG_ON_VMA(!rwsem_is_locked(&vma->vm_mm->mmap_sem), vma);
	if (pud_trans_huge(*pud) || pud_devmap(*pud))
		return __pud_trans_huge_lock(pud, vma);
	else
		return NULL;
}
179 180 181 182 183 184
static inline int hpage_nr_pages(struct page *page)
{
	if (unlikely(PageTransHuge(page)))
		return HPAGE_PMD_NR;
	return 1;
}
185

186 187 188 189 190
struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
		pmd_t *pmd, int flags);
struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr,
		pud_t *pud, int flags);

191
extern int do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t orig_pmd);
192

193 194 195 196 197 198 199
extern struct page *huge_zero_page;

static inline bool is_huge_zero_page(struct page *page)
{
	return ACCESS_ONCE(huge_zero_page) == page;
}

200 201 202 203 204
static inline bool is_huge_zero_pmd(pmd_t pmd)
{
	return is_huge_zero_page(pmd_page(pmd));
}

205 206 207 208 209
static inline bool is_huge_zero_pud(pud_t pud)
{
	return false;
}

210 211
struct page *mm_get_huge_zero_page(struct mm_struct *mm);
void mm_put_huge_zero_page(struct mm_struct *mm);
212

213 214
#define mk_huge_pmd(page, prot) pmd_mkhuge(mk_pmd(page, prot))

215
#else /* CONFIG_TRANSPARENT_HUGEPAGE */
216 217 218
#define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; })
#define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; })
#define HPAGE_PMD_SIZE ({ BUILD_BUG(); 0; })
219

220 221 222 223
#define HPAGE_PUD_SHIFT ({ BUILD_BUG(); 0; })
#define HPAGE_PUD_MASK ({ BUILD_BUG(); 0; })
#define HPAGE_PUD_SIZE ({ BUILD_BUG(); 0; })

224 225
#define hpage_nr_pages(x) 1

226 227
#define transparent_hugepage_enabled(__vma) 0

228 229
static inline void prep_transhuge_page(struct page *page) {}

230
#define transparent_hugepage_flags 0UL
231 232 233

#define thp_get_unmapped_area	NULL

234 235 236 237 238
static inline int
split_huge_page_to_list(struct page *page, struct list_head *list)
{
	return 0;
}
239 240 241 242
static inline int split_huge_page(struct page *page)
{
	return 0;
}
243
static inline void deferred_split_huge_page(struct page *page) {}
244
#define split_huge_pmd(__vma, __pmd, __address)	\
245
	do { } while (0)
246

247 248
static inline void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
		unsigned long address, bool freeze, struct page *page) {}
249
static inline void split_huge_pmd_address(struct vm_area_struct *vma,
250
		unsigned long address, bool freeze, struct page *page) {}
251

252 253 254
#define split_huge_pud(__vma, __pmd, __address)	\
	do { } while (0)

255 256
static inline int hugepage_madvise(struct vm_area_struct *vma,
				   unsigned long *vm_flags, int advice)
257 258 259 260
{
	BUG();
	return 0;
}
261 262 263 264 265 266
static inline void vma_adjust_trans_huge(struct vm_area_struct *vma,
					 unsigned long start,
					 unsigned long end,
					 long adjust_next)
{
}
267 268
static inline spinlock_t *pmd_trans_huge_lock(pmd_t *pmd,
		struct vm_area_struct *vma)
269
{
270
	return NULL;
271
}
272 273 274 275 276
static inline spinlock_t *pud_trans_huge_lock(pud_t *pud,
		struct vm_area_struct *vma)
{
	return NULL;
}
277

278
static inline int do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t orig_pmd)
279
{
280
	return 0;
281 282
}

283 284 285 286 287
static inline bool is_huge_zero_page(struct page *page)
{
	return false;
}

288 289 290 291 292
static inline bool is_huge_zero_pud(pud_t pud)
{
	return false;
}

293
static inline void mm_put_huge_zero_page(struct mm_struct *mm)
294
{
295
	return;
296
}
297 298 299 300 301 302

static inline struct page *follow_devmap_pmd(struct vm_area_struct *vma,
		unsigned long addr, pmd_t *pmd, int flags)
{
	return NULL;
}
303 304 305 306 307 308

static inline struct page *follow_devmap_pud(struct vm_area_struct *vma,
		unsigned long addr, pud_t *pud, int flags)
{
	return NULL;
}
309 310 311
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

#endif /* _LINUX_HUGE_MM_H */