Commit cf393195 authored by Linus Torvalds's avatar Linus Torvalds

Merge branch 'idr-4.11' of git://git.infradead.org/users/willy/linux-dax

Pull IDR rewrite from Matthew Wilcox:
 "The most significant part of the following is the patch to rewrite the
  IDR & IDA to be clients of the radix tree. But there's much more,
  including an enhancement of the IDA to be significantly more space
  efficient, an IDR & IDA test suite, some improvements to the IDR API
  (and driver changes to take advantage of those improvements), several
  improvements to the radix tree test suite and RCU annotations.

  The IDR & IDA rewrite had a good spin in linux-next and Andrew's tree
  for most of the last cycle. Coupled with the IDR test suite, I feel
  pretty confident that any remaining bugs are quite hard to hit. 0-day
  did a great job of watching my git tree and pointing out problems; as
  it hit them, I added new test-cases to be sure not to be caught the
  same way twice"

Willy goes on to expand a bit on the IDR rewrite rationale:
 "The radix tree and the IDR use very similar data structures.

  Merging the two codebases lets us share the memory allocation pools,
  and results in a net deletion of 500 lines of code. It also opens up
  the possibility of exposing more of the features of the radix tree to
  users of the IDR (and I have some interesting patches along those
  lines waiting for 4.12)

  It also shrinks the size of the 'struct idr' from 40 bytes to 24 which
  will shrink a fair few data structures that embed an IDR"

* 'idr-4.11' of git://git.infradead.org/users/willy/linux-dax: (32 commits)
  radix tree test suite: Add config option for map shift
  idr: Add missing __rcu annotations
  radix-tree: Fix __rcu annotations
  radix-tree: Add rcu_dereference and rcu_assign_pointer calls
  radix tree test suite: Run iteration tests for longer
  radix tree test suite: Fix split/join memory leaks
  radix tree test suite: Fix leaks in regression2.c
  radix tree test suite: Fix leaky tests
  radix tree test suite: Enable address sanitizer
  radix_tree_iter_resume: Fix out of bounds error
  radix-tree: Store a pointer to the root in each node
  radix-tree: Chain preallocated nodes through ->parent
  radix tree test suite: Dial down verbosity with -v
  radix tree test suite: Introduce kmalloc_verbose
  idr: Return the deleted entry from idr_remove
  radix tree test suite: Build separate binaries for some tests
  ida: Use exceptional entries for small IDAs
  ida: Move ida_bitmap to a percpu variable
  Reimplement IDR and IDA using the radix tree
  radix-tree: Add radix_tree_iter_delete
  ...
parents 5ecc5ac2 c6ce3e2f
......@@ -1980,13 +1980,12 @@ static void dequeue_rx(ns_dev * card, ns_rsqe * rsqe)
card->lbfqc = ns_stat_lfbqc_get(stat);
id = le32_to_cpu(rsqe->buffer_handle);
skb = idr_find(&card->idr, id);
skb = idr_remove(&card->idr, id);
if (!skb) {
RXPRINTK(KERN_ERR
"nicstar%d: idr_find() failed!\n", card->index);
"nicstar%d: skb not found!\n", card->index);
return;
}
idr_remove(&card->idr, id);
dma_sync_single_for_cpu(&card->pcidev->dev,
NS_PRV_DMA(skb),
(NS_PRV_BUFTYPE(skb) == BUF_SM
......
......@@ -2915,11 +2915,9 @@ enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsig
idr_remove(&connection->peer_devices, vnr);
out_idr_remove_from_resource:
for_each_connection(connection, resource) {
peer_device = idr_find(&connection->peer_devices, vnr);
if (peer_device) {
idr_remove(&connection->peer_devices, vnr);
peer_device = idr_remove(&connection->peer_devices, vnr);
if (peer_device)
kref_put(&connection->kref, drbd_destroy_connection);
}
}
for_each_peer_device_safe(peer_device, tmp_peer_device, device) {
list_del(&peer_device->peer_devices);
......
......@@ -1307,8 +1307,7 @@ static void iso_resource_work(struct work_struct *work)
*/
if (r->todo == ISO_RES_REALLOC && !success &&
!client->in_shutdown &&
idr_find(&client->resource_idr, r->resource.handle)) {
idr_remove(&client->resource_idr, r->resource.handle);
idr_remove(&client->resource_idr, r->resource.handle)) {
client_put(client);
free = true;
}
......
......@@ -70,10 +70,10 @@ static void amdgpu_bo_list_destroy(struct amdgpu_fpriv *fpriv, int id)
struct amdgpu_bo_list *list;
mutex_lock(&fpriv->bo_list_lock);
list = idr_find(&fpriv->bo_list_handles, id);
list = idr_remove(&fpriv->bo_list_handles, id);
if (list) {
/* Another user may have a reference to this list still */
mutex_lock(&list->lock);
idr_remove(&fpriv->bo_list_handles, id);
mutex_unlock(&list->lock);
amdgpu_bo_list_free(list);
}
......
......@@ -135,15 +135,11 @@ static int amdgpu_ctx_free(struct amdgpu_fpriv *fpriv, uint32_t id)
struct amdgpu_ctx *ctx;
mutex_lock(&mgr->lock);
ctx = idr_find(&mgr->ctx_handles, id);
if (ctx) {
idr_remove(&mgr->ctx_handles, id);
ctx = idr_remove(&mgr->ctx_handles, id);
if (ctx)
kref_put(&ctx->refcount, amdgpu_ctx_do_release);
mutex_unlock(&mgr->lock);
return 0;
}
mutex_unlock(&mgr->lock);
return -EINVAL;
return ctx ? 0 : -EINVAL;
}
static int amdgpu_ctx_query(struct amdgpu_device *adev,
......
......@@ -346,9 +346,7 @@ void mwifiex_parse_tx_status_event(struct mwifiex_private *priv,
return;
spin_lock_irqsave(&priv->ack_status_lock, flags);
ack_skb = idr_find(&priv->ack_status_frames, tx_status->tx_token_id);
if (ack_skb)
idr_remove(&priv->ack_status_frames, tx_status->tx_token_id);
ack_skb = idr_remove(&priv->ack_status_frames, tx_status->tx_token_id);
spin_unlock_irqrestore(&priv->ack_status_lock, flags);
if (ack_skb) {
......
......@@ -642,9 +642,7 @@ static unsigned int tcmu_handle_completions(struct tcmu_dev *udev)
WARN_ON(tcmu_hdr_get_op(entry->hdr.len_op) != TCMU_OP_CMD);
spin_lock(&udev->commands_lock);
cmd = idr_find(&udev->commands, entry->hdr.cmd_id);
if (cmd)
idr_remove(&udev->commands, cmd->cmd_id);
cmd = idr_remove(&udev->commands, entry->hdr.cmd_id);
spin_unlock(&udev->commands_lock);
if (!cmd) {
......
......@@ -12,47 +12,29 @@
#ifndef __IDR_H__
#define __IDR_H__
#include <linux/types.h>
#include <linux/bitops.h>
#include <linux/init.h>
#include <linux/rcupdate.h>
#include <linux/radix-tree.h>
#include <linux/gfp.h>
#include <linux/percpu.h>
struct idr {
struct radix_tree_root idr_rt;
unsigned int idr_next;
};
/*
* Using 6 bits at each layer allows us to allocate 7 layers out of each page.
* 8 bits only gave us 3 layers out of every pair of pages, which is less
* efficient except for trees with a largest element between 192-255 inclusive.
* The IDR API does not expose the tagging functionality of the radix tree
* to users. Use tag 0 to track whether a node has free space below it.
*/
#define IDR_BITS 6
#define IDR_SIZE (1 << IDR_BITS)
#define IDR_MASK ((1 << IDR_BITS)-1)
struct idr_layer {
int prefix; /* the ID prefix of this idr_layer */
int layer; /* distance from leaf */
struct idr_layer __rcu *ary[1<<IDR_BITS];
int count; /* When zero, we can release it */
union {
/* A zero bit means "space here" */
DECLARE_BITMAP(bitmap, IDR_SIZE);
struct rcu_head rcu_head;
};
};
#define IDR_FREE 0
struct idr {
struct idr_layer __rcu *hint; /* the last layer allocated from */
struct idr_layer __rcu *top;
int layers; /* only valid w/o concurrent changes */
int cur; /* current pos for cyclic allocation */
spinlock_t lock;
int id_free_cnt;
struct idr_layer *id_free;
};
/* Set the IDR flag and the IDR_FREE tag */
#define IDR_RT_MARKER ((__force gfp_t)(3 << __GFP_BITS_SHIFT))
#define IDR_INIT(name) \
#define IDR_INIT \
{ \
.lock = __SPIN_LOCK_UNLOCKED(name.lock), \
.idr_rt = RADIX_TREE_INIT(IDR_RT_MARKER) \
}
#define DEFINE_IDR(name) struct idr name = IDR_INIT(name)
#define DEFINE_IDR(name) struct idr name = IDR_INIT
/**
* idr_get_cursor - Return the current position of the cyclic allocator
......@@ -62,9 +44,9 @@ struct idr {
* idr_alloc_cyclic() if it is free (otherwise the search will start from
* this position).
*/
static inline unsigned int idr_get_cursor(struct idr *idr)
static inline unsigned int idr_get_cursor(const struct idr *idr)
{
return READ_ONCE(idr->cur);
return READ_ONCE(idr->idr_next);
}
/**
......@@ -77,7 +59,7 @@ static inline unsigned int idr_get_cursor(struct idr *idr)
*/
static inline void idr_set_cursor(struct idr *idr, unsigned int val)
{
WRITE_ONCE(idr->cur, val);
WRITE_ONCE(idr->idr_next, val);
}
/**
......@@ -97,22 +79,31 @@ static inline void idr_set_cursor(struct idr *idr, unsigned int val)
* period).
*/
/*
* This is what we export.
*/
void *idr_find_slowpath(struct idr *idp, int id);
void idr_preload(gfp_t gfp_mask);
int idr_alloc(struct idr *idp, void *ptr, int start, int end, gfp_t gfp_mask);
int idr_alloc_cyclic(struct idr *idr, void *ptr, int start, int end, gfp_t gfp_mask);
int idr_for_each(struct idr *idp,
int idr_alloc(struct idr *, void *entry, int start, int end, gfp_t);
int idr_alloc_cyclic(struct idr *, void *entry, int start, int end, gfp_t);
int idr_for_each(const struct idr *,
int (*fn)(int id, void *p, void *data), void *data);
void *idr_get_next(struct idr *idp, int *nextid);
void *idr_replace(struct idr *idp, void *ptr, int id);
void idr_remove(struct idr *idp, int id);
void idr_destroy(struct idr *idp);
void idr_init(struct idr *idp);
bool idr_is_empty(struct idr *idp);
void *idr_get_next(struct idr *, int *nextid);
void *idr_replace(struct idr *, void *, int id);
void idr_destroy(struct idr *);
static inline void *idr_remove(struct idr *idr, int id)
{
return radix_tree_delete_item(&idr->idr_rt, id, NULL);
}
static inline void idr_init(struct idr *idr)
{
INIT_RADIX_TREE(&idr->idr_rt, IDR_RT_MARKER);
idr->idr_next = 0;
}
static inline bool idr_is_empty(const struct idr *idr)
{
return radix_tree_empty(&idr->idr_rt) &&
radix_tree_tagged(&idr->idr_rt, IDR_FREE);
}
/**
* idr_preload_end - end preload section started with idr_preload()
......@@ -137,19 +128,14 @@ static inline void idr_preload_end(void)
* This function can be called under rcu_read_lock(), given that the leaf
* pointers lifetimes are correctly managed.
*/
static inline void *idr_find(struct idr *idr, int id)
static inline void *idr_find(const struct idr *idr, int id)
{
struct idr_layer *hint = rcu_dereference_raw(idr->hint);
if (hint && (id & ~IDR_MASK) == hint->prefix)
return rcu_dereference_raw(hint->ary[id & IDR_MASK]);
return idr_find_slowpath(idr, id);
return radix_tree_lookup(&idr->idr_rt, id);
}
/**
* idr_for_each_entry - iterate over an idr's elements of a given type
* @idp: idr handle
* @idr: idr handle
* @entry: the type * to use as cursor
* @id: id entry's key
*
......@@ -157,57 +143,60 @@ static inline void *idr_find(struct idr *idr, int id)
* after normal terminatinon @entry is left with the value NULL. This
* is convenient for a "not found" value.
*/
#define idr_for_each_entry(idp, entry, id) \
for (id = 0; ((entry) = idr_get_next(idp, &(id))) != NULL; ++id)
#define idr_for_each_entry(idr, entry, id) \
for (id = 0; ((entry) = idr_get_next(idr, &(id))) != NULL; ++id)
/**
* idr_for_each_entry - continue iteration over an idr's elements of a given type
* @idp: idr handle
* idr_for_each_entry_continue - continue iteration over an idr's elements of a given type
* @idr: idr handle
* @entry: the type * to use as cursor
* @id: id entry's key
*
* Continue to iterate over list of given type, continuing after
* the current position.
*/
#define idr_for_each_entry_continue(idp, entry, id) \
for ((entry) = idr_get_next((idp), &(id)); \
#define idr_for_each_entry_continue(idr, entry, id) \
for ((entry) = idr_get_next((idr), &(id)); \
entry; \
++id, (entry) = idr_get_next((idp), &(id)))
++id, (entry) = idr_get_next((idr), &(id)))
/*
* IDA - IDR based id allocator, use when translation from id to
* pointer isn't necessary.
*
* IDA_BITMAP_LONGS is calculated to be one less to accommodate
* ida_bitmap->nr_busy so that the whole struct fits in 128 bytes.
*/
#define IDA_CHUNK_SIZE 128 /* 128 bytes per chunk */
#define IDA_BITMAP_LONGS (IDA_CHUNK_SIZE / sizeof(long) - 1)
#define IDA_BITMAP_LONGS (IDA_CHUNK_SIZE / sizeof(long))
#define IDA_BITMAP_BITS (IDA_BITMAP_LONGS * sizeof(long) * 8)
struct ida_bitmap {
long nr_busy;
unsigned long bitmap[IDA_BITMAP_LONGS];
};
DECLARE_PER_CPU(struct ida_bitmap *, ida_bitmap);
struct ida {
struct idr idr;
struct ida_bitmap *free_bitmap;
struct radix_tree_root ida_rt;
};
#define IDA_INIT(name) { .idr = IDR_INIT((name).idr), .free_bitmap = NULL, }
#define DEFINE_IDA(name) struct ida name = IDA_INIT(name)
#define IDA_INIT { \
.ida_rt = RADIX_TREE_INIT(IDR_RT_MARKER | GFP_NOWAIT), \
}
#define DEFINE_IDA(name) struct ida name = IDA_INIT
int ida_pre_get(struct ida *ida, gfp_t gfp_mask);
int ida_get_new_above(struct ida *ida, int starting_id, int *p_id);
void ida_remove(struct ida *ida, int id);
void ida_destroy(struct ida *ida);
void ida_init(struct ida *ida);
int ida_simple_get(struct ida *ida, unsigned int start, unsigned int end,
gfp_t gfp_mask);
void ida_simple_remove(struct ida *ida, unsigned int id);
static inline void ida_init(struct ida *ida)
{
INIT_RADIX_TREE(&ida->ida_rt, IDR_RT_MARKER | GFP_NOWAIT);
}
/**
* ida_get_new - allocate new ID
* @ida: idr handle
......@@ -220,11 +209,8 @@ static inline int ida_get_new(struct ida *ida, int *p_id)
return ida_get_new_above(ida, 0, p_id);
}
static inline bool ida_is_empty(struct ida *ida)
static inline bool ida_is_empty(const struct ida *ida)
{
return idr_is_empty(&ida->idr);
return radix_tree_empty(&ida->ida_rt);
}
void __init idr_init_cache(void);
#endif /* __IDR_H__ */
This diff is collapsed.
......@@ -554,7 +554,7 @@ asmlinkage __visible void __init start_kernel(void)
if (WARN(!irqs_disabled(),
"Interrupts were enabled *very* early, fixing it\n"))
local_irq_disable();
idr_init_cache();
radix_tree_init();
/*
* Allow workqueue creation and work item queueing/cancelling
......@@ -569,7 +569,6 @@ asmlinkage __visible void __init start_kernel(void)
trace_init();
context_tracking_init();
radix_tree_init();
/* init some links before init_ISA_irqs() */
early_irq_init();
init_IRQ();
......
......@@ -25,6 +25,9 @@ lib-y := ctype.o string.o vsprintf.o cmdline.o \
earlycpio.o seq_buf.o siphash.o \
nmi_backtrace.o nodemask.o win_minmax.o
CFLAGS_radix-tree.o += -DCONFIG_SPARSE_RCU_POINTER
CFLAGS_idr.o += -DCONFIG_SPARSE_RCU_POINTER
lib-$(CONFIG_MMU) += ioremap.o
lib-$(CONFIG_SMP) += cpumask.o
lib-$(CONFIG_DMA_NOOP_OPS) += dma-noop.o
......
This diff is collapsed.
This diff is collapsed.
......@@ -355,10 +355,8 @@ void workingset_update_node(struct radix_tree_node *node, void *private)
* as node->private_list is protected by &mapping->tree_lock.
*/
if (node->count && node->count == node->exceptional) {
if (list_empty(&node->private_list)) {
node->private_data = mapping;
if (list_empty(&node->private_list))
list_lru_add(&shadow_nodes, &node->private_list);
}
} else {
if (!list_empty(&node->private_list))
list_lru_del(&shadow_nodes, &node->private_list);
......@@ -436,7 +434,7 @@ static enum lru_status shadow_lru_isolate(struct list_head *item,
*/
node = container_of(item, struct radix_tree_node, private_list);
mapping = node->private_data;
mapping = container_of(node->root, struct address_space, page_tree);
/* Coming from the list, invert the lock order */
if (!spin_trylock(&mapping->tree_lock)) {
......
......@@ -462,9 +462,7 @@ static void ieee80211_report_ack_skb(struct ieee80211_local *local,
unsigned long flags;
spin_lock_irqsave(&local->ack_status_lock, flags);
skb = idr_find(&local->ack_status_frames, info->ack_frame_id);
if (skb)
idr_remove(&local->ack_status_frames, info->ack_frame_id);
skb = idr_remove(&local->ack_status_frames, info->ack_frame_id);
spin_unlock_irqrestore(&local->ack_status_lock, flags);
if (!skb)
......
......@@ -20,4 +20,7 @@ static __always_inline int test_bit(unsigned int nr, const unsigned long *addr)
(((unsigned long *)addr)[nr / __BITS_PER_LONG])) != 0;
}
#define __set_bit(nr, addr) set_bit(nr, addr)
#define __clear_bit(nr, addr) clear_bit(nr, addr)
#endif /* _TOOLS_LINUX_ASM_GENERIC_BITOPS_ATOMIC_H_ */
......@@ -12,6 +12,14 @@
unlikely(__ret_warn_on); \
})
#define WARN_ON(condition) ({ \
int __ret_warn_on = !!(condition); \
if (unlikely(__ret_warn_on)) \
__WARN_printf("assertion failed at %s:%d\n", \
__FILE__, __LINE__); \
unlikely(__ret_warn_on); \
})
#define WARN_ON_ONCE(condition) ({ \
static int __warned; \
int __ret_warn_once = !!(condition); \
......
......@@ -4,6 +4,7 @@
#include <string.h>
#include <linux/bitops.h>
#include <stdlib.h>
#include <linux/kernel.h>
#define DECLARE_BITMAP(name,bits) \
unsigned long name[BITS_TO_LONGS(bits)]
......
......@@ -2,7 +2,6 @@
#define _TOOLS_LINUX_BITOPS_H_
#include <asm/types.h>
#include <linux/kernel.h>
#include <linux/compiler.h>
#ifndef __WORDSIZE
......
......@@ -25,6 +25,8 @@
#endif
#define __user
#define __rcu
#define __read_mostly
#ifndef __attribute_const__
# define __attribute_const__
......@@ -54,6 +56,8 @@
# define unlikely(x) __builtin_expect(!!(x), 0)
#endif
#define uninitialized_var(x) x = *(&(x))
#define ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x))
#include <linux/types.h>
......
#define spinlock_t pthread_mutex_t
#define DEFINE_SPINLOCK(x) pthread_mutex_t x = PTHREAD_MUTEX_INITIALIZER;
#define spin_lock_irqsave(x, f) (void)f, pthread_mutex_lock(x)
#define spin_unlock_irqrestore(x, f) (void)f, pthread_mutex_unlock(x)
generated/map-shift.h
idr.c
idr-test
main
multiorder
radix-tree.c
CFLAGS += -I. -I../../include -g -O2 -Wall -D_LGPL_SOURCE
CFLAGS += -I. -I../../include -g -O2 -Wall -D_LGPL_SOURCE -fsanitize=address
LDFLAGS += -lpthread -lurcu
TARGETS = main
OFILES = main.o radix-tree.o linux.o test.o tag_check.o find_next_bit.o \
regression1.o regression2.o regression3.o multiorder.o \
iteration_check.o benchmark.o
TARGETS = main idr-test multiorder
CORE_OFILES := radix-tree.o idr.o linux.o test.o find_bit.o
OFILES = main.o $(CORE_OFILES) regression1.o regression2.o regression3.o \
tag_check.o multiorder.o idr-test.o iteration_check.o benchmark.o
ifdef BENCHMARK
CFLAGS += -DBENCHMARK=1
ifndef SHIFT
SHIFT=3
endif
targets: $(TARGETS)
targets: mapshift $(TARGETS)
main: $(OFILES)
$(CC) $(CFLAGS) $(LDFLAGS) $(OFILES) -o main
$(CC) $(CFLAGS) $(LDFLAGS) $^ -o main
idr-test: idr-test.o $(CORE_OFILES)
$(CC) $(CFLAGS) $(LDFLAGS) $^ -o idr-test
multiorder: multiorder.o $(CORE_OFILES)
$(CC) $(CFLAGS) $(LDFLAGS) $^ -o multiorder
clean:
$(RM) -f $(TARGETS) *.o radix-tree.c
$(RM) $(TARGETS) *.o radix-tree.c idr.c generated/map-shift.h
find_next_bit.o: ../../lib/find_bit.c
$(CC) $(CFLAGS) -c -o $@ $<
vpath %.c ../../lib
$(OFILES): *.h */*.h \
$(OFILES): *.h */*.h generated/map-shift.h \
../../include/linux/*.h \
../../../include/linux/radix-tree.h
../../include/asm/*.h \
../../../include/linux/radix-tree.h \
../../../include/linux/idr.h
radix-tree.c: ../../../lib/radix-tree.c
sed -e 's/^static //' -e 's/__always_inline //' -e 's/inline //' < $< > $@
idr.c: ../../../lib/idr.c
sed -e 's/^static //' -e 's/__always_inline //' -e 's/inline //' < $< > $@
.PHONY: mapshift
mapshift:
@if ! grep -qw $(SHIFT) generated/map-shift.h; then \
echo "#define RADIX_TREE_MAP_SHIFT $(SHIFT)" > \
generated/map-shift.h; \
fi
......@@ -71,7 +71,7 @@ static void benchmark_size(unsigned long size, unsigned long step, int order)
tagged = benchmark_iter(&tree, true);
normal = benchmark_iter(&tree, false);
printf("Size %ld, step %6ld, order %d tagged %10lld ns, normal %10lld ns\n",
printv(2, "Size %ld, step %6ld, order %d tagged %10lld ns, normal %10lld ns\n",
size, step, order, tagged, normal);
item_kill_tree(&tree);
......@@ -85,8 +85,8 @@ void benchmark(void)
128, 256, 512, 12345, 0};
int c, s;
printf("starting benchmarks\n");
printf("RADIX_TREE_MAP_SHIFT = %d\n", RADIX_TREE_MAP_SHIFT);
printv(1, "starting benchmarks\n");
printv(1, "RADIX_TREE_MAP_SHIFT = %d\n", RADIX_TREE_MAP_SHIFT);
for (c = 0; size[c]; c++)
for (s = 0; step[s]; s++)
......
#define CONFIG_RADIX_TREE_MULTIORDER 1
#define CONFIG_SHMEM 1
#define CONFIG_SWAP 1
This diff is collapsed.
......@@ -177,7 +177,7 @@ void iteration_test(unsigned order, unsigned test_duration)
{
int i;
printf("Running %siteration tests for %d seconds\n",
printv(1, "Running %siteration tests for %d seconds\n",
order > 0 ? "multiorder " : "", test_duration);
max_order = order;
......
......@@ -5,7 +5,7 @@
#include <unistd.h>
#include <assert.h>
#include <linux/mempool.h>
#include <linux/gfp.h>
#include <linux/poison.h>
#include <linux/slab.h>
#include <linux/radix-tree.h>
......@@ -13,6 +13,8 @@
int nr_allocated;
int preempt_count;
int kmalloc_verbose;
int test_verbose;
struct kmem_cache {
pthread_mutex_t lock;
......@@ -22,27 +24,6 @@ struct kmem_cache {
void (*ctor)(void *);
};
void *mempool_alloc(mempool_t *pool, int gfp_mask)
{
return pool->alloc(gfp_mask, pool->data);
}
void mempool_free(void *element, mempool_t *pool)
{
pool->free(element, pool->data);
}
mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
mempool_free_t *free_fn, void *pool_data)
{
mempool_t *ret = malloc(sizeof(*ret));
ret->alloc = alloc_fn;
ret->free = free_fn;
ret->data = pool_data;
return ret;
}
void *kmem_cache_alloc(struct kmem_cache *cachep, int flags)
{
struct radix_tree_node *node;
......@@ -54,9 +35,9 @@ void *kmem_cache_alloc(struct kmem_cache *cachep, int flags)
if (cachep->nr_objs) {
cachep->nr_objs--;
node = cachep->objs;
cachep->objs = node->private_data;
cachep->objs = node->parent;
pthread_mutex_unlock(&cachep->lock);
node->private_data = NULL;
node->parent = NULL;
} else {
pthread_mutex_unlock(&cachep->lock);
node = malloc(cachep->size);
......@@ -65,6 +46,8 @@ void *kmem_cache_alloc(struct kmem_cache *cachep, int flags)
}
uatomic_inc(&nr_allocated);
if (kmalloc_verbose)
printf("Allocating %p from slab\n", node);
return node;
}
......@@ -72,6 +55,8 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp)
{
assert(objp);
uatomic_dec(&nr_allocated);
if (kmalloc_verbose)
printf("Freeing %p to slab\n", objp);
pthread_mutex_lock(&cachep->lock);
if (cachep->nr_objs > 10) {
memset(objp, POISON_FREE, cachep->size);
......@@ -79,7 +64,7 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp)
} else {
struct radix_tree_node *node = objp;
cachep->nr_objs++;
node->private_data = cachep->objs;
node->parent = cachep->objs;
cachep->objs = node;
}
pthread_mutex_unlock(&cachep->lock);
......@@ -89,6 +74,8 @@ void *kmalloc(size_t size, gfp_t gfp)
{
void *ret = malloc(size);
uatomic_inc(&nr_allocated);
if (kmalloc_verbose)
printf("Allocating %p from malloc\n", ret);
return ret;
}
......@@ -97,6 +84,8 @@ void kfree(void *p)
if (!p)
return;
uatomic_dec(&nr_allocated);
if (kmalloc_verbose)
printf("Freeing %p to malloc\n", p);
free(p);
}
......
#ifndef _ASM_GENERIC_BITOPS_NON_ATOMIC_H_
#define _ASM_GENERIC_BITOPS_NON_ATOMIC_H_
#include <linux/types.h>
#include <linux/bitops/find.h>
#include <linux/bitops/hweight.h>
#include <linux/kernel.h>
#define BIT_MASK(nr) (1UL << ((nr) % BITS_PER_LONG))
#define BIT_WORD(nr) ((nr) / BITS_PER_LONG)
#define BITS_PER_BYTE 8
#define BITS_TO_LONGS(nr) DIV_ROUND_UP(nr, BITS_PER_BYTE * sizeof(long))
/**
* __set_bit - Set a bit in memory
* @nr: the bit to set
* @addr: the address to start counting from
*
* Unlike set_bit(), this function is non-atomic and may be reordered.
* If it's called on the same region of memory simultaneously, the effect
* may be that only one operation succeeds.
*/
static inline void __set_bit(int nr, volatile unsigned long *addr)
{
unsigned long mask = BIT_MASK(nr);
unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
*p |= mask;
}
static inline void __clear_bit(int nr, volatile unsigned long *addr)
{
unsigned long mask = BIT_MASK(nr);
unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
*p &= ~mask;