debugobjects.c 28.3 KB
Newer Older
1 2 3 4 5 6 7 8 9
/*
 * Generic infrastructure for lifetime debugging of objects.
 *
 * Started by Thomas Gleixner
 *
 * Copyright (C) 2008, Thomas Gleixner <tglx@linutronix.de>
 *
 * For licencing details see kernel-base/COPYING
 */
10 11 12

#define pr_fmt(fmt) "ODEBUG: " fmt

13 14
#include <linux/debugobjects.h>
#include <linux/interrupt.h>
15
#include <linux/sched.h>
16
#include <linux/sched/task_stack.h>
17 18
#include <linux/seq_file.h>
#include <linux/debugfs.h>
19
#include <linux/slab.h>
20
#include <linux/hash.h>
21
#include <linux/kmemleak.h>
22 23 24 25

#define ODEBUG_HASH_BITS	14
#define ODEBUG_HASH_SIZE	(1 << ODEBUG_HASH_BITS)

26
#define ODEBUG_POOL_SIZE	1024
27 28 29 30 31 32 33 34
#define ODEBUG_POOL_MIN_LEVEL	256

#define ODEBUG_CHUNK_SHIFT	PAGE_SHIFT
#define ODEBUG_CHUNK_SIZE	(1 << ODEBUG_CHUNK_SHIFT)
#define ODEBUG_CHUNK_MASK	(~(ODEBUG_CHUNK_SIZE - 1))

struct debug_bucket {
	struct hlist_head	list;
35
	raw_spinlock_t		lock;
36 37 38 39
};

static struct debug_bucket	obj_hash[ODEBUG_HASH_SIZE];

40
static struct debug_obj		obj_static_pool[ODEBUG_POOL_SIZE] __initdata;
41

42
static DEFINE_RAW_SPINLOCK(pool_lock);
43 44 45 46 47 48 49 50 51 52 53 54

static HLIST_HEAD(obj_pool);

static int			obj_pool_min_free = ODEBUG_POOL_SIZE;
static int			obj_pool_free = ODEBUG_POOL_SIZE;
static int			obj_pool_used;
static int			obj_pool_max_used;
static struct kmem_cache	*obj_cache;

static int			debug_objects_maxchain __read_mostly;
static int			debug_objects_fixups __read_mostly;
static int			debug_objects_warnings __read_mostly;
55 56
static int			debug_objects_enabled __read_mostly
				= CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT;
57 58 59 60
static int			debug_objects_pool_size __read_mostly
				= ODEBUG_POOL_SIZE;
static int			debug_objects_pool_min_level __read_mostly
				= ODEBUG_POOL_MIN_LEVEL;
61 62
static struct debug_obj_descr	*descr_test  __read_mostly;

63
/*
64
 * Track numbers of kmem_cache_alloc()/free() calls done.
65
 */
66
static int			debug_objects_allocated;
67 68
static int			debug_objects_freed;

69 70 71
static void free_obj_work(struct work_struct *work);
static DECLARE_WORK(debug_obj_work, free_obj_work);

72 73 74 75 76
static int __init enable_object_debug(char *str)
{
	debug_objects_enabled = 1;
	return 0;
}
77 78 79 80 81 82 83

static int __init disable_object_debug(char *str)
{
	debug_objects_enabled = 0;
	return 0;
}

84
early_param("debug_objects", enable_object_debug);
85
early_param("no_debug_objects", disable_object_debug);
86 87 88 89 90 91 92 93 94 95

static const char *obj_states[ODEBUG_STATE_MAX] = {
	[ODEBUG_STATE_NONE]		= "none",
	[ODEBUG_STATE_INIT]		= "initialized",
	[ODEBUG_STATE_INACTIVE]		= "inactive",
	[ODEBUG_STATE_ACTIVE]		= "active",
	[ODEBUG_STATE_DESTROYED]	= "destroyed",
	[ODEBUG_STATE_NOTAVAILABLE]	= "not available",
};

96
static void fill_pool(void)
97 98 99
{
	gfp_t gfp = GFP_ATOMIC | __GFP_NORETRY | __GFP_NOWARN;
	struct debug_obj *new;
100
	unsigned long flags;
101

102
	if (likely(obj_pool_free >= debug_objects_pool_min_level))
103
		return;
104 105

	if (unlikely(!obj_cache))
106
		return;
107

108
	while (obj_pool_free < debug_objects_pool_min_level) {
109 110 111

		new = kmem_cache_zalloc(obj_cache, gfp);
		if (!new)
112
			return;
113

114
		kmemleak_ignore(new);
115
		raw_spin_lock_irqsave(&pool_lock, flags);
116
		hlist_add_head(&new->node, &obj_pool);
117
		debug_objects_allocated++;
118
		obj_pool_free++;
119
		raw_spin_unlock_irqrestore(&pool_lock, flags);
120 121 122 123 124 125 126 127 128 129 130
	}
}

/*
 * Lookup an object in the hash bucket.
 */
static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b)
{
	struct debug_obj *obj;
	int cnt = 0;

131
	hlist_for_each_entry(obj, &b->list, node) {
132 133 134 135 136 137 138 139 140 141 142
		cnt++;
		if (obj->object == addr)
			return obj;
	}
	if (cnt > debug_objects_maxchain)
		debug_objects_maxchain = cnt;

	return NULL;
}

/*
143
 * Allocate a new object. If the pool is empty, switch off the debugger.
144
 * Must be called with interrupts disabled.
145 146 147 148 149 150
 */
static struct debug_obj *
alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr)
{
	struct debug_obj *obj = NULL;

151
	raw_spin_lock(&pool_lock);
152 153 154 155 156 157
	if (obj_pool.first) {
		obj	    = hlist_entry(obj_pool.first, typeof(*obj), node);

		obj->object = addr;
		obj->descr  = descr;
		obj->state  = ODEBUG_STATE_NONE;
158
		obj->astate = 0;
159 160 161 162 163 164 165 166 167 168 169 170
		hlist_del(&obj->node);

		hlist_add_head(&obj->node, &b->list);

		obj_pool_used++;
		if (obj_pool_used > obj_pool_max_used)
			obj_pool_max_used = obj_pool_used;

		obj_pool_free--;
		if (obj_pool_free < obj_pool_min_free)
			obj_pool_min_free = obj_pool_free;
	}
171
	raw_spin_unlock(&pool_lock);
172 173 174 175 176

	return obj;
}

/*
177
 * workqueue function to free objects.
178 179 180 181
 *
 * To reduce contention on the global pool_lock, the actual freeing of
 * debug objects will be delayed if the pool_lock is busy. We also free
 * the objects in a batch of 4 for each lock/unlock cycle.
182
 */
183 184
#define ODEBUG_FREE_BATCH	4

185
static void free_obj_work(struct work_struct *work)
186
{
187
	struct debug_obj *objs[ODEBUG_FREE_BATCH];
188
	unsigned long flags;
189
	int i;
190

191 192 193 194 195 196 197 198 199 200 201
	if (!raw_spin_trylock_irqsave(&pool_lock, flags))
		return;
	while (obj_pool_free >= debug_objects_pool_size + ODEBUG_FREE_BATCH) {
		for (i = 0; i < ODEBUG_FREE_BATCH; i++) {
			objs[i] = hlist_entry(obj_pool.first,
					      typeof(*objs[0]), node);
			hlist_del(&objs[i]->node);
		}

		obj_pool_free -= ODEBUG_FREE_BATCH;
		debug_objects_freed += ODEBUG_FREE_BATCH;
202 203 204 205
		/*
		 * We release pool_lock across kmem_cache_free() to
		 * avoid contention on pool_lock.
		 */
206
		raw_spin_unlock_irqrestore(&pool_lock, flags);
207 208 209 210
		for (i = 0; i < ODEBUG_FREE_BATCH; i++)
			kmem_cache_free(obj_cache, objs[i]);
		if (!raw_spin_trylock_irqsave(&pool_lock, flags))
			return;
211
	}
212
	raw_spin_unlock_irqrestore(&pool_lock, flags);
213 214 215 216 217 218 219 220 221 222 223
}

/*
 * Put the object back into the pool and schedule work to free objects
 * if necessary.
 */
static void free_object(struct debug_obj *obj)
{
	unsigned long flags;
	int sched = 0;

224
	raw_spin_lock_irqsave(&pool_lock, flags);
225 226 227 228
	/*
	 * schedule work when the pool is filled and the cache is
	 * initialized:
	 */
229
	if (obj_pool_free > debug_objects_pool_size && obj_cache)
230
		sched = 1;
231 232 233
	hlist_add_head(&obj->node, &obj_pool);
	obj_pool_free++;
	obj_pool_used--;
234
	raw_spin_unlock_irqrestore(&pool_lock, flags);
235 236
	if (sched)
		schedule_work(&debug_obj_work);
237 238 239 240 241 242 243 244 245
}

/*
 * We run out of memory. That means we probably have tons of objects
 * allocated.
 */
static void debug_objects_oom(void)
{
	struct debug_bucket *db = obj_hash;
246
	struct hlist_node *tmp;
247
	HLIST_HEAD(freelist);
248 249 250 251
	struct debug_obj *obj;
	unsigned long flags;
	int i;

252
	pr_warn("Out of memory. ODEBUG disabled\n");
253 254

	for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
255
		raw_spin_lock_irqsave(&db->lock, flags);
256
		hlist_move_list(&db->list, &freelist);
257
		raw_spin_unlock_irqrestore(&db->lock, flags);
258 259

		/* Now free them */
260
		hlist_for_each_entry_safe(obj, tmp, &freelist, node) {
261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280
			hlist_del(&obj->node);
			free_object(obj);
		}
	}
}

/*
 * We use the pfn of the address for the hash. That way we can check
 * for freed objects simply by checking the affected bucket.
 */
static struct debug_bucket *get_bucket(unsigned long addr)
{
	unsigned long hash;

	hash = hash_long((addr >> ODEBUG_CHUNK_SHIFT), ODEBUG_HASH_BITS);
	return &obj_hash[hash];
}

static void debug_print_object(struct debug_obj *obj, char *msg)
{
281
	struct debug_obj_descr *descr = obj->descr;
282 283
	static int limit;

284 285 286
	if (limit < 5 && descr != descr_test) {
		void *hint = descr->debug_hint ?
			descr->debug_hint(obj->object) : NULL;
287
		limit++;
288
		WARN(1, KERN_ERR "ODEBUG: %s %s (active state %u) "
289
				 "object type: %s hint: %pS\n",
290
			msg, obj_states[obj->state], obj->astate,
291
			descr->name, hint);
292 293 294 295 296 297 298 299
	}
	debug_objects_warnings++;
}

/*
 * Try to repair the damage, so we have a better chance to get useful
 * debug output.
 */
300 301
static bool
debug_object_fixup(bool (*fixup)(void *addr, enum debug_obj_state state),
302 303
		   void * addr, enum debug_obj_state state)
{
304 305 306 307 308
	if (fixup && fixup(addr, state)) {
		debug_objects_fixups++;
		return true;
	}
	return false;
309 310 311 312 313 314 315 316 317 318
}

static void debug_object_is_on_stack(void *addr, int onstack)
{
	int is_on_stack;
	static int limit;

	if (limit > 4)
		return;

319
	is_on_stack = object_is_on_stack(addr);
320 321 322 323 324
	if (is_on_stack == onstack)
		return;

	limit++;
	if (is_on_stack)
325
		pr_warn("object is on stack, but not annotated\n");
326
	else
327
		pr_warn("object is not on stack, but annotated\n");
328 329 330 331 332 333 334 335 336 337 338
	WARN_ON(1);
}

static void
__debug_object_init(void *addr, struct debug_obj_descr *descr, int onstack)
{
	enum debug_obj_state state;
	struct debug_bucket *db;
	struct debug_obj *obj;
	unsigned long flags;

339 340
	fill_pool();

341 342
	db = get_bucket((unsigned long) addr);

343
	raw_spin_lock_irqsave(&db->lock, flags);
344 345 346 347 348 349

	obj = lookup_object(addr, db);
	if (!obj) {
		obj = alloc_object(addr, db, descr);
		if (!obj) {
			debug_objects_enabled = 0;
350
			raw_spin_unlock_irqrestore(&db->lock, flags);
351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366
			debug_objects_oom();
			return;
		}
		debug_object_is_on_stack(addr, onstack);
	}

	switch (obj->state) {
	case ODEBUG_STATE_NONE:
	case ODEBUG_STATE_INIT:
	case ODEBUG_STATE_INACTIVE:
		obj->state = ODEBUG_STATE_INIT;
		break;

	case ODEBUG_STATE_ACTIVE:
		debug_print_object(obj, "init");
		state = obj->state;
367
		raw_spin_unlock_irqrestore(&db->lock, flags);
368 369 370 371 372 373 374 375 376 377
		debug_object_fixup(descr->fixup_init, addr, state);
		return;

	case ODEBUG_STATE_DESTROYED:
		debug_print_object(obj, "init");
		break;
	default:
		break;
	}

378
	raw_spin_unlock_irqrestore(&db->lock, flags);
379 380 381 382 383 384 385 386 387 388 389 390 391 392
}

/**
 * debug_object_init - debug checks when an object is initialized
 * @addr:	address of the object
 * @descr:	pointer to an object specific debug description structure
 */
void debug_object_init(void *addr, struct debug_obj_descr *descr)
{
	if (!debug_objects_enabled)
		return;

	__debug_object_init(addr, descr, 0);
}
393
EXPORT_SYMBOL_GPL(debug_object_init);
394 395 396 397 398 399 400 401 402 403 404 405 406 407

/**
 * debug_object_init_on_stack - debug checks when an object on stack is
 *				initialized
 * @addr:	address of the object
 * @descr:	pointer to an object specific debug description structure
 */
void debug_object_init_on_stack(void *addr, struct debug_obj_descr *descr)
{
	if (!debug_objects_enabled)
		return;

	__debug_object_init(addr, descr, 1);
}
408
EXPORT_SYMBOL_GPL(debug_object_init_on_stack);
409 410 411 412 413

/**
 * debug_object_activate - debug checks when an object is activated
 * @addr:	address of the object
 * @descr:	pointer to an object specific debug description structure
414
 * Returns 0 for success, -EINVAL for check failed.
415
 */
416
int debug_object_activate(void *addr, struct debug_obj_descr *descr)
417 418 419 420 421
{
	enum debug_obj_state state;
	struct debug_bucket *db;
	struct debug_obj *obj;
	unsigned long flags;
422
	int ret;
423 424 425
	struct debug_obj o = { .object = addr,
			       .state = ODEBUG_STATE_NOTAVAILABLE,
			       .descr = descr };
426 427

	if (!debug_objects_enabled)
428
		return 0;
429 430 431

	db = get_bucket((unsigned long) addr);

432
	raw_spin_lock_irqsave(&db->lock, flags);
433 434 435 436 437 438 439

	obj = lookup_object(addr, db);
	if (obj) {
		switch (obj->state) {
		case ODEBUG_STATE_INIT:
		case ODEBUG_STATE_INACTIVE:
			obj->state = ODEBUG_STATE_ACTIVE;
440
			ret = 0;
441 442 443 444 445
			break;

		case ODEBUG_STATE_ACTIVE:
			debug_print_object(obj, "activate");
			state = obj->state;
446
			raw_spin_unlock_irqrestore(&db->lock, flags);
447
			ret = debug_object_fixup(descr->fixup_activate, addr, state);
448
			return ret ? 0 : -EINVAL;
449 450 451

		case ODEBUG_STATE_DESTROYED:
			debug_print_object(obj, "activate");
452
			ret = -EINVAL;
453 454
			break;
		default:
455
			ret = 0;
456 457
			break;
		}
458
		raw_spin_unlock_irqrestore(&db->lock, flags);
459
		return ret;
460 461
	}

462
	raw_spin_unlock_irqrestore(&db->lock, flags);
463
	/*
464 465 466 467 468
	 * We are here when a static object is activated. We
	 * let the type specific code confirm whether this is
	 * true or not. if true, we just make sure that the
	 * static object is tracked in the object tracker. If
	 * not, this must be a bug, so we try to fix it up.
469
	 */
470 471 472 473 474
	if (descr->is_static_object && descr->is_static_object(addr)) {
		/* track this static object */
		debug_object_init(addr, descr);
		debug_object_activate(addr, descr);
	} else {
475
		debug_print_object(&o, "activate");
476 477 478
		ret = debug_object_fixup(descr->fixup_activate, addr,
					ODEBUG_STATE_NOTAVAILABLE);
		return ret ? 0 : -EINVAL;
479 480
	}
	return 0;
481
}
482
EXPORT_SYMBOL_GPL(debug_object_activate);
483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499

/**
 * debug_object_deactivate - debug checks when an object is deactivated
 * @addr:	address of the object
 * @descr:	pointer to an object specific debug description structure
 */
void debug_object_deactivate(void *addr, struct debug_obj_descr *descr)
{
	struct debug_bucket *db;
	struct debug_obj *obj;
	unsigned long flags;

	if (!debug_objects_enabled)
		return;

	db = get_bucket((unsigned long) addr);

500
	raw_spin_lock_irqsave(&db->lock, flags);
501 502 503 504 505 506 507

	obj = lookup_object(addr, db);
	if (obj) {
		switch (obj->state) {
		case ODEBUG_STATE_INIT:
		case ODEBUG_STATE_INACTIVE:
		case ODEBUG_STATE_ACTIVE:
508 509 510 511
			if (!obj->astate)
				obj->state = ODEBUG_STATE_INACTIVE;
			else
				debug_print_object(obj, "deactivate");
512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527
			break;

		case ODEBUG_STATE_DESTROYED:
			debug_print_object(obj, "deactivate");
			break;
		default:
			break;
		}
	} else {
		struct debug_obj o = { .object = addr,
				       .state = ODEBUG_STATE_NOTAVAILABLE,
				       .descr = descr };

		debug_print_object(&o, "deactivate");
	}

528
	raw_spin_unlock_irqrestore(&db->lock, flags);
529
}
530
EXPORT_SYMBOL_GPL(debug_object_deactivate);
531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548

/**
 * debug_object_destroy - debug checks when an object is destroyed
 * @addr:	address of the object
 * @descr:	pointer to an object specific debug description structure
 */
void debug_object_destroy(void *addr, struct debug_obj_descr *descr)
{
	enum debug_obj_state state;
	struct debug_bucket *db;
	struct debug_obj *obj;
	unsigned long flags;

	if (!debug_objects_enabled)
		return;

	db = get_bucket((unsigned long) addr);

549
	raw_spin_lock_irqsave(&db->lock, flags);
550 551 552 553 554 555 556 557 558 559 560 561 562 563

	obj = lookup_object(addr, db);
	if (!obj)
		goto out_unlock;

	switch (obj->state) {
	case ODEBUG_STATE_NONE:
	case ODEBUG_STATE_INIT:
	case ODEBUG_STATE_INACTIVE:
		obj->state = ODEBUG_STATE_DESTROYED;
		break;
	case ODEBUG_STATE_ACTIVE:
		debug_print_object(obj, "destroy");
		state = obj->state;
564
		raw_spin_unlock_irqrestore(&db->lock, flags);
565 566 567 568 569 570 571 572 573 574
		debug_object_fixup(descr->fixup_destroy, addr, state);
		return;

	case ODEBUG_STATE_DESTROYED:
		debug_print_object(obj, "destroy");
		break;
	default:
		break;
	}
out_unlock:
575
	raw_spin_unlock_irqrestore(&db->lock, flags);
576
}
577
EXPORT_SYMBOL_GPL(debug_object_destroy);
578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595

/**
 * debug_object_free - debug checks when an object is freed
 * @addr:	address of the object
 * @descr:	pointer to an object specific debug description structure
 */
void debug_object_free(void *addr, struct debug_obj_descr *descr)
{
	enum debug_obj_state state;
	struct debug_bucket *db;
	struct debug_obj *obj;
	unsigned long flags;

	if (!debug_objects_enabled)
		return;

	db = get_bucket((unsigned long) addr);

596
	raw_spin_lock_irqsave(&db->lock, flags);
597 598 599 600 601 602 603 604 605

	obj = lookup_object(addr, db);
	if (!obj)
		goto out_unlock;

	switch (obj->state) {
	case ODEBUG_STATE_ACTIVE:
		debug_print_object(obj, "free");
		state = obj->state;
606
		raw_spin_unlock_irqrestore(&db->lock, flags);
607 608 609 610
		debug_object_fixup(descr->fixup_free, addr, state);
		return;
	default:
		hlist_del(&obj->node);
611
		raw_spin_unlock_irqrestore(&db->lock, flags);
612
		free_object(obj);
613
		return;
614 615
	}
out_unlock:
616
	raw_spin_unlock_irqrestore(&db->lock, flags);
617
}
618
EXPORT_SYMBOL_GPL(debug_object_free);
619

620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645
/**
 * debug_object_assert_init - debug checks when object should be init-ed
 * @addr:	address of the object
 * @descr:	pointer to an object specific debug description structure
 */
void debug_object_assert_init(void *addr, struct debug_obj_descr *descr)
{
	struct debug_bucket *db;
	struct debug_obj *obj;
	unsigned long flags;

	if (!debug_objects_enabled)
		return;

	db = get_bucket((unsigned long) addr);

	raw_spin_lock_irqsave(&db->lock, flags);

	obj = lookup_object(addr, db);
	if (!obj) {
		struct debug_obj o = { .object = addr,
				       .state = ODEBUG_STATE_NOTAVAILABLE,
				       .descr = descr };

		raw_spin_unlock_irqrestore(&db->lock, flags);
		/*
646 647 648
		 * Maybe the object is static, and we let the type specific
		 * code confirm. Track this static object if true, else invoke
		 * fixup.
649
		 */
650 651 652 653
		if (descr->is_static_object && descr->is_static_object(addr)) {
			/* Track this static object */
			debug_object_init(addr, descr);
		} else {
654
			debug_print_object(&o, "assert_init");
655 656 657
			debug_object_fixup(descr->fixup_assert_init, addr,
					   ODEBUG_STATE_NOTAVAILABLE);
		}
658 659 660 661 662
		return;
	}

	raw_spin_unlock_irqrestore(&db->lock, flags);
}
663
EXPORT_SYMBOL_GPL(debug_object_assert_init);
664

665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710
/**
 * debug_object_active_state - debug checks object usage state machine
 * @addr:	address of the object
 * @descr:	pointer to an object specific debug description structure
 * @expect:	expected state
 * @next:	state to move to if expected state is found
 */
void
debug_object_active_state(void *addr, struct debug_obj_descr *descr,
			  unsigned int expect, unsigned int next)
{
	struct debug_bucket *db;
	struct debug_obj *obj;
	unsigned long flags;

	if (!debug_objects_enabled)
		return;

	db = get_bucket((unsigned long) addr);

	raw_spin_lock_irqsave(&db->lock, flags);

	obj = lookup_object(addr, db);
	if (obj) {
		switch (obj->state) {
		case ODEBUG_STATE_ACTIVE:
			if (obj->astate == expect)
				obj->astate = next;
			else
				debug_print_object(obj, "active_state");
			break;

		default:
			debug_print_object(obj, "active_state");
			break;
		}
	} else {
		struct debug_obj o = { .object = addr,
				       .state = ODEBUG_STATE_NOTAVAILABLE,
				       .descr = descr };

		debug_print_object(&o, "active_state");
	}

	raw_spin_unlock_irqrestore(&db->lock, flags);
}
711
EXPORT_SYMBOL_GPL(debug_object_active_state);
712

713 714 715 716
#ifdef CONFIG_DEBUG_OBJECTS_FREE
static void __debug_check_no_obj_freed(const void *address, unsigned long size)
{
	unsigned long flags, oaddr, saddr, eaddr, paddr, chunks;
717
	struct hlist_node *tmp;
718
	HLIST_HEAD(freelist);
719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735
	struct debug_obj_descr *descr;
	enum debug_obj_state state;
	struct debug_bucket *db;
	struct debug_obj *obj;
	int cnt;

	saddr = (unsigned long) address;
	eaddr = saddr + size;
	paddr = saddr & ODEBUG_CHUNK_MASK;
	chunks = ((eaddr - paddr) + (ODEBUG_CHUNK_SIZE - 1));
	chunks >>= ODEBUG_CHUNK_SHIFT;

	for (;chunks > 0; chunks--, paddr += ODEBUG_CHUNK_SIZE) {
		db = get_bucket(paddr);

repeat:
		cnt = 0;
736
		raw_spin_lock_irqsave(&db->lock, flags);
737
		hlist_for_each_entry_safe(obj, tmp, &db->list, node) {
738 739 740 741 742 743 744 745 746 747
			cnt++;
			oaddr = (unsigned long) obj->object;
			if (oaddr < saddr || oaddr >= eaddr)
				continue;

			switch (obj->state) {
			case ODEBUG_STATE_ACTIVE:
				debug_print_object(obj, "free");
				descr = obj->descr;
				state = obj->state;
748
				raw_spin_unlock_irqrestore(&db->lock, flags);
749 750 751 752 753
				debug_object_fixup(descr->fixup_free,
						   (void *) oaddr, state);
				goto repeat;
			default:
				hlist_del(&obj->node);
754
				hlist_add_head(&obj->node, &freelist);
755 756 757
				break;
			}
		}
758
		raw_spin_unlock_irqrestore(&db->lock, flags);
759 760

		/* Now free them */
761
		hlist_for_each_entry_safe(obj, tmp, &freelist, node) {
762 763 764 765
			hlist_del(&obj->node);
			free_object(obj);
		}

766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788
		if (cnt > debug_objects_maxchain)
			debug_objects_maxchain = cnt;
	}
}

void debug_check_no_obj_freed(const void *address, unsigned long size)
{
	if (debug_objects_enabled)
		__debug_check_no_obj_freed(address, size);
}
#endif

#ifdef CONFIG_DEBUG_FS

static int debug_stats_show(struct seq_file *m, void *v)
{
	seq_printf(m, "max_chain     :%d\n", debug_objects_maxchain);
	seq_printf(m, "warnings      :%d\n", debug_objects_warnings);
	seq_printf(m, "fixups        :%d\n", debug_objects_fixups);
	seq_printf(m, "pool_free     :%d\n", obj_pool_free);
	seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free);
	seq_printf(m, "pool_used     :%d\n", obj_pool_used);
	seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used);
789 790
	seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated);
	seq_printf(m, "objs_freed    :%d\n", debug_objects_freed);
791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845
	return 0;
}

static int debug_stats_open(struct inode *inode, struct file *filp)
{
	return single_open(filp, debug_stats_show, NULL);
}

static const struct file_operations debug_stats_fops = {
	.open		= debug_stats_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
};

static int __init debug_objects_init_debugfs(void)
{
	struct dentry *dbgdir, *dbgstats;

	if (!debug_objects_enabled)
		return 0;

	dbgdir = debugfs_create_dir("debug_objects", NULL);
	if (!dbgdir)
		return -ENOMEM;

	dbgstats = debugfs_create_file("stats", 0444, dbgdir, NULL,
				       &debug_stats_fops);
	if (!dbgstats)
		goto err;

	return 0;

err:
	debugfs_remove(dbgdir);

	return -ENOMEM;
}
__initcall(debug_objects_init_debugfs);

#else
static inline void debug_objects_init_debugfs(void) { }
#endif

#ifdef CONFIG_DEBUG_OBJECTS_SELFTEST

/* Random data structure for the self test */
struct self_test {
	unsigned long	dummy1[6];
	int		static_init;
	unsigned long	dummy2[3];
};

static __initdata struct debug_obj_descr descr_type_test;

846 847 848 849 850 851 852
static bool __init is_static_object(void *addr)
{
	struct self_test *obj = addr;

	return obj->static_init;
}

853 854 855 856
/*
 * fixup_init is called when:
 * - an active object is initialized
 */
857
static bool __init fixup_init(void *addr, enum debug_obj_state state)
858 859 860 861 862 863 864
{
	struct self_test *obj = addr;

	switch (state) {
	case ODEBUG_STATE_ACTIVE:
		debug_object_deactivate(obj, &descr_type_test);
		debug_object_init(obj, &descr_type_test);
865
		return true;
866
	default:
867
		return false;
868 869 870 871 872 873
	}
}

/*
 * fixup_activate is called when:
 * - an active object is activated
874
 * - an unknown non-static object is activated
875
 */
876
static bool __init fixup_activate(void *addr, enum debug_obj_state state)
877 878 879 880 881
{
	struct self_test *obj = addr;

	switch (state) {
	case ODEBUG_STATE_NOTAVAILABLE:
882
		return true;
883 884 885
	case ODEBUG_STATE_ACTIVE:
		debug_object_deactivate(obj, &descr_type_test);
		debug_object_activate(obj, &descr_type_test);
886
		return true;
887 888

	default:
889
		return false;
890 891 892 893 894 895 896
	}
}

/*
 * fixup_destroy is called when:
 * - an active object is destroyed
 */
897
static bool __init fixup_destroy(void *addr, enum debug_obj_state state)
898 899 900 901 902 903 904
{
	struct self_test *obj = addr;

	switch (state) {
	case ODEBUG_STATE_ACTIVE:
		debug_object_deactivate(obj, &descr_type_test);
		debug_object_destroy(obj, &descr_type_test);
905
		return true;
906
	default:
907
		return false;
908 909 910 911 912 913 914
	}
}

/*
 * fixup_free is called when:
 * - an active object is freed
 */
915
static bool __init fixup_free(void *addr, enum debug_obj_state state)
916 917 918 919 920 921 922
{
	struct self_test *obj = addr;

	switch (state) {
	case ODEBUG_STATE_ACTIVE:
		debug_object_deactivate(obj, &descr_type_test);
		debug_object_free(obj, &descr_type_test);
923
		return true;
924
	default:
925
		return false;
926 927 928
	}
}

929
static int __init
930 931 932 933 934 935 936 937 938
check_results(void *addr, enum debug_obj_state state, int fixups, int warnings)
{
	struct debug_bucket *db;
	struct debug_obj *obj;
	unsigned long flags;
	int res = -EINVAL;

	db = get_bucket((unsigned long) addr);

939
	raw_spin_lock_irqsave(&db->lock, flags);
940 941 942

	obj = lookup_object(addr, db);
	if (!obj && state != ODEBUG_STATE_NONE) {
Arjan van de Ven's avatar
Arjan van de Ven committed
943
		WARN(1, KERN_ERR "ODEBUG: selftest object not found\n");
944 945 946
		goto out;
	}
	if (obj && obj->state != state) {
Arjan van de Ven's avatar
Arjan van de Ven committed
947
		WARN(1, KERN_ERR "ODEBUG: selftest wrong state: %d != %d\n",
948 949 950 951
		       obj->state, state);
		goto out;
	}
	if (fixups != debug_objects_fixups) {
Arjan van de Ven's avatar
Arjan van de Ven committed
952
		WARN(1, KERN_ERR "ODEBUG: selftest fixups failed %d != %d\n",
953 954 955 956
		       fixups, debug_objects_fixups);
		goto out;
	}
	if (warnings != debug_objects_warnings) {
Arjan van de Ven's avatar
Arjan van de Ven committed
957
		WARN(1, KERN_ERR "ODEBUG: selftest warnings failed %d != %d\n",
958 959 960 961 962
		       warnings, debug_objects_warnings);
		goto out;
	}
	res = 0;
out:
963
	raw_spin_unlock_irqrestore(&db->lock, flags);
964 965 966 967 968 969 970
	if (res)
		debug_objects_enabled = 0;
	return res;
}

static __initdata struct debug_obj_descr descr_type_test = {
	.name			= "selftest",
971
	.is_static_object	= is_static_object,
972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020
	.fixup_init		= fixup_init,
	.fixup_activate		= fixup_activate,
	.fixup_destroy		= fixup_destroy,
	.fixup_free		= fixup_free,
};

static __initdata struct self_test obj = { .static_init = 0 };

static void __init debug_objects_selftest(void)
{
	int fixups, oldfixups, warnings, oldwarnings;
	unsigned long flags;

	local_irq_save(flags);

	fixups = oldfixups = debug_objects_fixups;
	warnings = oldwarnings = debug_objects_warnings;
	descr_test = &descr_type_test;

	debug_object_init(&obj, &descr_type_test);
	if (check_results(&obj, ODEBUG_STATE_INIT, fixups, warnings))
		goto out;
	debug_object_activate(&obj, &descr_type_test);
	if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
		goto out;
	debug_object_activate(&obj, &descr_type_test);
	if (check_results(&obj, ODEBUG_STATE_ACTIVE, ++fixups, ++warnings))
		goto out;
	debug_object_deactivate(&obj, &descr_type_test);
	if (check_results(&obj, ODEBUG_STATE_INACTIVE, fixups, warnings))
		goto out;
	debug_object_destroy(&obj, &descr_type_test);
	if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, warnings))
		goto out;
	debug_object_init(&obj, &descr_type_test);
	if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
		goto out;
	debug_object_activate(&obj, &descr_type_test);
	if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
		goto out;
	debug_object_deactivate(&obj, &descr_type_test);
	if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
		goto out;
	debug_object_free(&obj, &descr_type_test);
	if (check_results(&obj, ODEBUG_STATE_NONE, fixups, warnings))
		goto out;

	obj.static_init = 1;
	debug_object_activate(&obj, &descr_type_test);
1021
	if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040
		goto out;
	debug_object_init(&obj, &descr_type_test);
	if (check_results(&obj, ODEBUG_STATE_INIT, ++fixups, ++warnings))
		goto out;
	debug_object_free(&obj, &descr_type_test);
	if (check_results(&obj, ODEBUG_STATE_NONE, fixups, warnings))
		goto out;

#ifdef CONFIG_DEBUG_OBJECTS_FREE
	debug_object_init(&obj, &descr_type_test);
	if (check_results(&obj, ODEBUG_STATE_INIT, fixups, warnings))
		goto out;
	debug_object_activate(&obj, &descr_type_test);
	if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
		goto out;
	__debug_check_no_obj_freed(&obj, sizeof(obj));
	if (check_results(&obj, ODEBUG_STATE_NONE, ++fixups, ++warnings))
		goto out;
#endif
1041
	pr_info("selftest passed\n");
1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063

out:
	debug_objects_fixups = oldfixups;
	debug_objects_warnings = oldwarnings;
	descr_test = NULL;

	local_irq_restore(flags);
}
#else
static inline void debug_objects_selftest(void) { }
#endif

/*
 * Called during early boot to initialize the hash buckets and link
 * the static object pool objects into the poll list. After this call
 * the object tracker is fully operational.
 */
void __init debug_objects_early_init(void)
{
	int i;

	for (i = 0; i < ODEBUG_HASH_SIZE; i++)
1064
		raw_spin_lock_init(&obj_hash[i].lock);
1065 1066 1067 1068 1069

	for (i = 0; i < ODEBUG_POOL_SIZE; i++)
		hlist_add_head(&obj_static_pool[i].node, &obj_pool);
}

1070 1071 1072
/*
 * Convert the statically allocated objects to dynamic ones:
 */
1073
static int __init debug_objects_replace_static_objects(void)
1074 1075
{
	struct debug_bucket *db = obj_hash;
1076
	struct hlist_node *tmp;
1077 1078 1079 1080 1081 1082 1083 1084
	struct debug_obj *obj, *new;
	HLIST_HEAD(objects);
	int i, cnt = 0;

	for (i = 0; i < ODEBUG_POOL_SIZE; i++) {
		obj = kmem_cache_zalloc(obj_cache, GFP_KERNEL);
		if (!obj)
			goto free;
1085
		kmemleak_ignore(obj);
1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096
		hlist_add_head(&obj->node, &objects);
	}

	/*
	 * When debug_objects_mem_init() is called we know that only
	 * one CPU is up, so disabling interrupts is enough
	 * protection. This avoids the lockdep hell of lock ordering.
	 */
	local_irq_disable();

	/* Remove the statically allocated objects from the pool */
1097
	hlist_for_each_entry_safe(obj, tmp, &obj_pool, node)
1098 1099 1100 1101 1102 1103 1104 1105
		hlist_del(&obj->node);
	/* Move the allocated objects to the pool */
	hlist_move_list(&objects, &obj_pool);

	/* Replace the active object references */
	for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
		hlist_move_list(&db->list, &objects);

1106
		hlist_for_each_entry(obj, &objects, node) {
1107 1108 1109 1110 1111 1112 1113 1114
			new = hlist_entry(obj_pool.first, typeof(*obj), node);
			hlist_del(&new->node);
			/* copy object data */
			*new = *obj;
			hlist_add_head(&new->node, &db->list);
			cnt++;
		}
	}
1115
	local_irq_enable();
1116

1117 1118
	pr_debug("%d of %d active objects replaced\n",
		 cnt, obj_pool_used);
1119 1120
	return 0;
free:
1121
	hlist_for_each_entry_safe(obj, tmp, &objects, node) {
1122 1123 1124 1125 1126 1127
		hlist_del(&obj->node);
		kmem_cache_free(obj_cache, obj);
	}
	return -ENOMEM;
}

1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142
/*
 * Called after the kmem_caches are functional to setup a dedicated
 * cache pool, which has the SLAB_DEBUG_OBJECTS flag set. This flag
 * prevents that the debug code is called on kmem_cache_free() for the
 * debug tracker objects to avoid recursive calls.
 */
void __init debug_objects_mem_init(void)
{
	if (!debug_objects_enabled)
		return;

	obj_cache = kmem_cache_create("debug_objects_cache",
				      sizeof (struct debug_obj), 0,
				      SLAB_DEBUG_OBJECTS, NULL);

1143
	if (!obj_cache || debug_objects_replace_static_objects()) {
1144
		debug_objects_enabled = 0;
1145 1146
		if (obj_cache)
			kmem_cache_destroy(obj_cache);
1147
		pr_warn("out of memory.\n");
1148
	} else
1149
		debug_objects_selftest();
1150 1151 1152 1153 1154 1155 1156

	/*
	 * Increase the thresholds for allocating and freeing objects
	 * according to the number of possible CPUs available in the system.
	 */
	debug_objects_pool_size += num_possible_cpus() * 32;
	debug_objects_pool_min_level += num_possible_cpus() * 4;
1157
}