mds_client.h 13.3 KB
Newer Older
Sage Weil's avatar
Sage Weil committed
1 2 3 4
#ifndef _FS_CEPH_MDS_CLIENT_H
#define _FS_CEPH_MDS_CLIENT_H

#include <linux/completion.h>
5
#include <linux/kref.h>
Sage Weil's avatar
Sage Weil committed
6 7
#include <linux/list.h>
#include <linux/mutex.h>
8
#include <linux/rbtree.h>
Sage Weil's avatar
Sage Weil committed
9 10
#include <linux/spinlock.h>

11 12 13
#include <linux/ceph/types.h>
#include <linux/ceph/messenger.h>
#include <linux/ceph/mdsmap.h>
14
#include <linux/ceph/auth.h>
Sage Weil's avatar
Sage Weil committed
15 16 17 18 19 20 21 22 23

/*
 * Some lock dependencies:
 *
 * session->s_mutex
 *         mdsc->mutex
 *
 *         mdsc->snap_rwsem
 *
24
 *         ci->i_ceph_lock
Sage Weil's avatar
Sage Weil committed
25 26 27 28 29
 *                 mdsc->snap_flush_lock
 *                 mdsc->cap_delay_lock
 *
 */

30
struct ceph_fs_client;
Sage Weil's avatar
Sage Weil committed
31 32 33 34 35 36 37 38
struct ceph_cap;

/*
 * parsed info about a single inode.  pointers are into the encoded
 * on-wire structures within the mds reply message payload.
 */
struct ceph_mds_reply_info_in {
	struct ceph_mds_reply_inode *in;
39
	struct ceph_dir_layout dir_layout;
Sage Weil's avatar
Sage Weil committed
40 41 42 43
	u32 symlink_len;
	char *symlink;
	u32 xattr_len;
	char *xattr_data;
44 45 46
	u64 inline_version;
	u32 inline_len;
	char *inline_data;
47
	u32 pool_ns_len;
48
	char *pool_ns_data;
Sage Weil's avatar
Sage Weil committed
49 50
};

51 52 53 54 55
struct ceph_mds_reply_dir_entry {
	char                          *name;
	u32                           name_len;
	struct ceph_mds_reply_lease   *lease;
	struct ceph_mds_reply_info_in inode;
56
	loff_t			      offset;
57 58
};

Sage Weil's avatar
Sage Weil committed
59
/*
60 61 62 63
 * parsed info about an mds reply, including information about
 * either: 1) the target inode and/or its parent directory and dentry,
 * and directory contents (for readdir results), or
 * 2) the file range lock info (for fcntl F_GETLK results).
Sage Weil's avatar
Sage Weil committed
64 65 66 67
 */
struct ceph_mds_reply_info_parsed {
	struct ceph_mds_reply_head    *head;

68
	/* trace */
Sage Weil's avatar
Sage Weil committed
69 70 71 72 73 74
	struct ceph_mds_reply_info_in diri, targeti;
	struct ceph_mds_reply_dirfrag *dirfrag;
	char                          *dname;
	u32                           dname_len;
	struct ceph_mds_reply_lease   *dlease;

75 76 77 78 79 80 81 82
	/* extra */
	union {
		/* for fcntl F_GETLK results */
		struct ceph_filelock *filelock_reply;

		/* for readdir results */
		struct {
			struct ceph_mds_reply_dirfrag *dir_dir;
83
			size_t			      dir_buf_size;
84
			int                           dir_nr;
85 86 87
			bool			      dir_complete;
			bool			      dir_end;
			bool			      hash_order;
88
			struct ceph_mds_reply_dir_entry  *dir_entries;
89
		};
90 91 92 93 94 95

		/* for create results */
		struct {
			bool has_create_ino;
			u64 ino;
		};
96
	};
Sage Weil's avatar
Sage Weil committed
97 98 99 100 101 102 103 104 105 106 107

	/* encoded blob describing snapshot contexts for certain
	   operations (e.g., open) */
	void *snapblob;
	int snapblob_len;
};


/*
 * cap releases are batched and sent to the MDS en masse.
 */
108
#define CEPH_CAPS_PER_RELEASE ((PAGE_SIZE -			\
Sage Weil's avatar
Sage Weil committed
109 110 111 112 113 114 115 116 117 118 119 120 121 122 123
				sizeof(struct ceph_mds_cap_release)) /	\
			       sizeof(struct ceph_mds_cap_item))


/*
 * state associated with each MDS<->client session
 */
enum {
	CEPH_MDS_SESSION_NEW = 1,
	CEPH_MDS_SESSION_OPENING = 2,
	CEPH_MDS_SESSION_OPEN = 3,
	CEPH_MDS_SESSION_HUNG = 4,
	CEPH_MDS_SESSION_CLOSING = 5,
	CEPH_MDS_SESSION_RESTARTING = 6,
	CEPH_MDS_SESSION_RECONNECTING = 7,
124
	CEPH_MDS_SESSION_REJECTED = 8,
Sage Weil's avatar
Sage Weil committed
125 126 127 128 129 130 131 132 133 134 135 136
};

struct ceph_mds_session {
	struct ceph_mds_client *s_mdsc;
	int               s_mds;
	int               s_state;
	unsigned long     s_ttl;      /* time until mds kills us */
	u64               s_seq;      /* incoming msg seq # */
	struct mutex      s_mutex;    /* serialize session messages */

	struct ceph_connection s_con;

137
	struct ceph_auth_handshake s_auth;
138

139 140
	/* protected by s_gen_ttl_lock */
	spinlock_t        s_gen_ttl_lock;
Sage Weil's avatar
Sage Weil committed
141 142
	u32               s_cap_gen;  /* inc each time we get mds stale msg */
	unsigned long     s_cap_ttl;  /* when session caps expire */
143 144 145

	/* protected by s_cap_lock */
	spinlock_t        s_cap_lock;
Sage Weil's avatar
Sage Weil committed
146 147 148
	struct list_head  s_caps;     /* all caps issued by this session */
	int               s_nr_caps, s_trim_caps;
	int               s_num_cap_releases;
149
	int		  s_cap_reconnect;
150
	int		  s_readonly;
Sage Weil's avatar
Sage Weil committed
151
	struct list_head  s_cap_releases; /* waiting cap_release messages */
152
	struct ceph_cap  *s_cap_iterator;
Sage Weil's avatar
Sage Weil committed
153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180

	/* protected by mutex */
	struct list_head  s_cap_flushing;     /* inodes w/ flushing caps */
	unsigned long     s_renew_requested; /* last time we sent a renew req */
	u64               s_renew_seq;

	atomic_t          s_ref;
	struct list_head  s_waiting;  /* waiting requests */
	struct list_head  s_unsafe;   /* unsafe requests */
};

/*
 * modes of choosing which MDS to send a request to
 */
enum {
	USE_ANY_MDS,
	USE_RANDOM_MDS,
	USE_AUTH_MDS,   /* prefer authoritative mds for this metadata item */
};

struct ceph_mds_request;
struct ceph_mds_client;

/*
 * request completion callback
 */
typedef void (*ceph_mds_request_callback_t) (struct ceph_mds_client *mdsc,
					     struct ceph_mds_request *req);
181 182 183 184 185
/*
 * wait for request completion callback
 */
typedef int (*ceph_mds_request_wait_callback_t) (struct ceph_mds_client *mdsc,
						 struct ceph_mds_request *req);
Sage Weil's avatar
Sage Weil committed
186 187 188 189 190 191

/*
 * an in-flight mds request
 */
struct ceph_mds_request {
	u64 r_tid;                   /* transaction id */
192
	struct rb_node r_node;
193
	struct ceph_mds_client *r_mdsc;
Sage Weil's avatar
Sage Weil committed
194 195 196 197 198 199 200

	int r_op;                    /* mds op code */

	/* operation on what? */
	struct inode *r_inode;              /* arg1 */
	struct dentry *r_dentry;            /* arg1 */
	struct dentry *r_old_dentry;        /* arg2: rename from or link from */
201
	struct inode *r_old_dentry_dir;     /* arg2: old dentry's parent dir */
Sage Weil's avatar
Sage Weil committed
202 203 204
	char *r_path1, *r_path2;
	struct ceph_vino r_ino1, r_ino2;

205
	struct inode *r_parent;		    /* parent dir inode */
Sage Weil's avatar
Sage Weil committed
206 207
	struct inode *r_target_inode;       /* resulting inode */

208 209 210 211 212 213
#define CEPH_MDS_R_DIRECT_IS_HASH	(1) /* r_direct_hash is valid */
#define CEPH_MDS_R_ABORTED		(2) /* call was aborted */
#define CEPH_MDS_R_GOT_UNSAFE		(3) /* got an unsafe reply */
#define CEPH_MDS_R_GOT_SAFE		(4) /* got a safe reply */
#define CEPH_MDS_R_GOT_RESULT		(5) /* got a result */
#define CEPH_MDS_R_DID_PREPOPULATE	(6) /* prepopulated readdir */
214
#define CEPH_MDS_R_PARENT_LOCKED	(7) /* is r_parent->i_rwsem wlocked? */
215 216
	unsigned long	r_req_flags;

217 218
	struct mutex r_fill_mutex;

Sage Weil's avatar
Sage Weil committed
219 220
	union ceph_mds_request_args r_args;
	int r_fmode;        /* file mode, if expecting cap */
221 222
	kuid_t r_uid;
	kgid_t r_gid;
223
	struct timespec r_stamp;
Sage Weil's avatar
Sage Weil committed
224 225 226 227 228 229

	/* for choosing which mds to send this request to */
	int r_direct_mode;
	u32 r_direct_hash;      /* choose dir frag based on this dentry hash */

	/* data payload is used for xattr ops */
230
	struct ceph_pagelist *r_pagelist;
Sage Weil's avatar
Sage Weil committed
231 232 233 234 235 236 237 238 239

	/* what caps shall we drop? */
	int r_inode_drop, r_inode_unless;
	int r_dentry_drop, r_dentry_unless;
	int r_old_dentry_drop, r_old_dentry_unless;
	struct inode *r_old_inode;
	int r_old_inode_drop, r_old_inode_unless;

	struct ceph_msg  *r_request;  /* original request */
240
	int r_request_release_offset;
Sage Weil's avatar
Sage Weil committed
241 242
	struct ceph_msg  *r_reply;
	struct ceph_mds_reply_info_parsed r_reply_info;
243
	struct page *r_locked_page;
Sage Weil's avatar
Sage Weil committed
244 245
	int r_err;

246
	unsigned long r_timeout;  /* optional.  jiffies, 0 is "wait forever" */
Sage Weil's avatar
Sage Weil committed
247 248 249 250 251 252 253 254
	unsigned long r_started;  /* start time to measure timeout against */
	unsigned long r_request_started; /* start time for mds request only,
					    used to measure lease durations */

	/* link unsafe requests to parent directory, for fsync */
	struct inode	*r_unsafe_dir;
	struct list_head r_unsafe_dir_item;

255 256 257
	/* unsafe requests that modify the target inode */
	struct list_head r_unsafe_target_item;

Sage Weil's avatar
Sage Weil committed
258 259 260 261 262
	struct ceph_mds_session *r_session;

	int               r_attempts;   /* resend attempts */
	int               r_num_fwd;    /* number of forward attempts */
	int               r_resend_mds; /* mds to resend to next, if any*/
263
	u32               r_sent_on_mseq; /* cap mseq request was sent at*/
Sage Weil's avatar
Sage Weil committed
264

265
	struct kref       r_kref;
Sage Weil's avatar
Sage Weil committed
266 267 268 269
	struct list_head  r_wait;
	struct completion r_completion;
	struct completion r_safe_completion;
	ceph_mds_request_callback_t r_callback;
270
	ceph_mds_request_wait_callback_t r_wait_for_completion;
Sage Weil's avatar
Sage Weil committed
271 272
	struct list_head  r_unsafe_item;  /* per-session unsafe list item */

Yan, Zheng's avatar
Yan, Zheng committed
273 274 275
	long long	  r_dir_release_cnt;
	long long	  r_dir_ordered_cnt;
	int		  r_readdir_cache_idx;
Sage Weil's avatar
Sage Weil committed
276 277 278 279 280 281
	u32               r_readdir_offset;

	struct ceph_cap_reservation r_caps_reservation;
	int r_num_caps;
};

282 283 284
struct ceph_pool_perm {
	struct rb_node node;
	int perm;
285
	s64 pool;
286 287
	size_t pool_ns_len;
	char pool_ns[];
288 289
};

Sage Weil's avatar
Sage Weil committed
290 291 292 293
/*
 * mds client state
 */
struct ceph_mds_client {
294
	struct ceph_fs_client  *fsc;
Sage Weil's avatar
Sage Weil committed
295 296 297
	struct mutex            mutex;         /* all nested structures */

	struct ceph_mdsmap      *mdsmap;
298 299
	struct completion       safe_umount_waiters;
	wait_queue_head_t       session_close_wq;
Sage Weil's avatar
Sage Weil committed
300
	struct list_head        waiting_for_map;
301
	int 			mdsmap_err;
Sage Weil's avatar
Sage Weil committed
302 303

	struct ceph_mds_session **sessions;    /* NULL for mds if no session */
304
	atomic_t		num_sessions;
Sage Weil's avatar
Sage Weil committed
305 306 307 308 309 310 311 312 313 314
	int                     max_sessions;  /* len of s_mds_sessions */
	int                     stopping;      /* true if shutting down */

	/*
	 * snap_rwsem will cover cap linkage into snaprealms, and
	 * realm snap contexts.  (later, we can do per-realm snap
	 * contexts locks..)  the empty list contains realms with no
	 * references (implying they contain no inodes with caps) that
	 * should be destroyed.
	 */
315
	u64			last_snap_seq;
Sage Weil's avatar
Sage Weil committed
316
	struct rw_semaphore     snap_rwsem;
Sage Weil's avatar
Sage Weil committed
317
	struct rb_root          snap_realms;
Sage Weil's avatar
Sage Weil committed
318 319 320 321
	struct list_head        snap_empty;
	spinlock_t              snap_empty_lock;  /* protect snap_empty */

	u64                    last_tid;      /* most recent mds request */
322 323
	u64                    oldest_tid;    /* oldest incomplete mds request,
						 excluding setfilelock requests */
324
	struct rb_root         request_tree;  /* pending mds requests */
Sage Weil's avatar
Sage Weil committed
325 326 327 328 329 330 331
	struct delayed_work    delayed_work;  /* delayed work */
	unsigned long    last_renew_caps;  /* last time we renewed our caps */
	struct list_head cap_delay_list;   /* caps with delayed release */
	spinlock_t       cap_delay_lock;   /* protects cap_delay_list */
	struct list_head snap_flush_list;  /* cap_snaps ready to flush */
	spinlock_t       snap_flush_lock;

332
	u64               last_cap_flush_tid;
333
	struct list_head  cap_flush_list;
Sage Weil's avatar
Sage Weil committed
334
	struct list_head  cap_dirty;        /* inodes with dirty caps */
335
	struct list_head  cap_dirty_migrating; /* ...that are migration... */
Sage Weil's avatar
Sage Weil committed
336 337 338 339
	int               num_cap_flushing; /* # caps we are flushing */
	spinlock_t        cap_dirty_lock;   /* protects above items */
	wait_queue_head_t cap_flushing_wq;

340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359
	/*
	 * Cap reservations
	 *
	 * Maintain a global pool of preallocated struct ceph_caps, referenced
	 * by struct ceph_caps_reservations.  This ensures that we preallocate
	 * memory needed to successfully process an MDS response.  (If an MDS
	 * sends us cap information and we fail to process it, we will have
	 * problems due to the client and MDS being out of sync.)
	 *
	 * Reservations are 'owned' by a ceph_cap_reservation context.
	 */
	spinlock_t	caps_list_lock;
	struct		list_head caps_list; /* unused (reserved or
						unreserved) */
	int		caps_total_count;    /* total caps allocated */
	int		caps_use_count;      /* in use */
	int		caps_reserve_count;  /* unused, reserved */
	int		caps_avail_count;    /* unused, unreserved */
	int		caps_min_count;      /* keep at least this many
						(unreserved) */
Sage Weil's avatar
Sage Weil committed
360 361 362
	spinlock_t	  dentry_lru_lock;
	struct list_head  dentry_lru;
	int		  num_dentry;
363 364 365

	struct rw_semaphore     pool_perm_rwsem;
	struct rb_root		pool_perm_tree;
Sage Weil's avatar
Sage Weil committed
366 367 368 369 370 371 372 373 374 375 376 377 378 379
};

extern const char *ceph_mds_op_name(int op);

extern struct ceph_mds_session *
__ceph_lookup_mds_session(struct ceph_mds_client *, int mds);

static inline struct ceph_mds_session *
ceph_get_mds_session(struct ceph_mds_session *s)
{
	atomic_inc(&s->s_ref);
	return s;
}

380 381
extern const char *ceph_session_state_name(int s);

Sage Weil's avatar
Sage Weil committed
382 383 384 385 386
extern void ceph_put_mds_session(struct ceph_mds_session *s);

extern int ceph_send_msg_mds(struct ceph_mds_client *mdsc,
			     struct ceph_msg *msg, int mds);

387
extern int ceph_mdsc_init(struct ceph_fs_client *fsc);
Sage Weil's avatar
Sage Weil committed
388
extern void ceph_mdsc_close_sessions(struct ceph_mds_client *mdsc);
389
extern void ceph_mdsc_force_umount(struct ceph_mds_client *mdsc);
390
extern void ceph_mdsc_destroy(struct ceph_fs_client *fsc);
Sage Weil's avatar
Sage Weil committed
391 392 393

extern void ceph_mdsc_sync(struct ceph_mds_client *mdsc);

394
extern void ceph_invalidate_dir_request(struct ceph_mds_request *req);
395 396
extern int ceph_alloc_readdir_reply_buffer(struct ceph_mds_request *req,
					   struct inode *dir);
Sage Weil's avatar
Sage Weil committed
397 398 399 400 401 402 403 404 405
extern struct ceph_mds_request *
ceph_mdsc_create_request(struct ceph_mds_client *mdsc, int op, int mode);
extern void ceph_mdsc_submit_request(struct ceph_mds_client *mdsc,
				     struct ceph_mds_request *req);
extern int ceph_mdsc_do_request(struct ceph_mds_client *mdsc,
				struct inode *dir,
				struct ceph_mds_request *req);
static inline void ceph_mdsc_get_request(struct ceph_mds_request *req)
{
406 407 408 409 410 411
	kref_get(&req->r_kref);
}
extern void ceph_mdsc_release_request(struct kref *kref);
static inline void ceph_mdsc_put_request(struct ceph_mds_request *req)
{
	kref_put(&req->r_kref, ceph_mdsc_release_request);
Sage Weil's avatar
Sage Weil committed
412 413
}

414 415 416
extern void ceph_send_cap_releases(struct ceph_mds_client *mdsc,
				   struct ceph_mds_session *session);

Sage Weil's avatar
Sage Weil committed
417 418 419 420 421 422 423 424 425 426 427
extern void ceph_mdsc_pre_umount(struct ceph_mds_client *mdsc);

extern char *ceph_mdsc_build_path(struct dentry *dentry, int *plen, u64 *base,
				  int stop_on_nosnap);

extern void __ceph_mdsc_drop_dentry_lease(struct dentry *dentry);
extern void ceph_mdsc_lease_send_msg(struct ceph_mds_session *session,
				     struct inode *inode,
				     struct dentry *dentry, char action,
				     u32 seq);

428 429 430 431
extern void ceph_mdsc_handle_mdsmap(struct ceph_mds_client *mdsc,
				    struct ceph_msg *msg);
extern void ceph_mdsc_handle_fsmap(struct ceph_mds_client *mdsc,
				   struct ceph_msg *msg);
Sage Weil's avatar
Sage Weil committed
432

433 434
extern struct ceph_mds_session *
ceph_mdsc_open_export_target_session(struct ceph_mds_client *mdsc, int target);
435 436 437
extern void ceph_mdsc_open_export_target_sessions(struct ceph_mds_client *mdsc,
					  struct ceph_mds_session *session);

Sage Weil's avatar
Sage Weil committed
438
#endif