fc.c 89.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
/*
 * Copyright (c) 2016 Avago Technologies.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of version 2 of the GNU General Public License as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful.
 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES,
 * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A
 * PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE DISCLAIMED, EXCEPT TO
 * THE EXTENT THAT SUCH DISCLAIMERS ARE HELD TO BE LEGALLY INVALID.
 * See the GNU General Public License for more details, a copy of which
 * can be found in the file COPYING included with this package
 *
 */
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/module.h>
#include <linux/parser.h>
#include <uapi/scsi/fc/fc_fs.h>
#include <uapi/scsi/fc/fc_els.h>
22
#include <linux/delay.h>
23 24 25 26 27 28 29 30 31 32 33

#include "nvme.h"
#include "fabrics.h"
#include <linux/nvme-fc-driver.h>
#include <linux/nvme-fc.h>


/* *************************** Data Structures/Defines ****************** */


enum nvme_fc_queue_flags {
34 35
	NVME_FC_Q_CONNECTED = 0,
	NVME_FC_Q_LIVE,
36 37
};

38 39
#define NVME_FC_DEFAULT_DEV_LOSS_TMO	60	/* seconds */

40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55
struct nvme_fc_queue {
	struct nvme_fc_ctrl	*ctrl;
	struct device		*dev;
	struct blk_mq_hw_ctx	*hctx;
	void			*lldd_handle;
	size_t			cmnd_capsule_len;
	u32			qnum;
	u32			rqcnt;
	u32			seqno;

	u64			connection_id;
	atomic_t		csn;

	unsigned long		flags;
} __aligned(sizeof(u64));	/* alignment for other things alloc'd with */

56 57
enum nvme_fcop_flags {
	FCOP_FLAGS_TERMIO	= (1 << 0),
58
	FCOP_FLAGS_AEN		= (1 << 1),
59 60
};

61 62 63
struct nvmefc_ls_req_op {
	struct nvmefc_ls_req	ls_req;

64
	struct nvme_fc_rport	*rport;
65 66
	struct nvme_fc_queue	*queue;
	struct request		*rq;
67
	u32			flags;
68 69 70

	int			ls_error;
	struct completion	ls_done;
71
	struct list_head	lsreq_list;	/* rport->ls_req_list */
72 73 74 75 76 77 78 79
	bool			req_queued;
};

enum nvme_fcpop_state {
	FCPOP_STATE_UNINIT	= 0,
	FCPOP_STATE_IDLE	= 1,
	FCPOP_STATE_ACTIVE	= 2,
	FCPOP_STATE_ABORTED	= 3,
80
	FCPOP_STATE_COMPLETE	= 4,
81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
};

struct nvme_fc_fcp_op {
	struct nvme_request	nreq;		/*
						 * nvme/host/core.c
						 * requires this to be
						 * the 1st element in the
						 * private structure
						 * associated with the
						 * request.
						 */
	struct nvmefc_fcp_req	fcp_req;

	struct nvme_fc_ctrl	*ctrl;
	struct nvme_fc_queue	*queue;
	struct request		*rq;

	atomic_t		state;
99
	u32			flags;
100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115
	u32			rqno;
	u32			nents;

	struct nvme_fc_cmd_iu	cmd_iu;
	struct nvme_fc_ersp_iu	rsp_iu;
};

struct nvme_fc_lport {
	struct nvme_fc_local_port	localport;

	struct ida			endp_cnt;
	struct list_head		port_list;	/* nvme_fc_port_list */
	struct list_head		endp_list;
	struct device			*dev;	/* physical device for dma */
	struct nvme_fc_port_template	*ops;
	struct kref			ref;
116
	atomic_t                        act_rport_cnt;
117 118 119 120 121 122 123
} __aligned(sizeof(u64));	/* alignment for other things alloc'd with */

struct nvme_fc_rport {
	struct nvme_fc_remote_port	remoteport;

	struct list_head		endp_list; /* for lport->endp_list */
	struct list_head		ctrl_list;
124 125 126
	struct list_head		ls_req_list;
	struct device			*dev;	/* physical device for dma */
	struct nvme_fc_lport		*lport;
127 128
	spinlock_t			lock;
	struct kref			ref;
129
	atomic_t                        act_ctrl_cnt;
130
	unsigned long			dev_loss_end;
131 132
} __aligned(sizeof(u64));	/* alignment for other things alloc'd with */

133 134
enum nvme_fcctrl_flags {
	FCCTRL_TERMIO		= (1 << 0),
135 136 137 138 139 140 141 142 143 144
};

struct nvme_fc_ctrl {
	spinlock_t		lock;
	struct nvme_fc_queue	*queues;
	struct device		*dev;
	struct nvme_fc_lport	*lport;
	struct nvme_fc_rport	*rport;
	u32			cnum;

145
	bool			assoc_active;
146 147 148 149 150 151 152
	u64			association_id;

	struct list_head	ctrl_list;	/* rport->ctrl_list */

	struct blk_mq_tag_set	admin_tag_set;
	struct blk_mq_tag_set	tag_set;

153 154
	struct delayed_work	connect_work;

155
	struct kref		ref;
156 157
	u32			flags;
	u32			iocnt;
158
	wait_queue_head_t	ioabort_wait;
159

160
	struct nvme_fc_fcp_op	aen_ops[NVME_NR_AEN_COMMANDS];
161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207

	struct nvme_ctrl	ctrl;
};

static inline struct nvme_fc_ctrl *
to_fc_ctrl(struct nvme_ctrl *ctrl)
{
	return container_of(ctrl, struct nvme_fc_ctrl, ctrl);
}

static inline struct nvme_fc_lport *
localport_to_lport(struct nvme_fc_local_port *portptr)
{
	return container_of(portptr, struct nvme_fc_lport, localport);
}

static inline struct nvme_fc_rport *
remoteport_to_rport(struct nvme_fc_remote_port *portptr)
{
	return container_of(portptr, struct nvme_fc_rport, remoteport);
}

static inline struct nvmefc_ls_req_op *
ls_req_to_lsop(struct nvmefc_ls_req *lsreq)
{
	return container_of(lsreq, struct nvmefc_ls_req_op, ls_req);
}

static inline struct nvme_fc_fcp_op *
fcp_req_to_fcp_op(struct nvmefc_fcp_req *fcpreq)
{
	return container_of(fcpreq, struct nvme_fc_fcp_op, fcp_req);
}



/* *************************** Globals **************************** */


static DEFINE_SPINLOCK(nvme_fc_lock);

static LIST_HEAD(nvme_fc_lport_list);
static DEFINE_IDA(nvme_fc_local_port_cnt);
static DEFINE_IDA(nvme_fc_ctrl_cnt);



208 209 210 211 212 213 214
/*
 * These items are short-term. They will eventually be moved into
 * a generic FC class. See comments in module init.
 */
static struct class *fc_class;
static struct device *fc_udev_device;

215 216 217 218 219 220

/* *********************** FC-NVME Port Management ************************ */

static void __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *,
			struct nvme_fc_queue *, unsigned int);

221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257
static void
nvme_fc_free_lport(struct kref *ref)
{
	struct nvme_fc_lport *lport =
		container_of(ref, struct nvme_fc_lport, ref);
	unsigned long flags;

	WARN_ON(lport->localport.port_state != FC_OBJSTATE_DELETED);
	WARN_ON(!list_empty(&lport->endp_list));

	/* remove from transport list */
	spin_lock_irqsave(&nvme_fc_lock, flags);
	list_del(&lport->port_list);
	spin_unlock_irqrestore(&nvme_fc_lock, flags);

	ida_simple_remove(&nvme_fc_local_port_cnt, lport->localport.port_num);
	ida_destroy(&lport->endp_cnt);

	put_device(lport->dev);

	kfree(lport);
}

static void
nvme_fc_lport_put(struct nvme_fc_lport *lport)
{
	kref_put(&lport->ref, nvme_fc_free_lport);
}

static int
nvme_fc_lport_get(struct nvme_fc_lport *lport)
{
	return kref_get_unless_zero(&lport->ref);
}


static struct nvme_fc_lport *
258 259 260
nvme_fc_attach_to_unreg_lport(struct nvme_fc_port_info *pinfo,
			struct nvme_fc_port_template *ops,
			struct device *dev)
261 262 263 264 265 266 267 268 269 270 271
{
	struct nvme_fc_lport *lport;
	unsigned long flags;

	spin_lock_irqsave(&nvme_fc_lock, flags);

	list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
		if (lport->localport.node_name != pinfo->node_name ||
		    lport->localport.port_name != pinfo->port_name)
			continue;

272 273 274 275 276
		if (lport->dev != dev) {
			lport = ERR_PTR(-EXDEV);
			goto out_done;
		}

277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292
		if (lport->localport.port_state != FC_OBJSTATE_DELETED) {
			lport = ERR_PTR(-EEXIST);
			goto out_done;
		}

		if (!nvme_fc_lport_get(lport)) {
			/*
			 * fails if ref cnt already 0. If so,
			 * act as if lport already deleted
			 */
			lport = NULL;
			goto out_done;
		}

		/* resume the lport */

293
		lport->ops = ops;
294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309
		lport->localport.port_role = pinfo->port_role;
		lport->localport.port_id = pinfo->port_id;
		lport->localport.port_state = FC_OBJSTATE_ONLINE;

		spin_unlock_irqrestore(&nvme_fc_lock, flags);

		return lport;
	}

	lport = NULL;

out_done:
	spin_unlock_irqrestore(&nvme_fc_lock, flags);

	return lport;
}
310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346

/**
 * nvme_fc_register_localport - transport entry point called by an
 *                              LLDD to register the existence of a NVME
 *                              host FC port.
 * @pinfo:     pointer to information about the port to be registered
 * @template:  LLDD entrypoints and operational parameters for the port
 * @dev:       physical hardware device node port corresponds to. Will be
 *             used for DMA mappings
 * @lport_p:   pointer to a local port pointer. Upon success, the routine
 *             will allocate a nvme_fc_local_port structure and place its
 *             address in the local port pointer. Upon failure, local port
 *             pointer will be set to 0.
 *
 * Returns:
 * a completion status. Must be 0 upon success; a negative errno
 * (ex: -ENXIO) upon failure.
 */
int
nvme_fc_register_localport(struct nvme_fc_port_info *pinfo,
			struct nvme_fc_port_template *template,
			struct device *dev,
			struct nvme_fc_local_port **portptr)
{
	struct nvme_fc_lport *newrec;
	unsigned long flags;
	int ret, idx;

	if (!template->localport_delete || !template->remoteport_delete ||
	    !template->ls_req || !template->fcp_io ||
	    !template->ls_abort || !template->fcp_abort ||
	    !template->max_hw_queues || !template->max_sgl_segments ||
	    !template->max_dif_sgl_segments || !template->dma_boundary) {
		ret = -EINVAL;
		goto out_reghost_failed;
	}

347 348 349 350 351 352 353
	/*
	 * look to see if there is already a localport that had been
	 * deregistered and in the process of waiting for all the
	 * references to fully be removed.  If the references haven't
	 * expired, we can simply re-enable the localport. Remoteports
	 * and controller reconnections should resume naturally.
	 */
354
	newrec = nvme_fc_attach_to_unreg_lport(pinfo, template, dev);
355 356 357 358 359 360 361 362 363 364 365 366 367 368

	/* found an lport, but something about its state is bad */
	if (IS_ERR(newrec)) {
		ret = PTR_ERR(newrec);
		goto out_reghost_failed;

	/* found existing lport, which was resumed */
	} else if (newrec) {
		*portptr = &newrec->localport;
		return 0;
	}

	/* nothing found - allocate a new localport struct */

369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389
	newrec = kmalloc((sizeof(*newrec) + template->local_priv_sz),
			 GFP_KERNEL);
	if (!newrec) {
		ret = -ENOMEM;
		goto out_reghost_failed;
	}

	idx = ida_simple_get(&nvme_fc_local_port_cnt, 0, 0, GFP_KERNEL);
	if (idx < 0) {
		ret = -ENOSPC;
		goto out_fail_kfree;
	}

	if (!get_device(dev) && dev) {
		ret = -ENODEV;
		goto out_ida_put;
	}

	INIT_LIST_HEAD(&newrec->port_list);
	INIT_LIST_HEAD(&newrec->endp_list);
	kref_init(&newrec->ref);
390
	atomic_set(&newrec->act_rport_cnt, 0);
391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452
	newrec->ops = template;
	newrec->dev = dev;
	ida_init(&newrec->endp_cnt);
	newrec->localport.private = &newrec[1];
	newrec->localport.node_name = pinfo->node_name;
	newrec->localport.port_name = pinfo->port_name;
	newrec->localport.port_role = pinfo->port_role;
	newrec->localport.port_id = pinfo->port_id;
	newrec->localport.port_state = FC_OBJSTATE_ONLINE;
	newrec->localport.port_num = idx;

	spin_lock_irqsave(&nvme_fc_lock, flags);
	list_add_tail(&newrec->port_list, &nvme_fc_lport_list);
	spin_unlock_irqrestore(&nvme_fc_lock, flags);

	if (dev)
		dma_set_seg_boundary(dev, template->dma_boundary);

	*portptr = &newrec->localport;
	return 0;

out_ida_put:
	ida_simple_remove(&nvme_fc_local_port_cnt, idx);
out_fail_kfree:
	kfree(newrec);
out_reghost_failed:
	*portptr = NULL;

	return ret;
}
EXPORT_SYMBOL_GPL(nvme_fc_register_localport);

/**
 * nvme_fc_unregister_localport - transport entry point called by an
 *                              LLDD to deregister/remove a previously
 *                              registered a NVME host FC port.
 * @localport: pointer to the (registered) local port that is to be
 *             deregistered.
 *
 * Returns:
 * a completion status. Must be 0 upon success; a negative errno
 * (ex: -ENXIO) upon failure.
 */
int
nvme_fc_unregister_localport(struct nvme_fc_local_port *portptr)
{
	struct nvme_fc_lport *lport = localport_to_lport(portptr);
	unsigned long flags;

	if (!portptr)
		return -EINVAL;

	spin_lock_irqsave(&nvme_fc_lock, flags);

	if (portptr->port_state != FC_OBJSTATE_ONLINE) {
		spin_unlock_irqrestore(&nvme_fc_lock, flags);
		return -EINVAL;
	}
	portptr->port_state = FC_OBJSTATE_DELETED;

	spin_unlock_irqrestore(&nvme_fc_lock, flags);

453 454 455
	if (atomic_read(&lport->act_rport_cnt) == 0)
		lport->ops->localport_delete(&lport->localport);

456 457 458 459 460 461
	nvme_fc_lport_put(lport);

	return 0;
}
EXPORT_SYMBOL_GPL(nvme_fc_unregister_localport);

462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491
/*
 * TRADDR strings, per FC-NVME are fixed format:
 *   "nn-0x<16hexdigits>:pn-0x<16hexdigits>" - 43 characters
 * udev event will only differ by prefix of what field is
 * being specified:
 *    "NVMEFC_HOST_TRADDR=" or "NVMEFC_TRADDR=" - 19 max characters
 *  19 + 43 + null_fudge = 64 characters
 */
#define FCNVME_TRADDR_LENGTH		64

static void
nvme_fc_signal_discovery_scan(struct nvme_fc_lport *lport,
		struct nvme_fc_rport *rport)
{
	char hostaddr[FCNVME_TRADDR_LENGTH];	/* NVMEFC_HOST_TRADDR=...*/
	char tgtaddr[FCNVME_TRADDR_LENGTH];	/* NVMEFC_TRADDR=...*/
	char *envp[4] = { "FC_EVENT=nvmediscovery", hostaddr, tgtaddr, NULL };

	if (!(rport->remoteport.port_role & FC_PORT_ROLE_NVME_DISCOVERY))
		return;

	snprintf(hostaddr, sizeof(hostaddr),
		"NVMEFC_HOST_TRADDR=nn-0x%016llx:pn-0x%016llx",
		lport->localport.node_name, lport->localport.port_name);
	snprintf(tgtaddr, sizeof(tgtaddr),
		"NVMEFC_TRADDR=nn-0x%016llx:pn-0x%016llx",
		rport->remoteport.node_name, rport->remoteport.port_name);
	kobject_uevent_env(&fc_udev_device->kobj, KOBJ_CHANGE, envp);
}

492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527
static void
nvme_fc_free_rport(struct kref *ref)
{
	struct nvme_fc_rport *rport =
		container_of(ref, struct nvme_fc_rport, ref);
	struct nvme_fc_lport *lport =
			localport_to_lport(rport->remoteport.localport);
	unsigned long flags;

	WARN_ON(rport->remoteport.port_state != FC_OBJSTATE_DELETED);
	WARN_ON(!list_empty(&rport->ctrl_list));

	/* remove from lport list */
	spin_lock_irqsave(&nvme_fc_lock, flags);
	list_del(&rport->endp_list);
	spin_unlock_irqrestore(&nvme_fc_lock, flags);

	ida_simple_remove(&lport->endp_cnt, rport->remoteport.port_num);

	kfree(rport);

	nvme_fc_lport_put(lport);
}

static void
nvme_fc_rport_put(struct nvme_fc_rport *rport)
{
	kref_put(&rport->ref, nvme_fc_free_rport);
}

static int
nvme_fc_rport_get(struct nvme_fc_rport *rport)
{
	return kref_get_unless_zero(&rport->ref);
}

528 529 530 531 532
static void
nvme_fc_resume_controller(struct nvme_fc_ctrl *ctrl)
{
	switch (ctrl->ctrl.state) {
	case NVME_CTRL_NEW:
533
	case NVME_CTRL_CONNECTING:
534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623
		/*
		 * As all reconnects were suppressed, schedule a
		 * connect.
		 */
		dev_info(ctrl->ctrl.device,
			"NVME-FC{%d}: connectivity re-established. "
			"Attempting reconnect\n", ctrl->cnum);

		queue_delayed_work(nvme_wq, &ctrl->connect_work, 0);
		break;

	case NVME_CTRL_RESETTING:
		/*
		 * Controller is already in the process of terminating the
		 * association. No need to do anything further. The reconnect
		 * step will naturally occur after the reset completes.
		 */
		break;

	default:
		/* no action to take - let it delete */
		break;
	}
}

static struct nvme_fc_rport *
nvme_fc_attach_to_suspended_rport(struct nvme_fc_lport *lport,
				struct nvme_fc_port_info *pinfo)
{
	struct nvme_fc_rport *rport;
	struct nvme_fc_ctrl *ctrl;
	unsigned long flags;

	spin_lock_irqsave(&nvme_fc_lock, flags);

	list_for_each_entry(rport, &lport->endp_list, endp_list) {
		if (rport->remoteport.node_name != pinfo->node_name ||
		    rport->remoteport.port_name != pinfo->port_name)
			continue;

		if (!nvme_fc_rport_get(rport)) {
			rport = ERR_PTR(-ENOLCK);
			goto out_done;
		}

		spin_unlock_irqrestore(&nvme_fc_lock, flags);

		spin_lock_irqsave(&rport->lock, flags);

		/* has it been unregistered */
		if (rport->remoteport.port_state != FC_OBJSTATE_DELETED) {
			/* means lldd called us twice */
			spin_unlock_irqrestore(&rport->lock, flags);
			nvme_fc_rport_put(rport);
			return ERR_PTR(-ESTALE);
		}

		rport->remoteport.port_state = FC_OBJSTATE_ONLINE;
		rport->dev_loss_end = 0;

		/*
		 * kick off a reconnect attempt on all associations to the
		 * remote port. A successful reconnects will resume i/o.
		 */
		list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list)
			nvme_fc_resume_controller(ctrl);

		spin_unlock_irqrestore(&rport->lock, flags);

		return rport;
	}

	rport = NULL;

out_done:
	spin_unlock_irqrestore(&nvme_fc_lock, flags);

	return rport;
}

static inline void
__nvme_fc_set_dev_loss_tmo(struct nvme_fc_rport *rport,
			struct nvme_fc_port_info *pinfo)
{
	if (pinfo->dev_loss_tmo)
		rport->remoteport.dev_loss_tmo = pinfo->dev_loss_tmo;
	else
		rport->remoteport.dev_loss_tmo = NVME_FC_DEFAULT_DEV_LOSS_TMO;
}

624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649
/**
 * nvme_fc_register_remoteport - transport entry point called by an
 *                              LLDD to register the existence of a NVME
 *                              subsystem FC port on its fabric.
 * @localport: pointer to the (registered) local port that the remote
 *             subsystem port is connected to.
 * @pinfo:     pointer to information about the port to be registered
 * @rport_p:   pointer to a remote port pointer. Upon success, the routine
 *             will allocate a nvme_fc_remote_port structure and place its
 *             address in the remote port pointer. Upon failure, remote port
 *             pointer will be set to 0.
 *
 * Returns:
 * a completion status. Must be 0 upon success; a negative errno
 * (ex: -ENXIO) upon failure.
 */
int
nvme_fc_register_remoteport(struct nvme_fc_local_port *localport,
				struct nvme_fc_port_info *pinfo,
				struct nvme_fc_remote_port **portptr)
{
	struct nvme_fc_lport *lport = localport_to_lport(localport);
	struct nvme_fc_rport *newrec;
	unsigned long flags;
	int ret, idx;

650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677
	if (!nvme_fc_lport_get(lport)) {
		ret = -ESHUTDOWN;
		goto out_reghost_failed;
	}

	/*
	 * look to see if there is already a remoteport that is waiting
	 * for a reconnect (within dev_loss_tmo) with the same WWN's.
	 * If so, transition to it and reconnect.
	 */
	newrec = nvme_fc_attach_to_suspended_rport(lport, pinfo);

	/* found an rport, but something about its state is bad */
	if (IS_ERR(newrec)) {
		ret = PTR_ERR(newrec);
		goto out_lport_put;

	/* found existing rport, which was resumed */
	} else if (newrec) {
		nvme_fc_lport_put(lport);
		__nvme_fc_set_dev_loss_tmo(newrec, pinfo);
		nvme_fc_signal_discovery_scan(lport, newrec);
		*portptr = &newrec->remoteport;
		return 0;
	}

	/* nothing found - allocate a new remoteport struct */

678 679 680 681
	newrec = kmalloc((sizeof(*newrec) + lport->ops->remote_priv_sz),
			 GFP_KERNEL);
	if (!newrec) {
		ret = -ENOMEM;
682
		goto out_lport_put;
683 684 685 686 687
	}

	idx = ida_simple_get(&lport->endp_cnt, 0, 0, GFP_KERNEL);
	if (idx < 0) {
		ret = -ENOSPC;
688
		goto out_kfree_rport;
689 690 691 692
	}

	INIT_LIST_HEAD(&newrec->endp_list);
	INIT_LIST_HEAD(&newrec->ctrl_list);
693
	INIT_LIST_HEAD(&newrec->ls_req_list);
694
	kref_init(&newrec->ref);
695
	atomic_set(&newrec->act_ctrl_cnt, 0);
696 697
	spin_lock_init(&newrec->lock);
	newrec->remoteport.localport = &lport->localport;
698 699
	newrec->dev = lport->dev;
	newrec->lport = lport;
700 701 702 703 704 705 706
	newrec->remoteport.private = &newrec[1];
	newrec->remoteport.port_role = pinfo->port_role;
	newrec->remoteport.node_name = pinfo->node_name;
	newrec->remoteport.port_name = pinfo->port_name;
	newrec->remoteport.port_id = pinfo->port_id;
	newrec->remoteport.port_state = FC_OBJSTATE_ONLINE;
	newrec->remoteport.port_num = idx;
707
	__nvme_fc_set_dev_loss_tmo(newrec, pinfo);
708 709 710 711 712

	spin_lock_irqsave(&nvme_fc_lock, flags);
	list_add_tail(&newrec->endp_list, &lport->endp_list);
	spin_unlock_irqrestore(&nvme_fc_lock, flags);

713 714
	nvme_fc_signal_discovery_scan(lport, newrec);

715 716 717 718 719
	*portptr = &newrec->remoteport;
	return 0;

out_kfree_rport:
	kfree(newrec);
720 721
out_lport_put:
	nvme_fc_lport_put(lport);
722 723 724 725 726 727
out_reghost_failed:
	*portptr = NULL;
	return ret;
}
EXPORT_SYMBOL_GPL(nvme_fc_register_remoteport);

728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751
static int
nvme_fc_abort_lsops(struct nvme_fc_rport *rport)
{
	struct nvmefc_ls_req_op *lsop;
	unsigned long flags;

restart:
	spin_lock_irqsave(&rport->lock, flags);

	list_for_each_entry(lsop, &rport->ls_req_list, lsreq_list) {
		if (!(lsop->flags & FCOP_FLAGS_TERMIO)) {
			lsop->flags |= FCOP_FLAGS_TERMIO;
			spin_unlock_irqrestore(&rport->lock, flags);
			rport->lport->ops->ls_abort(&rport->lport->localport,
						&rport->remoteport,
						&lsop->ls_req);
			goto restart;
		}
	}
	spin_unlock_irqrestore(&rport->lock, flags);

	return 0;
}

752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777
static void
nvme_fc_ctrl_connectivity_loss(struct nvme_fc_ctrl *ctrl)
{
	dev_info(ctrl->ctrl.device,
		"NVME-FC{%d}: controller connectivity lost. Awaiting "
		"Reconnect", ctrl->cnum);

	switch (ctrl->ctrl.state) {
	case NVME_CTRL_NEW:
	case NVME_CTRL_LIVE:
		/*
		 * Schedule a controller reset. The reset will terminate the
		 * association and schedule the reconnect timer.  Reconnects
		 * will be attempted until either the ctlr_loss_tmo
		 * (max_retries * connect_delay) expires or the remoteport's
		 * dev_loss_tmo expires.
		 */
		if (nvme_reset_ctrl(&ctrl->ctrl)) {
			dev_warn(ctrl->ctrl.device,
				"NVME-FC{%d}: Couldn't schedule reset. "
				"Deleting controller.\n",
				ctrl->cnum);
			nvme_delete_ctrl(&ctrl->ctrl);
		}
		break;

778
	case NVME_CTRL_CONNECTING:
779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803
		/*
		 * The association has already been terminated and the
		 * controller is attempting reconnects.  No need to do anything
		 * futher.  Reconnects will be attempted until either the
		 * ctlr_loss_tmo (max_retries * connect_delay) expires or the
		 * remoteport's dev_loss_tmo expires.
		 */
		break;

	case NVME_CTRL_RESETTING:
		/*
		 * Controller is already in the process of terminating the
		 * association.  No need to do anything further. The reconnect
		 * step will kick in naturally after the association is
		 * terminated.
		 */
		break;

	case NVME_CTRL_DELETING:
	default:
		/* no action to take - let it delete */
		break;
	}
}

804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832
/**
 * nvme_fc_unregister_remoteport - transport entry point called by an
 *                              LLDD to deregister/remove a previously
 *                              registered a NVME subsystem FC port.
 * @remoteport: pointer to the (registered) remote port that is to be
 *              deregistered.
 *
 * Returns:
 * a completion status. Must be 0 upon success; a negative errno
 * (ex: -ENXIO) upon failure.
 */
int
nvme_fc_unregister_remoteport(struct nvme_fc_remote_port *portptr)
{
	struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
	struct nvme_fc_ctrl *ctrl;
	unsigned long flags;

	if (!portptr)
		return -EINVAL;

	spin_lock_irqsave(&rport->lock, flags);

	if (portptr->port_state != FC_OBJSTATE_ONLINE) {
		spin_unlock_irqrestore(&rport->lock, flags);
		return -EINVAL;
	}
	portptr->port_state = FC_OBJSTATE_DELETED;

833 834 835 836 837 838 839 840 841 842 843 844 845
	rport->dev_loss_end = jiffies + (portptr->dev_loss_tmo * HZ);

	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
		/* if dev_loss_tmo==0, dev loss is immediate */
		if (!portptr->dev_loss_tmo) {
			dev_warn(ctrl->ctrl.device,
				"NVME-FC{%d}: controller connectivity lost. "
				"Deleting controller.\n",
				ctrl->cnum);
			nvme_delete_ctrl(&ctrl->ctrl);
		} else
			nvme_fc_ctrl_connectivity_loss(ctrl);
	}
846 847 848

	spin_unlock_irqrestore(&rport->lock, flags);

849 850
	nvme_fc_abort_lsops(rport);

851 852 853
	if (atomic_read(&rport->act_ctrl_cnt) == 0)
		rport->lport->ops->remoteport_delete(portptr);

854 855 856 857 858
	/*
	 * release the reference, which will allow, if all controllers
	 * go away, which should only occur after dev_loss_tmo occurs,
	 * for the rport to be torn down.
	 */
859
	nvme_fc_rport_put(rport);
860

861 862 863 864
	return 0;
}
EXPORT_SYMBOL_GPL(nvme_fc_unregister_remoteport);

865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881
/**
 * nvme_fc_rescan_remoteport - transport entry point called by an
 *                              LLDD to request a nvme device rescan.
 * @remoteport: pointer to the (registered) remote port that is to be
 *              rescanned.
 *
 * Returns: N/A
 */
void
nvme_fc_rescan_remoteport(struct nvme_fc_remote_port *remoteport)
{
	struct nvme_fc_rport *rport = remoteport_to_rport(remoteport);

	nvme_fc_signal_discovery_scan(rport->lport, rport);
}
EXPORT_SYMBOL_GPL(nvme_fc_rescan_remoteport);

882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904
int
nvme_fc_set_remoteport_devloss(struct nvme_fc_remote_port *portptr,
			u32 dev_loss_tmo)
{
	struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
	unsigned long flags;

	spin_lock_irqsave(&rport->lock, flags);

	if (portptr->port_state != FC_OBJSTATE_ONLINE) {
		spin_unlock_irqrestore(&rport->lock, flags);
		return -EINVAL;
	}

	/* a dev_loss_tmo of 0 (immediate) is allowed to be set */
	rport->remoteport.dev_loss_tmo = dev_loss_tmo;

	spin_unlock_irqrestore(&rport->lock, flags);

	return 0;
}
EXPORT_SYMBOL_GPL(nvme_fc_set_remoteport_devloss);

905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000

/* *********************** FC-NVME DMA Handling **************************** */

/*
 * The fcloop device passes in a NULL device pointer. Real LLD's will
 * pass in a valid device pointer. If NULL is passed to the dma mapping
 * routines, depending on the platform, it may or may not succeed, and
 * may crash.
 *
 * As such:
 * Wrapper all the dma routines and check the dev pointer.
 *
 * If simple mappings (return just a dma address, we'll noop them,
 * returning a dma address of 0.
 *
 * On more complex mappings (dma_map_sg), a pseudo routine fills
 * in the scatter list, setting all dma addresses to 0.
 */

static inline dma_addr_t
fc_dma_map_single(struct device *dev, void *ptr, size_t size,
		enum dma_data_direction dir)
{
	return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
}

static inline int
fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
{
	return dev ? dma_mapping_error(dev, dma_addr) : 0;
}

static inline void
fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
	enum dma_data_direction dir)
{
	if (dev)
		dma_unmap_single(dev, addr, size, dir);
}

static inline void
fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
		enum dma_data_direction dir)
{
	if (dev)
		dma_sync_single_for_cpu(dev, addr, size, dir);
}

static inline void
fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
		enum dma_data_direction dir)
{
	if (dev)
		dma_sync_single_for_device(dev, addr, size, dir);
}

/* pseudo dma_map_sg call */
static int
fc_map_sg(struct scatterlist *sg, int nents)
{
	struct scatterlist *s;
	int i;

	WARN_ON(nents == 0 || sg[0].length == 0);

	for_each_sg(sg, s, nents, i) {
		s->dma_address = 0L;
#ifdef CONFIG_NEED_SG_DMA_LENGTH
		s->dma_length = s->length;
#endif
	}
	return nents;
}

static inline int
fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
		enum dma_data_direction dir)
{
	return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
}

static inline void
fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
		enum dma_data_direction dir)
{
	if (dev)
		dma_unmap_sg(dev, sg, nents, dir);
}

/* *********************** FC-NVME LS Handling **************************** */

static void nvme_fc_ctrl_put(struct nvme_fc_ctrl *);
static int nvme_fc_ctrl_get(struct nvme_fc_ctrl *);


static void
1001
__nvme_fc_finish_ls_req(struct nvmefc_ls_req_op *lsop)
1002
{
1003
	struct nvme_fc_rport *rport = lsop->rport;
1004 1005 1006
	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
	unsigned long flags;

1007
	spin_lock_irqsave(&rport->lock, flags);
1008 1009

	if (!lsop->req_queued) {
1010
		spin_unlock_irqrestore(&rport->lock, flags);
1011 1012 1013 1014 1015 1016 1017
		return;
	}

	list_del(&lsop->lsreq_list);

	lsop->req_queued = false;

1018
	spin_unlock_irqrestore(&rport->lock, flags);
1019

1020
	fc_dma_unmap_single(rport->dev, lsreq->rqstdma,
1021 1022 1023
				  (lsreq->rqstlen + lsreq->rsplen),
				  DMA_BIDIRECTIONAL);

1024
	nvme_fc_rport_put(rport);
1025 1026 1027
}

static int
1028
__nvme_fc_send_ls_req(struct nvme_fc_rport *rport,
1029 1030 1031 1032 1033
		struct nvmefc_ls_req_op *lsop,
		void (*done)(struct nvmefc_ls_req *req, int status))
{
	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
	unsigned long flags;
1034
	int ret = 0;
1035

1036 1037 1038 1039
	if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
		return -ECONNREFUSED;

	if (!nvme_fc_rport_get(rport))
1040 1041 1042
		return -ESHUTDOWN;

	lsreq->done = done;
1043
	lsop->rport = rport;
1044 1045 1046 1047
	lsop->req_queued = false;
	INIT_LIST_HEAD(&lsop->lsreq_list);
	init_completion(&lsop->ls_done);

1048
	lsreq->rqstdma = fc_dma_map_single(rport->dev, lsreq->rqstaddr,
1049 1050
				  lsreq->rqstlen + lsreq->rsplen,
				  DMA_BIDIRECTIONAL);
1051 1052 1053
	if (fc_dma_mapping_error(rport->dev, lsreq->rqstdma)) {
		ret = -EFAULT;
		goto out_putrport;
1054 1055 1056
	}
	lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen;

1057
	spin_lock_irqsave(&rport->lock, flags);
1058

1059
	list_add_tail(&lsop->lsreq_list, &rport->ls_req_list);
1060 1061 1062

	lsop->req_queued = true;

1063
	spin_unlock_irqrestore(&rport->lock, flags);
1064

1065 1066
	ret = rport->lport->ops->ls_req(&rport->lport->localport,
					&rport->remoteport, lsreq);
1067
	if (ret)
1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082
		goto out_unlink;

	return 0;

out_unlink:
	lsop->ls_error = ret;
	spin_lock_irqsave(&rport->lock, flags);
	lsop->req_queued = false;
	list_del(&lsop->lsreq_list);
	spin_unlock_irqrestore(&rport->lock, flags);
	fc_dma_unmap_single(rport->dev, lsreq->rqstdma,
				  (lsreq->rqstlen + lsreq->rsplen),
				  DMA_BIDIRECTIONAL);
out_putrport:
	nvme_fc_rport_put(rport);
1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096

	return ret;
}

static void
nvme_fc_send_ls_req_done(struct nvmefc_ls_req *lsreq, int status)
{
	struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);

	lsop->ls_error = status;
	complete(&lsop->ls_done);
}

static int
1097
nvme_fc_send_ls_req(struct nvme_fc_rport *rport, struct nvmefc_ls_req_op *lsop)
1098 1099 1100 1101 1102
{
	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
	struct fcnvme_ls_rjt *rjt = lsreq->rspaddr;
	int ret;

1103
	ret = __nvme_fc_send_ls_req(rport, lsop, nvme_fc_send_ls_req_done);
1104

1105
	if (!ret) {
1106 1107 1108 1109 1110 1111 1112 1113
		/*
		 * No timeout/not interruptible as we need the struct
		 * to exist until the lldd calls us back. Thus mandate
		 * wait until driver calls back. lldd responsible for
		 * the timeout action
		 */
		wait_for_completion(&lsop->ls_done);

1114
		__nvme_fc_finish_ls_req(lsop);
1115

1116
		ret = lsop->ls_error;
1117 1118
	}

1119 1120 1121
	if (ret)
		return ret;

1122 1123 1124 1125 1126 1127 1128
	/* ACC or RJT payload ? */
	if (rjt->w0.ls_cmd == FCNVME_LS_RJT)
		return -ENXIO;

	return 0;
}

1129 1130
static int
nvme_fc_send_ls_req_async(struct nvme_fc_rport *rport,
1131 1132 1133 1134 1135
		struct nvmefc_ls_req_op *lsop,
		void (*done)(struct nvmefc_ls_req *req, int status))
{
	/* don't wait for completion */

1136
	return __nvme_fc_send_ls_req(rport, lsop, done);
1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208
}

/* Validation Error indexes into the string table below */
enum {
	VERR_NO_ERROR		= 0,
	VERR_LSACC		= 1,
	VERR_LSDESC_RQST	= 2,
	VERR_LSDESC_RQST_LEN	= 3,
	VERR_ASSOC_ID		= 4,
	VERR_ASSOC_ID_LEN	= 5,
	VERR_CONN_ID		= 6,
	VERR_CONN_ID_LEN	= 7,
	VERR_CR_ASSOC		= 8,
	VERR_CR_ASSOC_ACC_LEN	= 9,
	VERR_CR_CONN		= 10,
	VERR_CR_CONN_ACC_LEN	= 11,
	VERR_DISCONN		= 12,
	VERR_DISCONN_ACC_LEN	= 13,
};

static char *validation_errors[] = {
	"OK",
	"Not LS_ACC",
	"Not LSDESC_RQST",
	"Bad LSDESC_RQST Length",
	"Not Association ID",
	"Bad Association ID Length",
	"Not Connection ID",
	"Bad Connection ID Length",
	"Not CR_ASSOC Rqst",
	"Bad CR_ASSOC ACC Length",
	"Not CR_CONN Rqst",
	"Bad CR_CONN ACC Length",
	"Not Disconnect Rqst",
	"Bad Disconnect ACC Length",
};

static int
nvme_fc_connect_admin_queue(struct nvme_fc_ctrl *ctrl,
	struct nvme_fc_queue *queue, u16 qsize, u16 ersp_ratio)
{
	struct nvmefc_ls_req_op *lsop;
	struct nvmefc_ls_req *lsreq;
	struct fcnvme_ls_cr_assoc_rqst *assoc_rqst;
	struct fcnvme_ls_cr_assoc_acc *assoc_acc;
	int ret, fcret = 0;

	lsop = kzalloc((sizeof(*lsop) +
			 ctrl->lport->ops->lsrqst_priv_sz +
			 sizeof(*assoc_rqst) + sizeof(*assoc_acc)), GFP_KERNEL);
	if (!lsop) {
		ret = -ENOMEM;
		goto out_no_memory;
	}
	lsreq = &lsop->ls_req;

	lsreq->private = (void *)&lsop[1];
	assoc_rqst = (struct fcnvme_ls_cr_assoc_rqst *)
			(lsreq->private + ctrl->lport->ops->lsrqst_priv_sz);
	assoc_acc = (struct fcnvme_ls_cr_assoc_acc *)&assoc_rqst[1];

	assoc_rqst->w0.ls_cmd = FCNVME_LS_CREATE_ASSOCIATION;
	assoc_rqst->desc_list_len =
			cpu_to_be32(sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));

	assoc_rqst->assoc_cmd.desc_tag =
			cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD);
	assoc_rqst->assoc_cmd.desc_len =
			fcnvme_lsdesc_len(
				sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));

	assoc_rqst->assoc_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
1209
	assoc_rqst->assoc_cmd.sqsize = cpu_to_be16(qsize - 1);
1210 1211
	/* Linux supports only Dynamic controllers */
	assoc_rqst->assoc_cmd.cntlid = cpu_to_be16(0xffff);
1212
	uuid_copy(&assoc_rqst->assoc_cmd.hostid, &ctrl->ctrl.opts->host->id);
1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224
	strncpy(assoc_rqst->assoc_cmd.hostnqn, ctrl->ctrl.opts->host->nqn,
		min(FCNVME_ASSOC_HOSTNQN_LEN, NVMF_NQN_SIZE));
	strncpy(assoc_rqst->assoc_cmd.subnqn, ctrl->ctrl.opts->subsysnqn,
		min(FCNVME_ASSOC_SUBNQN_LEN, NVMF_NQN_SIZE));

	lsop->queue = queue;
	lsreq->rqstaddr = assoc_rqst;
	lsreq->rqstlen = sizeof(*assoc_rqst);
	lsreq->rspaddr = assoc_acc;
	lsreq->rsplen = sizeof(*assoc_acc);
	lsreq->timeout = NVME_FC_CONNECT_TIMEOUT_SEC;

1225
	ret = nvme_fc_send_ls_req(ctrl->rport, lsop);
1226 1227 1228 1229 1230 1231 1232 1233
	if (ret)
		goto out_free_buffer;

	/* process connect LS completion */

	/* validate the ACC response */
	if (assoc_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
		fcret = VERR_LSACC;
1234
	else if (assoc_acc->hdr.desc_list_len !=
1235 1236 1237
			fcnvme_lsdesc_len(
				sizeof(struct fcnvme_ls_cr_assoc_acc)))
		fcret = VERR_CR_ASSOC_ACC_LEN;
1238 1239
	else if (assoc_acc->hdr.rqst.desc_tag !=
			cpu_to_be32(FCNVME_LSDESC_RQST))
1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323
		fcret = VERR_LSDESC_RQST;
	else if (assoc_acc->hdr.rqst.desc_len !=
			fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)))
		fcret = VERR_LSDESC_RQST_LEN;
	else if (assoc_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_ASSOCIATION)
		fcret = VERR_CR_ASSOC;
	else if (assoc_acc->associd.desc_tag !=
			cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
		fcret = VERR_ASSOC_ID;
	else if (assoc_acc->associd.desc_len !=
			fcnvme_lsdesc_len(
				sizeof(struct fcnvme_lsdesc_assoc_id)))
		fcret = VERR_ASSOC_ID_LEN;
	else if (assoc_acc->connectid.desc_tag !=
			cpu_to_be32(FCNVME_LSDESC_CONN_ID))
		fcret = VERR_CONN_ID;
	else if (assoc_acc->connectid.desc_len !=
			fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id)))
		fcret = VERR_CONN_ID_LEN;

	if (fcret) {
		ret = -EBADF;
		dev_err(ctrl->dev,
			"q %d connect failed: %s\n",
			queue->qnum, validation_errors[fcret]);
	} else {
		ctrl->association_id =
			be64_to_cpu(assoc_acc->associd.association_id);
		queue->connection_id =
			be64_to_cpu(assoc_acc->connectid.connection_id);
		set_bit(NVME_FC_Q_CONNECTED, &queue->flags);
	}

out_free_buffer:
	kfree(lsop);
out_no_memory:
	if (ret)
		dev_err(ctrl->dev,
			"queue %d connect admin queue failed (%d).\n",
			queue->qnum, ret);
	return ret;
}

static int
nvme_fc_connect_queue(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
			u16 qsize, u16 ersp_ratio)
{
	struct nvmefc_ls_req_op *lsop;
	struct nvmefc_ls_req *lsreq;
	struct fcnvme_ls_cr_conn_rqst *conn_rqst;
	struct fcnvme_ls_cr_conn_acc *conn_acc;
	int ret, fcret = 0;

	lsop = kzalloc((sizeof(*lsop) +
			 ctrl->lport->ops->lsrqst_priv_sz +
			 sizeof(*conn_rqst) + sizeof(*conn_acc)), GFP_KERNEL);
	if (!lsop) {
		ret = -ENOMEM;
		goto out_no_memory;
	}
	lsreq = &lsop->ls_req;

	lsreq->private = (void *)&lsop[1];
	conn_rqst = (struct fcnvme_ls_cr_conn_rqst *)
			(lsreq->private + ctrl->lport->ops->lsrqst_priv_sz);
	conn_acc = (struct fcnvme_ls_cr_conn_acc *)&conn_rqst[1];

	conn_rqst->w0.ls_cmd = FCNVME_LS_CREATE_CONNECTION;
	conn_rqst->desc_list_len = cpu_to_be32(
				sizeof(struct fcnvme_lsdesc_assoc_id) +
				sizeof(struct fcnvme_lsdesc_cr_conn_cmd));

	conn_rqst->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
	conn_rqst->associd.desc_len =
			fcnvme_lsdesc_len(
				sizeof(struct fcnvme_lsdesc_assoc_id));
	conn_rqst->associd.association_id = cpu_to_be64(ctrl->association_id);
	conn_rqst->connect_cmd.desc_tag =
			cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD);
	conn_rqst->connect_cmd.desc_len =
			fcnvme_lsdesc_len(
				sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
	conn_rqst->connect_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
	conn_rqst->connect_cmd.qid  = cpu_to_be16(queue->qnum);
1324
	conn_rqst->connect_cmd.sqsize = cpu_to_be16(qsize - 1);
1325 1326 1327 1328 1329 1330 1331 1332

	lsop->queue = queue;
	lsreq->rqstaddr = conn_rqst;
	lsreq->rqstlen = sizeof(*conn_rqst);
	lsreq->rspaddr = conn_acc;
	lsreq->rsplen = sizeof(*conn_acc);
	lsreq->timeout = NVME_FC_CONNECT_TIMEOUT_SEC;

1333
	ret = nvme_fc_send_ls_req(ctrl->rport, lsop);
1334 1335 1336 1337 1338 1339 1340 1341
	if (ret)
		goto out_free_buffer;

	/* process connect LS completion */

	/* validate the ACC response */
	if (conn_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
		fcret = VERR_LSACC;
1342
	else if (conn_acc->hdr.desc_list_len !=
1343 1344
			fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)))
		fcret = VERR_CR_CONN_ACC_LEN;
1345
	else if (conn_acc->hdr.rqst.desc_tag != cpu_to_be32(FCNVME_LSDESC_RQST))
1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384
		fcret = VERR_LSDESC_RQST;
	else if (conn_acc->hdr.rqst.desc_len !=
			fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)))
		fcret = VERR_LSDESC_RQST_LEN;
	else if (conn_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_CONNECTION)
		fcret = VERR_CR_CONN;
	else if (conn_acc->connectid.desc_tag !=
			cpu_to_be32(FCNVME_LSDESC_CONN_ID))
		fcret = VERR_CONN_ID;
	else if (conn_acc->connectid.desc_len !=
			fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id)))
		fcret = VERR_CONN_ID_LEN;

	if (fcret) {
		ret = -EBADF;
		dev_err(ctrl->dev,
			"q %d connect failed: %s\n",
			queue->qnum, validation_errors[fcret]);
	} else {
		queue->connection_id =
			be64_to_cpu(conn_acc->connectid.connection_id);
		set_bit(NVME_FC_Q_CONNECTED, &queue->flags);
	}

out_free_buffer:
	kfree(lsop);
out_no_memory:
	if (ret)
		dev_err(ctrl->dev,
			"queue %d connect command failed (%d).\n",
			queue->qnum, ret);
	return ret;
}

static void
nvme_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status)
{
	struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);

1385
	__nvme_fc_finish_ls_req(lsop);
1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415

	/* fc-nvme iniator doesn't care about success or failure of cmd */

	kfree(lsop);
}

/*
 * This routine sends a FC-NVME LS to disconnect (aka terminate)
 * the FC-NVME Association.  Terminating the association also
 * terminates the FC-NVME connections (per queue, both admin and io
 * queues) that are part of the association. E.g. things are torn
 * down, and the related FC-NVME Association ID and Connection IDs
 * become invalid.
 *
 * The behavior of the fc-nvme initiator is such that it's
 * understanding of the association and connections will implicitly
 * be torn down. The action is implicit as it may be due to a loss of
 * connectivity with the fc-nvme target, so you may never get a
 * response even if you tried.  As such, the action of this routine
 * is to asynchronously send the LS, ignore any results of the LS, and
 * continue on with terminating the association. If the fc-nvme target
 * is present and receives the LS, it too can tear down.
 */
static void
nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl)
{
	struct fcnvme_ls_disconnect_rqst *discon_rqst;
	struct fcnvme_ls_disconnect_acc *discon_acc;
	struct nvmefc_ls_req_op *lsop;
	struct nvmefc_ls_req *lsreq;
1416
	int ret;
1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458

	lsop = kzalloc((sizeof(*lsop) +
			 ctrl->lport->ops->lsrqst_priv_sz +
			 sizeof(*discon_rqst) + sizeof(*discon_acc)),
			GFP_KERNEL);
	if (!lsop)
		/* couldn't sent it... too bad */
		return;

	lsreq = &lsop->ls_req;

	lsreq->private = (void *)&lsop[1];
	discon_rqst = (struct fcnvme_ls_disconnect_rqst *)
			(lsreq->private + ctrl->lport->ops->lsrqst_priv_sz);
	discon_acc = (struct fcnvme_ls_disconnect_acc *)&discon_rqst[1];

	discon_rqst->w0.ls_cmd = FCNVME_LS_DISCONNECT;
	discon_rqst->desc_list_len = cpu_to_be32(
				sizeof(struct fcnvme_lsdesc_assoc_id) +
				sizeof(struct fcnvme_lsdesc_disconn_cmd));

	discon_rqst->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
	discon_rqst->associd.desc_len =
			fcnvme_lsdesc_len(
				sizeof(struct fcnvme_lsdesc_assoc_id));

	discon_rqst->associd.association_id = cpu_to_be64(ctrl->association_id);

	discon_rqst->discon_cmd.desc_tag = cpu_to_be32(
						FCNVME_LSDESC_DISCONN_CMD);
	discon_rqst->discon_cmd.desc_len =
			fcnvme_lsdesc_len(
				sizeof(struct fcnvme_lsdesc_disconn_cmd));
	discon_rqst->discon_cmd.scope = FCNVME_DISCONN_ASSOCIATION;
	discon_rqst->discon_cmd.id = cpu_to_be64(ctrl->association_id);

	lsreq->rqstaddr = discon_rqst;
	lsreq->rqstlen = sizeof(*discon_rqst);
	lsreq->rspaddr = discon_acc;
	lsreq->rsplen = sizeof(*discon_acc);
	lsreq->timeout = NVME_FC_CONNECT_TIMEOUT_SEC;

1459 1460 1461 1462
	ret = nvme_fc_send_ls_req_async(ctrl->rport, lsop,
				nvme_fc_disconnect_assoc_done);
	if (ret)
		kfree(lsop);
1463 1464 1465 1466 1467 1468 1469 1470

	/* only meaningful part to terminating the association */
	ctrl->association_id = 0;
}


/* *********************** NVME Ctrl Routines **************************** */

1471
static void nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg);
1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500

static int
nvme_fc_reinit_request(void *data, struct request *rq)
{
	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
	struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;

	memset(cmdiu, 0, sizeof(*cmdiu));
	cmdiu->scsi_id = NVME_CMD_SCSI_ID;
	cmdiu->fc_id = NVME_CMD_FC_ID;
	cmdiu->iu_len = cpu_to_be16(sizeof(*cmdiu) / sizeof(u32));
	memset(&op->rsp_iu, 0, sizeof(op->rsp_iu));

	return 0;
}

static void
__nvme_fc_exit_request(struct nvme_fc_ctrl *ctrl,
		struct nvme_fc_fcp_op *op)
{
	fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.rspdma,
				sizeof(op->rsp_iu), DMA_FROM_DEVICE);
	fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.cmddma,
				sizeof(op->cmd_iu), DMA_TO_DEVICE);

	atomic_set(&op->state, FCPOP_STATE_UNINIT);
}

static void
1501 1502
nvme_fc_exit_request(struct blk_mq_tag_set *set, struct request *rq,
		unsigned int hctx_idx)
1503 1504 1505
{
	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);

1506
	return __nvme_fc_exit_request(set->driver_data, op);
1507 1508
}

1509 1510 1511
static int
__nvme_fc_abort_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_fcp_op *op)
{
1512 1513 1514 1515 1516 1517 1518 1519 1520 1521
	unsigned long flags;
	int opstate;

	spin_lock_irqsave(&ctrl->lock, flags);
	opstate = atomic_xchg(&op->state, FCPOP_STATE_ABORTED);
	if (opstate != FCPOP_STATE_ACTIVE)
		atomic_set(&op->state, opstate);
	else if (ctrl->flags & FCCTRL_TERMIO)
		ctrl->iocnt++;
	spin_unlock_irqrestore(&ctrl->lock, flags);
1522

1523
	if (opstate != FCPOP_STATE_ACTIVE)
1524 1525 1526 1527 1528 1529 1530 1531 1532 1533
		return -ECANCELED;

	ctrl->lport->ops->fcp_abort(&ctrl->lport->localport,
					&ctrl->rport->remoteport,
					op->queue->lldd_handle,
					&op->fcp_req);

	return 0;
}

1534
static void
1535
nvme_fc_abort_aen_ops(struct nvme_fc_ctrl *ctrl)
1536 1537
{
	struct nvme_fc_fcp_op *aen_op = ctrl->aen_ops;
1538
	int i;
1539

1540 1541
	for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++)
		__nvme_fc_abort_op(ctrl, aen_op);
1542 1543
}

1544
static inline void
1545
__nvme_fc_fcpop_chk_teardowns(struct nvme_fc_ctrl *ctrl,
1546
		struct nvme_fc_fcp_op *op, int opstate)
1547 1548 1549
{
	unsigned long flags;

1550 1551 1552 1553 1554 1555 1556
	if (opstate == FCPOP_STATE_ABORTED) {
		spin_lock_irqsave(&ctrl->lock, flags);
		if (ctrl->flags & FCCTRL_TERMIO) {
			if (!--ctrl->iocnt)
				wake_up(&ctrl->ioabort_wait);
		}
		spin_unlock_irqrestore(&ctrl->lock, flags);
1557
	}
1558 1559
}

1560
static void
1561 1562 1563 1564 1565 1566 1567 1568
nvme_fc_fcpio_done(struct nvmefc_fcp_req *req)
{
	struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(req);
	struct request *rq = op->rq;
	struct nvmefc_fcp_req *freq = &op->fcp_req;
	struct nvme_fc_ctrl *ctrl = op->ctrl;
	struct nvme_fc_queue *queue = op->queue;
	struct nvme_completion *cqe = &op->rsp_iu.cqe;
1569
	struct nvme_command *sqe = &op->cmd_iu.sqe;
1570
	__le16 status = cpu_to_le16(NVME_SC_SUCCESS << 1);
1571
	union nvme_result result;
1572
	bool terminate_assoc = true;
1573
	int opstate;
1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601

	/*
	 * WARNING:
	 * The current linux implementation of a nvme controller
	 * allocates a single tag set for all io queues and sizes
	 * the io queues to fully hold all possible tags. Thus, the
	 * implementation does not reference or care about the sqhd
	 * value as it never needs to use the sqhd/sqtail pointers
	 * for submission pacing.
	 *
	 * This affects the FC-NVME implementation in two ways:
	 * 1) As the value doesn't matter, we don't need to waste
	 *    cycles extracting it from ERSPs and stamping it in the
	 *    cases where the transport fabricates CQEs on successful
	 *    completions.
	 * 2) The FC-NVME implementation requires that delivery of
	 *    ERSP completions are to go back to the nvme layer in order
	 *    relative to the rsn, such that the sqhd value will always
	 *    be "in order" for the nvme layer. As the nvme layer in
	 *    linux doesn't care about sqhd, there's no need to return
	 *    them in order.
	 *
	 * Additionally:
	 * As the core nvme layer in linux currently does not look at
	 * every field in the cqe - in cases where the FC transport must
	 * fabricate a CQE, the following fields will not be set as they
	 * are not referenced:
	 *      cqe.sqid,  cqe.sqhd,  cqe.command_id
1602 1603 1604 1605 1606 1607 1608 1609
	 *
	 * Failure or error of an individual i/o, in a transport
	 * detected fashion unrelated to the nvme completion status,
	 * potentially cause the initiator and target sides to get out
	 * of sync on SQ head/tail (aka outstanding io count allowed).
	 * Per FC-NVME spec, failure of an individual command requires
	 * the connection to be terminated, which in turn requires the
	 * association to be terminated.
1610 1611
	 */

1612 1613
	opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE);

1614 1615 1616
	fc_dma_sync_single_for_cpu(ctrl->lport->dev, op->fcp_req.rspdma,
				sizeof(op->rsp_iu), DMA_FROM_DEVICE);

1617
	if (opstate == FCPOP_STATE_ABORTED)
1618
		status = cpu_to_le16(NVME_SC_ABORT_REQ << 1);
1619
	else if (freq->status)
1620
		status = cpu_to_le16(NVME_SC_INTERNAL << 1);
1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647

	/*
	 * For the linux implementation, if we have an unsuccesful
	 * status, they blk-mq layer can typically be called with the
	 * non-zero status and the content of the cqe isn't important.
	 */
	if (status)
		goto done;

	/*
	 * command completed successfully relative to the wire
	 * protocol. However, validate anything received and
	 * extract the status and result from the cqe (create it
	 * where necessary).
	 */

	switch (freq->rcv_rsplen) {

	case 0:
	case NVME_FC_SIZEOF_ZEROS_RSP:
		/*
		 * No response payload or 12 bytes of payload (which
		 * should all be zeros) are considered successful and
		 * no payload in the CQE by the transport.
		 */
		if (freq->transferred_length !=
			be32_to_cpu(op->cmd_iu.data_len)) {
1648
			status = cpu_to_le16(NVME_SC_INTERNAL << 1);
1649 1650
			goto done;
		}
1651
		result.u64 = 0;
1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662
		break;

	case sizeof(struct nvme_fc_ersp_iu):
		/*
		 * The ERSP IU contains a full completion with CQE.
		 * Validate ERSP IU and look at cqe.
		 */
		if (unlikely(be16_to_cpu(op->rsp_iu.iu_len) !=
					(freq->rcv_rsplen / 4) ||
			     be32_to_cpu(op->rsp_iu.xfrd_len) !=
					freq->transferred_length ||
1663
			     op->rsp_iu.status_code ||
1664
			     sqe->common.command_id != cqe->command_id)) {
1665
			status = cpu_to_le16(NVME_SC_INTERNAL << 1);
1666 1667
			goto done;
		}
1668
		result = cqe->result;
1669
		status = cqe->status;
1670 1671 1672
		break;

	default:
1673
		status = cpu_to_le16(NVME_SC_INTERNAL << 1);
1674 1675 1676
		goto done;
	}

1677 1678
	terminate_assoc = false;

1679
done:
1680
	if (op->flags & FCOP_FLAGS_AEN) {
1681
		nvme_complete_async_event(&queue->ctrl->ctrl, status, &result);
1682
		__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
1683 1684
		atomic_set(&op->state, FCPOP_STATE_IDLE);
		op->flags = FCOP_FLAGS_AEN;	/* clear other flags */
1685
		nvme_fc_ctrl_put(ctrl);
1686
		goto check_error;
1687 1688
	}

1689 1690 1691 1692 1693 1694 1695
	/*
	 * Force failures of commands if we're killing the controller
	 * or have an error on a command used to create an new association
	 */
	if (status &&
	    (blk_queue_dying(rq->q) ||
	     ctrl->ctrl.state == NVME_CTRL_NEW ||
1696
	     ctrl->ctrl.state == NVME_CTRL_CONNECTING))
1697 1698
		status |= cpu_to_le16(NVME_SC_DNR << 1);

1699 1700
	__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
	nvme_end_request(rq, status, result);
1701 1702 1703 1704

check_error:
	if (terminate_assoc)
		nvme_fc_error_recovery(ctrl, "transport detected io error");
1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755
}

static int
__nvme_fc_init_request(struct nvme_fc_ctrl *ctrl,
		struct nvme_fc_queue *queue, struct nvme_fc_fcp_op *op,
		struct request *rq, u32 rqno)
{
	struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
	int ret = 0;

	memset(op, 0, sizeof(*op));
	op->fcp_req.cmdaddr = &op->cmd_iu;
	op->fcp_req.cmdlen = sizeof(op->cmd_iu);
	op->fcp_req.rspaddr = &op->rsp_iu;
	op->fcp_req.rsplen = sizeof(op->rsp_iu);
	op->fcp_req.done = nvme_fc_fcpio_done;
	op->fcp_req.first_sgl = (struct scatterlist *)&op[1];
	op->fcp_req.private = &op->fcp_req.first_sgl[SG_CHUNK_SIZE];
	op->ctrl = ctrl;
	op->queue = queue;
	op->rq = rq;
	op->rqno = rqno;

	cmdiu->scsi_id = NVME_CMD_SCSI_ID;
	cmdiu->fc_id = NVME_CMD_FC_ID;
	cmdiu->iu_len = cpu_to_be16(sizeof(*cmdiu) / sizeof(u32));

	op->fcp_req.cmddma = fc_dma_map_single(ctrl->lport->dev,
				&op->cmd_iu, sizeof(op->cmd_iu), DMA_TO_DEVICE);
	if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.cmddma)) {
		dev_err(ctrl->dev,
			"FCP Op failed - cmdiu dma mapping failed.\n");
		ret = EFAULT;
		goto out_on_error;
	}

	op->fcp_req.rspdma = fc_dma_map_single(ctrl->lport->dev,
				&op->rsp_iu, sizeof(op->rsp_iu),
				DMA_FROM_DEVICE);
	if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.rspdma)) {
		dev_err(ctrl->dev,
			"FCP Op failed - rspiu dma mapping failed.\n");
		ret = EFAULT;
	}

	atomic_set(&op->state, FCPOP_STATE_IDLE);
out_on_error:
	return ret;
}

static int
1756 1757
nvme_fc_init_request(struct blk_mq_tag_set *set, struct request *rq,
		unsigned int hctx_idx, unsigned int numa_node)
1758
{
1759
	struct nvme_fc_ctrl *ctrl = set->driver_data;
1760
	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
1761 1762
	int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
	struct nvme_fc_queue *queue = &ctrl->queues[queue_idx];
1763 1764 1765 1766 1767 1768 1769 1770 1771 1772

	return __nvme_fc_init_request(ctrl, queue, op, rq, queue->rqcnt++);
}

static int
nvme_fc_init_aen_ops(struct nvme_fc_ctrl *ctrl)
{
	struct nvme_fc_fcp_op *aen_op;
	struct nvme_fc_cmd_iu *cmdiu;
	struct nvme_command *sqe;
1773
	void *private;
1774 1775 1776
	int i, ret;

	aen_op = ctrl->aen_ops;
1777
	for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
1778 1779 1780 1781 1782
		private = kzalloc(ctrl->lport->ops->fcprqst_priv_sz,
						GFP_KERNEL);
		if (!private)
			return -ENOMEM;

1783 1784 1785 1786
		cmdiu = &aen_op->cmd_iu;
		sqe = &cmdiu->sqe;
		ret = __nvme_fc_init_request(ctrl, &ctrl->queues[0],
				aen_op, (struct request *)NULL,
1787
				(NVME_AQ_BLK_MQ_DEPTH + i));
1788 1789
		if (ret) {
			kfree(private);
1790
			return ret;
1791
		}
1792

1793
		aen_op->flags = FCOP_FLAGS_AEN;
1794 1795
		aen_op->fcp_req.first_sgl = NULL; /* no sg list */
		aen_op->fcp_req.private = private;
1796

1797 1798
		memset(sqe, 0, sizeof(*sqe));
		sqe->common.opcode = nvme_admin_async_event;
1799
		/* Note: core layer may overwrite the sqe.command_id value */
1800
		sqe->common.command_id = NVME_AQ_BLK_MQ_DEPTH + i;
1801 1802 1803 1804
	}
	return 0;
}

1805 1806 1807 1808 1809 1810 1811
static void
nvme_fc_term_aen_ops(struct nvme_fc_ctrl *ctrl)
{
	struct nvme_fc_fcp_op *aen_op;
	int i;

	aen_op = ctrl->aen_ops;
1812
	for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
1813 1814 1815 1816 1817 1818 1819 1820 1821
		if (!aen_op->fcp_req.private)
			continue;

		__nvme_fc_exit_request(ctrl, aen_op);

		kfree(aen_op->fcp_req.private);
		aen_op->fcp_req.private = NULL;
	}
}
1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855

static inline void
__nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, struct nvme_fc_ctrl *ctrl,
		unsigned int qidx)
{
	struct nvme_fc_queue *queue = &ctrl->queues[qidx];

	hctx->driver_data = queue;
	queue->hctx = hctx;
}

static int
nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
		unsigned int hctx_idx)
{
	struct nvme_fc_ctrl *ctrl = data;

	__nvme_fc_init_hctx(hctx, ctrl, hctx_idx + 1);

	return 0;
}

static int
nvme_fc_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
		unsigned int hctx_idx)
{
	struct nvme_fc_ctrl *ctrl = data;

	__nvme_fc_init_hctx(hctx, ctrl, hctx_idx);

	return 0;
}

static void
1856
nvme_fc_init_queue(struct nvme_fc_ctrl *ctrl, int idx)
1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897
{
	struct nvme_fc_queue *queue;

	queue = &ctrl->queues[idx];
	memset(queue, 0, sizeof(*queue));
	queue->ctrl = ctrl;
	queue->qnum = idx;
	atomic_set(&queue->csn, 1);
	queue->dev = ctrl->dev;

	if (idx > 0)
		queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16;
	else
		queue->cmnd_capsule_len = sizeof(struct nvme_command);

	/*
	 * Considered whether we should allocate buffers for all SQEs
	 * and CQEs and dma map them - mapping their respective entries
	 * into the request structures (kernel vm addr and dma address)
	 * thus the driver could use the buffers/mappings directly.
	 * It only makes sense if the LLDD would use them for its
	 * messaging api. It's very unlikely most adapter api's would use
	 * a native NVME sqe/cqe. More reasonable if FC-NVME IU payload
	 * structures were used instead.
	 */
}

/*
 * This routine terminates a queue at the transport level.
 * The transport has already ensured that all outstanding ios on
 * the queue have been terminated.
 * The transport will send a Disconnect LS request to terminate
 * the queue's connection. Termination of the admin queue will also
 * terminate the association at the target.
 */
static void
nvme_fc_free_queue(struct nvme_fc_queue *queue)
{
	if (!test_and_clear_bit(NVME_FC_Q_CONNECTED, &queue->flags))
		return;

1898
	clear_bit(NVME_FC_Q_LIVE, &queue->flags);
1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922
	/*
	 * Current implementation never disconnects a single queue.
	 * It always terminates a whole association. So there is never
	 * a disconnect(queue) LS sent to the target.
	 */

	queue->connection_id = 0;
}

static void
__nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *ctrl,
	struct nvme_fc_queue *queue, unsigned int qidx)
{
	if (ctrl->lport->ops->delete_queue)
		ctrl->lport->ops->delete_queue(&ctrl->lport->localport, qidx,
				queue->lldd_handle);
	queue->lldd_handle = NULL;
}

static void
nvme_fc_free_io_queues(struct nvme_fc_ctrl *ctrl)
{
	int i;

1923
	for (i = 1; i < ctrl->ctrl.queue_count; i++)
1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943
		nvme_fc_free_queue(&ctrl->queues[i]);
}

static int
__nvme_fc_create_hw_queue(struct nvme_fc_ctrl *ctrl,
	struct nvme_fc_queue *queue, unsigned int qidx, u16 qsize)
{
	int ret = 0;

	queue->lldd_handle = NULL;
	if (ctrl->lport->ops->create_queue)
		ret = ctrl->lport->ops->create_queue(&ctrl->lport->localport,
				qidx, qsize, &queue->lldd_handle);

	return ret;
}

static void
nvme_fc_delete_hw_io_queues(struct nvme_fc_ctrl *ctrl)
{
1944
	struct nvme_fc_queue *queue = &ctrl->queues[ctrl->ctrl.queue_count - 1];
1945 1946
	int i;

1947
	for (i = ctrl->ctrl.queue_count - 1; i >= 1; i--, queue--)
1948 1949 1950 1951 1952 1953 1954
		__nvme_fc_delete_hw_queue(ctrl, queue, i);
}

static int
nvme_fc_create_hw_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
{
	struct nvme_fc_queue *queue = &ctrl->queues[1];
1955
	int i, ret;
1956

1957
	for (i = 1; i < ctrl->ctrl.queue_count; i++, queue++) {
1958
		ret = __nvme_fc_create_hw_queue(ctrl, queue, i, qsize);
1959 1960
		if (ret)
			goto delete_queues;
1961 1962 1963
	}

	return 0;
1964 1965 1966 1967 1968

delete_queues:
	for (; i >= 0; i--)
		__nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[i], i);
	return ret;
1969 1970 1971 1972 1973 1974 1975
}

static int
nvme_fc_connect_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
{
	int i, ret = 0;

1976
	for (i = 1; i < ctrl->ctrl.queue_count; i++) {
1977 1978 1979 1980 1981 1982 1983
		ret = nvme_fc_connect_queue(ctrl, &ctrl->queues[i], qsize,
					(qsize / 5));
		if (ret)
			break;
		ret = nvmf_connect_io_queue(&ctrl->ctrl, i);
		if (ret)
			break;
1984 1985

		set_bit(NVME_FC_Q_LIVE, &ctrl->queues[i].flags);
1986 1987 1988 1989 1990 1991 1992 1993 1994 1995
	}

	return ret;
}

static void
nvme_fc_init_io_queues(struct nvme_fc_ctrl *ctrl)
{
	int i;

1996
	for (i = 1; i < ctrl->ctrl.queue_count; i++)
1997
		nvme_fc_init_queue(ctrl, i);
1998 1999 2000 2001 2002 2003 2004 2005 2006
}

static void
nvme_fc_ctrl_free(struct kref *ref)
{
	struct nvme_fc_ctrl *ctrl =
		container_of(ref, struct nvme_fc_ctrl, ref);
	unsigned long flags;

2007 2008 2009
	if (ctrl->ctrl.tagset) {
		blk_cleanup_queue(ctrl->ctrl.connect_q);
		blk_mq_free_tag_set(&ctrl->tag_set);
2010 2011
	}

2012 2013 2014 2015 2016
	/* remove from rport list */
	spin_lock_irqsave(&ctrl->rport->lock, flags);
	list_del(&ctrl->ctrl_list);
	spin_unlock_irqrestore(&ctrl->rport->lock, flags);

2017
	blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);
2018 2019 2020 2021 2022
	blk_cleanup_queue(ctrl->ctrl.admin_q);
	blk_mq_free_tag_set(&ctrl->admin_tag_set);

	kfree(ctrl->queues);

2023 2024 2025 2026
	put_device(ctrl->dev);
	nvme_fc_rport_put(ctrl->rport);

	ida_simple_remove(&nvme_fc_ctrl_cnt, ctrl->cnum);
2027 2028
	if (ctrl->ctrl.opts)
		nvmf_free_options(ctrl->ctrl.opts);
2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048
	kfree(ctrl);
}

static void
nvme_fc_ctrl_put(struct nvme_fc_ctrl *ctrl)
{
	kref_put(&ctrl->ref, nvme_fc_ctrl_free);
}

static int
nvme_fc_ctrl_get(struct nvme_fc_ctrl *ctrl)
{
	return kref_get_unless_zero(&ctrl->ref);
}

/*
 * All accesses from nvme core layer done - can now free the
 * controller. Called after last nvme_put_ctrl() call
 */
static void
2049
nvme_fc_nvme_ctrl_freed(struct nvme_ctrl *nctrl)
2050 2051 2052 2053 2054
{
	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);

	WARN_ON(nctrl != &ctrl->ctrl);

2055 2056
	nvme_fc_ctrl_put(ctrl);
}
2057

2058 2059 2060
static void
nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg)
{
2061 2062 2063 2064
	/* only proceed if in LIVE state - e.g. on first error */
	if (ctrl->ctrl.state != NVME_CTRL_LIVE)
		return;

2065 2066 2067
	dev_warn(ctrl->ctrl.device,
		"NVME-FC{%d}: transport association error detected: %s\n",
		ctrl->cnum, errmsg);
2068
	dev_warn(ctrl->ctrl.device,
2069
		"NVME-FC{%d}: resetting controller\n", ctrl->cnum);
2070