xref: /haiku/src/add-ons/kernel/drivers/disk/nvme/libnvme/nvme_internal.h (revision 268f99dd7dc4bd7474a8bd2742d3f1ec1de6752a)

/*-
 *   BSD LICENSE
 *
 *   Copyright (c) Intel Corporation. All rights reserved.
 *   Copyright (c) 2017, Western Digital Corporation or its affiliates.
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef __NVME_INTERNAL_H__
#define __NVME_INTERNAL_H__

#include "nvme_common.h"
#include "nvme_pci.h"
#include "nvme_intel.h"
#include "nvme_mem.h"

#ifndef __HAIKU__
#include <pthread.h>
#include <sys/user.h> /* PAGE_SIZE */
#else
#include "nvme_platform.h"
#endif

/*
 * List functions.
 */
#define	LIST_FOREACH_SAFE(var, head, field, tvar)			\
	for ((var) = LIST_FIRST((head));				\
	     (var) && ((tvar) = LIST_NEXT((var), field), 1);		\
	     (var) = (tvar))

/*
 * Tail queue functions.
 */
#define	TAILQ_FOREACH_SAFE(var, head, field, tvar)			\
	for ((var) = TAILQ_FIRST((head));				\
	     (var) && ((tvar) = TAILQ_NEXT((var), field), 1);		\
	     (var) = (tvar))

#define INTEL_DC_P3X00_DEVID	0x09538086

#define NVME_TIMEOUT_INFINITE	UINT64_MAX

/*
 * Some Intel devices support vendor-unique read latency log page even
 * though the log page directory says otherwise.
 */
#define NVME_INTEL_QUIRK_READ_LATENCY   0x1

/*
 * Some Intel devices support vendor-unique write latency log page even
 * though the log page directory says otherwise.
 */
#define NVME_INTEL_QUIRK_WRITE_LATENCY  0x2

/*
 * Some controllers need a delay before starting to check the device
 * readiness, which is done by reading the controller status register rdy bit.
 */
#define NVME_QUIRK_DELAY_BEFORE_CHK_RDY	0x4

/*
 * Some controllers need a delay once the controller status register rdy bit
 * switches from 0 to 1.
 */
#define NVME_QUIRK_DELAY_AFTER_RDY	0x8

/*
 * Queues may consist of a contiguous block of physical
 * memory or optionally a non-contiguous set of physical
 * memory pages (defined by a Physical Region Pages List)
 */
#define NVME_MAX_PRP_LIST_ENTRIES       (506)

/*
 * For commands requiring more than 2 PRP entries, one PRP will be
 * embedded in the command (prp1), and the rest of the PRP entries
 * will be in a list pointed to by the command (prp2).  This means
 * that real max number of PRP entries we support is 506+1, which
 * results in a max xfer size of 506*PAGE_SIZE.
 */
#define NVME_MAX_XFER_SIZE	NVME_MAX_PRP_LIST_ENTRIES * PAGE_SIZE

#define NVME_ADMIN_TRACKERS	        (16)
#define NVME_ADMIN_ENTRIES	        (128)

/*
 * NVME_IO_ENTRIES defines the size of an I/O qpair's submission and completion
 * queues, while NVME_IO_TRACKERS defines the maximum number of I/O that we
 * will allow outstanding on an I/O qpair at any time. The only advantage in
 * having IO_ENTRIES > IO_TRACKERS is for debugging purposes - when dumping
 * the contents of the submission and completion queues, it will show a longer
 * history of data.
 */
#define NVME_IO_ENTRIES		        (1024U)
#define NVME_IO_TRACKERS	        (128U)
#define NVME_IO_ENTRIES_VS_TRACKERS_RATIO (NVME_IO_ENTRIES / NVME_IO_TRACKERS)

/*
 * NVME_MAX_SGL_DESCRIPTORS defines the maximum number of descriptors in one SGL
 * segment.
 */
#define NVME_MAX_SGL_DESCRIPTORS	(253)

/*
 * NVME_MAX_IO_ENTRIES is not defined, since it is specified in CC.MQES
 * for each controller.
 */

#define NVME_MAX_ASYNC_EVENTS	        (8)

/*
 * NVME_MAX_IO_QUEUES in nvme_spec.h defines the 64K spec-limit, but this
 * define specifies the maximum number of queues this driver will actually
 * try to configure, if available.
 */
#define DEFAULT_MAX_IO_QUEUES		(1024)

/*
 * Maximum of times a failed command can be retried.
 */
#define NVME_MAX_RETRY_COUNT		(3)

/*
 * I/O queue type.
 */
enum nvme_io_queue_type {

	NVME_IO_QTYPE_INVALID = 0,
	NVME_IO_SUBMISSION_QUEUE,
	NVME_IO_COMPLETION_QUEUE,
};

enum nvme_payload_type {

	NVME_PAYLOAD_TYPE_INVALID = 0,

	/*
	 * nvme_request::u.payload.contig_buffer is valid for this request.
	 */
	NVME_PAYLOAD_TYPE_CONTIG,

	/*
	 * nvme_request::u.sgl is valid for this request
	 */
	NVME_PAYLOAD_TYPE_SGL,
};

/*
 * Controller support flags.
 */
enum nvme_ctrlr_flags {

	/*
	 * The SGL is supported.
	 */
	NVME_CTRLR_SGL_SUPPORTED = 0x1,

};

/*
 * Descriptor for a request data payload.
 *
 * This struct is arranged so that it fits nicely in struct nvme_request.
 */
struct __attribute__((packed)) nvme_payload {

	union {
		/*
		 * Virtual memory address of a single
		 * physically contiguous buffer
		 */
		void *contig;

		/*
		 * Call back functions for retrieving physical
		 * addresses for scattered payloads.
		 */
		struct {
			nvme_req_reset_sgl_cb reset_sgl_fn;
			nvme_req_next_sge_cb next_sge_fn;
			void *cb_arg;
		} sgl;
	} u;

	/*
	 * Virtual memory address of a single physically
	 * contiguous metadata buffer
	 */
	void *md;

	/*
	 * Payload type.
	 */
	uint8_t type;

};

struct nvme_request {

	/*
	 * NVMe command: must be aligned on 64B.
	 */
	struct nvme_cmd		         cmd;

	/*
	 * Data payload for this request's command.
	 */
	struct nvme_payload              payload;

	uint8_t			         retries;

	/*
	 * Number of child requests still outstanding for this
	 * request which was split into multiple child requests.
	 */
	uint8_t			         child_reqs;
	uint32_t		         payload_size;

	/*
	 * Offset in bytes from the beginning of payload for this request.
	 * This is used for I/O commands that are split into multiple requests.
	 */
	uint32_t	                 payload_offset;
	uint32_t		         md_offset;

	nvme_cmd_cb		         cb_fn;
	void			         *cb_arg;

	/*
	 * The following members should not be reordered with members
	 * above.  These members are only needed when splitting
	 * requests which is done rarely, and the driver is careful
	 * to not touch the following fields until a split operation is
	 * needed, to avoid touching an extra cacheline.
	 */

	/*
	 * Points to the outstanding child requests for a parent request.
	 * Only valid if a request was split into multiple child
	 * requests, and is not initialized for non-split requests.
	 */
	TAILQ_HEAD(, nvme_request)	 children;

	/*
	 * Linked-list pointers for a child request in its parent's list.
	 */
	TAILQ_ENTRY(nvme_request)	 child_tailq;

	/*
	 * For queueing in qpair queued_req or free_req.
	 */
	struct nvme_qpair		 *qpair;
	STAILQ_ENTRY(nvme_request)	 stailq;

	/*
	 * Points to a parent request if part of a split request,
	 * NULL otherwise.
	 */
	struct nvme_request		 *parent;

	/*
	 * Completion status for a parent request.  Initialized to all 0's
	 * (SUCCESS) before child requests are submitted.  If a child
	 * request completes with error, the error status is copied here,
	 * to ensure that the parent request is also completed with error
	 * status once all child requests are completed.
	 */
	struct nvme_cpl		         parent_status;

} __attribute__((aligned(64)));

struct nvme_completion_poll_status {
	struct nvme_cpl		cpl;
	bool			done;
};

struct nvme_async_event_request {
	struct nvme_ctrlr	*ctrlr;
	struct nvme_request	*req;
	struct nvme_cpl		cpl;
};

struct nvme_tracker {

	LIST_ENTRY(nvme_tracker)	list;

	struct nvme_request		*req;
#if INTPTR_MAX == INT32_MAX
	int32_t __pad[3];
#elif !defined(INTPTR_MAX)
#	error Need definition of INTPTR_MAX!
#endif

	uint16_t			cid;

	uint16_t			rsvd1: 15;
	uint16_t			active: 1;

	uint32_t			rsvd2;

	uint64_t			prp_sgl_bus_addr;

	union {
		uint64_t			prp[NVME_MAX_PRP_LIST_ENTRIES];
		struct nvme_sgl_descriptor	sgl[NVME_MAX_SGL_DESCRIPTORS];
	} u;

	uint64_t			rsvd3;
};

/*
 * struct nvme_tracker must be exactly 4K so that the prp[] array does not
 * cross a page boundery and so that there is no padding required to meet
 * alignment requirements.
 */
nvme_static_assert(sizeof(struct nvme_tracker) == 4096,
		   "nvme_tracker is not 4K");
nvme_static_assert((offsetof(struct nvme_tracker, u.sgl) & 7) == 0,
		   "SGL must be Qword aligned");

struct nvme_qpair {
	/*
	 * Guards access to this structure.
	 */
	pthread_mutex_t				lock;

	volatile uint32_t	        *sq_tdbl;
	volatile uint32_t	        *cq_hdbl;

	/*
	 * Submission queue
	 */
	struct nvme_cmd		        *cmd;

	/*
	 * Completion queue
	 */
	struct nvme_cpl		        *cpl;

	LIST_HEAD(, nvme_tracker)	free_tr;
	LIST_HEAD(, nvme_tracker)	outstanding_tr;

	/*
	 * Array of trackers indexed by command ID.
	 */
	uint16_t			trackers;
	struct nvme_tracker		*tr;

	struct nvme_request		*reqs;
	unsigned int			num_reqs;
	STAILQ_HEAD(, nvme_request)	free_req;
	STAILQ_HEAD(, nvme_request)	queued_req;

	uint16_t			id;

	uint16_t			entries;
	uint16_t			sq_tail;
	uint16_t			cq_head;

	uint8_t				phase;

	bool				enabled;
	bool				sq_in_cmb;

	/*
	 * Fields below this point should not be touched on the
	 * normal I/O happy path.
	 */

	uint8_t				qprio;

	struct nvme_ctrlr		*ctrlr;

	/* List entry for nvme_ctrlr::free_io_qpairs and active_io_qpairs */
	TAILQ_ENTRY(nvme_qpair)		tailq;

	phys_addr_t			cmd_bus_addr;
	phys_addr_t			cpl_bus_addr;
};

struct nvme_ns {

	struct nvme_ctrlr		*ctrlr;

	uint32_t			stripe_size;
	uint32_t			sector_size;

	uint32_t			md_size;
	uint32_t			pi_type;

	uint32_t			sectors_per_max_io;
	uint32_t			sectors_per_stripe;

	uint16_t			id;
	uint16_t			flags;

	int				open_count;

};

/*
 * State of struct nvme_ctrlr (in particular, during initialization).
 */
enum nvme_ctrlr_state {

	/*
	 * Controller has not been initialized yet.
	 */
	NVME_CTRLR_STATE_INIT = 0,

	/*
	 * Waiting for CSTS.RDY to transition from 0 to 1
	 * so that CC.EN may be set to 0.
	 */
	NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1,

	/*
	 * Waiting for CSTS.RDY to transition from 1 to 0
	 * so that CC.EN may be set to 1.
	 */
	NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0,

	/*
	 * Waiting for CSTS.RDY to transition from 0 to 1
	 * after enabling the controller.
	 */
	NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1,

	/*
	 * Controller initialization has completed and
	 * the controller is ready.
	 */
	NVME_CTRLR_STATE_READY
};

/*
 * One of these per allocated PCI device.
 */
struct nvme_ctrlr {

	/*
	 * NVMe MMIO register space.
	 */
	volatile struct nvme_registers	*regs;

	/*
	 * Array of I/O queue pairs.
	 */
	struct nvme_qpair		*ioq;

	/*
	 * Size of the array of I/O queue pairs.
	 */
	unsigned int			io_queues;

	/*
	 * Maximum I/O queue pairs.
	 */
	unsigned int			max_io_queues;

	/*
	 * Number of I/O queue pairs enabled
	 */
	unsigned int			enabled_io_qpairs;

	/*
	 * Maximum entries for I/O qpairs
	 */
	unsigned int			io_qpairs_max_entries;

	/*
	 * Array of namespace IDs.
	 */
	unsigned int			nr_ns;
	struct nvme_ns		        *ns;

	/*
	 * Controller state.
	 */
	bool				resetting;
	bool				failed;

	/*
	 * Controller support flags.
	 */
	uint64_t			flags;

	/*
	 * Cold data (not accessed in normal I/O path) is after this point.
	 */
	enum nvme_ctrlr_state		state;
	uint64_t			state_timeout_ms;

	/*
	 * All the log pages supported.
	 */
	bool				log_page_supported[256];

	/*
	 * All the features supported.
	 */
	bool				feature_supported[256];

	/*
	 * Associated PCI device information.
	 */
	struct pci_device		*pci_dev;

	/*
	 * Maximum i/o size in bytes.
	 */
	uint32_t			max_xfer_size;

	/*
	 * Minimum page size supported by this controller in bytes.
	 */
	uint32_t			min_page_size;

	/*
	 * Stride in uint32_t units between doorbell registers
	 * (1 = 4 bytes, 2 = 8 bytes, ...).
	 */
	uint32_t			doorbell_stride_u32;

	uint32_t			num_aers;
	struct nvme_async_event_request	aer[NVME_MAX_ASYNC_EVENTS];
	nvme_aer_cb		        aer_cb_fn;
	void				*aer_cb_arg;

	/*
	 * Admin queue pair.
	 */
	struct nvme_qpair		adminq;

	/*
	 * Guards access to the controller itself.
	 */
	pthread_mutex_t			lock;

	/*
	 * Identify Controller data.
	 */
	struct nvme_ctrlr_data		cdata;

	/*
	 * Array of Identify Namespace data.
	 * Stored separately from ns since nsdata should
	 * not normally be accessed during I/O.
	 */
	struct nvme_ns_data	        *nsdata;

	TAILQ_HEAD(, nvme_qpair)	free_io_qpairs;
	TAILQ_HEAD(, nvme_qpair)	active_io_qpairs;

	/*
	 * Controller option set on open.
	 */
	struct nvme_ctrlr_opts		opts;

	/*
	 * BAR mapping address which contains controller memory buffer.
	 */
	void				*cmb_bar_virt_addr;

	/*
	 * BAR physical address which contains controller memory buffer.
	 */
	uint64_t			cmb_bar_phys_addr;

	/*
	 * Controller memory buffer size in Bytes.
	 */
	uint64_t			cmb_size;

	/*
	 * Current offset of controller memory buffer.
	 */
	uint64_t			cmb_current_offset;

	/*
	 * Quirks flags.
	 */
	unsigned int			quirks;

	/*
	 * For controller list.
	 */
	LIST_ENTRY(nvme_ctrlr)		link;

} __attribute__((aligned(PAGE_SIZE)));

/*
 * Admin functions.
 */
extern int nvme_admin_identify_ctrlr(struct nvme_ctrlr *ctrlr,
				     struct nvme_ctrlr_data *cdata);

extern int nvme_admin_get_feature(struct nvme_ctrlr *ctrlr,
				  enum nvme_feat_sel sel,
				  enum nvme_feat feature,
				  uint32_t cdw11, uint32_t *attributes);

extern int nvme_admin_set_feature(struct nvme_ctrlr *ctrlr,
				  bool save,
				  enum nvme_feat feature,
				  uint32_t cdw11, uint32_t cdw12,
				  uint32_t *attributes);

extern int nvme_admin_format_nvm(struct nvme_ctrlr *ctrlr,
				 unsigned int nsid,
				 struct nvme_format *format);

extern int nvme_admin_get_log_page(struct nvme_ctrlr *ctrlr,
				   uint8_t log_page, uint32_t nsid,
				   void *payload, uint32_t payload_size);

extern int nvme_admin_abort_cmd(struct nvme_ctrlr *ctrlr,
				uint16_t cid, uint16_t sqid);

extern int nvme_admin_create_ioq(struct nvme_ctrlr *ctrlr,
				 struct nvme_qpair *io_que,
				 enum nvme_io_queue_type io_qtype);

extern int nvme_admin_delete_ioq(struct nvme_ctrlr *ctrlr,
				 struct nvme_qpair *qpair,
				 enum nvme_io_queue_type io_qtype);

extern int nvme_admin_identify_ns(struct nvme_ctrlr *ctrlr,
				  uint16_t nsid,
				  struct nvme_ns_data *nsdata);

extern int nvme_admin_attach_ns(struct nvme_ctrlr *ctrlr,
				uint32_t nsid,
				struct nvme_ctrlr_list *clist);

extern int nvme_admin_detach_ns(struct nvme_ctrlr *ctrlr,
				uint32_t nsid,
				struct nvme_ctrlr_list *clist);

extern int nvme_admin_create_ns(struct nvme_ctrlr *ctrlr,
				struct nvme_ns_data *nsdata,
				unsigned int *nsid);

extern int nvme_admin_delete_ns(struct nvme_ctrlr *ctrlr,
				unsigned int nsid);

extern int nvme_admin_fw_commit(struct nvme_ctrlr *ctrlr,
				const struct nvme_fw_commit *fw_commit);

extern int nvme_admin_fw_image_dl(struct nvme_ctrlr *ctrlr,
				  void *fw, uint32_t size, uint32_t offset);

extern void nvme_request_completion_poll_cb(void *arg,
					    const struct nvme_cpl *cpl);

extern struct nvme_ctrlr *nvme_ctrlr_attach(struct pci_device *pci_dev,
					    struct nvme_ctrlr_opts *opts);

extern void nvme_ctrlr_detach(struct nvme_ctrlr *ctrlr);

extern int nvme_qpair_construct(struct nvme_ctrlr *ctrlr,
				struct nvme_qpair *qpair, enum nvme_qprio qprio,
				uint16_t entries, uint16_t trackers);

extern void nvme_qpair_destroy(struct nvme_qpair *qpair);
extern void nvme_qpair_enable(struct nvme_qpair *qpair);
extern void nvme_qpair_disable(struct nvme_qpair *qpair);
extern int  nvme_qpair_submit_request(struct nvme_qpair *qpair,
				      struct nvme_request *req);
extern void nvme_qpair_reset(struct nvme_qpair *qpair);
extern void nvme_qpair_fail(struct nvme_qpair *qpair);

extern int nvme_request_pool_construct(struct nvme_qpair *qpair);

extern void nvme_request_pool_destroy(struct nvme_qpair *qpair);

extern struct nvme_request *nvme_request_allocate(struct nvme_qpair *qpair,
		      const struct nvme_payload *payload, uint32_t payload_size,
		      nvme_cmd_cb cb_fn, void *cb_arg);

extern struct nvme_request *nvme_request_allocate_null(struct nvme_qpair *qpair,
						       nvme_cmd_cb cb_fn,
						       void *cb_arg);

extern struct nvme_request *
nvme_request_allocate_contig(struct nvme_qpair *qpair,
			     void *buffer, uint32_t payload_size,
			     nvme_cmd_cb cb_fn, void *cb_arg);

extern void nvme_request_free(struct nvme_request *req);
extern void nvme_request_free_locked(struct nvme_request *req);

extern void nvme_request_add_child(struct nvme_request *parent,
				   struct nvme_request *child);

extern void nvme_request_remove_child(struct nvme_request *parent,
				      struct nvme_request *child);

extern unsigned int nvme_ctrlr_get_quirks(struct pci_device *pdev);

extern int nvme_ns_construct(struct nvme_ctrlr *ctrlr,
			     struct nvme_ns *ns, unsigned int id);

/*
 * Registers mmio access.
 */
#define nvme_reg_mmio_read_4(sc, reg)		\
	nvme_mmio_read_4((__u32 *)&(sc)->regs->reg)

#define nvme_reg_mmio_read_8(sc, reg)		\
	nvme_mmio_read_8((__u64 *)&(sc)->regs->reg)

#define nvme_reg_mmio_write_4(sc, reg, val)	\
	nvme_mmio_write_4((__u32 *)&(sc)->regs->reg, val)

#define nvme_reg_mmio_write_8(sc, reg, val)	\
	nvme_mmio_write_8((__u64 *)&(sc)->regs->reg, val)

#endif /* __NVME_INTERNAL_H__ */