1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. All rights reserved. 5 * Copyright (c) 2017, Western Digital Corporation or its affiliates. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "nvme_internal.h" 35 36 /* 37 * Allocate a request descriptor from the queue pair free list. 38 */ 39 static struct nvme_request *nvme_alloc_request(struct nvme_qpair *qpair) 40 { 41 struct nvme_request *req; 42 43 req = STAILQ_FIRST(&qpair->free_req); 44 if (req) { 45 STAILQ_REMOVE_HEAD(&qpair->free_req, stailq); 46 memset(&req->cmd, 0, sizeof(struct nvme_cmd)); 47 } 48 49 return req; 50 } 51 52 static void nvme_request_cb_complete_child(void *child_arg, 53 const struct nvme_cpl *cpl) 54 { 55 struct nvme_request *child = child_arg; 56 struct nvme_request *parent = child->parent; 57 58 nvme_request_remove_child(parent, child); 59 60 if (nvme_cpl_is_error(cpl)) 61 memcpy(&parent->parent_status, cpl, sizeof(*cpl)); 62 63 if (parent->child_reqs == 0) { 64 if (parent->cb_fn) 65 parent->cb_fn(parent->cb_arg, &parent->parent_status); 66 nvme_request_free(parent); 67 } 68 } 69 70 void nvme_request_completion_poll_cb(void *arg, const struct nvme_cpl *cpl) 71 { 72 struct nvme_completion_poll_status *status = arg; 73 74 memcpy(&status->cpl, cpl, sizeof(*cpl)); 75 status->done = true; 76 } 77 78 int nvme_request_pool_construct(struct nvme_qpair *qpair) 79 { 80 struct nvme_request *req; 81 unsigned int i; 82 83 qpair->num_reqs = qpair->trackers * NVME_IO_ENTRIES_VS_TRACKERS_RATIO; 84 qpair->reqs = calloc(qpair->num_reqs, sizeof(struct nvme_request)); 85 if (!qpair->reqs) { 86 nvme_err("QPair %d: allocate %u requests failed\n", 87 (int)qpair->id, qpair->num_reqs); 88 return ENOMEM; 89 } 90 91 nvme_info("QPair %d: %d requests in pool\n", 92 (int)qpair->id, 93 (int)qpair->num_reqs); 94 95 for(i = 0; i < qpair->num_reqs; i++) { 96 req = &qpair->reqs[i]; 97 req->qpair = qpair; 98 STAILQ_INSERT_TAIL(&qpair->free_req, req, stailq); 99 req++; 100 } 101 102 return 0; 103 } 104 105 void nvme_request_pool_destroy(struct nvme_qpair *qpair) 106 { 107 struct nvme_request *req; 108 unsigned int n = 0; 109 110 while ((req = STAILQ_FIRST(&qpair->free_req))) { 111 STAILQ_REMOVE_HEAD(&qpair->free_req, stailq); 112 n++; 113 } 114 115 if (n != qpair->num_reqs) 116 nvme_err("QPair %d: Freed %d/%d requests\n", 117 (int)qpair->id, n, (int)qpair->num_reqs); 118 119 free(qpair->reqs); 120 } 121 122 struct nvme_request *nvme_request_allocate(struct nvme_qpair *qpair, 123 const struct nvme_payload *payload, 124 uint32_t payload_size, 125 nvme_cmd_cb cb_fn, 126 void *cb_arg) 127 { 128 struct nvme_request *req; 129 130 req = nvme_alloc_request(qpair); 131 if (req == NULL) 132 return NULL; 133 134 /* 135 * Only memset up to (but not including) the children TAILQ_ENTRY. 136 * Children, and following members, are only used as part of I/O 137 * splitting so we avoid memsetting them until it is actually needed. 138 * They will be initialized in nvme_request_add_child() 139 * if the request is split. 140 */ 141 memset(req, 0, offsetof(struct nvme_request, children)); 142 req->cb_fn = cb_fn; 143 req->cb_arg = cb_arg; 144 req->payload = *payload; 145 req->payload_size = payload_size; 146 147 return req; 148 } 149 150 struct nvme_request *nvme_request_allocate_contig(struct nvme_qpair *qpair, 151 void *buffer, 152 uint32_t payload_size, 153 nvme_cmd_cb cb_fn, 154 void *cb_arg) 155 { 156 struct nvme_payload payload; 157 158 payload.type = NVME_PAYLOAD_TYPE_CONTIG; 159 payload.u.contig = buffer; 160 payload.md = NULL; 161 162 return nvme_request_allocate(qpair, &payload, payload_size, 163 cb_fn, cb_arg); 164 } 165 166 struct nvme_request *nvme_request_allocate_null(struct nvme_qpair *qpair, 167 nvme_cmd_cb cb_fn, void *cb_arg) 168 { 169 return nvme_request_allocate_contig(qpair, NULL, 0, cb_fn, cb_arg); 170 } 171 172 void nvme_request_free(struct nvme_request *req) 173 { 174 struct nvme_qpair *qpair = req->qpair; 175 176 nvme_assert(req->child_reqs == 0, "Number of child request not 0\n"); 177 178 STAILQ_INSERT_HEAD(&qpair->free_req, req, stailq); 179 } 180 181 void nvme_request_add_child(struct nvme_request *parent, 182 struct nvme_request *child) 183 { 184 if (parent->child_reqs == 0) { 185 /* 186 * Defer initialization of the children TAILQ since it falls 187 * on a separate cacheline. This ensures we do not touch this 188 * cacheline except on request splitting cases, which are 189 * relatively rare. 190 */ 191 TAILQ_INIT(&parent->children); 192 parent->parent = NULL; 193 memset(&parent->parent_status, 0, sizeof(struct nvme_cpl)); 194 } 195 196 parent->child_reqs++; 197 TAILQ_INSERT_TAIL(&parent->children, child, child_tailq); 198 child->parent = parent; 199 child->cb_fn = nvme_request_cb_complete_child; 200 child->cb_arg = child; 201 } 202 203 void nvme_request_remove_child(struct nvme_request *parent, 204 struct nvme_request *child) 205 { 206 nvme_assert(child->parent == parent, "child->parent != parent\n"); 207 nvme_assert(parent->child_reqs != 0, "child_reqs is 0\n"); 208 209 parent->child_reqs--; 210 TAILQ_REMOVE(&parent->children, child, child_tailq); 211 } 212