1 /* 2 * Copyright 2007, Hugo Santos. All Rights Reserved. 3 * Copyright 2004, Marcus Overhagen. All Rights Reserved. 4 * Distributed under the terms of the MIT License. 5 */ 6 7 8 #include "device.h" 9 10 #include <stdint.h> 11 #include <string.h> 12 #include <slab/Slab.h> 13 14 #include <compat/sys/malloc.h> 15 #include <compat/sys/mbuf.h> 16 #include <compat/sys/kernel.h> 17 18 19 static object_cache *sMBufCache; 20 static object_cache *sChunkCache; 21 static object_cache *sJumbo9ChunkCache; 22 static object_cache *sJumboPageSizeCache; 23 24 25 int max_linkhdr = 16; 26 int max_protohdr = 40 + 20; /* ip6 + tcp */ 27 28 /* max_linkhdr + max_protohdr, but that's not allowed by gcc. */ 29 int max_hdr = 16 + 40 + 20; 30 31 /* MHLEN - max_hdr */ 32 int max_datalen = MHLEN - (16 + 40 + 20); 33 34 35 static int 36 m_to_oc_flags(int how) 37 { 38 if (how & M_NOWAIT) 39 return CACHE_DONT_WAIT_FOR_MEMORY; 40 41 return 0; 42 } 43 44 45 int 46 m_init(struct mbuf *m, int how, short type, int flags) 47 { 48 int error; 49 50 if (type == MT_NOINIT) 51 return 0; 52 53 m->m_next = NULL; 54 m->m_nextpkt = NULL; 55 m->m_data = m->m_dat; 56 m->m_len = 0; 57 m->m_flags = flags; 58 m->m_type = type; 59 if (flags & M_PKTHDR) 60 error = m_pkthdr_init(m, how); 61 else 62 error = 0; 63 64 return (error); 65 } 66 67 68 static void* 69 allocate_ext_buf(int how, int size, int* ext_type) 70 { 71 object_cache *cache; 72 int extType; 73 if (size != MCLBYTES && size != MJUM9BYTES && size != MJUMPAGESIZE) 74 panic("unsupported size"); 75 76 if (size == MCLBYTES) { 77 cache = sChunkCache; 78 extType = EXT_CLUSTER; 79 } else if (size == MJUM9BYTES) { 80 cache = sJumbo9ChunkCache; 81 extType = EXT_JUMBO9; 82 } else { 83 cache = sJumboPageSizeCache; 84 extType = EXT_JUMBOP; 85 } 86 87 if (ext_type != NULL) 88 *ext_type = extType; 89 return object_cache_alloc(cache, m_to_oc_flags(how)); 90 } 91 92 93 static int 94 construct_ext_sized_mbuf(struct mbuf *memoryBuffer, int how, int size) 95 { 96 int extType; 97 98 memoryBuffer->m_ext.ext_buf = allocate_ext_buf(how, size, &extType); 99 if (memoryBuffer->m_ext.ext_buf == NULL) 100 return B_NO_MEMORY; 101 102 memoryBuffer->m_data = memoryBuffer->m_ext.ext_buf; 103 memoryBuffer->m_flags |= M_EXT; 104 memoryBuffer->m_ext.ext_size = size; 105 memoryBuffer->m_ext.ext_type = extType; 106 memoryBuffer->m_ext.ext_flags = EXT_FLAG_EMBREF; 107 memoryBuffer->m_ext.ext_count = 1; 108 109 return 0; 110 } 111 112 113 static inline int 114 construct_ext_mbuf(struct mbuf *memoryBuffer, int how) 115 { 116 return construct_ext_sized_mbuf(memoryBuffer, how, MCLBYTES); 117 } 118 119 120 static int 121 construct_pkt_mbuf(int how, struct mbuf *memoryBuffer, short type, int flags) 122 { 123 if (m_init(memoryBuffer, how, type, flags) < 0) 124 return -1; 125 if (construct_ext_mbuf(memoryBuffer, how) < 0) 126 return -1; 127 memoryBuffer->m_ext.ext_type = EXT_CLUSTER; 128 return 0; 129 } 130 131 132 struct mbuf * 133 m_getcl(int how, short type, int flags) 134 { 135 struct mbuf *memoryBuffer = 136 (struct mbuf *)object_cache_alloc(sMBufCache, m_to_oc_flags(how)); 137 if (memoryBuffer == NULL) 138 return NULL; 139 140 if (construct_pkt_mbuf(how, memoryBuffer, type, flags) < 0) { 141 object_cache_free(sMBufCache, memoryBuffer, 0); 142 return NULL; 143 } 144 145 return memoryBuffer; 146 } 147 148 149 static struct mbuf * 150 _m_get(int how, short type, int flags) 151 { 152 struct mbuf *memoryBuffer = 153 (struct mbuf *)object_cache_alloc(sMBufCache, m_to_oc_flags(how)); 154 if (memoryBuffer == NULL) 155 return NULL; 156 157 m_init(memoryBuffer, how, type, flags); 158 159 return memoryBuffer; 160 } 161 162 163 struct mbuf * 164 m_get(int how, short type) 165 { 166 return _m_get(how, type, 0); 167 } 168 169 170 struct mbuf * 171 m_get2(int size, int how, short type, int flags) 172 { 173 if (size <= MHLEN || (size <= MLEN && (flags & M_PKTHDR) == 0)) { 174 return _m_get(how, type, flags); 175 } else if (size <= MCLBYTES) { 176 size = MCLBYTES; 177 } else if (size <= MJUMPAGESIZE) { 178 size = MJUMPAGESIZE; 179 } else if (size <= MJUM9BYTES) { 180 size = MJUM9BYTES; 181 } else /* (size > MJUM9BYTES) */ { 182 return NULL; 183 } 184 185 return m_getjcl(how, type, flags, size); 186 } 187 188 189 struct mbuf * 190 m_gethdr(int how, short type) 191 { 192 return _m_get(how, type, M_PKTHDR); 193 } 194 195 196 struct mbuf * 197 m_getjcl(int how, short type, int flags, int size) 198 { 199 struct mbuf *memoryBuffer = 200 (struct mbuf *)object_cache_alloc(sMBufCache, m_to_oc_flags(how)); 201 if (memoryBuffer == NULL) 202 return NULL; 203 if (m_init(memoryBuffer, how, type, flags) < 0) { 204 object_cache_free(sMBufCache, memoryBuffer, 0); 205 return NULL; 206 } 207 if (construct_ext_sized_mbuf(memoryBuffer, how, size) < 0) { 208 object_cache_free(sMBufCache, memoryBuffer, 0); 209 return NULL; 210 } 211 return memoryBuffer; 212 } 213 214 215 int 216 m_clget(struct mbuf *memoryBuffer, int how) 217 { 218 memoryBuffer->m_ext.ext_buf = NULL; 219 /* called checks for errors by looking for M_EXT */ 220 construct_ext_mbuf(memoryBuffer, how); 221 return memoryBuffer->m_flags & M_EXT; 222 } 223 224 225 void* 226 m_cljget(struct mbuf* memoryBuffer, int how, int size) 227 { 228 if (memoryBuffer == NULL) 229 return allocate_ext_buf(how, size, NULL); 230 231 memoryBuffer->m_ext.ext_buf = NULL; 232 construct_ext_sized_mbuf(memoryBuffer, how, size); 233 return memoryBuffer->m_ext.ext_buf; 234 } 235 236 237 static void 238 mb_free_ext(struct mbuf *memoryBuffer) 239 { 240 volatile u_int *refcnt; 241 struct mbuf *mref; 242 int freembuf; 243 244 KASSERT(memoryBuffer->m_flags & M_EXT, ("%s: M_EXT not set on %p", 245 __func__, memoryBuffer)); 246 247 /* See if this is the mbuf that holds the embedded refcount. */ 248 if (memoryBuffer->m_ext.ext_flags & EXT_FLAG_EMBREF) { 249 refcnt = &memoryBuffer->m_ext.ext_count; 250 mref = memoryBuffer; 251 } else { 252 KASSERT(memoryBuffer->m_ext.ext_cnt != NULL, 253 ("%s: no refcounting pointer on %p", __func__, memoryBuffer)); 254 refcnt = memoryBuffer->m_ext.ext_cnt; 255 mref = __containerof(refcnt, struct mbuf, m_ext.ext_count); 256 } 257 258 /* 259 * Check if the header is embedded in the cluster. It is 260 * important that we can't touch any of the mbuf fields 261 * after we have freed the external storage, since mbuf 262 * could have been embedded in it. For now, the mbufs 263 * embedded into the cluster are always of type EXT_EXTREF, 264 * and for this type we won't free the mref. 265 */ 266 if (memoryBuffer->m_flags & M_NOFREE) { 267 freembuf = 0; 268 KASSERT(memoryBuffer->m_ext.ext_type == EXT_EXTREF, 269 ("%s: no-free mbuf %p has wrong type", __func__, memoryBuffer)); 270 } else 271 freembuf = 1; 272 273 /* Free attached storage only if this mbuf is the only reference to it. */ 274 if (*refcnt == 1 || atomic_add((int32*)refcnt, -1) == 1) { 275 object_cache *cache = NULL; 276 277 if (memoryBuffer->m_ext.ext_type == EXT_CLUSTER) 278 cache = sChunkCache; 279 else if (memoryBuffer->m_ext.ext_type == EXT_JUMBO9) 280 cache = sJumbo9ChunkCache; 281 else if (memoryBuffer->m_ext.ext_type == EXT_JUMBOP) 282 cache = sJumboPageSizeCache; 283 else 284 panic("unknown mbuf ext_type %d", memoryBuffer->m_ext.ext_type); 285 286 object_cache_free(cache, memoryBuffer->m_ext.ext_buf, 0); 287 object_cache_free(sMBufCache, mref, 0); 288 } 289 290 if (freembuf && memoryBuffer != mref) 291 object_cache_free(sMBufCache, memoryBuffer, 0); 292 } 293 294 295 struct mbuf * 296 m_free(struct mbuf* memoryBuffer) 297 { 298 struct mbuf* next = memoryBuffer->m_next; 299 300 if ((memoryBuffer->m_flags & (M_PKTHDR|M_NOFREE)) == (M_PKTHDR|M_NOFREE)) 301 m_tag_delete_chain(memoryBuffer, NULL); 302 if (memoryBuffer->m_flags & M_EXT) 303 mb_free_ext(memoryBuffer); 304 else if ((memoryBuffer->m_flags & M_NOFREE) == 0) 305 object_cache_free(sMBufCache, memoryBuffer, 0); 306 307 return next; 308 } 309 310 311 status_t 312 init_mbufs() 313 { 314 sMBufCache = create_object_cache("mbufs", MSIZE, 8, NULL, NULL, NULL); 315 if (sMBufCache == NULL) 316 goto clean; 317 sChunkCache = create_object_cache("mbuf chunks", MCLBYTES, 0, NULL, NULL, 318 NULL); 319 if (sChunkCache == NULL) 320 goto clean; 321 sJumbo9ChunkCache = create_object_cache("mbuf jumbo9 chunks", MJUM9BYTES, 0, 322 NULL, NULL, NULL); 323 if (sJumbo9ChunkCache == NULL) 324 goto clean; 325 sJumboPageSizeCache = create_object_cache("mbuf jumbo page size chunks", 326 MJUMPAGESIZE, 0, NULL, NULL, NULL); 327 if (sJumboPageSizeCache == NULL) 328 goto clean; 329 return B_OK; 330 331 clean: 332 if (sJumbo9ChunkCache != NULL) 333 delete_object_cache(sJumbo9ChunkCache); 334 if (sChunkCache != NULL) 335 delete_object_cache(sChunkCache); 336 if (sMBufCache != NULL) 337 delete_object_cache(sMBufCache); 338 return B_NO_MEMORY; 339 } 340 341 342 void 343 uninit_mbufs() 344 { 345 delete_object_cache(sMBufCache); 346 delete_object_cache(sChunkCache); 347 delete_object_cache(sJumbo9ChunkCache); 348 delete_object_cache(sJumboPageSizeCache); 349 } 350