1 /* 2 * Copyright 2007, Hugo Santos. All Rights Reserved. 3 * Copyright 2004, Marcus Overhagen. All Rights Reserved. 4 * Distributed under the terms of the MIT License. 5 */ 6 7 8 #include "device.h" 9 10 #include <stdint.h> 11 #include <string.h> 12 #include <slab/Slab.h> 13 14 #include <compat/sys/malloc.h> 15 #include <compat/sys/mbuf.h> 16 #include <compat/sys/kernel.h> 17 18 19 static object_cache *sMBufCache; 20 static object_cache *sChunkCache; 21 static object_cache *sJumbo9ChunkCache; 22 static object_cache *sJumboPageSizeCache; 23 24 25 int max_linkhdr = 16; 26 int max_protohdr = 40 + 20; /* ip6 + tcp */ 27 28 /* max_linkhdr + max_protohdr, but that's not allowed by gcc. */ 29 int max_hdr = 16 + 40 + 20; 30 31 /* MHLEN - max_hdr */ 32 int max_datalen = MHLEN - (16 + 40 + 20); 33 34 35 static int 36 m_to_oc_flags(int how) 37 { 38 if (how & M_NOWAIT) 39 return CACHE_DONT_WAIT_FOR_MEMORY; 40 41 return 0; 42 } 43 44 45 int 46 m_init(struct mbuf *m, int how, short type, int flags) 47 { 48 int error; 49 50 if (type == MT_NOINIT) 51 return 0; 52 53 m->m_next = NULL; 54 m->m_nextpkt = NULL; 55 m->m_data = m->m_dat; 56 m->m_len = 0; 57 m->m_flags = flags; 58 m->m_type = type; 59 if (flags & M_PKTHDR) 60 error = m_pkthdr_init(m, how); 61 else 62 error = 0; 63 64 return (error); 65 } 66 67 68 static void* 69 allocate_ext_buf(int how, int size, int* ext_type) 70 { 71 object_cache *cache; 72 int extType; 73 if (size != MCLBYTES && size != MJUM9BYTES && size != MJUMPAGESIZE) 74 panic("unsupported size"); 75 76 if (size == MCLBYTES) { 77 cache = sChunkCache; 78 extType = EXT_CLUSTER; 79 } else if (size == MJUM9BYTES) { 80 cache = sJumbo9ChunkCache; 81 extType = EXT_JUMBO9; 82 } else { 83 cache = sJumboPageSizeCache; 84 extType = EXT_JUMBOP; 85 } 86 87 if (ext_type != NULL) 88 *ext_type = extType; 89 return object_cache_alloc(cache, m_to_oc_flags(how)); 90 } 91 92 93 static int 94 construct_ext_sized_mbuf(struct mbuf *memoryBuffer, int how, int size) 95 { 96 int extType; 97 98 memoryBuffer->m_ext.ext_buf = allocate_ext_buf(how, size, &extType); 99 if (memoryBuffer->m_ext.ext_buf == NULL) 100 return B_NO_MEMORY; 101 102 memoryBuffer->m_data = memoryBuffer->m_ext.ext_buf; 103 memoryBuffer->m_flags |= M_EXT; 104 memoryBuffer->m_ext.ext_size = size; 105 memoryBuffer->m_ext.ext_type = extType; 106 memoryBuffer->m_ext.ext_flags = EXT_FLAG_EMBREF; 107 memoryBuffer->m_ext.ext_count = 1; 108 109 return 0; 110 } 111 112 113 static inline int 114 construct_ext_mbuf(struct mbuf *memoryBuffer, int how) 115 { 116 return construct_ext_sized_mbuf(memoryBuffer, how, MCLBYTES); 117 } 118 119 120 static int 121 construct_pkt_mbuf(int how, struct mbuf *memoryBuffer, short type, int flags) 122 { 123 if (m_init(memoryBuffer, how, type, flags) < 0) 124 return -1; 125 if (construct_ext_mbuf(memoryBuffer, how) < 0) 126 return -1; 127 memoryBuffer->m_ext.ext_type = EXT_CLUSTER; 128 return 0; 129 } 130 131 132 struct mbuf * 133 m_getcl(int how, short type, int flags) 134 { 135 struct mbuf *memoryBuffer = 136 (struct mbuf *)object_cache_alloc(sMBufCache, m_to_oc_flags(how)); 137 if (memoryBuffer == NULL) 138 return NULL; 139 140 if (construct_pkt_mbuf(how, memoryBuffer, type, flags) < 0) { 141 object_cache_free(sMBufCache, memoryBuffer, 0); 142 return NULL; 143 } 144 145 return memoryBuffer; 146 } 147 148 149 static struct mbuf * 150 _m_get(int how, short type, int flags) 151 { 152 struct mbuf *memoryBuffer = 153 (struct mbuf *)object_cache_alloc(sMBufCache, m_to_oc_flags(how)); 154 if (memoryBuffer == NULL) 155 return NULL; 156 157 m_init(memoryBuffer, how, type, flags); 158 159 return memoryBuffer; 160 } 161 162 163 struct mbuf * 164 m_get(int how, short type) 165 { 166 return _m_get(how, type, 0); 167 } 168 169 170 struct mbuf * 171 m_get2(int size, int how, short type, int flags) 172 { 173 if (size <= MHLEN || (size <= MLEN && (flags & M_PKTHDR) == 0)) { 174 size = MCLBYTES; 175 } else if (size <= MJUMPAGESIZE) { 176 size = MJUMPAGESIZE; 177 } else if (size <= MJUM9BYTES) { 178 size = MJUM9BYTES; 179 } else /* (size > MJUM9BYTES) */ { 180 return NULL; 181 } 182 183 return m_getjcl(how, type, flags, size); 184 } 185 186 187 struct mbuf * 188 m_gethdr(int how, short type) 189 { 190 return _m_get(how, type, M_PKTHDR); 191 } 192 193 194 struct mbuf * 195 m_getjcl(int how, short type, int flags, int size) 196 { 197 struct mbuf *memoryBuffer = 198 (struct mbuf *)object_cache_alloc(sMBufCache, m_to_oc_flags(how)); 199 if (memoryBuffer == NULL) 200 return NULL; 201 if (m_init(memoryBuffer, how, type, flags) < 0) { 202 object_cache_free(sMBufCache, memoryBuffer, 0); 203 return NULL; 204 } 205 if (construct_ext_sized_mbuf(memoryBuffer, how, size) < 0) { 206 object_cache_free(sMBufCache, memoryBuffer, 0); 207 return NULL; 208 } 209 return memoryBuffer; 210 } 211 212 213 int 214 m_clget(struct mbuf *memoryBuffer, int how) 215 { 216 memoryBuffer->m_ext.ext_buf = NULL; 217 /* called checks for errors by looking for M_EXT */ 218 construct_ext_mbuf(memoryBuffer, how); 219 return memoryBuffer->m_flags & M_EXT; 220 } 221 222 223 void* 224 m_cljget(struct mbuf* memoryBuffer, int how, int size) 225 { 226 if (memoryBuffer == NULL) 227 return allocate_ext_buf(how, size, NULL); 228 229 memoryBuffer->m_ext.ext_buf = NULL; 230 construct_ext_sized_mbuf(memoryBuffer, how, size); 231 return memoryBuffer->m_ext.ext_buf; 232 } 233 234 235 static void 236 mb_free_ext(struct mbuf *memoryBuffer) 237 { 238 volatile u_int *refcnt; 239 struct mbuf *mref; 240 int freembuf; 241 242 KASSERT(memoryBuffer->m_flags & M_EXT, ("%s: M_EXT not set on %p", 243 __func__, memoryBuffer)); 244 245 /* See if this is the mbuf that holds the embedded refcount. */ 246 if (memoryBuffer->m_ext.ext_flags & EXT_FLAG_EMBREF) { 247 refcnt = &memoryBuffer->m_ext.ext_count; 248 mref = memoryBuffer; 249 } else { 250 KASSERT(memoryBuffer->m_ext.ext_cnt != NULL, 251 ("%s: no refcounting pointer on %p", __func__, memoryBuffer)); 252 refcnt = memoryBuffer->m_ext.ext_cnt; 253 mref = __containerof(refcnt, struct mbuf, m_ext.ext_count); 254 } 255 256 /* 257 * Check if the header is embedded in the cluster. It is 258 * important that we can't touch any of the mbuf fields 259 * after we have freed the external storage, since mbuf 260 * could have been embedded in it. For now, the mbufs 261 * embedded into the cluster are always of type EXT_EXTREF, 262 * and for this type we won't free the mref. 263 */ 264 if (memoryBuffer->m_flags & M_NOFREE) { 265 freembuf = 0; 266 KASSERT(memoryBuffer->m_ext.ext_type == EXT_EXTREF, 267 ("%s: no-free mbuf %p has wrong type", __func__, memoryBuffer)); 268 } else 269 freembuf = 1; 270 271 /* Free attached storage only if this mbuf is the only reference to it. */ 272 if (*refcnt == 1 || atomic_add((int32*)refcnt, -1) == 1) { 273 object_cache *cache = NULL; 274 275 if (memoryBuffer->m_ext.ext_type == EXT_CLUSTER) 276 cache = sChunkCache; 277 else if (memoryBuffer->m_ext.ext_type == EXT_JUMBO9) 278 cache = sJumbo9ChunkCache; 279 else if (memoryBuffer->m_ext.ext_type == EXT_JUMBOP) 280 cache = sJumboPageSizeCache; 281 else 282 panic("unknown mbuf ext_type %d", memoryBuffer->m_ext.ext_type); 283 284 object_cache_free(cache, memoryBuffer->m_ext.ext_buf, 0); 285 object_cache_free(sMBufCache, mref, 0); 286 } 287 288 if (freembuf && memoryBuffer != mref) 289 object_cache_free(sMBufCache, memoryBuffer, 0); 290 } 291 292 293 struct mbuf * 294 m_free(struct mbuf* memoryBuffer) 295 { 296 struct mbuf* next = memoryBuffer->m_next; 297 298 if ((memoryBuffer->m_flags & (M_PKTHDR|M_NOFREE)) == (M_PKTHDR|M_NOFREE)) 299 m_tag_delete_chain(memoryBuffer, NULL); 300 if (memoryBuffer->m_flags & M_EXT) 301 mb_free_ext(memoryBuffer); 302 else if ((memoryBuffer->m_flags & M_NOFREE) == 0) 303 object_cache_free(sMBufCache, memoryBuffer, 0); 304 305 return next; 306 } 307 308 309 status_t 310 init_mbufs() 311 { 312 sMBufCache = create_object_cache("mbufs", MSIZE, 8, NULL, NULL, NULL); 313 if (sMBufCache == NULL) 314 goto clean; 315 sChunkCache = create_object_cache("mbuf chunks", MCLBYTES, 0, NULL, NULL, 316 NULL); 317 if (sChunkCache == NULL) 318 goto clean; 319 sJumbo9ChunkCache = create_object_cache("mbuf jumbo9 chunks", MJUM9BYTES, 0, 320 NULL, NULL, NULL); 321 if (sJumbo9ChunkCache == NULL) 322 goto clean; 323 sJumboPageSizeCache = create_object_cache("mbuf jumbo page size chunks", 324 MJUMPAGESIZE, 0, NULL, NULL, NULL); 325 if (sJumboPageSizeCache == NULL) 326 goto clean; 327 return B_OK; 328 329 clean: 330 if (sJumbo9ChunkCache != NULL) 331 delete_object_cache(sJumbo9ChunkCache); 332 if (sChunkCache != NULL) 333 delete_object_cache(sChunkCache); 334 if (sMBufCache != NULL) 335 delete_object_cache(sMBufCache); 336 return B_NO_MEMORY; 337 } 338 339 340 void 341 uninit_mbufs() 342 { 343 delete_object_cache(sMBufCache); 344 delete_object_cache(sChunkCache); 345 delete_object_cache(sJumbo9ChunkCache); 346 delete_object_cache(sJumboPageSizeCache); 347 } 348