1 /* 2 * Copyright 2007, Hugo Santos. All Rights Reserved. 3 * Copyright 2004, Marcus Overhagen. All Rights Reserved. 4 * Distributed under the terms of the MIT License. 5 */ 6 7 8 #include "device.h" 9 10 #include <stdint.h> 11 #include <string.h> 12 #include <slab/Slab.h> 13 14 #include <compat/sys/malloc.h> 15 #include <compat/sys/mbuf.h> 16 #include <compat/sys/kernel.h> 17 18 19 static object_cache *sMBufCache; 20 static object_cache *sChunkCache; 21 static object_cache *sJumbo9ChunkCache; 22 static object_cache *sJumboPageSizeCache; 23 24 25 int max_linkhdr = 16; 26 int max_protohdr = 40 + 20; /* ip6 + tcp */ 27 28 /* max_linkhdr + max_protohdr, but that's not allowed by gcc. */ 29 int max_hdr = 16 + 40 + 20; 30 31 /* MHLEN - max_hdr */ 32 int max_datalen = MHLEN - (16 + 40 + 20); 33 34 35 static int 36 m_to_oc_flags(int how) 37 { 38 if (how & M_NOWAIT) 39 return CACHE_DONT_WAIT_FOR_MEMORY; 40 41 return 0; 42 } 43 44 45 int 46 m_init(struct mbuf *m, int how, short type, int flags) 47 { 48 int error; 49 50 if (type == MT_NOINIT) 51 return 0; 52 53 m->m_next = NULL; 54 m->m_nextpkt = NULL; 55 m->m_data = m->m_dat; 56 m->m_len = 0; 57 m->m_flags = flags; 58 m->m_type = type; 59 if (flags & M_PKTHDR) 60 error = m_pkthdr_init(m, how); 61 else 62 error = 0; 63 64 return (error); 65 } 66 67 68 static void* 69 allocate_ext_buf(int how, int size, int* ext_type) 70 { 71 object_cache *cache; 72 int extType; 73 if (size != MCLBYTES && size != MJUM9BYTES && size != MJUMPAGESIZE) 74 panic("unsupported size"); 75 76 if (size == MCLBYTES) { 77 cache = sChunkCache; 78 extType = EXT_CLUSTER; 79 } else if (size == MJUM9BYTES) { 80 cache = sJumbo9ChunkCache; 81 extType = EXT_JUMBO9; 82 } else { 83 cache = sJumboPageSizeCache; 84 extType = EXT_JUMBOP; 85 } 86 87 if (ext_type != NULL) 88 *ext_type = extType; 89 return object_cache_alloc(cache, m_to_oc_flags(how)); 90 } 91 92 93 static int 94 construct_ext_sized_mbuf(struct mbuf *memoryBuffer, int how, int size) 95 { 96 int extType; 97 98 memoryBuffer->m_ext.ext_buf = allocate_ext_buf(how, size, &extType); 99 if (memoryBuffer->m_ext.ext_buf == NULL) 100 return B_NO_MEMORY; 101 102 memoryBuffer->m_data = memoryBuffer->m_ext.ext_buf; 103 memoryBuffer->m_flags |= M_EXT; 104 memoryBuffer->m_ext.ext_size = size; 105 memoryBuffer->m_ext.ext_type = extType; 106 memoryBuffer->m_ext.ext_flags = EXT_FLAG_EMBREF; 107 memoryBuffer->m_ext.ext_count = 1; 108 109 return 0; 110 } 111 112 113 static inline int 114 construct_ext_mbuf(struct mbuf *memoryBuffer, int how) 115 { 116 return construct_ext_sized_mbuf(memoryBuffer, how, MCLBYTES); 117 } 118 119 120 static int 121 construct_pkt_mbuf(int how, struct mbuf *memoryBuffer, short type, int flags) 122 { 123 if (m_init(memoryBuffer, how, type, flags) < 0) 124 return -1; 125 if (construct_ext_mbuf(memoryBuffer, how) < 0) 126 return -1; 127 memoryBuffer->m_ext.ext_type = EXT_CLUSTER; 128 return 0; 129 } 130 131 132 struct mbuf * 133 m_getcl(int how, short type, int flags) 134 { 135 struct mbuf *memoryBuffer = 136 (struct mbuf *)object_cache_alloc(sMBufCache, m_to_oc_flags(how)); 137 if (memoryBuffer == NULL) 138 return NULL; 139 140 if (construct_pkt_mbuf(how, memoryBuffer, type, flags) < 0) { 141 object_cache_free(sMBufCache, memoryBuffer, 0); 142 return NULL; 143 } 144 145 return memoryBuffer; 146 } 147 148 149 static struct mbuf * 150 _m_get(int how, short type, int flags) 151 { 152 struct mbuf *memoryBuffer = 153 (struct mbuf *)object_cache_alloc(sMBufCache, m_to_oc_flags(how)); 154 if (memoryBuffer == NULL) 155 return NULL; 156 157 m_init(memoryBuffer, how, type, flags); 158 159 return memoryBuffer; 160 } 161 162 163 struct mbuf * 164 m_get(int how, short type) 165 { 166 return _m_get(how, type, 0); 167 } 168 169 170 struct mbuf * 171 m_get2(int size, int how, short type, int flags) 172 { 173 if (size <= MHLEN || (size <= MLEN && (flags & M_PKTHDR) == 0)) { 174 size = MCLBYTES; 175 } else if (size <= MJUMPAGESIZE) { 176 size = MJUMPAGESIZE; 177 } else if (size <= MJUM9BYTES) { 178 size = MJUM9BYTES; 179 } else /* (size > MJUM9BYTES) */ { 180 return NULL; 181 } 182 183 return m_getjcl(how, type, flags, size); 184 } 185 186 187 struct mbuf * 188 m_gethdr(int how, short type) 189 { 190 return _m_get(how, type, M_PKTHDR); 191 } 192 193 194 struct mbuf * 195 m_getjcl(int how, short type, int flags, int size) 196 { 197 struct mbuf *memoryBuffer = 198 (struct mbuf *)object_cache_alloc(sMBufCache, m_to_oc_flags(how)); 199 if (memoryBuffer == NULL) 200 return NULL; 201 if (m_init(memoryBuffer, how, type, flags) < 0) { 202 object_cache_free(sMBufCache, memoryBuffer, 0); 203 return NULL; 204 } 205 if (construct_ext_sized_mbuf(memoryBuffer, how, size) < 0) { 206 object_cache_free(sMBufCache, memoryBuffer, 0); 207 return NULL; 208 } 209 return memoryBuffer; 210 } 211 212 213 int 214 m_clget(struct mbuf *memoryBuffer, int how) 215 { 216 memoryBuffer->m_ext.ext_buf = NULL; 217 /* called checks for errors by looking for M_EXT */ 218 construct_ext_mbuf(memoryBuffer, how); 219 return memoryBuffer->m_flags & M_EXT; 220 } 221 222 223 void* 224 m_cljget(struct mbuf* memoryBuffer, int how, int size) 225 { 226 if (memoryBuffer == NULL) 227 return allocate_ext_buf(how, size, NULL); 228 229 memoryBuffer->m_ext.ext_buf = NULL; 230 construct_ext_sized_mbuf(memoryBuffer, how, size); 231 return memoryBuffer->m_ext.ext_buf; 232 } 233 234 235 static void 236 mb_free_ext(struct mbuf *memoryBuffer) 237 { 238 object_cache *cache = NULL; 239 volatile u_int *refcnt; 240 struct mbuf *mref; 241 int freembuf; 242 243 KASSERT(memoryBuffer->m_flags & M_EXT, ("%s: M_EXT not set on %p", 244 __func__, memoryBuffer)); 245 246 /* See if this is the mbuf that holds the embedded refcount. */ 247 if (memoryBuffer->m_ext.ext_flags & EXT_FLAG_EMBREF) { 248 refcnt = &memoryBuffer->m_ext.ext_count; 249 mref = memoryBuffer; 250 } else { 251 KASSERT(memoryBuffer->m_ext.ext_cnt != NULL, 252 ("%s: no refcounting pointer on %p", __func__, memoryBuffer)); 253 refcnt = memoryBuffer->m_ext.ext_cnt; 254 mref = __containerof(refcnt, struct mbuf, m_ext.ext_count); 255 } 256 257 /* 258 * Check if the header is embedded in the cluster. It is 259 * important that we can't touch any of the mbuf fields 260 * after we have freed the external storage, since mbuf 261 * could have been embedded in it. For now, the mbufs 262 * embedded into the cluster are always of type EXT_EXTREF, 263 * and for this type we won't free the mref. 264 */ 265 if (memoryBuffer->m_flags & M_NOFREE) { 266 freembuf = 0; 267 KASSERT(memoryBuffer->m_ext.ext_type == EXT_EXTREF, 268 ("%s: no-free mbuf %p has wrong type", __func__, memoryBuffer)); 269 } else 270 freembuf = 1; 271 272 /* Free attached storage only if this mbuf is the only reference to it. */ 273 if (!(*refcnt == 1 || atomic_add(refcnt, -1) == 1) 274 && !(freembuf && memoryBuffer != mref)) 275 return; 276 277 if (memoryBuffer->m_ext.ext_type == EXT_CLUSTER) 278 cache = sChunkCache; 279 else if (memoryBuffer->m_ext.ext_type == EXT_JUMBO9) 280 cache = sJumbo9ChunkCache; 281 else if (memoryBuffer->m_ext.ext_type == EXT_JUMBOP) 282 cache = sJumboPageSizeCache; 283 else 284 panic("unknown mbuf ext_type %d", memoryBuffer->m_ext.ext_type); 285 286 object_cache_free(cache, memoryBuffer->m_ext.ext_buf, 0); 287 memoryBuffer->m_ext.ext_buf = NULL; 288 object_cache_free(sMBufCache, memoryBuffer, 0); 289 } 290 291 292 struct mbuf * 293 m_free(struct mbuf* memoryBuffer) 294 { 295 struct mbuf* next = memoryBuffer->m_next; 296 297 if ((memoryBuffer->m_flags & (M_PKTHDR|M_NOFREE)) == (M_PKTHDR|M_NOFREE)) 298 m_tag_delete_chain(memoryBuffer, NULL); 299 if (memoryBuffer->m_flags & M_EXT) 300 mb_free_ext(memoryBuffer); 301 else if ((memoryBuffer->m_flags & M_NOFREE) == 0) 302 object_cache_free(sMBufCache, memoryBuffer, 0); 303 304 return next; 305 } 306 307 308 status_t 309 init_mbufs() 310 { 311 sMBufCache = create_object_cache("mbufs", MSIZE, 8, NULL, NULL, NULL); 312 if (sMBufCache == NULL) 313 goto clean; 314 sChunkCache = create_object_cache("mbuf chunks", MCLBYTES, 0, NULL, NULL, 315 NULL); 316 if (sChunkCache == NULL) 317 goto clean; 318 sJumbo9ChunkCache = create_object_cache("mbuf jumbo9 chunks", MJUM9BYTES, 0, 319 NULL, NULL, NULL); 320 if (sJumbo9ChunkCache == NULL) 321 goto clean; 322 sJumboPageSizeCache = create_object_cache("mbuf jumbo page size chunks", 323 MJUMPAGESIZE, 0, NULL, NULL, NULL); 324 if (sJumboPageSizeCache == NULL) 325 goto clean; 326 return B_OK; 327 328 clean: 329 if (sJumbo9ChunkCache != NULL) 330 delete_object_cache(sJumbo9ChunkCache); 331 if (sChunkCache != NULL) 332 delete_object_cache(sChunkCache); 333 if (sMBufCache != NULL) 334 delete_object_cache(sMBufCache); 335 return B_NO_MEMORY; 336 } 337 338 339 void 340 uninit_mbufs() 341 { 342 delete_object_cache(sMBufCache); 343 delete_object_cache(sChunkCache); 344 delete_object_cache(sJumbo9ChunkCache); 345 delete_object_cache(sJumboPageSizeCache); 346 } 347