1 /* 2 * Copyright 2007, Hugo Santos. All Rights Reserved. 3 * Copyright 2004, Marcus Overhagen. All Rights Reserved. 4 * Distributed under the terms of the MIT License. 5 */ 6 7 8 #include "device.h" 9 10 #include <stdint.h> 11 #include <string.h> 12 #include <slab/Slab.h> 13 14 #include <net_buffer.h> 15 16 #include <compat/sys/malloc.h> 17 #include <compat/sys/mbuf.h> 18 #include <compat/sys/kernel.h> 19 20 21 static object_cache *sMBufCache; 22 static object_cache *sChunkCache; 23 static object_cache *sJumbo9ChunkCache; 24 static object_cache *sJumboPageSizeCache; 25 26 27 int max_linkhdr = 16; 28 int max_protohdr = 40 + 20; /* ip6 + tcp */ 29 30 /* max_linkhdr + max_protohdr, but that's not allowed by gcc. */ 31 int max_hdr = 16 + 40 + 20; 32 33 /* MHLEN - max_hdr */ 34 int max_datalen = MHLEN - (16 + 40 + 20); 35 36 37 static int 38 m_to_oc_flags(int how) 39 { 40 if (how & M_NOWAIT) 41 return CACHE_DONT_WAIT_FOR_MEMORY; 42 43 return 0; 44 } 45 46 47 int 48 m_init(struct mbuf *m, int how, short type, int flags) 49 { 50 int error; 51 52 if (type == MT_NOINIT) 53 return 0; 54 55 m->m_next = NULL; 56 m->m_nextpkt = NULL; 57 m->m_data = m->m_dat; 58 m->m_len = 0; 59 m->m_flags = flags; 60 m->m_type = type; 61 if (flags & M_PKTHDR) 62 error = m_pkthdr_init(m, how); 63 else 64 error = 0; 65 66 return (error); 67 } 68 69 70 static void* 71 allocate_ext_buf(int how, int size, int* ext_type) 72 { 73 object_cache *cache; 74 int extType; 75 if (size != MCLBYTES && size != MJUM9BYTES && size != MJUMPAGESIZE) 76 panic("unsupported size"); 77 78 if (size == MCLBYTES) { 79 cache = sChunkCache; 80 extType = EXT_CLUSTER; 81 } else if (size == MJUM9BYTES) { 82 cache = sJumbo9ChunkCache; 83 extType = EXT_JUMBO9; 84 } else { 85 cache = sJumboPageSizeCache; 86 extType = EXT_JUMBOP; 87 } 88 89 if (ext_type != NULL) 90 *ext_type = extType; 91 return object_cache_alloc(cache, m_to_oc_flags(how)); 92 } 93 94 95 static int 96 construct_ext_sized_mbuf(struct mbuf *memoryBuffer, int how, int size) 97 { 98 int extType; 99 100 memoryBuffer->m_ext.ext_buf = allocate_ext_buf(how, size, &extType); 101 if (memoryBuffer->m_ext.ext_buf == NULL) 102 return B_NO_MEMORY; 103 104 memoryBuffer->m_data = memoryBuffer->m_ext.ext_buf; 105 memoryBuffer->m_flags |= M_EXT; 106 memoryBuffer->m_ext.ext_size = size; 107 memoryBuffer->m_ext.ext_type = extType; 108 memoryBuffer->m_ext.ext_flags = EXT_FLAG_EMBREF; 109 memoryBuffer->m_ext.ext_count = 1; 110 111 return 0; 112 } 113 114 115 static inline int 116 construct_ext_mbuf(struct mbuf *memoryBuffer, int how) 117 { 118 return construct_ext_sized_mbuf(memoryBuffer, how, MCLBYTES); 119 } 120 121 122 static int 123 construct_pkt_mbuf(int how, struct mbuf *memoryBuffer, short type, int flags) 124 { 125 if (m_init(memoryBuffer, how, type, flags) < 0) 126 return -1; 127 if (construct_ext_mbuf(memoryBuffer, how) < 0) 128 return -1; 129 memoryBuffer->m_ext.ext_type = EXT_CLUSTER; 130 return 0; 131 } 132 133 134 struct mbuf * 135 m_getcl(int how, short type, int flags) 136 { 137 struct mbuf *memoryBuffer = 138 (struct mbuf *)object_cache_alloc(sMBufCache, m_to_oc_flags(how)); 139 if (memoryBuffer == NULL) 140 return NULL; 141 142 if (construct_pkt_mbuf(how, memoryBuffer, type, flags) < 0) { 143 object_cache_free(sMBufCache, memoryBuffer, 0); 144 return NULL; 145 } 146 147 return memoryBuffer; 148 } 149 150 151 static struct mbuf * 152 _m_get(int how, short type, int flags) 153 { 154 struct mbuf *memoryBuffer = 155 (struct mbuf *)object_cache_alloc(sMBufCache, m_to_oc_flags(how)); 156 if (memoryBuffer == NULL) 157 return NULL; 158 159 m_init(memoryBuffer, how, type, flags); 160 161 return memoryBuffer; 162 } 163 164 165 struct mbuf * 166 m_get(int how, short type) 167 { 168 return _m_get(how, type, 0); 169 } 170 171 172 struct mbuf * 173 m_get2(int size, int how, short type, int flags) 174 { 175 if (size <= MHLEN || (size <= MLEN && (flags & M_PKTHDR) == 0)) { 176 return _m_get(how, type, flags); 177 } else if (size <= MCLBYTES) { 178 size = MCLBYTES; 179 } else if (size <= MJUMPAGESIZE) { 180 size = MJUMPAGESIZE; 181 } else if (size <= MJUM9BYTES) { 182 size = MJUM9BYTES; 183 } else /* (size > MJUM9BYTES) */ { 184 return NULL; 185 } 186 187 return m_getjcl(how, type, flags, size); 188 } 189 190 191 struct mbuf * 192 m_gethdr(int how, short type) 193 { 194 return _m_get(how, type, M_PKTHDR); 195 } 196 197 198 struct mbuf * 199 m_getjcl(int how, short type, int flags, int size) 200 { 201 struct mbuf *memoryBuffer = 202 (struct mbuf *)object_cache_alloc(sMBufCache, m_to_oc_flags(how)); 203 if (memoryBuffer == NULL) 204 return NULL; 205 if (m_init(memoryBuffer, how, type, flags) < 0) { 206 object_cache_free(sMBufCache, memoryBuffer, 0); 207 return NULL; 208 } 209 if (construct_ext_sized_mbuf(memoryBuffer, how, size) < 0) { 210 object_cache_free(sMBufCache, memoryBuffer, 0); 211 return NULL; 212 } 213 return memoryBuffer; 214 } 215 216 217 int 218 m_clget(struct mbuf *memoryBuffer, int how) 219 { 220 memoryBuffer->m_ext.ext_buf = NULL; 221 /* called checks for errors by looking for M_EXT */ 222 construct_ext_mbuf(memoryBuffer, how); 223 return memoryBuffer->m_flags & M_EXT; 224 } 225 226 227 void* 228 m_cljget(struct mbuf* memoryBuffer, int how, int size) 229 { 230 if (memoryBuffer == NULL) 231 return allocate_ext_buf(how, size, NULL); 232 233 memoryBuffer->m_ext.ext_buf = NULL; 234 construct_ext_sized_mbuf(memoryBuffer, how, size); 235 return memoryBuffer->m_ext.ext_buf; 236 } 237 238 239 static void 240 mb_free_ext(struct mbuf *memoryBuffer) 241 { 242 volatile u_int *refcnt; 243 struct mbuf *mref; 244 int freembuf; 245 246 KASSERT(memoryBuffer->m_flags & M_EXT, ("%s: M_EXT not set on %p", 247 __func__, memoryBuffer)); 248 249 /* See if this is the mbuf that holds the embedded refcount. */ 250 if (memoryBuffer->m_ext.ext_flags & EXT_FLAG_EMBREF) { 251 refcnt = &memoryBuffer->m_ext.ext_count; 252 mref = memoryBuffer; 253 } else { 254 KASSERT(memoryBuffer->m_ext.ext_cnt != NULL, 255 ("%s: no refcounting pointer on %p", __func__, memoryBuffer)); 256 refcnt = memoryBuffer->m_ext.ext_cnt; 257 mref = __containerof(refcnt, struct mbuf, m_ext.ext_count); 258 } 259 260 /* 261 * Check if the header is embedded in the cluster. It is 262 * important that we can't touch any of the mbuf fields 263 * after we have freed the external storage, since mbuf 264 * could have been embedded in it. For now, the mbufs 265 * embedded into the cluster are always of type EXT_EXTREF, 266 * and for this type we won't free the mref. 267 */ 268 if (memoryBuffer->m_flags & M_NOFREE) { 269 freembuf = 0; 270 KASSERT(memoryBuffer->m_ext.ext_type == EXT_EXTREF, 271 ("%s: no-free mbuf %p has wrong type", __func__, memoryBuffer)); 272 } else 273 freembuf = 1; 274 275 /* Free attached storage only if this mbuf is the only reference to it. */ 276 if (*refcnt == 1 || atomic_add((int32*)refcnt, -1) == 1) { 277 object_cache *cache = NULL; 278 279 if (memoryBuffer->m_ext.ext_type == EXT_CLUSTER) 280 cache = sChunkCache; 281 else if (memoryBuffer->m_ext.ext_type == EXT_JUMBO9) 282 cache = sJumbo9ChunkCache; 283 else if (memoryBuffer->m_ext.ext_type == EXT_JUMBOP) 284 cache = sJumboPageSizeCache; 285 else 286 panic("unknown mbuf ext_type %d", memoryBuffer->m_ext.ext_type); 287 288 object_cache_free(cache, memoryBuffer->m_ext.ext_buf, 0); 289 object_cache_free(sMBufCache, mref, 0); 290 } 291 292 if (freembuf && memoryBuffer != mref) 293 object_cache_free(sMBufCache, memoryBuffer, 0); 294 } 295 296 297 struct mbuf * 298 m_free(struct mbuf* memoryBuffer) 299 { 300 struct mbuf* next = memoryBuffer->m_next; 301 302 if ((memoryBuffer->m_flags & (M_PKTHDR|M_NOFREE)) == (M_PKTHDR|M_NOFREE)) 303 m_tag_delete_chain(memoryBuffer, NULL); 304 if (memoryBuffer->m_flags & M_EXT) 305 mb_free_ext(memoryBuffer); 306 else if ((memoryBuffer->m_flags & M_NOFREE) == 0) 307 object_cache_free(sMBufCache, memoryBuffer, 0); 308 309 return next; 310 } 311 312 313 status_t 314 init_mbufs() 315 { 316 status_t status = get_module(NET_BUFFER_MODULE_NAME, (module_info **)&gBufferModule); 317 if (status != B_OK) 318 goto clean; 319 320 sMBufCache = create_object_cache("mbufs", MSIZE, 8, NULL, NULL, NULL); 321 if (sMBufCache == NULL) 322 goto clean; 323 sChunkCache = create_object_cache("mbuf chunks", MCLBYTES, 0, NULL, NULL, 324 NULL); 325 if (sChunkCache == NULL) 326 goto clean; 327 sJumbo9ChunkCache = create_object_cache("mbuf jumbo9 chunks", MJUM9BYTES, 0, 328 NULL, NULL, NULL); 329 if (sJumbo9ChunkCache == NULL) 330 goto clean; 331 sJumboPageSizeCache = create_object_cache("mbuf page size chunks", 332 MJUMPAGESIZE, 0, NULL, NULL, NULL); 333 if (sJumboPageSizeCache == NULL) 334 goto clean; 335 return B_OK; 336 337 clean: 338 if (sJumbo9ChunkCache != NULL) 339 delete_object_cache(sJumbo9ChunkCache); 340 if (sChunkCache != NULL) 341 delete_object_cache(sChunkCache); 342 if (sMBufCache != NULL) 343 delete_object_cache(sMBufCache); 344 put_module(NET_BUFFER_MODULE_NAME); 345 return B_NO_MEMORY; 346 } 347 348 349 void 350 uninit_mbufs() 351 { 352 delete_object_cache(sMBufCache); 353 delete_object_cache(sChunkCache); 354 delete_object_cache(sJumbo9ChunkCache); 355 delete_object_cache(sJumboPageSizeCache); 356 } 357