1 /*
2 * Copyright 2008-2010, Axel Dörfler. All Rights Reserved.
3 * Copyright 2007, Hugo Santos. All Rights Reserved.
4 *
5 * Distributed under the terms of the MIT License.
6 */
7
8
9 #include "ObjectCache.h"
10
11 #include <string.h>
12
13 #include <smp.h>
14 #include <util/AutoLock.h>
15 #include <vm/vm.h>
16 #include <vm/VMAddressSpace.h>
17
18 #include "MemoryManager.h"
19 #include "slab_private.h"
20
21
RANGE_MARKER_FUNCTION_BEGIN(SlabObjectCache)22 RANGE_MARKER_FUNCTION_BEGIN(SlabObjectCache)
23
24
25 static void
26 object_cache_return_object_wrapper(object_depot* depot, void* cookie,
27 void* object, uint32 flags)
28 {
29 ObjectCache* cache = (ObjectCache*)cookie;
30
31 MutexLocker _(cache->lock);
32 cache->ReturnObjectToSlab(cache->ObjectSlab(object), object, flags);
33 }
34
35
36 // #pragma mark -
37
38
~ObjectCache()39 ObjectCache::~ObjectCache()
40 {
41 }
42
43
44 status_t
Init(const char * name,size_t objectSize,size_t alignment,size_t maximum,size_t magazineCapacity,size_t maxMagazineCount,uint32 flags,void * cookie,object_cache_constructor constructor,object_cache_destructor destructor,object_cache_reclaimer reclaimer)45 ObjectCache::Init(const char* name, size_t objectSize, size_t alignment,
46 size_t maximum, size_t magazineCapacity, size_t maxMagazineCount,
47 uint32 flags, void* cookie, object_cache_constructor constructor,
48 object_cache_destructor destructor, object_cache_reclaimer reclaimer)
49 {
50 strlcpy(this->name, name, sizeof(this->name));
51
52 mutex_init(&lock, this->name);
53
54 if (objectSize < sizeof(object_link))
55 objectSize = sizeof(object_link);
56
57 if (alignment < kMinObjectAlignment)
58 alignment = kMinObjectAlignment;
59
60 if (alignment > 0 && (objectSize & (alignment - 1)))
61 object_size = objectSize + alignment - (objectSize & (alignment - 1));
62 else
63 object_size = objectSize;
64
65 TRACE_CACHE(this, "init %lu, %lu -> %lu", objectSize, alignment,
66 object_size);
67
68 this->alignment = alignment;
69 cache_color_cycle = 0;
70 total_objects = 0;
71 used_count = 0;
72 empty_count = 0;
73 pressure = 0;
74 min_object_reserve = 0;
75
76 maintenance_pending = false;
77 maintenance_in_progress = false;
78 maintenance_resize = false;
79 maintenance_delete = false;
80
81 usage = 0;
82 this->maximum = maximum;
83
84 this->flags = flags;
85
86 resize_request = NULL;
87 resize_entry_can_wait = NULL;
88 resize_entry_dont_wait = NULL;
89
90 // no gain in using the depot in single cpu setups
91 if (smp_get_num_cpus() == 1)
92 this->flags |= CACHE_NO_DEPOT;
93
94 if (!(this->flags & CACHE_NO_DEPOT)) {
95 // Determine usable magazine configuration values if none had been given
96 if (magazineCapacity == 0) {
97 magazineCapacity = objectSize < 256
98 ? 32 : (objectSize < 512 ? 16 : 8);
99 }
100 if (maxMagazineCount == 0)
101 maxMagazineCount = magazineCapacity / 2;
102
103 status_t status = object_depot_init(&depot, magazineCapacity,
104 maxMagazineCount, flags, this, object_cache_return_object_wrapper);
105 if (status != B_OK) {
106 mutex_destroy(&lock);
107 return status;
108 }
109 }
110
111 this->cookie = cookie;
112 this->constructor = constructor;
113 this->destructor = destructor;
114 this->reclaimer = reclaimer;
115
116 return B_OK;
117 }
118
119
120 slab*
InitSlab(slab * slab,void * pages,size_t byteCount,uint32 flags)121 ObjectCache::InitSlab(slab* slab, void* pages, size_t byteCount, uint32 flags)
122 {
123 TRACE_CACHE(this, "construct (%p, %p .. %p, %lu)", slab, pages,
124 ((uint8*)pages) + byteCount, byteCount);
125
126 slab->pages = pages;
127 slab->count = slab->size = byteCount / object_size;
128 slab->free = NULL;
129
130 size_t spareBytes = byteCount - (slab->size * object_size);
131
132 slab->offset = cache_color_cycle;
133
134 cache_color_cycle += alignment;
135 if (cache_color_cycle > spareBytes)
136 cache_color_cycle = 0;
137
138 TRACE_CACHE(this, " %lu objects, %lu spare bytes, offset %lu",
139 slab->size, spareBytes, slab->offset);
140
141 uint8* data = ((uint8*)pages) + slab->offset;
142
143 CREATE_PARANOIA_CHECK_SET(slab, "slab");
144
145
146 for (size_t i = 0; i < slab->size; i++) {
147 status_t status = B_OK;
148 if (constructor)
149 status = constructor(cookie, data);
150
151 if (status != B_OK) {
152 data = ((uint8*)pages) + slab->offset;
153 for (size_t j = 0; j < i; j++) {
154 if (destructor)
155 destructor(cookie, data);
156 data += object_size;
157 }
158
159 DELETE_PARANOIA_CHECK_SET(slab);
160
161 return NULL;
162 }
163
164 _push(slab->free, object_to_link(data, object_size));
165
166 ADD_PARANOIA_CHECK(PARANOIA_SUSPICIOUS, slab,
167 &object_to_link(data, object_size)->next, sizeof(void*));
168
169 data += object_size;
170 }
171
172 return slab;
173 }
174
175
176 void
UninitSlab(slab * slab)177 ObjectCache::UninitSlab(slab* slab)
178 {
179 TRACE_CACHE(this, "destruct %p", slab);
180
181 if (slab->count != slab->size)
182 panic("cache: destroying a slab which isn't empty.");
183
184 usage -= slab_size;
185 total_objects -= slab->size;
186
187 DELETE_PARANOIA_CHECK_SET(slab);
188
189 uint8* data = ((uint8*)slab->pages) + slab->offset;
190
191 for (size_t i = 0; i < slab->size; i++) {
192 if (destructor)
193 destructor(cookie, data);
194 data += object_size;
195 }
196 }
197
198
199 void
ReturnObjectToSlab(slab * source,void * object,uint32 flags)200 ObjectCache::ReturnObjectToSlab(slab* source, void* object, uint32 flags)
201 {
202 if (source == NULL) {
203 panic("object_cache: free'd object %p has no slab", object);
204 return;
205 }
206
207 ParanoiaChecker _(source);
208
209 #if KDEBUG >= 1
210 uint8* objectsStart = (uint8*)source->pages + source->offset;
211 if (object < objectsStart
212 || object >= objectsStart + source->size * object_size
213 || ((uint8*)object - objectsStart) % object_size != 0) {
214 panic("object_cache: tried to free invalid object pointer %p", object);
215 return;
216 }
217 #endif // KDEBUG
218
219 object_link* link = object_to_link(object, object_size);
220
221 TRACE_CACHE(this, "returning %p (%p) to %p, %lu used (%lu empty slabs).",
222 object, link, source, source->size - source->count,
223 empty_count);
224
225 _push(source->free, link);
226 source->count++;
227 used_count--;
228
229 ADD_PARANOIA_CHECK(PARANOIA_SUSPICIOUS, source, &link->next, sizeof(void*));
230
231 if (source->count == source->size) {
232 partial.Remove(source);
233
234 if (empty_count < pressure
235 || (total_objects - (used_count + source->size))
236 < min_object_reserve) {
237 empty_count++;
238 empty.Add(source);
239 } else {
240 ReturnSlab(source, flags);
241 }
242 } else if (source->count == 1) {
243 full.Remove(source);
244 partial.Add(source);
245 }
246 }
247
248
249 void*
ObjectAtIndex(slab * source,int32 index) const250 ObjectCache::ObjectAtIndex(slab* source, int32 index) const
251 {
252 return (uint8*)source->pages + source->offset + index * object_size;
253 }
254
255
256 #if PARANOID_KERNEL_FREE
257
258 bool
AssertObjectNotFreed(void * object)259 ObjectCache::AssertObjectNotFreed(void* object)
260 {
261 MutexLocker locker(lock);
262
263 slab* source = ObjectSlab(object);
264 if (!partial.Contains(source) && !full.Contains(source)) {
265 panic("object_cache: to be freed object %p: slab not part of cache!",
266 object);
267 return false;
268 }
269
270 object_link* link = object_to_link(object, object_size);
271 for (object_link* freeLink = source->free; freeLink != NULL;
272 freeLink = freeLink->next) {
273 if (freeLink == link) {
274 panic("object_cache: double free of %p (slab %p, cache %p)",
275 object, source, this);
276 return false;
277 }
278 }
279
280 return true;
281 }
282
283 #endif // PARANOID_KERNEL_FREE
284
285
286 #if SLAB_OBJECT_CACHE_ALLOCATION_TRACKING
287
288 status_t
AllocateTrackingInfos(slab * slab,size_t byteCount,uint32 flags)289 ObjectCache::AllocateTrackingInfos(slab* slab, size_t byteCount, uint32 flags)
290 {
291 void* pages;
292 size_t objectCount = byteCount / object_size;
293 status_t result = MemoryManager::AllocateRaw(
294 objectCount * sizeof(AllocationTrackingInfo), flags, pages);
295 if (result == B_OK) {
296 slab->tracking = (AllocationTrackingInfo*)pages;
297 for (size_t i = 0; i < objectCount; i++)
298 slab->tracking[i].Clear();
299 }
300
301 return result;
302 }
303
304
305 void
FreeTrackingInfos(slab * slab,uint32 flags)306 ObjectCache::FreeTrackingInfos(slab* slab, uint32 flags)
307 {
308 MemoryManager::FreeRawOrReturnCache(slab->tracking, flags);
309 }
310
311
312 AllocationTrackingInfo*
TrackingInfoFor(void * object) const313 ObjectCache::TrackingInfoFor(void* object) const
314 {
315 slab* objectSlab = ObjectSlab(object);
316 return &objectSlab->tracking[((addr_t)object - objectSlab->offset
317 - (addr_t)objectSlab->pages) / object_size];
318 }
319
320 #endif // SLAB_OBJECT_CACHE_ALLOCATION_TRACKING
321
322
323 RANGE_MARKER_FUNCTION_END(SlabObjectCache)
324