1 /*
2 * Copyright 2002-2005, Axel Dörfler, axeld@pinc-software.de.
3 * Distributed under the terms of the MIT License.
4 *
5 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
6 * Distributed under the terms of the NewOS License.
7 */
8 #ifndef KERNEL_SMP_H
9 #define KERNEL_SMP_H
10
11
12 #include <arch/atomic.h>
13 #include <boot/kernel_args.h>
14 #include <kernel.h>
15
16 #include <KernelExport.h>
17
18 #include <string.h>
19
20
21 struct kernel_args;
22
23
24 // intercpu messages
25 enum {
26 SMP_MSG_INVALIDATE_PAGE_RANGE = 0,
27 SMP_MSG_INVALIDATE_PAGE_LIST,
28 SMP_MSG_USER_INVALIDATE_PAGES,
29 SMP_MSG_GLOBAL_INVALIDATE_PAGES,
30 SMP_MSG_CPU_HALT,
31 SMP_MSG_CALL_FUNCTION,
32 SMP_MSG_RESCHEDULE
33 };
34
35 enum {
36 SMP_MSG_FLAG_ASYNC = 0x0,
37 SMP_MSG_FLAG_SYNC = 0x1,
38 SMP_MSG_FLAG_FREE_ARG = 0x2,
39 };
40
41 typedef void (*smp_call_func)(addr_t data1, int32 currentCPU, addr_t data2, addr_t data3);
42
43 class CPUSet {
44 public:
45 inline CPUSet();
46
47 inline void ClearAll();
48 inline void SetAll();
49
50 inline void SetBit(int32 cpu);
51 inline void ClearBit(int32 cpu);
52
53 inline void SetBitAtomic(int32 cpu);
54 inline void ClearBitAtomic(int32 cpu);
55
56 inline bool GetBit(int32 cpu) const;
57
58 inline bool Matches(const CPUSet& mask) const;
59 inline CPUSet And(const CPUSet& mask) const;
60
61 inline bool IsEmpty() const;
62
Bits(uint32 index)63 inline uint32 Bits(uint32 index) const { return fBitmap[index];}
64 private:
65 static const int kArrayBits = 32;
66 static const int kArraySize = ROUNDUP(SMP_MAX_CPUS, kArrayBits) / kArrayBits;
67
68 uint32 fBitmap[kArraySize];
69 };
70
71
72 #ifdef __cplusplus
73 extern "C" {
74 #endif
75
76 bool try_acquire_spinlock(spinlock* lock);
77
78 status_t smp_init(struct kernel_args *args);
79 status_t smp_per_cpu_init(struct kernel_args *args, int32 cpu);
80 status_t smp_init_post_generic_syscalls(void);
81 bool smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous);
82 void smp_wake_up_non_boot_cpus(void);
83 void smp_cpu_rendezvous(uint32* var);
84 void smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, addr_t data3,
85 void *data_ptr, uint32 flags);
86 void smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data,
87 addr_t data2, addr_t data3, void *data_ptr, uint32 flags);
88 void smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3,
89 void *data_ptr, uint32 flags);
90 void smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message,
91 addr_t data, addr_t data2, addr_t data3, void *data_ptr, uint32 flags);
92
93 int32 smp_get_num_cpus(void);
94 void smp_set_num_cpus(int32 numCPUs);
95 int32 smp_get_current_cpu(void);
96
97 int smp_intercpu_int_handler(int32 cpu);
98
99 void call_single_cpu(uint32 targetCPU, void (*func)(void*, int), void* cookie);
100 void call_single_cpu_sync(uint32 targetCPU, void (*func)(void*, int),
101 void* cookie);
102
103
104 #ifdef __cplusplus
105 }
106 #endif
107
108
109 inline
CPUSet()110 CPUSet::CPUSet()
111 {
112 memset(fBitmap, 0, sizeof(fBitmap));
113 }
114
115
116 inline void
ClearAll()117 CPUSet::ClearAll()
118 {
119 memset(fBitmap, 0, sizeof(fBitmap));
120 }
121
122
123 inline void
SetAll()124 CPUSet::SetAll()
125 {
126 memset(fBitmap, ~uint8(0), sizeof(fBitmap));
127 }
128
129
130 inline void
SetBit(int32 cpu)131 CPUSet::SetBit(int32 cpu)
132 {
133 int32* element = (int32*)&fBitmap[cpu / kArrayBits];
134 *element |= 1u << (cpu % kArrayBits);
135 }
136
137
138 inline void
ClearBit(int32 cpu)139 CPUSet::ClearBit(int32 cpu)
140 {
141 int32* element = (int32*)&fBitmap[cpu / kArrayBits];
142 *element &= ~uint32(1u << (cpu % kArrayBits));
143 }
144
145
146 inline void
SetBitAtomic(int32 cpu)147 CPUSet::SetBitAtomic(int32 cpu)
148 {
149 int32* element = (int32*)&fBitmap[cpu / kArrayBits];
150 atomic_or(element, 1u << (cpu % kArrayBits));
151 }
152
153
154 inline void
ClearBitAtomic(int32 cpu)155 CPUSet::ClearBitAtomic(int32 cpu)
156 {
157 int32* element = (int32*)&fBitmap[cpu / kArrayBits];
158 atomic_and(element, ~uint32(1u << (cpu % kArrayBits)));
159 }
160
161
162 inline bool
GetBit(int32 cpu)163 CPUSet::GetBit(int32 cpu) const
164 {
165 int32* element = (int32*)&fBitmap[cpu / kArrayBits];
166 return ((uint32)atomic_get(element) & (1u << (cpu % kArrayBits))) != 0;
167 }
168
169
170 inline CPUSet
And(const CPUSet & mask)171 CPUSet::And(const CPUSet& mask) const
172 {
173 CPUSet andSet;
174 for (int i = 0; i < kArraySize; i++)
175 andSet.fBitmap[i] = fBitmap[i] & mask.fBitmap[i];
176 return andSet;
177 }
178
179
180 inline bool
Matches(const CPUSet & mask)181 CPUSet::Matches(const CPUSet& mask) const
182 {
183 for (int i = 0; i < kArraySize; i++) {
184 if ((fBitmap[i] & mask.fBitmap[i]) != 0)
185 return true;
186 }
187
188 return false;
189 }
190
191
192 inline bool
IsEmpty()193 CPUSet::IsEmpty() const
194 {
195 for (int i = 0; i < kArraySize; i++) {
196 if (fBitmap[i] != 0)
197 return false;
198 }
199
200 return true;
201 }
202
203
204 // Unless spinlock debug features are enabled, try to inline
205 // {acquire,release}_spinlock().
206 #if !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION
207
208
209 static inline bool
try_acquire_spinlock_inline(spinlock * lock)210 try_acquire_spinlock_inline(spinlock* lock)
211 {
212 return atomic_get_and_set(&lock->lock, 1) == 0;
213 }
214
215
216 static inline void
acquire_spinlock_inline(spinlock * lock)217 acquire_spinlock_inline(spinlock* lock)
218 {
219 if (try_acquire_spinlock_inline(lock))
220 return;
221 acquire_spinlock(lock);
222 }
223
224
225 static inline void
release_spinlock_inline(spinlock * lock)226 release_spinlock_inline(spinlock* lock)
227 {
228 atomic_set(&lock->lock, 0);
229 }
230
231
232 #define try_acquire_spinlock(lock) try_acquire_spinlock_inline(lock)
233 #define acquire_spinlock(lock) acquire_spinlock_inline(lock)
234 #define release_spinlock(lock) release_spinlock_inline(lock)
235
236
237 static inline bool
try_acquire_write_spinlock_inline(rw_spinlock * lock)238 try_acquire_write_spinlock_inline(rw_spinlock* lock)
239 {
240 return atomic_test_and_set(&lock->lock, 1u << 31, 0) == 0;
241 }
242
243
244 static inline void
acquire_write_spinlock_inline(rw_spinlock * lock)245 acquire_write_spinlock_inline(rw_spinlock* lock)
246 {
247 if (try_acquire_write_spinlock(lock))
248 return;
249 acquire_write_spinlock(lock);
250 }
251
252
253 static inline void
release_write_spinlock_inline(rw_spinlock * lock)254 release_write_spinlock_inline(rw_spinlock* lock)
255 {
256 atomic_set(&lock->lock, 0);
257 }
258
259
260 static inline bool
try_acquire_read_spinlock_inline(rw_spinlock * lock)261 try_acquire_read_spinlock_inline(rw_spinlock* lock)
262 {
263 uint32 previous = atomic_add(&lock->lock, 1);
264 return (previous & (1u << 31)) == 0;
265 }
266
267
268 static inline void
acquire_read_spinlock_inline(rw_spinlock * lock)269 acquire_read_spinlock_inline(rw_spinlock* lock)
270 {
271 if (try_acquire_read_spinlock(lock))
272 return;
273 acquire_read_spinlock(lock);
274 }
275
276
277 static inline void
release_read_spinlock_inline(rw_spinlock * lock)278 release_read_spinlock_inline(rw_spinlock* lock)
279 {
280 atomic_add(&lock->lock, -1);
281 }
282
283
284 #define try_acquire_read_spinlock(lock) try_acquire_read_spinlock_inline(lock)
285 #define acquire_read_spinlock(lock) acquire_read_spinlock_inline(lock)
286 #define release_read_spinlock(lock) release_read_spinlock_inline(lock)
287 #define try_acquire_write_spinlock(lock) \
288 try_acquire_write_spinlock(lock)
289 #define acquire_write_spinlock(lock) acquire_write_spinlock_inline(lock)
290 #define release_write_spinlock(lock) release_write_spinlock_inline(lock)
291
292
293 static inline bool
try_acquire_write_seqlock_inline(seqlock * lock)294 try_acquire_write_seqlock_inline(seqlock* lock) {
295 bool succeed = try_acquire_spinlock(&lock->lock);
296 if (succeed)
297 atomic_add((int32*)&lock->count, 1);
298 return succeed;
299 }
300
301
302 static inline void
acquire_write_seqlock_inline(seqlock * lock)303 acquire_write_seqlock_inline(seqlock* lock) {
304 acquire_spinlock(&lock->lock);
305 atomic_add((int32*)&lock->count, 1);
306 }
307
308
309 static inline void
release_write_seqlock_inline(seqlock * lock)310 release_write_seqlock_inline(seqlock* lock) {
311 atomic_add((int32*)&lock->count, 1);
312 release_spinlock(&lock->lock);
313 }
314
315
316 static inline uint32
acquire_read_seqlock_inline(seqlock * lock)317 acquire_read_seqlock_inline(seqlock* lock)
318 {
319 return (uint32)atomic_get((int32*)&lock->count);
320 }
321
322
323 static inline bool
release_read_seqlock_inline(seqlock * lock,uint32 count)324 release_read_seqlock_inline(seqlock* lock, uint32 count)
325 {
326 uint32 current = (uint32)atomic_get((int32*)&lock->count);
327
328 return count % 2 == 0 && current == count;
329 }
330
331
332 #define try_acquire_write_seqlock(lock) try_acquire_write_seqlock_inline(lock)
333 #define acquire_write_seqlock(lock) acquire_write_seqlock_inline(lock)
334 #define release_write_seqlock(lock) release_write_seqlock_inline(lock)
335 #define acquire_read_seqlock(lock) acquire_read_seqlock_inline(lock)
336 #define release_read_seqlock(lock, count) \
337 release_read_seqlock_inline(lock, count)
338
339
340 #endif // !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION
341
342
343 #endif /* KERNEL_SMP_H */
344