1 /* 2 * Copyright 2002-2005, Axel Dörfler, axeld@pinc-software.de. 3 * Distributed under the terms of the MIT License. 4 * 5 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 6 * Distributed under the terms of the NewOS License. 7 */ 8 #ifndef KERNEL_SMP_H 9 #define KERNEL_SMP_H 10 11 12 #include <arch/atomic.h> 13 #include <boot/kernel_args.h> 14 #include <kernel.h> 15 16 #include <KernelExport.h> 17 18 #include <string.h> 19 20 21 struct kernel_args; 22 23 24 // intercpu messages 25 enum { 26 SMP_MSG_INVALIDATE_PAGE_RANGE = 0, 27 SMP_MSG_INVALIDATE_PAGE_LIST, 28 SMP_MSG_USER_INVALIDATE_PAGES, 29 SMP_MSG_GLOBAL_INVALIDATE_PAGES, 30 SMP_MSG_CPU_HALT, 31 SMP_MSG_CALL_FUNCTION, 32 SMP_MSG_RESCHEDULE 33 }; 34 35 enum { 36 SMP_MSG_FLAG_ASYNC = 0x0, 37 SMP_MSG_FLAG_SYNC = 0x1, 38 SMP_MSG_FLAG_FREE_ARG = 0x2, 39 }; 40 41 typedef void (*smp_call_func)(addr_t data1, int32 currentCPU, addr_t data2, addr_t data3); 42 43 class CPUSet { 44 public: 45 inline CPUSet(); 46 47 inline void ClearAll(); 48 inline void SetAll(); 49 50 inline void SetBit(int32 cpu); 51 inline void ClearBit(int32 cpu); 52 53 inline void SetBitAtomic(int32 cpu); 54 inline void ClearBitAtomic(int32 cpu); 55 56 inline bool GetBit(int32 cpu) const; 57 58 inline bool Matches(const CPUSet& mask) const; 59 inline CPUSet And(const CPUSet& mask) const; 60 61 inline bool IsEmpty() const; 62 63 inline uint32 Bits(uint32 index) const { return fBitmap[index];} 64 private: 65 static const int kArrayBits = 32; 66 static const int kArraySize = ROUNDUP(SMP_MAX_CPUS, kArrayBits) / kArrayBits; 67 68 uint32 fBitmap[kArraySize]; 69 }; 70 71 72 #ifdef __cplusplus 73 extern "C" { 74 #endif 75 76 bool try_acquire_spinlock(spinlock* lock); 77 78 status_t smp_init(struct kernel_args *args); 79 status_t smp_per_cpu_init(struct kernel_args *args, int32 cpu); 80 status_t smp_init_post_generic_syscalls(void); 81 bool smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous); 82 void smp_wake_up_non_boot_cpus(void); 83 void smp_cpu_rendezvous(uint32* var); 84 void smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, addr_t data3, 85 void *data_ptr, uint32 flags); 86 void smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data, 87 addr_t data2, addr_t data3, void *data_ptr, uint32 flags); 88 void smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3, 89 void *data_ptr, uint32 flags); 90 void smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message, 91 addr_t data, addr_t data2, addr_t data3, void *data_ptr, uint32 flags); 92 93 int32 smp_get_num_cpus(void); 94 void smp_set_num_cpus(int32 numCPUs); 95 int32 smp_get_current_cpu(void); 96 97 int smp_intercpu_int_handler(int32 cpu); 98 99 void call_single_cpu(uint32 targetCPU, void (*func)(void*, int), void* cookie); 100 void call_single_cpu_sync(uint32 targetCPU, void (*func)(void*, int), 101 void* cookie); 102 103 104 #ifdef __cplusplus 105 } 106 #endif 107 108 109 inline 110 CPUSet::CPUSet() 111 { 112 memset(fBitmap, 0, sizeof(fBitmap)); 113 } 114 115 116 inline void 117 CPUSet::ClearAll() 118 { 119 memset(fBitmap, 0, sizeof(fBitmap)); 120 } 121 122 123 inline void 124 CPUSet::SetAll() 125 { 126 memset(fBitmap, ~uint8(0), sizeof(fBitmap)); 127 } 128 129 130 inline void 131 CPUSet::SetBit(int32 cpu) 132 { 133 int32* element = (int32*)&fBitmap[cpu / kArrayBits]; 134 *element |= 1u << (cpu % kArrayBits); 135 } 136 137 138 inline void 139 CPUSet::ClearBit(int32 cpu) 140 { 141 int32* element = (int32*)&fBitmap[cpu / kArrayBits]; 142 *element &= ~uint32(1u << (cpu % kArrayBits)); 143 } 144 145 146 inline void 147 CPUSet::SetBitAtomic(int32 cpu) 148 { 149 int32* element = (int32*)&fBitmap[cpu / kArrayBits]; 150 atomic_or(element, 1u << (cpu % kArrayBits)); 151 } 152 153 154 inline void 155 CPUSet::ClearBitAtomic(int32 cpu) 156 { 157 int32* element = (int32*)&fBitmap[cpu / kArrayBits]; 158 atomic_and(element, ~uint32(1u << (cpu % kArrayBits))); 159 } 160 161 162 inline bool 163 CPUSet::GetBit(int32 cpu) const 164 { 165 int32* element = (int32*)&fBitmap[cpu / kArrayBits]; 166 return ((uint32)atomic_get(element) & (1u << (cpu % kArrayBits))) != 0; 167 } 168 169 170 inline CPUSet 171 CPUSet::And(const CPUSet& mask) const 172 { 173 CPUSet andSet; 174 for (int i = 0; i < kArraySize; i++) 175 andSet.fBitmap[i] = fBitmap[i] & mask.fBitmap[i]; 176 return andSet; 177 } 178 179 180 inline bool 181 CPUSet::Matches(const CPUSet& mask) const 182 { 183 for (int i = 0; i < kArraySize; i++) { 184 if ((fBitmap[i] & mask.fBitmap[i]) != 0) 185 return true; 186 } 187 188 return false; 189 } 190 191 192 inline bool 193 CPUSet::IsEmpty() const 194 { 195 for (int i = 0; i < kArraySize; i++) { 196 if (fBitmap[i] != 0) 197 return false; 198 } 199 200 return true; 201 } 202 203 204 // Unless spinlock debug features are enabled, try to inline 205 // {acquire,release}_spinlock(). 206 #if !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION 207 208 209 static inline bool 210 try_acquire_spinlock_inline(spinlock* lock) 211 { 212 return atomic_get_and_set(&lock->lock, 1) == 0; 213 } 214 215 216 static inline void 217 acquire_spinlock_inline(spinlock* lock) 218 { 219 if (try_acquire_spinlock_inline(lock)) 220 return; 221 acquire_spinlock(lock); 222 } 223 224 225 static inline void 226 release_spinlock_inline(spinlock* lock) 227 { 228 atomic_set(&lock->lock, 0); 229 } 230 231 232 #define try_acquire_spinlock(lock) try_acquire_spinlock_inline(lock) 233 #define acquire_spinlock(lock) acquire_spinlock_inline(lock) 234 #define release_spinlock(lock) release_spinlock_inline(lock) 235 236 237 static inline bool 238 try_acquire_write_spinlock_inline(rw_spinlock* lock) 239 { 240 return atomic_test_and_set(&lock->lock, 1u << 31, 0) == 0; 241 } 242 243 244 static inline void 245 acquire_write_spinlock_inline(rw_spinlock* lock) 246 { 247 if (try_acquire_write_spinlock(lock)) 248 return; 249 acquire_write_spinlock(lock); 250 } 251 252 253 static inline void 254 release_write_spinlock_inline(rw_spinlock* lock) 255 { 256 atomic_set(&lock->lock, 0); 257 } 258 259 260 static inline bool 261 try_acquire_read_spinlock_inline(rw_spinlock* lock) 262 { 263 uint32 previous = atomic_add(&lock->lock, 1); 264 return (previous & (1u << 31)) == 0; 265 } 266 267 268 static inline void 269 acquire_read_spinlock_inline(rw_spinlock* lock) 270 { 271 if (try_acquire_read_spinlock(lock)) 272 return; 273 acquire_read_spinlock(lock); 274 } 275 276 277 static inline void 278 release_read_spinlock_inline(rw_spinlock* lock) 279 { 280 atomic_add(&lock->lock, -1); 281 } 282 283 284 #define try_acquire_read_spinlock(lock) try_acquire_read_spinlock_inline(lock) 285 #define acquire_read_spinlock(lock) acquire_read_spinlock_inline(lock) 286 #define release_read_spinlock(lock) release_read_spinlock_inline(lock) 287 #define try_acquire_write_spinlock(lock) \ 288 try_acquire_write_spinlock(lock) 289 #define acquire_write_spinlock(lock) acquire_write_spinlock_inline(lock) 290 #define release_write_spinlock(lock) release_write_spinlock_inline(lock) 291 292 293 static inline bool 294 try_acquire_write_seqlock_inline(seqlock* lock) { 295 bool succeed = try_acquire_spinlock(&lock->lock); 296 if (succeed) 297 atomic_add((int32*)&lock->count, 1); 298 return succeed; 299 } 300 301 302 static inline void 303 acquire_write_seqlock_inline(seqlock* lock) { 304 acquire_spinlock(&lock->lock); 305 atomic_add((int32*)&lock->count, 1); 306 } 307 308 309 static inline void 310 release_write_seqlock_inline(seqlock* lock) { 311 atomic_add((int32*)&lock->count, 1); 312 release_spinlock(&lock->lock); 313 } 314 315 316 static inline uint32 317 acquire_read_seqlock_inline(seqlock* lock) 318 { 319 return (uint32)atomic_get((int32*)&lock->count); 320 } 321 322 323 static inline bool 324 release_read_seqlock_inline(seqlock* lock, uint32 count) 325 { 326 uint32 current = (uint32)atomic_get((int32*)&lock->count); 327 328 return count % 2 == 0 && current == count; 329 } 330 331 332 #define try_acquire_write_seqlock(lock) try_acquire_write_seqlock_inline(lock) 333 #define acquire_write_seqlock(lock) acquire_write_seqlock_inline(lock) 334 #define release_write_seqlock(lock) release_write_seqlock_inline(lock) 335 #define acquire_read_seqlock(lock) acquire_read_seqlock_inline(lock) 336 #define release_read_seqlock(lock, count) \ 337 release_read_seqlock_inline(lock, count) 338 339 340 #endif // !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION 341 342 343 #endif /* KERNEL_SMP_H */ 344