1 /* 2 * Copyright 2002-2005, Axel Dörfler, axeld@pinc-software.de. 3 * Distributed under the terms of the MIT License. 4 * 5 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 6 * Distributed under the terms of the NewOS License. 7 */ 8 #ifndef KERNEL_SMP_H 9 #define KERNEL_SMP_H 10 11 12 #include <arch/atomic.h> 13 #include <boot/kernel_args.h> 14 #include <kernel.h> 15 16 #include <KernelExport.h> 17 18 #include <string.h> 19 20 21 struct kernel_args; 22 23 24 // intercpu messages 25 enum { 26 SMP_MSG_INVALIDATE_PAGE_RANGE = 0, 27 SMP_MSG_INVALIDATE_PAGE_LIST, 28 SMP_MSG_USER_INVALIDATE_PAGES, 29 SMP_MSG_GLOBAL_INVALIDATE_PAGES, 30 SMP_MSG_CPU_HALT, 31 SMP_MSG_CALL_FUNCTION, 32 SMP_MSG_RESCHEDULE 33 }; 34 35 enum { 36 SMP_MSG_FLAG_ASYNC = 0x0, 37 SMP_MSG_FLAG_SYNC = 0x1, 38 SMP_MSG_FLAG_FREE_ARG = 0x2, 39 }; 40 41 typedef void (*smp_call_func)(addr_t data1, int32 currentCPU, addr_t data2, addr_t data3); 42 43 class CPUSet { 44 public: 45 inline CPUSet(); 46 47 inline void ClearAll(); 48 inline void SetAll(); 49 50 inline void SetBit(int32 cpu); 51 inline void ClearBit(int32 cpu); 52 53 inline void SetBitAtomic(int32 cpu); 54 inline void ClearBitAtomic(int32 cpu); 55 56 inline bool GetBit(int32 cpu) const; 57 58 inline bool IsEmpty() const; 59 60 private: 61 static const int kArraySize = ROUNDUP(SMP_MAX_CPUS, 32) / 32; 62 63 uint32 fBitmap[kArraySize]; 64 }; 65 66 67 #ifdef __cplusplus 68 extern "C" { 69 #endif 70 71 bool try_acquire_spinlock(spinlock* lock); 72 73 status_t smp_init(struct kernel_args *args); 74 status_t smp_per_cpu_init(struct kernel_args *args, int32 cpu); 75 status_t smp_init_post_generic_syscalls(void); 76 bool smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous); 77 void smp_wake_up_non_boot_cpus(void); 78 void smp_cpu_rendezvous(uint32* var); 79 void smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, addr_t data3, 80 void *data_ptr, uint32 flags); 81 void smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data, 82 addr_t data2, addr_t data3, void *data_ptr, uint32 flags); 83 void smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3, 84 void *data_ptr, uint32 flags); 85 void smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message, 86 addr_t data, addr_t data2, addr_t data3, void *data_ptr, uint32 flags); 87 88 int32 smp_get_num_cpus(void); 89 void smp_set_num_cpus(int32 numCPUs); 90 int32 smp_get_current_cpu(void); 91 92 int smp_intercpu_int_handler(int32 cpu); 93 94 void call_single_cpu(uint32 targetCPU, void (*func)(void*, int), void* cookie); 95 void call_single_cpu_sync(uint32 targetCPU, void (*func)(void*, int), 96 void* cookie); 97 98 99 #ifdef __cplusplus 100 } 101 #endif 102 103 104 inline 105 CPUSet::CPUSet() 106 { 107 memset(fBitmap, 0, sizeof(fBitmap)); 108 } 109 110 111 inline void 112 CPUSet::ClearAll() 113 { 114 memset(fBitmap, 0, sizeof(fBitmap)); 115 } 116 117 118 inline void 119 CPUSet::SetAll() 120 { 121 memset(fBitmap, ~uint8(0), sizeof(fBitmap)); 122 } 123 124 125 inline void 126 CPUSet::SetBit(int32 cpu) 127 { 128 int32* element = (int32*)&fBitmap[cpu % kArraySize]; 129 *element |= 1u << (cpu / kArraySize); 130 } 131 132 133 inline void 134 CPUSet::ClearBit(int32 cpu) 135 { 136 int32* element = (int32*)&fBitmap[cpu % kArraySize]; 137 *element &= ~uint32(1u << (cpu / kArraySize)); 138 } 139 140 141 inline void 142 CPUSet::SetBitAtomic(int32 cpu) 143 { 144 int32* element = (int32*)&fBitmap[cpu % kArraySize]; 145 atomic_or(element, 1u << (cpu / kArraySize)); 146 } 147 148 149 inline void 150 CPUSet::ClearBitAtomic(int32 cpu) 151 { 152 int32* element = (int32*)&fBitmap[cpu % kArraySize]; 153 atomic_and(element, ~uint32(1u << (cpu / kArraySize))); 154 } 155 156 157 inline bool 158 CPUSet::GetBit(int32 cpu) const 159 { 160 int32* element = (int32*)&fBitmap[cpu % kArraySize]; 161 return ((uint32)atomic_get(element) & (1u << (cpu / kArraySize))) != 0; 162 } 163 164 165 inline bool 166 CPUSet::IsEmpty() const 167 { 168 for (int i = 0; i < kArraySize; i++) { 169 if (fBitmap[i] != 0) 170 return false; 171 } 172 173 return true; 174 } 175 176 177 // Unless spinlock debug features are enabled, try to inline 178 // {acquire,release}_spinlock(). 179 #if !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION 180 181 182 static inline bool 183 try_acquire_spinlock_inline(spinlock* lock) 184 { 185 return atomic_get_and_set(&lock->lock, 1) == 0; 186 } 187 188 189 static inline void 190 acquire_spinlock_inline(spinlock* lock) 191 { 192 if (try_acquire_spinlock_inline(lock)) 193 return; 194 acquire_spinlock(lock); 195 } 196 197 198 static inline void 199 release_spinlock_inline(spinlock* lock) 200 { 201 atomic_set(&lock->lock, 0); 202 } 203 204 205 #define try_acquire_spinlock(lock) try_acquire_spinlock_inline(lock) 206 #define acquire_spinlock(lock) acquire_spinlock_inline(lock) 207 #define release_spinlock(lock) release_spinlock_inline(lock) 208 209 210 static inline bool 211 try_acquire_write_spinlock_inline(rw_spinlock* lock) 212 { 213 return atomic_test_and_set(&lock->lock, 1u << 31, 0) == 0; 214 } 215 216 217 static inline void 218 acquire_write_spinlock_inline(rw_spinlock* lock) 219 { 220 if (try_acquire_write_spinlock(lock)) 221 return; 222 acquire_write_spinlock(lock); 223 } 224 225 226 static inline void 227 release_write_spinlock_inline(rw_spinlock* lock) 228 { 229 atomic_set(&lock->lock, 0); 230 } 231 232 233 static inline bool 234 try_acquire_read_spinlock_inline(rw_spinlock* lock) 235 { 236 uint32 previous = atomic_add(&lock->lock, 1); 237 return (previous & (1u << 31)) == 0; 238 } 239 240 241 static inline void 242 acquire_read_spinlock_inline(rw_spinlock* lock) 243 { 244 if (try_acquire_read_spinlock(lock)) 245 return; 246 acquire_read_spinlock(lock); 247 } 248 249 250 static inline void 251 release_read_spinlock_inline(rw_spinlock* lock) 252 { 253 atomic_add(&lock->lock, -1); 254 } 255 256 257 #define try_acquire_read_spinlock(lock) try_acquire_read_spinlock_inline(lock) 258 #define acquire_read_spinlock(lock) acquire_read_spinlock_inline(lock) 259 #define release_read_spinlock(lock) release_read_spinlock_inline(lock) 260 #define try_acquire_write_spinlock(lock) \ 261 try_acquire_write_spinlock(lock) 262 #define acquire_write_spinlock(lock) acquire_write_spinlock_inline(lock) 263 #define release_write_spinlock(lock) release_write_spinlock_inline(lock) 264 265 266 static inline bool 267 try_acquire_write_seqlock_inline(seqlock* lock) { 268 bool succeed = try_acquire_spinlock(&lock->lock); 269 if (succeed) 270 atomic_add((int32*)&lock->count, 1); 271 return succeed; 272 } 273 274 275 static inline void 276 acquire_write_seqlock_inline(seqlock* lock) { 277 acquire_spinlock(&lock->lock); 278 atomic_add((int32*)&lock->count, 1); 279 } 280 281 282 static inline void 283 release_write_seqlock_inline(seqlock* lock) { 284 atomic_add((int32*)&lock->count, 1); 285 release_spinlock(&lock->lock); 286 } 287 288 289 static inline uint32 290 acquire_read_seqlock_inline(seqlock* lock) 291 { 292 return (uint32)atomic_get((int32*)&lock->count); 293 } 294 295 296 static inline bool 297 release_read_seqlock_inline(seqlock* lock, uint32 count) 298 { 299 uint32 current = (uint32)atomic_get((int32*)&lock->count); 300 301 return count % 2 == 0 && current == count; 302 } 303 304 305 #define try_acquire_write_seqlock(lock) try_acquire_write_seqlock_inline(lock) 306 #define acquire_write_seqlock(lock) acquire_write_seqlock_inline(lock) 307 #define release_write_seqlock(lock) release_write_seqlock_inline(lock) 308 #define acquire_read_seqlock(lock) acquire_read_seqlock_inline(lock) 309 #define release_read_seqlock(lock, count) \ 310 release_read_seqlock_inline(lock, count) 311 312 313 #endif // !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION 314 315 316 #endif /* KERNEL_SMP_H */ 317