1 /* 2 * Copyright 2008-2009, Ingo Weinhold, ingo_weinhold@gmx.de. 3 * Copyright 2002-2008, Axel Dörfler, axeld@pinc-software.de. 4 * Distributed under the terms of the MIT License. 5 * 6 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 7 * Distributed under the terms of the NewOS License. 8 */ 9 10 /* Functionality for symetrical multi-processors */ 11 12 #include <smp.h> 13 14 #include <stdlib.h> 15 #include <string.h> 16 17 #include <arch/cpu.h> 18 #include <arch/debug.h> 19 #include <arch/int.h> 20 #include <arch/smp.h> 21 #include <cpu.h> 22 #include <generic_syscall.h> 23 #include <int.h> 24 #include <spinlock_contention.h> 25 #include <thread.h> 26 27 #include "kernel_debug_config.h" 28 29 30 //#define TRACE_SMP 31 32 #ifdef TRACE_SMP 33 # define TRACE(x) dprintf x 34 #else 35 # define TRACE(x) ; 36 #endif 37 38 #define MSG_POOL_SIZE (SMP_MAX_CPUS * 4) 39 40 // These macros define the number of unsuccessful iterations in 41 // acquire_spinlock() and acquire_spinlock_nocheck() after which the functions 42 // panic(), assuming a deadlock. 43 #define SPINLOCK_DEADLOCK_COUNT 100000000 44 #define SPINLOCK_DEADLOCK_COUNT_NO_CHECK 2000000000 45 46 47 struct smp_msg { 48 struct smp_msg *next; 49 int32 message; 50 uint32 data; 51 uint32 data2; 52 uint32 data3; 53 void *data_ptr; 54 uint32 flags; 55 int32 ref_count; 56 volatile bool done; 57 uint32 proc_bitmap; 58 }; 59 60 #define MAILBOX_LOCAL 1 61 #define MAILBOX_BCAST 2 62 63 static spinlock boot_cpu_spin[SMP_MAX_CPUS] = { }; 64 65 static struct smp_msg *sFreeMessages = NULL; 66 static volatile int sFreeMessageCount = 0; 67 static spinlock sFreeMessageSpinlock = B_SPINLOCK_INITIALIZER; 68 69 static struct smp_msg *sCPUMessages[SMP_MAX_CPUS] = { NULL, }; 70 static spinlock sCPUMessageSpinlock[SMP_MAX_CPUS]; 71 72 static struct smp_msg *sBroadcastMessages = NULL; 73 static spinlock sBroadcastMessageSpinlock = B_SPINLOCK_INITIALIZER; 74 75 static bool sICIEnabled = false; 76 static int32 sNumCPUs = 1; 77 78 static int32 process_pending_ici(int32 currentCPU); 79 80 81 #if DEBUG_SPINLOCKS 82 #define NUM_LAST_CALLERS 32 83 84 static struct { 85 void *caller; 86 spinlock *lock; 87 } sLastCaller[NUM_LAST_CALLERS]; 88 89 static vint32 sLastIndex = 0; 90 // Is incremented atomically. Must be % NUM_LAST_CALLERS before being used 91 // as index into sLastCaller. Note, that it has to be casted to uint32 92 // before applying the modulo operation, since otherwise after overflowing 93 // that would yield negative indices. 94 95 96 static void 97 push_lock_caller(void *caller, spinlock *lock) 98 { 99 int32 index = (uint32)atomic_add(&sLastIndex, 1) % NUM_LAST_CALLERS; 100 101 sLastCaller[index].caller = caller; 102 sLastCaller[index].lock = lock; 103 } 104 105 106 static void * 107 find_lock_caller(spinlock *lock) 108 { 109 int32 lastIndex = (uint32)sLastIndex % NUM_LAST_CALLERS; 110 111 for (int32 i = 0; i < NUM_LAST_CALLERS; i++) { 112 int32 index = (NUM_LAST_CALLERS + lastIndex - 1 - i) % NUM_LAST_CALLERS; 113 if (sLastCaller[index].lock == lock) 114 return sLastCaller[index].caller; 115 } 116 117 return NULL; 118 } 119 120 121 int 122 dump_spinlock(int argc, char** argv) 123 { 124 if (argc != 2) { 125 print_debugger_command_usage(argv[0]); 126 return 0; 127 } 128 129 uint64 address; 130 if (!evaluate_debug_expression(argv[1], &address, false)) 131 return 0; 132 133 spinlock* lock = (spinlock*)(addr_t)address; 134 kprintf("spinlock %p:\n", lock); 135 bool locked = B_SPINLOCK_IS_LOCKED(lock); 136 if (locked) { 137 kprintf(" locked from %p\n", find_lock_caller(lock)); 138 } else 139 kprintf(" not locked\n"); 140 141 return 0; 142 } 143 144 145 #endif // DEBUG_SPINLOCKS 146 147 148 int 149 dump_ici_messages(int argc, char** argv) 150 { 151 // count broadcast messages 152 int32 count = 0; 153 int32 doneCount = 0; 154 int32 unreferencedCount = 0; 155 smp_msg* message = sBroadcastMessages; 156 while (message != NULL) { 157 count++; 158 if (message->done) 159 doneCount++; 160 if (message->ref_count <= 0) 161 unreferencedCount++; 162 message = message->next; 163 } 164 165 kprintf("ICI broadcast messages: %ld, first: %p\n", count, 166 sBroadcastMessages); 167 kprintf(" done: %ld\n", doneCount); 168 kprintf(" unreferenced: %ld\n", unreferencedCount); 169 170 // count per-CPU messages 171 for (int32 i = 0; i < sNumCPUs; i++) { 172 count = 0; 173 message = sCPUMessages[i]; 174 while (message != NULL) { 175 count++; 176 message = message->next; 177 } 178 179 kprintf("CPU %ld messages: %ld, first: %p\n", i, count, 180 sCPUMessages[i]); 181 } 182 183 return 0; 184 } 185 186 187 int 188 dump_ici_message(int argc, char** argv) 189 { 190 if (argc != 2) { 191 print_debugger_command_usage(argv[0]); 192 return 0; 193 } 194 195 uint64 address; 196 if (!evaluate_debug_expression(argv[1], &address, false)) 197 return 0; 198 199 smp_msg* message = (smp_msg*)(addr_t)address; 200 kprintf("ICI message %p:\n", message); 201 kprintf(" next: %p\n", message->next); 202 kprintf(" message: %ld\n", message->message); 203 kprintf(" data: %ld\n", message->data); 204 kprintf(" data2: %ld\n", message->data2); 205 kprintf(" data3: %ld\n", message->data3); 206 kprintf(" data_ptr: %p\n", message->data_ptr); 207 kprintf(" flags: %lx\n", message->flags); 208 kprintf(" ref_count: %lx\n", message->ref_count); 209 kprintf(" done: %s\n", message->done ? "true" : "false"); 210 kprintf(" proc_bitmap: %lx\n", message->proc_bitmap); 211 212 return 0; 213 } 214 215 216 static inline void 217 process_all_pending_ici(int32 currentCPU) 218 { 219 while (process_pending_ici(currentCPU) != B_ENTRY_NOT_FOUND) 220 ; 221 } 222 223 224 void 225 acquire_spinlock(spinlock *lock) 226 { 227 #if DEBUG_SPINLOCKS 228 if (are_interrupts_enabled()) { 229 panic("acquire_spinlock: attempt to acquire lock %p with interrupts " 230 "enabled", lock); 231 } 232 #endif 233 234 if (sNumCPUs > 1) { 235 int currentCPU = smp_get_current_cpu(); 236 #if B_DEBUG_SPINLOCK_CONTENTION 237 while (atomic_add(&lock->lock, 1) != 0) 238 process_all_pending_ici(currentCPU); 239 #else 240 while (1) { 241 uint32 count = 0; 242 while (*lock != 0) { 243 if (++count == SPINLOCK_DEADLOCK_COUNT) { 244 panic("acquire_spinlock(): Failed to acquire spinlock %p " 245 "for a long time!", lock); 246 count = 0; 247 } 248 249 process_all_pending_ici(currentCPU); 250 PAUSE(); 251 } 252 if (atomic_set((int32 *)lock, 1) == 0) 253 break; 254 } 255 256 #if DEBUG_SPINLOCKS 257 push_lock_caller(arch_debug_get_caller(), lock); 258 #endif 259 #endif 260 } else { 261 #if DEBUG_SPINLOCKS 262 int32 oldValue; 263 oldValue = atomic_set((int32 *)lock, 1); 264 if (oldValue != 0) { 265 panic("acquire_spinlock: attempt to acquire lock %p twice on " 266 "non-SMP system (last caller: %p, value %ld)", lock, 267 find_lock_caller(lock), oldValue); 268 } 269 270 push_lock_caller(arch_debug_get_caller(), lock); 271 #endif 272 } 273 } 274 275 276 static void 277 acquire_spinlock_nocheck(spinlock *lock) 278 { 279 #if DEBUG_SPINLOCKS 280 if (are_interrupts_enabled()) { 281 panic("acquire_spinlock_nocheck: attempt to acquire lock %p with " 282 "interrupts enabled", lock); 283 } 284 #endif 285 286 if (sNumCPUs > 1) { 287 #if B_DEBUG_SPINLOCK_CONTENTION 288 while (atomic_add(&lock->lock, 1) != 0) { 289 } 290 #else 291 while (1) { 292 uint32 count = 0; 293 while (*lock != 0) { 294 if (++count == SPINLOCK_DEADLOCK_COUNT_NO_CHECK) { 295 panic("acquire_spinlock(): Failed to acquire spinlock %p " 296 "for a long time!", lock); 297 count = 0; 298 } 299 300 PAUSE(); 301 } 302 303 if (atomic_set((int32 *)lock, 1) == 0) 304 break; 305 } 306 #endif 307 } else { 308 #if DEBUG_SPINLOCKS 309 if (atomic_set((int32 *)lock, 1) != 0) { 310 panic("acquire_spinlock_nocheck: attempt to acquire lock %p twice " 311 "on non-SMP system\n", lock); 312 } 313 #endif 314 } 315 } 316 317 318 /*! Equivalent to acquire_spinlock(), save for currentCPU parameter. */ 319 static void 320 acquire_spinlock_cpu(int32 currentCPU, spinlock *lock) 321 { 322 #if DEBUG_SPINLOCKS 323 if (are_interrupts_enabled()) { 324 panic("acquire_spinlock_cpu: attempt to acquire lock %p with " 325 "interrupts enabled", lock); 326 } 327 #endif 328 329 if (sNumCPUs > 1) { 330 #if B_DEBUG_SPINLOCK_CONTENTION 331 while (atomic_add(&lock->lock, 1) != 0) 332 process_all_pending_ici(currentCPU); 333 #else 334 while (1) { 335 uint32 count = 0; 336 while (*lock != 0) { 337 if (++count == SPINLOCK_DEADLOCK_COUNT) { 338 panic("acquire_spinlock_cpu(): Failed to acquire spinlock " 339 "%p for a long time!", lock); 340 count = 0; 341 } 342 343 process_all_pending_ici(currentCPU); 344 PAUSE(); 345 } 346 if (atomic_set((int32 *)lock, 1) == 0) 347 break; 348 } 349 350 #if DEBUG_SPINLOCKS 351 push_lock_caller(arch_debug_get_caller(), lock); 352 #endif 353 #endif 354 } else { 355 #if DEBUG_SPINLOCKS 356 int32 oldValue; 357 oldValue = atomic_set((int32 *)lock, 1); 358 if (oldValue != 0) { 359 panic("acquire_spinlock_cpu(): attempt to acquire lock %p twice on " 360 "non-SMP system (last caller: %p, value %ld)", lock, 361 find_lock_caller(lock), oldValue); 362 } 363 364 push_lock_caller(arch_debug_get_caller(), lock); 365 #endif 366 } 367 } 368 369 370 void 371 release_spinlock(spinlock *lock) 372 { 373 if (sNumCPUs > 1) { 374 if (are_interrupts_enabled()) 375 panic("release_spinlock: attempt to release lock %p with interrupts enabled\n", lock); 376 #if B_DEBUG_SPINLOCK_CONTENTION 377 { 378 int32 count = atomic_set(&lock->lock, 0) - 1; 379 if (count < 0) { 380 panic("release_spinlock: lock %p was already released\n", lock); 381 } else { 382 // add to the total count -- deal with carry manually 383 if ((uint32)atomic_add(&lock->count_low, count) + count 384 < (uint32)count) { 385 atomic_add(&lock->count_high, 1); 386 } 387 } 388 } 389 #else 390 if (atomic_set((int32 *)lock, 0) != 1) 391 panic("release_spinlock: lock %p was already released\n", lock); 392 #endif 393 } else { 394 #if DEBUG_SPINLOCKS 395 if (are_interrupts_enabled()) 396 panic("release_spinlock: attempt to release lock %p with interrupts enabled\n", lock); 397 if (atomic_set((int32 *)lock, 0) != 1) 398 panic("release_spinlock: lock %p was already released\n", lock); 399 #endif 400 } 401 } 402 403 404 /** Finds a free message and gets it. 405 * NOTE: has side effect of disabling interrupts 406 * return value is the former interrupt state 407 */ 408 409 static cpu_status 410 find_free_message(struct smp_msg **msg) 411 { 412 cpu_status state; 413 414 TRACE(("find_free_message: entry\n")); 415 416 retry: 417 while (sFreeMessageCount <= 0) { 418 state = disable_interrupts(); 419 process_all_pending_ici(smp_get_current_cpu()); 420 restore_interrupts(state); 421 PAUSE(); 422 } 423 state = disable_interrupts(); 424 acquire_spinlock(&sFreeMessageSpinlock); 425 426 if (sFreeMessageCount <= 0) { 427 // someone grabbed one while we were getting the lock, 428 // go back to waiting for it 429 release_spinlock(&sFreeMessageSpinlock); 430 restore_interrupts(state); 431 goto retry; 432 } 433 434 *msg = sFreeMessages; 435 sFreeMessages = (*msg)->next; 436 sFreeMessageCount--; 437 438 release_spinlock(&sFreeMessageSpinlock); 439 440 TRACE(("find_free_message: returning msg %p\n", *msg)); 441 442 return state; 443 } 444 445 446 /*! Similar to find_free_message(), but expects the interrupts to be disabled 447 already. 448 */ 449 static void 450 find_free_message_interrupts_disabled(int32 currentCPU, 451 struct smp_msg** _message) 452 { 453 TRACE(("find_free_message_interrupts_disabled: entry\n")); 454 455 acquire_spinlock_cpu(currentCPU, &sFreeMessageSpinlock); 456 while (sFreeMessageCount <= 0) { 457 release_spinlock(&sFreeMessageSpinlock); 458 process_all_pending_ici(currentCPU); 459 PAUSE(); 460 acquire_spinlock_cpu(currentCPU, &sFreeMessageSpinlock); 461 } 462 463 *_message = sFreeMessages; 464 sFreeMessages = (*_message)->next; 465 sFreeMessageCount--; 466 467 release_spinlock(&sFreeMessageSpinlock); 468 469 TRACE(("find_free_message_interrupts_disabled: returning msg %p\n", 470 *_message)); 471 } 472 473 474 static void 475 return_free_message(struct smp_msg *msg) 476 { 477 TRACE(("return_free_message: returning msg %p\n", msg)); 478 479 acquire_spinlock_nocheck(&sFreeMessageSpinlock); 480 msg->next = sFreeMessages; 481 sFreeMessages = msg; 482 sFreeMessageCount++; 483 release_spinlock(&sFreeMessageSpinlock); 484 } 485 486 487 static struct smp_msg * 488 check_for_message(int currentCPU, int *source_mailbox) 489 { 490 struct smp_msg *msg; 491 492 if (!sICIEnabled) 493 return NULL; 494 495 acquire_spinlock_nocheck(&sCPUMessageSpinlock[currentCPU]); 496 msg = sCPUMessages[currentCPU]; 497 if (msg != NULL) { 498 sCPUMessages[currentCPU] = msg->next; 499 release_spinlock(&sCPUMessageSpinlock[currentCPU]); 500 TRACE((" cpu %d: found msg %p in cpu mailbox\n", currentCPU, msg)); 501 *source_mailbox = MAILBOX_LOCAL; 502 } else { 503 // try getting one from the broadcast mailbox 504 505 release_spinlock(&sCPUMessageSpinlock[currentCPU]); 506 acquire_spinlock_nocheck(&sBroadcastMessageSpinlock); 507 508 msg = sBroadcastMessages; 509 while (msg != NULL) { 510 if (CHECK_BIT(msg->proc_bitmap, currentCPU) != 0) { 511 // we have handled this one already 512 msg = msg->next; 513 continue; 514 } 515 516 // mark it so we wont try to process this one again 517 msg->proc_bitmap = SET_BIT(msg->proc_bitmap, currentCPU); 518 *source_mailbox = MAILBOX_BCAST; 519 break; 520 } 521 release_spinlock(&sBroadcastMessageSpinlock); 522 TRACE((" cpu %d: found msg %p in broadcast mailbox\n", currentCPU, msg)); 523 } 524 return msg; 525 } 526 527 528 static void 529 finish_message_processing(int currentCPU, struct smp_msg *msg, int source_mailbox) 530 { 531 int old_refcount; 532 533 old_refcount = atomic_add(&msg->ref_count, -1); 534 if (old_refcount == 1) { 535 // we were the last one to decrement the ref_count 536 // it's our job to remove it from the list & possibly clean it up 537 struct smp_msg **mbox = NULL; 538 spinlock *spinlock = NULL; 539 540 // clean up the message from one of the mailboxes 541 switch (source_mailbox) { 542 case MAILBOX_BCAST: 543 mbox = &sBroadcastMessages; 544 spinlock = &sBroadcastMessageSpinlock; 545 break; 546 case MAILBOX_LOCAL: 547 mbox = &sCPUMessages[currentCPU]; 548 spinlock = &sCPUMessageSpinlock[currentCPU]; 549 break; 550 } 551 552 acquire_spinlock_nocheck(spinlock); 553 554 TRACE(("cleaning up message %p\n", msg)); 555 556 if (source_mailbox != MAILBOX_BCAST) { 557 // local mailbox -- the message has already been removed in 558 // check_for_message() 559 } else if (msg == *mbox) { 560 (*mbox) = msg->next; 561 } else { 562 // we need to walk to find the message in the list. 563 // we can't use any data found when previously walking through 564 // the list, since the list may have changed. But, we are guaranteed 565 // to at least have msg in it. 566 struct smp_msg *last = NULL; 567 struct smp_msg *msg1; 568 569 msg1 = *mbox; 570 while (msg1 != NULL && msg1 != msg) { 571 last = msg1; 572 msg1 = msg1->next; 573 } 574 575 // by definition, last must be something 576 if (msg1 == msg && last != NULL) 577 last->next = msg->next; 578 else 579 panic("last == NULL or msg != msg1"); 580 } 581 582 release_spinlock(spinlock); 583 584 if ((msg->flags & SMP_MSG_FLAG_FREE_ARG) != 0 && msg->data_ptr != NULL) 585 free(msg->data_ptr); 586 587 if (msg->flags & SMP_MSG_FLAG_SYNC) { 588 msg->done = true; 589 // the caller cpu should now free the message 590 } else { 591 // in the !SYNC case, we get to free the message 592 return_free_message(msg); 593 } 594 } 595 } 596 597 598 static int32 599 process_pending_ici(int32 currentCPU) 600 { 601 struct smp_msg *msg; 602 bool haltCPU = false; 603 int sourceMailbox = 0; 604 int retval = B_HANDLED_INTERRUPT; 605 606 msg = check_for_message(currentCPU, &sourceMailbox); 607 if (msg == NULL) 608 return B_ENTRY_NOT_FOUND; 609 610 TRACE((" cpu %ld message = %ld\n", currentCPU, msg->message)); 611 612 switch (msg->message) { 613 case SMP_MSG_INVALIDATE_PAGE_RANGE: 614 arch_cpu_invalidate_TLB_range((addr_t)msg->data, (addr_t)msg->data2); 615 break; 616 case SMP_MSG_INVALIDATE_PAGE_LIST: 617 arch_cpu_invalidate_TLB_list((addr_t *)msg->data, (int)msg->data2); 618 break; 619 case SMP_MSG_USER_INVALIDATE_PAGES: 620 arch_cpu_user_TLB_invalidate(); 621 break; 622 case SMP_MSG_GLOBAL_INVALIDATE_PAGES: 623 arch_cpu_global_TLB_invalidate(); 624 break; 625 case SMP_MSG_CPU_HALT: 626 haltCPU = true; 627 break; 628 case SMP_MSG_CALL_FUNCTION: 629 { 630 smp_call_func func = (smp_call_func)msg->data_ptr; 631 func(msg->data, currentCPU, msg->data2, msg->data3); 632 break; 633 } 634 case SMP_MSG_RESCHEDULE: 635 thread_get_current_thread()->cpu->invoke_scheduler = true; 636 break; 637 default: 638 dprintf("smp_intercpu_int_handler: got unknown message %ld\n", msg->message); 639 } 640 641 // finish dealing with this message, possibly removing it from the list 642 finish_message_processing(currentCPU, msg, sourceMailbox); 643 644 // special case for the halt message 645 if (haltCPU) 646 debug_trap_cpu_in_kdl(currentCPU, false); 647 648 return retval; 649 } 650 651 652 #if B_DEBUG_SPINLOCK_CONTENTION 653 654 static uint64 655 get_spinlock_counter(spinlock* lock) 656 { 657 uint32 high; 658 uint32 low; 659 do { 660 high = (uint32)atomic_get(&lock->count_high); 661 low = (uint32)atomic_get(&lock->count_low); 662 } while (high != atomic_get(&lock->count_high)); 663 664 return ((uint64)high << 32) | low; 665 } 666 667 668 static status_t 669 spinlock_contention_syscall(const char* subsystem, uint32 function, 670 void* buffer, size_t bufferSize) 671 { 672 spinlock_contention_info info; 673 674 if (function != GET_SPINLOCK_CONTENTION_INFO) 675 return B_BAD_VALUE; 676 677 if (bufferSize < sizeof(spinlock_contention_info)) 678 return B_BAD_VALUE; 679 680 info.thread_spinlock_counter = get_spinlock_counter(&gThreadSpinlock); 681 info.team_spinlock_counter = get_spinlock_counter(&gTeamSpinlock); 682 683 if (!IS_USER_ADDRESS(buffer) 684 || user_memcpy(buffer, &info, sizeof(info)) != B_OK) { 685 return B_BAD_ADDRESS; 686 } 687 688 return B_OK; 689 } 690 691 #endif // B_DEBUG_SPINLOCK_CONTENTION 692 693 694 // #pragma mark - 695 696 697 int 698 smp_intercpu_int_handler(int32 cpu) 699 { 700 TRACE(("smp_intercpu_int_handler: entry on cpu %ld\n", cpu)); 701 702 process_all_pending_ici(cpu); 703 704 TRACE(("smp_intercpu_int_handler: done\n")); 705 706 return B_HANDLED_INTERRUPT; 707 } 708 709 710 void 711 smp_send_ici(int32 targetCPU, int32 message, uint32 data, uint32 data2, uint32 data3, 712 void *data_ptr, uint32 flags) 713 { 714 struct smp_msg *msg; 715 716 TRACE(("smp_send_ici: target 0x%lx, mess 0x%lx, data 0x%lx, data2 0x%lx, data3 0x%lx, ptr %p, flags 0x%lx\n", 717 targetCPU, message, data, data2, data3, data_ptr, flags)); 718 719 if (sICIEnabled) { 720 int state; 721 int currentCPU; 722 723 // find_free_message leaves interrupts disabled 724 state = find_free_message(&msg); 725 726 currentCPU = smp_get_current_cpu(); 727 if (targetCPU == currentCPU) { 728 return_free_message(msg); 729 restore_interrupts(state); 730 return; // nope, cant do that 731 } 732 733 // set up the message 734 msg->message = message; 735 msg->data = data; 736 msg->data2 = data2; 737 msg->data3 = data3; 738 msg->data_ptr = data_ptr; 739 msg->ref_count = 1; 740 msg->flags = flags; 741 msg->done = false; 742 743 // stick it in the appropriate cpu's mailbox 744 acquire_spinlock_nocheck(&sCPUMessageSpinlock[targetCPU]); 745 msg->next = sCPUMessages[targetCPU]; 746 sCPUMessages[targetCPU] = msg; 747 release_spinlock(&sCPUMessageSpinlock[targetCPU]); 748 749 arch_smp_send_ici(targetCPU); 750 751 if (flags & SMP_MSG_FLAG_SYNC) { 752 // wait for the other cpu to finish processing it 753 // the interrupt handler will ref count it to <0 754 // if the message is sync after it has removed it from the mailbox 755 while (msg->done == false) { 756 process_all_pending_ici(currentCPU); 757 PAUSE(); 758 } 759 // for SYNC messages, it's our responsibility to put it 760 // back into the free list 761 return_free_message(msg); 762 } 763 764 restore_interrupts(state); 765 } 766 } 767 768 769 void 770 smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, uint32 data, 771 uint32 data2, uint32 data3, void *data_ptr, uint32 flags) 772 { 773 if (!sICIEnabled) 774 return; 775 776 int currentCPU = smp_get_current_cpu(); 777 cpuMask &= ~((cpu_mask_t)1 << currentCPU) 778 & (((cpu_mask_t)1 << sNumCPUs) - 1); 779 if (cpuMask == 0) { 780 panic("smp_send_multicast_ici(): 0 CPU mask"); 781 return; 782 } 783 784 // count target CPUs 785 int32 targetCPUs = 0; 786 for (int32 i = 0; i < sNumCPUs; i++) { 787 if ((cpuMask & (cpu_mask_t)1 << i) != 0) 788 targetCPUs++; 789 } 790 791 // find_free_message leaves interrupts disabled 792 struct smp_msg *msg; 793 int state = find_free_message(&msg); 794 795 msg->message = message; 796 msg->data = data; 797 msg->data2 = data2; 798 msg->data3 = data3; 799 msg->data_ptr = data_ptr; 800 msg->ref_count = targetCPUs; 801 msg->flags = flags; 802 msg->proc_bitmap = ~cpuMask; 803 msg->done = false; 804 805 // stick it in the broadcast mailbox 806 acquire_spinlock_nocheck(&sBroadcastMessageSpinlock); 807 msg->next = sBroadcastMessages; 808 sBroadcastMessages = msg; 809 release_spinlock(&sBroadcastMessageSpinlock); 810 811 arch_smp_send_broadcast_ici(); 812 // TODO: Introduce a call that only bothers the target CPUs! 813 814 if (flags & SMP_MSG_FLAG_SYNC) { 815 // wait for the other cpus to finish processing it 816 // the interrupt handler will ref count it to <0 817 // if the message is sync after it has removed it from the mailbox 818 while (msg->done == false) { 819 process_all_pending_ici(currentCPU); 820 PAUSE(); 821 } 822 823 // for SYNC messages, it's our responsibility to put it 824 // back into the free list 825 return_free_message(msg); 826 } 827 828 restore_interrupts(state); 829 } 830 831 832 void 833 smp_send_broadcast_ici(int32 message, uint32 data, uint32 data2, uint32 data3, 834 void *data_ptr, uint32 flags) 835 { 836 struct smp_msg *msg; 837 838 TRACE(("smp_send_broadcast_ici: cpu %ld mess 0x%lx, data 0x%lx, data2 0x%lx, data3 0x%lx, ptr %p, flags 0x%lx\n", 839 smp_get_current_cpu(), message, data, data2, data3, data_ptr, flags)); 840 841 if (sICIEnabled) { 842 int state; 843 int currentCPU; 844 845 // find_free_message leaves interrupts disabled 846 state = find_free_message(&msg); 847 848 currentCPU = smp_get_current_cpu(); 849 850 msg->message = message; 851 msg->data = data; 852 msg->data2 = data2; 853 msg->data3 = data3; 854 msg->data_ptr = data_ptr; 855 msg->ref_count = sNumCPUs - 1; 856 msg->flags = flags; 857 msg->proc_bitmap = SET_BIT(0, currentCPU); 858 msg->done = false; 859 860 TRACE(("smp_send_broadcast_ici%d: inserting msg %p into broadcast mbox\n", 861 currentCPU, msg)); 862 863 // stick it in the appropriate cpu's mailbox 864 acquire_spinlock_nocheck(&sBroadcastMessageSpinlock); 865 msg->next = sBroadcastMessages; 866 sBroadcastMessages = msg; 867 release_spinlock(&sBroadcastMessageSpinlock); 868 869 arch_smp_send_broadcast_ici(); 870 871 TRACE(("smp_send_broadcast_ici: sent interrupt\n")); 872 873 if (flags & SMP_MSG_FLAG_SYNC) { 874 // wait for the other cpus to finish processing it 875 // the interrupt handler will ref count it to <0 876 // if the message is sync after it has removed it from the mailbox 877 TRACE(("smp_send_broadcast_ici: waiting for ack\n")); 878 879 while (msg->done == false) { 880 process_all_pending_ici(currentCPU); 881 PAUSE(); 882 } 883 884 TRACE(("smp_send_broadcast_ici: returning message to free list\n")); 885 886 // for SYNC messages, it's our responsibility to put it 887 // back into the free list 888 return_free_message(msg); 889 } 890 891 restore_interrupts(state); 892 } 893 894 TRACE(("smp_send_broadcast_ici: done\n")); 895 } 896 897 898 void 899 smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message, 900 uint32 data, uint32 data2, uint32 data3, void *data_ptr, uint32 flags) 901 { 902 if (!sICIEnabled) 903 return; 904 905 TRACE(("smp_send_broadcast_ici_interrupts_disabled: cpu %ld mess 0x%lx, " 906 "data 0x%lx, data2 0x%lx, data3 0x%lx, ptr %p, flags 0x%lx\n", 907 currentCPU, message, data, data2, data3, data_ptr, flags)); 908 909 struct smp_msg *msg; 910 find_free_message_interrupts_disabled(currentCPU, &msg); 911 912 msg->message = message; 913 msg->data = data; 914 msg->data2 = data2; 915 msg->data3 = data3; 916 msg->data_ptr = data_ptr; 917 msg->ref_count = sNumCPUs - 1; 918 msg->flags = flags; 919 msg->proc_bitmap = SET_BIT(0, currentCPU); 920 msg->done = false; 921 922 TRACE(("smp_send_broadcast_ici_interrupts_disabled %ld: inserting msg %p " 923 "into broadcast mbox\n", currentCPU, msg)); 924 925 // stick it in the appropriate cpu's mailbox 926 acquire_spinlock_nocheck(&sBroadcastMessageSpinlock); 927 msg->next = sBroadcastMessages; 928 sBroadcastMessages = msg; 929 release_spinlock(&sBroadcastMessageSpinlock); 930 931 arch_smp_send_broadcast_ici(); 932 933 TRACE(("smp_send_broadcast_ici_interrupts_disabled %ld: sent interrupt\n", 934 currentCPU)); 935 936 if (flags & SMP_MSG_FLAG_SYNC) { 937 // wait for the other cpus to finish processing it 938 // the interrupt handler will ref count it to <0 939 // if the message is sync after it has removed it from the mailbox 940 TRACE(("smp_send_broadcast_ici_interrupts_disabled %ld: waiting for " 941 "ack\n", currentCPU)); 942 943 while (msg->done == false) { 944 process_all_pending_ici(currentCPU); 945 PAUSE(); 946 } 947 948 TRACE(("smp_send_broadcast_ici_interrupts_disabled %ld: returning " 949 "message to free list\n", currentCPU)); 950 951 // for SYNC messages, it's our responsibility to put it 952 // back into the free list 953 return_free_message(msg); 954 } 955 956 TRACE(("smp_send_broadcast_ici_interrupts_disabled: done\n")); 957 } 958 959 960 bool 961 smp_trap_non_boot_cpus(int32 cpu) 962 { 963 if (cpu > 0) { 964 #if B_DEBUG_SPINLOCK_CONTENTION 965 boot_cpu_spin[cpu].lock = 1; 966 #else 967 boot_cpu_spin[cpu] = 1; 968 #endif 969 acquire_spinlock_nocheck(&boot_cpu_spin[cpu]); 970 return false; 971 } 972 973 return true; 974 } 975 976 977 void 978 smp_wake_up_non_boot_cpus() 979 { 980 int i; 981 982 // ICIs were previously being ignored 983 if (sNumCPUs > 1) 984 sICIEnabled = true; 985 986 // resume non boot CPUs 987 for (i = 1; i < sNumCPUs; i++) { 988 release_spinlock(&boot_cpu_spin[i]); 989 } 990 } 991 992 /* have all cpus spin until all have run */ 993 void 994 smp_cpu_rendezvous(volatile uint32 *var, int current_cpu) 995 { 996 atomic_or((vint32*)var, 1 << current_cpu); 997 998 while (*var != (((uint32)1 << sNumCPUs) - 1)) 999 PAUSE(); 1000 } 1001 1002 status_t 1003 smp_init(kernel_args *args) 1004 { 1005 struct smp_msg *msg; 1006 int i; 1007 1008 TRACE(("smp_init: entry\n")); 1009 1010 #if DEBUG_SPINLOCKS 1011 add_debugger_command_etc("spinlock", &dump_spinlock, 1012 "Dump info on a spinlock", 1013 "\n" 1014 "Dumps info on a spinlock.\n", 0); 1015 #endif 1016 add_debugger_command_etc("ici", &dump_ici_messages, 1017 "Dump info on pending ICI messages", 1018 "\n" 1019 "Dumps info on pending ICI messages.\n", 0); 1020 add_debugger_command_etc("ici_message", &dump_ici_message, 1021 "Dump info on an ICI message", 1022 "\n" 1023 "Dumps info on an ICI message.\n", 0); 1024 1025 if (args->num_cpus > 1) { 1026 sFreeMessages = NULL; 1027 sFreeMessageCount = 0; 1028 for (i = 0; i < MSG_POOL_SIZE; i++) { 1029 msg = (struct smp_msg *)malloc(sizeof(struct smp_msg)); 1030 if (msg == NULL) { 1031 panic("error creating smp mailboxes\n"); 1032 return B_ERROR; 1033 } 1034 memset(msg, 0, sizeof(struct smp_msg)); 1035 msg->next = sFreeMessages; 1036 sFreeMessages = msg; 1037 sFreeMessageCount++; 1038 } 1039 sNumCPUs = args->num_cpus; 1040 } 1041 TRACE(("smp_init: calling arch_smp_init\n")); 1042 1043 return arch_smp_init(args); 1044 } 1045 1046 1047 status_t 1048 smp_per_cpu_init(kernel_args *args, int32 cpu) 1049 { 1050 return arch_smp_per_cpu_init(args, cpu); 1051 } 1052 1053 1054 status_t 1055 smp_init_post_generic_syscalls(void) 1056 { 1057 #if B_DEBUG_SPINLOCK_CONTENTION 1058 return register_generic_syscall(SPINLOCK_CONTENTION, 1059 &spinlock_contention_syscall, 0, 0); 1060 #else 1061 return B_OK; 1062 #endif 1063 } 1064 1065 1066 void 1067 smp_set_num_cpus(int32 numCPUs) 1068 { 1069 sNumCPUs = numCPUs; 1070 } 1071 1072 1073 int32 1074 smp_get_num_cpus() 1075 { 1076 return sNumCPUs; 1077 } 1078 1079 1080 int32 1081 smp_get_current_cpu(void) 1082 { 1083 return thread_get_current_thread()->cpu->cpu_num; 1084 } 1085 1086 1087 // #pragma mark - 1088 // public exported functions 1089 1090 1091 void 1092 call_all_cpus(void (*func)(void *, int), void *cookie) 1093 { 1094 cpu_status state = disable_interrupts(); 1095 1096 if (smp_get_num_cpus() > 1) { 1097 smp_send_broadcast_ici(SMP_MSG_CALL_FUNCTION, (uint32)cookie, 1098 0, 0, (void *)func, SMP_MSG_FLAG_ASYNC); 1099 } 1100 1101 // we need to call this function ourselves as well 1102 func(cookie, smp_get_current_cpu()); 1103 1104 restore_interrupts(state); 1105 } 1106 1107 void 1108 call_all_cpus_sync(void (*func)(void *, int), void *cookie) 1109 { 1110 cpu_status state = disable_interrupts(); 1111 1112 if (smp_get_num_cpus() > 1) { 1113 smp_send_broadcast_ici(SMP_MSG_CALL_FUNCTION, (uint32)cookie, 1114 0, 0, (void *)func, SMP_MSG_FLAG_SYNC); 1115 } 1116 1117 // we need to call this function ourselves as well 1118 func(cookie, smp_get_current_cpu()); 1119 1120 restore_interrupts(state); 1121 } 1122 1123 1124 void 1125 memory_read_barrier(void) 1126 { 1127 arch_cpu_memory_read_barrier(); 1128 } 1129 1130 1131 void 1132 memory_write_barrier(void) 1133 { 1134 arch_cpu_memory_write_barrier(); 1135 } 1136 1137