1 /* 2 * Copyright 2011-2021, Haiku, Inc. All rights reserved. 3 * Distributed under the terms of the MIT License. 4 * 5 * Authors: 6 * Augustin Cavalier <waddlesplash> 7 * Jian Chiang <j.jian.chiang@gmail.com> 8 * Jérôme Duval <jerome.duval@gmail.com> 9 * Akshay Jaggi <akshay1994.leo@gmail.com> 10 * Michael Lotz <mmlr@mlotz.ch> 11 * Alexander von Gluck <kallisti5@unixzen.com> 12 */ 13 14 15 #include <stdio.h> 16 17 #include <bus/PCI.h> 18 #include <USB3.h> 19 #include <KernelExport.h> 20 21 #include <ByteOrder.h> 22 #include <util/AutoLock.h> 23 24 #include "xhci.h" 25 26 27 #define CALLED(x...) TRACE_MODULE("CALLED %s\n", __PRETTY_FUNCTION__) 28 29 30 #define USB_MODULE_NAME "xhci" 31 32 device_manager_info* gDeviceManager; 33 static usb_for_controller_interface* gUSB; 34 35 36 #define XHCI_PCI_DEVICE_MODULE_NAME "busses/usb/xhci/pci/driver_v1" 37 #define XHCI_PCI_USB_BUS_MODULE_NAME "busses/usb/xhci/device_v1" 38 39 40 typedef struct { 41 XHCI* xhci; 42 pci_device_module_info* pci; 43 pci_device* device; 44 45 pci_info pciinfo; 46 47 device_node* node; 48 device_node* driver_node; 49 } xhci_pci_sim_info; 50 51 52 // #pragma mark - 53 54 55 static status_t 56 init_bus(device_node* node, void** bus_cookie) 57 { 58 CALLED(); 59 60 driver_module_info* driver; 61 xhci_pci_sim_info* bus; 62 device_node* parent = gDeviceManager->get_parent_node(node); 63 gDeviceManager->get_driver(parent, &driver, (void**)&bus); 64 gDeviceManager->put_node(parent); 65 66 Stack *stack; 67 if (gUSB->get_stack((void**)&stack) != B_OK) 68 return B_ERROR; 69 70 XHCI *xhci = new(std::nothrow) XHCI(&bus->pciinfo, bus->pci, bus->device, stack, node); 71 if (xhci == NULL) { 72 return B_NO_MEMORY; 73 } 74 75 if (xhci->InitCheck() < B_OK) { 76 TRACE_MODULE_ERROR("bus failed init check\n"); 77 delete xhci; 78 return B_ERROR; 79 } 80 81 if (xhci->Start() != B_OK) { 82 delete xhci; 83 return B_ERROR; 84 } 85 86 *bus_cookie = xhci; 87 88 return B_OK; 89 } 90 91 92 static void 93 uninit_bus(void* bus_cookie) 94 { 95 CALLED(); 96 XHCI* xhci = (XHCI*)bus_cookie; 97 delete xhci; 98 } 99 100 101 static status_t 102 register_child_devices(void* cookie) 103 { 104 CALLED(); 105 xhci_pci_sim_info* bus = (xhci_pci_sim_info*)cookie; 106 device_node* node = bus->driver_node; 107 108 char prettyName[25]; 109 sprintf(prettyName, "XHCI Controller %" B_PRIu16, 0); 110 111 device_attr attrs[] = { 112 // properties of this controller for the usb bus manager 113 { B_DEVICE_PRETTY_NAME, B_STRING_TYPE, 114 { .string = prettyName }}, 115 { B_DEVICE_FIXED_CHILD, B_STRING_TYPE, 116 { .string = USB_FOR_CONTROLLER_MODULE_NAME }}, 117 118 // private data to identify the device 119 { NULL } 120 }; 121 122 return gDeviceManager->register_node(node, XHCI_PCI_USB_BUS_MODULE_NAME, 123 attrs, NULL, NULL); 124 } 125 126 127 static status_t 128 init_device(device_node* node, void** device_cookie) 129 { 130 CALLED(); 131 xhci_pci_sim_info* bus = (xhci_pci_sim_info*)calloc(1, 132 sizeof(xhci_pci_sim_info)); 133 if (bus == NULL) 134 return B_NO_MEMORY; 135 136 pci_device_module_info* pci; 137 pci_device* device; 138 { 139 device_node* pciParent = gDeviceManager->get_parent_node(node); 140 gDeviceManager->get_driver(pciParent, (driver_module_info**)&pci, 141 (void**)&device); 142 gDeviceManager->put_node(pciParent); 143 } 144 145 bus->pci = pci; 146 bus->device = device; 147 bus->driver_node = node; 148 149 pci_info *pciInfo = &bus->pciinfo; 150 pci->get_pci_info(device, pciInfo); 151 152 *device_cookie = bus; 153 return B_OK; 154 } 155 156 157 static void 158 uninit_device(void* device_cookie) 159 { 160 CALLED(); 161 xhci_pci_sim_info* bus = (xhci_pci_sim_info*)device_cookie; 162 free(bus); 163 } 164 165 166 static status_t 167 register_device(device_node* parent) 168 { 169 CALLED(); 170 device_attr attrs[] = { 171 {B_DEVICE_PRETTY_NAME, B_STRING_TYPE, {.string = "XHCI PCI"}}, 172 {} 173 }; 174 175 return gDeviceManager->register_node(parent, 176 XHCI_PCI_DEVICE_MODULE_NAME, attrs, NULL, NULL); 177 } 178 179 180 static float 181 supports_device(device_node* parent) 182 { 183 CALLED(); 184 const char* bus; 185 uint16 type, subType, api; 186 187 // make sure parent is a XHCI PCI device node 188 if (gDeviceManager->get_attr_string(parent, B_DEVICE_BUS, &bus, false) 189 < B_OK) { 190 return -1; 191 } 192 193 if (strcmp(bus, "pci") != 0) 194 return 0.0f; 195 196 if (gDeviceManager->get_attr_uint16(parent, B_DEVICE_SUB_TYPE, &subType, 197 false) < B_OK 198 || gDeviceManager->get_attr_uint16(parent, B_DEVICE_TYPE, &type, 199 false) < B_OK 200 || gDeviceManager->get_attr_uint16(parent, B_DEVICE_INTERFACE, &api, 201 false) < B_OK) { 202 TRACE_MODULE("Could not find type/subtype/interface attributes\n"); 203 return -1; 204 } 205 206 if (type == PCI_serial_bus && subType == PCI_usb && api == PCI_usb_xhci) { 207 pci_device_module_info* pci; 208 pci_device* device; 209 gDeviceManager->get_driver(parent, (driver_module_info**)&pci, 210 (void**)&device); 211 TRACE_MODULE("XHCI Device found!\n"); 212 213 return 0.8f; 214 } 215 216 return 0.0f; 217 } 218 219 220 static const char* 221 xhci_error_string(uint32 error) 222 { 223 switch (error) { 224 case COMP_INVALID: return "Invalid"; 225 case COMP_SUCCESS: return "Success"; 226 case COMP_DATA_BUFFER: return "Data buffer"; 227 case COMP_BABBLE: return "Babble detected"; 228 case COMP_USB_TRANSACTION: return "USB transaction"; 229 case COMP_TRB: return "TRB"; 230 case COMP_STALL: return "Stall"; 231 case COMP_RESOURCE: return "Resource"; 232 case COMP_BANDWIDTH: return "Bandwidth"; 233 case COMP_NO_SLOTS: return "No slots"; 234 case COMP_INVALID_STREAM: return "Invalid stream"; 235 case COMP_SLOT_NOT_ENABLED: return "Slot not enabled"; 236 case COMP_ENDPOINT_NOT_ENABLED: return "Endpoint not enabled"; 237 case COMP_SHORT_PACKET: return "Short packet"; 238 case COMP_RING_UNDERRUN: return "Ring underrun"; 239 case COMP_RING_OVERRUN: return "Ring overrun"; 240 case COMP_VF_RING_FULL: return "VF Event Ring Full"; 241 case COMP_PARAMETER: return "Parameter"; 242 case COMP_BANDWIDTH_OVERRUN: return "Bandwidth overrun"; 243 case COMP_CONTEXT_STATE: return "Context state"; 244 case COMP_NO_PING_RESPONSE: return "No ping response"; 245 case COMP_EVENT_RING_FULL: return "Event ring full"; 246 case COMP_INCOMPATIBLE_DEVICE: return "Incompatible device"; 247 case COMP_MISSED_SERVICE: return "Missed service"; 248 case COMP_COMMAND_RING_STOPPED: return "Command ring stopped"; 249 case COMP_COMMAND_ABORTED: return "Command aborted"; 250 case COMP_STOPPED: return "Stopped"; 251 case COMP_STOPPED_LENGTH_INVALID: return "Stopped (length invalid)"; 252 case COMP_MAX_EXIT_LATENCY: return "Max exit latency too large"; 253 case COMP_ISOC_OVERRUN: return "Isoch buffer overrun"; 254 case COMP_EVENT_LOST: return "Event lost"; 255 case COMP_UNDEFINED: return "Undefined"; 256 case COMP_INVALID_STREAM_ID: return "Invalid stream ID"; 257 case COMP_SECONDARY_BANDWIDTH: return "Secondary bandwidth"; 258 case COMP_SPLIT_TRANSACTION: return "Split transaction"; 259 260 default: return "Undefined"; 261 } 262 } 263 264 265 static status_t 266 xhci_error_status(uint32 error, bool directionIn) 267 { 268 switch (error) { 269 case COMP_SHORT_PACKET: 270 case COMP_SUCCESS: 271 return B_OK; 272 case COMP_DATA_BUFFER: 273 return directionIn ? B_DEV_WRITE_ERROR : B_DEV_READ_ERROR; 274 case COMP_BABBLE: 275 return directionIn ? B_DEV_DATA_OVERRUN : B_DEV_DATA_UNDERRUN; 276 case COMP_RING_UNDERRUN: 277 return B_DEV_FIFO_UNDERRUN; 278 case COMP_RING_OVERRUN: 279 return B_DEV_FIFO_OVERRUN; 280 case COMP_MISSED_SERVICE: 281 return B_DEV_TOO_LATE; 282 case COMP_USB_TRANSACTION: 283 return B_DEV_CRC_ERROR; 284 case COMP_STALL: 285 return B_DEV_STALLED; 286 default: 287 return B_DEV_STALLED; 288 } 289 } 290 291 292 module_dependency module_dependencies[] = { 293 { USB_FOR_CONTROLLER_MODULE_NAME, (module_info**)&gUSB }, 294 { B_DEVICE_MANAGER_MODULE_NAME, (module_info**)&gDeviceManager }, 295 {} 296 }; 297 298 299 static usb_bus_interface gXHCIPCIDeviceModule = { 300 { 301 { 302 XHCI_PCI_USB_BUS_MODULE_NAME, 303 0, 304 NULL 305 }, 306 NULL, // supports device 307 NULL, // register device 308 init_bus, 309 uninit_bus, 310 NULL, // register child devices 311 NULL, // rescan 312 NULL, // device removed 313 }, 314 }; 315 316 // Root device that binds to the PCI bus. It will register an usb_bus_interface 317 // node for each device. 318 static driver_module_info sXHCIDevice = { 319 { 320 XHCI_PCI_DEVICE_MODULE_NAME, 321 0, 322 NULL 323 }, 324 supports_device, 325 register_device, 326 init_device, 327 uninit_device, 328 register_child_devices, 329 NULL, // rescan 330 NULL, // device removed 331 }; 332 333 module_info* modules[] = { 334 (module_info* )&sXHCIDevice, 335 (module_info* )&gXHCIPCIDeviceModule, 336 NULL 337 }; 338 339 340 XHCI::XHCI(pci_info *info, pci_device_module_info* pci, pci_device* device, Stack *stack, 341 device_node* node) 342 : BusManager(stack, node), 343 fRegisterArea(-1), 344 fRegisters(NULL), 345 fPCIInfo(info), 346 fPci(pci), 347 fDevice(device), 348 fStack(stack), 349 fIRQ(0), 350 fUseMSI(false), 351 fErstArea(-1), 352 fDcbaArea(-1), 353 fCmdCompSem(-1), 354 fStopThreads(false), 355 fRootHub(NULL), 356 fPortCount(0), 357 fSlotCount(0), 358 fScratchpadCount(0), 359 fContextSizeShift(0), 360 fFinishedHead(NULL), 361 fFinishTransfersSem(-1), 362 fFinishThread(-1), 363 fEventSem(-1), 364 fEventThread(-1), 365 fEventIdx(0), 366 fCmdIdx(0), 367 fEventCcs(1), 368 fCmdCcs(1) 369 { 370 B_INITIALIZE_SPINLOCK(&fSpinlock); 371 mutex_init(&fFinishedLock, "XHCI finished transfers"); 372 mutex_init(&fEventLock, "XHCI event handler"); 373 374 if (BusManager::InitCheck() < B_OK) { 375 TRACE_ERROR("bus manager failed to init\n"); 376 return; 377 } 378 379 TRACE("constructing new XHCI host controller driver\n"); 380 fInitOK = false; 381 382 // enable busmaster and memory mapped access 383 uint16 command = fPci->read_pci_config(fDevice, PCI_command, 2); 384 command &= ~(PCI_command_io | PCI_command_int_disable); 385 command |= PCI_command_master | PCI_command_memory; 386 387 fPci->write_pci_config(fDevice, PCI_command, 2, command); 388 389 // map the registers (low + high for 64-bit when requested) 390 phys_addr_t physicalAddress = fPCIInfo->u.h0.base_registers[0]; 391 if ((fPCIInfo->u.h0.base_register_flags[0] & PCI_address_type) 392 == PCI_address_type_64) { 393 physicalAddress |= (uint64)fPCIInfo->u.h0.base_registers[1] << 32; 394 } 395 396 size_t mapSize = fPCIInfo->u.h0.base_register_sizes[0]; 397 398 TRACE("map registers %08" B_PRIxPHYSADDR ", size: %" B_PRIuSIZE "\n", 399 physicalAddress, mapSize); 400 401 fRegisterArea = map_physical_memory("XHCI memory mapped registers", 402 physicalAddress, mapSize, B_ANY_KERNEL_BLOCK_ADDRESS, 403 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 404 (void **)&fRegisters); 405 if (fRegisterArea < B_OK) { 406 TRACE_ERROR("failed to map register memory\n"); 407 return; 408 } 409 410 // determine the register offsets 411 fCapabilityRegisterOffset = 0; 412 fOperationalRegisterOffset = HCI_CAPLENGTH(ReadCapReg32(XHCI_HCI_CAPLENGTH)); 413 fRuntimeRegisterOffset = ReadCapReg32(XHCI_RTSOFF) & ~0x1F; 414 fDoorbellRegisterOffset = ReadCapReg32(XHCI_DBOFF) & ~0x3; 415 416 TRACE("mapped registers: %p\n", fRegisters); 417 TRACE("operational register offset: %" B_PRId32 "\n", fOperationalRegisterOffset); 418 TRACE("runtime register offset: %" B_PRId32 "\n", fRuntimeRegisterOffset); 419 TRACE("doorbell register offset: %" B_PRId32 "\n", fDoorbellRegisterOffset); 420 421 int32 interfaceVersion = HCI_VERSION(ReadCapReg32(XHCI_HCI_VERSION)); 422 if (interfaceVersion < 0x0090 || interfaceVersion > 0x0120) { 423 TRACE_ERROR("unsupported interface version: 0x%04" B_PRIx32 "\n", 424 interfaceVersion); 425 return; 426 } 427 TRACE_ALWAYS("interface version: 0x%04" B_PRIx32 "\n", interfaceVersion); 428 429 TRACE_ALWAYS("structural parameters: 1:0x%08" B_PRIx32 " 2:0x%08" 430 B_PRIx32 " 3:0x%08" B_PRIx32 "\n", ReadCapReg32(XHCI_HCSPARAMS1), 431 ReadCapReg32(XHCI_HCSPARAMS2), ReadCapReg32(XHCI_HCSPARAMS3)); 432 433 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 434 if (cparams == 0xffffffff) 435 return; 436 TRACE_ALWAYS("capability parameters: 0x%08" B_PRIx32 "\n", cparams); 437 438 // if 64 bytes context structures, then 1 439 fContextSizeShift = HCC_CSZ(cparams); 440 441 // Assume ownership of the controller from the BIOS. 442 uint32 eec = 0xffffffff; 443 uint32 eecp = HCS0_XECP(cparams) << 2; 444 for (; eecp != 0 && XECP_NEXT(eec); eecp += XECP_NEXT(eec) << 2) { 445 TRACE("eecp register: 0x%08" B_PRIx32 "\n", eecp); 446 447 eec = ReadCapReg32(eecp); 448 if (XECP_ID(eec) != XHCI_LEGSUP_CAPID) 449 continue; 450 451 if (eec & XHCI_LEGSUP_BIOSOWNED) { 452 TRACE_ALWAYS("the host controller is bios owned, claiming" 453 " ownership\n"); 454 WriteCapReg32(eecp, eec | XHCI_LEGSUP_OSOWNED); 455 456 for (int32 i = 0; i < 20; i++) { 457 eec = ReadCapReg32(eecp); 458 459 if ((eec & XHCI_LEGSUP_BIOSOWNED) == 0) 460 break; 461 462 TRACE_ALWAYS("controller is still bios owned, waiting\n"); 463 snooze(50000); 464 } 465 466 if (eec & XHCI_LEGSUP_BIOSOWNED) { 467 TRACE_ERROR("bios won't give up control over the host " 468 "controller (ignoring)\n"); 469 } else if (eec & XHCI_LEGSUP_OSOWNED) { 470 TRACE_ALWAYS("successfully took ownership of the host " 471 "controller\n"); 472 } 473 474 // Force off the BIOS owned flag, and clear all SMIs. Some BIOSes 475 // do indicate a successful handover but do not remove their SMIs 476 // and then freeze the system when interrupts are generated. 477 WriteCapReg32(eecp, eec & ~XHCI_LEGSUP_BIOSOWNED); 478 } 479 break; 480 } 481 uint32 legctlsts = ReadCapReg32(eecp + XHCI_LEGCTLSTS); 482 legctlsts &= XHCI_LEGCTLSTS_DISABLE_SMI; 483 legctlsts |= XHCI_LEGCTLSTS_EVENTS_SMI; 484 WriteCapReg32(eecp + XHCI_LEGCTLSTS, legctlsts); 485 486 // We need to explicitly take ownership of EHCI ports on earlier Intel chipsets. 487 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL) { 488 switch (fPCIInfo->device_id) { 489 case PCI_DEVICE_INTEL_PANTHER_POINT_XHCI: 490 case PCI_DEVICE_INTEL_LYNX_POINT_XHCI: 491 case PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI: 492 case PCI_DEVICE_INTEL_BAYTRAIL_XHCI: 493 case PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI: 494 case PCI_DEVICE_INTEL_WILDCAT_POINT_LP_XHCI: 495 _SwitchIntelPorts(); 496 break; 497 } 498 } 499 500 // halt the host controller 501 if (ControllerHalt() < B_OK) { 502 return; 503 } 504 505 // reset the host controller 506 if (ControllerReset() < B_OK) { 507 TRACE_ERROR("host controller failed to reset\n"); 508 return; 509 } 510 511 fCmdCompSem = create_sem(0, "XHCI Command Complete"); 512 fFinishTransfersSem = create_sem(0, "XHCI Finish Transfers"); 513 fEventSem = create_sem(0, "XHCI Event"); 514 if (fFinishTransfersSem < B_OK || fCmdCompSem < B_OK || fEventSem < B_OK) { 515 TRACE_ERROR("failed to create semaphores\n"); 516 return; 517 } 518 519 // create event handler thread 520 fEventThread = spawn_kernel_thread(EventThread, "xhci event thread", 521 B_URGENT_PRIORITY, (void *)this); 522 resume_thread(fEventThread); 523 524 // create finisher service thread 525 fFinishThread = spawn_kernel_thread(FinishThread, "xhci finish thread", 526 B_URGENT_PRIORITY - 1, (void *)this); 527 resume_thread(fFinishThread); 528 529 // Find the right interrupt vector, using MSIs if available. 530 fIRQ = fPCIInfo->u.h0.interrupt_line; 531 #if 0 532 if (fPci->get_msix_count(fDevice) >= 1) { 533 uint8 msiVector = 0; 534 if (fPci->configure_msix(fDevice, 1, &msiVector) == B_OK 535 && fPci->enable_msix(fDevice) == B_OK) { 536 TRACE_ALWAYS("using MSI-X\n"); 537 fIRQ = msiVector; 538 fUseMSI = true; 539 } 540 } else 541 #endif 542 if (fPci->get_msi_count(fDevice) >= 1) { 543 uint8 msiVector = 0; 544 if (fPci->configure_msi(fDevice, 1, &msiVector) == B_OK 545 && fPci->enable_msi(fDevice) == B_OK) { 546 TRACE_ALWAYS("using message signaled interrupts\n"); 547 fIRQ = msiVector; 548 fUseMSI = true; 549 } 550 } 551 552 if (fIRQ == 0 || fIRQ == 0xFF) { 553 TRACE_MODULE_ERROR("device PCI:%d:%d:%d was assigned an invalid IRQ\n", 554 fPCIInfo->bus, fPCIInfo->device, fPCIInfo->function); 555 return; 556 } 557 558 // Install the interrupt handler 559 TRACE("installing interrupt handler\n"); 560 install_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this, 0); 561 562 memset(fPortSpeeds, 0, sizeof(fPortSpeeds)); 563 memset(fDevices, 0, sizeof(fDevices)); 564 565 fInitOK = true; 566 TRACE("driver construction successful\n"); 567 } 568 569 570 XHCI::~XHCI() 571 { 572 TRACE("tear down XHCI host controller driver\n"); 573 574 WriteOpReg(XHCI_CMD, 0); 575 576 int32 result = 0; 577 fStopThreads = true; 578 delete_sem(fCmdCompSem); 579 delete_sem(fFinishTransfersSem); 580 delete_sem(fEventSem); 581 wait_for_thread(fFinishThread, &result); 582 wait_for_thread(fEventThread, &result); 583 584 mutex_destroy(&fFinishedLock); 585 mutex_destroy(&fEventLock); 586 587 remove_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this); 588 589 delete_area(fRegisterArea); 590 delete_area(fErstArea); 591 for (uint32 i = 0; i < fScratchpadCount; i++) 592 delete_area(fScratchpadArea[i]); 593 delete_area(fDcbaArea); 594 595 if (fUseMSI) { 596 fPci->disable_msi(fDevice); 597 fPci->unconfigure_msi(fDevice); 598 } 599 } 600 601 602 void 603 XHCI::_SwitchIntelPorts() 604 { 605 TRACE("Looking for EHCI owned ports\n"); 606 uint32 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_USB3PRM, 4); 607 TRACE("Superspeed Ports: 0x%" B_PRIx32 "\n", ports); 608 fPci->write_pci_config(fDevice, XHCI_INTEL_USB3_PSSEN, 4, ports); 609 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_USB3_PSSEN, 4); 610 TRACE("Superspeed ports now under XHCI : 0x%" B_PRIx32 "\n", ports); 611 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_USB2PRM, 4); 612 TRACE("USB 2.0 Ports : 0x%" B_PRIx32 "\n", ports); 613 fPci->write_pci_config(fDevice, XHCI_INTEL_XUSB2PR, 4, ports); 614 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_XUSB2PR, 4); 615 TRACE("USB 2.0 ports now under XHCI: 0x%" B_PRIx32 "\n", ports); 616 } 617 618 619 status_t 620 XHCI::Start() 621 { 622 TRACE_ALWAYS("starting XHCI host controller\n"); 623 TRACE("usbcmd: 0x%08" B_PRIx32 "; usbsts: 0x%08" B_PRIx32 "\n", 624 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 625 626 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 627 TRACE("Start() failed STS_CNR\n"); 628 } 629 630 if ((ReadOpReg(XHCI_CMD) & CMD_RUN) != 0) { 631 TRACE_ERROR("Start() warning, starting running XHCI controller!\n"); 632 } 633 634 if ((ReadOpReg(XHCI_PAGESIZE) & (1 << 0)) == 0) { 635 TRACE_ERROR("controller does not support 4K page size\n"); 636 return B_ERROR; 637 } 638 639 // read port count from capability register 640 uint32 capabilities = ReadCapReg32(XHCI_HCSPARAMS1); 641 fPortCount = HCS_MAX_PORTS(capabilities); 642 if (fPortCount == 0) { 643 TRACE_ERROR("invalid number of ports: %u\n", fPortCount); 644 return B_ERROR; 645 } 646 647 fSlotCount = HCS_MAX_SLOTS(capabilities); 648 if (fSlotCount > XHCI_MAX_DEVICES) 649 fSlotCount = XHCI_MAX_DEVICES; 650 WriteOpReg(XHCI_CONFIG, fSlotCount); 651 652 // find out which protocol is used for each port 653 uint8 portFound = 0; 654 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 655 uint32 eec = 0xffffffff; 656 uint32 eecp = HCS0_XECP(cparams) << 2; 657 for (; eecp != 0 && XECP_NEXT(eec) && portFound < fPortCount; 658 eecp += XECP_NEXT(eec) << 2) { 659 eec = ReadCapReg32(eecp); 660 if (XECP_ID(eec) != XHCI_SUPPORTED_PROTOCOLS_CAPID) 661 continue; 662 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) > 3) 663 continue; 664 uint32 temp = ReadCapReg32(eecp + 8); 665 uint32 offset = XHCI_SUPPORTED_PROTOCOLS_1_OFFSET(temp); 666 uint32 count = XHCI_SUPPORTED_PROTOCOLS_1_COUNT(temp); 667 if (offset == 0 || count == 0) 668 continue; 669 offset--; 670 for (uint32 i = offset; i < offset + count; i++) { 671 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) == 0x3) 672 fPortSpeeds[i] = USB_SPEED_SUPERSPEED; 673 else 674 fPortSpeeds[i] = USB_SPEED_HIGHSPEED; 675 676 TRACE("speed for port %" B_PRId32 " is %s\n", i, 677 fPortSpeeds[i] == USB_SPEED_SUPERSPEED ? "super" : "high"); 678 } 679 portFound += count; 680 } 681 682 uint32 params2 = ReadCapReg32(XHCI_HCSPARAMS2); 683 fScratchpadCount = HCS_MAX_SC_BUFFERS(params2); 684 if (fScratchpadCount > XHCI_MAX_SCRATCHPADS) { 685 TRACE_ERROR("invalid number of scratchpads: %" B_PRIu32 "\n", 686 fScratchpadCount); 687 return B_ERROR; 688 } 689 690 uint32 params3 = ReadCapReg32(XHCI_HCSPARAMS3); 691 fExitLatMax = HCS_U1_DEVICE_LATENCY(params3) 692 + HCS_U2_DEVICE_LATENCY(params3); 693 694 // clear interrupts & disable device notifications 695 WriteOpReg(XHCI_STS, ReadOpReg(XHCI_STS)); 696 WriteOpReg(XHCI_DNCTRL, 0); 697 698 // allocate Device Context Base Address array 699 phys_addr_t dmaAddress; 700 fDcbaArea = fStack->AllocateArea((void **)&fDcba, &dmaAddress, 701 sizeof(*fDcba), "DCBA Area"); 702 if (fDcbaArea < B_OK) { 703 TRACE_ERROR("unable to create the DCBA area\n"); 704 return B_ERROR; 705 } 706 memset(fDcba, 0, sizeof(*fDcba)); 707 memset(fScratchpadArea, 0, sizeof(fScratchpadArea)); 708 memset(fScratchpad, 0, sizeof(fScratchpad)); 709 710 // setting the first address to the scratchpad array address 711 fDcba->baseAddress[0] = dmaAddress 712 + offsetof(struct xhci_device_context_array, scratchpad); 713 714 // fill up the scratchpad array with scratchpad pages 715 for (uint32 i = 0; i < fScratchpadCount; i++) { 716 phys_addr_t scratchDmaAddress; 717 fScratchpadArea[i] = fStack->AllocateArea((void **)&fScratchpad[i], 718 &scratchDmaAddress, B_PAGE_SIZE, "Scratchpad Area"); 719 if (fScratchpadArea[i] < B_OK) { 720 TRACE_ERROR("unable to create the scratchpad area\n"); 721 return B_ERROR; 722 } 723 fDcba->scratchpad[i] = scratchDmaAddress; 724 } 725 726 TRACE("setting DCBAAP %" B_PRIxPHYSADDR "\n", dmaAddress); 727 WriteOpReg(XHCI_DCBAAP_LO, (uint32)dmaAddress); 728 WriteOpReg(XHCI_DCBAAP_HI, (uint32)(dmaAddress >> 32)); 729 730 // allocate Event Ring Segment Table 731 uint8 *addr; 732 fErstArea = fStack->AllocateArea((void **)&addr, &dmaAddress, 733 (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 734 + sizeof(xhci_erst_element), 735 "USB XHCI ERST CMD_RING and EVENT_RING Area"); 736 737 if (fErstArea < B_OK) { 738 TRACE_ERROR("unable to create the ERST AND RING area\n"); 739 delete_area(fDcbaArea); 740 return B_ERROR; 741 } 742 fErst = (xhci_erst_element *)addr; 743 memset(fErst, 0, (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 744 + sizeof(xhci_erst_element)); 745 746 // fill with Event Ring Segment Base Address and Event Ring Segment Size 747 fErst->rs_addr = dmaAddress + sizeof(xhci_erst_element); 748 fErst->rs_size = XHCI_MAX_EVENTS; 749 fErst->rsvdz = 0; 750 751 addr += sizeof(xhci_erst_element); 752 fEventRing = (xhci_trb *)addr; 753 addr += XHCI_MAX_EVENTS * sizeof(xhci_trb); 754 fCmdRing = (xhci_trb *)addr; 755 756 TRACE("setting ERST size\n"); 757 WriteRunReg32(XHCI_ERSTSZ(0), XHCI_ERSTS_SET(1)); 758 759 TRACE("setting ERDP addr = 0x%" B_PRIx64 "\n", fErst->rs_addr); 760 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)fErst->rs_addr); 761 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(fErst->rs_addr >> 32)); 762 763 TRACE("setting ERST base addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 764 WriteRunReg32(XHCI_ERSTBA_LO(0), (uint32)dmaAddress); 765 WriteRunReg32(XHCI_ERSTBA_HI(0), (uint32)(dmaAddress >> 32)); 766 767 dmaAddress += sizeof(xhci_erst_element) + XHCI_MAX_EVENTS 768 * sizeof(xhci_trb); 769 770 // Make sure the Command Ring is stopped 771 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 772 TRACE_ALWAYS("Command Ring is running, send stop/cancel\n"); 773 WriteOpReg(XHCI_CRCR_LO, CRCR_CS); 774 WriteOpReg(XHCI_CRCR_HI, 0); 775 WriteOpReg(XHCI_CRCR_LO, CRCR_CA); 776 WriteOpReg(XHCI_CRCR_HI, 0); 777 snooze(1000); 778 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 779 TRACE_ERROR("Command Ring still running after stop/cancel\n"); 780 } 781 } 782 TRACE("setting CRCR addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 783 WriteOpReg(XHCI_CRCR_LO, (uint32)dmaAddress | CRCR_RCS); 784 WriteOpReg(XHCI_CRCR_HI, (uint32)(dmaAddress >> 32)); 785 // link trb 786 fCmdRing[XHCI_MAX_COMMANDS - 1].address = dmaAddress; 787 788 TRACE("setting interrupt rate\n"); 789 790 // Setting IMOD below 0x3F8 on Intel Lynx Point can cause IRQ lockups 791 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL 792 && (fPCIInfo->device_id == PCI_DEVICE_INTEL_PANTHER_POINT_XHCI 793 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_XHCI 794 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI 795 || fPCIInfo->device_id == PCI_DEVICE_INTEL_BAYTRAIL_XHCI 796 || fPCIInfo->device_id == PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI)) { 797 WriteRunReg32(XHCI_IMOD(0), 0x000003f8); // 4000 irq/s 798 } else { 799 WriteRunReg32(XHCI_IMOD(0), 0x000001f4); // 8000 irq/s 800 } 801 802 TRACE("enabling interrupt\n"); 803 WriteRunReg32(XHCI_IMAN(0), ReadRunReg32(XHCI_IMAN(0)) | IMAN_INTR_ENA); 804 805 WriteOpReg(XHCI_CMD, CMD_RUN | CMD_INTE | CMD_HSEE); 806 807 // wait for start up state 808 if (WaitOpBits(XHCI_STS, STS_HCH, 0) != B_OK) { 809 TRACE_ERROR("HCH start up timeout\n"); 810 } 811 812 fRootHub = new(std::nothrow) XHCIRootHub(RootObject(), 1); 813 if (!fRootHub) { 814 TRACE_ERROR("no memory to allocate root hub\n"); 815 return B_NO_MEMORY; 816 } 817 818 if (fRootHub->InitCheck() < B_OK) { 819 TRACE_ERROR("root hub failed init check\n"); 820 return fRootHub->InitCheck(); 821 } 822 823 SetRootHub(fRootHub); 824 825 fRootHub->RegisterNode(Node()); 826 827 TRACE_ALWAYS("successfully started the controller\n"); 828 829 #ifdef TRACE_USB 830 TRACE("No-Op test...\n"); 831 Noop(); 832 #endif 833 834 return BusManager::Start(); 835 } 836 837 838 status_t 839 XHCI::SubmitTransfer(Transfer *transfer) 840 { 841 // short circuit the root hub 842 if (transfer->TransferPipe()->DeviceAddress() == 1) 843 return fRootHub->ProcessTransfer(this, transfer); 844 845 TRACE("SubmitTransfer(%p)\n", transfer); 846 Pipe *pipe = transfer->TransferPipe(); 847 if ((pipe->Type() & USB_OBJECT_CONTROL_PIPE) != 0) 848 return SubmitControlRequest(transfer); 849 return SubmitNormalRequest(transfer); 850 } 851 852 853 status_t 854 XHCI::SubmitControlRequest(Transfer *transfer) 855 { 856 Pipe *pipe = transfer->TransferPipe(); 857 usb_request_data *requestData = transfer->RequestData(); 858 bool directionIn = (requestData->RequestType & USB_REQTYPE_DEVICE_IN) != 0; 859 860 TRACE("SubmitControlRequest() length %d\n", requestData->Length); 861 862 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 863 if (endpoint == NULL) { 864 TRACE_ERROR("control pipe has no endpoint!\n"); 865 return B_BAD_VALUE; 866 } 867 if (endpoint->device == NULL) { 868 panic("endpoint is not initialized!"); 869 return B_NO_INIT; 870 } 871 872 status_t status = transfer->InitKernelAccess(); 873 if (status != B_OK) 874 return status; 875 876 xhci_td *descriptor = CreateDescriptor(3, 1, requestData->Length); 877 if (descriptor == NULL) 878 return B_NO_MEMORY; 879 descriptor->transfer = transfer; 880 881 // Setup Stage 882 uint8 index = 0; 883 memcpy(&descriptor->trbs[index].address, requestData, 884 sizeof(usb_request_data)); 885 descriptor->trbs[index].status = TRB_2_IRQ(0) | TRB_2_BYTES(8); 886 descriptor->trbs[index].flags 887 = TRB_3_TYPE(TRB_TYPE_SETUP_STAGE) | TRB_3_IDT_BIT | TRB_3_CYCLE_BIT; 888 if (requestData->Length > 0) { 889 descriptor->trbs[index].flags |= 890 directionIn ? TRB_3_TRT_IN : TRB_3_TRT_OUT; 891 } 892 893 index++; 894 895 // Data Stage (if any) 896 if (requestData->Length > 0) { 897 descriptor->trbs[index].address = descriptor->buffer_addrs[0]; 898 descriptor->trbs[index].status = TRB_2_IRQ(0) 899 | TRB_2_BYTES(requestData->Length) 900 | TRB_2_TD_SIZE(0); 901 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_DATA_STAGE) 902 | (directionIn ? TRB_3_DIR_IN : 0) 903 | TRB_3_CYCLE_BIT; 904 905 if (!directionIn) { 906 transfer->PrepareKernelAccess(); 907 WriteDescriptor(descriptor, transfer->Vector(), 908 transfer->VectorCount(), transfer->IsPhysical()); 909 } 910 911 index++; 912 } 913 914 // Status Stage 915 descriptor->trbs[index].address = 0; 916 descriptor->trbs[index].status = TRB_2_IRQ(0); 917 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_STATUS_STAGE) 918 | TRB_3_CHAIN_BIT | TRB_3_ENT_BIT | TRB_3_CYCLE_BIT; 919 // The CHAIN bit must be set when using an Event Data TRB 920 // (XHCI 1.2 § 6.4.1.2.3 Table 6-31 p472). 921 922 // Status Stage is an OUT transfer when the device is sending data 923 // (XHCI 1.2 § 4.11.2.2 Table 4-7 p213), otherwise set the IN bit. 924 if (requestData->Length == 0 || !directionIn) 925 descriptor->trbs[index].flags |= TRB_3_DIR_IN; 926 927 descriptor->trb_used = index + 1; 928 929 status = _LinkDescriptorForPipe(descriptor, endpoint); 930 if (status != B_OK) { 931 FreeDescriptor(descriptor); 932 return status; 933 } 934 935 return B_OK; 936 } 937 938 939 status_t 940 XHCI::SubmitNormalRequest(Transfer *transfer) 941 { 942 TRACE("SubmitNormalRequest() length %" B_PRIuSIZE "\n", transfer->FragmentLength()); 943 944 Pipe *pipe = transfer->TransferPipe(); 945 usb_isochronous_data *isochronousData = transfer->IsochronousData(); 946 bool directionIn = (pipe->Direction() == Pipe::In); 947 948 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 949 if (endpoint == NULL) { 950 TRACE_ERROR("pipe has no endpoint!\n"); 951 return B_BAD_VALUE; 952 } 953 if (endpoint->device == NULL) { 954 panic("endpoint is not initialized!"); 955 return B_NO_INIT; 956 } 957 958 status_t status = transfer->InitKernelAccess(); 959 if (status != B_OK) 960 return status; 961 962 // TRBs within a TD must be "grouped" into TD Fragments, which mostly means 963 // that a max_burst_payload boundary cannot be crossed within a TRB, but 964 // only between TRBs. More than one TRB can be in a TD Fragment, but we keep 965 // things simple by setting trbSize to the MBP. (XHCI 1.2 § 4.11.7.1 p235.) 966 size_t trbSize = endpoint->max_burst_payload; 967 968 if (isochronousData != NULL) { 969 if (isochronousData->packet_count == 0) 970 return B_BAD_VALUE; 971 972 // Isochronous transfers use more specifically sized packets. 973 trbSize = transfer->DataLength() / isochronousData->packet_count; 974 if (trbSize == 0 || trbSize > pipe->MaxPacketSize() || trbSize 975 != (size_t)isochronousData->packet_descriptors[0].request_length) 976 return B_BAD_VALUE; 977 } 978 979 // Now that we know trbSize, compute the count. 980 const int32 trbCount = (transfer->FragmentLength() + trbSize - 1) / trbSize; 981 982 xhci_td *td = CreateDescriptor(trbCount, trbCount, trbSize); 983 if (td == NULL) 984 return B_NO_MEMORY; 985 986 // Normal Stage 987 const size_t maxPacketSize = pipe->MaxPacketSize(); 988 size_t remaining = transfer->FragmentLength(); 989 for (int32 i = 0; i < trbCount; i++) { 990 int32 trbLength = (remaining < trbSize) ? remaining : trbSize; 991 remaining -= trbLength; 992 993 // The "TD Size" field of a transfer TRB indicates the number of 994 // remaining maximum-size *packets* in this TD, *not* including the 995 // packets in the current TRB, and capped at 31 if there are more 996 // than 31 packets remaining in the TD. (XHCI 1.2 § 4.11.2.4 p218.) 997 int32 tdSize = (remaining + maxPacketSize - 1) / maxPacketSize; 998 if (tdSize > 31) 999 tdSize = 31; 1000 1001 td->trbs[i].address = td->buffer_addrs[i]; 1002 td->trbs[i].status = TRB_2_IRQ(0) 1003 | TRB_2_BYTES(trbLength) 1004 | TRB_2_TD_SIZE(tdSize); 1005 td->trbs[i].flags = TRB_3_TYPE(TRB_TYPE_NORMAL) 1006 | TRB_3_CYCLE_BIT | TRB_3_CHAIN_BIT; 1007 1008 td->trb_used++; 1009 } 1010 1011 // Isochronous-specific. 1012 if (isochronousData != NULL) { 1013 // This is an isochronous transfer; it should have one TD per packet. 1014 for (uint32 i = 0; i < isochronousData->packet_count; i++) { 1015 td->trbs[i].flags &= ~(TRB_3_TYPE(TRB_TYPE_NORMAL)); 1016 td->trbs[i].flags |= TRB_3_TYPE(TRB_TYPE_ISOCH); 1017 1018 if (i != (isochronousData->packet_count - 1)) { 1019 // For all but the last TD, generate events (but not interrupts) on short packets. 1020 // (The last TD uses the regular Event Data TRB.) 1021 td->trbs[i].flags |= TRB_3_ISP_BIT | TRB_3_BEI_BIT; 1022 td->trbs[i].flags &= ~TRB_3_CHAIN_BIT; 1023 } 1024 } 1025 1026 // TODO: We do not currently take Mult into account at all! 1027 // How are we supposed to do that here? 1028 1029 // Determine the (starting) frame number: if ISO_ASAP is set, 1030 // we are queueing this "right away", and so want to reset 1031 // the starting_frame_number. Otherwise we use the passed one. 1032 uint32 frame; 1033 if ((isochronousData->flags & USB_ISO_ASAP) != 0 1034 || isochronousData->starting_frame_number == NULL) { 1035 // All reads from the microframe index register must be 1036 // incremented by 1. (XHCI 1.2 § 4.14.2.1.4 p265.) 1037 frame = (ReadRunReg32(XHCI_MFINDEX) + 1) >> 3; 1038 td->trbs[0].flags |= TRB_3_ISO_SIA_BIT; 1039 } else { 1040 frame = *isochronousData->starting_frame_number; 1041 td->trbs[0].flags |= TRB_3_FRID(frame); 1042 } 1043 if (isochronousData->starting_frame_number != NULL) 1044 *isochronousData->starting_frame_number = frame; 1045 } 1046 1047 // Set the ENT (Evaluate Next TRB) bit, so that the HC will not switch 1048 // contexts before evaluating the Link TRB that _LinkDescriptorForPipe 1049 // will insert, as otherwise there would be a race between us freeing 1050 // and unlinking the descriptor, and the controller evaluating the Link TRB 1051 // and thus getting back onto the main ring and executing the Event Data 1052 // TRB that generates the interrupt for this transfer. 1053 // 1054 // Note that we *do not* unset the CHAIN bit in this TRB, thus including 1055 // the Link TRB in this TD formally, which is required when using the 1056 // ENT bit. (XHCI 1.2 § 4.12.3 p250.) 1057 td->trbs[td->trb_used - 1].flags |= TRB_3_ENT_BIT; 1058 1059 if (!directionIn) { 1060 TRACE("copying out iov count %ld\n", transfer->VectorCount()); 1061 status_t status = transfer->PrepareKernelAccess(); 1062 if (status != B_OK) { 1063 FreeDescriptor(td); 1064 return status; 1065 } 1066 WriteDescriptor(td, transfer->Vector(), 1067 transfer->VectorCount(), transfer->IsPhysical()); 1068 } 1069 1070 td->transfer = transfer; 1071 status = _LinkDescriptorForPipe(td, endpoint); 1072 if (status != B_OK) { 1073 FreeDescriptor(td); 1074 return status; 1075 } 1076 1077 return B_OK; 1078 } 1079 1080 1081 status_t 1082 XHCI::CancelQueuedTransfers(Pipe *pipe, bool force) 1083 { 1084 xhci_endpoint* endpoint = (xhci_endpoint*)pipe->ControllerCookie(); 1085 if (endpoint == NULL || endpoint->trbs == NULL) { 1086 // Someone's de-allocated this pipe or endpoint in the meantime. 1087 // (Possibly AllocateDevice failed, and we were the temporary pipe.) 1088 return B_NO_INIT; 1089 } 1090 1091 #ifndef TRACE_USB 1092 if (force) 1093 #endif 1094 { 1095 TRACE_ALWAYS("cancel queued transfers (%" B_PRId8 ") for pipe %p (%d)\n", 1096 endpoint->used, pipe, pipe->EndpointAddress()); 1097 } 1098 1099 MutexLocker endpointLocker(endpoint->lock); 1100 1101 if (endpoint->td_head == NULL) { 1102 // There aren't any currently pending transfers to cancel. 1103 return B_OK; 1104 } 1105 1106 // Calling the callbacks while holding the endpoint lock could potentially 1107 // cause deadlocks, so we instead store them in a pointer array. We need 1108 // to do this separately from freeing the TDs, for in the case we fail 1109 // to stop the endpoint, we cancel the transfers but do not free the TDs. 1110 Transfer* transfers[XHCI_MAX_TRANSFERS]; 1111 int32 transfersCount = 0; 1112 1113 for (xhci_td* td = endpoint->td_head; td != NULL; td = td->next) { 1114 if (td->transfer == NULL) 1115 continue; 1116 1117 // We can't cancel or delete transfers under "force", as they probably 1118 // are not safe to use anymore. 1119 if (!force) { 1120 transfers[transfersCount] = td->transfer; 1121 transfersCount++; 1122 } 1123 td->transfer = NULL; 1124 } 1125 1126 // It is possible that while waiting for the stop-endpoint command to 1127 // complete, one of the queued transfers posts a completion event, so in 1128 // order to avoid a deadlock, we must unlock the endpoint. 1129 endpointLocker.Unlock(); 1130 status_t status = StopEndpoint(false, endpoint); 1131 if (status != B_OK && status != B_DEV_STALLED) { 1132 // It is possible that the endpoint was stopped by the controller at the 1133 // same time our STOP command was in progress, causing a "Context State" 1134 // error. In that case, try again; if the endpoint is already stopped, 1135 // StopEndpoint will notice this. (XHCI 1.2 § 4.6.9 p137.) 1136 status = StopEndpoint(false, endpoint); 1137 } 1138 if (status == B_DEV_STALLED) { 1139 // Only exit from a Halted state is a RESET. (XHCI 1.2 § 4.8.3 p163.) 1140 TRACE_ERROR("cancel queued transfers: halted endpoint, reset!\n"); 1141 status = ResetEndpoint(false, endpoint); 1142 } 1143 endpointLocker.Lock(); 1144 1145 // Detach the head TD from the endpoint. 1146 xhci_td* td_head = endpoint->td_head; 1147 endpoint->td_head = NULL; 1148 1149 if (status == B_OK) { 1150 // Clear the endpoint's TRBs. 1151 memset(endpoint->trbs, 0, sizeof(xhci_trb) * XHCI_ENDPOINT_RING_SIZE); 1152 endpoint->used = 0; 1153 endpoint->next = 0; 1154 1155 // Set dequeue pointer location to the beginning of the ring. 1156 SetTRDequeue(endpoint->trb_addr, 0, endpoint->id + 1, 1157 endpoint->device->slot); 1158 1159 // We don't need to do anything else to restart the ring, as it will resume 1160 // operation as normal upon the next doorbell. (XHCI 1.2 § 4.6.9 p136.) 1161 } else { 1162 // We couldn't stop the endpoint. Most likely the device has been 1163 // removed and the endpoint was stopped by the hardware, or is 1164 // for some reason busy and cannot be stopped. 1165 TRACE_ERROR("cancel queued transfers: could not stop endpoint: %s!\n", 1166 strerror(status)); 1167 1168 // Instead of freeing the TDs, we want to leave them in the endpoint 1169 // so that when/if the hardware returns, they can be properly unlinked, 1170 // as otherwise the endpoint could get "stuck" by having the "used" 1171 // slowly accumulate due to "dead" transfers. 1172 endpoint->td_head = td_head; 1173 td_head = NULL; 1174 } 1175 1176 endpointLocker.Unlock(); 1177 1178 for (int32 i = 0; i < transfersCount; i++) { 1179 transfers[i]->Finished(B_CANCELED, 0); 1180 delete transfers[i]; 1181 } 1182 1183 // This loop looks a bit strange because we need to store the "next" 1184 // pointer before freeing the descriptor. 1185 xhci_td* td; 1186 while ((td = td_head) != NULL) { 1187 td_head = td_head->next; 1188 FreeDescriptor(td); 1189 } 1190 1191 return B_OK; 1192 } 1193 1194 1195 status_t 1196 XHCI::StartDebugTransfer(Transfer *transfer) 1197 { 1198 Pipe *pipe = transfer->TransferPipe(); 1199 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 1200 if (endpoint == NULL) 1201 return B_BAD_VALUE; 1202 1203 // Check all locks that we are going to hit when running transfers. 1204 if (mutex_trylock(&endpoint->lock) != B_OK) 1205 return B_WOULD_BLOCK; 1206 if (mutex_trylock(&fFinishedLock) != B_OK) { 1207 mutex_unlock(&endpoint->lock); 1208 return B_WOULD_BLOCK; 1209 } 1210 if (mutex_trylock(&fEventLock) != B_OK) { 1211 mutex_unlock(&endpoint->lock); 1212 mutex_unlock(&fFinishedLock); 1213 return B_WOULD_BLOCK; 1214 } 1215 mutex_unlock(&endpoint->lock); 1216 mutex_unlock(&fFinishedLock); 1217 mutex_unlock(&fEventLock); 1218 1219 status_t status = SubmitTransfer(transfer); 1220 if (status != B_OK) 1221 return status; 1222 1223 // The endpoint's head TD is the TD of the just-submitted transfer. 1224 // Just like EHCI, abuse the callback cookie to hold the TD pointer. 1225 transfer->SetCallback(NULL, endpoint->td_head); 1226 1227 return B_OK; 1228 } 1229 1230 1231 status_t 1232 XHCI::CheckDebugTransfer(Transfer *transfer) 1233 { 1234 xhci_td *transfer_td = (xhci_td *)transfer->CallbackCookie(); 1235 if (transfer_td == NULL) 1236 return B_NO_INIT; 1237 1238 // Process events once, and then look for it in the finished list. 1239 ProcessEvents(); 1240 xhci_td *previous = NULL; 1241 for (xhci_td *td = fFinishedHead; td != NULL; td = td->next) { 1242 if (td != transfer_td) { 1243 previous = td; 1244 continue; 1245 } 1246 1247 // We've found it! 1248 if (previous == NULL) { 1249 fFinishedHead = fFinishedHead->next; 1250 } else { 1251 previous->next = td->next; 1252 } 1253 1254 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 1255 status_t status = (td->trb_completion_code == COMP_SUCCESS 1256 || td->trb_completion_code == COMP_SHORT_PACKET) ? B_OK : B_ERROR; 1257 1258 if (status == B_OK && directionIn) { 1259 ReadDescriptor(td, transfer->Vector(), transfer->VectorCount(), 1260 transfer->IsPhysical()); 1261 } 1262 1263 FreeDescriptor(td); 1264 transfer->SetCallback(NULL, NULL); 1265 return status; 1266 } 1267 1268 // We didn't find it. 1269 spin(75); 1270 return B_DEV_PENDING; 1271 } 1272 1273 1274 void 1275 XHCI::CancelDebugTransfer(Transfer *transfer) 1276 { 1277 while (CheckDebugTransfer(transfer) == B_DEV_PENDING) 1278 spin(100); 1279 } 1280 1281 1282 status_t 1283 XHCI::NotifyPipeChange(Pipe *pipe, usb_change change) 1284 { 1285 TRACE("pipe change %d for pipe %p (%d)\n", change, pipe, 1286 pipe->EndpointAddress()); 1287 1288 switch (change) { 1289 case USB_CHANGE_CREATED: 1290 return _InsertEndpointForPipe(pipe); 1291 case USB_CHANGE_DESTROYED: 1292 return _RemoveEndpointForPipe(pipe); 1293 1294 case USB_CHANGE_PIPE_POLICY_CHANGED: 1295 // We don't care about these, at least for now. 1296 return B_OK; 1297 } 1298 1299 TRACE_ERROR("unknown pipe change!\n"); 1300 return B_UNSUPPORTED; 1301 } 1302 1303 1304 xhci_td * 1305 XHCI::CreateDescriptor(uint32 trbCount, uint32 bufferCount, size_t bufferSize) 1306 { 1307 const bool inKDL = debug_debugger_running(); 1308 1309 xhci_td *result; 1310 if (!inKDL) { 1311 result = (xhci_td*)calloc(1, sizeof(xhci_td)); 1312 } else { 1313 // Just use the physical memory allocator while in KDL; it's less 1314 // secure than using the regular heap, but it's easier to deal with. 1315 phys_addr_t dummy; 1316 fStack->AllocateChunk((void **)&result, &dummy, sizeof(xhci_td)); 1317 } 1318 1319 if (result == NULL) { 1320 TRACE_ERROR("failed to allocate a transfer descriptor\n"); 1321 return NULL; 1322 } 1323 1324 // We always allocate 1 more TRB than requested, so that 1325 // _LinkDescriptorForPipe() has room to insert a link TRB. 1326 trbCount++; 1327 if (fStack->AllocateChunk((void **)&result->trbs, &result->trb_addr, 1328 (trbCount * sizeof(xhci_trb))) < B_OK) { 1329 TRACE_ERROR("failed to allocate TRBs\n"); 1330 FreeDescriptor(result); 1331 return NULL; 1332 } 1333 result->trb_count = trbCount; 1334 result->trb_used = 0; 1335 1336 if (bufferSize > 0) { 1337 // Due to how the USB stack allocates physical memory, we can't just 1338 // request one large chunk the size of the transfer, and so instead we 1339 // create a series of buffers as requested by our caller. 1340 1341 // We store the buffer pointers and addresses in one memory block. 1342 if (!inKDL) { 1343 result->buffers = (void**)calloc(bufferCount, 1344 (sizeof(void*) + sizeof(phys_addr_t))); 1345 } else { 1346 phys_addr_t dummy; 1347 fStack->AllocateChunk((void **)&result->buffers, &dummy, 1348 bufferCount * (sizeof(void*) + sizeof(phys_addr_t))); 1349 } 1350 if (result->buffers == NULL) { 1351 TRACE_ERROR("unable to allocate space for buffer infos\n"); 1352 FreeDescriptor(result); 1353 return NULL; 1354 } 1355 result->buffer_addrs = (phys_addr_t*)&result->buffers[bufferCount]; 1356 result->buffer_size = bufferSize; 1357 result->buffer_count = bufferCount; 1358 1359 // Optimization: If the requested total size of all buffers is less 1360 // than 32*B_PAGE_SIZE (the maximum size that the physical memory 1361 // allocator can handle), we allocate only one buffer and segment it. 1362 size_t totalSize = bufferSize * bufferCount; 1363 if (totalSize < (32 * B_PAGE_SIZE)) { 1364 if (fStack->AllocateChunk(&result->buffers[0], 1365 &result->buffer_addrs[0], totalSize) < B_OK) { 1366 TRACE_ERROR("unable to allocate space for large buffer (size %ld)\n", 1367 totalSize); 1368 FreeDescriptor(result); 1369 return NULL; 1370 } 1371 for (uint32 i = 1; i < bufferCount; i++) { 1372 result->buffers[i] = (void*)((addr_t)(result->buffers[i - 1]) 1373 + bufferSize); 1374 result->buffer_addrs[i] = result->buffer_addrs[i - 1] 1375 + bufferSize; 1376 } 1377 } else { 1378 // Otherwise, we allocate each buffer individually. 1379 for (uint32 i = 0; i < bufferCount; i++) { 1380 if (fStack->AllocateChunk(&result->buffers[i], 1381 &result->buffer_addrs[i], bufferSize) < B_OK) { 1382 TRACE_ERROR("unable to allocate space for a buffer (size " 1383 "%" B_PRIuSIZE ", count %" B_PRIu32 ")\n", 1384 bufferSize, bufferCount); 1385 FreeDescriptor(result); 1386 return NULL; 1387 } 1388 } 1389 } 1390 } else { 1391 result->buffers = NULL; 1392 result->buffer_addrs = NULL; 1393 } 1394 1395 // Initialize all other fields. 1396 result->transfer = NULL; 1397 result->trb_completion_code = 0; 1398 result->trb_left = 0; 1399 result->next = NULL; 1400 1401 TRACE("CreateDescriptor allocated %p, buffer_size %ld, buffer_count %" B_PRIu32 "\n", 1402 result, result->buffer_size, result->buffer_count); 1403 1404 return result; 1405 } 1406 1407 1408 void 1409 XHCI::FreeDescriptor(xhci_td *descriptor) 1410 { 1411 if (descriptor == NULL) 1412 return; 1413 1414 const bool inKDL = debug_debugger_running(); 1415 1416 if (descriptor->trbs != NULL) { 1417 fStack->FreeChunk(descriptor->trbs, descriptor->trb_addr, 1418 (descriptor->trb_count * sizeof(xhci_trb))); 1419 } 1420 if (descriptor->buffers != NULL) { 1421 size_t totalSize = descriptor->buffer_size * descriptor->buffer_count; 1422 if (totalSize < (32 * B_PAGE_SIZE)) { 1423 // This was allocated as one contiguous buffer. 1424 fStack->FreeChunk(descriptor->buffers[0], descriptor->buffer_addrs[0], 1425 totalSize); 1426 } else { 1427 for (uint32 i = 0; i < descriptor->buffer_count; i++) { 1428 if (descriptor->buffers[i] == NULL) 1429 continue; 1430 fStack->FreeChunk(descriptor->buffers[i], descriptor->buffer_addrs[i], 1431 descriptor->buffer_size); 1432 } 1433 } 1434 1435 if (!inKDL) { 1436 free(descriptor->buffers); 1437 } else { 1438 fStack->FreeChunk(descriptor->buffers, 0, 1439 descriptor->buffer_count * (sizeof(void*) + sizeof(phys_addr_t))); 1440 } 1441 } 1442 1443 if (!inKDL) 1444 free(descriptor); 1445 else 1446 fStack->FreeChunk(descriptor, 0, sizeof(xhci_td)); 1447 } 1448 1449 1450 size_t 1451 XHCI::WriteDescriptor(xhci_td *descriptor, generic_io_vec *vector, size_t vectorCount, bool physical) 1452 { 1453 size_t written = 0; 1454 1455 size_t bufIdx = 0, bufUsed = 0; 1456 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1457 size_t length = vector[vecIdx].length; 1458 1459 while (length > 0 && bufIdx < descriptor->buffer_count) { 1460 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1461 status_t status = generic_memcpy( 1462 (generic_addr_t)descriptor->buffers[bufIdx] + bufUsed, false, 1463 vector[vecIdx].base + (vector[vecIdx].length - length), physical, 1464 toCopy); 1465 ASSERT(status == B_OK); 1466 1467 written += toCopy; 1468 bufUsed += toCopy; 1469 length -= toCopy; 1470 if (bufUsed == descriptor->buffer_size) { 1471 bufIdx++; 1472 bufUsed = 0; 1473 } 1474 } 1475 } 1476 1477 TRACE("wrote descriptor (%" B_PRIuSIZE " bytes)\n", written); 1478 return written; 1479 } 1480 1481 1482 size_t 1483 XHCI::ReadDescriptor(xhci_td *descriptor, generic_io_vec *vector, size_t vectorCount, bool physical) 1484 { 1485 size_t read = 0; 1486 1487 size_t bufIdx = 0, bufUsed = 0; 1488 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1489 size_t length = vector[vecIdx].length; 1490 1491 while (length > 0 && bufIdx < descriptor->buffer_count) { 1492 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1493 status_t status = generic_memcpy( 1494 vector[vecIdx].base + (vector[vecIdx].length - length), physical, 1495 (generic_addr_t)descriptor->buffers[bufIdx] + bufUsed, false, toCopy); 1496 ASSERT(status == B_OK); 1497 1498 read += toCopy; 1499 bufUsed += toCopy; 1500 length -= toCopy; 1501 if (bufUsed == descriptor->buffer_size) { 1502 bufIdx++; 1503 bufUsed = 0; 1504 } 1505 } 1506 } 1507 1508 TRACE("read descriptor (%" B_PRIuSIZE " bytes)\n", read); 1509 return read; 1510 } 1511 1512 1513 Device * 1514 XHCI::AllocateDevice(Hub *parent, int8 hubAddress, uint8 hubPort, 1515 usb_speed speed) 1516 { 1517 TRACE("AllocateDevice hubAddress %d hubPort %d speed %d\n", hubAddress, 1518 hubPort, speed); 1519 1520 uint8 slot = XHCI_MAX_SLOTS; 1521 status_t status = EnableSlot(&slot); 1522 if (status != B_OK) { 1523 TRACE_ERROR("failed to enable slot: %s\n", strerror(status)); 1524 return NULL; 1525 } 1526 1527 if (slot == 0 || slot > fSlotCount) { 1528 TRACE_ERROR("AllocateDevice: bad slot\n"); 1529 return NULL; 1530 } 1531 1532 if (fDevices[slot].slot != 0) { 1533 TRACE_ERROR("AllocateDevice: slot already used\n"); 1534 return NULL; 1535 } 1536 1537 struct xhci_device *device = &fDevices[slot]; 1538 device->slot = slot; 1539 1540 device->input_ctx_area = fStack->AllocateArea((void **)&device->input_ctx, 1541 &device->input_ctx_addr, sizeof(*device->input_ctx) << fContextSizeShift, 1542 "XHCI input context"); 1543 if (device->input_ctx_area < B_OK) { 1544 TRACE_ERROR("unable to create a input context area\n"); 1545 CleanupDevice(device); 1546 return NULL; 1547 } 1548 if (fContextSizeShift == 1) { 1549 // 64-byte contexts have to be page-aligned in order for 1550 // _OffsetContextAddr to function properly. 1551 ASSERT((((addr_t)device->input_ctx) % B_PAGE_SIZE) == 0); 1552 } 1553 1554 memset(device->input_ctx, 0, sizeof(*device->input_ctx) << fContextSizeShift); 1555 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1556 _WriteContext(&device->input_ctx->input.addFlags, 3); 1557 1558 uint8 rhPort = hubPort; 1559 uint32 route = 0; 1560 for (Device *hubDevice = parent; hubDevice != RootObject(); 1561 hubDevice = (Device *)hubDevice->Parent()) { 1562 if (hubDevice->Parent() == RootObject()) 1563 break; 1564 1565 if (rhPort > 15) 1566 rhPort = 15; 1567 route = route << 4; 1568 route |= rhPort; 1569 1570 rhPort = hubDevice->HubPort(); 1571 } 1572 1573 uint32 dwslot0 = SLOT_0_NUM_ENTRIES(1) | SLOT_0_ROUTE(route); 1574 1575 // Get speed of port, only if device connected to root hub port 1576 // else we have to rely on value reported by the Hub Explore thread 1577 if (route == 0) { 1578 GetPortSpeed(hubPort - 1, &speed); 1579 TRACE("speed updated %d\n", speed); 1580 } 1581 1582 // add the speed 1583 switch (speed) { 1584 case USB_SPEED_LOWSPEED: 1585 dwslot0 |= SLOT_0_SPEED(2); 1586 break; 1587 case USB_SPEED_FULLSPEED: 1588 dwslot0 |= SLOT_0_SPEED(1); 1589 break; 1590 case USB_SPEED_HIGHSPEED: 1591 dwslot0 |= SLOT_0_SPEED(3); 1592 break; 1593 case USB_SPEED_SUPERSPEED: 1594 dwslot0 |= SLOT_0_SPEED(4); 1595 break; 1596 default: 1597 TRACE_ERROR("unknown usb speed\n"); 1598 break; 1599 } 1600 1601 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1602 // TODO enable power save 1603 _WriteContext(&device->input_ctx->slot.dwslot1, SLOT_1_RH_PORT(rhPort)); 1604 uint32 dwslot2 = SLOT_2_IRQ_TARGET(0); 1605 1606 // If LS/FS device connected to non-root HS device 1607 if (route != 0 && parent->Speed() == USB_SPEED_HIGHSPEED 1608 && (speed == USB_SPEED_LOWSPEED || speed == USB_SPEED_FULLSPEED)) { 1609 struct xhci_device *parenthub = (struct xhci_device *) 1610 parent->ControllerCookie(); 1611 dwslot2 |= SLOT_2_PORT_NUM(hubPort); 1612 dwslot2 |= SLOT_2_TT_HUB_SLOT(parenthub->slot); 1613 } 1614 1615 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1616 1617 _WriteContext(&device->input_ctx->slot.dwslot3, SLOT_3_SLOT_STATE(0) 1618 | SLOT_3_DEVICE_ADDRESS(0)); 1619 1620 TRACE("slot 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 1621 "\n", _ReadContext(&device->input_ctx->slot.dwslot0), 1622 _ReadContext(&device->input_ctx->slot.dwslot1), 1623 _ReadContext(&device->input_ctx->slot.dwslot2), 1624 _ReadContext(&device->input_ctx->slot.dwslot3)); 1625 1626 device->device_ctx_area = fStack->AllocateArea((void **)&device->device_ctx, 1627 &device->device_ctx_addr, sizeof(*device->device_ctx) << fContextSizeShift, 1628 "XHCI device context"); 1629 if (device->device_ctx_area < B_OK) { 1630 TRACE_ERROR("unable to create a device context area\n"); 1631 CleanupDevice(device); 1632 return NULL; 1633 } 1634 memset(device->device_ctx, 0, sizeof(*device->device_ctx) << fContextSizeShift); 1635 1636 device->trb_area = fStack->AllocateArea((void **)&device->trbs, 1637 &device->trb_addr, sizeof(xhci_trb) * (XHCI_MAX_ENDPOINTS - 1) 1638 * XHCI_ENDPOINT_RING_SIZE, "XHCI endpoint trbs"); 1639 if (device->trb_area < B_OK) { 1640 TRACE_ERROR("unable to create a device trbs area\n"); 1641 CleanupDevice(device); 1642 return NULL; 1643 } 1644 1645 // set up slot pointer to device context 1646 fDcba->baseAddress[slot] = device->device_ctx_addr; 1647 1648 size_t maxPacketSize; 1649 switch (speed) { 1650 case USB_SPEED_LOWSPEED: 1651 case USB_SPEED_FULLSPEED: 1652 maxPacketSize = 8; 1653 break; 1654 case USB_SPEED_HIGHSPEED: 1655 maxPacketSize = 64; 1656 break; 1657 default: 1658 maxPacketSize = 512; 1659 break; 1660 } 1661 1662 xhci_endpoint* endpoint0 = &device->endpoints[0]; 1663 mutex_init(&endpoint0->lock, "xhci endpoint lock"); 1664 endpoint0->device = device; 1665 endpoint0->id = 0; 1666 endpoint0->status = 0; 1667 endpoint0->td_head = NULL; 1668 endpoint0->used = 0; 1669 endpoint0->next = 0; 1670 endpoint0->trbs = device->trbs; 1671 endpoint0->trb_addr = device->trb_addr; 1672 1673 // configure the Control endpoint 0 1674 if (ConfigureEndpoint(endpoint0, slot, 0, USB_OBJECT_CONTROL_PIPE, false, 1675 0, maxPacketSize, speed, 0, 0) != B_OK) { 1676 TRACE_ERROR("unable to configure default control endpoint\n"); 1677 CleanupDevice(device); 1678 return NULL; 1679 } 1680 1681 // device should get to addressed state (bsr = 0) 1682 status = SetAddress(device->input_ctx_addr, false, slot); 1683 if (status != B_OK) { 1684 TRACE_ERROR("unable to set address: %s\n", strerror(status)); 1685 CleanupDevice(device); 1686 return NULL; 1687 } 1688 1689 device->address = SLOT_3_DEVICE_ADDRESS_GET(_ReadContext( 1690 &device->device_ctx->slot.dwslot3)); 1691 1692 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", device->address, 1693 SLOT_3_SLOT_STATE_GET(_ReadContext( 1694 &device->device_ctx->slot.dwslot3))); 1695 TRACE("endpoint0 state 0x%08" B_PRIx32 "\n", 1696 ENDPOINT_0_STATE_GET(_ReadContext( 1697 &device->device_ctx->endpoints[0].dwendpoint0))); 1698 1699 // Wait a bit for the device to complete addressing 1700 snooze(USB_DELAY_SET_ADDRESS); 1701 1702 // Create a temporary pipe with the new address 1703 ControlPipe pipe(parent); 1704 pipe.SetControllerCookie(endpoint0); 1705 pipe.InitCommon(device->address + 1, 0, speed, Pipe::Default, maxPacketSize, 0, 1706 hubAddress, hubPort); 1707 1708 // Get the device descriptor 1709 // Just retrieve the first 8 bytes of the descriptor -> minimum supported 1710 // size of any device. It is enough because it includes the device type. 1711 1712 size_t actualLength = 0; 1713 usb_device_descriptor deviceDescriptor; 1714 1715 TRACE("getting the device descriptor\n"); 1716 status = pipe.SendRequest( 1717 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_STANDARD, // type 1718 USB_REQUEST_GET_DESCRIPTOR, // request 1719 USB_DESCRIPTOR_DEVICE << 8, // value 1720 0, // index 1721 8, // length 1722 (void *)&deviceDescriptor, // buffer 1723 8, // buffer length 1724 &actualLength); // actual length 1725 1726 if (actualLength != 8) { 1727 TRACE_ERROR("failed to get the device descriptor: %s\n", 1728 strerror(status)); 1729 CleanupDevice(device); 1730 return NULL; 1731 } 1732 1733 TRACE("device_class: %d device_subclass %d device_protocol %d\n", 1734 deviceDescriptor.device_class, deviceDescriptor.device_subclass, 1735 deviceDescriptor.device_protocol); 1736 1737 if (speed == USB_SPEED_FULLSPEED && deviceDescriptor.max_packet_size_0 != 8) { 1738 TRACE("Full speed device with different max packet size for Endpoint 0\n"); 1739 uint32 dwendpoint1 = _ReadContext( 1740 &device->input_ctx->endpoints[0].dwendpoint1); 1741 dwendpoint1 &= ~ENDPOINT_1_MAXPACKETSIZE(0xffff); 1742 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE( 1743 deviceDescriptor.max_packet_size_0); 1744 _WriteContext(&device->input_ctx->endpoints[0].dwendpoint1, 1745 dwendpoint1); 1746 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1747 _WriteContext(&device->input_ctx->input.addFlags, (1 << 1)); 1748 EvaluateContext(device->input_ctx_addr, device->slot); 1749 } 1750 1751 Device *deviceObject = NULL; 1752 if (deviceDescriptor.device_class == 0x09) { 1753 TRACE("creating new Hub\n"); 1754 TRACE("getting the hub descriptor\n"); 1755 size_t actualLength = 0; 1756 usb_hub_descriptor hubDescriptor; 1757 status = pipe.SendRequest( 1758 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_CLASS, // type 1759 USB_REQUEST_GET_DESCRIPTOR, // request 1760 USB_DESCRIPTOR_HUB << 8, // value 1761 0, // index 1762 sizeof(usb_hub_descriptor), // length 1763 (void *)&hubDescriptor, // buffer 1764 sizeof(usb_hub_descriptor), // buffer length 1765 &actualLength); 1766 1767 if (actualLength != sizeof(usb_hub_descriptor)) { 1768 TRACE_ERROR("error while getting the hub descriptor: %s\n", 1769 strerror(status)); 1770 CleanupDevice(device); 1771 return NULL; 1772 } 1773 1774 uint32 dwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1775 dwslot0 |= SLOT_0_HUB_BIT; 1776 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1777 uint32 dwslot1 = _ReadContext(&device->input_ctx->slot.dwslot1); 1778 dwslot1 |= SLOT_1_NUM_PORTS(hubDescriptor.num_ports); 1779 _WriteContext(&device->input_ctx->slot.dwslot1, dwslot1); 1780 if (speed == USB_SPEED_HIGHSPEED) { 1781 uint32 dwslot2 = _ReadContext(&device->input_ctx->slot.dwslot2); 1782 dwslot2 |= SLOT_2_TT_TIME(HUB_TTT_GET(hubDescriptor.characteristics)); 1783 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1784 } 1785 1786 deviceObject = new(std::nothrow) Hub(parent, hubAddress, hubPort, 1787 deviceDescriptor, device->address + 1, speed, false, device); 1788 } else { 1789 TRACE("creating new device\n"); 1790 deviceObject = new(std::nothrow) Device(parent, hubAddress, hubPort, 1791 deviceDescriptor, device->address + 1, speed, false, device); 1792 } 1793 if (deviceObject == NULL || deviceObject->InitCheck() != B_OK) { 1794 if (deviceObject == NULL) { 1795 TRACE_ERROR("no memory to allocate device\n"); 1796 } else { 1797 TRACE_ERROR("device object failed to initialize\n"); 1798 } 1799 CleanupDevice(device); 1800 return NULL; 1801 } 1802 1803 // We don't want to disable the default endpoint, naturally, which would 1804 // otherwise happen when this Pipe object is destroyed. 1805 pipe.SetControllerCookie(NULL); 1806 1807 deviceObject->RegisterNode(); 1808 1809 TRACE("AllocateDevice() port %d slot %d\n", hubPort, slot); 1810 return deviceObject; 1811 } 1812 1813 1814 void 1815 XHCI::FreeDevice(Device *usbDevice) 1816 { 1817 xhci_device* device = (xhci_device*)usbDevice->ControllerCookie(); 1818 TRACE("FreeDevice() slot %d\n", device->slot); 1819 1820 // Delete the device first, so it cleans up its pipes and tells us 1821 // what we need to destroy before we tear down our internal state. 1822 delete usbDevice; 1823 1824 CleanupDevice(device); 1825 } 1826 1827 1828 void 1829 XHCI::CleanupDevice(xhci_device *device) 1830 { 1831 if (device->slot != 0) { 1832 DisableSlot(device->slot); 1833 fDcba->baseAddress[device->slot] = 0; 1834 } 1835 1836 if (device->trb_addr != 0) 1837 delete_area(device->trb_area); 1838 if (device->input_ctx_addr != 0) 1839 delete_area(device->input_ctx_area); 1840 if (device->device_ctx_addr != 0) 1841 delete_area(device->device_ctx_area); 1842 1843 memset(device, 0, sizeof(xhci_device)); 1844 } 1845 1846 1847 uint8 1848 XHCI::_GetEndpointState(xhci_endpoint* endpoint) 1849 { 1850 struct xhci_device_ctx* device_ctx = endpoint->device->device_ctx; 1851 return ENDPOINT_0_STATE_GET( 1852 _ReadContext(&device_ctx->endpoints[endpoint->id].dwendpoint0)); 1853 } 1854 1855 1856 status_t 1857 XHCI::_InsertEndpointForPipe(Pipe *pipe) 1858 { 1859 TRACE("insert endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1860 1861 if (pipe->ControllerCookie() != NULL 1862 || pipe->Parent()->Type() != USB_OBJECT_DEVICE) { 1863 // default pipe is already referenced 1864 return B_OK; 1865 } 1866 1867 Device* usbDevice = (Device *)pipe->Parent(); 1868 if (usbDevice->Parent() == RootObject()) { 1869 // root hub needs no initialization 1870 return B_OK; 1871 } 1872 1873 struct xhci_device *device = (struct xhci_device *) 1874 usbDevice->ControllerCookie(); 1875 if (device == NULL) { 1876 panic("device is NULL\n"); 1877 return B_NO_INIT; 1878 } 1879 1880 const uint8 id = (2 * pipe->EndpointAddress() 1881 + (pipe->Direction() != Pipe::Out ? 1 : 0)) - 1; 1882 if (id >= XHCI_MAX_ENDPOINTS - 1) 1883 return B_BAD_VALUE; 1884 1885 if (id > 0) { 1886 uint32 devicedwslot0 = _ReadContext(&device->device_ctx->slot.dwslot0); 1887 if (SLOT_0_NUM_ENTRIES_GET(devicedwslot0) == 1) { 1888 uint32 inputdwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1889 inputdwslot0 &= ~(SLOT_0_NUM_ENTRIES(0x1f)); 1890 inputdwslot0 |= SLOT_0_NUM_ENTRIES(XHCI_MAX_ENDPOINTS - 1); 1891 _WriteContext(&device->input_ctx->slot.dwslot0, inputdwslot0); 1892 EvaluateContext(device->input_ctx_addr, device->slot); 1893 } 1894 1895 xhci_endpoint* endpoint = &device->endpoints[id]; 1896 mutex_init(&endpoint->lock, "xhci endpoint lock"); 1897 MutexLocker endpointLocker(endpoint->lock); 1898 1899 endpoint->device = device; 1900 endpoint->id = id; 1901 endpoint->td_head = NULL; 1902 endpoint->used = 0; 1903 endpoint->next = 0; 1904 1905 endpoint->trbs = device->trbs + id * XHCI_ENDPOINT_RING_SIZE; 1906 endpoint->trb_addr = device->trb_addr 1907 + id * XHCI_ENDPOINT_RING_SIZE * sizeof(xhci_trb); 1908 memset(endpoint->trbs, 0, 1909 sizeof(xhci_trb) * XHCI_ENDPOINT_RING_SIZE); 1910 1911 TRACE("insert endpoint for pipe: trbs, device %p endpoint %p\n", 1912 device->trbs, endpoint->trbs); 1913 TRACE("insert endpoint for pipe: trb_addr, device 0x%" B_PRIxPHYSADDR 1914 " endpoint 0x%" B_PRIxPHYSADDR "\n", device->trb_addr, 1915 endpoint->trb_addr); 1916 1917 const uint8 endpointNum = id + 1; 1918 1919 status_t status = ConfigureEndpoint(endpoint, device->slot, id, pipe->Type(), 1920 pipe->Direction() == Pipe::In, pipe->Interval(), pipe->MaxPacketSize(), 1921 usbDevice->Speed(), pipe->MaxBurst(), pipe->BytesPerInterval()); 1922 if (status != B_OK) { 1923 TRACE_ERROR("unable to configure endpoint: %s\n", strerror(status)); 1924 return status; 1925 } 1926 1927 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1928 _WriteContext(&device->input_ctx->input.addFlags, 1929 (1 << endpointNum) | (1 << 0)); 1930 1931 ConfigureEndpoint(device->input_ctx_addr, false, device->slot); 1932 1933 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", 1934 device->address, SLOT_3_SLOT_STATE_GET(_ReadContext( 1935 &device->device_ctx->slot.dwslot3))); 1936 TRACE("endpoint[0] state 0x%08" B_PRIx32 "\n", 1937 ENDPOINT_0_STATE_GET(_ReadContext( 1938 &device->device_ctx->endpoints[0].dwendpoint0))); 1939 TRACE("endpoint[%d] state 0x%08" B_PRIx32 "\n", id, 1940 ENDPOINT_0_STATE_GET(_ReadContext( 1941 &device->device_ctx->endpoints[id].dwendpoint0))); 1942 } 1943 pipe->SetControllerCookie(&device->endpoints[id]); 1944 1945 return B_OK; 1946 } 1947 1948 1949 status_t 1950 XHCI::_RemoveEndpointForPipe(Pipe *pipe) 1951 { 1952 TRACE("remove endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1953 1954 if (pipe->Parent()->Type() != USB_OBJECT_DEVICE) 1955 return B_OK; 1956 Device* usbDevice = (Device *)pipe->Parent(); 1957 if (usbDevice->Parent() == RootObject()) 1958 return B_BAD_VALUE; 1959 1960 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 1961 if (endpoint == NULL || endpoint->trbs == NULL) 1962 return B_NO_INIT; 1963 1964 pipe->SetControllerCookie(NULL); 1965 1966 if (endpoint->id > 0) { 1967 xhci_device *device = endpoint->device; 1968 uint8 epNumber = endpoint->id + 1; 1969 StopEndpoint(true, endpoint); 1970 1971 mutex_lock(&endpoint->lock); 1972 1973 // See comment in CancelQueuedTransfers. 1974 xhci_td* td; 1975 while ((td = endpoint->td_head) != NULL) { 1976 endpoint->td_head = endpoint->td_head->next; 1977 FreeDescriptor(td); 1978 } 1979 1980 mutex_destroy(&endpoint->lock); 1981 memset(endpoint, 0, sizeof(xhci_endpoint)); 1982 1983 _WriteContext(&device->input_ctx->input.dropFlags, (1 << epNumber)); 1984 _WriteContext(&device->input_ctx->input.addFlags, (1 << 0)); 1985 1986 // The Deconfigure bit in the Configure Endpoint command indicates 1987 // that *all* endpoints are to be deconfigured, and not just the ones 1988 // specified in the context flags. (XHCI 1.2 § 4.6.6 p115.) 1989 ConfigureEndpoint(device->input_ctx_addr, false, device->slot); 1990 } 1991 1992 return B_OK; 1993 } 1994 1995 1996 status_t 1997 XHCI::_LinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 1998 { 1999 TRACE("link descriptor for pipe\n"); 2000 2001 // Use mutex_trylock first, in case we are in KDL. 2002 MutexLocker endpointLocker(&endpoint->lock, mutex_trylock(&endpoint->lock) == B_OK); 2003 2004 // "used" refers to the number of currently linked TDs, not the number of 2005 // used TRBs on the ring (we use 2 TRBs on the ring per transfer.) 2006 // Furthermore, we have to leave an empty item between the head and tail. 2007 if (endpoint->used >= (XHCI_MAX_TRANSFERS - 1)) { 2008 TRACE_ERROR("link descriptor for pipe: max transfers count exceeded\n"); 2009 return B_BAD_VALUE; 2010 } 2011 2012 // We do not support queuing other transfers in tandem with a fragmented one. 2013 if (endpoint->td_head != NULL && endpoint->td_head->transfer != NULL 2014 && endpoint->td_head->transfer->IsFragmented()) { 2015 TRACE_ERROR("cannot submit transfer: a fragmented transfer is queued\n"); 2016 return B_DEV_RESOURCE_CONFLICT; 2017 } 2018 2019 endpoint->used++; 2020 descriptor->next = endpoint->td_head; 2021 endpoint->td_head = descriptor; 2022 2023 uint32 link = endpoint->next, eventdata = link + 1, next = eventdata + 1; 2024 if (eventdata == XHCI_ENDPOINT_RING_SIZE || next == XHCI_ENDPOINT_RING_SIZE) { 2025 // If it's "next" not "eventdata" that got us here, we will be leaving 2026 // one TRB at the end of the ring unused. 2027 eventdata = 0; 2028 next = 1; 2029 } 2030 2031 TRACE("link descriptor for pipe: link %d, next %d\n", link, next); 2032 2033 // Add a Link TRB to the end of the descriptor. 2034 phys_addr_t addr = endpoint->trb_addr + (eventdata * sizeof(xhci_trb)); 2035 descriptor->trbs[descriptor->trb_used].address = addr; 2036 descriptor->trbs[descriptor->trb_used].status = TRB_2_IRQ(0); 2037 descriptor->trbs[descriptor->trb_used].flags = TRB_3_TYPE(TRB_TYPE_LINK) 2038 | TRB_3_CHAIN_BIT | TRB_3_CYCLE_BIT; 2039 // It is specified that (XHCI 1.2 § 4.12.3 Note 2 p251) if the TRB 2040 // following one with the ENT bit set is a Link TRB, the Link TRB 2041 // shall be evaluated *and* the subsequent TRB shall be. Thus a 2042 // TRB_3_ENT_BIT is unnecessary here; and from testing seems to 2043 // break all transfers on a (very) small number of controllers. 2044 2045 #if !B_HOST_IS_LENDIAN 2046 // Convert endianness. 2047 for (uint32 i = 0; i <= descriptor->trb_used; i++) { 2048 descriptor->trbs[i].address = 2049 B_HOST_TO_LENDIAN_INT64(descriptor->trbs[i].address); 2050 descriptor->trbs[i].status = 2051 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].status); 2052 descriptor->trbs[i].flags = 2053 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].flags); 2054 } 2055 #endif 2056 2057 // Link the descriptor. 2058 endpoint->trbs[link].address = 2059 B_HOST_TO_LENDIAN_INT64(descriptor->trb_addr); 2060 endpoint->trbs[link].status = 2061 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 2062 endpoint->trbs[link].flags = 2063 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_LINK)); 2064 2065 // Set up the Event Data TRB (XHCI 1.2 § 4.11.5.2 p230.) 2066 // 2067 // We do this on the main ring for two reasons: first, to avoid a small 2068 // potential race between the interrupt and the controller evaluating 2069 // the link TRB to get back onto the ring; and second, because many 2070 // controllers throw errors if the target of a Link TRB is not valid 2071 // (i.e. does not have its Cycle Bit set.) 2072 // 2073 // We also set the "address" field, which the controller will copy 2074 // verbatim into the TRB it posts to the event ring, to be the last 2075 // "real" TRB in the TD; this will allow us to determine what transfer 2076 // the resulting Transfer Event TRB refers to. 2077 endpoint->trbs[eventdata].address = 2078 B_HOST_TO_LENDIAN_INT64(descriptor->trb_addr 2079 + (descriptor->trb_used - 1) * sizeof(xhci_trb)); 2080 endpoint->trbs[eventdata].status = 2081 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 2082 endpoint->trbs[eventdata].flags = 2083 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_EVENT_DATA) 2084 | TRB_3_IOC_BIT | TRB_3_CYCLE_BIT); 2085 2086 endpoint->trbs[next].address = 0; 2087 endpoint->trbs[next].status = 0; 2088 endpoint->trbs[next].flags = 0; 2089 2090 memory_write_barrier(); 2091 2092 // Everything is ready, so write the cycle bit. 2093 endpoint->trbs[link].flags |= B_HOST_TO_LENDIAN_INT32(TRB_3_CYCLE_BIT); 2094 2095 TRACE("_LinkDescriptorForPipe pLink %p phys 0x%" B_PRIxPHYSADDR 2096 " 0x%" B_PRIxPHYSADDR " 0x%08" B_PRIx32 "\n", &endpoint->trbs[link], 2097 endpoint->trb_addr + link * sizeof(struct xhci_trb), 2098 endpoint->trbs[link].address, 2099 B_LENDIAN_TO_HOST_INT32(endpoint->trbs[link].flags)); 2100 2101 endpoint->next = next; 2102 endpointLocker.Unlock(); 2103 2104 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 2105 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 2106 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 2107 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 2108 2109 Ring(endpoint->device->slot, endpoint->id + 1); 2110 2111 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 2112 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 2113 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 2114 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 2115 2116 return B_OK; 2117 } 2118 2119 2120 status_t 2121 XHCI::_UnlinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 2122 { 2123 TRACE("unlink descriptor for pipe\n"); 2124 // We presume that the caller has already locked or owns the endpoint. 2125 2126 endpoint->used--; 2127 if (descriptor == endpoint->td_head) { 2128 endpoint->td_head = descriptor->next; 2129 descriptor->next = NULL; 2130 return B_OK; 2131 } else { 2132 for (xhci_td *td = endpoint->td_head; td->next != NULL; td = td->next) { 2133 if (td->next == descriptor) { 2134 td->next = descriptor->next; 2135 descriptor->next = NULL; 2136 return B_OK; 2137 } 2138 } 2139 } 2140 2141 endpoint->used++; 2142 return B_ERROR; 2143 } 2144 2145 2146 status_t 2147 XHCI::ConfigureEndpoint(xhci_endpoint* ep, uint8 slot, uint8 number, uint8 type, 2148 bool directionIn, uint16 interval, uint16 maxPacketSize, usb_speed speed, 2149 uint8 maxBurst, uint16 bytesPerInterval) 2150 { 2151 struct xhci_device* device = &fDevices[slot]; 2152 2153 uint32 dwendpoint0 = 0; 2154 uint32 dwendpoint1 = 0; 2155 uint64 qwendpoint2 = 0; 2156 uint32 dwendpoint4 = 0; 2157 2158 // Compute and assign the endpoint type. (XHCI 1.2 § 6.2.3 Table 6-9 p452.) 2159 uint8 xhciType = 4; 2160 if ((type & USB_OBJECT_INTERRUPT_PIPE) != 0) 2161 xhciType = 3; 2162 if ((type & USB_OBJECT_BULK_PIPE) != 0) 2163 xhciType = 2; 2164 if ((type & USB_OBJECT_ISO_PIPE) != 0) 2165 xhciType = 1; 2166 xhciType |= directionIn ? (1 << 2) : 0; 2167 dwendpoint1 |= ENDPOINT_1_EPTYPE(xhciType); 2168 2169 // Compute and assign interval. (XHCI 1.2 § 6.2.3.6 p456.) 2170 uint16 calcInterval; 2171 if ((type & USB_OBJECT_BULK_PIPE) != 0 2172 || (type & USB_OBJECT_CONTROL_PIPE) != 0) { 2173 // Bulk and Control endpoints never issue NAKs. 2174 calcInterval = 0; 2175 } else { 2176 switch (speed) { 2177 case USB_SPEED_FULLSPEED: 2178 if ((type & USB_OBJECT_ISO_PIPE) != 0) { 2179 // Convert 1-16 into 3-18. 2180 calcInterval = min_c(max_c(interval, 1), 16) + 2; 2181 break; 2182 } 2183 2184 // fall through 2185 case USB_SPEED_LOWSPEED: { 2186 // Convert 1ms-255ms into 3-10. 2187 2188 // Find the index of the highest set bit in "interval". 2189 uint32 temp = min_c(max_c(interval, 1), 255); 2190 for (calcInterval = 0; temp != 1; calcInterval++) 2191 temp = temp >> 1; 2192 calcInterval += 3; 2193 break; 2194 } 2195 2196 case USB_SPEED_HIGHSPEED: 2197 case USB_SPEED_SUPERSPEED: 2198 default: 2199 // Convert 1-16 into 0-15. 2200 calcInterval = min_c(max_c(interval, 1), 16) - 1; 2201 break; 2202 } 2203 } 2204 dwendpoint0 |= ENDPOINT_0_INTERVAL(calcInterval); 2205 2206 // For non-isochronous endpoints, we want the controller to retry failed 2207 // transfers, if possible. (XHCI 1.2 § 4.10.2.3 p197.) 2208 if ((type & USB_OBJECT_ISO_PIPE) == 0) 2209 dwendpoint1 |= ENDPOINT_1_CERR(3); 2210 2211 // Assign maximum burst size. For USB3 devices this is passed in; for 2212 // all other devices we compute it. (XHCI 1.2 § 4.8.2 p161.) 2213 if (speed == USB_SPEED_HIGHSPEED && (type & (USB_OBJECT_INTERRUPT_PIPE 2214 | USB_OBJECT_ISO_PIPE)) != 0) { 2215 maxBurst = (maxPacketSize & 0x1800) >> 11; 2216 } else if (speed != USB_SPEED_SUPERSPEED) { 2217 maxBurst = 0; 2218 } 2219 dwendpoint1 |= ENDPOINT_1_MAXBURST(maxBurst); 2220 2221 // Assign maximum packet size, set the ring address, and set the 2222 // "Dequeue Cycle State" bit. (XHCI 1.2 § 6.2.3 Table 6-10 p453.) 2223 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE(maxPacketSize); 2224 qwendpoint2 |= ENDPOINT_2_DCS_BIT | ep->trb_addr; 2225 2226 // The Max Burst Payload is the number of bytes moved by a 2227 // maximum sized burst. (XHCI 1.2 § 4.11.7.1 p236.) 2228 ep->max_burst_payload = (maxBurst + 1) * maxPacketSize; 2229 if (ep->max_burst_payload == 0) { 2230 TRACE_ERROR("ConfigureEndpoint() failed invalid max_burst_payload\n"); 2231 return B_BAD_VALUE; 2232 } 2233 2234 // Assign average TRB length. 2235 if ((type & USB_OBJECT_CONTROL_PIPE) != 0) { 2236 // Control pipes are a special case, as they rarely have 2237 // outbound transfers of any substantial size. 2238 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(8); 2239 } else if ((type & USB_OBJECT_ISO_PIPE) != 0) { 2240 // Isochronous pipes are another special case: the TRB size will be 2241 // one packet (which is normally smaller than the max packet size, 2242 // but we don't know what it is here.) 2243 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(maxPacketSize); 2244 } else { 2245 // Under all other circumstances, we put max_burst_payload in a TRB. 2246 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(ep->max_burst_payload); 2247 } 2248 2249 // Assign maximum ESIT payload. (XHCI 1.2 § 4.14.2 p259.) 2250 if ((type & (USB_OBJECT_INTERRUPT_PIPE | USB_OBJECT_ISO_PIPE)) != 0) { 2251 // TODO: For SuperSpeedPlus endpoints, there is yet another descriptor 2252 // for isochronous endpoints that specifies the maximum ESIT payload. 2253 // We don't fetch this yet, so just fall back to the USB2 computation 2254 // method if bytesPerInterval is 0. 2255 if (speed == USB_SPEED_SUPERSPEED && bytesPerInterval != 0) 2256 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD(bytesPerInterval); 2257 else if (speed >= USB_SPEED_HIGHSPEED) 2258 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD((maxBurst + 1) * maxPacketSize); 2259 } 2260 2261 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint0, 2262 dwendpoint0); 2263 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint1, 2264 dwendpoint1); 2265 _WriteContext(&device->input_ctx->endpoints[number].qwendpoint2, 2266 qwendpoint2); 2267 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint4, 2268 dwendpoint4); 2269 2270 TRACE("endpoint 0x%" B_PRIx32 " 0x%" B_PRIx32 " 0x%" B_PRIx64 " 0x%" 2271 B_PRIx32 "\n", 2272 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint0), 2273 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint1), 2274 _ReadContext(&device->input_ctx->endpoints[number].qwendpoint2), 2275 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint4)); 2276 2277 return B_OK; 2278 } 2279 2280 2281 status_t 2282 XHCI::GetPortSpeed(uint8 index, usb_speed* speed) 2283 { 2284 if (index >= fPortCount) 2285 return B_BAD_INDEX; 2286 2287 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 2288 2289 switch (PS_SPEED_GET(portStatus)) { 2290 case 2: 2291 *speed = USB_SPEED_LOWSPEED; 2292 break; 2293 case 1: 2294 *speed = USB_SPEED_FULLSPEED; 2295 break; 2296 case 3: 2297 *speed = USB_SPEED_HIGHSPEED; 2298 break; 2299 case 4: 2300 *speed = USB_SPEED_SUPERSPEED; 2301 break; 2302 default: 2303 TRACE_ALWAYS("nonstandard port speed %" B_PRId32 ", assuming SuperSpeed\n", 2304 PS_SPEED_GET(portStatus)); 2305 *speed = USB_SPEED_SUPERSPEED; 2306 break; 2307 } 2308 2309 return B_OK; 2310 } 2311 2312 2313 status_t 2314 XHCI::GetPortStatus(uint8 index, usb_port_status* status) 2315 { 2316 if (index >= fPortCount) 2317 return B_BAD_INDEX; 2318 2319 status->status = status->change = 0; 2320 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 2321 TRACE("port %" B_PRId8 " status=0x%08" B_PRIx32 "\n", index, portStatus); 2322 2323 // build the status 2324 switch (PS_SPEED_GET(portStatus)) { 2325 case 3: 2326 status->status |= PORT_STATUS_HIGH_SPEED; 2327 break; 2328 case 2: 2329 status->status |= PORT_STATUS_LOW_SPEED; 2330 break; 2331 default: 2332 break; 2333 } 2334 2335 if (portStatus & PS_CCS) 2336 status->status |= PORT_STATUS_CONNECTION; 2337 if (portStatus & PS_PED) 2338 status->status |= PORT_STATUS_ENABLE; 2339 if (portStatus & PS_OCA) 2340 status->status |= PORT_STATUS_OVER_CURRENT; 2341 if (portStatus & PS_PR) 2342 status->status |= PORT_STATUS_RESET; 2343 if (portStatus & PS_PP) { 2344 if (fPortSpeeds[index] == USB_SPEED_SUPERSPEED) 2345 status->status |= PORT_STATUS_SS_POWER; 2346 else 2347 status->status |= PORT_STATUS_POWER; 2348 } 2349 if (fPortSpeeds[index] == USB_SPEED_SUPERSPEED) 2350 status->status |= portStatus & PS_PLS_MASK; 2351 2352 // build the change 2353 if (portStatus & PS_CSC) 2354 status->change |= PORT_STATUS_CONNECTION; 2355 if (portStatus & PS_PEC) 2356 status->change |= PORT_STATUS_ENABLE; 2357 if (portStatus & PS_OCC) 2358 status->change |= PORT_STATUS_OVER_CURRENT; 2359 if (portStatus & PS_PRC) 2360 status->change |= PORT_STATUS_RESET; 2361 2362 if (fPortSpeeds[index] == USB_SPEED_SUPERSPEED) { 2363 if (portStatus & PS_PLC) 2364 status->change |= PORT_CHANGE_LINK_STATE; 2365 if (portStatus & PS_WRC) 2366 status->change |= PORT_CHANGE_BH_PORT_RESET; 2367 } 2368 2369 return B_OK; 2370 } 2371 2372 2373 status_t 2374 XHCI::SetPortFeature(uint8 index, uint16 feature) 2375 { 2376 TRACE("set port feature index %u feature %u\n", index, feature); 2377 if (index >= fPortCount) 2378 return B_BAD_INDEX; 2379 2380 uint32 portRegister = XHCI_PORTSC(index); 2381 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2382 2383 switch (feature) { 2384 case PORT_SUSPEND: 2385 if ((portStatus & PS_PED) == 0 || (portStatus & PS_PR) 2386 || (portStatus & PS_PLS_MASK) >= PS_XDEV_U3) { 2387 TRACE_ERROR("USB core suspending device not in U0/U1/U2.\n"); 2388 return B_BAD_VALUE; 2389 } 2390 portStatus &= ~PS_PLS_MASK; 2391 WriteOpReg(portRegister, portStatus | PS_LWS | PS_XDEV_U3); 2392 break; 2393 2394 case PORT_RESET: 2395 WriteOpReg(portRegister, portStatus | PS_PR); 2396 break; 2397 2398 case PORT_POWER: 2399 WriteOpReg(portRegister, portStatus | PS_PP); 2400 break; 2401 default: 2402 return B_BAD_VALUE; 2403 } 2404 ReadOpReg(portRegister); 2405 return B_OK; 2406 } 2407 2408 2409 status_t 2410 XHCI::ClearPortFeature(uint8 index, uint16 feature) 2411 { 2412 TRACE("clear port feature index %u feature %u\n", index, feature); 2413 if (index >= fPortCount) 2414 return B_BAD_INDEX; 2415 2416 uint32 portRegister = XHCI_PORTSC(index); 2417 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2418 2419 switch (feature) { 2420 case PORT_SUSPEND: 2421 portStatus = ReadOpReg(portRegister); 2422 if (portStatus & PS_PR) 2423 return B_BAD_VALUE; 2424 if (portStatus & PS_XDEV_U3) { 2425 if ((portStatus & PS_PED) == 0) 2426 return B_BAD_VALUE; 2427 portStatus &= ~PS_PLS_MASK; 2428 WriteOpReg(portRegister, portStatus | PS_XDEV_U0 | PS_LWS); 2429 } 2430 break; 2431 case PORT_ENABLE: 2432 WriteOpReg(portRegister, portStatus | PS_PED); 2433 break; 2434 case PORT_POWER: 2435 WriteOpReg(portRegister, portStatus & ~PS_PP); 2436 break; 2437 case C_PORT_CONNECTION: 2438 WriteOpReg(portRegister, portStatus | PS_CSC); 2439 break; 2440 case C_PORT_ENABLE: 2441 WriteOpReg(portRegister, portStatus | PS_PEC); 2442 break; 2443 case C_PORT_OVER_CURRENT: 2444 WriteOpReg(portRegister, portStatus | PS_OCC); 2445 break; 2446 case C_PORT_RESET: 2447 WriteOpReg(portRegister, portStatus | PS_PRC); 2448 break; 2449 case C_PORT_BH_PORT_RESET: 2450 WriteOpReg(portRegister, portStatus | PS_WRC); 2451 break; 2452 case C_PORT_LINK_STATE: 2453 WriteOpReg(portRegister, portStatus | PS_PLC); 2454 break; 2455 default: 2456 return B_BAD_VALUE; 2457 } 2458 2459 ReadOpReg(portRegister); 2460 return B_OK; 2461 } 2462 2463 2464 status_t 2465 XHCI::ControllerHalt() 2466 { 2467 // Mask off run state 2468 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) & ~CMD_RUN); 2469 2470 // wait for shutdown state 2471 if (WaitOpBits(XHCI_STS, STS_HCH, STS_HCH) != B_OK) { 2472 TRACE_ERROR("HCH shutdown timeout\n"); 2473 return B_ERROR; 2474 } 2475 return B_OK; 2476 } 2477 2478 2479 status_t 2480 XHCI::ControllerReset() 2481 { 2482 TRACE("ControllerReset() cmd: 0x%" B_PRIx32 " sts: 0x%" B_PRIx32 "\n", 2483 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 2484 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) | CMD_HCRST); 2485 2486 if (WaitOpBits(XHCI_CMD, CMD_HCRST, 0) != B_OK) { 2487 TRACE_ERROR("ControllerReset() failed CMD_HCRST\n"); 2488 return B_ERROR; 2489 } 2490 2491 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 2492 TRACE_ERROR("ControllerReset() failed STS_CNR\n"); 2493 return B_ERROR; 2494 } 2495 2496 return B_OK; 2497 } 2498 2499 2500 int32 2501 XHCI::InterruptHandler(void* data) 2502 { 2503 return ((XHCI*)data)->Interrupt(); 2504 } 2505 2506 2507 int32 2508 XHCI::Interrupt() 2509 { 2510 SpinLocker _(&fSpinlock); 2511 2512 uint32 status = ReadOpReg(XHCI_STS); 2513 uint32 temp = ReadRunReg32(XHCI_IMAN(0)); 2514 WriteOpReg(XHCI_STS, status); 2515 WriteRunReg32(XHCI_IMAN(0), temp); 2516 2517 int32 result = B_HANDLED_INTERRUPT; 2518 2519 if ((status & STS_HCH) != 0) { 2520 TRACE_ERROR("Host Controller halted\n"); 2521 return result; 2522 } 2523 if ((status & STS_HSE) != 0) { 2524 TRACE_ERROR("Host System Error\n"); 2525 return result; 2526 } 2527 if ((status & STS_HCE) != 0) { 2528 TRACE_ERROR("Host Controller Error\n"); 2529 return result; 2530 } 2531 2532 if ((status & STS_EINT) == 0) { 2533 TRACE("STS: 0x%" B_PRIx32 " IRQ_PENDING: 0x%" B_PRIx32 "\n", 2534 status, temp); 2535 return B_UNHANDLED_INTERRUPT; 2536 } 2537 2538 TRACE("Event Interrupt\n"); 2539 release_sem_etc(fEventSem, 1, B_DO_NOT_RESCHEDULE); 2540 return B_INVOKE_SCHEDULER; 2541 } 2542 2543 2544 void 2545 XHCI::Ring(uint8 slot, uint8 endpoint) 2546 { 2547 TRACE("Ding Dong! slot:%d endpoint %d\n", slot, endpoint) 2548 if ((slot == 0 && endpoint > 0) || (slot > 0 && endpoint == 0)) 2549 panic("Ring() invalid slot/endpoint combination\n"); 2550 if (slot > fSlotCount || endpoint >= XHCI_MAX_ENDPOINTS) 2551 panic("Ring() invalid slot or endpoint\n"); 2552 2553 WriteDoorReg32(XHCI_DOORBELL(slot), XHCI_DOORBELL_TARGET(endpoint) 2554 | XHCI_DOORBELL_STREAMID(0)); 2555 ReadDoorReg32(XHCI_DOORBELL(slot)); 2556 // Flush PCI writes 2557 } 2558 2559 2560 void 2561 XHCI::QueueCommand(xhci_trb* trb) 2562 { 2563 uint8 i, j; 2564 uint32 temp; 2565 2566 i = fCmdIdx; 2567 j = fCmdCcs; 2568 2569 TRACE("command[%u] = %" B_PRId32 " (0x%016" B_PRIx64 ", 0x%08" B_PRIx32 2570 ", 0x%08" B_PRIx32 ")\n", i, TRB_3_TYPE_GET(trb->flags), trb->address, 2571 trb->status, trb->flags); 2572 2573 fCmdRing[i].address = trb->address; 2574 fCmdRing[i].status = trb->status; 2575 temp = trb->flags; 2576 2577 if (j) 2578 temp |= TRB_3_CYCLE_BIT; 2579 else 2580 temp &= ~TRB_3_CYCLE_BIT; 2581 temp &= ~TRB_3_TC_BIT; 2582 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2583 2584 fCmdAddr = fErst->rs_addr + (XHCI_MAX_EVENTS + i) * sizeof(xhci_trb); 2585 2586 i++; 2587 2588 if (i == (XHCI_MAX_COMMANDS - 1)) { 2589 temp = TRB_3_TYPE(TRB_TYPE_LINK) | TRB_3_TC_BIT; 2590 if (j) 2591 temp |= TRB_3_CYCLE_BIT; 2592 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2593 2594 i = 0; 2595 j ^= 1; 2596 } 2597 2598 fCmdIdx = i; 2599 fCmdCcs = j; 2600 } 2601 2602 2603 void 2604 XHCI::HandleCmdComplete(xhci_trb* trb) 2605 { 2606 if (fCmdAddr == trb->address) { 2607 TRACE("Received command event\n"); 2608 fCmdResult[0] = trb->status; 2609 fCmdResult[1] = B_LENDIAN_TO_HOST_INT32(trb->flags); 2610 release_sem_etc(fCmdCompSem, 1, B_DO_NOT_RESCHEDULE); 2611 } else 2612 TRACE_ERROR("received command event for unknown command!\n") 2613 } 2614 2615 2616 void 2617 XHCI::HandleTransferComplete(xhci_trb* trb) 2618 { 2619 const uint32 flags = B_LENDIAN_TO_HOST_INT32(trb->flags); 2620 const uint8 endpointNumber = TRB_3_ENDPOINT_GET(flags), 2621 slot = TRB_3_SLOT_GET(flags); 2622 2623 if (slot > fSlotCount) 2624 TRACE_ERROR("invalid slot\n"); 2625 if (endpointNumber == 0 || endpointNumber >= XHCI_MAX_ENDPOINTS) { 2626 TRACE_ERROR("invalid endpoint\n"); 2627 return; 2628 } 2629 2630 xhci_device *device = &fDevices[slot]; 2631 xhci_endpoint *endpoint = &device->endpoints[endpointNumber - 1]; 2632 2633 if (endpoint->trbs == NULL) { 2634 TRACE_ERROR("got TRB but endpoint is not allocated!\n"); 2635 return; 2636 } 2637 2638 // Use mutex_trylock first, in case we are in KDL. 2639 MutexLocker endpointLocker(endpoint->lock, mutex_trylock(&endpoint->lock) == B_OK); 2640 if (!endpointLocker.IsLocked()) { 2641 // We failed to get the lock. Most likely it was destroyed 2642 // while we were waiting for it. 2643 return; 2644 } 2645 2646 TRACE("HandleTransferComplete: ed %" B_PRIu32 ", status %" B_PRId32 "\n", 2647 (flags & TRB_3_EVENT_DATA_BIT), trb->status); 2648 2649 uint8 completionCode = TRB_2_COMP_CODE_GET(trb->status); 2650 2651 if (completionCode == COMP_RING_OVERRUN || completionCode == COMP_RING_UNDERRUN) { 2652 // These occur on isochronous endpoints when there is no TRB ready to be 2653 // executed at the appropriate time. (XHCI 1.2 § 4.10.3.1 p204.) 2654 endpoint->status = completionCode; 2655 return; 2656 } 2657 2658 int32 remainder = TRB_2_REM_GET(trb->status), transferred = -1; 2659 if ((flags & TRB_3_EVENT_DATA_BIT) != 0) { 2660 // In the case of an Event Data TRB, value in the status field refers 2661 // to the actual number of bytes transferred across the whole TD. 2662 // (XHCI 1.2 § 6.4.2.1 Table 6-38 p478.) 2663 transferred = remainder; 2664 remainder = -1; 2665 } else { 2666 // This should only occur under error conditions, or for isochronous transfers. 2667 TRACE("got transfer event for a non-Event Data TRB!\n"); 2668 2669 if (completionCode == COMP_STOPPED_LENGTH_INVALID) 2670 remainder = -1; 2671 } 2672 2673 if (completionCode != COMP_SUCCESS && completionCode != COMP_SHORT_PACKET 2674 && completionCode != COMP_STOPPED && completionCode != COMP_STOPPED_LENGTH_INVALID) { 2675 TRACE_ALWAYS("transfer error on slot %" B_PRId8 " endpoint %" B_PRId8 2676 ": %s\n", slot, endpointNumber, xhci_error_string(completionCode)); 2677 } 2678 2679 phys_addr_t source = B_LENDIAN_TO_HOST_INT64(trb->address); 2680 if (source >= endpoint->trb_addr 2681 && (source - endpoint->trb_addr) < (XHCI_ENDPOINT_RING_SIZE * sizeof(xhci_trb))) { 2682 // The "source" address points to a TRB on the ring. 2683 // See if we can figure out what it really corresponds to. 2684 const int64 offset = (source - endpoint->trb_addr) / sizeof(xhci_trb); 2685 const int32 type = TRB_3_TYPE_GET(endpoint->trbs[offset].flags); 2686 if (type == TRB_TYPE_EVENT_DATA || type == TRB_TYPE_LINK) 2687 source = B_LENDIAN_TO_HOST_INT64(endpoint->trbs[offset].address); 2688 } 2689 2690 for (xhci_td *td = endpoint->td_head; td != NULL; td = td->next) { 2691 int64 offset = (source - td->trb_addr) / sizeof(xhci_trb); 2692 if (offset < 0 || offset >= td->trb_count) 2693 continue; 2694 2695 TRACE("HandleTransferComplete td %p trb %" B_PRId64 " found\n", 2696 td, offset); 2697 2698 if (td->transfer != NULL && td->transfer->IsochronousData() != NULL) { 2699 usb_isochronous_data* isochronousData = td->transfer->IsochronousData(); 2700 usb_iso_packet_descriptor& descriptor = isochronousData->packet_descriptors[offset]; 2701 if (transferred < 0) 2702 transferred = (TRB_2_BYTES_GET(td->trbs[offset].status) - remainder); 2703 descriptor.actual_length = transferred; 2704 descriptor.status = xhci_error_status(completionCode, 2705 (td->transfer->TransferPipe()->Direction() != Pipe::Out)); 2706 2707 // Don't double-report completion status. 2708 completionCode = COMP_SUCCESS; 2709 2710 if (offset != (td->trb_used - 1)) { 2711 // We'll be sent here again. 2712 return; 2713 } 2714 2715 // Compute the real transferred length. 2716 transferred = 0; 2717 for (int32 i = 0; i < offset; i++) { 2718 usb_iso_packet_descriptor& descriptor = isochronousData->packet_descriptors[i]; 2719 if (descriptor.status == B_NO_INIT) { 2720 // Assume success. 2721 descriptor.actual_length = descriptor.request_length; 2722 descriptor.status = B_OK; 2723 } 2724 transferred += descriptor.actual_length; 2725 } 2726 2727 // Report the endpoint status (if any.) 2728 if (endpoint->status != 0) { 2729 completionCode = endpoint->status; 2730 endpoint->status = 0; 2731 } 2732 } else if (completionCode == COMP_STOPPED_LENGTH_INVALID) { 2733 // To determine transferred length, sum up the lengths of all TRBs 2734 // prior to the referenced one. (XHCI 1.2 § 4.6.9 p136.) 2735 transferred = 0; 2736 for (int32 i = 0; i < offset; i++) 2737 transferred += TRB_2_BYTES_GET(td->trbs[i].status); 2738 } 2739 2740 // The TRB at offset trb_used will be the link TRB, which we do not 2741 // care about (and should not generate an interrupt at all.) We really 2742 // care about the properly last TRB, at index "count - 1", which the 2743 // Event Data TRB that _LinkDescriptorForPipe creates points to. 2744 // 2745 // But if we have an unsuccessful completion code, the transfer 2746 // likely failed midway; so just accept it anyway. 2747 if (offset == (td->trb_used - 1) || completionCode != COMP_SUCCESS) { 2748 _UnlinkDescriptorForPipe(td, endpoint); 2749 endpointLocker.Unlock(); 2750 2751 td->trb_completion_code = completionCode; 2752 td->td_transferred = transferred; 2753 td->trb_left = remainder; 2754 2755 // add descriptor to finished list 2756 if (mutex_trylock(&fFinishedLock) != B_OK) 2757 mutex_lock(&fFinishedLock); 2758 td->next = fFinishedHead; 2759 fFinishedHead = td; 2760 mutex_unlock(&fFinishedLock); 2761 2762 release_sem_etc(fFinishTransfersSem, 1, B_DO_NOT_RESCHEDULE); 2763 TRACE("HandleTransferComplete td %p done\n", td); 2764 } else { 2765 TRACE_ERROR("successful TRB 0x%" B_PRIxPHYSADDR " was found, but it wasn't " 2766 "the last in the TD!\n", source); 2767 } 2768 return; 2769 } 2770 TRACE_ERROR("TRB 0x%" B_PRIxPHYSADDR " was not found in the endpoint!\n", source); 2771 } 2772 2773 2774 void 2775 XHCI::DumpRing(xhci_trb *trbs, uint32 size) 2776 { 2777 if (!Lock()) { 2778 TRACE("Unable to get lock!\n"); 2779 return; 2780 } 2781 2782 for (uint32 i = 0; i < size; i++) { 2783 TRACE("command[%" B_PRId32 "] = %" B_PRId32 " (0x%016" B_PRIx64 "," 2784 " 0x%08" B_PRIx32 ", 0x%08" B_PRIx32 ")\n", i, 2785 TRB_3_TYPE_GET(B_LENDIAN_TO_HOST_INT32(trbs[i].flags)), 2786 trbs[i].address, trbs[i].status, trbs[i].flags); 2787 } 2788 2789 Unlock(); 2790 } 2791 2792 2793 status_t 2794 XHCI::DoCommand(xhci_trb* trb) 2795 { 2796 if (!Lock()) { 2797 TRACE("Unable to get lock!\n"); 2798 return B_ERROR; 2799 } 2800 2801 QueueCommand(trb); 2802 Ring(0, 0); 2803 2804 // Begin with a 50ms timeout. 2805 if (acquire_sem_etc(fCmdCompSem, 1, B_RELATIVE_TIMEOUT, 50 * 1000) != B_OK) { 2806 // We've hit the timeout. In some error cases, interrupts are not 2807 // generated; so here we force the event ring to be polled once. 2808 release_sem(fEventSem); 2809 2810 // Now try again, this time with a 750ms timeout. 2811 if (acquire_sem_etc(fCmdCompSem, 1, B_RELATIVE_TIMEOUT, 2812 750 * 1000) != B_OK) { 2813 TRACE("Unable to obtain fCmdCompSem!\n"); 2814 fCmdAddr = 0; 2815 Unlock(); 2816 return B_TIMED_OUT; 2817 } 2818 } 2819 2820 // eat up sems that have been released by multiple interrupts 2821 int32 semCount = 0; 2822 get_sem_count(fCmdCompSem, &semCount); 2823 if (semCount > 0) 2824 acquire_sem_etc(fCmdCompSem, semCount, B_RELATIVE_TIMEOUT, 0); 2825 2826 status_t status = B_OK; 2827 uint32 completionCode = TRB_2_COMP_CODE_GET(fCmdResult[0]); 2828 TRACE("command complete\n"); 2829 if (completionCode != COMP_SUCCESS) { 2830 TRACE_ERROR("unsuccessful command %" B_PRId32 ", error %s (%" B_PRId32 ")\n", 2831 TRB_3_TYPE_GET(trb->flags), xhci_error_string(completionCode), 2832 completionCode); 2833 status = B_IO_ERROR; 2834 } 2835 2836 trb->status = fCmdResult[0]; 2837 trb->flags = fCmdResult[1]; 2838 2839 fCmdAddr = 0; 2840 Unlock(); 2841 return status; 2842 } 2843 2844 2845 status_t 2846 XHCI::Noop() 2847 { 2848 TRACE("Issue No-Op\n"); 2849 xhci_trb trb; 2850 trb.address = 0; 2851 trb.status = 0; 2852 trb.flags = TRB_3_TYPE(TRB_TYPE_CMD_NOOP); 2853 2854 return DoCommand(&trb); 2855 } 2856 2857 2858 status_t 2859 XHCI::EnableSlot(uint8* slot) 2860 { 2861 TRACE("Enable Slot\n"); 2862 xhci_trb trb; 2863 trb.address = 0; 2864 trb.status = 0; 2865 trb.flags = TRB_3_TYPE(TRB_TYPE_ENABLE_SLOT); 2866 2867 status_t status = DoCommand(&trb); 2868 if (status != B_OK) 2869 return status; 2870 2871 *slot = TRB_3_SLOT_GET(trb.flags); 2872 return *slot != 0 ? B_OK : B_BAD_VALUE; 2873 } 2874 2875 2876 status_t 2877 XHCI::DisableSlot(uint8 slot) 2878 { 2879 TRACE("Disable Slot\n"); 2880 xhci_trb trb; 2881 trb.address = 0; 2882 trb.status = 0; 2883 trb.flags = TRB_3_TYPE(TRB_TYPE_DISABLE_SLOT) | TRB_3_SLOT(slot); 2884 2885 return DoCommand(&trb); 2886 } 2887 2888 2889 status_t 2890 XHCI::SetAddress(uint64 inputContext, bool bsr, uint8 slot) 2891 { 2892 TRACE("Set Address\n"); 2893 xhci_trb trb; 2894 trb.address = inputContext; 2895 trb.status = 0; 2896 trb.flags = TRB_3_TYPE(TRB_TYPE_ADDRESS_DEVICE) | TRB_3_SLOT(slot); 2897 2898 if (bsr) 2899 trb.flags |= TRB_3_BSR_BIT; 2900 2901 return DoCommand(&trb); 2902 } 2903 2904 2905 status_t 2906 XHCI::ConfigureEndpoint(uint64 inputContext, bool deconfigure, uint8 slot) 2907 { 2908 TRACE("Configure Endpoint\n"); 2909 xhci_trb trb; 2910 trb.address = inputContext; 2911 trb.status = 0; 2912 trb.flags = TRB_3_TYPE(TRB_TYPE_CONFIGURE_ENDPOINT) | TRB_3_SLOT(slot); 2913 2914 if (deconfigure) 2915 trb.flags |= TRB_3_DCEP_BIT; 2916 2917 return DoCommand(&trb); 2918 } 2919 2920 2921 status_t 2922 XHCI::EvaluateContext(uint64 inputContext, uint8 slot) 2923 { 2924 TRACE("Evaluate Context\n"); 2925 xhci_trb trb; 2926 trb.address = inputContext; 2927 trb.status = 0; 2928 trb.flags = TRB_3_TYPE(TRB_TYPE_EVALUATE_CONTEXT) | TRB_3_SLOT(slot); 2929 2930 return DoCommand(&trb); 2931 } 2932 2933 2934 status_t 2935 XHCI::ResetEndpoint(bool preserve, xhci_endpoint* endpoint) 2936 { 2937 TRACE("Reset Endpoint\n"); 2938 2939 switch (_GetEndpointState(endpoint)) { 2940 case ENDPOINT_STATE_STOPPED: 2941 TRACE("Reset Endpoint: already stopped"); 2942 return B_OK; 2943 case ENDPOINT_STATE_HALTED: 2944 TRACE("Reset Endpoint: warning, weird state!"); 2945 default: 2946 break; 2947 } 2948 2949 xhci_trb trb; 2950 trb.address = 0; 2951 trb.status = 0; 2952 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_ENDPOINT) 2953 | TRB_3_SLOT(endpoint->device->slot) | TRB_3_ENDPOINT(endpoint->id + 1); 2954 if (preserve) 2955 trb.flags |= TRB_3_PRSV_BIT; 2956 2957 return DoCommand(&trb); 2958 } 2959 2960 2961 status_t 2962 XHCI::StopEndpoint(bool suspend, xhci_endpoint* endpoint) 2963 { 2964 TRACE("Stop Endpoint\n"); 2965 2966 switch (_GetEndpointState(endpoint)) { 2967 case ENDPOINT_STATE_HALTED: 2968 TRACE("Stop Endpoint: error, halted"); 2969 return B_DEV_STALLED; 2970 case ENDPOINT_STATE_STOPPED: 2971 TRACE("Stop Endpoint: already stopped"); 2972 return B_OK; 2973 default: 2974 break; 2975 } 2976 2977 xhci_trb trb; 2978 trb.address = 0; 2979 trb.status = 0; 2980 trb.flags = TRB_3_TYPE(TRB_TYPE_STOP_ENDPOINT) 2981 | TRB_3_SLOT(endpoint->device->slot) | TRB_3_ENDPOINT(endpoint->id + 1); 2982 if (suspend) 2983 trb.flags |= TRB_3_SUSPEND_ENDPOINT_BIT; 2984 2985 return DoCommand(&trb); 2986 } 2987 2988 2989 status_t 2990 XHCI::SetTRDequeue(uint64 dequeue, uint16 stream, uint8 endpoint, uint8 slot) 2991 { 2992 TRACE("Set TR Dequeue\n"); 2993 xhci_trb trb; 2994 trb.address = dequeue | ENDPOINT_2_DCS_BIT; 2995 // The DCS bit is copied from the address field as in ConfigureEndpoint. 2996 // (XHCI 1.2 § 4.6.10 p142.) 2997 trb.status = TRB_2_STREAM(stream); 2998 trb.flags = TRB_3_TYPE(TRB_TYPE_SET_TR_DEQUEUE) 2999 | TRB_3_SLOT(slot) | TRB_3_ENDPOINT(endpoint); 3000 3001 return DoCommand(&trb); 3002 } 3003 3004 3005 status_t 3006 XHCI::ResetDevice(uint8 slot) 3007 { 3008 TRACE("Reset Device\n"); 3009 xhci_trb trb; 3010 trb.address = 0; 3011 trb.status = 0; 3012 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_DEVICE) | TRB_3_SLOT(slot); 3013 3014 return DoCommand(&trb); 3015 } 3016 3017 3018 int32 3019 XHCI::EventThread(void* data) 3020 { 3021 ((XHCI *)data)->CompleteEvents(); 3022 return B_OK; 3023 } 3024 3025 3026 void 3027 XHCI::CompleteEvents() 3028 { 3029 while (!fStopThreads) { 3030 if (acquire_sem(fEventSem) < B_OK) 3031 continue; 3032 3033 // eat up sems that have been released by multiple interrupts 3034 int32 semCount = 0; 3035 get_sem_count(fEventSem, &semCount); 3036 if (semCount > 0) 3037 acquire_sem_etc(fEventSem, semCount, B_RELATIVE_TIMEOUT, 0); 3038 3039 ProcessEvents(); 3040 } 3041 } 3042 3043 3044 void 3045 XHCI::ProcessEvents() 3046 { 3047 // Use mutex_trylock first, in case we are in KDL. 3048 MutexLocker locker(fEventLock, mutex_trylock(&fEventLock) == B_OK); 3049 if (!locker.IsLocked()) { 3050 // We failed to get the lock. This really should not happen. 3051 TRACE_ERROR("failed to acquire event lock!\n"); 3052 return; 3053 } 3054 3055 uint16 i = fEventIdx; 3056 uint8 j = fEventCcs; 3057 uint8 t = 2; 3058 3059 while (1) { 3060 uint32 temp = B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags); 3061 uint8 event = TRB_3_TYPE_GET(temp); 3062 TRACE("event[%u] = %u (0x%016" B_PRIx64 " 0x%08" B_PRIx32 " 0x%08" 3063 B_PRIx32 ")\n", i, event, fEventRing[i].address, 3064 fEventRing[i].status, B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags)); 3065 uint8 k = (temp & TRB_3_CYCLE_BIT) ? 1 : 0; 3066 if (j != k) 3067 break; 3068 3069 switch (event) { 3070 case TRB_TYPE_COMMAND_COMPLETION: 3071 HandleCmdComplete(&fEventRing[i]); 3072 break; 3073 case TRB_TYPE_TRANSFER: 3074 HandleTransferComplete(&fEventRing[i]); 3075 break; 3076 case TRB_TYPE_PORT_STATUS_CHANGE: 3077 TRACE("port change detected\n"); 3078 break; 3079 default: 3080 TRACE_ERROR("Unhandled event = %u\n", event); 3081 break; 3082 } 3083 3084 i++; 3085 if (i == XHCI_MAX_EVENTS) { 3086 i = 0; 3087 j ^= 1; 3088 if (!--t) 3089 break; 3090 } 3091 } 3092 3093 fEventIdx = i; 3094 fEventCcs = j; 3095 3096 uint64 addr = fErst->rs_addr + i * sizeof(xhci_trb); 3097 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)addr | ERDP_BUSY); 3098 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(addr >> 32)); 3099 } 3100 3101 3102 int32 3103 XHCI::FinishThread(void* data) 3104 { 3105 ((XHCI *)data)->FinishTransfers(); 3106 return B_OK; 3107 } 3108 3109 3110 void 3111 XHCI::FinishTransfers() 3112 { 3113 while (!fStopThreads) { 3114 if (acquire_sem(fFinishTransfersSem) < B_OK) 3115 continue; 3116 3117 // eat up sems that have been released by multiple interrupts 3118 int32 semCount = 0; 3119 get_sem_count(fFinishTransfersSem, &semCount); 3120 if (semCount > 0) 3121 acquire_sem_etc(fFinishTransfersSem, semCount, B_RELATIVE_TIMEOUT, 0); 3122 3123 mutex_lock(&fFinishedLock); 3124 TRACE("finishing transfers\n"); 3125 while (fFinishedHead != NULL) { 3126 xhci_td* td = fFinishedHead; 3127 fFinishedHead = td->next; 3128 td->next = NULL; 3129 mutex_unlock(&fFinishedLock); 3130 3131 TRACE("finishing transfer td %p\n", td); 3132 3133 Transfer* transfer = td->transfer; 3134 if (transfer == NULL) { 3135 // No transfer? Quick way out. 3136 FreeDescriptor(td); 3137 mutex_lock(&fFinishedLock); 3138 continue; 3139 } 3140 3141 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 3142 3143 const uint8 completionCode = td->trb_completion_code; 3144 status_t callbackStatus = xhci_error_status(completionCode, directionIn); 3145 3146 size_t actualLength = transfer->FragmentLength(); 3147 if (completionCode != COMP_SUCCESS) { 3148 actualLength = td->td_transferred; 3149 if (td->td_transferred == -1) 3150 actualLength = transfer->FragmentLength() - td->trb_left; 3151 TRACE("transfer not successful, actualLength=%" B_PRIuSIZE "\n", 3152 actualLength); 3153 } 3154 3155 if (directionIn && actualLength > 0) { 3156 TRACE("copying in iov count %ld\n", transfer->VectorCount()); 3157 status_t status = transfer->PrepareKernelAccess(); 3158 if (status == B_OK) { 3159 ReadDescriptor(td, transfer->Vector(), 3160 transfer->VectorCount(), transfer->IsPhysical()); 3161 } else { 3162 callbackStatus = status; 3163 } 3164 } 3165 3166 FreeDescriptor(td); 3167 3168 // this transfer may still have data left 3169 bool finished = true; 3170 transfer->AdvanceByFragment(actualLength); 3171 if (completionCode == COMP_SUCCESS 3172 && transfer->FragmentLength() > 0) { 3173 TRACE("still %" B_PRIuSIZE " bytes left on transfer\n", 3174 transfer->FragmentLength()); 3175 callbackStatus = SubmitTransfer(transfer); 3176 finished = (callbackStatus != B_OK); 3177 } 3178 if (finished) { 3179 // The actualLength was already handled in AdvanceByFragment. 3180 transfer->Finished(callbackStatus, 0); 3181 delete transfer; 3182 } 3183 3184 mutex_lock(&fFinishedLock); 3185 } 3186 mutex_unlock(&fFinishedLock); 3187 } 3188 } 3189 3190 3191 inline void 3192 XHCI::WriteOpReg(uint32 reg, uint32 value) 3193 { 3194 *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg) = value; 3195 } 3196 3197 3198 inline uint32 3199 XHCI::ReadOpReg(uint32 reg) 3200 { 3201 return *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg); 3202 } 3203 3204 3205 inline status_t 3206 XHCI::WaitOpBits(uint32 reg, uint32 mask, uint32 expected) 3207 { 3208 int loops = 0; 3209 uint32 value = ReadOpReg(reg); 3210 while ((value & mask) != expected) { 3211 snooze(1000); 3212 value = ReadOpReg(reg); 3213 if (loops == 100) { 3214 TRACE("delay waiting on reg 0x%" B_PRIX32 " match 0x%" B_PRIX32 3215 " (0x%" B_PRIX32 ")\n", reg, expected, mask); 3216 } else if (loops > 250) { 3217 TRACE_ERROR("timeout waiting on reg 0x%" B_PRIX32 3218 " match 0x%" B_PRIX32 " (0x%" B_PRIX32 ")\n", reg, expected, 3219 mask); 3220 return B_ERROR; 3221 } 3222 loops++; 3223 } 3224 return B_OK; 3225 } 3226 3227 3228 inline uint32 3229 XHCI::ReadCapReg32(uint32 reg) 3230 { 3231 return *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg); 3232 } 3233 3234 3235 inline void 3236 XHCI::WriteCapReg32(uint32 reg, uint32 value) 3237 { 3238 *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg) = value; 3239 } 3240 3241 3242 inline uint32 3243 XHCI::ReadRunReg32(uint32 reg) 3244 { 3245 return *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg); 3246 } 3247 3248 3249 inline void 3250 XHCI::WriteRunReg32(uint32 reg, uint32 value) 3251 { 3252 *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg) = value; 3253 } 3254 3255 3256 inline uint32 3257 XHCI::ReadDoorReg32(uint32 reg) 3258 { 3259 return *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg); 3260 } 3261 3262 3263 inline void 3264 XHCI::WriteDoorReg32(uint32 reg, uint32 value) 3265 { 3266 *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg) = value; 3267 } 3268 3269 3270 inline addr_t 3271 XHCI::_OffsetContextAddr(addr_t p) 3272 { 3273 if (fContextSizeShift == 1) { 3274 // each structure is page aligned, each pointer is 32 bits aligned 3275 uint32 offset = p & ((B_PAGE_SIZE - 1) & ~31U); 3276 p += offset; 3277 } 3278 return p; 3279 } 3280 3281 inline uint32 3282 XHCI::_ReadContext(uint32* p) 3283 { 3284 p = (uint32*)_OffsetContextAddr((addr_t)p); 3285 return *p; 3286 } 3287 3288 3289 inline void 3290 XHCI::_WriteContext(uint32* p, uint32 value) 3291 { 3292 p = (uint32*)_OffsetContextAddr((addr_t)p); 3293 *p = value; 3294 } 3295 3296 3297 inline uint64 3298 XHCI::_ReadContext(uint64* p) 3299 { 3300 p = (uint64*)_OffsetContextAddr((addr_t)p); 3301 return *p; 3302 } 3303 3304 3305 inline void 3306 XHCI::_WriteContext(uint64* p, uint64 value) 3307 { 3308 p = (uint64*)_OffsetContextAddr((addr_t)p); 3309 *p = value; 3310 } 3311