1 /* 2 * Copyright 2011-2021, Haiku, Inc. All rights reserved. 3 * Distributed under the terms of the MIT License. 4 * 5 * Authors: 6 * Augustin Cavalier <waddlesplash> 7 * Jian Chiang <j.jian.chiang@gmail.com> 8 * Jérôme Duval <jerome.duval@gmail.com> 9 * Akshay Jaggi <akshay1994.leo@gmail.com> 10 * Michael Lotz <mmlr@mlotz.ch> 11 * Alexander von Gluck <kallisti5@unixzen.com> 12 */ 13 14 15 #include <stdio.h> 16 17 #include <bus/PCI.h> 18 #include <USB3.h> 19 #include <KernelExport.h> 20 21 #include <ByteOrder.h> 22 #include <util/AutoLock.h> 23 24 #include "xhci.h" 25 26 27 #define CALLED(x...) TRACE_MODULE("CALLED %s\n", __PRETTY_FUNCTION__) 28 29 30 #define USB_MODULE_NAME "xhci" 31 32 device_manager_info* gDeviceManager; 33 static usb_for_controller_interface* gUSB; 34 35 36 #define XHCI_PCI_DEVICE_MODULE_NAME "busses/usb/xhci/pci/driver_v1" 37 #define XHCI_PCI_USB_BUS_MODULE_NAME "busses/usb/xhci/device_v1" 38 39 40 typedef struct { 41 XHCI* xhci; 42 pci_device_module_info* pci; 43 pci_device* device; 44 45 pci_info pciinfo; 46 47 device_node* node; 48 device_node* driver_node; 49 } xhci_pci_sim_info; 50 51 52 // #pragma mark - 53 54 55 static status_t 56 init_bus(device_node* node, void** bus_cookie) 57 { 58 CALLED(); 59 60 driver_module_info* driver; 61 xhci_pci_sim_info* bus; 62 device_node* parent = gDeviceManager->get_parent_node(node); 63 gDeviceManager->get_driver(parent, &driver, (void**)&bus); 64 gDeviceManager->put_node(parent); 65 66 Stack *stack; 67 if (gUSB->get_stack((void**)&stack) != B_OK) 68 return B_ERROR; 69 70 XHCI *xhci = new(std::nothrow) XHCI(&bus->pciinfo, bus->pci, bus->device, stack, node); 71 if (xhci == NULL) { 72 return B_NO_MEMORY; 73 } 74 75 if (xhci->InitCheck() < B_OK) { 76 TRACE_MODULE_ERROR("bus failed init check\n"); 77 delete xhci; 78 return B_ERROR; 79 } 80 81 if (xhci->Start() != B_OK) { 82 delete xhci; 83 return B_ERROR; 84 } 85 86 *bus_cookie = xhci; 87 88 return B_OK; 89 } 90 91 92 static void 93 uninit_bus(void* bus_cookie) 94 { 95 CALLED(); 96 XHCI* xhci = (XHCI*)bus_cookie; 97 delete xhci; 98 } 99 100 101 static status_t 102 register_child_devices(void* cookie) 103 { 104 CALLED(); 105 xhci_pci_sim_info* bus = (xhci_pci_sim_info*)cookie; 106 device_node* node = bus->driver_node; 107 108 char prettyName[25]; 109 sprintf(prettyName, "XHCI Controller %" B_PRIu16, 0); 110 111 device_attr attrs[] = { 112 // properties of this controller for the usb bus manager 113 { B_DEVICE_PRETTY_NAME, B_STRING_TYPE, 114 { .string = prettyName }}, 115 { B_DEVICE_FIXED_CHILD, B_STRING_TYPE, 116 { .string = USB_FOR_CONTROLLER_MODULE_NAME }}, 117 118 // private data to identify the device 119 { NULL } 120 }; 121 122 return gDeviceManager->register_node(node, XHCI_PCI_USB_BUS_MODULE_NAME, 123 attrs, NULL, NULL); 124 } 125 126 127 static status_t 128 init_device(device_node* node, void** device_cookie) 129 { 130 CALLED(); 131 xhci_pci_sim_info* bus = (xhci_pci_sim_info*)calloc(1, 132 sizeof(xhci_pci_sim_info)); 133 if (bus == NULL) 134 return B_NO_MEMORY; 135 136 pci_device_module_info* pci; 137 pci_device* device; 138 { 139 device_node* pciParent = gDeviceManager->get_parent_node(node); 140 gDeviceManager->get_driver(pciParent, (driver_module_info**)&pci, 141 (void**)&device); 142 gDeviceManager->put_node(pciParent); 143 } 144 145 bus->pci = pci; 146 bus->device = device; 147 bus->driver_node = node; 148 149 pci_info *pciInfo = &bus->pciinfo; 150 pci->get_pci_info(device, pciInfo); 151 152 *device_cookie = bus; 153 return B_OK; 154 } 155 156 157 static void 158 uninit_device(void* device_cookie) 159 { 160 CALLED(); 161 xhci_pci_sim_info* bus = (xhci_pci_sim_info*)device_cookie; 162 free(bus); 163 } 164 165 166 static status_t 167 register_device(device_node* parent) 168 { 169 CALLED(); 170 device_attr attrs[] = { 171 {B_DEVICE_PRETTY_NAME, B_STRING_TYPE, {.string = "XHCI PCI"}}, 172 {} 173 }; 174 175 return gDeviceManager->register_node(parent, 176 XHCI_PCI_DEVICE_MODULE_NAME, attrs, NULL, NULL); 177 } 178 179 180 static float 181 supports_device(device_node* parent) 182 { 183 CALLED(); 184 const char* bus; 185 uint16 type, subType, api; 186 187 // make sure parent is a XHCI PCI device node 188 if (gDeviceManager->get_attr_string(parent, B_DEVICE_BUS, &bus, false) 189 < B_OK) { 190 return -1; 191 } 192 193 if (strcmp(bus, "pci") != 0) 194 return 0.0f; 195 196 if (gDeviceManager->get_attr_uint16(parent, B_DEVICE_SUB_TYPE, &subType, 197 false) < B_OK 198 || gDeviceManager->get_attr_uint16(parent, B_DEVICE_TYPE, &type, 199 false) < B_OK 200 || gDeviceManager->get_attr_uint16(parent, B_DEVICE_INTERFACE, &api, 201 false) < B_OK) { 202 TRACE_MODULE("Could not find type/subtype/interface attributes\n"); 203 return -1; 204 } 205 206 if (type == PCI_serial_bus && subType == PCI_usb && api == PCI_usb_xhci) { 207 pci_device_module_info* pci; 208 pci_device* device; 209 gDeviceManager->get_driver(parent, (driver_module_info**)&pci, 210 (void**)&device); 211 TRACE_MODULE("XHCI Device found!\n"); 212 213 return 0.8f; 214 } 215 216 return 0.0f; 217 } 218 219 220 static const char* 221 xhci_error_string(uint32 error) 222 { 223 switch (error) { 224 case COMP_INVALID: return "Invalid"; 225 case COMP_SUCCESS: return "Success"; 226 case COMP_DATA_BUFFER: return "Data buffer"; 227 case COMP_BABBLE: return "Babble detected"; 228 case COMP_USB_TRANSACTION: return "USB transaction"; 229 case COMP_TRB: return "TRB"; 230 case COMP_STALL: return "Stall"; 231 case COMP_RESOURCE: return "Resource"; 232 case COMP_BANDWIDTH: return "Bandwidth"; 233 case COMP_NO_SLOTS: return "No slots"; 234 case COMP_INVALID_STREAM: return "Invalid stream"; 235 case COMP_SLOT_NOT_ENABLED: return "Slot not enabled"; 236 case COMP_ENDPOINT_NOT_ENABLED: return "Endpoint not enabled"; 237 case COMP_SHORT_PACKET: return "Short packet"; 238 case COMP_RING_UNDERRUN: return "Ring underrun"; 239 case COMP_RING_OVERRUN: return "Ring overrun"; 240 case COMP_VF_RING_FULL: return "VF Event Ring Full"; 241 case COMP_PARAMETER: return "Parameter"; 242 case COMP_BANDWIDTH_OVERRUN: return "Bandwidth overrun"; 243 case COMP_CONTEXT_STATE: return "Context state"; 244 case COMP_NO_PING_RESPONSE: return "No ping response"; 245 case COMP_EVENT_RING_FULL: return "Event ring full"; 246 case COMP_INCOMPATIBLE_DEVICE: return "Incompatible device"; 247 case COMP_MISSED_SERVICE: return "Missed service"; 248 case COMP_COMMAND_RING_STOPPED: return "Command ring stopped"; 249 case COMP_COMMAND_ABORTED: return "Command aborted"; 250 case COMP_STOPPED: return "Stopped"; 251 case COMP_STOPPED_LENGTH_INVALID: return "Stopped (length invalid)"; 252 case COMP_MAX_EXIT_LATENCY: return "Max exit latency too large"; 253 case COMP_ISOC_OVERRUN: return "Isoch buffer overrun"; 254 case COMP_EVENT_LOST: return "Event lost"; 255 case COMP_UNDEFINED: return "Undefined"; 256 case COMP_INVALID_STREAM_ID: return "Invalid stream ID"; 257 case COMP_SECONDARY_BANDWIDTH: return "Secondary bandwidth"; 258 case COMP_SPLIT_TRANSACTION: return "Split transaction"; 259 260 default: return "Undefined"; 261 } 262 } 263 264 265 static status_t 266 xhci_error_status(uint32 error, bool directionIn) 267 { 268 switch (error) { 269 case COMP_SHORT_PACKET: 270 case COMP_SUCCESS: 271 return B_OK; 272 case COMP_DATA_BUFFER: 273 return directionIn ? B_DEV_WRITE_ERROR : B_DEV_READ_ERROR; 274 case COMP_BABBLE: 275 return directionIn ? B_DEV_DATA_OVERRUN : B_DEV_DATA_UNDERRUN; 276 case COMP_RING_UNDERRUN: 277 return B_DEV_FIFO_UNDERRUN; 278 case COMP_RING_OVERRUN: 279 return B_DEV_FIFO_OVERRUN; 280 case COMP_MISSED_SERVICE: 281 return B_DEV_TOO_LATE; 282 case COMP_USB_TRANSACTION: 283 return B_DEV_CRC_ERROR; 284 case COMP_STALL: 285 return B_DEV_STALLED; 286 default: 287 return B_DEV_STALLED; 288 } 289 } 290 291 292 module_dependency module_dependencies[] = { 293 { USB_FOR_CONTROLLER_MODULE_NAME, (module_info**)&gUSB }, 294 { B_DEVICE_MANAGER_MODULE_NAME, (module_info**)&gDeviceManager }, 295 {} 296 }; 297 298 299 static usb_bus_interface gXHCIPCIDeviceModule = { 300 { 301 { 302 XHCI_PCI_USB_BUS_MODULE_NAME, 303 0, 304 NULL 305 }, 306 NULL, // supports device 307 NULL, // register device 308 init_bus, 309 uninit_bus, 310 NULL, // register child devices 311 NULL, // rescan 312 NULL, // device removed 313 }, 314 }; 315 316 // Root device that binds to the PCI bus. It will register an usb_bus_interface 317 // node for each device. 318 static driver_module_info sXHCIDevice = { 319 { 320 XHCI_PCI_DEVICE_MODULE_NAME, 321 0, 322 NULL 323 }, 324 supports_device, 325 register_device, 326 init_device, 327 uninit_device, 328 register_child_devices, 329 NULL, // rescan 330 NULL, // device removed 331 }; 332 333 module_info* modules[] = { 334 (module_info* )&sXHCIDevice, 335 (module_info* )&gXHCIPCIDeviceModule, 336 NULL 337 }; 338 339 340 XHCI::XHCI(pci_info *info, pci_device_module_info* pci, pci_device* device, Stack *stack, 341 device_node* node) 342 : BusManager(stack, node), 343 fRegisterArea(-1), 344 fRegisters(NULL), 345 fPCIInfo(info), 346 fPci(pci), 347 fDevice(device), 348 fStack(stack), 349 fIRQ(0), 350 fUseMSI(false), 351 fErstArea(-1), 352 fDcbaArea(-1), 353 fCmdCompSem(-1), 354 fStopThreads(false), 355 fRootHub(NULL), 356 fPortCount(0), 357 fSlotCount(0), 358 fScratchpadCount(0), 359 fContextSizeShift(0), 360 fFinishedHead(NULL), 361 fFinishTransfersSem(-1), 362 fFinishThread(-1), 363 fEventSem(-1), 364 fEventThread(-1), 365 fEventIdx(0), 366 fCmdIdx(0), 367 fEventCcs(1), 368 fCmdCcs(1) 369 { 370 B_INITIALIZE_SPINLOCK(&fSpinlock); 371 mutex_init(&fFinishedLock, "XHCI finished transfers"); 372 mutex_init(&fEventLock, "XHCI event handler"); 373 374 if (BusManager::InitCheck() < B_OK) { 375 TRACE_ERROR("bus manager failed to init\n"); 376 return; 377 } 378 379 TRACE("constructing new XHCI host controller driver\n"); 380 fInitOK = false; 381 382 // enable busmaster and memory mapped access 383 uint16 command = fPci->read_pci_config(fDevice, PCI_command, 2); 384 command &= ~(PCI_command_io | PCI_command_int_disable); 385 command |= PCI_command_master | PCI_command_memory; 386 387 fPci->write_pci_config(fDevice, PCI_command, 2, command); 388 389 // map the registers (low + high for 64-bit when requested) 390 phys_addr_t physicalAddress = fPCIInfo->u.h0.base_registers[0]; 391 if ((fPCIInfo->u.h0.base_register_flags[0] & PCI_address_type) 392 == PCI_address_type_64) { 393 physicalAddress |= (uint64)fPCIInfo->u.h0.base_registers[1] << 32; 394 } 395 396 size_t mapSize = fPCIInfo->u.h0.base_register_sizes[0]; 397 398 TRACE("map registers %08" B_PRIxPHYSADDR ", size: %" B_PRIuSIZE "\n", 399 physicalAddress, mapSize); 400 401 fRegisterArea = map_physical_memory("XHCI memory mapped registers", 402 physicalAddress, mapSize, B_ANY_KERNEL_BLOCK_ADDRESS, 403 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 404 (void **)&fRegisters); 405 if (fRegisterArea < B_OK) { 406 TRACE_ERROR("failed to map register memory\n"); 407 return; 408 } 409 410 // determine the register offsets 411 fCapabilityRegisterOffset = 0; 412 fOperationalRegisterOffset = HCI_CAPLENGTH(ReadCapReg32(XHCI_HCI_CAPLENGTH)); 413 fRuntimeRegisterOffset = ReadCapReg32(XHCI_RTSOFF) & ~0x1F; 414 fDoorbellRegisterOffset = ReadCapReg32(XHCI_DBOFF) & ~0x3; 415 416 TRACE("mapped registers: %p\n", fRegisters); 417 TRACE("operational register offset: %" B_PRId32 "\n", fOperationalRegisterOffset); 418 TRACE("runtime register offset: %" B_PRId32 "\n", fRuntimeRegisterOffset); 419 TRACE("doorbell register offset: %" B_PRId32 "\n", fDoorbellRegisterOffset); 420 421 int32 interfaceVersion = HCI_VERSION(ReadCapReg32(XHCI_HCI_VERSION)); 422 if (interfaceVersion < 0x0090 || interfaceVersion > 0x0120) { 423 TRACE_ERROR("unsupported interface version: 0x%04" B_PRIx32 "\n", 424 interfaceVersion); 425 return; 426 } 427 TRACE_ALWAYS("interface version: 0x%04" B_PRIx32 "\n", interfaceVersion); 428 429 TRACE_ALWAYS("structural parameters: 1:0x%08" B_PRIx32 " 2:0x%08" 430 B_PRIx32 " 3:0x%08" B_PRIx32 "\n", ReadCapReg32(XHCI_HCSPARAMS1), 431 ReadCapReg32(XHCI_HCSPARAMS2), ReadCapReg32(XHCI_HCSPARAMS3)); 432 433 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 434 if (cparams == 0xffffffff) 435 return; 436 TRACE_ALWAYS("capability parameters: 0x%08" B_PRIx32 "\n", cparams); 437 438 // if 64 bytes context structures, then 1 439 fContextSizeShift = HCC_CSZ(cparams); 440 441 // Assume ownership of the controller from the BIOS. 442 uint32 eec = 0xffffffff; 443 uint32 eecp = HCS0_XECP(cparams) << 2; 444 for (; eecp != 0 && XECP_NEXT(eec); eecp += XECP_NEXT(eec) << 2) { 445 TRACE("eecp register: 0x%08" B_PRIx32 "\n", eecp); 446 447 eec = ReadCapReg32(eecp); 448 if (XECP_ID(eec) != XHCI_LEGSUP_CAPID) 449 continue; 450 451 if (eec & XHCI_LEGSUP_BIOSOWNED) { 452 TRACE_ALWAYS("the host controller is bios owned, claiming" 453 " ownership\n"); 454 WriteCapReg32(eecp, eec | XHCI_LEGSUP_OSOWNED); 455 456 for (int32 i = 0; i < 20; i++) { 457 eec = ReadCapReg32(eecp); 458 459 if ((eec & XHCI_LEGSUP_BIOSOWNED) == 0) 460 break; 461 462 TRACE_ALWAYS("controller is still bios owned, waiting\n"); 463 snooze(50000); 464 } 465 466 if (eec & XHCI_LEGSUP_BIOSOWNED) { 467 TRACE_ERROR("bios won't give up control over the host " 468 "controller (ignoring)\n"); 469 } else if (eec & XHCI_LEGSUP_OSOWNED) { 470 TRACE_ALWAYS("successfully took ownership of the host " 471 "controller\n"); 472 } 473 474 // Force off the BIOS owned flag, and clear all SMIs. Some BIOSes 475 // do indicate a successful handover but do not remove their SMIs 476 // and then freeze the system when interrupts are generated. 477 WriteCapReg32(eecp, eec & ~XHCI_LEGSUP_BIOSOWNED); 478 } 479 break; 480 } 481 uint32 legctlsts = ReadCapReg32(eecp + XHCI_LEGCTLSTS); 482 legctlsts &= XHCI_LEGCTLSTS_DISABLE_SMI; 483 legctlsts |= XHCI_LEGCTLSTS_EVENTS_SMI; 484 WriteCapReg32(eecp + XHCI_LEGCTLSTS, legctlsts); 485 486 // We need to explicitly take ownership of EHCI ports on earlier Intel chipsets. 487 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL) { 488 switch (fPCIInfo->device_id) { 489 case PCI_DEVICE_INTEL_PANTHER_POINT_XHCI: 490 case PCI_DEVICE_INTEL_LYNX_POINT_XHCI: 491 case PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI: 492 case PCI_DEVICE_INTEL_BAYTRAIL_XHCI: 493 case PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI: 494 case PCI_DEVICE_INTEL_WILDCAT_POINT_LP_XHCI: 495 _SwitchIntelPorts(); 496 break; 497 } 498 } 499 500 // halt the host controller 501 if (ControllerHalt() < B_OK) { 502 return; 503 } 504 505 // reset the host controller 506 if (ControllerReset() < B_OK) { 507 TRACE_ERROR("host controller failed to reset\n"); 508 return; 509 } 510 511 fCmdCompSem = create_sem(0, "XHCI Command Complete"); 512 fFinishTransfersSem = create_sem(0, "XHCI Finish Transfers"); 513 fEventSem = create_sem(0, "XHCI Event"); 514 if (fFinishTransfersSem < B_OK || fCmdCompSem < B_OK || fEventSem < B_OK) { 515 TRACE_ERROR("failed to create semaphores\n"); 516 return; 517 } 518 519 // create event handler thread 520 fEventThread = spawn_kernel_thread(EventThread, "xhci event thread", 521 B_URGENT_PRIORITY, (void *)this); 522 resume_thread(fEventThread); 523 524 // create finisher service thread 525 fFinishThread = spawn_kernel_thread(FinishThread, "xhci finish thread", 526 B_URGENT_PRIORITY - 1, (void *)this); 527 resume_thread(fFinishThread); 528 529 // Find the right interrupt vector, using MSIs if available. 530 fIRQ = fPCIInfo->u.h0.interrupt_line; 531 if (fIRQ == 0xFF) 532 fIRQ = 0; 533 534 #if 0 535 if (fPci->get_msix_count(fDevice) >= 1) { 536 uint8 msiVector = 0; 537 if (fPci->configure_msix(fDevice, 1, &msiVector) == B_OK 538 && fPci->enable_msix(fDevice) == B_OK) { 539 TRACE_ALWAYS("using MSI-X\n"); 540 fIRQ = msiVector; 541 fUseMSI = true; 542 } 543 } else 544 #endif 545 if (fPci->get_msi_count(fDevice) >= 1) { 546 uint32 msiVector = 0; 547 if (fPci->configure_msi(fDevice, 1, &msiVector) == B_OK 548 && fPci->enable_msi(fDevice) == B_OK) { 549 TRACE_ALWAYS("using message signaled interrupts\n"); 550 fIRQ = msiVector; 551 fUseMSI = true; 552 } 553 } 554 555 if (fIRQ == 0) { 556 TRACE_MODULE_ERROR("device PCI:%d:%d:%d was assigned an invalid IRQ\n", 557 fPCIInfo->bus, fPCIInfo->device, fPCIInfo->function); 558 return; 559 } 560 561 // Install the interrupt handler 562 TRACE("installing interrupt handler\n"); 563 install_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this, 0); 564 565 memset(fPortSpeeds, 0, sizeof(fPortSpeeds)); 566 memset(fDevices, 0, sizeof(fDevices)); 567 568 fInitOK = true; 569 TRACE("driver construction successful\n"); 570 } 571 572 573 XHCI::~XHCI() 574 { 575 TRACE("tear down XHCI host controller driver\n"); 576 577 WriteOpReg(XHCI_CMD, 0); 578 579 int32 result = 0; 580 fStopThreads = true; 581 delete_sem(fCmdCompSem); 582 delete_sem(fFinishTransfersSem); 583 delete_sem(fEventSem); 584 wait_for_thread(fFinishThread, &result); 585 wait_for_thread(fEventThread, &result); 586 587 mutex_destroy(&fFinishedLock); 588 mutex_destroy(&fEventLock); 589 590 remove_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this); 591 592 delete_area(fRegisterArea); 593 delete_area(fErstArea); 594 for (uint32 i = 0; i < fScratchpadCount; i++) 595 delete_area(fScratchpadArea[i]); 596 delete_area(fDcbaArea); 597 598 if (fUseMSI) { 599 fPci->disable_msi(fDevice); 600 fPci->unconfigure_msi(fDevice); 601 } 602 } 603 604 605 void 606 XHCI::_SwitchIntelPorts() 607 { 608 TRACE("Looking for EHCI owned ports\n"); 609 uint32 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_USB3PRM, 4); 610 TRACE("Superspeed Ports: 0x%" B_PRIx32 "\n", ports); 611 fPci->write_pci_config(fDevice, XHCI_INTEL_USB3_PSSEN, 4, ports); 612 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_USB3_PSSEN, 4); 613 TRACE("Superspeed ports now under XHCI : 0x%" B_PRIx32 "\n", ports); 614 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_USB2PRM, 4); 615 TRACE("USB 2.0 Ports : 0x%" B_PRIx32 "\n", ports); 616 fPci->write_pci_config(fDevice, XHCI_INTEL_XUSB2PR, 4, ports); 617 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_XUSB2PR, 4); 618 TRACE("USB 2.0 ports now under XHCI: 0x%" B_PRIx32 "\n", ports); 619 } 620 621 622 status_t 623 XHCI::Start() 624 { 625 TRACE_ALWAYS("starting XHCI host controller\n"); 626 TRACE("usbcmd: 0x%08" B_PRIx32 "; usbsts: 0x%08" B_PRIx32 "\n", 627 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 628 629 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 630 TRACE("Start() failed STS_CNR\n"); 631 } 632 633 if ((ReadOpReg(XHCI_CMD) & CMD_RUN) != 0) { 634 TRACE_ERROR("Start() warning, starting running XHCI controller!\n"); 635 } 636 637 if ((ReadOpReg(XHCI_PAGESIZE) & (1 << 0)) == 0) { 638 TRACE_ERROR("controller does not support 4K page size\n"); 639 return B_ERROR; 640 } 641 642 // read port count from capability register 643 uint32 capabilities = ReadCapReg32(XHCI_HCSPARAMS1); 644 fPortCount = HCS_MAX_PORTS(capabilities); 645 if (fPortCount == 0) { 646 TRACE_ERROR("invalid number of ports: %u\n", fPortCount); 647 return B_ERROR; 648 } 649 650 fSlotCount = HCS_MAX_SLOTS(capabilities); 651 if (fSlotCount > XHCI_MAX_DEVICES) 652 fSlotCount = XHCI_MAX_DEVICES; 653 WriteOpReg(XHCI_CONFIG, fSlotCount); 654 655 // find out which protocol is used for each port 656 uint8 portFound = 0; 657 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 658 uint32 eec = 0xffffffff; 659 uint32 eecp = HCS0_XECP(cparams) << 2; 660 for (; eecp != 0 && XECP_NEXT(eec) && portFound < fPortCount; 661 eecp += XECP_NEXT(eec) << 2) { 662 eec = ReadCapReg32(eecp); 663 if (XECP_ID(eec) != XHCI_SUPPORTED_PROTOCOLS_CAPID) 664 continue; 665 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) > 3) 666 continue; 667 uint32 temp = ReadCapReg32(eecp + 8); 668 uint32 offset = XHCI_SUPPORTED_PROTOCOLS_1_OFFSET(temp); 669 uint32 count = XHCI_SUPPORTED_PROTOCOLS_1_COUNT(temp); 670 if (offset == 0 || count == 0) 671 continue; 672 offset--; 673 for (uint32 i = offset; i < offset + count; i++) { 674 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) == 0x3) 675 fPortSpeeds[i] = USB_SPEED_SUPERSPEED; 676 else 677 fPortSpeeds[i] = USB_SPEED_HIGHSPEED; 678 679 TRACE("speed for port %" B_PRId32 " is %s\n", i, 680 fPortSpeeds[i] == USB_SPEED_SUPERSPEED ? "super" : "high"); 681 } 682 portFound += count; 683 } 684 685 uint32 params2 = ReadCapReg32(XHCI_HCSPARAMS2); 686 fScratchpadCount = HCS_MAX_SC_BUFFERS(params2); 687 if (fScratchpadCount > XHCI_MAX_SCRATCHPADS) { 688 TRACE_ERROR("invalid number of scratchpads: %" B_PRIu32 "\n", 689 fScratchpadCount); 690 return B_ERROR; 691 } 692 693 uint32 params3 = ReadCapReg32(XHCI_HCSPARAMS3); 694 fExitLatMax = HCS_U1_DEVICE_LATENCY(params3) 695 + HCS_U2_DEVICE_LATENCY(params3); 696 697 // clear interrupts & disable device notifications 698 WriteOpReg(XHCI_STS, ReadOpReg(XHCI_STS)); 699 WriteOpReg(XHCI_DNCTRL, 0); 700 701 // allocate Device Context Base Address array 702 phys_addr_t dmaAddress; 703 fDcbaArea = fStack->AllocateArea((void **)&fDcba, &dmaAddress, 704 sizeof(*fDcba), "DCBA Area"); 705 if (fDcbaArea < B_OK) { 706 TRACE_ERROR("unable to create the DCBA area\n"); 707 return B_ERROR; 708 } 709 memset(fDcba, 0, sizeof(*fDcba)); 710 memset(fScratchpadArea, 0, sizeof(fScratchpadArea)); 711 memset(fScratchpad, 0, sizeof(fScratchpad)); 712 713 // setting the first address to the scratchpad array address 714 fDcba->baseAddress[0] = dmaAddress 715 + offsetof(struct xhci_device_context_array, scratchpad); 716 717 // fill up the scratchpad array with scratchpad pages 718 for (uint32 i = 0; i < fScratchpadCount; i++) { 719 phys_addr_t scratchDmaAddress; 720 fScratchpadArea[i] = fStack->AllocateArea((void **)&fScratchpad[i], 721 &scratchDmaAddress, B_PAGE_SIZE, "Scratchpad Area"); 722 if (fScratchpadArea[i] < B_OK) { 723 TRACE_ERROR("unable to create the scratchpad area\n"); 724 return B_ERROR; 725 } 726 fDcba->scratchpad[i] = scratchDmaAddress; 727 } 728 729 TRACE("setting DCBAAP %" B_PRIxPHYSADDR "\n", dmaAddress); 730 WriteOpReg(XHCI_DCBAAP_LO, (uint32)dmaAddress); 731 WriteOpReg(XHCI_DCBAAP_HI, (uint32)(dmaAddress >> 32)); 732 733 // allocate Event Ring Segment Table 734 uint8 *addr; 735 fErstArea = fStack->AllocateArea((void **)&addr, &dmaAddress, 736 (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 737 + sizeof(xhci_erst_element), 738 "USB XHCI ERST CMD_RING and EVENT_RING Area"); 739 740 if (fErstArea < B_OK) { 741 TRACE_ERROR("unable to create the ERST AND RING area\n"); 742 delete_area(fDcbaArea); 743 return B_ERROR; 744 } 745 fErst = (xhci_erst_element *)addr; 746 memset(fErst, 0, (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 747 + sizeof(xhci_erst_element)); 748 749 // fill with Event Ring Segment Base Address and Event Ring Segment Size 750 fErst->rs_addr = dmaAddress + sizeof(xhci_erst_element); 751 fErst->rs_size = XHCI_MAX_EVENTS; 752 fErst->rsvdz = 0; 753 754 addr += sizeof(xhci_erst_element); 755 fEventRing = (xhci_trb *)addr; 756 addr += XHCI_MAX_EVENTS * sizeof(xhci_trb); 757 fCmdRing = (xhci_trb *)addr; 758 759 TRACE("setting ERST size\n"); 760 WriteRunReg32(XHCI_ERSTSZ(0), XHCI_ERSTS_SET(1)); 761 762 TRACE("setting ERDP addr = 0x%" B_PRIx64 "\n", fErst->rs_addr); 763 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)fErst->rs_addr); 764 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(fErst->rs_addr >> 32)); 765 766 TRACE("setting ERST base addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 767 WriteRunReg32(XHCI_ERSTBA_LO(0), (uint32)dmaAddress); 768 WriteRunReg32(XHCI_ERSTBA_HI(0), (uint32)(dmaAddress >> 32)); 769 770 dmaAddress += sizeof(xhci_erst_element) + XHCI_MAX_EVENTS 771 * sizeof(xhci_trb); 772 773 // Make sure the Command Ring is stopped 774 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 775 TRACE_ALWAYS("Command Ring is running, send stop/cancel\n"); 776 WriteOpReg(XHCI_CRCR_LO, CRCR_CS); 777 WriteOpReg(XHCI_CRCR_HI, 0); 778 WriteOpReg(XHCI_CRCR_LO, CRCR_CA); 779 WriteOpReg(XHCI_CRCR_HI, 0); 780 snooze(1000); 781 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 782 TRACE_ERROR("Command Ring still running after stop/cancel\n"); 783 } 784 } 785 TRACE("setting CRCR addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 786 WriteOpReg(XHCI_CRCR_LO, (uint32)dmaAddress | CRCR_RCS); 787 WriteOpReg(XHCI_CRCR_HI, (uint32)(dmaAddress >> 32)); 788 // link trb 789 fCmdRing[XHCI_MAX_COMMANDS - 1].address = dmaAddress; 790 791 TRACE("setting interrupt rate\n"); 792 793 // Setting IMOD below 0x3F8 on Intel Lynx Point can cause IRQ lockups 794 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL 795 && (fPCIInfo->device_id == PCI_DEVICE_INTEL_PANTHER_POINT_XHCI 796 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_XHCI 797 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI 798 || fPCIInfo->device_id == PCI_DEVICE_INTEL_BAYTRAIL_XHCI 799 || fPCIInfo->device_id == PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI)) { 800 WriteRunReg32(XHCI_IMOD(0), 0x000003f8); // 4000 irq/s 801 } else { 802 WriteRunReg32(XHCI_IMOD(0), 0x000001f4); // 8000 irq/s 803 } 804 805 TRACE("enabling interrupt\n"); 806 WriteRunReg32(XHCI_IMAN(0), ReadRunReg32(XHCI_IMAN(0)) | IMAN_INTR_ENA); 807 808 WriteOpReg(XHCI_CMD, CMD_RUN | CMD_INTE | CMD_HSEE); 809 810 // wait for start up state 811 if (WaitOpBits(XHCI_STS, STS_HCH, 0) != B_OK) { 812 TRACE_ERROR("HCH start up timeout\n"); 813 } 814 815 fRootHub = new(std::nothrow) XHCIRootHub(RootObject(), 1); 816 if (!fRootHub) { 817 TRACE_ERROR("no memory to allocate root hub\n"); 818 return B_NO_MEMORY; 819 } 820 821 if (fRootHub->InitCheck() < B_OK) { 822 TRACE_ERROR("root hub failed init check\n"); 823 return fRootHub->InitCheck(); 824 } 825 826 SetRootHub(fRootHub); 827 828 fRootHub->RegisterNode(Node()); 829 830 TRACE_ALWAYS("successfully started the controller\n"); 831 832 #ifdef TRACE_USB 833 TRACE("No-Op test...\n"); 834 Noop(); 835 #endif 836 837 return BusManager::Start(); 838 } 839 840 841 status_t 842 XHCI::SubmitTransfer(Transfer *transfer) 843 { 844 // short circuit the root hub 845 if (transfer->TransferPipe()->DeviceAddress() == 1) 846 return fRootHub->ProcessTransfer(this, transfer); 847 848 TRACE("SubmitTransfer(%p)\n", transfer); 849 Pipe *pipe = transfer->TransferPipe(); 850 if ((pipe->Type() & USB_OBJECT_CONTROL_PIPE) != 0) 851 return SubmitControlRequest(transfer); 852 return SubmitNormalRequest(transfer); 853 } 854 855 856 status_t 857 XHCI::SubmitControlRequest(Transfer *transfer) 858 { 859 Pipe *pipe = transfer->TransferPipe(); 860 usb_request_data *requestData = transfer->RequestData(); 861 bool directionIn = (requestData->RequestType & USB_REQTYPE_DEVICE_IN) != 0; 862 863 TRACE("SubmitControlRequest() length %d\n", requestData->Length); 864 865 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 866 if (endpoint == NULL) { 867 TRACE_ERROR("control pipe has no endpoint!\n"); 868 return B_BAD_VALUE; 869 } 870 if (endpoint->device == NULL) { 871 panic("endpoint is not initialized!"); 872 return B_NO_INIT; 873 } 874 875 status_t status = transfer->InitKernelAccess(); 876 if (status != B_OK) 877 return status; 878 879 xhci_td *descriptor = CreateDescriptor(3, 1, requestData->Length); 880 if (descriptor == NULL) 881 return B_NO_MEMORY; 882 descriptor->transfer = transfer; 883 884 // Setup Stage 885 uint8 index = 0; 886 memcpy(&descriptor->trbs[index].address, requestData, 887 sizeof(usb_request_data)); 888 descriptor->trbs[index].status = TRB_2_IRQ(0) | TRB_2_BYTES(8); 889 descriptor->trbs[index].flags 890 = TRB_3_TYPE(TRB_TYPE_SETUP_STAGE) | TRB_3_IDT_BIT | TRB_3_CYCLE_BIT; 891 if (requestData->Length > 0) { 892 descriptor->trbs[index].flags |= 893 directionIn ? TRB_3_TRT_IN : TRB_3_TRT_OUT; 894 } 895 896 index++; 897 898 // Data Stage (if any) 899 if (requestData->Length > 0) { 900 descriptor->trbs[index].address = descriptor->buffer_addrs[0]; 901 descriptor->trbs[index].status = TRB_2_IRQ(0) 902 | TRB_2_BYTES(requestData->Length) 903 | TRB_2_TD_SIZE(0); 904 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_DATA_STAGE) 905 | (directionIn ? TRB_3_DIR_IN : 0) 906 | TRB_3_CYCLE_BIT; 907 908 if (!directionIn) { 909 transfer->PrepareKernelAccess(); 910 WriteDescriptor(descriptor, transfer->Vector(), 911 transfer->VectorCount(), transfer->IsPhysical()); 912 } 913 914 index++; 915 } 916 917 // Status Stage 918 descriptor->trbs[index].address = 0; 919 descriptor->trbs[index].status = TRB_2_IRQ(0); 920 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_STATUS_STAGE) 921 | TRB_3_CHAIN_BIT | TRB_3_ENT_BIT | TRB_3_CYCLE_BIT; 922 // The CHAIN bit must be set when using an Event Data TRB 923 // (XHCI 1.2 § 6.4.1.2.3 Table 6-31 p472). 924 925 // Status Stage is an OUT transfer when the device is sending data 926 // (XHCI 1.2 § 4.11.2.2 Table 4-7 p213), otherwise set the IN bit. 927 if (requestData->Length == 0 || !directionIn) 928 descriptor->trbs[index].flags |= TRB_3_DIR_IN; 929 930 descriptor->trb_used = index + 1; 931 932 status = _LinkDescriptorForPipe(descriptor, endpoint); 933 if (status != B_OK) { 934 FreeDescriptor(descriptor); 935 return status; 936 } 937 938 return B_OK; 939 } 940 941 942 status_t 943 XHCI::SubmitNormalRequest(Transfer *transfer) 944 { 945 TRACE("SubmitNormalRequest() length %" B_PRIuSIZE "\n", transfer->FragmentLength()); 946 947 Pipe *pipe = transfer->TransferPipe(); 948 usb_isochronous_data *isochronousData = transfer->IsochronousData(); 949 bool directionIn = (pipe->Direction() == Pipe::In); 950 951 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 952 if (endpoint == NULL) { 953 TRACE_ERROR("pipe has no endpoint!\n"); 954 return B_BAD_VALUE; 955 } 956 if (endpoint->device == NULL) { 957 panic("endpoint is not initialized!"); 958 return B_NO_INIT; 959 } 960 961 status_t status = transfer->InitKernelAccess(); 962 if (status != B_OK) 963 return status; 964 965 // TRBs within a TD must be "grouped" into TD Fragments, which mostly means 966 // that a max_burst_payload boundary cannot be crossed within a TRB, but 967 // only between TRBs. More than one TRB can be in a TD Fragment, but we keep 968 // things simple by setting trbSize to the MBP. (XHCI 1.2 § 4.11.7.1 p235.) 969 size_t trbSize = endpoint->max_burst_payload; 970 971 if (isochronousData != NULL) { 972 if (isochronousData->packet_count == 0) 973 return B_BAD_VALUE; 974 975 // Isochronous transfers use more specifically sized packets. 976 trbSize = transfer->DataLength() / isochronousData->packet_count; 977 if (trbSize == 0 || trbSize > pipe->MaxPacketSize() || trbSize 978 != (size_t)isochronousData->packet_descriptors[0].request_length) 979 return B_BAD_VALUE; 980 } 981 982 // Now that we know trbSize, compute the count. 983 const int32 trbCount = (transfer->FragmentLength() + trbSize - 1) / trbSize; 984 985 xhci_td *td = CreateDescriptor(trbCount, trbCount, trbSize); 986 if (td == NULL) 987 return B_NO_MEMORY; 988 989 // Normal Stage 990 const size_t maxPacketSize = pipe->MaxPacketSize(); 991 size_t remaining = transfer->FragmentLength(); 992 for (int32 i = 0; i < trbCount; i++) { 993 int32 trbLength = (remaining < trbSize) ? remaining : trbSize; 994 remaining -= trbLength; 995 996 // The "TD Size" field of a transfer TRB indicates the number of 997 // remaining maximum-size *packets* in this TD, *not* including the 998 // packets in the current TRB, and capped at 31 if there are more 999 // than 31 packets remaining in the TD. (XHCI 1.2 § 4.11.2.4 p218.) 1000 int32 tdSize = (remaining + maxPacketSize - 1) / maxPacketSize; 1001 if (tdSize > 31) 1002 tdSize = 31; 1003 1004 td->trbs[i].address = td->buffer_addrs[i]; 1005 td->trbs[i].status = TRB_2_IRQ(0) 1006 | TRB_2_BYTES(trbLength) 1007 | TRB_2_TD_SIZE(tdSize); 1008 td->trbs[i].flags = TRB_3_TYPE(TRB_TYPE_NORMAL) 1009 | TRB_3_CYCLE_BIT | TRB_3_CHAIN_BIT; 1010 1011 td->trb_used++; 1012 } 1013 1014 // Isochronous-specific. 1015 if (isochronousData != NULL) { 1016 // This is an isochronous transfer; it should have one TD per packet. 1017 for (uint32 i = 0; i < isochronousData->packet_count; i++) { 1018 td->trbs[i].flags &= ~(TRB_3_TYPE(TRB_TYPE_NORMAL)); 1019 td->trbs[i].flags |= TRB_3_TYPE(TRB_TYPE_ISOCH); 1020 1021 if (i != (isochronousData->packet_count - 1)) { 1022 // For all but the last TD, generate events (but not interrupts) on short packets. 1023 // (The last TD uses the regular Event Data TRB.) 1024 td->trbs[i].flags |= TRB_3_ISP_BIT | TRB_3_BEI_BIT; 1025 td->trbs[i].flags &= ~TRB_3_CHAIN_BIT; 1026 } 1027 } 1028 1029 // TODO: We do not currently take Mult into account at all! 1030 // How are we supposed to do that here? 1031 1032 // Determine the (starting) frame number: if ISO_ASAP is set, 1033 // we are queueing this "right away", and so want to reset 1034 // the starting_frame_number. Otherwise we use the passed one. 1035 uint32 frame; 1036 if ((isochronousData->flags & USB_ISO_ASAP) != 0 1037 || isochronousData->starting_frame_number == NULL) { 1038 // All reads from the microframe index register must be 1039 // incremented by 1. (XHCI 1.2 § 4.14.2.1.4 p265.) 1040 frame = (ReadRunReg32(XHCI_MFINDEX) + 1) >> 3; 1041 td->trbs[0].flags |= TRB_3_ISO_SIA_BIT; 1042 } else { 1043 frame = *isochronousData->starting_frame_number; 1044 td->trbs[0].flags |= TRB_3_FRID(frame); 1045 } 1046 if (isochronousData->starting_frame_number != NULL) 1047 *isochronousData->starting_frame_number = frame; 1048 } 1049 1050 // Set the ENT (Evaluate Next TRB) bit, so that the HC will not switch 1051 // contexts before evaluating the Link TRB that _LinkDescriptorForPipe 1052 // will insert, as otherwise there would be a race between us freeing 1053 // and unlinking the descriptor, and the controller evaluating the Link TRB 1054 // and thus getting back onto the main ring and executing the Event Data 1055 // TRB that generates the interrupt for this transfer. 1056 // 1057 // Note that we *do not* unset the CHAIN bit in this TRB, thus including 1058 // the Link TRB in this TD formally, which is required when using the 1059 // ENT bit. (XHCI 1.2 § 4.12.3 p250.) 1060 td->trbs[td->trb_used - 1].flags |= TRB_3_ENT_BIT; 1061 1062 if (!directionIn) { 1063 TRACE("copying out iov count %ld\n", transfer->VectorCount()); 1064 status_t status = transfer->PrepareKernelAccess(); 1065 if (status != B_OK) { 1066 FreeDescriptor(td); 1067 return status; 1068 } 1069 WriteDescriptor(td, transfer->Vector(), 1070 transfer->VectorCount(), transfer->IsPhysical()); 1071 } 1072 1073 td->transfer = transfer; 1074 status = _LinkDescriptorForPipe(td, endpoint); 1075 if (status != B_OK) { 1076 FreeDescriptor(td); 1077 return status; 1078 } 1079 1080 return B_OK; 1081 } 1082 1083 1084 status_t 1085 XHCI::CancelQueuedTransfers(Pipe *pipe, bool force) 1086 { 1087 xhci_endpoint* endpoint = (xhci_endpoint*)pipe->ControllerCookie(); 1088 if (endpoint == NULL || endpoint->trbs == NULL) { 1089 // Someone's de-allocated this pipe or endpoint in the meantime. 1090 // (Possibly AllocateDevice failed, and we were the temporary pipe.) 1091 return B_NO_INIT; 1092 } 1093 1094 #ifndef TRACE_USB 1095 if (force) 1096 #endif 1097 { 1098 TRACE_ALWAYS("cancel queued transfers (%" B_PRId8 ") for pipe %p (%d)\n", 1099 endpoint->used, pipe, pipe->EndpointAddress()); 1100 } 1101 1102 MutexLocker endpointLocker(endpoint->lock); 1103 1104 if (endpoint->td_head == NULL) { 1105 // There aren't any currently pending transfers to cancel. 1106 return B_OK; 1107 } 1108 1109 // Calling the callbacks while holding the endpoint lock could potentially 1110 // cause deadlocks, so we instead store them in a pointer array. We need 1111 // to do this separately from freeing the TDs, for in the case we fail 1112 // to stop the endpoint, we cancel the transfers but do not free the TDs. 1113 Transfer* transfers[XHCI_MAX_TRANSFERS]; 1114 int32 transfersCount = 0; 1115 1116 for (xhci_td* td = endpoint->td_head; td != NULL; td = td->next) { 1117 if (td->transfer == NULL) 1118 continue; 1119 1120 // We can't cancel or delete transfers under "force", as they probably 1121 // are not safe to use anymore. 1122 if (!force) { 1123 transfers[transfersCount] = td->transfer; 1124 transfersCount++; 1125 } 1126 td->transfer = NULL; 1127 } 1128 1129 // It is possible that while waiting for the stop-endpoint command to 1130 // complete, one of the queued transfers posts a completion event, so in 1131 // order to avoid a deadlock, we must unlock the endpoint. 1132 endpointLocker.Unlock(); 1133 status_t status = StopEndpoint(false, endpoint); 1134 if (status != B_OK && status != B_DEV_STALLED) { 1135 // It is possible that the endpoint was stopped by the controller at the 1136 // same time our STOP command was in progress, causing a "Context State" 1137 // error. In that case, try again; if the endpoint is already stopped, 1138 // StopEndpoint will notice this. (XHCI 1.2 § 4.6.9 p137.) 1139 status = StopEndpoint(false, endpoint); 1140 } 1141 if (status == B_DEV_STALLED) { 1142 // Only exit from a Halted state is a RESET. (XHCI 1.2 § 4.8.3 p163.) 1143 TRACE_ERROR("cancel queued transfers: halted endpoint, reset!\n"); 1144 status = ResetEndpoint(false, endpoint); 1145 } 1146 endpointLocker.Lock(); 1147 1148 // Detach the head TD from the endpoint. 1149 xhci_td* td_head = endpoint->td_head; 1150 endpoint->td_head = NULL; 1151 1152 if (status == B_OK) { 1153 // Clear the endpoint's TRBs. 1154 memset(endpoint->trbs, 0, sizeof(xhci_trb) * XHCI_ENDPOINT_RING_SIZE); 1155 endpoint->used = 0; 1156 endpoint->next = 0; 1157 1158 // Set dequeue pointer location to the beginning of the ring. 1159 SetTRDequeue(endpoint->trb_addr, 0, endpoint->id + 1, 1160 endpoint->device->slot); 1161 1162 // We don't need to do anything else to restart the ring, as it will resume 1163 // operation as normal upon the next doorbell. (XHCI 1.2 § 4.6.9 p136.) 1164 } else { 1165 // We couldn't stop the endpoint. Most likely the device has been 1166 // removed and the endpoint was stopped by the hardware, or is 1167 // for some reason busy and cannot be stopped. 1168 TRACE_ERROR("cancel queued transfers: could not stop endpoint: %s!\n", 1169 strerror(status)); 1170 1171 // Instead of freeing the TDs, we want to leave them in the endpoint 1172 // so that when/if the hardware returns, they can be properly unlinked, 1173 // as otherwise the endpoint could get "stuck" by having the "used" 1174 // slowly accumulate due to "dead" transfers. 1175 endpoint->td_head = td_head; 1176 td_head = NULL; 1177 } 1178 1179 endpointLocker.Unlock(); 1180 1181 for (int32 i = 0; i < transfersCount; i++) { 1182 transfers[i]->Finished(B_CANCELED, 0); 1183 delete transfers[i]; 1184 } 1185 1186 // This loop looks a bit strange because we need to store the "next" 1187 // pointer before freeing the descriptor. 1188 xhci_td* td; 1189 while ((td = td_head) != NULL) { 1190 td_head = td_head->next; 1191 FreeDescriptor(td); 1192 } 1193 1194 return B_OK; 1195 } 1196 1197 1198 status_t 1199 XHCI::StartDebugTransfer(Transfer *transfer) 1200 { 1201 Pipe *pipe = transfer->TransferPipe(); 1202 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 1203 if (endpoint == NULL) 1204 return B_BAD_VALUE; 1205 1206 // Check all locks that we are going to hit when running transfers. 1207 if (mutex_trylock(&endpoint->lock) != B_OK) 1208 return B_WOULD_BLOCK; 1209 if (mutex_trylock(&fFinishedLock) != B_OK) { 1210 mutex_unlock(&endpoint->lock); 1211 return B_WOULD_BLOCK; 1212 } 1213 if (mutex_trylock(&fEventLock) != B_OK) { 1214 mutex_unlock(&endpoint->lock); 1215 mutex_unlock(&fFinishedLock); 1216 return B_WOULD_BLOCK; 1217 } 1218 mutex_unlock(&endpoint->lock); 1219 mutex_unlock(&fFinishedLock); 1220 mutex_unlock(&fEventLock); 1221 1222 status_t status = SubmitTransfer(transfer); 1223 if (status != B_OK) 1224 return status; 1225 1226 // The endpoint's head TD is the TD of the just-submitted transfer. 1227 // Just like EHCI, abuse the callback cookie to hold the TD pointer. 1228 transfer->SetCallback(NULL, endpoint->td_head); 1229 1230 return B_OK; 1231 } 1232 1233 1234 status_t 1235 XHCI::CheckDebugTransfer(Transfer *transfer) 1236 { 1237 xhci_td *transfer_td = (xhci_td *)transfer->CallbackCookie(); 1238 if (transfer_td == NULL) 1239 return B_NO_INIT; 1240 1241 // Process events once, and then look for it in the finished list. 1242 ProcessEvents(); 1243 xhci_td *previous = NULL; 1244 for (xhci_td *td = fFinishedHead; td != NULL; td = td->next) { 1245 if (td != transfer_td) { 1246 previous = td; 1247 continue; 1248 } 1249 1250 // We've found it! 1251 if (previous == NULL) { 1252 fFinishedHead = fFinishedHead->next; 1253 } else { 1254 previous->next = td->next; 1255 } 1256 1257 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 1258 status_t status = (td->trb_completion_code == COMP_SUCCESS 1259 || td->trb_completion_code == COMP_SHORT_PACKET) ? B_OK : B_ERROR; 1260 1261 if (status == B_OK && directionIn) { 1262 ReadDescriptor(td, transfer->Vector(), transfer->VectorCount(), 1263 transfer->IsPhysical()); 1264 } 1265 1266 FreeDescriptor(td); 1267 transfer->SetCallback(NULL, NULL); 1268 return status; 1269 } 1270 1271 // We didn't find it. 1272 spin(75); 1273 return B_DEV_PENDING; 1274 } 1275 1276 1277 void 1278 XHCI::CancelDebugTransfer(Transfer *transfer) 1279 { 1280 while (CheckDebugTransfer(transfer) == B_DEV_PENDING) 1281 spin(100); 1282 } 1283 1284 1285 status_t 1286 XHCI::NotifyPipeChange(Pipe *pipe, usb_change change) 1287 { 1288 TRACE("pipe change %d for pipe %p (%d)\n", change, pipe, 1289 pipe->EndpointAddress()); 1290 1291 switch (change) { 1292 case USB_CHANGE_CREATED: 1293 return _InsertEndpointForPipe(pipe); 1294 case USB_CHANGE_DESTROYED: 1295 return _RemoveEndpointForPipe(pipe); 1296 1297 case USB_CHANGE_PIPE_POLICY_CHANGED: 1298 // We don't care about these, at least for now. 1299 return B_OK; 1300 } 1301 1302 TRACE_ERROR("unknown pipe change!\n"); 1303 return B_UNSUPPORTED; 1304 } 1305 1306 1307 xhci_td * 1308 XHCI::CreateDescriptor(uint32 trbCount, uint32 bufferCount, size_t bufferSize) 1309 { 1310 const bool inKDL = debug_debugger_running(); 1311 1312 xhci_td *result; 1313 if (!inKDL) { 1314 result = (xhci_td*)calloc(1, sizeof(xhci_td)); 1315 } else { 1316 // Just use the physical memory allocator while in KDL; it's less 1317 // secure than using the regular heap, but it's easier to deal with. 1318 phys_addr_t dummy; 1319 fStack->AllocateChunk((void **)&result, &dummy, sizeof(xhci_td)); 1320 } 1321 1322 if (result == NULL) { 1323 TRACE_ERROR("failed to allocate a transfer descriptor\n"); 1324 return NULL; 1325 } 1326 1327 // We always allocate 1 more TRB than requested, so that 1328 // _LinkDescriptorForPipe() has room to insert a link TRB. 1329 trbCount++; 1330 if (fStack->AllocateChunk((void **)&result->trbs, &result->trb_addr, 1331 (trbCount * sizeof(xhci_trb))) < B_OK) { 1332 TRACE_ERROR("failed to allocate TRBs\n"); 1333 FreeDescriptor(result); 1334 return NULL; 1335 } 1336 result->trb_count = trbCount; 1337 result->trb_used = 0; 1338 1339 if (bufferSize > 0) { 1340 // Due to how the USB stack allocates physical memory, we can't just 1341 // request one large chunk the size of the transfer, and so instead we 1342 // create a series of buffers as requested by our caller. 1343 1344 // We store the buffer pointers and addresses in one memory block. 1345 if (!inKDL) { 1346 result->buffers = (void**)calloc(bufferCount, 1347 (sizeof(void*) + sizeof(phys_addr_t))); 1348 } else { 1349 phys_addr_t dummy; 1350 fStack->AllocateChunk((void **)&result->buffers, &dummy, 1351 bufferCount * (sizeof(void*) + sizeof(phys_addr_t))); 1352 } 1353 if (result->buffers == NULL) { 1354 TRACE_ERROR("unable to allocate space for buffer infos\n"); 1355 FreeDescriptor(result); 1356 return NULL; 1357 } 1358 result->buffer_addrs = (phys_addr_t*)&result->buffers[bufferCount]; 1359 result->buffer_size = bufferSize; 1360 result->buffer_count = bufferCount; 1361 1362 // Optimization: If the requested total size of all buffers is less 1363 // than 32*B_PAGE_SIZE (the maximum size that the physical memory 1364 // allocator can handle), we allocate only one buffer and segment it. 1365 size_t totalSize = bufferSize * bufferCount; 1366 if (totalSize < (32 * B_PAGE_SIZE)) { 1367 if (fStack->AllocateChunk(&result->buffers[0], 1368 &result->buffer_addrs[0], totalSize) < B_OK) { 1369 TRACE_ERROR("unable to allocate space for large buffer (size %ld)\n", 1370 totalSize); 1371 FreeDescriptor(result); 1372 return NULL; 1373 } 1374 for (uint32 i = 1; i < bufferCount; i++) { 1375 result->buffers[i] = (void*)((addr_t)(result->buffers[i - 1]) 1376 + bufferSize); 1377 result->buffer_addrs[i] = result->buffer_addrs[i - 1] 1378 + bufferSize; 1379 } 1380 } else { 1381 // Otherwise, we allocate each buffer individually. 1382 for (uint32 i = 0; i < bufferCount; i++) { 1383 if (fStack->AllocateChunk(&result->buffers[i], 1384 &result->buffer_addrs[i], bufferSize) < B_OK) { 1385 TRACE_ERROR("unable to allocate space for a buffer (size " 1386 "%" B_PRIuSIZE ", count %" B_PRIu32 ")\n", 1387 bufferSize, bufferCount); 1388 FreeDescriptor(result); 1389 return NULL; 1390 } 1391 } 1392 } 1393 } else { 1394 result->buffers = NULL; 1395 result->buffer_addrs = NULL; 1396 } 1397 1398 // Initialize all other fields. 1399 result->transfer = NULL; 1400 result->trb_completion_code = 0; 1401 result->trb_left = 0; 1402 result->next = NULL; 1403 1404 TRACE("CreateDescriptor allocated %p, buffer_size %ld, buffer_count %" B_PRIu32 "\n", 1405 result, result->buffer_size, result->buffer_count); 1406 1407 return result; 1408 } 1409 1410 1411 void 1412 XHCI::FreeDescriptor(xhci_td *descriptor) 1413 { 1414 if (descriptor == NULL) 1415 return; 1416 1417 const bool inKDL = debug_debugger_running(); 1418 1419 if (descriptor->trbs != NULL) { 1420 fStack->FreeChunk(descriptor->trbs, descriptor->trb_addr, 1421 (descriptor->trb_count * sizeof(xhci_trb))); 1422 } 1423 if (descriptor->buffers != NULL) { 1424 size_t totalSize = descriptor->buffer_size * descriptor->buffer_count; 1425 if (totalSize < (32 * B_PAGE_SIZE)) { 1426 // This was allocated as one contiguous buffer. 1427 fStack->FreeChunk(descriptor->buffers[0], descriptor->buffer_addrs[0], 1428 totalSize); 1429 } else { 1430 for (uint32 i = 0; i < descriptor->buffer_count; i++) { 1431 if (descriptor->buffers[i] == NULL) 1432 continue; 1433 fStack->FreeChunk(descriptor->buffers[i], descriptor->buffer_addrs[i], 1434 descriptor->buffer_size); 1435 } 1436 } 1437 1438 if (!inKDL) { 1439 free(descriptor->buffers); 1440 } else { 1441 fStack->FreeChunk(descriptor->buffers, 0, 1442 descriptor->buffer_count * (sizeof(void*) + sizeof(phys_addr_t))); 1443 } 1444 } 1445 1446 if (!inKDL) 1447 free(descriptor); 1448 else 1449 fStack->FreeChunk(descriptor, 0, sizeof(xhci_td)); 1450 } 1451 1452 1453 size_t 1454 XHCI::WriteDescriptor(xhci_td *descriptor, generic_io_vec *vector, size_t vectorCount, bool physical) 1455 { 1456 size_t written = 0; 1457 1458 size_t bufIdx = 0, bufUsed = 0; 1459 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1460 size_t length = vector[vecIdx].length; 1461 1462 while (length > 0 && bufIdx < descriptor->buffer_count) { 1463 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1464 status_t status = generic_memcpy( 1465 (generic_addr_t)descriptor->buffers[bufIdx] + bufUsed, false, 1466 vector[vecIdx].base + (vector[vecIdx].length - length), physical, 1467 toCopy); 1468 ASSERT_ALWAYS(status == B_OK); 1469 1470 written += toCopy; 1471 bufUsed += toCopy; 1472 length -= toCopy; 1473 if (bufUsed == descriptor->buffer_size) { 1474 bufIdx++; 1475 bufUsed = 0; 1476 } 1477 } 1478 } 1479 1480 TRACE("wrote descriptor (%" B_PRIuSIZE " bytes)\n", written); 1481 return written; 1482 } 1483 1484 1485 size_t 1486 XHCI::ReadDescriptor(xhci_td *descriptor, generic_io_vec *vector, size_t vectorCount, bool physical) 1487 { 1488 size_t read = 0; 1489 1490 size_t bufIdx = 0, bufUsed = 0; 1491 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1492 size_t length = vector[vecIdx].length; 1493 1494 while (length > 0 && bufIdx < descriptor->buffer_count) { 1495 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1496 status_t status = generic_memcpy( 1497 vector[vecIdx].base + (vector[vecIdx].length - length), physical, 1498 (generic_addr_t)descriptor->buffers[bufIdx] + bufUsed, false, toCopy); 1499 ASSERT_ALWAYS(status == B_OK); 1500 1501 read += toCopy; 1502 bufUsed += toCopy; 1503 length -= toCopy; 1504 if (bufUsed == descriptor->buffer_size) { 1505 bufIdx++; 1506 bufUsed = 0; 1507 } 1508 } 1509 } 1510 1511 TRACE("read descriptor (%" B_PRIuSIZE " bytes)\n", read); 1512 return read; 1513 } 1514 1515 1516 Device * 1517 XHCI::AllocateDevice(Hub *parent, int8 hubAddress, uint8 hubPort, 1518 usb_speed speed) 1519 { 1520 TRACE("AllocateDevice hubAddress %d hubPort %d speed %d\n", hubAddress, 1521 hubPort, speed); 1522 1523 uint8 slot = XHCI_MAX_SLOTS; 1524 status_t status = EnableSlot(&slot); 1525 if (status != B_OK) { 1526 TRACE_ERROR("failed to enable slot: %s\n", strerror(status)); 1527 return NULL; 1528 } 1529 1530 if (slot == 0 || slot > fSlotCount) { 1531 TRACE_ERROR("AllocateDevice: bad slot\n"); 1532 return NULL; 1533 } 1534 1535 if (fDevices[slot].slot != 0) { 1536 TRACE_ERROR("AllocateDevice: slot already used\n"); 1537 return NULL; 1538 } 1539 1540 struct xhci_device *device = &fDevices[slot]; 1541 device->slot = slot; 1542 1543 device->input_ctx_area = fStack->AllocateArea((void **)&device->input_ctx, 1544 &device->input_ctx_addr, sizeof(*device->input_ctx) << fContextSizeShift, 1545 "XHCI input context"); 1546 if (device->input_ctx_area < B_OK) { 1547 TRACE_ERROR("unable to create a input context area\n"); 1548 CleanupDevice(device); 1549 return NULL; 1550 } 1551 if (fContextSizeShift == 1) { 1552 // 64-byte contexts have to be page-aligned in order for 1553 // _OffsetContextAddr to function properly. 1554 ASSERT((((addr_t)device->input_ctx) % B_PAGE_SIZE) == 0); 1555 } 1556 1557 memset(device->input_ctx, 0, sizeof(*device->input_ctx) << fContextSizeShift); 1558 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1559 _WriteContext(&device->input_ctx->input.addFlags, 3); 1560 1561 uint8 rhPort = hubPort; 1562 uint32 route = 0; 1563 for (Device *hubDevice = parent; hubDevice != RootObject(); 1564 hubDevice = (Device *)hubDevice->Parent()) { 1565 if (hubDevice->Parent() == RootObject()) 1566 break; 1567 1568 if (rhPort > 15) 1569 rhPort = 15; 1570 route = route << 4; 1571 route |= rhPort; 1572 1573 rhPort = hubDevice->HubPort(); 1574 } 1575 1576 uint32 dwslot0 = SLOT_0_NUM_ENTRIES(1) | SLOT_0_ROUTE(route); 1577 1578 // Get speed of port, only if device connected to root hub port 1579 // else we have to rely on value reported by the Hub Explore thread 1580 if (route == 0) { 1581 GetPortSpeed(hubPort - 1, &speed); 1582 TRACE("speed updated %d\n", speed); 1583 } 1584 1585 // add the speed 1586 switch (speed) { 1587 case USB_SPEED_LOWSPEED: 1588 dwslot0 |= SLOT_0_SPEED(2); 1589 break; 1590 case USB_SPEED_FULLSPEED: 1591 dwslot0 |= SLOT_0_SPEED(1); 1592 break; 1593 case USB_SPEED_HIGHSPEED: 1594 dwslot0 |= SLOT_0_SPEED(3); 1595 break; 1596 case USB_SPEED_SUPERSPEED: 1597 dwslot0 |= SLOT_0_SPEED(4); 1598 break; 1599 case USB_SPEED_SUPERSPEEDPLUS: 1600 dwslot0 |= SLOT_0_SPEED(5); 1601 break; 1602 default: 1603 TRACE_ERROR("unknown usb speed\n"); 1604 break; 1605 } 1606 1607 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1608 // TODO enable power save 1609 _WriteContext(&device->input_ctx->slot.dwslot1, SLOT_1_RH_PORT(rhPort)); 1610 uint32 dwslot2 = SLOT_2_IRQ_TARGET(0); 1611 1612 // If LS/FS device connected to non-root HS device 1613 if (route != 0 && parent->Speed() == USB_SPEED_HIGHSPEED 1614 && (speed == USB_SPEED_LOWSPEED || speed == USB_SPEED_FULLSPEED)) { 1615 struct xhci_device *parenthub = (struct xhci_device *) 1616 parent->ControllerCookie(); 1617 dwslot2 |= SLOT_2_PORT_NUM(hubPort); 1618 dwslot2 |= SLOT_2_TT_HUB_SLOT(parenthub->slot); 1619 } 1620 1621 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1622 1623 _WriteContext(&device->input_ctx->slot.dwslot3, SLOT_3_SLOT_STATE(0) 1624 | SLOT_3_DEVICE_ADDRESS(0)); 1625 1626 TRACE("slot 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 1627 "\n", _ReadContext(&device->input_ctx->slot.dwslot0), 1628 _ReadContext(&device->input_ctx->slot.dwslot1), 1629 _ReadContext(&device->input_ctx->slot.dwslot2), 1630 _ReadContext(&device->input_ctx->slot.dwslot3)); 1631 1632 device->device_ctx_area = fStack->AllocateArea((void **)&device->device_ctx, 1633 &device->device_ctx_addr, sizeof(*device->device_ctx) << fContextSizeShift, 1634 "XHCI device context"); 1635 if (device->device_ctx_area < B_OK) { 1636 TRACE_ERROR("unable to create a device context area\n"); 1637 CleanupDevice(device); 1638 return NULL; 1639 } 1640 memset(device->device_ctx, 0, sizeof(*device->device_ctx) << fContextSizeShift); 1641 1642 device->trb_area = fStack->AllocateArea((void **)&device->trbs, 1643 &device->trb_addr, sizeof(xhci_trb) * (XHCI_MAX_ENDPOINTS - 1) 1644 * XHCI_ENDPOINT_RING_SIZE, "XHCI endpoint trbs"); 1645 if (device->trb_area < B_OK) { 1646 TRACE_ERROR("unable to create a device trbs area\n"); 1647 CleanupDevice(device); 1648 return NULL; 1649 } 1650 1651 // set up slot pointer to device context 1652 fDcba->baseAddress[slot] = device->device_ctx_addr; 1653 1654 size_t maxPacketSize; 1655 switch (speed) { 1656 case USB_SPEED_LOWSPEED: 1657 case USB_SPEED_FULLSPEED: 1658 maxPacketSize = 8; 1659 break; 1660 case USB_SPEED_HIGHSPEED: 1661 maxPacketSize = 64; 1662 break; 1663 default: 1664 maxPacketSize = 512; 1665 break; 1666 } 1667 1668 xhci_endpoint* endpoint0 = &device->endpoints[0]; 1669 mutex_init(&endpoint0->lock, "xhci endpoint lock"); 1670 endpoint0->device = device; 1671 endpoint0->id = 0; 1672 endpoint0->status = 0; 1673 endpoint0->td_head = NULL; 1674 endpoint0->used = 0; 1675 endpoint0->next = 0; 1676 endpoint0->trbs = device->trbs; 1677 endpoint0->trb_addr = device->trb_addr; 1678 1679 // configure the Control endpoint 0 1680 if (ConfigureEndpoint(endpoint0, slot, 0, USB_OBJECT_CONTROL_PIPE, false, 1681 0, maxPacketSize, speed, 0, 0) != B_OK) { 1682 TRACE_ERROR("unable to configure default control endpoint\n"); 1683 CleanupDevice(device); 1684 return NULL; 1685 } 1686 1687 // device should get to addressed state (bsr = 0) 1688 status = SetAddress(device->input_ctx_addr, false, slot); 1689 if (status != B_OK) { 1690 TRACE_ERROR("unable to set address: %s\n", strerror(status)); 1691 CleanupDevice(device); 1692 return NULL; 1693 } 1694 1695 device->address = SLOT_3_DEVICE_ADDRESS_GET(_ReadContext( 1696 &device->device_ctx->slot.dwslot3)); 1697 1698 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", device->address, 1699 SLOT_3_SLOT_STATE_GET(_ReadContext( 1700 &device->device_ctx->slot.dwslot3))); 1701 TRACE("endpoint0 state 0x%08" B_PRIx32 "\n", 1702 ENDPOINT_0_STATE_GET(_ReadContext( 1703 &device->device_ctx->endpoints[0].dwendpoint0))); 1704 1705 // Wait a bit for the device to complete addressing 1706 snooze(USB_DELAY_SET_ADDRESS); 1707 1708 // Create a temporary pipe with the new address 1709 ControlPipe pipe(parent); 1710 pipe.SetControllerCookie(endpoint0); 1711 pipe.InitCommon(device->address + 1, 0, speed, Pipe::Default, maxPacketSize, 0, 1712 hubAddress, hubPort); 1713 1714 // Get the device descriptor 1715 // Just retrieve the first 8 bytes of the descriptor -> minimum supported 1716 // size of any device. It is enough because it includes the device type. 1717 1718 size_t actualLength = 0; 1719 usb_device_descriptor deviceDescriptor; 1720 1721 TRACE("getting the device descriptor\n"); 1722 status = pipe.SendRequest( 1723 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_STANDARD, // type 1724 USB_REQUEST_GET_DESCRIPTOR, // request 1725 USB_DESCRIPTOR_DEVICE << 8, // value 1726 0, // index 1727 8, // length 1728 (void *)&deviceDescriptor, // buffer 1729 8, // buffer length 1730 &actualLength); // actual length 1731 1732 if (actualLength != 8) { 1733 TRACE_ERROR("failed to get the device descriptor: %s\n", 1734 strerror(status)); 1735 CleanupDevice(device); 1736 return NULL; 1737 } 1738 1739 TRACE("device_class: %d device_subclass %d device_protocol %d\n", 1740 deviceDescriptor.device_class, deviceDescriptor.device_subclass, 1741 deviceDescriptor.device_protocol); 1742 1743 if (speed == USB_SPEED_FULLSPEED && deviceDescriptor.max_packet_size_0 != 8) { 1744 TRACE("Full speed device with different max packet size for Endpoint 0\n"); 1745 uint32 dwendpoint1 = _ReadContext( 1746 &device->input_ctx->endpoints[0].dwendpoint1); 1747 dwendpoint1 &= ~ENDPOINT_1_MAXPACKETSIZE(0xffff); 1748 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE( 1749 deviceDescriptor.max_packet_size_0); 1750 _WriteContext(&device->input_ctx->endpoints[0].dwendpoint1, 1751 dwendpoint1); 1752 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1753 _WriteContext(&device->input_ctx->input.addFlags, (1 << 1)); 1754 EvaluateContext(device->input_ctx_addr, device->slot); 1755 } 1756 1757 Device *deviceObject = NULL; 1758 if (deviceDescriptor.device_class == 0x09) { 1759 TRACE("creating new Hub\n"); 1760 TRACE("getting the hub descriptor\n"); 1761 size_t actualLength = 0; 1762 usb_hub_descriptor hubDescriptor; 1763 status = pipe.SendRequest( 1764 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_CLASS, // type 1765 USB_REQUEST_GET_DESCRIPTOR, // request 1766 USB_DESCRIPTOR_HUB << 8, // value 1767 0, // index 1768 sizeof(usb_hub_descriptor), // length 1769 (void *)&hubDescriptor, // buffer 1770 sizeof(usb_hub_descriptor), // buffer length 1771 &actualLength); 1772 1773 if (actualLength != sizeof(usb_hub_descriptor)) { 1774 TRACE_ERROR("error while getting the hub descriptor: %s\n", 1775 strerror(status)); 1776 CleanupDevice(device); 1777 return NULL; 1778 } 1779 1780 uint32 dwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1781 dwslot0 |= SLOT_0_HUB_BIT; 1782 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1783 uint32 dwslot1 = _ReadContext(&device->input_ctx->slot.dwslot1); 1784 dwslot1 |= SLOT_1_NUM_PORTS(hubDescriptor.num_ports); 1785 _WriteContext(&device->input_ctx->slot.dwslot1, dwslot1); 1786 if (speed == USB_SPEED_HIGHSPEED) { 1787 uint32 dwslot2 = _ReadContext(&device->input_ctx->slot.dwslot2); 1788 dwslot2 |= SLOT_2_TT_TIME(HUB_TTT_GET(hubDescriptor.characteristics)); 1789 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1790 } 1791 1792 deviceObject = new(std::nothrow) Hub(parent, hubAddress, hubPort, 1793 deviceDescriptor, device->address + 1, speed, false, device); 1794 } else { 1795 TRACE("creating new device\n"); 1796 deviceObject = new(std::nothrow) Device(parent, hubAddress, hubPort, 1797 deviceDescriptor, device->address + 1, speed, false, device); 1798 } 1799 if (deviceObject == NULL || deviceObject->InitCheck() != B_OK) { 1800 if (deviceObject == NULL) { 1801 TRACE_ERROR("no memory to allocate device\n"); 1802 } else { 1803 TRACE_ERROR("device object failed to initialize\n"); 1804 } 1805 CleanupDevice(device); 1806 return NULL; 1807 } 1808 1809 // We don't want to disable the default endpoint, naturally, which would 1810 // otherwise happen when this Pipe object is destroyed. 1811 pipe.SetControllerCookie(NULL); 1812 1813 deviceObject->RegisterNode(); 1814 1815 TRACE("AllocateDevice() port %d slot %d\n", hubPort, slot); 1816 return deviceObject; 1817 } 1818 1819 1820 void 1821 XHCI::FreeDevice(Device *usbDevice) 1822 { 1823 xhci_device* device = (xhci_device*)usbDevice->ControllerCookie(); 1824 TRACE("FreeDevice() slot %d\n", device->slot); 1825 1826 // Delete the device first, so it cleans up its pipes and tells us 1827 // what we need to destroy before we tear down our internal state. 1828 delete usbDevice; 1829 1830 CleanupDevice(device); 1831 } 1832 1833 1834 void 1835 XHCI::CleanupDevice(xhci_device *device) 1836 { 1837 if (device->slot != 0) { 1838 DisableSlot(device->slot); 1839 fDcba->baseAddress[device->slot] = 0; 1840 } 1841 1842 if (device->trb_addr != 0) 1843 delete_area(device->trb_area); 1844 if (device->input_ctx_addr != 0) 1845 delete_area(device->input_ctx_area); 1846 if (device->device_ctx_addr != 0) 1847 delete_area(device->device_ctx_area); 1848 1849 memset(device, 0, sizeof(xhci_device)); 1850 } 1851 1852 1853 uint8 1854 XHCI::_GetEndpointState(xhci_endpoint* endpoint) 1855 { 1856 struct xhci_device_ctx* device_ctx = endpoint->device->device_ctx; 1857 return ENDPOINT_0_STATE_GET( 1858 _ReadContext(&device_ctx->endpoints[endpoint->id].dwendpoint0)); 1859 } 1860 1861 1862 status_t 1863 XHCI::_InsertEndpointForPipe(Pipe *pipe) 1864 { 1865 TRACE("insert endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1866 1867 if (pipe->ControllerCookie() != NULL 1868 || pipe->Parent()->Type() != USB_OBJECT_DEVICE) { 1869 // default pipe is already referenced 1870 return B_OK; 1871 } 1872 1873 Device* usbDevice = (Device *)pipe->Parent(); 1874 if (usbDevice->Parent() == RootObject()) { 1875 // root hub needs no initialization 1876 return B_OK; 1877 } 1878 1879 struct xhci_device *device = (struct xhci_device *) 1880 usbDevice->ControllerCookie(); 1881 if (device == NULL) { 1882 panic("device is NULL\n"); 1883 return B_NO_INIT; 1884 } 1885 1886 const uint8 id = (2 * pipe->EndpointAddress() 1887 + (pipe->Direction() != Pipe::Out ? 1 : 0)) - 1; 1888 if (id >= XHCI_MAX_ENDPOINTS - 1) 1889 return B_BAD_VALUE; 1890 1891 if (id > 0) { 1892 uint32 devicedwslot0 = _ReadContext(&device->device_ctx->slot.dwslot0); 1893 if (SLOT_0_NUM_ENTRIES_GET(devicedwslot0) == 1) { 1894 uint32 inputdwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1895 inputdwslot0 &= ~(SLOT_0_NUM_ENTRIES(0x1f)); 1896 inputdwslot0 |= SLOT_0_NUM_ENTRIES(XHCI_MAX_ENDPOINTS - 1); 1897 _WriteContext(&device->input_ctx->slot.dwslot0, inputdwslot0); 1898 EvaluateContext(device->input_ctx_addr, device->slot); 1899 } 1900 1901 xhci_endpoint* endpoint = &device->endpoints[id]; 1902 mutex_init(&endpoint->lock, "xhci endpoint lock"); 1903 MutexLocker endpointLocker(endpoint->lock); 1904 1905 endpoint->device = device; 1906 endpoint->id = id; 1907 endpoint->td_head = NULL; 1908 endpoint->used = 0; 1909 endpoint->next = 0; 1910 1911 endpoint->trbs = device->trbs + id * XHCI_ENDPOINT_RING_SIZE; 1912 endpoint->trb_addr = device->trb_addr 1913 + id * XHCI_ENDPOINT_RING_SIZE * sizeof(xhci_trb); 1914 memset(endpoint->trbs, 0, 1915 sizeof(xhci_trb) * XHCI_ENDPOINT_RING_SIZE); 1916 1917 TRACE("insert endpoint for pipe: trbs, device %p endpoint %p\n", 1918 device->trbs, endpoint->trbs); 1919 TRACE("insert endpoint for pipe: trb_addr, device 0x%" B_PRIxPHYSADDR 1920 " endpoint 0x%" B_PRIxPHYSADDR "\n", device->trb_addr, 1921 endpoint->trb_addr); 1922 1923 const uint8 endpointNum = id + 1; 1924 1925 status_t status = ConfigureEndpoint(endpoint, device->slot, id, pipe->Type(), 1926 pipe->Direction() == Pipe::In, pipe->Interval(), pipe->MaxPacketSize(), 1927 usbDevice->Speed(), pipe->MaxBurst(), pipe->BytesPerInterval()); 1928 if (status != B_OK) { 1929 TRACE_ERROR("unable to configure endpoint: %s\n", strerror(status)); 1930 return status; 1931 } 1932 1933 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1934 _WriteContext(&device->input_ctx->input.addFlags, 1935 (1 << endpointNum) | (1 << 0)); 1936 1937 ConfigureEndpoint(device->input_ctx_addr, false, device->slot); 1938 1939 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", 1940 device->address, SLOT_3_SLOT_STATE_GET(_ReadContext( 1941 &device->device_ctx->slot.dwslot3))); 1942 TRACE("endpoint[0] state 0x%08" B_PRIx32 "\n", 1943 ENDPOINT_0_STATE_GET(_ReadContext( 1944 &device->device_ctx->endpoints[0].dwendpoint0))); 1945 TRACE("endpoint[%d] state 0x%08" B_PRIx32 "\n", id, 1946 ENDPOINT_0_STATE_GET(_ReadContext( 1947 &device->device_ctx->endpoints[id].dwendpoint0))); 1948 } 1949 pipe->SetControllerCookie(&device->endpoints[id]); 1950 1951 return B_OK; 1952 } 1953 1954 1955 status_t 1956 XHCI::_RemoveEndpointForPipe(Pipe *pipe) 1957 { 1958 TRACE("remove endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1959 1960 if (pipe->Parent()->Type() != USB_OBJECT_DEVICE) 1961 return B_OK; 1962 Device* usbDevice = (Device *)pipe->Parent(); 1963 if (usbDevice->Parent() == RootObject()) 1964 return B_BAD_VALUE; 1965 1966 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 1967 if (endpoint == NULL || endpoint->trbs == NULL) 1968 return B_NO_INIT; 1969 1970 pipe->SetControllerCookie(NULL); 1971 1972 if (endpoint->id > 0) { 1973 xhci_device *device = endpoint->device; 1974 uint8 epNumber = endpoint->id + 1; 1975 StopEndpoint(true, endpoint); 1976 1977 mutex_lock(&endpoint->lock); 1978 1979 // See comment in CancelQueuedTransfers. 1980 xhci_td* td; 1981 while ((td = endpoint->td_head) != NULL) { 1982 endpoint->td_head = endpoint->td_head->next; 1983 FreeDescriptor(td); 1984 } 1985 1986 mutex_destroy(&endpoint->lock); 1987 memset(endpoint, 0, sizeof(xhci_endpoint)); 1988 1989 _WriteContext(&device->input_ctx->input.dropFlags, (1 << epNumber)); 1990 _WriteContext(&device->input_ctx->input.addFlags, (1 << 0)); 1991 1992 // The Deconfigure bit in the Configure Endpoint command indicates 1993 // that *all* endpoints are to be deconfigured, and not just the ones 1994 // specified in the context flags. (XHCI 1.2 § 4.6.6 p115.) 1995 ConfigureEndpoint(device->input_ctx_addr, false, device->slot); 1996 } 1997 1998 return B_OK; 1999 } 2000 2001 2002 status_t 2003 XHCI::_LinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 2004 { 2005 TRACE("link descriptor for pipe\n"); 2006 2007 // Use mutex_trylock first, in case we are in KDL. 2008 MutexLocker endpointLocker(&endpoint->lock, mutex_trylock(&endpoint->lock) == B_OK); 2009 2010 // "used" refers to the number of currently linked TDs, not the number of 2011 // used TRBs on the ring (we use 2 TRBs on the ring per transfer.) 2012 // Furthermore, we have to leave an empty item between the head and tail. 2013 if (endpoint->used >= (XHCI_MAX_TRANSFERS - 1)) { 2014 TRACE_ERROR("link descriptor for pipe: max transfers count exceeded\n"); 2015 return B_BAD_VALUE; 2016 } 2017 2018 // We do not support queuing other transfers in tandem with a fragmented one. 2019 if (endpoint->td_head != NULL && endpoint->td_head->transfer != NULL 2020 && endpoint->td_head->transfer->IsFragmented()) { 2021 TRACE_ERROR("cannot submit transfer: a fragmented transfer is queued\n"); 2022 return B_DEV_RESOURCE_CONFLICT; 2023 } 2024 2025 endpoint->used++; 2026 descriptor->next = endpoint->td_head; 2027 endpoint->td_head = descriptor; 2028 2029 uint32 link = endpoint->next, eventdata = link + 1, next = eventdata + 1; 2030 if (eventdata == XHCI_ENDPOINT_RING_SIZE || next == XHCI_ENDPOINT_RING_SIZE) { 2031 // If it's "next" not "eventdata" that got us here, we will be leaving 2032 // one TRB at the end of the ring unused. 2033 eventdata = 0; 2034 next = 1; 2035 } 2036 2037 TRACE("link descriptor for pipe: link %d, next %d\n", link, next); 2038 2039 // Add a Link TRB to the end of the descriptor. 2040 phys_addr_t addr = endpoint->trb_addr + (eventdata * sizeof(xhci_trb)); 2041 descriptor->trbs[descriptor->trb_used].address = addr; 2042 descriptor->trbs[descriptor->trb_used].status = TRB_2_IRQ(0); 2043 descriptor->trbs[descriptor->trb_used].flags = TRB_3_TYPE(TRB_TYPE_LINK) 2044 | TRB_3_CHAIN_BIT | TRB_3_CYCLE_BIT; 2045 // It is specified that (XHCI 1.2 § 4.12.3 Note 2 p251) if the TRB 2046 // following one with the ENT bit set is a Link TRB, the Link TRB 2047 // shall be evaluated *and* the subsequent TRB shall be. Thus a 2048 // TRB_3_ENT_BIT is unnecessary here; and from testing seems to 2049 // break all transfers on a (very) small number of controllers. 2050 2051 #if !B_HOST_IS_LENDIAN 2052 // Convert endianness. 2053 for (uint32 i = 0; i <= descriptor->trb_used; i++) { 2054 descriptor->trbs[i].address = 2055 B_HOST_TO_LENDIAN_INT64(descriptor->trbs[i].address); 2056 descriptor->trbs[i].status = 2057 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].status); 2058 descriptor->trbs[i].flags = 2059 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].flags); 2060 } 2061 #endif 2062 2063 // Link the descriptor. 2064 endpoint->trbs[link].address = 2065 B_HOST_TO_LENDIAN_INT64(descriptor->trb_addr); 2066 endpoint->trbs[link].status = 2067 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 2068 endpoint->trbs[link].flags = 2069 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_LINK)); 2070 2071 // Set up the Event Data TRB (XHCI 1.2 § 4.11.5.2 p230.) 2072 // 2073 // We do this on the main ring for two reasons: first, to avoid a small 2074 // potential race between the interrupt and the controller evaluating 2075 // the link TRB to get back onto the ring; and second, because many 2076 // controllers throw errors if the target of a Link TRB is not valid 2077 // (i.e. does not have its Cycle Bit set.) 2078 // 2079 // We also set the "address" field, which the controller will copy 2080 // verbatim into the TRB it posts to the event ring, to be the last 2081 // "real" TRB in the TD; this will allow us to determine what transfer 2082 // the resulting Transfer Event TRB refers to. 2083 endpoint->trbs[eventdata].address = 2084 B_HOST_TO_LENDIAN_INT64(descriptor->trb_addr 2085 + (descriptor->trb_used - 1) * sizeof(xhci_trb)); 2086 endpoint->trbs[eventdata].status = 2087 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 2088 endpoint->trbs[eventdata].flags = 2089 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_EVENT_DATA) 2090 | TRB_3_IOC_BIT | TRB_3_CYCLE_BIT); 2091 2092 endpoint->trbs[next].address = 0; 2093 endpoint->trbs[next].status = 0; 2094 endpoint->trbs[next].flags = 0; 2095 2096 memory_write_barrier(); 2097 2098 // Everything is ready, so write the cycle bit. 2099 endpoint->trbs[link].flags |= B_HOST_TO_LENDIAN_INT32(TRB_3_CYCLE_BIT); 2100 2101 TRACE("_LinkDescriptorForPipe pLink %p phys 0x%" B_PRIxPHYSADDR 2102 " 0x%" B_PRIxPHYSADDR " 0x%08" B_PRIx32 "\n", &endpoint->trbs[link], 2103 endpoint->trb_addr + link * sizeof(struct xhci_trb), 2104 endpoint->trbs[link].address, 2105 B_LENDIAN_TO_HOST_INT32(endpoint->trbs[link].flags)); 2106 2107 endpoint->next = next; 2108 endpointLocker.Unlock(); 2109 2110 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 2111 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 2112 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 2113 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 2114 2115 Ring(endpoint->device->slot, endpoint->id + 1); 2116 2117 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 2118 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 2119 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 2120 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 2121 2122 return B_OK; 2123 } 2124 2125 2126 status_t 2127 XHCI::_UnlinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 2128 { 2129 TRACE("unlink descriptor for pipe\n"); 2130 // We presume that the caller has already locked or owns the endpoint. 2131 2132 endpoint->used--; 2133 if (descriptor == endpoint->td_head) { 2134 endpoint->td_head = descriptor->next; 2135 descriptor->next = NULL; 2136 return B_OK; 2137 } else { 2138 for (xhci_td *td = endpoint->td_head; td->next != NULL; td = td->next) { 2139 if (td->next == descriptor) { 2140 td->next = descriptor->next; 2141 descriptor->next = NULL; 2142 return B_OK; 2143 } 2144 } 2145 } 2146 2147 endpoint->used++; 2148 return B_ERROR; 2149 } 2150 2151 2152 status_t 2153 XHCI::ConfigureEndpoint(xhci_endpoint* ep, uint8 slot, uint8 number, uint8 type, 2154 bool directionIn, uint16 interval, uint16 maxPacketSize, usb_speed speed, 2155 uint8 maxBurst, uint16 bytesPerInterval) 2156 { 2157 struct xhci_device* device = &fDevices[slot]; 2158 2159 uint32 dwendpoint0 = 0; 2160 uint32 dwendpoint1 = 0; 2161 uint64 qwendpoint2 = 0; 2162 uint32 dwendpoint4 = 0; 2163 2164 // Compute and assign the endpoint type. (XHCI 1.2 § 6.2.3 Table 6-9 p452.) 2165 uint8 xhciType = 4; 2166 if ((type & USB_OBJECT_INTERRUPT_PIPE) != 0) 2167 xhciType = 3; 2168 if ((type & USB_OBJECT_BULK_PIPE) != 0) 2169 xhciType = 2; 2170 if ((type & USB_OBJECT_ISO_PIPE) != 0) 2171 xhciType = 1; 2172 xhciType |= directionIn ? (1 << 2) : 0; 2173 dwendpoint1 |= ENDPOINT_1_EPTYPE(xhciType); 2174 2175 // Compute and assign interval. (XHCI 1.2 § 6.2.3.6 p456.) 2176 uint16 calcInterval; 2177 if ((type & USB_OBJECT_BULK_PIPE) != 0 2178 || (type & USB_OBJECT_CONTROL_PIPE) != 0) { 2179 // Bulk and Control endpoints never issue NAKs. 2180 calcInterval = 0; 2181 } else { 2182 switch (speed) { 2183 case USB_SPEED_FULLSPEED: 2184 if ((type & USB_OBJECT_ISO_PIPE) != 0) { 2185 // Convert 1-16 into 3-18. 2186 calcInterval = min_c(max_c(interval, 1), 16) + 2; 2187 break; 2188 } 2189 2190 // fall through 2191 case USB_SPEED_LOWSPEED: { 2192 // Convert 1ms-255ms into 3-10. 2193 2194 // Find the index of the highest set bit in "interval". 2195 uint32 temp = min_c(max_c(interval, 1), 255); 2196 for (calcInterval = 0; temp != 1; calcInterval++) 2197 temp = temp >> 1; 2198 calcInterval += 3; 2199 break; 2200 } 2201 2202 case USB_SPEED_HIGHSPEED: 2203 case USB_SPEED_SUPERSPEED: 2204 case USB_SPEED_SUPERSPEEDPLUS: 2205 default: 2206 // Convert 1-16 into 0-15. 2207 calcInterval = min_c(max_c(interval, 1), 16) - 1; 2208 break; 2209 } 2210 } 2211 dwendpoint0 |= ENDPOINT_0_INTERVAL(calcInterval); 2212 2213 // For non-isochronous endpoints, we want the controller to retry failed 2214 // transfers, if possible. (XHCI 1.2 § 4.10.2.3 p197.) 2215 if ((type & USB_OBJECT_ISO_PIPE) == 0) 2216 dwendpoint1 |= ENDPOINT_1_CERR(3); 2217 2218 // Assign maximum burst size. For USB3 devices this is passed in; for 2219 // all other devices we compute it. (XHCI 1.2 § 4.8.2 p161.) 2220 if (speed == USB_SPEED_HIGHSPEED && (type & (USB_OBJECT_INTERRUPT_PIPE 2221 | USB_OBJECT_ISO_PIPE)) != 0) { 2222 maxBurst = (maxPacketSize & 0x1800) >> 11; 2223 } else if (speed < USB_SPEED_SUPERSPEED) { 2224 maxBurst = 0; 2225 } 2226 dwendpoint1 |= ENDPOINT_1_MAXBURST(maxBurst); 2227 2228 // Assign maximum packet size, set the ring address, and set the 2229 // "Dequeue Cycle State" bit. (XHCI 1.2 § 6.2.3 Table 6-10 p453.) 2230 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE(maxPacketSize); 2231 qwendpoint2 |= ENDPOINT_2_DCS_BIT | ep->trb_addr; 2232 2233 // The Max Burst Payload is the number of bytes moved by a 2234 // maximum sized burst. (XHCI 1.2 § 4.11.7.1 p236.) 2235 ep->max_burst_payload = (maxBurst + 1) * maxPacketSize; 2236 if (ep->max_burst_payload == 0) { 2237 TRACE_ERROR("ConfigureEndpoint() failed invalid max_burst_payload\n"); 2238 return B_BAD_VALUE; 2239 } 2240 2241 // Assign average TRB length. 2242 if ((type & USB_OBJECT_CONTROL_PIPE) != 0) { 2243 // Control pipes are a special case, as they rarely have 2244 // outbound transfers of any substantial size. 2245 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(8); 2246 } else if ((type & USB_OBJECT_ISO_PIPE) != 0) { 2247 // Isochronous pipes are another special case: the TRB size will be 2248 // one packet (which is normally smaller than the max packet size, 2249 // but we don't know what it is here.) 2250 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(maxPacketSize); 2251 } else { 2252 // Under all other circumstances, we put max_burst_payload in a TRB. 2253 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(ep->max_burst_payload); 2254 } 2255 2256 // Assign maximum ESIT payload. (XHCI 1.2 § 4.14.2 p259.) 2257 if ((type & (USB_OBJECT_INTERRUPT_PIPE | USB_OBJECT_ISO_PIPE)) != 0) { 2258 // TODO: For SuperSpeedPlus endpoints, there is yet another descriptor 2259 // for isochronous endpoints that specifies the maximum ESIT payload. 2260 // We don't fetch this yet, so just fall back to the USB2 computation 2261 // method if bytesPerInterval is 0. 2262 if (speed >= USB_SPEED_SUPERSPEED && bytesPerInterval != 0) 2263 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD(bytesPerInterval); 2264 else if (speed >= USB_SPEED_HIGHSPEED) 2265 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD((maxBurst + 1) * maxPacketSize); 2266 } 2267 2268 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint0, 2269 dwendpoint0); 2270 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint1, 2271 dwendpoint1); 2272 _WriteContext(&device->input_ctx->endpoints[number].qwendpoint2, 2273 qwendpoint2); 2274 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint4, 2275 dwendpoint4); 2276 2277 TRACE("endpoint 0x%" B_PRIx32 " 0x%" B_PRIx32 " 0x%" B_PRIx64 " 0x%" 2278 B_PRIx32 "\n", 2279 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint0), 2280 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint1), 2281 _ReadContext(&device->input_ctx->endpoints[number].qwendpoint2), 2282 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint4)); 2283 2284 return B_OK; 2285 } 2286 2287 2288 status_t 2289 XHCI::GetPortSpeed(uint8 index, usb_speed* speed) 2290 { 2291 if (index >= fPortCount) 2292 return B_BAD_INDEX; 2293 2294 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 2295 2296 switch (PS_SPEED_GET(portStatus)) { 2297 case 2: 2298 *speed = USB_SPEED_LOWSPEED; 2299 break; 2300 case 1: 2301 *speed = USB_SPEED_FULLSPEED; 2302 break; 2303 case 3: 2304 *speed = USB_SPEED_HIGHSPEED; 2305 break; 2306 case 4: 2307 *speed = USB_SPEED_SUPERSPEED; 2308 break; 2309 case 5: 2310 *speed = USB_SPEED_SUPERSPEEDPLUS; 2311 break; 2312 default: 2313 TRACE_ALWAYS("nonstandard port speed %" B_PRId32 ", assuming SuperSpeed\n", 2314 PS_SPEED_GET(portStatus)); 2315 *speed = USB_SPEED_SUPERSPEED; 2316 break; 2317 } 2318 2319 return B_OK; 2320 } 2321 2322 2323 status_t 2324 XHCI::GetPortStatus(uint8 index, usb_port_status* status) 2325 { 2326 if (index >= fPortCount) 2327 return B_BAD_INDEX; 2328 2329 status->status = status->change = 0; 2330 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 2331 TRACE("port %" B_PRId8 " status=0x%08" B_PRIx32 "\n", index, portStatus); 2332 2333 // build the status 2334 switch (PS_SPEED_GET(portStatus)) { 2335 case 3: 2336 status->status |= PORT_STATUS_HIGH_SPEED; 2337 break; 2338 case 2: 2339 status->status |= PORT_STATUS_LOW_SPEED; 2340 break; 2341 default: 2342 break; 2343 } 2344 2345 if (portStatus & PS_CCS) 2346 status->status |= PORT_STATUS_CONNECTION; 2347 if (portStatus & PS_PED) 2348 status->status |= PORT_STATUS_ENABLE; 2349 if (portStatus & PS_OCA) 2350 status->status |= PORT_STATUS_OVER_CURRENT; 2351 if (portStatus & PS_PR) 2352 status->status |= PORT_STATUS_RESET; 2353 if (portStatus & PS_PP) { 2354 if (fPortSpeeds[index] >= USB_SPEED_SUPERSPEED) 2355 status->status |= PORT_STATUS_SS_POWER; 2356 else 2357 status->status |= PORT_STATUS_POWER; 2358 } 2359 if (fPortSpeeds[index] >= USB_SPEED_SUPERSPEED) 2360 status->status |= portStatus & PS_PLS_MASK; 2361 2362 // build the change 2363 if (portStatus & PS_CSC) 2364 status->change |= PORT_STATUS_CONNECTION; 2365 if (portStatus & PS_PEC) 2366 status->change |= PORT_STATUS_ENABLE; 2367 if (portStatus & PS_OCC) 2368 status->change |= PORT_STATUS_OVER_CURRENT; 2369 if (portStatus & PS_PRC) 2370 status->change |= PORT_STATUS_RESET; 2371 2372 if (fPortSpeeds[index] >= USB_SPEED_SUPERSPEED) { 2373 if (portStatus & PS_PLC) 2374 status->change |= PORT_CHANGE_LINK_STATE; 2375 if (portStatus & PS_WRC) 2376 status->change |= PORT_CHANGE_BH_PORT_RESET; 2377 } 2378 2379 return B_OK; 2380 } 2381 2382 2383 status_t 2384 XHCI::SetPortFeature(uint8 index, uint16 feature) 2385 { 2386 TRACE("set port feature index %u feature %u\n", index, feature); 2387 if (index >= fPortCount) 2388 return B_BAD_INDEX; 2389 2390 uint32 portRegister = XHCI_PORTSC(index); 2391 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2392 2393 switch (feature) { 2394 case PORT_SUSPEND: 2395 if ((portStatus & PS_PED) == 0 || (portStatus & PS_PR) 2396 || (portStatus & PS_PLS_MASK) >= PS_XDEV_U3) { 2397 TRACE_ERROR("USB core suspending device not in U0/U1/U2.\n"); 2398 return B_BAD_VALUE; 2399 } 2400 portStatus &= ~PS_PLS_MASK; 2401 WriteOpReg(portRegister, portStatus | PS_LWS | PS_XDEV_U3); 2402 break; 2403 2404 case PORT_RESET: 2405 WriteOpReg(portRegister, portStatus | PS_PR); 2406 break; 2407 2408 case PORT_POWER: 2409 WriteOpReg(portRegister, portStatus | PS_PP); 2410 break; 2411 default: 2412 return B_BAD_VALUE; 2413 } 2414 ReadOpReg(portRegister); 2415 return B_OK; 2416 } 2417 2418 2419 status_t 2420 XHCI::ClearPortFeature(uint8 index, uint16 feature) 2421 { 2422 TRACE("clear port feature index %u feature %u\n", index, feature); 2423 if (index >= fPortCount) 2424 return B_BAD_INDEX; 2425 2426 uint32 portRegister = XHCI_PORTSC(index); 2427 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2428 2429 switch (feature) { 2430 case PORT_SUSPEND: 2431 portStatus = ReadOpReg(portRegister); 2432 if (portStatus & PS_PR) 2433 return B_BAD_VALUE; 2434 if (portStatus & PS_XDEV_U3) { 2435 if ((portStatus & PS_PED) == 0) 2436 return B_BAD_VALUE; 2437 portStatus &= ~PS_PLS_MASK; 2438 WriteOpReg(portRegister, portStatus | PS_XDEV_U0 | PS_LWS); 2439 } 2440 break; 2441 case PORT_ENABLE: 2442 WriteOpReg(portRegister, portStatus | PS_PED); 2443 break; 2444 case PORT_POWER: 2445 WriteOpReg(portRegister, portStatus & ~PS_PP); 2446 break; 2447 case C_PORT_CONNECTION: 2448 WriteOpReg(portRegister, portStatus | PS_CSC); 2449 break; 2450 case C_PORT_ENABLE: 2451 WriteOpReg(portRegister, portStatus | PS_PEC); 2452 break; 2453 case C_PORT_OVER_CURRENT: 2454 WriteOpReg(portRegister, portStatus | PS_OCC); 2455 break; 2456 case C_PORT_RESET: 2457 WriteOpReg(portRegister, portStatus | PS_PRC); 2458 break; 2459 case C_PORT_BH_PORT_RESET: 2460 WriteOpReg(portRegister, portStatus | PS_WRC); 2461 break; 2462 case C_PORT_LINK_STATE: 2463 WriteOpReg(portRegister, portStatus | PS_PLC); 2464 break; 2465 default: 2466 return B_BAD_VALUE; 2467 } 2468 2469 ReadOpReg(portRegister); 2470 return B_OK; 2471 } 2472 2473 2474 status_t 2475 XHCI::ControllerHalt() 2476 { 2477 // Mask off run state 2478 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) & ~CMD_RUN); 2479 2480 // wait for shutdown state 2481 if (WaitOpBits(XHCI_STS, STS_HCH, STS_HCH) != B_OK) { 2482 TRACE_ERROR("HCH shutdown timeout\n"); 2483 return B_ERROR; 2484 } 2485 return B_OK; 2486 } 2487 2488 2489 status_t 2490 XHCI::ControllerReset() 2491 { 2492 TRACE("ControllerReset() cmd: 0x%" B_PRIx32 " sts: 0x%" B_PRIx32 "\n", 2493 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 2494 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) | CMD_HCRST); 2495 2496 if (WaitOpBits(XHCI_CMD, CMD_HCRST, 0) != B_OK) { 2497 TRACE_ERROR("ControllerReset() failed CMD_HCRST\n"); 2498 return B_ERROR; 2499 } 2500 2501 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 2502 TRACE_ERROR("ControllerReset() failed STS_CNR\n"); 2503 return B_ERROR; 2504 } 2505 2506 return B_OK; 2507 } 2508 2509 2510 int32 2511 XHCI::InterruptHandler(void* data) 2512 { 2513 return ((XHCI*)data)->Interrupt(); 2514 } 2515 2516 2517 int32 2518 XHCI::Interrupt() 2519 { 2520 SpinLocker _(&fSpinlock); 2521 2522 uint32 status = ReadOpReg(XHCI_STS); 2523 uint32 temp = ReadRunReg32(XHCI_IMAN(0)); 2524 WriteOpReg(XHCI_STS, status); 2525 WriteRunReg32(XHCI_IMAN(0), temp); 2526 2527 int32 result = B_HANDLED_INTERRUPT; 2528 2529 if ((status & STS_HCH) != 0) { 2530 TRACE_ERROR("Host Controller halted\n"); 2531 return result; 2532 } 2533 if ((status & STS_HSE) != 0) { 2534 TRACE_ERROR("Host System Error\n"); 2535 return result; 2536 } 2537 if ((status & STS_HCE) != 0) { 2538 TRACE_ERROR("Host Controller Error\n"); 2539 return result; 2540 } 2541 2542 if ((status & STS_EINT) == 0) { 2543 TRACE("STS: 0x%" B_PRIx32 " IRQ_PENDING: 0x%" B_PRIx32 "\n", 2544 status, temp); 2545 return B_UNHANDLED_INTERRUPT; 2546 } 2547 2548 TRACE("Event Interrupt\n"); 2549 release_sem_etc(fEventSem, 1, B_DO_NOT_RESCHEDULE); 2550 return B_INVOKE_SCHEDULER; 2551 } 2552 2553 2554 void 2555 XHCI::Ring(uint8 slot, uint8 endpoint) 2556 { 2557 TRACE("Ding Dong! slot:%d endpoint %d\n", slot, endpoint) 2558 if ((slot == 0 && endpoint > 0) || (slot > 0 && endpoint == 0)) 2559 panic("Ring() invalid slot/endpoint combination\n"); 2560 if (slot > fSlotCount || endpoint >= XHCI_MAX_ENDPOINTS) 2561 panic("Ring() invalid slot or endpoint\n"); 2562 2563 WriteDoorReg32(XHCI_DOORBELL(slot), XHCI_DOORBELL_TARGET(endpoint) 2564 | XHCI_DOORBELL_STREAMID(0)); 2565 ReadDoorReg32(XHCI_DOORBELL(slot)); 2566 // Flush PCI writes 2567 } 2568 2569 2570 void 2571 XHCI::QueueCommand(xhci_trb* trb) 2572 { 2573 uint8 i, j; 2574 uint32 temp; 2575 2576 i = fCmdIdx; 2577 j = fCmdCcs; 2578 2579 TRACE("command[%u] = %" B_PRId32 " (0x%016" B_PRIx64 ", 0x%08" B_PRIx32 2580 ", 0x%08" B_PRIx32 ")\n", i, TRB_3_TYPE_GET(trb->flags), trb->address, 2581 trb->status, trb->flags); 2582 2583 fCmdRing[i].address = trb->address; 2584 fCmdRing[i].status = trb->status; 2585 temp = trb->flags; 2586 2587 if (j) 2588 temp |= TRB_3_CYCLE_BIT; 2589 else 2590 temp &= ~TRB_3_CYCLE_BIT; 2591 temp &= ~TRB_3_TC_BIT; 2592 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2593 2594 fCmdAddr = fErst->rs_addr + (XHCI_MAX_EVENTS + i) * sizeof(xhci_trb); 2595 2596 i++; 2597 2598 if (i == (XHCI_MAX_COMMANDS - 1)) { 2599 temp = TRB_3_TYPE(TRB_TYPE_LINK) | TRB_3_TC_BIT; 2600 if (j) 2601 temp |= TRB_3_CYCLE_BIT; 2602 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2603 2604 i = 0; 2605 j ^= 1; 2606 } 2607 2608 fCmdIdx = i; 2609 fCmdCcs = j; 2610 } 2611 2612 2613 void 2614 XHCI::HandleCmdComplete(xhci_trb* trb) 2615 { 2616 if (fCmdAddr == trb->address) { 2617 TRACE("Received command event\n"); 2618 fCmdResult[0] = trb->status; 2619 fCmdResult[1] = B_LENDIAN_TO_HOST_INT32(trb->flags); 2620 release_sem_etc(fCmdCompSem, 1, B_DO_NOT_RESCHEDULE); 2621 } else 2622 TRACE_ERROR("received command event for unknown command!\n") 2623 } 2624 2625 2626 void 2627 XHCI::HandleTransferComplete(xhci_trb* trb) 2628 { 2629 const uint32 flags = B_LENDIAN_TO_HOST_INT32(trb->flags); 2630 const uint8 endpointNumber = TRB_3_ENDPOINT_GET(flags), 2631 slot = TRB_3_SLOT_GET(flags); 2632 2633 if (slot > fSlotCount) 2634 TRACE_ERROR("invalid slot\n"); 2635 if (endpointNumber == 0 || endpointNumber >= XHCI_MAX_ENDPOINTS) { 2636 TRACE_ERROR("invalid endpoint\n"); 2637 return; 2638 } 2639 2640 xhci_device *device = &fDevices[slot]; 2641 xhci_endpoint *endpoint = &device->endpoints[endpointNumber - 1]; 2642 2643 if (endpoint->trbs == NULL) { 2644 TRACE_ERROR("got TRB but endpoint is not allocated!\n"); 2645 return; 2646 } 2647 2648 // Use mutex_trylock first, in case we are in KDL. 2649 MutexLocker endpointLocker(endpoint->lock, mutex_trylock(&endpoint->lock) == B_OK); 2650 if (!endpointLocker.IsLocked()) { 2651 // We failed to get the lock. Most likely it was destroyed 2652 // while we were waiting for it. 2653 return; 2654 } 2655 2656 TRACE("HandleTransferComplete: ed %" B_PRIu32 ", status %" B_PRId32 "\n", 2657 (flags & TRB_3_EVENT_DATA_BIT), trb->status); 2658 2659 uint8 completionCode = TRB_2_COMP_CODE_GET(trb->status); 2660 2661 if (completionCode == COMP_RING_OVERRUN || completionCode == COMP_RING_UNDERRUN) { 2662 // These occur on isochronous endpoints when there is no TRB ready to be 2663 // executed at the appropriate time. (XHCI 1.2 § 4.10.3.1 p204.) 2664 endpoint->status = completionCode; 2665 return; 2666 } 2667 2668 int32 remainder = TRB_2_REM_GET(trb->status), transferred = -1; 2669 if ((flags & TRB_3_EVENT_DATA_BIT) != 0) { 2670 // In the case of an Event Data TRB, value in the status field refers 2671 // to the actual number of bytes transferred across the whole TD. 2672 // (XHCI 1.2 § 6.4.2.1 Table 6-38 p478.) 2673 transferred = remainder; 2674 remainder = -1; 2675 } else { 2676 // This should only occur under error conditions, or for isochronous transfers. 2677 TRACE("got transfer event for a non-Event Data TRB!\n"); 2678 2679 if (completionCode == COMP_STOPPED_LENGTH_INVALID) 2680 remainder = -1; 2681 } 2682 2683 if (completionCode != COMP_SUCCESS && completionCode != COMP_SHORT_PACKET 2684 && completionCode != COMP_STOPPED && completionCode != COMP_STOPPED_LENGTH_INVALID) { 2685 TRACE_ALWAYS("transfer error on slot %" B_PRId8 " endpoint %" B_PRId8 2686 ": %s\n", slot, endpointNumber, xhci_error_string(completionCode)); 2687 } 2688 2689 phys_addr_t source = B_LENDIAN_TO_HOST_INT64(trb->address); 2690 if (source >= endpoint->trb_addr 2691 && (source - endpoint->trb_addr) < (XHCI_ENDPOINT_RING_SIZE * sizeof(xhci_trb))) { 2692 // The "source" address points to a TRB on the ring. 2693 // See if we can figure out what it really corresponds to. 2694 const int64 offset = (source - endpoint->trb_addr) / sizeof(xhci_trb); 2695 const int32 type = TRB_3_TYPE_GET(endpoint->trbs[offset].flags); 2696 if (type == TRB_TYPE_EVENT_DATA || type == TRB_TYPE_LINK) 2697 source = B_LENDIAN_TO_HOST_INT64(endpoint->trbs[offset].address); 2698 } 2699 2700 for (xhci_td *td = endpoint->td_head; td != NULL; td = td->next) { 2701 int64 offset = (source - td->trb_addr) / sizeof(xhci_trb); 2702 if (offset < 0 || offset >= td->trb_count) 2703 continue; 2704 2705 TRACE("HandleTransferComplete td %p trb %" B_PRId64 " found\n", 2706 td, offset); 2707 2708 if (td->transfer != NULL && td->transfer->IsochronousData() != NULL) { 2709 usb_isochronous_data* isochronousData = td->transfer->IsochronousData(); 2710 usb_iso_packet_descriptor& descriptor = isochronousData->packet_descriptors[offset]; 2711 if (transferred < 0) 2712 transferred = (TRB_2_BYTES_GET(td->trbs[offset].status) - remainder); 2713 descriptor.actual_length = transferred; 2714 descriptor.status = xhci_error_status(completionCode, 2715 (td->transfer->TransferPipe()->Direction() != Pipe::Out)); 2716 2717 // Don't double-report completion status. 2718 completionCode = COMP_SUCCESS; 2719 2720 if (offset != (td->trb_used - 1)) { 2721 // We'll be sent here again. 2722 return; 2723 } 2724 2725 // Compute the real transferred length. 2726 transferred = 0; 2727 for (int32 i = 0; i < offset; i++) { 2728 usb_iso_packet_descriptor& descriptor = isochronousData->packet_descriptors[i]; 2729 if (descriptor.status == B_NO_INIT) { 2730 // Assume success. 2731 descriptor.actual_length = descriptor.request_length; 2732 descriptor.status = B_OK; 2733 } 2734 transferred += descriptor.actual_length; 2735 } 2736 2737 // Report the endpoint status (if any.) 2738 if (endpoint->status != 0) { 2739 completionCode = endpoint->status; 2740 endpoint->status = 0; 2741 } 2742 } else if (completionCode == COMP_STOPPED_LENGTH_INVALID) { 2743 // To determine transferred length, sum up the lengths of all TRBs 2744 // prior to the referenced one. (XHCI 1.2 § 4.6.9 p136.) 2745 transferred = 0; 2746 for (int32 i = 0; i < offset; i++) 2747 transferred += TRB_2_BYTES_GET(td->trbs[i].status); 2748 } 2749 2750 // The TRB at offset trb_used will be the link TRB, which we do not 2751 // care about (and should not generate an interrupt at all.) We really 2752 // care about the properly last TRB, at index "count - 1", which the 2753 // Event Data TRB that _LinkDescriptorForPipe creates points to. 2754 // 2755 // But if we have an unsuccessful completion code, the transfer 2756 // likely failed midway; so just accept it anyway. 2757 if (offset == (td->trb_used - 1) || completionCode != COMP_SUCCESS) { 2758 _UnlinkDescriptorForPipe(td, endpoint); 2759 endpointLocker.Unlock(); 2760 2761 td->trb_completion_code = completionCode; 2762 td->td_transferred = transferred; 2763 td->trb_left = remainder; 2764 2765 // add descriptor to finished list 2766 if (mutex_trylock(&fFinishedLock) != B_OK) 2767 mutex_lock(&fFinishedLock); 2768 td->next = fFinishedHead; 2769 fFinishedHead = td; 2770 mutex_unlock(&fFinishedLock); 2771 2772 release_sem_etc(fFinishTransfersSem, 1, B_DO_NOT_RESCHEDULE); 2773 TRACE("HandleTransferComplete td %p done\n", td); 2774 } else { 2775 TRACE_ERROR("successful TRB 0x%" B_PRIxPHYSADDR " was found, but it wasn't " 2776 "the last in the TD!\n", source); 2777 } 2778 return; 2779 } 2780 TRACE_ERROR("TRB 0x%" B_PRIxPHYSADDR " was not found in the endpoint!\n", source); 2781 } 2782 2783 2784 void 2785 XHCI::DumpRing(xhci_trb *trbs, uint32 size) 2786 { 2787 if (!Lock()) { 2788 TRACE("Unable to get lock!\n"); 2789 return; 2790 } 2791 2792 for (uint32 i = 0; i < size; i++) { 2793 TRACE("command[%" B_PRId32 "] = %" B_PRId32 " (0x%016" B_PRIx64 "," 2794 " 0x%08" B_PRIx32 ", 0x%08" B_PRIx32 ")\n", i, 2795 TRB_3_TYPE_GET(B_LENDIAN_TO_HOST_INT32(trbs[i].flags)), 2796 trbs[i].address, trbs[i].status, trbs[i].flags); 2797 } 2798 2799 Unlock(); 2800 } 2801 2802 2803 status_t 2804 XHCI::DoCommand(xhci_trb* trb) 2805 { 2806 if (!Lock()) { 2807 TRACE("Unable to get lock!\n"); 2808 return B_ERROR; 2809 } 2810 2811 QueueCommand(trb); 2812 Ring(0, 0); 2813 2814 // Begin with a 50ms timeout. 2815 if (acquire_sem_etc(fCmdCompSem, 1, B_RELATIVE_TIMEOUT, 50 * 1000) != B_OK) { 2816 // We've hit the timeout. In some error cases, interrupts are not 2817 // generated; so here we force the event ring to be polled once. 2818 release_sem(fEventSem); 2819 2820 // Now try again, this time with a 750ms timeout. 2821 if (acquire_sem_etc(fCmdCompSem, 1, B_RELATIVE_TIMEOUT, 2822 750 * 1000) != B_OK) { 2823 TRACE("Unable to obtain fCmdCompSem!\n"); 2824 fCmdAddr = 0; 2825 Unlock(); 2826 return B_TIMED_OUT; 2827 } 2828 } 2829 2830 // eat up sems that have been released by multiple interrupts 2831 int32 semCount = 0; 2832 get_sem_count(fCmdCompSem, &semCount); 2833 if (semCount > 0) 2834 acquire_sem_etc(fCmdCompSem, semCount, B_RELATIVE_TIMEOUT, 0); 2835 2836 status_t status = B_OK; 2837 uint32 completionCode = TRB_2_COMP_CODE_GET(fCmdResult[0]); 2838 TRACE("command complete\n"); 2839 if (completionCode != COMP_SUCCESS) { 2840 TRACE_ERROR("unsuccessful command %" B_PRId32 ", error %s (%" B_PRId32 ")\n", 2841 TRB_3_TYPE_GET(trb->flags), xhci_error_string(completionCode), 2842 completionCode); 2843 status = B_IO_ERROR; 2844 } 2845 2846 trb->status = fCmdResult[0]; 2847 trb->flags = fCmdResult[1]; 2848 2849 fCmdAddr = 0; 2850 Unlock(); 2851 return status; 2852 } 2853 2854 2855 status_t 2856 XHCI::Noop() 2857 { 2858 TRACE("Issue No-Op\n"); 2859 xhci_trb trb; 2860 trb.address = 0; 2861 trb.status = 0; 2862 trb.flags = TRB_3_TYPE(TRB_TYPE_CMD_NOOP); 2863 2864 return DoCommand(&trb); 2865 } 2866 2867 2868 status_t 2869 XHCI::EnableSlot(uint8* slot) 2870 { 2871 TRACE("Enable Slot\n"); 2872 xhci_trb trb; 2873 trb.address = 0; 2874 trb.status = 0; 2875 trb.flags = TRB_3_TYPE(TRB_TYPE_ENABLE_SLOT); 2876 2877 status_t status = DoCommand(&trb); 2878 if (status != B_OK) 2879 return status; 2880 2881 *slot = TRB_3_SLOT_GET(trb.flags); 2882 return *slot != 0 ? B_OK : B_BAD_VALUE; 2883 } 2884 2885 2886 status_t 2887 XHCI::DisableSlot(uint8 slot) 2888 { 2889 TRACE("Disable Slot\n"); 2890 xhci_trb trb; 2891 trb.address = 0; 2892 trb.status = 0; 2893 trb.flags = TRB_3_TYPE(TRB_TYPE_DISABLE_SLOT) | TRB_3_SLOT(slot); 2894 2895 return DoCommand(&trb); 2896 } 2897 2898 2899 status_t 2900 XHCI::SetAddress(uint64 inputContext, bool bsr, uint8 slot) 2901 { 2902 TRACE("Set Address\n"); 2903 xhci_trb trb; 2904 trb.address = inputContext; 2905 trb.status = 0; 2906 trb.flags = TRB_3_TYPE(TRB_TYPE_ADDRESS_DEVICE) | TRB_3_SLOT(slot); 2907 2908 if (bsr) 2909 trb.flags |= TRB_3_BSR_BIT; 2910 2911 return DoCommand(&trb); 2912 } 2913 2914 2915 status_t 2916 XHCI::ConfigureEndpoint(uint64 inputContext, bool deconfigure, uint8 slot) 2917 { 2918 TRACE("Configure Endpoint\n"); 2919 xhci_trb trb; 2920 trb.address = inputContext; 2921 trb.status = 0; 2922 trb.flags = TRB_3_TYPE(TRB_TYPE_CONFIGURE_ENDPOINT) | TRB_3_SLOT(slot); 2923 2924 if (deconfigure) 2925 trb.flags |= TRB_3_DCEP_BIT; 2926 2927 return DoCommand(&trb); 2928 } 2929 2930 2931 status_t 2932 XHCI::EvaluateContext(uint64 inputContext, uint8 slot) 2933 { 2934 TRACE("Evaluate Context\n"); 2935 xhci_trb trb; 2936 trb.address = inputContext; 2937 trb.status = 0; 2938 trb.flags = TRB_3_TYPE(TRB_TYPE_EVALUATE_CONTEXT) | TRB_3_SLOT(slot); 2939 2940 return DoCommand(&trb); 2941 } 2942 2943 2944 status_t 2945 XHCI::ResetEndpoint(bool preserve, xhci_endpoint* endpoint) 2946 { 2947 TRACE("Reset Endpoint\n"); 2948 2949 switch (_GetEndpointState(endpoint)) { 2950 case ENDPOINT_STATE_STOPPED: 2951 TRACE("Reset Endpoint: already stopped"); 2952 return B_OK; 2953 case ENDPOINT_STATE_HALTED: 2954 TRACE("Reset Endpoint: warning, weird state!"); 2955 default: 2956 break; 2957 } 2958 2959 xhci_trb trb; 2960 trb.address = 0; 2961 trb.status = 0; 2962 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_ENDPOINT) 2963 | TRB_3_SLOT(endpoint->device->slot) | TRB_3_ENDPOINT(endpoint->id + 1); 2964 if (preserve) 2965 trb.flags |= TRB_3_PRSV_BIT; 2966 2967 return DoCommand(&trb); 2968 } 2969 2970 2971 status_t 2972 XHCI::StopEndpoint(bool suspend, xhci_endpoint* endpoint) 2973 { 2974 TRACE("Stop Endpoint\n"); 2975 2976 switch (_GetEndpointState(endpoint)) { 2977 case ENDPOINT_STATE_HALTED: 2978 TRACE("Stop Endpoint: error, halted"); 2979 return B_DEV_STALLED; 2980 case ENDPOINT_STATE_STOPPED: 2981 TRACE("Stop Endpoint: already stopped"); 2982 return B_OK; 2983 default: 2984 break; 2985 } 2986 2987 xhci_trb trb; 2988 trb.address = 0; 2989 trb.status = 0; 2990 trb.flags = TRB_3_TYPE(TRB_TYPE_STOP_ENDPOINT) 2991 | TRB_3_SLOT(endpoint->device->slot) | TRB_3_ENDPOINT(endpoint->id + 1); 2992 if (suspend) 2993 trb.flags |= TRB_3_SUSPEND_ENDPOINT_BIT; 2994 2995 return DoCommand(&trb); 2996 } 2997 2998 2999 status_t 3000 XHCI::SetTRDequeue(uint64 dequeue, uint16 stream, uint8 endpoint, uint8 slot) 3001 { 3002 TRACE("Set TR Dequeue\n"); 3003 xhci_trb trb; 3004 trb.address = dequeue | ENDPOINT_2_DCS_BIT; 3005 // The DCS bit is copied from the address field as in ConfigureEndpoint. 3006 // (XHCI 1.2 § 4.6.10 p142.) 3007 trb.status = TRB_2_STREAM(stream); 3008 trb.flags = TRB_3_TYPE(TRB_TYPE_SET_TR_DEQUEUE) 3009 | TRB_3_SLOT(slot) | TRB_3_ENDPOINT(endpoint); 3010 3011 return DoCommand(&trb); 3012 } 3013 3014 3015 status_t 3016 XHCI::ResetDevice(uint8 slot) 3017 { 3018 TRACE("Reset Device\n"); 3019 xhci_trb trb; 3020 trb.address = 0; 3021 trb.status = 0; 3022 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_DEVICE) | TRB_3_SLOT(slot); 3023 3024 return DoCommand(&trb); 3025 } 3026 3027 3028 int32 3029 XHCI::EventThread(void* data) 3030 { 3031 ((XHCI *)data)->CompleteEvents(); 3032 return B_OK; 3033 } 3034 3035 3036 void 3037 XHCI::CompleteEvents() 3038 { 3039 while (!fStopThreads) { 3040 if (acquire_sem(fEventSem) < B_OK) 3041 continue; 3042 3043 // eat up sems that have been released by multiple interrupts 3044 int32 semCount = 0; 3045 get_sem_count(fEventSem, &semCount); 3046 if (semCount > 0) 3047 acquire_sem_etc(fEventSem, semCount, B_RELATIVE_TIMEOUT, 0); 3048 3049 ProcessEvents(); 3050 } 3051 } 3052 3053 3054 void 3055 XHCI::ProcessEvents() 3056 { 3057 // Use mutex_trylock first, in case we are in KDL. 3058 MutexLocker locker(fEventLock, mutex_trylock(&fEventLock) == B_OK); 3059 if (!locker.IsLocked()) { 3060 // We failed to get the lock. This really should not happen. 3061 TRACE_ERROR("failed to acquire event lock!\n"); 3062 return; 3063 } 3064 3065 uint16 i = fEventIdx; 3066 uint8 j = fEventCcs; 3067 uint8 t = 2; 3068 3069 while (1) { 3070 uint32 temp = B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags); 3071 uint8 event = TRB_3_TYPE_GET(temp); 3072 TRACE("event[%u] = %u (0x%016" B_PRIx64 " 0x%08" B_PRIx32 " 0x%08" 3073 B_PRIx32 ")\n", i, event, fEventRing[i].address, 3074 fEventRing[i].status, B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags)); 3075 uint8 k = (temp & TRB_3_CYCLE_BIT) ? 1 : 0; 3076 if (j != k) 3077 break; 3078 3079 switch (event) { 3080 case TRB_TYPE_COMMAND_COMPLETION: 3081 HandleCmdComplete(&fEventRing[i]); 3082 break; 3083 case TRB_TYPE_TRANSFER: 3084 HandleTransferComplete(&fEventRing[i]); 3085 break; 3086 case TRB_TYPE_PORT_STATUS_CHANGE: 3087 TRACE("port change detected\n"); 3088 break; 3089 default: 3090 TRACE_ERROR("Unhandled event = %u\n", event); 3091 break; 3092 } 3093 3094 i++; 3095 if (i == XHCI_MAX_EVENTS) { 3096 i = 0; 3097 j ^= 1; 3098 if (!--t) 3099 break; 3100 } 3101 } 3102 3103 fEventIdx = i; 3104 fEventCcs = j; 3105 3106 uint64 addr = fErst->rs_addr + i * sizeof(xhci_trb); 3107 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)addr | ERDP_BUSY); 3108 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(addr >> 32)); 3109 } 3110 3111 3112 int32 3113 XHCI::FinishThread(void* data) 3114 { 3115 ((XHCI *)data)->FinishTransfers(); 3116 return B_OK; 3117 } 3118 3119 3120 void 3121 XHCI::FinishTransfers() 3122 { 3123 while (!fStopThreads) { 3124 if (acquire_sem(fFinishTransfersSem) < B_OK) 3125 continue; 3126 3127 // eat up sems that have been released by multiple interrupts 3128 int32 semCount = 0; 3129 get_sem_count(fFinishTransfersSem, &semCount); 3130 if (semCount > 0) 3131 acquire_sem_etc(fFinishTransfersSem, semCount, B_RELATIVE_TIMEOUT, 0); 3132 3133 mutex_lock(&fFinishedLock); 3134 TRACE("finishing transfers\n"); 3135 while (fFinishedHead != NULL) { 3136 xhci_td* td = fFinishedHead; 3137 fFinishedHead = td->next; 3138 td->next = NULL; 3139 mutex_unlock(&fFinishedLock); 3140 3141 TRACE("finishing transfer td %p\n", td); 3142 3143 Transfer* transfer = td->transfer; 3144 if (transfer == NULL) { 3145 // No transfer? Quick way out. 3146 FreeDescriptor(td); 3147 mutex_lock(&fFinishedLock); 3148 continue; 3149 } 3150 3151 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 3152 3153 const uint8 completionCode = td->trb_completion_code; 3154 status_t callbackStatus = xhci_error_status(completionCode, directionIn); 3155 3156 size_t actualLength = transfer->FragmentLength(); 3157 if (completionCode != COMP_SUCCESS) { 3158 actualLength = td->td_transferred; 3159 if (td->td_transferred == -1) 3160 actualLength = transfer->FragmentLength() - td->trb_left; 3161 TRACE("transfer not successful, actualLength=%" B_PRIuSIZE "\n", 3162 actualLength); 3163 } 3164 3165 if (directionIn && actualLength > 0) { 3166 TRACE("copying in iov count %ld\n", transfer->VectorCount()); 3167 status_t status = transfer->PrepareKernelAccess(); 3168 if (status == B_OK) { 3169 ReadDescriptor(td, transfer->Vector(), 3170 transfer->VectorCount(), transfer->IsPhysical()); 3171 } else { 3172 callbackStatus = status; 3173 } 3174 } 3175 3176 FreeDescriptor(td); 3177 3178 // this transfer may still have data left 3179 bool finished = true; 3180 transfer->AdvanceByFragment(actualLength); 3181 if (completionCode == COMP_SUCCESS 3182 && transfer->FragmentLength() > 0) { 3183 TRACE("still %" B_PRIuSIZE " bytes left on transfer\n", 3184 transfer->FragmentLength()); 3185 callbackStatus = SubmitTransfer(transfer); 3186 finished = (callbackStatus != B_OK); 3187 } 3188 if (finished) { 3189 // The actualLength was already handled in AdvanceByFragment. 3190 transfer->Finished(callbackStatus, 0); 3191 delete transfer; 3192 } 3193 3194 mutex_lock(&fFinishedLock); 3195 } 3196 mutex_unlock(&fFinishedLock); 3197 } 3198 } 3199 3200 3201 inline void 3202 XHCI::WriteOpReg(uint32 reg, uint32 value) 3203 { 3204 *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg) = value; 3205 } 3206 3207 3208 inline uint32 3209 XHCI::ReadOpReg(uint32 reg) 3210 { 3211 return *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg); 3212 } 3213 3214 3215 inline status_t 3216 XHCI::WaitOpBits(uint32 reg, uint32 mask, uint32 expected) 3217 { 3218 int loops = 0; 3219 uint32 value = ReadOpReg(reg); 3220 while ((value & mask) != expected) { 3221 snooze(1000); 3222 value = ReadOpReg(reg); 3223 if (loops == 100) { 3224 TRACE("delay waiting on reg 0x%" B_PRIX32 " match 0x%" B_PRIX32 3225 " (0x%" B_PRIX32 ")\n", reg, expected, mask); 3226 } else if (loops > 250) { 3227 TRACE_ERROR("timeout waiting on reg 0x%" B_PRIX32 3228 " match 0x%" B_PRIX32 " (0x%" B_PRIX32 ")\n", reg, expected, 3229 mask); 3230 return B_ERROR; 3231 } 3232 loops++; 3233 } 3234 return B_OK; 3235 } 3236 3237 3238 inline uint32 3239 XHCI::ReadCapReg32(uint32 reg) 3240 { 3241 return *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg); 3242 } 3243 3244 3245 inline void 3246 XHCI::WriteCapReg32(uint32 reg, uint32 value) 3247 { 3248 *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg) = value; 3249 } 3250 3251 3252 inline uint32 3253 XHCI::ReadRunReg32(uint32 reg) 3254 { 3255 return *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg); 3256 } 3257 3258 3259 inline void 3260 XHCI::WriteRunReg32(uint32 reg, uint32 value) 3261 { 3262 *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg) = value; 3263 } 3264 3265 3266 inline uint32 3267 XHCI::ReadDoorReg32(uint32 reg) 3268 { 3269 return *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg); 3270 } 3271 3272 3273 inline void 3274 XHCI::WriteDoorReg32(uint32 reg, uint32 value) 3275 { 3276 *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg) = value; 3277 } 3278 3279 3280 inline addr_t 3281 XHCI::_OffsetContextAddr(addr_t p) 3282 { 3283 if (fContextSizeShift == 1) { 3284 // each structure is page aligned, each pointer is 32 bits aligned 3285 uint32 offset = p & ((B_PAGE_SIZE - 1) & ~31U); 3286 p += offset; 3287 } 3288 return p; 3289 } 3290 3291 inline uint32 3292 XHCI::_ReadContext(uint32* p) 3293 { 3294 p = (uint32*)_OffsetContextAddr((addr_t)p); 3295 return *p; 3296 } 3297 3298 3299 inline void 3300 XHCI::_WriteContext(uint32* p, uint32 value) 3301 { 3302 p = (uint32*)_OffsetContextAddr((addr_t)p); 3303 *p = value; 3304 } 3305 3306 3307 inline uint64 3308 XHCI::_ReadContext(uint64* p) 3309 { 3310 p = (uint64*)_OffsetContextAddr((addr_t)p); 3311 return *p; 3312 } 3313 3314 3315 inline void 3316 XHCI::_WriteContext(uint64* p, uint64 value) 3317 { 3318 p = (uint64*)_OffsetContextAddr((addr_t)p); 3319 *p = value; 3320 } 3321