1 /* 2 * Copyright 2011-2021, Haiku, Inc. All rights reserved. 3 * Distributed under the terms of the MIT License. 4 * 5 * Authors: 6 * Augustin Cavalier <waddlesplash> 7 * Jian Chiang <j.jian.chiang@gmail.com> 8 * Jérôme Duval <jerome.duval@gmail.com> 9 * Akshay Jaggi <akshay1994.leo@gmail.com> 10 * Michael Lotz <mmlr@mlotz.ch> 11 * Alexander von Gluck <kallisti5@unixzen.com> 12 */ 13 14 15 #include <stdio.h> 16 17 #include <bus/PCI.h> 18 #include <USB3.h> 19 #include <KernelExport.h> 20 21 #include <ByteOrder.h> 22 #include <util/AutoLock.h> 23 24 #include "xhci.h" 25 26 27 #define CALLED(x...) TRACE_MODULE("CALLED %s\n", __PRETTY_FUNCTION__) 28 29 30 #define USB_MODULE_NAME "xhci" 31 32 device_manager_info* gDeviceManager; 33 static usb_for_controller_interface* gUSB; 34 35 36 #define XHCI_PCI_DEVICE_MODULE_NAME "busses/usb/xhci/pci/driver_v1" 37 #define XHCI_PCI_USB_BUS_MODULE_NAME "busses/usb/xhci/device_v1" 38 39 40 typedef struct { 41 XHCI* xhci; 42 pci_device_module_info* pci; 43 pci_device* device; 44 45 pci_info pciinfo; 46 47 device_node* node; 48 device_node* driver_node; 49 } xhci_pci_sim_info; 50 51 52 // #pragma mark - 53 54 55 static status_t 56 init_bus(device_node* node, void** bus_cookie) 57 { 58 CALLED(); 59 60 driver_module_info* driver; 61 xhci_pci_sim_info* bus; 62 device_node* parent = gDeviceManager->get_parent_node(node); 63 gDeviceManager->get_driver(parent, &driver, (void**)&bus); 64 gDeviceManager->put_node(parent); 65 66 Stack *stack; 67 if (gUSB->get_stack((void**)&stack) != B_OK) 68 return B_ERROR; 69 70 XHCI *xhci = new(std::nothrow) XHCI(&bus->pciinfo, bus->pci, bus->device, stack, node); 71 if (xhci == NULL) { 72 return B_NO_MEMORY; 73 } 74 75 if (xhci->InitCheck() < B_OK) { 76 TRACE_MODULE_ERROR("bus failed init check\n"); 77 delete xhci; 78 return B_ERROR; 79 } 80 81 if (xhci->Start() != B_OK) { 82 delete xhci; 83 return B_ERROR; 84 } 85 86 *bus_cookie = xhci; 87 88 return B_OK; 89 } 90 91 92 static void 93 uninit_bus(void* bus_cookie) 94 { 95 CALLED(); 96 XHCI* xhci = (XHCI*)bus_cookie; 97 delete xhci; 98 } 99 100 101 static status_t 102 register_child_devices(void* cookie) 103 { 104 CALLED(); 105 xhci_pci_sim_info* bus = (xhci_pci_sim_info*)cookie; 106 device_node* node = bus->driver_node; 107 108 char prettyName[25]; 109 sprintf(prettyName, "XHCI Controller %" B_PRIu16, 0); 110 111 device_attr attrs[] = { 112 // properties of this controller for the usb bus manager 113 { B_DEVICE_PRETTY_NAME, B_STRING_TYPE, 114 { .string = prettyName }}, 115 { B_DEVICE_FIXED_CHILD, B_STRING_TYPE, 116 { .string = USB_FOR_CONTROLLER_MODULE_NAME }}, 117 118 // private data to identify the device 119 { NULL } 120 }; 121 122 return gDeviceManager->register_node(node, XHCI_PCI_USB_BUS_MODULE_NAME, 123 attrs, NULL, NULL); 124 } 125 126 127 static status_t 128 init_device(device_node* node, void** device_cookie) 129 { 130 CALLED(); 131 xhci_pci_sim_info* bus = (xhci_pci_sim_info*)calloc(1, 132 sizeof(xhci_pci_sim_info)); 133 if (bus == NULL) 134 return B_NO_MEMORY; 135 136 pci_device_module_info* pci; 137 pci_device* device; 138 { 139 device_node* pciParent = gDeviceManager->get_parent_node(node); 140 gDeviceManager->get_driver(pciParent, (driver_module_info**)&pci, 141 (void**)&device); 142 gDeviceManager->put_node(pciParent); 143 } 144 145 bus->pci = pci; 146 bus->device = device; 147 bus->driver_node = node; 148 149 pci_info *pciInfo = &bus->pciinfo; 150 pci->get_pci_info(device, pciInfo); 151 152 *device_cookie = bus; 153 return B_OK; 154 } 155 156 157 static void 158 uninit_device(void* device_cookie) 159 { 160 CALLED(); 161 xhci_pci_sim_info* bus = (xhci_pci_sim_info*)device_cookie; 162 free(bus); 163 } 164 165 166 static status_t 167 register_device(device_node* parent) 168 { 169 CALLED(); 170 device_attr attrs[] = { 171 {B_DEVICE_PRETTY_NAME, B_STRING_TYPE, {.string = "XHCI PCI"}}, 172 {} 173 }; 174 175 return gDeviceManager->register_node(parent, 176 XHCI_PCI_DEVICE_MODULE_NAME, attrs, NULL, NULL); 177 } 178 179 180 static float 181 supports_device(device_node* parent) 182 { 183 CALLED(); 184 const char* bus; 185 uint16 type, subType, api; 186 187 // make sure parent is a XHCI PCI device node 188 if (gDeviceManager->get_attr_string(parent, B_DEVICE_BUS, &bus, false) 189 < B_OK) { 190 return -1; 191 } 192 193 if (strcmp(bus, "pci") != 0) 194 return 0.0f; 195 196 if (gDeviceManager->get_attr_uint16(parent, B_DEVICE_SUB_TYPE, &subType, 197 false) < B_OK 198 || gDeviceManager->get_attr_uint16(parent, B_DEVICE_TYPE, &type, 199 false) < B_OK 200 || gDeviceManager->get_attr_uint16(parent, B_DEVICE_INTERFACE, &api, 201 false) < B_OK) { 202 TRACE_MODULE("Could not find type/subtype/interface attributes\n"); 203 return -1; 204 } 205 206 if (type == PCI_serial_bus && subType == PCI_usb && api == PCI_usb_xhci) { 207 pci_device_module_info* pci; 208 pci_device* device; 209 gDeviceManager->get_driver(parent, (driver_module_info**)&pci, 210 (void**)&device); 211 TRACE_MODULE("XHCI Device found!\n"); 212 213 return 0.8f; 214 } 215 216 return 0.0f; 217 } 218 219 220 static const char* 221 xhci_error_string(uint32 error) 222 { 223 switch (error) { 224 case COMP_INVALID: return "Invalid"; 225 case COMP_SUCCESS: return "Success"; 226 case COMP_DATA_BUFFER: return "Data buffer"; 227 case COMP_BABBLE: return "Babble detected"; 228 case COMP_USB_TRANSACTION: return "USB transaction"; 229 case COMP_TRB: return "TRB"; 230 case COMP_STALL: return "Stall"; 231 case COMP_RESOURCE: return "Resource"; 232 case COMP_BANDWIDTH: return "Bandwidth"; 233 case COMP_NO_SLOTS: return "No slots"; 234 case COMP_INVALID_STREAM: return "Invalid stream"; 235 case COMP_SLOT_NOT_ENABLED: return "Slot not enabled"; 236 case COMP_ENDPOINT_NOT_ENABLED: return "Endpoint not enabled"; 237 case COMP_SHORT_PACKET: return "Short packet"; 238 case COMP_RING_UNDERRUN: return "Ring underrun"; 239 case COMP_RING_OVERRUN: return "Ring overrun"; 240 case COMP_VF_RING_FULL: return "VF Event Ring Full"; 241 case COMP_PARAMETER: return "Parameter"; 242 case COMP_BANDWIDTH_OVERRUN: return "Bandwidth overrun"; 243 case COMP_CONTEXT_STATE: return "Context state"; 244 case COMP_NO_PING_RESPONSE: return "No ping response"; 245 case COMP_EVENT_RING_FULL: return "Event ring full"; 246 case COMP_INCOMPATIBLE_DEVICE: return "Incompatible device"; 247 case COMP_MISSED_SERVICE: return "Missed service"; 248 case COMP_COMMAND_RING_STOPPED: return "Command ring stopped"; 249 case COMP_COMMAND_ABORTED: return "Command aborted"; 250 case COMP_STOPPED: return "Stopped"; 251 case COMP_STOPPED_LENGTH_INVALID: return "Stopped (length invalid)"; 252 case COMP_MAX_EXIT_LATENCY: return "Max exit latency too large"; 253 case COMP_ISOC_OVERRUN: return "Isoch buffer overrun"; 254 case COMP_EVENT_LOST: return "Event lost"; 255 case COMP_UNDEFINED: return "Undefined"; 256 case COMP_INVALID_STREAM_ID: return "Invalid stream ID"; 257 case COMP_SECONDARY_BANDWIDTH: return "Secondary bandwidth"; 258 case COMP_SPLIT_TRANSACTION: return "Split transaction"; 259 260 default: return "Undefined"; 261 } 262 } 263 264 265 static status_t 266 xhci_error_status(uint32 error, bool directionIn) 267 { 268 switch (error) { 269 case COMP_SHORT_PACKET: 270 case COMP_SUCCESS: 271 return B_OK; 272 case COMP_DATA_BUFFER: 273 return directionIn ? B_DEV_WRITE_ERROR : B_DEV_READ_ERROR; 274 case COMP_BABBLE: 275 return directionIn ? B_DEV_DATA_OVERRUN : B_DEV_DATA_UNDERRUN; 276 case COMP_RING_UNDERRUN: 277 return B_DEV_FIFO_UNDERRUN; 278 case COMP_RING_OVERRUN: 279 return B_DEV_FIFO_OVERRUN; 280 case COMP_MISSED_SERVICE: 281 return B_DEV_TOO_LATE; 282 case COMP_USB_TRANSACTION: 283 return B_DEV_CRC_ERROR; 284 case COMP_STALL: 285 return B_DEV_STALLED; 286 default: 287 return B_DEV_STALLED; 288 } 289 } 290 291 292 module_dependency module_dependencies[] = { 293 { USB_FOR_CONTROLLER_MODULE_NAME, (module_info**)&gUSB }, 294 { B_DEVICE_MANAGER_MODULE_NAME, (module_info**)&gDeviceManager }, 295 {} 296 }; 297 298 299 static usb_bus_interface gXHCIPCIDeviceModule = { 300 { 301 { 302 XHCI_PCI_USB_BUS_MODULE_NAME, 303 0, 304 NULL 305 }, 306 NULL, // supports device 307 NULL, // register device 308 init_bus, 309 uninit_bus, 310 NULL, // register child devices 311 NULL, // rescan 312 NULL, // device removed 313 }, 314 }; 315 316 // Root device that binds to the PCI bus. It will register an usb_bus_interface 317 // node for each device. 318 static driver_module_info sXHCIDevice = { 319 { 320 XHCI_PCI_DEVICE_MODULE_NAME, 321 0, 322 NULL 323 }, 324 supports_device, 325 register_device, 326 init_device, 327 uninit_device, 328 register_child_devices, 329 NULL, // rescan 330 NULL, // device removed 331 }; 332 333 module_info* modules[] = { 334 (module_info* )&sXHCIDevice, 335 (module_info* )&gXHCIPCIDeviceModule, 336 NULL 337 }; 338 339 340 XHCI::XHCI(pci_info *info, pci_device_module_info* pci, pci_device* device, Stack *stack, 341 device_node* node) 342 : BusManager(stack, node), 343 fRegisterArea(-1), 344 fRegisters(NULL), 345 fPCIInfo(info), 346 fPci(pci), 347 fDevice(device), 348 fStack(stack), 349 fIRQ(0), 350 fUseMSI(false), 351 fErstArea(-1), 352 fDcbaArea(-1), 353 fCmdCompSem(-1), 354 fStopThreads(false), 355 fRootHub(NULL), 356 fPortCount(0), 357 fSlotCount(0), 358 fScratchpadCount(0), 359 fContextSizeShift(0), 360 fFinishedHead(NULL), 361 fFinishTransfersSem(-1), 362 fFinishThread(-1), 363 fEventSem(-1), 364 fEventThread(-1), 365 fEventIdx(0), 366 fCmdIdx(0), 367 fEventCcs(1), 368 fCmdCcs(1) 369 { 370 B_INITIALIZE_SPINLOCK(&fSpinlock); 371 mutex_init(&fFinishedLock, "XHCI finished transfers"); 372 mutex_init(&fEventLock, "XHCI event handler"); 373 374 if (BusManager::InitCheck() < B_OK) { 375 TRACE_ERROR("bus manager failed to init\n"); 376 return; 377 } 378 379 TRACE("constructing new XHCI host controller driver\n"); 380 fInitOK = false; 381 382 // enable busmaster and memory mapped access 383 uint16 command = fPci->read_pci_config(fDevice, PCI_command, 2); 384 command &= ~(PCI_command_io | PCI_command_int_disable); 385 command |= PCI_command_master | PCI_command_memory; 386 387 fPci->write_pci_config(fDevice, PCI_command, 2, command); 388 389 // map the registers (low + high for 64-bit when requested) 390 phys_addr_t physicalAddress = fPCIInfo->u.h0.base_registers[0]; 391 if ((fPCIInfo->u.h0.base_register_flags[0] & PCI_address_type) 392 == PCI_address_type_64) { 393 physicalAddress |= (uint64)fPCIInfo->u.h0.base_registers[1] << 32; 394 } 395 396 size_t mapSize = fPCIInfo->u.h0.base_register_sizes[0]; 397 398 TRACE("map registers %08" B_PRIxPHYSADDR ", size: %" B_PRIuSIZE "\n", 399 physicalAddress, mapSize); 400 401 fRegisterArea = map_physical_memory("XHCI memory mapped registers", 402 physicalAddress, mapSize, B_ANY_KERNEL_BLOCK_ADDRESS, 403 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 404 (void **)&fRegisters); 405 if (fRegisterArea < B_OK) { 406 TRACE_ERROR("failed to map register memory\n"); 407 return; 408 } 409 410 // determine the register offsets 411 fCapabilityRegisterOffset = 0; 412 fOperationalRegisterOffset = HCI_CAPLENGTH(ReadCapReg32(XHCI_HCI_CAPLENGTH)); 413 fRuntimeRegisterOffset = ReadCapReg32(XHCI_RTSOFF) & ~0x1F; 414 fDoorbellRegisterOffset = ReadCapReg32(XHCI_DBOFF) & ~0x3; 415 416 TRACE("mapped registers: %p\n", fRegisters); 417 TRACE("operational register offset: %" B_PRId32 "\n", fOperationalRegisterOffset); 418 TRACE("runtime register offset: %" B_PRId32 "\n", fRuntimeRegisterOffset); 419 TRACE("doorbell register offset: %" B_PRId32 "\n", fDoorbellRegisterOffset); 420 421 int32 interfaceVersion = HCI_VERSION(ReadCapReg32(XHCI_HCI_VERSION)); 422 if (interfaceVersion < 0x0090 || interfaceVersion > 0x0120) { 423 TRACE_ERROR("unsupported interface version: 0x%04" B_PRIx32 "\n", 424 interfaceVersion); 425 return; 426 } 427 TRACE_ALWAYS("interface version: 0x%04" B_PRIx32 "\n", interfaceVersion); 428 429 TRACE_ALWAYS("structural parameters: 1:0x%08" B_PRIx32 " 2:0x%08" 430 B_PRIx32 " 3:0x%08" B_PRIx32 "\n", ReadCapReg32(XHCI_HCSPARAMS1), 431 ReadCapReg32(XHCI_HCSPARAMS2), ReadCapReg32(XHCI_HCSPARAMS3)); 432 433 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 434 if (cparams == 0xffffffff) 435 return; 436 TRACE_ALWAYS("capability parameters: 0x%08" B_PRIx32 "\n", cparams); 437 438 // if 64 bytes context structures, then 1 439 fContextSizeShift = HCC_CSZ(cparams); 440 441 // Assume ownership of the controller from the BIOS. 442 uint32 eec = 0xffffffff; 443 uint32 eecp = HCS0_XECP(cparams) << 2; 444 for (; eecp != 0 && XECP_NEXT(eec); eecp += XECP_NEXT(eec) << 2) { 445 TRACE("eecp register: 0x%08" B_PRIx32 "\n", eecp); 446 447 eec = ReadCapReg32(eecp); 448 if (XECP_ID(eec) != XHCI_LEGSUP_CAPID) 449 continue; 450 451 if (eec & XHCI_LEGSUP_BIOSOWNED) { 452 TRACE_ALWAYS("the host controller is bios owned, claiming" 453 " ownership\n"); 454 WriteCapReg32(eecp, eec | XHCI_LEGSUP_OSOWNED); 455 456 for (int32 i = 0; i < 20; i++) { 457 eec = ReadCapReg32(eecp); 458 459 if ((eec & XHCI_LEGSUP_BIOSOWNED) == 0) 460 break; 461 462 TRACE_ALWAYS("controller is still bios owned, waiting\n"); 463 snooze(50000); 464 } 465 466 if (eec & XHCI_LEGSUP_BIOSOWNED) { 467 TRACE_ERROR("bios won't give up control over the host " 468 "controller (ignoring)\n"); 469 } else if (eec & XHCI_LEGSUP_OSOWNED) { 470 TRACE_ALWAYS("successfully took ownership of the host " 471 "controller\n"); 472 } 473 474 // Force off the BIOS owned flag, and clear all SMIs. Some BIOSes 475 // do indicate a successful handover but do not remove their SMIs 476 // and then freeze the system when interrupts are generated. 477 WriteCapReg32(eecp, eec & ~XHCI_LEGSUP_BIOSOWNED); 478 } 479 break; 480 } 481 uint32 legctlsts = ReadCapReg32(eecp + XHCI_LEGCTLSTS); 482 legctlsts &= XHCI_LEGCTLSTS_DISABLE_SMI; 483 legctlsts |= XHCI_LEGCTLSTS_EVENTS_SMI; 484 WriteCapReg32(eecp + XHCI_LEGCTLSTS, legctlsts); 485 486 // We need to explicitly take ownership of EHCI ports on earlier Intel chipsets. 487 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL) { 488 switch (fPCIInfo->device_id) { 489 case PCI_DEVICE_INTEL_PANTHER_POINT_XHCI: 490 case PCI_DEVICE_INTEL_LYNX_POINT_XHCI: 491 case PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI: 492 case PCI_DEVICE_INTEL_BAYTRAIL_XHCI: 493 case PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI: 494 case PCI_DEVICE_INTEL_WILDCAT_POINT_LP_XHCI: 495 _SwitchIntelPorts(); 496 break; 497 } 498 } 499 500 // halt the host controller 501 if (ControllerHalt() < B_OK) { 502 return; 503 } 504 505 // reset the host controller 506 if (ControllerReset() < B_OK) { 507 TRACE_ERROR("host controller failed to reset\n"); 508 return; 509 } 510 511 fCmdCompSem = create_sem(0, "XHCI Command Complete"); 512 fFinishTransfersSem = create_sem(0, "XHCI Finish Transfers"); 513 fEventSem = create_sem(0, "XHCI Event"); 514 if (fFinishTransfersSem < B_OK || fCmdCompSem < B_OK || fEventSem < B_OK) { 515 TRACE_ERROR("failed to create semaphores\n"); 516 return; 517 } 518 519 // create event handler thread 520 fEventThread = spawn_kernel_thread(EventThread, "xhci event thread", 521 B_URGENT_PRIORITY, (void *)this); 522 resume_thread(fEventThread); 523 524 // create finisher service thread 525 fFinishThread = spawn_kernel_thread(FinishThread, "xhci finish thread", 526 B_URGENT_PRIORITY - 1, (void *)this); 527 resume_thread(fFinishThread); 528 529 // Find the right interrupt vector, using MSIs if available. 530 fIRQ = fPCIInfo->u.h0.interrupt_line; 531 if (fIRQ == 0xFF) 532 fIRQ = 0; 533 534 #if 0 535 if (fPci->get_msix_count(fDevice) >= 1) { 536 uint8 msiVector = 0; 537 if (fPci->configure_msix(fDevice, 1, &msiVector) == B_OK 538 && fPci->enable_msix(fDevice) == B_OK) { 539 TRACE_ALWAYS("using MSI-X\n"); 540 fIRQ = msiVector; 541 fUseMSI = true; 542 } 543 } else 544 #endif 545 if (fPci->get_msi_count(fDevice) >= 1) { 546 uint32 msiVector = 0; 547 if (fPci->configure_msi(fDevice, 1, &msiVector) == B_OK 548 && fPci->enable_msi(fDevice) == B_OK) { 549 TRACE_ALWAYS("using message signaled interrupts\n"); 550 fIRQ = msiVector; 551 fUseMSI = true; 552 } 553 } 554 555 if (fIRQ == 0) { 556 TRACE_MODULE_ERROR("device PCI:%d:%d:%d was assigned an invalid IRQ\n", 557 fPCIInfo->bus, fPCIInfo->device, fPCIInfo->function); 558 return; 559 } 560 561 // Install the interrupt handler 562 TRACE("installing interrupt handler\n"); 563 install_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this, 0); 564 565 memset(fPortSpeeds, 0, sizeof(fPortSpeeds)); 566 memset(fDevices, 0, sizeof(fDevices)); 567 568 fInitOK = true; 569 TRACE("driver construction successful\n"); 570 } 571 572 573 XHCI::~XHCI() 574 { 575 TRACE("tear down XHCI host controller driver\n"); 576 577 WriteOpReg(XHCI_CMD, 0); 578 579 int32 result = 0; 580 fStopThreads = true; 581 delete_sem(fCmdCompSem); 582 delete_sem(fFinishTransfersSem); 583 delete_sem(fEventSem); 584 wait_for_thread(fFinishThread, &result); 585 wait_for_thread(fEventThread, &result); 586 587 mutex_destroy(&fFinishedLock); 588 mutex_destroy(&fEventLock); 589 590 remove_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this); 591 592 delete_area(fRegisterArea); 593 delete_area(fErstArea); 594 for (uint32 i = 0; i < fScratchpadCount; i++) 595 delete_area(fScratchpadArea[i]); 596 delete_area(fDcbaArea); 597 598 if (fUseMSI) { 599 fPci->disable_msi(fDevice); 600 fPci->unconfigure_msi(fDevice); 601 } 602 } 603 604 605 void 606 XHCI::_SwitchIntelPorts() 607 { 608 TRACE("Looking for EHCI owned ports\n"); 609 uint32 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_USB3PRM, 4); 610 TRACE("Superspeed Ports: 0x%" B_PRIx32 "\n", ports); 611 fPci->write_pci_config(fDevice, XHCI_INTEL_USB3_PSSEN, 4, ports); 612 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_USB3_PSSEN, 4); 613 TRACE("Superspeed ports now under XHCI : 0x%" B_PRIx32 "\n", ports); 614 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_USB2PRM, 4); 615 TRACE("USB 2.0 Ports : 0x%" B_PRIx32 "\n", ports); 616 fPci->write_pci_config(fDevice, XHCI_INTEL_XUSB2PR, 4, ports); 617 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_XUSB2PR, 4); 618 TRACE("USB 2.0 ports now under XHCI: 0x%" B_PRIx32 "\n", ports); 619 } 620 621 622 status_t 623 XHCI::Start() 624 { 625 TRACE_ALWAYS("starting XHCI host controller\n"); 626 TRACE("usbcmd: 0x%08" B_PRIx32 "; usbsts: 0x%08" B_PRIx32 "\n", 627 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 628 629 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 630 TRACE("Start() failed STS_CNR\n"); 631 } 632 633 if ((ReadOpReg(XHCI_CMD) & CMD_RUN) != 0) { 634 TRACE_ERROR("Start() warning, starting running XHCI controller!\n"); 635 } 636 637 if ((ReadOpReg(XHCI_PAGESIZE) & (1 << 0)) == 0) { 638 TRACE_ERROR("controller does not support 4K page size\n"); 639 return B_ERROR; 640 } 641 642 // read port count from capability register 643 uint32 capabilities = ReadCapReg32(XHCI_HCSPARAMS1); 644 fPortCount = HCS_MAX_PORTS(capabilities); 645 if (fPortCount == 0) { 646 TRACE_ERROR("invalid number of ports: %u\n", fPortCount); 647 return B_ERROR; 648 } 649 650 fSlotCount = HCS_MAX_SLOTS(capabilities); 651 if (fSlotCount > XHCI_MAX_DEVICES) 652 fSlotCount = XHCI_MAX_DEVICES; 653 WriteOpReg(XHCI_CONFIG, fSlotCount); 654 655 // find out which protocol is used for each port 656 uint8 portFound = 0; 657 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 658 uint32 eec = 0xffffffff; 659 uint32 eecp = HCS0_XECP(cparams) << 2; 660 for (; eecp != 0 && XECP_NEXT(eec) && portFound < fPortCount; 661 eecp += XECP_NEXT(eec) << 2) { 662 eec = ReadCapReg32(eecp); 663 if (XECP_ID(eec) != XHCI_SUPPORTED_PROTOCOLS_CAPID) 664 continue; 665 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) > 3) 666 continue; 667 uint32 temp = ReadCapReg32(eecp + 8); 668 uint32 offset = XHCI_SUPPORTED_PROTOCOLS_1_OFFSET(temp); 669 uint32 count = XHCI_SUPPORTED_PROTOCOLS_1_COUNT(temp); 670 if (offset == 0 || count == 0) 671 continue; 672 offset--; 673 for (uint32 i = offset; i < offset + count; i++) { 674 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) == 0x3) 675 fPortSpeeds[i] = USB_SPEED_SUPERSPEED; 676 else 677 fPortSpeeds[i] = USB_SPEED_HIGHSPEED; 678 679 TRACE("speed for port %" B_PRId32 " is %s\n", i, 680 fPortSpeeds[i] == USB_SPEED_SUPERSPEED ? "super" : "high"); 681 } 682 portFound += count; 683 } 684 685 uint32 params2 = ReadCapReg32(XHCI_HCSPARAMS2); 686 fScratchpadCount = HCS_MAX_SC_BUFFERS(params2); 687 if (fScratchpadCount > XHCI_MAX_SCRATCHPADS) { 688 TRACE_ERROR("invalid number of scratchpads: %" B_PRIu32 "\n", 689 fScratchpadCount); 690 return B_ERROR; 691 } 692 693 uint32 params3 = ReadCapReg32(XHCI_HCSPARAMS3); 694 fExitLatMax = HCS_U1_DEVICE_LATENCY(params3) 695 + HCS_U2_DEVICE_LATENCY(params3); 696 697 // clear interrupts & disable device notifications 698 WriteOpReg(XHCI_STS, ReadOpReg(XHCI_STS)); 699 WriteOpReg(XHCI_DNCTRL, 0); 700 701 // allocate Device Context Base Address array 702 phys_addr_t dmaAddress; 703 fDcbaArea = fStack->AllocateArea((void **)&fDcba, &dmaAddress, 704 sizeof(*fDcba), "DCBA Area"); 705 if (fDcbaArea < B_OK) { 706 TRACE_ERROR("unable to create the DCBA area\n"); 707 return B_ERROR; 708 } 709 memset(fDcba, 0, sizeof(*fDcba)); 710 memset(fScratchpadArea, 0, sizeof(fScratchpadArea)); 711 memset(fScratchpad, 0, sizeof(fScratchpad)); 712 713 // setting the first address to the scratchpad array address 714 fDcba->baseAddress[0] = dmaAddress 715 + offsetof(struct xhci_device_context_array, scratchpad); 716 717 // fill up the scratchpad array with scratchpad pages 718 for (uint32 i = 0; i < fScratchpadCount; i++) { 719 phys_addr_t scratchDmaAddress; 720 fScratchpadArea[i] = fStack->AllocateArea((void **)&fScratchpad[i], 721 &scratchDmaAddress, B_PAGE_SIZE, "Scratchpad Area"); 722 if (fScratchpadArea[i] < B_OK) { 723 TRACE_ERROR("unable to create the scratchpad area\n"); 724 return B_ERROR; 725 } 726 fDcba->scratchpad[i] = scratchDmaAddress; 727 } 728 729 TRACE("setting DCBAAP %" B_PRIxPHYSADDR "\n", dmaAddress); 730 WriteOpReg(XHCI_DCBAAP_LO, (uint32)dmaAddress); 731 WriteOpReg(XHCI_DCBAAP_HI, (uint32)(dmaAddress >> 32)); 732 733 // allocate Event Ring Segment Table 734 uint8 *addr; 735 fErstArea = fStack->AllocateArea((void **)&addr, &dmaAddress, 736 (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 737 + sizeof(xhci_erst_element), 738 "USB XHCI ERST CMD_RING and EVENT_RING Area"); 739 740 if (fErstArea < B_OK) { 741 TRACE_ERROR("unable to create the ERST AND RING area\n"); 742 delete_area(fDcbaArea); 743 return B_ERROR; 744 } 745 fErst = (xhci_erst_element *)addr; 746 memset(fErst, 0, (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 747 + sizeof(xhci_erst_element)); 748 749 // fill with Event Ring Segment Base Address and Event Ring Segment Size 750 fErst->rs_addr = dmaAddress + sizeof(xhci_erst_element); 751 fErst->rs_size = XHCI_MAX_EVENTS; 752 fErst->rsvdz = 0; 753 754 addr += sizeof(xhci_erst_element); 755 fEventRing = (xhci_trb *)addr; 756 addr += XHCI_MAX_EVENTS * sizeof(xhci_trb); 757 fCmdRing = (xhci_trb *)addr; 758 759 TRACE("setting ERST size\n"); 760 WriteRunReg32(XHCI_ERSTSZ(0), XHCI_ERSTS_SET(1)); 761 762 TRACE("setting ERDP addr = 0x%" B_PRIx64 "\n", fErst->rs_addr); 763 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)fErst->rs_addr); 764 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(fErst->rs_addr >> 32)); 765 766 TRACE("setting ERST base addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 767 WriteRunReg32(XHCI_ERSTBA_LO(0), (uint32)dmaAddress); 768 WriteRunReg32(XHCI_ERSTBA_HI(0), (uint32)(dmaAddress >> 32)); 769 770 dmaAddress += sizeof(xhci_erst_element) + XHCI_MAX_EVENTS 771 * sizeof(xhci_trb); 772 773 // Make sure the Command Ring is stopped 774 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 775 TRACE_ALWAYS("Command Ring is running, send stop/cancel\n"); 776 WriteOpReg(XHCI_CRCR_LO, CRCR_CS); 777 WriteOpReg(XHCI_CRCR_HI, 0); 778 WriteOpReg(XHCI_CRCR_LO, CRCR_CA); 779 WriteOpReg(XHCI_CRCR_HI, 0); 780 snooze(1000); 781 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 782 TRACE_ERROR("Command Ring still running after stop/cancel\n"); 783 } 784 } 785 TRACE("setting CRCR addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 786 WriteOpReg(XHCI_CRCR_LO, (uint32)dmaAddress | CRCR_RCS); 787 WriteOpReg(XHCI_CRCR_HI, (uint32)(dmaAddress >> 32)); 788 // link trb 789 fCmdRing[XHCI_MAX_COMMANDS - 1].address = dmaAddress; 790 791 TRACE("setting interrupt rate\n"); 792 793 // Setting IMOD below 0x3F8 on Intel Lynx Point can cause IRQ lockups 794 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL 795 && (fPCIInfo->device_id == PCI_DEVICE_INTEL_PANTHER_POINT_XHCI 796 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_XHCI 797 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI 798 || fPCIInfo->device_id == PCI_DEVICE_INTEL_BAYTRAIL_XHCI 799 || fPCIInfo->device_id == PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI)) { 800 WriteRunReg32(XHCI_IMOD(0), 0x000003f8); // 4000 irq/s 801 } else { 802 WriteRunReg32(XHCI_IMOD(0), 0x000001f4); // 8000 irq/s 803 } 804 805 TRACE("enabling interrupt\n"); 806 WriteRunReg32(XHCI_IMAN(0), ReadRunReg32(XHCI_IMAN(0)) | IMAN_INTR_ENA); 807 808 WriteOpReg(XHCI_CMD, CMD_RUN | CMD_INTE | CMD_HSEE); 809 810 // wait for start up state 811 if (WaitOpBits(XHCI_STS, STS_HCH, 0) != B_OK) { 812 TRACE_ERROR("HCH start up timeout\n"); 813 } 814 815 fRootHub = new(std::nothrow) XHCIRootHub(RootObject(), 1); 816 if (!fRootHub) { 817 TRACE_ERROR("no memory to allocate root hub\n"); 818 return B_NO_MEMORY; 819 } 820 821 if (fRootHub->InitCheck() < B_OK) { 822 TRACE_ERROR("root hub failed init check\n"); 823 return fRootHub->InitCheck(); 824 } 825 826 SetRootHub(fRootHub); 827 828 fRootHub->RegisterNode(Node()); 829 830 TRACE_ALWAYS("successfully started the controller\n"); 831 832 #ifdef TRACE_USB 833 TRACE("No-Op test...\n"); 834 Noop(); 835 #endif 836 837 return BusManager::Start(); 838 } 839 840 841 status_t 842 XHCI::SubmitTransfer(Transfer *transfer) 843 { 844 // short circuit the root hub 845 if (transfer->TransferPipe()->DeviceAddress() == 1) 846 return fRootHub->ProcessTransfer(this, transfer); 847 848 TRACE("SubmitTransfer(%p)\n", transfer); 849 Pipe *pipe = transfer->TransferPipe(); 850 if ((pipe->Type() & USB_OBJECT_CONTROL_PIPE) != 0) 851 return SubmitControlRequest(transfer); 852 return SubmitNormalRequest(transfer); 853 } 854 855 856 status_t 857 XHCI::SubmitControlRequest(Transfer *transfer) 858 { 859 Pipe *pipe = transfer->TransferPipe(); 860 usb_request_data *requestData = transfer->RequestData(); 861 bool directionIn = (requestData->RequestType & USB_REQTYPE_DEVICE_IN) != 0; 862 863 TRACE("SubmitControlRequest() length %d\n", requestData->Length); 864 865 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 866 if (endpoint == NULL) { 867 TRACE_ERROR("control pipe has no endpoint!\n"); 868 return B_BAD_VALUE; 869 } 870 if (endpoint->device == NULL) { 871 panic("endpoint is not initialized!"); 872 return B_NO_INIT; 873 } 874 875 status_t status = transfer->InitKernelAccess(); 876 if (status != B_OK) 877 return status; 878 879 xhci_td *descriptor = CreateDescriptor(3, 1, requestData->Length); 880 if (descriptor == NULL) 881 return B_NO_MEMORY; 882 descriptor->transfer = transfer; 883 884 // Setup Stage 885 uint8 index = 0; 886 memcpy(&descriptor->trbs[index].address, requestData, 887 sizeof(usb_request_data)); 888 descriptor->trbs[index].status = TRB_2_IRQ(0) | TRB_2_BYTES(8); 889 descriptor->trbs[index].flags 890 = TRB_3_TYPE(TRB_TYPE_SETUP_STAGE) | TRB_3_IDT_BIT | TRB_3_CYCLE_BIT; 891 if (requestData->Length > 0) { 892 descriptor->trbs[index].flags |= 893 directionIn ? TRB_3_TRT_IN : TRB_3_TRT_OUT; 894 } 895 896 index++; 897 898 // Data Stage (if any) 899 if (requestData->Length > 0) { 900 descriptor->trbs[index].address = descriptor->buffer_addrs[0]; 901 descriptor->trbs[index].status = TRB_2_IRQ(0) 902 | TRB_2_BYTES(requestData->Length) 903 | TRB_2_TD_SIZE(0); 904 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_DATA_STAGE) 905 | (directionIn ? TRB_3_DIR_IN : 0) 906 | TRB_3_CYCLE_BIT; 907 908 if (!directionIn) { 909 transfer->PrepareKernelAccess(); 910 WriteDescriptor(descriptor, transfer->Vector(), 911 transfer->VectorCount(), transfer->IsPhysical()); 912 } 913 914 index++; 915 } 916 917 // Status Stage 918 descriptor->trbs[index].address = 0; 919 descriptor->trbs[index].status = TRB_2_IRQ(0); 920 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_STATUS_STAGE) 921 | TRB_3_CHAIN_BIT | TRB_3_ENT_BIT | TRB_3_CYCLE_BIT; 922 // The CHAIN bit must be set when using an Event Data TRB 923 // (XHCI 1.2 § 6.4.1.2.3 Table 6-31 p472). 924 925 // Status Stage is an OUT transfer when the device is sending data 926 // (XHCI 1.2 § 4.11.2.2 Table 4-7 p213), otherwise set the IN bit. 927 if (requestData->Length == 0 || !directionIn) 928 descriptor->trbs[index].flags |= TRB_3_DIR_IN; 929 930 descriptor->trb_used = index + 1; 931 932 status = _LinkDescriptorForPipe(descriptor, endpoint); 933 if (status != B_OK) { 934 FreeDescriptor(descriptor); 935 return status; 936 } 937 938 return B_OK; 939 } 940 941 942 status_t 943 XHCI::SubmitNormalRequest(Transfer *transfer) 944 { 945 TRACE("SubmitNormalRequest() length %" B_PRIuSIZE "\n", transfer->FragmentLength()); 946 947 Pipe *pipe = transfer->TransferPipe(); 948 usb_isochronous_data *isochronousData = transfer->IsochronousData(); 949 bool directionIn = (pipe->Direction() == Pipe::In); 950 951 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 952 if (endpoint == NULL) { 953 TRACE_ERROR("pipe has no endpoint!\n"); 954 return B_BAD_VALUE; 955 } 956 if (endpoint->device == NULL) { 957 panic("endpoint is not initialized!"); 958 return B_NO_INIT; 959 } 960 961 status_t status = transfer->InitKernelAccess(); 962 if (status != B_OK) 963 return status; 964 965 // TRBs within a TD must be "grouped" into TD Fragments, which mostly means 966 // that a max_burst_payload boundary cannot be crossed within a TRB, but 967 // only between TRBs. More than one TRB can be in a TD Fragment, but we keep 968 // things simple by setting trbSize to the MBP. (XHCI 1.2 § 4.11.7.1 p235.) 969 size_t trbSize = endpoint->max_burst_payload; 970 971 if (isochronousData != NULL) { 972 if (isochronousData->packet_count == 0) 973 return B_BAD_VALUE; 974 975 // Isochronous transfers use more specifically sized packets. 976 trbSize = transfer->DataLength() / isochronousData->packet_count; 977 if (trbSize == 0 || trbSize > pipe->MaxPacketSize() || trbSize 978 != (size_t)isochronousData->packet_descriptors[0].request_length) 979 return B_BAD_VALUE; 980 } 981 982 // Now that we know trbSize, compute the count. 983 const int32 trbCount = (transfer->FragmentLength() + trbSize - 1) / trbSize; 984 985 xhci_td *td = CreateDescriptor(trbCount, trbCount, trbSize); 986 if (td == NULL) 987 return B_NO_MEMORY; 988 989 // Normal Stage 990 const size_t maxPacketSize = pipe->MaxPacketSize(); 991 size_t remaining = transfer->FragmentLength(); 992 for (int32 i = 0; i < trbCount; i++) { 993 int32 trbLength = (remaining < trbSize) ? remaining : trbSize; 994 remaining -= trbLength; 995 996 // The "TD Size" field of a transfer TRB indicates the number of 997 // remaining maximum-size *packets* in this TD, *not* including the 998 // packets in the current TRB, and capped at 31 if there are more 999 // than 31 packets remaining in the TD. (XHCI 1.2 § 4.11.2.4 p218.) 1000 int32 tdSize = (remaining + maxPacketSize - 1) / maxPacketSize; 1001 if (tdSize > 31) 1002 tdSize = 31; 1003 1004 td->trbs[i].address = td->buffer_addrs[i]; 1005 td->trbs[i].status = TRB_2_IRQ(0) 1006 | TRB_2_BYTES(trbLength) 1007 | TRB_2_TD_SIZE(tdSize); 1008 td->trbs[i].flags = TRB_3_TYPE(TRB_TYPE_NORMAL) 1009 | TRB_3_CYCLE_BIT | TRB_3_CHAIN_BIT; 1010 1011 td->trb_used++; 1012 } 1013 1014 // Isochronous-specific. 1015 if (isochronousData != NULL) { 1016 // This is an isochronous transfer; it should have one TD per packet. 1017 for (uint32 i = 0; i < isochronousData->packet_count; i++) { 1018 td->trbs[i].flags &= ~(TRB_3_TYPE(TRB_TYPE_NORMAL)); 1019 td->trbs[i].flags |= TRB_3_TYPE(TRB_TYPE_ISOCH); 1020 1021 if (i != (isochronousData->packet_count - 1)) { 1022 // For all but the last TD, generate events (but not interrupts) on short packets. 1023 // (The last TD uses the regular Event Data TRB.) 1024 td->trbs[i].flags |= TRB_3_ISP_BIT | TRB_3_BEI_BIT; 1025 td->trbs[i].flags &= ~TRB_3_CHAIN_BIT; 1026 } 1027 } 1028 1029 // TODO: We do not currently take Mult into account at all! 1030 // How are we supposed to do that here? 1031 1032 // Determine the (starting) frame number: if ISO_ASAP is set, 1033 // we are queueing this "right away", and so want to reset 1034 // the starting_frame_number. Otherwise we use the passed one. 1035 uint32 frame; 1036 if ((isochronousData->flags & USB_ISO_ASAP) != 0 1037 || isochronousData->starting_frame_number == NULL) { 1038 // All reads from the microframe index register must be 1039 // incremented by 1. (XHCI 1.2 § 4.14.2.1.4 p265.) 1040 frame = (ReadRunReg32(XHCI_MFINDEX) + 1) >> 3; 1041 td->trbs[0].flags |= TRB_3_ISO_SIA_BIT; 1042 } else { 1043 frame = *isochronousData->starting_frame_number; 1044 td->trbs[0].flags |= TRB_3_FRID(frame); 1045 } 1046 if (isochronousData->starting_frame_number != NULL) 1047 *isochronousData->starting_frame_number = frame; 1048 } 1049 1050 // Set the ENT (Evaluate Next TRB) bit, so that the HC will not switch 1051 // contexts before evaluating the Link TRB that _LinkDescriptorForPipe 1052 // will insert, as otherwise there would be a race between us freeing 1053 // and unlinking the descriptor, and the controller evaluating the Link TRB 1054 // and thus getting back onto the main ring and executing the Event Data 1055 // TRB that generates the interrupt for this transfer. 1056 // 1057 // Note that we *do not* unset the CHAIN bit in this TRB, thus including 1058 // the Link TRB in this TD formally, which is required when using the 1059 // ENT bit. (XHCI 1.2 § 4.12.3 p250.) 1060 td->trbs[td->trb_used - 1].flags |= TRB_3_ENT_BIT; 1061 1062 if (!directionIn) { 1063 TRACE("copying out iov count %ld\n", transfer->VectorCount()); 1064 status_t status = transfer->PrepareKernelAccess(); 1065 if (status != B_OK) { 1066 FreeDescriptor(td); 1067 return status; 1068 } 1069 WriteDescriptor(td, transfer->Vector(), 1070 transfer->VectorCount(), transfer->IsPhysical()); 1071 } 1072 1073 td->transfer = transfer; 1074 status = _LinkDescriptorForPipe(td, endpoint); 1075 if (status != B_OK) { 1076 FreeDescriptor(td); 1077 return status; 1078 } 1079 1080 return B_OK; 1081 } 1082 1083 1084 status_t 1085 XHCI::CancelQueuedTransfers(Pipe *pipe, bool force) 1086 { 1087 xhci_endpoint* endpoint = (xhci_endpoint*)pipe->ControllerCookie(); 1088 if (endpoint == NULL || endpoint->trbs == NULL) { 1089 // Someone's de-allocated this pipe or endpoint in the meantime. 1090 // (Possibly AllocateDevice failed, and we were the temporary pipe.) 1091 return B_NO_INIT; 1092 } 1093 1094 #ifndef TRACE_USB 1095 if (force) 1096 #endif 1097 { 1098 TRACE_ALWAYS("cancel queued transfers (%" B_PRId8 ") for pipe %p (%d)\n", 1099 endpoint->used, pipe, pipe->EndpointAddress()); 1100 } 1101 1102 MutexLocker endpointLocker(endpoint->lock); 1103 1104 if (endpoint->td_head == NULL) { 1105 // There aren't any currently pending transfers to cancel. 1106 return B_OK; 1107 } 1108 1109 // Calling the callbacks while holding the endpoint lock could potentially 1110 // cause deadlocks, so we instead store them in a pointer array. We need 1111 // to do this separately from freeing the TDs, for in the case we fail 1112 // to stop the endpoint, we cancel the transfers but do not free the TDs. 1113 Transfer* transfers[XHCI_MAX_TRANSFERS]; 1114 int32 transfersCount = 0; 1115 1116 for (xhci_td* td = endpoint->td_head; td != NULL; td = td->next) { 1117 if (td->transfer == NULL) 1118 continue; 1119 1120 transfers[transfersCount] = td->transfer; 1121 transfersCount++; 1122 td->transfer = NULL; 1123 } 1124 1125 // It is possible that while waiting for the stop-endpoint command to 1126 // complete, one of the queued transfers posts a completion event, so in 1127 // order to avoid a deadlock, we must unlock the endpoint. 1128 endpointLocker.Unlock(); 1129 status_t status = StopEndpoint(false, endpoint); 1130 if (status != B_OK && status != B_DEV_STALLED) { 1131 // It is possible that the endpoint was stopped by the controller at the 1132 // same time our STOP command was in progress, causing a "Context State" 1133 // error. In that case, try again; if the endpoint is already stopped, 1134 // StopEndpoint will notice this. (XHCI 1.2 § 4.6.9 p137.) 1135 status = StopEndpoint(false, endpoint); 1136 } 1137 if (status == B_DEV_STALLED) { 1138 // Only exit from a Halted state is a RESET. (XHCI 1.2 § 4.8.3 p163.) 1139 TRACE_ERROR("cancel queued transfers: halted endpoint, reset!\n"); 1140 status = ResetEndpoint(false, endpoint); 1141 } 1142 endpointLocker.Lock(); 1143 1144 // Detach the head TD from the endpoint. 1145 xhci_td* td_head = endpoint->td_head; 1146 endpoint->td_head = NULL; 1147 1148 if (status == B_OK) { 1149 // Clear the endpoint's TRBs. 1150 memset(endpoint->trbs, 0, sizeof(xhci_trb) * XHCI_ENDPOINT_RING_SIZE); 1151 endpoint->used = 0; 1152 endpoint->next = 0; 1153 1154 // Set dequeue pointer location to the beginning of the ring. 1155 SetTRDequeue(endpoint->trb_addr, 0, endpoint->id + 1, 1156 endpoint->device->slot); 1157 1158 // We don't need to do anything else to restart the ring, as it will resume 1159 // operation as normal upon the next doorbell. (XHCI 1.2 § 4.6.9 p136.) 1160 } else { 1161 // We couldn't stop the endpoint. Most likely the device has been 1162 // removed and the endpoint was stopped by the hardware, or is 1163 // for some reason busy and cannot be stopped. 1164 TRACE_ERROR("cancel queued transfers: could not stop endpoint: %s!\n", 1165 strerror(status)); 1166 1167 // Instead of freeing the TDs, we want to leave them in the endpoint 1168 // so that when/if the hardware returns, they can be properly unlinked, 1169 // as otherwise the endpoint could get "stuck" by having the "used" 1170 // slowly accumulate due to "dead" transfers. 1171 endpoint->td_head = td_head; 1172 td_head = NULL; 1173 } 1174 1175 endpointLocker.Unlock(); 1176 1177 for (int32 i = 0; i < transfersCount; i++) { 1178 // If the transfer is canceled by force, the one causing the 1179 // cancel is possibly not the one who initiated the transfer 1180 // and the callback is likely not safe anymore. 1181 if (!force) 1182 transfers[i]->Finished(B_CANCELED, 0); 1183 1184 delete transfers[i]; 1185 } 1186 1187 // This loop looks a bit strange because we need to store the "next" 1188 // pointer before freeing the descriptor. 1189 xhci_td* td; 1190 while ((td = td_head) != NULL) { 1191 td_head = td_head->next; 1192 FreeDescriptor(td); 1193 } 1194 1195 return B_OK; 1196 } 1197 1198 1199 status_t 1200 XHCI::StartDebugTransfer(Transfer *transfer) 1201 { 1202 Pipe *pipe = transfer->TransferPipe(); 1203 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 1204 if (endpoint == NULL) 1205 return B_BAD_VALUE; 1206 1207 // Check all locks that we are going to hit when running transfers. 1208 if (mutex_trylock(&endpoint->lock) != B_OK) 1209 return B_WOULD_BLOCK; 1210 if (mutex_trylock(&fFinishedLock) != B_OK) { 1211 mutex_unlock(&endpoint->lock); 1212 return B_WOULD_BLOCK; 1213 } 1214 if (mutex_trylock(&fEventLock) != B_OK) { 1215 mutex_unlock(&endpoint->lock); 1216 mutex_unlock(&fFinishedLock); 1217 return B_WOULD_BLOCK; 1218 } 1219 mutex_unlock(&endpoint->lock); 1220 mutex_unlock(&fFinishedLock); 1221 mutex_unlock(&fEventLock); 1222 1223 status_t status = SubmitTransfer(transfer); 1224 if (status != B_OK) 1225 return status; 1226 1227 // The endpoint's head TD is the TD of the just-submitted transfer. 1228 // Just like EHCI, abuse the callback cookie to hold the TD pointer. 1229 transfer->SetCallback(NULL, endpoint->td_head); 1230 1231 return B_OK; 1232 } 1233 1234 1235 status_t 1236 XHCI::CheckDebugTransfer(Transfer *transfer) 1237 { 1238 xhci_td *transfer_td = (xhci_td *)transfer->CallbackCookie(); 1239 if (transfer_td == NULL) 1240 return B_NO_INIT; 1241 1242 // Process events once, and then look for it in the finished list. 1243 ProcessEvents(); 1244 xhci_td *previous = NULL; 1245 for (xhci_td *td = fFinishedHead; td != NULL; td = td->next) { 1246 if (td != transfer_td) { 1247 previous = td; 1248 continue; 1249 } 1250 1251 // We've found it! 1252 if (previous == NULL) { 1253 fFinishedHead = fFinishedHead->next; 1254 } else { 1255 previous->next = td->next; 1256 } 1257 1258 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 1259 status_t status = (td->trb_completion_code == COMP_SUCCESS 1260 || td->trb_completion_code == COMP_SHORT_PACKET) ? B_OK : B_ERROR; 1261 1262 if (status == B_OK && directionIn) { 1263 ReadDescriptor(td, transfer->Vector(), transfer->VectorCount(), 1264 transfer->IsPhysical()); 1265 } 1266 1267 FreeDescriptor(td); 1268 transfer->SetCallback(NULL, NULL); 1269 return status; 1270 } 1271 1272 // We didn't find it. 1273 spin(75); 1274 return B_DEV_PENDING; 1275 } 1276 1277 1278 void 1279 XHCI::CancelDebugTransfer(Transfer *transfer) 1280 { 1281 while (CheckDebugTransfer(transfer) == B_DEV_PENDING) 1282 spin(100); 1283 } 1284 1285 1286 status_t 1287 XHCI::NotifyPipeChange(Pipe *pipe, usb_change change) 1288 { 1289 TRACE("pipe change %d for pipe %p (%d)\n", change, pipe, 1290 pipe->EndpointAddress()); 1291 1292 switch (change) { 1293 case USB_CHANGE_CREATED: 1294 return _InsertEndpointForPipe(pipe); 1295 case USB_CHANGE_DESTROYED: 1296 return _RemoveEndpointForPipe(pipe); 1297 1298 case USB_CHANGE_PIPE_POLICY_CHANGED: 1299 // We don't care about these, at least for now. 1300 return B_OK; 1301 } 1302 1303 TRACE_ERROR("unknown pipe change!\n"); 1304 return B_UNSUPPORTED; 1305 } 1306 1307 1308 xhci_td * 1309 XHCI::CreateDescriptor(uint32 trbCount, uint32 bufferCount, size_t bufferSize) 1310 { 1311 const bool inKDL = debug_debugger_running(); 1312 1313 xhci_td *result; 1314 if (!inKDL) { 1315 result = (xhci_td*)calloc(1, sizeof(xhci_td)); 1316 } else { 1317 // Just use the physical memory allocator while in KDL; it's less 1318 // secure than using the regular heap, but it's easier to deal with. 1319 phys_addr_t dummy; 1320 fStack->AllocateChunk((void **)&result, &dummy, sizeof(xhci_td)); 1321 } 1322 1323 if (result == NULL) { 1324 TRACE_ERROR("failed to allocate a transfer descriptor\n"); 1325 return NULL; 1326 } 1327 1328 // We always allocate 1 more TRB than requested, so that 1329 // _LinkDescriptorForPipe() has room to insert a link TRB. 1330 trbCount++; 1331 if (fStack->AllocateChunk((void **)&result->trbs, &result->trb_addr, 1332 (trbCount * sizeof(xhci_trb))) < B_OK) { 1333 TRACE_ERROR("failed to allocate TRBs\n"); 1334 FreeDescriptor(result); 1335 return NULL; 1336 } 1337 result->trb_count = trbCount; 1338 result->trb_used = 0; 1339 1340 if (bufferSize > 0) { 1341 // Due to how the USB stack allocates physical memory, we can't just 1342 // request one large chunk the size of the transfer, and so instead we 1343 // create a series of buffers as requested by our caller. 1344 1345 // We store the buffer pointers and addresses in one memory block. 1346 if (!inKDL) { 1347 result->buffers = (void**)calloc(bufferCount, 1348 (sizeof(void*) + sizeof(phys_addr_t))); 1349 } else { 1350 phys_addr_t dummy; 1351 fStack->AllocateChunk((void **)&result->buffers, &dummy, 1352 bufferCount * (sizeof(void*) + sizeof(phys_addr_t))); 1353 } 1354 if (result->buffers == NULL) { 1355 TRACE_ERROR("unable to allocate space for buffer infos\n"); 1356 FreeDescriptor(result); 1357 return NULL; 1358 } 1359 result->buffer_addrs = (phys_addr_t*)&result->buffers[bufferCount]; 1360 result->buffer_size = bufferSize; 1361 result->buffer_count = bufferCount; 1362 1363 // Optimization: If the requested total size of all buffers is less 1364 // than 32*B_PAGE_SIZE (the maximum size that the physical memory 1365 // allocator can handle), we allocate only one buffer and segment it. 1366 size_t totalSize = bufferSize * bufferCount; 1367 if (totalSize < (32 * B_PAGE_SIZE)) { 1368 if (fStack->AllocateChunk(&result->buffers[0], 1369 &result->buffer_addrs[0], totalSize) < B_OK) { 1370 TRACE_ERROR("unable to allocate space for large buffer (size %ld)\n", 1371 totalSize); 1372 FreeDescriptor(result); 1373 return NULL; 1374 } 1375 for (uint32 i = 1; i < bufferCount; i++) { 1376 result->buffers[i] = (void*)((addr_t)(result->buffers[i - 1]) 1377 + bufferSize); 1378 result->buffer_addrs[i] = result->buffer_addrs[i - 1] 1379 + bufferSize; 1380 } 1381 } else { 1382 // Otherwise, we allocate each buffer individually. 1383 for (uint32 i = 0; i < bufferCount; i++) { 1384 if (fStack->AllocateChunk(&result->buffers[i], 1385 &result->buffer_addrs[i], bufferSize) < B_OK) { 1386 TRACE_ERROR("unable to allocate space for a buffer (size " 1387 "%" B_PRIuSIZE ", count %" B_PRIu32 ")\n", 1388 bufferSize, bufferCount); 1389 FreeDescriptor(result); 1390 return NULL; 1391 } 1392 } 1393 } 1394 } else { 1395 result->buffers = NULL; 1396 result->buffer_addrs = NULL; 1397 } 1398 1399 // Initialize all other fields. 1400 result->transfer = NULL; 1401 result->trb_completion_code = 0; 1402 result->trb_left = 0; 1403 result->next = NULL; 1404 1405 TRACE("CreateDescriptor allocated %p, buffer_size %ld, buffer_count %" B_PRIu32 "\n", 1406 result, result->buffer_size, result->buffer_count); 1407 1408 return result; 1409 } 1410 1411 1412 void 1413 XHCI::FreeDescriptor(xhci_td *descriptor) 1414 { 1415 if (descriptor == NULL) 1416 return; 1417 1418 const bool inKDL = debug_debugger_running(); 1419 1420 if (descriptor->trbs != NULL) { 1421 fStack->FreeChunk(descriptor->trbs, descriptor->trb_addr, 1422 (descriptor->trb_count * sizeof(xhci_trb))); 1423 } 1424 if (descriptor->buffers != NULL) { 1425 size_t totalSize = descriptor->buffer_size * descriptor->buffer_count; 1426 if (totalSize < (32 * B_PAGE_SIZE)) { 1427 // This was allocated as one contiguous buffer. 1428 fStack->FreeChunk(descriptor->buffers[0], descriptor->buffer_addrs[0], 1429 totalSize); 1430 } else { 1431 for (uint32 i = 0; i < descriptor->buffer_count; i++) { 1432 if (descriptor->buffers[i] == NULL) 1433 continue; 1434 fStack->FreeChunk(descriptor->buffers[i], descriptor->buffer_addrs[i], 1435 descriptor->buffer_size); 1436 } 1437 } 1438 1439 if (!inKDL) { 1440 free(descriptor->buffers); 1441 } else { 1442 fStack->FreeChunk(descriptor->buffers, 0, 1443 descriptor->buffer_count * (sizeof(void*) + sizeof(phys_addr_t))); 1444 } 1445 } 1446 1447 if (!inKDL) 1448 free(descriptor); 1449 else 1450 fStack->FreeChunk(descriptor, 0, sizeof(xhci_td)); 1451 } 1452 1453 1454 size_t 1455 XHCI::WriteDescriptor(xhci_td *descriptor, generic_io_vec *vector, size_t vectorCount, bool physical) 1456 { 1457 size_t written = 0; 1458 1459 size_t bufIdx = 0, bufUsed = 0; 1460 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1461 size_t length = vector[vecIdx].length; 1462 1463 while (length > 0 && bufIdx < descriptor->buffer_count) { 1464 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1465 status_t status = generic_memcpy( 1466 (generic_addr_t)descriptor->buffers[bufIdx] + bufUsed, false, 1467 vector[vecIdx].base + (vector[vecIdx].length - length), physical, 1468 toCopy); 1469 ASSERT_ALWAYS(status == B_OK); 1470 1471 written += toCopy; 1472 bufUsed += toCopy; 1473 length -= toCopy; 1474 if (bufUsed == descriptor->buffer_size) { 1475 bufIdx++; 1476 bufUsed = 0; 1477 } 1478 } 1479 } 1480 1481 TRACE("wrote descriptor (%" B_PRIuSIZE " bytes)\n", written); 1482 return written; 1483 } 1484 1485 1486 size_t 1487 XHCI::ReadDescriptor(xhci_td *descriptor, generic_io_vec *vector, size_t vectorCount, bool physical) 1488 { 1489 size_t read = 0; 1490 1491 size_t bufIdx = 0, bufUsed = 0; 1492 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1493 size_t length = vector[vecIdx].length; 1494 1495 while (length > 0 && bufIdx < descriptor->buffer_count) { 1496 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1497 status_t status = generic_memcpy( 1498 vector[vecIdx].base + (vector[vecIdx].length - length), physical, 1499 (generic_addr_t)descriptor->buffers[bufIdx] + bufUsed, false, toCopy); 1500 ASSERT_ALWAYS(status == B_OK); 1501 1502 read += toCopy; 1503 bufUsed += toCopy; 1504 length -= toCopy; 1505 if (bufUsed == descriptor->buffer_size) { 1506 bufIdx++; 1507 bufUsed = 0; 1508 } 1509 } 1510 } 1511 1512 TRACE("read descriptor (%" B_PRIuSIZE " bytes)\n", read); 1513 return read; 1514 } 1515 1516 1517 Device * 1518 XHCI::AllocateDevice(Hub *parent, int8 hubAddress, uint8 hubPort, 1519 usb_speed speed) 1520 { 1521 TRACE("AllocateDevice hubAddress %d hubPort %d speed %d\n", hubAddress, 1522 hubPort, speed); 1523 1524 uint8 slot = XHCI_MAX_SLOTS; 1525 status_t status = EnableSlot(&slot); 1526 if (status != B_OK) { 1527 TRACE_ERROR("failed to enable slot: %s\n", strerror(status)); 1528 return NULL; 1529 } 1530 1531 if (slot == 0 || slot > fSlotCount) { 1532 TRACE_ERROR("AllocateDevice: bad slot\n"); 1533 return NULL; 1534 } 1535 1536 if (fDevices[slot].slot != 0) { 1537 TRACE_ERROR("AllocateDevice: slot already used\n"); 1538 return NULL; 1539 } 1540 1541 struct xhci_device *device = &fDevices[slot]; 1542 device->slot = slot; 1543 1544 device->input_ctx_area = fStack->AllocateArea((void **)&device->input_ctx, 1545 &device->input_ctx_addr, sizeof(*device->input_ctx) << fContextSizeShift, 1546 "XHCI input context"); 1547 if (device->input_ctx_area < B_OK) { 1548 TRACE_ERROR("unable to create a input context area\n"); 1549 CleanupDevice(device); 1550 return NULL; 1551 } 1552 if (fContextSizeShift == 1) { 1553 // 64-byte contexts have to be page-aligned in order for 1554 // _OffsetContextAddr to function properly. 1555 ASSERT((((addr_t)device->input_ctx) % B_PAGE_SIZE) == 0); 1556 } 1557 1558 memset(device->input_ctx, 0, sizeof(*device->input_ctx) << fContextSizeShift); 1559 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1560 _WriteContext(&device->input_ctx->input.addFlags, 3); 1561 1562 uint8 rhPort = hubPort; 1563 uint32 route = 0; 1564 for (Device *hubDevice = parent; hubDevice != RootObject(); 1565 hubDevice = (Device *)hubDevice->Parent()) { 1566 if (hubDevice->Parent() == RootObject()) 1567 break; 1568 1569 if (rhPort > 15) 1570 rhPort = 15; 1571 route = route << 4; 1572 route |= rhPort; 1573 1574 rhPort = hubDevice->HubPort(); 1575 } 1576 1577 uint32 dwslot0 = SLOT_0_NUM_ENTRIES(1) | SLOT_0_ROUTE(route); 1578 1579 // Get speed of port, only if device connected to root hub port 1580 // else we have to rely on value reported by the Hub Explore thread 1581 if (route == 0) { 1582 GetPortSpeed(hubPort - 1, &speed); 1583 TRACE("speed updated %d\n", speed); 1584 } 1585 1586 // add the speed 1587 switch (speed) { 1588 case USB_SPEED_LOWSPEED: 1589 dwslot0 |= SLOT_0_SPEED(2); 1590 break; 1591 case USB_SPEED_FULLSPEED: 1592 dwslot0 |= SLOT_0_SPEED(1); 1593 break; 1594 case USB_SPEED_HIGHSPEED: 1595 dwslot0 |= SLOT_0_SPEED(3); 1596 break; 1597 case USB_SPEED_SUPERSPEED: 1598 dwslot0 |= SLOT_0_SPEED(4); 1599 break; 1600 case USB_SPEED_SUPERSPEEDPLUS: 1601 dwslot0 |= SLOT_0_SPEED(5); 1602 break; 1603 default: 1604 TRACE_ERROR("unknown usb speed\n"); 1605 break; 1606 } 1607 1608 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1609 // TODO enable power save 1610 _WriteContext(&device->input_ctx->slot.dwslot1, SLOT_1_RH_PORT(rhPort)); 1611 uint32 dwslot2 = SLOT_2_IRQ_TARGET(0); 1612 1613 // If LS/FS device connected to non-root HS device 1614 if (route != 0 && parent->Speed() == USB_SPEED_HIGHSPEED 1615 && (speed == USB_SPEED_LOWSPEED || speed == USB_SPEED_FULLSPEED)) { 1616 struct xhci_device *parenthub = (struct xhci_device *) 1617 parent->ControllerCookie(); 1618 dwslot2 |= SLOT_2_PORT_NUM(hubPort); 1619 dwslot2 |= SLOT_2_TT_HUB_SLOT(parenthub->slot); 1620 } 1621 1622 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1623 1624 _WriteContext(&device->input_ctx->slot.dwslot3, SLOT_3_SLOT_STATE(0) 1625 | SLOT_3_DEVICE_ADDRESS(0)); 1626 1627 TRACE("slot 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 1628 "\n", _ReadContext(&device->input_ctx->slot.dwslot0), 1629 _ReadContext(&device->input_ctx->slot.dwslot1), 1630 _ReadContext(&device->input_ctx->slot.dwslot2), 1631 _ReadContext(&device->input_ctx->slot.dwslot3)); 1632 1633 device->device_ctx_area = fStack->AllocateArea((void **)&device->device_ctx, 1634 &device->device_ctx_addr, sizeof(*device->device_ctx) << fContextSizeShift, 1635 "XHCI device context"); 1636 if (device->device_ctx_area < B_OK) { 1637 TRACE_ERROR("unable to create a device context area\n"); 1638 CleanupDevice(device); 1639 return NULL; 1640 } 1641 memset(device->device_ctx, 0, sizeof(*device->device_ctx) << fContextSizeShift); 1642 1643 device->trb_area = fStack->AllocateArea((void **)&device->trbs, 1644 &device->trb_addr, sizeof(xhci_trb) * (XHCI_MAX_ENDPOINTS - 1) 1645 * XHCI_ENDPOINT_RING_SIZE, "XHCI endpoint trbs"); 1646 if (device->trb_area < B_OK) { 1647 TRACE_ERROR("unable to create a device trbs area\n"); 1648 CleanupDevice(device); 1649 return NULL; 1650 } 1651 1652 // set up slot pointer to device context 1653 fDcba->baseAddress[slot] = device->device_ctx_addr; 1654 1655 size_t maxPacketSize; 1656 switch (speed) { 1657 case USB_SPEED_LOWSPEED: 1658 case USB_SPEED_FULLSPEED: 1659 maxPacketSize = 8; 1660 break; 1661 case USB_SPEED_HIGHSPEED: 1662 maxPacketSize = 64; 1663 break; 1664 default: 1665 maxPacketSize = 512; 1666 break; 1667 } 1668 1669 xhci_endpoint* endpoint0 = &device->endpoints[0]; 1670 mutex_init(&endpoint0->lock, "xhci endpoint lock"); 1671 endpoint0->device = device; 1672 endpoint0->id = 0; 1673 endpoint0->status = 0; 1674 endpoint0->td_head = NULL; 1675 endpoint0->used = 0; 1676 endpoint0->next = 0; 1677 endpoint0->trbs = device->trbs; 1678 endpoint0->trb_addr = device->trb_addr; 1679 1680 // configure the Control endpoint 0 1681 if (ConfigureEndpoint(endpoint0, slot, 0, USB_OBJECT_CONTROL_PIPE, false, 1682 0, maxPacketSize, speed, 0, 0) != B_OK) { 1683 TRACE_ERROR("unable to configure default control endpoint\n"); 1684 CleanupDevice(device); 1685 return NULL; 1686 } 1687 1688 // device should get to addressed state (bsr = 0) 1689 status = SetAddress(device->input_ctx_addr, false, slot); 1690 if (status != B_OK) { 1691 TRACE_ERROR("unable to set address: %s\n", strerror(status)); 1692 CleanupDevice(device); 1693 return NULL; 1694 } 1695 1696 device->address = SLOT_3_DEVICE_ADDRESS_GET(_ReadContext( 1697 &device->device_ctx->slot.dwslot3)); 1698 1699 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", device->address, 1700 SLOT_3_SLOT_STATE_GET(_ReadContext( 1701 &device->device_ctx->slot.dwslot3))); 1702 TRACE("endpoint0 state 0x%08" B_PRIx32 "\n", 1703 ENDPOINT_0_STATE_GET(_ReadContext( 1704 &device->device_ctx->endpoints[0].dwendpoint0))); 1705 1706 // Wait a bit for the device to complete addressing 1707 snooze(USB_DELAY_SET_ADDRESS); 1708 1709 // Create a temporary pipe with the new address 1710 ControlPipe pipe(parent); 1711 pipe.SetControllerCookie(endpoint0); 1712 pipe.InitCommon(device->address + 1, 0, speed, Pipe::Default, maxPacketSize, 0, 1713 hubAddress, hubPort); 1714 1715 // Get the device descriptor 1716 // Just retrieve the first 8 bytes of the descriptor -> minimum supported 1717 // size of any device. It is enough because it includes the device type. 1718 1719 size_t actualLength = 0; 1720 usb_device_descriptor deviceDescriptor; 1721 1722 TRACE("getting the device descriptor\n"); 1723 status = pipe.SendRequest( 1724 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_STANDARD, // type 1725 USB_REQUEST_GET_DESCRIPTOR, // request 1726 USB_DESCRIPTOR_DEVICE << 8, // value 1727 0, // index 1728 8, // length 1729 (void *)&deviceDescriptor, // buffer 1730 8, // buffer length 1731 &actualLength); // actual length 1732 1733 if (actualLength != 8) { 1734 TRACE_ERROR("failed to get the device descriptor: %s\n", 1735 strerror(status)); 1736 CleanupDevice(device); 1737 return NULL; 1738 } 1739 1740 TRACE("device_class: %d device_subclass %d device_protocol %d\n", 1741 deviceDescriptor.device_class, deviceDescriptor.device_subclass, 1742 deviceDescriptor.device_protocol); 1743 1744 if (speed == USB_SPEED_FULLSPEED && deviceDescriptor.max_packet_size_0 != 8) { 1745 TRACE("Full speed device with different max packet size for Endpoint 0\n"); 1746 uint32 dwendpoint1 = _ReadContext( 1747 &device->input_ctx->endpoints[0].dwendpoint1); 1748 dwendpoint1 &= ~ENDPOINT_1_MAXPACKETSIZE(0xffff); 1749 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE( 1750 deviceDescriptor.max_packet_size_0); 1751 _WriteContext(&device->input_ctx->endpoints[0].dwendpoint1, 1752 dwendpoint1); 1753 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1754 _WriteContext(&device->input_ctx->input.addFlags, (1 << 1)); 1755 EvaluateContext(device->input_ctx_addr, device->slot); 1756 } 1757 1758 Device *deviceObject = NULL; 1759 if (deviceDescriptor.device_class == 0x09) { 1760 TRACE("creating new Hub\n"); 1761 TRACE("getting the hub descriptor\n"); 1762 size_t actualLength = 0; 1763 usb_hub_descriptor hubDescriptor; 1764 status = pipe.SendRequest( 1765 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_CLASS, // type 1766 USB_REQUEST_GET_DESCRIPTOR, // request 1767 USB_DESCRIPTOR_HUB << 8, // value 1768 0, // index 1769 sizeof(usb_hub_descriptor), // length 1770 (void *)&hubDescriptor, // buffer 1771 sizeof(usb_hub_descriptor), // buffer length 1772 &actualLength); 1773 1774 if (actualLength != sizeof(usb_hub_descriptor)) { 1775 TRACE_ERROR("error while getting the hub descriptor: %s\n", 1776 strerror(status)); 1777 CleanupDevice(device); 1778 return NULL; 1779 } 1780 1781 uint32 dwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1782 dwslot0 |= SLOT_0_HUB_BIT; 1783 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1784 uint32 dwslot1 = _ReadContext(&device->input_ctx->slot.dwslot1); 1785 dwslot1 |= SLOT_1_NUM_PORTS(hubDescriptor.num_ports); 1786 _WriteContext(&device->input_ctx->slot.dwslot1, dwslot1); 1787 if (speed == USB_SPEED_HIGHSPEED) { 1788 uint32 dwslot2 = _ReadContext(&device->input_ctx->slot.dwslot2); 1789 dwslot2 |= SLOT_2_TT_TIME(HUB_TTT_GET(hubDescriptor.characteristics)); 1790 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1791 } 1792 1793 deviceObject = new(std::nothrow) Hub(parent, hubAddress, hubPort, 1794 deviceDescriptor, device->address + 1, speed, false, device); 1795 } else { 1796 TRACE("creating new device\n"); 1797 deviceObject = new(std::nothrow) Device(parent, hubAddress, hubPort, 1798 deviceDescriptor, device->address + 1, speed, false, device); 1799 } 1800 if (deviceObject == NULL || deviceObject->InitCheck() != B_OK) { 1801 if (deviceObject == NULL) { 1802 TRACE_ERROR("no memory to allocate device\n"); 1803 } else { 1804 TRACE_ERROR("device object failed to initialize\n"); 1805 } 1806 CleanupDevice(device); 1807 return NULL; 1808 } 1809 1810 // We don't want to disable the default endpoint, naturally, which would 1811 // otherwise happen when this Pipe object is destroyed. 1812 pipe.SetControllerCookie(NULL); 1813 1814 deviceObject->RegisterNode(); 1815 1816 TRACE("AllocateDevice() port %d slot %d\n", hubPort, slot); 1817 return deviceObject; 1818 } 1819 1820 1821 void 1822 XHCI::FreeDevice(Device *usbDevice) 1823 { 1824 xhci_device* device = (xhci_device*)usbDevice->ControllerCookie(); 1825 TRACE("FreeDevice() slot %d\n", device->slot); 1826 1827 // Delete the device first, so it cleans up its pipes and tells us 1828 // what we need to destroy before we tear down our internal state. 1829 delete usbDevice; 1830 1831 CleanupDevice(device); 1832 } 1833 1834 1835 void 1836 XHCI::CleanupDevice(xhci_device *device) 1837 { 1838 if (device->slot != 0) { 1839 DisableSlot(device->slot); 1840 fDcba->baseAddress[device->slot] = 0; 1841 } 1842 1843 if (device->trb_addr != 0) 1844 delete_area(device->trb_area); 1845 if (device->input_ctx_addr != 0) 1846 delete_area(device->input_ctx_area); 1847 if (device->device_ctx_addr != 0) 1848 delete_area(device->device_ctx_area); 1849 1850 memset(device, 0, sizeof(xhci_device)); 1851 } 1852 1853 1854 uint8 1855 XHCI::_GetEndpointState(xhci_endpoint* endpoint) 1856 { 1857 struct xhci_device_ctx* device_ctx = endpoint->device->device_ctx; 1858 return ENDPOINT_0_STATE_GET( 1859 _ReadContext(&device_ctx->endpoints[endpoint->id].dwendpoint0)); 1860 } 1861 1862 1863 status_t 1864 XHCI::_InsertEndpointForPipe(Pipe *pipe) 1865 { 1866 TRACE("insert endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1867 1868 if (pipe->ControllerCookie() != NULL 1869 || pipe->Parent()->Type() != USB_OBJECT_DEVICE) { 1870 // default pipe is already referenced 1871 return B_OK; 1872 } 1873 1874 Device* usbDevice = (Device *)pipe->Parent(); 1875 if (usbDevice->Parent() == RootObject()) { 1876 // root hub needs no initialization 1877 return B_OK; 1878 } 1879 1880 struct xhci_device *device = (struct xhci_device *) 1881 usbDevice->ControllerCookie(); 1882 if (device == NULL) { 1883 panic("device is NULL\n"); 1884 return B_NO_INIT; 1885 } 1886 1887 const uint8 id = (2 * pipe->EndpointAddress() 1888 + (pipe->Direction() != Pipe::Out ? 1 : 0)) - 1; 1889 if (id >= XHCI_MAX_ENDPOINTS - 1) 1890 return B_BAD_VALUE; 1891 1892 if (id > 0) { 1893 uint32 devicedwslot0 = _ReadContext(&device->device_ctx->slot.dwslot0); 1894 if (SLOT_0_NUM_ENTRIES_GET(devicedwslot0) == 1) { 1895 uint32 inputdwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1896 inputdwslot0 &= ~(SLOT_0_NUM_ENTRIES(0x1f)); 1897 inputdwslot0 |= SLOT_0_NUM_ENTRIES(XHCI_MAX_ENDPOINTS - 1); 1898 _WriteContext(&device->input_ctx->slot.dwslot0, inputdwslot0); 1899 EvaluateContext(device->input_ctx_addr, device->slot); 1900 } 1901 1902 xhci_endpoint* endpoint = &device->endpoints[id]; 1903 mutex_init(&endpoint->lock, "xhci endpoint lock"); 1904 MutexLocker endpointLocker(endpoint->lock); 1905 1906 endpoint->device = device; 1907 endpoint->id = id; 1908 endpoint->td_head = NULL; 1909 endpoint->used = 0; 1910 endpoint->next = 0; 1911 1912 endpoint->trbs = device->trbs + id * XHCI_ENDPOINT_RING_SIZE; 1913 endpoint->trb_addr = device->trb_addr 1914 + id * XHCI_ENDPOINT_RING_SIZE * sizeof(xhci_trb); 1915 memset(endpoint->trbs, 0, 1916 sizeof(xhci_trb) * XHCI_ENDPOINT_RING_SIZE); 1917 1918 TRACE("insert endpoint for pipe: trbs, device %p endpoint %p\n", 1919 device->trbs, endpoint->trbs); 1920 TRACE("insert endpoint for pipe: trb_addr, device 0x%" B_PRIxPHYSADDR 1921 " endpoint 0x%" B_PRIxPHYSADDR "\n", device->trb_addr, 1922 endpoint->trb_addr); 1923 1924 const uint8 endpointNum = id + 1; 1925 1926 status_t status = ConfigureEndpoint(endpoint, device->slot, id, pipe->Type(), 1927 pipe->Direction() == Pipe::In, pipe->Interval(), pipe->MaxPacketSize(), 1928 usbDevice->Speed(), pipe->MaxBurst(), pipe->BytesPerInterval()); 1929 if (status != B_OK) { 1930 TRACE_ERROR("unable to configure endpoint: %s\n", strerror(status)); 1931 return status; 1932 } 1933 1934 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1935 _WriteContext(&device->input_ctx->input.addFlags, 1936 (1 << endpointNum) | (1 << 0)); 1937 1938 ConfigureEndpoint(device->input_ctx_addr, false, device->slot); 1939 1940 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", 1941 device->address, SLOT_3_SLOT_STATE_GET(_ReadContext( 1942 &device->device_ctx->slot.dwslot3))); 1943 TRACE("endpoint[0] state 0x%08" B_PRIx32 "\n", 1944 ENDPOINT_0_STATE_GET(_ReadContext( 1945 &device->device_ctx->endpoints[0].dwendpoint0))); 1946 TRACE("endpoint[%d] state 0x%08" B_PRIx32 "\n", id, 1947 ENDPOINT_0_STATE_GET(_ReadContext( 1948 &device->device_ctx->endpoints[id].dwendpoint0))); 1949 } 1950 pipe->SetControllerCookie(&device->endpoints[id]); 1951 1952 return B_OK; 1953 } 1954 1955 1956 status_t 1957 XHCI::_RemoveEndpointForPipe(Pipe *pipe) 1958 { 1959 TRACE("remove endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1960 1961 if (pipe->Parent()->Type() != USB_OBJECT_DEVICE) 1962 return B_OK; 1963 Device* usbDevice = (Device *)pipe->Parent(); 1964 if (usbDevice->Parent() == RootObject()) 1965 return B_BAD_VALUE; 1966 1967 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 1968 if (endpoint == NULL || endpoint->trbs == NULL) 1969 return B_NO_INIT; 1970 1971 pipe->SetControllerCookie(NULL); 1972 1973 if (endpoint->id > 0) { 1974 xhci_device *device = endpoint->device; 1975 uint8 epNumber = endpoint->id + 1; 1976 StopEndpoint(true, endpoint); 1977 1978 mutex_lock(&endpoint->lock); 1979 1980 // See comment in CancelQueuedTransfers. 1981 xhci_td* td; 1982 while ((td = endpoint->td_head) != NULL) { 1983 endpoint->td_head = endpoint->td_head->next; 1984 FreeDescriptor(td); 1985 } 1986 1987 mutex_destroy(&endpoint->lock); 1988 memset(endpoint, 0, sizeof(xhci_endpoint)); 1989 1990 _WriteContext(&device->input_ctx->input.dropFlags, (1 << epNumber)); 1991 _WriteContext(&device->input_ctx->input.addFlags, (1 << 0)); 1992 1993 // The Deconfigure bit in the Configure Endpoint command indicates 1994 // that *all* endpoints are to be deconfigured, and not just the ones 1995 // specified in the context flags. (XHCI 1.2 § 4.6.6 p115.) 1996 ConfigureEndpoint(device->input_ctx_addr, false, device->slot); 1997 } 1998 1999 return B_OK; 2000 } 2001 2002 2003 status_t 2004 XHCI::_LinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 2005 { 2006 TRACE("link descriptor for pipe\n"); 2007 2008 // Use mutex_trylock first, in case we are in KDL. 2009 MutexLocker endpointLocker(&endpoint->lock, mutex_trylock(&endpoint->lock) == B_OK); 2010 2011 // "used" refers to the number of currently linked TDs, not the number of 2012 // used TRBs on the ring (we use 2 TRBs on the ring per transfer.) 2013 // Furthermore, we have to leave an empty item between the head and tail. 2014 if (endpoint->used >= (XHCI_MAX_TRANSFERS - 1)) { 2015 TRACE_ERROR("link descriptor for pipe: max transfers count exceeded\n"); 2016 return B_BAD_VALUE; 2017 } 2018 2019 // We do not support queuing other transfers in tandem with a fragmented one. 2020 if (endpoint->td_head != NULL && endpoint->td_head->transfer != NULL 2021 && endpoint->td_head->transfer->IsFragmented()) { 2022 TRACE_ERROR("cannot submit transfer: a fragmented transfer is queued\n"); 2023 return B_DEV_RESOURCE_CONFLICT; 2024 } 2025 2026 endpoint->used++; 2027 descriptor->next = endpoint->td_head; 2028 endpoint->td_head = descriptor; 2029 2030 uint32 link = endpoint->next, eventdata = link + 1, next = eventdata + 1; 2031 if (eventdata == XHCI_ENDPOINT_RING_SIZE || next == XHCI_ENDPOINT_RING_SIZE) { 2032 // If it's "next" not "eventdata" that got us here, we will be leaving 2033 // one TRB at the end of the ring unused. 2034 eventdata = 0; 2035 next = 1; 2036 } 2037 2038 TRACE("link descriptor for pipe: link %d, next %d\n", link, next); 2039 2040 // Add a Link TRB to the end of the descriptor. 2041 phys_addr_t addr = endpoint->trb_addr + (eventdata * sizeof(xhci_trb)); 2042 descriptor->trbs[descriptor->trb_used].address = addr; 2043 descriptor->trbs[descriptor->trb_used].status = TRB_2_IRQ(0); 2044 descriptor->trbs[descriptor->trb_used].flags = TRB_3_TYPE(TRB_TYPE_LINK) 2045 | TRB_3_CHAIN_BIT | TRB_3_CYCLE_BIT; 2046 // It is specified that (XHCI 1.2 § 4.12.3 Note 2 p251) if the TRB 2047 // following one with the ENT bit set is a Link TRB, the Link TRB 2048 // shall be evaluated *and* the subsequent TRB shall be. Thus a 2049 // TRB_3_ENT_BIT is unnecessary here; and from testing seems to 2050 // break all transfers on a (very) small number of controllers. 2051 2052 #if !B_HOST_IS_LENDIAN 2053 // Convert endianness. 2054 for (uint32 i = 0; i <= descriptor->trb_used; i++) { 2055 descriptor->trbs[i].address = 2056 B_HOST_TO_LENDIAN_INT64(descriptor->trbs[i].address); 2057 descriptor->trbs[i].status = 2058 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].status); 2059 descriptor->trbs[i].flags = 2060 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].flags); 2061 } 2062 #endif 2063 2064 // Link the descriptor. 2065 endpoint->trbs[link].address = 2066 B_HOST_TO_LENDIAN_INT64(descriptor->trb_addr); 2067 endpoint->trbs[link].status = 2068 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 2069 endpoint->trbs[link].flags = 2070 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_LINK)); 2071 2072 // Set up the Event Data TRB (XHCI 1.2 § 4.11.5.2 p230.) 2073 // 2074 // We do this on the main ring for two reasons: first, to avoid a small 2075 // potential race between the interrupt and the controller evaluating 2076 // the link TRB to get back onto the ring; and second, because many 2077 // controllers throw errors if the target of a Link TRB is not valid 2078 // (i.e. does not have its Cycle Bit set.) 2079 // 2080 // We also set the "address" field, which the controller will copy 2081 // verbatim into the TRB it posts to the event ring, to be the last 2082 // "real" TRB in the TD; this will allow us to determine what transfer 2083 // the resulting Transfer Event TRB refers to. 2084 endpoint->trbs[eventdata].address = 2085 B_HOST_TO_LENDIAN_INT64(descriptor->trb_addr 2086 + (descriptor->trb_used - 1) * sizeof(xhci_trb)); 2087 endpoint->trbs[eventdata].status = 2088 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 2089 endpoint->trbs[eventdata].flags = 2090 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_EVENT_DATA) 2091 | TRB_3_IOC_BIT | TRB_3_CYCLE_BIT); 2092 2093 endpoint->trbs[next].address = 0; 2094 endpoint->trbs[next].status = 0; 2095 endpoint->trbs[next].flags = 0; 2096 2097 memory_write_barrier(); 2098 2099 // Everything is ready, so write the cycle bit. 2100 endpoint->trbs[link].flags |= B_HOST_TO_LENDIAN_INT32(TRB_3_CYCLE_BIT); 2101 2102 TRACE("_LinkDescriptorForPipe pLink %p phys 0x%" B_PRIxPHYSADDR 2103 " 0x%" B_PRIxPHYSADDR " 0x%08" B_PRIx32 "\n", &endpoint->trbs[link], 2104 endpoint->trb_addr + link * sizeof(struct xhci_trb), 2105 endpoint->trbs[link].address, 2106 B_LENDIAN_TO_HOST_INT32(endpoint->trbs[link].flags)); 2107 2108 endpoint->next = next; 2109 endpointLocker.Unlock(); 2110 2111 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 2112 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 2113 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 2114 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 2115 2116 Ring(endpoint->device->slot, endpoint->id + 1); 2117 2118 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 2119 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 2120 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 2121 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 2122 2123 return B_OK; 2124 } 2125 2126 2127 status_t 2128 XHCI::_UnlinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 2129 { 2130 TRACE("unlink descriptor for pipe\n"); 2131 // We presume that the caller has already locked or owns the endpoint. 2132 2133 endpoint->used--; 2134 if (descriptor == endpoint->td_head) { 2135 endpoint->td_head = descriptor->next; 2136 descriptor->next = NULL; 2137 return B_OK; 2138 } else { 2139 for (xhci_td *td = endpoint->td_head; td->next != NULL; td = td->next) { 2140 if (td->next == descriptor) { 2141 td->next = descriptor->next; 2142 descriptor->next = NULL; 2143 return B_OK; 2144 } 2145 } 2146 } 2147 2148 endpoint->used++; 2149 return B_ERROR; 2150 } 2151 2152 2153 status_t 2154 XHCI::ConfigureEndpoint(xhci_endpoint* ep, uint8 slot, uint8 number, uint8 type, 2155 bool directionIn, uint16 interval, uint16 maxPacketSize, usb_speed speed, 2156 uint8 maxBurst, uint16 bytesPerInterval) 2157 { 2158 struct xhci_device* device = &fDevices[slot]; 2159 2160 uint32 dwendpoint0 = 0; 2161 uint32 dwendpoint1 = 0; 2162 uint64 qwendpoint2 = 0; 2163 uint32 dwendpoint4 = 0; 2164 2165 // Compute and assign the endpoint type. (XHCI 1.2 § 6.2.3 Table 6-9 p452.) 2166 uint8 xhciType = 4; 2167 if ((type & USB_OBJECT_INTERRUPT_PIPE) != 0) 2168 xhciType = 3; 2169 if ((type & USB_OBJECT_BULK_PIPE) != 0) 2170 xhciType = 2; 2171 if ((type & USB_OBJECT_ISO_PIPE) != 0) 2172 xhciType = 1; 2173 xhciType |= directionIn ? (1 << 2) : 0; 2174 dwendpoint1 |= ENDPOINT_1_EPTYPE(xhciType); 2175 2176 // Compute and assign interval. (XHCI 1.2 § 6.2.3.6 p456.) 2177 uint16 calcInterval; 2178 if ((type & USB_OBJECT_BULK_PIPE) != 0 2179 || (type & USB_OBJECT_CONTROL_PIPE) != 0) { 2180 // Bulk and Control endpoints never issue NAKs. 2181 calcInterval = 0; 2182 } else { 2183 switch (speed) { 2184 case USB_SPEED_FULLSPEED: 2185 if ((type & USB_OBJECT_ISO_PIPE) != 0) { 2186 // Convert 1-16 into 3-18. 2187 calcInterval = min_c(max_c(interval, 1), 16) + 2; 2188 break; 2189 } 2190 2191 // fall through 2192 case USB_SPEED_LOWSPEED: { 2193 // Convert 1ms-255ms into 3-10. 2194 2195 // Find the index of the highest set bit in "interval". 2196 uint32 temp = min_c(max_c(interval, 1), 255); 2197 for (calcInterval = 0; temp != 1; calcInterval++) 2198 temp = temp >> 1; 2199 calcInterval += 3; 2200 break; 2201 } 2202 2203 case USB_SPEED_HIGHSPEED: 2204 case USB_SPEED_SUPERSPEED: 2205 case USB_SPEED_SUPERSPEEDPLUS: 2206 default: 2207 // Convert 1-16 into 0-15. 2208 calcInterval = min_c(max_c(interval, 1), 16) - 1; 2209 break; 2210 } 2211 } 2212 dwendpoint0 |= ENDPOINT_0_INTERVAL(calcInterval); 2213 2214 // For non-isochronous endpoints, we want the controller to retry failed 2215 // transfers, if possible. (XHCI 1.2 § 4.10.2.3 p197.) 2216 if ((type & USB_OBJECT_ISO_PIPE) == 0) 2217 dwendpoint1 |= ENDPOINT_1_CERR(3); 2218 2219 // Assign maximum burst size. For USB3 devices this is passed in; for 2220 // all other devices we compute it. (XHCI 1.2 § 4.8.2 p161.) 2221 if (speed == USB_SPEED_HIGHSPEED && (type & (USB_OBJECT_INTERRUPT_PIPE 2222 | USB_OBJECT_ISO_PIPE)) != 0) { 2223 maxBurst = (maxPacketSize & 0x1800) >> 11; 2224 } else if (speed < USB_SPEED_SUPERSPEED) { 2225 maxBurst = 0; 2226 } 2227 dwendpoint1 |= ENDPOINT_1_MAXBURST(maxBurst); 2228 2229 // Assign maximum packet size, set the ring address, and set the 2230 // "Dequeue Cycle State" bit. (XHCI 1.2 § 6.2.3 Table 6-10 p453.) 2231 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE(maxPacketSize); 2232 qwendpoint2 |= ENDPOINT_2_DCS_BIT | ep->trb_addr; 2233 2234 // The Max Burst Payload is the number of bytes moved by a 2235 // maximum sized burst. (XHCI 1.2 § 4.11.7.1 p236.) 2236 ep->max_burst_payload = (maxBurst + 1) * maxPacketSize; 2237 if (ep->max_burst_payload == 0) { 2238 TRACE_ERROR("ConfigureEndpoint() failed invalid max_burst_payload\n"); 2239 return B_BAD_VALUE; 2240 } 2241 2242 // Assign average TRB length. 2243 if ((type & USB_OBJECT_CONTROL_PIPE) != 0) { 2244 // Control pipes are a special case, as they rarely have 2245 // outbound transfers of any substantial size. 2246 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(8); 2247 } else if ((type & USB_OBJECT_ISO_PIPE) != 0) { 2248 // Isochronous pipes are another special case: the TRB size will be 2249 // one packet (which is normally smaller than the max packet size, 2250 // but we don't know what it is here.) 2251 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(maxPacketSize); 2252 } else { 2253 // Under all other circumstances, we put max_burst_payload in a TRB. 2254 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(ep->max_burst_payload); 2255 } 2256 2257 // Assign maximum ESIT payload. (XHCI 1.2 § 4.14.2 p259.) 2258 if ((type & (USB_OBJECT_INTERRUPT_PIPE | USB_OBJECT_ISO_PIPE)) != 0) { 2259 // TODO: For SuperSpeedPlus endpoints, there is yet another descriptor 2260 // for isochronous endpoints that specifies the maximum ESIT payload. 2261 // We don't fetch this yet, so just fall back to the USB2 computation 2262 // method if bytesPerInterval is 0. 2263 if (speed >= USB_SPEED_SUPERSPEED && bytesPerInterval != 0) 2264 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD(bytesPerInterval); 2265 else if (speed >= USB_SPEED_HIGHSPEED) 2266 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD((maxBurst + 1) * maxPacketSize); 2267 } 2268 2269 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint0, 2270 dwendpoint0); 2271 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint1, 2272 dwendpoint1); 2273 _WriteContext(&device->input_ctx->endpoints[number].qwendpoint2, 2274 qwendpoint2); 2275 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint4, 2276 dwendpoint4); 2277 2278 TRACE("endpoint 0x%" B_PRIx32 " 0x%" B_PRIx32 " 0x%" B_PRIx64 " 0x%" 2279 B_PRIx32 "\n", 2280 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint0), 2281 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint1), 2282 _ReadContext(&device->input_ctx->endpoints[number].qwendpoint2), 2283 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint4)); 2284 2285 return B_OK; 2286 } 2287 2288 2289 status_t 2290 XHCI::GetPortSpeed(uint8 index, usb_speed* speed) 2291 { 2292 if (index >= fPortCount) 2293 return B_BAD_INDEX; 2294 2295 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 2296 2297 switch (PS_SPEED_GET(portStatus)) { 2298 case 2: 2299 *speed = USB_SPEED_LOWSPEED; 2300 break; 2301 case 1: 2302 *speed = USB_SPEED_FULLSPEED; 2303 break; 2304 case 3: 2305 *speed = USB_SPEED_HIGHSPEED; 2306 break; 2307 case 4: 2308 *speed = USB_SPEED_SUPERSPEED; 2309 break; 2310 case 5: 2311 *speed = USB_SPEED_SUPERSPEEDPLUS; 2312 break; 2313 default: 2314 TRACE_ALWAYS("nonstandard port speed %" B_PRId32 ", assuming SuperSpeed\n", 2315 PS_SPEED_GET(portStatus)); 2316 *speed = USB_SPEED_SUPERSPEED; 2317 break; 2318 } 2319 2320 return B_OK; 2321 } 2322 2323 2324 status_t 2325 XHCI::GetPortStatus(uint8 index, usb_port_status* status) 2326 { 2327 if (index >= fPortCount) 2328 return B_BAD_INDEX; 2329 2330 status->status = status->change = 0; 2331 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 2332 TRACE("port %" B_PRId8 " status=0x%08" B_PRIx32 "\n", index, portStatus); 2333 2334 // build the status 2335 switch (PS_SPEED_GET(portStatus)) { 2336 case 3: 2337 status->status |= PORT_STATUS_HIGH_SPEED; 2338 break; 2339 case 2: 2340 status->status |= PORT_STATUS_LOW_SPEED; 2341 break; 2342 default: 2343 break; 2344 } 2345 2346 if (portStatus & PS_CCS) 2347 status->status |= PORT_STATUS_CONNECTION; 2348 if (portStatus & PS_PED) 2349 status->status |= PORT_STATUS_ENABLE; 2350 if (portStatus & PS_OCA) 2351 status->status |= PORT_STATUS_OVER_CURRENT; 2352 if (portStatus & PS_PR) 2353 status->status |= PORT_STATUS_RESET; 2354 if (portStatus & PS_PP) { 2355 if (fPortSpeeds[index] >= USB_SPEED_SUPERSPEED) 2356 status->status |= PORT_STATUS_SS_POWER; 2357 else 2358 status->status |= PORT_STATUS_POWER; 2359 } 2360 if (fPortSpeeds[index] >= USB_SPEED_SUPERSPEED) 2361 status->status |= portStatus & PS_PLS_MASK; 2362 2363 // build the change 2364 if (portStatus & PS_CSC) 2365 status->change |= PORT_STATUS_CONNECTION; 2366 if (portStatus & PS_PEC) 2367 status->change |= PORT_STATUS_ENABLE; 2368 if (portStatus & PS_OCC) 2369 status->change |= PORT_STATUS_OVER_CURRENT; 2370 if (portStatus & PS_PRC) 2371 status->change |= PORT_STATUS_RESET; 2372 2373 if (fPortSpeeds[index] >= USB_SPEED_SUPERSPEED) { 2374 if (portStatus & PS_PLC) 2375 status->change |= PORT_CHANGE_LINK_STATE; 2376 if (portStatus & PS_WRC) 2377 status->change |= PORT_CHANGE_BH_PORT_RESET; 2378 } 2379 2380 return B_OK; 2381 } 2382 2383 2384 status_t 2385 XHCI::SetPortFeature(uint8 index, uint16 feature) 2386 { 2387 TRACE("set port feature index %u feature %u\n", index, feature); 2388 if (index >= fPortCount) 2389 return B_BAD_INDEX; 2390 2391 uint32 portRegister = XHCI_PORTSC(index); 2392 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2393 2394 switch (feature) { 2395 case PORT_SUSPEND: 2396 if ((portStatus & PS_PED) == 0 || (portStatus & PS_PR) 2397 || (portStatus & PS_PLS_MASK) >= PS_XDEV_U3) { 2398 TRACE_ERROR("USB core suspending device not in U0/U1/U2.\n"); 2399 return B_BAD_VALUE; 2400 } 2401 portStatus &= ~PS_PLS_MASK; 2402 WriteOpReg(portRegister, portStatus | PS_LWS | PS_XDEV_U3); 2403 break; 2404 2405 case PORT_RESET: 2406 WriteOpReg(portRegister, portStatus | PS_PR); 2407 break; 2408 2409 case PORT_POWER: 2410 WriteOpReg(portRegister, portStatus | PS_PP); 2411 break; 2412 default: 2413 return B_BAD_VALUE; 2414 } 2415 ReadOpReg(portRegister); 2416 return B_OK; 2417 } 2418 2419 2420 status_t 2421 XHCI::ClearPortFeature(uint8 index, uint16 feature) 2422 { 2423 TRACE("clear port feature index %u feature %u\n", index, feature); 2424 if (index >= fPortCount) 2425 return B_BAD_INDEX; 2426 2427 uint32 portRegister = XHCI_PORTSC(index); 2428 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2429 2430 switch (feature) { 2431 case PORT_SUSPEND: 2432 portStatus = ReadOpReg(portRegister); 2433 if (portStatus & PS_PR) 2434 return B_BAD_VALUE; 2435 if (portStatus & PS_XDEV_U3) { 2436 if ((portStatus & PS_PED) == 0) 2437 return B_BAD_VALUE; 2438 portStatus &= ~PS_PLS_MASK; 2439 WriteOpReg(portRegister, portStatus | PS_XDEV_U0 | PS_LWS); 2440 } 2441 break; 2442 case PORT_ENABLE: 2443 WriteOpReg(portRegister, portStatus | PS_PED); 2444 break; 2445 case PORT_POWER: 2446 WriteOpReg(portRegister, portStatus & ~PS_PP); 2447 break; 2448 case C_PORT_CONNECTION: 2449 WriteOpReg(portRegister, portStatus | PS_CSC); 2450 break; 2451 case C_PORT_ENABLE: 2452 WriteOpReg(portRegister, portStatus | PS_PEC); 2453 break; 2454 case C_PORT_OVER_CURRENT: 2455 WriteOpReg(portRegister, portStatus | PS_OCC); 2456 break; 2457 case C_PORT_RESET: 2458 WriteOpReg(portRegister, portStatus | PS_PRC); 2459 break; 2460 case C_PORT_BH_PORT_RESET: 2461 WriteOpReg(portRegister, portStatus | PS_WRC); 2462 break; 2463 case C_PORT_LINK_STATE: 2464 WriteOpReg(portRegister, portStatus | PS_PLC); 2465 break; 2466 default: 2467 return B_BAD_VALUE; 2468 } 2469 2470 ReadOpReg(portRegister); 2471 return B_OK; 2472 } 2473 2474 2475 status_t 2476 XHCI::ControllerHalt() 2477 { 2478 // Mask off run state 2479 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) & ~CMD_RUN); 2480 2481 // wait for shutdown state 2482 if (WaitOpBits(XHCI_STS, STS_HCH, STS_HCH) != B_OK) { 2483 TRACE_ERROR("HCH shutdown timeout\n"); 2484 return B_ERROR; 2485 } 2486 return B_OK; 2487 } 2488 2489 2490 status_t 2491 XHCI::ControllerReset() 2492 { 2493 TRACE("ControllerReset() cmd: 0x%" B_PRIx32 " sts: 0x%" B_PRIx32 "\n", 2494 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 2495 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) | CMD_HCRST); 2496 2497 if (WaitOpBits(XHCI_CMD, CMD_HCRST, 0) != B_OK) { 2498 TRACE_ERROR("ControllerReset() failed CMD_HCRST\n"); 2499 return B_ERROR; 2500 } 2501 2502 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 2503 TRACE_ERROR("ControllerReset() failed STS_CNR\n"); 2504 return B_ERROR; 2505 } 2506 2507 return B_OK; 2508 } 2509 2510 2511 int32 2512 XHCI::InterruptHandler(void* data) 2513 { 2514 return ((XHCI*)data)->Interrupt(); 2515 } 2516 2517 2518 int32 2519 XHCI::Interrupt() 2520 { 2521 SpinLocker _(&fSpinlock); 2522 2523 uint32 status = ReadOpReg(XHCI_STS); 2524 uint32 temp = ReadRunReg32(XHCI_IMAN(0)); 2525 WriteOpReg(XHCI_STS, status); 2526 WriteRunReg32(XHCI_IMAN(0), temp); 2527 2528 int32 result = B_HANDLED_INTERRUPT; 2529 2530 if ((status & STS_HCH) != 0) { 2531 TRACE_ERROR("Host Controller halted\n"); 2532 return result; 2533 } 2534 if ((status & STS_HSE) != 0) { 2535 TRACE_ERROR("Host System Error\n"); 2536 return result; 2537 } 2538 if ((status & STS_HCE) != 0) { 2539 TRACE_ERROR("Host Controller Error\n"); 2540 return result; 2541 } 2542 2543 if ((status & STS_EINT) == 0) { 2544 TRACE("STS: 0x%" B_PRIx32 " IRQ_PENDING: 0x%" B_PRIx32 "\n", 2545 status, temp); 2546 return B_UNHANDLED_INTERRUPT; 2547 } 2548 2549 TRACE("Event Interrupt\n"); 2550 release_sem_etc(fEventSem, 1, B_DO_NOT_RESCHEDULE); 2551 return B_INVOKE_SCHEDULER; 2552 } 2553 2554 2555 void 2556 XHCI::Ring(uint8 slot, uint8 endpoint) 2557 { 2558 TRACE("Ding Dong! slot:%d endpoint %d\n", slot, endpoint) 2559 if ((slot == 0 && endpoint > 0) || (slot > 0 && endpoint == 0)) 2560 panic("Ring() invalid slot/endpoint combination\n"); 2561 if (slot > fSlotCount || endpoint >= XHCI_MAX_ENDPOINTS) 2562 panic("Ring() invalid slot or endpoint\n"); 2563 2564 WriteDoorReg32(XHCI_DOORBELL(slot), XHCI_DOORBELL_TARGET(endpoint) 2565 | XHCI_DOORBELL_STREAMID(0)); 2566 ReadDoorReg32(XHCI_DOORBELL(slot)); 2567 // Flush PCI writes 2568 } 2569 2570 2571 void 2572 XHCI::QueueCommand(xhci_trb* trb) 2573 { 2574 uint8 i, j; 2575 uint32 temp; 2576 2577 i = fCmdIdx; 2578 j = fCmdCcs; 2579 2580 TRACE("command[%u] = %" B_PRId32 " (0x%016" B_PRIx64 ", 0x%08" B_PRIx32 2581 ", 0x%08" B_PRIx32 ")\n", i, TRB_3_TYPE_GET(trb->flags), trb->address, 2582 trb->status, trb->flags); 2583 2584 fCmdRing[i].address = trb->address; 2585 fCmdRing[i].status = trb->status; 2586 temp = trb->flags; 2587 2588 if (j) 2589 temp |= TRB_3_CYCLE_BIT; 2590 else 2591 temp &= ~TRB_3_CYCLE_BIT; 2592 temp &= ~TRB_3_TC_BIT; 2593 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2594 2595 fCmdAddr = fErst->rs_addr + (XHCI_MAX_EVENTS + i) * sizeof(xhci_trb); 2596 2597 i++; 2598 2599 if (i == (XHCI_MAX_COMMANDS - 1)) { 2600 temp = TRB_3_TYPE(TRB_TYPE_LINK) | TRB_3_TC_BIT; 2601 if (j) 2602 temp |= TRB_3_CYCLE_BIT; 2603 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2604 2605 i = 0; 2606 j ^= 1; 2607 } 2608 2609 fCmdIdx = i; 2610 fCmdCcs = j; 2611 } 2612 2613 2614 void 2615 XHCI::HandleCmdComplete(xhci_trb* trb) 2616 { 2617 if (fCmdAddr == trb->address) { 2618 TRACE("Received command event\n"); 2619 fCmdResult[0] = trb->status; 2620 fCmdResult[1] = B_LENDIAN_TO_HOST_INT32(trb->flags); 2621 release_sem_etc(fCmdCompSem, 1, B_DO_NOT_RESCHEDULE); 2622 } else 2623 TRACE_ERROR("received command event for unknown command!\n") 2624 } 2625 2626 2627 void 2628 XHCI::HandleTransferComplete(xhci_trb* trb) 2629 { 2630 const uint32 flags = B_LENDIAN_TO_HOST_INT32(trb->flags); 2631 const uint8 endpointNumber = TRB_3_ENDPOINT_GET(flags), 2632 slot = TRB_3_SLOT_GET(flags); 2633 2634 if (slot > fSlotCount) 2635 TRACE_ERROR("invalid slot\n"); 2636 if (endpointNumber == 0 || endpointNumber >= XHCI_MAX_ENDPOINTS) { 2637 TRACE_ERROR("invalid endpoint\n"); 2638 return; 2639 } 2640 2641 xhci_device *device = &fDevices[slot]; 2642 xhci_endpoint *endpoint = &device->endpoints[endpointNumber - 1]; 2643 2644 if (endpoint->trbs == NULL) { 2645 TRACE_ERROR("got TRB but endpoint is not allocated!\n"); 2646 return; 2647 } 2648 2649 // Use mutex_trylock first, in case we are in KDL. 2650 MutexLocker endpointLocker(endpoint->lock, mutex_trylock(&endpoint->lock) == B_OK); 2651 if (!endpointLocker.IsLocked()) { 2652 // We failed to get the lock. Most likely it was destroyed 2653 // while we were waiting for it. 2654 return; 2655 } 2656 2657 TRACE("HandleTransferComplete: ed %" B_PRIu32 ", status %" B_PRId32 "\n", 2658 (flags & TRB_3_EVENT_DATA_BIT), trb->status); 2659 2660 uint8 completionCode = TRB_2_COMP_CODE_GET(trb->status); 2661 2662 if (completionCode == COMP_RING_OVERRUN || completionCode == COMP_RING_UNDERRUN) { 2663 // These occur on isochronous endpoints when there is no TRB ready to be 2664 // executed at the appropriate time. (XHCI 1.2 § 4.10.3.1 p204.) 2665 endpoint->status = completionCode; 2666 return; 2667 } 2668 2669 int32 remainder = TRB_2_REM_GET(trb->status), transferred = -1; 2670 if ((flags & TRB_3_EVENT_DATA_BIT) != 0) { 2671 // In the case of an Event Data TRB, value in the status field refers 2672 // to the actual number of bytes transferred across the whole TD. 2673 // (XHCI 1.2 § 6.4.2.1 Table 6-38 p478.) 2674 transferred = remainder; 2675 remainder = -1; 2676 } else { 2677 // This should only occur under error conditions, or for isochronous transfers. 2678 TRACE("got transfer event for a non-Event Data TRB!\n"); 2679 2680 if (completionCode == COMP_STOPPED_LENGTH_INVALID) 2681 remainder = -1; 2682 } 2683 2684 if (completionCode != COMP_SUCCESS && completionCode != COMP_SHORT_PACKET 2685 && completionCode != COMP_STOPPED && completionCode != COMP_STOPPED_LENGTH_INVALID) { 2686 TRACE_ALWAYS("transfer error on slot %" B_PRId8 " endpoint %" B_PRId8 2687 ": %s\n", slot, endpointNumber, xhci_error_string(completionCode)); 2688 } 2689 2690 phys_addr_t source = B_LENDIAN_TO_HOST_INT64(trb->address); 2691 if (source >= endpoint->trb_addr 2692 && (source - endpoint->trb_addr) < (XHCI_ENDPOINT_RING_SIZE * sizeof(xhci_trb))) { 2693 // The "source" address points to a TRB on the ring. 2694 // See if we can figure out what it really corresponds to. 2695 const int64 offset = (source - endpoint->trb_addr) / sizeof(xhci_trb); 2696 const int32 type = TRB_3_TYPE_GET(endpoint->trbs[offset].flags); 2697 if (type == TRB_TYPE_EVENT_DATA || type == TRB_TYPE_LINK) 2698 source = B_LENDIAN_TO_HOST_INT64(endpoint->trbs[offset].address); 2699 } 2700 2701 for (xhci_td *td = endpoint->td_head; td != NULL; td = td->next) { 2702 int64 offset = (source - td->trb_addr) / sizeof(xhci_trb); 2703 if (offset < 0 || offset >= td->trb_count) 2704 continue; 2705 2706 TRACE("HandleTransferComplete td %p trb %" B_PRId64 " found\n", 2707 td, offset); 2708 2709 if (td->transfer != NULL && td->transfer->IsochronousData() != NULL) { 2710 usb_isochronous_data* isochronousData = td->transfer->IsochronousData(); 2711 usb_iso_packet_descriptor& descriptor = isochronousData->packet_descriptors[offset]; 2712 if (transferred < 0) 2713 transferred = (TRB_2_BYTES_GET(td->trbs[offset].status) - remainder); 2714 descriptor.actual_length = transferred; 2715 descriptor.status = xhci_error_status(completionCode, 2716 (td->transfer->TransferPipe()->Direction() != Pipe::Out)); 2717 2718 // Don't double-report completion status. 2719 completionCode = COMP_SUCCESS; 2720 2721 if (offset != (td->trb_used - 1)) { 2722 // We'll be sent here again. 2723 return; 2724 } 2725 2726 // Compute the real transferred length. 2727 transferred = 0; 2728 for (int32 i = 0; i < offset; i++) { 2729 usb_iso_packet_descriptor& descriptor = isochronousData->packet_descriptors[i]; 2730 if (descriptor.status == B_NO_INIT) { 2731 // Assume success. 2732 descriptor.actual_length = descriptor.request_length; 2733 descriptor.status = B_OK; 2734 } 2735 transferred += descriptor.actual_length; 2736 } 2737 2738 // Report the endpoint status (if any.) 2739 if (endpoint->status != 0) { 2740 completionCode = endpoint->status; 2741 endpoint->status = 0; 2742 } 2743 } else if (completionCode == COMP_STOPPED_LENGTH_INVALID) { 2744 // To determine transferred length, sum up the lengths of all TRBs 2745 // prior to the referenced one. (XHCI 1.2 § 4.6.9 p136.) 2746 transferred = 0; 2747 for (int32 i = 0; i < offset; i++) 2748 transferred += TRB_2_BYTES_GET(td->trbs[i].status); 2749 } 2750 2751 // The TRB at offset trb_used will be the link TRB, which we do not 2752 // care about (and should not generate an interrupt at all.) We really 2753 // care about the properly last TRB, at index "count - 1", which the 2754 // Event Data TRB that _LinkDescriptorForPipe creates points to. 2755 // 2756 // But if we have an unsuccessful completion code, the transfer 2757 // likely failed midway; so just accept it anyway. 2758 if (offset == (td->trb_used - 1) || completionCode != COMP_SUCCESS) { 2759 _UnlinkDescriptorForPipe(td, endpoint); 2760 endpointLocker.Unlock(); 2761 2762 td->trb_completion_code = completionCode; 2763 td->td_transferred = transferred; 2764 td->trb_left = remainder; 2765 2766 // add descriptor to finished list 2767 if (mutex_trylock(&fFinishedLock) != B_OK) 2768 mutex_lock(&fFinishedLock); 2769 td->next = fFinishedHead; 2770 fFinishedHead = td; 2771 mutex_unlock(&fFinishedLock); 2772 2773 release_sem_etc(fFinishTransfersSem, 1, B_DO_NOT_RESCHEDULE); 2774 TRACE("HandleTransferComplete td %p done\n", td); 2775 } else { 2776 TRACE_ERROR("successful TRB 0x%" B_PRIxPHYSADDR " was found, but it wasn't " 2777 "the last in the TD!\n", source); 2778 } 2779 return; 2780 } 2781 TRACE_ERROR("TRB 0x%" B_PRIxPHYSADDR " was not found in the endpoint!\n", source); 2782 } 2783 2784 2785 void 2786 XHCI::DumpRing(xhci_trb *trbs, uint32 size) 2787 { 2788 if (!Lock()) { 2789 TRACE("Unable to get lock!\n"); 2790 return; 2791 } 2792 2793 for (uint32 i = 0; i < size; i++) { 2794 TRACE("command[%" B_PRId32 "] = %" B_PRId32 " (0x%016" B_PRIx64 "," 2795 " 0x%08" B_PRIx32 ", 0x%08" B_PRIx32 ")\n", i, 2796 TRB_3_TYPE_GET(B_LENDIAN_TO_HOST_INT32(trbs[i].flags)), 2797 trbs[i].address, trbs[i].status, trbs[i].flags); 2798 } 2799 2800 Unlock(); 2801 } 2802 2803 2804 status_t 2805 XHCI::DoCommand(xhci_trb* trb) 2806 { 2807 if (!Lock()) { 2808 TRACE("Unable to get lock!\n"); 2809 return B_ERROR; 2810 } 2811 2812 QueueCommand(trb); 2813 Ring(0, 0); 2814 2815 // Begin with a 50ms timeout. 2816 if (acquire_sem_etc(fCmdCompSem, 1, B_RELATIVE_TIMEOUT, 50 * 1000) != B_OK) { 2817 // We've hit the timeout. In some error cases, interrupts are not 2818 // generated; so here we force the event ring to be polled once. 2819 release_sem(fEventSem); 2820 2821 // Now try again, this time with a 750ms timeout. 2822 if (acquire_sem_etc(fCmdCompSem, 1, B_RELATIVE_TIMEOUT, 2823 750 * 1000) != B_OK) { 2824 TRACE("Unable to obtain fCmdCompSem!\n"); 2825 fCmdAddr = 0; 2826 Unlock(); 2827 return B_TIMED_OUT; 2828 } 2829 } 2830 2831 // eat up sems that have been released by multiple interrupts 2832 int32 semCount = 0; 2833 get_sem_count(fCmdCompSem, &semCount); 2834 if (semCount > 0) 2835 acquire_sem_etc(fCmdCompSem, semCount, B_RELATIVE_TIMEOUT, 0); 2836 2837 status_t status = B_OK; 2838 uint32 completionCode = TRB_2_COMP_CODE_GET(fCmdResult[0]); 2839 TRACE("command complete\n"); 2840 if (completionCode != COMP_SUCCESS) { 2841 TRACE_ERROR("unsuccessful command %" B_PRId32 ", error %s (%" B_PRId32 ")\n", 2842 TRB_3_TYPE_GET(trb->flags), xhci_error_string(completionCode), 2843 completionCode); 2844 status = B_IO_ERROR; 2845 } 2846 2847 trb->status = fCmdResult[0]; 2848 trb->flags = fCmdResult[1]; 2849 2850 fCmdAddr = 0; 2851 Unlock(); 2852 return status; 2853 } 2854 2855 2856 status_t 2857 XHCI::Noop() 2858 { 2859 TRACE("Issue No-Op\n"); 2860 xhci_trb trb; 2861 trb.address = 0; 2862 trb.status = 0; 2863 trb.flags = TRB_3_TYPE(TRB_TYPE_CMD_NOOP); 2864 2865 return DoCommand(&trb); 2866 } 2867 2868 2869 status_t 2870 XHCI::EnableSlot(uint8* slot) 2871 { 2872 TRACE("Enable Slot\n"); 2873 xhci_trb trb; 2874 trb.address = 0; 2875 trb.status = 0; 2876 trb.flags = TRB_3_TYPE(TRB_TYPE_ENABLE_SLOT); 2877 2878 status_t status = DoCommand(&trb); 2879 if (status != B_OK) 2880 return status; 2881 2882 *slot = TRB_3_SLOT_GET(trb.flags); 2883 return *slot != 0 ? B_OK : B_BAD_VALUE; 2884 } 2885 2886 2887 status_t 2888 XHCI::DisableSlot(uint8 slot) 2889 { 2890 TRACE("Disable Slot\n"); 2891 xhci_trb trb; 2892 trb.address = 0; 2893 trb.status = 0; 2894 trb.flags = TRB_3_TYPE(TRB_TYPE_DISABLE_SLOT) | TRB_3_SLOT(slot); 2895 2896 return DoCommand(&trb); 2897 } 2898 2899 2900 status_t 2901 XHCI::SetAddress(uint64 inputContext, bool bsr, uint8 slot) 2902 { 2903 TRACE("Set Address\n"); 2904 xhci_trb trb; 2905 trb.address = inputContext; 2906 trb.status = 0; 2907 trb.flags = TRB_3_TYPE(TRB_TYPE_ADDRESS_DEVICE) | TRB_3_SLOT(slot); 2908 2909 if (bsr) 2910 trb.flags |= TRB_3_BSR_BIT; 2911 2912 return DoCommand(&trb); 2913 } 2914 2915 2916 status_t 2917 XHCI::ConfigureEndpoint(uint64 inputContext, bool deconfigure, uint8 slot) 2918 { 2919 TRACE("Configure Endpoint\n"); 2920 xhci_trb trb; 2921 trb.address = inputContext; 2922 trb.status = 0; 2923 trb.flags = TRB_3_TYPE(TRB_TYPE_CONFIGURE_ENDPOINT) | TRB_3_SLOT(slot); 2924 2925 if (deconfigure) 2926 trb.flags |= TRB_3_DCEP_BIT; 2927 2928 return DoCommand(&trb); 2929 } 2930 2931 2932 status_t 2933 XHCI::EvaluateContext(uint64 inputContext, uint8 slot) 2934 { 2935 TRACE("Evaluate Context\n"); 2936 xhci_trb trb; 2937 trb.address = inputContext; 2938 trb.status = 0; 2939 trb.flags = TRB_3_TYPE(TRB_TYPE_EVALUATE_CONTEXT) | TRB_3_SLOT(slot); 2940 2941 return DoCommand(&trb); 2942 } 2943 2944 2945 status_t 2946 XHCI::ResetEndpoint(bool preserve, xhci_endpoint* endpoint) 2947 { 2948 TRACE("Reset Endpoint\n"); 2949 2950 switch (_GetEndpointState(endpoint)) { 2951 case ENDPOINT_STATE_STOPPED: 2952 TRACE("Reset Endpoint: already stopped"); 2953 return B_OK; 2954 case ENDPOINT_STATE_HALTED: 2955 TRACE("Reset Endpoint: warning, weird state!"); 2956 default: 2957 break; 2958 } 2959 2960 xhci_trb trb; 2961 trb.address = 0; 2962 trb.status = 0; 2963 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_ENDPOINT) 2964 | TRB_3_SLOT(endpoint->device->slot) | TRB_3_ENDPOINT(endpoint->id + 1); 2965 if (preserve) 2966 trb.flags |= TRB_3_PRSV_BIT; 2967 2968 return DoCommand(&trb); 2969 } 2970 2971 2972 status_t 2973 XHCI::StopEndpoint(bool suspend, xhci_endpoint* endpoint) 2974 { 2975 TRACE("Stop Endpoint\n"); 2976 2977 switch (_GetEndpointState(endpoint)) { 2978 case ENDPOINT_STATE_HALTED: 2979 TRACE("Stop Endpoint: error, halted"); 2980 return B_DEV_STALLED; 2981 case ENDPOINT_STATE_STOPPED: 2982 TRACE("Stop Endpoint: already stopped"); 2983 return B_OK; 2984 default: 2985 break; 2986 } 2987 2988 xhci_trb trb; 2989 trb.address = 0; 2990 trb.status = 0; 2991 trb.flags = TRB_3_TYPE(TRB_TYPE_STOP_ENDPOINT) 2992 | TRB_3_SLOT(endpoint->device->slot) | TRB_3_ENDPOINT(endpoint->id + 1); 2993 if (suspend) 2994 trb.flags |= TRB_3_SUSPEND_ENDPOINT_BIT; 2995 2996 return DoCommand(&trb); 2997 } 2998 2999 3000 status_t 3001 XHCI::SetTRDequeue(uint64 dequeue, uint16 stream, uint8 endpoint, uint8 slot) 3002 { 3003 TRACE("Set TR Dequeue\n"); 3004 xhci_trb trb; 3005 trb.address = dequeue | ENDPOINT_2_DCS_BIT; 3006 // The DCS bit is copied from the address field as in ConfigureEndpoint. 3007 // (XHCI 1.2 § 4.6.10 p142.) 3008 trb.status = TRB_2_STREAM(stream); 3009 trb.flags = TRB_3_TYPE(TRB_TYPE_SET_TR_DEQUEUE) 3010 | TRB_3_SLOT(slot) | TRB_3_ENDPOINT(endpoint); 3011 3012 return DoCommand(&trb); 3013 } 3014 3015 3016 status_t 3017 XHCI::ResetDevice(uint8 slot) 3018 { 3019 TRACE("Reset Device\n"); 3020 xhci_trb trb; 3021 trb.address = 0; 3022 trb.status = 0; 3023 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_DEVICE) | TRB_3_SLOT(slot); 3024 3025 return DoCommand(&trb); 3026 } 3027 3028 3029 int32 3030 XHCI::EventThread(void* data) 3031 { 3032 ((XHCI *)data)->CompleteEvents(); 3033 return B_OK; 3034 } 3035 3036 3037 void 3038 XHCI::CompleteEvents() 3039 { 3040 while (!fStopThreads) { 3041 if (acquire_sem(fEventSem) < B_OK) 3042 continue; 3043 3044 // eat up sems that have been released by multiple interrupts 3045 int32 semCount = 0; 3046 get_sem_count(fEventSem, &semCount); 3047 if (semCount > 0) 3048 acquire_sem_etc(fEventSem, semCount, B_RELATIVE_TIMEOUT, 0); 3049 3050 ProcessEvents(); 3051 } 3052 } 3053 3054 3055 void 3056 XHCI::ProcessEvents() 3057 { 3058 // Use mutex_trylock first, in case we are in KDL. 3059 MutexLocker locker(fEventLock, mutex_trylock(&fEventLock) == B_OK); 3060 if (!locker.IsLocked()) { 3061 // We failed to get the lock. This really should not happen. 3062 TRACE_ERROR("failed to acquire event lock!\n"); 3063 return; 3064 } 3065 3066 uint16 i = fEventIdx; 3067 uint8 j = fEventCcs; 3068 uint8 t = 2; 3069 3070 while (1) { 3071 uint32 temp = B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags); 3072 uint8 event = TRB_3_TYPE_GET(temp); 3073 TRACE("event[%u] = %u (0x%016" B_PRIx64 " 0x%08" B_PRIx32 " 0x%08" 3074 B_PRIx32 ")\n", i, event, fEventRing[i].address, 3075 fEventRing[i].status, B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags)); 3076 uint8 k = (temp & TRB_3_CYCLE_BIT) ? 1 : 0; 3077 if (j != k) 3078 break; 3079 3080 switch (event) { 3081 case TRB_TYPE_COMMAND_COMPLETION: 3082 HandleCmdComplete(&fEventRing[i]); 3083 break; 3084 case TRB_TYPE_TRANSFER: 3085 HandleTransferComplete(&fEventRing[i]); 3086 break; 3087 case TRB_TYPE_PORT_STATUS_CHANGE: 3088 TRACE("port change detected\n"); 3089 break; 3090 default: 3091 TRACE_ERROR("Unhandled event = %u\n", event); 3092 break; 3093 } 3094 3095 i++; 3096 if (i == XHCI_MAX_EVENTS) { 3097 i = 0; 3098 j ^= 1; 3099 if (!--t) 3100 break; 3101 } 3102 } 3103 3104 fEventIdx = i; 3105 fEventCcs = j; 3106 3107 uint64 addr = fErst->rs_addr + i * sizeof(xhci_trb); 3108 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)addr | ERDP_BUSY); 3109 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(addr >> 32)); 3110 } 3111 3112 3113 int32 3114 XHCI::FinishThread(void* data) 3115 { 3116 ((XHCI *)data)->FinishTransfers(); 3117 return B_OK; 3118 } 3119 3120 3121 void 3122 XHCI::FinishTransfers() 3123 { 3124 while (!fStopThreads) { 3125 if (acquire_sem(fFinishTransfersSem) < B_OK) 3126 continue; 3127 3128 // eat up sems that have been released by multiple interrupts 3129 int32 semCount = 0; 3130 get_sem_count(fFinishTransfersSem, &semCount); 3131 if (semCount > 0) 3132 acquire_sem_etc(fFinishTransfersSem, semCount, B_RELATIVE_TIMEOUT, 0); 3133 3134 mutex_lock(&fFinishedLock); 3135 TRACE("finishing transfers\n"); 3136 while (fFinishedHead != NULL) { 3137 xhci_td* td = fFinishedHead; 3138 fFinishedHead = td->next; 3139 td->next = NULL; 3140 mutex_unlock(&fFinishedLock); 3141 3142 TRACE("finishing transfer td %p\n", td); 3143 3144 Transfer* transfer = td->transfer; 3145 if (transfer == NULL) { 3146 // No transfer? Quick way out. 3147 FreeDescriptor(td); 3148 mutex_lock(&fFinishedLock); 3149 continue; 3150 } 3151 3152 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 3153 3154 const uint8 completionCode = td->trb_completion_code; 3155 status_t callbackStatus = xhci_error_status(completionCode, directionIn); 3156 3157 size_t actualLength = transfer->FragmentLength(); 3158 if (completionCode != COMP_SUCCESS) { 3159 actualLength = td->td_transferred; 3160 if (td->td_transferred == -1) 3161 actualLength = transfer->FragmentLength() - td->trb_left; 3162 TRACE("transfer not successful, actualLength=%" B_PRIuSIZE "\n", 3163 actualLength); 3164 } 3165 3166 if (directionIn && actualLength > 0) { 3167 TRACE("copying in iov count %ld\n", transfer->VectorCount()); 3168 status_t status = transfer->PrepareKernelAccess(); 3169 if (status == B_OK) { 3170 ReadDescriptor(td, transfer->Vector(), 3171 transfer->VectorCount(), transfer->IsPhysical()); 3172 } else { 3173 callbackStatus = status; 3174 } 3175 } 3176 3177 FreeDescriptor(td); 3178 3179 // this transfer may still have data left 3180 bool finished = true; 3181 transfer->AdvanceByFragment(actualLength); 3182 if (completionCode == COMP_SUCCESS 3183 && transfer->FragmentLength() > 0) { 3184 TRACE("still %" B_PRIuSIZE " bytes left on transfer\n", 3185 transfer->FragmentLength()); 3186 callbackStatus = SubmitTransfer(transfer); 3187 finished = (callbackStatus != B_OK); 3188 } 3189 if (finished) { 3190 // The actualLength was already handled in AdvanceByFragment. 3191 transfer->Finished(callbackStatus, 0); 3192 delete transfer; 3193 } 3194 3195 mutex_lock(&fFinishedLock); 3196 } 3197 mutex_unlock(&fFinishedLock); 3198 } 3199 } 3200 3201 3202 inline void 3203 XHCI::WriteOpReg(uint32 reg, uint32 value) 3204 { 3205 *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg) = value; 3206 } 3207 3208 3209 inline uint32 3210 XHCI::ReadOpReg(uint32 reg) 3211 { 3212 return *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg); 3213 } 3214 3215 3216 inline status_t 3217 XHCI::WaitOpBits(uint32 reg, uint32 mask, uint32 expected) 3218 { 3219 int loops = 0; 3220 uint32 value = ReadOpReg(reg); 3221 while ((value & mask) != expected) { 3222 snooze(1000); 3223 value = ReadOpReg(reg); 3224 if (loops == 100) { 3225 TRACE("delay waiting on reg 0x%" B_PRIX32 " match 0x%" B_PRIX32 3226 " (0x%" B_PRIX32 ")\n", reg, expected, mask); 3227 } else if (loops > 250) { 3228 TRACE_ERROR("timeout waiting on reg 0x%" B_PRIX32 3229 " match 0x%" B_PRIX32 " (0x%" B_PRIX32 ")\n", reg, expected, 3230 mask); 3231 return B_ERROR; 3232 } 3233 loops++; 3234 } 3235 return B_OK; 3236 } 3237 3238 3239 inline uint32 3240 XHCI::ReadCapReg32(uint32 reg) 3241 { 3242 return *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg); 3243 } 3244 3245 3246 inline void 3247 XHCI::WriteCapReg32(uint32 reg, uint32 value) 3248 { 3249 *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg) = value; 3250 } 3251 3252 3253 inline uint32 3254 XHCI::ReadRunReg32(uint32 reg) 3255 { 3256 return *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg); 3257 } 3258 3259 3260 inline void 3261 XHCI::WriteRunReg32(uint32 reg, uint32 value) 3262 { 3263 *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg) = value; 3264 } 3265 3266 3267 inline uint32 3268 XHCI::ReadDoorReg32(uint32 reg) 3269 { 3270 return *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg); 3271 } 3272 3273 3274 inline void 3275 XHCI::WriteDoorReg32(uint32 reg, uint32 value) 3276 { 3277 *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg) = value; 3278 } 3279 3280 3281 inline addr_t 3282 XHCI::_OffsetContextAddr(addr_t p) 3283 { 3284 if (fContextSizeShift == 1) { 3285 // each structure is page aligned, each pointer is 32 bits aligned 3286 uint32 offset = p & ((B_PAGE_SIZE - 1) & ~31U); 3287 p += offset; 3288 } 3289 return p; 3290 } 3291 3292 inline uint32 3293 XHCI::_ReadContext(uint32* p) 3294 { 3295 p = (uint32*)_OffsetContextAddr((addr_t)p); 3296 return *p; 3297 } 3298 3299 3300 inline void 3301 XHCI::_WriteContext(uint32* p, uint32 value) 3302 { 3303 p = (uint32*)_OffsetContextAddr((addr_t)p); 3304 *p = value; 3305 } 3306 3307 3308 inline uint64 3309 XHCI::_ReadContext(uint64* p) 3310 { 3311 p = (uint64*)_OffsetContextAddr((addr_t)p); 3312 return *p; 3313 } 3314 3315 3316 inline void 3317 XHCI::_WriteContext(uint64* p, uint64 value) 3318 { 3319 p = (uint64*)_OffsetContextAddr((addr_t)p); 3320 *p = value; 3321 } 3322