1 /* 2 * Copyright 2011-2021, Haiku, Inc. All rights reserved. 3 * Distributed under the terms of the MIT License. 4 * 5 * Authors: 6 * Augustin Cavalier <waddlesplash> 7 * Jian Chiang <j.jian.chiang@gmail.com> 8 * Jérôme Duval <jerome.duval@gmail.com> 9 * Akshay Jaggi <akshay1994.leo@gmail.com> 10 * Michael Lotz <mmlr@mlotz.ch> 11 * Alexander von Gluck <kallisti5@unixzen.com> 12 */ 13 14 15 #include <stdio.h> 16 17 #include <bus/PCI.h> 18 #include <USB3.h> 19 #include <KernelExport.h> 20 21 #include <ByteOrder.h> 22 #include <util/AutoLock.h> 23 24 #include "xhci.h" 25 26 27 #define CALLED(x...) TRACE_MODULE("CALLED %s\n", __PRETTY_FUNCTION__) 28 29 30 #define USB_MODULE_NAME "xhci" 31 32 device_manager_info* gDeviceManager; 33 static usb_for_controller_interface* gUSB; 34 35 36 #define XHCI_PCI_DEVICE_MODULE_NAME "busses/usb/xhci/pci/driver_v1" 37 #define XHCI_PCI_USB_BUS_MODULE_NAME "busses/usb/xhci/device_v1" 38 39 40 typedef struct { 41 XHCI* xhci; 42 pci_device_module_info* pci; 43 pci_device* device; 44 45 pci_info pciinfo; 46 47 device_node* node; 48 device_node* driver_node; 49 } xhci_pci_sim_info; 50 51 52 // #pragma mark - 53 54 55 static status_t 56 init_bus(device_node* node, void** bus_cookie) 57 { 58 CALLED(); 59 60 driver_module_info* driver; 61 xhci_pci_sim_info* bus; 62 device_node* parent = gDeviceManager->get_parent_node(node); 63 gDeviceManager->get_driver(parent, &driver, (void**)&bus); 64 gDeviceManager->put_node(parent); 65 66 Stack *stack; 67 if (gUSB->get_stack((void**)&stack) != B_OK) 68 return B_ERROR; 69 70 XHCI *xhci = new(std::nothrow) XHCI(&bus->pciinfo, bus->pci, bus->device, stack, node); 71 if (xhci == NULL) { 72 return B_NO_MEMORY; 73 } 74 75 if (xhci->InitCheck() < B_OK) { 76 TRACE_MODULE_ERROR("bus failed init check\n"); 77 delete xhci; 78 return B_ERROR; 79 } 80 81 if (xhci->Start() != B_OK) { 82 delete xhci; 83 return B_ERROR; 84 } 85 86 *bus_cookie = xhci; 87 88 return B_OK; 89 } 90 91 92 static void 93 uninit_bus(void* bus_cookie) 94 { 95 CALLED(); 96 XHCI* xhci = (XHCI*)bus_cookie; 97 delete xhci; 98 } 99 100 101 static status_t 102 register_child_devices(void* cookie) 103 { 104 CALLED(); 105 xhci_pci_sim_info* bus = (xhci_pci_sim_info*)cookie; 106 device_node* node = bus->driver_node; 107 108 char prettyName[25]; 109 sprintf(prettyName, "XHCI Controller %" B_PRIu16, 0); 110 111 device_attr attrs[] = { 112 // properties of this controller for the usb bus manager 113 { B_DEVICE_PRETTY_NAME, B_STRING_TYPE, 114 { .string = prettyName }}, 115 { B_DEVICE_FIXED_CHILD, B_STRING_TYPE, 116 { .string = USB_FOR_CONTROLLER_MODULE_NAME }}, 117 118 // private data to identify the device 119 { NULL } 120 }; 121 122 return gDeviceManager->register_node(node, XHCI_PCI_USB_BUS_MODULE_NAME, 123 attrs, NULL, NULL); 124 } 125 126 127 static status_t 128 init_device(device_node* node, void** device_cookie) 129 { 130 CALLED(); 131 xhci_pci_sim_info* bus = (xhci_pci_sim_info*)calloc(1, 132 sizeof(xhci_pci_sim_info)); 133 if (bus == NULL) 134 return B_NO_MEMORY; 135 136 pci_device_module_info* pci; 137 pci_device* device; 138 { 139 device_node* pciParent = gDeviceManager->get_parent_node(node); 140 gDeviceManager->get_driver(pciParent, (driver_module_info**)&pci, 141 (void**)&device); 142 gDeviceManager->put_node(pciParent); 143 } 144 145 bus->pci = pci; 146 bus->device = device; 147 bus->driver_node = node; 148 149 pci_info *pciInfo = &bus->pciinfo; 150 pci->get_pci_info(device, pciInfo); 151 152 *device_cookie = bus; 153 return B_OK; 154 } 155 156 157 static void 158 uninit_device(void* device_cookie) 159 { 160 CALLED(); 161 xhci_pci_sim_info* bus = (xhci_pci_sim_info*)device_cookie; 162 free(bus); 163 } 164 165 166 static status_t 167 register_device(device_node* parent) 168 { 169 CALLED(); 170 device_attr attrs[] = { 171 {B_DEVICE_PRETTY_NAME, B_STRING_TYPE, {.string = "XHCI PCI"}}, 172 {} 173 }; 174 175 return gDeviceManager->register_node(parent, 176 XHCI_PCI_DEVICE_MODULE_NAME, attrs, NULL, NULL); 177 } 178 179 180 static float 181 supports_device(device_node* parent) 182 { 183 CALLED(); 184 const char* bus; 185 uint16 type, subType, api; 186 187 // make sure parent is a XHCI PCI device node 188 if (gDeviceManager->get_attr_string(parent, B_DEVICE_BUS, &bus, false) 189 < B_OK) { 190 return -1; 191 } 192 193 if (strcmp(bus, "pci") != 0) 194 return 0.0f; 195 196 if (gDeviceManager->get_attr_uint16(parent, B_DEVICE_SUB_TYPE, &subType, 197 false) < B_OK 198 || gDeviceManager->get_attr_uint16(parent, B_DEVICE_TYPE, &type, 199 false) < B_OK 200 || gDeviceManager->get_attr_uint16(parent, B_DEVICE_INTERFACE, &api, 201 false) < B_OK) { 202 TRACE_MODULE("Could not find type/subtype/interface attributes\n"); 203 return -1; 204 } 205 206 if (type == PCI_serial_bus && subType == PCI_usb && api == PCI_usb_xhci) { 207 pci_device_module_info* pci; 208 pci_device* device; 209 gDeviceManager->get_driver(parent, (driver_module_info**)&pci, 210 (void**)&device); 211 TRACE_MODULE("XHCI Device found!\n"); 212 213 return 0.8f; 214 } 215 216 return 0.0f; 217 } 218 219 220 static const char* 221 xhci_error_string(uint32 error) 222 { 223 switch (error) { 224 case COMP_INVALID: return "Invalid"; 225 case COMP_SUCCESS: return "Success"; 226 case COMP_DATA_BUFFER: return "Data buffer"; 227 case COMP_BABBLE: return "Babble detected"; 228 case COMP_USB_TRANSACTION: return "USB transaction"; 229 case COMP_TRB: return "TRB"; 230 case COMP_STALL: return "Stall"; 231 case COMP_RESOURCE: return "Resource"; 232 case COMP_BANDWIDTH: return "Bandwidth"; 233 case COMP_NO_SLOTS: return "No slots"; 234 case COMP_INVALID_STREAM: return "Invalid stream"; 235 case COMP_SLOT_NOT_ENABLED: return "Slot not enabled"; 236 case COMP_ENDPOINT_NOT_ENABLED: return "Endpoint not enabled"; 237 case COMP_SHORT_PACKET: return "Short packet"; 238 case COMP_RING_UNDERRUN: return "Ring underrun"; 239 case COMP_RING_OVERRUN: return "Ring overrun"; 240 case COMP_VF_RING_FULL: return "VF Event Ring Full"; 241 case COMP_PARAMETER: return "Parameter"; 242 case COMP_BANDWIDTH_OVERRUN: return "Bandwidth overrun"; 243 case COMP_CONTEXT_STATE: return "Context state"; 244 case COMP_NO_PING_RESPONSE: return "No ping response"; 245 case COMP_EVENT_RING_FULL: return "Event ring full"; 246 case COMP_INCOMPATIBLE_DEVICE: return "Incompatible device"; 247 case COMP_MISSED_SERVICE: return "Missed service"; 248 case COMP_COMMAND_RING_STOPPED: return "Command ring stopped"; 249 case COMP_COMMAND_ABORTED: return "Command aborted"; 250 case COMP_STOPPED: return "Stopped"; 251 case COMP_STOPPED_LENGTH_INVALID: return "Stopped (length invalid)"; 252 case COMP_MAX_EXIT_LATENCY: return "Max exit latency too large"; 253 case COMP_ISOC_OVERRUN: return "Isoch buffer overrun"; 254 case COMP_EVENT_LOST: return "Event lost"; 255 case COMP_UNDEFINED: return "Undefined"; 256 case COMP_INVALID_STREAM_ID: return "Invalid stream ID"; 257 case COMP_SECONDARY_BANDWIDTH: return "Secondary bandwidth"; 258 case COMP_SPLIT_TRANSACTION: return "Split transaction"; 259 260 default: return "Undefined"; 261 } 262 } 263 264 265 module_dependency module_dependencies[] = { 266 { USB_FOR_CONTROLLER_MODULE_NAME, (module_info**)&gUSB }, 267 { B_DEVICE_MANAGER_MODULE_NAME, (module_info**)&gDeviceManager }, 268 {} 269 }; 270 271 272 static usb_bus_interface gXHCIPCIDeviceModule = { 273 { 274 { 275 XHCI_PCI_USB_BUS_MODULE_NAME, 276 0, 277 NULL 278 }, 279 NULL, // supports device 280 NULL, // register device 281 init_bus, 282 uninit_bus, 283 NULL, // register child devices 284 NULL, // rescan 285 NULL, // device removed 286 }, 287 }; 288 289 // Root device that binds to the PCI bus. It will register an usb_bus_interface 290 // node for each device. 291 static driver_module_info sXHCIDevice = { 292 { 293 XHCI_PCI_DEVICE_MODULE_NAME, 294 0, 295 NULL 296 }, 297 supports_device, 298 register_device, 299 init_device, 300 uninit_device, 301 register_child_devices, 302 NULL, // rescan 303 NULL, // device removed 304 }; 305 306 module_info* modules[] = { 307 (module_info* )&sXHCIDevice, 308 (module_info* )&gXHCIPCIDeviceModule, 309 NULL 310 }; 311 312 313 XHCI::XHCI(pci_info *info, pci_device_module_info* pci, pci_device* device, Stack *stack, 314 device_node* node) 315 : BusManager(stack, node), 316 fRegisterArea(-1), 317 fRegisters(NULL), 318 fPCIInfo(info), 319 fPci(pci), 320 fDevice(device), 321 fStack(stack), 322 fIRQ(0), 323 fUseMSI(false), 324 fErstArea(-1), 325 fDcbaArea(-1), 326 fCmdCompSem(-1), 327 fStopThreads(false), 328 fRootHub(NULL), 329 fPortCount(0), 330 fSlotCount(0), 331 fScratchpadCount(0), 332 fContextSizeShift(0), 333 fFinishedHead(NULL), 334 fFinishTransfersSem(-1), 335 fFinishThread(-1), 336 fEventSem(-1), 337 fEventThread(-1), 338 fEventIdx(0), 339 fCmdIdx(0), 340 fEventCcs(1), 341 fCmdCcs(1) 342 { 343 B_INITIALIZE_SPINLOCK(&fSpinlock); 344 mutex_init(&fFinishedLock, "XHCI finished transfers"); 345 mutex_init(&fEventLock, "XHCI event handler"); 346 347 if (BusManager::InitCheck() < B_OK) { 348 TRACE_ERROR("bus manager failed to init\n"); 349 return; 350 } 351 352 TRACE("constructing new XHCI host controller driver\n"); 353 fInitOK = false; 354 355 // enable busmaster and memory mapped access 356 uint16 command = fPci->read_pci_config(fDevice, PCI_command, 2); 357 command &= ~(PCI_command_io | PCI_command_int_disable); 358 command |= PCI_command_master | PCI_command_memory; 359 360 fPci->write_pci_config(fDevice, PCI_command, 2, command); 361 362 // map the registers (low + high for 64-bit when requested) 363 phys_addr_t physicalAddress = fPCIInfo->u.h0.base_registers[0]; 364 if ((fPCIInfo->u.h0.base_register_flags[0] & PCI_address_type) 365 == PCI_address_type_64) { 366 physicalAddress |= (uint64)fPCIInfo->u.h0.base_registers[1] << 32; 367 } 368 369 size_t mapSize = fPCIInfo->u.h0.base_register_sizes[0]; 370 371 TRACE("map registers %08" B_PRIxPHYSADDR ", size: %" B_PRIuSIZE "\n", 372 physicalAddress, mapSize); 373 374 fRegisterArea = map_physical_memory("XHCI memory mapped registers", 375 physicalAddress, mapSize, B_ANY_KERNEL_BLOCK_ADDRESS, 376 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 377 (void **)&fRegisters); 378 if (fRegisterArea < B_OK) { 379 TRACE_ERROR("failed to map register memory\n"); 380 return; 381 } 382 383 // determine the register offsets 384 fCapabilityRegisterOffset = 0; 385 fOperationalRegisterOffset = HCI_CAPLENGTH(ReadCapReg32(XHCI_HCI_CAPLENGTH)); 386 fRuntimeRegisterOffset = ReadCapReg32(XHCI_RTSOFF) & ~0x1F; 387 fDoorbellRegisterOffset = ReadCapReg32(XHCI_DBOFF) & ~0x3; 388 389 TRACE("mapped registers: %p\n", fRegisters); 390 TRACE("operational register offset: %" B_PRId32 "\n", fOperationalRegisterOffset); 391 TRACE("runtime register offset: %" B_PRId32 "\n", fRuntimeRegisterOffset); 392 TRACE("doorbell register offset: %" B_PRId32 "\n", fDoorbellRegisterOffset); 393 394 int32 interfaceVersion = HCI_VERSION(ReadCapReg32(XHCI_HCI_VERSION)); 395 if (interfaceVersion < 0x0090 || interfaceVersion > 0x0120) { 396 TRACE_ERROR("unsupported interface version: 0x%04" B_PRIx32 "\n", 397 interfaceVersion); 398 return; 399 } 400 TRACE_ALWAYS("interface version: 0x%04" B_PRIx32 "\n", interfaceVersion); 401 402 TRACE_ALWAYS("structural parameters: 1:0x%08" B_PRIx32 " 2:0x%08" 403 B_PRIx32 " 3:0x%08" B_PRIx32 "\n", ReadCapReg32(XHCI_HCSPARAMS1), 404 ReadCapReg32(XHCI_HCSPARAMS2), ReadCapReg32(XHCI_HCSPARAMS3)); 405 406 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 407 if (cparams == 0xffffffff) 408 return; 409 TRACE_ALWAYS("capability parameters: 0x%08" B_PRIx32 "\n", cparams); 410 411 // if 64 bytes context structures, then 1 412 fContextSizeShift = HCC_CSZ(cparams); 413 414 // Assume ownership of the controller from the BIOS. 415 uint32 eec = 0xffffffff; 416 uint32 eecp = HCS0_XECP(cparams) << 2; 417 for (; eecp != 0 && XECP_NEXT(eec); eecp += XECP_NEXT(eec) << 2) { 418 TRACE("eecp register: 0x%08" B_PRIx32 "\n", eecp); 419 420 eec = ReadCapReg32(eecp); 421 if (XECP_ID(eec) != XHCI_LEGSUP_CAPID) 422 continue; 423 424 if (eec & XHCI_LEGSUP_BIOSOWNED) { 425 TRACE_ALWAYS("the host controller is bios owned, claiming" 426 " ownership\n"); 427 WriteCapReg32(eecp, eec | XHCI_LEGSUP_OSOWNED); 428 429 for (int32 i = 0; i < 20; i++) { 430 eec = ReadCapReg32(eecp); 431 432 if ((eec & XHCI_LEGSUP_BIOSOWNED) == 0) 433 break; 434 435 TRACE_ALWAYS("controller is still bios owned, waiting\n"); 436 snooze(50000); 437 } 438 439 if (eec & XHCI_LEGSUP_BIOSOWNED) { 440 TRACE_ERROR("bios won't give up control over the host " 441 "controller (ignoring)\n"); 442 } else if (eec & XHCI_LEGSUP_OSOWNED) { 443 TRACE_ALWAYS("successfully took ownership of the host " 444 "controller\n"); 445 } 446 447 // Force off the BIOS owned flag, and clear all SMIs. Some BIOSes 448 // do indicate a successful handover but do not remove their SMIs 449 // and then freeze the system when interrupts are generated. 450 WriteCapReg32(eecp, eec & ~XHCI_LEGSUP_BIOSOWNED); 451 } 452 break; 453 } 454 uint32 legctlsts = ReadCapReg32(eecp + XHCI_LEGCTLSTS); 455 legctlsts &= XHCI_LEGCTLSTS_DISABLE_SMI; 456 legctlsts |= XHCI_LEGCTLSTS_EVENTS_SMI; 457 WriteCapReg32(eecp + XHCI_LEGCTLSTS, legctlsts); 458 459 // We need to explicitly take ownership of EHCI ports on earlier Intel chipsets. 460 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL) { 461 switch (fPCIInfo->device_id) { 462 case PCI_DEVICE_INTEL_PANTHER_POINT_XHCI: 463 case PCI_DEVICE_INTEL_LYNX_POINT_XHCI: 464 case PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI: 465 case PCI_DEVICE_INTEL_BAYTRAIL_XHCI: 466 case PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI: 467 case PCI_DEVICE_INTEL_WILDCAT_POINT_LP_XHCI: 468 _SwitchIntelPorts(); 469 break; 470 } 471 } 472 473 // halt the host controller 474 if (ControllerHalt() < B_OK) { 475 return; 476 } 477 478 // reset the host controller 479 if (ControllerReset() < B_OK) { 480 TRACE_ERROR("host controller failed to reset\n"); 481 return; 482 } 483 484 fCmdCompSem = create_sem(0, "XHCI Command Complete"); 485 fFinishTransfersSem = create_sem(0, "XHCI Finish Transfers"); 486 fEventSem = create_sem(0, "XHCI Event"); 487 if (fFinishTransfersSem < B_OK || fCmdCompSem < B_OK || fEventSem < B_OK) { 488 TRACE_ERROR("failed to create semaphores\n"); 489 return; 490 } 491 492 // create event handler thread 493 fEventThread = spawn_kernel_thread(EventThread, "xhci event thread", 494 B_URGENT_PRIORITY, (void *)this); 495 resume_thread(fEventThread); 496 497 // create finisher service thread 498 fFinishThread = spawn_kernel_thread(FinishThread, "xhci finish thread", 499 B_URGENT_PRIORITY - 1, (void *)this); 500 resume_thread(fFinishThread); 501 502 // Find the right interrupt vector, using MSIs if available. 503 fIRQ = fPCIInfo->u.h0.interrupt_line; 504 #if 0 505 if (fPci->get_msix_count(fDevice) >= 1) { 506 uint8 msiVector = 0; 507 if (fPci->configure_msix(fDevice, 1, &msiVector) == B_OK 508 && fPci->enable_msix(fDevice) == B_OK) { 509 TRACE_ALWAYS("using MSI-X\n"); 510 fIRQ = msiVector; 511 fUseMSI = true; 512 } 513 } else 514 #endif 515 if (fPci->get_msi_count(fDevice) >= 1) { 516 uint8 msiVector = 0; 517 if (fPci->configure_msi(fDevice, 1, &msiVector) == B_OK 518 && fPci->enable_msi(fDevice) == B_OK) { 519 TRACE_ALWAYS("using message signaled interrupts\n"); 520 fIRQ = msiVector; 521 fUseMSI = true; 522 } 523 } 524 525 if (fIRQ == 0 || fIRQ == 0xFF) { 526 TRACE_MODULE_ERROR("device PCI:%d:%d:%d was assigned an invalid IRQ\n", 527 fPCIInfo->bus, fPCIInfo->device, fPCIInfo->function); 528 return; 529 } 530 531 // Install the interrupt handler 532 TRACE("installing interrupt handler\n"); 533 install_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this, 0); 534 535 memset(fPortSpeeds, 0, sizeof(fPortSpeeds)); 536 memset(fDevices, 0, sizeof(fDevices)); 537 538 fInitOK = true; 539 TRACE("driver construction successful\n"); 540 } 541 542 543 XHCI::~XHCI() 544 { 545 TRACE("tear down XHCI host controller driver\n"); 546 547 WriteOpReg(XHCI_CMD, 0); 548 549 int32 result = 0; 550 fStopThreads = true; 551 delete_sem(fCmdCompSem); 552 delete_sem(fFinishTransfersSem); 553 delete_sem(fEventSem); 554 wait_for_thread(fFinishThread, &result); 555 wait_for_thread(fEventThread, &result); 556 557 mutex_destroy(&fFinishedLock); 558 mutex_destroy(&fEventLock); 559 560 remove_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this); 561 562 delete_area(fRegisterArea); 563 delete_area(fErstArea); 564 for (uint32 i = 0; i < fScratchpadCount; i++) 565 delete_area(fScratchpadArea[i]); 566 delete_area(fDcbaArea); 567 568 if (fUseMSI) { 569 fPci->disable_msi(fDevice); 570 fPci->unconfigure_msi(fDevice); 571 } 572 } 573 574 575 void 576 XHCI::_SwitchIntelPorts() 577 { 578 TRACE("Looking for EHCI owned ports\n"); 579 uint32 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_USB3PRM, 4); 580 TRACE("Superspeed Ports: 0x%" B_PRIx32 "\n", ports); 581 fPci->write_pci_config(fDevice, XHCI_INTEL_USB3_PSSEN, 4, ports); 582 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_USB3_PSSEN, 4); 583 TRACE("Superspeed ports now under XHCI : 0x%" B_PRIx32 "\n", ports); 584 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_USB2PRM, 4); 585 TRACE("USB 2.0 Ports : 0x%" B_PRIx32 "\n", ports); 586 fPci->write_pci_config(fDevice, XHCI_INTEL_XUSB2PR, 4, ports); 587 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_XUSB2PR, 4); 588 TRACE("USB 2.0 ports now under XHCI: 0x%" B_PRIx32 "\n", ports); 589 } 590 591 592 status_t 593 XHCI::Start() 594 { 595 TRACE_ALWAYS("starting XHCI host controller\n"); 596 TRACE("usbcmd: 0x%08" B_PRIx32 "; usbsts: 0x%08" B_PRIx32 "\n", 597 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 598 599 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 600 TRACE("Start() failed STS_CNR\n"); 601 } 602 603 if ((ReadOpReg(XHCI_CMD) & CMD_RUN) != 0) { 604 TRACE_ERROR("Start() warning, starting running XHCI controller!\n"); 605 } 606 607 if ((ReadOpReg(XHCI_PAGESIZE) & (1 << 0)) == 0) { 608 TRACE_ERROR("controller does not support 4K page size\n"); 609 return B_ERROR; 610 } 611 612 // read port count from capability register 613 uint32 capabilities = ReadCapReg32(XHCI_HCSPARAMS1); 614 fPortCount = HCS_MAX_PORTS(capabilities); 615 if (fPortCount == 0) { 616 TRACE_ERROR("invalid number of ports: %u\n", fPortCount); 617 return B_ERROR; 618 } 619 620 fSlotCount = HCS_MAX_SLOTS(capabilities); 621 if (fSlotCount > XHCI_MAX_DEVICES) 622 fSlotCount = XHCI_MAX_DEVICES; 623 WriteOpReg(XHCI_CONFIG, fSlotCount); 624 625 // find out which protocol is used for each port 626 uint8 portFound = 0; 627 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 628 uint32 eec = 0xffffffff; 629 uint32 eecp = HCS0_XECP(cparams) << 2; 630 for (; eecp != 0 && XECP_NEXT(eec) && portFound < fPortCount; 631 eecp += XECP_NEXT(eec) << 2) { 632 eec = ReadCapReg32(eecp); 633 if (XECP_ID(eec) != XHCI_SUPPORTED_PROTOCOLS_CAPID) 634 continue; 635 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) > 3) 636 continue; 637 uint32 temp = ReadCapReg32(eecp + 8); 638 uint32 offset = XHCI_SUPPORTED_PROTOCOLS_1_OFFSET(temp); 639 uint32 count = XHCI_SUPPORTED_PROTOCOLS_1_COUNT(temp); 640 if (offset == 0 || count == 0) 641 continue; 642 offset--; 643 for (uint32 i = offset; i < offset + count; i++) { 644 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) == 0x3) 645 fPortSpeeds[i] = USB_SPEED_SUPERSPEED; 646 else 647 fPortSpeeds[i] = USB_SPEED_HIGHSPEED; 648 649 TRACE("speed for port %" B_PRId32 " is %s\n", i, 650 fPortSpeeds[i] == USB_SPEED_SUPERSPEED ? "super" : "high"); 651 } 652 portFound += count; 653 } 654 655 uint32 params2 = ReadCapReg32(XHCI_HCSPARAMS2); 656 fScratchpadCount = HCS_MAX_SC_BUFFERS(params2); 657 if (fScratchpadCount > XHCI_MAX_SCRATCHPADS) { 658 TRACE_ERROR("invalid number of scratchpads: %" B_PRIu32 "\n", 659 fScratchpadCount); 660 return B_ERROR; 661 } 662 663 uint32 params3 = ReadCapReg32(XHCI_HCSPARAMS3); 664 fExitLatMax = HCS_U1_DEVICE_LATENCY(params3) 665 + HCS_U2_DEVICE_LATENCY(params3); 666 667 // clear interrupts & disable device notifications 668 WriteOpReg(XHCI_STS, ReadOpReg(XHCI_STS)); 669 WriteOpReg(XHCI_DNCTRL, 0); 670 671 // allocate Device Context Base Address array 672 phys_addr_t dmaAddress; 673 fDcbaArea = fStack->AllocateArea((void **)&fDcba, &dmaAddress, 674 sizeof(*fDcba), "DCBA Area"); 675 if (fDcbaArea < B_OK) { 676 TRACE_ERROR("unable to create the DCBA area\n"); 677 return B_ERROR; 678 } 679 memset(fDcba, 0, sizeof(*fDcba)); 680 memset(fScratchpadArea, 0, sizeof(fScratchpadArea)); 681 memset(fScratchpad, 0, sizeof(fScratchpad)); 682 683 // setting the first address to the scratchpad array address 684 fDcba->baseAddress[0] = dmaAddress 685 + offsetof(struct xhci_device_context_array, scratchpad); 686 687 // fill up the scratchpad array with scratchpad pages 688 for (uint32 i = 0; i < fScratchpadCount; i++) { 689 phys_addr_t scratchDmaAddress; 690 fScratchpadArea[i] = fStack->AllocateArea((void **)&fScratchpad[i], 691 &scratchDmaAddress, B_PAGE_SIZE, "Scratchpad Area"); 692 if (fScratchpadArea[i] < B_OK) { 693 TRACE_ERROR("unable to create the scratchpad area\n"); 694 return B_ERROR; 695 } 696 fDcba->scratchpad[i] = scratchDmaAddress; 697 } 698 699 TRACE("setting DCBAAP %" B_PRIxPHYSADDR "\n", dmaAddress); 700 WriteOpReg(XHCI_DCBAAP_LO, (uint32)dmaAddress); 701 WriteOpReg(XHCI_DCBAAP_HI, (uint32)(dmaAddress >> 32)); 702 703 // allocate Event Ring Segment Table 704 uint8 *addr; 705 fErstArea = fStack->AllocateArea((void **)&addr, &dmaAddress, 706 (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 707 + sizeof(xhci_erst_element), 708 "USB XHCI ERST CMD_RING and EVENT_RING Area"); 709 710 if (fErstArea < B_OK) { 711 TRACE_ERROR("unable to create the ERST AND RING area\n"); 712 delete_area(fDcbaArea); 713 return B_ERROR; 714 } 715 fErst = (xhci_erst_element *)addr; 716 memset(fErst, 0, (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 717 + sizeof(xhci_erst_element)); 718 719 // fill with Event Ring Segment Base Address and Event Ring Segment Size 720 fErst->rs_addr = dmaAddress + sizeof(xhci_erst_element); 721 fErst->rs_size = XHCI_MAX_EVENTS; 722 fErst->rsvdz = 0; 723 724 addr += sizeof(xhci_erst_element); 725 fEventRing = (xhci_trb *)addr; 726 addr += XHCI_MAX_EVENTS * sizeof(xhci_trb); 727 fCmdRing = (xhci_trb *)addr; 728 729 TRACE("setting ERST size\n"); 730 WriteRunReg32(XHCI_ERSTSZ(0), XHCI_ERSTS_SET(1)); 731 732 TRACE("setting ERDP addr = 0x%" B_PRIx64 "\n", fErst->rs_addr); 733 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)fErst->rs_addr); 734 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(fErst->rs_addr >> 32)); 735 736 TRACE("setting ERST base addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 737 WriteRunReg32(XHCI_ERSTBA_LO(0), (uint32)dmaAddress); 738 WriteRunReg32(XHCI_ERSTBA_HI(0), (uint32)(dmaAddress >> 32)); 739 740 dmaAddress += sizeof(xhci_erst_element) + XHCI_MAX_EVENTS 741 * sizeof(xhci_trb); 742 743 // Make sure the Command Ring is stopped 744 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 745 TRACE_ALWAYS("Command Ring is running, send stop/cancel\n"); 746 WriteOpReg(XHCI_CRCR_LO, CRCR_CS); 747 WriteOpReg(XHCI_CRCR_HI, 0); 748 WriteOpReg(XHCI_CRCR_LO, CRCR_CA); 749 WriteOpReg(XHCI_CRCR_HI, 0); 750 snooze(1000); 751 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 752 TRACE_ERROR("Command Ring still running after stop/cancel\n"); 753 } 754 } 755 TRACE("setting CRCR addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 756 WriteOpReg(XHCI_CRCR_LO, (uint32)dmaAddress | CRCR_RCS); 757 WriteOpReg(XHCI_CRCR_HI, (uint32)(dmaAddress >> 32)); 758 // link trb 759 fCmdRing[XHCI_MAX_COMMANDS - 1].address = dmaAddress; 760 761 TRACE("setting interrupt rate\n"); 762 763 // Setting IMOD below 0x3F8 on Intel Lynx Point can cause IRQ lockups 764 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL 765 && (fPCIInfo->device_id == PCI_DEVICE_INTEL_PANTHER_POINT_XHCI 766 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_XHCI 767 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI 768 || fPCIInfo->device_id == PCI_DEVICE_INTEL_BAYTRAIL_XHCI 769 || fPCIInfo->device_id == PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI)) { 770 WriteRunReg32(XHCI_IMOD(0), 0x000003f8); // 4000 irq/s 771 } else { 772 WriteRunReg32(XHCI_IMOD(0), 0x000001f4); // 8000 irq/s 773 } 774 775 TRACE("enabling interrupt\n"); 776 WriteRunReg32(XHCI_IMAN(0), ReadRunReg32(XHCI_IMAN(0)) | IMAN_INTR_ENA); 777 778 WriteOpReg(XHCI_CMD, CMD_RUN | CMD_INTE | CMD_HSEE); 779 780 // wait for start up state 781 if (WaitOpBits(XHCI_STS, STS_HCH, 0) != B_OK) { 782 TRACE_ERROR("HCH start up timeout\n"); 783 } 784 785 fRootHub = new(std::nothrow) XHCIRootHub(RootObject(), 1); 786 if (!fRootHub) { 787 TRACE_ERROR("no memory to allocate root hub\n"); 788 return B_NO_MEMORY; 789 } 790 791 if (fRootHub->InitCheck() < B_OK) { 792 TRACE_ERROR("root hub failed init check\n"); 793 return fRootHub->InitCheck(); 794 } 795 796 SetRootHub(fRootHub); 797 798 fRootHub->RegisterNode(Node()); 799 800 TRACE_ALWAYS("successfully started the controller\n"); 801 802 #ifdef TRACE_USB 803 TRACE("No-Op test...\n"); 804 Noop(); 805 #endif 806 807 return BusManager::Start(); 808 } 809 810 811 status_t 812 XHCI::SubmitTransfer(Transfer *transfer) 813 { 814 // short circuit the root hub 815 if (transfer->TransferPipe()->DeviceAddress() == 1) 816 return fRootHub->ProcessTransfer(this, transfer); 817 818 TRACE("SubmitTransfer(%p)\n", transfer); 819 Pipe *pipe = transfer->TransferPipe(); 820 if ((pipe->Type() & USB_OBJECT_CONTROL_PIPE) != 0) 821 return SubmitControlRequest(transfer); 822 return SubmitNormalRequest(transfer); 823 } 824 825 826 status_t 827 XHCI::SubmitControlRequest(Transfer *transfer) 828 { 829 Pipe *pipe = transfer->TransferPipe(); 830 usb_request_data *requestData = transfer->RequestData(); 831 bool directionIn = (requestData->RequestType & USB_REQTYPE_DEVICE_IN) != 0; 832 833 TRACE("SubmitControlRequest() length %d\n", requestData->Length); 834 835 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 836 if (endpoint == NULL) { 837 TRACE_ERROR("control pipe has no endpoint!\n"); 838 return B_BAD_VALUE; 839 } 840 if (endpoint->device == NULL) { 841 panic("endpoint is not initialized!"); 842 return B_NO_INIT; 843 } 844 845 status_t status = transfer->InitKernelAccess(); 846 if (status != B_OK) 847 return status; 848 849 xhci_td *descriptor = CreateDescriptor(3, 1, requestData->Length); 850 if (descriptor == NULL) 851 return B_NO_MEMORY; 852 descriptor->transfer = transfer; 853 854 // Setup Stage 855 uint8 index = 0; 856 memcpy(&descriptor->trbs[index].address, requestData, 857 sizeof(usb_request_data)); 858 descriptor->trbs[index].status = TRB_2_IRQ(0) | TRB_2_BYTES(8); 859 descriptor->trbs[index].flags 860 = TRB_3_TYPE(TRB_TYPE_SETUP_STAGE) | TRB_3_IDT_BIT | TRB_3_CYCLE_BIT; 861 if (requestData->Length > 0) { 862 descriptor->trbs[index].flags |= 863 directionIn ? TRB_3_TRT_IN : TRB_3_TRT_OUT; 864 } 865 866 index++; 867 868 // Data Stage (if any) 869 if (requestData->Length > 0) { 870 descriptor->trbs[index].address = descriptor->buffer_addrs[0]; 871 descriptor->trbs[index].status = TRB_2_IRQ(0) 872 | TRB_2_BYTES(requestData->Length) 873 | TRB_2_TD_SIZE(0); 874 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_DATA_STAGE) 875 | (directionIn ? TRB_3_DIR_IN : 0) 876 | TRB_3_CYCLE_BIT; 877 878 if (!directionIn) { 879 transfer->PrepareKernelAccess(); 880 WriteDescriptor(descriptor, transfer->Vector(), 881 transfer->VectorCount(), transfer->IsPhysical()); 882 } 883 884 index++; 885 } 886 887 // Status Stage 888 descriptor->trbs[index].address = 0; 889 descriptor->trbs[index].status = TRB_2_IRQ(0); 890 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_STATUS_STAGE) 891 | TRB_3_CHAIN_BIT | TRB_3_ENT_BIT | TRB_3_CYCLE_BIT; 892 // The CHAIN bit must be set when using an Event Data TRB 893 // (XHCI 1.2 § 6.4.1.2.3 Table 6-31 p472). 894 895 // Status Stage is an OUT transfer when the device is sending data 896 // (XHCI 1.2 § 4.11.2.2 Table 4-7 p213), otherwise set the IN bit. 897 if (requestData->Length == 0 || !directionIn) 898 descriptor->trbs[index].flags |= TRB_3_DIR_IN; 899 900 descriptor->trb_used = index + 1; 901 902 status = _LinkDescriptorForPipe(descriptor, endpoint); 903 if (status != B_OK) { 904 FreeDescriptor(descriptor); 905 return status; 906 } 907 908 return B_OK; 909 } 910 911 912 status_t 913 XHCI::SubmitNormalRequest(Transfer *transfer) 914 { 915 TRACE("SubmitNormalRequest() length %" B_PRIuSIZE "\n", transfer->FragmentLength()); 916 917 Pipe *pipe = transfer->TransferPipe(); 918 usb_isochronous_data *isochronousData = transfer->IsochronousData(); 919 bool directionIn = (pipe->Direction() == Pipe::In); 920 921 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 922 if (endpoint == NULL) { 923 TRACE_ERROR("pipe has no endpoint!\n"); 924 return B_BAD_VALUE; 925 } 926 if (endpoint->device == NULL) { 927 panic("endpoint is not initialized!"); 928 return B_NO_INIT; 929 } 930 931 status_t status = transfer->InitKernelAccess(); 932 if (status != B_OK) 933 return status; 934 935 // TRBs within a TD must be "grouped" into TD Fragments, which mostly means 936 // that a max_burst_payload boundary cannot be crossed within a TRB, but 937 // only between TRBs. More than one TRB can be in a TD Fragment, but we keep 938 // things simple by setting trbSize to the MBP. (XHCI 1.2 § 4.11.7.1 p235.) 939 size_t trbSize = endpoint->max_burst_payload; 940 941 if (isochronousData != NULL) { 942 if (isochronousData->packet_count == 0) 943 return B_BAD_VALUE; 944 945 // Isochronous transfers use more specifically sized packets. 946 trbSize = transfer->DataLength() / isochronousData->packet_count; 947 if (trbSize == 0 || trbSize > pipe->MaxPacketSize() || trbSize 948 != (size_t)isochronousData->packet_descriptors[0].request_length) 949 return B_BAD_VALUE; 950 } 951 952 // Now that we know trbSize, compute the count. 953 const int32 trbCount = (transfer->FragmentLength() + trbSize - 1) / trbSize; 954 955 xhci_td *td = CreateDescriptor(trbCount, trbCount, trbSize); 956 if (td == NULL) 957 return B_NO_MEMORY; 958 959 // Normal Stage 960 const size_t maxPacketSize = pipe->MaxPacketSize(); 961 size_t remaining = transfer->FragmentLength(); 962 for (int32 i = 0; i < trbCount; i++) { 963 int32 trbLength = (remaining < trbSize) ? remaining : trbSize; 964 remaining -= trbLength; 965 966 // The "TD Size" field of a transfer TRB indicates the number of 967 // remaining maximum-size *packets* in this TD, *not* including the 968 // packets in the current TRB, and capped at 31 if there are more 969 // than 31 packets remaining in the TD. (XHCI 1.2 § 4.11.2.4 p218.) 970 int32 tdSize = (remaining + maxPacketSize - 1) / maxPacketSize; 971 if (tdSize > 31) 972 tdSize = 31; 973 974 td->trbs[i].address = td->buffer_addrs[i]; 975 td->trbs[i].status = TRB_2_IRQ(0) 976 | TRB_2_BYTES(trbLength) 977 | TRB_2_TD_SIZE(tdSize); 978 td->trbs[i].flags = TRB_3_TYPE(TRB_TYPE_NORMAL) 979 | TRB_3_CYCLE_BIT | TRB_3_CHAIN_BIT; 980 981 td->trb_used++; 982 } 983 984 // Isochronous-specific 985 if (isochronousData != NULL) { 986 // This is an isochronous transfer; we need to make the first TRB 987 // an isochronous TRB. 988 td->trbs[0].flags &= ~(TRB_3_TYPE(TRB_TYPE_NORMAL)); 989 td->trbs[0].flags |= TRB_3_TYPE(TRB_TYPE_ISOCH); 990 991 // Isochronous pipes are scheduled by microframes, one of which 992 // is 125us for USB 2 and above. But for USB 1 it was 1ms, so 993 // we need to use a different frame delta for that case. 994 uint8 frameDelta = 1; 995 if (transfer->TransferPipe()->Speed() == USB_SPEED_FULLSPEED) 996 frameDelta = 8; 997 998 // TODO: We do not currently take Mult into account at all! 999 // How are we supposed to do that here? 1000 1001 // Determine the (starting) frame number: if ISO_ASAP is set, 1002 // we are queueing this "right away", and so want to reset 1003 // the starting_frame_number. Otherwise we use the passed one. 1004 uint32 frame; 1005 if ((isochronousData->flags & USB_ISO_ASAP) != 0 1006 || isochronousData->starting_frame_number == NULL) { 1007 // All reads from the microframe index register must be 1008 // incremented by 1. (XHCI 1.2 § 4.14.2.1.4 p265.) 1009 frame = ReadRunReg32(XHCI_MFINDEX) + 1; 1010 td->trbs[0].flags |= TRB_3_ISO_SIA_BIT; 1011 } else { 1012 frame = *isochronousData->starting_frame_number; 1013 td->trbs[0].flags |= TRB_3_FRID(frame); 1014 } 1015 frame = (frame + frameDelta) % 2048; 1016 if (isochronousData->starting_frame_number != NULL) 1017 *isochronousData->starting_frame_number = frame; 1018 1019 // TODO: The OHCI bus driver seems to also do this for inbound 1020 // isochronous transfers. Perhaps it should be moved into the stack? 1021 if (directionIn) { 1022 for (uint32 i = 0; i < isochronousData->packet_count; i++) { 1023 isochronousData->packet_descriptors[i].actual_length = 0; 1024 isochronousData->packet_descriptors[i].status = B_NO_INIT; 1025 } 1026 } 1027 } 1028 1029 // Set the ENT (Evaluate Next TRB) bit, so that the HC will not switch 1030 // contexts before evaluating the Link TRB that _LinkDescriptorForPipe 1031 // will insert, as otherwise there would be a race between us freeing 1032 // and unlinking the descriptor, and the controller evaluating the Link TRB 1033 // and thus getting back onto the main ring and executing the Event Data 1034 // TRB that generates the interrupt for this transfer. 1035 // 1036 // Note that we *do not* unset the CHAIN bit in this TRB, thus including 1037 // the Link TRB in this TD formally, which is required when using the 1038 // ENT bit. (XHCI 1.2 § 4.12.3 p250.) 1039 td->trbs[td->trb_used - 1].flags |= TRB_3_ENT_BIT; 1040 1041 if (!directionIn) { 1042 TRACE("copying out iov count %ld\n", transfer->VectorCount()); 1043 status_t status = transfer->PrepareKernelAccess(); 1044 if (status != B_OK) { 1045 FreeDescriptor(td); 1046 return status; 1047 } 1048 WriteDescriptor(td, transfer->Vector(), 1049 transfer->VectorCount(), transfer->IsPhysical()); 1050 } 1051 1052 td->transfer = transfer; 1053 status = _LinkDescriptorForPipe(td, endpoint); 1054 if (status != B_OK) { 1055 FreeDescriptor(td); 1056 return status; 1057 } 1058 1059 return B_OK; 1060 } 1061 1062 1063 status_t 1064 XHCI::CancelQueuedTransfers(Pipe *pipe, bool force) 1065 { 1066 xhci_endpoint* endpoint = (xhci_endpoint*)pipe->ControllerCookie(); 1067 if (endpoint == NULL || endpoint->trbs == NULL) { 1068 // Someone's de-allocated this pipe or endpoint in the meantime. 1069 // (Possibly AllocateDevice failed, and we were the temporary pipe.) 1070 return B_NO_INIT; 1071 } 1072 1073 #ifndef TRACE_USB 1074 if (force) 1075 #endif 1076 { 1077 TRACE_ALWAYS("cancel queued transfers (%" B_PRId8 ") for pipe %p (%d)\n", 1078 endpoint->used, pipe, pipe->EndpointAddress()); 1079 } 1080 1081 MutexLocker endpointLocker(endpoint->lock); 1082 1083 if (endpoint->td_head == NULL) { 1084 // There aren't any currently pending transfers to cancel. 1085 return B_OK; 1086 } 1087 1088 // Calling the callbacks while holding the endpoint lock could potentially 1089 // cause deadlocks, so we instead store them in a pointer array. We need 1090 // to do this separately from freeing the TDs, for in the case we fail 1091 // to stop the endpoint, we cancel the transfers but do not free the TDs. 1092 Transfer* transfers[XHCI_MAX_TRANSFERS]; 1093 int32 transfersCount = 0; 1094 1095 for (xhci_td* td = endpoint->td_head; td != NULL; td = td->next) { 1096 if (td->transfer == NULL) 1097 continue; 1098 1099 // We can't cancel or delete transfers under "force", as they probably 1100 // are not safe to use anymore. 1101 if (!force) { 1102 transfers[transfersCount] = td->transfer; 1103 transfersCount++; 1104 } 1105 td->transfer = NULL; 1106 } 1107 1108 // It is possible that while waiting for the stop-endpoint command to 1109 // complete, one of the queued transfers posts a completion event, so in 1110 // order to avoid a deadlock, we must unlock the endpoint. 1111 endpointLocker.Unlock(); 1112 status_t status = StopEndpoint(false, endpoint); 1113 if (status != B_OK && status != B_DEV_STALLED) { 1114 // It is possible that the endpoint was stopped by the controller at the 1115 // same time our STOP command was in progress, causing a "Context State" 1116 // error. In that case, try again; if the endpoint is already stopped, 1117 // StopEndpoint will notice this. (XHCI 1.2 § 4.6.9 p137.) 1118 status = StopEndpoint(false, endpoint); 1119 } 1120 if (status == B_DEV_STALLED) { 1121 // Only exit from a Halted state is a RESET. (XHCI 1.2 § 4.8.3 p163.) 1122 TRACE_ERROR("cancel queued transfers: halted endpoint, reset!\n"); 1123 status = ResetEndpoint(false, endpoint); 1124 } 1125 endpointLocker.Lock(); 1126 1127 // Detach the head TD from the endpoint. 1128 xhci_td* td_head = endpoint->td_head; 1129 endpoint->td_head = NULL; 1130 1131 if (status == B_OK) { 1132 // Clear the endpoint's TRBs. 1133 memset(endpoint->trbs, 0, sizeof(xhci_trb) * XHCI_ENDPOINT_RING_SIZE); 1134 endpoint->used = 0; 1135 endpoint->next = 0; 1136 1137 // Set dequeue pointer location to the beginning of the ring. 1138 SetTRDequeue(endpoint->trb_addr, 0, endpoint->id + 1, 1139 endpoint->device->slot); 1140 1141 // We don't need to do anything else to restart the ring, as it will resume 1142 // operation as normal upon the next doorbell. (XHCI 1.2 § 4.6.9 p136.) 1143 } else { 1144 // We couldn't stop the endpoint. Most likely the device has been 1145 // removed and the endpoint was stopped by the hardware, or is 1146 // for some reason busy and cannot be stopped. 1147 TRACE_ERROR("cancel queued transfers: could not stop endpoint: %s!\n", 1148 strerror(status)); 1149 1150 // Instead of freeing the TDs, we want to leave them in the endpoint 1151 // so that when/if the hardware returns, they can be properly unlinked, 1152 // as otherwise the endpoint could get "stuck" by having the "used" 1153 // slowly accumulate due to "dead" transfers. 1154 endpoint->td_head = td_head; 1155 td_head = NULL; 1156 } 1157 1158 endpointLocker.Unlock(); 1159 1160 for (int32 i = 0; i < transfersCount; i++) { 1161 transfers[i]->Finished(B_CANCELED, 0); 1162 delete transfers[i]; 1163 } 1164 1165 // This loop looks a bit strange because we need to store the "next" 1166 // pointer before freeing the descriptor. 1167 xhci_td* td; 1168 while ((td = td_head) != NULL) { 1169 td_head = td_head->next; 1170 FreeDescriptor(td); 1171 } 1172 1173 return B_OK; 1174 } 1175 1176 1177 status_t 1178 XHCI::StartDebugTransfer(Transfer *transfer) 1179 { 1180 Pipe *pipe = transfer->TransferPipe(); 1181 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 1182 if (endpoint == NULL) 1183 return B_BAD_VALUE; 1184 1185 // Check all locks that we are going to hit when running transfers. 1186 if (mutex_trylock(&endpoint->lock) != B_OK) 1187 return B_WOULD_BLOCK; 1188 if (mutex_trylock(&fFinishedLock) != B_OK) { 1189 mutex_unlock(&endpoint->lock); 1190 return B_WOULD_BLOCK; 1191 } 1192 if (mutex_trylock(&fEventLock) != B_OK) { 1193 mutex_unlock(&endpoint->lock); 1194 mutex_unlock(&fFinishedLock); 1195 return B_WOULD_BLOCK; 1196 } 1197 mutex_unlock(&endpoint->lock); 1198 mutex_unlock(&fFinishedLock); 1199 mutex_unlock(&fEventLock); 1200 1201 status_t status = SubmitTransfer(transfer); 1202 if (status != B_OK) 1203 return status; 1204 1205 // The endpoint's head TD is the TD of the just-submitted transfer. 1206 // Just like EHCI, abuse the callback cookie to hold the TD pointer. 1207 transfer->SetCallback(NULL, endpoint->td_head); 1208 1209 return B_OK; 1210 } 1211 1212 1213 status_t 1214 XHCI::CheckDebugTransfer(Transfer *transfer) 1215 { 1216 xhci_td *transfer_td = (xhci_td *)transfer->CallbackCookie(); 1217 if (transfer_td == NULL) 1218 return B_NO_INIT; 1219 1220 // Process events once, and then look for it in the finished list. 1221 ProcessEvents(); 1222 xhci_td *previous = NULL; 1223 for (xhci_td *td = fFinishedHead; td != NULL; td = td->next) { 1224 if (td != transfer_td) { 1225 previous = td; 1226 continue; 1227 } 1228 1229 // We've found it! 1230 if (previous == NULL) { 1231 fFinishedHead = fFinishedHead->next; 1232 } else { 1233 previous->next = td->next; 1234 } 1235 1236 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 1237 status_t status = (td->trb_completion_code == COMP_SUCCESS 1238 || td->trb_completion_code == COMP_SHORT_PACKET) ? B_OK : B_ERROR; 1239 1240 if (status == B_OK && directionIn) { 1241 ReadDescriptor(td, transfer->Vector(), transfer->VectorCount(), 1242 transfer->IsPhysical()); 1243 } 1244 1245 FreeDescriptor(td); 1246 transfer->SetCallback(NULL, NULL); 1247 return status; 1248 } 1249 1250 // We didn't find it. 1251 spin(75); 1252 return B_DEV_PENDING; 1253 } 1254 1255 1256 void 1257 XHCI::CancelDebugTransfer(Transfer *transfer) 1258 { 1259 while (CheckDebugTransfer(transfer) == B_DEV_PENDING) 1260 spin(100); 1261 } 1262 1263 1264 status_t 1265 XHCI::NotifyPipeChange(Pipe *pipe, usb_change change) 1266 { 1267 TRACE("pipe change %d for pipe %p (%d)\n", change, pipe, 1268 pipe->EndpointAddress()); 1269 1270 switch (change) { 1271 case USB_CHANGE_CREATED: 1272 return _InsertEndpointForPipe(pipe); 1273 case USB_CHANGE_DESTROYED: 1274 return _RemoveEndpointForPipe(pipe); 1275 1276 case USB_CHANGE_PIPE_POLICY_CHANGED: 1277 // We don't care about these, at least for now. 1278 return B_OK; 1279 } 1280 1281 TRACE_ERROR("unknown pipe change!\n"); 1282 return B_UNSUPPORTED; 1283 } 1284 1285 1286 xhci_td * 1287 XHCI::CreateDescriptor(uint32 trbCount, uint32 bufferCount, size_t bufferSize) 1288 { 1289 const bool inKDL = debug_debugger_running(); 1290 1291 xhci_td *result; 1292 if (!inKDL) { 1293 result = (xhci_td*)calloc(1, sizeof(xhci_td)); 1294 } else { 1295 // Just use the physical memory allocator while in KDL; it's less 1296 // secure than using the regular heap, but it's easier to deal with. 1297 phys_addr_t dummy; 1298 fStack->AllocateChunk((void **)&result, &dummy, sizeof(xhci_td)); 1299 } 1300 1301 if (result == NULL) { 1302 TRACE_ERROR("failed to allocate a transfer descriptor\n"); 1303 return NULL; 1304 } 1305 1306 // We always allocate 1 more TRB than requested, so that 1307 // _LinkDescriptorForPipe() has room to insert a link TRB. 1308 trbCount++; 1309 if (fStack->AllocateChunk((void **)&result->trbs, &result->trb_addr, 1310 (trbCount * sizeof(xhci_trb))) < B_OK) { 1311 TRACE_ERROR("failed to allocate TRBs\n"); 1312 FreeDescriptor(result); 1313 return NULL; 1314 } 1315 result->trb_count = trbCount; 1316 result->trb_used = 0; 1317 1318 if (bufferSize > 0) { 1319 // Due to how the USB stack allocates physical memory, we can't just 1320 // request one large chunk the size of the transfer, and so instead we 1321 // create a series of buffers as requested by our caller. 1322 1323 // We store the buffer pointers and addresses in one memory block. 1324 if (!inKDL) { 1325 result->buffers = (void**)calloc(bufferCount, 1326 (sizeof(void*) + sizeof(phys_addr_t))); 1327 } else { 1328 phys_addr_t dummy; 1329 fStack->AllocateChunk((void **)&result->buffers, &dummy, 1330 bufferCount * (sizeof(void*) + sizeof(phys_addr_t))); 1331 } 1332 if (result->buffers == NULL) { 1333 TRACE_ERROR("unable to allocate space for buffer infos\n"); 1334 FreeDescriptor(result); 1335 return NULL; 1336 } 1337 result->buffer_addrs = (phys_addr_t*)&result->buffers[bufferCount]; 1338 result->buffer_size = bufferSize; 1339 result->buffer_count = bufferCount; 1340 1341 // Optimization: If the requested total size of all buffers is less 1342 // than 32*B_PAGE_SIZE (the maximum size that the physical memory 1343 // allocator can handle), we allocate only one buffer and segment it. 1344 size_t totalSize = bufferSize * bufferCount; 1345 if (totalSize < (32 * B_PAGE_SIZE)) { 1346 if (fStack->AllocateChunk(&result->buffers[0], 1347 &result->buffer_addrs[0], totalSize) < B_OK) { 1348 TRACE_ERROR("unable to allocate space for large buffer (size %ld)\n", 1349 totalSize); 1350 FreeDescriptor(result); 1351 return NULL; 1352 } 1353 for (uint32 i = 1; i < bufferCount; i++) { 1354 result->buffers[i] = (void*)((addr_t)(result->buffers[i - 1]) 1355 + bufferSize); 1356 result->buffer_addrs[i] = result->buffer_addrs[i - 1] 1357 + bufferSize; 1358 } 1359 } else { 1360 // Otherwise, we allocate each buffer individually. 1361 for (uint32 i = 0; i < bufferCount; i++) { 1362 if (fStack->AllocateChunk(&result->buffers[i], 1363 &result->buffer_addrs[i], bufferSize) < B_OK) { 1364 TRACE_ERROR("unable to allocate space for a buffer (size " 1365 "%" B_PRIuSIZE ", count %" B_PRIu32 ")\n", 1366 bufferSize, bufferCount); 1367 FreeDescriptor(result); 1368 return NULL; 1369 } 1370 } 1371 } 1372 } else { 1373 result->buffers = NULL; 1374 result->buffer_addrs = NULL; 1375 } 1376 1377 // Initialize all other fields. 1378 result->transfer = NULL; 1379 result->trb_completion_code = 0; 1380 result->trb_left = 0; 1381 result->next = NULL; 1382 1383 TRACE("CreateDescriptor allocated %p, buffer_size %ld, buffer_count %" B_PRIu32 "\n", 1384 result, result->buffer_size, result->buffer_count); 1385 1386 return result; 1387 } 1388 1389 1390 void 1391 XHCI::FreeDescriptor(xhci_td *descriptor) 1392 { 1393 if (descriptor == NULL) 1394 return; 1395 1396 const bool inKDL = debug_debugger_running(); 1397 1398 if (descriptor->trbs != NULL) { 1399 fStack->FreeChunk(descriptor->trbs, descriptor->trb_addr, 1400 (descriptor->trb_count * sizeof(xhci_trb))); 1401 } 1402 if (descriptor->buffers != NULL) { 1403 size_t totalSize = descriptor->buffer_size * descriptor->buffer_count; 1404 if (totalSize < (32 * B_PAGE_SIZE)) { 1405 // This was allocated as one contiguous buffer. 1406 fStack->FreeChunk(descriptor->buffers[0], descriptor->buffer_addrs[0], 1407 totalSize); 1408 } else { 1409 for (uint32 i = 0; i < descriptor->buffer_count; i++) { 1410 if (descriptor->buffers[i] == NULL) 1411 continue; 1412 fStack->FreeChunk(descriptor->buffers[i], descriptor->buffer_addrs[i], 1413 descriptor->buffer_size); 1414 } 1415 } 1416 1417 if (!inKDL) { 1418 free(descriptor->buffers); 1419 } else { 1420 fStack->FreeChunk(descriptor->buffers, 0, 1421 descriptor->buffer_count * (sizeof(void*) + sizeof(phys_addr_t))); 1422 } 1423 } 1424 1425 if (!inKDL) 1426 free(descriptor); 1427 else 1428 fStack->FreeChunk(descriptor, 0, sizeof(xhci_td)); 1429 } 1430 1431 1432 size_t 1433 XHCI::WriteDescriptor(xhci_td *descriptor, generic_io_vec *vector, size_t vectorCount, bool physical) 1434 { 1435 size_t written = 0; 1436 1437 size_t bufIdx = 0, bufUsed = 0; 1438 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1439 size_t length = vector[vecIdx].length; 1440 1441 while (length > 0 && bufIdx < descriptor->buffer_count) { 1442 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1443 status_t status = generic_memcpy( 1444 (generic_addr_t)descriptor->buffers[bufIdx] + bufUsed, false, 1445 vector[vecIdx].base + (vector[vecIdx].length - length), physical, 1446 toCopy); 1447 ASSERT(status == B_OK); 1448 1449 written += toCopy; 1450 bufUsed += toCopy; 1451 length -= toCopy; 1452 if (bufUsed == descriptor->buffer_size) { 1453 bufIdx++; 1454 bufUsed = 0; 1455 } 1456 } 1457 } 1458 1459 TRACE("wrote descriptor (%" B_PRIuSIZE " bytes)\n", written); 1460 return written; 1461 } 1462 1463 1464 size_t 1465 XHCI::ReadDescriptor(xhci_td *descriptor, generic_io_vec *vector, size_t vectorCount, bool physical) 1466 { 1467 size_t read = 0; 1468 1469 size_t bufIdx = 0, bufUsed = 0; 1470 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1471 size_t length = vector[vecIdx].length; 1472 1473 while (length > 0 && bufIdx < descriptor->buffer_count) { 1474 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1475 status_t status = generic_memcpy( 1476 vector[vecIdx].base + (vector[vecIdx].length - length), physical, 1477 (generic_addr_t)descriptor->buffers[bufIdx] + bufUsed, false, toCopy); 1478 ASSERT(status == B_OK); 1479 1480 read += toCopy; 1481 bufUsed += toCopy; 1482 length -= toCopy; 1483 if (bufUsed == descriptor->buffer_size) { 1484 bufIdx++; 1485 bufUsed = 0; 1486 } 1487 } 1488 } 1489 1490 TRACE("read descriptor (%" B_PRIuSIZE " bytes)\n", read); 1491 return read; 1492 } 1493 1494 1495 Device * 1496 XHCI::AllocateDevice(Hub *parent, int8 hubAddress, uint8 hubPort, 1497 usb_speed speed) 1498 { 1499 TRACE("AllocateDevice hubAddress %d hubPort %d speed %d\n", hubAddress, 1500 hubPort, speed); 1501 1502 uint8 slot = XHCI_MAX_SLOTS; 1503 status_t status = EnableSlot(&slot); 1504 if (status != B_OK) { 1505 TRACE_ERROR("failed to enable slot: %s\n", strerror(status)); 1506 return NULL; 1507 } 1508 1509 if (slot == 0 || slot > fSlotCount) { 1510 TRACE_ERROR("AllocateDevice: bad slot\n"); 1511 return NULL; 1512 } 1513 1514 if (fDevices[slot].slot != 0) { 1515 TRACE_ERROR("AllocateDevice: slot already used\n"); 1516 return NULL; 1517 } 1518 1519 struct xhci_device *device = &fDevices[slot]; 1520 device->slot = slot; 1521 1522 device->input_ctx_area = fStack->AllocateArea((void **)&device->input_ctx, 1523 &device->input_ctx_addr, sizeof(*device->input_ctx) << fContextSizeShift, 1524 "XHCI input context"); 1525 if (device->input_ctx_area < B_OK) { 1526 TRACE_ERROR("unable to create a input context area\n"); 1527 CleanupDevice(device); 1528 return NULL; 1529 } 1530 if (fContextSizeShift == 1) { 1531 // 64-byte contexts have to be page-aligned in order for 1532 // _OffsetContextAddr to function properly. 1533 ASSERT((((addr_t)device->input_ctx) % B_PAGE_SIZE) == 0); 1534 } 1535 1536 memset(device->input_ctx, 0, sizeof(*device->input_ctx) << fContextSizeShift); 1537 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1538 _WriteContext(&device->input_ctx->input.addFlags, 3); 1539 1540 uint8 rhPort = hubPort; 1541 uint32 route = 0; 1542 for (Device *hubDevice = parent; hubDevice != RootObject(); 1543 hubDevice = (Device *)hubDevice->Parent()) { 1544 if (hubDevice->Parent() == RootObject()) 1545 break; 1546 1547 if (rhPort > 15) 1548 rhPort = 15; 1549 route = route << 4; 1550 route |= rhPort; 1551 1552 rhPort = hubDevice->HubPort(); 1553 } 1554 1555 uint32 dwslot0 = SLOT_0_NUM_ENTRIES(1) | SLOT_0_ROUTE(route); 1556 1557 // Get speed of port, only if device connected to root hub port 1558 // else we have to rely on value reported by the Hub Explore thread 1559 if (route == 0) { 1560 GetPortSpeed(hubPort - 1, &speed); 1561 TRACE("speed updated %d\n", speed); 1562 } 1563 1564 // add the speed 1565 switch (speed) { 1566 case USB_SPEED_LOWSPEED: 1567 dwslot0 |= SLOT_0_SPEED(2); 1568 break; 1569 case USB_SPEED_FULLSPEED: 1570 dwslot0 |= SLOT_0_SPEED(1); 1571 break; 1572 case USB_SPEED_HIGHSPEED: 1573 dwslot0 |= SLOT_0_SPEED(3); 1574 break; 1575 case USB_SPEED_SUPERSPEED: 1576 dwslot0 |= SLOT_0_SPEED(4); 1577 break; 1578 default: 1579 TRACE_ERROR("unknown usb speed\n"); 1580 break; 1581 } 1582 1583 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1584 // TODO enable power save 1585 _WriteContext(&device->input_ctx->slot.dwslot1, SLOT_1_RH_PORT(rhPort)); 1586 uint32 dwslot2 = SLOT_2_IRQ_TARGET(0); 1587 1588 // If LS/FS device connected to non-root HS device 1589 if (route != 0 && parent->Speed() == USB_SPEED_HIGHSPEED 1590 && (speed == USB_SPEED_LOWSPEED || speed == USB_SPEED_FULLSPEED)) { 1591 struct xhci_device *parenthub = (struct xhci_device *) 1592 parent->ControllerCookie(); 1593 dwslot2 |= SLOT_2_PORT_NUM(hubPort); 1594 dwslot2 |= SLOT_2_TT_HUB_SLOT(parenthub->slot); 1595 } 1596 1597 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1598 1599 _WriteContext(&device->input_ctx->slot.dwslot3, SLOT_3_SLOT_STATE(0) 1600 | SLOT_3_DEVICE_ADDRESS(0)); 1601 1602 TRACE("slot 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 1603 "\n", _ReadContext(&device->input_ctx->slot.dwslot0), 1604 _ReadContext(&device->input_ctx->slot.dwslot1), 1605 _ReadContext(&device->input_ctx->slot.dwslot2), 1606 _ReadContext(&device->input_ctx->slot.dwslot3)); 1607 1608 device->device_ctx_area = fStack->AllocateArea((void **)&device->device_ctx, 1609 &device->device_ctx_addr, sizeof(*device->device_ctx) << fContextSizeShift, 1610 "XHCI device context"); 1611 if (device->device_ctx_area < B_OK) { 1612 TRACE_ERROR("unable to create a device context area\n"); 1613 CleanupDevice(device); 1614 return NULL; 1615 } 1616 memset(device->device_ctx, 0, sizeof(*device->device_ctx) << fContextSizeShift); 1617 1618 device->trb_area = fStack->AllocateArea((void **)&device->trbs, 1619 &device->trb_addr, sizeof(xhci_trb) * (XHCI_MAX_ENDPOINTS - 1) 1620 * XHCI_ENDPOINT_RING_SIZE, "XHCI endpoint trbs"); 1621 if (device->trb_area < B_OK) { 1622 TRACE_ERROR("unable to create a device trbs area\n"); 1623 CleanupDevice(device); 1624 return NULL; 1625 } 1626 1627 // set up slot pointer to device context 1628 fDcba->baseAddress[slot] = device->device_ctx_addr; 1629 1630 size_t maxPacketSize; 1631 switch (speed) { 1632 case USB_SPEED_LOWSPEED: 1633 case USB_SPEED_FULLSPEED: 1634 maxPacketSize = 8; 1635 break; 1636 case USB_SPEED_HIGHSPEED: 1637 maxPacketSize = 64; 1638 break; 1639 default: 1640 maxPacketSize = 512; 1641 break; 1642 } 1643 1644 xhci_endpoint* endpoint0 = &device->endpoints[0]; 1645 mutex_init(&endpoint0->lock, "xhci endpoint lock"); 1646 endpoint0->device = device; 1647 endpoint0->id = 0; 1648 endpoint0->td_head = NULL; 1649 endpoint0->used = 0; 1650 endpoint0->next = 0; 1651 endpoint0->trbs = device->trbs; 1652 endpoint0->trb_addr = device->trb_addr; 1653 1654 // configure the Control endpoint 0 1655 if (ConfigureEndpoint(endpoint0, slot, 0, USB_OBJECT_CONTROL_PIPE, false, 1656 0, maxPacketSize, speed, 0, 0) != B_OK) { 1657 TRACE_ERROR("unable to configure default control endpoint\n"); 1658 CleanupDevice(device); 1659 return NULL; 1660 } 1661 1662 // device should get to addressed state (bsr = 0) 1663 status = SetAddress(device->input_ctx_addr, false, slot); 1664 if (status != B_OK) { 1665 TRACE_ERROR("unable to set address: %s\n", strerror(status)); 1666 CleanupDevice(device); 1667 return NULL; 1668 } 1669 1670 device->address = SLOT_3_DEVICE_ADDRESS_GET(_ReadContext( 1671 &device->device_ctx->slot.dwslot3)); 1672 1673 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", device->address, 1674 SLOT_3_SLOT_STATE_GET(_ReadContext( 1675 &device->device_ctx->slot.dwslot3))); 1676 TRACE("endpoint0 state 0x%08" B_PRIx32 "\n", 1677 ENDPOINT_0_STATE_GET(_ReadContext( 1678 &device->device_ctx->endpoints[0].dwendpoint0))); 1679 1680 // Wait a bit for the device to complete addressing 1681 snooze(USB_DELAY_SET_ADDRESS); 1682 1683 // Create a temporary pipe with the new address 1684 ControlPipe pipe(parent); 1685 pipe.SetControllerCookie(endpoint0); 1686 pipe.InitCommon(device->address + 1, 0, speed, Pipe::Default, maxPacketSize, 0, 1687 hubAddress, hubPort); 1688 1689 // Get the device descriptor 1690 // Just retrieve the first 8 bytes of the descriptor -> minimum supported 1691 // size of any device. It is enough because it includes the device type. 1692 1693 size_t actualLength = 0; 1694 usb_device_descriptor deviceDescriptor; 1695 1696 TRACE("getting the device descriptor\n"); 1697 status = pipe.SendRequest( 1698 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_STANDARD, // type 1699 USB_REQUEST_GET_DESCRIPTOR, // request 1700 USB_DESCRIPTOR_DEVICE << 8, // value 1701 0, // index 1702 8, // length 1703 (void *)&deviceDescriptor, // buffer 1704 8, // buffer length 1705 &actualLength); // actual length 1706 1707 if (actualLength != 8) { 1708 TRACE_ERROR("failed to get the device descriptor: %s\n", 1709 strerror(status)); 1710 CleanupDevice(device); 1711 return NULL; 1712 } 1713 1714 TRACE("device_class: %d device_subclass %d device_protocol %d\n", 1715 deviceDescriptor.device_class, deviceDescriptor.device_subclass, 1716 deviceDescriptor.device_protocol); 1717 1718 if (speed == USB_SPEED_FULLSPEED && deviceDescriptor.max_packet_size_0 != 8) { 1719 TRACE("Full speed device with different max packet size for Endpoint 0\n"); 1720 uint32 dwendpoint1 = _ReadContext( 1721 &device->input_ctx->endpoints[0].dwendpoint1); 1722 dwendpoint1 &= ~ENDPOINT_1_MAXPACKETSIZE(0xffff); 1723 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE( 1724 deviceDescriptor.max_packet_size_0); 1725 _WriteContext(&device->input_ctx->endpoints[0].dwendpoint1, 1726 dwendpoint1); 1727 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1728 _WriteContext(&device->input_ctx->input.addFlags, (1 << 1)); 1729 EvaluateContext(device->input_ctx_addr, device->slot); 1730 } 1731 1732 Device *deviceObject = NULL; 1733 if (deviceDescriptor.device_class == 0x09) { 1734 TRACE("creating new Hub\n"); 1735 TRACE("getting the hub descriptor\n"); 1736 size_t actualLength = 0; 1737 usb_hub_descriptor hubDescriptor; 1738 status = pipe.SendRequest( 1739 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_CLASS, // type 1740 USB_REQUEST_GET_DESCRIPTOR, // request 1741 USB_DESCRIPTOR_HUB << 8, // value 1742 0, // index 1743 sizeof(usb_hub_descriptor), // length 1744 (void *)&hubDescriptor, // buffer 1745 sizeof(usb_hub_descriptor), // buffer length 1746 &actualLength); 1747 1748 if (actualLength != sizeof(usb_hub_descriptor)) { 1749 TRACE_ERROR("error while getting the hub descriptor: %s\n", 1750 strerror(status)); 1751 CleanupDevice(device); 1752 return NULL; 1753 } 1754 1755 uint32 dwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1756 dwslot0 |= SLOT_0_HUB_BIT; 1757 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1758 uint32 dwslot1 = _ReadContext(&device->input_ctx->slot.dwslot1); 1759 dwslot1 |= SLOT_1_NUM_PORTS(hubDescriptor.num_ports); 1760 _WriteContext(&device->input_ctx->slot.dwslot1, dwslot1); 1761 if (speed == USB_SPEED_HIGHSPEED) { 1762 uint32 dwslot2 = _ReadContext(&device->input_ctx->slot.dwslot2); 1763 dwslot2 |= SLOT_2_TT_TIME(HUB_TTT_GET(hubDescriptor.characteristics)); 1764 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1765 } 1766 1767 deviceObject = new(std::nothrow) Hub(parent, hubAddress, hubPort, 1768 deviceDescriptor, device->address + 1, speed, false, device); 1769 } else { 1770 TRACE("creating new device\n"); 1771 deviceObject = new(std::nothrow) Device(parent, hubAddress, hubPort, 1772 deviceDescriptor, device->address + 1, speed, false, device); 1773 } 1774 if (deviceObject == NULL || deviceObject->InitCheck() != B_OK) { 1775 if (deviceObject == NULL) { 1776 TRACE_ERROR("no memory to allocate device\n"); 1777 } else { 1778 TRACE_ERROR("device object failed to initialize\n"); 1779 } 1780 CleanupDevice(device); 1781 return NULL; 1782 } 1783 1784 // We don't want to disable the default endpoint, naturally, which would 1785 // otherwise happen when this Pipe object is destroyed. 1786 pipe.SetControllerCookie(NULL); 1787 1788 deviceObject->RegisterNode(); 1789 1790 TRACE("AllocateDevice() port %d slot %d\n", hubPort, slot); 1791 return deviceObject; 1792 } 1793 1794 1795 void 1796 XHCI::FreeDevice(Device *usbDevice) 1797 { 1798 xhci_device* device = (xhci_device*)usbDevice->ControllerCookie(); 1799 TRACE("FreeDevice() slot %d\n", device->slot); 1800 1801 // Delete the device first, so it cleans up its pipes and tells us 1802 // what we need to destroy before we tear down our internal state. 1803 delete usbDevice; 1804 1805 CleanupDevice(device); 1806 } 1807 1808 1809 void 1810 XHCI::CleanupDevice(xhci_device *device) 1811 { 1812 if (device->slot != 0) { 1813 DisableSlot(device->slot); 1814 fDcba->baseAddress[device->slot] = 0; 1815 } 1816 1817 if (device->trb_addr != 0) 1818 delete_area(device->trb_area); 1819 if (device->input_ctx_addr != 0) 1820 delete_area(device->input_ctx_area); 1821 if (device->device_ctx_addr != 0) 1822 delete_area(device->device_ctx_area); 1823 1824 memset(device, 0, sizeof(xhci_device)); 1825 } 1826 1827 1828 uint8 1829 XHCI::_GetEndpointState(xhci_endpoint* endpoint) 1830 { 1831 struct xhci_device_ctx* device_ctx = endpoint->device->device_ctx; 1832 return ENDPOINT_0_STATE_GET( 1833 _ReadContext(&device_ctx->endpoints[endpoint->id].dwendpoint0)); 1834 } 1835 1836 1837 status_t 1838 XHCI::_InsertEndpointForPipe(Pipe *pipe) 1839 { 1840 TRACE("insert endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1841 1842 if (pipe->ControllerCookie() != NULL 1843 || pipe->Parent()->Type() != USB_OBJECT_DEVICE) { 1844 // default pipe is already referenced 1845 return B_OK; 1846 } 1847 1848 Device* usbDevice = (Device *)pipe->Parent(); 1849 if (usbDevice->Parent() == RootObject()) { 1850 // root hub needs no initialization 1851 return B_OK; 1852 } 1853 1854 struct xhci_device *device = (struct xhci_device *) 1855 usbDevice->ControllerCookie(); 1856 if (device == NULL) { 1857 panic("device is NULL\n"); 1858 return B_NO_INIT; 1859 } 1860 1861 const uint8 id = (2 * pipe->EndpointAddress() 1862 + (pipe->Direction() != Pipe::Out ? 1 : 0)) - 1; 1863 if (id >= XHCI_MAX_ENDPOINTS - 1) 1864 return B_BAD_VALUE; 1865 1866 if (id > 0) { 1867 uint32 devicedwslot0 = _ReadContext(&device->device_ctx->slot.dwslot0); 1868 if (SLOT_0_NUM_ENTRIES_GET(devicedwslot0) == 1) { 1869 uint32 inputdwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1870 inputdwslot0 &= ~(SLOT_0_NUM_ENTRIES(0x1f)); 1871 inputdwslot0 |= SLOT_0_NUM_ENTRIES(XHCI_MAX_ENDPOINTS - 1); 1872 _WriteContext(&device->input_ctx->slot.dwslot0, inputdwslot0); 1873 EvaluateContext(device->input_ctx_addr, device->slot); 1874 } 1875 1876 xhci_endpoint* endpoint = &device->endpoints[id]; 1877 mutex_init(&endpoint->lock, "xhci endpoint lock"); 1878 MutexLocker endpointLocker(endpoint->lock); 1879 1880 endpoint->device = device; 1881 endpoint->id = id; 1882 endpoint->td_head = NULL; 1883 endpoint->used = 0; 1884 endpoint->next = 0; 1885 1886 endpoint->trbs = device->trbs + id * XHCI_ENDPOINT_RING_SIZE; 1887 endpoint->trb_addr = device->trb_addr 1888 + id * XHCI_ENDPOINT_RING_SIZE * sizeof(xhci_trb); 1889 memset(endpoint->trbs, 0, 1890 sizeof(xhci_trb) * XHCI_ENDPOINT_RING_SIZE); 1891 1892 TRACE("insert endpoint for pipe: trbs, device %p endpoint %p\n", 1893 device->trbs, endpoint->trbs); 1894 TRACE("insert endpoint for pipe: trb_addr, device 0x%" B_PRIxPHYSADDR 1895 " endpoint 0x%" B_PRIxPHYSADDR "\n", device->trb_addr, 1896 endpoint->trb_addr); 1897 1898 const uint8 endpointNum = id + 1; 1899 1900 status_t status = ConfigureEndpoint(endpoint, device->slot, id, pipe->Type(), 1901 pipe->Direction() == Pipe::In, pipe->Interval(), pipe->MaxPacketSize(), 1902 usbDevice->Speed(), pipe->MaxBurst(), pipe->BytesPerInterval()); 1903 if (status != B_OK) { 1904 TRACE_ERROR("unable to configure endpoint: %s\n", strerror(status)); 1905 return status; 1906 } 1907 1908 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1909 _WriteContext(&device->input_ctx->input.addFlags, 1910 (1 << endpointNum) | (1 << 0)); 1911 1912 ConfigureEndpoint(device->input_ctx_addr, false, device->slot); 1913 1914 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", 1915 device->address, SLOT_3_SLOT_STATE_GET(_ReadContext( 1916 &device->device_ctx->slot.dwslot3))); 1917 TRACE("endpoint[0] state 0x%08" B_PRIx32 "\n", 1918 ENDPOINT_0_STATE_GET(_ReadContext( 1919 &device->device_ctx->endpoints[0].dwendpoint0))); 1920 TRACE("endpoint[%d] state 0x%08" B_PRIx32 "\n", id, 1921 ENDPOINT_0_STATE_GET(_ReadContext( 1922 &device->device_ctx->endpoints[id].dwendpoint0))); 1923 } 1924 pipe->SetControllerCookie(&device->endpoints[id]); 1925 1926 return B_OK; 1927 } 1928 1929 1930 status_t 1931 XHCI::_RemoveEndpointForPipe(Pipe *pipe) 1932 { 1933 TRACE("remove endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1934 1935 if (pipe->Parent()->Type() != USB_OBJECT_DEVICE) 1936 return B_OK; 1937 Device* usbDevice = (Device *)pipe->Parent(); 1938 if (usbDevice->Parent() == RootObject()) 1939 return B_BAD_VALUE; 1940 1941 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 1942 if (endpoint == NULL || endpoint->trbs == NULL) 1943 return B_NO_INIT; 1944 1945 pipe->SetControllerCookie(NULL); 1946 1947 if (endpoint->id > 0) { 1948 xhci_device *device = endpoint->device; 1949 uint8 epNumber = endpoint->id + 1; 1950 StopEndpoint(true, endpoint); 1951 1952 mutex_lock(&endpoint->lock); 1953 1954 // See comment in CancelQueuedTransfers. 1955 xhci_td* td; 1956 while ((td = endpoint->td_head) != NULL) { 1957 endpoint->td_head = endpoint->td_head->next; 1958 FreeDescriptor(td); 1959 } 1960 1961 mutex_destroy(&endpoint->lock); 1962 memset(endpoint, 0, sizeof(xhci_endpoint)); 1963 1964 _WriteContext(&device->input_ctx->input.dropFlags, (1 << epNumber)); 1965 _WriteContext(&device->input_ctx->input.addFlags, (1 << 0)); 1966 1967 // The Deconfigure bit in the Configure Endpoint command indicates 1968 // that *all* endpoints are to be deconfigured, and not just the ones 1969 // specified in the context flags. (XHCI 1.2 § 4.6.6 p115.) 1970 ConfigureEndpoint(device->input_ctx_addr, false, device->slot); 1971 } 1972 1973 return B_OK; 1974 } 1975 1976 1977 status_t 1978 XHCI::_LinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 1979 { 1980 TRACE("link descriptor for pipe\n"); 1981 1982 // Use mutex_trylock first, in case we are in KDL. 1983 MutexLocker endpointLocker(&endpoint->lock, mutex_trylock(&endpoint->lock) == B_OK); 1984 1985 // "used" refers to the number of currently linked TDs, not the number of 1986 // used TRBs on the ring (we use 2 TRBs on the ring per transfer.) 1987 // Furthermore, we have to leave an empty item between the head and tail. 1988 if (endpoint->used >= (XHCI_MAX_TRANSFERS - 1)) { 1989 TRACE_ERROR("link descriptor for pipe: max transfers count exceeded\n"); 1990 return B_BAD_VALUE; 1991 } 1992 1993 // We do not support queuing other transfers in tandem with a fragmented one. 1994 if (endpoint->td_head != NULL && endpoint->td_head->transfer != NULL 1995 && endpoint->td_head->transfer->IsFragmented()) { 1996 TRACE_ERROR("cannot submit transfer: a fragmented transfer is queued\n"); 1997 return B_DEV_RESOURCE_CONFLICT; 1998 } 1999 2000 endpoint->used++; 2001 descriptor->next = endpoint->td_head; 2002 endpoint->td_head = descriptor; 2003 2004 uint32 link = endpoint->next, eventdata = link + 1, next = eventdata + 1; 2005 if (eventdata == XHCI_ENDPOINT_RING_SIZE || next == XHCI_ENDPOINT_RING_SIZE) { 2006 // If it's "next" not "eventdata" that got us here, we will be leaving 2007 // one TRB at the end of the ring unused. 2008 eventdata = 0; 2009 next = 1; 2010 } 2011 2012 TRACE("link descriptor for pipe: link %d, next %d\n", link, next); 2013 2014 // Add a Link TRB to the end of the descriptor. 2015 phys_addr_t addr = endpoint->trb_addr + (eventdata * sizeof(xhci_trb)); 2016 descriptor->trbs[descriptor->trb_used].address = addr; 2017 descriptor->trbs[descriptor->trb_used].status = TRB_2_IRQ(0); 2018 descriptor->trbs[descriptor->trb_used].flags = TRB_3_TYPE(TRB_TYPE_LINK) 2019 | TRB_3_CHAIN_BIT | TRB_3_CYCLE_BIT; 2020 // It is specified that (XHCI 1.2 § 4.12.3 Note 2 p251) if the TRB 2021 // following one with the ENT bit set is a Link TRB, the Link TRB 2022 // shall be evaluated *and* the subsequent TRB shall be. Thus a 2023 // TRB_3_ENT_BIT is unnecessary here; and from testing seems to 2024 // break all transfers on a (very) small number of controllers. 2025 2026 #if !B_HOST_IS_LENDIAN 2027 // Convert endianness. 2028 for (uint32 i = 0; i <= descriptor->trb_used; i++) { 2029 descriptor->trbs[i].address = 2030 B_HOST_TO_LENDIAN_INT64(descriptor->trbs[i].address); 2031 descriptor->trbs[i].status = 2032 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].status); 2033 descriptor->trbs[i].flags = 2034 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].flags); 2035 } 2036 #endif 2037 2038 // Link the descriptor. 2039 endpoint->trbs[link].address = 2040 B_HOST_TO_LENDIAN_INT64(descriptor->trb_addr); 2041 endpoint->trbs[link].status = 2042 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 2043 endpoint->trbs[link].flags = 2044 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_LINK)); 2045 2046 // Set up the Event Data TRB (XHCI 1.2 § 4.11.5.2 p230.) 2047 // 2048 // We do this on the main ring for two reasons: first, to avoid a small 2049 // potential race between the interrupt and the controller evaluating 2050 // the link TRB to get back onto the ring; and second, because many 2051 // controllers throw errors if the target of a Link TRB is not valid 2052 // (i.e. does not have its Cycle Bit set.) 2053 // 2054 // We also set the "address" field, which the controller will copy 2055 // verbatim into the TRB it posts to the event ring, to be the last 2056 // "real" TRB in the TD; this will allow us to determine what transfer 2057 // the resulting Transfer Event TRB refers to. 2058 endpoint->trbs[eventdata].address = 2059 B_HOST_TO_LENDIAN_INT64(descriptor->trb_addr 2060 + (descriptor->trb_used - 1) * sizeof(xhci_trb)); 2061 endpoint->trbs[eventdata].status = 2062 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 2063 endpoint->trbs[eventdata].flags = 2064 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_EVENT_DATA) 2065 | TRB_3_IOC_BIT | TRB_3_CYCLE_BIT); 2066 2067 endpoint->trbs[next].address = 0; 2068 endpoint->trbs[next].status = 0; 2069 endpoint->trbs[next].flags = 0; 2070 2071 memory_write_barrier(); 2072 2073 // Everything is ready, so write the cycle bit. 2074 endpoint->trbs[link].flags |= B_HOST_TO_LENDIAN_INT32(TRB_3_CYCLE_BIT); 2075 2076 TRACE("_LinkDescriptorForPipe pLink %p phys 0x%" B_PRIxPHYSADDR 2077 " 0x%" B_PRIxPHYSADDR " 0x%08" B_PRIx32 "\n", &endpoint->trbs[link], 2078 endpoint->trb_addr + link * sizeof(struct xhci_trb), 2079 endpoint->trbs[link].address, 2080 B_LENDIAN_TO_HOST_INT32(endpoint->trbs[link].flags)); 2081 2082 endpoint->next = next; 2083 endpointLocker.Unlock(); 2084 2085 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 2086 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 2087 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 2088 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 2089 2090 Ring(endpoint->device->slot, endpoint->id + 1); 2091 2092 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 2093 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 2094 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 2095 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 2096 2097 return B_OK; 2098 } 2099 2100 2101 status_t 2102 XHCI::_UnlinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 2103 { 2104 TRACE("unlink descriptor for pipe\n"); 2105 // We presume that the caller has already locked or owns the endpoint. 2106 2107 endpoint->used--; 2108 if (descriptor == endpoint->td_head) { 2109 endpoint->td_head = descriptor->next; 2110 descriptor->next = NULL; 2111 return B_OK; 2112 } else { 2113 for (xhci_td *td = endpoint->td_head; td->next != NULL; td = td->next) { 2114 if (td->next == descriptor) { 2115 td->next = descriptor->next; 2116 descriptor->next = NULL; 2117 return B_OK; 2118 } 2119 } 2120 } 2121 2122 endpoint->used++; 2123 return B_ERROR; 2124 } 2125 2126 2127 status_t 2128 XHCI::ConfigureEndpoint(xhci_endpoint* ep, uint8 slot, uint8 number, uint8 type, 2129 bool directionIn, uint16 interval, uint16 maxPacketSize, usb_speed speed, 2130 uint8 maxBurst, uint16 bytesPerInterval) 2131 { 2132 struct xhci_device* device = &fDevices[slot]; 2133 2134 uint32 dwendpoint0 = 0; 2135 uint32 dwendpoint1 = 0; 2136 uint64 qwendpoint2 = 0; 2137 uint32 dwendpoint4 = 0; 2138 2139 // Compute and assign the endpoint type. (XHCI 1.2 § 6.2.3 Table 6-9 p452.) 2140 uint8 xhciType = 4; 2141 if ((type & USB_OBJECT_INTERRUPT_PIPE) != 0) 2142 xhciType = 3; 2143 if ((type & USB_OBJECT_BULK_PIPE) != 0) 2144 xhciType = 2; 2145 if ((type & USB_OBJECT_ISO_PIPE) != 0) 2146 xhciType = 1; 2147 xhciType |= directionIn ? (1 << 2) : 0; 2148 dwendpoint1 |= ENDPOINT_1_EPTYPE(xhciType); 2149 2150 // Compute and assign interval. (XHCI 1.2 § 6.2.3.6 p456.) 2151 uint16 calcInterval; 2152 if ((type & USB_OBJECT_BULK_PIPE) != 0 2153 || (type & USB_OBJECT_CONTROL_PIPE) != 0) { 2154 // Bulk and Control endpoints never issue NAKs. 2155 calcInterval = 0; 2156 } else { 2157 switch (speed) { 2158 case USB_SPEED_FULLSPEED: 2159 if ((type & USB_OBJECT_ISO_PIPE) != 0) { 2160 // Convert 1-16 into 3-18. 2161 calcInterval = min_c(max_c(interval, 1), 16) + 2; 2162 break; 2163 } 2164 2165 // fall through 2166 case USB_SPEED_LOWSPEED: { 2167 // Convert 1ms-255ms into 3-10. 2168 2169 // Find the index of the highest set bit in "interval". 2170 uint32 temp = min_c(max_c(interval, 1), 255); 2171 for (calcInterval = 0; temp != 1; calcInterval++) 2172 temp = temp >> 1; 2173 calcInterval += 3; 2174 break; 2175 } 2176 2177 case USB_SPEED_HIGHSPEED: 2178 case USB_SPEED_SUPERSPEED: 2179 default: 2180 // Convert 1-16 into 0-15. 2181 calcInterval = min_c(max_c(interval, 1), 16) - 1; 2182 break; 2183 } 2184 } 2185 dwendpoint0 |= ENDPOINT_0_INTERVAL(calcInterval); 2186 2187 // For non-isochronous endpoints, we want the controller to retry failed 2188 // transfers, if possible. (XHCI 1.2 § 4.10.2.3 p197.) 2189 if ((type & USB_OBJECT_ISO_PIPE) == 0) 2190 dwendpoint1 |= ENDPOINT_1_CERR(3); 2191 2192 // Assign maximum burst size. For USB3 devices this is passed in; for 2193 // all other devices we compute it. (XHCI 1.2 § 4.8.2 p161.) 2194 if (speed == USB_SPEED_HIGHSPEED && (type & (USB_OBJECT_INTERRUPT_PIPE 2195 | USB_OBJECT_ISO_PIPE)) != 0) { 2196 maxBurst = (maxPacketSize & 0x1800) >> 11; 2197 } else if (speed != USB_SPEED_SUPERSPEED) { 2198 maxBurst = 0; 2199 } 2200 dwendpoint1 |= ENDPOINT_1_MAXBURST(maxBurst); 2201 2202 // Assign maximum packet size, set the ring address, and set the 2203 // "Dequeue Cycle State" bit. (XHCI 1.2 § 6.2.3 Table 6-10 p453.) 2204 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE(maxPacketSize); 2205 qwendpoint2 |= ENDPOINT_2_DCS_BIT | ep->trb_addr; 2206 2207 // The Max Burst Payload is the number of bytes moved by a 2208 // maximum sized burst. (XHCI 1.2 § 4.11.7.1 p236.) 2209 ep->max_burst_payload = (maxBurst + 1) * maxPacketSize; 2210 if (ep->max_burst_payload == 0) { 2211 TRACE_ERROR("ConfigureEndpoint() failed invalid max_burst_payload\n"); 2212 return B_BAD_VALUE; 2213 } 2214 2215 // Assign average TRB length. 2216 if ((type & USB_OBJECT_CONTROL_PIPE) != 0) { 2217 // Control pipes are a special case, as they rarely have 2218 // outbound transfers of any substantial size. 2219 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(8); 2220 } else if ((type & USB_OBJECT_ISO_PIPE) != 0) { 2221 // Isochronous pipes are another special case: the TRB size will be 2222 // one packet (which is normally smaller than the max packet size, 2223 // but we don't know what it is here.) 2224 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(maxPacketSize); 2225 } else { 2226 // Under all other circumstances, we put max_burst_payload in a TRB. 2227 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(ep->max_burst_payload); 2228 } 2229 2230 // Assign maximum ESIT payload. (XHCI 1.2 § 4.14.2 p259.) 2231 if ((type & (USB_OBJECT_INTERRUPT_PIPE | USB_OBJECT_ISO_PIPE)) != 0) { 2232 // TODO: For SuperSpeedPlus endpoints, there is yet another descriptor 2233 // for isochronous endpoints that specifies the maximum ESIT payload. 2234 // We don't fetch this yet, so just fall back to the USB2 computation 2235 // method if bytesPerInterval is 0. 2236 if (speed == USB_SPEED_SUPERSPEED && bytesPerInterval != 0) 2237 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD(bytesPerInterval); 2238 else if (speed >= USB_SPEED_HIGHSPEED) 2239 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD((maxBurst + 1) * maxPacketSize); 2240 } 2241 2242 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint0, 2243 dwendpoint0); 2244 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint1, 2245 dwendpoint1); 2246 _WriteContext(&device->input_ctx->endpoints[number].qwendpoint2, 2247 qwendpoint2); 2248 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint4, 2249 dwendpoint4); 2250 2251 TRACE("endpoint 0x%" B_PRIx32 " 0x%" B_PRIx32 " 0x%" B_PRIx64 " 0x%" 2252 B_PRIx32 "\n", 2253 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint0), 2254 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint1), 2255 _ReadContext(&device->input_ctx->endpoints[number].qwendpoint2), 2256 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint4)); 2257 2258 return B_OK; 2259 } 2260 2261 2262 status_t 2263 XHCI::GetPortSpeed(uint8 index, usb_speed* speed) 2264 { 2265 if (index >= fPortCount) 2266 return B_BAD_INDEX; 2267 2268 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 2269 2270 switch (PS_SPEED_GET(portStatus)) { 2271 case 2: 2272 *speed = USB_SPEED_LOWSPEED; 2273 break; 2274 case 1: 2275 *speed = USB_SPEED_FULLSPEED; 2276 break; 2277 case 3: 2278 *speed = USB_SPEED_HIGHSPEED; 2279 break; 2280 case 4: 2281 *speed = USB_SPEED_SUPERSPEED; 2282 break; 2283 default: 2284 TRACE_ALWAYS("nonstandard port speed %" B_PRId32 ", assuming SuperSpeed\n", 2285 PS_SPEED_GET(portStatus)); 2286 *speed = USB_SPEED_SUPERSPEED; 2287 break; 2288 } 2289 2290 return B_OK; 2291 } 2292 2293 2294 status_t 2295 XHCI::GetPortStatus(uint8 index, usb_port_status* status) 2296 { 2297 if (index >= fPortCount) 2298 return B_BAD_INDEX; 2299 2300 status->status = status->change = 0; 2301 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 2302 TRACE("port %" B_PRId8 " status=0x%08" B_PRIx32 "\n", index, portStatus); 2303 2304 // build the status 2305 switch (PS_SPEED_GET(portStatus)) { 2306 case 3: 2307 status->status |= PORT_STATUS_HIGH_SPEED; 2308 break; 2309 case 2: 2310 status->status |= PORT_STATUS_LOW_SPEED; 2311 break; 2312 default: 2313 break; 2314 } 2315 2316 if (portStatus & PS_CCS) 2317 status->status |= PORT_STATUS_CONNECTION; 2318 if (portStatus & PS_PED) 2319 status->status |= PORT_STATUS_ENABLE; 2320 if (portStatus & PS_OCA) 2321 status->status |= PORT_STATUS_OVER_CURRENT; 2322 if (portStatus & PS_PR) 2323 status->status |= PORT_STATUS_RESET; 2324 if (portStatus & PS_PP) { 2325 if (fPortSpeeds[index] == USB_SPEED_SUPERSPEED) 2326 status->status |= PORT_STATUS_SS_POWER; 2327 else 2328 status->status |= PORT_STATUS_POWER; 2329 } 2330 if (fPortSpeeds[index] == USB_SPEED_SUPERSPEED) 2331 status->status |= portStatus & PS_PLS_MASK; 2332 2333 // build the change 2334 if (portStatus & PS_CSC) 2335 status->change |= PORT_STATUS_CONNECTION; 2336 if (portStatus & PS_PEC) 2337 status->change |= PORT_STATUS_ENABLE; 2338 if (portStatus & PS_OCC) 2339 status->change |= PORT_STATUS_OVER_CURRENT; 2340 if (portStatus & PS_PRC) 2341 status->change |= PORT_STATUS_RESET; 2342 2343 if (fPortSpeeds[index] == USB_SPEED_SUPERSPEED) { 2344 if (portStatus & PS_PLC) 2345 status->change |= PORT_CHANGE_LINK_STATE; 2346 if (portStatus & PS_WRC) 2347 status->change |= PORT_CHANGE_BH_PORT_RESET; 2348 } 2349 2350 return B_OK; 2351 } 2352 2353 2354 status_t 2355 XHCI::SetPortFeature(uint8 index, uint16 feature) 2356 { 2357 TRACE("set port feature index %u feature %u\n", index, feature); 2358 if (index >= fPortCount) 2359 return B_BAD_INDEX; 2360 2361 uint32 portRegister = XHCI_PORTSC(index); 2362 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2363 2364 switch (feature) { 2365 case PORT_SUSPEND: 2366 if ((portStatus & PS_PED) == 0 || (portStatus & PS_PR) 2367 || (portStatus & PS_PLS_MASK) >= PS_XDEV_U3) { 2368 TRACE_ERROR("USB core suspending device not in U0/U1/U2.\n"); 2369 return B_BAD_VALUE; 2370 } 2371 portStatus &= ~PS_PLS_MASK; 2372 WriteOpReg(portRegister, portStatus | PS_LWS | PS_XDEV_U3); 2373 break; 2374 2375 case PORT_RESET: 2376 WriteOpReg(portRegister, portStatus | PS_PR); 2377 break; 2378 2379 case PORT_POWER: 2380 WriteOpReg(portRegister, portStatus | PS_PP); 2381 break; 2382 default: 2383 return B_BAD_VALUE; 2384 } 2385 ReadOpReg(portRegister); 2386 return B_OK; 2387 } 2388 2389 2390 status_t 2391 XHCI::ClearPortFeature(uint8 index, uint16 feature) 2392 { 2393 TRACE("clear port feature index %u feature %u\n", index, feature); 2394 if (index >= fPortCount) 2395 return B_BAD_INDEX; 2396 2397 uint32 portRegister = XHCI_PORTSC(index); 2398 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2399 2400 switch (feature) { 2401 case PORT_SUSPEND: 2402 portStatus = ReadOpReg(portRegister); 2403 if (portStatus & PS_PR) 2404 return B_BAD_VALUE; 2405 if (portStatus & PS_XDEV_U3) { 2406 if ((portStatus & PS_PED) == 0) 2407 return B_BAD_VALUE; 2408 portStatus &= ~PS_PLS_MASK; 2409 WriteOpReg(portRegister, portStatus | PS_XDEV_U0 | PS_LWS); 2410 } 2411 break; 2412 case PORT_ENABLE: 2413 WriteOpReg(portRegister, portStatus | PS_PED); 2414 break; 2415 case PORT_POWER: 2416 WriteOpReg(portRegister, portStatus & ~PS_PP); 2417 break; 2418 case C_PORT_CONNECTION: 2419 WriteOpReg(portRegister, portStatus | PS_CSC); 2420 break; 2421 case C_PORT_ENABLE: 2422 WriteOpReg(portRegister, portStatus | PS_PEC); 2423 break; 2424 case C_PORT_OVER_CURRENT: 2425 WriteOpReg(portRegister, portStatus | PS_OCC); 2426 break; 2427 case C_PORT_RESET: 2428 WriteOpReg(portRegister, portStatus | PS_PRC); 2429 break; 2430 case C_PORT_BH_PORT_RESET: 2431 WriteOpReg(portRegister, portStatus | PS_WRC); 2432 break; 2433 case C_PORT_LINK_STATE: 2434 WriteOpReg(portRegister, portStatus | PS_PLC); 2435 break; 2436 default: 2437 return B_BAD_VALUE; 2438 } 2439 2440 ReadOpReg(portRegister); 2441 return B_OK; 2442 } 2443 2444 2445 status_t 2446 XHCI::ControllerHalt() 2447 { 2448 // Mask off run state 2449 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) & ~CMD_RUN); 2450 2451 // wait for shutdown state 2452 if (WaitOpBits(XHCI_STS, STS_HCH, STS_HCH) != B_OK) { 2453 TRACE_ERROR("HCH shutdown timeout\n"); 2454 return B_ERROR; 2455 } 2456 return B_OK; 2457 } 2458 2459 2460 status_t 2461 XHCI::ControllerReset() 2462 { 2463 TRACE("ControllerReset() cmd: 0x%" B_PRIx32 " sts: 0x%" B_PRIx32 "\n", 2464 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 2465 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) | CMD_HCRST); 2466 2467 if (WaitOpBits(XHCI_CMD, CMD_HCRST, 0) != B_OK) { 2468 TRACE_ERROR("ControllerReset() failed CMD_HCRST\n"); 2469 return B_ERROR; 2470 } 2471 2472 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 2473 TRACE_ERROR("ControllerReset() failed STS_CNR\n"); 2474 return B_ERROR; 2475 } 2476 2477 return B_OK; 2478 } 2479 2480 2481 int32 2482 XHCI::InterruptHandler(void* data) 2483 { 2484 return ((XHCI*)data)->Interrupt(); 2485 } 2486 2487 2488 int32 2489 XHCI::Interrupt() 2490 { 2491 SpinLocker _(&fSpinlock); 2492 2493 uint32 status = ReadOpReg(XHCI_STS); 2494 uint32 temp = ReadRunReg32(XHCI_IMAN(0)); 2495 WriteOpReg(XHCI_STS, status); 2496 WriteRunReg32(XHCI_IMAN(0), temp); 2497 2498 int32 result = B_HANDLED_INTERRUPT; 2499 2500 if ((status & STS_HCH) != 0) { 2501 TRACE_ERROR("Host Controller halted\n"); 2502 return result; 2503 } 2504 if ((status & STS_HSE) != 0) { 2505 TRACE_ERROR("Host System Error\n"); 2506 return result; 2507 } 2508 if ((status & STS_HCE) != 0) { 2509 TRACE_ERROR("Host Controller Error\n"); 2510 return result; 2511 } 2512 2513 if ((status & STS_EINT) == 0) { 2514 TRACE("STS: 0x%" B_PRIx32 " IRQ_PENDING: 0x%" B_PRIx32 "\n", 2515 status, temp); 2516 return B_UNHANDLED_INTERRUPT; 2517 } 2518 2519 TRACE("Event Interrupt\n"); 2520 release_sem_etc(fEventSem, 1, B_DO_NOT_RESCHEDULE); 2521 return B_INVOKE_SCHEDULER; 2522 } 2523 2524 2525 void 2526 XHCI::Ring(uint8 slot, uint8 endpoint) 2527 { 2528 TRACE("Ding Dong! slot:%d endpoint %d\n", slot, endpoint) 2529 if ((slot == 0 && endpoint > 0) || (slot > 0 && endpoint == 0)) 2530 panic("Ring() invalid slot/endpoint combination\n"); 2531 if (slot > fSlotCount || endpoint >= XHCI_MAX_ENDPOINTS) 2532 panic("Ring() invalid slot or endpoint\n"); 2533 2534 WriteDoorReg32(XHCI_DOORBELL(slot), XHCI_DOORBELL_TARGET(endpoint) 2535 | XHCI_DOORBELL_STREAMID(0)); 2536 ReadDoorReg32(XHCI_DOORBELL(slot)); 2537 // Flush PCI writes 2538 } 2539 2540 2541 void 2542 XHCI::QueueCommand(xhci_trb* trb) 2543 { 2544 uint8 i, j; 2545 uint32 temp; 2546 2547 i = fCmdIdx; 2548 j = fCmdCcs; 2549 2550 TRACE("command[%u] = %" B_PRId32 " (0x%016" B_PRIx64 ", 0x%08" B_PRIx32 2551 ", 0x%08" B_PRIx32 ")\n", i, TRB_3_TYPE_GET(trb->flags), trb->address, 2552 trb->status, trb->flags); 2553 2554 fCmdRing[i].address = trb->address; 2555 fCmdRing[i].status = trb->status; 2556 temp = trb->flags; 2557 2558 if (j) 2559 temp |= TRB_3_CYCLE_BIT; 2560 else 2561 temp &= ~TRB_3_CYCLE_BIT; 2562 temp &= ~TRB_3_TC_BIT; 2563 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2564 2565 fCmdAddr = fErst->rs_addr + (XHCI_MAX_EVENTS + i) * sizeof(xhci_trb); 2566 2567 i++; 2568 2569 if (i == (XHCI_MAX_COMMANDS - 1)) { 2570 temp = TRB_3_TYPE(TRB_TYPE_LINK) | TRB_3_TC_BIT; 2571 if (j) 2572 temp |= TRB_3_CYCLE_BIT; 2573 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2574 2575 i = 0; 2576 j ^= 1; 2577 } 2578 2579 fCmdIdx = i; 2580 fCmdCcs = j; 2581 } 2582 2583 2584 void 2585 XHCI::HandleCmdComplete(xhci_trb* trb) 2586 { 2587 if (fCmdAddr == trb->address) { 2588 TRACE("Received command event\n"); 2589 fCmdResult[0] = trb->status; 2590 fCmdResult[1] = B_LENDIAN_TO_HOST_INT32(trb->flags); 2591 release_sem_etc(fCmdCompSem, 1, B_DO_NOT_RESCHEDULE); 2592 } else 2593 TRACE_ERROR("received command event for unknown command!\n") 2594 } 2595 2596 2597 void 2598 XHCI::HandleTransferComplete(xhci_trb* trb) 2599 { 2600 const uint32 flags = B_LENDIAN_TO_HOST_INT32(trb->flags); 2601 const uint8 endpointNumber = TRB_3_ENDPOINT_GET(flags), 2602 slot = TRB_3_SLOT_GET(flags); 2603 2604 if (slot > fSlotCount) 2605 TRACE_ERROR("invalid slot\n"); 2606 if (endpointNumber == 0 || endpointNumber >= XHCI_MAX_ENDPOINTS) { 2607 TRACE_ERROR("invalid endpoint\n"); 2608 return; 2609 } 2610 2611 xhci_device *device = &fDevices[slot]; 2612 xhci_endpoint *endpoint = &device->endpoints[endpointNumber - 1]; 2613 2614 if (endpoint->trbs == NULL) { 2615 TRACE_ERROR("got TRB but endpoint is not allocated!\n"); 2616 return; 2617 } 2618 2619 // Use mutex_trylock first, in case we are in KDL. 2620 MutexLocker endpointLocker(endpoint->lock, mutex_trylock(&endpoint->lock) == B_OK); 2621 if (!endpointLocker.IsLocked()) { 2622 // We failed to get the lock. Most likely it was destroyed 2623 // while we were waiting for it. 2624 return; 2625 } 2626 2627 TRACE("HandleTransferComplete: ed %" B_PRIu32 ", status %" B_PRId32 "\n", 2628 (flags & TRB_3_EVENT_DATA_BIT), trb->status); 2629 2630 const uint8 completionCode = TRB_2_COMP_CODE_GET(trb->status); 2631 int32 remainder = TRB_2_REM_GET(trb->status), transferred = -1; 2632 if ((flags & TRB_3_EVENT_DATA_BIT) != 0) { 2633 // In the case of an Event Data TRB, value in the status field refers 2634 // to the actual number of bytes transferred across the whole TD. 2635 // (XHCI 1.2 § 6.4.2.1 Table 6-38 p478.) 2636 transferred = remainder; 2637 remainder = -1; 2638 } else { 2639 // This should only occur under error conditions. 2640 TRACE("got an interrupt for a non-Event Data TRB!\n"); 2641 2642 if (completionCode == COMP_STOPPED_LENGTH_INVALID) 2643 remainder = -1; 2644 } 2645 2646 if (completionCode != COMP_SUCCESS && completionCode != COMP_SHORT_PACKET 2647 && completionCode != COMP_STOPPED && completionCode != COMP_STOPPED_LENGTH_INVALID) { 2648 TRACE_ALWAYS("transfer error on slot %" B_PRId8 " endpoint %" B_PRId8 2649 ": %s\n", slot, endpointNumber, xhci_error_string(completionCode)); 2650 } 2651 2652 phys_addr_t source = B_LENDIAN_TO_HOST_INT64(trb->address); 2653 if (source >= endpoint->trb_addr 2654 && (source - endpoint->trb_addr) < (XHCI_ENDPOINT_RING_SIZE * sizeof(xhci_trb))) { 2655 // The "source" address points to a TRB on the ring. 2656 // See if we can figure out what it really corresponds to. 2657 const int64 offset = (source - endpoint->trb_addr) / sizeof(xhci_trb); 2658 const int32 type = TRB_3_TYPE_GET(endpoint->trbs[offset].flags); 2659 if (type == TRB_TYPE_EVENT_DATA || type == TRB_TYPE_LINK) 2660 source = B_LENDIAN_TO_HOST_INT64(endpoint->trbs[offset].address); 2661 } 2662 2663 for (xhci_td *td = endpoint->td_head; td != NULL; td = td->next) { 2664 int64 offset = (source - td->trb_addr) / sizeof(xhci_trb); 2665 if (offset < 0 || offset >= td->trb_count) 2666 continue; 2667 2668 TRACE("HandleTransferComplete td %p trb %" B_PRId64 " found\n", 2669 td, offset); 2670 2671 if (completionCode == COMP_STOPPED_LENGTH_INVALID) { 2672 // To determine transferred length, sum up the lengths of all TRBs 2673 // prior to the referenced one. (XHCI 1.2 § 4.6.9 p136.) 2674 transferred = 0; 2675 for (int32 i = 0; i < offset; i++) 2676 transferred += TRB_2_BYTES_GET(td->trbs[i].status); 2677 } 2678 2679 // The TRB at offset trb_used will be the link TRB, which we do not 2680 // care about (and should not generate an interrupt at all.) We really 2681 // care about the properly last TRB, at index "count - 1", which the 2682 // Event Data TRB that _LinkDescriptorForPipe creates points to. 2683 // 2684 // But if we have an unsuccessful completion code, the transfer 2685 // likely failed midway; so just accept it anyway. 2686 if (offset == (td->trb_used - 1) || completionCode != COMP_SUCCESS) { 2687 _UnlinkDescriptorForPipe(td, endpoint); 2688 endpointLocker.Unlock(); 2689 2690 td->trb_completion_code = completionCode; 2691 td->td_transferred = transferred; 2692 td->trb_left = remainder; 2693 2694 // add descriptor to finished list 2695 if (mutex_trylock(&fFinishedLock) != B_OK) 2696 mutex_lock(&fFinishedLock); 2697 td->next = fFinishedHead; 2698 fFinishedHead = td; 2699 mutex_unlock(&fFinishedLock); 2700 2701 release_sem_etc(fFinishTransfersSem, 1, B_DO_NOT_RESCHEDULE); 2702 TRACE("HandleTransferComplete td %p done\n", td); 2703 } else { 2704 TRACE_ERROR("successful TRB 0x%" B_PRIxPHYSADDR " was found, but it wasn't " 2705 "the last in the TD!\n", source); 2706 } 2707 return; 2708 } 2709 TRACE_ERROR("TRB 0x%" B_PRIxPHYSADDR " was not found in the endpoint!\n", source); 2710 } 2711 2712 2713 void 2714 XHCI::DumpRing(xhci_trb *trbs, uint32 size) 2715 { 2716 if (!Lock()) { 2717 TRACE("Unable to get lock!\n"); 2718 return; 2719 } 2720 2721 for (uint32 i = 0; i < size; i++) { 2722 TRACE("command[%" B_PRId32 "] = %" B_PRId32 " (0x%016" B_PRIx64 "," 2723 " 0x%08" B_PRIx32 ", 0x%08" B_PRIx32 ")\n", i, 2724 TRB_3_TYPE_GET(B_LENDIAN_TO_HOST_INT32(trbs[i].flags)), 2725 trbs[i].address, trbs[i].status, trbs[i].flags); 2726 } 2727 2728 Unlock(); 2729 } 2730 2731 2732 status_t 2733 XHCI::DoCommand(xhci_trb* trb) 2734 { 2735 if (!Lock()) { 2736 TRACE("Unable to get lock!\n"); 2737 return B_ERROR; 2738 } 2739 2740 QueueCommand(trb); 2741 Ring(0, 0); 2742 2743 // Begin with a 50ms timeout. 2744 if (acquire_sem_etc(fCmdCompSem, 1, B_RELATIVE_TIMEOUT, 50 * 1000) != B_OK) { 2745 // We've hit the timeout. In some error cases, interrupts are not 2746 // generated; so here we force the event ring to be polled once. 2747 release_sem(fEventSem); 2748 2749 // Now try again, this time with a 750ms timeout. 2750 if (acquire_sem_etc(fCmdCompSem, 1, B_RELATIVE_TIMEOUT, 2751 750 * 1000) != B_OK) { 2752 TRACE("Unable to obtain fCmdCompSem!\n"); 2753 fCmdAddr = 0; 2754 Unlock(); 2755 return B_TIMED_OUT; 2756 } 2757 } 2758 2759 // eat up sems that have been released by multiple interrupts 2760 int32 semCount = 0; 2761 get_sem_count(fCmdCompSem, &semCount); 2762 if (semCount > 0) 2763 acquire_sem_etc(fCmdCompSem, semCount, B_RELATIVE_TIMEOUT, 0); 2764 2765 status_t status = B_OK; 2766 uint32 completionCode = TRB_2_COMP_CODE_GET(fCmdResult[0]); 2767 TRACE("command complete\n"); 2768 if (completionCode != COMP_SUCCESS) { 2769 TRACE_ERROR("unsuccessful command %" B_PRId32 ", error %s (%" B_PRId32 ")\n", 2770 TRB_3_TYPE_GET(trb->flags), xhci_error_string(completionCode), 2771 completionCode); 2772 status = B_IO_ERROR; 2773 } 2774 2775 trb->status = fCmdResult[0]; 2776 trb->flags = fCmdResult[1]; 2777 2778 fCmdAddr = 0; 2779 Unlock(); 2780 return status; 2781 } 2782 2783 2784 status_t 2785 XHCI::Noop() 2786 { 2787 TRACE("Issue No-Op\n"); 2788 xhci_trb trb; 2789 trb.address = 0; 2790 trb.status = 0; 2791 trb.flags = TRB_3_TYPE(TRB_TYPE_CMD_NOOP); 2792 2793 return DoCommand(&trb); 2794 } 2795 2796 2797 status_t 2798 XHCI::EnableSlot(uint8* slot) 2799 { 2800 TRACE("Enable Slot\n"); 2801 xhci_trb trb; 2802 trb.address = 0; 2803 trb.status = 0; 2804 trb.flags = TRB_3_TYPE(TRB_TYPE_ENABLE_SLOT); 2805 2806 status_t status = DoCommand(&trb); 2807 if (status != B_OK) 2808 return status; 2809 2810 *slot = TRB_3_SLOT_GET(trb.flags); 2811 return *slot != 0 ? B_OK : B_BAD_VALUE; 2812 } 2813 2814 2815 status_t 2816 XHCI::DisableSlot(uint8 slot) 2817 { 2818 TRACE("Disable Slot\n"); 2819 xhci_trb trb; 2820 trb.address = 0; 2821 trb.status = 0; 2822 trb.flags = TRB_3_TYPE(TRB_TYPE_DISABLE_SLOT) | TRB_3_SLOT(slot); 2823 2824 return DoCommand(&trb); 2825 } 2826 2827 2828 status_t 2829 XHCI::SetAddress(uint64 inputContext, bool bsr, uint8 slot) 2830 { 2831 TRACE("Set Address\n"); 2832 xhci_trb trb; 2833 trb.address = inputContext; 2834 trb.status = 0; 2835 trb.flags = TRB_3_TYPE(TRB_TYPE_ADDRESS_DEVICE) | TRB_3_SLOT(slot); 2836 2837 if (bsr) 2838 trb.flags |= TRB_3_BSR_BIT; 2839 2840 return DoCommand(&trb); 2841 } 2842 2843 2844 status_t 2845 XHCI::ConfigureEndpoint(uint64 inputContext, bool deconfigure, uint8 slot) 2846 { 2847 TRACE("Configure Endpoint\n"); 2848 xhci_trb trb; 2849 trb.address = inputContext; 2850 trb.status = 0; 2851 trb.flags = TRB_3_TYPE(TRB_TYPE_CONFIGURE_ENDPOINT) | TRB_3_SLOT(slot); 2852 2853 if (deconfigure) 2854 trb.flags |= TRB_3_DCEP_BIT; 2855 2856 return DoCommand(&trb); 2857 } 2858 2859 2860 status_t 2861 XHCI::EvaluateContext(uint64 inputContext, uint8 slot) 2862 { 2863 TRACE("Evaluate Context\n"); 2864 xhci_trb trb; 2865 trb.address = inputContext; 2866 trb.status = 0; 2867 trb.flags = TRB_3_TYPE(TRB_TYPE_EVALUATE_CONTEXT) | TRB_3_SLOT(slot); 2868 2869 return DoCommand(&trb); 2870 } 2871 2872 2873 status_t 2874 XHCI::ResetEndpoint(bool preserve, xhci_endpoint* endpoint) 2875 { 2876 TRACE("Reset Endpoint\n"); 2877 2878 switch (_GetEndpointState(endpoint)) { 2879 case ENDPOINT_STATE_STOPPED: 2880 TRACE("Reset Endpoint: already stopped"); 2881 return B_OK; 2882 case ENDPOINT_STATE_HALTED: 2883 TRACE("Reset Endpoint: warning, weird state!"); 2884 default: 2885 break; 2886 } 2887 2888 xhci_trb trb; 2889 trb.address = 0; 2890 trb.status = 0; 2891 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_ENDPOINT) 2892 | TRB_3_SLOT(endpoint->device->slot) | TRB_3_ENDPOINT(endpoint->id + 1); 2893 if (preserve) 2894 trb.flags |= TRB_3_PRSV_BIT; 2895 2896 return DoCommand(&trb); 2897 } 2898 2899 2900 status_t 2901 XHCI::StopEndpoint(bool suspend, xhci_endpoint* endpoint) 2902 { 2903 TRACE("Stop Endpoint\n"); 2904 2905 switch (_GetEndpointState(endpoint)) { 2906 case ENDPOINT_STATE_HALTED: 2907 TRACE("Stop Endpoint: error, halted"); 2908 return B_DEV_STALLED; 2909 case ENDPOINT_STATE_STOPPED: 2910 TRACE("Stop Endpoint: already stopped"); 2911 return B_OK; 2912 default: 2913 break; 2914 } 2915 2916 xhci_trb trb; 2917 trb.address = 0; 2918 trb.status = 0; 2919 trb.flags = TRB_3_TYPE(TRB_TYPE_STOP_ENDPOINT) 2920 | TRB_3_SLOT(endpoint->device->slot) | TRB_3_ENDPOINT(endpoint->id + 1); 2921 if (suspend) 2922 trb.flags |= TRB_3_SUSPEND_ENDPOINT_BIT; 2923 2924 return DoCommand(&trb); 2925 } 2926 2927 2928 status_t 2929 XHCI::SetTRDequeue(uint64 dequeue, uint16 stream, uint8 endpoint, uint8 slot) 2930 { 2931 TRACE("Set TR Dequeue\n"); 2932 xhci_trb trb; 2933 trb.address = dequeue | ENDPOINT_2_DCS_BIT; 2934 // The DCS bit is copied from the address field as in ConfigureEndpoint. 2935 // (XHCI 1.2 § 4.6.10 p142.) 2936 trb.status = TRB_2_STREAM(stream); 2937 trb.flags = TRB_3_TYPE(TRB_TYPE_SET_TR_DEQUEUE) 2938 | TRB_3_SLOT(slot) | TRB_3_ENDPOINT(endpoint); 2939 2940 return DoCommand(&trb); 2941 } 2942 2943 2944 status_t 2945 XHCI::ResetDevice(uint8 slot) 2946 { 2947 TRACE("Reset Device\n"); 2948 xhci_trb trb; 2949 trb.address = 0; 2950 trb.status = 0; 2951 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_DEVICE) | TRB_3_SLOT(slot); 2952 2953 return DoCommand(&trb); 2954 } 2955 2956 2957 int32 2958 XHCI::EventThread(void* data) 2959 { 2960 ((XHCI *)data)->CompleteEvents(); 2961 return B_OK; 2962 } 2963 2964 2965 void 2966 XHCI::CompleteEvents() 2967 { 2968 while (!fStopThreads) { 2969 if (acquire_sem(fEventSem) < B_OK) 2970 continue; 2971 2972 // eat up sems that have been released by multiple interrupts 2973 int32 semCount = 0; 2974 get_sem_count(fEventSem, &semCount); 2975 if (semCount > 0) 2976 acquire_sem_etc(fEventSem, semCount, B_RELATIVE_TIMEOUT, 0); 2977 2978 ProcessEvents(); 2979 } 2980 } 2981 2982 2983 void 2984 XHCI::ProcessEvents() 2985 { 2986 // Use mutex_trylock first, in case we are in KDL. 2987 MutexLocker locker(fEventLock, mutex_trylock(&fEventLock) == B_OK); 2988 if (!locker.IsLocked()) { 2989 // We failed to get the lock. This really should not happen. 2990 TRACE_ERROR("failed to acquire event lock!\n"); 2991 return; 2992 } 2993 2994 uint16 i = fEventIdx; 2995 uint8 j = fEventCcs; 2996 uint8 t = 2; 2997 2998 while (1) { 2999 uint32 temp = B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags); 3000 uint8 event = TRB_3_TYPE_GET(temp); 3001 TRACE("event[%u] = %u (0x%016" B_PRIx64 " 0x%08" B_PRIx32 " 0x%08" 3002 B_PRIx32 ")\n", i, event, fEventRing[i].address, 3003 fEventRing[i].status, B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags)); 3004 uint8 k = (temp & TRB_3_CYCLE_BIT) ? 1 : 0; 3005 if (j != k) 3006 break; 3007 3008 switch (event) { 3009 case TRB_TYPE_COMMAND_COMPLETION: 3010 HandleCmdComplete(&fEventRing[i]); 3011 break; 3012 case TRB_TYPE_TRANSFER: 3013 HandleTransferComplete(&fEventRing[i]); 3014 break; 3015 case TRB_TYPE_PORT_STATUS_CHANGE: 3016 TRACE("port change detected\n"); 3017 break; 3018 default: 3019 TRACE_ERROR("Unhandled event = %u\n", event); 3020 break; 3021 } 3022 3023 i++; 3024 if (i == XHCI_MAX_EVENTS) { 3025 i = 0; 3026 j ^= 1; 3027 if (!--t) 3028 break; 3029 } 3030 } 3031 3032 fEventIdx = i; 3033 fEventCcs = j; 3034 3035 uint64 addr = fErst->rs_addr + i * sizeof(xhci_trb); 3036 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)addr | ERDP_BUSY); 3037 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(addr >> 32)); 3038 } 3039 3040 3041 int32 3042 XHCI::FinishThread(void* data) 3043 { 3044 ((XHCI *)data)->FinishTransfers(); 3045 return B_OK; 3046 } 3047 3048 3049 void 3050 XHCI::FinishTransfers() 3051 { 3052 while (!fStopThreads) { 3053 if (acquire_sem(fFinishTransfersSem) < B_OK) 3054 continue; 3055 3056 // eat up sems that have been released by multiple interrupts 3057 int32 semCount = 0; 3058 get_sem_count(fFinishTransfersSem, &semCount); 3059 if (semCount > 0) 3060 acquire_sem_etc(fFinishTransfersSem, semCount, B_RELATIVE_TIMEOUT, 0); 3061 3062 mutex_lock(&fFinishedLock); 3063 TRACE("finishing transfers\n"); 3064 while (fFinishedHead != NULL) { 3065 xhci_td* td = fFinishedHead; 3066 fFinishedHead = td->next; 3067 td->next = NULL; 3068 mutex_unlock(&fFinishedLock); 3069 3070 TRACE("finishing transfer td %p\n", td); 3071 3072 Transfer* transfer = td->transfer; 3073 if (transfer == NULL) { 3074 // No transfer? Quick way out. 3075 FreeDescriptor(td); 3076 mutex_lock(&fFinishedLock); 3077 continue; 3078 } 3079 3080 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 3081 3082 status_t callbackStatus = B_OK; 3083 const uint8 completionCode = td->trb_completion_code; 3084 switch (completionCode) { 3085 case COMP_SHORT_PACKET: 3086 case COMP_SUCCESS: 3087 callbackStatus = B_OK; 3088 break; 3089 case COMP_DATA_BUFFER: 3090 callbackStatus = directionIn ? B_DEV_DATA_OVERRUN 3091 : B_DEV_DATA_UNDERRUN; 3092 break; 3093 case COMP_BABBLE: 3094 callbackStatus = directionIn ? B_DEV_FIFO_OVERRUN 3095 : B_DEV_FIFO_UNDERRUN; 3096 break; 3097 case COMP_USB_TRANSACTION: 3098 callbackStatus = B_DEV_CRC_ERROR; 3099 break; 3100 case COMP_STALL: 3101 callbackStatus = B_DEV_STALLED; 3102 break; 3103 default: 3104 callbackStatus = B_DEV_STALLED; 3105 break; 3106 } 3107 3108 size_t actualLength = transfer->FragmentLength(); 3109 if (completionCode != COMP_SUCCESS) { 3110 actualLength = td->td_transferred; 3111 if (td->td_transferred == -1) 3112 actualLength = transfer->FragmentLength() - td->trb_left; 3113 TRACE("transfer not successful, actualLength=%" B_PRIuSIZE "\n", 3114 actualLength); 3115 } 3116 3117 usb_isochronous_data* isochronousData = transfer->IsochronousData(); 3118 if (isochronousData != NULL) { 3119 size_t packetSize = transfer->DataLength() 3120 / isochronousData->packet_count, 3121 left = actualLength; 3122 for (uint32 i = 0; i < isochronousData->packet_count; i++) { 3123 size_t size = min_c(packetSize, left); 3124 isochronousData->packet_descriptors[i].actual_length = size; 3125 isochronousData->packet_descriptors[i].status = (size > 0) 3126 ? B_OK : B_DEV_FIFO_UNDERRUN; 3127 left -= size; 3128 } 3129 } 3130 3131 if (callbackStatus == B_OK && directionIn && actualLength > 0) { 3132 TRACE("copying in iov count %ld\n", transfer->VectorCount()); 3133 status_t status = transfer->PrepareKernelAccess(); 3134 if (status == B_OK) { 3135 ReadDescriptor(td, transfer->Vector(), 3136 transfer->VectorCount(), transfer->IsPhysical()); 3137 } else { 3138 callbackStatus = status; 3139 } 3140 } 3141 3142 FreeDescriptor(td); 3143 3144 // this transfer may still have data left 3145 bool finished = true; 3146 transfer->AdvanceByFragment(actualLength); 3147 if (completionCode == COMP_SUCCESS 3148 && transfer->FragmentLength() > 0) { 3149 TRACE("still %" B_PRIuSIZE " bytes left on transfer\n", 3150 transfer->FragmentLength()); 3151 callbackStatus = SubmitTransfer(transfer); 3152 finished = (callbackStatus != B_OK); 3153 } 3154 if (finished) { 3155 // The actualLength was already handled in AdvanceByFragment. 3156 transfer->Finished(callbackStatus, 0); 3157 delete transfer; 3158 } 3159 3160 mutex_lock(&fFinishedLock); 3161 } 3162 mutex_unlock(&fFinishedLock); 3163 } 3164 } 3165 3166 3167 inline void 3168 XHCI::WriteOpReg(uint32 reg, uint32 value) 3169 { 3170 *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg) = value; 3171 } 3172 3173 3174 inline uint32 3175 XHCI::ReadOpReg(uint32 reg) 3176 { 3177 return *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg); 3178 } 3179 3180 3181 inline status_t 3182 XHCI::WaitOpBits(uint32 reg, uint32 mask, uint32 expected) 3183 { 3184 int loops = 0; 3185 uint32 value = ReadOpReg(reg); 3186 while ((value & mask) != expected) { 3187 snooze(1000); 3188 value = ReadOpReg(reg); 3189 if (loops == 100) { 3190 TRACE("delay waiting on reg 0x%" B_PRIX32 " match 0x%" B_PRIX32 3191 " (0x%" B_PRIX32 ")\n", reg, expected, mask); 3192 } else if (loops > 250) { 3193 TRACE_ERROR("timeout waiting on reg 0x%" B_PRIX32 3194 " match 0x%" B_PRIX32 " (0x%" B_PRIX32 ")\n", reg, expected, 3195 mask); 3196 return B_ERROR; 3197 } 3198 loops++; 3199 } 3200 return B_OK; 3201 } 3202 3203 3204 inline uint32 3205 XHCI::ReadCapReg32(uint32 reg) 3206 { 3207 return *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg); 3208 } 3209 3210 3211 inline void 3212 XHCI::WriteCapReg32(uint32 reg, uint32 value) 3213 { 3214 *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg) = value; 3215 } 3216 3217 3218 inline uint32 3219 XHCI::ReadRunReg32(uint32 reg) 3220 { 3221 return *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg); 3222 } 3223 3224 3225 inline void 3226 XHCI::WriteRunReg32(uint32 reg, uint32 value) 3227 { 3228 *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg) = value; 3229 } 3230 3231 3232 inline uint32 3233 XHCI::ReadDoorReg32(uint32 reg) 3234 { 3235 return *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg); 3236 } 3237 3238 3239 inline void 3240 XHCI::WriteDoorReg32(uint32 reg, uint32 value) 3241 { 3242 *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg) = value; 3243 } 3244 3245 3246 inline addr_t 3247 XHCI::_OffsetContextAddr(addr_t p) 3248 { 3249 if (fContextSizeShift == 1) { 3250 // each structure is page aligned, each pointer is 32 bits aligned 3251 uint32 offset = p & ((B_PAGE_SIZE - 1) & ~31U); 3252 p += offset; 3253 } 3254 return p; 3255 } 3256 3257 inline uint32 3258 XHCI::_ReadContext(uint32* p) 3259 { 3260 p = (uint32*)_OffsetContextAddr((addr_t)p); 3261 return *p; 3262 } 3263 3264 3265 inline void 3266 XHCI::_WriteContext(uint32* p, uint32 value) 3267 { 3268 p = (uint32*)_OffsetContextAddr((addr_t)p); 3269 *p = value; 3270 } 3271 3272 3273 inline uint64 3274 XHCI::_ReadContext(uint64* p) 3275 { 3276 p = (uint64*)_OffsetContextAddr((addr_t)p); 3277 return *p; 3278 } 3279 3280 3281 inline void 3282 XHCI::_WriteContext(uint64* p, uint64 value) 3283 { 3284 p = (uint64*)_OffsetContextAddr((addr_t)p); 3285 *p = value; 3286 } 3287