1 /* 2 * Copyright 2011-2021, Haiku, Inc. All rights reserved. 3 * Distributed under the terms of the MIT License. 4 * 5 * Authors: 6 * Augustin Cavalier <waddlesplash> 7 * Jian Chiang <j.jian.chiang@gmail.com> 8 * Jérôme Duval <jerome.duval@gmail.com> 9 * Akshay Jaggi <akshay1994.leo@gmail.com> 10 * Michael Lotz <mmlr@mlotz.ch> 11 * Alexander von Gluck <kallisti5@unixzen.com> 12 */ 13 14 15 #include <stdio.h> 16 17 #include <bus/PCI.h> 18 #include <USB3.h> 19 #include <KernelExport.h> 20 21 #include <ByteOrder.h> 22 #include <util/AutoLock.h> 23 24 #include "xhci.h" 25 26 27 #define CALLED(x...) TRACE_MODULE("CALLED %s\n", __PRETTY_FUNCTION__) 28 29 30 #define USB_MODULE_NAME "xhci" 31 32 device_manager_info* gDeviceManager; 33 static usb_for_controller_interface* gUSB; 34 35 36 #define XHCI_PCI_DEVICE_MODULE_NAME "busses/usb/xhci/pci/driver_v1" 37 #define XHCI_PCI_USB_BUS_MODULE_NAME "busses/usb/xhci/device_v1" 38 39 40 typedef struct { 41 XHCI* xhci; 42 pci_device_module_info* pci; 43 pci_device* device; 44 45 pci_info pciinfo; 46 47 device_node* node; 48 device_node* driver_node; 49 } xhci_pci_sim_info; 50 51 52 // #pragma mark - 53 54 55 static status_t 56 init_bus(device_node* node, void** bus_cookie) 57 { 58 CALLED(); 59 60 driver_module_info* driver; 61 xhci_pci_sim_info* bus; 62 device_node* parent = gDeviceManager->get_parent_node(node); 63 gDeviceManager->get_driver(parent, &driver, (void**)&bus); 64 gDeviceManager->put_node(parent); 65 66 Stack *stack; 67 if (gUSB->get_stack((void**)&stack) != B_OK) 68 return B_ERROR; 69 70 XHCI *xhci = new(std::nothrow) XHCI(&bus->pciinfo, bus->pci, bus->device, stack, node); 71 if (xhci == NULL) { 72 return B_NO_MEMORY; 73 } 74 75 if (xhci->InitCheck() < B_OK) { 76 TRACE_MODULE_ERROR("bus failed init check\n"); 77 delete xhci; 78 return B_ERROR; 79 } 80 81 if (xhci->Start() != B_OK) { 82 delete xhci; 83 return B_ERROR; 84 } 85 86 *bus_cookie = xhci; 87 88 return B_OK; 89 } 90 91 92 static void 93 uninit_bus(void* bus_cookie) 94 { 95 CALLED(); 96 XHCI* xhci = (XHCI*)bus_cookie; 97 delete xhci; 98 } 99 100 101 static status_t 102 register_child_devices(void* cookie) 103 { 104 CALLED(); 105 xhci_pci_sim_info* bus = (xhci_pci_sim_info*)cookie; 106 device_node* node = bus->driver_node; 107 108 char prettyName[25]; 109 sprintf(prettyName, "XHCI Controller %" B_PRIu16, 0); 110 111 device_attr attrs[] = { 112 // properties of this controller for the usb bus manager 113 { B_DEVICE_PRETTY_NAME, B_STRING_TYPE, 114 { .string = prettyName }}, 115 { B_DEVICE_FIXED_CHILD, B_STRING_TYPE, 116 { .string = USB_FOR_CONTROLLER_MODULE_NAME }}, 117 118 // private data to identify the device 119 { NULL } 120 }; 121 122 return gDeviceManager->register_node(node, XHCI_PCI_USB_BUS_MODULE_NAME, 123 attrs, NULL, NULL); 124 } 125 126 127 static status_t 128 init_device(device_node* node, void** device_cookie) 129 { 130 CALLED(); 131 xhci_pci_sim_info* bus = (xhci_pci_sim_info*)calloc(1, 132 sizeof(xhci_pci_sim_info)); 133 if (bus == NULL) 134 return B_NO_MEMORY; 135 136 pci_device_module_info* pci; 137 pci_device* device; 138 { 139 device_node* pciParent = gDeviceManager->get_parent_node(node); 140 gDeviceManager->get_driver(pciParent, (driver_module_info**)&pci, 141 (void**)&device); 142 gDeviceManager->put_node(pciParent); 143 } 144 145 bus->pci = pci; 146 bus->device = device; 147 bus->driver_node = node; 148 149 pci_info *pciInfo = &bus->pciinfo; 150 pci->get_pci_info(device, pciInfo); 151 152 *device_cookie = bus; 153 return B_OK; 154 } 155 156 157 static void 158 uninit_device(void* device_cookie) 159 { 160 CALLED(); 161 xhci_pci_sim_info* bus = (xhci_pci_sim_info*)device_cookie; 162 free(bus); 163 } 164 165 166 static status_t 167 register_device(device_node* parent) 168 { 169 CALLED(); 170 device_attr attrs[] = { 171 {B_DEVICE_PRETTY_NAME, B_STRING_TYPE, {.string = "XHCI PCI"}}, 172 {} 173 }; 174 175 return gDeviceManager->register_node(parent, 176 XHCI_PCI_DEVICE_MODULE_NAME, attrs, NULL, NULL); 177 } 178 179 180 static float 181 supports_device(device_node* parent) 182 { 183 CALLED(); 184 const char* bus; 185 uint16 type, subType, api; 186 187 // make sure parent is a XHCI PCI device node 188 if (gDeviceManager->get_attr_string(parent, B_DEVICE_BUS, &bus, false) 189 < B_OK) { 190 return -1; 191 } 192 193 if (strcmp(bus, "pci") != 0) 194 return 0.0f; 195 196 if (gDeviceManager->get_attr_uint16(parent, B_DEVICE_SUB_TYPE, &subType, 197 false) < B_OK 198 || gDeviceManager->get_attr_uint16(parent, B_DEVICE_TYPE, &type, 199 false) < B_OK 200 || gDeviceManager->get_attr_uint16(parent, B_DEVICE_INTERFACE, &api, 201 false) < B_OK) { 202 TRACE_MODULE("Could not find type/subtype/interface attributes\n"); 203 return -1; 204 } 205 206 if (type == PCI_serial_bus && subType == PCI_usb && api == PCI_usb_xhci) { 207 pci_device_module_info* pci; 208 pci_device* device; 209 gDeviceManager->get_driver(parent, (driver_module_info**)&pci, 210 (void**)&device); 211 TRACE_MODULE("XHCI Device found!\n"); 212 213 return 0.8f; 214 } 215 216 return 0.0f; 217 } 218 219 220 static const char* 221 xhci_error_string(uint32 error) 222 { 223 switch (error) { 224 case COMP_INVALID: return "Invalid"; 225 case COMP_SUCCESS: return "Success"; 226 case COMP_DATA_BUFFER: return "Data buffer"; 227 case COMP_BABBLE: return "Babble detected"; 228 case COMP_USB_TRANSACTION: return "USB transaction"; 229 case COMP_TRB: return "TRB"; 230 case COMP_STALL: return "Stall"; 231 case COMP_RESOURCE: return "Resource"; 232 case COMP_BANDWIDTH: return "Bandwidth"; 233 case COMP_NO_SLOTS: return "No slots"; 234 case COMP_INVALID_STREAM: return "Invalid stream"; 235 case COMP_SLOT_NOT_ENABLED: return "Slot not enabled"; 236 case COMP_ENDPOINT_NOT_ENABLED: return "Endpoint not enabled"; 237 case COMP_SHORT_PACKET: return "Short packet"; 238 case COMP_RING_UNDERRUN: return "Ring underrun"; 239 case COMP_RING_OVERRUN: return "Ring overrun"; 240 case COMP_VF_RING_FULL: return "VF Event Ring Full"; 241 case COMP_PARAMETER: return "Parameter"; 242 case COMP_BANDWIDTH_OVERRUN: return "Bandwidth overrun"; 243 case COMP_CONTEXT_STATE: return "Context state"; 244 case COMP_NO_PING_RESPONSE: return "No ping response"; 245 case COMP_EVENT_RING_FULL: return "Event ring full"; 246 case COMP_INCOMPATIBLE_DEVICE: return "Incompatible device"; 247 case COMP_MISSED_SERVICE: return "Missed service"; 248 case COMP_COMMAND_RING_STOPPED: return "Command ring stopped"; 249 case COMP_COMMAND_ABORTED: return "Command aborted"; 250 case COMP_STOPPED: return "Stopped"; 251 case COMP_LENGTH_INVALID: return "Length invalid"; 252 case COMP_MAX_EXIT_LATENCY: return "Max exit latency too large"; 253 case COMP_ISOC_OVERRUN: return "Isoch buffer overrun"; 254 case COMP_EVENT_LOST: return "Event lost"; 255 case COMP_UNDEFINED: return "Undefined"; 256 case COMP_INVALID_STREAM_ID: return "Invalid stream ID"; 257 case COMP_SECONDARY_BANDWIDTH: return "Secondary bandwidth"; 258 case COMP_SPLIT_TRANSACTION: return "Split transaction"; 259 260 default: return "Undefined"; 261 } 262 } 263 264 265 module_dependency module_dependencies[] = { 266 { USB_FOR_CONTROLLER_MODULE_NAME, (module_info**)&gUSB }, 267 { B_DEVICE_MANAGER_MODULE_NAME, (module_info**)&gDeviceManager }, 268 {} 269 }; 270 271 272 static usb_bus_interface gXHCIPCIDeviceModule = { 273 { 274 { 275 XHCI_PCI_USB_BUS_MODULE_NAME, 276 0, 277 NULL 278 }, 279 NULL, // supports device 280 NULL, // register device 281 init_bus, 282 uninit_bus, 283 NULL, // register child devices 284 NULL, // rescan 285 NULL, // device removed 286 }, 287 }; 288 289 // Root device that binds to the PCI bus. It will register an usb_bus_interface 290 // node for each device. 291 static driver_module_info sXHCIDevice = { 292 { 293 XHCI_PCI_DEVICE_MODULE_NAME, 294 0, 295 NULL 296 }, 297 supports_device, 298 register_device, 299 init_device, 300 uninit_device, 301 register_child_devices, 302 NULL, // rescan 303 NULL, // device removed 304 }; 305 306 module_info* modules[] = { 307 (module_info* )&sXHCIDevice, 308 (module_info* )&gXHCIPCIDeviceModule, 309 NULL 310 }; 311 312 313 XHCI::XHCI(pci_info *info, pci_device_module_info* pci, pci_device* device, Stack *stack, 314 device_node* node) 315 : BusManager(stack, node), 316 fRegisterArea(-1), 317 fRegisters(NULL), 318 fPCIInfo(info), 319 fPci(pci), 320 fDevice(device), 321 fStack(stack), 322 fIRQ(0), 323 fUseMSI(false), 324 fErstArea(-1), 325 fDcbaArea(-1), 326 fCmdCompSem(-1), 327 fStopThreads(false), 328 fRootHub(NULL), 329 fPortCount(0), 330 fSlotCount(0), 331 fScratchpadCount(0), 332 fContextSizeShift(0), 333 fFinishedHead(NULL), 334 fFinishTransfersSem(-1), 335 fFinishThread(-1), 336 fEventSem(-1), 337 fEventThread(-1), 338 fEventIdx(0), 339 fCmdIdx(0), 340 fEventCcs(1), 341 fCmdCcs(1) 342 { 343 B_INITIALIZE_SPINLOCK(&fSpinlock); 344 mutex_init(&fFinishedLock, "XHCI finished transfers"); 345 mutex_init(&fEventLock, "XHCI event handler"); 346 347 if (BusManager::InitCheck() < B_OK) { 348 TRACE_ERROR("bus manager failed to init\n"); 349 return; 350 } 351 352 TRACE("constructing new XHCI host controller driver\n"); 353 fInitOK = false; 354 355 // enable busmaster and memory mapped access 356 uint16 command = fPci->read_pci_config(fDevice, PCI_command, 2); 357 command &= ~(PCI_command_io | PCI_command_int_disable); 358 command |= PCI_command_master | PCI_command_memory; 359 360 fPci->write_pci_config(fDevice, PCI_command, 2, command); 361 362 // map the registers (low + high for 64-bit when requested) 363 phys_addr_t physicalAddress = fPCIInfo->u.h0.base_registers[0]; 364 if ((fPCIInfo->u.h0.base_register_flags[0] & PCI_address_type) 365 == PCI_address_type_64) { 366 physicalAddress |= (uint64)fPCIInfo->u.h0.base_registers[1] << 32; 367 } 368 369 size_t mapSize = fPCIInfo->u.h0.base_register_sizes[0]; 370 371 TRACE("map registers %08" B_PRIxPHYSADDR ", size: %" B_PRIuSIZE "\n", 372 physicalAddress, mapSize); 373 374 fRegisterArea = map_physical_memory("XHCI memory mapped registers", 375 physicalAddress, mapSize, B_ANY_KERNEL_BLOCK_ADDRESS, 376 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 377 (void **)&fRegisters); 378 if (fRegisterArea < B_OK) { 379 TRACE_ERROR("failed to map register memory\n"); 380 return; 381 } 382 383 // determine the register offsets 384 fCapabilityRegisterOffset = 0; 385 fOperationalRegisterOffset = HCI_CAPLENGTH(ReadCapReg32(XHCI_HCI_CAPLENGTH)); 386 fRuntimeRegisterOffset = ReadCapReg32(XHCI_RTSOFF) & ~0x1F; 387 fDoorbellRegisterOffset = ReadCapReg32(XHCI_DBOFF) & ~0x3; 388 389 TRACE("mapped registers: %p\n", fRegisters); 390 TRACE("operational register offset: %" B_PRId32 "\n", fOperationalRegisterOffset); 391 TRACE("runtime register offset: %" B_PRId32 "\n", fRuntimeRegisterOffset); 392 TRACE("doorbell register offset: %" B_PRId32 "\n", fDoorbellRegisterOffset); 393 394 int32 interfaceVersion = HCI_VERSION(ReadCapReg32(XHCI_HCI_VERSION)); 395 if (interfaceVersion < 0x0090 || interfaceVersion > 0x0120) { 396 TRACE_ERROR("unsupported interface version: 0x%04" B_PRIx32 "\n", 397 interfaceVersion); 398 return; 399 } 400 TRACE_ALWAYS("interface version: 0x%04" B_PRIx32 "\n", interfaceVersion); 401 402 TRACE_ALWAYS("structural parameters: 1:0x%08" B_PRIx32 " 2:0x%08" 403 B_PRIx32 " 3:0x%08" B_PRIx32 "\n", ReadCapReg32(XHCI_HCSPARAMS1), 404 ReadCapReg32(XHCI_HCSPARAMS2), ReadCapReg32(XHCI_HCSPARAMS3)); 405 406 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 407 if (cparams == 0xffffffff) 408 return; 409 TRACE_ALWAYS("capability parameters: 0x%08" B_PRIx32 "\n", cparams); 410 411 // if 64 bytes context structures, then 1 412 fContextSizeShift = HCC_CSZ(cparams); 413 414 // Assume ownership of the controller from the BIOS. 415 uint32 eec = 0xffffffff; 416 uint32 eecp = HCS0_XECP(cparams) << 2; 417 for (; eecp != 0 && XECP_NEXT(eec); eecp += XECP_NEXT(eec) << 2) { 418 TRACE("eecp register: 0x%08" B_PRIx32 "\n", eecp); 419 420 eec = ReadCapReg32(eecp); 421 if (XECP_ID(eec) != XHCI_LEGSUP_CAPID) 422 continue; 423 424 if (eec & XHCI_LEGSUP_BIOSOWNED) { 425 TRACE_ALWAYS("the host controller is bios owned, claiming" 426 " ownership\n"); 427 WriteCapReg32(eecp, eec | XHCI_LEGSUP_OSOWNED); 428 429 for (int32 i = 0; i < 20; i++) { 430 eec = ReadCapReg32(eecp); 431 432 if ((eec & XHCI_LEGSUP_BIOSOWNED) == 0) 433 break; 434 435 TRACE_ALWAYS("controller is still bios owned, waiting\n"); 436 snooze(50000); 437 } 438 439 if (eec & XHCI_LEGSUP_BIOSOWNED) { 440 TRACE_ERROR("bios won't give up control over the host " 441 "controller (ignoring)\n"); 442 } else if (eec & XHCI_LEGSUP_OSOWNED) { 443 TRACE_ALWAYS("successfully took ownership of the host " 444 "controller\n"); 445 } 446 447 // Force off the BIOS owned flag, and clear all SMIs. Some BIOSes 448 // do indicate a successful handover but do not remove their SMIs 449 // and then freeze the system when interrupts are generated. 450 WriteCapReg32(eecp, eec & ~XHCI_LEGSUP_BIOSOWNED); 451 } 452 break; 453 } 454 uint32 legctlsts = ReadCapReg32(eecp + XHCI_LEGCTLSTS); 455 legctlsts &= XHCI_LEGCTLSTS_DISABLE_SMI; 456 legctlsts |= XHCI_LEGCTLSTS_EVENTS_SMI; 457 WriteCapReg32(eecp + XHCI_LEGCTLSTS, legctlsts); 458 459 // We need to explicitly take ownership of EHCI ports on earlier Intel chipsets. 460 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL) { 461 switch (fPCIInfo->device_id) { 462 case PCI_DEVICE_INTEL_PANTHER_POINT_XHCI: 463 case PCI_DEVICE_INTEL_LYNX_POINT_XHCI: 464 case PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI: 465 case PCI_DEVICE_INTEL_BAYTRAIL_XHCI: 466 case PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI: 467 case PCI_DEVICE_INTEL_WILDCAT_POINT_LP_XHCI: 468 _SwitchIntelPorts(); 469 break; 470 } 471 } 472 473 // halt the host controller 474 if (ControllerHalt() < B_OK) { 475 return; 476 } 477 478 // reset the host controller 479 if (ControllerReset() < B_OK) { 480 TRACE_ERROR("host controller failed to reset\n"); 481 return; 482 } 483 484 fCmdCompSem = create_sem(0, "XHCI Command Complete"); 485 fFinishTransfersSem = create_sem(0, "XHCI Finish Transfers"); 486 fEventSem = create_sem(0, "XHCI Event"); 487 if (fFinishTransfersSem < B_OK || fCmdCompSem < B_OK || fEventSem < B_OK) { 488 TRACE_ERROR("failed to create semaphores\n"); 489 return; 490 } 491 492 // create event handler thread 493 fEventThread = spawn_kernel_thread(EventThread, "xhci event thread", 494 B_URGENT_PRIORITY, (void *)this); 495 resume_thread(fEventThread); 496 497 // create finisher service thread 498 fFinishThread = spawn_kernel_thread(FinishThread, "xhci finish thread", 499 B_URGENT_PRIORITY - 1, (void *)this); 500 resume_thread(fFinishThread); 501 502 // Find the right interrupt vector, using MSIs if available. 503 fIRQ = fPCIInfo->u.h0.interrupt_line; 504 if (fPci->get_msi_count(fDevice) >= 1) { 505 uint8 msiVector = 0; 506 if (fPci->configure_msi(fDevice, 1, &msiVector) == B_OK 507 && fPci->enable_msi(fDevice) == B_OK) { 508 TRACE_ALWAYS("using message signaled interrupts\n"); 509 fIRQ = msiVector; 510 fUseMSI = true; 511 } 512 } 513 514 if (fIRQ == 0 || fIRQ == 0xFF) { 515 TRACE_MODULE_ERROR("device PCI:%d:%d:%d was assigned an invalid IRQ\n", 516 fPCIInfo->bus, fPCIInfo->device, fPCIInfo->function); 517 return; 518 } 519 520 // Install the interrupt handler 521 TRACE("installing interrupt handler\n"); 522 install_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this, 0); 523 524 memset(fPortSpeeds, 0, sizeof(fPortSpeeds)); 525 memset(fDevices, 0, sizeof(fDevices)); 526 527 fInitOK = true; 528 TRACE("driver construction successful\n"); 529 } 530 531 532 XHCI::~XHCI() 533 { 534 TRACE("tear down XHCI host controller driver\n"); 535 536 WriteOpReg(XHCI_CMD, 0); 537 538 int32 result = 0; 539 fStopThreads = true; 540 delete_sem(fCmdCompSem); 541 delete_sem(fFinishTransfersSem); 542 delete_sem(fEventSem); 543 wait_for_thread(fFinishThread, &result); 544 wait_for_thread(fEventThread, &result); 545 546 mutex_destroy(&fFinishedLock); 547 mutex_destroy(&fEventLock); 548 549 remove_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this); 550 551 delete_area(fRegisterArea); 552 delete_area(fErstArea); 553 for (uint32 i = 0; i < fScratchpadCount; i++) 554 delete_area(fScratchpadArea[i]); 555 delete_area(fDcbaArea); 556 557 if (fUseMSI) { 558 fPci->disable_msi(fDevice); 559 fPci->unconfigure_msi(fDevice); 560 } 561 } 562 563 564 void 565 XHCI::_SwitchIntelPorts() 566 { 567 TRACE("Looking for EHCI owned ports\n"); 568 uint32 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_USB3PRM, 4); 569 TRACE("Superspeed Ports: 0x%" B_PRIx32 "\n", ports); 570 fPci->write_pci_config(fDevice, XHCI_INTEL_USB3_PSSEN, 4, ports); 571 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_USB3_PSSEN, 4); 572 TRACE("Superspeed ports now under XHCI : 0x%" B_PRIx32 "\n", ports); 573 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_USB2PRM, 4); 574 TRACE("USB 2.0 Ports : 0x%" B_PRIx32 "\n", ports); 575 fPci->write_pci_config(fDevice, XHCI_INTEL_XUSB2PR, 4, ports); 576 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_XUSB2PR, 4); 577 TRACE("USB 2.0 ports now under XHCI: 0x%" B_PRIx32 "\n", ports); 578 } 579 580 581 status_t 582 XHCI::Start() 583 { 584 TRACE_ALWAYS("starting XHCI host controller\n"); 585 TRACE("usbcmd: 0x%08" B_PRIx32 "; usbsts: 0x%08" B_PRIx32 "\n", 586 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 587 588 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 589 TRACE("Start() failed STS_CNR\n"); 590 } 591 592 if ((ReadOpReg(XHCI_CMD) & CMD_RUN) != 0) { 593 TRACE_ERROR("Start() warning, starting running XHCI controller!\n"); 594 } 595 596 if ((ReadOpReg(XHCI_PAGESIZE) & (1 << 0)) == 0) { 597 TRACE_ERROR("controller does not support 4K page size\n"); 598 return B_ERROR; 599 } 600 601 // read port count from capability register 602 uint32 capabilities = ReadCapReg32(XHCI_HCSPARAMS1); 603 fPortCount = HCS_MAX_PORTS(capabilities); 604 if (fPortCount == 0) { 605 TRACE_ERROR("invalid number of ports: %u\n", fPortCount); 606 return B_ERROR; 607 } 608 609 fSlotCount = HCS_MAX_SLOTS(capabilities); 610 if (fSlotCount > XHCI_MAX_DEVICES) 611 fSlotCount = XHCI_MAX_DEVICES; 612 WriteOpReg(XHCI_CONFIG, fSlotCount); 613 614 // find out which protocol is used for each port 615 uint8 portFound = 0; 616 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 617 uint32 eec = 0xffffffff; 618 uint32 eecp = HCS0_XECP(cparams) << 2; 619 for (; eecp != 0 && XECP_NEXT(eec) && portFound < fPortCount; 620 eecp += XECP_NEXT(eec) << 2) { 621 eec = ReadCapReg32(eecp); 622 if (XECP_ID(eec) != XHCI_SUPPORTED_PROTOCOLS_CAPID) 623 continue; 624 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) > 3) 625 continue; 626 uint32 temp = ReadCapReg32(eecp + 8); 627 uint32 offset = XHCI_SUPPORTED_PROTOCOLS_1_OFFSET(temp); 628 uint32 count = XHCI_SUPPORTED_PROTOCOLS_1_COUNT(temp); 629 if (offset == 0 || count == 0) 630 continue; 631 offset--; 632 for (uint32 i = offset; i < offset + count; i++) { 633 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) == 0x3) 634 fPortSpeeds[i] = USB_SPEED_SUPERSPEED; 635 else 636 fPortSpeeds[i] = USB_SPEED_HIGHSPEED; 637 638 TRACE("speed for port %" B_PRId32 " is %s\n", i, 639 fPortSpeeds[i] == USB_SPEED_SUPERSPEED ? "super" : "high"); 640 } 641 portFound += count; 642 } 643 644 uint32 params2 = ReadCapReg32(XHCI_HCSPARAMS2); 645 fScratchpadCount = HCS_MAX_SC_BUFFERS(params2); 646 if (fScratchpadCount > XHCI_MAX_SCRATCHPADS) { 647 TRACE_ERROR("invalid number of scratchpads: %" B_PRIu32 "\n", 648 fScratchpadCount); 649 return B_ERROR; 650 } 651 652 uint32 params3 = ReadCapReg32(XHCI_HCSPARAMS3); 653 fExitLatMax = HCS_U1_DEVICE_LATENCY(params3) 654 + HCS_U2_DEVICE_LATENCY(params3); 655 656 // clear interrupts & disable device notifications 657 WriteOpReg(XHCI_STS, ReadOpReg(XHCI_STS)); 658 WriteOpReg(XHCI_DNCTRL, 0); 659 660 // allocate Device Context Base Address array 661 phys_addr_t dmaAddress; 662 fDcbaArea = fStack->AllocateArea((void **)&fDcba, &dmaAddress, 663 sizeof(*fDcba), "DCBA Area"); 664 if (fDcbaArea < B_OK) { 665 TRACE_ERROR("unable to create the DCBA area\n"); 666 return B_ERROR; 667 } 668 memset(fDcba, 0, sizeof(*fDcba)); 669 memset(fScratchpadArea, 0, sizeof(fScratchpadArea)); 670 memset(fScratchpad, 0, sizeof(fScratchpad)); 671 672 // setting the first address to the scratchpad array address 673 fDcba->baseAddress[0] = dmaAddress 674 + offsetof(struct xhci_device_context_array, scratchpad); 675 676 // fill up the scratchpad array with scratchpad pages 677 for (uint32 i = 0; i < fScratchpadCount; i++) { 678 phys_addr_t scratchDmaAddress; 679 fScratchpadArea[i] = fStack->AllocateArea((void **)&fScratchpad[i], 680 &scratchDmaAddress, B_PAGE_SIZE, "Scratchpad Area"); 681 if (fScratchpadArea[i] < B_OK) { 682 TRACE_ERROR("unable to create the scratchpad area\n"); 683 return B_ERROR; 684 } 685 fDcba->scratchpad[i] = scratchDmaAddress; 686 } 687 688 TRACE("setting DCBAAP %" B_PRIxPHYSADDR "\n", dmaAddress); 689 WriteOpReg(XHCI_DCBAAP_LO, (uint32)dmaAddress); 690 WriteOpReg(XHCI_DCBAAP_HI, (uint32)(dmaAddress >> 32)); 691 692 // allocate Event Ring Segment Table 693 uint8 *addr; 694 fErstArea = fStack->AllocateArea((void **)&addr, &dmaAddress, 695 (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 696 + sizeof(xhci_erst_element), 697 "USB XHCI ERST CMD_RING and EVENT_RING Area"); 698 699 if (fErstArea < B_OK) { 700 TRACE_ERROR("unable to create the ERST AND RING area\n"); 701 delete_area(fDcbaArea); 702 return B_ERROR; 703 } 704 fErst = (xhci_erst_element *)addr; 705 memset(fErst, 0, (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 706 + sizeof(xhci_erst_element)); 707 708 // fill with Event Ring Segment Base Address and Event Ring Segment Size 709 fErst->rs_addr = dmaAddress + sizeof(xhci_erst_element); 710 fErst->rs_size = XHCI_MAX_EVENTS; 711 fErst->rsvdz = 0; 712 713 addr += sizeof(xhci_erst_element); 714 fEventRing = (xhci_trb *)addr; 715 addr += XHCI_MAX_EVENTS * sizeof(xhci_trb); 716 fCmdRing = (xhci_trb *)addr; 717 718 TRACE("setting ERST size\n"); 719 WriteRunReg32(XHCI_ERSTSZ(0), XHCI_ERSTS_SET(1)); 720 721 TRACE("setting ERDP addr = 0x%" B_PRIx64 "\n", fErst->rs_addr); 722 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)fErst->rs_addr); 723 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(fErst->rs_addr >> 32)); 724 725 TRACE("setting ERST base addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 726 WriteRunReg32(XHCI_ERSTBA_LO(0), (uint32)dmaAddress); 727 WriteRunReg32(XHCI_ERSTBA_HI(0), (uint32)(dmaAddress >> 32)); 728 729 dmaAddress += sizeof(xhci_erst_element) + XHCI_MAX_EVENTS 730 * sizeof(xhci_trb); 731 732 // Make sure the Command Ring is stopped 733 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 734 TRACE_ALWAYS("Command Ring is running, send stop/cancel\n"); 735 WriteOpReg(XHCI_CRCR_LO, CRCR_CS); 736 WriteOpReg(XHCI_CRCR_HI, 0); 737 WriteOpReg(XHCI_CRCR_LO, CRCR_CA); 738 WriteOpReg(XHCI_CRCR_HI, 0); 739 snooze(1000); 740 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 741 TRACE_ERROR("Command Ring still running after stop/cancel\n"); 742 } 743 } 744 TRACE("setting CRCR addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 745 WriteOpReg(XHCI_CRCR_LO, (uint32)dmaAddress | CRCR_RCS); 746 WriteOpReg(XHCI_CRCR_HI, (uint32)(dmaAddress >> 32)); 747 // link trb 748 fCmdRing[XHCI_MAX_COMMANDS - 1].address = dmaAddress; 749 750 TRACE("setting interrupt rate\n"); 751 752 // Setting IMOD below 0x3F8 on Intel Lynx Point can cause IRQ lockups 753 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL 754 && (fPCIInfo->device_id == PCI_DEVICE_INTEL_PANTHER_POINT_XHCI 755 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_XHCI 756 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI 757 || fPCIInfo->device_id == PCI_DEVICE_INTEL_BAYTRAIL_XHCI 758 || fPCIInfo->device_id == PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI)) { 759 WriteRunReg32(XHCI_IMOD(0), 0x000003f8); // 4000 irq/s 760 } else { 761 WriteRunReg32(XHCI_IMOD(0), 0x000001f4); // 8000 irq/s 762 } 763 764 TRACE("enabling interrupt\n"); 765 WriteRunReg32(XHCI_IMAN(0), ReadRunReg32(XHCI_IMAN(0)) | IMAN_INTR_ENA); 766 767 WriteOpReg(XHCI_CMD, CMD_RUN | CMD_INTE | CMD_HSEE); 768 769 // wait for start up state 770 if (WaitOpBits(XHCI_STS, STS_HCH, 0) != B_OK) { 771 TRACE_ERROR("HCH start up timeout\n"); 772 } 773 774 fRootHub = new(std::nothrow) XHCIRootHub(RootObject(), 1); 775 if (!fRootHub) { 776 TRACE_ERROR("no memory to allocate root hub\n"); 777 return B_NO_MEMORY; 778 } 779 780 if (fRootHub->InitCheck() < B_OK) { 781 TRACE_ERROR("root hub failed init check\n"); 782 return fRootHub->InitCheck(); 783 } 784 785 SetRootHub(fRootHub); 786 787 fRootHub->RegisterNode(Node()); 788 789 TRACE_ALWAYS("successfully started the controller\n"); 790 791 #ifdef TRACE_USB 792 TRACE("No-Op test...\n"); 793 Noop(); 794 #endif 795 796 return BusManager::Start(); 797 } 798 799 800 status_t 801 XHCI::SubmitTransfer(Transfer *transfer) 802 { 803 // short circuit the root hub 804 if (transfer->TransferPipe()->DeviceAddress() == 1) 805 return fRootHub->ProcessTransfer(this, transfer); 806 807 TRACE("SubmitTransfer(%p)\n", transfer); 808 Pipe *pipe = transfer->TransferPipe(); 809 if ((pipe->Type() & USB_OBJECT_CONTROL_PIPE) != 0) 810 return SubmitControlRequest(transfer); 811 return SubmitNormalRequest(transfer); 812 } 813 814 815 status_t 816 XHCI::SubmitControlRequest(Transfer *transfer) 817 { 818 Pipe *pipe = transfer->TransferPipe(); 819 usb_request_data *requestData = transfer->RequestData(); 820 bool directionIn = (requestData->RequestType & USB_REQTYPE_DEVICE_IN) != 0; 821 822 TRACE("SubmitControlRequest() length %d\n", requestData->Length); 823 824 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 825 if (endpoint == NULL) { 826 TRACE_ERROR("control pipe has no endpoint!\n"); 827 return B_BAD_VALUE; 828 } 829 if (endpoint->device == NULL) { 830 panic("endpoint is not initialized!"); 831 return B_NO_INIT; 832 } 833 834 status_t status = transfer->InitKernelAccess(); 835 if (status != B_OK) 836 return status; 837 838 xhci_td *descriptor = CreateDescriptor(3, 1, requestData->Length); 839 if (descriptor == NULL) 840 return B_NO_MEMORY; 841 descriptor->transfer = transfer; 842 843 // Setup Stage 844 uint8 index = 0; 845 memcpy(&descriptor->trbs[index].address, requestData, 846 sizeof(usb_request_data)); 847 descriptor->trbs[index].status = TRB_2_IRQ(0) | TRB_2_BYTES(8); 848 descriptor->trbs[index].flags 849 = TRB_3_TYPE(TRB_TYPE_SETUP_STAGE) | TRB_3_IDT_BIT | TRB_3_CYCLE_BIT; 850 if (requestData->Length > 0) { 851 descriptor->trbs[index].flags |= 852 directionIn ? TRB_3_TRT_IN : TRB_3_TRT_OUT; 853 } 854 855 index++; 856 857 // Data Stage (if any) 858 if (requestData->Length > 0) { 859 descriptor->trbs[index].address = descriptor->buffer_addrs[0]; 860 descriptor->trbs[index].status = TRB_2_IRQ(0) 861 | TRB_2_BYTES(requestData->Length) 862 | TRB_2_TD_SIZE(0); 863 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_DATA_STAGE) 864 | (directionIn ? TRB_3_DIR_IN : 0) 865 | TRB_3_CYCLE_BIT; 866 867 if (!directionIn) { 868 transfer->PrepareKernelAccess(); 869 memcpy(descriptor->buffers[0], 870 (uint8 *)transfer->Vector()[0].iov_base, requestData->Length); 871 } 872 873 index++; 874 } 875 876 // Status Stage 877 descriptor->trbs[index].address = 0; 878 descriptor->trbs[index].status = TRB_2_IRQ(0); 879 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_STATUS_STAGE) 880 | TRB_3_CHAIN_BIT | TRB_3_ENT_BIT | TRB_3_CYCLE_BIT; 881 // The CHAIN bit must be set when using an Event Data TRB 882 // (XHCI 1.2 § 6.4.1.2.3 Table 6-31 p472). 883 884 // Status Stage is an OUT transfer when the device is sending data 885 // (XHCI 1.2 § 4.11.2.2 Table 4-7 p213), otherwise set the IN bit. 886 if (requestData->Length == 0 || !directionIn) 887 descriptor->trbs[index].flags |= TRB_3_DIR_IN; 888 889 descriptor->trb_used = index + 1; 890 891 status = _LinkDescriptorForPipe(descriptor, endpoint); 892 if (status != B_OK) { 893 FreeDescriptor(descriptor); 894 return status; 895 } 896 897 return B_OK; 898 } 899 900 901 status_t 902 XHCI::SubmitNormalRequest(Transfer *transfer) 903 { 904 TRACE("SubmitNormalRequest() length %" B_PRIuSIZE "\n", transfer->FragmentLength()); 905 906 Pipe *pipe = transfer->TransferPipe(); 907 usb_isochronous_data *isochronousData = transfer->IsochronousData(); 908 bool directionIn = (pipe->Direction() == Pipe::In); 909 910 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 911 if (endpoint == NULL) { 912 TRACE_ERROR("pipe has no endpoint!\n"); 913 return B_BAD_VALUE; 914 } 915 if (endpoint->device == NULL) { 916 panic("endpoint is not initialized!"); 917 return B_NO_INIT; 918 } 919 920 status_t status = transfer->InitKernelAccess(); 921 if (status != B_OK) 922 return status; 923 924 // TRBs within a TD must be "grouped" into TD Fragments, which mostly means 925 // that a max_burst_payload boundary cannot be crossed within a TRB, but 926 // only between TRBs. More than one TRB can be in a TD Fragment, but we keep 927 // things simple by setting trbSize to the MBP. (XHCI 1.2 § 4.11.7.1 p235.) 928 size_t trbSize = endpoint->max_burst_payload; 929 930 if (isochronousData != NULL) { 931 if (isochronousData->packet_count == 0) 932 return B_BAD_VALUE; 933 934 // Isochronous transfers use more specifically sized packets. 935 trbSize = transfer->DataLength() / isochronousData->packet_count; 936 if (trbSize == 0 || trbSize > pipe->MaxPacketSize() || trbSize 937 != (size_t)isochronousData->packet_descriptors[0].request_length) 938 return B_BAD_VALUE; 939 } 940 941 // Now that we know trbSize, compute the count. 942 const int32 trbCount = (transfer->FragmentLength() + trbSize - 1) / trbSize; 943 944 xhci_td *td = CreateDescriptor(trbCount, trbCount, trbSize); 945 if (td == NULL) 946 return B_NO_MEMORY; 947 948 // Normal Stage 949 const size_t maxPacketSize = pipe->MaxPacketSize(); 950 size_t remaining = transfer->FragmentLength(); 951 for (int32 i = 0; i < trbCount; i++) { 952 int32 trbLength = (remaining < trbSize) ? remaining : trbSize; 953 remaining -= trbLength; 954 955 // The "TD Size" field of a transfer TRB indicates the number of 956 // remaining maximum-size *packets* in this TD, *not* including the 957 // packets in the current TRB, and capped at 31 if there are more 958 // than 31 packets remaining in the TD. (XHCI 1.2 § 4.11.2.4 p218.) 959 int32 tdSize = (remaining + maxPacketSize - 1) / maxPacketSize; 960 if (tdSize > 31) 961 tdSize = 31; 962 963 td->trbs[i].address = td->buffer_addrs[i]; 964 td->trbs[i].status = TRB_2_IRQ(0) 965 | TRB_2_BYTES(trbLength) 966 | TRB_2_TD_SIZE(tdSize); 967 td->trbs[i].flags = TRB_3_TYPE(TRB_TYPE_NORMAL) 968 | TRB_3_CYCLE_BIT | TRB_3_CHAIN_BIT; 969 970 td->trb_used++; 971 } 972 973 // Isochronous-specific 974 if (isochronousData != NULL) { 975 // This is an isochronous transfer; we need to make the first TRB 976 // an isochronous TRB. 977 td->trbs[0].flags &= ~(TRB_3_TYPE(TRB_TYPE_NORMAL)); 978 td->trbs[0].flags |= TRB_3_TYPE(TRB_TYPE_ISOCH); 979 980 // Isochronous pipes are scheduled by microframes, one of which 981 // is 125us for USB 2 and above. But for USB 1 it was 1ms, so 982 // we need to use a different frame delta for that case. 983 uint8 frameDelta = 1; 984 if (transfer->TransferPipe()->Speed() == USB_SPEED_FULLSPEED) 985 frameDelta = 8; 986 987 // TODO: We do not currently take Mult into account at all! 988 // How are we supposed to do that here? 989 990 // Determine the (starting) frame number: if ISO_ASAP is set, 991 // we are queueing this "right away", and so want to reset 992 // the starting_frame_number. Otherwise we use the passed one. 993 uint32 frame; 994 if ((isochronousData->flags & USB_ISO_ASAP) != 0 995 || isochronousData->starting_frame_number == NULL) { 996 // All reads from the microframe index register must be 997 // incremented by 1. (XHCI 1.2 § 4.14.2.1.4 p265.) 998 frame = ReadRunReg32(XHCI_MFINDEX) + 1; 999 td->trbs[0].flags |= TRB_3_ISO_SIA_BIT; 1000 } else { 1001 frame = *isochronousData->starting_frame_number; 1002 td->trbs[0].flags |= TRB_3_FRID(frame); 1003 } 1004 frame = (frame + frameDelta) % 2048; 1005 if (isochronousData->starting_frame_number != NULL) 1006 *isochronousData->starting_frame_number = frame; 1007 1008 // TODO: The OHCI bus driver seems to also do this for inbound 1009 // isochronous transfers. Perhaps it should be moved into the stack? 1010 if (directionIn) { 1011 for (uint32 i = 0; i < isochronousData->packet_count; i++) { 1012 isochronousData->packet_descriptors[i].actual_length = 0; 1013 isochronousData->packet_descriptors[i].status = B_NO_INIT; 1014 } 1015 } 1016 } 1017 1018 // Set the ENT (Evaluate Next TRB) bit, so that the HC will not switch 1019 // contexts before evaluating the Link TRB that _LinkDescriptorForPipe 1020 // will insert, as otherwise there would be a race between us freeing 1021 // and unlinking the descriptor, and the controller evaluating the Link TRB 1022 // and thus getting back onto the main ring and executing the Event Data 1023 // TRB that generates the interrupt for this transfer. 1024 // 1025 // Note that we *do not* unset the CHAIN bit in this TRB, thus including 1026 // the Link TRB in this TD formally, which is required when using the 1027 // ENT bit. (XHCI 1.2 § 4.12.3 p250.) 1028 td->trbs[td->trb_used - 1].flags |= TRB_3_ENT_BIT; 1029 1030 if (!directionIn) { 1031 TRACE("copying out iov count %ld\n", transfer->VectorCount()); 1032 status_t status = transfer->PrepareKernelAccess(); 1033 if (status != B_OK) { 1034 FreeDescriptor(td); 1035 return status; 1036 } 1037 WriteDescriptor(td, transfer->Vector(), transfer->VectorCount()); 1038 } 1039 1040 td->transfer = transfer; 1041 status = _LinkDescriptorForPipe(td, endpoint); 1042 if (status != B_OK) { 1043 FreeDescriptor(td); 1044 return status; 1045 } 1046 1047 return B_OK; 1048 } 1049 1050 1051 status_t 1052 XHCI::CancelQueuedTransfers(Pipe *pipe, bool force) 1053 { 1054 xhci_endpoint* endpoint = (xhci_endpoint*)pipe->ControllerCookie(); 1055 if (endpoint == NULL || endpoint->trbs == NULL) { 1056 // Someone's de-allocated this pipe or endpoint in the meantime. 1057 // (Possibly AllocateDevice failed, and we were the temporary pipe.) 1058 return B_NO_INIT; 1059 } 1060 1061 #ifndef TRACE_USB 1062 if (force) 1063 #endif 1064 { 1065 TRACE_ALWAYS("cancel queued transfers (%" B_PRId8 ") for pipe %p (%d)\n", 1066 endpoint->used, pipe, pipe->EndpointAddress()); 1067 } 1068 1069 MutexLocker endpointLocker(endpoint->lock); 1070 1071 if (endpoint->td_head == NULL) { 1072 // There aren't any currently pending transfers to cancel. 1073 return B_OK; 1074 } 1075 1076 // Calling the callbacks while holding the endpoint lock could potentially 1077 // cause deadlocks, so we instead store them in a pointer array. We need 1078 // to do this separately from freeing the TDs, for in the case we fail 1079 // to stop the endpoint, we cancel the transfers but do not free the TDs. 1080 Transfer* transfers[XHCI_MAX_TRANSFERS]; 1081 int32 transfersCount = 0; 1082 1083 for (xhci_td* td = endpoint->td_head; td != NULL; td = td->next) { 1084 if (td->transfer == NULL) 1085 continue; 1086 1087 // We can't cancel or delete transfers under "force", as they probably 1088 // are not safe to use anymore. 1089 if (!force) { 1090 transfers[transfersCount] = td->transfer; 1091 transfersCount++; 1092 } 1093 td->transfer = NULL; 1094 } 1095 1096 // It is possible that while waiting for the stop-endpoint command to 1097 // complete, one of the queued transfers posts a completion event, so in 1098 // order to avoid a deadlock, we must unlock the endpoint. 1099 endpointLocker.Unlock(); 1100 status_t status = StopEndpoint(false, endpoint); 1101 if (status == B_DEV_STALLED) { 1102 // Only exit from a Halted state is a reset. (XHCI 1.2 § 4.8.3 p163.) 1103 TRACE_ERROR("cancel queued transfers: halted endpoint, reset!\n"); 1104 status = ResetEndpoint(false, endpoint); 1105 } 1106 endpointLocker.Lock(); 1107 1108 // Detach the head TD from the endpoint. 1109 xhci_td* td_head = endpoint->td_head; 1110 endpoint->td_head = NULL; 1111 1112 if (status == B_OK) { 1113 // Clear the endpoint's TRBs. 1114 memset(endpoint->trbs, 0, sizeof(xhci_trb) * XHCI_ENDPOINT_RING_SIZE); 1115 endpoint->used = 0; 1116 endpoint->current = 0; 1117 1118 // Set dequeue pointer location to the beginning of the ring. 1119 SetTRDequeue(endpoint->trb_addr, 0, endpoint->id + 1, 1120 endpoint->device->slot); 1121 1122 // We don't need to do anything else to restart the ring, as it will resume 1123 // operation as normal upon the next doorbell. (XHCI 1.2 § 4.6.9 p136.) 1124 } else { 1125 // We couldn't stop the endpoint. Most likely the device has been 1126 // removed and the endpoint was stopped by the hardware, or is 1127 // for some reason busy and cannot be stopped. 1128 TRACE_ERROR("cancel queued transfers: could not stop endpoint: %s!\n", 1129 strerror(status)); 1130 1131 // Instead of freeing the TDs, we want to leave them in the endpoint 1132 // so that when/if the hardware returns, they can be properly unlinked, 1133 // as otherwise the endpoint could get "stuck" by having the "used" 1134 // slowly accumulate due to "dead" transfers. 1135 endpoint->td_head = td_head; 1136 td_head = NULL; 1137 } 1138 1139 endpointLocker.Unlock(); 1140 1141 for (int32 i = 0; i < transfersCount; i++) { 1142 transfers[i]->Finished(B_CANCELED, 0); 1143 delete transfers[i]; 1144 } 1145 1146 // This loop looks a bit strange because we need to store the "next" 1147 // pointer before freeing the descriptor. 1148 xhci_td* td; 1149 while ((td = td_head) != NULL) { 1150 td_head = td_head->next; 1151 FreeDescriptor(td); 1152 } 1153 1154 return B_OK; 1155 } 1156 1157 1158 status_t 1159 XHCI::StartDebugTransfer(Transfer *transfer) 1160 { 1161 Pipe *pipe = transfer->TransferPipe(); 1162 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 1163 if (endpoint == NULL) 1164 return B_BAD_VALUE; 1165 1166 // Check all locks that we are going to hit when running transfers. 1167 if (mutex_trylock(&endpoint->lock) != B_OK) 1168 return B_WOULD_BLOCK; 1169 if (mutex_trylock(&fFinishedLock) != B_OK) { 1170 mutex_unlock(&endpoint->lock); 1171 return B_WOULD_BLOCK; 1172 } 1173 if (mutex_trylock(&fEventLock) != B_OK) { 1174 mutex_unlock(&endpoint->lock); 1175 mutex_unlock(&fFinishedLock); 1176 return B_WOULD_BLOCK; 1177 } 1178 mutex_unlock(&endpoint->lock); 1179 mutex_unlock(&fFinishedLock); 1180 mutex_unlock(&fEventLock); 1181 1182 status_t status = SubmitTransfer(transfer); 1183 if (status != B_OK) 1184 return status; 1185 1186 // The endpoint's head TD is the TD of the just-submitted transfer. 1187 // Just like EHCI, abuse the callback cookie to hold the TD pointer. 1188 transfer->SetCallback(NULL, endpoint->td_head); 1189 1190 return B_OK; 1191 } 1192 1193 1194 status_t 1195 XHCI::CheckDebugTransfer(Transfer *transfer) 1196 { 1197 xhci_td *transfer_td = (xhci_td *)transfer->CallbackCookie(); 1198 if (transfer_td == NULL) 1199 return B_NO_INIT; 1200 1201 // Process events once, and then look for it in the finished list. 1202 ProcessEvents(); 1203 xhci_td *previous = NULL; 1204 for (xhci_td *td = fFinishedHead; td != NULL; td = td->next) { 1205 if (td != transfer_td) { 1206 previous = td; 1207 continue; 1208 } 1209 1210 // We've found it! 1211 if (previous == NULL) { 1212 fFinishedHead = fFinishedHead->next; 1213 } else { 1214 previous->next = td->next; 1215 } 1216 1217 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 1218 status_t status = (td->trb_completion_code == COMP_SUCCESS 1219 || td->trb_completion_code == COMP_SHORT_PACKET) ? B_OK : B_ERROR; 1220 1221 if (status == B_OK && directionIn) 1222 ReadDescriptor(td, transfer->Vector(), transfer->VectorCount()); 1223 1224 FreeDescriptor(td); 1225 transfer->SetCallback(NULL, NULL); 1226 return status; 1227 } 1228 1229 // We didn't find it. 1230 spin(75); 1231 return B_DEV_PENDING; 1232 } 1233 1234 1235 void 1236 XHCI::CancelDebugTransfer(Transfer *transfer) 1237 { 1238 while (CheckDebugTransfer(transfer) == B_DEV_PENDING) 1239 spin(100); 1240 } 1241 1242 1243 status_t 1244 XHCI::NotifyPipeChange(Pipe *pipe, usb_change change) 1245 { 1246 TRACE("pipe change %d for pipe %p (%d)\n", change, pipe, 1247 pipe->EndpointAddress()); 1248 1249 switch (change) { 1250 case USB_CHANGE_CREATED: 1251 return _InsertEndpointForPipe(pipe); 1252 case USB_CHANGE_DESTROYED: 1253 return _RemoveEndpointForPipe(pipe); 1254 1255 case USB_CHANGE_PIPE_POLICY_CHANGED: 1256 // We don't care about these, at least for now. 1257 return B_OK; 1258 } 1259 1260 TRACE_ERROR("unknown pipe change!\n"); 1261 return B_UNSUPPORTED; 1262 } 1263 1264 1265 xhci_td * 1266 XHCI::CreateDescriptor(uint32 trbCount, uint32 bufferCount, size_t bufferSize) 1267 { 1268 const bool inKDL = debug_debugger_running(); 1269 1270 xhci_td *result; 1271 if (!inKDL) { 1272 result = (xhci_td*)calloc(1, sizeof(xhci_td)); 1273 } else { 1274 // Just use the physical memory allocator while in KDL; it's less 1275 // secure than using the regular heap, but it's easier to deal with. 1276 phys_addr_t dummy; 1277 fStack->AllocateChunk((void **)&result, &dummy, sizeof(xhci_td)); 1278 } 1279 1280 if (result == NULL) { 1281 TRACE_ERROR("failed to allocate a transfer descriptor\n"); 1282 return NULL; 1283 } 1284 1285 // We always allocate 1 more TRB than requested, so that 1286 // _LinkDescriptorForPipe() has room to insert a link TRB. 1287 trbCount++; 1288 if (fStack->AllocateChunk((void **)&result->trbs, &result->trb_addr, 1289 (trbCount * sizeof(xhci_trb))) < B_OK) { 1290 TRACE_ERROR("failed to allocate TRBs\n"); 1291 FreeDescriptor(result); 1292 return NULL; 1293 } 1294 result->trb_count = trbCount; 1295 result->trb_used = 0; 1296 1297 if (bufferSize > 0) { 1298 // Due to how the USB stack allocates physical memory, we can't just 1299 // request one large chunk the size of the transfer, and so instead we 1300 // create a series of buffers as requested by our caller. 1301 1302 // We store the buffer pointers and addresses in one memory block. 1303 if (!inKDL) { 1304 result->buffers = (void**)calloc(bufferCount, 1305 (sizeof(void*) + sizeof(phys_addr_t))); 1306 } else { 1307 phys_addr_t dummy; 1308 fStack->AllocateChunk((void **)&result->buffers, &dummy, 1309 bufferCount * (sizeof(void*) + sizeof(phys_addr_t))); 1310 } 1311 if (result->buffers == NULL) { 1312 TRACE_ERROR("unable to allocate space for buffer infos\n"); 1313 FreeDescriptor(result); 1314 return NULL; 1315 } 1316 result->buffer_addrs = (phys_addr_t*)&result->buffers[bufferCount]; 1317 result->buffer_size = bufferSize; 1318 result->buffer_count = bufferCount; 1319 1320 // Optimization: If the requested total size of all buffers is less 1321 // than 32*B_PAGE_SIZE (the maximum size that the physical memory 1322 // allocator can handle), we allocate only one buffer and segment it. 1323 size_t totalSize = bufferSize * bufferCount; 1324 if (totalSize < (32 * B_PAGE_SIZE)) { 1325 if (fStack->AllocateChunk(&result->buffers[0], 1326 &result->buffer_addrs[0], totalSize) < B_OK) { 1327 TRACE_ERROR("unable to allocate space for large buffer (size %ld)\n", 1328 totalSize); 1329 FreeDescriptor(result); 1330 return NULL; 1331 } 1332 for (uint32 i = 1; i < bufferCount; i++) { 1333 result->buffers[i] = (void*)((addr_t)(result->buffers[i - 1]) 1334 + bufferSize); 1335 result->buffer_addrs[i] = result->buffer_addrs[i - 1] 1336 + bufferSize; 1337 } 1338 } else { 1339 // Otherwise, we allocate each buffer individually. 1340 for (uint32 i = 0; i < bufferCount; i++) { 1341 if (fStack->AllocateChunk(&result->buffers[i], 1342 &result->buffer_addrs[i], bufferSize) < B_OK) { 1343 TRACE_ERROR("unable to allocate space for a buffer (size " 1344 "%" B_PRIuSIZE ", count %" B_PRIu32 ")\n", 1345 bufferSize, bufferCount); 1346 FreeDescriptor(result); 1347 return NULL; 1348 } 1349 } 1350 } 1351 } else { 1352 result->buffers = NULL; 1353 result->buffer_addrs = NULL; 1354 } 1355 1356 // Initialize all other fields. 1357 result->transfer = NULL; 1358 result->trb_completion_code = 0; 1359 result->trb_left = 0; 1360 result->next = NULL; 1361 1362 TRACE("CreateDescriptor allocated %p, buffer_size %ld, buffer_count %" B_PRIu32 "\n", 1363 result, result->buffer_size, result->buffer_count); 1364 1365 return result; 1366 } 1367 1368 1369 void 1370 XHCI::FreeDescriptor(xhci_td *descriptor) 1371 { 1372 if (descriptor == NULL) 1373 return; 1374 1375 const bool inKDL = debug_debugger_running(); 1376 1377 if (descriptor->trbs != NULL) { 1378 fStack->FreeChunk(descriptor->trbs, descriptor->trb_addr, 1379 (descriptor->trb_count * sizeof(xhci_trb))); 1380 } 1381 if (descriptor->buffers != NULL) { 1382 size_t totalSize = descriptor->buffer_size * descriptor->buffer_count; 1383 if (totalSize < (32 * B_PAGE_SIZE)) { 1384 // This was allocated as one contiguous buffer. 1385 fStack->FreeChunk(descriptor->buffers[0], descriptor->buffer_addrs[0], 1386 totalSize); 1387 } else { 1388 for (uint32 i = 0; i < descriptor->buffer_count; i++) { 1389 if (descriptor->buffers[i] == NULL) 1390 continue; 1391 fStack->FreeChunk(descriptor->buffers[i], descriptor->buffer_addrs[i], 1392 descriptor->buffer_size); 1393 } 1394 } 1395 1396 if (!inKDL) { 1397 free(descriptor->buffers); 1398 } else { 1399 fStack->FreeChunk(descriptor->buffers, 0, 1400 descriptor->buffer_count * (sizeof(void*) + sizeof(phys_addr_t))); 1401 } 1402 } 1403 1404 if (!inKDL) 1405 free(descriptor); 1406 else 1407 fStack->FreeChunk(descriptor, 0, sizeof(xhci_td)); 1408 } 1409 1410 1411 size_t 1412 XHCI::WriteDescriptor(xhci_td *descriptor, iovec *vector, size_t vectorCount) 1413 { 1414 size_t written = 0; 1415 1416 size_t bufIdx = 0, bufUsed = 0; 1417 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1418 size_t length = vector[vecIdx].iov_len; 1419 1420 while (length > 0 && bufIdx < descriptor->buffer_count) { 1421 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1422 memcpy((uint8 *)descriptor->buffers[bufIdx] + bufUsed, 1423 (uint8 *)vector[vecIdx].iov_base + (vector[vecIdx].iov_len - length), 1424 toCopy); 1425 1426 written += toCopy; 1427 bufUsed += toCopy; 1428 length -= toCopy; 1429 if (bufUsed == descriptor->buffer_size) { 1430 bufIdx++; 1431 bufUsed = 0; 1432 } 1433 } 1434 } 1435 1436 TRACE("wrote descriptor (%" B_PRIuSIZE " bytes)\n", written); 1437 return written; 1438 } 1439 1440 1441 size_t 1442 XHCI::ReadDescriptor(xhci_td *descriptor, iovec *vector, size_t vectorCount) 1443 { 1444 size_t read = 0; 1445 1446 size_t bufIdx = 0, bufUsed = 0; 1447 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1448 size_t length = vector[vecIdx].iov_len; 1449 1450 while (length > 0 && bufIdx < descriptor->buffer_count) { 1451 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1452 memcpy((uint8 *)vector[vecIdx].iov_base + (vector[vecIdx].iov_len - length), 1453 (uint8 *)descriptor->buffers[bufIdx] + bufUsed, toCopy); 1454 1455 read += toCopy; 1456 bufUsed += toCopy; 1457 length -= toCopy; 1458 if (bufUsed == descriptor->buffer_size) { 1459 bufIdx++; 1460 bufUsed = 0; 1461 } 1462 } 1463 } 1464 1465 TRACE("read descriptor (%" B_PRIuSIZE " bytes)\n", read); 1466 return read; 1467 } 1468 1469 1470 Device * 1471 XHCI::AllocateDevice(Hub *parent, int8 hubAddress, uint8 hubPort, 1472 usb_speed speed) 1473 { 1474 TRACE("AllocateDevice hubAddress %d hubPort %d speed %d\n", hubAddress, 1475 hubPort, speed); 1476 1477 uint8 slot = XHCI_MAX_SLOTS; 1478 status_t status = EnableSlot(&slot); 1479 if (status != B_OK) { 1480 TRACE_ERROR("failed to enable slot: %s\n", strerror(status)); 1481 return NULL; 1482 } 1483 1484 if (slot == 0 || slot > fSlotCount) { 1485 TRACE_ERROR("AllocateDevice: bad slot\n"); 1486 return NULL; 1487 } 1488 1489 if (fDevices[slot].slot != 0) { 1490 TRACE_ERROR("AllocateDevice: slot already used\n"); 1491 return NULL; 1492 } 1493 1494 struct xhci_device *device = &fDevices[slot]; 1495 device->slot = slot; 1496 1497 device->input_ctx_area = fStack->AllocateArea((void **)&device->input_ctx, 1498 &device->input_ctx_addr, sizeof(*device->input_ctx) << fContextSizeShift, 1499 "XHCI input context"); 1500 if (device->input_ctx_area < B_OK) { 1501 TRACE_ERROR("unable to create a input context area\n"); 1502 CleanupDevice(device); 1503 return NULL; 1504 } 1505 if (fContextSizeShift == 1) { 1506 // 64-byte contexts have to be page-aligned in order for 1507 // _OffsetContextAddr to function properly. 1508 ASSERT((((addr_t)device->input_ctx) % B_PAGE_SIZE) == 0); 1509 } 1510 1511 memset(device->input_ctx, 0, sizeof(*device->input_ctx) << fContextSizeShift); 1512 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1513 _WriteContext(&device->input_ctx->input.addFlags, 3); 1514 1515 uint8 rhPort = hubPort; 1516 uint32 route = 0; 1517 for (Device *hubDevice = parent; hubDevice != RootObject(); 1518 hubDevice = (Device *)hubDevice->Parent()) { 1519 if (hubDevice->Parent() == RootObject()) 1520 break; 1521 1522 if (rhPort > 15) 1523 rhPort = 15; 1524 route = route << 4; 1525 route |= rhPort; 1526 1527 rhPort = hubDevice->HubPort(); 1528 } 1529 1530 uint32 dwslot0 = SLOT_0_NUM_ENTRIES(1) | SLOT_0_ROUTE(route); 1531 1532 // Get speed of port, only if device connected to root hub port 1533 // else we have to rely on value reported by the Hub Explore thread 1534 if (route == 0) { 1535 GetPortSpeed(hubPort - 1, &speed); 1536 TRACE("speed updated %d\n", speed); 1537 } 1538 1539 // add the speed 1540 switch (speed) { 1541 case USB_SPEED_LOWSPEED: 1542 dwslot0 |= SLOT_0_SPEED(2); 1543 break; 1544 case USB_SPEED_FULLSPEED: 1545 dwslot0 |= SLOT_0_SPEED(1); 1546 break; 1547 case USB_SPEED_HIGHSPEED: 1548 dwslot0 |= SLOT_0_SPEED(3); 1549 break; 1550 case USB_SPEED_SUPERSPEED: 1551 dwslot0 |= SLOT_0_SPEED(4); 1552 break; 1553 default: 1554 TRACE_ERROR("unknown usb speed\n"); 1555 break; 1556 } 1557 1558 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1559 // TODO enable power save 1560 _WriteContext(&device->input_ctx->slot.dwslot1, SLOT_1_RH_PORT(rhPort)); 1561 uint32 dwslot2 = SLOT_2_IRQ_TARGET(0); 1562 1563 // If LS/FS device connected to non-root HS device 1564 if (route != 0 && parent->Speed() == USB_SPEED_HIGHSPEED 1565 && (speed == USB_SPEED_LOWSPEED || speed == USB_SPEED_FULLSPEED)) { 1566 struct xhci_device *parenthub = (struct xhci_device *) 1567 parent->ControllerCookie(); 1568 dwslot2 |= SLOT_2_PORT_NUM(hubPort); 1569 dwslot2 |= SLOT_2_TT_HUB_SLOT(parenthub->slot); 1570 } 1571 1572 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1573 1574 _WriteContext(&device->input_ctx->slot.dwslot3, SLOT_3_SLOT_STATE(0) 1575 | SLOT_3_DEVICE_ADDRESS(0)); 1576 1577 TRACE("slot 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 1578 "\n", _ReadContext(&device->input_ctx->slot.dwslot0), 1579 _ReadContext(&device->input_ctx->slot.dwslot1), 1580 _ReadContext(&device->input_ctx->slot.dwslot2), 1581 _ReadContext(&device->input_ctx->slot.dwslot3)); 1582 1583 device->device_ctx_area = fStack->AllocateArea((void **)&device->device_ctx, 1584 &device->device_ctx_addr, sizeof(*device->device_ctx) << fContextSizeShift, 1585 "XHCI device context"); 1586 if (device->device_ctx_area < B_OK) { 1587 TRACE_ERROR("unable to create a device context area\n"); 1588 CleanupDevice(device); 1589 return NULL; 1590 } 1591 memset(device->device_ctx, 0, sizeof(*device->device_ctx) << fContextSizeShift); 1592 1593 device->trb_area = fStack->AllocateArea((void **)&device->trbs, 1594 &device->trb_addr, sizeof(xhci_trb) * (XHCI_MAX_ENDPOINTS - 1) 1595 * XHCI_ENDPOINT_RING_SIZE, "XHCI endpoint trbs"); 1596 if (device->trb_area < B_OK) { 1597 TRACE_ERROR("unable to create a device trbs area\n"); 1598 CleanupDevice(device); 1599 return NULL; 1600 } 1601 1602 // set up slot pointer to device context 1603 fDcba->baseAddress[slot] = device->device_ctx_addr; 1604 1605 size_t maxPacketSize; 1606 switch (speed) { 1607 case USB_SPEED_LOWSPEED: 1608 case USB_SPEED_FULLSPEED: 1609 maxPacketSize = 8; 1610 break; 1611 case USB_SPEED_HIGHSPEED: 1612 maxPacketSize = 64; 1613 break; 1614 default: 1615 maxPacketSize = 512; 1616 break; 1617 } 1618 1619 xhci_endpoint* endpoint0 = &device->endpoints[0]; 1620 mutex_init(&endpoint0->lock, "xhci endpoint lock"); 1621 endpoint0->device = device; 1622 endpoint0->id = 0; 1623 endpoint0->td_head = NULL; 1624 endpoint0->used = 0; 1625 endpoint0->current = 0; 1626 endpoint0->trbs = device->trbs; 1627 endpoint0->trb_addr = device->trb_addr; 1628 1629 // configure the Control endpoint 0 1630 if (ConfigureEndpoint(endpoint0, slot, 0, USB_OBJECT_CONTROL_PIPE, false, 1631 0, maxPacketSize, speed, 0, 0) != B_OK) { 1632 TRACE_ERROR("unable to configure default control endpoint\n"); 1633 CleanupDevice(device); 1634 return NULL; 1635 } 1636 1637 // device should get to addressed state (bsr = 0) 1638 status = SetAddress(device->input_ctx_addr, false, slot); 1639 if (status != B_OK) { 1640 TRACE_ERROR("unable to set address: %s\n", strerror(status)); 1641 CleanupDevice(device); 1642 return NULL; 1643 } 1644 1645 device->address = SLOT_3_DEVICE_ADDRESS_GET(_ReadContext( 1646 &device->device_ctx->slot.dwslot3)); 1647 1648 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", device->address, 1649 SLOT_3_SLOT_STATE_GET(_ReadContext( 1650 &device->device_ctx->slot.dwslot3))); 1651 TRACE("endpoint0 state 0x%08" B_PRIx32 "\n", 1652 ENDPOINT_0_STATE_GET(_ReadContext( 1653 &device->device_ctx->endpoints[0].dwendpoint0))); 1654 1655 // Wait a bit for the device to complete addressing 1656 snooze(USB_DELAY_SET_ADDRESS); 1657 1658 // Create a temporary pipe with the new address 1659 ControlPipe pipe(parent); 1660 pipe.SetControllerCookie(endpoint0); 1661 pipe.InitCommon(device->address + 1, 0, speed, Pipe::Default, maxPacketSize, 0, 1662 hubAddress, hubPort); 1663 1664 // Get the device descriptor 1665 // Just retrieve the first 8 bytes of the descriptor -> minimum supported 1666 // size of any device. It is enough because it includes the device type. 1667 1668 size_t actualLength = 0; 1669 usb_device_descriptor deviceDescriptor; 1670 1671 TRACE("getting the device descriptor\n"); 1672 status = pipe.SendRequest( 1673 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_STANDARD, // type 1674 USB_REQUEST_GET_DESCRIPTOR, // request 1675 USB_DESCRIPTOR_DEVICE << 8, // value 1676 0, // index 1677 8, // length 1678 (void *)&deviceDescriptor, // buffer 1679 8, // buffer length 1680 &actualLength); // actual length 1681 1682 if (actualLength != 8) { 1683 TRACE_ERROR("failed to get the device descriptor: %s\n", 1684 strerror(status)); 1685 CleanupDevice(device); 1686 return NULL; 1687 } 1688 1689 TRACE("device_class: %d device_subclass %d device_protocol %d\n", 1690 deviceDescriptor.device_class, deviceDescriptor.device_subclass, 1691 deviceDescriptor.device_protocol); 1692 1693 if (speed == USB_SPEED_FULLSPEED && deviceDescriptor.max_packet_size_0 != 8) { 1694 TRACE("Full speed device with different max packet size for Endpoint 0\n"); 1695 uint32 dwendpoint1 = _ReadContext( 1696 &device->input_ctx->endpoints[0].dwendpoint1); 1697 dwendpoint1 &= ~ENDPOINT_1_MAXPACKETSIZE(0xffff); 1698 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE( 1699 deviceDescriptor.max_packet_size_0); 1700 _WriteContext(&device->input_ctx->endpoints[0].dwendpoint1, 1701 dwendpoint1); 1702 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1703 _WriteContext(&device->input_ctx->input.addFlags, (1 << 1)); 1704 EvaluateContext(device->input_ctx_addr, device->slot); 1705 } 1706 1707 Device *deviceObject = NULL; 1708 if (deviceDescriptor.device_class == 0x09) { 1709 TRACE("creating new Hub\n"); 1710 TRACE("getting the hub descriptor\n"); 1711 size_t actualLength = 0; 1712 usb_hub_descriptor hubDescriptor; 1713 status = pipe.SendRequest( 1714 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_CLASS, // type 1715 USB_REQUEST_GET_DESCRIPTOR, // request 1716 USB_DESCRIPTOR_HUB << 8, // value 1717 0, // index 1718 sizeof(usb_hub_descriptor), // length 1719 (void *)&hubDescriptor, // buffer 1720 sizeof(usb_hub_descriptor), // buffer length 1721 &actualLength); 1722 1723 if (actualLength != sizeof(usb_hub_descriptor)) { 1724 TRACE_ERROR("error while getting the hub descriptor: %s\n", 1725 strerror(status)); 1726 CleanupDevice(device); 1727 return NULL; 1728 } 1729 1730 uint32 dwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1731 dwslot0 |= SLOT_0_HUB_BIT; 1732 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1733 uint32 dwslot1 = _ReadContext(&device->input_ctx->slot.dwslot1); 1734 dwslot1 |= SLOT_1_NUM_PORTS(hubDescriptor.num_ports); 1735 _WriteContext(&device->input_ctx->slot.dwslot1, dwslot1); 1736 if (speed == USB_SPEED_HIGHSPEED) { 1737 uint32 dwslot2 = _ReadContext(&device->input_ctx->slot.dwslot2); 1738 dwslot2 |= SLOT_2_TT_TIME(HUB_TTT_GET(hubDescriptor.characteristics)); 1739 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1740 } 1741 1742 deviceObject = new(std::nothrow) Hub(parent, hubAddress, hubPort, 1743 deviceDescriptor, device->address + 1, speed, false, device); 1744 } else { 1745 TRACE("creating new device\n"); 1746 deviceObject = new(std::nothrow) Device(parent, hubAddress, hubPort, 1747 deviceDescriptor, device->address + 1, speed, false, device); 1748 } 1749 if (deviceObject == NULL || deviceObject->InitCheck() != B_OK) { 1750 if (deviceObject == NULL) { 1751 TRACE_ERROR("no memory to allocate device\n"); 1752 } else { 1753 TRACE_ERROR("device object failed to initialize\n"); 1754 } 1755 CleanupDevice(device); 1756 return NULL; 1757 } 1758 1759 // We don't want to disable the default endpoint, naturally, which would 1760 // otherwise happen when this Pipe object is destroyed. 1761 pipe.SetControllerCookie(NULL); 1762 1763 deviceObject->RegisterNode(); 1764 1765 TRACE("AllocateDevice() port %d slot %d\n", hubPort, slot); 1766 return deviceObject; 1767 } 1768 1769 1770 void 1771 XHCI::FreeDevice(Device *usbDevice) 1772 { 1773 xhci_device* device = (xhci_device*)usbDevice->ControllerCookie(); 1774 TRACE("FreeDevice() slot %d\n", device->slot); 1775 1776 // Delete the device first, so it cleans up its pipes and tells us 1777 // what we need to destroy before we tear down our internal state. 1778 delete usbDevice; 1779 1780 CleanupDevice(device); 1781 } 1782 1783 1784 void 1785 XHCI::CleanupDevice(xhci_device *device) 1786 { 1787 if (device->slot != 0) { 1788 DisableSlot(device->slot); 1789 fDcba->baseAddress[device->slot] = 0; 1790 } 1791 1792 if (device->trb_addr != 0) 1793 delete_area(device->trb_area); 1794 if (device->input_ctx_addr != 0) 1795 delete_area(device->input_ctx_area); 1796 if (device->device_ctx_addr != 0) 1797 delete_area(device->device_ctx_area); 1798 1799 memset(device, 0, sizeof(xhci_device)); 1800 } 1801 1802 1803 uint8 1804 XHCI::_GetEndpointState(xhci_endpoint* endpoint) 1805 { 1806 struct xhci_device_ctx* device_ctx = endpoint->device->device_ctx; 1807 return ENDPOINT_0_STATE_GET( 1808 _ReadContext(&device_ctx->endpoints[endpoint->id].dwendpoint0)); 1809 } 1810 1811 1812 status_t 1813 XHCI::_InsertEndpointForPipe(Pipe *pipe) 1814 { 1815 TRACE("insert endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1816 1817 if (pipe->ControllerCookie() != NULL 1818 || pipe->Parent()->Type() != USB_OBJECT_DEVICE) { 1819 // default pipe is already referenced 1820 return B_OK; 1821 } 1822 1823 Device* usbDevice = (Device *)pipe->Parent(); 1824 if (usbDevice->Parent() == RootObject()) { 1825 // root hub needs no initialization 1826 return B_OK; 1827 } 1828 1829 struct xhci_device *device = (struct xhci_device *) 1830 usbDevice->ControllerCookie(); 1831 if (device == NULL) { 1832 panic("device is NULL\n"); 1833 return B_NO_INIT; 1834 } 1835 1836 const uint8 id = (2 * pipe->EndpointAddress() 1837 + (pipe->Direction() != Pipe::Out ? 1 : 0)) - 1; 1838 if (id >= XHCI_MAX_ENDPOINTS - 1) 1839 return B_BAD_VALUE; 1840 1841 if (id > 0) { 1842 uint32 devicedwslot0 = _ReadContext(&device->device_ctx->slot.dwslot0); 1843 if (SLOT_0_NUM_ENTRIES_GET(devicedwslot0) == 1) { 1844 uint32 inputdwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1845 inputdwslot0 &= ~(SLOT_0_NUM_ENTRIES(0x1f)); 1846 inputdwslot0 |= SLOT_0_NUM_ENTRIES(XHCI_MAX_ENDPOINTS - 1); 1847 _WriteContext(&device->input_ctx->slot.dwslot0, inputdwslot0); 1848 EvaluateContext(device->input_ctx_addr, device->slot); 1849 } 1850 1851 xhci_endpoint* endpoint = &device->endpoints[id]; 1852 mutex_init(&endpoint->lock, "xhci endpoint lock"); 1853 MutexLocker endpointLocker(endpoint->lock); 1854 1855 endpoint->device = device; 1856 endpoint->id = id; 1857 endpoint->td_head = NULL; 1858 endpoint->used = 0; 1859 endpoint->current = 0; 1860 1861 endpoint->trbs = device->trbs + id * XHCI_ENDPOINT_RING_SIZE; 1862 endpoint->trb_addr = device->trb_addr 1863 + id * XHCI_ENDPOINT_RING_SIZE * sizeof(xhci_trb); 1864 memset(endpoint->trbs, 0, 1865 sizeof(xhci_trb) * XHCI_ENDPOINT_RING_SIZE); 1866 1867 TRACE("insert endpoint for pipe: trbs, device %p endpoint %p\n", 1868 device->trbs, endpoint->trbs); 1869 TRACE("insert endpoint for pipe: trb_addr, device 0x%" B_PRIxPHYSADDR 1870 " endpoint 0x%" B_PRIxPHYSADDR "\n", device->trb_addr, 1871 endpoint->trb_addr); 1872 1873 const uint8 endpointNum = id + 1; 1874 1875 status_t status = ConfigureEndpoint(endpoint, device->slot, id, pipe->Type(), 1876 pipe->Direction() == Pipe::In, pipe->Interval(), pipe->MaxPacketSize(), 1877 usbDevice->Speed(), pipe->MaxBurst(), pipe->BytesPerInterval()); 1878 if (status != B_OK) { 1879 TRACE_ERROR("unable to configure endpoint: %s\n", strerror(status)); 1880 return status; 1881 } 1882 1883 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1884 _WriteContext(&device->input_ctx->input.addFlags, 1885 (1 << endpointNum) | (1 << 0)); 1886 1887 ConfigureEndpoint(device->input_ctx_addr, false, device->slot); 1888 1889 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", 1890 device->address, SLOT_3_SLOT_STATE_GET(_ReadContext( 1891 &device->device_ctx->slot.dwslot3))); 1892 TRACE("endpoint[0] state 0x%08" B_PRIx32 "\n", 1893 ENDPOINT_0_STATE_GET(_ReadContext( 1894 &device->device_ctx->endpoints[0].dwendpoint0))); 1895 TRACE("endpoint[%d] state 0x%08" B_PRIx32 "\n", id, 1896 ENDPOINT_0_STATE_GET(_ReadContext( 1897 &device->device_ctx->endpoints[id].dwendpoint0))); 1898 } 1899 pipe->SetControllerCookie(&device->endpoints[id]); 1900 1901 return B_OK; 1902 } 1903 1904 1905 status_t 1906 XHCI::_RemoveEndpointForPipe(Pipe *pipe) 1907 { 1908 TRACE("remove endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1909 1910 if (pipe->Parent()->Type() != USB_OBJECT_DEVICE) 1911 return B_OK; 1912 Device* usbDevice = (Device *)pipe->Parent(); 1913 if (usbDevice->Parent() == RootObject()) 1914 return B_BAD_VALUE; 1915 1916 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 1917 if (endpoint == NULL || endpoint->trbs == NULL) 1918 return B_NO_INIT; 1919 1920 pipe->SetControllerCookie(NULL); 1921 1922 if (endpoint->id > 0) { 1923 xhci_device *device = endpoint->device; 1924 uint8 epNumber = endpoint->id + 1; 1925 StopEndpoint(true, endpoint); 1926 1927 mutex_lock(&endpoint->lock); 1928 1929 // See comment in CancelQueuedTransfers. 1930 xhci_td* td; 1931 while ((td = endpoint->td_head) != NULL) { 1932 endpoint->td_head = endpoint->td_head->next; 1933 FreeDescriptor(td); 1934 } 1935 1936 mutex_destroy(&endpoint->lock); 1937 memset(endpoint, 0, sizeof(xhci_endpoint)); 1938 1939 _WriteContext(&device->input_ctx->input.dropFlags, (1 << epNumber)); 1940 _WriteContext(&device->input_ctx->input.addFlags, (1 << 0)); 1941 1942 // The Deconfigure bit in the Configure Endpoint command indicates 1943 // that *all* endpoints are to be deconfigured, and not just the ones 1944 // specified in the context flags. (XHCI 1.2 § 4.6.6 p115.) 1945 ConfigureEndpoint(device->input_ctx_addr, false, device->slot); 1946 } 1947 1948 return B_OK; 1949 } 1950 1951 1952 status_t 1953 XHCI::_LinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 1954 { 1955 TRACE("link descriptor for pipe\n"); 1956 1957 // Use mutex_trylock first, in case we are in KDL. 1958 MutexLocker endpointLocker(&endpoint->lock, mutex_trylock(&endpoint->lock) == B_OK); 1959 1960 // "used" refers to the number of currently linked TDs, not the number of 1961 // used TRBs on the ring (we use 2 TRBs on the ring per transfer.) 1962 if (endpoint->used >= (XHCI_MAX_TRANSFERS - 1)) { 1963 TRACE_ERROR("link descriptor for pipe: max transfers count exceeded\n"); 1964 return B_BAD_VALUE; 1965 } 1966 1967 // We do not support queuing other transfers in tandem with a fragmented one. 1968 if (endpoint->td_head != NULL && endpoint->td_head->transfer != NULL 1969 && endpoint->td_head->transfer->IsFragmented()) { 1970 TRACE_ERROR("cannot submit transfer: a fragmented transfer is queued\n"); 1971 return B_DEV_RESOURCE_CONFLICT; 1972 } 1973 1974 endpoint->used++; 1975 descriptor->next = endpoint->td_head; 1976 endpoint->td_head = descriptor; 1977 1978 const uint32 current = endpoint->current, 1979 eventdata = current + 1, 1980 last = XHCI_ENDPOINT_RING_SIZE - 1; 1981 uint32 next = eventdata + 1; 1982 1983 TRACE("link descriptor for pipe: current %d, next %d\n", current, next); 1984 1985 // Add a Link TRB to the end of the descriptor. 1986 phys_addr_t addr = endpoint->trb_addr + eventdata * sizeof(xhci_trb); 1987 descriptor->trbs[descriptor->trb_used].address = addr; 1988 descriptor->trbs[descriptor->trb_used].status = TRB_2_IRQ(0); 1989 descriptor->trbs[descriptor->trb_used].flags = TRB_3_TYPE(TRB_TYPE_LINK) 1990 | TRB_3_CHAIN_BIT | TRB_3_CYCLE_BIT; 1991 // It is specified that (XHCI 1.2 § 4.12.3 Note 2 p251) if the TRB 1992 // following one with the ENT bit set is a Link TRB, the Link TRB 1993 // shall be evaluated *and* the subsequent TRB shall be. Thus a 1994 // TRB_3_ENT_BIT is unnecessary here; and from testing seems to 1995 // break all transfers on a (very) small number of controllers. 1996 1997 #if !B_HOST_IS_LENDIAN 1998 // Convert endianness. 1999 for (uint32 i = 0; i <= descriptor->trb_used; i++) { 2000 descriptor->trbs[i].address = 2001 B_HOST_TO_LENDIAN_INT64(descriptor->trbs[i].address); 2002 descriptor->trbs[i].status = 2003 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].status); 2004 descriptor->trbs[i].flags = 2005 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].flags); 2006 } 2007 #endif 2008 2009 // Link the descriptor. 2010 endpoint->trbs[current].address = 2011 B_HOST_TO_LENDIAN_INT64(descriptor->trb_addr); 2012 endpoint->trbs[current].status = 2013 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 2014 endpoint->trbs[current].flags = 2015 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_LINK)); 2016 2017 // Set up the Event Data TRB (XHCI 1.2 § 4.11.5.2 p230.) 2018 // 2019 // We do this on the main ring for two reasons: first, to avoid a small 2020 // potential race between the interrupt and the controller evaluating 2021 // the link TRB to get back onto the ring; and second, because many 2022 // controllers throw errors if the target of a Link TRB is not valid 2023 // (i.e. does not have its Cycle Bit set.) 2024 // 2025 // We also set the "address" field, which the controller will copy 2026 // verbatim into the TRB it posts to the event ring, to be the last 2027 // "real" TRB in the TD; this will allow us to determine what transfer 2028 // the resulting Transfer Event TRB refers to. 2029 endpoint->trbs[eventdata].address = 2030 B_HOST_TO_LENDIAN_INT64(descriptor->trb_addr 2031 + (descriptor->trb_used - 1) * sizeof(xhci_trb)); 2032 endpoint->trbs[eventdata].status = 2033 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 2034 endpoint->trbs[eventdata].flags = 2035 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_EVENT_DATA) 2036 | TRB_3_IOC_BIT | TRB_3_CYCLE_BIT); 2037 2038 if (next == last) { 2039 // We always use 2 TRBs per _Link..() call, so if "next" is the last 2040 // TRB in the ring, we need to generate a link TRB at "next", and 2041 // then wrap it to 0. (We write the cycle bit later, after wrapping, 2042 // for the reason noted in the previous comment.) 2043 endpoint->trbs[next].address = 2044 B_HOST_TO_LENDIAN_INT64(endpoint->trb_addr); 2045 endpoint->trbs[next].status = 2046 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 2047 endpoint->trbs[next].flags = 2048 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_LINK)); 2049 2050 next = 0; 2051 } 2052 2053 endpoint->trbs[next].address = 0; 2054 endpoint->trbs[next].status = 0; 2055 endpoint->trbs[next].flags = 0; 2056 2057 memory_write_barrier(); 2058 2059 // Everything is ready, so write the cycle bit(s). 2060 endpoint->trbs[current].flags |= B_HOST_TO_LENDIAN_INT32(TRB_3_CYCLE_BIT); 2061 if (current == 0 && endpoint->trbs[last].address != 0) 2062 endpoint->trbs[last].flags |= B_HOST_TO_LENDIAN_INT32(TRB_3_CYCLE_BIT); 2063 2064 TRACE("_LinkDescriptorForPipe pCurrent %p phys 0x%" B_PRIxPHYSADDR 2065 " 0x%" B_PRIxPHYSADDR " 0x%08" B_PRIx32 "\n", &endpoint->trbs[current], 2066 endpoint->trb_addr + current * sizeof(struct xhci_trb), 2067 endpoint->trbs[current].address, 2068 B_LENDIAN_TO_HOST_INT32(endpoint->trbs[current].flags)); 2069 2070 endpoint->current = next; 2071 endpointLocker.Unlock(); 2072 2073 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 2074 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 2075 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 2076 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 2077 2078 Ring(endpoint->device->slot, endpoint->id + 1); 2079 2080 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 2081 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 2082 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 2083 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 2084 2085 return B_OK; 2086 } 2087 2088 2089 status_t 2090 XHCI::_UnlinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 2091 { 2092 TRACE("unlink descriptor for pipe\n"); 2093 // We presume that the caller has already locked or owns the endpoint. 2094 2095 endpoint->used--; 2096 if (descriptor == endpoint->td_head) { 2097 endpoint->td_head = descriptor->next; 2098 descriptor->next = NULL; 2099 return B_OK; 2100 } else { 2101 for (xhci_td *td = endpoint->td_head; td->next != NULL; td = td->next) { 2102 if (td->next == descriptor) { 2103 td->next = descriptor->next; 2104 descriptor->next = NULL; 2105 return B_OK; 2106 } 2107 } 2108 } 2109 2110 endpoint->used++; 2111 return B_ERROR; 2112 } 2113 2114 2115 status_t 2116 XHCI::ConfigureEndpoint(xhci_endpoint* ep, uint8 slot, uint8 number, uint8 type, 2117 bool directionIn, uint16 interval, uint16 maxPacketSize, usb_speed speed, 2118 uint8 maxBurst, uint16 bytesPerInterval) 2119 { 2120 struct xhci_device* device = &fDevices[slot]; 2121 2122 uint32 dwendpoint0 = 0; 2123 uint32 dwendpoint1 = 0; 2124 uint64 qwendpoint2 = 0; 2125 uint32 dwendpoint4 = 0; 2126 2127 // Compute and assign the endpoint type. (XHCI 1.2 § 6.2.3 Table 6-9 p452.) 2128 uint8 xhciType = 4; 2129 if ((type & USB_OBJECT_INTERRUPT_PIPE) != 0) 2130 xhciType = 3; 2131 if ((type & USB_OBJECT_BULK_PIPE) != 0) 2132 xhciType = 2; 2133 if ((type & USB_OBJECT_ISO_PIPE) != 0) 2134 xhciType = 1; 2135 xhciType |= directionIn ? (1 << 2) : 0; 2136 dwendpoint1 |= ENDPOINT_1_EPTYPE(xhciType); 2137 2138 // Compute and assign interval. (XHCI 1.2 § 6.2.3.6 p456.) 2139 uint16 calcInterval; 2140 if ((type & USB_OBJECT_BULK_PIPE) != 0 2141 || (type & USB_OBJECT_CONTROL_PIPE) != 0) { 2142 // Bulk and Control endpoints never issue NAKs. 2143 calcInterval = 0; 2144 } else { 2145 switch (speed) { 2146 case USB_SPEED_FULLSPEED: 2147 if ((type & USB_OBJECT_ISO_PIPE) != 0) { 2148 // Convert 1-16 into 3-18. 2149 calcInterval = min_c(max_c(interval, 1), 16) + 2; 2150 break; 2151 } 2152 2153 // fall through 2154 case USB_SPEED_LOWSPEED: { 2155 // Convert 1ms-255ms into 3-10. 2156 2157 // Find the index of the highest set bit in "interval". 2158 uint32 temp = min_c(max_c(interval, 1), 255); 2159 for (calcInterval = 0; temp != 1; calcInterval++) 2160 temp = temp >> 1; 2161 calcInterval += 3; 2162 break; 2163 } 2164 2165 case USB_SPEED_HIGHSPEED: 2166 case USB_SPEED_SUPERSPEED: 2167 default: 2168 // Convert 1-16 into 0-15. 2169 calcInterval = min_c(max_c(interval, 1), 16) - 1; 2170 break; 2171 } 2172 } 2173 dwendpoint0 |= ENDPOINT_0_INTERVAL(calcInterval); 2174 2175 // For non-isochronous endpoints, we want the controller to retry failed 2176 // transfers, if possible. (XHCI 1.2 § 4.10.2.3 p197.) 2177 if ((type & USB_OBJECT_ISO_PIPE) == 0) 2178 dwendpoint1 |= ENDPOINT_1_CERR(3); 2179 2180 // Assign maximum burst size. For USB3 devices this is passed in; for 2181 // all other devices we compute it. (XHCI 1.2 § 4.8.2 p161.) 2182 if (speed == USB_SPEED_HIGHSPEED && (type & (USB_OBJECT_INTERRUPT_PIPE 2183 | USB_OBJECT_ISO_PIPE)) != 0) { 2184 maxBurst = (maxPacketSize & 0x1800) >> 11; 2185 } else if (speed != USB_SPEED_SUPERSPEED) { 2186 maxBurst = 0; 2187 } 2188 dwendpoint1 |= ENDPOINT_1_MAXBURST(maxBurst); 2189 2190 // Assign maximum packet size, set the ring address, and set the 2191 // "Dequeue Cycle State" bit. (XHCI 1.2 § 6.2.3 Table 6-10 p453.) 2192 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE(maxPacketSize); 2193 qwendpoint2 |= ENDPOINT_2_DCS_BIT | ep->trb_addr; 2194 2195 // The Max Burst Payload is the number of bytes moved by a 2196 // maximum sized burst. (XHCI 1.2 § 4.11.7.1 p236.) 2197 ep->max_burst_payload = (maxBurst + 1) * maxPacketSize; 2198 if (ep->max_burst_payload == 0) { 2199 TRACE_ERROR("ConfigureEndpoint() failed invalid max_burst_payload\n"); 2200 return B_BAD_VALUE; 2201 } 2202 2203 // Assign average TRB length. 2204 if ((type & USB_OBJECT_CONTROL_PIPE) != 0) { 2205 // Control pipes are a special case, as they rarely have 2206 // outbound transfers of any substantial size. 2207 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(8); 2208 } else if ((type & USB_OBJECT_ISO_PIPE) != 0) { 2209 // Isochronous pipes are another special case: the TRB size will be 2210 // one packet (which is normally smaller than the max packet size, 2211 // but we don't know what it is here.) 2212 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(maxPacketSize); 2213 } else { 2214 // Under all other circumstances, we put max_burst_payload in a TRB. 2215 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(ep->max_burst_payload); 2216 } 2217 2218 // Assign maximum ESIT payload. (XHCI 1.2 § 4.14.2 p259.) 2219 if ((type & (USB_OBJECT_INTERRUPT_PIPE | USB_OBJECT_ISO_PIPE)) != 0) { 2220 // TODO: For SuperSpeedPlus endpoints, there is yet another descriptor 2221 // for isochronous endpoints that specifies the maximum ESIT payload. 2222 // We don't fetch this yet, so just fall back to the USB2 computation 2223 // method if bytesPerInterval is 0. 2224 if (speed == USB_SPEED_SUPERSPEED && bytesPerInterval != 0) 2225 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD(bytesPerInterval); 2226 else if (speed >= USB_SPEED_HIGHSPEED) 2227 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD((maxBurst + 1) * maxPacketSize); 2228 } 2229 2230 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint0, 2231 dwendpoint0); 2232 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint1, 2233 dwendpoint1); 2234 _WriteContext(&device->input_ctx->endpoints[number].qwendpoint2, 2235 qwendpoint2); 2236 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint4, 2237 dwendpoint4); 2238 2239 TRACE("endpoint 0x%" B_PRIx32 " 0x%" B_PRIx32 " 0x%" B_PRIx64 " 0x%" 2240 B_PRIx32 "\n", 2241 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint0), 2242 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint1), 2243 _ReadContext(&device->input_ctx->endpoints[number].qwendpoint2), 2244 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint4)); 2245 2246 return B_OK; 2247 } 2248 2249 2250 status_t 2251 XHCI::GetPortSpeed(uint8 index, usb_speed* speed) 2252 { 2253 if (index >= fPortCount) 2254 return B_BAD_INDEX; 2255 2256 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 2257 2258 switch (PS_SPEED_GET(portStatus)) { 2259 case 2: 2260 *speed = USB_SPEED_LOWSPEED; 2261 break; 2262 case 1: 2263 *speed = USB_SPEED_FULLSPEED; 2264 break; 2265 case 3: 2266 *speed = USB_SPEED_HIGHSPEED; 2267 break; 2268 case 4: 2269 *speed = USB_SPEED_SUPERSPEED; 2270 break; 2271 default: 2272 TRACE_ALWAYS("nonstandard port speed %" B_PRId32 ", assuming SuperSpeed\n", 2273 PS_SPEED_GET(portStatus)); 2274 *speed = USB_SPEED_SUPERSPEED; 2275 break; 2276 } 2277 2278 return B_OK; 2279 } 2280 2281 2282 status_t 2283 XHCI::GetPortStatus(uint8 index, usb_port_status* status) 2284 { 2285 if (index >= fPortCount) 2286 return B_BAD_INDEX; 2287 2288 status->status = status->change = 0; 2289 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 2290 TRACE("port %" B_PRId8 " status=0x%08" B_PRIx32 "\n", index, portStatus); 2291 2292 // build the status 2293 switch (PS_SPEED_GET(portStatus)) { 2294 case 3: 2295 status->status |= PORT_STATUS_HIGH_SPEED; 2296 break; 2297 case 2: 2298 status->status |= PORT_STATUS_LOW_SPEED; 2299 break; 2300 default: 2301 break; 2302 } 2303 2304 if (portStatus & PS_CCS) 2305 status->status |= PORT_STATUS_CONNECTION; 2306 if (portStatus & PS_PED) 2307 status->status |= PORT_STATUS_ENABLE; 2308 if (portStatus & PS_OCA) 2309 status->status |= PORT_STATUS_OVER_CURRENT; 2310 if (portStatus & PS_PR) 2311 status->status |= PORT_STATUS_RESET; 2312 if (portStatus & PS_PP) { 2313 if (fPortSpeeds[index] == USB_SPEED_SUPERSPEED) 2314 status->status |= PORT_STATUS_SS_POWER; 2315 else 2316 status->status |= PORT_STATUS_POWER; 2317 } 2318 2319 // build the change 2320 if (portStatus & PS_CSC) 2321 status->change |= PORT_STATUS_CONNECTION; 2322 if (portStatus & PS_PEC) 2323 status->change |= PORT_STATUS_ENABLE; 2324 if (portStatus & PS_OCC) 2325 status->change |= PORT_STATUS_OVER_CURRENT; 2326 if (portStatus & PS_PRC) 2327 status->change |= PORT_STATUS_RESET; 2328 2329 if (fPortSpeeds[index] == USB_SPEED_SUPERSPEED) { 2330 if (portStatus & PS_PLC) 2331 status->change |= PORT_CHANGE_LINK_STATE; 2332 if (portStatus & PS_WRC) 2333 status->change |= PORT_CHANGE_BH_PORT_RESET; 2334 } 2335 2336 return B_OK; 2337 } 2338 2339 2340 status_t 2341 XHCI::SetPortFeature(uint8 index, uint16 feature) 2342 { 2343 TRACE("set port feature index %u feature %u\n", index, feature); 2344 if (index >= fPortCount) 2345 return B_BAD_INDEX; 2346 2347 uint32 portRegister = XHCI_PORTSC(index); 2348 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2349 2350 switch (feature) { 2351 case PORT_SUSPEND: 2352 if ((portStatus & PS_PED) == 0 || (portStatus & PS_PR) 2353 || (portStatus & PS_PLS_MASK) >= PS_XDEV_U3) { 2354 TRACE_ERROR("USB core suspending device not in U0/U1/U2.\n"); 2355 return B_BAD_VALUE; 2356 } 2357 portStatus &= ~PS_PLS_MASK; 2358 WriteOpReg(portRegister, portStatus | PS_LWS | PS_XDEV_U3); 2359 break; 2360 2361 case PORT_RESET: 2362 WriteOpReg(portRegister, portStatus | PS_PR); 2363 break; 2364 2365 case PORT_POWER: 2366 WriteOpReg(portRegister, portStatus | PS_PP); 2367 break; 2368 default: 2369 return B_BAD_VALUE; 2370 } 2371 ReadOpReg(portRegister); 2372 return B_OK; 2373 } 2374 2375 2376 status_t 2377 XHCI::ClearPortFeature(uint8 index, uint16 feature) 2378 { 2379 TRACE("clear port feature index %u feature %u\n", index, feature); 2380 if (index >= fPortCount) 2381 return B_BAD_INDEX; 2382 2383 uint32 portRegister = XHCI_PORTSC(index); 2384 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2385 2386 switch (feature) { 2387 case PORT_SUSPEND: 2388 portStatus = ReadOpReg(portRegister); 2389 if (portStatus & PS_PR) 2390 return B_BAD_VALUE; 2391 if (portStatus & PS_XDEV_U3) { 2392 if ((portStatus & PS_PED) == 0) 2393 return B_BAD_VALUE; 2394 portStatus &= ~PS_PLS_MASK; 2395 WriteOpReg(portRegister, portStatus | PS_XDEV_U0 | PS_LWS); 2396 } 2397 break; 2398 case PORT_ENABLE: 2399 WriteOpReg(portRegister, portStatus | PS_PED); 2400 break; 2401 case PORT_POWER: 2402 WriteOpReg(portRegister, portStatus & ~PS_PP); 2403 break; 2404 case C_PORT_CONNECTION: 2405 WriteOpReg(portRegister, portStatus | PS_CSC); 2406 break; 2407 case C_PORT_ENABLE: 2408 WriteOpReg(portRegister, portStatus | PS_PEC); 2409 break; 2410 case C_PORT_OVER_CURRENT: 2411 WriteOpReg(portRegister, portStatus | PS_OCC); 2412 break; 2413 case C_PORT_RESET: 2414 WriteOpReg(portRegister, portStatus | PS_PRC); 2415 break; 2416 case C_PORT_BH_PORT_RESET: 2417 WriteOpReg(portRegister, portStatus | PS_WRC); 2418 break; 2419 case C_PORT_LINK_STATE: 2420 WriteOpReg(portRegister, portStatus | PS_PLC); 2421 break; 2422 default: 2423 return B_BAD_VALUE; 2424 } 2425 2426 ReadOpReg(portRegister); 2427 return B_OK; 2428 } 2429 2430 2431 status_t 2432 XHCI::ControllerHalt() 2433 { 2434 // Mask off run state 2435 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) & ~CMD_RUN); 2436 2437 // wait for shutdown state 2438 if (WaitOpBits(XHCI_STS, STS_HCH, STS_HCH) != B_OK) { 2439 TRACE_ERROR("HCH shutdown timeout\n"); 2440 return B_ERROR; 2441 } 2442 return B_OK; 2443 } 2444 2445 2446 status_t 2447 XHCI::ControllerReset() 2448 { 2449 TRACE("ControllerReset() cmd: 0x%" B_PRIx32 " sts: 0x%" B_PRIx32 "\n", 2450 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 2451 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) | CMD_HCRST); 2452 2453 if (WaitOpBits(XHCI_CMD, CMD_HCRST, 0) != B_OK) { 2454 TRACE_ERROR("ControllerReset() failed CMD_HCRST\n"); 2455 return B_ERROR; 2456 } 2457 2458 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 2459 TRACE_ERROR("ControllerReset() failed STS_CNR\n"); 2460 return B_ERROR; 2461 } 2462 2463 return B_OK; 2464 } 2465 2466 2467 int32 2468 XHCI::InterruptHandler(void* data) 2469 { 2470 return ((XHCI*)data)->Interrupt(); 2471 } 2472 2473 2474 int32 2475 XHCI::Interrupt() 2476 { 2477 SpinLocker _(&fSpinlock); 2478 2479 uint32 status = ReadOpReg(XHCI_STS); 2480 uint32 temp = ReadRunReg32(XHCI_IMAN(0)); 2481 WriteOpReg(XHCI_STS, status); 2482 WriteRunReg32(XHCI_IMAN(0), temp); 2483 2484 int32 result = B_HANDLED_INTERRUPT; 2485 2486 if ((status & STS_HCH) != 0) { 2487 TRACE_ERROR("Host Controller halted\n"); 2488 return result; 2489 } 2490 if ((status & STS_HSE) != 0) { 2491 TRACE_ERROR("Host System Error\n"); 2492 return result; 2493 } 2494 if ((status & STS_HCE) != 0) { 2495 TRACE_ERROR("Host Controller Error\n"); 2496 return result; 2497 } 2498 2499 if ((status & STS_EINT) == 0) { 2500 TRACE("STS: 0x%" B_PRIx32 " IRQ_PENDING: 0x%" B_PRIx32 "\n", 2501 status, temp); 2502 return B_UNHANDLED_INTERRUPT; 2503 } 2504 2505 TRACE("Event Interrupt\n"); 2506 release_sem_etc(fEventSem, 1, B_DO_NOT_RESCHEDULE); 2507 return B_INVOKE_SCHEDULER; 2508 } 2509 2510 2511 void 2512 XHCI::Ring(uint8 slot, uint8 endpoint) 2513 { 2514 TRACE("Ding Dong! slot:%d endpoint %d\n", slot, endpoint) 2515 if ((slot == 0 && endpoint > 0) || (slot > 0 && endpoint == 0)) 2516 panic("Ring() invalid slot/endpoint combination\n"); 2517 if (slot > fSlotCount || endpoint >= XHCI_MAX_ENDPOINTS) 2518 panic("Ring() invalid slot or endpoint\n"); 2519 2520 WriteDoorReg32(XHCI_DOORBELL(slot), XHCI_DOORBELL_TARGET(endpoint) 2521 | XHCI_DOORBELL_STREAMID(0)); 2522 ReadDoorReg32(XHCI_DOORBELL(slot)); 2523 // Flush PCI writes 2524 } 2525 2526 2527 void 2528 XHCI::QueueCommand(xhci_trb* trb) 2529 { 2530 uint8 i, j; 2531 uint32 temp; 2532 2533 i = fCmdIdx; 2534 j = fCmdCcs; 2535 2536 TRACE("command[%u] = %" B_PRId32 " (0x%016" B_PRIx64 ", 0x%08" B_PRIx32 2537 ", 0x%08" B_PRIx32 ")\n", i, TRB_3_TYPE_GET(trb->flags), trb->address, 2538 trb->status, trb->flags); 2539 2540 fCmdRing[i].address = trb->address; 2541 fCmdRing[i].status = trb->status; 2542 temp = trb->flags; 2543 2544 if (j) 2545 temp |= TRB_3_CYCLE_BIT; 2546 else 2547 temp &= ~TRB_3_CYCLE_BIT; 2548 temp &= ~TRB_3_TC_BIT; 2549 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2550 2551 fCmdAddr = fErst->rs_addr + (XHCI_MAX_EVENTS + i) * sizeof(xhci_trb); 2552 2553 i++; 2554 2555 if (i == (XHCI_MAX_COMMANDS - 1)) { 2556 temp = TRB_3_TYPE(TRB_TYPE_LINK) | TRB_3_TC_BIT; 2557 if (j) 2558 temp |= TRB_3_CYCLE_BIT; 2559 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2560 2561 i = 0; 2562 j ^= 1; 2563 } 2564 2565 fCmdIdx = i; 2566 fCmdCcs = j; 2567 } 2568 2569 2570 void 2571 XHCI::HandleCmdComplete(xhci_trb* trb) 2572 { 2573 if (fCmdAddr == trb->address) { 2574 TRACE("Received command event\n"); 2575 fCmdResult[0] = trb->status; 2576 fCmdResult[1] = B_LENDIAN_TO_HOST_INT32(trb->flags); 2577 release_sem_etc(fCmdCompSem, 1, B_DO_NOT_RESCHEDULE); 2578 } else 2579 TRACE_ERROR("received command event for unknown command!\n") 2580 } 2581 2582 2583 void 2584 XHCI::HandleTransferComplete(xhci_trb* trb) 2585 { 2586 const uint32 flags = B_LENDIAN_TO_HOST_INT32(trb->flags); 2587 const uint8 endpointNumber = TRB_3_ENDPOINT_GET(flags), 2588 slot = TRB_3_SLOT_GET(flags); 2589 2590 if (slot > fSlotCount) 2591 TRACE_ERROR("invalid slot\n"); 2592 if (endpointNumber == 0 || endpointNumber >= XHCI_MAX_ENDPOINTS) { 2593 TRACE_ERROR("invalid endpoint\n"); 2594 return; 2595 } 2596 2597 xhci_device *device = &fDevices[slot]; 2598 xhci_endpoint *endpoint = &device->endpoints[endpointNumber - 1]; 2599 2600 if (endpoint->trbs == NULL) { 2601 TRACE_ERROR("got TRB but endpoint is not allocated!\n"); 2602 return; 2603 } 2604 2605 // Use mutex_trylock first, in case we are in KDL. 2606 MutexLocker endpointLocker(endpoint->lock, mutex_trylock(&endpoint->lock) == B_OK); 2607 if (!endpointLocker.IsLocked()) { 2608 // We failed to get the lock. Most likely it was destroyed 2609 // while we were waiting for it. 2610 return; 2611 } 2612 2613 // In the case of an Event Data TRB, the "transferred" field refers 2614 // to the actual number of bytes transferred across the whole TD. 2615 // (XHCI 1.2 § 6.4.2.1 Table 6-38 p478.) 2616 const uint8 completionCode = TRB_2_COMP_CODE_GET(trb->status); 2617 int32 transferred = TRB_2_REM_GET(trb->status), remainder = -1; 2618 2619 TRACE("HandleTransferComplete: ed %" B_PRIu32 ", code %" B_PRIu8 ", transferred %" B_PRId32 "\n", 2620 (flags & TRB_3_EVENT_DATA_BIT), completionCode, transferred); 2621 2622 if ((flags & TRB_3_EVENT_DATA_BIT) == 0) { 2623 // This should only occur under error conditions. 2624 TRACE("got an interrupt for a non-Event Data TRB!\n"); 2625 remainder = transferred; 2626 transferred = -1; 2627 } 2628 2629 if (completionCode != COMP_SUCCESS && completionCode != COMP_SHORT_PACKET 2630 && completionCode != COMP_STOPPED) { 2631 TRACE_ALWAYS("transfer error on slot %" B_PRId8 " endpoint %" B_PRId8 2632 ": %s\n", slot, endpointNumber, xhci_error_string(completionCode)); 2633 } 2634 2635 const phys_addr_t source = B_LENDIAN_TO_HOST_INT64(trb->address); 2636 for (xhci_td *td = endpoint->td_head; td != NULL; td = td->next) { 2637 int64 offset = (source - td->trb_addr) / sizeof(xhci_trb); 2638 if (offset < 0 || offset >= td->trb_count) 2639 continue; 2640 2641 TRACE("HandleTransferComplete td %p trb %" B_PRId64 " found\n", 2642 td, offset); 2643 2644 // The TRB at offset trb_used will be the link TRB, which we do not 2645 // care about (and should not generate an interrupt at all.) We really 2646 // care about the properly last TRB, at index "count - 1", which the 2647 // Event Data TRB that _LinkDescriptorForPipe creates points to. 2648 // 2649 // But if we have an unsuccessful completion code, the transfer 2650 // likely failed midway; so just accept it anyway. 2651 if (offset == (td->trb_used - 1) || completionCode != COMP_SUCCESS) { 2652 _UnlinkDescriptorForPipe(td, endpoint); 2653 endpointLocker.Unlock(); 2654 2655 td->trb_completion_code = completionCode; 2656 td->td_transferred = transferred; 2657 td->trb_left = remainder; 2658 2659 // add descriptor to finished list 2660 if (mutex_trylock(&fFinishedLock) != B_OK) 2661 mutex_lock(&fFinishedLock); 2662 td->next = fFinishedHead; 2663 fFinishedHead = td; 2664 mutex_unlock(&fFinishedLock); 2665 2666 release_sem_etc(fFinishTransfersSem, 1, B_DO_NOT_RESCHEDULE); 2667 TRACE("HandleTransferComplete td %p done\n", td); 2668 } else { 2669 TRACE_ERROR("successful TRB 0x%" B_PRIxPHYSADDR " was found, but it wasn't " 2670 "the last in the TD!\n", source); 2671 } 2672 return; 2673 } 2674 TRACE_ERROR("TRB 0x%" B_PRIxPHYSADDR " was not found in the endpoint!\n", source); 2675 } 2676 2677 2678 void 2679 XHCI::DumpRing(xhci_trb *trbs, uint32 size) 2680 { 2681 if (!Lock()) { 2682 TRACE("Unable to get lock!\n"); 2683 return; 2684 } 2685 2686 for (uint32 i = 0; i < size; i++) { 2687 TRACE("command[%" B_PRId32 "] = %" B_PRId32 " (0x%016" B_PRIx64 "," 2688 " 0x%08" B_PRIx32 ", 0x%08" B_PRIx32 ")\n", i, 2689 TRB_3_TYPE_GET(B_LENDIAN_TO_HOST_INT32(trbs[i].flags)), 2690 trbs[i].address, trbs[i].status, trbs[i].flags); 2691 } 2692 2693 Unlock(); 2694 } 2695 2696 2697 status_t 2698 XHCI::DoCommand(xhci_trb* trb) 2699 { 2700 if (!Lock()) { 2701 TRACE("Unable to get lock!\n"); 2702 return B_ERROR; 2703 } 2704 2705 QueueCommand(trb); 2706 Ring(0, 0); 2707 2708 // Begin with a 50ms timeout. 2709 if (acquire_sem_etc(fCmdCompSem, 1, B_RELATIVE_TIMEOUT, 50 * 1000) != B_OK) { 2710 // We've hit the timeout. In some error cases, interrupts are not 2711 // generated; so here we force the event ring to be polled once. 2712 release_sem(fEventSem); 2713 2714 // Now try again, this time with a 750ms timeout. 2715 if (acquire_sem_etc(fCmdCompSem, 1, B_RELATIVE_TIMEOUT, 2716 750 * 1000) != B_OK) { 2717 TRACE("Unable to obtain fCmdCompSem!\n"); 2718 fCmdAddr = 0; 2719 Unlock(); 2720 return B_TIMED_OUT; 2721 } 2722 } 2723 2724 // eat up sems that have been released by multiple interrupts 2725 int32 semCount = 0; 2726 get_sem_count(fCmdCompSem, &semCount); 2727 if (semCount > 0) 2728 acquire_sem_etc(fCmdCompSem, semCount, B_RELATIVE_TIMEOUT, 0); 2729 2730 status_t status = B_OK; 2731 uint32 completionCode = TRB_2_COMP_CODE_GET(fCmdResult[0]); 2732 TRACE("command complete\n"); 2733 if (completionCode != COMP_SUCCESS) { 2734 TRACE_ERROR("unsuccessful command %" B_PRId32 ", error %s (%" B_PRId32 ")\n", 2735 TRB_3_TYPE_GET(trb->flags), xhci_error_string(completionCode), 2736 completionCode); 2737 status = B_IO_ERROR; 2738 } 2739 2740 trb->status = fCmdResult[0]; 2741 trb->flags = fCmdResult[1]; 2742 2743 fCmdAddr = 0; 2744 Unlock(); 2745 return status; 2746 } 2747 2748 2749 status_t 2750 XHCI::Noop() 2751 { 2752 TRACE("Issue No-Op\n"); 2753 xhci_trb trb; 2754 trb.address = 0; 2755 trb.status = 0; 2756 trb.flags = TRB_3_TYPE(TRB_TYPE_CMD_NOOP); 2757 2758 return DoCommand(&trb); 2759 } 2760 2761 2762 status_t 2763 XHCI::EnableSlot(uint8* slot) 2764 { 2765 TRACE("Enable Slot\n"); 2766 xhci_trb trb; 2767 trb.address = 0; 2768 trb.status = 0; 2769 trb.flags = TRB_3_TYPE(TRB_TYPE_ENABLE_SLOT); 2770 2771 status_t status = DoCommand(&trb); 2772 if (status != B_OK) 2773 return status; 2774 2775 *slot = TRB_3_SLOT_GET(trb.flags); 2776 return *slot != 0 ? B_OK : B_BAD_VALUE; 2777 } 2778 2779 2780 status_t 2781 XHCI::DisableSlot(uint8 slot) 2782 { 2783 TRACE("Disable Slot\n"); 2784 xhci_trb trb; 2785 trb.address = 0; 2786 trb.status = 0; 2787 trb.flags = TRB_3_TYPE(TRB_TYPE_DISABLE_SLOT) | TRB_3_SLOT(slot); 2788 2789 return DoCommand(&trb); 2790 } 2791 2792 2793 status_t 2794 XHCI::SetAddress(uint64 inputContext, bool bsr, uint8 slot) 2795 { 2796 TRACE("Set Address\n"); 2797 xhci_trb trb; 2798 trb.address = inputContext; 2799 trb.status = 0; 2800 trb.flags = TRB_3_TYPE(TRB_TYPE_ADDRESS_DEVICE) | TRB_3_SLOT(slot); 2801 2802 if (bsr) 2803 trb.flags |= TRB_3_BSR_BIT; 2804 2805 return DoCommand(&trb); 2806 } 2807 2808 2809 status_t 2810 XHCI::ConfigureEndpoint(uint64 inputContext, bool deconfigure, uint8 slot) 2811 { 2812 TRACE("Configure Endpoint\n"); 2813 xhci_trb trb; 2814 trb.address = inputContext; 2815 trb.status = 0; 2816 trb.flags = TRB_3_TYPE(TRB_TYPE_CONFIGURE_ENDPOINT) | TRB_3_SLOT(slot); 2817 2818 if (deconfigure) 2819 trb.flags |= TRB_3_DCEP_BIT; 2820 2821 return DoCommand(&trb); 2822 } 2823 2824 2825 status_t 2826 XHCI::EvaluateContext(uint64 inputContext, uint8 slot) 2827 { 2828 TRACE("Evaluate Context\n"); 2829 xhci_trb trb; 2830 trb.address = inputContext; 2831 trb.status = 0; 2832 trb.flags = TRB_3_TYPE(TRB_TYPE_EVALUATE_CONTEXT) | TRB_3_SLOT(slot); 2833 2834 return DoCommand(&trb); 2835 } 2836 2837 2838 status_t 2839 XHCI::ResetEndpoint(bool preserve, xhci_endpoint* endpoint) 2840 { 2841 TRACE("Reset Endpoint\n"); 2842 2843 switch (_GetEndpointState(endpoint)) { 2844 case ENDPOINT_STATE_STOPPED: 2845 TRACE("Reset Endpoint: already stopped"); 2846 return B_OK; 2847 case ENDPOINT_STATE_HALTED: 2848 TRACE("Reset Endpoint: warning, weird state!"); 2849 default: 2850 break; 2851 } 2852 2853 xhci_trb trb; 2854 trb.address = 0; 2855 trb.status = 0; 2856 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_ENDPOINT) 2857 | TRB_3_SLOT(endpoint->device->slot) | TRB_3_ENDPOINT(endpoint->id + 1); 2858 if (preserve) 2859 trb.flags |= TRB_3_PRSV_BIT; 2860 2861 return DoCommand(&trb); 2862 } 2863 2864 2865 status_t 2866 XHCI::StopEndpoint(bool suspend, xhci_endpoint* endpoint) 2867 { 2868 TRACE("Stop Endpoint\n"); 2869 2870 switch (_GetEndpointState(endpoint)) { 2871 case ENDPOINT_STATE_HALTED: 2872 TRACE("Stop Endpoint: error, halted"); 2873 return B_DEV_STALLED; 2874 case ENDPOINT_STATE_STOPPED: 2875 TRACE("Stop Endpoint: already stopped"); 2876 return B_OK; 2877 default: 2878 break; 2879 } 2880 2881 xhci_trb trb; 2882 trb.address = 0; 2883 trb.status = 0; 2884 trb.flags = TRB_3_TYPE(TRB_TYPE_STOP_ENDPOINT) 2885 | TRB_3_SLOT(endpoint->device->slot) | TRB_3_ENDPOINT(endpoint->id + 1); 2886 if (suspend) 2887 trb.flags |= TRB_3_SUSPEND_ENDPOINT_BIT; 2888 2889 return DoCommand(&trb); 2890 } 2891 2892 2893 status_t 2894 XHCI::SetTRDequeue(uint64 dequeue, uint16 stream, uint8 endpoint, uint8 slot) 2895 { 2896 TRACE("Set TR Dequeue\n"); 2897 xhci_trb trb; 2898 trb.address = dequeue | ENDPOINT_2_DCS_BIT; 2899 // The DCS bit is copied from the address field as in ConfigureEndpoint. 2900 // (XHCI 1.2 § 4.6.10 p142.) 2901 trb.status = TRB_2_STREAM(stream); 2902 trb.flags = TRB_3_TYPE(TRB_TYPE_SET_TR_DEQUEUE) 2903 | TRB_3_SLOT(slot) | TRB_3_ENDPOINT(endpoint); 2904 2905 return DoCommand(&trb); 2906 } 2907 2908 2909 status_t 2910 XHCI::ResetDevice(uint8 slot) 2911 { 2912 TRACE("Reset Device\n"); 2913 xhci_trb trb; 2914 trb.address = 0; 2915 trb.status = 0; 2916 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_DEVICE) | TRB_3_SLOT(slot); 2917 2918 return DoCommand(&trb); 2919 } 2920 2921 2922 int32 2923 XHCI::EventThread(void* data) 2924 { 2925 ((XHCI *)data)->CompleteEvents(); 2926 return B_OK; 2927 } 2928 2929 2930 void 2931 XHCI::CompleteEvents() 2932 { 2933 while (!fStopThreads) { 2934 if (acquire_sem(fEventSem) < B_OK) 2935 continue; 2936 2937 // eat up sems that have been released by multiple interrupts 2938 int32 semCount = 0; 2939 get_sem_count(fEventSem, &semCount); 2940 if (semCount > 0) 2941 acquire_sem_etc(fEventSem, semCount, B_RELATIVE_TIMEOUT, 0); 2942 2943 ProcessEvents(); 2944 } 2945 } 2946 2947 2948 void 2949 XHCI::ProcessEvents() 2950 { 2951 // Use mutex_trylock first, in case we are in KDL. 2952 MutexLocker locker(fEventLock, mutex_trylock(&fEventLock) == B_OK); 2953 if (!locker.IsLocked()) { 2954 // We failed to get the lock. This really should not happen. 2955 TRACE_ERROR("failed to acquire event lock!\n"); 2956 return; 2957 } 2958 2959 uint16 i = fEventIdx; 2960 uint8 j = fEventCcs; 2961 uint8 t = 2; 2962 2963 while (1) { 2964 uint32 temp = B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags); 2965 uint8 event = TRB_3_TYPE_GET(temp); 2966 TRACE("event[%u] = %u (0x%016" B_PRIx64 " 0x%08" B_PRIx32 " 0x%08" 2967 B_PRIx32 ")\n", i, event, fEventRing[i].address, 2968 fEventRing[i].status, B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags)); 2969 uint8 k = (temp & TRB_3_CYCLE_BIT) ? 1 : 0; 2970 if (j != k) 2971 break; 2972 2973 switch (event) { 2974 case TRB_TYPE_COMMAND_COMPLETION: 2975 HandleCmdComplete(&fEventRing[i]); 2976 break; 2977 case TRB_TYPE_TRANSFER: 2978 HandleTransferComplete(&fEventRing[i]); 2979 break; 2980 case TRB_TYPE_PORT_STATUS_CHANGE: 2981 TRACE("port change detected\n"); 2982 break; 2983 default: 2984 TRACE_ERROR("Unhandled event = %u\n", event); 2985 break; 2986 } 2987 2988 i++; 2989 if (i == XHCI_MAX_EVENTS) { 2990 i = 0; 2991 j ^= 1; 2992 if (!--t) 2993 break; 2994 } 2995 } 2996 2997 fEventIdx = i; 2998 fEventCcs = j; 2999 3000 uint64 addr = fErst->rs_addr + i * sizeof(xhci_trb); 3001 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)addr | ERDP_BUSY); 3002 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(addr >> 32)); 3003 } 3004 3005 3006 int32 3007 XHCI::FinishThread(void* data) 3008 { 3009 ((XHCI *)data)->FinishTransfers(); 3010 return B_OK; 3011 } 3012 3013 3014 void 3015 XHCI::FinishTransfers() 3016 { 3017 while (!fStopThreads) { 3018 if (acquire_sem(fFinishTransfersSem) < B_OK) 3019 continue; 3020 3021 // eat up sems that have been released by multiple interrupts 3022 int32 semCount = 0; 3023 get_sem_count(fFinishTransfersSem, &semCount); 3024 if (semCount > 0) 3025 acquire_sem_etc(fFinishTransfersSem, semCount, B_RELATIVE_TIMEOUT, 0); 3026 3027 mutex_lock(&fFinishedLock); 3028 TRACE("finishing transfers\n"); 3029 while (fFinishedHead != NULL) { 3030 xhci_td* td = fFinishedHead; 3031 fFinishedHead = td->next; 3032 td->next = NULL; 3033 mutex_unlock(&fFinishedLock); 3034 3035 TRACE("finishing transfer td %p\n", td); 3036 3037 Transfer* transfer = td->transfer; 3038 if (transfer == NULL) { 3039 // No transfer? Quick way out. 3040 FreeDescriptor(td); 3041 mutex_lock(&fFinishedLock); 3042 continue; 3043 } 3044 3045 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 3046 3047 status_t callbackStatus = B_OK; 3048 const uint8 completionCode = td->trb_completion_code; 3049 switch (completionCode) { 3050 case COMP_SHORT_PACKET: 3051 case COMP_SUCCESS: 3052 callbackStatus = B_OK; 3053 break; 3054 case COMP_DATA_BUFFER: 3055 callbackStatus = directionIn ? B_DEV_DATA_OVERRUN 3056 : B_DEV_DATA_UNDERRUN; 3057 break; 3058 case COMP_BABBLE: 3059 callbackStatus = directionIn ? B_DEV_FIFO_OVERRUN 3060 : B_DEV_FIFO_UNDERRUN; 3061 break; 3062 case COMP_USB_TRANSACTION: 3063 callbackStatus = B_DEV_CRC_ERROR; 3064 break; 3065 case COMP_STALL: 3066 callbackStatus = B_DEV_STALLED; 3067 break; 3068 default: 3069 callbackStatus = B_DEV_STALLED; 3070 break; 3071 } 3072 3073 size_t actualLength = transfer->FragmentLength(); 3074 if (completionCode != COMP_SUCCESS) { 3075 actualLength = td->td_transferred; 3076 if (td->td_transferred == -1) 3077 actualLength = transfer->FragmentLength() - td->trb_left; 3078 TRACE("transfer not successful, actualLength=%" B_PRIuSIZE "\n", 3079 actualLength); 3080 } 3081 3082 usb_isochronous_data* isochronousData = transfer->IsochronousData(); 3083 if (isochronousData != NULL) { 3084 size_t packetSize = transfer->DataLength() 3085 / isochronousData->packet_count, 3086 left = actualLength; 3087 for (uint32 i = 0; i < isochronousData->packet_count; i++) { 3088 size_t size = min_c(packetSize, left); 3089 isochronousData->packet_descriptors[i].actual_length = size; 3090 isochronousData->packet_descriptors[i].status = (size > 0) 3091 ? B_OK : B_DEV_FIFO_UNDERRUN; 3092 left -= size; 3093 } 3094 } 3095 3096 if (callbackStatus == B_OK && directionIn && actualLength > 0) { 3097 TRACE("copying in iov count %ld\n", transfer->VectorCount()); 3098 status_t status = transfer->PrepareKernelAccess(); 3099 if (status == B_OK) { 3100 ReadDescriptor(td, transfer->Vector(), 3101 transfer->VectorCount()); 3102 } else { 3103 callbackStatus = status; 3104 } 3105 } 3106 3107 FreeDescriptor(td); 3108 3109 // this transfer may still have data left 3110 bool finished = true; 3111 transfer->AdvanceByFragment(actualLength); 3112 if (completionCode == COMP_SUCCESS 3113 && transfer->FragmentLength() > 0) { 3114 TRACE("still %" B_PRIuSIZE " bytes left on transfer\n", 3115 transfer->FragmentLength()); 3116 callbackStatus = SubmitTransfer(transfer); 3117 finished = (callbackStatus != B_OK); 3118 } 3119 if (finished) { 3120 // The actualLength was already handled in AdvanceByFragment. 3121 transfer->Finished(callbackStatus, 0); 3122 delete transfer; 3123 } 3124 3125 mutex_lock(&fFinishedLock); 3126 } 3127 mutex_unlock(&fFinishedLock); 3128 } 3129 } 3130 3131 3132 inline void 3133 XHCI::WriteOpReg(uint32 reg, uint32 value) 3134 { 3135 *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg) = value; 3136 } 3137 3138 3139 inline uint32 3140 XHCI::ReadOpReg(uint32 reg) 3141 { 3142 return *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg); 3143 } 3144 3145 3146 inline status_t 3147 XHCI::WaitOpBits(uint32 reg, uint32 mask, uint32 expected) 3148 { 3149 int loops = 0; 3150 uint32 value = ReadOpReg(reg); 3151 while ((value & mask) != expected) { 3152 snooze(1000); 3153 value = ReadOpReg(reg); 3154 if (loops == 100) { 3155 TRACE("delay waiting on reg 0x%" B_PRIX32 " match 0x%" B_PRIX32 3156 " (0x%" B_PRIX32 ")\n", reg, expected, mask); 3157 } else if (loops > 250) { 3158 TRACE_ERROR("timeout waiting on reg 0x%" B_PRIX32 3159 " match 0x%" B_PRIX32 " (0x%" B_PRIX32 ")\n", reg, expected, 3160 mask); 3161 return B_ERROR; 3162 } 3163 loops++; 3164 } 3165 return B_OK; 3166 } 3167 3168 3169 inline uint32 3170 XHCI::ReadCapReg32(uint32 reg) 3171 { 3172 return *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg); 3173 } 3174 3175 3176 inline void 3177 XHCI::WriteCapReg32(uint32 reg, uint32 value) 3178 { 3179 *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg) = value; 3180 } 3181 3182 3183 inline uint32 3184 XHCI::ReadRunReg32(uint32 reg) 3185 { 3186 return *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg); 3187 } 3188 3189 3190 inline void 3191 XHCI::WriteRunReg32(uint32 reg, uint32 value) 3192 { 3193 *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg) = value; 3194 } 3195 3196 3197 inline uint32 3198 XHCI::ReadDoorReg32(uint32 reg) 3199 { 3200 return *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg); 3201 } 3202 3203 3204 inline void 3205 XHCI::WriteDoorReg32(uint32 reg, uint32 value) 3206 { 3207 *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg) = value; 3208 } 3209 3210 3211 inline addr_t 3212 XHCI::_OffsetContextAddr(addr_t p) 3213 { 3214 if (fContextSizeShift == 1) { 3215 // each structure is page aligned, each pointer is 32 bits aligned 3216 uint32 offset = p & ((B_PAGE_SIZE - 1) & ~31U); 3217 p += offset; 3218 } 3219 return p; 3220 } 3221 3222 inline uint32 3223 XHCI::_ReadContext(uint32* p) 3224 { 3225 p = (uint32*)_OffsetContextAddr((addr_t)p); 3226 return *p; 3227 } 3228 3229 3230 inline void 3231 XHCI::_WriteContext(uint32* p, uint32 value) 3232 { 3233 p = (uint32*)_OffsetContextAddr((addr_t)p); 3234 *p = value; 3235 } 3236 3237 3238 inline uint64 3239 XHCI::_ReadContext(uint64* p) 3240 { 3241 p = (uint64*)_OffsetContextAddr((addr_t)p); 3242 return *p; 3243 } 3244 3245 3246 inline void 3247 XHCI::_WriteContext(uint64* p, uint64 value) 3248 { 3249 p = (uint64*)_OffsetContextAddr((addr_t)p); 3250 *p = value; 3251 } 3252