1 /* 2 * Copyright 2011-2021, Haiku, Inc. All rights reserved. 3 * Distributed under the terms of the MIT License. 4 * 5 * Authors: 6 * Augustin Cavalier <waddlesplash> 7 * Jian Chiang <j.jian.chiang@gmail.com> 8 * Jérôme Duval <jerome.duval@gmail.com> 9 * Akshay Jaggi <akshay1994.leo@gmail.com> 10 * Michael Lotz <mmlr@mlotz.ch> 11 * Alexander von Gluck <kallisti5@unixzen.com> 12 */ 13 14 15 #include <stdio.h> 16 17 #include <PCI_x86.h> 18 #include <bus/PCI.h> 19 #include <USB3.h> 20 #include <KernelExport.h> 21 22 #include <ByteOrder.h> 23 #include <util/AutoLock.h> 24 25 #include "xhci.h" 26 27 28 #define CALLED(x...) TRACE_MODULE("CALLED %s\n", __PRETTY_FUNCTION__) 29 30 31 #define USB_MODULE_NAME "xhci" 32 33 static pci_x86_module_info* sPCIx86Module = NULL; 34 device_manager_info* gDeviceManager; 35 static usb_for_controller_interface* gUSB; 36 37 38 #define XHCI_PCI_DEVICE_MODULE_NAME "busses/usb/xhci/pci/driver_v1" 39 #define XHCI_PCI_USB_BUS_MODULE_NAME "busses/usb/xhci/device_v1" 40 41 42 typedef struct { 43 XHCI* xhci; 44 pci_device_module_info* pci; 45 pci_device* device; 46 47 pci_info pciinfo; 48 49 device_node* node; 50 device_node* driver_node; 51 } xhci_pci_sim_info; 52 53 54 // #pragma mark - 55 56 57 static status_t 58 init_bus(device_node* node, void** bus_cookie) 59 { 60 CALLED(); 61 62 driver_module_info* driver; 63 xhci_pci_sim_info* bus; 64 device_node* parent = gDeviceManager->get_parent_node(node); 65 gDeviceManager->get_driver(parent, &driver, (void**)&bus); 66 gDeviceManager->put_node(parent); 67 68 Stack *stack; 69 if (gUSB->get_stack((void**)&stack) != B_OK) 70 return B_ERROR; 71 72 XHCI *xhci = new(std::nothrow) XHCI(&bus->pciinfo, bus->pci, bus->device, stack, node); 73 if (xhci == NULL) { 74 return B_NO_MEMORY; 75 } 76 77 if (xhci->InitCheck() < B_OK) { 78 TRACE_MODULE_ERROR("bus failed init check\n"); 79 delete xhci; 80 return B_ERROR; 81 } 82 83 if (xhci->Start() != B_OK) { 84 delete xhci; 85 return B_ERROR; 86 } 87 88 *bus_cookie = xhci; 89 90 return B_OK; 91 } 92 93 94 static void 95 uninit_bus(void* bus_cookie) 96 { 97 CALLED(); 98 XHCI* xhci = (XHCI*)bus_cookie; 99 delete xhci; 100 } 101 102 103 static status_t 104 register_child_devices(void* cookie) 105 { 106 CALLED(); 107 xhci_pci_sim_info* bus = (xhci_pci_sim_info*)cookie; 108 device_node* node = bus->driver_node; 109 110 char prettyName[25]; 111 sprintf(prettyName, "XHCI Controller %" B_PRIu16, 0); 112 113 device_attr attrs[] = { 114 // properties of this controller for the usb bus manager 115 { B_DEVICE_PRETTY_NAME, B_STRING_TYPE, 116 { string: prettyName }}, 117 { B_DEVICE_FIXED_CHILD, B_STRING_TYPE, 118 { string: USB_FOR_CONTROLLER_MODULE_NAME }}, 119 120 // private data to identify the device 121 { NULL } 122 }; 123 124 return gDeviceManager->register_node(node, XHCI_PCI_USB_BUS_MODULE_NAME, 125 attrs, NULL, NULL); 126 } 127 128 129 static status_t 130 init_device(device_node* node, void** device_cookie) 131 { 132 CALLED(); 133 xhci_pci_sim_info* bus = (xhci_pci_sim_info*)calloc(1, 134 sizeof(xhci_pci_sim_info)); 135 if (bus == NULL) 136 return B_NO_MEMORY; 137 138 pci_device_module_info* pci; 139 pci_device* device; 140 { 141 device_node* pciParent = gDeviceManager->get_parent_node(node); 142 gDeviceManager->get_driver(pciParent, (driver_module_info**)&pci, 143 (void**)&device); 144 gDeviceManager->put_node(pciParent); 145 } 146 147 bus->pci = pci; 148 bus->device = device; 149 bus->driver_node = node; 150 151 pci_info *pciInfo = &bus->pciinfo; 152 pci->get_pci_info(device, pciInfo); 153 154 if (sPCIx86Module == NULL) { 155 if (get_module(B_PCI_X86_MODULE_NAME, (module_info**)&sPCIx86Module) != B_OK) 156 sPCIx86Module = NULL; 157 } 158 159 *device_cookie = bus; 160 return B_OK; 161 } 162 163 164 static void 165 uninit_device(void* device_cookie) 166 { 167 CALLED(); 168 xhci_pci_sim_info* bus = (xhci_pci_sim_info*)device_cookie; 169 free(bus); 170 171 if (sPCIx86Module != NULL) { 172 put_module(B_PCI_X86_MODULE_NAME); 173 sPCIx86Module = NULL; 174 } 175 } 176 177 178 static status_t 179 register_device(device_node* parent) 180 { 181 CALLED(); 182 device_attr attrs[] = { 183 {B_DEVICE_PRETTY_NAME, B_STRING_TYPE, {string: "XHCI PCI"}}, 184 {} 185 }; 186 187 return gDeviceManager->register_node(parent, 188 XHCI_PCI_DEVICE_MODULE_NAME, attrs, NULL, NULL); 189 } 190 191 192 static float 193 supports_device(device_node* parent) 194 { 195 CALLED(); 196 const char* bus; 197 uint16 type, subType, api; 198 199 // make sure parent is a XHCI PCI device node 200 if (gDeviceManager->get_attr_string(parent, B_DEVICE_BUS, &bus, false) 201 < B_OK) { 202 return -1; 203 } 204 205 if (strcmp(bus, "pci") != 0) 206 return 0.0f; 207 208 if (gDeviceManager->get_attr_uint16(parent, B_DEVICE_SUB_TYPE, &subType, 209 false) < B_OK 210 || gDeviceManager->get_attr_uint16(parent, B_DEVICE_TYPE, &type, 211 false) < B_OK 212 || gDeviceManager->get_attr_uint16(parent, B_DEVICE_INTERFACE, &api, 213 false) < B_OK) { 214 TRACE_MODULE("Could not find type/subtype/interface attributes\n"); 215 return -1; 216 } 217 218 if (type == PCI_serial_bus && subType == PCI_usb && api == PCI_usb_xhci) { 219 pci_device_module_info* pci; 220 pci_device* device; 221 gDeviceManager->get_driver(parent, (driver_module_info**)&pci, 222 (void**)&device); 223 TRACE_MODULE("XHCI Device found!\n"); 224 225 return 0.8f; 226 } 227 228 return 0.0f; 229 } 230 231 232 static const char* 233 xhci_error_string(uint32 error) 234 { 235 switch (error) { 236 case COMP_INVALID: return "Invalid"; 237 case COMP_SUCCESS: return "Success"; 238 case COMP_DATA_BUFFER: return "Data buffer"; 239 case COMP_BABBLE: return "Babble detected"; 240 case COMP_USB_TRANSACTION: return "USB transaction"; 241 case COMP_TRB: return "TRB"; 242 case COMP_STALL: return "Stall"; 243 case COMP_RESOURCE: return "Resource"; 244 case COMP_BANDWIDTH: return "Bandwidth"; 245 case COMP_NO_SLOTS: return "No slots"; 246 case COMP_INVALID_STREAM: return "Invalid stream"; 247 case COMP_SLOT_NOT_ENABLED: return "Slot not enabled"; 248 case COMP_ENDPOINT_NOT_ENABLED: return "Endpoint not enabled"; 249 case COMP_SHORT_PACKET: return "Short packet"; 250 case COMP_RING_UNDERRUN: return "Ring underrun"; 251 case COMP_RING_OVERRUN: return "Ring overrun"; 252 case COMP_VF_RING_FULL: return "VF Event Ring Full"; 253 case COMP_PARAMETER: return "Parameter"; 254 case COMP_BANDWIDTH_OVERRUN: return "Bandwidth overrun"; 255 case COMP_CONTEXT_STATE: return "Context state"; 256 case COMP_NO_PING_RESPONSE: return "No ping response"; 257 case COMP_EVENT_RING_FULL: return "Event ring full"; 258 case COMP_INCOMPATIBLE_DEVICE: return "Incompatible device"; 259 case COMP_MISSED_SERVICE: return "Missed service"; 260 case COMP_COMMAND_RING_STOPPED: return "Command ring stopped"; 261 case COMP_COMMAND_ABORTED: return "Command aborted"; 262 case COMP_STOPPED: return "Stopped"; 263 case COMP_LENGTH_INVALID: return "Length invalid"; 264 case COMP_MAX_EXIT_LATENCY: return "Max exit latency too large"; 265 case COMP_ISOC_OVERRUN: return "Isoch buffer overrun"; 266 case COMP_EVENT_LOST: return "Event lost"; 267 case COMP_UNDEFINED: return "Undefined"; 268 case COMP_INVALID_STREAM_ID: return "Invalid stream ID"; 269 case COMP_SECONDARY_BANDWIDTH: return "Secondary bandwidth"; 270 case COMP_SPLIT_TRANSACTION: return "Split transaction"; 271 272 default: return "Undefined"; 273 } 274 } 275 276 277 module_dependency module_dependencies[] = { 278 { USB_FOR_CONTROLLER_MODULE_NAME, (module_info**)&gUSB }, 279 { B_DEVICE_MANAGER_MODULE_NAME, (module_info**)&gDeviceManager }, 280 {} 281 }; 282 283 284 static usb_bus_interface gXHCIPCIDeviceModule = { 285 { 286 { 287 XHCI_PCI_USB_BUS_MODULE_NAME, 288 0, 289 NULL 290 }, 291 NULL, // supports device 292 NULL, // register device 293 init_bus, 294 uninit_bus, 295 NULL, // register child devices 296 NULL, // rescan 297 NULL, // device removed 298 }, 299 }; 300 301 // Root device that binds to the PCI bus. It will register an usb_bus_interface 302 // node for each device. 303 static driver_module_info sXHCIDevice = { 304 { 305 XHCI_PCI_DEVICE_MODULE_NAME, 306 0, 307 NULL 308 }, 309 supports_device, 310 register_device, 311 init_device, 312 uninit_device, 313 register_child_devices, 314 NULL, // rescan 315 NULL, // device removed 316 }; 317 318 module_info* modules[] = { 319 (module_info* )&sXHCIDevice, 320 (module_info* )&gXHCIPCIDeviceModule, 321 NULL 322 }; 323 324 325 XHCI::XHCI(pci_info *info, pci_device_module_info* pci, pci_device* device, Stack *stack, 326 device_node* node) 327 : BusManager(stack, node), 328 fRegisterArea(-1), 329 fRegisters(NULL), 330 fPCIInfo(info), 331 fPci(pci), 332 fDevice(device), 333 fStack(stack), 334 fIRQ(0), 335 fUseMSI(false), 336 fErstArea(-1), 337 fDcbaArea(-1), 338 fCmdCompSem(-1), 339 fStopThreads(false), 340 fRootHub(NULL), 341 fPortCount(0), 342 fSlotCount(0), 343 fScratchpadCount(0), 344 fContextSizeShift(0), 345 fFinishedHead(NULL), 346 fFinishTransfersSem(-1), 347 fFinishThread(-1), 348 fEventSem(-1), 349 fEventThread(-1), 350 fEventIdx(0), 351 fCmdIdx(0), 352 fEventCcs(1), 353 fCmdCcs(1) 354 { 355 B_INITIALIZE_SPINLOCK(&fSpinlock); 356 mutex_init(&fFinishedLock, "XHCI finished transfers"); 357 mutex_init(&fEventLock, "XHCI event handler"); 358 359 if (BusManager::InitCheck() < B_OK) { 360 TRACE_ERROR("bus manager failed to init\n"); 361 return; 362 } 363 364 TRACE("constructing new XHCI host controller driver\n"); 365 fInitOK = false; 366 367 // enable busmaster and memory mapped access 368 uint16 command = fPci->read_pci_config(fDevice, PCI_command, 2); 369 command &= ~(PCI_command_io | PCI_command_int_disable); 370 command |= PCI_command_master | PCI_command_memory; 371 372 fPci->write_pci_config(fDevice, PCI_command, 2, command); 373 374 // map the registers (low + high for 64-bit when requested) 375 phys_addr_t physicalAddress = fPCIInfo->u.h0.base_registers[0]; 376 if ((fPCIInfo->u.h0.base_register_flags[0] & PCI_address_type) 377 == PCI_address_type_64) { 378 physicalAddress |= (uint64)fPCIInfo->u.h0.base_registers[1] << 32; 379 } 380 381 size_t mapSize = fPCIInfo->u.h0.base_register_sizes[0]; 382 383 TRACE("map registers %08" B_PRIxPHYSADDR ", size: %" B_PRIuSIZE "\n", 384 physicalAddress, mapSize); 385 386 fRegisterArea = map_physical_memory("XHCI memory mapped registers", 387 physicalAddress, mapSize, B_ANY_KERNEL_BLOCK_ADDRESS, 388 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 389 (void **)&fRegisters); 390 if (fRegisterArea < B_OK) { 391 TRACE_ERROR("failed to map register memory\n"); 392 return; 393 } 394 395 // determine the register offsets 396 fCapabilityRegisterOffset = 0; 397 fOperationalRegisterOffset = HCI_CAPLENGTH(ReadCapReg32(XHCI_HCI_CAPLENGTH)); 398 fRuntimeRegisterOffset = ReadCapReg32(XHCI_RTSOFF) & ~0x1F; 399 fDoorbellRegisterOffset = ReadCapReg32(XHCI_DBOFF) & ~0x3; 400 401 TRACE("mapped registers: %p\n", fRegisters); 402 TRACE("operational register offset: %" B_PRId32 "\n", fOperationalRegisterOffset); 403 TRACE("runtime register offset: %" B_PRId32 "\n", fRuntimeRegisterOffset); 404 TRACE("doorbell register offset: %" B_PRId32 "\n", fDoorbellRegisterOffset); 405 406 int32 interfaceVersion = HCI_VERSION(ReadCapReg32(XHCI_HCI_VERSION)); 407 if (interfaceVersion < 0x0090 || interfaceVersion > 0x0120) { 408 TRACE_ERROR("unsupported interface version: 0x%04" B_PRIx32 "\n", 409 interfaceVersion); 410 return; 411 } 412 TRACE_ALWAYS("interface version: 0x%04" B_PRIx32 "\n", interfaceVersion); 413 414 TRACE_ALWAYS("structural parameters: 1:0x%08" B_PRIx32 " 2:0x%08" 415 B_PRIx32 " 3:0x%08" B_PRIx32 "\n", ReadCapReg32(XHCI_HCSPARAMS1), 416 ReadCapReg32(XHCI_HCSPARAMS2), ReadCapReg32(XHCI_HCSPARAMS3)); 417 418 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 419 if (cparams == 0xffffffff) 420 return; 421 TRACE_ALWAYS("capability parameters: 0x%08" B_PRIx32 "\n", cparams); 422 423 // if 64 bytes context structures, then 1 424 fContextSizeShift = HCC_CSZ(cparams); 425 426 // Assume ownership of the controller from the BIOS. 427 uint32 eec = 0xffffffff; 428 uint32 eecp = HCS0_XECP(cparams) << 2; 429 for (; eecp != 0 && XECP_NEXT(eec); eecp += XECP_NEXT(eec) << 2) { 430 TRACE("eecp register: 0x%08" B_PRIx32 "\n", eecp); 431 432 eec = ReadCapReg32(eecp); 433 if (XECP_ID(eec) != XHCI_LEGSUP_CAPID) 434 continue; 435 436 if (eec & XHCI_LEGSUP_BIOSOWNED) { 437 TRACE_ALWAYS("the host controller is bios owned, claiming" 438 " ownership\n"); 439 WriteCapReg32(eecp, eec | XHCI_LEGSUP_OSOWNED); 440 441 for (int32 i = 0; i < 20; i++) { 442 eec = ReadCapReg32(eecp); 443 444 if ((eec & XHCI_LEGSUP_BIOSOWNED) == 0) 445 break; 446 447 TRACE_ALWAYS("controller is still bios owned, waiting\n"); 448 snooze(50000); 449 } 450 451 if (eec & XHCI_LEGSUP_BIOSOWNED) { 452 TRACE_ERROR("bios won't give up control over the host " 453 "controller (ignoring)\n"); 454 } else if (eec & XHCI_LEGSUP_OSOWNED) { 455 TRACE_ALWAYS("successfully took ownership of the host " 456 "controller\n"); 457 } 458 459 // Force off the BIOS owned flag, and clear all SMIs. Some BIOSes 460 // do indicate a successful handover but do not remove their SMIs 461 // and then freeze the system when interrupts are generated. 462 WriteCapReg32(eecp, eec & ~XHCI_LEGSUP_BIOSOWNED); 463 } 464 break; 465 } 466 uint32 legctlsts = ReadCapReg32(eecp + XHCI_LEGCTLSTS); 467 legctlsts &= XHCI_LEGCTLSTS_DISABLE_SMI; 468 legctlsts |= XHCI_LEGCTLSTS_EVENTS_SMI; 469 WriteCapReg32(eecp + XHCI_LEGCTLSTS, legctlsts); 470 471 // We need to explicitly take ownership of EHCI ports on earlier Intel chipsets. 472 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL) { 473 switch (fPCIInfo->device_id) { 474 case PCI_DEVICE_INTEL_PANTHER_POINT_XHCI: 475 case PCI_DEVICE_INTEL_LYNX_POINT_XHCI: 476 case PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI: 477 case PCI_DEVICE_INTEL_BAYTRAIL_XHCI: 478 case PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI: 479 case PCI_DEVICE_INTEL_WILDCAT_POINT_LP_XHCI: 480 _SwitchIntelPorts(); 481 break; 482 } 483 } 484 485 // halt the host controller 486 if (ControllerHalt() < B_OK) { 487 return; 488 } 489 490 // reset the host controller 491 if (ControllerReset() < B_OK) { 492 TRACE_ERROR("host controller failed to reset\n"); 493 return; 494 } 495 496 fCmdCompSem = create_sem(0, "XHCI Command Complete"); 497 fFinishTransfersSem = create_sem(0, "XHCI Finish Transfers"); 498 fEventSem = create_sem(0, "XHCI Event"); 499 if (fFinishTransfersSem < B_OK || fCmdCompSem < B_OK || fEventSem < B_OK) { 500 TRACE_ERROR("failed to create semaphores\n"); 501 return; 502 } 503 504 // create event handler thread 505 fEventThread = spawn_kernel_thread(EventThread, "xhci event thread", 506 B_URGENT_PRIORITY, (void *)this); 507 resume_thread(fEventThread); 508 509 // create finisher service thread 510 fFinishThread = spawn_kernel_thread(FinishThread, "xhci finish thread", 511 B_URGENT_PRIORITY - 1, (void *)this); 512 resume_thread(fFinishThread); 513 514 // Find the right interrupt vector, using MSIs if available. 515 fIRQ = fPCIInfo->u.h0.interrupt_line; 516 if (sPCIx86Module != NULL && sPCIx86Module->get_msi_count(fPCIInfo->bus, 517 fPCIInfo->device, fPCIInfo->function) >= 1) { 518 uint8 msiVector = 0; 519 if (sPCIx86Module->configure_msi(fPCIInfo->bus, fPCIInfo->device, 520 fPCIInfo->function, 1, &msiVector) == B_OK 521 && sPCIx86Module->enable_msi(fPCIInfo->bus, fPCIInfo->device, 522 fPCIInfo->function) == B_OK) { 523 TRACE_ALWAYS("using message signaled interrupts\n"); 524 fIRQ = msiVector; 525 fUseMSI = true; 526 } 527 } 528 529 if (fIRQ == 0 || fIRQ == 0xFF) { 530 TRACE_MODULE_ERROR("device PCI:%d:%d:%d was assigned an invalid IRQ\n", 531 fPCIInfo->bus, fPCIInfo->device, fPCIInfo->function); 532 return; 533 } 534 535 // Install the interrupt handler 536 TRACE("installing interrupt handler\n"); 537 install_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this, 0); 538 539 memset(fPortSpeeds, 0, sizeof(fPortSpeeds)); 540 memset(fDevices, 0, sizeof(fDevices)); 541 542 fInitOK = true; 543 TRACE("driver construction successful\n"); 544 } 545 546 547 XHCI::~XHCI() 548 { 549 TRACE("tear down XHCI host controller driver\n"); 550 551 WriteOpReg(XHCI_CMD, 0); 552 553 int32 result = 0; 554 fStopThreads = true; 555 delete_sem(fCmdCompSem); 556 delete_sem(fFinishTransfersSem); 557 delete_sem(fEventSem); 558 wait_for_thread(fFinishThread, &result); 559 wait_for_thread(fEventThread, &result); 560 561 mutex_destroy(&fFinishedLock); 562 mutex_destroy(&fEventLock); 563 564 remove_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this); 565 566 delete_area(fRegisterArea); 567 delete_area(fErstArea); 568 for (uint32 i = 0; i < fScratchpadCount; i++) 569 delete_area(fScratchpadArea[i]); 570 delete_area(fDcbaArea); 571 572 if (fUseMSI && sPCIx86Module != NULL) { 573 sPCIx86Module->disable_msi(fPCIInfo->bus, 574 fPCIInfo->device, fPCIInfo->function); 575 sPCIx86Module->unconfigure_msi(fPCIInfo->bus, 576 fPCIInfo->device, fPCIInfo->function); 577 } 578 } 579 580 581 void 582 XHCI::_SwitchIntelPorts() 583 { 584 TRACE("Looking for EHCI owned ports\n"); 585 uint32 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_USB3PRM, 4); 586 TRACE("Superspeed Ports: 0x%" B_PRIx32 "\n", ports); 587 fPci->write_pci_config(fDevice, XHCI_INTEL_USB3_PSSEN, 4, ports); 588 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_USB3_PSSEN, 4); 589 TRACE("Superspeed ports now under XHCI : 0x%" B_PRIx32 "\n", ports); 590 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_USB2PRM, 4); 591 TRACE("USB 2.0 Ports : 0x%" B_PRIx32 "\n", ports); 592 fPci->write_pci_config(fDevice, XHCI_INTEL_XUSB2PR, 4, ports); 593 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_XUSB2PR, 4); 594 TRACE("USB 2.0 ports now under XHCI: 0x%" B_PRIx32 "\n", ports); 595 } 596 597 598 status_t 599 XHCI::Start() 600 { 601 TRACE_ALWAYS("starting XHCI host controller\n"); 602 TRACE("usbcmd: 0x%08" B_PRIx32 "; usbsts: 0x%08" B_PRIx32 "\n", 603 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 604 605 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 606 TRACE("Start() failed STS_CNR\n"); 607 } 608 609 if ((ReadOpReg(XHCI_CMD) & CMD_RUN) != 0) { 610 TRACE_ERROR("Start() warning, starting running XHCI controller!\n"); 611 } 612 613 if ((ReadOpReg(XHCI_PAGESIZE) & (1 << 0)) == 0) { 614 TRACE_ERROR("controller does not support 4K page size\n"); 615 return B_ERROR; 616 } 617 618 // read port count from capability register 619 uint32 capabilities = ReadCapReg32(XHCI_HCSPARAMS1); 620 fPortCount = HCS_MAX_PORTS(capabilities); 621 if (fPortCount == 0) { 622 TRACE_ERROR("invalid number of ports: %u\n", fPortCount); 623 return B_ERROR; 624 } 625 626 fSlotCount = HCS_MAX_SLOTS(capabilities); 627 if (fSlotCount > XHCI_MAX_DEVICES) 628 fSlotCount = XHCI_MAX_DEVICES; 629 WriteOpReg(XHCI_CONFIG, fSlotCount); 630 631 // find out which protocol is used for each port 632 uint8 portFound = 0; 633 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 634 uint32 eec = 0xffffffff; 635 uint32 eecp = HCS0_XECP(cparams) << 2; 636 for (; eecp != 0 && XECP_NEXT(eec) && portFound < fPortCount; 637 eecp += XECP_NEXT(eec) << 2) { 638 eec = ReadCapReg32(eecp); 639 if (XECP_ID(eec) != XHCI_SUPPORTED_PROTOCOLS_CAPID) 640 continue; 641 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) > 3) 642 continue; 643 uint32 temp = ReadCapReg32(eecp + 8); 644 uint32 offset = XHCI_SUPPORTED_PROTOCOLS_1_OFFSET(temp); 645 uint32 count = XHCI_SUPPORTED_PROTOCOLS_1_COUNT(temp); 646 if (offset == 0 || count == 0) 647 continue; 648 offset--; 649 for (uint32 i = offset; i < offset + count; i++) { 650 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) == 0x3) 651 fPortSpeeds[i] = USB_SPEED_SUPERSPEED; 652 else 653 fPortSpeeds[i] = USB_SPEED_HIGHSPEED; 654 655 TRACE("speed for port %" B_PRId32 " is %s\n", i, 656 fPortSpeeds[i] == USB_SPEED_SUPERSPEED ? "super" : "high"); 657 } 658 portFound += count; 659 } 660 661 uint32 params2 = ReadCapReg32(XHCI_HCSPARAMS2); 662 fScratchpadCount = HCS_MAX_SC_BUFFERS(params2); 663 if (fScratchpadCount > XHCI_MAX_SCRATCHPADS) { 664 TRACE_ERROR("invalid number of scratchpads: %" B_PRIu32 "\n", 665 fScratchpadCount); 666 return B_ERROR; 667 } 668 669 uint32 params3 = ReadCapReg32(XHCI_HCSPARAMS3); 670 fExitLatMax = HCS_U1_DEVICE_LATENCY(params3) 671 + HCS_U2_DEVICE_LATENCY(params3); 672 673 // clear interrupts & disable device notifications 674 WriteOpReg(XHCI_STS, ReadOpReg(XHCI_STS)); 675 WriteOpReg(XHCI_DNCTRL, 0); 676 677 // allocate Device Context Base Address array 678 phys_addr_t dmaAddress; 679 fDcbaArea = fStack->AllocateArea((void **)&fDcba, &dmaAddress, 680 sizeof(*fDcba), "DCBA Area"); 681 if (fDcbaArea < B_OK) { 682 TRACE_ERROR("unable to create the DCBA area\n"); 683 return B_ERROR; 684 } 685 memset(fDcba, 0, sizeof(*fDcba)); 686 memset(fScratchpadArea, 0, sizeof(fScratchpadArea)); 687 memset(fScratchpad, 0, sizeof(fScratchpad)); 688 689 // setting the first address to the scratchpad array address 690 fDcba->baseAddress[0] = dmaAddress 691 + offsetof(struct xhci_device_context_array, scratchpad); 692 693 // fill up the scratchpad array with scratchpad pages 694 for (uint32 i = 0; i < fScratchpadCount; i++) { 695 phys_addr_t scratchDmaAddress; 696 fScratchpadArea[i] = fStack->AllocateArea((void **)&fScratchpad[i], 697 &scratchDmaAddress, B_PAGE_SIZE, "Scratchpad Area"); 698 if (fScratchpadArea[i] < B_OK) { 699 TRACE_ERROR("unable to create the scratchpad area\n"); 700 return B_ERROR; 701 } 702 fDcba->scratchpad[i] = scratchDmaAddress; 703 } 704 705 TRACE("setting DCBAAP %" B_PRIxPHYSADDR "\n", dmaAddress); 706 WriteOpReg(XHCI_DCBAAP_LO, (uint32)dmaAddress); 707 WriteOpReg(XHCI_DCBAAP_HI, (uint32)(dmaAddress >> 32)); 708 709 // allocate Event Ring Segment Table 710 uint8 *addr; 711 fErstArea = fStack->AllocateArea((void **)&addr, &dmaAddress, 712 (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 713 + sizeof(xhci_erst_element), 714 "USB XHCI ERST CMD_RING and EVENT_RING Area"); 715 716 if (fErstArea < B_OK) { 717 TRACE_ERROR("unable to create the ERST AND RING area\n"); 718 delete_area(fDcbaArea); 719 return B_ERROR; 720 } 721 fErst = (xhci_erst_element *)addr; 722 memset(fErst, 0, (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 723 + sizeof(xhci_erst_element)); 724 725 // fill with Event Ring Segment Base Address and Event Ring Segment Size 726 fErst->rs_addr = dmaAddress + sizeof(xhci_erst_element); 727 fErst->rs_size = XHCI_MAX_EVENTS; 728 fErst->rsvdz = 0; 729 730 addr += sizeof(xhci_erst_element); 731 fEventRing = (xhci_trb *)addr; 732 addr += XHCI_MAX_EVENTS * sizeof(xhci_trb); 733 fCmdRing = (xhci_trb *)addr; 734 735 TRACE("setting ERST size\n"); 736 WriteRunReg32(XHCI_ERSTSZ(0), XHCI_ERSTS_SET(1)); 737 738 TRACE("setting ERDP addr = 0x%" B_PRIx64 "\n", fErst->rs_addr); 739 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)fErst->rs_addr); 740 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(fErst->rs_addr >> 32)); 741 742 TRACE("setting ERST base addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 743 WriteRunReg32(XHCI_ERSTBA_LO(0), (uint32)dmaAddress); 744 WriteRunReg32(XHCI_ERSTBA_HI(0), (uint32)(dmaAddress >> 32)); 745 746 dmaAddress += sizeof(xhci_erst_element) + XHCI_MAX_EVENTS 747 * sizeof(xhci_trb); 748 749 // Make sure the Command Ring is stopped 750 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 751 TRACE_ALWAYS("Command Ring is running, send stop/cancel\n"); 752 WriteOpReg(XHCI_CRCR_LO, CRCR_CS); 753 WriteOpReg(XHCI_CRCR_HI, 0); 754 WriteOpReg(XHCI_CRCR_LO, CRCR_CA); 755 WriteOpReg(XHCI_CRCR_HI, 0); 756 snooze(1000); 757 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 758 TRACE_ERROR("Command Ring still running after stop/cancel\n"); 759 } 760 } 761 TRACE("setting CRCR addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 762 WriteOpReg(XHCI_CRCR_LO, (uint32)dmaAddress | CRCR_RCS); 763 WriteOpReg(XHCI_CRCR_HI, (uint32)(dmaAddress >> 32)); 764 // link trb 765 fCmdRing[XHCI_MAX_COMMANDS - 1].address = dmaAddress; 766 767 TRACE("setting interrupt rate\n"); 768 769 // Setting IMOD below 0x3F8 on Intel Lynx Point can cause IRQ lockups 770 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL 771 && (fPCIInfo->device_id == PCI_DEVICE_INTEL_PANTHER_POINT_XHCI 772 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_XHCI 773 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI 774 || fPCIInfo->device_id == PCI_DEVICE_INTEL_BAYTRAIL_XHCI 775 || fPCIInfo->device_id == PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI)) { 776 WriteRunReg32(XHCI_IMOD(0), 0x000003f8); // 4000 irq/s 777 } else { 778 WriteRunReg32(XHCI_IMOD(0), 0x000001f4); // 8000 irq/s 779 } 780 781 TRACE("enabling interrupt\n"); 782 WriteRunReg32(XHCI_IMAN(0), ReadRunReg32(XHCI_IMAN(0)) | IMAN_INTR_ENA); 783 784 WriteOpReg(XHCI_CMD, CMD_RUN | CMD_INTE | CMD_HSEE); 785 786 // wait for start up state 787 if (WaitOpBits(XHCI_STS, STS_HCH, 0) != B_OK) { 788 TRACE_ERROR("HCH start up timeout\n"); 789 } 790 791 fRootHub = new(std::nothrow) XHCIRootHub(RootObject(), 1); 792 if (!fRootHub) { 793 TRACE_ERROR("no memory to allocate root hub\n"); 794 return B_NO_MEMORY; 795 } 796 797 if (fRootHub->InitCheck() < B_OK) { 798 TRACE_ERROR("root hub failed init check\n"); 799 return fRootHub->InitCheck(); 800 } 801 802 SetRootHub(fRootHub); 803 804 fRootHub->RegisterNode(Node()); 805 806 TRACE_ALWAYS("successfully started the controller\n"); 807 808 #ifdef TRACE_USB 809 TRACE("No-Op test...\n"); 810 Noop(); 811 #endif 812 813 return BusManager::Start(); 814 } 815 816 817 status_t 818 XHCI::SubmitTransfer(Transfer *transfer) 819 { 820 // short circuit the root hub 821 if (transfer->TransferPipe()->DeviceAddress() == 1) 822 return fRootHub->ProcessTransfer(this, transfer); 823 824 TRACE("SubmitTransfer(%p)\n", transfer); 825 Pipe *pipe = transfer->TransferPipe(); 826 if ((pipe->Type() & USB_OBJECT_CONTROL_PIPE) != 0) 827 return SubmitControlRequest(transfer); 828 return SubmitNormalRequest(transfer); 829 } 830 831 832 status_t 833 XHCI::SubmitControlRequest(Transfer *transfer) 834 { 835 Pipe *pipe = transfer->TransferPipe(); 836 usb_request_data *requestData = transfer->RequestData(); 837 bool directionIn = (requestData->RequestType & USB_REQTYPE_DEVICE_IN) != 0; 838 839 TRACE("SubmitControlRequest() length %d\n", requestData->Length); 840 841 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 842 if (endpoint == NULL) { 843 TRACE_ERROR("control pipe has no endpoint!\n"); 844 return B_BAD_VALUE; 845 } 846 if (endpoint->device == NULL) { 847 panic("endpoint is not initialized!"); 848 return B_NO_INIT; 849 } 850 851 status_t status = transfer->InitKernelAccess(); 852 if (status != B_OK) 853 return status; 854 855 xhci_td *descriptor = CreateDescriptor(3, 1, requestData->Length); 856 if (descriptor == NULL) 857 return B_NO_MEMORY; 858 descriptor->transfer = transfer; 859 860 // Setup Stage 861 uint8 index = 0; 862 memcpy(&descriptor->trbs[index].address, requestData, 863 sizeof(usb_request_data)); 864 descriptor->trbs[index].status = TRB_2_IRQ(0) | TRB_2_BYTES(8); 865 descriptor->trbs[index].flags 866 = TRB_3_TYPE(TRB_TYPE_SETUP_STAGE) | TRB_3_IDT_BIT | TRB_3_CYCLE_BIT; 867 if (requestData->Length > 0) { 868 descriptor->trbs[index].flags |= 869 directionIn ? TRB_3_TRT_IN : TRB_3_TRT_OUT; 870 } 871 872 index++; 873 874 // Data Stage (if any) 875 if (requestData->Length > 0) { 876 descriptor->trbs[index].address = descriptor->buffer_addrs[0]; 877 descriptor->trbs[index].status = TRB_2_IRQ(0) 878 | TRB_2_BYTES(requestData->Length) 879 | TRB_2_TD_SIZE(0); 880 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_DATA_STAGE) 881 | (directionIn ? TRB_3_DIR_IN : 0) 882 | TRB_3_CYCLE_BIT; 883 884 if (!directionIn) { 885 transfer->PrepareKernelAccess(); 886 memcpy(descriptor->buffers[0], 887 (uint8 *)transfer->Vector()[0].iov_base, requestData->Length); 888 } 889 890 index++; 891 } 892 893 // Status Stage 894 descriptor->trbs[index].address = 0; 895 descriptor->trbs[index].status = TRB_2_IRQ(0); 896 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_STATUS_STAGE) 897 | TRB_3_CHAIN_BIT | TRB_3_ENT_BIT | TRB_3_CYCLE_BIT; 898 // The CHAIN bit must be set when using an Event Data TRB 899 // (XHCI 1.2 § 6.4.1.2.3 Table 6-31 p472). 900 901 // Status Stage is an OUT transfer when the device is sending data 902 // (XHCI 1.2 § 4.11.2.2 Table 4-7 p213), otherwise set the IN bit. 903 if (requestData->Length == 0 || !directionIn) 904 descriptor->trbs[index].flags |= TRB_3_DIR_IN; 905 906 descriptor->trb_used = index + 1; 907 908 status = _LinkDescriptorForPipe(descriptor, endpoint); 909 if (status != B_OK) { 910 FreeDescriptor(descriptor); 911 return status; 912 } 913 914 return B_OK; 915 } 916 917 918 status_t 919 XHCI::SubmitNormalRequest(Transfer *transfer) 920 { 921 TRACE("SubmitNormalRequest() length %" B_PRIuSIZE "\n", transfer->FragmentLength()); 922 923 Pipe *pipe = transfer->TransferPipe(); 924 usb_isochronous_data *isochronousData = transfer->IsochronousData(); 925 bool directionIn = (pipe->Direction() == Pipe::In); 926 927 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 928 if (endpoint == NULL) { 929 TRACE_ERROR("pipe has no endpoint!\n"); 930 return B_BAD_VALUE; 931 } 932 if (endpoint->device == NULL) { 933 panic("endpoint is not initialized!"); 934 return B_NO_INIT; 935 } 936 937 status_t status = transfer->InitKernelAccess(); 938 if (status != B_OK) 939 return status; 940 941 // TRBs within a TD must be "grouped" into TD Fragments, which mostly means 942 // that a max_burst_payload boundary cannot be crossed within a TRB, but 943 // only between TRBs. More than one TRB can be in a TD Fragment, but we keep 944 // things simple by setting trbSize to the MBP. (XHCI 1.2 § 4.11.7.1 p235.) 945 size_t trbSize = endpoint->max_burst_payload; 946 947 if (isochronousData != NULL) { 948 if (isochronousData->packet_count == 0) 949 return B_BAD_VALUE; 950 951 // Isochronous transfers use more specifically sized packets. 952 trbSize = transfer->DataLength() / isochronousData->packet_count; 953 if (trbSize == 0 || trbSize > pipe->MaxPacketSize() || trbSize 954 != (size_t)isochronousData->packet_descriptors[0].request_length) 955 return B_BAD_VALUE; 956 } 957 958 // Now that we know trbSize, compute the count. 959 const int32 trbCount = (transfer->FragmentLength() + trbSize - 1) / trbSize; 960 961 xhci_td *td = CreateDescriptor(trbCount, trbCount, trbSize); 962 if (td == NULL) 963 return B_NO_MEMORY; 964 965 // Normal Stage 966 const size_t maxPacketSize = pipe->MaxPacketSize(); 967 size_t remaining = transfer->FragmentLength(); 968 for (int32 i = 0; i < trbCount; i++) { 969 int32 trbLength = (remaining < trbSize) ? remaining : trbSize; 970 remaining -= trbLength; 971 972 // The "TD Size" field of a transfer TRB indicates the number of 973 // remaining maximum-size *packets* in this TD, *not* including the 974 // packets in the current TRB, and capped at 31 if there are more 975 // than 31 packets remaining in the TD. (XHCI 1.2 § 4.11.2.4 p218.) 976 int32 tdSize = (remaining + maxPacketSize - 1) / maxPacketSize; 977 if (tdSize > 31) 978 tdSize = 31; 979 980 td->trbs[i].address = td->buffer_addrs[i]; 981 td->trbs[i].status = TRB_2_IRQ(0) 982 | TRB_2_BYTES(trbLength) 983 | TRB_2_TD_SIZE(tdSize); 984 td->trbs[i].flags = TRB_3_TYPE(TRB_TYPE_NORMAL) 985 | TRB_3_CYCLE_BIT | TRB_3_CHAIN_BIT; 986 987 td->trb_used++; 988 } 989 990 // Isochronous-specific 991 if (isochronousData != NULL) { 992 // This is an isochronous transfer; we need to make the first TRB 993 // an isochronous TRB. 994 td->trbs[0].flags &= ~(TRB_3_TYPE(TRB_TYPE_NORMAL)); 995 td->trbs[0].flags |= TRB_3_TYPE(TRB_TYPE_ISOCH); 996 997 // Isochronous pipes are scheduled by microframes, one of which 998 // is 125us for USB 2 and above. But for USB 1 it was 1ms, so 999 // we need to use a different frame delta for that case. 1000 uint8 frameDelta = 1; 1001 if (transfer->TransferPipe()->Speed() == USB_SPEED_FULLSPEED) 1002 frameDelta = 8; 1003 1004 // TODO: We do not currently take Mult into account at all! 1005 // How are we supposed to do that here? 1006 1007 // Determine the (starting) frame number: if ISO_ASAP is set, 1008 // we are queueing this "right away", and so want to reset 1009 // the starting_frame_number. Otherwise we use the passed one. 1010 uint32 frame; 1011 if ((isochronousData->flags & USB_ISO_ASAP) != 0 1012 || isochronousData->starting_frame_number == NULL) { 1013 // All reads from the microframe index register must be 1014 // incremented by 1. (XHCI 1.2 § 4.14.2.1.4 p265.) 1015 frame = ReadRunReg32(XHCI_MFINDEX) + 1; 1016 td->trbs[0].flags |= TRB_3_ISO_SIA_BIT; 1017 } else { 1018 frame = *isochronousData->starting_frame_number; 1019 td->trbs[0].flags |= TRB_3_FRID(frame); 1020 } 1021 frame = (frame + frameDelta) % 2048; 1022 if (isochronousData->starting_frame_number != NULL) 1023 *isochronousData->starting_frame_number = frame; 1024 1025 // TODO: The OHCI bus driver seems to also do this for inbound 1026 // isochronous transfers. Perhaps it should be moved into the stack? 1027 if (directionIn) { 1028 for (uint32 i = 0; i < isochronousData->packet_count; i++) { 1029 isochronousData->packet_descriptors[i].actual_length = 0; 1030 isochronousData->packet_descriptors[i].status = B_NO_INIT; 1031 } 1032 } 1033 } 1034 1035 // Set the ENT (Evaluate Next TRB) bit, so that the HC will not switch 1036 // contexts before evaluating the Link TRB that _LinkDescriptorForPipe 1037 // will insert, as otherwise there would be a race between us freeing 1038 // and unlinking the descriptor, and the controller evaluating the Link TRB 1039 // and thus getting back onto the main ring and executing the Event Data 1040 // TRB that generates the interrupt for this transfer. 1041 // 1042 // Note that we *do not* unset the CHAIN bit in this TRB, thus including 1043 // the Link TRB in this TD formally, which is required when using the 1044 // ENT bit. (XHCI 1.2 § 4.12.3 p250.) 1045 td->trbs[td->trb_used - 1].flags |= TRB_3_ENT_BIT; 1046 1047 if (!directionIn) { 1048 TRACE("copying out iov count %ld\n", transfer->VectorCount()); 1049 status_t status = transfer->PrepareKernelAccess(); 1050 if (status != B_OK) { 1051 FreeDescriptor(td); 1052 return status; 1053 } 1054 WriteDescriptor(td, transfer->Vector(), transfer->VectorCount()); 1055 } 1056 1057 td->transfer = transfer; 1058 status = _LinkDescriptorForPipe(td, endpoint); 1059 if (status != B_OK) { 1060 FreeDescriptor(td); 1061 return status; 1062 } 1063 1064 return B_OK; 1065 } 1066 1067 1068 status_t 1069 XHCI::CancelQueuedTransfers(Pipe *pipe, bool force) 1070 { 1071 xhci_endpoint* endpoint = (xhci_endpoint*)pipe->ControllerCookie(); 1072 if (endpoint == NULL || endpoint->trbs == NULL) { 1073 // Someone's de-allocated this pipe or endpoint in the meantime. 1074 // (Possibly AllocateDevice failed, and we were the temporary pipe.) 1075 return B_NO_INIT; 1076 } 1077 1078 #ifndef TRACE_USB 1079 if (force) 1080 #endif 1081 { 1082 TRACE_ALWAYS("cancel queued transfers (%" B_PRId8 ") for pipe %p (%d)\n", 1083 endpoint->used, pipe, pipe->EndpointAddress()); 1084 } 1085 1086 MutexLocker endpointLocker(endpoint->lock); 1087 1088 if (endpoint->td_head == NULL) { 1089 // There aren't any currently pending transfers to cancel. 1090 return B_OK; 1091 } 1092 1093 // Calling the callbacks while holding the endpoint lock could potentially 1094 // cause deadlocks, so we instead store them in a pointer array. We need 1095 // to do this separately from freeing the TDs, for in the case we fail 1096 // to stop the endpoint, we cancel the transfers but do not free the TDs. 1097 Transfer* transfers[XHCI_MAX_TRANSFERS]; 1098 int32 transfersCount = 0; 1099 1100 for (xhci_td* td = endpoint->td_head; td != NULL; td = td->next) { 1101 if (td->transfer == NULL) 1102 continue; 1103 1104 // We can't cancel or delete transfers under "force", as they probably 1105 // are not safe to use anymore. 1106 if (!force) { 1107 transfers[transfersCount] = td->transfer; 1108 transfersCount++; 1109 } 1110 td->transfer = NULL; 1111 } 1112 1113 // It is possible that while waiting for the stop-endpoint command to 1114 // complete, one of the queued transfers posts a completion event, so in 1115 // order to avoid a deadlock, we must unlock the endpoint. 1116 endpointLocker.Unlock(); 1117 status_t status = StopEndpoint(false, endpoint); 1118 if (status == B_DEV_STALLED) { 1119 // Only exit from a Halted state is a reset. (XHCI 1.2 § 4.8.3 p163.) 1120 TRACE_ERROR("cancel queued transfers: halted endpoint, reset!\n"); 1121 status = ResetEndpoint(false, endpoint); 1122 } 1123 endpointLocker.Lock(); 1124 1125 // Detach the head TD from the endpoint. 1126 xhci_td* td_head = endpoint->td_head; 1127 endpoint->td_head = NULL; 1128 1129 if (status == B_OK) { 1130 // Clear the endpoint's TRBs. 1131 memset(endpoint->trbs, 0, sizeof(xhci_trb) * XHCI_ENDPOINT_RING_SIZE); 1132 endpoint->used = 0; 1133 endpoint->current = 0; 1134 1135 // Set dequeue pointer location to the beginning of the ring. 1136 SetTRDequeue(endpoint->trb_addr, 0, endpoint->id + 1, 1137 endpoint->device->slot); 1138 1139 // We don't need to do anything else to restart the ring, as it will resume 1140 // operation as normal upon the next doorbell. (XHCI 1.2 § 4.6.9 p136.) 1141 } else { 1142 // We couldn't stop the endpoint. Most likely the device has been 1143 // removed and the endpoint was stopped by the hardware, or is 1144 // for some reason busy and cannot be stopped. 1145 TRACE_ERROR("cancel queued transfers: could not stop endpoint: %s!\n", 1146 strerror(status)); 1147 1148 // Instead of freeing the TDs, we want to leave them in the endpoint 1149 // so that when/if the hardware returns, they can be properly unlinked, 1150 // as otherwise the endpoint could get "stuck" by having the "used" 1151 // slowly accumulate due to "dead" transfers. 1152 endpoint->td_head = td_head; 1153 td_head = NULL; 1154 } 1155 1156 endpointLocker.Unlock(); 1157 1158 for (int32 i = 0; i < transfersCount; i++) { 1159 transfers[i]->Finished(B_CANCELED, 0); 1160 delete transfers[i]; 1161 } 1162 1163 // This loop looks a bit strange because we need to store the "next" 1164 // pointer before freeing the descriptor. 1165 xhci_td* td; 1166 while ((td = td_head) != NULL) { 1167 td_head = td_head->next; 1168 FreeDescriptor(td); 1169 } 1170 1171 return B_OK; 1172 } 1173 1174 1175 status_t 1176 XHCI::StartDebugTransfer(Transfer *transfer) 1177 { 1178 Pipe *pipe = transfer->TransferPipe(); 1179 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 1180 if (endpoint == NULL) 1181 return B_BAD_VALUE; 1182 1183 // Check all locks that we are going to hit when running transfers. 1184 if (mutex_trylock(&endpoint->lock) != B_OK) 1185 return B_WOULD_BLOCK; 1186 if (mutex_trylock(&fFinishedLock) != B_OK) { 1187 mutex_unlock(&endpoint->lock); 1188 return B_WOULD_BLOCK; 1189 } 1190 if (mutex_trylock(&fEventLock) != B_OK) { 1191 mutex_unlock(&endpoint->lock); 1192 mutex_unlock(&fFinishedLock); 1193 return B_WOULD_BLOCK; 1194 } 1195 mutex_unlock(&endpoint->lock); 1196 mutex_unlock(&fFinishedLock); 1197 mutex_unlock(&fEventLock); 1198 1199 status_t status = SubmitTransfer(transfer); 1200 if (status != B_OK) 1201 return status; 1202 1203 // The endpoint's head TD is the TD of the just-submitted transfer. 1204 // Just like EHCI, abuse the callback cookie to hold the TD pointer. 1205 transfer->SetCallback(NULL, endpoint->td_head); 1206 1207 return B_OK; 1208 } 1209 1210 1211 status_t 1212 XHCI::CheckDebugTransfer(Transfer *transfer) 1213 { 1214 xhci_td *transfer_td = (xhci_td *)transfer->CallbackCookie(); 1215 if (transfer_td == NULL) 1216 return B_NO_INIT; 1217 1218 // Process events once, and then look for it in the finished list. 1219 ProcessEvents(); 1220 xhci_td *previous = NULL; 1221 for (xhci_td *td = fFinishedHead; td != NULL; td = td->next) { 1222 if (td != transfer_td) { 1223 previous = td; 1224 continue; 1225 } 1226 1227 // We've found it! 1228 if (previous == NULL) { 1229 fFinishedHead = fFinishedHead->next; 1230 } else { 1231 previous->next = td->next; 1232 } 1233 1234 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 1235 status_t status = (td->trb_completion_code == COMP_SUCCESS 1236 || td->trb_completion_code == COMP_SHORT_PACKET) ? B_OK : B_ERROR; 1237 1238 if (status == B_OK && directionIn) 1239 ReadDescriptor(td, transfer->Vector(), transfer->VectorCount()); 1240 1241 FreeDescriptor(td); 1242 transfer->SetCallback(NULL, NULL); 1243 return status; 1244 } 1245 1246 // We didn't find it. 1247 spin(75); 1248 return B_DEV_PENDING; 1249 } 1250 1251 1252 void 1253 XHCI::CancelDebugTransfer(Transfer *transfer) 1254 { 1255 while (CheckDebugTransfer(transfer) == B_DEV_PENDING) 1256 spin(100); 1257 } 1258 1259 1260 status_t 1261 XHCI::NotifyPipeChange(Pipe *pipe, usb_change change) 1262 { 1263 TRACE("pipe change %d for pipe %p (%d)\n", change, pipe, 1264 pipe->EndpointAddress()); 1265 1266 switch (change) { 1267 case USB_CHANGE_CREATED: 1268 return _InsertEndpointForPipe(pipe); 1269 case USB_CHANGE_DESTROYED: 1270 return _RemoveEndpointForPipe(pipe); 1271 1272 case USB_CHANGE_PIPE_POLICY_CHANGED: 1273 // We don't care about these, at least for now. 1274 return B_OK; 1275 } 1276 1277 TRACE_ERROR("unknown pipe change!\n"); 1278 return B_UNSUPPORTED; 1279 } 1280 1281 1282 xhci_td * 1283 XHCI::CreateDescriptor(uint32 trbCount, uint32 bufferCount, size_t bufferSize) 1284 { 1285 const bool inKDL = debug_debugger_running(); 1286 1287 xhci_td *result; 1288 if (!inKDL) { 1289 result = (xhci_td*)calloc(1, sizeof(xhci_td)); 1290 } else { 1291 // Just use the physical memory allocator while in KDL; it's less 1292 // secure than using the regular heap, but it's easier to deal with. 1293 phys_addr_t dummy; 1294 fStack->AllocateChunk((void **)&result, &dummy, sizeof(xhci_td)); 1295 } 1296 1297 if (result == NULL) { 1298 TRACE_ERROR("failed to allocate a transfer descriptor\n"); 1299 return NULL; 1300 } 1301 1302 // We always allocate 1 more TRB than requested, so that 1303 // _LinkDescriptorForPipe() has room to insert a link TRB. 1304 trbCount++; 1305 if (fStack->AllocateChunk((void **)&result->trbs, &result->trb_addr, 1306 (trbCount * sizeof(xhci_trb))) < B_OK) { 1307 TRACE_ERROR("failed to allocate TRBs\n"); 1308 FreeDescriptor(result); 1309 return NULL; 1310 } 1311 result->trb_count = trbCount; 1312 result->trb_used = 0; 1313 1314 if (bufferSize > 0) { 1315 // Due to how the USB stack allocates physical memory, we can't just 1316 // request one large chunk the size of the transfer, and so instead we 1317 // create a series of buffers as requested by our caller. 1318 1319 // We store the buffer pointers and addresses in one memory block. 1320 if (!inKDL) { 1321 result->buffers = (void**)calloc(bufferCount, 1322 (sizeof(void*) + sizeof(phys_addr_t))); 1323 } else { 1324 phys_addr_t dummy; 1325 fStack->AllocateChunk((void **)&result->buffers, &dummy, 1326 bufferCount * (sizeof(void*) + sizeof(phys_addr_t))); 1327 } 1328 if (result->buffers == NULL) { 1329 TRACE_ERROR("unable to allocate space for buffer infos\n"); 1330 FreeDescriptor(result); 1331 return NULL; 1332 } 1333 result->buffer_addrs = (phys_addr_t*)&result->buffers[bufferCount]; 1334 result->buffer_size = bufferSize; 1335 result->buffer_count = bufferCount; 1336 1337 // Optimization: If the requested total size of all buffers is less 1338 // than 32*B_PAGE_SIZE (the maximum size that the physical memory 1339 // allocator can handle), we allocate only one buffer and segment it. 1340 size_t totalSize = bufferSize * bufferCount; 1341 if (totalSize < (32 * B_PAGE_SIZE)) { 1342 if (fStack->AllocateChunk(&result->buffers[0], 1343 &result->buffer_addrs[0], totalSize) < B_OK) { 1344 TRACE_ERROR("unable to allocate space for large buffer (size %ld)\n", 1345 totalSize); 1346 FreeDescriptor(result); 1347 return NULL; 1348 } 1349 for (uint32 i = 1; i < bufferCount; i++) { 1350 result->buffers[i] = (void*)((addr_t)(result->buffers[i - 1]) 1351 + bufferSize); 1352 result->buffer_addrs[i] = result->buffer_addrs[i - 1] 1353 + bufferSize; 1354 } 1355 } else { 1356 // Otherwise, we allocate each buffer individually. 1357 for (uint32 i = 0; i < bufferCount; i++) { 1358 if (fStack->AllocateChunk(&result->buffers[i], 1359 &result->buffer_addrs[i], bufferSize) < B_OK) { 1360 TRACE_ERROR("unable to allocate space for a buffer (size " 1361 "%" B_PRIuSIZE ", count %" B_PRIu32 ")\n", 1362 bufferSize, bufferCount); 1363 FreeDescriptor(result); 1364 return NULL; 1365 } 1366 } 1367 } 1368 } else { 1369 result->buffers = NULL; 1370 result->buffer_addrs = NULL; 1371 } 1372 1373 // Initialize all other fields. 1374 result->transfer = NULL; 1375 result->trb_completion_code = 0; 1376 result->trb_left = 0; 1377 result->next = NULL; 1378 1379 TRACE("CreateDescriptor allocated %p, buffer_size %ld, buffer_count %" B_PRIu32 "\n", 1380 result, result->buffer_size, result->buffer_count); 1381 1382 return result; 1383 } 1384 1385 1386 void 1387 XHCI::FreeDescriptor(xhci_td *descriptor) 1388 { 1389 if (descriptor == NULL) 1390 return; 1391 1392 const bool inKDL = debug_debugger_running(); 1393 1394 if (descriptor->trbs != NULL) { 1395 fStack->FreeChunk(descriptor->trbs, descriptor->trb_addr, 1396 (descriptor->trb_count * sizeof(xhci_trb))); 1397 } 1398 if (descriptor->buffers != NULL) { 1399 size_t totalSize = descriptor->buffer_size * descriptor->buffer_count; 1400 if (totalSize < (32 * B_PAGE_SIZE)) { 1401 // This was allocated as one contiguous buffer. 1402 fStack->FreeChunk(descriptor->buffers[0], descriptor->buffer_addrs[0], 1403 totalSize); 1404 } else { 1405 for (uint32 i = 0; i < descriptor->buffer_count; i++) { 1406 if (descriptor->buffers[i] == NULL) 1407 continue; 1408 fStack->FreeChunk(descriptor->buffers[i], descriptor->buffer_addrs[i], 1409 descriptor->buffer_size); 1410 } 1411 } 1412 1413 if (!inKDL) { 1414 free(descriptor->buffers); 1415 } else { 1416 fStack->FreeChunk(descriptor->buffers, 0, 1417 descriptor->buffer_count * (sizeof(void*) + sizeof(phys_addr_t))); 1418 } 1419 } 1420 1421 if (!inKDL) 1422 free(descriptor); 1423 else 1424 fStack->FreeChunk(descriptor, 0, sizeof(xhci_td)); 1425 } 1426 1427 1428 size_t 1429 XHCI::WriteDescriptor(xhci_td *descriptor, iovec *vector, size_t vectorCount) 1430 { 1431 size_t written = 0; 1432 1433 size_t bufIdx = 0, bufUsed = 0; 1434 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1435 size_t length = vector[vecIdx].iov_len; 1436 1437 while (length > 0 && bufIdx < descriptor->buffer_count) { 1438 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1439 memcpy((uint8 *)descriptor->buffers[bufIdx] + bufUsed, 1440 (uint8 *)vector[vecIdx].iov_base + (vector[vecIdx].iov_len - length), 1441 toCopy); 1442 1443 written += toCopy; 1444 bufUsed += toCopy; 1445 length -= toCopy; 1446 if (bufUsed == descriptor->buffer_size) { 1447 bufIdx++; 1448 bufUsed = 0; 1449 } 1450 } 1451 } 1452 1453 TRACE("wrote descriptor (%" B_PRIuSIZE " bytes)\n", written); 1454 return written; 1455 } 1456 1457 1458 size_t 1459 XHCI::ReadDescriptor(xhci_td *descriptor, iovec *vector, size_t vectorCount) 1460 { 1461 size_t read = 0; 1462 1463 size_t bufIdx = 0, bufUsed = 0; 1464 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1465 size_t length = vector[vecIdx].iov_len; 1466 1467 while (length > 0 && bufIdx < descriptor->buffer_count) { 1468 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1469 memcpy((uint8 *)vector[vecIdx].iov_base + (vector[vecIdx].iov_len - length), 1470 (uint8 *)descriptor->buffers[bufIdx] + bufUsed, toCopy); 1471 1472 read += toCopy; 1473 bufUsed += toCopy; 1474 length -= toCopy; 1475 if (bufUsed == descriptor->buffer_size) { 1476 bufIdx++; 1477 bufUsed = 0; 1478 } 1479 } 1480 } 1481 1482 TRACE("read descriptor (%" B_PRIuSIZE " bytes)\n", read); 1483 return read; 1484 } 1485 1486 1487 Device * 1488 XHCI::AllocateDevice(Hub *parent, int8 hubAddress, uint8 hubPort, 1489 usb_speed speed) 1490 { 1491 TRACE("AllocateDevice hubAddress %d hubPort %d speed %d\n", hubAddress, 1492 hubPort, speed); 1493 1494 uint8 slot = XHCI_MAX_SLOTS; 1495 status_t status = EnableSlot(&slot); 1496 if (status != B_OK) { 1497 TRACE_ERROR("failed to enable slot: %s\n", strerror(status)); 1498 return NULL; 1499 } 1500 1501 if (slot == 0 || slot > fSlotCount) { 1502 TRACE_ERROR("AllocateDevice: bad slot\n"); 1503 return NULL; 1504 } 1505 1506 if (fDevices[slot].slot != 0) { 1507 TRACE_ERROR("AllocateDevice: slot already used\n"); 1508 return NULL; 1509 } 1510 1511 struct xhci_device *device = &fDevices[slot]; 1512 device->slot = slot; 1513 1514 device->input_ctx_area = fStack->AllocateArea((void **)&device->input_ctx, 1515 &device->input_ctx_addr, sizeof(*device->input_ctx) << fContextSizeShift, 1516 "XHCI input context"); 1517 if (device->input_ctx_area < B_OK) { 1518 TRACE_ERROR("unable to create a input context area\n"); 1519 CleanupDevice(device); 1520 return NULL; 1521 } 1522 if (fContextSizeShift == 1) { 1523 // 64-byte contexts have to be page-aligned in order for 1524 // _OffsetContextAddr to function properly. 1525 ASSERT((((addr_t)device->input_ctx) % B_PAGE_SIZE) == 0); 1526 } 1527 1528 memset(device->input_ctx, 0, sizeof(*device->input_ctx) << fContextSizeShift); 1529 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1530 _WriteContext(&device->input_ctx->input.addFlags, 3); 1531 1532 uint8 rhPort = hubPort; 1533 uint32 route = 0; 1534 for (Device *hubDevice = parent; hubDevice != RootObject(); 1535 hubDevice = (Device *)hubDevice->Parent()) { 1536 if (hubDevice->Parent() == RootObject()) 1537 break; 1538 1539 if (rhPort > 15) 1540 rhPort = 15; 1541 route = route << 4; 1542 route |= rhPort; 1543 1544 rhPort = hubDevice->HubPort(); 1545 } 1546 1547 uint32 dwslot0 = SLOT_0_NUM_ENTRIES(1) | SLOT_0_ROUTE(route); 1548 1549 // Get speed of port, only if device connected to root hub port 1550 // else we have to rely on value reported by the Hub Explore thread 1551 if (route == 0) { 1552 GetPortSpeed(hubPort - 1, &speed); 1553 TRACE("speed updated %d\n", speed); 1554 } 1555 1556 // add the speed 1557 switch (speed) { 1558 case USB_SPEED_LOWSPEED: 1559 dwslot0 |= SLOT_0_SPEED(2); 1560 break; 1561 case USB_SPEED_FULLSPEED: 1562 dwslot0 |= SLOT_0_SPEED(1); 1563 break; 1564 case USB_SPEED_HIGHSPEED: 1565 dwslot0 |= SLOT_0_SPEED(3); 1566 break; 1567 case USB_SPEED_SUPERSPEED: 1568 dwslot0 |= SLOT_0_SPEED(4); 1569 break; 1570 default: 1571 TRACE_ERROR("unknown usb speed\n"); 1572 break; 1573 } 1574 1575 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1576 // TODO enable power save 1577 _WriteContext(&device->input_ctx->slot.dwslot1, SLOT_1_RH_PORT(rhPort)); 1578 uint32 dwslot2 = SLOT_2_IRQ_TARGET(0); 1579 1580 // If LS/FS device connected to non-root HS device 1581 if (route != 0 && parent->Speed() == USB_SPEED_HIGHSPEED 1582 && (speed == USB_SPEED_LOWSPEED || speed == USB_SPEED_FULLSPEED)) { 1583 struct xhci_device *parenthub = (struct xhci_device *) 1584 parent->ControllerCookie(); 1585 dwslot2 |= SLOT_2_PORT_NUM(hubPort); 1586 dwslot2 |= SLOT_2_TT_HUB_SLOT(parenthub->slot); 1587 } 1588 1589 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1590 1591 _WriteContext(&device->input_ctx->slot.dwslot3, SLOT_3_SLOT_STATE(0) 1592 | SLOT_3_DEVICE_ADDRESS(0)); 1593 1594 TRACE("slot 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 1595 "\n", _ReadContext(&device->input_ctx->slot.dwslot0), 1596 _ReadContext(&device->input_ctx->slot.dwslot1), 1597 _ReadContext(&device->input_ctx->slot.dwslot2), 1598 _ReadContext(&device->input_ctx->slot.dwslot3)); 1599 1600 device->device_ctx_area = fStack->AllocateArea((void **)&device->device_ctx, 1601 &device->device_ctx_addr, sizeof(*device->device_ctx) << fContextSizeShift, 1602 "XHCI device context"); 1603 if (device->device_ctx_area < B_OK) { 1604 TRACE_ERROR("unable to create a device context area\n"); 1605 CleanupDevice(device); 1606 return NULL; 1607 } 1608 memset(device->device_ctx, 0, sizeof(*device->device_ctx) << fContextSizeShift); 1609 1610 device->trb_area = fStack->AllocateArea((void **)&device->trbs, 1611 &device->trb_addr, sizeof(xhci_trb) * (XHCI_MAX_ENDPOINTS - 1) 1612 * XHCI_ENDPOINT_RING_SIZE, "XHCI endpoint trbs"); 1613 if (device->trb_area < B_OK) { 1614 TRACE_ERROR("unable to create a device trbs area\n"); 1615 CleanupDevice(device); 1616 return NULL; 1617 } 1618 1619 // set up slot pointer to device context 1620 fDcba->baseAddress[slot] = device->device_ctx_addr; 1621 1622 size_t maxPacketSize; 1623 switch (speed) { 1624 case USB_SPEED_LOWSPEED: 1625 case USB_SPEED_FULLSPEED: 1626 maxPacketSize = 8; 1627 break; 1628 case USB_SPEED_HIGHSPEED: 1629 maxPacketSize = 64; 1630 break; 1631 default: 1632 maxPacketSize = 512; 1633 break; 1634 } 1635 1636 xhci_endpoint* endpoint0 = &device->endpoints[0]; 1637 mutex_init(&endpoint0->lock, "xhci endpoint lock"); 1638 endpoint0->device = device; 1639 endpoint0->id = 0; 1640 endpoint0->td_head = NULL; 1641 endpoint0->used = 0; 1642 endpoint0->current = 0; 1643 endpoint0->trbs = device->trbs; 1644 endpoint0->trb_addr = device->trb_addr; 1645 1646 // configure the Control endpoint 0 1647 if (ConfigureEndpoint(endpoint0, slot, 0, USB_OBJECT_CONTROL_PIPE, false, 1648 0, maxPacketSize, speed, 0, 0) != B_OK) { 1649 TRACE_ERROR("unable to configure default control endpoint\n"); 1650 CleanupDevice(device); 1651 return NULL; 1652 } 1653 1654 // device should get to addressed state (bsr = 0) 1655 status = SetAddress(device->input_ctx_addr, false, slot); 1656 if (status != B_OK) { 1657 TRACE_ERROR("unable to set address: %s\n", strerror(status)); 1658 CleanupDevice(device); 1659 return NULL; 1660 } 1661 1662 device->address = SLOT_3_DEVICE_ADDRESS_GET(_ReadContext( 1663 &device->device_ctx->slot.dwslot3)); 1664 1665 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", device->address, 1666 SLOT_3_SLOT_STATE_GET(_ReadContext( 1667 &device->device_ctx->slot.dwslot3))); 1668 TRACE("endpoint0 state 0x%08" B_PRIx32 "\n", 1669 ENDPOINT_0_STATE_GET(_ReadContext( 1670 &device->device_ctx->endpoints[0].dwendpoint0))); 1671 1672 // Wait a bit for the device to complete addressing 1673 snooze(USB_DELAY_SET_ADDRESS); 1674 1675 // Create a temporary pipe with the new address 1676 ControlPipe pipe(parent); 1677 pipe.SetControllerCookie(endpoint0); 1678 pipe.InitCommon(device->address + 1, 0, speed, Pipe::Default, maxPacketSize, 0, 1679 hubAddress, hubPort); 1680 1681 // Get the device descriptor 1682 // Just retrieve the first 8 bytes of the descriptor -> minimum supported 1683 // size of any device. It is enough because it includes the device type. 1684 1685 size_t actualLength = 0; 1686 usb_device_descriptor deviceDescriptor; 1687 1688 TRACE("getting the device descriptor\n"); 1689 status = pipe.SendRequest( 1690 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_STANDARD, // type 1691 USB_REQUEST_GET_DESCRIPTOR, // request 1692 USB_DESCRIPTOR_DEVICE << 8, // value 1693 0, // index 1694 8, // length 1695 (void *)&deviceDescriptor, // buffer 1696 8, // buffer length 1697 &actualLength); // actual length 1698 1699 if (actualLength != 8) { 1700 TRACE_ERROR("failed to get the device descriptor: %s\n", 1701 strerror(status)); 1702 CleanupDevice(device); 1703 return NULL; 1704 } 1705 1706 TRACE("device_class: %d device_subclass %d device_protocol %d\n", 1707 deviceDescriptor.device_class, deviceDescriptor.device_subclass, 1708 deviceDescriptor.device_protocol); 1709 1710 if (speed == USB_SPEED_FULLSPEED && deviceDescriptor.max_packet_size_0 != 8) { 1711 TRACE("Full speed device with different max packet size for Endpoint 0\n"); 1712 uint32 dwendpoint1 = _ReadContext( 1713 &device->input_ctx->endpoints[0].dwendpoint1); 1714 dwendpoint1 &= ~ENDPOINT_1_MAXPACKETSIZE(0xffff); 1715 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE( 1716 deviceDescriptor.max_packet_size_0); 1717 _WriteContext(&device->input_ctx->endpoints[0].dwendpoint1, 1718 dwendpoint1); 1719 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1720 _WriteContext(&device->input_ctx->input.addFlags, (1 << 1)); 1721 EvaluateContext(device->input_ctx_addr, device->slot); 1722 } 1723 1724 Device *deviceObject = NULL; 1725 if (deviceDescriptor.device_class == 0x09) { 1726 TRACE("creating new Hub\n"); 1727 TRACE("getting the hub descriptor\n"); 1728 size_t actualLength = 0; 1729 usb_hub_descriptor hubDescriptor; 1730 status = pipe.SendRequest( 1731 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_CLASS, // type 1732 USB_REQUEST_GET_DESCRIPTOR, // request 1733 USB_DESCRIPTOR_HUB << 8, // value 1734 0, // index 1735 sizeof(usb_hub_descriptor), // length 1736 (void *)&hubDescriptor, // buffer 1737 sizeof(usb_hub_descriptor), // buffer length 1738 &actualLength); 1739 1740 if (actualLength != sizeof(usb_hub_descriptor)) { 1741 TRACE_ERROR("error while getting the hub descriptor: %s\n", 1742 strerror(status)); 1743 CleanupDevice(device); 1744 return NULL; 1745 } 1746 1747 uint32 dwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1748 dwslot0 |= SLOT_0_HUB_BIT; 1749 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1750 uint32 dwslot1 = _ReadContext(&device->input_ctx->slot.dwslot1); 1751 dwslot1 |= SLOT_1_NUM_PORTS(hubDescriptor.num_ports); 1752 _WriteContext(&device->input_ctx->slot.dwslot1, dwslot1); 1753 if (speed == USB_SPEED_HIGHSPEED) { 1754 uint32 dwslot2 = _ReadContext(&device->input_ctx->slot.dwslot2); 1755 dwslot2 |= SLOT_2_TT_TIME(HUB_TTT_GET(hubDescriptor.characteristics)); 1756 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1757 } 1758 1759 deviceObject = new(std::nothrow) Hub(parent, hubAddress, hubPort, 1760 deviceDescriptor, device->address + 1, speed, false, device); 1761 } else { 1762 TRACE("creating new device\n"); 1763 deviceObject = new(std::nothrow) Device(parent, hubAddress, hubPort, 1764 deviceDescriptor, device->address + 1, speed, false, device); 1765 } 1766 if (deviceObject == NULL || deviceObject->InitCheck() != B_OK) { 1767 if (deviceObject == NULL) { 1768 TRACE_ERROR("no memory to allocate device\n"); 1769 } else { 1770 TRACE_ERROR("device object failed to initialize\n"); 1771 } 1772 CleanupDevice(device); 1773 return NULL; 1774 } 1775 1776 // We don't want to disable the default endpoint, naturally, which would 1777 // otherwise happen when this Pipe object is destroyed. 1778 pipe.SetControllerCookie(NULL); 1779 1780 deviceObject->RegisterNode(); 1781 1782 TRACE("AllocateDevice() port %d slot %d\n", hubPort, slot); 1783 return deviceObject; 1784 } 1785 1786 1787 void 1788 XHCI::FreeDevice(Device *usbDevice) 1789 { 1790 xhci_device* device = (xhci_device*)usbDevice->ControllerCookie(); 1791 TRACE("FreeDevice() slot %d\n", device->slot); 1792 1793 // Delete the device first, so it cleans up its pipes and tells us 1794 // what we need to destroy before we tear down our internal state. 1795 delete usbDevice; 1796 1797 CleanupDevice(device); 1798 } 1799 1800 1801 void 1802 XHCI::CleanupDevice(xhci_device *device) 1803 { 1804 if (device->slot != 0) { 1805 DisableSlot(device->slot); 1806 fDcba->baseAddress[device->slot] = 0; 1807 } 1808 1809 if (device->trb_addr != 0) 1810 delete_area(device->trb_area); 1811 if (device->input_ctx_addr != 0) 1812 delete_area(device->input_ctx_area); 1813 if (device->device_ctx_addr != 0) 1814 delete_area(device->device_ctx_area); 1815 1816 memset(device, 0, sizeof(xhci_device)); 1817 } 1818 1819 1820 uint8 1821 XHCI::_GetEndpointState(xhci_endpoint* endpoint) 1822 { 1823 struct xhci_device_ctx* device_ctx = endpoint->device->device_ctx; 1824 return ENDPOINT_0_STATE_GET( 1825 _ReadContext(&device_ctx->endpoints[endpoint->id].dwendpoint0)); 1826 } 1827 1828 1829 status_t 1830 XHCI::_InsertEndpointForPipe(Pipe *pipe) 1831 { 1832 TRACE("insert endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1833 1834 if (pipe->ControllerCookie() != NULL 1835 || pipe->Parent()->Type() != USB_OBJECT_DEVICE) { 1836 // default pipe is already referenced 1837 return B_OK; 1838 } 1839 1840 Device* usbDevice = (Device *)pipe->Parent(); 1841 if (usbDevice->Parent() == RootObject()) { 1842 // root hub needs no initialization 1843 return B_OK; 1844 } 1845 1846 struct xhci_device *device = (struct xhci_device *) 1847 usbDevice->ControllerCookie(); 1848 if (device == NULL) { 1849 panic("device is NULL\n"); 1850 return B_NO_INIT; 1851 } 1852 1853 const uint8 id = (2 * pipe->EndpointAddress() 1854 + (pipe->Direction() != Pipe::Out ? 1 : 0)) - 1; 1855 if (id >= XHCI_MAX_ENDPOINTS - 1) 1856 return B_BAD_VALUE; 1857 1858 if (id > 0) { 1859 uint32 devicedwslot0 = _ReadContext(&device->device_ctx->slot.dwslot0); 1860 if (SLOT_0_NUM_ENTRIES_GET(devicedwslot0) == 1) { 1861 uint32 inputdwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1862 inputdwslot0 &= ~(SLOT_0_NUM_ENTRIES(0x1f)); 1863 inputdwslot0 |= SLOT_0_NUM_ENTRIES(XHCI_MAX_ENDPOINTS - 1); 1864 _WriteContext(&device->input_ctx->slot.dwslot0, inputdwslot0); 1865 EvaluateContext(device->input_ctx_addr, device->slot); 1866 } 1867 1868 xhci_endpoint* endpoint = &device->endpoints[id]; 1869 mutex_init(&endpoint->lock, "xhci endpoint lock"); 1870 MutexLocker endpointLocker(endpoint->lock); 1871 1872 endpoint->device = device; 1873 endpoint->id = id; 1874 endpoint->td_head = NULL; 1875 endpoint->used = 0; 1876 endpoint->current = 0; 1877 1878 endpoint->trbs = device->trbs + id * XHCI_ENDPOINT_RING_SIZE; 1879 endpoint->trb_addr = device->trb_addr 1880 + id * XHCI_ENDPOINT_RING_SIZE * sizeof(xhci_trb); 1881 memset(endpoint->trbs, 0, 1882 sizeof(xhci_trb) * XHCI_ENDPOINT_RING_SIZE); 1883 1884 TRACE("insert endpoint for pipe: trbs, device %p endpoint %p\n", 1885 device->trbs, endpoint->trbs); 1886 TRACE("insert endpoint for pipe: trb_addr, device 0x%" B_PRIxPHYSADDR 1887 " endpoint 0x%" B_PRIxPHYSADDR "\n", device->trb_addr, 1888 endpoint->trb_addr); 1889 1890 const uint8 endpointNum = id + 1; 1891 1892 status_t status = ConfigureEndpoint(endpoint, device->slot, id, pipe->Type(), 1893 pipe->Direction() == Pipe::In, pipe->Interval(), pipe->MaxPacketSize(), 1894 usbDevice->Speed(), pipe->MaxBurst(), pipe->BytesPerInterval()); 1895 if (status != B_OK) { 1896 TRACE_ERROR("unable to configure endpoint: %s\n", strerror(status)); 1897 return status; 1898 } 1899 1900 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1901 _WriteContext(&device->input_ctx->input.addFlags, 1902 (1 << endpointNum) | (1 << 0)); 1903 1904 ConfigureEndpoint(device->input_ctx_addr, false, device->slot); 1905 1906 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", 1907 device->address, SLOT_3_SLOT_STATE_GET(_ReadContext( 1908 &device->device_ctx->slot.dwslot3))); 1909 TRACE("endpoint[0] state 0x%08" B_PRIx32 "\n", 1910 ENDPOINT_0_STATE_GET(_ReadContext( 1911 &device->device_ctx->endpoints[0].dwendpoint0))); 1912 TRACE("endpoint[%d] state 0x%08" B_PRIx32 "\n", id, 1913 ENDPOINT_0_STATE_GET(_ReadContext( 1914 &device->device_ctx->endpoints[id].dwendpoint0))); 1915 } 1916 pipe->SetControllerCookie(&device->endpoints[id]); 1917 1918 return B_OK; 1919 } 1920 1921 1922 status_t 1923 XHCI::_RemoveEndpointForPipe(Pipe *pipe) 1924 { 1925 TRACE("remove endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1926 1927 if (pipe->Parent()->Type() != USB_OBJECT_DEVICE) 1928 return B_OK; 1929 Device* usbDevice = (Device *)pipe->Parent(); 1930 if (usbDevice->Parent() == RootObject()) 1931 return B_BAD_VALUE; 1932 1933 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 1934 if (endpoint == NULL || endpoint->trbs == NULL) 1935 return B_NO_INIT; 1936 1937 pipe->SetControllerCookie(NULL); 1938 1939 if (endpoint->id > 0) { 1940 xhci_device *device = endpoint->device; 1941 uint8 epNumber = endpoint->id + 1; 1942 StopEndpoint(true, endpoint); 1943 1944 mutex_lock(&endpoint->lock); 1945 1946 // See comment in CancelQueuedTransfers. 1947 xhci_td* td; 1948 while ((td = endpoint->td_head) != NULL) { 1949 endpoint->td_head = endpoint->td_head->next; 1950 FreeDescriptor(td); 1951 } 1952 1953 mutex_destroy(&endpoint->lock); 1954 memset(endpoint, 0, sizeof(xhci_endpoint)); 1955 1956 _WriteContext(&device->input_ctx->input.dropFlags, (1 << epNumber)); 1957 _WriteContext(&device->input_ctx->input.addFlags, (1 << 0)); 1958 1959 // The Deconfigure bit in the Configure Endpoint command indicates 1960 // that *all* endpoints are to be deconfigured, and not just the ones 1961 // specified in the context flags. (XHCI 1.2 § 4.6.6 p115.) 1962 ConfigureEndpoint(device->input_ctx_addr, false, device->slot); 1963 } 1964 1965 return B_OK; 1966 } 1967 1968 1969 status_t 1970 XHCI::_LinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 1971 { 1972 TRACE("link descriptor for pipe\n"); 1973 1974 // Use mutex_trylock first, in case we are in KDL. 1975 MutexLocker endpointLocker(&endpoint->lock, mutex_trylock(&endpoint->lock) == B_OK); 1976 1977 // "used" refers to the number of currently linked TDs, not the number of 1978 // used TRBs on the ring (we use 2 TRBs on the ring per transfer.) 1979 if (endpoint->used >= (XHCI_MAX_TRANSFERS - 1)) { 1980 TRACE_ERROR("link descriptor for pipe: max transfers count exceeded\n"); 1981 return B_BAD_VALUE; 1982 } 1983 1984 // We do not support queuing other transfers in tandem with a fragmented one. 1985 if (endpoint->td_head != NULL && endpoint->td_head->transfer != NULL 1986 && endpoint->td_head->transfer->IsFragmented()) { 1987 TRACE_ERROR("cannot submit transfer: a fragmented transfer is queued\n"); 1988 return B_DEV_RESOURCE_CONFLICT; 1989 } 1990 1991 endpoint->used++; 1992 descriptor->next = endpoint->td_head; 1993 endpoint->td_head = descriptor; 1994 1995 const uint32 current = endpoint->current, 1996 eventdata = current + 1, 1997 last = XHCI_ENDPOINT_RING_SIZE - 1; 1998 uint32 next = eventdata + 1; 1999 2000 TRACE("link descriptor for pipe: current %d, next %d\n", current, next); 2001 2002 // Add a Link TRB to the end of the descriptor. 2003 phys_addr_t addr = endpoint->trb_addr + eventdata * sizeof(xhci_trb); 2004 descriptor->trbs[descriptor->trb_used].address = addr; 2005 descriptor->trbs[descriptor->trb_used].status = TRB_2_IRQ(0); 2006 descriptor->trbs[descriptor->trb_used].flags = TRB_3_TYPE(TRB_TYPE_LINK) 2007 | TRB_3_CHAIN_BIT | TRB_3_CYCLE_BIT; 2008 // It is specified that (XHCI 1.2 § 4.12.3 Note 2 p251) if the TRB 2009 // following one with the ENT bit set is a Link TRB, the Link TRB 2010 // shall be evaluated *and* the subsequent TRB shall be. Thus a 2011 // TRB_3_ENT_BIT is unnecessary here; and from testing seems to 2012 // break all transfers on a (very) small number of controllers. 2013 2014 #if !B_HOST_IS_LENDIAN 2015 // Convert endianness. 2016 for (uint32 i = 0; i <= descriptor->trb_used; i++) { 2017 descriptor->trbs[i].address = 2018 B_HOST_TO_LENDIAN_INT64(descriptor->trbs[i].address); 2019 descriptor->trbs[i].status = 2020 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].status); 2021 descriptor->trbs[i].flags = 2022 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].flags); 2023 } 2024 #endif 2025 2026 // Link the descriptor. 2027 endpoint->trbs[current].address = 2028 B_HOST_TO_LENDIAN_INT64(descriptor->trb_addr); 2029 endpoint->trbs[current].status = 2030 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 2031 endpoint->trbs[current].flags = 2032 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_LINK)); 2033 2034 // Set up the Event Data TRB (XHCI 1.2 § 4.11.5.2 p230.) 2035 // 2036 // We do this on the main ring for two reasons: first, to avoid a small 2037 // potential race between the interrupt and the controller evaluating 2038 // the link TRB to get back onto the ring; and second, because many 2039 // controllers throw errors if the target of a Link TRB is not valid 2040 // (i.e. does not have its Cycle Bit set.) 2041 // 2042 // We also set the "address" field, which the controller will copy 2043 // verbatim into the TRB it posts to the event ring, to be the last 2044 // "real" TRB in the TD; this will allow us to determine what transfer 2045 // the resulting Transfer Event TRB refers to. 2046 endpoint->trbs[eventdata].address = 2047 B_HOST_TO_LENDIAN_INT64(descriptor->trb_addr 2048 + (descriptor->trb_used - 1) * sizeof(xhci_trb)); 2049 endpoint->trbs[eventdata].status = 2050 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 2051 endpoint->trbs[eventdata].flags = 2052 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_EVENT_DATA) 2053 | TRB_3_IOC_BIT | TRB_3_CYCLE_BIT); 2054 2055 if (next == last) { 2056 // We always use 2 TRBs per _Link..() call, so if "next" is the last 2057 // TRB in the ring, we need to generate a link TRB at "next", and 2058 // then wrap it to 0. (We write the cycle bit later, after wrapping, 2059 // for the reason noted in the previous comment.) 2060 endpoint->trbs[next].address = 2061 B_HOST_TO_LENDIAN_INT64(endpoint->trb_addr); 2062 endpoint->trbs[next].status = 2063 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 2064 endpoint->trbs[next].flags = 2065 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_LINK)); 2066 2067 next = 0; 2068 } 2069 2070 endpoint->trbs[next].address = 0; 2071 endpoint->trbs[next].status = 0; 2072 endpoint->trbs[next].flags = 0; 2073 2074 memory_write_barrier(); 2075 2076 // Everything is ready, so write the cycle bit(s). 2077 endpoint->trbs[current].flags |= B_HOST_TO_LENDIAN_INT32(TRB_3_CYCLE_BIT); 2078 if (current == 0 && endpoint->trbs[last].address != 0) 2079 endpoint->trbs[last].flags |= B_HOST_TO_LENDIAN_INT32(TRB_3_CYCLE_BIT); 2080 2081 TRACE("_LinkDescriptorForPipe pCurrent %p phys 0x%" B_PRIxPHYSADDR 2082 " 0x%" B_PRIxPHYSADDR " 0x%08" B_PRIx32 "\n", &endpoint->trbs[current], 2083 endpoint->trb_addr + current * sizeof(struct xhci_trb), 2084 endpoint->trbs[current].address, 2085 B_LENDIAN_TO_HOST_INT32(endpoint->trbs[current].flags)); 2086 2087 endpoint->current = next; 2088 endpointLocker.Unlock(); 2089 2090 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 2091 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 2092 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 2093 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 2094 2095 Ring(endpoint->device->slot, endpoint->id + 1); 2096 2097 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 2098 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 2099 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 2100 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 2101 2102 return B_OK; 2103 } 2104 2105 2106 status_t 2107 XHCI::_UnlinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 2108 { 2109 TRACE("unlink descriptor for pipe\n"); 2110 // We presume that the caller has already locked or owns the endpoint. 2111 2112 endpoint->used--; 2113 if (descriptor == endpoint->td_head) { 2114 endpoint->td_head = descriptor->next; 2115 descriptor->next = NULL; 2116 return B_OK; 2117 } else { 2118 for (xhci_td *td = endpoint->td_head; td->next != NULL; td = td->next) { 2119 if (td->next == descriptor) { 2120 td->next = descriptor->next; 2121 descriptor->next = NULL; 2122 return B_OK; 2123 } 2124 } 2125 } 2126 2127 endpoint->used++; 2128 return B_ERROR; 2129 } 2130 2131 2132 status_t 2133 XHCI::ConfigureEndpoint(xhci_endpoint* ep, uint8 slot, uint8 number, uint8 type, 2134 bool directionIn, uint16 interval, uint16 maxPacketSize, usb_speed speed, 2135 uint8 maxBurst, uint16 bytesPerInterval) 2136 { 2137 struct xhci_device* device = &fDevices[slot]; 2138 2139 uint32 dwendpoint0 = 0; 2140 uint32 dwendpoint1 = 0; 2141 uint64 qwendpoint2 = 0; 2142 uint32 dwendpoint4 = 0; 2143 2144 // Compute and assign the endpoint type. (XHCI 1.2 § 6.2.3 Table 6-9 p452.) 2145 uint8 xhciType = 4; 2146 if ((type & USB_OBJECT_INTERRUPT_PIPE) != 0) 2147 xhciType = 3; 2148 if ((type & USB_OBJECT_BULK_PIPE) != 0) 2149 xhciType = 2; 2150 if ((type & USB_OBJECT_ISO_PIPE) != 0) 2151 xhciType = 1; 2152 xhciType |= directionIn ? (1 << 2) : 0; 2153 dwendpoint1 |= ENDPOINT_1_EPTYPE(xhciType); 2154 2155 // Compute and assign interval. (XHCI 1.2 § 6.2.3.6 p456.) 2156 uint16 calcInterval; 2157 if ((type & USB_OBJECT_BULK_PIPE) != 0 2158 || (type & USB_OBJECT_CONTROL_PIPE) != 0) { 2159 // Bulk and Control endpoints never issue NAKs. 2160 calcInterval = 0; 2161 } else { 2162 switch (speed) { 2163 case USB_SPEED_FULLSPEED: 2164 if ((type & USB_OBJECT_ISO_PIPE) != 0) { 2165 // Convert 1-16 into 3-18. 2166 calcInterval = min_c(max_c(interval, 1), 16) + 2; 2167 break; 2168 } 2169 2170 // fall through 2171 case USB_SPEED_LOWSPEED: { 2172 // Convert 1ms-255ms into 3-10. 2173 2174 // Find the index of the highest set bit in "interval". 2175 uint32 temp = min_c(max_c(interval, 1), 255); 2176 for (calcInterval = 0; temp != 1; calcInterval++) 2177 temp = temp >> 1; 2178 calcInterval += 3; 2179 break; 2180 } 2181 2182 case USB_SPEED_HIGHSPEED: 2183 case USB_SPEED_SUPERSPEED: 2184 default: 2185 // Convert 1-16 into 0-15. 2186 calcInterval = min_c(max_c(interval, 1), 16) - 1; 2187 break; 2188 } 2189 } 2190 dwendpoint0 |= ENDPOINT_0_INTERVAL(calcInterval); 2191 2192 // For non-isochronous endpoints, we want the controller to retry failed 2193 // transfers, if possible. (XHCI 1.2 § 4.10.2.3 p197.) 2194 if ((type & USB_OBJECT_ISO_PIPE) == 0) 2195 dwendpoint1 |= ENDPOINT_1_CERR(3); 2196 2197 // Assign maximum burst size. For USB3 devices this is passed in; for 2198 // all other devices we compute it. (XHCI 1.2 § 4.8.2 p161.) 2199 if (speed == USB_SPEED_HIGHSPEED && (type & (USB_OBJECT_INTERRUPT_PIPE 2200 | USB_OBJECT_ISO_PIPE)) != 0) { 2201 maxBurst = (maxPacketSize & 0x1800) >> 11; 2202 } else if (speed != USB_SPEED_SUPERSPEED) { 2203 maxBurst = 0; 2204 } 2205 dwendpoint1 |= ENDPOINT_1_MAXBURST(maxBurst); 2206 2207 // Assign maximum packet size, set the ring address, and set the 2208 // "Dequeue Cycle State" bit. (XHCI 1.2 § 6.2.3 Table 6-10 p453.) 2209 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE(maxPacketSize); 2210 qwendpoint2 |= ENDPOINT_2_DCS_BIT | ep->trb_addr; 2211 2212 // The Max Burst Payload is the number of bytes moved by a 2213 // maximum sized burst. (XHCI 1.2 § 4.11.7.1 p236.) 2214 ep->max_burst_payload = (maxBurst + 1) * maxPacketSize; 2215 if (ep->max_burst_payload == 0) { 2216 TRACE_ERROR("ConfigureEndpoint() failed invalid max_burst_payload\n"); 2217 return B_BAD_VALUE; 2218 } 2219 2220 // Assign average TRB length. 2221 if ((type & USB_OBJECT_CONTROL_PIPE) != 0) { 2222 // Control pipes are a special case, as they rarely have 2223 // outbound transfers of any substantial size. 2224 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(8); 2225 } else if ((type & USB_OBJECT_ISO_PIPE) != 0) { 2226 // Isochronous pipes are another special case: the TRB size will be 2227 // one packet (which is normally smaller than the max packet size, 2228 // but we don't know what it is here.) 2229 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(maxPacketSize); 2230 } else { 2231 // Under all other circumstances, we put max_burst_payload in a TRB. 2232 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(ep->max_burst_payload); 2233 } 2234 2235 // Assign maximum ESIT payload. (XHCI 1.2 § 4.14.2 p259.) 2236 if ((type & (USB_OBJECT_INTERRUPT_PIPE | USB_OBJECT_ISO_PIPE)) != 0) { 2237 // TODO: For SuperSpeedPlus endpoints, there is yet another descriptor 2238 // for isochronous endpoints that specifies the maximum ESIT payload. 2239 // We don't fetch this yet, so just fall back to the USB2 computation 2240 // method if bytesPerInterval is 0. 2241 if (speed == USB_SPEED_SUPERSPEED && bytesPerInterval != 0) 2242 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD(bytesPerInterval); 2243 else if (speed >= USB_SPEED_HIGHSPEED) 2244 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD((maxBurst + 1) * maxPacketSize); 2245 } 2246 2247 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint0, 2248 dwendpoint0); 2249 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint1, 2250 dwendpoint1); 2251 _WriteContext(&device->input_ctx->endpoints[number].qwendpoint2, 2252 qwendpoint2); 2253 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint4, 2254 dwendpoint4); 2255 2256 TRACE("endpoint 0x%" B_PRIx32 " 0x%" B_PRIx32 " 0x%" B_PRIx64 " 0x%" 2257 B_PRIx32 "\n", 2258 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint0), 2259 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint1), 2260 _ReadContext(&device->input_ctx->endpoints[number].qwendpoint2), 2261 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint4)); 2262 2263 return B_OK; 2264 } 2265 2266 2267 status_t 2268 XHCI::GetPortSpeed(uint8 index, usb_speed* speed) 2269 { 2270 if (index >= fPortCount) 2271 return B_BAD_INDEX; 2272 2273 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 2274 2275 switch (PS_SPEED_GET(portStatus)) { 2276 case 2: 2277 *speed = USB_SPEED_LOWSPEED; 2278 break; 2279 case 1: 2280 *speed = USB_SPEED_FULLSPEED; 2281 break; 2282 case 3: 2283 *speed = USB_SPEED_HIGHSPEED; 2284 break; 2285 case 4: 2286 *speed = USB_SPEED_SUPERSPEED; 2287 break; 2288 default: 2289 TRACE_ALWAYS("nonstandard port speed %" B_PRId32 ", assuming SuperSpeed\n", 2290 PS_SPEED_GET(portStatus)); 2291 *speed = USB_SPEED_SUPERSPEED; 2292 break; 2293 } 2294 2295 return B_OK; 2296 } 2297 2298 2299 status_t 2300 XHCI::GetPortStatus(uint8 index, usb_port_status* status) 2301 { 2302 if (index >= fPortCount) 2303 return B_BAD_INDEX; 2304 2305 status->status = status->change = 0; 2306 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 2307 TRACE("port %" B_PRId8 " status=0x%08" B_PRIx32 "\n", index, portStatus); 2308 2309 // build the status 2310 switch (PS_SPEED_GET(portStatus)) { 2311 case 3: 2312 status->status |= PORT_STATUS_HIGH_SPEED; 2313 break; 2314 case 2: 2315 status->status |= PORT_STATUS_LOW_SPEED; 2316 break; 2317 default: 2318 break; 2319 } 2320 2321 if (portStatus & PS_CCS) 2322 status->status |= PORT_STATUS_CONNECTION; 2323 if (portStatus & PS_PED) 2324 status->status |= PORT_STATUS_ENABLE; 2325 if (portStatus & PS_OCA) 2326 status->status |= PORT_STATUS_OVER_CURRENT; 2327 if (portStatus & PS_PR) 2328 status->status |= PORT_STATUS_RESET; 2329 if (portStatus & PS_PP) { 2330 if (fPortSpeeds[index] == USB_SPEED_SUPERSPEED) 2331 status->status |= PORT_STATUS_SS_POWER; 2332 else 2333 status->status |= PORT_STATUS_POWER; 2334 } 2335 2336 // build the change 2337 if (portStatus & PS_CSC) 2338 status->change |= PORT_STATUS_CONNECTION; 2339 if (portStatus & PS_PEC) 2340 status->change |= PORT_STATUS_ENABLE; 2341 if (portStatus & PS_OCC) 2342 status->change |= PORT_STATUS_OVER_CURRENT; 2343 if (portStatus & PS_PRC) 2344 status->change |= PORT_STATUS_RESET; 2345 2346 if (fPortSpeeds[index] == USB_SPEED_SUPERSPEED) { 2347 if (portStatus & PS_PLC) 2348 status->change |= PORT_CHANGE_LINK_STATE; 2349 if (portStatus & PS_WRC) 2350 status->change |= PORT_CHANGE_BH_PORT_RESET; 2351 } 2352 2353 return B_OK; 2354 } 2355 2356 2357 status_t 2358 XHCI::SetPortFeature(uint8 index, uint16 feature) 2359 { 2360 TRACE("set port feature index %u feature %u\n", index, feature); 2361 if (index >= fPortCount) 2362 return B_BAD_INDEX; 2363 2364 uint32 portRegister = XHCI_PORTSC(index); 2365 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2366 2367 switch (feature) { 2368 case PORT_SUSPEND: 2369 if ((portStatus & PS_PED) == 0 || (portStatus & PS_PR) 2370 || (portStatus & PS_PLS_MASK) >= PS_XDEV_U3) { 2371 TRACE_ERROR("USB core suspending device not in U0/U1/U2.\n"); 2372 return B_BAD_VALUE; 2373 } 2374 portStatus &= ~PS_PLS_MASK; 2375 WriteOpReg(portRegister, portStatus | PS_LWS | PS_XDEV_U3); 2376 break; 2377 2378 case PORT_RESET: 2379 WriteOpReg(portRegister, portStatus | PS_PR); 2380 break; 2381 2382 case PORT_POWER: 2383 WriteOpReg(portRegister, portStatus | PS_PP); 2384 break; 2385 default: 2386 return B_BAD_VALUE; 2387 } 2388 ReadOpReg(portRegister); 2389 return B_OK; 2390 } 2391 2392 2393 status_t 2394 XHCI::ClearPortFeature(uint8 index, uint16 feature) 2395 { 2396 TRACE("clear port feature index %u feature %u\n", index, feature); 2397 if (index >= fPortCount) 2398 return B_BAD_INDEX; 2399 2400 uint32 portRegister = XHCI_PORTSC(index); 2401 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2402 2403 switch (feature) { 2404 case PORT_SUSPEND: 2405 portStatus = ReadOpReg(portRegister); 2406 if (portStatus & PS_PR) 2407 return B_BAD_VALUE; 2408 if (portStatus & PS_XDEV_U3) { 2409 if ((portStatus & PS_PED) == 0) 2410 return B_BAD_VALUE; 2411 portStatus &= ~PS_PLS_MASK; 2412 WriteOpReg(portRegister, portStatus | PS_XDEV_U0 | PS_LWS); 2413 } 2414 break; 2415 case PORT_ENABLE: 2416 WriteOpReg(portRegister, portStatus | PS_PED); 2417 break; 2418 case PORT_POWER: 2419 WriteOpReg(portRegister, portStatus & ~PS_PP); 2420 break; 2421 case C_PORT_CONNECTION: 2422 WriteOpReg(portRegister, portStatus | PS_CSC); 2423 break; 2424 case C_PORT_ENABLE: 2425 WriteOpReg(portRegister, portStatus | PS_PEC); 2426 break; 2427 case C_PORT_OVER_CURRENT: 2428 WriteOpReg(portRegister, portStatus | PS_OCC); 2429 break; 2430 case C_PORT_RESET: 2431 WriteOpReg(portRegister, portStatus | PS_PRC); 2432 break; 2433 case C_PORT_BH_PORT_RESET: 2434 WriteOpReg(portRegister, portStatus | PS_WRC); 2435 break; 2436 case C_PORT_LINK_STATE: 2437 WriteOpReg(portRegister, portStatus | PS_PLC); 2438 break; 2439 default: 2440 return B_BAD_VALUE; 2441 } 2442 2443 ReadOpReg(portRegister); 2444 return B_OK; 2445 } 2446 2447 2448 status_t 2449 XHCI::ControllerHalt() 2450 { 2451 // Mask off run state 2452 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) & ~CMD_RUN); 2453 2454 // wait for shutdown state 2455 if (WaitOpBits(XHCI_STS, STS_HCH, STS_HCH) != B_OK) { 2456 TRACE_ERROR("HCH shutdown timeout\n"); 2457 return B_ERROR; 2458 } 2459 return B_OK; 2460 } 2461 2462 2463 status_t 2464 XHCI::ControllerReset() 2465 { 2466 TRACE("ControllerReset() cmd: 0x%" B_PRIx32 " sts: 0x%" B_PRIx32 "\n", 2467 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 2468 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) | CMD_HCRST); 2469 2470 if (WaitOpBits(XHCI_CMD, CMD_HCRST, 0) != B_OK) { 2471 TRACE_ERROR("ControllerReset() failed CMD_HCRST\n"); 2472 return B_ERROR; 2473 } 2474 2475 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 2476 TRACE_ERROR("ControllerReset() failed STS_CNR\n"); 2477 return B_ERROR; 2478 } 2479 2480 return B_OK; 2481 } 2482 2483 2484 int32 2485 XHCI::InterruptHandler(void* data) 2486 { 2487 return ((XHCI*)data)->Interrupt(); 2488 } 2489 2490 2491 int32 2492 XHCI::Interrupt() 2493 { 2494 SpinLocker _(&fSpinlock); 2495 2496 uint32 status = ReadOpReg(XHCI_STS); 2497 uint32 temp = ReadRunReg32(XHCI_IMAN(0)); 2498 WriteOpReg(XHCI_STS, status); 2499 WriteRunReg32(XHCI_IMAN(0), temp); 2500 2501 int32 result = B_HANDLED_INTERRUPT; 2502 2503 if ((status & STS_HCH) != 0) { 2504 TRACE_ERROR("Host Controller halted\n"); 2505 return result; 2506 } 2507 if ((status & STS_HSE) != 0) { 2508 TRACE_ERROR("Host System Error\n"); 2509 return result; 2510 } 2511 if ((status & STS_HCE) != 0) { 2512 TRACE_ERROR("Host Controller Error\n"); 2513 return result; 2514 } 2515 2516 if ((status & STS_EINT) == 0) { 2517 TRACE("STS: 0x%" B_PRIx32 " IRQ_PENDING: 0x%" B_PRIx32 "\n", 2518 status, temp); 2519 return B_UNHANDLED_INTERRUPT; 2520 } 2521 2522 TRACE("Event Interrupt\n"); 2523 release_sem_etc(fEventSem, 1, B_DO_NOT_RESCHEDULE); 2524 return B_INVOKE_SCHEDULER; 2525 } 2526 2527 2528 void 2529 XHCI::Ring(uint8 slot, uint8 endpoint) 2530 { 2531 TRACE("Ding Dong! slot:%d endpoint %d\n", slot, endpoint) 2532 if ((slot == 0 && endpoint > 0) || (slot > 0 && endpoint == 0)) 2533 panic("Ring() invalid slot/endpoint combination\n"); 2534 if (slot > fSlotCount || endpoint >= XHCI_MAX_ENDPOINTS) 2535 panic("Ring() invalid slot or endpoint\n"); 2536 2537 WriteDoorReg32(XHCI_DOORBELL(slot), XHCI_DOORBELL_TARGET(endpoint) 2538 | XHCI_DOORBELL_STREAMID(0)); 2539 ReadDoorReg32(XHCI_DOORBELL(slot)); 2540 // Flush PCI writes 2541 } 2542 2543 2544 void 2545 XHCI::QueueCommand(xhci_trb* trb) 2546 { 2547 uint8 i, j; 2548 uint32 temp; 2549 2550 i = fCmdIdx; 2551 j = fCmdCcs; 2552 2553 TRACE("command[%u] = %" B_PRId32 " (0x%016" B_PRIx64 ", 0x%08" B_PRIx32 2554 ", 0x%08" B_PRIx32 ")\n", i, TRB_3_TYPE_GET(trb->flags), trb->address, 2555 trb->status, trb->flags); 2556 2557 fCmdRing[i].address = trb->address; 2558 fCmdRing[i].status = trb->status; 2559 temp = trb->flags; 2560 2561 if (j) 2562 temp |= TRB_3_CYCLE_BIT; 2563 else 2564 temp &= ~TRB_3_CYCLE_BIT; 2565 temp &= ~TRB_3_TC_BIT; 2566 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2567 2568 fCmdAddr = fErst->rs_addr + (XHCI_MAX_EVENTS + i) * sizeof(xhci_trb); 2569 2570 i++; 2571 2572 if (i == (XHCI_MAX_COMMANDS - 1)) { 2573 temp = TRB_3_TYPE(TRB_TYPE_LINK) | TRB_3_TC_BIT; 2574 if (j) 2575 temp |= TRB_3_CYCLE_BIT; 2576 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2577 2578 i = 0; 2579 j ^= 1; 2580 } 2581 2582 fCmdIdx = i; 2583 fCmdCcs = j; 2584 } 2585 2586 2587 void 2588 XHCI::HandleCmdComplete(xhci_trb* trb) 2589 { 2590 if (fCmdAddr == trb->address) { 2591 TRACE("Received command event\n"); 2592 fCmdResult[0] = trb->status; 2593 fCmdResult[1] = B_LENDIAN_TO_HOST_INT32(trb->flags); 2594 release_sem_etc(fCmdCompSem, 1, B_DO_NOT_RESCHEDULE); 2595 } else 2596 TRACE_ERROR("received command event for unknown command!\n") 2597 } 2598 2599 2600 void 2601 XHCI::HandleTransferComplete(xhci_trb* trb) 2602 { 2603 const uint32 flags = B_LENDIAN_TO_HOST_INT32(trb->flags); 2604 const uint8 endpointNumber = TRB_3_ENDPOINT_GET(flags), 2605 slot = TRB_3_SLOT_GET(flags); 2606 2607 if (slot > fSlotCount) 2608 TRACE_ERROR("invalid slot\n"); 2609 if (endpointNumber == 0 || endpointNumber >= XHCI_MAX_ENDPOINTS) { 2610 TRACE_ERROR("invalid endpoint\n"); 2611 return; 2612 } 2613 2614 xhci_device *device = &fDevices[slot]; 2615 xhci_endpoint *endpoint = &device->endpoints[endpointNumber - 1]; 2616 2617 if (endpoint->trbs == NULL) { 2618 TRACE_ERROR("got TRB but endpoint is not allocated!\n"); 2619 return; 2620 } 2621 2622 // Use mutex_trylock first, in case we are in KDL. 2623 MutexLocker endpointLocker(endpoint->lock, mutex_trylock(&endpoint->lock) == B_OK); 2624 if (!endpointLocker.IsLocked()) { 2625 // We failed to get the lock. Most likely it was destroyed 2626 // while we were waiting for it. 2627 return; 2628 } 2629 2630 // In the case of an Event Data TRB, the "transferred" field refers 2631 // to the actual number of bytes transferred across the whole TD. 2632 // (XHCI 1.2 § 6.4.2.1 Table 6-38 p478.) 2633 const uint8 completionCode = TRB_2_COMP_CODE_GET(trb->status); 2634 int32 transferred = TRB_2_REM_GET(trb->status), remainder = -1; 2635 2636 TRACE("HandleTransferComplete: ed %" B_PRIu32 ", code %" B_PRIu8 ", transferred %" B_PRId32 "\n", 2637 (flags & TRB_3_EVENT_DATA_BIT), completionCode, transferred); 2638 2639 if ((flags & TRB_3_EVENT_DATA_BIT) == 0) { 2640 // This should only occur under error conditions. 2641 TRACE("got an interrupt for a non-Event Data TRB!\n"); 2642 remainder = transferred; 2643 transferred = -1; 2644 } 2645 2646 if (completionCode != COMP_SUCCESS && completionCode != COMP_SHORT_PACKET 2647 && completionCode != COMP_STOPPED) { 2648 TRACE_ALWAYS("transfer error on slot %" B_PRId8 " endpoint %" B_PRId8 2649 ": %s\n", slot, endpointNumber, xhci_error_string(completionCode)); 2650 } 2651 2652 const phys_addr_t source = B_LENDIAN_TO_HOST_INT64(trb->address); 2653 for (xhci_td *td = endpoint->td_head; td != NULL; td = td->next) { 2654 int64 offset = (source - td->trb_addr) / sizeof(xhci_trb); 2655 if (offset < 0 || offset >= td->trb_count) 2656 continue; 2657 2658 TRACE("HandleTransferComplete td %p trb %" B_PRId64 " found\n", 2659 td, offset); 2660 2661 // The TRB at offset trb_used will be the link TRB, which we do not 2662 // care about (and should not generate an interrupt at all.) We really 2663 // care about the properly last TRB, at index "count - 1", which the 2664 // Event Data TRB that _LinkDescriptorForPipe creates points to. 2665 // 2666 // But if we have an unsuccessful completion code, the transfer 2667 // likely failed midway; so just accept it anyway. 2668 if (offset == (td->trb_used - 1) || completionCode != COMP_SUCCESS) { 2669 _UnlinkDescriptorForPipe(td, endpoint); 2670 endpointLocker.Unlock(); 2671 2672 td->trb_completion_code = completionCode; 2673 td->td_transferred = transferred; 2674 td->trb_left = remainder; 2675 2676 // add descriptor to finished list 2677 if (mutex_trylock(&fFinishedLock) != B_OK) 2678 mutex_lock(&fFinishedLock); 2679 td->next = fFinishedHead; 2680 fFinishedHead = td; 2681 mutex_unlock(&fFinishedLock); 2682 2683 release_sem_etc(fFinishTransfersSem, 1, B_DO_NOT_RESCHEDULE); 2684 TRACE("HandleTransferComplete td %p done\n", td); 2685 } else { 2686 TRACE_ERROR("successful TRB 0x%" B_PRIxPHYSADDR " was found, but it wasn't " 2687 "the last in the TD!\n", source); 2688 } 2689 return; 2690 } 2691 TRACE_ERROR("TRB 0x%" B_PRIxPHYSADDR " was not found in the endpoint!\n", source); 2692 } 2693 2694 2695 void 2696 XHCI::DumpRing(xhci_trb *trbs, uint32 size) 2697 { 2698 if (!Lock()) { 2699 TRACE("Unable to get lock!\n"); 2700 return; 2701 } 2702 2703 for (uint32 i = 0; i < size; i++) { 2704 TRACE("command[%" B_PRId32 "] = %" B_PRId32 " (0x%016" B_PRIx64 "," 2705 " 0x%08" B_PRIx32 ", 0x%08" B_PRIx32 ")\n", i, 2706 TRB_3_TYPE_GET(B_LENDIAN_TO_HOST_INT32(trbs[i].flags)), 2707 trbs[i].address, trbs[i].status, trbs[i].flags); 2708 } 2709 2710 Unlock(); 2711 } 2712 2713 2714 status_t 2715 XHCI::DoCommand(xhci_trb* trb) 2716 { 2717 if (!Lock()) { 2718 TRACE("Unable to get lock!\n"); 2719 return B_ERROR; 2720 } 2721 2722 QueueCommand(trb); 2723 Ring(0, 0); 2724 2725 // Begin with a 50ms timeout. 2726 if (acquire_sem_etc(fCmdCompSem, 1, B_RELATIVE_TIMEOUT, 50 * 1000) != B_OK) { 2727 // We've hit the timeout. In some error cases, interrupts are not 2728 // generated; so here we force the event ring to be polled once. 2729 release_sem(fEventSem); 2730 2731 // Now try again, this time with a 750ms timeout. 2732 if (acquire_sem_etc(fCmdCompSem, 1, B_RELATIVE_TIMEOUT, 2733 750 * 1000) != B_OK) { 2734 TRACE("Unable to obtain fCmdCompSem!\n"); 2735 fCmdAddr = 0; 2736 Unlock(); 2737 return B_TIMED_OUT; 2738 } 2739 } 2740 2741 // eat up sems that have been released by multiple interrupts 2742 int32 semCount = 0; 2743 get_sem_count(fCmdCompSem, &semCount); 2744 if (semCount > 0) 2745 acquire_sem_etc(fCmdCompSem, semCount, B_RELATIVE_TIMEOUT, 0); 2746 2747 status_t status = B_OK; 2748 uint32 completionCode = TRB_2_COMP_CODE_GET(fCmdResult[0]); 2749 TRACE("command complete\n"); 2750 if (completionCode != COMP_SUCCESS) { 2751 TRACE_ERROR("unsuccessful command %" B_PRId32 ", error %s (%" B_PRId32 ")\n", 2752 TRB_3_TYPE_GET(trb->flags), xhci_error_string(completionCode), 2753 completionCode); 2754 status = B_IO_ERROR; 2755 } 2756 2757 trb->status = fCmdResult[0]; 2758 trb->flags = fCmdResult[1]; 2759 2760 fCmdAddr = 0; 2761 Unlock(); 2762 return status; 2763 } 2764 2765 2766 status_t 2767 XHCI::Noop() 2768 { 2769 TRACE("Issue No-Op\n"); 2770 xhci_trb trb; 2771 trb.address = 0; 2772 trb.status = 0; 2773 trb.flags = TRB_3_TYPE(TRB_TYPE_CMD_NOOP); 2774 2775 return DoCommand(&trb); 2776 } 2777 2778 2779 status_t 2780 XHCI::EnableSlot(uint8* slot) 2781 { 2782 TRACE("Enable Slot\n"); 2783 xhci_trb trb; 2784 trb.address = 0; 2785 trb.status = 0; 2786 trb.flags = TRB_3_TYPE(TRB_TYPE_ENABLE_SLOT); 2787 2788 status_t status = DoCommand(&trb); 2789 if (status != B_OK) 2790 return status; 2791 2792 *slot = TRB_3_SLOT_GET(trb.flags); 2793 return *slot != 0 ? B_OK : B_BAD_VALUE; 2794 } 2795 2796 2797 status_t 2798 XHCI::DisableSlot(uint8 slot) 2799 { 2800 TRACE("Disable Slot\n"); 2801 xhci_trb trb; 2802 trb.address = 0; 2803 trb.status = 0; 2804 trb.flags = TRB_3_TYPE(TRB_TYPE_DISABLE_SLOT) | TRB_3_SLOT(slot); 2805 2806 return DoCommand(&trb); 2807 } 2808 2809 2810 status_t 2811 XHCI::SetAddress(uint64 inputContext, bool bsr, uint8 slot) 2812 { 2813 TRACE("Set Address\n"); 2814 xhci_trb trb; 2815 trb.address = inputContext; 2816 trb.status = 0; 2817 trb.flags = TRB_3_TYPE(TRB_TYPE_ADDRESS_DEVICE) | TRB_3_SLOT(slot); 2818 2819 if (bsr) 2820 trb.flags |= TRB_3_BSR_BIT; 2821 2822 return DoCommand(&trb); 2823 } 2824 2825 2826 status_t 2827 XHCI::ConfigureEndpoint(uint64 inputContext, bool deconfigure, uint8 slot) 2828 { 2829 TRACE("Configure Endpoint\n"); 2830 xhci_trb trb; 2831 trb.address = inputContext; 2832 trb.status = 0; 2833 trb.flags = TRB_3_TYPE(TRB_TYPE_CONFIGURE_ENDPOINT) | TRB_3_SLOT(slot); 2834 2835 if (deconfigure) 2836 trb.flags |= TRB_3_DCEP_BIT; 2837 2838 return DoCommand(&trb); 2839 } 2840 2841 2842 status_t 2843 XHCI::EvaluateContext(uint64 inputContext, uint8 slot) 2844 { 2845 TRACE("Evaluate Context\n"); 2846 xhci_trb trb; 2847 trb.address = inputContext; 2848 trb.status = 0; 2849 trb.flags = TRB_3_TYPE(TRB_TYPE_EVALUATE_CONTEXT) | TRB_3_SLOT(slot); 2850 2851 return DoCommand(&trb); 2852 } 2853 2854 2855 status_t 2856 XHCI::ResetEndpoint(bool preserve, xhci_endpoint* endpoint) 2857 { 2858 TRACE("Reset Endpoint\n"); 2859 2860 switch (_GetEndpointState(endpoint)) { 2861 case ENDPOINT_STATE_STOPPED: 2862 TRACE("Reset Endpoint: already stopped"); 2863 return B_OK; 2864 case ENDPOINT_STATE_HALTED: 2865 TRACE("Reset Endpoint: warning, weird state!"); 2866 default: 2867 break; 2868 } 2869 2870 xhci_trb trb; 2871 trb.address = 0; 2872 trb.status = 0; 2873 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_ENDPOINT) 2874 | TRB_3_SLOT(endpoint->device->slot) | TRB_3_ENDPOINT(endpoint->id + 1); 2875 if (preserve) 2876 trb.flags |= TRB_3_PRSV_BIT; 2877 2878 return DoCommand(&trb); 2879 } 2880 2881 2882 status_t 2883 XHCI::StopEndpoint(bool suspend, xhci_endpoint* endpoint) 2884 { 2885 TRACE("Stop Endpoint\n"); 2886 2887 switch (_GetEndpointState(endpoint)) { 2888 case ENDPOINT_STATE_HALTED: 2889 TRACE("Stop Endpoint: error, halted"); 2890 return B_DEV_STALLED; 2891 case ENDPOINT_STATE_STOPPED: 2892 TRACE("Stop Endpoint: already stopped"); 2893 return B_OK; 2894 default: 2895 break; 2896 } 2897 2898 xhci_trb trb; 2899 trb.address = 0; 2900 trb.status = 0; 2901 trb.flags = TRB_3_TYPE(TRB_TYPE_STOP_ENDPOINT) 2902 | TRB_3_SLOT(endpoint->device->slot) | TRB_3_ENDPOINT(endpoint->id + 1); 2903 if (suspend) 2904 trb.flags |= TRB_3_SUSPEND_ENDPOINT_BIT; 2905 2906 return DoCommand(&trb); 2907 } 2908 2909 2910 status_t 2911 XHCI::SetTRDequeue(uint64 dequeue, uint16 stream, uint8 endpoint, uint8 slot) 2912 { 2913 TRACE("Set TR Dequeue\n"); 2914 xhci_trb trb; 2915 trb.address = dequeue | ENDPOINT_2_DCS_BIT; 2916 // The DCS bit is copied from the address field as in ConfigureEndpoint. 2917 // (XHCI 1.2 § 4.6.10 p142.) 2918 trb.status = TRB_2_STREAM(stream); 2919 trb.flags = TRB_3_TYPE(TRB_TYPE_SET_TR_DEQUEUE) 2920 | TRB_3_SLOT(slot) | TRB_3_ENDPOINT(endpoint); 2921 2922 return DoCommand(&trb); 2923 } 2924 2925 2926 status_t 2927 XHCI::ResetDevice(uint8 slot) 2928 { 2929 TRACE("Reset Device\n"); 2930 xhci_trb trb; 2931 trb.address = 0; 2932 trb.status = 0; 2933 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_DEVICE) | TRB_3_SLOT(slot); 2934 2935 return DoCommand(&trb); 2936 } 2937 2938 2939 int32 2940 XHCI::EventThread(void* data) 2941 { 2942 ((XHCI *)data)->CompleteEvents(); 2943 return B_OK; 2944 } 2945 2946 2947 void 2948 XHCI::CompleteEvents() 2949 { 2950 while (!fStopThreads) { 2951 if (acquire_sem(fEventSem) < B_OK) 2952 continue; 2953 2954 // eat up sems that have been released by multiple interrupts 2955 int32 semCount = 0; 2956 get_sem_count(fEventSem, &semCount); 2957 if (semCount > 0) 2958 acquire_sem_etc(fEventSem, semCount, B_RELATIVE_TIMEOUT, 0); 2959 2960 ProcessEvents(); 2961 } 2962 } 2963 2964 2965 void 2966 XHCI::ProcessEvents() 2967 { 2968 // Use mutex_trylock first, in case we are in KDL. 2969 MutexLocker locker(fEventLock, mutex_trylock(&fEventLock) == B_OK); 2970 if (!locker.IsLocked()) { 2971 // We failed to get the lock. This really should not happen. 2972 TRACE_ERROR("failed to acquire event lock!\n"); 2973 return; 2974 } 2975 2976 uint16 i = fEventIdx; 2977 uint8 j = fEventCcs; 2978 uint8 t = 2; 2979 2980 while (1) { 2981 uint32 temp = B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags); 2982 uint8 event = TRB_3_TYPE_GET(temp); 2983 TRACE("event[%u] = %u (0x%016" B_PRIx64 " 0x%08" B_PRIx32 " 0x%08" 2984 B_PRIx32 ")\n", i, event, fEventRing[i].address, 2985 fEventRing[i].status, B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags)); 2986 uint8 k = (temp & TRB_3_CYCLE_BIT) ? 1 : 0; 2987 if (j != k) 2988 break; 2989 2990 switch (event) { 2991 case TRB_TYPE_COMMAND_COMPLETION: 2992 HandleCmdComplete(&fEventRing[i]); 2993 break; 2994 case TRB_TYPE_TRANSFER: 2995 HandleTransferComplete(&fEventRing[i]); 2996 break; 2997 case TRB_TYPE_PORT_STATUS_CHANGE: 2998 TRACE("port change detected\n"); 2999 break; 3000 default: 3001 TRACE_ERROR("Unhandled event = %u\n", event); 3002 break; 3003 } 3004 3005 i++; 3006 if (i == XHCI_MAX_EVENTS) { 3007 i = 0; 3008 j ^= 1; 3009 if (!--t) 3010 break; 3011 } 3012 } 3013 3014 fEventIdx = i; 3015 fEventCcs = j; 3016 3017 uint64 addr = fErst->rs_addr + i * sizeof(xhci_trb); 3018 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)addr | ERDP_BUSY); 3019 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(addr >> 32)); 3020 } 3021 3022 3023 int32 3024 XHCI::FinishThread(void* data) 3025 { 3026 ((XHCI *)data)->FinishTransfers(); 3027 return B_OK; 3028 } 3029 3030 3031 void 3032 XHCI::FinishTransfers() 3033 { 3034 while (!fStopThreads) { 3035 if (acquire_sem(fFinishTransfersSem) < B_OK) 3036 continue; 3037 3038 // eat up sems that have been released by multiple interrupts 3039 int32 semCount = 0; 3040 get_sem_count(fFinishTransfersSem, &semCount); 3041 if (semCount > 0) 3042 acquire_sem_etc(fFinishTransfersSem, semCount, B_RELATIVE_TIMEOUT, 0); 3043 3044 mutex_lock(&fFinishedLock); 3045 TRACE("finishing transfers\n"); 3046 while (fFinishedHead != NULL) { 3047 xhci_td* td = fFinishedHead; 3048 fFinishedHead = td->next; 3049 td->next = NULL; 3050 mutex_unlock(&fFinishedLock); 3051 3052 TRACE("finishing transfer td %p\n", td); 3053 3054 Transfer* transfer = td->transfer; 3055 if (transfer == NULL) { 3056 // No transfer? Quick way out. 3057 FreeDescriptor(td); 3058 mutex_lock(&fFinishedLock); 3059 continue; 3060 } 3061 3062 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 3063 3064 status_t callbackStatus = B_OK; 3065 const uint8 completionCode = td->trb_completion_code; 3066 switch (completionCode) { 3067 case COMP_SHORT_PACKET: 3068 case COMP_SUCCESS: 3069 callbackStatus = B_OK; 3070 break; 3071 case COMP_DATA_BUFFER: 3072 callbackStatus = directionIn ? B_DEV_DATA_OVERRUN 3073 : B_DEV_DATA_UNDERRUN; 3074 break; 3075 case COMP_BABBLE: 3076 callbackStatus = directionIn ? B_DEV_FIFO_OVERRUN 3077 : B_DEV_FIFO_UNDERRUN; 3078 break; 3079 case COMP_USB_TRANSACTION: 3080 callbackStatus = B_DEV_CRC_ERROR; 3081 break; 3082 case COMP_STALL: 3083 callbackStatus = B_DEV_STALLED; 3084 break; 3085 default: 3086 callbackStatus = B_DEV_STALLED; 3087 break; 3088 } 3089 3090 size_t actualLength = transfer->FragmentLength(); 3091 if (completionCode != COMP_SUCCESS) { 3092 actualLength = td->td_transferred; 3093 if (td->td_transferred == -1) 3094 actualLength = transfer->FragmentLength() - td->trb_left; 3095 TRACE("transfer not successful, actualLength=%" B_PRIuSIZE "\n", 3096 actualLength); 3097 } 3098 3099 usb_isochronous_data* isochronousData = transfer->IsochronousData(); 3100 if (isochronousData != NULL) { 3101 size_t packetSize = transfer->DataLength() 3102 / isochronousData->packet_count, 3103 left = actualLength; 3104 for (uint32 i = 0; i < isochronousData->packet_count; i++) { 3105 size_t size = min_c(packetSize, left); 3106 isochronousData->packet_descriptors[i].actual_length = size; 3107 isochronousData->packet_descriptors[i].status = (size > 0) 3108 ? B_OK : B_DEV_FIFO_UNDERRUN; 3109 left -= size; 3110 } 3111 } 3112 3113 if (callbackStatus == B_OK && directionIn && actualLength > 0) { 3114 TRACE("copying in iov count %ld\n", transfer->VectorCount()); 3115 status_t status = transfer->PrepareKernelAccess(); 3116 if (status == B_OK) { 3117 ReadDescriptor(td, transfer->Vector(), 3118 transfer->VectorCount()); 3119 } else { 3120 callbackStatus = status; 3121 } 3122 } 3123 3124 FreeDescriptor(td); 3125 3126 // this transfer may still have data left 3127 bool finished = true; 3128 transfer->AdvanceByFragment(actualLength); 3129 if (completionCode == COMP_SUCCESS 3130 && transfer->FragmentLength() > 0) { 3131 TRACE("still %" B_PRIuSIZE " bytes left on transfer\n", 3132 transfer->FragmentLength()); 3133 callbackStatus = SubmitTransfer(transfer); 3134 finished = (callbackStatus != B_OK); 3135 } 3136 if (finished) { 3137 // The actualLength was already handled in AdvanceByFragment. 3138 transfer->Finished(callbackStatus, 0); 3139 delete transfer; 3140 } 3141 3142 mutex_lock(&fFinishedLock); 3143 } 3144 mutex_unlock(&fFinishedLock); 3145 } 3146 } 3147 3148 3149 inline void 3150 XHCI::WriteOpReg(uint32 reg, uint32 value) 3151 { 3152 *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg) = value; 3153 } 3154 3155 3156 inline uint32 3157 XHCI::ReadOpReg(uint32 reg) 3158 { 3159 return *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg); 3160 } 3161 3162 3163 inline status_t 3164 XHCI::WaitOpBits(uint32 reg, uint32 mask, uint32 expected) 3165 { 3166 int loops = 0; 3167 uint32 value = ReadOpReg(reg); 3168 while ((value & mask) != expected) { 3169 snooze(1000); 3170 value = ReadOpReg(reg); 3171 if (loops == 100) { 3172 TRACE("delay waiting on reg 0x%" B_PRIX32 " match 0x%" B_PRIX32 3173 " (0x%" B_PRIX32 ")\n", reg, expected, mask); 3174 } else if (loops > 250) { 3175 TRACE_ERROR("timeout waiting on reg 0x%" B_PRIX32 3176 " match 0x%" B_PRIX32 " (0x%" B_PRIX32 ")\n", reg, expected, 3177 mask); 3178 return B_ERROR; 3179 } 3180 loops++; 3181 } 3182 return B_OK; 3183 } 3184 3185 3186 inline uint32 3187 XHCI::ReadCapReg32(uint32 reg) 3188 { 3189 return *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg); 3190 } 3191 3192 3193 inline void 3194 XHCI::WriteCapReg32(uint32 reg, uint32 value) 3195 { 3196 *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg) = value; 3197 } 3198 3199 3200 inline uint32 3201 XHCI::ReadRunReg32(uint32 reg) 3202 { 3203 return *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg); 3204 } 3205 3206 3207 inline void 3208 XHCI::WriteRunReg32(uint32 reg, uint32 value) 3209 { 3210 *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg) = value; 3211 } 3212 3213 3214 inline uint32 3215 XHCI::ReadDoorReg32(uint32 reg) 3216 { 3217 return *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg); 3218 } 3219 3220 3221 inline void 3222 XHCI::WriteDoorReg32(uint32 reg, uint32 value) 3223 { 3224 *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg) = value; 3225 } 3226 3227 3228 inline addr_t 3229 XHCI::_OffsetContextAddr(addr_t p) 3230 { 3231 if (fContextSizeShift == 1) { 3232 // each structure is page aligned, each pointer is 32 bits aligned 3233 uint32 offset = p & ((B_PAGE_SIZE - 1) & ~31U); 3234 p += offset; 3235 } 3236 return p; 3237 } 3238 3239 inline uint32 3240 XHCI::_ReadContext(uint32* p) 3241 { 3242 p = (uint32*)_OffsetContextAddr((addr_t)p); 3243 return *p; 3244 } 3245 3246 3247 inline void 3248 XHCI::_WriteContext(uint32* p, uint32 value) 3249 { 3250 p = (uint32*)_OffsetContextAddr((addr_t)p); 3251 *p = value; 3252 } 3253 3254 3255 inline uint64 3256 XHCI::_ReadContext(uint64* p) 3257 { 3258 p = (uint64*)_OffsetContextAddr((addr_t)p); 3259 return *p; 3260 } 3261 3262 3263 inline void 3264 XHCI::_WriteContext(uint64* p, uint64 value) 3265 { 3266 p = (uint64*)_OffsetContextAddr((addr_t)p); 3267 *p = value; 3268 } 3269