1 /* 2 * Copyright 2011-2019, Haiku, Inc. All rights reserved. 3 * Distributed under the terms of the MIT License. 4 * 5 * Authors: 6 * Augustin Cavalier <waddlesplash> 7 * Jian Chiang <j.jian.chiang@gmail.com> 8 * Jérôme Duval <jerome.duval@gmail.com> 9 * Akshay Jaggi <akshay1994.leo@gmail.com> 10 * Michael Lotz <mmlr@mlotz.ch> 11 */ 12 13 14 #include <module.h> 15 #include <PCI.h> 16 #include <PCI_x86.h> 17 #include <USB3.h> 18 #include <KernelExport.h> 19 20 #include <util/AutoLock.h> 21 22 #include "xhci.h" 23 24 #define USB_MODULE_NAME "xhci" 25 26 pci_module_info *XHCI::sPCIModule = NULL; 27 pci_x86_module_info *XHCI::sPCIx86Module = NULL; 28 29 30 static int32 31 xhci_std_ops(int32 op, ...) 32 { 33 switch (op) { 34 case B_MODULE_INIT: 35 TRACE_MODULE("xhci init module\n"); 36 return B_OK; 37 case B_MODULE_UNINIT: 38 TRACE_MODULE("xhci uninit module\n"); 39 return B_OK; 40 } 41 42 return EINVAL; 43 } 44 45 46 static const char* 47 xhci_error_string(uint32 error) 48 { 49 switch (error) { 50 case COMP_INVALID: return "Invalid"; 51 case COMP_SUCCESS: return "Success"; 52 case COMP_DATA_BUFFER: return "Data buffer"; 53 case COMP_BABBLE: return "Babble detected"; 54 case COMP_USB_TRANSACTION: return "USB transaction"; 55 case COMP_TRB: return "TRB"; 56 case COMP_STALL: return "Stall"; 57 case COMP_RESOURCE: return "Resource"; 58 case COMP_BANDWIDTH: return "Bandwidth"; 59 case COMP_NO_SLOTS: return "No slots"; 60 case COMP_INVALID_STREAM: return "Invalid stream"; 61 case COMP_SLOT_NOT_ENABLED: return "Slot not enabled"; 62 case COMP_ENDPOINT_NOT_ENABLED: return "Endpoint not enabled"; 63 case COMP_SHORT_PACKET: return "Short packet"; 64 case COMP_RING_UNDERRUN: return "Ring underrun"; 65 case COMP_RING_OVERRUN: return "Ring overrun"; 66 case COMP_VF_RING_FULL: return "VF Event Ring Full"; 67 case COMP_PARAMETER: return "Parameter"; 68 case COMP_BANDWIDTH_OVERRUN: return "Bandwidth overrun"; 69 case COMP_CONTEXT_STATE: return "Context state"; 70 case COMP_NO_PING_RESPONSE: return "No ping response"; 71 case COMP_EVENT_RING_FULL: return "Event ring full"; 72 case COMP_INCOMPATIBLE_DEVICE: return "Incompatible device"; 73 case COMP_MISSED_SERVICE: return "Missed service"; 74 case COMP_COMMAND_RING_STOPPED: return "Command ring stopped"; 75 case COMP_COMMAND_ABORTED: return "Command aborted"; 76 case COMP_STOPPED: return "Stopped"; 77 case COMP_LENGTH_INVALID: return "Length invalid"; 78 case COMP_MAX_EXIT_LATENCY: return "Max exit latency too large"; 79 case COMP_ISOC_OVERRUN: return "Isoch buffer overrun"; 80 case COMP_EVENT_LOST: return "Event lost"; 81 case COMP_UNDEFINED: return "Undefined"; 82 case COMP_INVALID_STREAM_ID: return "Invalid stream ID"; 83 case COMP_SECONDARY_BANDWIDTH: return "Secondary bandwidth"; 84 case COMP_SPLIT_TRANSACTION: return "Split transaction"; 85 86 default: return "Undefined"; 87 } 88 } 89 90 91 usb_host_controller_info xhci_module = { 92 { 93 "busses/usb/xhci", 94 0, 95 xhci_std_ops 96 }, 97 NULL, 98 XHCI::AddTo 99 }; 100 101 102 module_info *modules[] = { 103 (module_info *)&xhci_module, 104 NULL 105 }; 106 107 108 status_t 109 XHCI::AddTo(Stack *stack) 110 { 111 if (!sPCIModule) { 112 status_t status = get_module(B_PCI_MODULE_NAME, 113 (module_info **)&sPCIModule); 114 if (status < B_OK) { 115 TRACE_MODULE_ERROR("getting pci module failed! 0x%08" B_PRIx32 116 "\n", status); 117 return status; 118 } 119 } 120 121 TRACE_MODULE("searching devices\n"); 122 bool found = false; 123 pci_info *item = new(std::nothrow) pci_info; 124 if (item == NULL) { 125 sPCIModule = NULL; 126 put_module(B_PCI_MODULE_NAME); 127 return B_NO_MEMORY; 128 } 129 130 // Try to get the PCI x86 module as well so we can enable possible MSIs. 131 if (sPCIx86Module == NULL && get_module(B_PCI_X86_MODULE_NAME, 132 (module_info **)&sPCIx86Module) != B_OK) { 133 // If it isn't there, that's not critical though. 134 TRACE_MODULE_ERROR("failed to get pci x86 module\n"); 135 sPCIx86Module = NULL; 136 } 137 138 for (int32 i = 0; sPCIModule->get_nth_pci_info(i, item) >= B_OK; i++) { 139 if (item->class_base == PCI_serial_bus && item->class_sub == PCI_usb 140 && item->class_api == PCI_usb_xhci) { 141 TRACE_MODULE("found device at PCI:%d:%d:%d\n", 142 item->bus, item->device, item->function); 143 XHCI *bus = new(std::nothrow) XHCI(item, stack); 144 if (bus == NULL) { 145 delete item; 146 sPCIModule = NULL; 147 put_module(B_PCI_MODULE_NAME); 148 if (sPCIx86Module != NULL) 149 put_module(B_PCI_X86_MODULE_NAME); 150 return B_NO_MEMORY; 151 } 152 153 // The bus will put the PCI modules when it is destroyed, so get 154 // them again to increase their reference count. 155 get_module(B_PCI_MODULE_NAME, (module_info **)&sPCIModule); 156 if (sPCIx86Module != NULL) 157 get_module(B_PCI_X86_MODULE_NAME, (module_info **)&sPCIx86Module); 158 159 if (bus->InitCheck() < B_OK) { 160 TRACE_MODULE_ERROR("bus failed init check\n"); 161 delete bus; 162 continue; 163 } 164 165 // the bus took it away 166 item = new(std::nothrow) pci_info; 167 168 if (bus->Start() != B_OK) { 169 delete bus; 170 continue; 171 } 172 found = true; 173 } 174 } 175 176 // The modules will have been gotten again if we successfully 177 // initialized a bus, so we should put them here. 178 put_module(B_PCI_MODULE_NAME); 179 if (sPCIx86Module != NULL) 180 put_module(B_PCI_X86_MODULE_NAME); 181 182 if (!found) 183 TRACE_MODULE_ERROR("no devices found\n"); 184 delete item; 185 return found ? B_OK : ENODEV; 186 } 187 188 189 XHCI::XHCI(pci_info *info, Stack *stack) 190 : BusManager(stack), 191 fRegisterArea(-1), 192 fRegisters(NULL), 193 fPCIInfo(info), 194 fStack(stack), 195 fIRQ(0), 196 fUseMSI(false), 197 fErstArea(-1), 198 fDcbaArea(-1), 199 fCmdCompSem(-1), 200 fStopThreads(false), 201 fRootHub(NULL), 202 fRootHubAddress(0), 203 fPortCount(0), 204 fSlotCount(0), 205 fScratchpadCount(0), 206 fContextSizeShift(0), 207 fFinishedHead(NULL), 208 fFinishTransfersSem(-1), 209 fFinishThread(-1), 210 fEventSem(-1), 211 fEventThread(-1), 212 fEventIdx(0), 213 fCmdIdx(0), 214 fEventCcs(1), 215 fCmdCcs(1) 216 { 217 B_INITIALIZE_SPINLOCK(&fSpinlock); 218 mutex_init(&fFinishedLock, "XHCI finished transfers"); 219 mutex_init(&fEventLock, "XHCI event handler"); 220 221 if (BusManager::InitCheck() < B_OK) { 222 TRACE_ERROR("bus manager failed to init\n"); 223 return; 224 } 225 226 TRACE("constructing new XHCI host controller driver\n"); 227 fInitOK = false; 228 229 // enable busmaster and memory mapped access 230 uint16 command = sPCIModule->read_pci_config(fPCIInfo->bus, 231 fPCIInfo->device, fPCIInfo->function, PCI_command, 2); 232 command &= ~(PCI_command_io | PCI_command_int_disable); 233 command |= PCI_command_master | PCI_command_memory; 234 235 sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device, 236 fPCIInfo->function, PCI_command, 2, command); 237 238 // map the registers (low + high for 64-bit when requested) 239 phys_addr_t physicalAddress = fPCIInfo->u.h0.base_registers[0]; 240 physicalAddress &= PCI_address_memory_32_mask; 241 if ((fPCIInfo->u.h0.base_register_flags[0] & 0xC) == PCI_address_type_64) 242 physicalAddress += (phys_addr_t)fPCIInfo->u.h0.base_registers[1] << 32; 243 244 size_t mapSize = fPCIInfo->u.h0.base_register_sizes[0]; 245 246 TRACE("map physical memory %08" B_PRIxPHYSADDR ", size: %" B_PRIuSIZE "\n", 247 physicalAddress, mapSize); 248 249 fRegisterArea = map_physical_memory("XHCI memory mapped registers", 250 physicalAddress, mapSize, B_ANY_KERNEL_BLOCK_ADDRESS, 251 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 252 (void **)&fRegisters); 253 if (fRegisterArea < B_OK) { 254 TRACE_ERROR("failed to map register memory\n"); 255 return; 256 } 257 258 // determine the register offsets 259 fCapabilityRegisterOffset = 0; 260 fOperationalRegisterOffset = HCI_CAPLENGTH(ReadCapReg32(XHCI_HCI_CAPLENGTH)); 261 fRuntimeRegisterOffset = ReadCapReg32(XHCI_RTSOFF) & ~0x1F; 262 fDoorbellRegisterOffset = ReadCapReg32(XHCI_DBOFF) & ~0x3; 263 264 TRACE("mapped registers: %p\n", fRegisters); 265 TRACE("operational register offset: %" B_PRId32 "\n", fOperationalRegisterOffset); 266 TRACE("runtime register offset: %" B_PRId32 "\n", fRuntimeRegisterOffset); 267 TRACE("doorbell register offset: %" B_PRId32 "\n", fDoorbellRegisterOffset); 268 269 TRACE_ALWAYS("interface version: 0x%04" B_PRIx32 "\n", 270 HCI_VERSION(ReadCapReg32(XHCI_HCI_VERSION))); 271 TRACE_ALWAYS("structural parameters: 1:0x%08" B_PRIx32 " 2:0x%08" 272 B_PRIx32 " 3:0x%08" B_PRIx32 "\n", ReadCapReg32(XHCI_HCSPARAMS1), 273 ReadCapReg32(XHCI_HCSPARAMS2), ReadCapReg32(XHCI_HCSPARAMS3)); 274 275 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 276 if (cparams == 0xffffffff) 277 return; 278 TRACE_ALWAYS("capability params: 0x%08" B_PRIx32 "\n", cparams); 279 280 // if 64 bytes context structures, then 1 281 fContextSizeShift = HCC_CSZ(cparams); 282 283 // Assume ownership of the controller from the BIOS. 284 uint32 eec = 0xffffffff; 285 uint32 eecp = HCS0_XECP(cparams) << 2; 286 for (; eecp != 0 && XECP_NEXT(eec); eecp += XECP_NEXT(eec) << 2) { 287 TRACE("eecp register: 0x%08" B_PRIx32 "\n", eecp); 288 289 eec = ReadCapReg32(eecp); 290 if (XECP_ID(eec) != XHCI_LEGSUP_CAPID) 291 continue; 292 293 if (eec & XHCI_LEGSUP_BIOSOWNED) { 294 TRACE_ALWAYS("the host controller is bios owned, claiming" 295 " ownership\n"); 296 WriteCapReg32(eecp, eec | XHCI_LEGSUP_OSOWNED); 297 298 for (int32 i = 0; i < 20; i++) { 299 eec = ReadCapReg32(eecp); 300 301 if ((eec & XHCI_LEGSUP_BIOSOWNED) == 0) 302 break; 303 304 TRACE_ALWAYS("controller is still bios owned, waiting\n"); 305 snooze(50000); 306 } 307 308 if (eec & XHCI_LEGSUP_BIOSOWNED) { 309 TRACE_ERROR("bios won't give up control over the host " 310 "controller (ignoring)\n"); 311 } else if (eec & XHCI_LEGSUP_OSOWNED) { 312 TRACE_ALWAYS("successfully took ownership of the host " 313 "controller\n"); 314 } 315 316 // Force off the BIOS owned flag, and clear all SMIs. Some BIOSes 317 // do indicate a successful handover but do not remove their SMIs 318 // and then freeze the system when interrupts are generated. 319 WriteCapReg32(eecp, eec & ~XHCI_LEGSUP_BIOSOWNED); 320 } 321 break; 322 } 323 uint32 legctlsts = ReadCapReg32(eecp + XHCI_LEGCTLSTS); 324 legctlsts &= XHCI_LEGCTLSTS_DISABLE_SMI; 325 legctlsts |= XHCI_LEGCTLSTS_EVENTS_SMI; 326 WriteCapReg32(eecp + XHCI_LEGCTLSTS, legctlsts); 327 328 // On Intel's Panther Point and Lynx Point Chipset taking ownership 329 // of EHCI owned ports, is what we do here. 330 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL) { 331 switch (fPCIInfo->device_id) { 332 case PCI_DEVICE_INTEL_PANTHER_POINT_XHCI: 333 case PCI_DEVICE_INTEL_LYNX_POINT_XHCI: 334 case PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI: 335 case PCI_DEVICE_INTEL_BAYTRAIL_XHCI: 336 case PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI: 337 case PCI_DEVICE_INTEL_WILDCAT_POINT_LP_XHCI: 338 _SwitchIntelPorts(); 339 break; 340 } 341 } 342 343 // halt the host controller 344 if (ControllerHalt() < B_OK) { 345 return; 346 } 347 348 // reset the host controller 349 if (ControllerReset() < B_OK) { 350 TRACE_ERROR("host controller failed to reset\n"); 351 return; 352 } 353 354 fCmdCompSem = create_sem(0, "XHCI Command Complete"); 355 fFinishTransfersSem = create_sem(0, "XHCI Finish Transfers"); 356 fEventSem = create_sem(0, "XHCI Event"); 357 if (fFinishTransfersSem < B_OK || fCmdCompSem < B_OK || fEventSem < B_OK) { 358 TRACE_ERROR("failed to create semaphores\n"); 359 return; 360 } 361 362 // create finisher service thread 363 fFinishThread = spawn_kernel_thread(FinishThread, "xhci finish thread", 364 B_NORMAL_PRIORITY, (void *)this); 365 resume_thread(fFinishThread); 366 367 // create finisher service thread 368 fEventThread = spawn_kernel_thread(EventThread, "xhci event thread", 369 B_NORMAL_PRIORITY, (void *)this); 370 resume_thread(fEventThread); 371 372 // Find the right interrupt vector, using MSIs if available. 373 fIRQ = fPCIInfo->u.h0.interrupt_line; 374 if (sPCIx86Module != NULL && sPCIx86Module->get_msi_count(fPCIInfo->bus, 375 fPCIInfo->device, fPCIInfo->function) >= 1) { 376 uint8 msiVector = 0; 377 if (sPCIx86Module->configure_msi(fPCIInfo->bus, fPCIInfo->device, 378 fPCIInfo->function, 1, &msiVector) == B_OK 379 && sPCIx86Module->enable_msi(fPCIInfo->bus, fPCIInfo->device, 380 fPCIInfo->function) == B_OK) { 381 TRACE_ALWAYS("using message signaled interrupts\n"); 382 fIRQ = msiVector; 383 fUseMSI = true; 384 } 385 } 386 387 if (fIRQ == 0 || fIRQ == 0xFF) { 388 TRACE_MODULE_ERROR("device PCI:%d:%d:%d was assigned an invalid IRQ\n", 389 fPCIInfo->bus, fPCIInfo->device, fPCIInfo->function); 390 return; 391 } 392 393 // Install the interrupt handler 394 TRACE("installing interrupt handler\n"); 395 install_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this, 0); 396 397 memset(fPortSpeeds, 0, sizeof(fPortSpeeds)); 398 memset(fPortSlots, 0, sizeof(fPortSlots)); 399 memset(fDevices, 0, sizeof(fDevices)); 400 401 fInitOK = true; 402 TRACE("XHCI host controller driver constructed\n"); 403 } 404 405 406 XHCI::~XHCI() 407 { 408 TRACE("tear down XHCI host controller driver\n"); 409 410 WriteOpReg(XHCI_CMD, 0); 411 412 int32 result = 0; 413 fStopThreads = true; 414 delete_sem(fCmdCompSem); 415 delete_sem(fFinishTransfersSem); 416 delete_sem(fEventSem); 417 wait_for_thread(fFinishThread, &result); 418 wait_for_thread(fEventThread, &result); 419 420 mutex_destroy(&fFinishedLock); 421 mutex_destroy(&fEventLock); 422 423 remove_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this); 424 425 delete_area(fRegisterArea); 426 delete_area(fErstArea); 427 for (uint32 i = 0; i < fScratchpadCount; i++) 428 delete_area(fScratchpadArea[i]); 429 delete_area(fDcbaArea); 430 431 if (fUseMSI && sPCIx86Module != NULL) { 432 sPCIx86Module->disable_msi(fPCIInfo->bus, 433 fPCIInfo->device, fPCIInfo->function); 434 sPCIx86Module->unconfigure_msi(fPCIInfo->bus, 435 fPCIInfo->device, fPCIInfo->function); 436 } 437 put_module(B_PCI_MODULE_NAME); 438 if (sPCIx86Module != NULL) 439 put_module(B_PCI_X86_MODULE_NAME); 440 } 441 442 443 void 444 XHCI::_SwitchIntelPorts() 445 { 446 TRACE("Intel xHC Controller\n"); 447 TRACE("Looking for EHCI owned ports\n"); 448 uint32 ports = sPCIModule->read_pci_config(fPCIInfo->bus, 449 fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_USB3PRM, 4); 450 TRACE("Superspeed Ports: 0x%" B_PRIx32 "\n", ports); 451 sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device, 452 fPCIInfo->function, XHCI_INTEL_USB3_PSSEN, 4, ports); 453 ports = sPCIModule->read_pci_config(fPCIInfo->bus, 454 fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_USB3_PSSEN, 4); 455 TRACE("Superspeed ports now under XHCI : 0x%" B_PRIx32 "\n", ports); 456 ports = sPCIModule->read_pci_config(fPCIInfo->bus, 457 fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_USB2PRM, 4); 458 TRACE("USB 2.0 Ports : 0x%" B_PRIx32 "\n", ports); 459 sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device, 460 fPCIInfo->function, XHCI_INTEL_XUSB2PR, 4, ports); 461 ports = sPCIModule->read_pci_config(fPCIInfo->bus, 462 fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_XUSB2PR, 4); 463 TRACE("USB 2.0 ports now under XHCI: 0x%" B_PRIx32 "\n", ports); 464 } 465 466 467 status_t 468 XHCI::Start() 469 { 470 TRACE_ALWAYS("starting XHCI host controller\n"); 471 TRACE("usbcmd: 0x%08" B_PRIx32 "; usbsts: 0x%08" B_PRIx32 "\n", 472 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 473 474 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 475 TRACE("Start() failed STS_CNR\n"); 476 } 477 478 if ((ReadOpReg(XHCI_CMD) & CMD_RUN) != 0) { 479 TRACE_ERROR("Start() warning, starting running XHCI controller!\n"); 480 } 481 482 if ((ReadOpReg(XHCI_PAGESIZE) & (1 << 0)) == 0) { 483 TRACE_ERROR("Controller does not support 4K page size.\n"); 484 return B_ERROR; 485 } 486 487 // read port count from capability register 488 uint32 capabilities = ReadCapReg32(XHCI_HCSPARAMS1); 489 fPortCount = HCS_MAX_PORTS(capabilities); 490 if (fPortCount == 0) { 491 TRACE_ERROR("Invalid number of ports: %u\n", fPortCount); 492 return B_ERROR; 493 } 494 495 fSlotCount = HCS_MAX_SLOTS(capabilities); 496 if (fSlotCount > XHCI_MAX_DEVICES) 497 fSlotCount = XHCI_MAX_DEVICES; 498 WriteOpReg(XHCI_CONFIG, fSlotCount); 499 500 // find out which protocol is used for each port 501 uint8 portFound = 0; 502 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 503 uint32 eec = 0xffffffff; 504 uint32 eecp = HCS0_XECP(cparams) << 2; 505 for (; eecp != 0 && XECP_NEXT(eec) && portFound < fPortCount; 506 eecp += XECP_NEXT(eec) << 2) { 507 eec = ReadCapReg32(eecp); 508 if (XECP_ID(eec) != XHCI_SUPPORTED_PROTOCOLS_CAPID) 509 continue; 510 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) > 3) 511 continue; 512 uint32 temp = ReadCapReg32(eecp + 8); 513 uint32 offset = XHCI_SUPPORTED_PROTOCOLS_1_OFFSET(temp); 514 uint32 count = XHCI_SUPPORTED_PROTOCOLS_1_COUNT(temp); 515 if (offset == 0 || count == 0) 516 continue; 517 offset--; 518 for (uint32 i = offset; i < offset + count; i++) { 519 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) == 0x3) 520 fPortSpeeds[i] = USB_SPEED_SUPER; 521 else 522 fPortSpeeds[i] = USB_SPEED_HIGHSPEED; 523 TRACE("speed for port %" B_PRId32 " is %s\n", i, 524 fPortSpeeds[i] == USB_SPEED_SUPER ? "super" : "high"); 525 } 526 portFound += count; 527 } 528 529 uint32 params2 = ReadCapReg32(XHCI_HCSPARAMS2); 530 fScratchpadCount = HCS_MAX_SC_BUFFERS(params2); 531 if (fScratchpadCount > XHCI_MAX_SCRATCHPADS) { 532 TRACE_ERROR("Invalid number of scratchpads: %" B_PRIu32 "\n", 533 fScratchpadCount); 534 return B_ERROR; 535 } 536 537 uint32 params3 = ReadCapReg32(XHCI_HCSPARAMS3); 538 fExitLatMax = HCS_U1_DEVICE_LATENCY(params3) 539 + HCS_U2_DEVICE_LATENCY(params3); 540 541 // clear interrupts & disable device notifications 542 WriteOpReg(XHCI_STS, ReadOpReg(XHCI_STS)); 543 WriteOpReg(XHCI_DNCTRL, 0); 544 545 // allocate Device Context Base Address array 546 phys_addr_t dmaAddress; 547 fDcbaArea = fStack->AllocateArea((void **)&fDcba, &dmaAddress, 548 sizeof(*fDcba), "DCBA Area"); 549 if (fDcbaArea < B_OK) { 550 TRACE_ERROR("unable to create the DCBA area\n"); 551 return B_ERROR; 552 } 553 memset(fDcba, 0, sizeof(*fDcba)); 554 memset(fScratchpadArea, 0, sizeof(fScratchpadArea)); 555 memset(fScratchpad, 0, sizeof(fScratchpad)); 556 557 // setting the first address to the scratchpad array address 558 fDcba->baseAddress[0] = dmaAddress 559 + offsetof(struct xhci_device_context_array, scratchpad); 560 561 // fill up the scratchpad array with scratchpad pages 562 for (uint32 i = 0; i < fScratchpadCount; i++) { 563 phys_addr_t scratchDmaAddress; 564 fScratchpadArea[i] = fStack->AllocateArea((void **)&fScratchpad[i], 565 &scratchDmaAddress, B_PAGE_SIZE, "Scratchpad Area"); 566 if (fScratchpadArea[i] < B_OK) { 567 TRACE_ERROR("unable to create the scratchpad area\n"); 568 return B_ERROR; 569 } 570 fDcba->scratchpad[i] = scratchDmaAddress; 571 } 572 573 TRACE("setting DCBAAP %" B_PRIxPHYSADDR "\n", dmaAddress); 574 WriteOpReg(XHCI_DCBAAP_LO, (uint32)dmaAddress); 575 WriteOpReg(XHCI_DCBAAP_HI, (uint32)(dmaAddress >> 32)); 576 577 // allocate Event Ring Segment Table 578 uint8 *addr; 579 fErstArea = fStack->AllocateArea((void **)&addr, &dmaAddress, 580 (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 581 + sizeof(xhci_erst_element), 582 "USB XHCI ERST CMD_RING and EVENT_RING Area"); 583 584 if (fErstArea < B_OK) { 585 TRACE_ERROR("unable to create the ERST AND RING area\n"); 586 delete_area(fDcbaArea); 587 return B_ERROR; 588 } 589 fErst = (xhci_erst_element *)addr; 590 memset(fErst, 0, (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 591 + sizeof(xhci_erst_element)); 592 593 // fill with Event Ring Segment Base Address and Event Ring Segment Size 594 fErst->rs_addr = dmaAddress + sizeof(xhci_erst_element); 595 fErst->rs_size = XHCI_MAX_EVENTS; 596 fErst->rsvdz = 0; 597 598 addr += sizeof(xhci_erst_element); 599 fEventRing = (xhci_trb *)addr; 600 addr += XHCI_MAX_EVENTS * sizeof(xhci_trb); 601 fCmdRing = (xhci_trb *)addr; 602 603 TRACE("setting ERST size\n"); 604 WriteRunReg32(XHCI_ERSTSZ(0), XHCI_ERSTS_SET(1)); 605 606 TRACE("setting ERDP addr = 0x%" B_PRIx64 "\n", fErst->rs_addr); 607 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)fErst->rs_addr); 608 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(fErst->rs_addr >> 32)); 609 610 TRACE("setting ERST base addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 611 WriteRunReg32(XHCI_ERSTBA_LO(0), (uint32)dmaAddress); 612 WriteRunReg32(XHCI_ERSTBA_HI(0), (uint32)(dmaAddress >> 32)); 613 614 dmaAddress += sizeof(xhci_erst_element) + XHCI_MAX_EVENTS 615 * sizeof(xhci_trb); 616 617 // Make sure the Command Ring is stopped 618 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 619 TRACE_ALWAYS("Command Ring is running, send stop/cancel\n"); 620 WriteOpReg(XHCI_CRCR_LO, CRCR_CS); 621 WriteOpReg(XHCI_CRCR_HI, 0); 622 WriteOpReg(XHCI_CRCR_LO, CRCR_CA); 623 WriteOpReg(XHCI_CRCR_HI, 0); 624 snooze(1000); 625 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 626 TRACE_ERROR("Command Ring still running after stop/cancel\n"); 627 } 628 } 629 TRACE("setting CRCR addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 630 WriteOpReg(XHCI_CRCR_LO, (uint32)dmaAddress | CRCR_RCS); 631 WriteOpReg(XHCI_CRCR_HI, (uint32)(dmaAddress >> 32)); 632 // link trb 633 fCmdRing[XHCI_MAX_COMMANDS - 1].address = dmaAddress; 634 635 TRACE("setting interrupt rate\n"); 636 637 // Setting IMOD below 0x3F8 on Intel Lynx Point can cause IRQ lockups 638 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL 639 && (fPCIInfo->device_id == PCI_DEVICE_INTEL_PANTHER_POINT_XHCI 640 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_XHCI 641 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI 642 || fPCIInfo->device_id == PCI_DEVICE_INTEL_BAYTRAIL_XHCI 643 || fPCIInfo->device_id == PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI)) { 644 WriteRunReg32(XHCI_IMOD(0), 0x000003f8); // 4000 irq/s 645 } else { 646 WriteRunReg32(XHCI_IMOD(0), 0x000001f4); // 8000 irq/s 647 } 648 649 TRACE("enabling interrupt\n"); 650 WriteRunReg32(XHCI_IMAN(0), ReadRunReg32(XHCI_IMAN(0)) | IMAN_INTR_ENA); 651 652 WriteOpReg(XHCI_CMD, CMD_RUN | CMD_INTE | CMD_HSEE); 653 654 // wait for start up state 655 if (WaitOpBits(XHCI_STS, STS_HCH, 0) != B_OK) { 656 TRACE_ERROR("HCH start up timeout\n"); 657 } 658 659 fRootHubAddress = AllocateAddress(); 660 fRootHub = new(std::nothrow) XHCIRootHub(RootObject(), fRootHubAddress); 661 if (!fRootHub) { 662 TRACE_ERROR("no memory to allocate root hub\n"); 663 return B_NO_MEMORY; 664 } 665 666 if (fRootHub->InitCheck() < B_OK) { 667 TRACE_ERROR("root hub failed init check\n"); 668 return fRootHub->InitCheck(); 669 } 670 671 SetRootHub(fRootHub); 672 673 TRACE_ALWAYS("successfully started the controller\n"); 674 #ifdef TRACE_USB 675 TRACE("No-Op test...\n"); 676 status_t noopResult = Noop(); 677 TRACE("No-Op %ssuccessful\n", noopResult < B_OK ? "un" : ""); 678 #endif 679 680 //DumpRing(fCmdRing, (XHCI_MAX_COMMANDS - 1)); 681 682 return BusManager::Start(); 683 } 684 685 686 status_t 687 XHCI::SubmitTransfer(Transfer *transfer) 688 { 689 // short circuit the root hub 690 if (transfer->TransferPipe()->DeviceAddress() == fRootHubAddress) 691 return fRootHub->ProcessTransfer(this, transfer); 692 693 TRACE("SubmitTransfer()\n"); 694 Pipe *pipe = transfer->TransferPipe(); 695 if ((pipe->Type() & USB_OBJECT_ISO_PIPE) != 0) 696 return B_UNSUPPORTED; 697 if ((pipe->Type() & USB_OBJECT_CONTROL_PIPE) != 0) 698 return SubmitControlRequest(transfer); 699 return SubmitNormalRequest(transfer); 700 } 701 702 703 status_t 704 XHCI::SubmitControlRequest(Transfer *transfer) 705 { 706 Pipe *pipe = transfer->TransferPipe(); 707 usb_request_data *requestData = transfer->RequestData(); 708 bool directionIn = (requestData->RequestType & USB_REQTYPE_DEVICE_IN) != 0; 709 710 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 711 if (endpoint == NULL) { 712 TRACE_ERROR("invalid endpoint!\n"); 713 return B_BAD_VALUE; 714 } 715 status_t status = transfer->InitKernelAccess(); 716 if (status != B_OK) 717 return status; 718 719 TRACE("SubmitControlRequest() length %d\n", requestData->Length); 720 721 xhci_td *descriptor = CreateDescriptor(3, 1, requestData->Length); 722 if (descriptor == NULL) 723 return B_NO_MEMORY; 724 descriptor->transfer = transfer; 725 726 // Setup Stage 727 uint8 index = 0; 728 memcpy(&descriptor->trbs[index].address, requestData, 729 sizeof(usb_request_data)); 730 descriptor->trbs[index].status = TRB_2_IRQ(0) | TRB_2_BYTES(8); 731 descriptor->trbs[index].flags 732 = TRB_3_TYPE(TRB_TYPE_SETUP_STAGE) | TRB_3_IDT_BIT | TRB_3_CYCLE_BIT; 733 if (requestData->Length > 0) { 734 descriptor->trbs[index].flags |= 735 directionIn ? TRB_3_TRT_IN : TRB_3_TRT_OUT; 736 } 737 738 index++; 739 740 // Data Stage (if any) 741 if (requestData->Length > 0) { 742 descriptor->trbs[index].address = descriptor->buffer_addrs[0]; 743 descriptor->trbs[index].status = TRB_2_IRQ(0) 744 | TRB_2_BYTES(requestData->Length) 745 | TRB_2_TD_SIZE(0); 746 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_DATA_STAGE) 747 | (directionIn ? (TRB_3_DIR_IN | TRB_3_ISP_BIT) : 0) 748 | TRB_3_CYCLE_BIT; 749 750 if (!directionIn) { 751 transfer->PrepareKernelAccess(); 752 memcpy(descriptor->buffers[0], 753 (uint8 *)transfer->Vector()[0].iov_base, requestData->Length); 754 } 755 756 index++; 757 } 758 759 // Status Stage 760 descriptor->trbs[index].address = 0; 761 descriptor->trbs[index].status = TRB_2_IRQ(0); 762 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_STATUS_STAGE) 763 | ((directionIn && requestData->Length > 0) ? 0 : TRB_3_DIR_IN) 764 | TRB_3_IOC_BIT | TRB_3_CYCLE_BIT; 765 // Status Stage is an OUT transfer when the device is sending data. 766 // (XHCI 1.1 § 4.11.2.2 Table 4-6 p205.) 767 768 descriptor->trb_used = index + 1; 769 770 status = _LinkDescriptorForPipe(descriptor, endpoint); 771 if (status != B_OK) { 772 FreeDescriptor(descriptor); 773 return status; 774 } 775 TRACE("SubmitControlRequest() request linked\n"); 776 777 return B_OK; 778 } 779 780 781 status_t 782 XHCI::SubmitNormalRequest(Transfer *transfer) 783 { 784 TRACE("SubmitNormalRequest() length %ld\n", transfer->DataLength()); 785 786 Pipe *pipe = transfer->TransferPipe(); 787 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 788 if (endpoint == NULL) 789 return B_BAD_VALUE; 790 bool directionIn = (pipe->Direction() == Pipe::In); 791 792 status_t status = transfer->InitKernelAccess(); 793 if (status != B_OK) 794 return status; 795 796 // Compute the size to use for the TRBs, and then how many TRBs 797 // of this size we will need. We always need at least 1, of course. 798 const size_t dataLength = transfer->DataLength(), 799 maxPacketSize = pipe->MaxPacketSize(), 800 packetsPerTrb = 4; 801 const size_t trbSize = packetsPerTrb * maxPacketSize; 802 int32 trbCount = (dataLength + trbSize - 1) / trbSize; 803 804 xhci_td *td = CreateDescriptor(trbCount, trbCount, trbSize); 805 if (td == NULL) 806 return B_NO_MEMORY; 807 808 // Normal Stage 809 size_t remaining = dataLength; 810 int32 remainingPackets = (remaining - trbSize) / maxPacketSize; 811 for (int32 i = 0; i < trbCount; i++) { 812 // The "TD Size" field of a transfer TRB indicates the number of 813 // remaining maximum-size *packets* in this TD, *not* including the 814 // packets in the current TRB, and capped at 31 if there are more 815 // than 31 packets remaining in the TD. (XHCI 1.1 § 4.11.2.4 p210.) 816 int32 tdSize = remainingPackets > 31 ? 31 : remainingPackets; 817 if (tdSize < 0) 818 tdSize = 0; 819 int32 trbLength = remaining < trbSize ? remaining : trbSize; 820 821 td->trbs[i].address = td->buffer_addrs[i]; 822 td->trbs[i].status = TRB_2_IRQ(0) 823 | TRB_2_BYTES(trbLength) 824 | TRB_2_TD_SIZE(tdSize); 825 td->trbs[i].flags = TRB_3_TYPE(TRB_TYPE_NORMAL) 826 | TRB_3_CYCLE_BIT | TRB_3_CHAIN_BIT 827 | (directionIn ? TRB_3_ISP_BIT : 0); 828 829 td->trb_used++; 830 remaining -= trbLength; 831 remainingPackets -= packetsPerTrb; 832 } 833 834 // Set the IOC (Interrupt On Completion) bit so that we will get an event 835 // and interrupt for this TRB as the transfer will be finished. 836 // (XHCI 1.1 § 6.4.1.1 Table 6-22 p443.) 837 td->trbs[td->trb_used - 1].flags |= TRB_3_IOC_BIT; 838 839 // Set the ENT (Evaluate Next TRB) bit, so that the HC will not switch 840 // contexts before evaluating the Link TRB that _LinkDescriptorForPipe 841 // will insert, as otherwise there would be a race between us freeing 842 // and unlinking the descriptor, and the HC evaluating the Link TRB 843 // and thus getting back onto the main ring. 844 // 845 // Note that we *do not* unset the CHAIN bit in this TRB, thus including 846 // the Link TRB in this TD formally, which is required when using the 847 // ENT bit. (XHCI 1.1 § 4.12.3 p241.) 848 td->trbs[td->trb_used - 1].flags |= TRB_3_ENT_BIT; 849 850 if (!directionIn) { 851 TRACE("copying out iov count %ld\n", transfer->VectorCount()); 852 transfer->PrepareKernelAccess(); 853 WriteDescriptor(td, transfer->Vector(), transfer->VectorCount()); 854 } 855 856 td->transfer = transfer; 857 status = _LinkDescriptorForPipe(td, endpoint); 858 if (status != B_OK) { 859 FreeDescriptor(td); 860 return status; 861 } 862 TRACE("SubmitNormalRequest() request linked\n"); 863 864 return B_OK; 865 } 866 867 868 status_t 869 XHCI::CancelQueuedTransfers(Pipe *pipe, bool force) 870 { 871 TRACE_ALWAYS("cancel queued transfers for pipe %p (%d)\n", pipe, 872 pipe->EndpointAddress()); 873 874 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 875 if (endpoint == NULL || endpoint->trbs == NULL) { 876 // Someone's de-allocated this pipe or endpoint in the meantime. 877 // (Possibly AllocateDevice failed, and we were the temporary pipe.) 878 return B_NO_INIT; 879 } 880 881 MutexLocker endpointLocker(endpoint->lock); 882 883 if (endpoint->td_head == NULL) { 884 // There aren't any currently pending transfers to cancel. 885 return B_OK; 886 } 887 888 // Get the head TD from the endpoint. We don't want to free the TDs while 889 // holding the endpoint lock, as the callbacks could potentially cause 890 // deadlocks. 891 xhci_td* td_head = endpoint->td_head; 892 endpoint->td_head = NULL; 893 894 if (StopEndpoint(false, endpoint->id + 1, endpoint->device->slot) == B_OK) { 895 // Clear the endpoint's TRBs. 896 memset(endpoint->trbs, 0, sizeof(xhci_trb) * XHCI_MAX_TRANSFERS); 897 endpoint->used = 0; 898 endpoint->current = 0; 899 900 // Set dequeue pointer location to the beginning of the ring. 901 SetTRDequeue(endpoint->trb_addr, 0, endpoint->id + 1, 902 endpoint->device->slot); 903 904 // We don't need to do anything else to restart the ring, as it will resume 905 // operation as normal upon the next doorbell. (XHCI 1.1 § 4.6.9 p132.) 906 } else { 907 // We couldn't stop the endpoint. Most likely the device has been 908 // removed and the endpoint was stopped by the hardware. 909 TRACE("CancelQueuedTransfers: could not stop endpoint\n"); 910 } 911 912 endpointLocker.Unlock(); 913 914 xhci_td* td; 915 while ((td = td_head) != NULL) { 916 td_head = td_head->next; 917 918 // We can't cancel or delete transfers under "force", as they probably 919 // are not safe to use anymore. 920 if (!force && td->transfer != NULL) { 921 td->transfer->Finished(B_CANCELED, 0); 922 delete td->transfer; 923 td->transfer = NULL; 924 } 925 FreeDescriptor(td); 926 } 927 928 return B_OK; 929 } 930 931 932 status_t 933 XHCI::StartDebugTransfer(Transfer *transfer) 934 { 935 Pipe *pipe = transfer->TransferPipe(); 936 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 937 if (endpoint == NULL) 938 return B_BAD_VALUE; 939 940 // Check all locks that we are going to hit when running transfers. 941 if (mutex_trylock(&endpoint->lock) != B_OK) 942 return B_WOULD_BLOCK; 943 if (mutex_trylock(&fFinishedLock) != B_OK) { 944 mutex_unlock(&endpoint->lock); 945 return B_WOULD_BLOCK; 946 } 947 if (mutex_trylock(&fEventLock) != B_OK) { 948 mutex_unlock(&endpoint->lock); 949 mutex_unlock(&fFinishedLock); 950 return B_WOULD_BLOCK; 951 } 952 mutex_unlock(&endpoint->lock); 953 mutex_unlock(&fFinishedLock); 954 mutex_unlock(&fEventLock); 955 956 status_t status = SubmitTransfer(transfer); 957 if (status != B_OK) 958 return status; 959 960 // The endpoint's head TD is the TD of the just-submitted transfer. 961 // Just like EHCI, abuse the callback cookie to hold the TD pointer. 962 transfer->SetCallback(NULL, endpoint->td_head); 963 964 return B_OK; 965 } 966 967 968 status_t 969 XHCI::CheckDebugTransfer(Transfer *transfer) 970 { 971 xhci_td *transfer_td = (xhci_td *)transfer->CallbackCookie(); 972 if (transfer_td == NULL) 973 return B_NO_INIT; 974 975 // Process events once, and then look for it in the finished list. 976 ProcessEvents(); 977 xhci_td *previous = NULL; 978 for (xhci_td *td = fFinishedHead; td != NULL; td = td->next) { 979 if (td != transfer_td) { 980 previous = td; 981 continue; 982 } 983 984 // We've found it! 985 if (previous == NULL) { 986 fFinishedHead = fFinishedHead->next; 987 } else { 988 previous->next = td->next; 989 } 990 991 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 992 status_t status = (td->trb_completion_code == COMP_SUCCESS 993 || td->trb_completion_code == COMP_SHORT_PACKET) ? B_OK : B_ERROR; 994 995 if (status == B_OK && directionIn) 996 ReadDescriptor(td, transfer->Vector(), transfer->VectorCount()); 997 998 FreeDescriptor(td); 999 transfer->SetCallback(NULL, NULL); 1000 return status; 1001 } 1002 1003 // We didn't find it. 1004 spin(75); 1005 return B_DEV_PENDING; 1006 } 1007 1008 1009 void 1010 XHCI::CancelDebugTransfer(Transfer *transfer) 1011 { 1012 while (CheckDebugTransfer(transfer) == B_DEV_PENDING) 1013 spin(100); 1014 } 1015 1016 1017 status_t 1018 XHCI::NotifyPipeChange(Pipe *pipe, usb_change change) 1019 { 1020 TRACE("pipe change %d for pipe %p (%d)\n", change, pipe, 1021 pipe->EndpointAddress()); 1022 1023 switch (change) { 1024 case USB_CHANGE_CREATED: 1025 return _InsertEndpointForPipe(pipe); 1026 case USB_CHANGE_DESTROYED: 1027 return _RemoveEndpointForPipe(pipe); 1028 1029 case USB_CHANGE_PIPE_POLICY_CHANGED: 1030 // We don't care about these, at least for now. 1031 return B_OK; 1032 } 1033 1034 TRACE_ERROR("unknown pipe change!\n"); 1035 return B_UNSUPPORTED; 1036 } 1037 1038 1039 xhci_td * 1040 XHCI::CreateDescriptor(uint32 trbCount, uint32 bufferCount, size_t bufferSize) 1041 { 1042 const bool inKDL = debug_debugger_running(); 1043 1044 xhci_td *result; 1045 if (!inKDL) { 1046 result = (xhci_td*)calloc(1, sizeof(xhci_td)); 1047 } else { 1048 // Just use the physical memory allocator while in KDL; it's less 1049 // secure than using the regular heap, but it's easier to deal with. 1050 phys_addr_t dummy; 1051 fStack->AllocateChunk((void **)&result, &dummy, sizeof(xhci_td)); 1052 } 1053 1054 if (result == NULL) { 1055 TRACE_ERROR("failed to allocate a transfer descriptor\n"); 1056 return NULL; 1057 } 1058 1059 // We always allocate 1 more TRB than requested, so that 1060 // _LinkDescriptorForPipe() has room to insert a link TRB. 1061 trbCount++; 1062 if (fStack->AllocateChunk((void **)&result->trbs, &result->trb_addr, 1063 (trbCount * sizeof(xhci_trb))) < B_OK) { 1064 TRACE_ERROR("failed to allocate TRBs\n"); 1065 FreeDescriptor(result); 1066 return NULL; 1067 } 1068 result->trb_count = trbCount; 1069 result->trb_used = 0; 1070 1071 if (bufferSize > 0) { 1072 // Due to how the USB stack allocates physical memory, we can't just 1073 // request one large chunk the size of the transfer, and so instead we 1074 // create a series of buffers as requested by our caller. 1075 1076 // We store the buffer pointers and addresses in one memory block. 1077 if (!inKDL) { 1078 result->buffers = (void**)calloc(bufferCount, 1079 (sizeof(void*) + sizeof(phys_addr_t))); 1080 } else { 1081 phys_addr_t dummy; 1082 fStack->AllocateChunk((void **)&result->buffers, &dummy, 1083 bufferCount * (sizeof(void*) + sizeof(phys_addr_t))); 1084 } 1085 if (result->buffers == NULL) { 1086 TRACE_ERROR("unable to allocate space for buffer infos\n"); 1087 FreeDescriptor(result); 1088 return NULL; 1089 } 1090 result->buffer_addrs = (phys_addr_t*)&result->buffers[bufferCount]; 1091 1092 // Optimization: If the requested total size of all buffers is less 1093 // than 32*B_PAGE_SIZE (the maximum size that the physical memory 1094 // allocator can handle), we allocate only one buffer and segment it. 1095 size_t totalSize = bufferSize * bufferCount; 1096 if (totalSize < (32 * B_PAGE_SIZE)) { 1097 if (fStack->AllocateChunk(&result->buffers[0], 1098 &result->buffer_addrs[0], totalSize) < B_OK) { 1099 TRACE_ERROR("unable to allocate space for large buffer (size %ld)\n", 1100 totalSize); 1101 FreeDescriptor(result); 1102 return NULL; 1103 } 1104 for (uint32 i = 1; i < bufferCount; i++) { 1105 result->buffers[i] = (void*)((addr_t)(result->buffers[i - 1]) 1106 + bufferSize); 1107 result->buffer_addrs[i] = result->buffer_addrs[i - 1] 1108 + bufferSize; 1109 } 1110 } else { 1111 // Otherwise, we allocate each buffer individually. 1112 for (uint32 i = 0; i < bufferCount; i++) { 1113 if (fStack->AllocateChunk(&result->buffers[i], 1114 &result->buffer_addrs[i], bufferSize) < B_OK) { 1115 TRACE_ERROR("unable to allocate space for the buffer (size %ld)\n", 1116 bufferSize); 1117 FreeDescriptor(result); 1118 return NULL; 1119 } 1120 } 1121 } 1122 } else { 1123 result->buffers = NULL; 1124 result->buffer_addrs = NULL; 1125 } 1126 result->buffer_size = bufferSize; 1127 result->buffer_count = bufferCount; 1128 1129 // Initialize all other fields. 1130 result->transfer = NULL; 1131 result->trb_completion_code = 0; 1132 result->trb_left = 0; 1133 result->next = NULL; 1134 1135 TRACE("CreateDescriptor allocated %p, buffer_size %ld, buffer_count %ld\n", 1136 result, result->buffer_size, result->buffer_count); 1137 1138 return result; 1139 } 1140 1141 1142 void 1143 XHCI::FreeDescriptor(xhci_td *descriptor) 1144 { 1145 if (descriptor == NULL) 1146 return; 1147 1148 const bool inKDL = debug_debugger_running(); 1149 1150 if (descriptor->trbs != NULL) { 1151 fStack->FreeChunk(descriptor->trbs, descriptor->trb_addr, 1152 (descriptor->trb_count * sizeof(xhci_trb))); 1153 } 1154 if (descriptor->buffers != NULL) { 1155 size_t totalSize = descriptor->buffer_size * descriptor->buffer_count; 1156 if (totalSize < (32 * B_PAGE_SIZE)) { 1157 // This was allocated as one contiguous buffer. 1158 fStack->FreeChunk(descriptor->buffers[0], descriptor->buffer_addrs[0], 1159 totalSize); 1160 } else { 1161 for (uint32 i = 0; i < descriptor->buffer_count; i++) { 1162 if (descriptor->buffers[i] == NULL) 1163 continue; 1164 fStack->FreeChunk(descriptor->buffers[i], descriptor->buffer_addrs[i], 1165 descriptor->buffer_size); 1166 } 1167 } 1168 1169 if (!inKDL) { 1170 free(descriptor->buffers); 1171 } else { 1172 fStack->FreeChunk(descriptor->buffers, 0, 1173 descriptor->buffer_count * (sizeof(void*) + sizeof(phys_addr_t))); 1174 } 1175 } 1176 1177 if (!inKDL) 1178 free(descriptor); 1179 else 1180 fStack->FreeChunk(descriptor, 0, sizeof(xhci_td)); 1181 } 1182 1183 1184 size_t 1185 XHCI::WriteDescriptor(xhci_td *descriptor, iovec *vector, size_t vectorCount) 1186 { 1187 size_t written = 0; 1188 1189 size_t bufIdx = 0, bufUsed = 0; 1190 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1191 size_t length = vector[vecIdx].iov_len; 1192 1193 while (length > 0 && bufIdx < descriptor->buffer_count) { 1194 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1195 memcpy((uint8 *)descriptor->buffers[bufIdx] + bufUsed, 1196 (uint8 *)vector[vecIdx].iov_base + (vector[vecIdx].iov_len - length), 1197 toCopy); 1198 1199 written += toCopy; 1200 bufUsed += toCopy; 1201 length -= toCopy; 1202 if (bufUsed == descriptor->buffer_size) { 1203 bufIdx++; 1204 bufUsed = 0; 1205 } 1206 } 1207 } 1208 1209 TRACE("wrote descriptor (%" B_PRIuSIZE " bytes)\n", written); 1210 return written; 1211 } 1212 1213 1214 size_t 1215 XHCI::ReadDescriptor(xhci_td *descriptor, iovec *vector, size_t vectorCount) 1216 { 1217 size_t read = 0; 1218 1219 size_t bufIdx = 0, bufUsed = 0; 1220 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1221 size_t length = vector[vecIdx].iov_len; 1222 1223 while (length > 0 && bufIdx < descriptor->buffer_count) { 1224 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1225 memcpy((uint8 *)vector[vecIdx].iov_base + (vector[vecIdx].iov_len - length), 1226 (uint8 *)descriptor->buffers[bufIdx] + bufUsed, toCopy); 1227 1228 read += toCopy; 1229 bufUsed += toCopy; 1230 length -= toCopy; 1231 if (bufUsed == descriptor->buffer_size) { 1232 bufIdx++; 1233 bufUsed = 0; 1234 } 1235 } 1236 } 1237 1238 TRACE("read descriptor (%" B_PRIuSIZE " bytes)\n", read); 1239 return read; 1240 } 1241 1242 1243 Device * 1244 XHCI::AllocateDevice(Hub *parent, int8 hubAddress, uint8 hubPort, 1245 usb_speed speed) 1246 { 1247 TRACE("AllocateDevice hubAddress %d hubPort %d speed %d\n", hubAddress, 1248 hubPort, speed); 1249 1250 uint8 slot = XHCI_MAX_SLOTS; 1251 if (EnableSlot(&slot) != B_OK) { 1252 TRACE_ERROR("AllocateDevice() failed enable slot\n"); 1253 return NULL; 1254 } 1255 1256 if (slot == 0 || slot > fSlotCount) { 1257 TRACE_ERROR("AllocateDevice() bad slot\n"); 1258 return NULL; 1259 } 1260 1261 if (fDevices[slot].state != XHCI_STATE_DISABLED) { 1262 TRACE_ERROR("AllocateDevice() slot already used\n"); 1263 return NULL; 1264 } 1265 1266 struct xhci_device *device = &fDevices[slot]; 1267 memset(device, 0, sizeof(struct xhci_device)); 1268 device->state = XHCI_STATE_ENABLED; 1269 device->slot = slot; 1270 1271 device->input_ctx_area = fStack->AllocateArea((void **)&device->input_ctx, 1272 &device->input_ctx_addr, sizeof(*device->input_ctx) << fContextSizeShift, 1273 "XHCI input context"); 1274 if (device->input_ctx_area < B_OK) { 1275 TRACE_ERROR("unable to create a input context area\n"); 1276 device->state = XHCI_STATE_DISABLED; 1277 return NULL; 1278 } 1279 1280 memset(device->input_ctx, 0, sizeof(*device->input_ctx) << fContextSizeShift); 1281 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1282 _WriteContext(&device->input_ctx->input.addFlags, 3); 1283 1284 uint32 route = 0; 1285 uint8 routePort = hubPort; 1286 uint8 rhPort = hubPort; 1287 for (Device *hubDevice = parent; hubDevice != RootObject(); 1288 hubDevice = (Device *)hubDevice->Parent()) { 1289 1290 rhPort = routePort; 1291 if (hubDevice->Parent() == RootObject()) 1292 break; 1293 route *= 16; 1294 if (hubPort > 15) 1295 route += 15; 1296 else 1297 route += routePort; 1298 1299 routePort = hubDevice->HubPort(); 1300 } 1301 1302 // Get speed of port, only if device connected to root hub port 1303 // else we have to rely on value reported by the Hub Explore thread 1304 if (route == 0) { 1305 GetPortSpeed(hubPort - 1, &speed); 1306 TRACE("speed updated %d\n", speed); 1307 } 1308 1309 uint32 dwslot0 = SLOT_0_NUM_ENTRIES(1) | SLOT_0_ROUTE(route); 1310 1311 // add the speed 1312 switch (speed) { 1313 case USB_SPEED_LOWSPEED: 1314 dwslot0 |= SLOT_0_SPEED(2); 1315 break; 1316 case USB_SPEED_HIGHSPEED: 1317 dwslot0 |= SLOT_0_SPEED(3); 1318 break; 1319 case USB_SPEED_FULLSPEED: 1320 dwslot0 |= SLOT_0_SPEED(1); 1321 break; 1322 case USB_SPEED_SUPER: 1323 dwslot0 |= SLOT_0_SPEED(4); 1324 break; 1325 default: 1326 TRACE_ERROR("unknown usb speed\n"); 1327 break; 1328 } 1329 1330 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1331 // TODO enable power save 1332 _WriteContext(&device->input_ctx->slot.dwslot1, SLOT_1_RH_PORT(rhPort)); 1333 uint32 dwslot2 = SLOT_2_IRQ_TARGET(0); 1334 1335 // If LS/FS device connected to non-root HS device 1336 if (route != 0 && parent->Speed() == USB_SPEED_HIGHSPEED 1337 && (speed == USB_SPEED_LOWSPEED || speed == USB_SPEED_FULLSPEED)) { 1338 struct xhci_device *parenthub = (struct xhci_device *) 1339 parent->ControllerCookie(); 1340 dwslot2 |= SLOT_2_PORT_NUM(hubPort); 1341 dwslot2 |= SLOT_2_TT_HUB_SLOT(parenthub->slot); 1342 } 1343 1344 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1345 1346 _WriteContext(&device->input_ctx->slot.dwslot3, SLOT_3_SLOT_STATE(0) 1347 | SLOT_3_DEVICE_ADDRESS(0)); 1348 1349 TRACE("slot 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 1350 "\n", _ReadContext(&device->input_ctx->slot.dwslot0), 1351 _ReadContext(&device->input_ctx->slot.dwslot1), 1352 _ReadContext(&device->input_ctx->slot.dwslot2), 1353 _ReadContext(&device->input_ctx->slot.dwslot3)); 1354 1355 device->device_ctx_area = fStack->AllocateArea((void **)&device->device_ctx, 1356 &device->device_ctx_addr, sizeof(*device->device_ctx) << fContextSizeShift, 1357 "XHCI device context"); 1358 if (device->device_ctx_area < B_OK) { 1359 TRACE_ERROR("unable to create a device context area\n"); 1360 delete_area(device->input_ctx_area); 1361 memset(device, 0, sizeof(xhci_device)); 1362 device->state = XHCI_STATE_DISABLED; 1363 return NULL; 1364 } 1365 memset(device->device_ctx, 0, sizeof(*device->device_ctx) << fContextSizeShift); 1366 1367 device->trb_area = fStack->AllocateArea((void **)&device->trbs, 1368 &device->trb_addr, sizeof(xhci_trb) * (XHCI_MAX_ENDPOINTS - 1) 1369 * XHCI_MAX_TRANSFERS, "XHCI endpoint trbs"); 1370 if (device->trb_area < B_OK) { 1371 TRACE_ERROR("unable to create a device trbs area\n"); 1372 delete_area(device->input_ctx_area); 1373 delete_area(device->device_ctx_area); 1374 memset(device, 0, sizeof(xhci_device)); 1375 device->state = XHCI_STATE_DISABLED; 1376 return NULL; 1377 } 1378 1379 // set up slot pointer to device context 1380 fDcba->baseAddress[slot] = device->device_ctx_addr; 1381 1382 size_t maxPacketSize; 1383 switch (speed) { 1384 case USB_SPEED_LOWSPEED: 1385 case USB_SPEED_FULLSPEED: 1386 maxPacketSize = 8; 1387 break; 1388 case USB_SPEED_HIGHSPEED: 1389 maxPacketSize = 64; 1390 break; 1391 default: 1392 maxPacketSize = 512; 1393 break; 1394 } 1395 1396 // configure the Control endpoint 0 1397 if (ConfigureEndpoint(slot, 0, USB_OBJECT_CONTROL_PIPE, false, 1398 device->trb_addr, 0, maxPacketSize, maxPacketSize & 0x7ff, 1399 speed) != B_OK) { 1400 TRACE_ERROR("unable to configure default control endpoint\n"); 1401 delete_area(device->input_ctx_area); 1402 delete_area(device->device_ctx_area); 1403 delete_area(device->trb_area); 1404 memset(device, 0, sizeof(xhci_device)); 1405 device->state = XHCI_STATE_DISABLED; 1406 return NULL; 1407 } 1408 1409 mutex_init(&device->endpoints[0].lock, "xhci endpoint lock"); 1410 device->endpoints[0].device = device; 1411 device->endpoints[0].id = 0; 1412 device->endpoints[0].td_head = NULL; 1413 device->endpoints[0].used = 0; 1414 device->endpoints[0].current = 0; 1415 device->endpoints[0].trbs = device->trbs; 1416 device->endpoints[0].trb_addr = device->trb_addr; 1417 1418 // device should get to addressed state (bsr = 0) 1419 if (SetAddress(device->input_ctx_addr, false, slot) != B_OK) { 1420 TRACE_ERROR("unable to set address\n"); 1421 delete_area(device->input_ctx_area); 1422 delete_area(device->device_ctx_area); 1423 delete_area(device->trb_area); 1424 memset(device, 0, sizeof(xhci_device)); 1425 device->state = XHCI_STATE_DISABLED; 1426 return NULL; 1427 } 1428 1429 device->state = XHCI_STATE_ADDRESSED; 1430 device->address = SLOT_3_DEVICE_ADDRESS_GET(_ReadContext( 1431 &device->device_ctx->slot.dwslot3)); 1432 1433 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", device->address, 1434 SLOT_3_SLOT_STATE_GET(_ReadContext( 1435 &device->device_ctx->slot.dwslot3))); 1436 TRACE("endpoint0 state 0x%08" B_PRIx32 "\n", 1437 ENDPOINT_0_STATE_GET(_ReadContext( 1438 &device->device_ctx->endpoints[0].dwendpoint0))); 1439 1440 // Create a temporary pipe with the new address 1441 ControlPipe pipe(parent); 1442 pipe.SetControllerCookie(&device->endpoints[0]); 1443 pipe.InitCommon(device->address + 1, 0, speed, Pipe::Default, maxPacketSize, 0, 1444 hubAddress, hubPort); 1445 1446 // Get the device descriptor 1447 // Just retrieve the first 8 bytes of the descriptor -> minimum supported 1448 // size of any device. It is enough because it includes the device type. 1449 1450 size_t actualLength = 0; 1451 usb_device_descriptor deviceDescriptor; 1452 1453 TRACE("getting the device descriptor\n"); 1454 status_t status = pipe.SendRequest( 1455 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_STANDARD, // type 1456 USB_REQUEST_GET_DESCRIPTOR, // request 1457 USB_DESCRIPTOR_DEVICE << 8, // value 1458 0, // index 1459 8, // length 1460 (void *)&deviceDescriptor, // buffer 1461 8, // buffer length 1462 &actualLength); // actual length 1463 1464 if (actualLength != 8) { 1465 TRACE_ERROR("error while getting the device descriptor: %s\n", 1466 strerror(status)); 1467 delete_area(device->input_ctx_area); 1468 delete_area(device->device_ctx_area); 1469 delete_area(device->trb_area); 1470 memset(device, 0, sizeof(xhci_device)); 1471 device->state = XHCI_STATE_DISABLED; 1472 return NULL; 1473 } 1474 1475 TRACE("device_class: %d device_subclass %d device_protocol %d\n", 1476 deviceDescriptor.device_class, deviceDescriptor.device_subclass, 1477 deviceDescriptor.device_protocol); 1478 1479 if (speed == USB_SPEED_FULLSPEED && deviceDescriptor.max_packet_size_0 != 8) { 1480 TRACE("Full speed device with different max packet size for Endpoint 0\n"); 1481 uint32 dwendpoint1 = _ReadContext( 1482 &device->input_ctx->endpoints[0].dwendpoint1); 1483 dwendpoint1 &= ~ENDPOINT_1_MAXPACKETSIZE(0xffff); 1484 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE( 1485 deviceDescriptor.max_packet_size_0); 1486 _WriteContext(&device->input_ctx->endpoints[0].dwendpoint1, 1487 dwendpoint1); 1488 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1489 _WriteContext(&device->input_ctx->input.addFlags, (1 << 1)); 1490 EvaluateContext(device->input_ctx_addr, device->slot); 1491 } 1492 1493 Device *deviceObject = NULL; 1494 if (deviceDescriptor.device_class == 0x09) { 1495 TRACE("creating new Hub\n"); 1496 TRACE("getting the hub descriptor\n"); 1497 size_t actualLength = 0; 1498 usb_hub_descriptor hubDescriptor; 1499 status = pipe.SendRequest( 1500 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_CLASS, // type 1501 USB_REQUEST_GET_DESCRIPTOR, // request 1502 USB_DESCRIPTOR_HUB << 8, // value 1503 0, // index 1504 sizeof(usb_hub_descriptor), // length 1505 (void *)&hubDescriptor, // buffer 1506 sizeof(usb_hub_descriptor), // buffer length 1507 &actualLength); 1508 1509 if (actualLength != sizeof(usb_hub_descriptor)) { 1510 TRACE_ERROR("error while getting the hub descriptor: %s\n", 1511 strerror(status)); 1512 delete_area(device->input_ctx_area); 1513 delete_area(device->device_ctx_area); 1514 delete_area(device->trb_area); 1515 memset(device, 0, sizeof(xhci_device)); 1516 device->state = XHCI_STATE_DISABLED; 1517 return NULL; 1518 } 1519 1520 uint32 dwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1521 dwslot0 |= SLOT_0_HUB_BIT; 1522 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1523 uint32 dwslot1 = _ReadContext(&device->input_ctx->slot.dwslot1); 1524 dwslot1 |= SLOT_1_NUM_PORTS(hubDescriptor.num_ports); 1525 _WriteContext(&device->input_ctx->slot.dwslot1, dwslot1); 1526 if (speed == USB_SPEED_HIGHSPEED) { 1527 uint32 dwslot2 = _ReadContext(&device->input_ctx->slot.dwslot2); 1528 dwslot2 |= SLOT_2_TT_TIME(HUB_TTT_GET(hubDescriptor.characteristics)); 1529 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1530 } 1531 1532 deviceObject = new(std::nothrow) Hub(parent, hubAddress, hubPort, 1533 deviceDescriptor, device->address + 1, speed, false, device); 1534 } else { 1535 TRACE("creating new device\n"); 1536 deviceObject = new(std::nothrow) Device(parent, hubAddress, hubPort, 1537 deviceDescriptor, device->address + 1, speed, false, device); 1538 } 1539 if (deviceObject == NULL || deviceObject->InitCheck() != B_OK) { 1540 if (deviceObject == NULL) { 1541 TRACE_ERROR("no memory to allocate device\n"); 1542 } else { 1543 TRACE_ERROR("device object failed to initialize\n"); 1544 } 1545 delete_area(device->input_ctx_area); 1546 delete_area(device->device_ctx_area); 1547 delete_area(device->trb_area); 1548 memset(device, 0, sizeof(xhci_device)); 1549 device->state = XHCI_STATE_DISABLED; 1550 return NULL; 1551 } 1552 1553 // We don't want to disable the default endpoint, naturally, which would 1554 // otherwise happen when this Pipe object is destroyed. 1555 pipe.SetControllerCookie(NULL); 1556 1557 fPortSlots[hubPort] = slot; 1558 TRACE("AllocateDevice() port %d slot %d\n", hubPort, slot); 1559 return deviceObject; 1560 } 1561 1562 1563 void 1564 XHCI::FreeDevice(Device *device) 1565 { 1566 uint8 slot = fPortSlots[device->HubPort()]; 1567 TRACE("FreeDevice() port %d slot %d\n", device->HubPort(), slot); 1568 1569 // Delete the device first, so it cleans up its pipes and tells us 1570 // what we need to destroy before we tear down our internal state. 1571 delete device; 1572 1573 DisableSlot(slot); 1574 fDcba->baseAddress[slot] = 0; 1575 fPortSlots[device->HubPort()] = 0; 1576 delete_area(fDevices[slot].trb_area); 1577 delete_area(fDevices[slot].input_ctx_area); 1578 delete_area(fDevices[slot].device_ctx_area); 1579 1580 memset(&fDevices[slot], 0, sizeof(xhci_device)); 1581 fDevices[slot].state = XHCI_STATE_DISABLED; 1582 } 1583 1584 1585 status_t 1586 XHCI::_InsertEndpointForPipe(Pipe *pipe) 1587 { 1588 TRACE("insert endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1589 1590 if (pipe->ControllerCookie() != NULL 1591 || pipe->Parent()->Type() != USB_OBJECT_DEVICE) { 1592 // default pipe is already referenced 1593 return B_OK; 1594 } 1595 1596 Device* usbDevice = (Device *)pipe->Parent(); 1597 struct xhci_device *device = (struct xhci_device *) 1598 usbDevice->ControllerCookie(); 1599 if (usbDevice->Parent() == RootObject()) 1600 return B_OK; 1601 if (device == NULL) { 1602 panic("_InsertEndpointForPipe device is NULL\n"); 1603 return B_NO_INIT; 1604 } 1605 1606 uint8 id = (2 * pipe->EndpointAddress() 1607 + (pipe->Direction() != Pipe::Out ? 1 : 0)) - 1; 1608 if (id >= XHCI_MAX_ENDPOINTS - 1) 1609 return B_BAD_VALUE; 1610 1611 if (id > 0) { 1612 uint32 devicedwslot0 = _ReadContext(&device->device_ctx->slot.dwslot0); 1613 if (SLOT_0_NUM_ENTRIES_GET(devicedwslot0) == 1) { 1614 uint32 inputdwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1615 inputdwslot0 &= ~(SLOT_0_NUM_ENTRIES(0x1f)); 1616 inputdwslot0 |= SLOT_0_NUM_ENTRIES(XHCI_MAX_ENDPOINTS - 1); 1617 _WriteContext(&device->input_ctx->slot.dwslot0, inputdwslot0); 1618 EvaluateContext(device->input_ctx_addr, device->slot); 1619 } 1620 1621 mutex_init(&device->endpoints[id].lock, "xhci endpoint lock"); 1622 MutexLocker endpointLocker(device->endpoints[id].lock); 1623 1624 device->endpoints[id].device = device; 1625 device->endpoints[id].id = id; 1626 device->endpoints[id].td_head = NULL; 1627 device->endpoints[id].used = 0; 1628 device->endpoints[id].current = 0; 1629 1630 device->endpoints[id].trbs = device->trbs 1631 + id * XHCI_MAX_TRANSFERS; 1632 device->endpoints[id].trb_addr = device->trb_addr 1633 + id * XHCI_MAX_TRANSFERS * sizeof(xhci_trb); 1634 memset(device->endpoints[id].trbs, 0, 1635 sizeof(xhci_trb) * XHCI_MAX_TRANSFERS); 1636 1637 TRACE("_InsertEndpointForPipe trbs device %p endpoint %p\n", 1638 device->trbs, device->endpoints[id].trbs); 1639 TRACE("_InsertEndpointForPipe trb_addr device 0x%" B_PRIxPHYSADDR 1640 " endpoint 0x%" B_PRIxPHYSADDR "\n", device->trb_addr, 1641 device->endpoints[id].trb_addr); 1642 1643 uint8 endpoint = id + 1; 1644 1645 TRACE("trb_addr 0x%" B_PRIxPHYSADDR "\n", device->endpoints[id].trb_addr); 1646 1647 status_t status = ConfigureEndpoint(device->slot, id, pipe->Type(), 1648 pipe->Direction() == Pipe::In, device->endpoints[id].trb_addr, 1649 pipe->Interval(), pipe->MaxPacketSize(), 1650 pipe->MaxPacketSize() & 0x7ff, usbDevice->Speed()); 1651 if (status != B_OK) { 1652 TRACE_ERROR("unable to configure endpoint\n"); 1653 return status; 1654 } 1655 1656 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1657 _WriteContext(&device->input_ctx->input.addFlags, 1658 (1 << endpoint) | (1 << 0)); 1659 1660 if (endpoint > 1) 1661 ConfigureEndpoint(device->input_ctx_addr, false, device->slot); 1662 else 1663 EvaluateContext(device->input_ctx_addr, device->slot); 1664 1665 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", 1666 device->address, SLOT_3_SLOT_STATE_GET(_ReadContext( 1667 &device->device_ctx->slot.dwslot3))); 1668 TRACE("endpoint[0] state 0x%08" B_PRIx32 "\n", 1669 ENDPOINT_0_STATE_GET(_ReadContext( 1670 &device->device_ctx->endpoints[0].dwendpoint0))); 1671 TRACE("endpoint[%d] state 0x%08" B_PRIx32 "\n", id, 1672 ENDPOINT_0_STATE_GET(_ReadContext( 1673 &device->device_ctx->endpoints[id].dwendpoint0))); 1674 1675 device->state = XHCI_STATE_CONFIGURED; 1676 } 1677 pipe->SetControllerCookie(&device->endpoints[id]); 1678 1679 TRACE("_InsertEndpointForPipe for pipe %p at id %d\n", pipe, id); 1680 1681 return B_OK; 1682 } 1683 1684 1685 status_t 1686 XHCI::_RemoveEndpointForPipe(Pipe *pipe) 1687 { 1688 TRACE("remove endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1689 1690 if (pipe->Parent()->Type() != USB_OBJECT_DEVICE) 1691 return B_OK; 1692 Device* usbDevice = (Device *)pipe->Parent(); 1693 if (usbDevice->Parent() == RootObject()) 1694 return B_BAD_VALUE; 1695 1696 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 1697 if (endpoint == NULL || endpoint->trbs == NULL) 1698 return B_NO_INIT; 1699 1700 xhci_device *device = endpoint->device; 1701 1702 if (endpoint->id > 0) { 1703 mutex_lock(&endpoint->lock); 1704 1705 uint8 epNumber = endpoint->id + 1; 1706 StopEndpoint(true, epNumber, device->slot); 1707 1708 mutex_destroy(&endpoint->lock); 1709 memset(endpoint, 0, sizeof(xhci_endpoint)); 1710 1711 _WriteContext(&device->input_ctx->input.dropFlags, (1 << epNumber)); 1712 _WriteContext(&device->input_ctx->input.addFlags, 0); 1713 1714 if (epNumber > 1) 1715 ConfigureEndpoint(device->input_ctx_addr, true, device->slot); 1716 else 1717 EvaluateContext(device->input_ctx_addr, device->slot); 1718 1719 device->state = XHCI_STATE_ADDRESSED; 1720 } 1721 pipe->SetControllerCookie(NULL); 1722 1723 return B_OK; 1724 } 1725 1726 1727 status_t 1728 XHCI::_LinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 1729 { 1730 TRACE("_LinkDescriptorForPipe\n"); 1731 1732 // We must check this before we lock the endpoint, because if it is 1733 // NULL, the mutex is probably uninitialized, too. 1734 if (endpoint->device == NULL) { 1735 TRACE_ERROR("trying to submit a transfer to a non-existent endpoint!\n"); 1736 return B_NO_INIT; 1737 } 1738 1739 // Use mutex_trylock first, in case we are in KDL. 1740 if (mutex_trylock(&endpoint->lock) != B_OK) 1741 mutex_lock(&endpoint->lock); 1742 1743 // We will be modifying 2 TRBs as part of linking a new descriptor: 1744 // the "current" TRB (which will link to the passed descriptor), and 1745 // the "next" (current + 1) TRB (which will be zeroed, as we have 1746 // likely used it before.) Hence the "+ 1" in this check. 1747 if ((endpoint->used + 1) >= XHCI_MAX_TRANSFERS) { 1748 TRACE_ERROR("_LinkDescriptorForPipe max transfers count exceeded\n"); 1749 mutex_unlock(&endpoint->lock); 1750 return B_BAD_VALUE; 1751 } 1752 1753 endpoint->used++; 1754 descriptor->next = endpoint->td_head; 1755 endpoint->td_head = descriptor; 1756 1757 uint8 current = endpoint->current; 1758 uint8 next = (current + 1) % (XHCI_MAX_TRANSFERS); 1759 1760 TRACE("_LinkDescriptorForPipe current %d, next %d\n", current, next); 1761 1762 // Compute next link. 1763 addr_t addr = endpoint->trb_addr + next * sizeof(xhci_trb); 1764 descriptor->trbs[descriptor->trb_used].address = addr; 1765 descriptor->trbs[descriptor->trb_used].status = TRB_2_IRQ(0); 1766 descriptor->trbs[descriptor->trb_used].flags = TRB_3_TYPE(TRB_TYPE_LINK) 1767 | TRB_3_CYCLE_BIT; 1768 1769 #if !B_HOST_IS_LENDIAN 1770 // Convert endianness. 1771 for (uint32 i = 0; i <= descriptor->trb_used; i++) { 1772 descriptor->trbs[i].address = 1773 B_HOST_TO_LENDIAN_INT64(descriptor->trbs[i].address); 1774 descriptor->trbs[i].status = 1775 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].status); 1776 descriptor->trbs[i].flags = 1777 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].flags); 1778 } 1779 #endif 1780 1781 // Link the descriptor. 1782 endpoint->trbs[next].address = 0; 1783 endpoint->trbs[next].status = 0; 1784 endpoint->trbs[next].flags = 0; 1785 1786 endpoint->trbs[current].address = 1787 B_HOST_TO_LENDIAN_INT64(descriptor->trb_addr); 1788 endpoint->trbs[current].status = 1789 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 1790 endpoint->trbs[current].flags = 1791 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_LINK)); 1792 1793 // Everything is ready, so write the cycle bit. 1794 endpoint->trbs[current].flags |= B_HOST_TO_LENDIAN_INT32(TRB_3_CYCLE_BIT); 1795 1796 TRACE("_LinkDescriptorForPipe pCurrent %p phys 0x%" B_PRIxPHYSADDR 1797 " 0x%" B_PRIxPHYSADDR " 0x%08" B_PRIx32 "\n", &endpoint->trbs[current], 1798 endpoint->trb_addr + current * sizeof(struct xhci_trb), 1799 endpoint->trbs[current].address, 1800 B_LENDIAN_TO_HOST_INT32(endpoint->trbs[current].flags)); 1801 1802 endpoint->current = next; 1803 mutex_unlock(&endpoint->lock); 1804 1805 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 1806 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 1807 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 1808 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 1809 1810 Ring(endpoint->device->slot, endpoint->id + 1); 1811 1812 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 1813 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 1814 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 1815 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 1816 1817 return B_OK; 1818 } 1819 1820 1821 status_t 1822 XHCI::_UnlinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 1823 { 1824 TRACE("_UnlinkDescriptorForPipe\n"); 1825 // We presume that the caller has already locked or owns the endpoint. 1826 1827 endpoint->used--; 1828 if (descriptor == endpoint->td_head) { 1829 endpoint->td_head = descriptor->next; 1830 descriptor->next = NULL; 1831 return B_OK; 1832 } else { 1833 for (xhci_td *td = endpoint->td_head; td->next != NULL; td = td->next) { 1834 if (td->next == descriptor) { 1835 td->next = descriptor->next; 1836 descriptor->next = NULL; 1837 return B_OK; 1838 } 1839 } 1840 } 1841 1842 endpoint->used++; 1843 return B_ERROR; 1844 } 1845 1846 1847 status_t 1848 XHCI::ConfigureEndpoint(uint8 slot, uint8 number, uint8 type, bool directionIn, 1849 uint64 ringAddr, uint16 interval, uint16 maxPacketSize, uint16 maxFrameSize, 1850 usb_speed speed) 1851 { 1852 struct xhci_device* device = &fDevices[slot]; 1853 1854 uint32 dwendpoint0 = 0; 1855 uint32 dwendpoint1 = 0; 1856 uint64 qwendpoint2 = 0; 1857 uint32 dwendpoint4 = 0; 1858 1859 // Compute and assign the endpoint type. (XHCI 1.1 § 6.2.3 Table 6-9 p429.) 1860 uint8 xhciType = 4; 1861 if ((type & USB_OBJECT_INTERRUPT_PIPE) != 0) 1862 xhciType = 3; 1863 if ((type & USB_OBJECT_BULK_PIPE) != 0) 1864 xhciType = 2; 1865 if ((type & USB_OBJECT_ISO_PIPE) != 0) 1866 xhciType = 1; 1867 xhciType |= directionIn ? (1 << 2) : 0; 1868 dwendpoint1 |= ENDPOINT_1_EPTYPE(xhciType); 1869 1870 // Compute and assign interval. (XHCI 1.1 § 6.2.3.6 p433.) 1871 uint16 calcInterval; 1872 if ((type & USB_OBJECT_BULK_PIPE) != 0 1873 || (type & USB_OBJECT_CONTROL_PIPE) != 0) { 1874 // Bulk and Control endpoints never issue NAKs. 1875 calcInterval = 0; 1876 } else { 1877 switch (speed) { 1878 case USB_SPEED_FULLSPEED: 1879 if ((type & USB_OBJECT_ISO_PIPE) != 0) { 1880 // Convert 1-16 into 3-18. 1881 calcInterval = min_c(max_c(interval, 1), 16) + 2; 1882 break; 1883 } 1884 1885 // fall through 1886 case USB_SPEED_LOWSPEED: { 1887 // Convert 1ms-255ms into 3-10. 1888 1889 // Find the index of the highest set bit in "interval". 1890 uint32 temp = min_c(max_c(interval, 1), 255); 1891 for (calcInterval = 0; temp != 1; calcInterval++) 1892 temp = temp >> 1; 1893 calcInterval += 3; 1894 break; 1895 } 1896 1897 case USB_SPEED_HIGHSPEED: 1898 case USB_SPEED_SUPER: 1899 default: 1900 // Convert 1-16 into 0-15. 1901 calcInterval = min_c(max_c(interval, 1), 16) - 1; 1902 break; 1903 } 1904 } 1905 dwendpoint0 |= ENDPOINT_0_INTERVAL(calcInterval); 1906 1907 // For non-isochronous endpoints, we want the controller to retry failed 1908 // transfers, if possible. (XHCI 1.1 § 4.10.2.3 p189.) 1909 if (!(type & USB_OBJECT_ISO_PIPE)) 1910 dwendpoint1 |= ENDPOINT_1_CERR(3); 1911 1912 // Assign maximum burst size. 1913 // TODO: While computing the maximum burst this way is correct for USB2 1914 // devices, it is merely acceptable for USB3 devices, which have a more 1915 // correct value stored in the Companion Descriptor. (Further, this value 1916 // in the USB3 Companion Descriptor is to be used for *all* endpoints, not 1917 // just Interrupt and Isoch ones.) 1918 uint8 maxBurst = (maxPacketSize & 0x1800) >> 11; 1919 if (speed >= USB_SPEED_HIGHSPEED 1920 && (((type & USB_OBJECT_INTERRUPT_PIPE) != 0) 1921 || (type & USB_OBJECT_ISO_PIPE) != 0)) { 1922 dwendpoint1 |= ENDPOINT_1_MAXBURST(maxBurst); 1923 } 1924 1925 // Assign maximum packet size, set the ring address, and set the 1926 // "Dequeue Cycle State" bit. (XHCI 1.1 § 6.2.3 Table 6-10 p430.) 1927 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE(maxPacketSize); 1928 qwendpoint2 |= ENDPOINT_2_DCS_BIT | ringAddr; 1929 1930 // Assign average TRB length. 1931 if ((type & USB_OBJECT_CONTROL_PIPE) != 0) { 1932 // Control pipes are a special case, as they rarely have 1933 // outbound transfers of any substantial size. 1934 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(8); 1935 } else { 1936 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(maxPacketSize * 4); 1937 } 1938 1939 // Assign maximum ESIT payload. (XHCI 1.1 § 4.14.2 p250.) 1940 // TODO: This computation is *only* correct for USB2 devices. 1941 if (((type & USB_OBJECT_INTERRUPT_PIPE) != 0) 1942 || ((type & USB_OBJECT_ISO_PIPE) != 0)) { 1943 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD((maxBurst + 1) * maxPacketSize); 1944 } 1945 1946 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint0, 1947 dwendpoint0); 1948 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint1, 1949 dwendpoint1); 1950 _WriteContext(&device->input_ctx->endpoints[number].qwendpoint2, 1951 qwendpoint2); 1952 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint4, 1953 dwendpoint4); 1954 1955 TRACE("endpoint 0x%" B_PRIx32 " 0x%" B_PRIx32 " 0x%" B_PRIx64 " 0x%" 1956 B_PRIx32 "\n", 1957 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint0), 1958 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint1), 1959 _ReadContext(&device->input_ctx->endpoints[number].qwendpoint2), 1960 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint4)); 1961 1962 return B_OK; 1963 } 1964 1965 1966 status_t 1967 XHCI::GetPortSpeed(uint8 index, usb_speed* speed) 1968 { 1969 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 1970 1971 switch (PS_SPEED_GET(portStatus)) { 1972 case 3: 1973 *speed = USB_SPEED_HIGHSPEED; 1974 break; 1975 case 2: 1976 *speed = USB_SPEED_LOWSPEED; 1977 break; 1978 case 1: 1979 *speed = USB_SPEED_FULLSPEED; 1980 break; 1981 case 4: 1982 *speed = USB_SPEED_SUPER; 1983 break; 1984 default: 1985 TRACE("Non Standard Port Speed\n"); 1986 TRACE("Assuming Superspeed\n"); 1987 *speed = USB_SPEED_SUPER; 1988 break; 1989 } 1990 1991 return B_OK; 1992 } 1993 1994 1995 status_t 1996 XHCI::GetPortStatus(uint8 index, usb_port_status* status) 1997 { 1998 if (index >= fPortCount) 1999 return B_BAD_INDEX; 2000 2001 status->status = status->change = 0; 2002 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 2003 TRACE("port %" B_PRId8 " status=0x%08" B_PRIx32 "\n", index, portStatus); 2004 2005 // build the status 2006 switch (PS_SPEED_GET(portStatus)) { 2007 case 3: 2008 status->status |= PORT_STATUS_HIGH_SPEED; 2009 break; 2010 case 2: 2011 status->status |= PORT_STATUS_LOW_SPEED; 2012 break; 2013 default: 2014 break; 2015 } 2016 2017 if (portStatus & PS_CCS) 2018 status->status |= PORT_STATUS_CONNECTION; 2019 if (portStatus & PS_PED) 2020 status->status |= PORT_STATUS_ENABLE; 2021 if (portStatus & PS_OCA) 2022 status->status |= PORT_STATUS_OVER_CURRENT; 2023 if (portStatus & PS_PR) 2024 status->status |= PORT_STATUS_RESET; 2025 if (portStatus & PS_PP) { 2026 if (fPortSpeeds[index] == USB_SPEED_SUPER) 2027 status->status |= PORT_STATUS_SS_POWER; 2028 else 2029 status->status |= PORT_STATUS_POWER; 2030 } 2031 2032 // build the change 2033 if (portStatus & PS_CSC) 2034 status->change |= PORT_STATUS_CONNECTION; 2035 if (portStatus & PS_PEC) 2036 status->change |= PORT_STATUS_ENABLE; 2037 if (portStatus & PS_OCC) 2038 status->change |= PORT_STATUS_OVER_CURRENT; 2039 if (portStatus & PS_PRC) 2040 status->change |= PORT_STATUS_RESET; 2041 2042 if (fPortSpeeds[index] == USB_SPEED_SUPER) { 2043 if (portStatus & PS_PLC) 2044 status->change |= PORT_CHANGE_LINK_STATE; 2045 if (portStatus & PS_WRC) 2046 status->change |= PORT_CHANGE_BH_PORT_RESET; 2047 } 2048 2049 return B_OK; 2050 } 2051 2052 2053 status_t 2054 XHCI::SetPortFeature(uint8 index, uint16 feature) 2055 { 2056 TRACE("set port feature index %u feature %u\n", index, feature); 2057 if (index >= fPortCount) 2058 return B_BAD_INDEX; 2059 2060 uint32 portRegister = XHCI_PORTSC(index); 2061 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2062 2063 switch (feature) { 2064 case PORT_SUSPEND: 2065 if ((portStatus & PS_PED) == 0 || (portStatus & PS_PR) 2066 || (portStatus & PS_PLS_MASK) >= PS_XDEV_U3) { 2067 TRACE_ERROR("USB core suspending device not in U0/U1/U2.\n"); 2068 return B_BAD_VALUE; 2069 } 2070 portStatus &= ~PS_PLS_MASK; 2071 WriteOpReg(portRegister, portStatus | PS_LWS | PS_XDEV_U3); 2072 break; 2073 2074 case PORT_RESET: 2075 WriteOpReg(portRegister, portStatus | PS_PR); 2076 break; 2077 2078 case PORT_POWER: 2079 WriteOpReg(portRegister, portStatus | PS_PP); 2080 break; 2081 default: 2082 return B_BAD_VALUE; 2083 } 2084 ReadOpReg(portRegister); 2085 return B_OK; 2086 } 2087 2088 2089 status_t 2090 XHCI::ClearPortFeature(uint8 index, uint16 feature) 2091 { 2092 TRACE("clear port feature index %u feature %u\n", index, feature); 2093 if (index >= fPortCount) 2094 return B_BAD_INDEX; 2095 2096 uint32 portRegister = XHCI_PORTSC(index); 2097 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2098 2099 switch (feature) { 2100 case PORT_SUSPEND: 2101 portStatus = ReadOpReg(portRegister); 2102 if (portStatus & PS_PR) 2103 return B_BAD_VALUE; 2104 if (portStatus & PS_XDEV_U3) { 2105 if ((portStatus & PS_PED) == 0) 2106 return B_BAD_VALUE; 2107 portStatus &= ~PS_PLS_MASK; 2108 WriteOpReg(portRegister, portStatus | PS_XDEV_U0 | PS_LWS); 2109 } 2110 break; 2111 case PORT_ENABLE: 2112 WriteOpReg(portRegister, portStatus | PS_PED); 2113 break; 2114 case PORT_POWER: 2115 WriteOpReg(portRegister, portStatus & ~PS_PP); 2116 break; 2117 case C_PORT_CONNECTION: 2118 WriteOpReg(portRegister, portStatus | PS_CSC); 2119 break; 2120 case C_PORT_ENABLE: 2121 WriteOpReg(portRegister, portStatus | PS_PEC); 2122 break; 2123 case C_PORT_OVER_CURRENT: 2124 WriteOpReg(portRegister, portStatus | PS_OCC); 2125 break; 2126 case C_PORT_RESET: 2127 WriteOpReg(portRegister, portStatus | PS_PRC); 2128 break; 2129 case C_PORT_BH_PORT_RESET: 2130 WriteOpReg(portRegister, portStatus | PS_WRC); 2131 break; 2132 case C_PORT_LINK_STATE: 2133 WriteOpReg(portRegister, portStatus | PS_PLC); 2134 break; 2135 default: 2136 return B_BAD_VALUE; 2137 } 2138 2139 ReadOpReg(portRegister); 2140 return B_OK; 2141 } 2142 2143 2144 status_t 2145 XHCI::ControllerHalt() 2146 { 2147 // Mask off run state 2148 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) & ~CMD_RUN); 2149 2150 // wait for shutdown state 2151 if (WaitOpBits(XHCI_STS, STS_HCH, STS_HCH) != B_OK) { 2152 TRACE_ERROR("HCH shutdown timeout\n"); 2153 return B_ERROR; 2154 } 2155 return B_OK; 2156 } 2157 2158 2159 status_t 2160 XHCI::ControllerReset() 2161 { 2162 TRACE("ControllerReset() cmd: 0x%" B_PRIx32 " sts: 0x%" B_PRIx32 "\n", 2163 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 2164 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) | CMD_HCRST); 2165 2166 if (WaitOpBits(XHCI_CMD, CMD_HCRST, 0) != B_OK) { 2167 TRACE_ERROR("ControllerReset() failed CMD_HCRST\n"); 2168 return B_ERROR; 2169 } 2170 2171 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 2172 TRACE_ERROR("ControllerReset() failed STS_CNR\n"); 2173 return B_ERROR; 2174 } 2175 2176 return B_OK; 2177 } 2178 2179 2180 int32 2181 XHCI::InterruptHandler(void* data) 2182 { 2183 return ((XHCI*)data)->Interrupt(); 2184 } 2185 2186 2187 int32 2188 XHCI::Interrupt() 2189 { 2190 SpinLocker _(&fSpinlock); 2191 2192 uint32 status = ReadOpReg(XHCI_STS); 2193 uint32 temp = ReadRunReg32(XHCI_IMAN(0)); 2194 WriteOpReg(XHCI_STS, status); 2195 WriteRunReg32(XHCI_IMAN(0), temp); 2196 2197 int32 result = B_HANDLED_INTERRUPT; 2198 2199 if ((status & STS_HCH) != 0) { 2200 TRACE_ERROR("Host Controller halted\n"); 2201 return result; 2202 } 2203 if ((status & STS_HSE) != 0) { 2204 TRACE_ERROR("Host System Error\n"); 2205 return result; 2206 } 2207 if ((status & STS_HCE) != 0) { 2208 TRACE_ERROR("Host Controller Error\n"); 2209 return result; 2210 } 2211 2212 if ((status & STS_EINT) == 0) { 2213 TRACE("STS: 0x%" B_PRIx32 " IRQ_PENDING: 0x%" B_PRIx32 "\n", 2214 status, temp); 2215 return B_UNHANDLED_INTERRUPT; 2216 } 2217 2218 TRACE("Event Interrupt\n"); 2219 release_sem_etc(fEventSem, 1, B_DO_NOT_RESCHEDULE); 2220 return B_INVOKE_SCHEDULER; 2221 } 2222 2223 2224 void 2225 XHCI::Ring(uint8 slot, uint8 endpoint) 2226 { 2227 TRACE("Ding Dong! slot:%d endpoint %d\n", slot, endpoint) 2228 if ((slot == 0 && endpoint > 0) || (slot > 0 && endpoint == 0)) 2229 panic("Ring() invalid slot/endpoint combination\n"); 2230 if (slot > fSlotCount || endpoint >= XHCI_MAX_ENDPOINTS) 2231 panic("Ring() invalid slot or endpoint\n"); 2232 2233 WriteDoorReg32(XHCI_DOORBELL(slot), XHCI_DOORBELL_TARGET(endpoint) 2234 | XHCI_DOORBELL_STREAMID(0)); 2235 /* Flush PCI posted writes */ 2236 ReadDoorReg32(XHCI_DOORBELL(slot)); 2237 } 2238 2239 2240 void 2241 XHCI::QueueCommand(xhci_trb* trb) 2242 { 2243 uint8 i, j; 2244 uint32 temp; 2245 2246 i = fCmdIdx; 2247 j = fCmdCcs; 2248 2249 TRACE("command[%u] = %" B_PRId32 " (0x%016" B_PRIx64 ", 0x%08" B_PRIx32 2250 ", 0x%08" B_PRIx32 ")\n", i, TRB_3_TYPE_GET(trb->flags), trb->address, 2251 trb->status, trb->flags); 2252 2253 fCmdRing[i].address = trb->address; 2254 fCmdRing[i].status = trb->status; 2255 temp = trb->flags; 2256 2257 if (j) 2258 temp |= TRB_3_CYCLE_BIT; 2259 else 2260 temp &= ~TRB_3_CYCLE_BIT; 2261 temp &= ~TRB_3_TC_BIT; 2262 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2263 2264 fCmdAddr = fErst->rs_addr + (XHCI_MAX_EVENTS + i) * sizeof(xhci_trb); 2265 2266 i++; 2267 2268 if (i == (XHCI_MAX_COMMANDS - 1)) { 2269 temp = TRB_3_TYPE(TRB_TYPE_LINK) | TRB_3_TC_BIT; 2270 if (j) 2271 temp |= TRB_3_CYCLE_BIT; 2272 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2273 2274 i = 0; 2275 j ^= 1; 2276 } 2277 2278 fCmdIdx = i; 2279 fCmdCcs = j; 2280 } 2281 2282 2283 void 2284 XHCI::HandleCmdComplete(xhci_trb* trb) 2285 { 2286 TRACE("HandleCmdComplete trb %p\n", trb); 2287 2288 if (fCmdAddr == trb->address) { 2289 TRACE("Received command event\n"); 2290 fCmdResult[0] = trb->status; 2291 fCmdResult[1] = B_LENDIAN_TO_HOST_INT32(trb->flags); 2292 release_sem_etc(fCmdCompSem, 1, B_DO_NOT_RESCHEDULE); 2293 } 2294 } 2295 2296 2297 void 2298 XHCI::HandleTransferComplete(xhci_trb* trb) 2299 { 2300 TRACE("HandleTransferComplete trb %p\n", trb); 2301 2302 uint8 endpointNumber 2303 = TRB_3_ENDPOINT_GET(B_LENDIAN_TO_HOST_INT32(trb->flags)); 2304 uint8 slot = TRB_3_SLOT_GET(B_LENDIAN_TO_HOST_INT32(trb->flags)); 2305 uint8 type = TRB_3_TYPE_GET(B_LENDIAN_TO_HOST_INT32(trb->flags)); 2306 2307 if (slot > fSlotCount) 2308 TRACE_ERROR("invalid slot\n"); 2309 if (endpointNumber == 0 || endpointNumber >= XHCI_MAX_ENDPOINTS) 2310 TRACE_ERROR("invalid endpoint\n"); 2311 if (type == TRB_TYPE_EVENT_DATA) { 2312 // TODO: Implement these. (Do we trigger any at present?) 2313 TRACE_ERROR("event data TRBs are not handled yet!\n"); 2314 return; 2315 } 2316 2317 xhci_device *device = &fDevices[slot]; 2318 xhci_endpoint *endpoint = &device->endpoints[endpointNumber - 1]; 2319 2320 if (endpoint->trbs == NULL) { 2321 TRACE_ERROR("got TRB but endpoint is not allocated!\n"); 2322 return; 2323 } 2324 2325 // Use mutex_trylock first, in case we are in KDL. 2326 MutexLocker endpointLocker(endpoint->lock, 2327 mutex_trylock(&endpoint->lock) == B_OK); 2328 if (!endpointLocker.IsLocked()) { 2329 // We failed to get the lock. Most likely it was destroyed 2330 // while we were waiting for it. 2331 return; 2332 } 2333 2334 addr_t source = trb->address; 2335 uint8 completionCode = TRB_2_COMP_CODE_GET(trb->status); 2336 uint32 remainder = TRB_2_REM_GET(trb->status); 2337 2338 for (xhci_td *td = endpoint->td_head; td != NULL; td = td->next) { 2339 int64 offset = (source - td->trb_addr) / sizeof(xhci_trb); 2340 if (offset < 0 || offset >= td->trb_count) 2341 continue; 2342 2343 TRACE("HandleTransferComplete td %p trb %" B_PRId64 " found\n", 2344 td, offset); 2345 2346 // The TRB at offset trb_used will be the link TRB, which we do not 2347 // care about (and should not generate an interrupt at all.) 2348 // We really care about the properly last TRB, at index "count - 1". 2349 // Additionally, if we have an unsuccessful completion code, the transfer 2350 // likely failed midway; so just accept it anyway. 2351 if (offset == (td->trb_used - 1) || completionCode != COMP_SUCCESS) { 2352 _UnlinkDescriptorForPipe(td, endpoint); 2353 endpointLocker.Unlock(); 2354 2355 td->trb_completion_code = completionCode; 2356 td->trb_left = remainder; 2357 2358 // add descriptor to finished list 2359 if (mutex_trylock(&fFinishedLock) != B_OK) 2360 mutex_lock(&fFinishedLock); 2361 td->next = fFinishedHead; 2362 fFinishedHead = td; 2363 mutex_unlock(&fFinishedLock); 2364 2365 release_sem_etc(fFinishTransfersSem, 1, B_DO_NOT_RESCHEDULE); 2366 TRACE("HandleTransferComplete td %p done\n", td); 2367 } else { 2368 TRACE_ERROR("successful TRB %" B_PRIxADDR " was found, but it wasn't " 2369 "the last in the TD!\n", source); 2370 } 2371 return; 2372 } 2373 TRACE_ERROR("TRB 0x%" B_PRIxADDR " was not found in the endpoint!\n", source); 2374 } 2375 2376 2377 void 2378 XHCI::DumpRing(xhci_trb *trbs, uint32 size) 2379 { 2380 if (!Lock()) { 2381 TRACE("Unable to get lock!\n"); 2382 return; 2383 } 2384 2385 for (uint32 i = 0; i < size; i++) { 2386 TRACE("command[%" B_PRId32 "] = %" B_PRId32 " (0x%016" B_PRIx64 "," 2387 " 0x%08" B_PRIx32 ", 0x%08" B_PRIx32 ")\n", i, 2388 TRB_3_TYPE_GET(B_LENDIAN_TO_HOST_INT32(trbs[i].flags)), 2389 trbs[i].address, trbs[i].status, trbs[i].flags); 2390 } 2391 2392 Unlock(); 2393 } 2394 2395 2396 status_t 2397 XHCI::DoCommand(xhci_trb* trb) 2398 { 2399 if (!Lock()) { 2400 TRACE("Unable to get lock!\n"); 2401 return B_ERROR; 2402 } 2403 2404 QueueCommand(trb); 2405 Ring(0, 0); 2406 2407 if (acquire_sem_etc(fCmdCompSem, 1, B_RELATIVE_TIMEOUT, 1 * 1000 * 1000) < B_OK) { 2408 TRACE("Unable to obtain fCmdCompSem!\n"); 2409 Unlock(); 2410 return B_TIMED_OUT; 2411 } 2412 // eat up sems that have been released by multiple interrupts 2413 int32 semCount = 0; 2414 get_sem_count(fCmdCompSem, &semCount); 2415 if (semCount > 0) 2416 acquire_sem_etc(fCmdCompSem, semCount, B_RELATIVE_TIMEOUT, 0); 2417 2418 status_t status = B_OK; 2419 uint32 completionCode = TRB_2_COMP_CODE_GET(fCmdResult[0]); 2420 TRACE("Command Complete. Result: %" B_PRId32 "\n", completionCode); 2421 if (completionCode != COMP_SUCCESS) { 2422 TRACE_ERROR("unsuccessful command %" B_PRId32 ", error %s (%" B_PRId32 ")\n", 2423 TRB_3_TYPE_GET(trb->flags), xhci_error_string(completionCode), 2424 completionCode); 2425 status = B_IO_ERROR; 2426 } 2427 2428 trb->status = fCmdResult[0]; 2429 trb->flags = fCmdResult[1]; 2430 TRACE("Storing trb 0x%08" B_PRIx32 " 0x%08" B_PRIx32 "\n", trb->status, 2431 trb->flags); 2432 2433 Unlock(); 2434 return status; 2435 } 2436 2437 2438 status_t 2439 XHCI::Noop() 2440 { 2441 TRACE("Issue No-Op\n"); 2442 xhci_trb trb; 2443 trb.address = 0; 2444 trb.status = 0; 2445 trb.flags = TRB_3_TYPE(TRB_TYPE_CMD_NOOP); 2446 2447 return DoCommand(&trb); 2448 } 2449 2450 2451 status_t 2452 XHCI::EnableSlot(uint8* slot) 2453 { 2454 TRACE("Enable Slot\n"); 2455 xhci_trb trb; 2456 trb.address = 0; 2457 trb.status = 0; 2458 trb.flags = TRB_3_TYPE(TRB_TYPE_ENABLE_SLOT); 2459 2460 status_t status = DoCommand(&trb); 2461 if (status != B_OK) 2462 return status; 2463 2464 *slot = TRB_3_SLOT_GET(trb.flags); 2465 return *slot != 0 ? B_OK : B_BAD_VALUE; 2466 } 2467 2468 2469 status_t 2470 XHCI::DisableSlot(uint8 slot) 2471 { 2472 TRACE("Disable Slot\n"); 2473 xhci_trb trb; 2474 trb.address = 0; 2475 trb.status = 0; 2476 trb.flags = TRB_3_TYPE(TRB_TYPE_DISABLE_SLOT) | TRB_3_SLOT(slot); 2477 2478 return DoCommand(&trb); 2479 } 2480 2481 2482 status_t 2483 XHCI::SetAddress(uint64 inputContext, bool bsr, uint8 slot) 2484 { 2485 TRACE("Set Address\n"); 2486 xhci_trb trb; 2487 trb.address = inputContext; 2488 trb.status = 0; 2489 trb.flags = TRB_3_TYPE(TRB_TYPE_ADDRESS_DEVICE) | TRB_3_SLOT(slot); 2490 2491 if (bsr) 2492 trb.flags |= TRB_3_BSR_BIT; 2493 2494 return DoCommand(&trb); 2495 } 2496 2497 2498 status_t 2499 XHCI::ConfigureEndpoint(uint64 inputContext, bool deconfigure, uint8 slot) 2500 { 2501 TRACE("Configure Endpoint\n"); 2502 xhci_trb trb; 2503 trb.address = inputContext; 2504 trb.status = 0; 2505 trb.flags = TRB_3_TYPE(TRB_TYPE_CONFIGURE_ENDPOINT) | TRB_3_SLOT(slot); 2506 2507 if (deconfigure) 2508 trb.flags |= TRB_3_DCEP_BIT; 2509 2510 return DoCommand(&trb); 2511 } 2512 2513 2514 status_t 2515 XHCI::EvaluateContext(uint64 inputContext, uint8 slot) 2516 { 2517 TRACE("Evaluate Context\n"); 2518 xhci_trb trb; 2519 trb.address = inputContext; 2520 trb.status = 0; 2521 trb.flags = TRB_3_TYPE(TRB_TYPE_EVALUATE_CONTEXT) | TRB_3_SLOT(slot); 2522 2523 return DoCommand(&trb); 2524 } 2525 2526 2527 status_t 2528 XHCI::ResetEndpoint(bool preserve, uint8 endpoint, uint8 slot) 2529 { 2530 TRACE("Reset Endpoint\n"); 2531 xhci_trb trb; 2532 trb.address = 0; 2533 trb.status = 0; 2534 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_ENDPOINT) 2535 | TRB_3_SLOT(slot) | TRB_3_ENDPOINT(endpoint); 2536 if (preserve) 2537 trb.flags |= TRB_3_PRSV_BIT; 2538 2539 return DoCommand(&trb); 2540 } 2541 2542 2543 status_t 2544 XHCI::StopEndpoint(bool suspend, uint8 endpoint, uint8 slot) 2545 { 2546 TRACE("Stop Endpoint\n"); 2547 xhci_trb trb; 2548 trb.address = 0; 2549 trb.status = 0; 2550 trb.flags = TRB_3_TYPE(TRB_TYPE_STOP_ENDPOINT) 2551 | TRB_3_SLOT(slot) | TRB_3_ENDPOINT(endpoint); 2552 if (suspend) 2553 trb.flags |= TRB_3_SUSPEND_ENDPOINT_BIT; 2554 2555 return DoCommand(&trb); 2556 } 2557 2558 2559 status_t 2560 XHCI::SetTRDequeue(uint64 dequeue, uint16 stream, uint8 endpoint, uint8 slot) 2561 { 2562 TRACE("Set TR Dequeue\n"); 2563 xhci_trb trb; 2564 trb.address = dequeue; 2565 trb.status = TRB_2_STREAM(stream); 2566 trb.flags = TRB_3_TYPE(TRB_TYPE_SET_TR_DEQUEUE) 2567 | TRB_3_SLOT(slot) | TRB_3_ENDPOINT(endpoint); 2568 2569 return DoCommand(&trb); 2570 } 2571 2572 2573 status_t 2574 XHCI::ResetDevice(uint8 slot) 2575 { 2576 TRACE("Reset Device\n"); 2577 xhci_trb trb; 2578 trb.address = 0; 2579 trb.status = 0; 2580 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_DEVICE) | TRB_3_SLOT(slot); 2581 2582 return DoCommand(&trb); 2583 } 2584 2585 2586 int32 2587 XHCI::EventThread(void* data) 2588 { 2589 ((XHCI *)data)->CompleteEvents(); 2590 return B_OK; 2591 } 2592 2593 2594 void 2595 XHCI::CompleteEvents() 2596 { 2597 while (!fStopThreads) { 2598 if (acquire_sem(fEventSem) < B_OK) 2599 continue; 2600 2601 // eat up sems that have been released by multiple interrupts 2602 int32 semCount = 0; 2603 get_sem_count(fEventSem, &semCount); 2604 if (semCount > 0) 2605 acquire_sem_etc(fEventSem, semCount, B_RELATIVE_TIMEOUT, 0); 2606 2607 ProcessEvents(); 2608 } 2609 } 2610 2611 2612 void 2613 XHCI::ProcessEvents() 2614 { 2615 // Use mutex_trylock first, in case we are in KDL. 2616 MutexLocker locker(fEventLock, mutex_trylock(&fEventLock) == B_OK); 2617 if (!locker.IsLocked()) { 2618 // We failed to get the lock. This really should not happen. 2619 TRACE_ERROR("failed to acquire event lock!\n"); 2620 return; 2621 } 2622 2623 uint16 i = fEventIdx; 2624 uint8 j = fEventCcs; 2625 uint8 t = 2; 2626 2627 while (1) { 2628 uint32 temp = B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags); 2629 uint8 event = TRB_3_TYPE_GET(temp); 2630 TRACE("event[%u] = %u (0x%016" B_PRIx64 " 0x%08" B_PRIx32 " 0x%08" 2631 B_PRIx32 ")\n", i, event, fEventRing[i].address, 2632 fEventRing[i].status, B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags)); 2633 uint8 k = (temp & TRB_3_CYCLE_BIT) ? 1 : 0; 2634 if (j != k) 2635 break; 2636 2637 switch (event) { 2638 case TRB_TYPE_COMMAND_COMPLETION: 2639 HandleCmdComplete(&fEventRing[i]); 2640 break; 2641 case TRB_TYPE_TRANSFER: 2642 HandleTransferComplete(&fEventRing[i]); 2643 break; 2644 case TRB_TYPE_PORT_STATUS_CHANGE: 2645 TRACE("port change detected\n"); 2646 break; 2647 default: 2648 TRACE_ERROR("Unhandled event = %u\n", event); 2649 break; 2650 } 2651 2652 i++; 2653 if (i == XHCI_MAX_EVENTS) { 2654 i = 0; 2655 j ^= 1; 2656 if (!--t) 2657 break; 2658 } 2659 } 2660 2661 fEventIdx = i; 2662 fEventCcs = j; 2663 2664 uint64 addr = fErst->rs_addr + i * sizeof(xhci_trb); 2665 addr |= ERST_EHB; 2666 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)addr); 2667 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(addr >> 32)); 2668 } 2669 2670 2671 int32 2672 XHCI::FinishThread(void* data) 2673 { 2674 ((XHCI *)data)->FinishTransfers(); 2675 return B_OK; 2676 } 2677 2678 2679 void 2680 XHCI::FinishTransfers() 2681 { 2682 while (!fStopThreads) { 2683 if (acquire_sem(fFinishTransfersSem) < B_OK) 2684 continue; 2685 2686 // eat up sems that have been released by multiple interrupts 2687 int32 semCount = 0; 2688 get_sem_count(fFinishTransfersSem, &semCount); 2689 if (semCount > 0) 2690 acquire_sem_etc(fFinishTransfersSem, semCount, B_RELATIVE_TIMEOUT, 0); 2691 2692 mutex_lock(&fFinishedLock); 2693 TRACE("finishing transfers\n"); 2694 while (fFinishedHead != NULL) { 2695 xhci_td* td = fFinishedHead; 2696 fFinishedHead = td->next; 2697 td->next = NULL; 2698 mutex_unlock(&fFinishedLock); 2699 2700 TRACE("finishing transfer td %p\n", td); 2701 2702 Transfer* transfer = td->transfer; 2703 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 2704 2705 status_t callbackStatus = B_OK; 2706 switch (td->trb_completion_code) { 2707 case COMP_SHORT_PACKET: 2708 case COMP_SUCCESS: 2709 callbackStatus = B_OK; 2710 break; 2711 case COMP_DATA_BUFFER: 2712 callbackStatus = directionIn ? B_DEV_DATA_OVERRUN 2713 : B_DEV_DATA_UNDERRUN; 2714 break; 2715 case COMP_BABBLE: 2716 callbackStatus = directionIn ? B_DEV_FIFO_OVERRUN 2717 : B_DEV_FIFO_UNDERRUN; 2718 break; 2719 case COMP_USB_TRANSACTION: 2720 callbackStatus = B_DEV_CRC_ERROR; 2721 break; 2722 case COMP_STALL: 2723 callbackStatus = B_DEV_STALLED; 2724 break; 2725 default: 2726 callbackStatus = B_DEV_STALLED; 2727 break; 2728 } 2729 2730 size_t actualLength = 0; 2731 if (callbackStatus == B_OK) { 2732 actualLength = transfer->DataLength(); 2733 2734 if (td->trb_completion_code == COMP_SHORT_PACKET) 2735 actualLength -= td->trb_left; 2736 2737 if (directionIn && actualLength > 0) { 2738 TRACE("copying in iov count %ld\n", transfer->VectorCount()); 2739 transfer->PrepareKernelAccess(); 2740 ReadDescriptor(td, transfer->Vector(), 2741 transfer->VectorCount()); 2742 } 2743 } 2744 transfer->Finished(callbackStatus, actualLength); 2745 delete transfer; 2746 FreeDescriptor(td); 2747 mutex_lock(&fFinishedLock); 2748 } 2749 mutex_unlock(&fFinishedLock); 2750 } 2751 } 2752 2753 2754 inline void 2755 XHCI::WriteOpReg(uint32 reg, uint32 value) 2756 { 2757 *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg) = value; 2758 } 2759 2760 2761 inline uint32 2762 XHCI::ReadOpReg(uint32 reg) 2763 { 2764 return *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg); 2765 } 2766 2767 2768 inline status_t 2769 XHCI::WaitOpBits(uint32 reg, uint32 mask, uint32 expected) 2770 { 2771 int loops = 0; 2772 uint32 value = ReadOpReg(reg); 2773 while ((value & mask) != expected) { 2774 snooze(1000); 2775 value = ReadOpReg(reg); 2776 if (loops == 100) { 2777 TRACE("delay waiting on reg 0x%" B_PRIX32 " match 0x%" B_PRIX32 2778 " (0x%" B_PRIX32 ")\n", reg, expected, mask); 2779 } else if (loops > 250) { 2780 TRACE_ERROR("timeout waiting on reg 0x%" B_PRIX32 2781 " match 0x%" B_PRIX32 " (0x%" B_PRIX32 ")\n", reg, expected, 2782 mask); 2783 return B_ERROR; 2784 } 2785 loops++; 2786 } 2787 return B_OK; 2788 } 2789 2790 2791 inline uint32 2792 XHCI::ReadCapReg32(uint32 reg) 2793 { 2794 return *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg); 2795 } 2796 2797 2798 inline void 2799 XHCI::WriteCapReg32(uint32 reg, uint32 value) 2800 { 2801 *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg) = value; 2802 } 2803 2804 2805 inline uint32 2806 XHCI::ReadRunReg32(uint32 reg) 2807 { 2808 return *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg); 2809 } 2810 2811 2812 inline void 2813 XHCI::WriteRunReg32(uint32 reg, uint32 value) 2814 { 2815 *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg) = value; 2816 } 2817 2818 2819 inline uint32 2820 XHCI::ReadDoorReg32(uint32 reg) 2821 { 2822 return *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg); 2823 } 2824 2825 2826 inline void 2827 XHCI::WriteDoorReg32(uint32 reg, uint32 value) 2828 { 2829 *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg) = value; 2830 } 2831 2832 2833 inline addr_t 2834 XHCI::_OffsetContextAddr(addr_t p) 2835 { 2836 if (fContextSizeShift == 1) { 2837 // each structure is page aligned, each pointer is 32 bits aligned 2838 uint32 offset = p & ((B_PAGE_SIZE - 1) & ~31U); 2839 p += offset; 2840 } 2841 return p; 2842 } 2843 2844 inline uint32 2845 XHCI::_ReadContext(uint32* p) 2846 { 2847 p = (uint32*)_OffsetContextAddr((addr_t)p); 2848 return *p; 2849 } 2850 2851 2852 inline void 2853 XHCI::_WriteContext(uint32* p, uint32 value) 2854 { 2855 p = (uint32*)_OffsetContextAddr((addr_t)p); 2856 *p = value; 2857 } 2858 2859 2860 inline uint64 2861 XHCI::_ReadContext(uint64* p) 2862 { 2863 p = (uint64*)_OffsetContextAddr((addr_t)p); 2864 return *p; 2865 } 2866 2867 2868 inline void 2869 XHCI::_WriteContext(uint64* p, uint64 value) 2870 { 2871 p = (uint64*)_OffsetContextAddr((addr_t)p); 2872 *p = value; 2873 } 2874