1 /* 2 * Copyright 2011-2019, Haiku, Inc. All rights reserved. 3 * Distributed under the terms of the MIT License. 4 * 5 * Authors: 6 * Augustin Cavalier <waddlesplash> 7 * Jian Chiang <j.jian.chiang@gmail.com> 8 * Jérôme Duval <jerome.duval@gmail.com> 9 * Akshay Jaggi <akshay1994.leo@gmail.com> 10 * Michael Lotz <mmlr@mlotz.ch> 11 */ 12 13 14 #include <module.h> 15 #include <PCI.h> 16 #include <PCI_x86.h> 17 #include <USB3.h> 18 #include <KernelExport.h> 19 20 #include <util/AutoLock.h> 21 22 #include "xhci.h" 23 24 #define USB_MODULE_NAME "xhci" 25 26 pci_module_info *XHCI::sPCIModule = NULL; 27 pci_x86_module_info *XHCI::sPCIx86Module = NULL; 28 29 30 static int32 31 xhci_std_ops(int32 op, ...) 32 { 33 switch (op) { 34 case B_MODULE_INIT: 35 TRACE_MODULE("xhci init module\n"); 36 return B_OK; 37 case B_MODULE_UNINIT: 38 TRACE_MODULE("xhci uninit module\n"); 39 return B_OK; 40 } 41 42 return EINVAL; 43 } 44 45 46 static const char* 47 xhci_error_string(uint32 error) 48 { 49 switch (error) { 50 case COMP_INVALID: return "Invalid"; 51 case COMP_SUCCESS: return "Success"; 52 case COMP_DATA_BUFFER: return "Data buffer"; 53 case COMP_BABBLE: return "Babble detected"; 54 case COMP_USB_TRANSACTION: return "USB transaction"; 55 case COMP_TRB: return "TRB"; 56 case COMP_STALL: return "Stall"; 57 case COMP_RESOURCE: return "Resource"; 58 case COMP_BANDWIDTH: return "Bandwidth"; 59 case COMP_NO_SLOTS: return "No slots"; 60 case COMP_INVALID_STREAM: return "Invalid stream"; 61 case COMP_SLOT_NOT_ENABLED: return "Slot not enabled"; 62 case COMP_ENDPOINT_NOT_ENABLED: return "Endpoint not enabled"; 63 case COMP_SHORT_PACKET: return "Short packet"; 64 case COMP_RING_UNDERRUN: return "Ring underrun"; 65 case COMP_RING_OVERRUN: return "Ring overrun"; 66 case COMP_VF_RING_FULL: return "VF Event Ring Full"; 67 case COMP_PARAMETER: return "Parameter"; 68 case COMP_BANDWIDTH_OVERRUN: return "Bandwidth overrun"; 69 case COMP_CONTEXT_STATE: return "Context state"; 70 case COMP_NO_PING_RESPONSE: return "No ping response"; 71 case COMP_EVENT_RING_FULL: return "Event ring full"; 72 case COMP_INCOMPATIBLE_DEVICE: return "Incompatible device"; 73 case COMP_MISSED_SERVICE: return "Missed service"; 74 case COMP_COMMAND_RING_STOPPED: return "Command ring stopped"; 75 case COMP_COMMAND_ABORTED: return "Command aborted"; 76 case COMP_STOPPED: return "Stopped"; 77 case COMP_LENGTH_INVALID: return "Length invalid"; 78 case COMP_MAX_EXIT_LATENCY: return "Max exit latency too large"; 79 case COMP_ISOC_OVERRUN: return "Isoch buffer overrun"; 80 case COMP_EVENT_LOST: return "Event lost"; 81 case COMP_UNDEFINED: return "Undefined"; 82 case COMP_INVALID_STREAM_ID: return "Invalid stream ID"; 83 case COMP_SECONDARY_BANDWIDTH: return "Secondary bandwidth"; 84 case COMP_SPLIT_TRANSACTION: return "Split transaction"; 85 86 default: return "Undefined"; 87 } 88 } 89 90 91 usb_host_controller_info xhci_module = { 92 { 93 "busses/usb/xhci", 94 0, 95 xhci_std_ops 96 }, 97 NULL, 98 XHCI::AddTo 99 }; 100 101 102 module_info *modules[] = { 103 (module_info *)&xhci_module, 104 NULL 105 }; 106 107 108 status_t 109 XHCI::AddTo(Stack *stack) 110 { 111 if (!sPCIModule) { 112 status_t status = get_module(B_PCI_MODULE_NAME, 113 (module_info **)&sPCIModule); 114 if (status < B_OK) { 115 TRACE_MODULE_ERROR("getting pci module failed! 0x%08" B_PRIx32 116 "\n", status); 117 return status; 118 } 119 } 120 121 TRACE_MODULE("searching devices\n"); 122 bool found = false; 123 pci_info *item = new(std::nothrow) pci_info; 124 if (item == NULL) { 125 sPCIModule = NULL; 126 put_module(B_PCI_MODULE_NAME); 127 return B_NO_MEMORY; 128 } 129 130 // Try to get the PCI x86 module as well so we can enable possible MSIs. 131 if (sPCIx86Module == NULL && get_module(B_PCI_X86_MODULE_NAME, 132 (module_info **)&sPCIx86Module) != B_OK) { 133 // If it isn't there, that's not critical though. 134 TRACE_MODULE_ERROR("failed to get pci x86 module\n"); 135 sPCIx86Module = NULL; 136 } 137 138 for (int32 i = 0; sPCIModule->get_nth_pci_info(i, item) >= B_OK; i++) { 139 if (item->class_base == PCI_serial_bus && item->class_sub == PCI_usb 140 && item->class_api == PCI_usb_xhci) { 141 TRACE_MODULE("found device at PCI:%d:%d:%d\n", 142 item->bus, item->device, item->function); 143 XHCI *bus = new(std::nothrow) XHCI(item, stack); 144 if (bus == NULL) { 145 delete item; 146 sPCIModule = NULL; 147 put_module(B_PCI_MODULE_NAME); 148 if (sPCIx86Module != NULL) 149 put_module(B_PCI_X86_MODULE_NAME); 150 return B_NO_MEMORY; 151 } 152 153 // The bus will put the PCI modules when it is destroyed, so get 154 // them again to increase their reference count. 155 get_module(B_PCI_MODULE_NAME, (module_info **)&sPCIModule); 156 if (sPCIx86Module != NULL) 157 get_module(B_PCI_X86_MODULE_NAME, (module_info **)&sPCIx86Module); 158 159 if (bus->InitCheck() < B_OK) { 160 TRACE_MODULE_ERROR("bus failed init check\n"); 161 delete bus; 162 continue; 163 } 164 165 // the bus took it away 166 item = new(std::nothrow) pci_info; 167 168 if (bus->Start() != B_OK) { 169 delete bus; 170 continue; 171 } 172 found = true; 173 } 174 } 175 176 // The modules will have been gotten again if we successfully 177 // initialized a bus, so we should put them here. 178 put_module(B_PCI_MODULE_NAME); 179 if (sPCIx86Module != NULL) 180 put_module(B_PCI_X86_MODULE_NAME); 181 182 if (!found) 183 TRACE_MODULE_ERROR("no devices found\n"); 184 delete item; 185 return found ? B_OK : ENODEV; 186 } 187 188 189 XHCI::XHCI(pci_info *info, Stack *stack) 190 : BusManager(stack), 191 fRegisterArea(-1), 192 fRegisters(NULL), 193 fPCIInfo(info), 194 fStack(stack), 195 fIRQ(0), 196 fUseMSI(false), 197 fErstArea(-1), 198 fDcbaArea(-1), 199 fCmdCompSem(-1), 200 fStopThreads(false), 201 fRootHub(NULL), 202 fRootHubAddress(0), 203 fPortCount(0), 204 fSlotCount(0), 205 fScratchpadCount(0), 206 fContextSizeShift(0), 207 fFinishedHead(NULL), 208 fFinishTransfersSem(-1), 209 fFinishThread(-1), 210 fEventSem(-1), 211 fEventThread(-1), 212 fEventIdx(0), 213 fCmdIdx(0), 214 fEventCcs(1), 215 fCmdCcs(1) 216 { 217 B_INITIALIZE_SPINLOCK(&fSpinlock); 218 mutex_init(&fFinishedLock, "XHCI finished transfers"); 219 mutex_init(&fEventLock, "XHCI event handler"); 220 221 if (BusManager::InitCheck() < B_OK) { 222 TRACE_ERROR("bus manager failed to init\n"); 223 return; 224 } 225 226 TRACE("constructing new XHCI host controller driver\n"); 227 fInitOK = false; 228 229 // enable busmaster and memory mapped access 230 uint16 command = sPCIModule->read_pci_config(fPCIInfo->bus, 231 fPCIInfo->device, fPCIInfo->function, PCI_command, 2); 232 command &= ~(PCI_command_io | PCI_command_int_disable); 233 command |= PCI_command_master | PCI_command_memory; 234 235 sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device, 236 fPCIInfo->function, PCI_command, 2, command); 237 238 // map the registers (low + high for 64-bit when requested) 239 phys_addr_t physicalAddress = fPCIInfo->u.h0.base_registers[0]; 240 physicalAddress &= PCI_address_memory_32_mask; 241 if ((fPCIInfo->u.h0.base_register_flags[0] & 0xC) == PCI_address_type_64) 242 physicalAddress += (phys_addr_t)fPCIInfo->u.h0.base_registers[1] << 32; 243 244 size_t mapSize = fPCIInfo->u.h0.base_register_sizes[0]; 245 246 TRACE("map physical memory %08" B_PRIxPHYSADDR ", size: %" B_PRIuSIZE "\n", 247 physicalAddress, mapSize); 248 249 fRegisterArea = map_physical_memory("XHCI memory mapped registers", 250 physicalAddress, mapSize, B_ANY_KERNEL_BLOCK_ADDRESS, 251 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 252 (void **)&fRegisters); 253 if (fRegisterArea < B_OK) { 254 TRACE_ERROR("failed to map register memory\n"); 255 return; 256 } 257 258 // determine the register offsets 259 fCapabilityRegisterOffset = 0; 260 fOperationalRegisterOffset = HCI_CAPLENGTH(ReadCapReg32(XHCI_HCI_CAPLENGTH)); 261 fRuntimeRegisterOffset = ReadCapReg32(XHCI_RTSOFF) & ~0x1F; 262 fDoorbellRegisterOffset = ReadCapReg32(XHCI_DBOFF) & ~0x3; 263 264 TRACE("mapped registers: %p\n", fRegisters); 265 TRACE("operational register offset: %" B_PRId32 "\n", fOperationalRegisterOffset); 266 TRACE("runtime register offset: %" B_PRId32 "\n", fRuntimeRegisterOffset); 267 TRACE("doorbell register offset: %" B_PRId32 "\n", fDoorbellRegisterOffset); 268 269 TRACE_ALWAYS("interface version: 0x%04" B_PRIx32 "\n", 270 HCI_VERSION(ReadCapReg32(XHCI_HCI_VERSION))); 271 TRACE_ALWAYS("structural parameters: 1:0x%08" B_PRIx32 " 2:0x%08" 272 B_PRIx32 " 3:0x%08" B_PRIx32 "\n", ReadCapReg32(XHCI_HCSPARAMS1), 273 ReadCapReg32(XHCI_HCSPARAMS2), ReadCapReg32(XHCI_HCSPARAMS3)); 274 275 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 276 if (cparams == 0xffffffff) 277 return; 278 TRACE_ALWAYS("capability params: 0x%08" B_PRIx32 "\n", cparams); 279 280 // if 64 bytes context structures, then 1 281 fContextSizeShift = HCC_CSZ(cparams); 282 283 // Assume ownership of the controller from the BIOS. 284 uint32 eec = 0xffffffff; 285 uint32 eecp = HCS0_XECP(cparams) << 2; 286 for (; eecp != 0 && XECP_NEXT(eec); eecp += XECP_NEXT(eec) << 2) { 287 TRACE("eecp register: 0x%08" B_PRIx32 "\n", eecp); 288 289 eec = ReadCapReg32(eecp); 290 if (XECP_ID(eec) != XHCI_LEGSUP_CAPID) 291 continue; 292 293 if (eec & XHCI_LEGSUP_BIOSOWNED) { 294 TRACE_ALWAYS("the host controller is bios owned, claiming" 295 " ownership\n"); 296 WriteCapReg32(eecp, eec | XHCI_LEGSUP_OSOWNED); 297 298 for (int32 i = 0; i < 20; i++) { 299 eec = ReadCapReg32(eecp); 300 301 if ((eec & XHCI_LEGSUP_BIOSOWNED) == 0) 302 break; 303 304 TRACE_ALWAYS("controller is still bios owned, waiting\n"); 305 snooze(50000); 306 } 307 308 if (eec & XHCI_LEGSUP_BIOSOWNED) { 309 TRACE_ERROR("bios won't give up control over the host " 310 "controller (ignoring)\n"); 311 } else if (eec & XHCI_LEGSUP_OSOWNED) { 312 TRACE_ALWAYS("successfully took ownership of the host " 313 "controller\n"); 314 } 315 316 // Force off the BIOS owned flag, and clear all SMIs. Some BIOSes 317 // do indicate a successful handover but do not remove their SMIs 318 // and then freeze the system when interrupts are generated. 319 WriteCapReg32(eecp, eec & ~XHCI_LEGSUP_BIOSOWNED); 320 } 321 break; 322 } 323 uint32 legctlsts = ReadCapReg32(eecp + XHCI_LEGCTLSTS); 324 legctlsts &= XHCI_LEGCTLSTS_DISABLE_SMI; 325 legctlsts |= XHCI_LEGCTLSTS_EVENTS_SMI; 326 WriteCapReg32(eecp + XHCI_LEGCTLSTS, legctlsts); 327 328 // On Intel's Panther Point and Lynx Point Chipset taking ownership 329 // of EHCI owned ports, is what we do here. 330 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL) { 331 switch (fPCIInfo->device_id) { 332 case PCI_DEVICE_INTEL_PANTHER_POINT_XHCI: 333 case PCI_DEVICE_INTEL_LYNX_POINT_XHCI: 334 case PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI: 335 case PCI_DEVICE_INTEL_BAYTRAIL_XHCI: 336 case PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI: 337 case PCI_DEVICE_INTEL_WILDCAT_POINT_LP_XHCI: 338 _SwitchIntelPorts(); 339 break; 340 } 341 } 342 343 // halt the host controller 344 if (ControllerHalt() < B_OK) { 345 return; 346 } 347 348 // reset the host controller 349 if (ControllerReset() < B_OK) { 350 TRACE_ERROR("host controller failed to reset\n"); 351 return; 352 } 353 354 fCmdCompSem = create_sem(0, "XHCI Command Complete"); 355 fFinishTransfersSem = create_sem(0, "XHCI Finish Transfers"); 356 fEventSem = create_sem(0, "XHCI Event"); 357 if (fFinishTransfersSem < B_OK || fCmdCompSem < B_OK || fEventSem < B_OK) { 358 TRACE_ERROR("failed to create semaphores\n"); 359 return; 360 } 361 362 // create finisher service thread 363 fFinishThread = spawn_kernel_thread(FinishThread, "xhci finish thread", 364 B_NORMAL_PRIORITY, (void *)this); 365 resume_thread(fFinishThread); 366 367 // create finisher service thread 368 fEventThread = spawn_kernel_thread(EventThread, "xhci event thread", 369 B_NORMAL_PRIORITY, (void *)this); 370 resume_thread(fEventThread); 371 372 // Find the right interrupt vector, using MSIs if available. 373 fIRQ = fPCIInfo->u.h0.interrupt_line; 374 if (sPCIx86Module != NULL && sPCIx86Module->get_msi_count(fPCIInfo->bus, 375 fPCIInfo->device, fPCIInfo->function) >= 1) { 376 uint8 msiVector = 0; 377 if (sPCIx86Module->configure_msi(fPCIInfo->bus, fPCIInfo->device, 378 fPCIInfo->function, 1, &msiVector) == B_OK 379 && sPCIx86Module->enable_msi(fPCIInfo->bus, fPCIInfo->device, 380 fPCIInfo->function) == B_OK) { 381 TRACE_ALWAYS("using message signaled interrupts\n"); 382 fIRQ = msiVector; 383 fUseMSI = true; 384 } 385 } 386 387 if (fIRQ == 0 || fIRQ == 0xFF) { 388 TRACE_MODULE_ERROR("device PCI:%d:%d:%d was assigned an invalid IRQ\n", 389 fPCIInfo->bus, fPCIInfo->device, fPCIInfo->function); 390 return; 391 } 392 393 // Install the interrupt handler 394 TRACE("installing interrupt handler\n"); 395 install_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this, 0); 396 397 memset(fPortSpeeds, 0, sizeof(fPortSpeeds)); 398 memset(fPortSlots, 0, sizeof(fPortSlots)); 399 memset(fDevices, 0, sizeof(fDevices)); 400 401 fInitOK = true; 402 TRACE("XHCI host controller driver constructed\n"); 403 } 404 405 406 XHCI::~XHCI() 407 { 408 TRACE("tear down XHCI host controller driver\n"); 409 410 WriteOpReg(XHCI_CMD, 0); 411 412 int32 result = 0; 413 fStopThreads = true; 414 delete_sem(fCmdCompSem); 415 delete_sem(fFinishTransfersSem); 416 delete_sem(fEventSem); 417 wait_for_thread(fFinishThread, &result); 418 wait_for_thread(fEventThread, &result); 419 420 mutex_destroy(&fFinishedLock); 421 mutex_destroy(&fEventLock); 422 423 remove_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this); 424 425 delete_area(fRegisterArea); 426 delete_area(fErstArea); 427 for (uint32 i = 0; i < fScratchpadCount; i++) 428 delete_area(fScratchpadArea[i]); 429 delete_area(fDcbaArea); 430 431 if (fUseMSI && sPCIx86Module != NULL) { 432 sPCIx86Module->disable_msi(fPCIInfo->bus, 433 fPCIInfo->device, fPCIInfo->function); 434 sPCIx86Module->unconfigure_msi(fPCIInfo->bus, 435 fPCIInfo->device, fPCIInfo->function); 436 } 437 put_module(B_PCI_MODULE_NAME); 438 if (sPCIx86Module != NULL) 439 put_module(B_PCI_X86_MODULE_NAME); 440 } 441 442 443 void 444 XHCI::_SwitchIntelPorts() 445 { 446 TRACE("Intel xHC Controller\n"); 447 TRACE("Looking for EHCI owned ports\n"); 448 uint32 ports = sPCIModule->read_pci_config(fPCIInfo->bus, 449 fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_USB3PRM, 4); 450 TRACE("Superspeed Ports: 0x%" B_PRIx32 "\n", ports); 451 sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device, 452 fPCIInfo->function, XHCI_INTEL_USB3_PSSEN, 4, ports); 453 ports = sPCIModule->read_pci_config(fPCIInfo->bus, 454 fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_USB3_PSSEN, 4); 455 TRACE("Superspeed ports now under XHCI : 0x%" B_PRIx32 "\n", ports); 456 ports = sPCIModule->read_pci_config(fPCIInfo->bus, 457 fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_USB2PRM, 4); 458 TRACE("USB 2.0 Ports : 0x%" B_PRIx32 "\n", ports); 459 sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device, 460 fPCIInfo->function, XHCI_INTEL_XUSB2PR, 4, ports); 461 ports = sPCIModule->read_pci_config(fPCIInfo->bus, 462 fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_XUSB2PR, 4); 463 TRACE("USB 2.0 ports now under XHCI: 0x%" B_PRIx32 "\n", ports); 464 } 465 466 467 status_t 468 XHCI::Start() 469 { 470 TRACE_ALWAYS("starting XHCI host controller\n"); 471 TRACE("usbcmd: 0x%08" B_PRIx32 "; usbsts: 0x%08" B_PRIx32 "\n", 472 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 473 474 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 475 TRACE("Start() failed STS_CNR\n"); 476 } 477 478 if ((ReadOpReg(XHCI_CMD) & CMD_RUN) != 0) { 479 TRACE_ERROR("Start() warning, starting running XHCI controller!\n"); 480 } 481 482 if ((ReadOpReg(XHCI_PAGESIZE) & (1 << 0)) == 0) { 483 TRACE_ERROR("Controller does not support 4K page size.\n"); 484 return B_ERROR; 485 } 486 487 // read port count from capability register 488 uint32 capabilities = ReadCapReg32(XHCI_HCSPARAMS1); 489 fPortCount = HCS_MAX_PORTS(capabilities); 490 if (fPortCount == 0) { 491 TRACE_ERROR("Invalid number of ports: %u\n", fPortCount); 492 return B_ERROR; 493 } 494 495 fSlotCount = HCS_MAX_SLOTS(capabilities); 496 if (fSlotCount > XHCI_MAX_DEVICES) 497 fSlotCount = XHCI_MAX_DEVICES; 498 WriteOpReg(XHCI_CONFIG, fSlotCount); 499 500 // find out which protocol is used for each port 501 uint8 portFound = 0; 502 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 503 uint32 eec = 0xffffffff; 504 uint32 eecp = HCS0_XECP(cparams) << 2; 505 for (; eecp != 0 && XECP_NEXT(eec) && portFound < fPortCount; 506 eecp += XECP_NEXT(eec) << 2) { 507 eec = ReadCapReg32(eecp); 508 if (XECP_ID(eec) != XHCI_SUPPORTED_PROTOCOLS_CAPID) 509 continue; 510 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) > 3) 511 continue; 512 uint32 temp = ReadCapReg32(eecp + 8); 513 uint32 offset = XHCI_SUPPORTED_PROTOCOLS_1_OFFSET(temp); 514 uint32 count = XHCI_SUPPORTED_PROTOCOLS_1_COUNT(temp); 515 if (offset == 0 || count == 0) 516 continue; 517 offset--; 518 for (uint32 i = offset; i < offset + count; i++) { 519 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) == 0x3) 520 fPortSpeeds[i] = USB_SPEED_SUPER; 521 else 522 fPortSpeeds[i] = USB_SPEED_HIGHSPEED; 523 TRACE("speed for port %" B_PRId32 " is %s\n", i, 524 fPortSpeeds[i] == USB_SPEED_SUPER ? "super" : "high"); 525 } 526 portFound += count; 527 } 528 529 uint32 params2 = ReadCapReg32(XHCI_HCSPARAMS2); 530 fScratchpadCount = HCS_MAX_SC_BUFFERS(params2); 531 if (fScratchpadCount > XHCI_MAX_SCRATCHPADS) { 532 TRACE_ERROR("Invalid number of scratchpads: %" B_PRIu32 "\n", 533 fScratchpadCount); 534 return B_ERROR; 535 } 536 537 uint32 params3 = ReadCapReg32(XHCI_HCSPARAMS3); 538 fExitLatMax = HCS_U1_DEVICE_LATENCY(params3) 539 + HCS_U2_DEVICE_LATENCY(params3); 540 541 // clear interrupts & disable device notifications 542 WriteOpReg(XHCI_STS, ReadOpReg(XHCI_STS)); 543 WriteOpReg(XHCI_DNCTRL, 0); 544 545 // allocate Device Context Base Address array 546 phys_addr_t dmaAddress; 547 fDcbaArea = fStack->AllocateArea((void **)&fDcba, &dmaAddress, 548 sizeof(*fDcba), "DCBA Area"); 549 if (fDcbaArea < B_OK) { 550 TRACE_ERROR("unable to create the DCBA area\n"); 551 return B_ERROR; 552 } 553 memset(fDcba, 0, sizeof(*fDcba)); 554 memset(fScratchpadArea, 0, sizeof(fScratchpadArea)); 555 memset(fScratchpad, 0, sizeof(fScratchpad)); 556 557 // setting the first address to the scratchpad array address 558 fDcba->baseAddress[0] = dmaAddress 559 + offsetof(struct xhci_device_context_array, scratchpad); 560 561 // fill up the scratchpad array with scratchpad pages 562 for (uint32 i = 0; i < fScratchpadCount; i++) { 563 phys_addr_t scratchDmaAddress; 564 fScratchpadArea[i] = fStack->AllocateArea((void **)&fScratchpad[i], 565 &scratchDmaAddress, B_PAGE_SIZE, "Scratchpad Area"); 566 if (fScratchpadArea[i] < B_OK) { 567 TRACE_ERROR("unable to create the scratchpad area\n"); 568 return B_ERROR; 569 } 570 fDcba->scratchpad[i] = scratchDmaAddress; 571 } 572 573 TRACE("setting DCBAAP %" B_PRIxPHYSADDR "\n", dmaAddress); 574 WriteOpReg(XHCI_DCBAAP_LO, (uint32)dmaAddress); 575 WriteOpReg(XHCI_DCBAAP_HI, (uint32)(dmaAddress >> 32)); 576 577 // allocate Event Ring Segment Table 578 uint8 *addr; 579 fErstArea = fStack->AllocateArea((void **)&addr, &dmaAddress, 580 (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 581 + sizeof(xhci_erst_element), 582 "USB XHCI ERST CMD_RING and EVENT_RING Area"); 583 584 if (fErstArea < B_OK) { 585 TRACE_ERROR("unable to create the ERST AND RING area\n"); 586 delete_area(fDcbaArea); 587 return B_ERROR; 588 } 589 fErst = (xhci_erst_element *)addr; 590 memset(fErst, 0, (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 591 + sizeof(xhci_erst_element)); 592 593 // fill with Event Ring Segment Base Address and Event Ring Segment Size 594 fErst->rs_addr = dmaAddress + sizeof(xhci_erst_element); 595 fErst->rs_size = XHCI_MAX_EVENTS; 596 fErst->rsvdz = 0; 597 598 addr += sizeof(xhci_erst_element); 599 fEventRing = (xhci_trb *)addr; 600 addr += XHCI_MAX_EVENTS * sizeof(xhci_trb); 601 fCmdRing = (xhci_trb *)addr; 602 603 TRACE("setting ERST size\n"); 604 WriteRunReg32(XHCI_ERSTSZ(0), XHCI_ERSTS_SET(1)); 605 606 TRACE("setting ERDP addr = 0x%" B_PRIx64 "\n", fErst->rs_addr); 607 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)fErst->rs_addr); 608 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(fErst->rs_addr >> 32)); 609 610 TRACE("setting ERST base addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 611 WriteRunReg32(XHCI_ERSTBA_LO(0), (uint32)dmaAddress); 612 WriteRunReg32(XHCI_ERSTBA_HI(0), (uint32)(dmaAddress >> 32)); 613 614 dmaAddress += sizeof(xhci_erst_element) + XHCI_MAX_EVENTS 615 * sizeof(xhci_trb); 616 617 // Make sure the Command Ring is stopped 618 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 619 TRACE_ALWAYS("Command Ring is running, send stop/cancel\n"); 620 WriteOpReg(XHCI_CRCR_LO, CRCR_CS); 621 WriteOpReg(XHCI_CRCR_HI, 0); 622 WriteOpReg(XHCI_CRCR_LO, CRCR_CA); 623 WriteOpReg(XHCI_CRCR_HI, 0); 624 snooze(1000); 625 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 626 TRACE_ERROR("Command Ring still running after stop/cancel\n"); 627 } 628 } 629 TRACE("setting CRCR addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 630 WriteOpReg(XHCI_CRCR_LO, (uint32)dmaAddress | CRCR_RCS); 631 WriteOpReg(XHCI_CRCR_HI, (uint32)(dmaAddress >> 32)); 632 // link trb 633 fCmdRing[XHCI_MAX_COMMANDS - 1].address = dmaAddress; 634 635 TRACE("setting interrupt rate\n"); 636 637 // Setting IMOD below 0x3F8 on Intel Lynx Point can cause IRQ lockups 638 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL 639 && (fPCIInfo->device_id == PCI_DEVICE_INTEL_PANTHER_POINT_XHCI 640 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_XHCI 641 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI 642 || fPCIInfo->device_id == PCI_DEVICE_INTEL_BAYTRAIL_XHCI 643 || fPCIInfo->device_id == PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI)) { 644 WriteRunReg32(XHCI_IMOD(0), 0x000003f8); // 4000 irq/s 645 } else { 646 WriteRunReg32(XHCI_IMOD(0), 0x000001f4); // 8000 irq/s 647 } 648 649 TRACE("enabling interrupt\n"); 650 WriteRunReg32(XHCI_IMAN(0), ReadRunReg32(XHCI_IMAN(0)) | IMAN_INTR_ENA); 651 652 WriteOpReg(XHCI_CMD, CMD_RUN | CMD_INTE | CMD_HSEE); 653 654 // wait for start up state 655 if (WaitOpBits(XHCI_STS, STS_HCH, 0) != B_OK) { 656 TRACE_ERROR("HCH start up timeout\n"); 657 } 658 659 fRootHubAddress = AllocateAddress(); 660 fRootHub = new(std::nothrow) XHCIRootHub(RootObject(), fRootHubAddress); 661 if (!fRootHub) { 662 TRACE_ERROR("no memory to allocate root hub\n"); 663 return B_NO_MEMORY; 664 } 665 666 if (fRootHub->InitCheck() < B_OK) { 667 TRACE_ERROR("root hub failed init check\n"); 668 return fRootHub->InitCheck(); 669 } 670 671 SetRootHub(fRootHub); 672 673 TRACE_ALWAYS("successfully started the controller\n"); 674 #ifdef TRACE_USB 675 TRACE("No-Op test...\n"); 676 status_t noopResult = Noop(); 677 TRACE("No-Op %ssuccessful\n", noopResult < B_OK ? "un" : ""); 678 #endif 679 680 //DumpRing(fCmdRing, (XHCI_MAX_COMMANDS - 1)); 681 682 return BusManager::Start(); 683 } 684 685 686 status_t 687 XHCI::SubmitTransfer(Transfer *transfer) 688 { 689 // short circuit the root hub 690 if (transfer->TransferPipe()->DeviceAddress() == fRootHubAddress) 691 return fRootHub->ProcessTransfer(this, transfer); 692 693 TRACE("SubmitTransfer()\n"); 694 Pipe *pipe = transfer->TransferPipe(); 695 if ((pipe->Type() & USB_OBJECT_ISO_PIPE) != 0) 696 return B_UNSUPPORTED; 697 if ((pipe->Type() & USB_OBJECT_CONTROL_PIPE) != 0) 698 return SubmitControlRequest(transfer); 699 return SubmitNormalRequest(transfer); 700 } 701 702 703 status_t 704 XHCI::SubmitControlRequest(Transfer *transfer) 705 { 706 Pipe *pipe = transfer->TransferPipe(); 707 usb_request_data *requestData = transfer->RequestData(); 708 bool directionIn = (requestData->RequestType & USB_REQTYPE_DEVICE_IN) != 0; 709 710 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 711 if (endpoint == NULL) { 712 TRACE_ERROR("invalid endpoint!\n"); 713 return B_BAD_VALUE; 714 } 715 status_t status = transfer->InitKernelAccess(); 716 if (status != B_OK) 717 return status; 718 719 TRACE("SubmitControlRequest() length %d\n", requestData->Length); 720 721 xhci_td *descriptor = CreateDescriptor(3, 1, requestData->Length); 722 if (descriptor == NULL) 723 return B_NO_MEMORY; 724 descriptor->transfer = transfer; 725 726 // Setup Stage 727 uint8 index = 0; 728 memcpy(&descriptor->trbs[index].address, requestData, 729 sizeof(usb_request_data)); 730 descriptor->trbs[index].status = TRB_2_IRQ(0) | TRB_2_BYTES(8); 731 descriptor->trbs[index].flags 732 = TRB_3_TYPE(TRB_TYPE_SETUP_STAGE) | TRB_3_IDT_BIT | TRB_3_CYCLE_BIT; 733 if (requestData->Length > 0) { 734 descriptor->trbs[index].flags |= 735 directionIn ? TRB_3_TRT_IN : TRB_3_TRT_OUT; 736 } 737 738 index++; 739 740 // Data Stage (if any) 741 if (requestData->Length > 0) { 742 descriptor->trbs[index].address = descriptor->buffer_addrs[0]; 743 descriptor->trbs[index].status = TRB_2_IRQ(0) 744 | TRB_2_BYTES(requestData->Length) 745 | TRB_2_TD_SIZE(0); 746 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_DATA_STAGE) 747 | (directionIn ? (TRB_3_DIR_IN | TRB_3_ISP_BIT) : 0) 748 | TRB_3_CYCLE_BIT; 749 750 if (!directionIn) { 751 transfer->PrepareKernelAccess(); 752 memcpy(descriptor->buffers[0], 753 (uint8 *)transfer->Vector()[0].iov_base, requestData->Length); 754 } 755 756 index++; 757 } 758 759 // Status Stage 760 descriptor->trbs[index].address = 0; 761 descriptor->trbs[index].status = TRB_2_IRQ(0); 762 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_STATUS_STAGE) 763 | ((directionIn && requestData->Length > 0) ? 0 : TRB_3_DIR_IN) 764 | TRB_3_IOC_BIT | TRB_3_CYCLE_BIT; 765 // Status Stage is an OUT transfer when the device is sending data. 766 // (XHCI 1.1 § 4.11.2.2 Table 4-6 p205.) 767 768 descriptor->trb_used = index + 1; 769 770 status = _LinkDescriptorForPipe(descriptor, endpoint); 771 if (status != B_OK) { 772 FreeDescriptor(descriptor); 773 return status; 774 } 775 TRACE("SubmitControlRequest() request linked\n"); 776 777 return B_OK; 778 } 779 780 781 status_t 782 XHCI::SubmitNormalRequest(Transfer *transfer) 783 { 784 TRACE("SubmitNormalRequest() length %ld\n", transfer->DataLength()); 785 786 Pipe *pipe = transfer->TransferPipe(); 787 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 788 if (endpoint == NULL) 789 return B_BAD_VALUE; 790 bool directionIn = (pipe->Direction() == Pipe::In); 791 792 status_t status = transfer->InitKernelAccess(); 793 if (status != B_OK) 794 return status; 795 796 // Compute the size to use for the TRBs, and then how many TRBs 797 // of this size we will need. We always need at least 1, of course. 798 const size_t dataLength = transfer->DataLength(), 799 maxPacketSize = pipe->MaxPacketSize(), 800 packetsPerTrb = 4; 801 const size_t trbSize = packetsPerTrb * maxPacketSize; 802 int32 trbCount = (dataLength + trbSize - 1) / trbSize; 803 804 xhci_td *td = CreateDescriptor(trbCount, trbCount, trbSize); 805 if (td == NULL) 806 return B_NO_MEMORY; 807 808 // Normal Stage 809 size_t remaining = dataLength; 810 int32 remainingPackets = (remaining - trbSize) / maxPacketSize; 811 for (int32 i = 0; i < trbCount; i++) { 812 // The "TD Size" field of a transfer TRB indicates the number of 813 // remaining maximum-size *packets* in this TD, *not* including the 814 // packets in the current TRB, and capped at 31 if there are more 815 // than 31 packets remaining in the TD. (XHCI 1.1 § 4.11.2.4 p210.) 816 int32 tdSize = remainingPackets > 31 ? 31 : remainingPackets; 817 if (tdSize < 0) 818 tdSize = 0; 819 int32 trbLength = remaining < trbSize ? remaining : trbSize; 820 821 td->trbs[i].address = td->buffer_addrs[i]; 822 td->trbs[i].status = TRB_2_IRQ(0) 823 | TRB_2_BYTES(trbLength) 824 | TRB_2_TD_SIZE(tdSize); 825 td->trbs[i].flags = TRB_3_TYPE(TRB_TYPE_NORMAL) 826 | TRB_3_CYCLE_BIT | TRB_3_CHAIN_BIT 827 | (directionIn ? TRB_3_ISP_BIT : 0); 828 829 td->trb_used++; 830 remaining -= trbLength; 831 remainingPackets -= packetsPerTrb; 832 } 833 834 // Set the IOC (Interrupt On Completion) bit so that we will get an event 835 // and interrupt for this TRB as the transfer will be finished. 836 // (XHCI 1.1 § 6.4.1.1 Table 6-22 p443.) 837 td->trbs[td->trb_used - 1].flags |= TRB_3_IOC_BIT; 838 839 // Set the ENT (Evaluate Next TRB) bit, so that the HC will not switch 840 // contexts before evaluating the Link TRB that _LinkDescriptorForPipe 841 // will insert, as otherwise there would be a race between us freeing 842 // and unlinking the descriptor, and the HC evaluating the Link TRB 843 // and thus getting back onto the main ring. 844 // 845 // Note that we *do not* unset the CHAIN bit in this TRB, thus including 846 // the Link TRB in this TD formally, which is required when using the 847 // ENT bit. (XHCI 1.1 § 4.12.3 p241.) 848 td->trbs[td->trb_used - 1].flags |= TRB_3_ENT_BIT; 849 850 if (!directionIn) { 851 TRACE("copying out iov count %ld\n", transfer->VectorCount()); 852 transfer->PrepareKernelAccess(); 853 WriteDescriptor(td, transfer->Vector(), transfer->VectorCount()); 854 } 855 856 td->transfer = transfer; 857 status = _LinkDescriptorForPipe(td, endpoint); 858 if (status != B_OK) { 859 FreeDescriptor(td); 860 return status; 861 } 862 TRACE("SubmitNormalRequest() request linked\n"); 863 864 return B_OK; 865 } 866 867 868 status_t 869 XHCI::CancelQueuedTransfers(Pipe *pipe, bool force) 870 { 871 TRACE_ALWAYS("cancel queued transfers for pipe %p (%d)\n", pipe, 872 pipe->EndpointAddress()); 873 874 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 875 if (endpoint == NULL || endpoint->trbs == NULL) { 876 // Someone's de-allocated this pipe or endpoint in the meantime. 877 // (Possibly AllocateDevice failed, and we were the temporary pipe.) 878 return B_NO_INIT; 879 } 880 881 MutexLocker endpointLocker(endpoint->lock); 882 883 if (endpoint->td_head == NULL) { 884 // There aren't any currently pending transfers to cancel. 885 return B_OK; 886 } 887 888 // Get the head TD from the endpoint. We don't want to free the TDs while 889 // holding the endpoint lock, as the callbacks could potentially cause 890 // deadlocks. 891 xhci_td* td_head = endpoint->td_head; 892 endpoint->td_head = NULL; 893 894 if (StopEndpoint(false, endpoint->id + 1, endpoint->device->slot) == B_OK) { 895 // Clear the endpoint's TRBs. 896 memset(endpoint->trbs, 0, sizeof(xhci_trb) * XHCI_MAX_TRANSFERS); 897 endpoint->used = 0; 898 endpoint->current = 0; 899 900 // Set dequeue pointer location to the beginning of the ring. 901 SetTRDequeue(endpoint->trb_addr, 0, endpoint->id + 1, 902 endpoint->device->slot); 903 904 // We don't need to do anything else to restart the ring, as it will resume 905 // operation as normal upon the next doorbell. (XHCI 1.1 § 4.6.9 p132.) 906 } else { 907 // We couldn't stop the endpoint. Most likely the device has been 908 // removed and the endpoint was stopped by the hardware. 909 TRACE("CancelQueuedTransfers: could not stop endpoint\n"); 910 } 911 912 endpointLocker.Unlock(); 913 914 xhci_td* td; 915 while ((td = td_head) != NULL) { 916 td_head = td_head->next; 917 918 // We can't cancel or delete transfers under "force", as they probably 919 // are not safe to use anymore. 920 if (!force && td->transfer != NULL) { 921 td->transfer->Finished(B_CANCELED, 0); 922 delete td->transfer; 923 td->transfer = NULL; 924 } 925 FreeDescriptor(td); 926 } 927 928 return B_OK; 929 } 930 931 932 status_t 933 XHCI::StartDebugTransfer(Transfer *transfer) 934 { 935 Pipe *pipe = transfer->TransferPipe(); 936 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 937 if (endpoint == NULL) 938 return B_BAD_VALUE; 939 940 // Check all locks that we are going to hit when running transfers. 941 if (mutex_trylock(&endpoint->lock) != B_OK) 942 return B_WOULD_BLOCK; 943 if (mutex_trylock(&fFinishedLock) != B_OK) { 944 mutex_unlock(&endpoint->lock); 945 return B_WOULD_BLOCK; 946 } 947 if (mutex_trylock(&fEventLock) != B_OK) { 948 mutex_unlock(&endpoint->lock); 949 mutex_unlock(&fFinishedLock); 950 return B_WOULD_BLOCK; 951 } 952 mutex_unlock(&endpoint->lock); 953 mutex_unlock(&fFinishedLock); 954 mutex_unlock(&fEventLock); 955 956 status_t status = SubmitTransfer(transfer); 957 if (status != B_OK) 958 return status; 959 960 // The endpoint's head TD is the TD of the just-submitted transfer. 961 // Just like EHCI, abuse the callback cookie to hold the TD pointer. 962 transfer->SetCallback(NULL, endpoint->td_head); 963 964 return B_OK; 965 } 966 967 968 status_t 969 XHCI::CheckDebugTransfer(Transfer *transfer) 970 { 971 xhci_td *transfer_td = (xhci_td *)transfer->CallbackCookie(); 972 if (transfer_td == NULL) 973 return B_NO_INIT; 974 975 // Process events once, and then look for it in the finished list. 976 ProcessEvents(); 977 xhci_td *previous = NULL; 978 for (xhci_td *td = fFinishedHead; td != NULL; td = td->next) { 979 if (td != transfer_td) { 980 previous = td; 981 continue; 982 } 983 984 // We've found it! 985 if (previous == NULL) { 986 fFinishedHead = fFinishedHead->next; 987 } else { 988 previous->next = td->next; 989 } 990 991 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 992 status_t status = (td->trb_completion_code == COMP_SUCCESS 993 || td->trb_completion_code == COMP_SHORT_PACKET) ? B_OK : B_ERROR; 994 995 if (status == B_OK && directionIn) 996 ReadDescriptor(td, transfer->Vector(), transfer->VectorCount()); 997 998 FreeDescriptor(td); 999 transfer->SetCallback(NULL, NULL); 1000 return status; 1001 } 1002 1003 // We didn't find it. 1004 spin(75); 1005 return B_DEV_PENDING; 1006 } 1007 1008 1009 void 1010 XHCI::CancelDebugTransfer(Transfer *transfer) 1011 { 1012 while (CheckDebugTransfer(transfer) == B_DEV_PENDING) 1013 spin(100); 1014 } 1015 1016 1017 status_t 1018 XHCI::NotifyPipeChange(Pipe *pipe, usb_change change) 1019 { 1020 TRACE("pipe change %d for pipe %p (%d)\n", change, pipe, 1021 pipe->EndpointAddress()); 1022 1023 switch (change) { 1024 case USB_CHANGE_CREATED: 1025 return _InsertEndpointForPipe(pipe); 1026 case USB_CHANGE_DESTROYED: 1027 return _RemoveEndpointForPipe(pipe); 1028 1029 case USB_CHANGE_PIPE_POLICY_CHANGED: 1030 // We don't care about these, at least for now. 1031 return B_OK; 1032 } 1033 1034 TRACE_ERROR("unknown pipe change!\n"); 1035 return B_UNSUPPORTED; 1036 } 1037 1038 1039 xhci_td * 1040 XHCI::CreateDescriptor(uint32 trbCount, uint32 bufferCount, size_t bufferSize) 1041 { 1042 const bool inKDL = debug_debugger_running(); 1043 1044 xhci_td *result; 1045 if (!inKDL) { 1046 result = (xhci_td*)calloc(1, sizeof(xhci_td)); 1047 } else { 1048 // Just use the physical memory allocator while in KDL; it's less 1049 // secure than using the regular heap, but it's easier to deal with. 1050 phys_addr_t dummy; 1051 fStack->AllocateChunk((void **)&result, &dummy, sizeof(xhci_td)); 1052 } 1053 1054 if (result == NULL) { 1055 TRACE_ERROR("failed to allocate a transfer descriptor\n"); 1056 return NULL; 1057 } 1058 1059 // We always allocate 1 more TRB than requested, so that 1060 // _LinkDescriptorForPipe() has room to insert a link TRB. 1061 trbCount++; 1062 if (fStack->AllocateChunk((void **)&result->trbs, &result->trb_addr, 1063 (trbCount * sizeof(xhci_trb))) < B_OK) { 1064 TRACE_ERROR("failed to allocate TRBs\n"); 1065 FreeDescriptor(result); 1066 return NULL; 1067 } 1068 result->trb_count = trbCount; 1069 result->trb_used = 0; 1070 1071 if (bufferSize > 0) { 1072 // Due to how the USB stack allocates physical memory, we can't just 1073 // request one large chunk the size of the transfer, and so instead we 1074 // create a series of buffers as requested by our caller. 1075 1076 // We store the buffer pointers and addresses in one memory block. 1077 if (!inKDL) { 1078 result->buffers = (void**)calloc(bufferCount, 1079 (sizeof(void*) + sizeof(phys_addr_t))); 1080 } else { 1081 phys_addr_t dummy; 1082 fStack->AllocateChunk((void **)&result->buffers, &dummy, 1083 bufferCount * (sizeof(void*) + sizeof(phys_addr_t))); 1084 } 1085 if (result->buffers == NULL) { 1086 TRACE_ERROR("unable to allocate space for buffer infos\n"); 1087 FreeDescriptor(result); 1088 return NULL; 1089 } 1090 result->buffer_addrs = (phys_addr_t*)&result->buffers[bufferCount]; 1091 1092 // Optimization: If the requested total size of all buffers is less 1093 // than 32*B_PAGE_SIZE (the maximum size that the physical memory 1094 // allocator can handle), we allocate only one buffer and segment it. 1095 size_t totalSize = bufferSize * bufferCount; 1096 if (totalSize < (32 * B_PAGE_SIZE)) { 1097 if (fStack->AllocateChunk(&result->buffers[0], 1098 &result->buffer_addrs[0], totalSize) < B_OK) { 1099 TRACE_ERROR("unable to allocate space for large buffer (size %ld)\n", 1100 totalSize); 1101 FreeDescriptor(result); 1102 return NULL; 1103 } 1104 for (uint32 i = 1; i < bufferCount; i++) { 1105 result->buffers[i] = (void*)((addr_t)(result->buffers[i - 1]) 1106 + bufferSize); 1107 result->buffer_addrs[i] = result->buffer_addrs[i - 1] 1108 + bufferSize; 1109 } 1110 } else { 1111 // Otherwise, we allocate each buffer individually. 1112 for (uint32 i = 0; i < bufferCount; i++) { 1113 if (fStack->AllocateChunk(&result->buffers[i], 1114 &result->buffer_addrs[i], bufferSize) < B_OK) { 1115 TRACE_ERROR("unable to allocate space for the buffer (size %ld)\n", 1116 bufferSize); 1117 FreeDescriptor(result); 1118 return NULL; 1119 } 1120 } 1121 } 1122 } else { 1123 result->buffers = NULL; 1124 result->buffer_addrs = NULL; 1125 } 1126 result->buffer_size = bufferSize; 1127 result->buffer_count = bufferCount; 1128 1129 // Initialize all other fields. 1130 result->transfer = NULL; 1131 result->trb_completion_code = 0; 1132 result->trb_left = 0; 1133 result->next = NULL; 1134 1135 TRACE("CreateDescriptor allocated %p, buffer_size %ld, buffer_count %ld\n", 1136 result, result->buffer_size, result->buffer_count); 1137 1138 return result; 1139 } 1140 1141 1142 void 1143 XHCI::FreeDescriptor(xhci_td *descriptor) 1144 { 1145 if (descriptor == NULL) 1146 return; 1147 1148 const bool inKDL = debug_debugger_running(); 1149 1150 if (descriptor->trbs != NULL) { 1151 fStack->FreeChunk(descriptor->trbs, descriptor->trb_addr, 1152 (descriptor->trb_count * sizeof(xhci_trb))); 1153 } 1154 if (descriptor->buffers != NULL) { 1155 size_t totalSize = descriptor->buffer_size * descriptor->buffer_count; 1156 if (totalSize < (32 * B_PAGE_SIZE)) { 1157 // This was allocated as one contiguous buffer. 1158 fStack->FreeChunk(descriptor->buffers[0], descriptor->buffer_addrs[0], 1159 totalSize); 1160 } else { 1161 for (uint32 i = 0; i < descriptor->buffer_count; i++) { 1162 if (descriptor->buffers[i] == NULL) 1163 continue; 1164 fStack->FreeChunk(descriptor->buffers[i], descriptor->buffer_addrs[i], 1165 descriptor->buffer_size); 1166 } 1167 } 1168 1169 if (!inKDL) { 1170 free(descriptor->buffers); 1171 } else { 1172 fStack->FreeChunk(descriptor->buffers, 0, 1173 descriptor->buffer_count * (sizeof(void*) + sizeof(phys_addr_t))); 1174 } 1175 } 1176 1177 if (!inKDL) 1178 free(descriptor); 1179 else 1180 fStack->FreeChunk(descriptor, 0, sizeof(xhci_td)); 1181 } 1182 1183 1184 size_t 1185 XHCI::WriteDescriptor(xhci_td *descriptor, iovec *vector, size_t vectorCount) 1186 { 1187 size_t written = 0; 1188 1189 size_t bufIdx = 0, bufUsed = 0; 1190 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1191 size_t length = vector[vecIdx].iov_len; 1192 1193 while (length > 0 && bufIdx < descriptor->buffer_count) { 1194 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1195 memcpy((uint8 *)descriptor->buffers[bufIdx] + bufUsed, 1196 (uint8 *)vector[vecIdx].iov_base + (vector[vecIdx].iov_len - length), 1197 toCopy); 1198 1199 written += toCopy; 1200 bufUsed += toCopy; 1201 length -= toCopy; 1202 if (bufUsed == descriptor->buffer_size) { 1203 bufIdx++; 1204 bufUsed = 0; 1205 } 1206 } 1207 } 1208 1209 TRACE("wrote descriptor (%" B_PRIuSIZE " bytes)\n", written); 1210 return written; 1211 } 1212 1213 1214 size_t 1215 XHCI::ReadDescriptor(xhci_td *descriptor, iovec *vector, size_t vectorCount) 1216 { 1217 size_t read = 0; 1218 1219 size_t bufIdx = 0, bufUsed = 0; 1220 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1221 size_t length = vector[vecIdx].iov_len; 1222 1223 while (length > 0 && bufIdx < descriptor->buffer_count) { 1224 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1225 memcpy((uint8 *)vector[vecIdx].iov_base + (vector[vecIdx].iov_len - length), 1226 (uint8 *)descriptor->buffers[bufIdx] + bufUsed, toCopy); 1227 1228 read += toCopy; 1229 bufUsed += toCopy; 1230 length -= toCopy; 1231 if (bufUsed == descriptor->buffer_size) { 1232 bufIdx++; 1233 bufUsed = 0; 1234 } 1235 } 1236 } 1237 1238 TRACE("read descriptor (%" B_PRIuSIZE " bytes)\n", read); 1239 return read; 1240 } 1241 1242 1243 Device * 1244 XHCI::AllocateDevice(Hub *parent, int8 hubAddress, uint8 hubPort, 1245 usb_speed speed) 1246 { 1247 TRACE("AllocateDevice hubAddress %d hubPort %d speed %d\n", hubAddress, 1248 hubPort, speed); 1249 1250 uint8 slot = XHCI_MAX_SLOTS; 1251 if (EnableSlot(&slot) != B_OK) { 1252 TRACE_ERROR("AllocateDevice() failed enable slot\n"); 1253 return NULL; 1254 } 1255 1256 if (slot == 0 || slot > fSlotCount) { 1257 TRACE_ERROR("AllocateDevice() bad slot\n"); 1258 return NULL; 1259 } 1260 1261 if (fDevices[slot].state != XHCI_STATE_DISABLED) { 1262 TRACE_ERROR("AllocateDevice() slot already used\n"); 1263 return NULL; 1264 } 1265 1266 struct xhci_device *device = &fDevices[slot]; 1267 memset(device, 0, sizeof(struct xhci_device)); 1268 device->state = XHCI_STATE_ENABLED; 1269 device->slot = slot; 1270 1271 device->input_ctx_area = fStack->AllocateArea((void **)&device->input_ctx, 1272 &device->input_ctx_addr, sizeof(*device->input_ctx) << fContextSizeShift, 1273 "XHCI input context"); 1274 if (device->input_ctx_area < B_OK) { 1275 TRACE_ERROR("unable to create a input context area\n"); 1276 device->state = XHCI_STATE_DISABLED; 1277 return NULL; 1278 } 1279 1280 memset(device->input_ctx, 0, sizeof(*device->input_ctx) << fContextSizeShift); 1281 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1282 _WriteContext(&device->input_ctx->input.addFlags, 3); 1283 1284 uint32 route = 0; 1285 uint8 routePort = hubPort; 1286 uint8 rhPort = hubPort; 1287 for (Device *hubDevice = parent; hubDevice != RootObject(); 1288 hubDevice = (Device *)hubDevice->Parent()) { 1289 1290 rhPort = routePort; 1291 if (hubDevice->Parent() == RootObject()) 1292 break; 1293 route *= 16; 1294 if (hubPort > 15) 1295 route += 15; 1296 else 1297 route += routePort; 1298 1299 routePort = hubDevice->HubPort(); 1300 } 1301 1302 // Get speed of port, only if device connected to root hub port 1303 // else we have to rely on value reported by the Hub Explore thread 1304 if (route == 0) { 1305 GetPortSpeed(hubPort - 1, &speed); 1306 TRACE("speed updated %d\n", speed); 1307 } 1308 1309 uint32 dwslot0 = SLOT_0_NUM_ENTRIES(1) | SLOT_0_ROUTE(route); 1310 1311 // add the speed 1312 switch (speed) { 1313 case USB_SPEED_LOWSPEED: 1314 dwslot0 |= SLOT_0_SPEED(2); 1315 break; 1316 case USB_SPEED_HIGHSPEED: 1317 dwslot0 |= SLOT_0_SPEED(3); 1318 break; 1319 case USB_SPEED_FULLSPEED: 1320 dwslot0 |= SLOT_0_SPEED(1); 1321 break; 1322 case USB_SPEED_SUPER: 1323 dwslot0 |= SLOT_0_SPEED(4); 1324 break; 1325 default: 1326 TRACE_ERROR("unknown usb speed\n"); 1327 break; 1328 } 1329 1330 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1331 // TODO enable power save 1332 _WriteContext(&device->input_ctx->slot.dwslot1, SLOT_1_RH_PORT(rhPort)); 1333 uint32 dwslot2 = SLOT_2_IRQ_TARGET(0); 1334 1335 // If LS/FS device connected to non-root HS device 1336 if (route != 0 && parent->Speed() == USB_SPEED_HIGHSPEED 1337 && (speed == USB_SPEED_LOWSPEED || speed == USB_SPEED_FULLSPEED)) { 1338 struct xhci_device *parenthub = (struct xhci_device *) 1339 parent->ControllerCookie(); 1340 dwslot2 |= SLOT_2_PORT_NUM(hubPort); 1341 dwslot2 |= SLOT_2_TT_HUB_SLOT(parenthub->slot); 1342 } 1343 1344 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1345 1346 _WriteContext(&device->input_ctx->slot.dwslot3, SLOT_3_SLOT_STATE(0) 1347 | SLOT_3_DEVICE_ADDRESS(0)); 1348 1349 TRACE("slot 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 1350 "\n", _ReadContext(&device->input_ctx->slot.dwslot0), 1351 _ReadContext(&device->input_ctx->slot.dwslot1), 1352 _ReadContext(&device->input_ctx->slot.dwslot2), 1353 _ReadContext(&device->input_ctx->slot.dwslot3)); 1354 1355 device->device_ctx_area = fStack->AllocateArea((void **)&device->device_ctx, 1356 &device->device_ctx_addr, sizeof(*device->device_ctx) << fContextSizeShift, 1357 "XHCI device context"); 1358 if (device->device_ctx_area < B_OK) { 1359 TRACE_ERROR("unable to create a device context area\n"); 1360 delete_area(device->input_ctx_area); 1361 memset(device, 0, sizeof(xhci_device)); 1362 device->state = XHCI_STATE_DISABLED; 1363 return NULL; 1364 } 1365 memset(device->device_ctx, 0, sizeof(*device->device_ctx) << fContextSizeShift); 1366 1367 device->trb_area = fStack->AllocateArea((void **)&device->trbs, 1368 &device->trb_addr, sizeof(xhci_trb) * (XHCI_MAX_ENDPOINTS - 1) 1369 * XHCI_MAX_TRANSFERS, "XHCI endpoint trbs"); 1370 if (device->trb_area < B_OK) { 1371 TRACE_ERROR("unable to create a device trbs area\n"); 1372 delete_area(device->input_ctx_area); 1373 delete_area(device->device_ctx_area); 1374 memset(device, 0, sizeof(xhci_device)); 1375 device->state = XHCI_STATE_DISABLED; 1376 return NULL; 1377 } 1378 1379 // set up slot pointer to device context 1380 fDcba->baseAddress[slot] = device->device_ctx_addr; 1381 1382 size_t maxPacketSize; 1383 switch (speed) { 1384 case USB_SPEED_LOWSPEED: 1385 case USB_SPEED_FULLSPEED: 1386 maxPacketSize = 8; 1387 break; 1388 case USB_SPEED_HIGHSPEED: 1389 maxPacketSize = 64; 1390 break; 1391 default: 1392 maxPacketSize = 512; 1393 break; 1394 } 1395 1396 // configure the Control endpoint 0 1397 if (ConfigureEndpoint(slot, 0, USB_OBJECT_CONTROL_PIPE, false, 1398 device->trb_addr, 0, maxPacketSize, speed, 0, 0) != B_OK) { 1399 TRACE_ERROR("unable to configure default control endpoint\n"); 1400 delete_area(device->input_ctx_area); 1401 delete_area(device->device_ctx_area); 1402 delete_area(device->trb_area); 1403 memset(device, 0, sizeof(xhci_device)); 1404 device->state = XHCI_STATE_DISABLED; 1405 return NULL; 1406 } 1407 1408 mutex_init(&device->endpoints[0].lock, "xhci endpoint lock"); 1409 device->endpoints[0].device = device; 1410 device->endpoints[0].id = 0; 1411 device->endpoints[0].td_head = NULL; 1412 device->endpoints[0].used = 0; 1413 device->endpoints[0].current = 0; 1414 device->endpoints[0].trbs = device->trbs; 1415 device->endpoints[0].trb_addr = device->trb_addr; 1416 1417 // device should get to addressed state (bsr = 0) 1418 if (SetAddress(device->input_ctx_addr, false, slot) != B_OK) { 1419 TRACE_ERROR("unable to set address\n"); 1420 delete_area(device->input_ctx_area); 1421 delete_area(device->device_ctx_area); 1422 delete_area(device->trb_area); 1423 memset(device, 0, sizeof(xhci_device)); 1424 device->state = XHCI_STATE_DISABLED; 1425 return NULL; 1426 } 1427 1428 device->state = XHCI_STATE_ADDRESSED; 1429 device->address = SLOT_3_DEVICE_ADDRESS_GET(_ReadContext( 1430 &device->device_ctx->slot.dwslot3)); 1431 1432 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", device->address, 1433 SLOT_3_SLOT_STATE_GET(_ReadContext( 1434 &device->device_ctx->slot.dwslot3))); 1435 TRACE("endpoint0 state 0x%08" B_PRIx32 "\n", 1436 ENDPOINT_0_STATE_GET(_ReadContext( 1437 &device->device_ctx->endpoints[0].dwendpoint0))); 1438 1439 // Create a temporary pipe with the new address 1440 ControlPipe pipe(parent); 1441 pipe.SetControllerCookie(&device->endpoints[0]); 1442 pipe.InitCommon(device->address + 1, 0, speed, Pipe::Default, maxPacketSize, 0, 1443 hubAddress, hubPort); 1444 1445 // Get the device descriptor 1446 // Just retrieve the first 8 bytes of the descriptor -> minimum supported 1447 // size of any device. It is enough because it includes the device type. 1448 1449 size_t actualLength = 0; 1450 usb_device_descriptor deviceDescriptor; 1451 1452 TRACE("getting the device descriptor\n"); 1453 status_t status = pipe.SendRequest( 1454 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_STANDARD, // type 1455 USB_REQUEST_GET_DESCRIPTOR, // request 1456 USB_DESCRIPTOR_DEVICE << 8, // value 1457 0, // index 1458 8, // length 1459 (void *)&deviceDescriptor, // buffer 1460 8, // buffer length 1461 &actualLength); // actual length 1462 1463 if (actualLength != 8) { 1464 TRACE_ERROR("error while getting the device descriptor: %s\n", 1465 strerror(status)); 1466 delete_area(device->input_ctx_area); 1467 delete_area(device->device_ctx_area); 1468 delete_area(device->trb_area); 1469 memset(device, 0, sizeof(xhci_device)); 1470 device->state = XHCI_STATE_DISABLED; 1471 return NULL; 1472 } 1473 1474 TRACE("device_class: %d device_subclass %d device_protocol %d\n", 1475 deviceDescriptor.device_class, deviceDescriptor.device_subclass, 1476 deviceDescriptor.device_protocol); 1477 1478 if (speed == USB_SPEED_FULLSPEED && deviceDescriptor.max_packet_size_0 != 8) { 1479 TRACE("Full speed device with different max packet size for Endpoint 0\n"); 1480 uint32 dwendpoint1 = _ReadContext( 1481 &device->input_ctx->endpoints[0].dwendpoint1); 1482 dwendpoint1 &= ~ENDPOINT_1_MAXPACKETSIZE(0xffff); 1483 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE( 1484 deviceDescriptor.max_packet_size_0); 1485 _WriteContext(&device->input_ctx->endpoints[0].dwendpoint1, 1486 dwendpoint1); 1487 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1488 _WriteContext(&device->input_ctx->input.addFlags, (1 << 1)); 1489 EvaluateContext(device->input_ctx_addr, device->slot); 1490 } 1491 1492 Device *deviceObject = NULL; 1493 if (deviceDescriptor.device_class == 0x09) { 1494 TRACE("creating new Hub\n"); 1495 TRACE("getting the hub descriptor\n"); 1496 size_t actualLength = 0; 1497 usb_hub_descriptor hubDescriptor; 1498 status = pipe.SendRequest( 1499 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_CLASS, // type 1500 USB_REQUEST_GET_DESCRIPTOR, // request 1501 USB_DESCRIPTOR_HUB << 8, // value 1502 0, // index 1503 sizeof(usb_hub_descriptor), // length 1504 (void *)&hubDescriptor, // buffer 1505 sizeof(usb_hub_descriptor), // buffer length 1506 &actualLength); 1507 1508 if (actualLength != sizeof(usb_hub_descriptor)) { 1509 TRACE_ERROR("error while getting the hub descriptor: %s\n", 1510 strerror(status)); 1511 delete_area(device->input_ctx_area); 1512 delete_area(device->device_ctx_area); 1513 delete_area(device->trb_area); 1514 memset(device, 0, sizeof(xhci_device)); 1515 device->state = XHCI_STATE_DISABLED; 1516 return NULL; 1517 } 1518 1519 uint32 dwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1520 dwslot0 |= SLOT_0_HUB_BIT; 1521 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1522 uint32 dwslot1 = _ReadContext(&device->input_ctx->slot.dwslot1); 1523 dwslot1 |= SLOT_1_NUM_PORTS(hubDescriptor.num_ports); 1524 _WriteContext(&device->input_ctx->slot.dwslot1, dwslot1); 1525 if (speed == USB_SPEED_HIGHSPEED) { 1526 uint32 dwslot2 = _ReadContext(&device->input_ctx->slot.dwslot2); 1527 dwslot2 |= SLOT_2_TT_TIME(HUB_TTT_GET(hubDescriptor.characteristics)); 1528 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1529 } 1530 1531 deviceObject = new(std::nothrow) Hub(parent, hubAddress, hubPort, 1532 deviceDescriptor, device->address + 1, speed, false, device); 1533 } else { 1534 TRACE("creating new device\n"); 1535 deviceObject = new(std::nothrow) Device(parent, hubAddress, hubPort, 1536 deviceDescriptor, device->address + 1, speed, false, device); 1537 } 1538 if (deviceObject == NULL || deviceObject->InitCheck() != B_OK) { 1539 if (deviceObject == NULL) { 1540 TRACE_ERROR("no memory to allocate device\n"); 1541 } else { 1542 TRACE_ERROR("device object failed to initialize\n"); 1543 } 1544 delete_area(device->input_ctx_area); 1545 delete_area(device->device_ctx_area); 1546 delete_area(device->trb_area); 1547 memset(device, 0, sizeof(xhci_device)); 1548 device->state = XHCI_STATE_DISABLED; 1549 return NULL; 1550 } 1551 1552 // We don't want to disable the default endpoint, naturally, which would 1553 // otherwise happen when this Pipe object is destroyed. 1554 pipe.SetControllerCookie(NULL); 1555 1556 fPortSlots[hubPort] = slot; 1557 TRACE("AllocateDevice() port %d slot %d\n", hubPort, slot); 1558 return deviceObject; 1559 } 1560 1561 1562 void 1563 XHCI::FreeDevice(Device *device) 1564 { 1565 uint8 slot = fPortSlots[device->HubPort()]; 1566 TRACE("FreeDevice() port %d slot %d\n", device->HubPort(), slot); 1567 1568 // Delete the device first, so it cleans up its pipes and tells us 1569 // what we need to destroy before we tear down our internal state. 1570 delete device; 1571 1572 DisableSlot(slot); 1573 fDcba->baseAddress[slot] = 0; 1574 fPortSlots[device->HubPort()] = 0; 1575 delete_area(fDevices[slot].trb_area); 1576 delete_area(fDevices[slot].input_ctx_area); 1577 delete_area(fDevices[slot].device_ctx_area); 1578 1579 memset(&fDevices[slot], 0, sizeof(xhci_device)); 1580 fDevices[slot].state = XHCI_STATE_DISABLED; 1581 } 1582 1583 1584 status_t 1585 XHCI::_InsertEndpointForPipe(Pipe *pipe) 1586 { 1587 TRACE("insert endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1588 1589 if (pipe->ControllerCookie() != NULL 1590 || pipe->Parent()->Type() != USB_OBJECT_DEVICE) { 1591 // default pipe is already referenced 1592 return B_OK; 1593 } 1594 1595 Device* usbDevice = (Device *)pipe->Parent(); 1596 struct xhci_device *device = (struct xhci_device *) 1597 usbDevice->ControllerCookie(); 1598 if (usbDevice->Parent() == RootObject()) 1599 return B_OK; 1600 if (device == NULL) { 1601 panic("_InsertEndpointForPipe device is NULL\n"); 1602 return B_NO_INIT; 1603 } 1604 1605 uint8 id = (2 * pipe->EndpointAddress() 1606 + (pipe->Direction() != Pipe::Out ? 1 : 0)) - 1; 1607 if (id >= XHCI_MAX_ENDPOINTS - 1) 1608 return B_BAD_VALUE; 1609 1610 if (id > 0) { 1611 uint32 devicedwslot0 = _ReadContext(&device->device_ctx->slot.dwslot0); 1612 if (SLOT_0_NUM_ENTRIES_GET(devicedwslot0) == 1) { 1613 uint32 inputdwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1614 inputdwslot0 &= ~(SLOT_0_NUM_ENTRIES(0x1f)); 1615 inputdwslot0 |= SLOT_0_NUM_ENTRIES(XHCI_MAX_ENDPOINTS - 1); 1616 _WriteContext(&device->input_ctx->slot.dwslot0, inputdwslot0); 1617 EvaluateContext(device->input_ctx_addr, device->slot); 1618 } 1619 1620 mutex_init(&device->endpoints[id].lock, "xhci endpoint lock"); 1621 MutexLocker endpointLocker(device->endpoints[id].lock); 1622 1623 device->endpoints[id].device = device; 1624 device->endpoints[id].id = id; 1625 device->endpoints[id].td_head = NULL; 1626 device->endpoints[id].used = 0; 1627 device->endpoints[id].current = 0; 1628 1629 device->endpoints[id].trbs = device->trbs 1630 + id * XHCI_MAX_TRANSFERS; 1631 device->endpoints[id].trb_addr = device->trb_addr 1632 + id * XHCI_MAX_TRANSFERS * sizeof(xhci_trb); 1633 memset(device->endpoints[id].trbs, 0, 1634 sizeof(xhci_trb) * XHCI_MAX_TRANSFERS); 1635 1636 TRACE("_InsertEndpointForPipe trbs device %p endpoint %p\n", 1637 device->trbs, device->endpoints[id].trbs); 1638 TRACE("_InsertEndpointForPipe trb_addr device 0x%" B_PRIxPHYSADDR 1639 " endpoint 0x%" B_PRIxPHYSADDR "\n", device->trb_addr, 1640 device->endpoints[id].trb_addr); 1641 1642 uint8 endpoint = id + 1; 1643 1644 TRACE("trb_addr 0x%" B_PRIxPHYSADDR "\n", device->endpoints[id].trb_addr); 1645 1646 status_t status = ConfigureEndpoint(device->slot, id, pipe->Type(), 1647 pipe->Direction() == Pipe::In, device->endpoints[id].trb_addr, 1648 pipe->Interval(), pipe->MaxPacketSize(), usbDevice->Speed(), 1649 pipe->MaxBurst(), pipe->BytesPerInterval()); 1650 if (status != B_OK) { 1651 TRACE_ERROR("unable to configure endpoint\n"); 1652 return status; 1653 } 1654 1655 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1656 _WriteContext(&device->input_ctx->input.addFlags, 1657 (1 << endpoint) | (1 << 0)); 1658 1659 if (endpoint > 1) 1660 ConfigureEndpoint(device->input_ctx_addr, false, device->slot); 1661 else 1662 EvaluateContext(device->input_ctx_addr, device->slot); 1663 1664 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", 1665 device->address, SLOT_3_SLOT_STATE_GET(_ReadContext( 1666 &device->device_ctx->slot.dwslot3))); 1667 TRACE("endpoint[0] state 0x%08" B_PRIx32 "\n", 1668 ENDPOINT_0_STATE_GET(_ReadContext( 1669 &device->device_ctx->endpoints[0].dwendpoint0))); 1670 TRACE("endpoint[%d] state 0x%08" B_PRIx32 "\n", id, 1671 ENDPOINT_0_STATE_GET(_ReadContext( 1672 &device->device_ctx->endpoints[id].dwendpoint0))); 1673 1674 device->state = XHCI_STATE_CONFIGURED; 1675 } 1676 pipe->SetControllerCookie(&device->endpoints[id]); 1677 1678 TRACE("_InsertEndpointForPipe for pipe %p at id %d\n", pipe, id); 1679 1680 return B_OK; 1681 } 1682 1683 1684 status_t 1685 XHCI::_RemoveEndpointForPipe(Pipe *pipe) 1686 { 1687 TRACE("remove endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1688 1689 if (pipe->Parent()->Type() != USB_OBJECT_DEVICE) 1690 return B_OK; 1691 Device* usbDevice = (Device *)pipe->Parent(); 1692 if (usbDevice->Parent() == RootObject()) 1693 return B_BAD_VALUE; 1694 1695 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 1696 if (endpoint == NULL || endpoint->trbs == NULL) 1697 return B_NO_INIT; 1698 1699 xhci_device *device = endpoint->device; 1700 1701 if (endpoint->id > 0) { 1702 mutex_lock(&endpoint->lock); 1703 1704 uint8 epNumber = endpoint->id + 1; 1705 StopEndpoint(true, epNumber, device->slot); 1706 1707 mutex_destroy(&endpoint->lock); 1708 memset(endpoint, 0, sizeof(xhci_endpoint)); 1709 1710 _WriteContext(&device->input_ctx->input.dropFlags, (1 << epNumber)); 1711 _WriteContext(&device->input_ctx->input.addFlags, 0); 1712 1713 if (epNumber > 1) 1714 ConfigureEndpoint(device->input_ctx_addr, true, device->slot); 1715 else 1716 EvaluateContext(device->input_ctx_addr, device->slot); 1717 1718 device->state = XHCI_STATE_ADDRESSED; 1719 } 1720 pipe->SetControllerCookie(NULL); 1721 1722 return B_OK; 1723 } 1724 1725 1726 status_t 1727 XHCI::_LinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 1728 { 1729 TRACE("_LinkDescriptorForPipe\n"); 1730 1731 // We must check this before we lock the endpoint, because if it is 1732 // NULL, the mutex is probably uninitialized, too. 1733 if (endpoint->device == NULL) { 1734 TRACE_ERROR("trying to submit a transfer to a non-existent endpoint!\n"); 1735 return B_NO_INIT; 1736 } 1737 1738 // Use mutex_trylock first, in case we are in KDL. 1739 if (mutex_trylock(&endpoint->lock) != B_OK) 1740 mutex_lock(&endpoint->lock); 1741 1742 // We will be modifying 2 TRBs as part of linking a new descriptor: 1743 // the "current" TRB (which will link to the passed descriptor), and 1744 // the "next" (current + 1) TRB (which will be zeroed, as we have 1745 // likely used it before.) Hence the "+ 1" in this check. 1746 if ((endpoint->used + 1) >= XHCI_MAX_TRANSFERS) { 1747 TRACE_ERROR("_LinkDescriptorForPipe max transfers count exceeded\n"); 1748 mutex_unlock(&endpoint->lock); 1749 return B_BAD_VALUE; 1750 } 1751 1752 endpoint->used++; 1753 descriptor->next = endpoint->td_head; 1754 endpoint->td_head = descriptor; 1755 1756 uint8 current = endpoint->current; 1757 uint8 next = (current + 1) % (XHCI_MAX_TRANSFERS); 1758 1759 TRACE("_LinkDescriptorForPipe current %d, next %d\n", current, next); 1760 1761 // Compute next link. 1762 addr_t addr = endpoint->trb_addr + next * sizeof(xhci_trb); 1763 descriptor->trbs[descriptor->trb_used].address = addr; 1764 descriptor->trbs[descriptor->trb_used].status = TRB_2_IRQ(0); 1765 descriptor->trbs[descriptor->trb_used].flags = TRB_3_TYPE(TRB_TYPE_LINK) 1766 | TRB_3_CYCLE_BIT; 1767 1768 #if !B_HOST_IS_LENDIAN 1769 // Convert endianness. 1770 for (uint32 i = 0; i <= descriptor->trb_used; i++) { 1771 descriptor->trbs[i].address = 1772 B_HOST_TO_LENDIAN_INT64(descriptor->trbs[i].address); 1773 descriptor->trbs[i].status = 1774 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].status); 1775 descriptor->trbs[i].flags = 1776 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].flags); 1777 } 1778 #endif 1779 1780 // Link the descriptor. 1781 endpoint->trbs[next].address = 0; 1782 endpoint->trbs[next].status = 0; 1783 endpoint->trbs[next].flags = 0; 1784 1785 endpoint->trbs[current].address = 1786 B_HOST_TO_LENDIAN_INT64(descriptor->trb_addr); 1787 endpoint->trbs[current].status = 1788 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 1789 endpoint->trbs[current].flags = 1790 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_LINK)); 1791 1792 // Everything is ready, so write the cycle bit. 1793 endpoint->trbs[current].flags |= B_HOST_TO_LENDIAN_INT32(TRB_3_CYCLE_BIT); 1794 1795 TRACE("_LinkDescriptorForPipe pCurrent %p phys 0x%" B_PRIxPHYSADDR 1796 " 0x%" B_PRIxPHYSADDR " 0x%08" B_PRIx32 "\n", &endpoint->trbs[current], 1797 endpoint->trb_addr + current * sizeof(struct xhci_trb), 1798 endpoint->trbs[current].address, 1799 B_LENDIAN_TO_HOST_INT32(endpoint->trbs[current].flags)); 1800 1801 endpoint->current = next; 1802 mutex_unlock(&endpoint->lock); 1803 1804 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 1805 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 1806 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 1807 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 1808 1809 Ring(endpoint->device->slot, endpoint->id + 1); 1810 1811 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 1812 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 1813 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 1814 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 1815 1816 return B_OK; 1817 } 1818 1819 1820 status_t 1821 XHCI::_UnlinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 1822 { 1823 TRACE("_UnlinkDescriptorForPipe\n"); 1824 // We presume that the caller has already locked or owns the endpoint. 1825 1826 endpoint->used--; 1827 if (descriptor == endpoint->td_head) { 1828 endpoint->td_head = descriptor->next; 1829 descriptor->next = NULL; 1830 return B_OK; 1831 } else { 1832 for (xhci_td *td = endpoint->td_head; td->next != NULL; td = td->next) { 1833 if (td->next == descriptor) { 1834 td->next = descriptor->next; 1835 descriptor->next = NULL; 1836 return B_OK; 1837 } 1838 } 1839 } 1840 1841 endpoint->used++; 1842 return B_ERROR; 1843 } 1844 1845 1846 status_t 1847 XHCI::ConfigureEndpoint(uint8 slot, uint8 number, uint8 type, bool directionIn, 1848 uint64 ringAddr, uint16 interval, uint16 maxPacketSize, usb_speed speed, 1849 uint8 maxBurst, uint16 bytesPerInterval) 1850 { 1851 struct xhci_device* device = &fDevices[slot]; 1852 1853 uint32 dwendpoint0 = 0; 1854 uint32 dwendpoint1 = 0; 1855 uint64 qwendpoint2 = 0; 1856 uint32 dwendpoint4 = 0; 1857 1858 // Compute and assign the endpoint type. (XHCI 1.1 § 6.2.3 Table 6-9 p429.) 1859 uint8 xhciType = 4; 1860 if ((type & USB_OBJECT_INTERRUPT_PIPE) != 0) 1861 xhciType = 3; 1862 if ((type & USB_OBJECT_BULK_PIPE) != 0) 1863 xhciType = 2; 1864 if ((type & USB_OBJECT_ISO_PIPE) != 0) 1865 xhciType = 1; 1866 xhciType |= directionIn ? (1 << 2) : 0; 1867 dwendpoint1 |= ENDPOINT_1_EPTYPE(xhciType); 1868 1869 // Compute and assign interval. (XHCI 1.1 § 6.2.3.6 p433.) 1870 uint16 calcInterval; 1871 if ((type & USB_OBJECT_BULK_PIPE) != 0 1872 || (type & USB_OBJECT_CONTROL_PIPE) != 0) { 1873 // Bulk and Control endpoints never issue NAKs. 1874 calcInterval = 0; 1875 } else { 1876 switch (speed) { 1877 case USB_SPEED_FULLSPEED: 1878 if ((type & USB_OBJECT_ISO_PIPE) != 0) { 1879 // Convert 1-16 into 3-18. 1880 calcInterval = min_c(max_c(interval, 1), 16) + 2; 1881 break; 1882 } 1883 1884 // fall through 1885 case USB_SPEED_LOWSPEED: { 1886 // Convert 1ms-255ms into 3-10. 1887 1888 // Find the index of the highest set bit in "interval". 1889 uint32 temp = min_c(max_c(interval, 1), 255); 1890 for (calcInterval = 0; temp != 1; calcInterval++) 1891 temp = temp >> 1; 1892 calcInterval += 3; 1893 break; 1894 } 1895 1896 case USB_SPEED_HIGHSPEED: 1897 case USB_SPEED_SUPER: 1898 default: 1899 // Convert 1-16 into 0-15. 1900 calcInterval = min_c(max_c(interval, 1), 16) - 1; 1901 break; 1902 } 1903 } 1904 dwendpoint0 |= ENDPOINT_0_INTERVAL(calcInterval); 1905 1906 // For non-isochronous endpoints, we want the controller to retry failed 1907 // transfers, if possible. (XHCI 1.1 § 4.10.2.3 p189.) 1908 if ((type & USB_OBJECT_ISO_PIPE) == 0) 1909 dwendpoint1 |= ENDPOINT_1_CERR(3); 1910 1911 // Assign maximum burst size. For USB3 devices this is passed in; for 1912 // all other devices we compute it. (XHCI 1.1 § 4.8.2 p154.) 1913 if (speed == USB_SPEED_HIGHSPEED && (type & (USB_OBJECT_INTERRUPT_PIPE 1914 | USB_OBJECT_ISO_PIPE)) != 0) { 1915 maxBurst = (maxPacketSize & 0x1800) >> 11; 1916 } else if (speed != USB_SPEED_SUPER) { 1917 maxBurst = 0; 1918 } 1919 dwendpoint1 |= ENDPOINT_1_MAXBURST(maxBurst); 1920 1921 // Assign maximum packet size, set the ring address, and set the 1922 // "Dequeue Cycle State" bit. (XHCI 1.1 § 6.2.3 Table 6-10 p430.) 1923 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE(maxPacketSize); 1924 qwendpoint2 |= ENDPOINT_2_DCS_BIT | ringAddr; 1925 1926 // Assign average TRB length. 1927 if ((type & USB_OBJECT_CONTROL_PIPE) != 0) { 1928 // Control pipes are a special case, as they rarely have 1929 // outbound transfers of any substantial size. 1930 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(8); 1931 } else { 1932 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(maxPacketSize * 4); 1933 } 1934 1935 // Assign maximum ESIT payload. (XHCI 1.1 § 4.14.2 p250.) 1936 if ((type & (USB_OBJECT_INTERRUPT_PIPE | USB_OBJECT_ISO_PIPE)) != 0) { 1937 // TODO: For SuperSpeedPlus endpoints, there is yet another descriptor 1938 // for isochronous endpoints that specifies the maximum ESIT payload. 1939 // We don't fetch this yet, so just fall back to the USB2 computation 1940 // method if bytesPerInterval is 0. 1941 if (speed == USB_SPEED_SUPER && bytesPerInterval != 0) 1942 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD(bytesPerInterval); 1943 else if (speed >= USB_SPEED_HIGHSPEED) 1944 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD((maxBurst + 1) * maxPacketSize); 1945 } 1946 1947 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint0, 1948 dwendpoint0); 1949 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint1, 1950 dwendpoint1); 1951 _WriteContext(&device->input_ctx->endpoints[number].qwendpoint2, 1952 qwendpoint2); 1953 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint4, 1954 dwendpoint4); 1955 1956 TRACE("endpoint 0x%" B_PRIx32 " 0x%" B_PRIx32 " 0x%" B_PRIx64 " 0x%" 1957 B_PRIx32 "\n", 1958 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint0), 1959 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint1), 1960 _ReadContext(&device->input_ctx->endpoints[number].qwendpoint2), 1961 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint4)); 1962 1963 return B_OK; 1964 } 1965 1966 1967 status_t 1968 XHCI::GetPortSpeed(uint8 index, usb_speed* speed) 1969 { 1970 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 1971 1972 switch (PS_SPEED_GET(portStatus)) { 1973 case 3: 1974 *speed = USB_SPEED_HIGHSPEED; 1975 break; 1976 case 2: 1977 *speed = USB_SPEED_LOWSPEED; 1978 break; 1979 case 1: 1980 *speed = USB_SPEED_FULLSPEED; 1981 break; 1982 case 4: 1983 *speed = USB_SPEED_SUPER; 1984 break; 1985 default: 1986 TRACE("Non Standard Port Speed\n"); 1987 TRACE("Assuming Superspeed\n"); 1988 *speed = USB_SPEED_SUPER; 1989 break; 1990 } 1991 1992 return B_OK; 1993 } 1994 1995 1996 status_t 1997 XHCI::GetPortStatus(uint8 index, usb_port_status* status) 1998 { 1999 if (index >= fPortCount) 2000 return B_BAD_INDEX; 2001 2002 status->status = status->change = 0; 2003 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 2004 TRACE("port %" B_PRId8 " status=0x%08" B_PRIx32 "\n", index, portStatus); 2005 2006 // build the status 2007 switch (PS_SPEED_GET(portStatus)) { 2008 case 3: 2009 status->status |= PORT_STATUS_HIGH_SPEED; 2010 break; 2011 case 2: 2012 status->status |= PORT_STATUS_LOW_SPEED; 2013 break; 2014 default: 2015 break; 2016 } 2017 2018 if (portStatus & PS_CCS) 2019 status->status |= PORT_STATUS_CONNECTION; 2020 if (portStatus & PS_PED) 2021 status->status |= PORT_STATUS_ENABLE; 2022 if (portStatus & PS_OCA) 2023 status->status |= PORT_STATUS_OVER_CURRENT; 2024 if (portStatus & PS_PR) 2025 status->status |= PORT_STATUS_RESET; 2026 if (portStatus & PS_PP) { 2027 if (fPortSpeeds[index] == USB_SPEED_SUPER) 2028 status->status |= PORT_STATUS_SS_POWER; 2029 else 2030 status->status |= PORT_STATUS_POWER; 2031 } 2032 2033 // build the change 2034 if (portStatus & PS_CSC) 2035 status->change |= PORT_STATUS_CONNECTION; 2036 if (portStatus & PS_PEC) 2037 status->change |= PORT_STATUS_ENABLE; 2038 if (portStatus & PS_OCC) 2039 status->change |= PORT_STATUS_OVER_CURRENT; 2040 if (portStatus & PS_PRC) 2041 status->change |= PORT_STATUS_RESET; 2042 2043 if (fPortSpeeds[index] == USB_SPEED_SUPER) { 2044 if (portStatus & PS_PLC) 2045 status->change |= PORT_CHANGE_LINK_STATE; 2046 if (portStatus & PS_WRC) 2047 status->change |= PORT_CHANGE_BH_PORT_RESET; 2048 } 2049 2050 return B_OK; 2051 } 2052 2053 2054 status_t 2055 XHCI::SetPortFeature(uint8 index, uint16 feature) 2056 { 2057 TRACE("set port feature index %u feature %u\n", index, feature); 2058 if (index >= fPortCount) 2059 return B_BAD_INDEX; 2060 2061 uint32 portRegister = XHCI_PORTSC(index); 2062 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2063 2064 switch (feature) { 2065 case PORT_SUSPEND: 2066 if ((portStatus & PS_PED) == 0 || (portStatus & PS_PR) 2067 || (portStatus & PS_PLS_MASK) >= PS_XDEV_U3) { 2068 TRACE_ERROR("USB core suspending device not in U0/U1/U2.\n"); 2069 return B_BAD_VALUE; 2070 } 2071 portStatus &= ~PS_PLS_MASK; 2072 WriteOpReg(portRegister, portStatus | PS_LWS | PS_XDEV_U3); 2073 break; 2074 2075 case PORT_RESET: 2076 WriteOpReg(portRegister, portStatus | PS_PR); 2077 break; 2078 2079 case PORT_POWER: 2080 WriteOpReg(portRegister, portStatus | PS_PP); 2081 break; 2082 default: 2083 return B_BAD_VALUE; 2084 } 2085 ReadOpReg(portRegister); 2086 return B_OK; 2087 } 2088 2089 2090 status_t 2091 XHCI::ClearPortFeature(uint8 index, uint16 feature) 2092 { 2093 TRACE("clear port feature index %u feature %u\n", index, feature); 2094 if (index >= fPortCount) 2095 return B_BAD_INDEX; 2096 2097 uint32 portRegister = XHCI_PORTSC(index); 2098 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2099 2100 switch (feature) { 2101 case PORT_SUSPEND: 2102 portStatus = ReadOpReg(portRegister); 2103 if (portStatus & PS_PR) 2104 return B_BAD_VALUE; 2105 if (portStatus & PS_XDEV_U3) { 2106 if ((portStatus & PS_PED) == 0) 2107 return B_BAD_VALUE; 2108 portStatus &= ~PS_PLS_MASK; 2109 WriteOpReg(portRegister, portStatus | PS_XDEV_U0 | PS_LWS); 2110 } 2111 break; 2112 case PORT_ENABLE: 2113 WriteOpReg(portRegister, portStatus | PS_PED); 2114 break; 2115 case PORT_POWER: 2116 WriteOpReg(portRegister, portStatus & ~PS_PP); 2117 break; 2118 case C_PORT_CONNECTION: 2119 WriteOpReg(portRegister, portStatus | PS_CSC); 2120 break; 2121 case C_PORT_ENABLE: 2122 WriteOpReg(portRegister, portStatus | PS_PEC); 2123 break; 2124 case C_PORT_OVER_CURRENT: 2125 WriteOpReg(portRegister, portStatus | PS_OCC); 2126 break; 2127 case C_PORT_RESET: 2128 WriteOpReg(portRegister, portStatus | PS_PRC); 2129 break; 2130 case C_PORT_BH_PORT_RESET: 2131 WriteOpReg(portRegister, portStatus | PS_WRC); 2132 break; 2133 case C_PORT_LINK_STATE: 2134 WriteOpReg(portRegister, portStatus | PS_PLC); 2135 break; 2136 default: 2137 return B_BAD_VALUE; 2138 } 2139 2140 ReadOpReg(portRegister); 2141 return B_OK; 2142 } 2143 2144 2145 status_t 2146 XHCI::ControllerHalt() 2147 { 2148 // Mask off run state 2149 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) & ~CMD_RUN); 2150 2151 // wait for shutdown state 2152 if (WaitOpBits(XHCI_STS, STS_HCH, STS_HCH) != B_OK) { 2153 TRACE_ERROR("HCH shutdown timeout\n"); 2154 return B_ERROR; 2155 } 2156 return B_OK; 2157 } 2158 2159 2160 status_t 2161 XHCI::ControllerReset() 2162 { 2163 TRACE("ControllerReset() cmd: 0x%" B_PRIx32 " sts: 0x%" B_PRIx32 "\n", 2164 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 2165 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) | CMD_HCRST); 2166 2167 if (WaitOpBits(XHCI_CMD, CMD_HCRST, 0) != B_OK) { 2168 TRACE_ERROR("ControllerReset() failed CMD_HCRST\n"); 2169 return B_ERROR; 2170 } 2171 2172 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 2173 TRACE_ERROR("ControllerReset() failed STS_CNR\n"); 2174 return B_ERROR; 2175 } 2176 2177 return B_OK; 2178 } 2179 2180 2181 int32 2182 XHCI::InterruptHandler(void* data) 2183 { 2184 return ((XHCI*)data)->Interrupt(); 2185 } 2186 2187 2188 int32 2189 XHCI::Interrupt() 2190 { 2191 SpinLocker _(&fSpinlock); 2192 2193 uint32 status = ReadOpReg(XHCI_STS); 2194 uint32 temp = ReadRunReg32(XHCI_IMAN(0)); 2195 WriteOpReg(XHCI_STS, status); 2196 WriteRunReg32(XHCI_IMAN(0), temp); 2197 2198 int32 result = B_HANDLED_INTERRUPT; 2199 2200 if ((status & STS_HCH) != 0) { 2201 TRACE_ERROR("Host Controller halted\n"); 2202 return result; 2203 } 2204 if ((status & STS_HSE) != 0) { 2205 TRACE_ERROR("Host System Error\n"); 2206 return result; 2207 } 2208 if ((status & STS_HCE) != 0) { 2209 TRACE_ERROR("Host Controller Error\n"); 2210 return result; 2211 } 2212 2213 if ((status & STS_EINT) == 0) { 2214 TRACE("STS: 0x%" B_PRIx32 " IRQ_PENDING: 0x%" B_PRIx32 "\n", 2215 status, temp); 2216 return B_UNHANDLED_INTERRUPT; 2217 } 2218 2219 TRACE("Event Interrupt\n"); 2220 release_sem_etc(fEventSem, 1, B_DO_NOT_RESCHEDULE); 2221 return B_INVOKE_SCHEDULER; 2222 } 2223 2224 2225 void 2226 XHCI::Ring(uint8 slot, uint8 endpoint) 2227 { 2228 TRACE("Ding Dong! slot:%d endpoint %d\n", slot, endpoint) 2229 if ((slot == 0 && endpoint > 0) || (slot > 0 && endpoint == 0)) 2230 panic("Ring() invalid slot/endpoint combination\n"); 2231 if (slot > fSlotCount || endpoint >= XHCI_MAX_ENDPOINTS) 2232 panic("Ring() invalid slot or endpoint\n"); 2233 2234 WriteDoorReg32(XHCI_DOORBELL(slot), XHCI_DOORBELL_TARGET(endpoint) 2235 | XHCI_DOORBELL_STREAMID(0)); 2236 /* Flush PCI posted writes */ 2237 ReadDoorReg32(XHCI_DOORBELL(slot)); 2238 } 2239 2240 2241 void 2242 XHCI::QueueCommand(xhci_trb* trb) 2243 { 2244 uint8 i, j; 2245 uint32 temp; 2246 2247 i = fCmdIdx; 2248 j = fCmdCcs; 2249 2250 TRACE("command[%u] = %" B_PRId32 " (0x%016" B_PRIx64 ", 0x%08" B_PRIx32 2251 ", 0x%08" B_PRIx32 ")\n", i, TRB_3_TYPE_GET(trb->flags), trb->address, 2252 trb->status, trb->flags); 2253 2254 fCmdRing[i].address = trb->address; 2255 fCmdRing[i].status = trb->status; 2256 temp = trb->flags; 2257 2258 if (j) 2259 temp |= TRB_3_CYCLE_BIT; 2260 else 2261 temp &= ~TRB_3_CYCLE_BIT; 2262 temp &= ~TRB_3_TC_BIT; 2263 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2264 2265 fCmdAddr = fErst->rs_addr + (XHCI_MAX_EVENTS + i) * sizeof(xhci_trb); 2266 2267 i++; 2268 2269 if (i == (XHCI_MAX_COMMANDS - 1)) { 2270 temp = TRB_3_TYPE(TRB_TYPE_LINK) | TRB_3_TC_BIT; 2271 if (j) 2272 temp |= TRB_3_CYCLE_BIT; 2273 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2274 2275 i = 0; 2276 j ^= 1; 2277 } 2278 2279 fCmdIdx = i; 2280 fCmdCcs = j; 2281 } 2282 2283 2284 void 2285 XHCI::HandleCmdComplete(xhci_trb* trb) 2286 { 2287 TRACE("HandleCmdComplete trb %p\n", trb); 2288 2289 if (fCmdAddr == trb->address) { 2290 TRACE("Received command event\n"); 2291 fCmdResult[0] = trb->status; 2292 fCmdResult[1] = B_LENDIAN_TO_HOST_INT32(trb->flags); 2293 release_sem_etc(fCmdCompSem, 1, B_DO_NOT_RESCHEDULE); 2294 } 2295 } 2296 2297 2298 void 2299 XHCI::HandleTransferComplete(xhci_trb* trb) 2300 { 2301 TRACE("HandleTransferComplete trb %p\n", trb); 2302 2303 uint8 endpointNumber 2304 = TRB_3_ENDPOINT_GET(B_LENDIAN_TO_HOST_INT32(trb->flags)); 2305 uint8 slot = TRB_3_SLOT_GET(B_LENDIAN_TO_HOST_INT32(trb->flags)); 2306 uint8 type = TRB_3_TYPE_GET(B_LENDIAN_TO_HOST_INT32(trb->flags)); 2307 2308 if (slot > fSlotCount) 2309 TRACE_ERROR("invalid slot\n"); 2310 if (endpointNumber == 0 || endpointNumber >= XHCI_MAX_ENDPOINTS) 2311 TRACE_ERROR("invalid endpoint\n"); 2312 if (type == TRB_TYPE_EVENT_DATA) { 2313 // TODO: Implement these. (Do we trigger any at present?) 2314 TRACE_ERROR("event data TRBs are not handled yet!\n"); 2315 return; 2316 } 2317 2318 xhci_device *device = &fDevices[slot]; 2319 xhci_endpoint *endpoint = &device->endpoints[endpointNumber - 1]; 2320 2321 if (endpoint->trbs == NULL) { 2322 TRACE_ERROR("got TRB but endpoint is not allocated!\n"); 2323 return; 2324 } 2325 2326 // Use mutex_trylock first, in case we are in KDL. 2327 MutexLocker endpointLocker(endpoint->lock, 2328 mutex_trylock(&endpoint->lock) == B_OK); 2329 if (!endpointLocker.IsLocked()) { 2330 // We failed to get the lock. Most likely it was destroyed 2331 // while we were waiting for it. 2332 return; 2333 } 2334 2335 addr_t source = trb->address; 2336 uint8 completionCode = TRB_2_COMP_CODE_GET(trb->status); 2337 uint32 remainder = TRB_2_REM_GET(trb->status); 2338 2339 for (xhci_td *td = endpoint->td_head; td != NULL; td = td->next) { 2340 int64 offset = (source - td->trb_addr) / sizeof(xhci_trb); 2341 if (offset < 0 || offset >= td->trb_count) 2342 continue; 2343 2344 TRACE("HandleTransferComplete td %p trb %" B_PRId64 " found\n", 2345 td, offset); 2346 2347 // The TRB at offset trb_used will be the link TRB, which we do not 2348 // care about (and should not generate an interrupt at all.) 2349 // We really care about the properly last TRB, at index "count - 1". 2350 // Additionally, if we have an unsuccessful completion code, the transfer 2351 // likely failed midway; so just accept it anyway. 2352 if (offset == (td->trb_used - 1) || completionCode != COMP_SUCCESS) { 2353 _UnlinkDescriptorForPipe(td, endpoint); 2354 endpointLocker.Unlock(); 2355 2356 td->trb_completion_code = completionCode; 2357 td->trb_left = remainder; 2358 2359 // add descriptor to finished list 2360 if (mutex_trylock(&fFinishedLock) != B_OK) 2361 mutex_lock(&fFinishedLock); 2362 td->next = fFinishedHead; 2363 fFinishedHead = td; 2364 mutex_unlock(&fFinishedLock); 2365 2366 release_sem_etc(fFinishTransfersSem, 1, B_DO_NOT_RESCHEDULE); 2367 TRACE("HandleTransferComplete td %p done\n", td); 2368 } else { 2369 TRACE_ERROR("successful TRB %" B_PRIxADDR " was found, but it wasn't " 2370 "the last in the TD!\n", source); 2371 } 2372 return; 2373 } 2374 TRACE_ERROR("TRB 0x%" B_PRIxADDR " was not found in the endpoint!\n", source); 2375 } 2376 2377 2378 void 2379 XHCI::DumpRing(xhci_trb *trbs, uint32 size) 2380 { 2381 if (!Lock()) { 2382 TRACE("Unable to get lock!\n"); 2383 return; 2384 } 2385 2386 for (uint32 i = 0; i < size; i++) { 2387 TRACE("command[%" B_PRId32 "] = %" B_PRId32 " (0x%016" B_PRIx64 "," 2388 " 0x%08" B_PRIx32 ", 0x%08" B_PRIx32 ")\n", i, 2389 TRB_3_TYPE_GET(B_LENDIAN_TO_HOST_INT32(trbs[i].flags)), 2390 trbs[i].address, trbs[i].status, trbs[i].flags); 2391 } 2392 2393 Unlock(); 2394 } 2395 2396 2397 status_t 2398 XHCI::DoCommand(xhci_trb* trb) 2399 { 2400 if (!Lock()) { 2401 TRACE("Unable to get lock!\n"); 2402 return B_ERROR; 2403 } 2404 2405 QueueCommand(trb); 2406 Ring(0, 0); 2407 2408 if (acquire_sem_etc(fCmdCompSem, 1, B_RELATIVE_TIMEOUT, 1 * 1000 * 1000) < B_OK) { 2409 TRACE("Unable to obtain fCmdCompSem!\n"); 2410 Unlock(); 2411 return B_TIMED_OUT; 2412 } 2413 // eat up sems that have been released by multiple interrupts 2414 int32 semCount = 0; 2415 get_sem_count(fCmdCompSem, &semCount); 2416 if (semCount > 0) 2417 acquire_sem_etc(fCmdCompSem, semCount, B_RELATIVE_TIMEOUT, 0); 2418 2419 status_t status = B_OK; 2420 uint32 completionCode = TRB_2_COMP_CODE_GET(fCmdResult[0]); 2421 TRACE("Command Complete. Result: %" B_PRId32 "\n", completionCode); 2422 if (completionCode != COMP_SUCCESS) { 2423 TRACE_ERROR("unsuccessful command %" B_PRId32 ", error %s (%" B_PRId32 ")\n", 2424 TRB_3_TYPE_GET(trb->flags), xhci_error_string(completionCode), 2425 completionCode); 2426 status = B_IO_ERROR; 2427 } 2428 2429 trb->status = fCmdResult[0]; 2430 trb->flags = fCmdResult[1]; 2431 TRACE("Storing trb 0x%08" B_PRIx32 " 0x%08" B_PRIx32 "\n", trb->status, 2432 trb->flags); 2433 2434 Unlock(); 2435 return status; 2436 } 2437 2438 2439 status_t 2440 XHCI::Noop() 2441 { 2442 TRACE("Issue No-Op\n"); 2443 xhci_trb trb; 2444 trb.address = 0; 2445 trb.status = 0; 2446 trb.flags = TRB_3_TYPE(TRB_TYPE_CMD_NOOP); 2447 2448 return DoCommand(&trb); 2449 } 2450 2451 2452 status_t 2453 XHCI::EnableSlot(uint8* slot) 2454 { 2455 TRACE("Enable Slot\n"); 2456 xhci_trb trb; 2457 trb.address = 0; 2458 trb.status = 0; 2459 trb.flags = TRB_3_TYPE(TRB_TYPE_ENABLE_SLOT); 2460 2461 status_t status = DoCommand(&trb); 2462 if (status != B_OK) 2463 return status; 2464 2465 *slot = TRB_3_SLOT_GET(trb.flags); 2466 return *slot != 0 ? B_OK : B_BAD_VALUE; 2467 } 2468 2469 2470 status_t 2471 XHCI::DisableSlot(uint8 slot) 2472 { 2473 TRACE("Disable Slot\n"); 2474 xhci_trb trb; 2475 trb.address = 0; 2476 trb.status = 0; 2477 trb.flags = TRB_3_TYPE(TRB_TYPE_DISABLE_SLOT) | TRB_3_SLOT(slot); 2478 2479 return DoCommand(&trb); 2480 } 2481 2482 2483 status_t 2484 XHCI::SetAddress(uint64 inputContext, bool bsr, uint8 slot) 2485 { 2486 TRACE("Set Address\n"); 2487 xhci_trb trb; 2488 trb.address = inputContext; 2489 trb.status = 0; 2490 trb.flags = TRB_3_TYPE(TRB_TYPE_ADDRESS_DEVICE) | TRB_3_SLOT(slot); 2491 2492 if (bsr) 2493 trb.flags |= TRB_3_BSR_BIT; 2494 2495 return DoCommand(&trb); 2496 } 2497 2498 2499 status_t 2500 XHCI::ConfigureEndpoint(uint64 inputContext, bool deconfigure, uint8 slot) 2501 { 2502 TRACE("Configure Endpoint\n"); 2503 xhci_trb trb; 2504 trb.address = inputContext; 2505 trb.status = 0; 2506 trb.flags = TRB_3_TYPE(TRB_TYPE_CONFIGURE_ENDPOINT) | TRB_3_SLOT(slot); 2507 2508 if (deconfigure) 2509 trb.flags |= TRB_3_DCEP_BIT; 2510 2511 return DoCommand(&trb); 2512 } 2513 2514 2515 status_t 2516 XHCI::EvaluateContext(uint64 inputContext, uint8 slot) 2517 { 2518 TRACE("Evaluate Context\n"); 2519 xhci_trb trb; 2520 trb.address = inputContext; 2521 trb.status = 0; 2522 trb.flags = TRB_3_TYPE(TRB_TYPE_EVALUATE_CONTEXT) | TRB_3_SLOT(slot); 2523 2524 return DoCommand(&trb); 2525 } 2526 2527 2528 status_t 2529 XHCI::ResetEndpoint(bool preserve, uint8 endpoint, uint8 slot) 2530 { 2531 TRACE("Reset Endpoint\n"); 2532 xhci_trb trb; 2533 trb.address = 0; 2534 trb.status = 0; 2535 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_ENDPOINT) 2536 | TRB_3_SLOT(slot) | TRB_3_ENDPOINT(endpoint); 2537 if (preserve) 2538 trb.flags |= TRB_3_PRSV_BIT; 2539 2540 return DoCommand(&trb); 2541 } 2542 2543 2544 status_t 2545 XHCI::StopEndpoint(bool suspend, uint8 endpoint, uint8 slot) 2546 { 2547 TRACE("Stop Endpoint\n"); 2548 xhci_trb trb; 2549 trb.address = 0; 2550 trb.status = 0; 2551 trb.flags = TRB_3_TYPE(TRB_TYPE_STOP_ENDPOINT) 2552 | TRB_3_SLOT(slot) | TRB_3_ENDPOINT(endpoint); 2553 if (suspend) 2554 trb.flags |= TRB_3_SUSPEND_ENDPOINT_BIT; 2555 2556 return DoCommand(&trb); 2557 } 2558 2559 2560 status_t 2561 XHCI::SetTRDequeue(uint64 dequeue, uint16 stream, uint8 endpoint, uint8 slot) 2562 { 2563 TRACE("Set TR Dequeue\n"); 2564 xhci_trb trb; 2565 trb.address = dequeue; 2566 trb.status = TRB_2_STREAM(stream); 2567 trb.flags = TRB_3_TYPE(TRB_TYPE_SET_TR_DEQUEUE) 2568 | TRB_3_SLOT(slot) | TRB_3_ENDPOINT(endpoint); 2569 2570 return DoCommand(&trb); 2571 } 2572 2573 2574 status_t 2575 XHCI::ResetDevice(uint8 slot) 2576 { 2577 TRACE("Reset Device\n"); 2578 xhci_trb trb; 2579 trb.address = 0; 2580 trb.status = 0; 2581 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_DEVICE) | TRB_3_SLOT(slot); 2582 2583 return DoCommand(&trb); 2584 } 2585 2586 2587 int32 2588 XHCI::EventThread(void* data) 2589 { 2590 ((XHCI *)data)->CompleteEvents(); 2591 return B_OK; 2592 } 2593 2594 2595 void 2596 XHCI::CompleteEvents() 2597 { 2598 while (!fStopThreads) { 2599 if (acquire_sem(fEventSem) < B_OK) 2600 continue; 2601 2602 // eat up sems that have been released by multiple interrupts 2603 int32 semCount = 0; 2604 get_sem_count(fEventSem, &semCount); 2605 if (semCount > 0) 2606 acquire_sem_etc(fEventSem, semCount, B_RELATIVE_TIMEOUT, 0); 2607 2608 ProcessEvents(); 2609 } 2610 } 2611 2612 2613 void 2614 XHCI::ProcessEvents() 2615 { 2616 // Use mutex_trylock first, in case we are in KDL. 2617 MutexLocker locker(fEventLock, mutex_trylock(&fEventLock) == B_OK); 2618 if (!locker.IsLocked()) { 2619 // We failed to get the lock. This really should not happen. 2620 TRACE_ERROR("failed to acquire event lock!\n"); 2621 return; 2622 } 2623 2624 uint16 i = fEventIdx; 2625 uint8 j = fEventCcs; 2626 uint8 t = 2; 2627 2628 while (1) { 2629 uint32 temp = B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags); 2630 uint8 event = TRB_3_TYPE_GET(temp); 2631 TRACE("event[%u] = %u (0x%016" B_PRIx64 " 0x%08" B_PRIx32 " 0x%08" 2632 B_PRIx32 ")\n", i, event, fEventRing[i].address, 2633 fEventRing[i].status, B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags)); 2634 uint8 k = (temp & TRB_3_CYCLE_BIT) ? 1 : 0; 2635 if (j != k) 2636 break; 2637 2638 switch (event) { 2639 case TRB_TYPE_COMMAND_COMPLETION: 2640 HandleCmdComplete(&fEventRing[i]); 2641 break; 2642 case TRB_TYPE_TRANSFER: 2643 HandleTransferComplete(&fEventRing[i]); 2644 break; 2645 case TRB_TYPE_PORT_STATUS_CHANGE: 2646 TRACE("port change detected\n"); 2647 break; 2648 default: 2649 TRACE_ERROR("Unhandled event = %u\n", event); 2650 break; 2651 } 2652 2653 i++; 2654 if (i == XHCI_MAX_EVENTS) { 2655 i = 0; 2656 j ^= 1; 2657 if (!--t) 2658 break; 2659 } 2660 } 2661 2662 fEventIdx = i; 2663 fEventCcs = j; 2664 2665 uint64 addr = fErst->rs_addr + i * sizeof(xhci_trb); 2666 addr |= ERST_EHB; 2667 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)addr); 2668 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(addr >> 32)); 2669 } 2670 2671 2672 int32 2673 XHCI::FinishThread(void* data) 2674 { 2675 ((XHCI *)data)->FinishTransfers(); 2676 return B_OK; 2677 } 2678 2679 2680 void 2681 XHCI::FinishTransfers() 2682 { 2683 while (!fStopThreads) { 2684 if (acquire_sem(fFinishTransfersSem) < B_OK) 2685 continue; 2686 2687 // eat up sems that have been released by multiple interrupts 2688 int32 semCount = 0; 2689 get_sem_count(fFinishTransfersSem, &semCount); 2690 if (semCount > 0) 2691 acquire_sem_etc(fFinishTransfersSem, semCount, B_RELATIVE_TIMEOUT, 0); 2692 2693 mutex_lock(&fFinishedLock); 2694 TRACE("finishing transfers\n"); 2695 while (fFinishedHead != NULL) { 2696 xhci_td* td = fFinishedHead; 2697 fFinishedHead = td->next; 2698 td->next = NULL; 2699 mutex_unlock(&fFinishedLock); 2700 2701 TRACE("finishing transfer td %p\n", td); 2702 2703 Transfer* transfer = td->transfer; 2704 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 2705 2706 status_t callbackStatus = B_OK; 2707 switch (td->trb_completion_code) { 2708 case COMP_SHORT_PACKET: 2709 case COMP_SUCCESS: 2710 callbackStatus = B_OK; 2711 break; 2712 case COMP_DATA_BUFFER: 2713 callbackStatus = directionIn ? B_DEV_DATA_OVERRUN 2714 : B_DEV_DATA_UNDERRUN; 2715 break; 2716 case COMP_BABBLE: 2717 callbackStatus = directionIn ? B_DEV_FIFO_OVERRUN 2718 : B_DEV_FIFO_UNDERRUN; 2719 break; 2720 case COMP_USB_TRANSACTION: 2721 callbackStatus = B_DEV_CRC_ERROR; 2722 break; 2723 case COMP_STALL: 2724 callbackStatus = B_DEV_STALLED; 2725 break; 2726 default: 2727 callbackStatus = B_DEV_STALLED; 2728 break; 2729 } 2730 2731 size_t actualLength = 0; 2732 if (callbackStatus == B_OK) { 2733 actualLength = transfer->DataLength(); 2734 2735 if (td->trb_completion_code == COMP_SHORT_PACKET) 2736 actualLength -= td->trb_left; 2737 2738 if (directionIn && actualLength > 0) { 2739 TRACE("copying in iov count %ld\n", transfer->VectorCount()); 2740 transfer->PrepareKernelAccess(); 2741 ReadDescriptor(td, transfer->Vector(), 2742 transfer->VectorCount()); 2743 } 2744 } 2745 transfer->Finished(callbackStatus, actualLength); 2746 delete transfer; 2747 FreeDescriptor(td); 2748 mutex_lock(&fFinishedLock); 2749 } 2750 mutex_unlock(&fFinishedLock); 2751 } 2752 } 2753 2754 2755 inline void 2756 XHCI::WriteOpReg(uint32 reg, uint32 value) 2757 { 2758 *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg) = value; 2759 } 2760 2761 2762 inline uint32 2763 XHCI::ReadOpReg(uint32 reg) 2764 { 2765 return *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg); 2766 } 2767 2768 2769 inline status_t 2770 XHCI::WaitOpBits(uint32 reg, uint32 mask, uint32 expected) 2771 { 2772 int loops = 0; 2773 uint32 value = ReadOpReg(reg); 2774 while ((value & mask) != expected) { 2775 snooze(1000); 2776 value = ReadOpReg(reg); 2777 if (loops == 100) { 2778 TRACE("delay waiting on reg 0x%" B_PRIX32 " match 0x%" B_PRIX32 2779 " (0x%" B_PRIX32 ")\n", reg, expected, mask); 2780 } else if (loops > 250) { 2781 TRACE_ERROR("timeout waiting on reg 0x%" B_PRIX32 2782 " match 0x%" B_PRIX32 " (0x%" B_PRIX32 ")\n", reg, expected, 2783 mask); 2784 return B_ERROR; 2785 } 2786 loops++; 2787 } 2788 return B_OK; 2789 } 2790 2791 2792 inline uint32 2793 XHCI::ReadCapReg32(uint32 reg) 2794 { 2795 return *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg); 2796 } 2797 2798 2799 inline void 2800 XHCI::WriteCapReg32(uint32 reg, uint32 value) 2801 { 2802 *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg) = value; 2803 } 2804 2805 2806 inline uint32 2807 XHCI::ReadRunReg32(uint32 reg) 2808 { 2809 return *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg); 2810 } 2811 2812 2813 inline void 2814 XHCI::WriteRunReg32(uint32 reg, uint32 value) 2815 { 2816 *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg) = value; 2817 } 2818 2819 2820 inline uint32 2821 XHCI::ReadDoorReg32(uint32 reg) 2822 { 2823 return *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg); 2824 } 2825 2826 2827 inline void 2828 XHCI::WriteDoorReg32(uint32 reg, uint32 value) 2829 { 2830 *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg) = value; 2831 } 2832 2833 2834 inline addr_t 2835 XHCI::_OffsetContextAddr(addr_t p) 2836 { 2837 if (fContextSizeShift == 1) { 2838 // each structure is page aligned, each pointer is 32 bits aligned 2839 uint32 offset = p & ((B_PAGE_SIZE - 1) & ~31U); 2840 p += offset; 2841 } 2842 return p; 2843 } 2844 2845 inline uint32 2846 XHCI::_ReadContext(uint32* p) 2847 { 2848 p = (uint32*)_OffsetContextAddr((addr_t)p); 2849 return *p; 2850 } 2851 2852 2853 inline void 2854 XHCI::_WriteContext(uint32* p, uint32 value) 2855 { 2856 p = (uint32*)_OffsetContextAddr((addr_t)p); 2857 *p = value; 2858 } 2859 2860 2861 inline uint64 2862 XHCI::_ReadContext(uint64* p) 2863 { 2864 p = (uint64*)_OffsetContextAddr((addr_t)p); 2865 return *p; 2866 } 2867 2868 2869 inline void 2870 XHCI::_WriteContext(uint64* p, uint64 value) 2871 { 2872 p = (uint64*)_OffsetContextAddr((addr_t)p); 2873 *p = value; 2874 } 2875