1 /* 2 * Copyright 2011-2021, Haiku, Inc. All rights reserved. 3 * Distributed under the terms of the MIT License. 4 * 5 * Authors: 6 * Augustin Cavalier <waddlesplash> 7 * Jian Chiang <j.jian.chiang@gmail.com> 8 * Jérôme Duval <jerome.duval@gmail.com> 9 * Akshay Jaggi <akshay1994.leo@gmail.com> 10 * Michael Lotz <mmlr@mlotz.ch> 11 * Alexander von Gluck <kallisti5@unixzen.com> 12 */ 13 14 15 #include <module.h> 16 #include <PCI.h> 17 #include <PCI_x86.h> 18 #include <USB3.h> 19 #include <KernelExport.h> 20 21 #include <ByteOrder.h> 22 #include <util/AutoLock.h> 23 24 #include "xhci.h" 25 26 #define USB_MODULE_NAME "xhci" 27 28 pci_module_info *XHCI::sPCIModule = NULL; 29 pci_x86_module_info *XHCI::sPCIx86Module = NULL; 30 31 32 static int32 33 xhci_std_ops(int32 op, ...) 34 { 35 switch (op) { 36 case B_MODULE_INIT: 37 TRACE_MODULE("xhci init module\n"); 38 return B_OK; 39 case B_MODULE_UNINIT: 40 TRACE_MODULE("xhci uninit module\n"); 41 return B_OK; 42 } 43 44 return EINVAL; 45 } 46 47 48 static const char* 49 xhci_error_string(uint32 error) 50 { 51 switch (error) { 52 case COMP_INVALID: return "Invalid"; 53 case COMP_SUCCESS: return "Success"; 54 case COMP_DATA_BUFFER: return "Data buffer"; 55 case COMP_BABBLE: return "Babble detected"; 56 case COMP_USB_TRANSACTION: return "USB transaction"; 57 case COMP_TRB: return "TRB"; 58 case COMP_STALL: return "Stall"; 59 case COMP_RESOURCE: return "Resource"; 60 case COMP_BANDWIDTH: return "Bandwidth"; 61 case COMP_NO_SLOTS: return "No slots"; 62 case COMP_INVALID_STREAM: return "Invalid stream"; 63 case COMP_SLOT_NOT_ENABLED: return "Slot not enabled"; 64 case COMP_ENDPOINT_NOT_ENABLED: return "Endpoint not enabled"; 65 case COMP_SHORT_PACKET: return "Short packet"; 66 case COMP_RING_UNDERRUN: return "Ring underrun"; 67 case COMP_RING_OVERRUN: return "Ring overrun"; 68 case COMP_VF_RING_FULL: return "VF Event Ring Full"; 69 case COMP_PARAMETER: return "Parameter"; 70 case COMP_BANDWIDTH_OVERRUN: return "Bandwidth overrun"; 71 case COMP_CONTEXT_STATE: return "Context state"; 72 case COMP_NO_PING_RESPONSE: return "No ping response"; 73 case COMP_EVENT_RING_FULL: return "Event ring full"; 74 case COMP_INCOMPATIBLE_DEVICE: return "Incompatible device"; 75 case COMP_MISSED_SERVICE: return "Missed service"; 76 case COMP_COMMAND_RING_STOPPED: return "Command ring stopped"; 77 case COMP_COMMAND_ABORTED: return "Command aborted"; 78 case COMP_STOPPED: return "Stopped"; 79 case COMP_LENGTH_INVALID: return "Length invalid"; 80 case COMP_MAX_EXIT_LATENCY: return "Max exit latency too large"; 81 case COMP_ISOC_OVERRUN: return "Isoch buffer overrun"; 82 case COMP_EVENT_LOST: return "Event lost"; 83 case COMP_UNDEFINED: return "Undefined"; 84 case COMP_INVALID_STREAM_ID: return "Invalid stream ID"; 85 case COMP_SECONDARY_BANDWIDTH: return "Secondary bandwidth"; 86 case COMP_SPLIT_TRANSACTION: return "Split transaction"; 87 88 default: return "Undefined"; 89 } 90 } 91 92 93 usb_host_controller_info xhci_module = { 94 { 95 "busses/usb/xhci", 96 0, 97 xhci_std_ops 98 }, 99 NULL, 100 XHCI::AddTo 101 }; 102 103 104 module_info *modules[] = { 105 (module_info *)&xhci_module, 106 NULL 107 }; 108 109 110 status_t 111 XHCI::AddTo(Stack *stack) 112 { 113 if (!sPCIModule) { 114 status_t status = get_module(B_PCI_MODULE_NAME, 115 (module_info **)&sPCIModule); 116 if (status < B_OK) { 117 TRACE_MODULE_ERROR("getting pci module failed! 0x%08" B_PRIx32 118 "\n", status); 119 return status; 120 } 121 } 122 123 TRACE_MODULE("searching devices\n"); 124 bool found = false; 125 pci_info *item = new(std::nothrow) pci_info; 126 if (item == NULL) { 127 sPCIModule = NULL; 128 put_module(B_PCI_MODULE_NAME); 129 return B_NO_MEMORY; 130 } 131 132 // Try to get the PCI x86 module as well so we can enable possible MSIs. 133 if (sPCIx86Module == NULL && get_module(B_PCI_X86_MODULE_NAME, 134 (module_info **)&sPCIx86Module) != B_OK) { 135 // If it isn't there, that's not critical though. 136 TRACE_MODULE_ERROR("failed to get pci x86 module\n"); 137 sPCIx86Module = NULL; 138 } 139 140 for (int32 i = 0; sPCIModule->get_nth_pci_info(i, item) >= B_OK; i++) { 141 if (item->class_base == PCI_serial_bus && item->class_sub == PCI_usb 142 && item->class_api == PCI_usb_xhci) { 143 TRACE_MODULE("found device at PCI:%d:%d:%d\n", 144 item->bus, item->device, item->function); 145 XHCI *bus = new(std::nothrow) XHCI(item, stack); 146 if (bus == NULL) { 147 delete item; 148 sPCIModule = NULL; 149 put_module(B_PCI_MODULE_NAME); 150 if (sPCIx86Module != NULL) 151 put_module(B_PCI_X86_MODULE_NAME); 152 return B_NO_MEMORY; 153 } 154 155 // The bus will put the PCI modules when it is destroyed, so get 156 // them again to increase their reference count. 157 get_module(B_PCI_MODULE_NAME, (module_info **)&sPCIModule); 158 if (sPCIx86Module != NULL) 159 get_module(B_PCI_X86_MODULE_NAME, (module_info **)&sPCIx86Module); 160 161 if (bus->InitCheck() < B_OK) { 162 TRACE_MODULE_ERROR("bus failed init check\n"); 163 delete bus; 164 continue; 165 } 166 167 // the bus took it away 168 item = new(std::nothrow) pci_info; 169 170 if (bus->Start() != B_OK) { 171 delete bus; 172 continue; 173 } 174 found = true; 175 } 176 } 177 178 // The modules will have been gotten again if we successfully 179 // initialized a bus, so we should put them here. 180 put_module(B_PCI_MODULE_NAME); 181 if (sPCIx86Module != NULL) 182 put_module(B_PCI_X86_MODULE_NAME); 183 184 if (!found) 185 TRACE_MODULE_ERROR("no devices found\n"); 186 delete item; 187 return found ? B_OK : ENODEV; 188 } 189 190 191 XHCI::XHCI(pci_info *info, Stack *stack) 192 : BusManager(stack), 193 fRegisterArea(-1), 194 fRegisters(NULL), 195 fPCIInfo(info), 196 fStack(stack), 197 fIRQ(0), 198 fUseMSI(false), 199 fErstArea(-1), 200 fDcbaArea(-1), 201 fCmdCompSem(-1), 202 fStopThreads(false), 203 fRootHub(NULL), 204 fRootHubAddress(0), 205 fPortCount(0), 206 fSlotCount(0), 207 fScratchpadCount(0), 208 fContextSizeShift(0), 209 fFinishedHead(NULL), 210 fFinishTransfersSem(-1), 211 fFinishThread(-1), 212 fEventSem(-1), 213 fEventThread(-1), 214 fEventIdx(0), 215 fCmdIdx(0), 216 fEventCcs(1), 217 fCmdCcs(1) 218 { 219 B_INITIALIZE_SPINLOCK(&fSpinlock); 220 mutex_init(&fFinishedLock, "XHCI finished transfers"); 221 mutex_init(&fEventLock, "XHCI event handler"); 222 223 if (BusManager::InitCheck() < B_OK) { 224 TRACE_ERROR("bus manager failed to init\n"); 225 return; 226 } 227 228 TRACE("constructing new XHCI host controller driver\n"); 229 fInitOK = false; 230 231 // enable busmaster and memory mapped access 232 uint16 command = sPCIModule->read_pci_config(fPCIInfo->bus, 233 fPCIInfo->device, fPCIInfo->function, PCI_command, 2); 234 command &= ~(PCI_command_io | PCI_command_int_disable); 235 command |= PCI_command_master | PCI_command_memory; 236 237 sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device, 238 fPCIInfo->function, PCI_command, 2, command); 239 240 // map the registers (low + high for 64-bit when requested) 241 phys_addr_t physicalAddress = fPCIInfo->u.h0.base_registers[0]; 242 if ((fPCIInfo->u.h0.base_register_flags[0] & PCI_address_type) 243 == PCI_address_type_64) { 244 physicalAddress |= (uint64)fPCIInfo->u.h0.base_registers[1] << 32; 245 } 246 247 size_t mapSize = fPCIInfo->u.h0.base_register_sizes[0]; 248 249 TRACE("map registers %08" B_PRIxPHYSADDR ", size: %" B_PRIuSIZE "\n", 250 physicalAddress, mapSize); 251 252 fRegisterArea = map_physical_memory("XHCI memory mapped registers", 253 physicalAddress, mapSize, B_ANY_KERNEL_BLOCK_ADDRESS, 254 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 255 (void **)&fRegisters); 256 if (fRegisterArea < B_OK) { 257 TRACE_ERROR("failed to map register memory\n"); 258 return; 259 } 260 261 // determine the register offsets 262 fCapabilityRegisterOffset = 0; 263 fOperationalRegisterOffset = HCI_CAPLENGTH(ReadCapReg32(XHCI_HCI_CAPLENGTH)); 264 fRuntimeRegisterOffset = ReadCapReg32(XHCI_RTSOFF) & ~0x1F; 265 fDoorbellRegisterOffset = ReadCapReg32(XHCI_DBOFF) & ~0x3; 266 267 TRACE("mapped registers: %p\n", fRegisters); 268 TRACE("operational register offset: %" B_PRId32 "\n", fOperationalRegisterOffset); 269 TRACE("runtime register offset: %" B_PRId32 "\n", fRuntimeRegisterOffset); 270 TRACE("doorbell register offset: %" B_PRId32 "\n", fDoorbellRegisterOffset); 271 272 int32 interfaceVersion = HCI_VERSION(ReadCapReg32(XHCI_HCI_VERSION)); 273 if (interfaceVersion < 0x0090 || interfaceVersion > 0x0120) { 274 TRACE_ERROR("unsupported interface version: 0x%04" B_PRIx32 "\n", 275 interfaceVersion); 276 return; 277 } 278 TRACE_ALWAYS("interface version: 0x%04" B_PRIx32 "\n", interfaceVersion); 279 280 TRACE_ALWAYS("structural parameters: 1:0x%08" B_PRIx32 " 2:0x%08" 281 B_PRIx32 " 3:0x%08" B_PRIx32 "\n", ReadCapReg32(XHCI_HCSPARAMS1), 282 ReadCapReg32(XHCI_HCSPARAMS2), ReadCapReg32(XHCI_HCSPARAMS3)); 283 284 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 285 if (cparams == 0xffffffff) 286 return; 287 TRACE_ALWAYS("capability parameters: 0x%08" B_PRIx32 "\n", cparams); 288 289 // if 64 bytes context structures, then 1 290 fContextSizeShift = HCC_CSZ(cparams); 291 292 // Assume ownership of the controller from the BIOS. 293 uint32 eec = 0xffffffff; 294 uint32 eecp = HCS0_XECP(cparams) << 2; 295 for (; eecp != 0 && XECP_NEXT(eec); eecp += XECP_NEXT(eec) << 2) { 296 TRACE("eecp register: 0x%08" B_PRIx32 "\n", eecp); 297 298 eec = ReadCapReg32(eecp); 299 if (XECP_ID(eec) != XHCI_LEGSUP_CAPID) 300 continue; 301 302 if (eec & XHCI_LEGSUP_BIOSOWNED) { 303 TRACE_ALWAYS("the host controller is bios owned, claiming" 304 " ownership\n"); 305 WriteCapReg32(eecp, eec | XHCI_LEGSUP_OSOWNED); 306 307 for (int32 i = 0; i < 20; i++) { 308 eec = ReadCapReg32(eecp); 309 310 if ((eec & XHCI_LEGSUP_BIOSOWNED) == 0) 311 break; 312 313 TRACE_ALWAYS("controller is still bios owned, waiting\n"); 314 snooze(50000); 315 } 316 317 if (eec & XHCI_LEGSUP_BIOSOWNED) { 318 TRACE_ERROR("bios won't give up control over the host " 319 "controller (ignoring)\n"); 320 } else if (eec & XHCI_LEGSUP_OSOWNED) { 321 TRACE_ALWAYS("successfully took ownership of the host " 322 "controller\n"); 323 } 324 325 // Force off the BIOS owned flag, and clear all SMIs. Some BIOSes 326 // do indicate a successful handover but do not remove their SMIs 327 // and then freeze the system when interrupts are generated. 328 WriteCapReg32(eecp, eec & ~XHCI_LEGSUP_BIOSOWNED); 329 } 330 break; 331 } 332 uint32 legctlsts = ReadCapReg32(eecp + XHCI_LEGCTLSTS); 333 legctlsts &= XHCI_LEGCTLSTS_DISABLE_SMI; 334 legctlsts |= XHCI_LEGCTLSTS_EVENTS_SMI; 335 WriteCapReg32(eecp + XHCI_LEGCTLSTS, legctlsts); 336 337 // We need to explicitly take ownership of EHCI ports on earlier Intel chipsets. 338 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL) { 339 switch (fPCIInfo->device_id) { 340 case PCI_DEVICE_INTEL_PANTHER_POINT_XHCI: 341 case PCI_DEVICE_INTEL_LYNX_POINT_XHCI: 342 case PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI: 343 case PCI_DEVICE_INTEL_BAYTRAIL_XHCI: 344 case PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI: 345 case PCI_DEVICE_INTEL_WILDCAT_POINT_LP_XHCI: 346 _SwitchIntelPorts(); 347 break; 348 } 349 } 350 351 // halt the host controller 352 if (ControllerHalt() < B_OK) { 353 return; 354 } 355 356 // reset the host controller 357 if (ControllerReset() < B_OK) { 358 TRACE_ERROR("host controller failed to reset\n"); 359 return; 360 } 361 362 fCmdCompSem = create_sem(0, "XHCI Command Complete"); 363 fFinishTransfersSem = create_sem(0, "XHCI Finish Transfers"); 364 fEventSem = create_sem(0, "XHCI Event"); 365 if (fFinishTransfersSem < B_OK || fCmdCompSem < B_OK || fEventSem < B_OK) { 366 TRACE_ERROR("failed to create semaphores\n"); 367 return; 368 } 369 370 // create event handler thread 371 fEventThread = spawn_kernel_thread(EventThread, "xhci event thread", 372 B_URGENT_PRIORITY, (void *)this); 373 resume_thread(fEventThread); 374 375 // create finisher service thread 376 fFinishThread = spawn_kernel_thread(FinishThread, "xhci finish thread", 377 B_URGENT_PRIORITY - 1, (void *)this); 378 resume_thread(fFinishThread); 379 380 // Find the right interrupt vector, using MSIs if available. 381 fIRQ = fPCIInfo->u.h0.interrupt_line; 382 if (sPCIx86Module != NULL && sPCIx86Module->get_msi_count(fPCIInfo->bus, 383 fPCIInfo->device, fPCIInfo->function) >= 1) { 384 uint8 msiVector = 0; 385 if (sPCIx86Module->configure_msi(fPCIInfo->bus, fPCIInfo->device, 386 fPCIInfo->function, 1, &msiVector) == B_OK 387 && sPCIx86Module->enable_msi(fPCIInfo->bus, fPCIInfo->device, 388 fPCIInfo->function) == B_OK) { 389 TRACE_ALWAYS("using message signaled interrupts\n"); 390 fIRQ = msiVector; 391 fUseMSI = true; 392 } 393 } 394 395 if (fIRQ == 0 || fIRQ == 0xFF) { 396 TRACE_MODULE_ERROR("device PCI:%d:%d:%d was assigned an invalid IRQ\n", 397 fPCIInfo->bus, fPCIInfo->device, fPCIInfo->function); 398 return; 399 } 400 401 // Install the interrupt handler 402 TRACE("installing interrupt handler\n"); 403 install_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this, 0); 404 405 memset(fPortSpeeds, 0, sizeof(fPortSpeeds)); 406 memset(fPortSlots, 0, sizeof(fPortSlots)); 407 memset(fDevices, 0, sizeof(fDevices)); 408 409 fInitOK = true; 410 TRACE("driver construction successful\n"); 411 } 412 413 414 XHCI::~XHCI() 415 { 416 TRACE("tear down XHCI host controller driver\n"); 417 418 WriteOpReg(XHCI_CMD, 0); 419 420 int32 result = 0; 421 fStopThreads = true; 422 delete_sem(fCmdCompSem); 423 delete_sem(fFinishTransfersSem); 424 delete_sem(fEventSem); 425 wait_for_thread(fFinishThread, &result); 426 wait_for_thread(fEventThread, &result); 427 428 mutex_destroy(&fFinishedLock); 429 mutex_destroy(&fEventLock); 430 431 remove_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this); 432 433 delete_area(fRegisterArea); 434 delete_area(fErstArea); 435 for (uint32 i = 0; i < fScratchpadCount; i++) 436 delete_area(fScratchpadArea[i]); 437 delete_area(fDcbaArea); 438 439 if (fUseMSI && sPCIx86Module != NULL) { 440 sPCIx86Module->disable_msi(fPCIInfo->bus, 441 fPCIInfo->device, fPCIInfo->function); 442 sPCIx86Module->unconfigure_msi(fPCIInfo->bus, 443 fPCIInfo->device, fPCIInfo->function); 444 } 445 put_module(B_PCI_MODULE_NAME); 446 if (sPCIx86Module != NULL) 447 put_module(B_PCI_X86_MODULE_NAME); 448 } 449 450 451 void 452 XHCI::_SwitchIntelPorts() 453 { 454 TRACE("Looking for EHCI owned ports\n"); 455 uint32 ports = sPCIModule->read_pci_config(fPCIInfo->bus, 456 fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_USB3PRM, 4); 457 TRACE("Superspeed Ports: 0x%" B_PRIx32 "\n", ports); 458 sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device, 459 fPCIInfo->function, XHCI_INTEL_USB3_PSSEN, 4, ports); 460 ports = sPCIModule->read_pci_config(fPCIInfo->bus, 461 fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_USB3_PSSEN, 4); 462 TRACE("Superspeed ports now under XHCI : 0x%" B_PRIx32 "\n", ports); 463 ports = sPCIModule->read_pci_config(fPCIInfo->bus, 464 fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_USB2PRM, 4); 465 TRACE("USB 2.0 Ports : 0x%" B_PRIx32 "\n", ports); 466 sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device, 467 fPCIInfo->function, XHCI_INTEL_XUSB2PR, 4, ports); 468 ports = sPCIModule->read_pci_config(fPCIInfo->bus, 469 fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_XUSB2PR, 4); 470 TRACE("USB 2.0 ports now under XHCI: 0x%" B_PRIx32 "\n", ports); 471 } 472 473 474 status_t 475 XHCI::Start() 476 { 477 TRACE_ALWAYS("starting XHCI host controller\n"); 478 TRACE("usbcmd: 0x%08" B_PRIx32 "; usbsts: 0x%08" B_PRIx32 "\n", 479 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 480 481 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 482 TRACE("Start() failed STS_CNR\n"); 483 } 484 485 if ((ReadOpReg(XHCI_CMD) & CMD_RUN) != 0) { 486 TRACE_ERROR("Start() warning, starting running XHCI controller!\n"); 487 } 488 489 if ((ReadOpReg(XHCI_PAGESIZE) & (1 << 0)) == 0) { 490 TRACE_ERROR("controller does not support 4K page size\n"); 491 return B_ERROR; 492 } 493 494 // read port count from capability register 495 uint32 capabilities = ReadCapReg32(XHCI_HCSPARAMS1); 496 fPortCount = HCS_MAX_PORTS(capabilities); 497 if (fPortCount == 0) { 498 TRACE_ERROR("invalid number of ports: %u\n", fPortCount); 499 return B_ERROR; 500 } 501 502 fSlotCount = HCS_MAX_SLOTS(capabilities); 503 if (fSlotCount > XHCI_MAX_DEVICES) 504 fSlotCount = XHCI_MAX_DEVICES; 505 WriteOpReg(XHCI_CONFIG, fSlotCount); 506 507 // find out which protocol is used for each port 508 uint8 portFound = 0; 509 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 510 uint32 eec = 0xffffffff; 511 uint32 eecp = HCS0_XECP(cparams) << 2; 512 for (; eecp != 0 && XECP_NEXT(eec) && portFound < fPortCount; 513 eecp += XECP_NEXT(eec) << 2) { 514 eec = ReadCapReg32(eecp); 515 if (XECP_ID(eec) != XHCI_SUPPORTED_PROTOCOLS_CAPID) 516 continue; 517 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) > 3) 518 continue; 519 uint32 temp = ReadCapReg32(eecp + 8); 520 uint32 offset = XHCI_SUPPORTED_PROTOCOLS_1_OFFSET(temp); 521 uint32 count = XHCI_SUPPORTED_PROTOCOLS_1_COUNT(temp); 522 if (offset == 0 || count == 0) 523 continue; 524 offset--; 525 for (uint32 i = offset; i < offset + count; i++) { 526 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) == 0x3) 527 fPortSpeeds[i] = USB_SPEED_SUPERSPEED; 528 else 529 fPortSpeeds[i] = USB_SPEED_HIGHSPEED; 530 531 TRACE("speed for port %" B_PRId32 " is %s\n", i, 532 fPortSpeeds[i] == USB_SPEED_SUPERSPEED ? "super" : "high"); 533 } 534 portFound += count; 535 } 536 537 uint32 params2 = ReadCapReg32(XHCI_HCSPARAMS2); 538 fScratchpadCount = HCS_MAX_SC_BUFFERS(params2); 539 if (fScratchpadCount > XHCI_MAX_SCRATCHPADS) { 540 TRACE_ERROR("invalid number of scratchpads: %" B_PRIu32 "\n", 541 fScratchpadCount); 542 return B_ERROR; 543 } 544 545 uint32 params3 = ReadCapReg32(XHCI_HCSPARAMS3); 546 fExitLatMax = HCS_U1_DEVICE_LATENCY(params3) 547 + HCS_U2_DEVICE_LATENCY(params3); 548 549 // clear interrupts & disable device notifications 550 WriteOpReg(XHCI_STS, ReadOpReg(XHCI_STS)); 551 WriteOpReg(XHCI_DNCTRL, 0); 552 553 // allocate Device Context Base Address array 554 phys_addr_t dmaAddress; 555 fDcbaArea = fStack->AllocateArea((void **)&fDcba, &dmaAddress, 556 sizeof(*fDcba), "DCBA Area"); 557 if (fDcbaArea < B_OK) { 558 TRACE_ERROR("unable to create the DCBA area\n"); 559 return B_ERROR; 560 } 561 memset(fDcba, 0, sizeof(*fDcba)); 562 memset(fScratchpadArea, 0, sizeof(fScratchpadArea)); 563 memset(fScratchpad, 0, sizeof(fScratchpad)); 564 565 // setting the first address to the scratchpad array address 566 fDcba->baseAddress[0] = dmaAddress 567 + offsetof(struct xhci_device_context_array, scratchpad); 568 569 // fill up the scratchpad array with scratchpad pages 570 for (uint32 i = 0; i < fScratchpadCount; i++) { 571 phys_addr_t scratchDmaAddress; 572 fScratchpadArea[i] = fStack->AllocateArea((void **)&fScratchpad[i], 573 &scratchDmaAddress, B_PAGE_SIZE, "Scratchpad Area"); 574 if (fScratchpadArea[i] < B_OK) { 575 TRACE_ERROR("unable to create the scratchpad area\n"); 576 return B_ERROR; 577 } 578 fDcba->scratchpad[i] = scratchDmaAddress; 579 } 580 581 TRACE("setting DCBAAP %" B_PRIxPHYSADDR "\n", dmaAddress); 582 WriteOpReg(XHCI_DCBAAP_LO, (uint32)dmaAddress); 583 WriteOpReg(XHCI_DCBAAP_HI, (uint32)(dmaAddress >> 32)); 584 585 // allocate Event Ring Segment Table 586 uint8 *addr; 587 fErstArea = fStack->AllocateArea((void **)&addr, &dmaAddress, 588 (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 589 + sizeof(xhci_erst_element), 590 "USB XHCI ERST CMD_RING and EVENT_RING Area"); 591 592 if (fErstArea < B_OK) { 593 TRACE_ERROR("unable to create the ERST AND RING area\n"); 594 delete_area(fDcbaArea); 595 return B_ERROR; 596 } 597 fErst = (xhci_erst_element *)addr; 598 memset(fErst, 0, (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 599 + sizeof(xhci_erst_element)); 600 601 // fill with Event Ring Segment Base Address and Event Ring Segment Size 602 fErst->rs_addr = dmaAddress + sizeof(xhci_erst_element); 603 fErst->rs_size = XHCI_MAX_EVENTS; 604 fErst->rsvdz = 0; 605 606 addr += sizeof(xhci_erst_element); 607 fEventRing = (xhci_trb *)addr; 608 addr += XHCI_MAX_EVENTS * sizeof(xhci_trb); 609 fCmdRing = (xhci_trb *)addr; 610 611 TRACE("setting ERST size\n"); 612 WriteRunReg32(XHCI_ERSTSZ(0), XHCI_ERSTS_SET(1)); 613 614 TRACE("setting ERDP addr = 0x%" B_PRIx64 "\n", fErst->rs_addr); 615 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)fErst->rs_addr); 616 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(fErst->rs_addr >> 32)); 617 618 TRACE("setting ERST base addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 619 WriteRunReg32(XHCI_ERSTBA_LO(0), (uint32)dmaAddress); 620 WriteRunReg32(XHCI_ERSTBA_HI(0), (uint32)(dmaAddress >> 32)); 621 622 dmaAddress += sizeof(xhci_erst_element) + XHCI_MAX_EVENTS 623 * sizeof(xhci_trb); 624 625 // Make sure the Command Ring is stopped 626 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 627 TRACE_ALWAYS("Command Ring is running, send stop/cancel\n"); 628 WriteOpReg(XHCI_CRCR_LO, CRCR_CS); 629 WriteOpReg(XHCI_CRCR_HI, 0); 630 WriteOpReg(XHCI_CRCR_LO, CRCR_CA); 631 WriteOpReg(XHCI_CRCR_HI, 0); 632 snooze(1000); 633 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 634 TRACE_ERROR("Command Ring still running after stop/cancel\n"); 635 } 636 } 637 TRACE("setting CRCR addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 638 WriteOpReg(XHCI_CRCR_LO, (uint32)dmaAddress | CRCR_RCS); 639 WriteOpReg(XHCI_CRCR_HI, (uint32)(dmaAddress >> 32)); 640 // link trb 641 fCmdRing[XHCI_MAX_COMMANDS - 1].address = dmaAddress; 642 643 TRACE("setting interrupt rate\n"); 644 645 // Setting IMOD below 0x3F8 on Intel Lynx Point can cause IRQ lockups 646 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL 647 && (fPCIInfo->device_id == PCI_DEVICE_INTEL_PANTHER_POINT_XHCI 648 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_XHCI 649 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI 650 || fPCIInfo->device_id == PCI_DEVICE_INTEL_BAYTRAIL_XHCI 651 || fPCIInfo->device_id == PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI)) { 652 WriteRunReg32(XHCI_IMOD(0), 0x000003f8); // 4000 irq/s 653 } else { 654 WriteRunReg32(XHCI_IMOD(0), 0x000001f4); // 8000 irq/s 655 } 656 657 TRACE("enabling interrupt\n"); 658 WriteRunReg32(XHCI_IMAN(0), ReadRunReg32(XHCI_IMAN(0)) | IMAN_INTR_ENA); 659 660 WriteOpReg(XHCI_CMD, CMD_RUN | CMD_INTE | CMD_HSEE); 661 662 // wait for start up state 663 if (WaitOpBits(XHCI_STS, STS_HCH, 0) != B_OK) { 664 TRACE_ERROR("HCH start up timeout\n"); 665 } 666 667 fRootHubAddress = AllocateAddress(); 668 fRootHub = new(std::nothrow) XHCIRootHub(RootObject(), fRootHubAddress); 669 if (!fRootHub) { 670 TRACE_ERROR("no memory to allocate root hub\n"); 671 return B_NO_MEMORY; 672 } 673 674 if (fRootHub->InitCheck() < B_OK) { 675 TRACE_ERROR("root hub failed init check\n"); 676 return fRootHub->InitCheck(); 677 } 678 679 SetRootHub(fRootHub); 680 681 TRACE_ALWAYS("successfully started the controller\n"); 682 683 #ifdef TRACE_USB 684 TRACE("No-Op test...\n"); 685 Noop(); 686 #endif 687 688 return BusManager::Start(); 689 } 690 691 692 status_t 693 XHCI::SubmitTransfer(Transfer *transfer) 694 { 695 // short circuit the root hub 696 if (transfer->TransferPipe()->DeviceAddress() == fRootHubAddress) 697 return fRootHub->ProcessTransfer(this, transfer); 698 699 TRACE("SubmitTransfer(%p)\n", transfer); 700 Pipe *pipe = transfer->TransferPipe(); 701 if ((pipe->Type() & USB_OBJECT_CONTROL_PIPE) != 0) 702 return SubmitControlRequest(transfer); 703 return SubmitNormalRequest(transfer); 704 } 705 706 707 status_t 708 XHCI::SubmitControlRequest(Transfer *transfer) 709 { 710 Pipe *pipe = transfer->TransferPipe(); 711 usb_request_data *requestData = transfer->RequestData(); 712 bool directionIn = (requestData->RequestType & USB_REQTYPE_DEVICE_IN) != 0; 713 714 TRACE("SubmitControlRequest() length %d\n", requestData->Length); 715 716 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 717 if (endpoint == NULL) { 718 TRACE_ERROR("control pipe has no endpoint!\n"); 719 return B_BAD_VALUE; 720 } 721 if (endpoint->device == NULL) { 722 panic("endpoint is not initialized!"); 723 return B_NO_INIT; 724 } 725 726 status_t status = transfer->InitKernelAccess(); 727 if (status != B_OK) 728 return status; 729 730 xhci_td *descriptor = CreateDescriptor(3, 1, requestData->Length); 731 if (descriptor == NULL) 732 return B_NO_MEMORY; 733 descriptor->transfer = transfer; 734 735 // Setup Stage 736 uint8 index = 0; 737 memcpy(&descriptor->trbs[index].address, requestData, 738 sizeof(usb_request_data)); 739 descriptor->trbs[index].status = TRB_2_IRQ(0) | TRB_2_BYTES(8); 740 descriptor->trbs[index].flags 741 = TRB_3_TYPE(TRB_TYPE_SETUP_STAGE) | TRB_3_IDT_BIT | TRB_3_CYCLE_BIT; 742 if (requestData->Length > 0) { 743 descriptor->trbs[index].flags |= 744 directionIn ? TRB_3_TRT_IN : TRB_3_TRT_OUT; 745 } 746 747 index++; 748 749 // Data Stage (if any) 750 if (requestData->Length > 0) { 751 descriptor->trbs[index].address = descriptor->buffer_addrs[0]; 752 descriptor->trbs[index].status = TRB_2_IRQ(0) 753 | TRB_2_BYTES(requestData->Length) 754 | TRB_2_TD_SIZE(0); 755 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_DATA_STAGE) 756 | (directionIn ? TRB_3_DIR_IN : 0) 757 | TRB_3_CYCLE_BIT; 758 759 if (!directionIn) { 760 transfer->PrepareKernelAccess(); 761 memcpy(descriptor->buffers[0], 762 (uint8 *)transfer->Vector()[0].iov_base, requestData->Length); 763 } 764 765 index++; 766 } 767 768 // Status Stage 769 descriptor->trbs[index].address = 0; 770 descriptor->trbs[index].status = TRB_2_IRQ(0); 771 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_STATUS_STAGE) 772 | ((directionIn && requestData->Length > 0) ? 0 : TRB_3_DIR_IN) 773 | TRB_3_CHAIN_BIT | TRB_3_ENT_BIT | TRB_3_CYCLE_BIT; 774 // Status Stage is an OUT transfer when the device is sending data 775 // (XHCI 1.2 § 4.11.2.2 Table 4-7 p213), and the CHAIN bit must be 776 // set when using an Event Data TRB (as _LinkDescriptorForPipe does) 777 // (XHCI 1.2 § 6.4.1.2.3 Table 6-31 p472) 778 779 descriptor->trb_used = index + 1; 780 781 status = _LinkDescriptorForPipe(descriptor, endpoint); 782 if (status != B_OK) { 783 FreeDescriptor(descriptor); 784 return status; 785 } 786 787 return B_OK; 788 } 789 790 791 status_t 792 XHCI::SubmitNormalRequest(Transfer *transfer) 793 { 794 TRACE("SubmitNormalRequest() length %ld\n", transfer->DataLength()); 795 796 Pipe *pipe = transfer->TransferPipe(); 797 usb_isochronous_data *isochronousData = transfer->IsochronousData(); 798 bool directionIn = (pipe->Direction() == Pipe::In); 799 800 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 801 if (endpoint == NULL) { 802 TRACE_ERROR("pipe has no endpoint!\n"); 803 return B_BAD_VALUE; 804 } 805 if (endpoint->device == NULL) { 806 panic("endpoint is not initialized!"); 807 return B_NO_INIT; 808 } 809 810 status_t status = transfer->InitKernelAccess(); 811 if (status != B_OK) 812 return status; 813 814 // TRBs within a TD must be "grouped" into TD Fragments, which mostly means 815 // that a max_burst_payload boundary cannot be crossed within a TRB, but 816 // only between TRBs. More than one TRB can be in a TD Fragment, but we keep 817 // things simple by setting trbSize to the MBP. (XHCI 1.2 § 4.11.7.1 p235.) 818 size_t trbSize = endpoint->max_burst_payload; 819 820 if (isochronousData != NULL) { 821 if (isochronousData->packet_count == 0) 822 return B_BAD_VALUE; 823 824 // Isochronous transfers use more specifically sized packets. 825 trbSize = transfer->DataLength() / isochronousData->packet_count; 826 if (trbSize == 0 || trbSize > pipe->MaxPacketSize() || trbSize 827 != (size_t)isochronousData->packet_descriptors[0].request_length) 828 return B_BAD_VALUE; 829 } 830 831 // Now that we know trbSize, compute the count. 832 const int32 trbCount = (transfer->DataLength() + trbSize - 1) / trbSize; 833 834 xhci_td *td = CreateDescriptor(trbCount, trbCount, trbSize); 835 if (td == NULL) 836 return B_NO_MEMORY; 837 838 // Normal Stage 839 const size_t maxPacketSize = pipe->MaxPacketSize(); 840 size_t remaining = transfer->DataLength(); 841 for (int32 i = 0; i < trbCount; i++) { 842 int32 trbLength = (remaining < trbSize) ? remaining : trbSize; 843 remaining -= trbLength; 844 845 // The "TD Size" field of a transfer TRB indicates the number of 846 // remaining maximum-size *packets* in this TD, *not* including the 847 // packets in the current TRB, and capped at 31 if there are more 848 // than 31 packets remaining in the TD. (XHCI 1.2 § 4.11.2.4 p218.) 849 int32 tdSize = (remaining + maxPacketSize - 1) / maxPacketSize; 850 if (tdSize > 31) 851 tdSize = 31; 852 853 td->trbs[i].address = td->buffer_addrs[i]; 854 td->trbs[i].status = TRB_2_IRQ(0) 855 | TRB_2_BYTES(trbLength) 856 | TRB_2_TD_SIZE(tdSize); 857 td->trbs[i].flags = TRB_3_TYPE(TRB_TYPE_NORMAL) 858 | TRB_3_CYCLE_BIT | TRB_3_CHAIN_BIT; 859 860 td->trb_used++; 861 } 862 863 // Isochronous-specific 864 if (isochronousData != NULL) { 865 // This is an isochronous transfer; we need to make the first TRB 866 // an isochronous TRB. 867 td->trbs[0].flags &= ~(TRB_3_TYPE(TRB_TYPE_NORMAL)); 868 td->trbs[0].flags |= TRB_3_TYPE(TRB_TYPE_ISOCH); 869 870 // Isochronous pipes are scheduled by microframes, one of which 871 // is 125us for USB 2 and above. But for USB 1 it was 1ms, so 872 // we need to use a different frame delta for that case. 873 uint8 frameDelta = 1; 874 if (transfer->TransferPipe()->Speed() == USB_SPEED_FULLSPEED) 875 frameDelta = 8; 876 877 // TODO: We do not currently take Mult into account at all! 878 // How are we supposed to do that here? 879 880 // Determine the (starting) frame number: if ISO_ASAP is set, 881 // we are queueing this "right away", and so want to reset 882 // the starting_frame_number. Otherwise we use the passed one. 883 uint32 frame; 884 if ((isochronousData->flags & USB_ISO_ASAP) != 0 885 || isochronousData->starting_frame_number == NULL) { 886 // All reads from the microframe index register must be 887 // incremented by 1. (XHCI 1.2 § 4.14.2.1.4 p265.) 888 frame = ReadRunReg32(XHCI_MFINDEX) + 1; 889 td->trbs[0].flags |= TRB_3_ISO_SIA_BIT; 890 } else { 891 frame = *isochronousData->starting_frame_number; 892 td->trbs[0].flags |= TRB_3_FRID(frame); 893 } 894 frame = (frame + frameDelta) % 2048; 895 if (isochronousData->starting_frame_number != NULL) 896 *isochronousData->starting_frame_number = frame; 897 898 // TODO: The OHCI bus driver seems to also do this for inbound 899 // isochronous transfers. Perhaps it should be moved into the stack? 900 if (directionIn) { 901 for (uint32 i = 0; i < isochronousData->packet_count; i++) { 902 isochronousData->packet_descriptors[i].actual_length = 0; 903 isochronousData->packet_descriptors[i].status = B_NO_INIT; 904 } 905 } 906 } 907 908 // Set the ENT (Evaluate Next TRB) bit, so that the HC will not switch 909 // contexts before evaluating the Link TRB that _LinkDescriptorForPipe 910 // will insert, as otherwise there would be a race between us freeing 911 // and unlinking the descriptor, and the controller evaluating the Link TRB 912 // and thus getting back onto the main ring and executing the Event Data 913 // TRB that generates the interrupt for this transfer. 914 // 915 // Note that we *do not* unset the CHAIN bit in this TRB, thus including 916 // the Link TRB in this TD formally, which is required when using the 917 // ENT bit. (XHCI 1.2 § 4.12.3 p250.) 918 td->trbs[td->trb_used - 1].flags |= TRB_3_ENT_BIT; 919 920 if (!directionIn) { 921 TRACE("copying out iov count %ld\n", transfer->VectorCount()); 922 status_t status = transfer->PrepareKernelAccess(); 923 if (status != B_OK) { 924 FreeDescriptor(td); 925 return status; 926 } 927 WriteDescriptor(td, transfer->Vector(), transfer->VectorCount()); 928 } 929 930 td->transfer = transfer; 931 status = _LinkDescriptorForPipe(td, endpoint); 932 if (status != B_OK) { 933 FreeDescriptor(td); 934 return status; 935 } 936 937 return B_OK; 938 } 939 940 941 status_t 942 XHCI::CancelQueuedTransfers(Pipe *pipe, bool force) 943 { 944 xhci_endpoint* endpoint = (xhci_endpoint*)pipe->ControllerCookie(); 945 if (endpoint == NULL || endpoint->trbs == NULL) { 946 // Someone's de-allocated this pipe or endpoint in the meantime. 947 // (Possibly AllocateDevice failed, and we were the temporary pipe.) 948 return B_NO_INIT; 949 } 950 951 TRACE_ALWAYS("cancel queued transfers (%" B_PRId8 ") for pipe %p (%d)\n", 952 endpoint->used, pipe, pipe->EndpointAddress()); 953 954 MutexLocker endpointLocker(endpoint->lock); 955 956 if (endpoint->td_head == NULL) { 957 // There aren't any currently pending transfers to cancel. 958 return B_OK; 959 } 960 961 // Calling the callbacks while holding the endpoint lock could potentially 962 // cause deadlocks, so we instead store them in a pointer array. We need 963 // to do this separately from freeing the TDs, for in the case we fail 964 // to stop the endpoint, we cancel the transfers but do not free the TDs. 965 Transfer* transfers[XHCI_MAX_TRANSFERS]; 966 int32 transfersCount = 0; 967 968 for (xhci_td* td = endpoint->td_head; td != NULL; td = td->next) { 969 if (td->transfer == NULL) 970 continue; 971 972 // We can't cancel or delete transfers under "force", as they probably 973 // are not safe to use anymore. 974 if (!force) { 975 transfers[transfersCount] = td->transfer; 976 transfersCount++; 977 } 978 td->transfer = NULL; 979 } 980 981 // It is possible that while waiting for the stop-endpoint command to 982 // complete, one of the queued transfers posts a completion event, so in 983 // order to avoid a deadlock, we must unlock the endpoint. 984 endpointLocker.Unlock(); 985 status_t status = StopEndpoint(false, endpoint); 986 if (status == B_DEV_STALLED) { 987 // Only exit from a Halted state is a reset. (XHCI 1.2 § 4.8.3 p163.) 988 TRACE_ERROR("cancel queued transfers: halted endpoint, reset!\n"); 989 status = ResetEndpoint(false, endpoint); 990 } 991 endpointLocker.Lock(); 992 993 // Detach the head TD from the endpoint. 994 xhci_td* td_head = endpoint->td_head; 995 endpoint->td_head = NULL; 996 997 if (status == B_OK) { 998 // Clear the endpoint's TRBs. 999 memset(endpoint->trbs, 0, sizeof(xhci_trb) * XHCI_ENDPOINT_RING_SIZE); 1000 endpoint->used = 0; 1001 endpoint->current = 0; 1002 1003 // Set dequeue pointer location to the beginning of the ring. 1004 SetTRDequeue(endpoint->trb_addr, 0, endpoint->id + 1, 1005 endpoint->device->slot); 1006 1007 // We don't need to do anything else to restart the ring, as it will resume 1008 // operation as normal upon the next doorbell. (XHCI 1.2 § 4.6.9 p136.) 1009 } else { 1010 // We couldn't stop the endpoint. Most likely the device has been 1011 // removed and the endpoint was stopped by the hardware, or is 1012 // for some reason busy and cannot be stopped. 1013 TRACE_ERROR("cancel queued transfers: could not stop endpoint: %s!\n", 1014 strerror(status)); 1015 1016 // Instead of freeing the TDs, we want to leave them in the endpoint 1017 // so that when/if the hardware returns, they can be properly unlinked, 1018 // as otherwise the endpoint could get "stuck" by having the "used" 1019 // slowly accumulate due to "dead" transfers. 1020 endpoint->td_head = td_head; 1021 td_head = NULL; 1022 } 1023 1024 endpointLocker.Unlock(); 1025 1026 for (int32 i = 0; i < transfersCount; i++) { 1027 transfers[i]->Finished(B_CANCELED, 0); 1028 delete transfers[i]; 1029 } 1030 1031 // This loop looks a bit strange because we need to store the "next" 1032 // pointer before freeing the descriptor. 1033 xhci_td* td; 1034 while ((td = td_head) != NULL) { 1035 td_head = td_head->next; 1036 FreeDescriptor(td); 1037 } 1038 1039 return B_OK; 1040 } 1041 1042 1043 status_t 1044 XHCI::StartDebugTransfer(Transfer *transfer) 1045 { 1046 Pipe *pipe = transfer->TransferPipe(); 1047 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 1048 if (endpoint == NULL) 1049 return B_BAD_VALUE; 1050 1051 // Check all locks that we are going to hit when running transfers. 1052 if (mutex_trylock(&endpoint->lock) != B_OK) 1053 return B_WOULD_BLOCK; 1054 if (mutex_trylock(&fFinishedLock) != B_OK) { 1055 mutex_unlock(&endpoint->lock); 1056 return B_WOULD_BLOCK; 1057 } 1058 if (mutex_trylock(&fEventLock) != B_OK) { 1059 mutex_unlock(&endpoint->lock); 1060 mutex_unlock(&fFinishedLock); 1061 return B_WOULD_BLOCK; 1062 } 1063 mutex_unlock(&endpoint->lock); 1064 mutex_unlock(&fFinishedLock); 1065 mutex_unlock(&fEventLock); 1066 1067 status_t status = SubmitTransfer(transfer); 1068 if (status != B_OK) 1069 return status; 1070 1071 // The endpoint's head TD is the TD of the just-submitted transfer. 1072 // Just like EHCI, abuse the callback cookie to hold the TD pointer. 1073 transfer->SetCallback(NULL, endpoint->td_head); 1074 1075 return B_OK; 1076 } 1077 1078 1079 status_t 1080 XHCI::CheckDebugTransfer(Transfer *transfer) 1081 { 1082 xhci_td *transfer_td = (xhci_td *)transfer->CallbackCookie(); 1083 if (transfer_td == NULL) 1084 return B_NO_INIT; 1085 1086 // Process events once, and then look for it in the finished list. 1087 ProcessEvents(); 1088 xhci_td *previous = NULL; 1089 for (xhci_td *td = fFinishedHead; td != NULL; td = td->next) { 1090 if (td != transfer_td) { 1091 previous = td; 1092 continue; 1093 } 1094 1095 // We've found it! 1096 if (previous == NULL) { 1097 fFinishedHead = fFinishedHead->next; 1098 } else { 1099 previous->next = td->next; 1100 } 1101 1102 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 1103 status_t status = (td->trb_completion_code == COMP_SUCCESS 1104 || td->trb_completion_code == COMP_SHORT_PACKET) ? B_OK : B_ERROR; 1105 1106 if (status == B_OK && directionIn) 1107 ReadDescriptor(td, transfer->Vector(), transfer->VectorCount()); 1108 1109 FreeDescriptor(td); 1110 transfer->SetCallback(NULL, NULL); 1111 return status; 1112 } 1113 1114 // We didn't find it. 1115 spin(75); 1116 return B_DEV_PENDING; 1117 } 1118 1119 1120 void 1121 XHCI::CancelDebugTransfer(Transfer *transfer) 1122 { 1123 while (CheckDebugTransfer(transfer) == B_DEV_PENDING) 1124 spin(100); 1125 } 1126 1127 1128 status_t 1129 XHCI::NotifyPipeChange(Pipe *pipe, usb_change change) 1130 { 1131 TRACE("pipe change %d for pipe %p (%d)\n", change, pipe, 1132 pipe->EndpointAddress()); 1133 1134 switch (change) { 1135 case USB_CHANGE_CREATED: 1136 return _InsertEndpointForPipe(pipe); 1137 case USB_CHANGE_DESTROYED: 1138 return _RemoveEndpointForPipe(pipe); 1139 1140 case USB_CHANGE_PIPE_POLICY_CHANGED: 1141 // We don't care about these, at least for now. 1142 return B_OK; 1143 } 1144 1145 TRACE_ERROR("unknown pipe change!\n"); 1146 return B_UNSUPPORTED; 1147 } 1148 1149 1150 xhci_td * 1151 XHCI::CreateDescriptor(uint32 trbCount, uint32 bufferCount, size_t bufferSize) 1152 { 1153 const bool inKDL = debug_debugger_running(); 1154 1155 xhci_td *result; 1156 if (!inKDL) { 1157 result = (xhci_td*)calloc(1, sizeof(xhci_td)); 1158 } else { 1159 // Just use the physical memory allocator while in KDL; it's less 1160 // secure than using the regular heap, but it's easier to deal with. 1161 phys_addr_t dummy; 1162 fStack->AllocateChunk((void **)&result, &dummy, sizeof(xhci_td)); 1163 } 1164 1165 if (result == NULL) { 1166 TRACE_ERROR("failed to allocate a transfer descriptor\n"); 1167 return NULL; 1168 } 1169 1170 // We always allocate 1 more TRB than requested, so that 1171 // _LinkDescriptorForPipe() has room to insert a link TRB. 1172 trbCount++; 1173 if (fStack->AllocateChunk((void **)&result->trbs, &result->trb_addr, 1174 (trbCount * sizeof(xhci_trb))) < B_OK) { 1175 TRACE_ERROR("failed to allocate TRBs\n"); 1176 FreeDescriptor(result); 1177 return NULL; 1178 } 1179 result->trb_count = trbCount; 1180 result->trb_used = 0; 1181 1182 if (bufferSize > 0) { 1183 // Due to how the USB stack allocates physical memory, we can't just 1184 // request one large chunk the size of the transfer, and so instead we 1185 // create a series of buffers as requested by our caller. 1186 1187 // We store the buffer pointers and addresses in one memory block. 1188 if (!inKDL) { 1189 result->buffers = (void**)calloc(bufferCount, 1190 (sizeof(void*) + sizeof(phys_addr_t))); 1191 } else { 1192 phys_addr_t dummy; 1193 fStack->AllocateChunk((void **)&result->buffers, &dummy, 1194 bufferCount * (sizeof(void*) + sizeof(phys_addr_t))); 1195 } 1196 if (result->buffers == NULL) { 1197 TRACE_ERROR("unable to allocate space for buffer infos\n"); 1198 FreeDescriptor(result); 1199 return NULL; 1200 } 1201 result->buffer_addrs = (phys_addr_t*)&result->buffers[bufferCount]; 1202 result->buffer_size = bufferSize; 1203 result->buffer_count = bufferCount; 1204 1205 // Optimization: If the requested total size of all buffers is less 1206 // than 32*B_PAGE_SIZE (the maximum size that the physical memory 1207 // allocator can handle), we allocate only one buffer and segment it. 1208 size_t totalSize = bufferSize * bufferCount; 1209 if (totalSize < (32 * B_PAGE_SIZE)) { 1210 if (fStack->AllocateChunk(&result->buffers[0], 1211 &result->buffer_addrs[0], totalSize) < B_OK) { 1212 TRACE_ERROR("unable to allocate space for large buffer (size %ld)\n", 1213 totalSize); 1214 FreeDescriptor(result); 1215 return NULL; 1216 } 1217 for (uint32 i = 1; i < bufferCount; i++) { 1218 result->buffers[i] = (void*)((addr_t)(result->buffers[i - 1]) 1219 + bufferSize); 1220 result->buffer_addrs[i] = result->buffer_addrs[i - 1] 1221 + bufferSize; 1222 } 1223 } else { 1224 // Otherwise, we allocate each buffer individually. 1225 for (uint32 i = 0; i < bufferCount; i++) { 1226 if (fStack->AllocateChunk(&result->buffers[i], 1227 &result->buffer_addrs[i], bufferSize) < B_OK) { 1228 TRACE_ERROR("unable to allocate space for a buffer (size " 1229 "%" B_PRIuSIZE ", count %" B_PRIu32 ")\n", 1230 bufferSize, bufferCount); 1231 FreeDescriptor(result); 1232 return NULL; 1233 } 1234 } 1235 } 1236 } else { 1237 result->buffers = NULL; 1238 result->buffer_addrs = NULL; 1239 } 1240 1241 // Initialize all other fields. 1242 result->transfer = NULL; 1243 result->trb_completion_code = 0; 1244 result->trb_left = 0; 1245 result->next = NULL; 1246 1247 TRACE("CreateDescriptor allocated %p, buffer_size %ld, buffer_count %" B_PRIu32 "\n", 1248 result, result->buffer_size, result->buffer_count); 1249 1250 return result; 1251 } 1252 1253 1254 void 1255 XHCI::FreeDescriptor(xhci_td *descriptor) 1256 { 1257 if (descriptor == NULL) 1258 return; 1259 1260 const bool inKDL = debug_debugger_running(); 1261 1262 if (descriptor->trbs != NULL) { 1263 fStack->FreeChunk(descriptor->trbs, descriptor->trb_addr, 1264 (descriptor->trb_count * sizeof(xhci_trb))); 1265 } 1266 if (descriptor->buffers != NULL) { 1267 size_t totalSize = descriptor->buffer_size * descriptor->buffer_count; 1268 if (totalSize < (32 * B_PAGE_SIZE)) { 1269 // This was allocated as one contiguous buffer. 1270 fStack->FreeChunk(descriptor->buffers[0], descriptor->buffer_addrs[0], 1271 totalSize); 1272 } else { 1273 for (uint32 i = 0; i < descriptor->buffer_count; i++) { 1274 if (descriptor->buffers[i] == NULL) 1275 continue; 1276 fStack->FreeChunk(descriptor->buffers[i], descriptor->buffer_addrs[i], 1277 descriptor->buffer_size); 1278 } 1279 } 1280 1281 if (!inKDL) { 1282 free(descriptor->buffers); 1283 } else { 1284 fStack->FreeChunk(descriptor->buffers, 0, 1285 descriptor->buffer_count * (sizeof(void*) + sizeof(phys_addr_t))); 1286 } 1287 } 1288 1289 if (!inKDL) 1290 free(descriptor); 1291 else 1292 fStack->FreeChunk(descriptor, 0, sizeof(xhci_td)); 1293 } 1294 1295 1296 size_t 1297 XHCI::WriteDescriptor(xhci_td *descriptor, iovec *vector, size_t vectorCount) 1298 { 1299 size_t written = 0; 1300 1301 size_t bufIdx = 0, bufUsed = 0; 1302 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1303 size_t length = vector[vecIdx].iov_len; 1304 1305 while (length > 0 && bufIdx < descriptor->buffer_count) { 1306 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1307 memcpy((uint8 *)descriptor->buffers[bufIdx] + bufUsed, 1308 (uint8 *)vector[vecIdx].iov_base + (vector[vecIdx].iov_len - length), 1309 toCopy); 1310 1311 written += toCopy; 1312 bufUsed += toCopy; 1313 length -= toCopy; 1314 if (bufUsed == descriptor->buffer_size) { 1315 bufIdx++; 1316 bufUsed = 0; 1317 } 1318 } 1319 } 1320 1321 TRACE("wrote descriptor (%" B_PRIuSIZE " bytes)\n", written); 1322 return written; 1323 } 1324 1325 1326 size_t 1327 XHCI::ReadDescriptor(xhci_td *descriptor, iovec *vector, size_t vectorCount) 1328 { 1329 size_t read = 0; 1330 1331 size_t bufIdx = 0, bufUsed = 0; 1332 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1333 size_t length = vector[vecIdx].iov_len; 1334 1335 while (length > 0 && bufIdx < descriptor->buffer_count) { 1336 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1337 memcpy((uint8 *)vector[vecIdx].iov_base + (vector[vecIdx].iov_len - length), 1338 (uint8 *)descriptor->buffers[bufIdx] + bufUsed, toCopy); 1339 1340 read += toCopy; 1341 bufUsed += toCopy; 1342 length -= toCopy; 1343 if (bufUsed == descriptor->buffer_size) { 1344 bufIdx++; 1345 bufUsed = 0; 1346 } 1347 } 1348 } 1349 1350 TRACE("read descriptor (%" B_PRIuSIZE " bytes)\n", read); 1351 return read; 1352 } 1353 1354 1355 Device * 1356 XHCI::AllocateDevice(Hub *parent, int8 hubAddress, uint8 hubPort, 1357 usb_speed speed) 1358 { 1359 TRACE("AllocateDevice hubAddress %d hubPort %d speed %d\n", hubAddress, 1360 hubPort, speed); 1361 1362 uint8 slot = XHCI_MAX_SLOTS; 1363 if (EnableSlot(&slot) != B_OK) { 1364 TRACE_ERROR("AllocateDevice() failed enable slot\n"); 1365 return NULL; 1366 } 1367 1368 if (slot == 0 || slot > fSlotCount) { 1369 TRACE_ERROR("AllocateDevice() bad slot\n"); 1370 return NULL; 1371 } 1372 1373 if (fDevices[slot].slot != 0) { 1374 TRACE_ERROR("AllocateDevice() slot already used\n"); 1375 return NULL; 1376 } 1377 1378 struct xhci_device *device = &fDevices[slot]; 1379 memset(device, 0, sizeof(struct xhci_device)); 1380 device->slot = slot; 1381 1382 device->input_ctx_area = fStack->AllocateArea((void **)&device->input_ctx, 1383 &device->input_ctx_addr, sizeof(*device->input_ctx) << fContextSizeShift, 1384 "XHCI input context"); 1385 if (device->input_ctx_area < B_OK) { 1386 TRACE_ERROR("unable to create a input context area\n"); 1387 return NULL; 1388 } 1389 1390 memset(device->input_ctx, 0, sizeof(*device->input_ctx) << fContextSizeShift); 1391 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1392 _WriteContext(&device->input_ctx->input.addFlags, 3); 1393 1394 uint32 route = 0; 1395 uint8 routePort = hubPort; 1396 uint8 rhPort = hubPort; 1397 for (Device *hubDevice = parent; hubDevice != RootObject(); 1398 hubDevice = (Device *)hubDevice->Parent()) { 1399 1400 rhPort = routePort; 1401 if (hubDevice->Parent() == RootObject()) 1402 break; 1403 route *= 16; 1404 if (hubPort > 15) 1405 route += 15; 1406 else 1407 route += routePort; 1408 1409 routePort = hubDevice->HubPort(); 1410 } 1411 1412 // Get speed of port, only if device connected to root hub port 1413 // else we have to rely on value reported by the Hub Explore thread 1414 if (route == 0) { 1415 GetPortSpeed(hubPort - 1, &speed); 1416 TRACE("speed updated %d\n", speed); 1417 } 1418 1419 uint32 dwslot0 = SLOT_0_NUM_ENTRIES(1) | SLOT_0_ROUTE(route); 1420 1421 // add the speed 1422 switch (speed) { 1423 case USB_SPEED_LOWSPEED: 1424 dwslot0 |= SLOT_0_SPEED(2); 1425 break; 1426 case USB_SPEED_HIGHSPEED: 1427 dwslot0 |= SLOT_0_SPEED(3); 1428 break; 1429 case USB_SPEED_FULLSPEED: 1430 dwslot0 |= SLOT_0_SPEED(1); 1431 break; 1432 case USB_SPEED_SUPERSPEED: 1433 dwslot0 |= SLOT_0_SPEED(4); 1434 break; 1435 default: 1436 TRACE_ERROR("unknown usb speed\n"); 1437 break; 1438 } 1439 1440 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1441 // TODO enable power save 1442 _WriteContext(&device->input_ctx->slot.dwslot1, SLOT_1_RH_PORT(rhPort)); 1443 uint32 dwslot2 = SLOT_2_IRQ_TARGET(0); 1444 1445 // If LS/FS device connected to non-root HS device 1446 if (route != 0 && parent->Speed() == USB_SPEED_HIGHSPEED 1447 && (speed == USB_SPEED_LOWSPEED || speed == USB_SPEED_FULLSPEED)) { 1448 struct xhci_device *parenthub = (struct xhci_device *) 1449 parent->ControllerCookie(); 1450 dwslot2 |= SLOT_2_PORT_NUM(hubPort); 1451 dwslot2 |= SLOT_2_TT_HUB_SLOT(parenthub->slot); 1452 } 1453 1454 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1455 1456 _WriteContext(&device->input_ctx->slot.dwslot3, SLOT_3_SLOT_STATE(0) 1457 | SLOT_3_DEVICE_ADDRESS(0)); 1458 1459 TRACE("slot 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 1460 "\n", _ReadContext(&device->input_ctx->slot.dwslot0), 1461 _ReadContext(&device->input_ctx->slot.dwslot1), 1462 _ReadContext(&device->input_ctx->slot.dwslot2), 1463 _ReadContext(&device->input_ctx->slot.dwslot3)); 1464 1465 device->device_ctx_area = fStack->AllocateArea((void **)&device->device_ctx, 1466 &device->device_ctx_addr, sizeof(*device->device_ctx) << fContextSizeShift, 1467 "XHCI device context"); 1468 if (device->device_ctx_area < B_OK) { 1469 TRACE_ERROR("unable to create a device context area\n"); 1470 delete_area(device->input_ctx_area); 1471 memset(device, 0, sizeof(xhci_device)); 1472 return NULL; 1473 } 1474 memset(device->device_ctx, 0, sizeof(*device->device_ctx) << fContextSizeShift); 1475 1476 device->trb_area = fStack->AllocateArea((void **)&device->trbs, 1477 &device->trb_addr, sizeof(xhci_trb) * (XHCI_MAX_ENDPOINTS - 1) 1478 * XHCI_ENDPOINT_RING_SIZE, "XHCI endpoint trbs"); 1479 if (device->trb_area < B_OK) { 1480 TRACE_ERROR("unable to create a device trbs area\n"); 1481 delete_area(device->input_ctx_area); 1482 delete_area(device->device_ctx_area); 1483 memset(device, 0, sizeof(xhci_device)); 1484 return NULL; 1485 } 1486 1487 // set up slot pointer to device context 1488 fDcba->baseAddress[slot] = device->device_ctx_addr; 1489 1490 size_t maxPacketSize; 1491 switch (speed) { 1492 case USB_SPEED_LOWSPEED: 1493 case USB_SPEED_FULLSPEED: 1494 maxPacketSize = 8; 1495 break; 1496 case USB_SPEED_HIGHSPEED: 1497 maxPacketSize = 64; 1498 break; 1499 default: 1500 maxPacketSize = 512; 1501 break; 1502 } 1503 1504 xhci_endpoint* endpoint0 = &device->endpoints[0]; 1505 mutex_init(&endpoint0->lock, "xhci endpoint lock"); 1506 endpoint0->device = device; 1507 endpoint0->id = 0; 1508 endpoint0->td_head = NULL; 1509 endpoint0->used = 0; 1510 endpoint0->current = 0; 1511 endpoint0->trbs = device->trbs; 1512 endpoint0->trb_addr = device->trb_addr; 1513 1514 // configure the Control endpoint 0 1515 if (ConfigureEndpoint(endpoint0, slot, 0, USB_OBJECT_CONTROL_PIPE, false, 1516 0, maxPacketSize, speed, 0, 0) != B_OK) { 1517 TRACE_ERROR("unable to configure default control endpoint\n"); 1518 delete_area(device->input_ctx_area); 1519 delete_area(device->device_ctx_area); 1520 delete_area(device->trb_area); 1521 memset(device, 0, sizeof(xhci_device)); 1522 return NULL; 1523 } 1524 1525 // device should get to addressed state (bsr = 0) 1526 if (SetAddress(device->input_ctx_addr, false, slot) != B_OK) { 1527 TRACE_ERROR("unable to set address\n"); 1528 delete_area(device->input_ctx_area); 1529 delete_area(device->device_ctx_area); 1530 delete_area(device->trb_area); 1531 memset(device, 0, sizeof(xhci_device)); 1532 return NULL; 1533 } 1534 1535 device->address = SLOT_3_DEVICE_ADDRESS_GET(_ReadContext( 1536 &device->device_ctx->slot.dwslot3)); 1537 1538 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", device->address, 1539 SLOT_3_SLOT_STATE_GET(_ReadContext( 1540 &device->device_ctx->slot.dwslot3))); 1541 TRACE("endpoint0 state 0x%08" B_PRIx32 "\n", 1542 ENDPOINT_0_STATE_GET(_ReadContext( 1543 &device->device_ctx->endpoints[0].dwendpoint0))); 1544 1545 // Create a temporary pipe with the new address 1546 ControlPipe pipe(parent); 1547 pipe.SetControllerCookie(endpoint0); 1548 pipe.InitCommon(device->address + 1, 0, speed, Pipe::Default, maxPacketSize, 0, 1549 hubAddress, hubPort); 1550 1551 // Get the device descriptor 1552 // Just retrieve the first 8 bytes of the descriptor -> minimum supported 1553 // size of any device. It is enough because it includes the device type. 1554 1555 size_t actualLength = 0; 1556 usb_device_descriptor deviceDescriptor; 1557 1558 TRACE("getting the device descriptor\n"); 1559 status_t status = pipe.SendRequest( 1560 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_STANDARD, // type 1561 USB_REQUEST_GET_DESCRIPTOR, // request 1562 USB_DESCRIPTOR_DEVICE << 8, // value 1563 0, // index 1564 8, // length 1565 (void *)&deviceDescriptor, // buffer 1566 8, // buffer length 1567 &actualLength); // actual length 1568 1569 if (actualLength != 8) { 1570 TRACE_ERROR("failed to get the device descriptor: %s\n", 1571 strerror(status)); 1572 delete_area(device->input_ctx_area); 1573 delete_area(device->device_ctx_area); 1574 delete_area(device->trb_area); 1575 memset(device, 0, sizeof(xhci_device)); 1576 return NULL; 1577 } 1578 1579 TRACE("device_class: %d device_subclass %d device_protocol %d\n", 1580 deviceDescriptor.device_class, deviceDescriptor.device_subclass, 1581 deviceDescriptor.device_protocol); 1582 1583 if (speed == USB_SPEED_FULLSPEED && deviceDescriptor.max_packet_size_0 != 8) { 1584 TRACE("Full speed device with different max packet size for Endpoint 0\n"); 1585 uint32 dwendpoint1 = _ReadContext( 1586 &device->input_ctx->endpoints[0].dwendpoint1); 1587 dwendpoint1 &= ~ENDPOINT_1_MAXPACKETSIZE(0xffff); 1588 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE( 1589 deviceDescriptor.max_packet_size_0); 1590 _WriteContext(&device->input_ctx->endpoints[0].dwendpoint1, 1591 dwendpoint1); 1592 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1593 _WriteContext(&device->input_ctx->input.addFlags, (1 << 1)); 1594 EvaluateContext(device->input_ctx_addr, device->slot); 1595 } 1596 1597 Device *deviceObject = NULL; 1598 if (deviceDescriptor.device_class == 0x09) { 1599 TRACE("creating new Hub\n"); 1600 TRACE("getting the hub descriptor\n"); 1601 size_t actualLength = 0; 1602 usb_hub_descriptor hubDescriptor; 1603 status = pipe.SendRequest( 1604 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_CLASS, // type 1605 USB_REQUEST_GET_DESCRIPTOR, // request 1606 USB_DESCRIPTOR_HUB << 8, // value 1607 0, // index 1608 sizeof(usb_hub_descriptor), // length 1609 (void *)&hubDescriptor, // buffer 1610 sizeof(usb_hub_descriptor), // buffer length 1611 &actualLength); 1612 1613 if (actualLength != sizeof(usb_hub_descriptor)) { 1614 TRACE_ERROR("error while getting the hub descriptor: %s\n", 1615 strerror(status)); 1616 delete_area(device->input_ctx_area); 1617 delete_area(device->device_ctx_area); 1618 delete_area(device->trb_area); 1619 memset(device, 0, sizeof(xhci_device)); 1620 return NULL; 1621 } 1622 1623 uint32 dwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1624 dwslot0 |= SLOT_0_HUB_BIT; 1625 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1626 uint32 dwslot1 = _ReadContext(&device->input_ctx->slot.dwslot1); 1627 dwslot1 |= SLOT_1_NUM_PORTS(hubDescriptor.num_ports); 1628 _WriteContext(&device->input_ctx->slot.dwslot1, dwslot1); 1629 if (speed == USB_SPEED_HIGHSPEED) { 1630 uint32 dwslot2 = _ReadContext(&device->input_ctx->slot.dwslot2); 1631 dwslot2 |= SLOT_2_TT_TIME(HUB_TTT_GET(hubDescriptor.characteristics)); 1632 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1633 } 1634 1635 deviceObject = new(std::nothrow) Hub(parent, hubAddress, hubPort, 1636 deviceDescriptor, device->address + 1, speed, false, device); 1637 } else { 1638 TRACE("creating new device\n"); 1639 deviceObject = new(std::nothrow) Device(parent, hubAddress, hubPort, 1640 deviceDescriptor, device->address + 1, speed, false, device); 1641 } 1642 if (deviceObject == NULL || deviceObject->InitCheck() != B_OK) { 1643 if (deviceObject == NULL) { 1644 TRACE_ERROR("no memory to allocate device\n"); 1645 } else { 1646 TRACE_ERROR("device object failed to initialize\n"); 1647 } 1648 delete_area(device->input_ctx_area); 1649 delete_area(device->device_ctx_area); 1650 delete_area(device->trb_area); 1651 memset(device, 0, sizeof(xhci_device)); 1652 return NULL; 1653 } 1654 1655 // We don't want to disable the default endpoint, naturally, which would 1656 // otherwise happen when this Pipe object is destroyed. 1657 pipe.SetControllerCookie(NULL); 1658 1659 fPortSlots[hubPort] = slot; 1660 TRACE("AllocateDevice() port %d slot %d\n", hubPort, slot); 1661 return deviceObject; 1662 } 1663 1664 1665 void 1666 XHCI::FreeDevice(Device *device) 1667 { 1668 uint8 hubPort = device->HubPort(); 1669 uint8 slot = fPortSlots[hubPort]; 1670 TRACE("FreeDevice() port %d slot %d\n", hubPort, slot); 1671 1672 // Delete the device first, so it cleans up its pipes and tells us 1673 // what we need to destroy before we tear down our internal state. 1674 delete device; 1675 1676 DisableSlot(slot); 1677 fDcba->baseAddress[slot] = 0; 1678 fPortSlots[hubPort] = 0; 1679 delete_area(fDevices[slot].trb_area); 1680 delete_area(fDevices[slot].input_ctx_area); 1681 delete_area(fDevices[slot].device_ctx_area); 1682 1683 memset(&fDevices[slot], 0, sizeof(xhci_device)); 1684 } 1685 1686 1687 uint8 1688 XHCI::_GetEndpointState(xhci_endpoint* endpoint) 1689 { 1690 struct xhci_device_ctx* device_ctx = endpoint->device->device_ctx; 1691 return ENDPOINT_0_STATE_GET( 1692 _ReadContext(&device_ctx->endpoints[endpoint->id].dwendpoint0)); 1693 } 1694 1695 1696 status_t 1697 XHCI::_InsertEndpointForPipe(Pipe *pipe) 1698 { 1699 TRACE("insert endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1700 1701 if (pipe->ControllerCookie() != NULL 1702 || pipe->Parent()->Type() != USB_OBJECT_DEVICE) { 1703 // default pipe is already referenced 1704 return B_OK; 1705 } 1706 1707 Device* usbDevice = (Device *)pipe->Parent(); 1708 struct xhci_device *device = (struct xhci_device *) 1709 usbDevice->ControllerCookie(); 1710 if (usbDevice->Parent() == RootObject()) 1711 return B_OK; 1712 if (device == NULL) { 1713 panic("device is NULL\n"); 1714 return B_NO_INIT; 1715 } 1716 1717 const uint8 id = (2 * pipe->EndpointAddress() 1718 + (pipe->Direction() != Pipe::Out ? 1 : 0)) - 1; 1719 if (id >= XHCI_MAX_ENDPOINTS - 1) 1720 return B_BAD_VALUE; 1721 1722 if (id > 0) { 1723 uint32 devicedwslot0 = _ReadContext(&device->device_ctx->slot.dwslot0); 1724 if (SLOT_0_NUM_ENTRIES_GET(devicedwslot0) == 1) { 1725 uint32 inputdwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1726 inputdwslot0 &= ~(SLOT_0_NUM_ENTRIES(0x1f)); 1727 inputdwslot0 |= SLOT_0_NUM_ENTRIES(XHCI_MAX_ENDPOINTS - 1); 1728 _WriteContext(&device->input_ctx->slot.dwslot0, inputdwslot0); 1729 EvaluateContext(device->input_ctx_addr, device->slot); 1730 } 1731 1732 xhci_endpoint* endpoint = &device->endpoints[id]; 1733 mutex_init(&endpoint->lock, "xhci endpoint lock"); 1734 MutexLocker endpointLocker(endpoint->lock); 1735 1736 endpoint->device = device; 1737 endpoint->id = id; 1738 endpoint->td_head = NULL; 1739 endpoint->used = 0; 1740 endpoint->current = 0; 1741 1742 endpoint->trbs = device->trbs + id * XHCI_ENDPOINT_RING_SIZE; 1743 endpoint->trb_addr = device->trb_addr 1744 + id * XHCI_ENDPOINT_RING_SIZE * sizeof(xhci_trb); 1745 memset(endpoint->trbs, 0, 1746 sizeof(xhci_trb) * XHCI_ENDPOINT_RING_SIZE); 1747 1748 TRACE("insert endpoint for pipe: trbs, device %p endpoint %p\n", 1749 device->trbs, endpoint->trbs); 1750 TRACE("insert endpoint for pipe: trb_addr, device 0x%" B_PRIxPHYSADDR 1751 " endpoint 0x%" B_PRIxPHYSADDR "\n", device->trb_addr, 1752 endpoint->trb_addr); 1753 1754 const uint8 endpointNum = id + 1; 1755 1756 status_t status = ConfigureEndpoint(endpoint, device->slot, id, pipe->Type(), 1757 pipe->Direction() == Pipe::In, pipe->Interval(), pipe->MaxPacketSize(), 1758 usbDevice->Speed(), pipe->MaxBurst(), pipe->BytesPerInterval()); 1759 if (status != B_OK) { 1760 TRACE_ERROR("unable to configure endpoint: %s\n", strerror(status)); 1761 return status; 1762 } 1763 1764 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1765 _WriteContext(&device->input_ctx->input.addFlags, 1766 (1 << endpointNum) | (1 << 0)); 1767 1768 ConfigureEndpoint(device->input_ctx_addr, false, device->slot); 1769 1770 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", 1771 device->address, SLOT_3_SLOT_STATE_GET(_ReadContext( 1772 &device->device_ctx->slot.dwslot3))); 1773 TRACE("endpoint[0] state 0x%08" B_PRIx32 "\n", 1774 ENDPOINT_0_STATE_GET(_ReadContext( 1775 &device->device_ctx->endpoints[0].dwendpoint0))); 1776 TRACE("endpoint[%d] state 0x%08" B_PRIx32 "\n", id, 1777 ENDPOINT_0_STATE_GET(_ReadContext( 1778 &device->device_ctx->endpoints[id].dwendpoint0))); 1779 } 1780 pipe->SetControllerCookie(&device->endpoints[id]); 1781 1782 return B_OK; 1783 } 1784 1785 1786 status_t 1787 XHCI::_RemoveEndpointForPipe(Pipe *pipe) 1788 { 1789 TRACE("remove endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1790 1791 if (pipe->Parent()->Type() != USB_OBJECT_DEVICE) 1792 return B_OK; 1793 Device* usbDevice = (Device *)pipe->Parent(); 1794 if (usbDevice->Parent() == RootObject()) 1795 return B_BAD_VALUE; 1796 1797 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 1798 if (endpoint == NULL || endpoint->trbs == NULL) 1799 return B_NO_INIT; 1800 1801 pipe->SetControllerCookie(NULL); 1802 1803 if (endpoint->id > 0) { 1804 xhci_device *device = endpoint->device; 1805 uint8 epNumber = endpoint->id + 1; 1806 StopEndpoint(true, endpoint); 1807 1808 mutex_lock(&endpoint->lock); 1809 1810 // See comment in CancelQueuedTransfers. 1811 xhci_td* td; 1812 while ((td = endpoint->td_head) != NULL) { 1813 endpoint->td_head = endpoint->td_head->next; 1814 FreeDescriptor(td); 1815 } 1816 1817 mutex_destroy(&endpoint->lock); 1818 memset(endpoint, 0, sizeof(xhci_endpoint)); 1819 1820 _WriteContext(&device->input_ctx->input.dropFlags, (1 << epNumber)); 1821 _WriteContext(&device->input_ctx->input.addFlags, (1 << 0)); 1822 1823 // The Deconfigure bit in the Configure Endpoint command indicates 1824 // that *all* endpoints are to be deconfigured, and not just the ones 1825 // specified in the context flags. (XHCI 1.2 § 4.6.6 p115.) 1826 ConfigureEndpoint(device->input_ctx_addr, false, device->slot); 1827 } 1828 1829 return B_OK; 1830 } 1831 1832 1833 status_t 1834 XHCI::_LinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 1835 { 1836 TRACE("link descriptor for pipe\n"); 1837 1838 // Use mutex_trylock first, in case we are in KDL. 1839 if (mutex_trylock(&endpoint->lock) != B_OK) 1840 mutex_lock(&endpoint->lock); 1841 1842 // "used" refers to the number of currently linked TDs, not the number of 1843 // used TRBs on the ring (we use 2 TRBs on the ring per transfer.) 1844 if (endpoint->used >= (XHCI_MAX_TRANSFERS - 1)) { 1845 TRACE_ERROR("link descriptor for pipe: max transfers count exceeded\n"); 1846 mutex_unlock(&endpoint->lock); 1847 return B_BAD_VALUE; 1848 } 1849 1850 endpoint->used++; 1851 descriptor->next = endpoint->td_head; 1852 endpoint->td_head = descriptor; 1853 1854 const uint8 current = endpoint->current, 1855 eventdata = current + 1; 1856 uint8 next = eventdata + 1; 1857 1858 TRACE("link descriptor for pipe: current %d, next %d\n", current, next); 1859 1860 // Add a Link TRB to the end of the descriptor. 1861 phys_addr_t addr = endpoint->trb_addr + eventdata * sizeof(xhci_trb); 1862 descriptor->trbs[descriptor->trb_used].address = addr; 1863 descriptor->trbs[descriptor->trb_used].status = TRB_2_IRQ(0); 1864 descriptor->trbs[descriptor->trb_used].flags = TRB_3_TYPE(TRB_TYPE_LINK) 1865 | TRB_3_CHAIN_BIT | TRB_3_CYCLE_BIT; 1866 // It is specified that (XHCI 1.2 § 4.12.3 Note 2 p251) if the TRB 1867 // following one with the ENT bit set is a Link TRB, the Link TRB 1868 // shall be evaluated *and* the subsequent TRB shall be. Thus a 1869 // TRB_3_ENT_BIT is unnecessary here; and from testing seems to 1870 // break all transfers on a (very) small number of controllers. 1871 1872 #if !B_HOST_IS_LENDIAN 1873 // Convert endianness. 1874 for (uint32 i = 0; i <= descriptor->trb_used; i++) { 1875 descriptor->trbs[i].address = 1876 B_HOST_TO_LENDIAN_INT64(descriptor->trbs[i].address); 1877 descriptor->trbs[i].status = 1878 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].status); 1879 descriptor->trbs[i].flags = 1880 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].flags); 1881 } 1882 #endif 1883 1884 // Link the descriptor. 1885 endpoint->trbs[current].address = 1886 B_HOST_TO_LENDIAN_INT64(descriptor->trb_addr); 1887 endpoint->trbs[current].status = 1888 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 1889 endpoint->trbs[current].flags = 1890 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_LINK)); 1891 1892 // Set up the Event Data TRB (XHCI 1.2 § 4.11.5.2 p230.) 1893 // 1894 // We do this on the main ring for two reasons: first, to avoid a small 1895 // potential race between the interrupt and the controller evaluating 1896 // the link TRB to get back onto the ring; and second, because many 1897 // controllers throw errors if the target of a Link TRB is not valid 1898 // (i.e. does not have its Cycle Bit set.) 1899 // 1900 // We also set the "address" field, which the controller will copy 1901 // verbatim into the TRB it posts to the event ring, to be the last 1902 // "real" TRB in the TD; this will allow us to determine what transfer 1903 // the resulting Transfer Event TRB refers to. 1904 endpoint->trbs[eventdata].address = 1905 B_HOST_TO_LENDIAN_INT64(descriptor->trb_addr 1906 + (descriptor->trb_used - 1) * sizeof(xhci_trb)); 1907 endpoint->trbs[eventdata].status = 1908 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 1909 endpoint->trbs[eventdata].flags = 1910 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_EVENT_DATA) 1911 | TRB_3_IOC_BIT | TRB_3_CYCLE_BIT); 1912 1913 if (next == (XHCI_ENDPOINT_RING_SIZE - 1)) { 1914 // We always use 2 TRBs per _Link..() call, so if "next" is the last 1915 // TRB in the ring, we need to generate a link TRB at "next", and 1916 // then wrap it to 0. 1917 endpoint->trbs[next].address = 1918 B_HOST_TO_LENDIAN_INT64(endpoint->trb_addr); 1919 endpoint->trbs[next].status = 1920 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 1921 endpoint->trbs[next].flags = 1922 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_LINK) | TRB_3_CYCLE_BIT); 1923 1924 next = 0; 1925 } 1926 1927 endpoint->trbs[next].address = 0; 1928 endpoint->trbs[next].status = 0; 1929 endpoint->trbs[next].flags = 0; 1930 1931 // Everything is ready, so write the cycle bit. 1932 endpoint->trbs[current].flags |= B_HOST_TO_LENDIAN_INT32(TRB_3_CYCLE_BIT); 1933 1934 TRACE("_LinkDescriptorForPipe pCurrent %p phys 0x%" B_PRIxPHYSADDR 1935 " 0x%" B_PRIxPHYSADDR " 0x%08" B_PRIx32 "\n", &endpoint->trbs[current], 1936 endpoint->trb_addr + current * sizeof(struct xhci_trb), 1937 endpoint->trbs[current].address, 1938 B_LENDIAN_TO_HOST_INT32(endpoint->trbs[current].flags)); 1939 1940 endpoint->current = next; 1941 mutex_unlock(&endpoint->lock); 1942 1943 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 1944 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 1945 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 1946 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 1947 1948 Ring(endpoint->device->slot, endpoint->id + 1); 1949 1950 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 1951 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 1952 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 1953 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 1954 1955 return B_OK; 1956 } 1957 1958 1959 status_t 1960 XHCI::_UnlinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 1961 { 1962 TRACE("unlink descriptor for pipe\n"); 1963 // We presume that the caller has already locked or owns the endpoint. 1964 1965 endpoint->used--; 1966 if (descriptor == endpoint->td_head) { 1967 endpoint->td_head = descriptor->next; 1968 descriptor->next = NULL; 1969 return B_OK; 1970 } else { 1971 for (xhci_td *td = endpoint->td_head; td->next != NULL; td = td->next) { 1972 if (td->next == descriptor) { 1973 td->next = descriptor->next; 1974 descriptor->next = NULL; 1975 return B_OK; 1976 } 1977 } 1978 } 1979 1980 endpoint->used++; 1981 return B_ERROR; 1982 } 1983 1984 1985 status_t 1986 XHCI::ConfigureEndpoint(xhci_endpoint* ep, uint8 slot, uint8 number, uint8 type, 1987 bool directionIn, uint16 interval, uint16 maxPacketSize, usb_speed speed, 1988 uint8 maxBurst, uint16 bytesPerInterval) 1989 { 1990 struct xhci_device* device = &fDevices[slot]; 1991 1992 uint32 dwendpoint0 = 0; 1993 uint32 dwendpoint1 = 0; 1994 uint64 qwendpoint2 = 0; 1995 uint32 dwendpoint4 = 0; 1996 1997 // Compute and assign the endpoint type. (XHCI 1.2 § 6.2.3 Table 6-9 p452.) 1998 uint8 xhciType = 4; 1999 if ((type & USB_OBJECT_INTERRUPT_PIPE) != 0) 2000 xhciType = 3; 2001 if ((type & USB_OBJECT_BULK_PIPE) != 0) 2002 xhciType = 2; 2003 if ((type & USB_OBJECT_ISO_PIPE) != 0) 2004 xhciType = 1; 2005 xhciType |= directionIn ? (1 << 2) : 0; 2006 dwendpoint1 |= ENDPOINT_1_EPTYPE(xhciType); 2007 2008 // Compute and assign interval. (XHCI 1.2 § 6.2.3.6 p456.) 2009 uint16 calcInterval; 2010 if ((type & USB_OBJECT_BULK_PIPE) != 0 2011 || (type & USB_OBJECT_CONTROL_PIPE) != 0) { 2012 // Bulk and Control endpoints never issue NAKs. 2013 calcInterval = 0; 2014 } else { 2015 switch (speed) { 2016 case USB_SPEED_FULLSPEED: 2017 if ((type & USB_OBJECT_ISO_PIPE) != 0) { 2018 // Convert 1-16 into 3-18. 2019 calcInterval = min_c(max_c(interval, 1), 16) + 2; 2020 break; 2021 } 2022 2023 // fall through 2024 case USB_SPEED_LOWSPEED: { 2025 // Convert 1ms-255ms into 3-10. 2026 2027 // Find the index of the highest set bit in "interval". 2028 uint32 temp = min_c(max_c(interval, 1), 255); 2029 for (calcInterval = 0; temp != 1; calcInterval++) 2030 temp = temp >> 1; 2031 calcInterval += 3; 2032 break; 2033 } 2034 2035 case USB_SPEED_HIGHSPEED: 2036 case USB_SPEED_SUPERSPEED: 2037 default: 2038 // Convert 1-16 into 0-15. 2039 calcInterval = min_c(max_c(interval, 1), 16) - 1; 2040 break; 2041 } 2042 } 2043 dwendpoint0 |= ENDPOINT_0_INTERVAL(calcInterval); 2044 2045 // For non-isochronous endpoints, we want the controller to retry failed 2046 // transfers, if possible. (XHCI 1.2 § 4.10.2.3 p197.) 2047 if ((type & USB_OBJECT_ISO_PIPE) == 0) 2048 dwendpoint1 |= ENDPOINT_1_CERR(3); 2049 2050 // Assign maximum burst size. For USB3 devices this is passed in; for 2051 // all other devices we compute it. (XHCI 1.2 § 4.8.2 p161.) 2052 if (speed == USB_SPEED_HIGHSPEED && (type & (USB_OBJECT_INTERRUPT_PIPE 2053 | USB_OBJECT_ISO_PIPE)) != 0) { 2054 maxBurst = (maxPacketSize & 0x1800) >> 11; 2055 } else if (speed != USB_SPEED_SUPERSPEED) { 2056 maxBurst = 0; 2057 } 2058 dwendpoint1 |= ENDPOINT_1_MAXBURST(maxBurst); 2059 2060 // Assign maximum packet size, set the ring address, and set the 2061 // "Dequeue Cycle State" bit. (XHCI 1.2 § 6.2.3 Table 6-10 p453.) 2062 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE(maxPacketSize); 2063 qwendpoint2 |= ENDPOINT_2_DCS_BIT | ep->trb_addr; 2064 2065 // The Max Burst Payload is the number of bytes moved by a 2066 // maximum sized burst. (XHCI 1.2 § 4.11.7.1 p236.) 2067 ep->max_burst_payload = (maxBurst + 1) * maxPacketSize; 2068 if (ep->max_burst_payload == 0) { 2069 TRACE_ERROR("ConfigureEndpoint() failed invalid max_burst_payload\n"); 2070 return B_BAD_VALUE; 2071 } 2072 2073 // Assign average TRB length. 2074 if ((type & USB_OBJECT_CONTROL_PIPE) != 0) { 2075 // Control pipes are a special case, as they rarely have 2076 // outbound transfers of any substantial size. 2077 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(8); 2078 } else if ((type & USB_OBJECT_ISO_PIPE) != 0) { 2079 // Isochronous pipes are another special case: the TRB size will be 2080 // one packet (which is normally smaller than the max packet size, 2081 // but we don't know what it is here.) 2082 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(maxPacketSize); 2083 } else { 2084 // Under all other circumstances, we put max_burst_payload in a TRB. 2085 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(ep->max_burst_payload); 2086 } 2087 2088 // Assign maximum ESIT payload. (XHCI 1.2 § 4.14.2 p259.) 2089 if ((type & (USB_OBJECT_INTERRUPT_PIPE | USB_OBJECT_ISO_PIPE)) != 0) { 2090 // TODO: For SuperSpeedPlus endpoints, there is yet another descriptor 2091 // for isochronous endpoints that specifies the maximum ESIT payload. 2092 // We don't fetch this yet, so just fall back to the USB2 computation 2093 // method if bytesPerInterval is 0. 2094 if (speed == USB_SPEED_SUPERSPEED && bytesPerInterval != 0) 2095 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD(bytesPerInterval); 2096 else if (speed >= USB_SPEED_HIGHSPEED) 2097 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD((maxBurst + 1) * maxPacketSize); 2098 } 2099 2100 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint0, 2101 dwendpoint0); 2102 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint1, 2103 dwendpoint1); 2104 _WriteContext(&device->input_ctx->endpoints[number].qwendpoint2, 2105 qwendpoint2); 2106 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint4, 2107 dwendpoint4); 2108 2109 TRACE("endpoint 0x%" B_PRIx32 " 0x%" B_PRIx32 " 0x%" B_PRIx64 " 0x%" 2110 B_PRIx32 "\n", 2111 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint0), 2112 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint1), 2113 _ReadContext(&device->input_ctx->endpoints[number].qwendpoint2), 2114 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint4)); 2115 2116 return B_OK; 2117 } 2118 2119 2120 status_t 2121 XHCI::GetPortSpeed(uint8 index, usb_speed* speed) 2122 { 2123 if (index >= fPortCount) 2124 return B_BAD_INDEX; 2125 2126 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 2127 2128 switch (PS_SPEED_GET(portStatus)) { 2129 case 3: 2130 *speed = USB_SPEED_HIGHSPEED; 2131 break; 2132 case 2: 2133 *speed = USB_SPEED_LOWSPEED; 2134 break; 2135 case 1: 2136 *speed = USB_SPEED_FULLSPEED; 2137 break; 2138 case 4: 2139 *speed = USB_SPEED_SUPERSPEED; 2140 break; 2141 default: 2142 TRACE_ALWAYS("nonstandard port speed %" B_PRId32 ", assuming SuperSpeed\n", 2143 PS_SPEED_GET(portStatus)); 2144 *speed = USB_SPEED_SUPERSPEED; 2145 break; 2146 } 2147 2148 return B_OK; 2149 } 2150 2151 2152 status_t 2153 XHCI::GetPortStatus(uint8 index, usb_port_status* status) 2154 { 2155 if (index >= fPortCount) 2156 return B_BAD_INDEX; 2157 2158 status->status = status->change = 0; 2159 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 2160 TRACE("port %" B_PRId8 " status=0x%08" B_PRIx32 "\n", index, portStatus); 2161 2162 // build the status 2163 switch (PS_SPEED_GET(portStatus)) { 2164 case 3: 2165 status->status |= PORT_STATUS_HIGH_SPEED; 2166 break; 2167 case 2: 2168 status->status |= PORT_STATUS_LOW_SPEED; 2169 break; 2170 default: 2171 break; 2172 } 2173 2174 if (portStatus & PS_CCS) 2175 status->status |= PORT_STATUS_CONNECTION; 2176 if (portStatus & PS_PED) 2177 status->status |= PORT_STATUS_ENABLE; 2178 if (portStatus & PS_OCA) 2179 status->status |= PORT_STATUS_OVER_CURRENT; 2180 if (portStatus & PS_PR) 2181 status->status |= PORT_STATUS_RESET; 2182 if (portStatus & PS_PP) { 2183 if (fPortSpeeds[index] == USB_SPEED_SUPERSPEED) 2184 status->status |= PORT_STATUS_SS_POWER; 2185 else 2186 status->status |= PORT_STATUS_POWER; 2187 } 2188 2189 // build the change 2190 if (portStatus & PS_CSC) 2191 status->change |= PORT_STATUS_CONNECTION; 2192 if (portStatus & PS_PEC) 2193 status->change |= PORT_STATUS_ENABLE; 2194 if (portStatus & PS_OCC) 2195 status->change |= PORT_STATUS_OVER_CURRENT; 2196 if (portStatus & PS_PRC) 2197 status->change |= PORT_STATUS_RESET; 2198 2199 if (fPortSpeeds[index] == USB_SPEED_SUPERSPEED) { 2200 if (portStatus & PS_PLC) 2201 status->change |= PORT_CHANGE_LINK_STATE; 2202 if (portStatus & PS_WRC) 2203 status->change |= PORT_CHANGE_BH_PORT_RESET; 2204 } 2205 2206 return B_OK; 2207 } 2208 2209 2210 status_t 2211 XHCI::SetPortFeature(uint8 index, uint16 feature) 2212 { 2213 TRACE("set port feature index %u feature %u\n", index, feature); 2214 if (index >= fPortCount) 2215 return B_BAD_INDEX; 2216 2217 uint32 portRegister = XHCI_PORTSC(index); 2218 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2219 2220 switch (feature) { 2221 case PORT_SUSPEND: 2222 if ((portStatus & PS_PED) == 0 || (portStatus & PS_PR) 2223 || (portStatus & PS_PLS_MASK) >= PS_XDEV_U3) { 2224 TRACE_ERROR("USB core suspending device not in U0/U1/U2.\n"); 2225 return B_BAD_VALUE; 2226 } 2227 portStatus &= ~PS_PLS_MASK; 2228 WriteOpReg(portRegister, portStatus | PS_LWS | PS_XDEV_U3); 2229 break; 2230 2231 case PORT_RESET: 2232 WriteOpReg(portRegister, portStatus | PS_PR); 2233 break; 2234 2235 case PORT_POWER: 2236 WriteOpReg(portRegister, portStatus | PS_PP); 2237 break; 2238 default: 2239 return B_BAD_VALUE; 2240 } 2241 ReadOpReg(portRegister); 2242 return B_OK; 2243 } 2244 2245 2246 status_t 2247 XHCI::ClearPortFeature(uint8 index, uint16 feature) 2248 { 2249 TRACE("clear port feature index %u feature %u\n", index, feature); 2250 if (index >= fPortCount) 2251 return B_BAD_INDEX; 2252 2253 uint32 portRegister = XHCI_PORTSC(index); 2254 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2255 2256 switch (feature) { 2257 case PORT_SUSPEND: 2258 portStatus = ReadOpReg(portRegister); 2259 if (portStatus & PS_PR) 2260 return B_BAD_VALUE; 2261 if (portStatus & PS_XDEV_U3) { 2262 if ((portStatus & PS_PED) == 0) 2263 return B_BAD_VALUE; 2264 portStatus &= ~PS_PLS_MASK; 2265 WriteOpReg(portRegister, portStatus | PS_XDEV_U0 | PS_LWS); 2266 } 2267 break; 2268 case PORT_ENABLE: 2269 WriteOpReg(portRegister, portStatus | PS_PED); 2270 break; 2271 case PORT_POWER: 2272 WriteOpReg(portRegister, portStatus & ~PS_PP); 2273 break; 2274 case C_PORT_CONNECTION: 2275 WriteOpReg(portRegister, portStatus | PS_CSC); 2276 break; 2277 case C_PORT_ENABLE: 2278 WriteOpReg(portRegister, portStatus | PS_PEC); 2279 break; 2280 case C_PORT_OVER_CURRENT: 2281 WriteOpReg(portRegister, portStatus | PS_OCC); 2282 break; 2283 case C_PORT_RESET: 2284 WriteOpReg(portRegister, portStatus | PS_PRC); 2285 break; 2286 case C_PORT_BH_PORT_RESET: 2287 WriteOpReg(portRegister, portStatus | PS_WRC); 2288 break; 2289 case C_PORT_LINK_STATE: 2290 WriteOpReg(portRegister, portStatus | PS_PLC); 2291 break; 2292 default: 2293 return B_BAD_VALUE; 2294 } 2295 2296 ReadOpReg(portRegister); 2297 return B_OK; 2298 } 2299 2300 2301 status_t 2302 XHCI::ControllerHalt() 2303 { 2304 // Mask off run state 2305 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) & ~CMD_RUN); 2306 2307 // wait for shutdown state 2308 if (WaitOpBits(XHCI_STS, STS_HCH, STS_HCH) != B_OK) { 2309 TRACE_ERROR("HCH shutdown timeout\n"); 2310 return B_ERROR; 2311 } 2312 return B_OK; 2313 } 2314 2315 2316 status_t 2317 XHCI::ControllerReset() 2318 { 2319 TRACE("ControllerReset() cmd: 0x%" B_PRIx32 " sts: 0x%" B_PRIx32 "\n", 2320 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 2321 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) | CMD_HCRST); 2322 2323 if (WaitOpBits(XHCI_CMD, CMD_HCRST, 0) != B_OK) { 2324 TRACE_ERROR("ControllerReset() failed CMD_HCRST\n"); 2325 return B_ERROR; 2326 } 2327 2328 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 2329 TRACE_ERROR("ControllerReset() failed STS_CNR\n"); 2330 return B_ERROR; 2331 } 2332 2333 return B_OK; 2334 } 2335 2336 2337 int32 2338 XHCI::InterruptHandler(void* data) 2339 { 2340 return ((XHCI*)data)->Interrupt(); 2341 } 2342 2343 2344 int32 2345 XHCI::Interrupt() 2346 { 2347 SpinLocker _(&fSpinlock); 2348 2349 uint32 status = ReadOpReg(XHCI_STS); 2350 uint32 temp = ReadRunReg32(XHCI_IMAN(0)); 2351 WriteOpReg(XHCI_STS, status); 2352 WriteRunReg32(XHCI_IMAN(0), temp); 2353 2354 int32 result = B_HANDLED_INTERRUPT; 2355 2356 if ((status & STS_HCH) != 0) { 2357 TRACE_ERROR("Host Controller halted\n"); 2358 return result; 2359 } 2360 if ((status & STS_HSE) != 0) { 2361 TRACE_ERROR("Host System Error\n"); 2362 return result; 2363 } 2364 if ((status & STS_HCE) != 0) { 2365 TRACE_ERROR("Host Controller Error\n"); 2366 return result; 2367 } 2368 2369 if ((status & STS_EINT) == 0) { 2370 TRACE("STS: 0x%" B_PRIx32 " IRQ_PENDING: 0x%" B_PRIx32 "\n", 2371 status, temp); 2372 return B_UNHANDLED_INTERRUPT; 2373 } 2374 2375 TRACE("Event Interrupt\n"); 2376 release_sem_etc(fEventSem, 1, B_DO_NOT_RESCHEDULE); 2377 return B_INVOKE_SCHEDULER; 2378 } 2379 2380 2381 void 2382 XHCI::Ring(uint8 slot, uint8 endpoint) 2383 { 2384 TRACE("Ding Dong! slot:%d endpoint %d\n", slot, endpoint) 2385 if ((slot == 0 && endpoint > 0) || (slot > 0 && endpoint == 0)) 2386 panic("Ring() invalid slot/endpoint combination\n"); 2387 if (slot > fSlotCount || endpoint >= XHCI_MAX_ENDPOINTS) 2388 panic("Ring() invalid slot or endpoint\n"); 2389 2390 WriteDoorReg32(XHCI_DOORBELL(slot), XHCI_DOORBELL_TARGET(endpoint) 2391 | XHCI_DOORBELL_STREAMID(0)); 2392 ReadDoorReg32(XHCI_DOORBELL(slot)); 2393 // Flush PCI writes 2394 } 2395 2396 2397 void 2398 XHCI::QueueCommand(xhci_trb* trb) 2399 { 2400 uint8 i, j; 2401 uint32 temp; 2402 2403 i = fCmdIdx; 2404 j = fCmdCcs; 2405 2406 TRACE("command[%u] = %" B_PRId32 " (0x%016" B_PRIx64 ", 0x%08" B_PRIx32 2407 ", 0x%08" B_PRIx32 ")\n", i, TRB_3_TYPE_GET(trb->flags), trb->address, 2408 trb->status, trb->flags); 2409 2410 fCmdRing[i].address = trb->address; 2411 fCmdRing[i].status = trb->status; 2412 temp = trb->flags; 2413 2414 if (j) 2415 temp |= TRB_3_CYCLE_BIT; 2416 else 2417 temp &= ~TRB_3_CYCLE_BIT; 2418 temp &= ~TRB_3_TC_BIT; 2419 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2420 2421 fCmdAddr = fErst->rs_addr + (XHCI_MAX_EVENTS + i) * sizeof(xhci_trb); 2422 2423 i++; 2424 2425 if (i == (XHCI_MAX_COMMANDS - 1)) { 2426 temp = TRB_3_TYPE(TRB_TYPE_LINK) | TRB_3_TC_BIT; 2427 if (j) 2428 temp |= TRB_3_CYCLE_BIT; 2429 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2430 2431 i = 0; 2432 j ^= 1; 2433 } 2434 2435 fCmdIdx = i; 2436 fCmdCcs = j; 2437 } 2438 2439 2440 void 2441 XHCI::HandleCmdComplete(xhci_trb* trb) 2442 { 2443 if (fCmdAddr == trb->address) { 2444 TRACE("Received command event\n"); 2445 fCmdResult[0] = trb->status; 2446 fCmdResult[1] = B_LENDIAN_TO_HOST_INT32(trb->flags); 2447 release_sem_etc(fCmdCompSem, 1, B_DO_NOT_RESCHEDULE); 2448 } else 2449 TRACE_ERROR("received command event for unknown command!\n") 2450 } 2451 2452 2453 void 2454 XHCI::HandleTransferComplete(xhci_trb* trb) 2455 { 2456 const uint32 flags = B_LENDIAN_TO_HOST_INT32(trb->flags); 2457 const uint8 endpointNumber = TRB_3_ENDPOINT_GET(flags), 2458 slot = TRB_3_SLOT_GET(flags); 2459 2460 if (slot > fSlotCount) 2461 TRACE_ERROR("invalid slot\n"); 2462 if (endpointNumber == 0 || endpointNumber >= XHCI_MAX_ENDPOINTS) { 2463 TRACE_ERROR("invalid endpoint\n"); 2464 return; 2465 } 2466 2467 xhci_device *device = &fDevices[slot]; 2468 xhci_endpoint *endpoint = &device->endpoints[endpointNumber - 1]; 2469 2470 if (endpoint->trbs == NULL) { 2471 TRACE_ERROR("got TRB but endpoint is not allocated!\n"); 2472 return; 2473 } 2474 2475 // Use mutex_trylock first, in case we are in KDL. 2476 MutexLocker endpointLocker(endpoint->lock, 2477 mutex_trylock(&endpoint->lock) == B_OK); 2478 if (!endpointLocker.IsLocked()) { 2479 // We failed to get the lock. Most likely it was destroyed 2480 // while we were waiting for it. 2481 return; 2482 } 2483 2484 // In the case of an Event Data TRB, the "transferred" field refers 2485 // to the actual number of bytes transferred across the whole TD. 2486 // (XHCI 1.2 § 6.4.2.1 Table 6-38 p478.) 2487 const uint8 completionCode = TRB_2_COMP_CODE_GET(trb->status); 2488 int32 transferred = TRB_2_REM_GET(trb->status), remainder = -1; 2489 2490 TRACE("HandleTransferComplete: ed %d, code %d, transferred %d\n", 2491 (flags & TRB_3_EVENT_DATA_BIT), completionCode, transferred); 2492 2493 if ((flags & TRB_3_EVENT_DATA_BIT) == 0) { 2494 // This should only occur under error conditions. 2495 TRACE("got an interrupt for a non-Event Data TRB!\n"); 2496 remainder = transferred; 2497 transferred = -1; 2498 } 2499 2500 if (completionCode != COMP_SUCCESS && completionCode != COMP_SHORT_PACKET) { 2501 TRACE_ALWAYS("transfer error on slot %" B_PRId8 " endpoint %" B_PRId8 2502 ": %s\n", slot, endpointNumber, xhci_error_string(completionCode)); 2503 } 2504 2505 const phys_addr_t source = B_LENDIAN_TO_HOST_INT64(trb->address); 2506 for (xhci_td *td = endpoint->td_head; td != NULL; td = td->next) { 2507 int64 offset = (source - td->trb_addr) / sizeof(xhci_trb); 2508 if (offset < 0 || offset >= td->trb_count) 2509 continue; 2510 2511 TRACE("HandleTransferComplete td %p trb %" B_PRId64 " found\n", 2512 td, offset); 2513 2514 // The TRB at offset trb_used will be the link TRB, which we do not 2515 // care about (and should not generate an interrupt at all.) We really 2516 // care about the properly last TRB, at index "count - 1", which the 2517 // Event Data TRB that _LinkDescriptorForPipe creates points to. 2518 // 2519 // But if we have an unsuccessful completion code, the transfer 2520 // likely failed midway; so just accept it anyway. 2521 if (offset == (td->trb_used - 1) || completionCode != COMP_SUCCESS) { 2522 _UnlinkDescriptorForPipe(td, endpoint); 2523 endpointLocker.Unlock(); 2524 2525 td->trb_completion_code = completionCode; 2526 td->td_transferred = transferred; 2527 td->trb_left = remainder; 2528 2529 // add descriptor to finished list 2530 if (mutex_trylock(&fFinishedLock) != B_OK) 2531 mutex_lock(&fFinishedLock); 2532 td->next = fFinishedHead; 2533 fFinishedHead = td; 2534 mutex_unlock(&fFinishedLock); 2535 2536 release_sem_etc(fFinishTransfersSem, 1, B_DO_NOT_RESCHEDULE); 2537 TRACE("HandleTransferComplete td %p done\n", td); 2538 } else { 2539 TRACE_ERROR("successful TRB 0x%" B_PRIxPHYSADDR " was found, but it wasn't " 2540 "the last in the TD!\n", source); 2541 } 2542 return; 2543 } 2544 TRACE_ERROR("TRB 0x%" B_PRIxPHYSADDR " was not found in the endpoint!\n", source); 2545 } 2546 2547 2548 void 2549 XHCI::DumpRing(xhci_trb *trbs, uint32 size) 2550 { 2551 if (!Lock()) { 2552 TRACE("Unable to get lock!\n"); 2553 return; 2554 } 2555 2556 for (uint32 i = 0; i < size; i++) { 2557 TRACE("command[%" B_PRId32 "] = %" B_PRId32 " (0x%016" B_PRIx64 "," 2558 " 0x%08" B_PRIx32 ", 0x%08" B_PRIx32 ")\n", i, 2559 TRB_3_TYPE_GET(B_LENDIAN_TO_HOST_INT32(trbs[i].flags)), 2560 trbs[i].address, trbs[i].status, trbs[i].flags); 2561 } 2562 2563 Unlock(); 2564 } 2565 2566 2567 status_t 2568 XHCI::DoCommand(xhci_trb* trb) 2569 { 2570 if (!Lock()) { 2571 TRACE("Unable to get lock!\n"); 2572 return B_ERROR; 2573 } 2574 2575 QueueCommand(trb); 2576 Ring(0, 0); 2577 2578 // Begin with a 50ms timeout. 2579 if (acquire_sem_etc(fCmdCompSem, 1, B_RELATIVE_TIMEOUT, 50 * 1000) != B_OK) { 2580 // We've hit the timeout. In some error cases, interrupts are not 2581 // generated; so here we force the event ring to be polled once. 2582 release_sem(fEventSem); 2583 2584 // Now try again, this time with a 750ms timeout. 2585 if (acquire_sem_etc(fCmdCompSem, 1, B_RELATIVE_TIMEOUT, 2586 750 * 1000) != B_OK) { 2587 TRACE("Unable to obtain fCmdCompSem!\n"); 2588 fCmdAddr = 0; 2589 Unlock(); 2590 return B_TIMED_OUT; 2591 } 2592 } 2593 2594 // eat up sems that have been released by multiple interrupts 2595 int32 semCount = 0; 2596 get_sem_count(fCmdCompSem, &semCount); 2597 if (semCount > 0) 2598 acquire_sem_etc(fCmdCompSem, semCount, B_RELATIVE_TIMEOUT, 0); 2599 2600 status_t status = B_OK; 2601 uint32 completionCode = TRB_2_COMP_CODE_GET(fCmdResult[0]); 2602 TRACE("command complete\n"); 2603 if (completionCode != COMP_SUCCESS) { 2604 TRACE_ERROR("unsuccessful command %" B_PRId32 ", error %s (%" B_PRId32 ")\n", 2605 TRB_3_TYPE_GET(trb->flags), xhci_error_string(completionCode), 2606 completionCode); 2607 status = B_IO_ERROR; 2608 } 2609 2610 trb->status = fCmdResult[0]; 2611 trb->flags = fCmdResult[1]; 2612 2613 fCmdAddr = 0; 2614 Unlock(); 2615 return status; 2616 } 2617 2618 2619 status_t 2620 XHCI::Noop() 2621 { 2622 TRACE("Issue No-Op\n"); 2623 xhci_trb trb; 2624 trb.address = 0; 2625 trb.status = 0; 2626 trb.flags = TRB_3_TYPE(TRB_TYPE_CMD_NOOP); 2627 2628 return DoCommand(&trb); 2629 } 2630 2631 2632 status_t 2633 XHCI::EnableSlot(uint8* slot) 2634 { 2635 TRACE("Enable Slot\n"); 2636 xhci_trb trb; 2637 trb.address = 0; 2638 trb.status = 0; 2639 trb.flags = TRB_3_TYPE(TRB_TYPE_ENABLE_SLOT); 2640 2641 status_t status = DoCommand(&trb); 2642 if (status != B_OK) 2643 return status; 2644 2645 *slot = TRB_3_SLOT_GET(trb.flags); 2646 return *slot != 0 ? B_OK : B_BAD_VALUE; 2647 } 2648 2649 2650 status_t 2651 XHCI::DisableSlot(uint8 slot) 2652 { 2653 TRACE("Disable Slot\n"); 2654 xhci_trb trb; 2655 trb.address = 0; 2656 trb.status = 0; 2657 trb.flags = TRB_3_TYPE(TRB_TYPE_DISABLE_SLOT) | TRB_3_SLOT(slot); 2658 2659 return DoCommand(&trb); 2660 } 2661 2662 2663 status_t 2664 XHCI::SetAddress(uint64 inputContext, bool bsr, uint8 slot) 2665 { 2666 TRACE("Set Address\n"); 2667 xhci_trb trb; 2668 trb.address = inputContext; 2669 trb.status = 0; 2670 trb.flags = TRB_3_TYPE(TRB_TYPE_ADDRESS_DEVICE) | TRB_3_SLOT(slot); 2671 2672 if (bsr) 2673 trb.flags |= TRB_3_BSR_BIT; 2674 2675 return DoCommand(&trb); 2676 } 2677 2678 2679 status_t 2680 XHCI::ConfigureEndpoint(uint64 inputContext, bool deconfigure, uint8 slot) 2681 { 2682 TRACE("Configure Endpoint\n"); 2683 xhci_trb trb; 2684 trb.address = inputContext; 2685 trb.status = 0; 2686 trb.flags = TRB_3_TYPE(TRB_TYPE_CONFIGURE_ENDPOINT) | TRB_3_SLOT(slot); 2687 2688 if (deconfigure) 2689 trb.flags |= TRB_3_DCEP_BIT; 2690 2691 return DoCommand(&trb); 2692 } 2693 2694 2695 status_t 2696 XHCI::EvaluateContext(uint64 inputContext, uint8 slot) 2697 { 2698 TRACE("Evaluate Context\n"); 2699 xhci_trb trb; 2700 trb.address = inputContext; 2701 trb.status = 0; 2702 trb.flags = TRB_3_TYPE(TRB_TYPE_EVALUATE_CONTEXT) | TRB_3_SLOT(slot); 2703 2704 return DoCommand(&trb); 2705 } 2706 2707 2708 status_t 2709 XHCI::ResetEndpoint(bool preserve, xhci_endpoint* endpoint) 2710 { 2711 TRACE("Reset Endpoint\n"); 2712 2713 switch (_GetEndpointState(endpoint)) { 2714 case ENDPOINT_STATE_STOPPED: 2715 TRACE("Reset Endpoint: already stopped"); 2716 return B_OK; 2717 case ENDPOINT_STATE_HALTED: 2718 TRACE("Reset Endpoint: warning, weird state!"); 2719 default: 2720 break; 2721 } 2722 2723 xhci_trb trb; 2724 trb.address = 0; 2725 trb.status = 0; 2726 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_ENDPOINT) 2727 | TRB_3_SLOT(endpoint->device->slot) | TRB_3_ENDPOINT(endpoint->id + 1); 2728 if (preserve) 2729 trb.flags |= TRB_3_PRSV_BIT; 2730 2731 return DoCommand(&trb); 2732 } 2733 2734 2735 status_t 2736 XHCI::StopEndpoint(bool suspend, xhci_endpoint* endpoint) 2737 { 2738 TRACE("Stop Endpoint\n"); 2739 2740 switch (_GetEndpointState(endpoint)) { 2741 case ENDPOINT_STATE_HALTED: 2742 TRACE("Stop Endpoint: error, halted"); 2743 return B_DEV_STALLED; 2744 case ENDPOINT_STATE_STOPPED: 2745 TRACE("Stop Endpoint: already stopped"); 2746 return B_OK; 2747 default: 2748 break; 2749 } 2750 2751 xhci_trb trb; 2752 trb.address = 0; 2753 trb.status = 0; 2754 trb.flags = TRB_3_TYPE(TRB_TYPE_STOP_ENDPOINT) 2755 | TRB_3_SLOT(endpoint->device->slot) | TRB_3_ENDPOINT(endpoint->id + 1); 2756 if (suspend) 2757 trb.flags |= TRB_3_SUSPEND_ENDPOINT_BIT; 2758 2759 return DoCommand(&trb); 2760 } 2761 2762 2763 status_t 2764 XHCI::SetTRDequeue(uint64 dequeue, uint16 stream, uint8 endpoint, uint8 slot) 2765 { 2766 TRACE("Set TR Dequeue\n"); 2767 xhci_trb trb; 2768 trb.address = dequeue | ENDPOINT_2_DCS_BIT; 2769 // The DCS bit is copied from the address field as in ConfigureEndpoint. 2770 // (XHCI 1.2 § 4.6.10 p142.) 2771 trb.status = TRB_2_STREAM(stream); 2772 trb.flags = TRB_3_TYPE(TRB_TYPE_SET_TR_DEQUEUE) 2773 | TRB_3_SLOT(slot) | TRB_3_ENDPOINT(endpoint); 2774 2775 return DoCommand(&trb); 2776 } 2777 2778 2779 status_t 2780 XHCI::ResetDevice(uint8 slot) 2781 { 2782 TRACE("Reset Device\n"); 2783 xhci_trb trb; 2784 trb.address = 0; 2785 trb.status = 0; 2786 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_DEVICE) | TRB_3_SLOT(slot); 2787 2788 return DoCommand(&trb); 2789 } 2790 2791 2792 int32 2793 XHCI::EventThread(void* data) 2794 { 2795 ((XHCI *)data)->CompleteEvents(); 2796 return B_OK; 2797 } 2798 2799 2800 void 2801 XHCI::CompleteEvents() 2802 { 2803 while (!fStopThreads) { 2804 if (acquire_sem(fEventSem) < B_OK) 2805 continue; 2806 2807 // eat up sems that have been released by multiple interrupts 2808 int32 semCount = 0; 2809 get_sem_count(fEventSem, &semCount); 2810 if (semCount > 0) 2811 acquire_sem_etc(fEventSem, semCount, B_RELATIVE_TIMEOUT, 0); 2812 2813 ProcessEvents(); 2814 } 2815 } 2816 2817 2818 void 2819 XHCI::ProcessEvents() 2820 { 2821 // Use mutex_trylock first, in case we are in KDL. 2822 MutexLocker locker(fEventLock, mutex_trylock(&fEventLock) == B_OK); 2823 if (!locker.IsLocked()) { 2824 // We failed to get the lock. This really should not happen. 2825 TRACE_ERROR("failed to acquire event lock!\n"); 2826 return; 2827 } 2828 2829 uint16 i = fEventIdx; 2830 uint8 j = fEventCcs; 2831 uint8 t = 2; 2832 2833 while (1) { 2834 uint32 temp = B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags); 2835 uint8 event = TRB_3_TYPE_GET(temp); 2836 TRACE("event[%u] = %u (0x%016" B_PRIx64 " 0x%08" B_PRIx32 " 0x%08" 2837 B_PRIx32 ")\n", i, event, fEventRing[i].address, 2838 fEventRing[i].status, B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags)); 2839 uint8 k = (temp & TRB_3_CYCLE_BIT) ? 1 : 0; 2840 if (j != k) 2841 break; 2842 2843 switch (event) { 2844 case TRB_TYPE_COMMAND_COMPLETION: 2845 HandleCmdComplete(&fEventRing[i]); 2846 break; 2847 case TRB_TYPE_TRANSFER: 2848 HandleTransferComplete(&fEventRing[i]); 2849 break; 2850 case TRB_TYPE_PORT_STATUS_CHANGE: 2851 TRACE("port change detected\n"); 2852 break; 2853 default: 2854 TRACE_ERROR("Unhandled event = %u\n", event); 2855 break; 2856 } 2857 2858 i++; 2859 if (i == XHCI_MAX_EVENTS) { 2860 i = 0; 2861 j ^= 1; 2862 if (!--t) 2863 break; 2864 } 2865 } 2866 2867 fEventIdx = i; 2868 fEventCcs = j; 2869 2870 uint64 addr = fErst->rs_addr + i * sizeof(xhci_trb); 2871 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)addr | ERDP_BUSY); 2872 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(addr >> 32)); 2873 } 2874 2875 2876 int32 2877 XHCI::FinishThread(void* data) 2878 { 2879 ((XHCI *)data)->FinishTransfers(); 2880 return B_OK; 2881 } 2882 2883 2884 void 2885 XHCI::FinishTransfers() 2886 { 2887 while (!fStopThreads) { 2888 if (acquire_sem(fFinishTransfersSem) < B_OK) 2889 continue; 2890 2891 // eat up sems that have been released by multiple interrupts 2892 int32 semCount = 0; 2893 get_sem_count(fFinishTransfersSem, &semCount); 2894 if (semCount > 0) 2895 acquire_sem_etc(fFinishTransfersSem, semCount, B_RELATIVE_TIMEOUT, 0); 2896 2897 mutex_lock(&fFinishedLock); 2898 TRACE("finishing transfers\n"); 2899 while (fFinishedHead != NULL) { 2900 xhci_td* td = fFinishedHead; 2901 fFinishedHead = td->next; 2902 td->next = NULL; 2903 mutex_unlock(&fFinishedLock); 2904 2905 TRACE("finishing transfer td %p\n", td); 2906 2907 Transfer* transfer = td->transfer; 2908 if (transfer == NULL) { 2909 // No transfer? Quick way out. 2910 FreeDescriptor(td); 2911 mutex_lock(&fFinishedLock); 2912 continue; 2913 } 2914 2915 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 2916 2917 status_t callbackStatus = B_OK; 2918 switch (td->trb_completion_code) { 2919 case COMP_SHORT_PACKET: 2920 case COMP_SUCCESS: 2921 callbackStatus = B_OK; 2922 break; 2923 case COMP_DATA_BUFFER: 2924 callbackStatus = directionIn ? B_DEV_DATA_OVERRUN 2925 : B_DEV_DATA_UNDERRUN; 2926 break; 2927 case COMP_BABBLE: 2928 callbackStatus = directionIn ? B_DEV_FIFO_OVERRUN 2929 : B_DEV_FIFO_UNDERRUN; 2930 break; 2931 case COMP_USB_TRANSACTION: 2932 callbackStatus = B_DEV_CRC_ERROR; 2933 break; 2934 case COMP_STALL: 2935 callbackStatus = B_DEV_STALLED; 2936 break; 2937 default: 2938 callbackStatus = B_DEV_STALLED; 2939 break; 2940 } 2941 2942 size_t actualLength = transfer->DataLength(); 2943 if (td->trb_completion_code != COMP_SUCCESS) { 2944 actualLength = td->td_transferred; 2945 if (td->td_transferred == -1) 2946 actualLength = transfer->DataLength() - td->trb_left; 2947 TRACE("transfer not successful, actualLength=%" B_PRIuSIZE "\n", 2948 actualLength); 2949 } 2950 2951 usb_isochronous_data* isochronousData = transfer->IsochronousData(); 2952 if (isochronousData != NULL) { 2953 size_t packetSize = transfer->DataLength() / isochronousData->packet_count, 2954 left = actualLength; 2955 for (uint32 i = 0; i < isochronousData->packet_count; i++) { 2956 size_t size = min_c(packetSize, left); 2957 isochronousData->packet_descriptors[i].actual_length = size; 2958 isochronousData->packet_descriptors[i].status = (size > 0) 2959 ? B_OK : B_DEV_FIFO_UNDERRUN; 2960 left -= size; 2961 } 2962 } 2963 2964 if (callbackStatus == B_OK && directionIn && actualLength > 0) { 2965 TRACE("copying in iov count %ld\n", transfer->VectorCount()); 2966 status_t status = transfer->PrepareKernelAccess(); 2967 if (status == B_OK) { 2968 ReadDescriptor(td, transfer->Vector(), 2969 transfer->VectorCount()); 2970 } else { 2971 callbackStatus = status; 2972 } 2973 } 2974 transfer->Finished(callbackStatus, actualLength); 2975 delete transfer; 2976 FreeDescriptor(td); 2977 mutex_lock(&fFinishedLock); 2978 } 2979 mutex_unlock(&fFinishedLock); 2980 } 2981 } 2982 2983 2984 inline void 2985 XHCI::WriteOpReg(uint32 reg, uint32 value) 2986 { 2987 *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg) = value; 2988 } 2989 2990 2991 inline uint32 2992 XHCI::ReadOpReg(uint32 reg) 2993 { 2994 return *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg); 2995 } 2996 2997 2998 inline status_t 2999 XHCI::WaitOpBits(uint32 reg, uint32 mask, uint32 expected) 3000 { 3001 int loops = 0; 3002 uint32 value = ReadOpReg(reg); 3003 while ((value & mask) != expected) { 3004 snooze(1000); 3005 value = ReadOpReg(reg); 3006 if (loops == 100) { 3007 TRACE("delay waiting on reg 0x%" B_PRIX32 " match 0x%" B_PRIX32 3008 " (0x%" B_PRIX32 ")\n", reg, expected, mask); 3009 } else if (loops > 250) { 3010 TRACE_ERROR("timeout waiting on reg 0x%" B_PRIX32 3011 " match 0x%" B_PRIX32 " (0x%" B_PRIX32 ")\n", reg, expected, 3012 mask); 3013 return B_ERROR; 3014 } 3015 loops++; 3016 } 3017 return B_OK; 3018 } 3019 3020 3021 inline uint32 3022 XHCI::ReadCapReg32(uint32 reg) 3023 { 3024 return *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg); 3025 } 3026 3027 3028 inline void 3029 XHCI::WriteCapReg32(uint32 reg, uint32 value) 3030 { 3031 *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg) = value; 3032 } 3033 3034 3035 inline uint32 3036 XHCI::ReadRunReg32(uint32 reg) 3037 { 3038 return *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg); 3039 } 3040 3041 3042 inline void 3043 XHCI::WriteRunReg32(uint32 reg, uint32 value) 3044 { 3045 *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg) = value; 3046 } 3047 3048 3049 inline uint32 3050 XHCI::ReadDoorReg32(uint32 reg) 3051 { 3052 return *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg); 3053 } 3054 3055 3056 inline void 3057 XHCI::WriteDoorReg32(uint32 reg, uint32 value) 3058 { 3059 *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg) = value; 3060 } 3061 3062 3063 inline addr_t 3064 XHCI::_OffsetContextAddr(addr_t p) 3065 { 3066 if (fContextSizeShift == 1) { 3067 // each structure is page aligned, each pointer is 32 bits aligned 3068 uint32 offset = p & ((B_PAGE_SIZE - 1) & ~31U); 3069 p += offset; 3070 } 3071 return p; 3072 } 3073 3074 inline uint32 3075 XHCI::_ReadContext(uint32* p) 3076 { 3077 p = (uint32*)_OffsetContextAddr((addr_t)p); 3078 return *p; 3079 } 3080 3081 3082 inline void 3083 XHCI::_WriteContext(uint32* p, uint32 value) 3084 { 3085 p = (uint32*)_OffsetContextAddr((addr_t)p); 3086 *p = value; 3087 } 3088 3089 3090 inline uint64 3091 XHCI::_ReadContext(uint64* p) 3092 { 3093 p = (uint64*)_OffsetContextAddr((addr_t)p); 3094 return *p; 3095 } 3096 3097 3098 inline void 3099 XHCI::_WriteContext(uint64* p, uint64 value) 3100 { 3101 p = (uint64*)_OffsetContextAddr((addr_t)p); 3102 *p = value; 3103 } 3104