1 /* 2 * Copyright 2011-2021, Haiku, Inc. All rights reserved. 3 * Distributed under the terms of the MIT License. 4 * 5 * Authors: 6 * Augustin Cavalier <waddlesplash> 7 * Jian Chiang <j.jian.chiang@gmail.com> 8 * Jérôme Duval <jerome.duval@gmail.com> 9 * Akshay Jaggi <akshay1994.leo@gmail.com> 10 * Michael Lotz <mmlr@mlotz.ch> 11 * Alexander von Gluck <kallisti5@unixzen.com> 12 */ 13 14 15 #include <module.h> 16 #include <PCI.h> 17 #include <PCI_x86.h> 18 #include <USB3.h> 19 #include <KernelExport.h> 20 21 #include <util/AutoLock.h> 22 23 #include "xhci.h" 24 25 #define USB_MODULE_NAME "xhci" 26 27 pci_module_info *XHCI::sPCIModule = NULL; 28 pci_x86_module_info *XHCI::sPCIx86Module = NULL; 29 30 31 static int32 32 xhci_std_ops(int32 op, ...) 33 { 34 switch (op) { 35 case B_MODULE_INIT: 36 TRACE_MODULE("xhci init module\n"); 37 return B_OK; 38 case B_MODULE_UNINIT: 39 TRACE_MODULE("xhci uninit module\n"); 40 return B_OK; 41 } 42 43 return EINVAL; 44 } 45 46 47 static const char* 48 xhci_error_string(uint32 error) 49 { 50 switch (error) { 51 case COMP_INVALID: return "Invalid"; 52 case COMP_SUCCESS: return "Success"; 53 case COMP_DATA_BUFFER: return "Data buffer"; 54 case COMP_BABBLE: return "Babble detected"; 55 case COMP_USB_TRANSACTION: return "USB transaction"; 56 case COMP_TRB: return "TRB"; 57 case COMP_STALL: return "Stall"; 58 case COMP_RESOURCE: return "Resource"; 59 case COMP_BANDWIDTH: return "Bandwidth"; 60 case COMP_NO_SLOTS: return "No slots"; 61 case COMP_INVALID_STREAM: return "Invalid stream"; 62 case COMP_SLOT_NOT_ENABLED: return "Slot not enabled"; 63 case COMP_ENDPOINT_NOT_ENABLED: return "Endpoint not enabled"; 64 case COMP_SHORT_PACKET: return "Short packet"; 65 case COMP_RING_UNDERRUN: return "Ring underrun"; 66 case COMP_RING_OVERRUN: return "Ring overrun"; 67 case COMP_VF_RING_FULL: return "VF Event Ring Full"; 68 case COMP_PARAMETER: return "Parameter"; 69 case COMP_BANDWIDTH_OVERRUN: return "Bandwidth overrun"; 70 case COMP_CONTEXT_STATE: return "Context state"; 71 case COMP_NO_PING_RESPONSE: return "No ping response"; 72 case COMP_EVENT_RING_FULL: return "Event ring full"; 73 case COMP_INCOMPATIBLE_DEVICE: return "Incompatible device"; 74 case COMP_MISSED_SERVICE: return "Missed service"; 75 case COMP_COMMAND_RING_STOPPED: return "Command ring stopped"; 76 case COMP_COMMAND_ABORTED: return "Command aborted"; 77 case COMP_STOPPED: return "Stopped"; 78 case COMP_LENGTH_INVALID: return "Length invalid"; 79 case COMP_MAX_EXIT_LATENCY: return "Max exit latency too large"; 80 case COMP_ISOC_OVERRUN: return "Isoch buffer overrun"; 81 case COMP_EVENT_LOST: return "Event lost"; 82 case COMP_UNDEFINED: return "Undefined"; 83 case COMP_INVALID_STREAM_ID: return "Invalid stream ID"; 84 case COMP_SECONDARY_BANDWIDTH: return "Secondary bandwidth"; 85 case COMP_SPLIT_TRANSACTION: return "Split transaction"; 86 87 default: return "Undefined"; 88 } 89 } 90 91 92 usb_host_controller_info xhci_module = { 93 { 94 "busses/usb/xhci", 95 0, 96 xhci_std_ops 97 }, 98 NULL, 99 XHCI::AddTo 100 }; 101 102 103 module_info *modules[] = { 104 (module_info *)&xhci_module, 105 NULL 106 }; 107 108 109 status_t 110 XHCI::AddTo(Stack *stack) 111 { 112 if (!sPCIModule) { 113 status_t status = get_module(B_PCI_MODULE_NAME, 114 (module_info **)&sPCIModule); 115 if (status < B_OK) { 116 TRACE_MODULE_ERROR("getting pci module failed! 0x%08" B_PRIx32 117 "\n", status); 118 return status; 119 } 120 } 121 122 TRACE_MODULE("searching devices\n"); 123 bool found = false; 124 pci_info *item = new(std::nothrow) pci_info; 125 if (item == NULL) { 126 sPCIModule = NULL; 127 put_module(B_PCI_MODULE_NAME); 128 return B_NO_MEMORY; 129 } 130 131 // Try to get the PCI x86 module as well so we can enable possible MSIs. 132 if (sPCIx86Module == NULL && get_module(B_PCI_X86_MODULE_NAME, 133 (module_info **)&sPCIx86Module) != B_OK) { 134 // If it isn't there, that's not critical though. 135 TRACE_MODULE_ERROR("failed to get pci x86 module\n"); 136 sPCIx86Module = NULL; 137 } 138 139 for (int32 i = 0; sPCIModule->get_nth_pci_info(i, item) >= B_OK; i++) { 140 if (item->class_base == PCI_serial_bus && item->class_sub == PCI_usb 141 && item->class_api == PCI_usb_xhci) { 142 TRACE_MODULE("found device at PCI:%d:%d:%d\n", 143 item->bus, item->device, item->function); 144 XHCI *bus = new(std::nothrow) XHCI(item, stack); 145 if (bus == NULL) { 146 delete item; 147 sPCIModule = NULL; 148 put_module(B_PCI_MODULE_NAME); 149 if (sPCIx86Module != NULL) 150 put_module(B_PCI_X86_MODULE_NAME); 151 return B_NO_MEMORY; 152 } 153 154 // The bus will put the PCI modules when it is destroyed, so get 155 // them again to increase their reference count. 156 get_module(B_PCI_MODULE_NAME, (module_info **)&sPCIModule); 157 if (sPCIx86Module != NULL) 158 get_module(B_PCI_X86_MODULE_NAME, (module_info **)&sPCIx86Module); 159 160 if (bus->InitCheck() < B_OK) { 161 TRACE_MODULE_ERROR("bus failed init check\n"); 162 delete bus; 163 continue; 164 } 165 166 // the bus took it away 167 item = new(std::nothrow) pci_info; 168 169 if (bus->Start() != B_OK) { 170 delete bus; 171 continue; 172 } 173 found = true; 174 } 175 } 176 177 // The modules will have been gotten again if we successfully 178 // initialized a bus, so we should put them here. 179 put_module(B_PCI_MODULE_NAME); 180 if (sPCIx86Module != NULL) 181 put_module(B_PCI_X86_MODULE_NAME); 182 183 if (!found) 184 TRACE_MODULE_ERROR("no devices found\n"); 185 delete item; 186 return found ? B_OK : ENODEV; 187 } 188 189 190 XHCI::XHCI(pci_info *info, Stack *stack) 191 : BusManager(stack), 192 fRegisterArea(-1), 193 fRegisters(NULL), 194 fPCIInfo(info), 195 fStack(stack), 196 fIRQ(0), 197 fUseMSI(false), 198 fErstArea(-1), 199 fDcbaArea(-1), 200 fCmdCompSem(-1), 201 fStopThreads(false), 202 fRootHub(NULL), 203 fRootHubAddress(0), 204 fPortCount(0), 205 fSlotCount(0), 206 fScratchpadCount(0), 207 fContextSizeShift(0), 208 fFinishedHead(NULL), 209 fFinishTransfersSem(-1), 210 fFinishThread(-1), 211 fEventSem(-1), 212 fEventThread(-1), 213 fEventIdx(0), 214 fCmdIdx(0), 215 fEventCcs(1), 216 fCmdCcs(1) 217 { 218 B_INITIALIZE_SPINLOCK(&fSpinlock); 219 mutex_init(&fFinishedLock, "XHCI finished transfers"); 220 mutex_init(&fEventLock, "XHCI event handler"); 221 222 if (BusManager::InitCheck() < B_OK) { 223 TRACE_ERROR("bus manager failed to init\n"); 224 return; 225 } 226 227 TRACE("constructing new XHCI host controller driver\n"); 228 fInitOK = false; 229 230 // enable busmaster and memory mapped access 231 uint16 command = sPCIModule->read_pci_config(fPCIInfo->bus, 232 fPCIInfo->device, fPCIInfo->function, PCI_command, 2); 233 command &= ~(PCI_command_io | PCI_command_int_disable); 234 command |= PCI_command_master | PCI_command_memory; 235 236 sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device, 237 fPCIInfo->function, PCI_command, 2, command); 238 239 // map the registers (low + high for 64-bit when requested) 240 phys_addr_t physicalAddress = fPCIInfo->u.h0.base_registers[0]; 241 if ((fPCIInfo->u.h0.base_register_flags[0] & PCI_address_type) 242 == PCI_address_type_64) { 243 physicalAddress |= (uint64)fPCIInfo->u.h0.base_registers[1] << 32; 244 } 245 246 size_t mapSize = fPCIInfo->u.h0.base_register_sizes[0]; 247 248 TRACE("map registers %08" B_PRIxPHYSADDR ", size: %" B_PRIuSIZE "\n", 249 physicalAddress, mapSize); 250 251 fRegisterArea = map_physical_memory("XHCI memory mapped registers", 252 physicalAddress, mapSize, B_ANY_KERNEL_BLOCK_ADDRESS, 253 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 254 (void **)&fRegisters); 255 if (fRegisterArea < B_OK) { 256 TRACE_ERROR("failed to map register memory\n"); 257 return; 258 } 259 260 // determine the register offsets 261 fCapabilityRegisterOffset = 0; 262 fOperationalRegisterOffset = HCI_CAPLENGTH(ReadCapReg32(XHCI_HCI_CAPLENGTH)); 263 fRuntimeRegisterOffset = ReadCapReg32(XHCI_RTSOFF) & ~0x1F; 264 fDoorbellRegisterOffset = ReadCapReg32(XHCI_DBOFF) & ~0x3; 265 266 TRACE("mapped registers: %p\n", fRegisters); 267 TRACE("operational register offset: %" B_PRId32 "\n", fOperationalRegisterOffset); 268 TRACE("runtime register offset: %" B_PRId32 "\n", fRuntimeRegisterOffset); 269 TRACE("doorbell register offset: %" B_PRId32 "\n", fDoorbellRegisterOffset); 270 271 int32 interfaceVersion = HCI_VERSION(ReadCapReg32(XHCI_HCI_VERSION)); 272 if (interfaceVersion < 0x0090 || interfaceVersion > 0x0120) { 273 TRACE_ERROR("unsupported interface version: 0x%04" B_PRIx32 "\n", 274 interfaceVersion); 275 return; 276 } 277 TRACE_ALWAYS("interface version: 0x%04" B_PRIx32 "\n", interfaceVersion); 278 279 TRACE_ALWAYS("structural parameters: 1:0x%08" B_PRIx32 " 2:0x%08" 280 B_PRIx32 " 3:0x%08" B_PRIx32 "\n", ReadCapReg32(XHCI_HCSPARAMS1), 281 ReadCapReg32(XHCI_HCSPARAMS2), ReadCapReg32(XHCI_HCSPARAMS3)); 282 283 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 284 if (cparams == 0xffffffff) 285 return; 286 TRACE_ALWAYS("capability parameters: 0x%08" B_PRIx32 "\n", cparams); 287 288 // if 64 bytes context structures, then 1 289 fContextSizeShift = HCC_CSZ(cparams); 290 291 // Assume ownership of the controller from the BIOS. 292 uint32 eec = 0xffffffff; 293 uint32 eecp = HCS0_XECP(cparams) << 2; 294 for (; eecp != 0 && XECP_NEXT(eec); eecp += XECP_NEXT(eec) << 2) { 295 TRACE("eecp register: 0x%08" B_PRIx32 "\n", eecp); 296 297 eec = ReadCapReg32(eecp); 298 if (XECP_ID(eec) != XHCI_LEGSUP_CAPID) 299 continue; 300 301 if (eec & XHCI_LEGSUP_BIOSOWNED) { 302 TRACE_ALWAYS("the host controller is bios owned, claiming" 303 " ownership\n"); 304 WriteCapReg32(eecp, eec | XHCI_LEGSUP_OSOWNED); 305 306 for (int32 i = 0; i < 20; i++) { 307 eec = ReadCapReg32(eecp); 308 309 if ((eec & XHCI_LEGSUP_BIOSOWNED) == 0) 310 break; 311 312 TRACE_ALWAYS("controller is still bios owned, waiting\n"); 313 snooze(50000); 314 } 315 316 if (eec & XHCI_LEGSUP_BIOSOWNED) { 317 TRACE_ERROR("bios won't give up control over the host " 318 "controller (ignoring)\n"); 319 } else if (eec & XHCI_LEGSUP_OSOWNED) { 320 TRACE_ALWAYS("successfully took ownership of the host " 321 "controller\n"); 322 } 323 324 // Force off the BIOS owned flag, and clear all SMIs. Some BIOSes 325 // do indicate a successful handover but do not remove their SMIs 326 // and then freeze the system when interrupts are generated. 327 WriteCapReg32(eecp, eec & ~XHCI_LEGSUP_BIOSOWNED); 328 } 329 break; 330 } 331 uint32 legctlsts = ReadCapReg32(eecp + XHCI_LEGCTLSTS); 332 legctlsts &= XHCI_LEGCTLSTS_DISABLE_SMI; 333 legctlsts |= XHCI_LEGCTLSTS_EVENTS_SMI; 334 WriteCapReg32(eecp + XHCI_LEGCTLSTS, legctlsts); 335 336 // We need to explicitly take ownership of EHCI ports on earlier Intel chipsets. 337 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL) { 338 switch (fPCIInfo->device_id) { 339 case PCI_DEVICE_INTEL_PANTHER_POINT_XHCI: 340 case PCI_DEVICE_INTEL_LYNX_POINT_XHCI: 341 case PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI: 342 case PCI_DEVICE_INTEL_BAYTRAIL_XHCI: 343 case PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI: 344 case PCI_DEVICE_INTEL_WILDCAT_POINT_LP_XHCI: 345 _SwitchIntelPorts(); 346 break; 347 } 348 } 349 350 // halt the host controller 351 if (ControllerHalt() < B_OK) { 352 return; 353 } 354 355 // reset the host controller 356 if (ControllerReset() < B_OK) { 357 TRACE_ERROR("host controller failed to reset\n"); 358 return; 359 } 360 361 fCmdCompSem = create_sem(0, "XHCI Command Complete"); 362 fFinishTransfersSem = create_sem(0, "XHCI Finish Transfers"); 363 fEventSem = create_sem(0, "XHCI Event"); 364 if (fFinishTransfersSem < B_OK || fCmdCompSem < B_OK || fEventSem < B_OK) { 365 TRACE_ERROR("failed to create semaphores\n"); 366 return; 367 } 368 369 // create event handler thread 370 fEventThread = spawn_kernel_thread(EventThread, "xhci event thread", 371 B_URGENT_PRIORITY, (void *)this); 372 resume_thread(fEventThread); 373 374 // create finisher service thread 375 fFinishThread = spawn_kernel_thread(FinishThread, "xhci finish thread", 376 B_URGENT_PRIORITY - 1, (void *)this); 377 resume_thread(fFinishThread); 378 379 // Find the right interrupt vector, using MSIs if available. 380 fIRQ = fPCIInfo->u.h0.interrupt_line; 381 if (sPCIx86Module != NULL && sPCIx86Module->get_msi_count(fPCIInfo->bus, 382 fPCIInfo->device, fPCIInfo->function) >= 1) { 383 uint8 msiVector = 0; 384 if (sPCIx86Module->configure_msi(fPCIInfo->bus, fPCIInfo->device, 385 fPCIInfo->function, 1, &msiVector) == B_OK 386 && sPCIx86Module->enable_msi(fPCIInfo->bus, fPCIInfo->device, 387 fPCIInfo->function) == B_OK) { 388 TRACE_ALWAYS("using message signaled interrupts\n"); 389 fIRQ = msiVector; 390 fUseMSI = true; 391 } 392 } 393 394 if (fIRQ == 0 || fIRQ == 0xFF) { 395 TRACE_MODULE_ERROR("device PCI:%d:%d:%d was assigned an invalid IRQ\n", 396 fPCIInfo->bus, fPCIInfo->device, fPCIInfo->function); 397 return; 398 } 399 400 // Install the interrupt handler 401 TRACE("installing interrupt handler\n"); 402 install_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this, 0); 403 404 memset(fPortSpeeds, 0, sizeof(fPortSpeeds)); 405 memset(fPortSlots, 0, sizeof(fPortSlots)); 406 memset(fDevices, 0, sizeof(fDevices)); 407 408 fInitOK = true; 409 TRACE("driver construction successful\n"); 410 } 411 412 413 XHCI::~XHCI() 414 { 415 TRACE("tear down XHCI host controller driver\n"); 416 417 WriteOpReg(XHCI_CMD, 0); 418 419 int32 result = 0; 420 fStopThreads = true; 421 delete_sem(fCmdCompSem); 422 delete_sem(fFinishTransfersSem); 423 delete_sem(fEventSem); 424 wait_for_thread(fFinishThread, &result); 425 wait_for_thread(fEventThread, &result); 426 427 mutex_destroy(&fFinishedLock); 428 mutex_destroy(&fEventLock); 429 430 remove_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this); 431 432 delete_area(fRegisterArea); 433 delete_area(fErstArea); 434 for (uint32 i = 0; i < fScratchpadCount; i++) 435 delete_area(fScratchpadArea[i]); 436 delete_area(fDcbaArea); 437 438 if (fUseMSI && sPCIx86Module != NULL) { 439 sPCIx86Module->disable_msi(fPCIInfo->bus, 440 fPCIInfo->device, fPCIInfo->function); 441 sPCIx86Module->unconfigure_msi(fPCIInfo->bus, 442 fPCIInfo->device, fPCIInfo->function); 443 } 444 put_module(B_PCI_MODULE_NAME); 445 if (sPCIx86Module != NULL) 446 put_module(B_PCI_X86_MODULE_NAME); 447 } 448 449 450 void 451 XHCI::_SwitchIntelPorts() 452 { 453 TRACE("Looking for EHCI owned ports\n"); 454 uint32 ports = sPCIModule->read_pci_config(fPCIInfo->bus, 455 fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_USB3PRM, 4); 456 TRACE("Superspeed Ports: 0x%" B_PRIx32 "\n", ports); 457 sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device, 458 fPCIInfo->function, XHCI_INTEL_USB3_PSSEN, 4, ports); 459 ports = sPCIModule->read_pci_config(fPCIInfo->bus, 460 fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_USB3_PSSEN, 4); 461 TRACE("Superspeed ports now under XHCI : 0x%" B_PRIx32 "\n", ports); 462 ports = sPCIModule->read_pci_config(fPCIInfo->bus, 463 fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_USB2PRM, 4); 464 TRACE("USB 2.0 Ports : 0x%" B_PRIx32 "\n", ports); 465 sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device, 466 fPCIInfo->function, XHCI_INTEL_XUSB2PR, 4, ports); 467 ports = sPCIModule->read_pci_config(fPCIInfo->bus, 468 fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_XUSB2PR, 4); 469 TRACE("USB 2.0 ports now under XHCI: 0x%" B_PRIx32 "\n", ports); 470 } 471 472 473 status_t 474 XHCI::Start() 475 { 476 TRACE_ALWAYS("starting XHCI host controller\n"); 477 TRACE("usbcmd: 0x%08" B_PRIx32 "; usbsts: 0x%08" B_PRIx32 "\n", 478 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 479 480 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 481 TRACE("Start() failed STS_CNR\n"); 482 } 483 484 if ((ReadOpReg(XHCI_CMD) & CMD_RUN) != 0) { 485 TRACE_ERROR("Start() warning, starting running XHCI controller!\n"); 486 } 487 488 if ((ReadOpReg(XHCI_PAGESIZE) & (1 << 0)) == 0) { 489 TRACE_ERROR("controller does not support 4K page size\n"); 490 return B_ERROR; 491 } 492 493 // read port count from capability register 494 uint32 capabilities = ReadCapReg32(XHCI_HCSPARAMS1); 495 fPortCount = HCS_MAX_PORTS(capabilities); 496 if (fPortCount == 0) { 497 TRACE_ERROR("invalid number of ports: %u\n", fPortCount); 498 return B_ERROR; 499 } 500 501 fSlotCount = HCS_MAX_SLOTS(capabilities); 502 if (fSlotCount > XHCI_MAX_DEVICES) 503 fSlotCount = XHCI_MAX_DEVICES; 504 WriteOpReg(XHCI_CONFIG, fSlotCount); 505 506 // find out which protocol is used for each port 507 uint8 portFound = 0; 508 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 509 uint32 eec = 0xffffffff; 510 uint32 eecp = HCS0_XECP(cparams) << 2; 511 for (; eecp != 0 && XECP_NEXT(eec) && portFound < fPortCount; 512 eecp += XECP_NEXT(eec) << 2) { 513 eec = ReadCapReg32(eecp); 514 if (XECP_ID(eec) != XHCI_SUPPORTED_PROTOCOLS_CAPID) 515 continue; 516 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) > 3) 517 continue; 518 uint32 temp = ReadCapReg32(eecp + 8); 519 uint32 offset = XHCI_SUPPORTED_PROTOCOLS_1_OFFSET(temp); 520 uint32 count = XHCI_SUPPORTED_PROTOCOLS_1_COUNT(temp); 521 if (offset == 0 || count == 0) 522 continue; 523 offset--; 524 for (uint32 i = offset; i < offset + count; i++) { 525 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) == 0x3) 526 fPortSpeeds[i] = USB_SPEED_SUPERSPEED; 527 else 528 fPortSpeeds[i] = USB_SPEED_HIGHSPEED; 529 530 TRACE("speed for port %" B_PRId32 " is %s\n", i, 531 fPortSpeeds[i] == USB_SPEED_SUPERSPEED ? "super" : "high"); 532 } 533 portFound += count; 534 } 535 536 uint32 params2 = ReadCapReg32(XHCI_HCSPARAMS2); 537 fScratchpadCount = HCS_MAX_SC_BUFFERS(params2); 538 if (fScratchpadCount > XHCI_MAX_SCRATCHPADS) { 539 TRACE_ERROR("invalid number of scratchpads: %" B_PRIu32 "\n", 540 fScratchpadCount); 541 return B_ERROR; 542 } 543 544 uint32 params3 = ReadCapReg32(XHCI_HCSPARAMS3); 545 fExitLatMax = HCS_U1_DEVICE_LATENCY(params3) 546 + HCS_U2_DEVICE_LATENCY(params3); 547 548 // clear interrupts & disable device notifications 549 WriteOpReg(XHCI_STS, ReadOpReg(XHCI_STS)); 550 WriteOpReg(XHCI_DNCTRL, 0); 551 552 // allocate Device Context Base Address array 553 phys_addr_t dmaAddress; 554 fDcbaArea = fStack->AllocateArea((void **)&fDcba, &dmaAddress, 555 sizeof(*fDcba), "DCBA Area"); 556 if (fDcbaArea < B_OK) { 557 TRACE_ERROR("unable to create the DCBA area\n"); 558 return B_ERROR; 559 } 560 memset(fDcba, 0, sizeof(*fDcba)); 561 memset(fScratchpadArea, 0, sizeof(fScratchpadArea)); 562 memset(fScratchpad, 0, sizeof(fScratchpad)); 563 564 // setting the first address to the scratchpad array address 565 fDcba->baseAddress[0] = dmaAddress 566 + offsetof(struct xhci_device_context_array, scratchpad); 567 568 // fill up the scratchpad array with scratchpad pages 569 for (uint32 i = 0; i < fScratchpadCount; i++) { 570 phys_addr_t scratchDmaAddress; 571 fScratchpadArea[i] = fStack->AllocateArea((void **)&fScratchpad[i], 572 &scratchDmaAddress, B_PAGE_SIZE, "Scratchpad Area"); 573 if (fScratchpadArea[i] < B_OK) { 574 TRACE_ERROR("unable to create the scratchpad area\n"); 575 return B_ERROR; 576 } 577 fDcba->scratchpad[i] = scratchDmaAddress; 578 } 579 580 TRACE("setting DCBAAP %" B_PRIxPHYSADDR "\n", dmaAddress); 581 WriteOpReg(XHCI_DCBAAP_LO, (uint32)dmaAddress); 582 WriteOpReg(XHCI_DCBAAP_HI, (uint32)(dmaAddress >> 32)); 583 584 // allocate Event Ring Segment Table 585 uint8 *addr; 586 fErstArea = fStack->AllocateArea((void **)&addr, &dmaAddress, 587 (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 588 + sizeof(xhci_erst_element), 589 "USB XHCI ERST CMD_RING and EVENT_RING Area"); 590 591 if (fErstArea < B_OK) { 592 TRACE_ERROR("unable to create the ERST AND RING area\n"); 593 delete_area(fDcbaArea); 594 return B_ERROR; 595 } 596 fErst = (xhci_erst_element *)addr; 597 memset(fErst, 0, (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 598 + sizeof(xhci_erst_element)); 599 600 // fill with Event Ring Segment Base Address and Event Ring Segment Size 601 fErst->rs_addr = dmaAddress + sizeof(xhci_erst_element); 602 fErst->rs_size = XHCI_MAX_EVENTS; 603 fErst->rsvdz = 0; 604 605 addr += sizeof(xhci_erst_element); 606 fEventRing = (xhci_trb *)addr; 607 addr += XHCI_MAX_EVENTS * sizeof(xhci_trb); 608 fCmdRing = (xhci_trb *)addr; 609 610 TRACE("setting ERST size\n"); 611 WriteRunReg32(XHCI_ERSTSZ(0), XHCI_ERSTS_SET(1)); 612 613 TRACE("setting ERDP addr = 0x%" B_PRIx64 "\n", fErst->rs_addr); 614 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)fErst->rs_addr); 615 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(fErst->rs_addr >> 32)); 616 617 TRACE("setting ERST base addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 618 WriteRunReg32(XHCI_ERSTBA_LO(0), (uint32)dmaAddress); 619 WriteRunReg32(XHCI_ERSTBA_HI(0), (uint32)(dmaAddress >> 32)); 620 621 dmaAddress += sizeof(xhci_erst_element) + XHCI_MAX_EVENTS 622 * sizeof(xhci_trb); 623 624 // Make sure the Command Ring is stopped 625 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 626 TRACE_ALWAYS("Command Ring is running, send stop/cancel\n"); 627 WriteOpReg(XHCI_CRCR_LO, CRCR_CS); 628 WriteOpReg(XHCI_CRCR_HI, 0); 629 WriteOpReg(XHCI_CRCR_LO, CRCR_CA); 630 WriteOpReg(XHCI_CRCR_HI, 0); 631 snooze(1000); 632 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 633 TRACE_ERROR("Command Ring still running after stop/cancel\n"); 634 } 635 } 636 TRACE("setting CRCR addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 637 WriteOpReg(XHCI_CRCR_LO, (uint32)dmaAddress | CRCR_RCS); 638 WriteOpReg(XHCI_CRCR_HI, (uint32)(dmaAddress >> 32)); 639 // link trb 640 fCmdRing[XHCI_MAX_COMMANDS - 1].address = dmaAddress; 641 642 TRACE("setting interrupt rate\n"); 643 644 // Setting IMOD below 0x3F8 on Intel Lynx Point can cause IRQ lockups 645 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL 646 && (fPCIInfo->device_id == PCI_DEVICE_INTEL_PANTHER_POINT_XHCI 647 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_XHCI 648 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI 649 || fPCIInfo->device_id == PCI_DEVICE_INTEL_BAYTRAIL_XHCI 650 || fPCIInfo->device_id == PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI)) { 651 WriteRunReg32(XHCI_IMOD(0), 0x000003f8); // 4000 irq/s 652 } else { 653 WriteRunReg32(XHCI_IMOD(0), 0x000001f4); // 8000 irq/s 654 } 655 656 TRACE("enabling interrupt\n"); 657 WriteRunReg32(XHCI_IMAN(0), ReadRunReg32(XHCI_IMAN(0)) | IMAN_INTR_ENA); 658 659 WriteOpReg(XHCI_CMD, CMD_RUN | CMD_INTE | CMD_HSEE); 660 661 // wait for start up state 662 if (WaitOpBits(XHCI_STS, STS_HCH, 0) != B_OK) { 663 TRACE_ERROR("HCH start up timeout\n"); 664 } 665 666 fRootHubAddress = AllocateAddress(); 667 fRootHub = new(std::nothrow) XHCIRootHub(RootObject(), fRootHubAddress); 668 if (!fRootHub) { 669 TRACE_ERROR("no memory to allocate root hub\n"); 670 return B_NO_MEMORY; 671 } 672 673 if (fRootHub->InitCheck() < B_OK) { 674 TRACE_ERROR("root hub failed init check\n"); 675 return fRootHub->InitCheck(); 676 } 677 678 SetRootHub(fRootHub); 679 680 TRACE_ALWAYS("successfully started the controller\n"); 681 682 #ifdef TRACE_USB 683 TRACE("No-Op test...\n"); 684 Noop(); 685 #endif 686 687 return BusManager::Start(); 688 } 689 690 691 status_t 692 XHCI::SubmitTransfer(Transfer *transfer) 693 { 694 // short circuit the root hub 695 if (transfer->TransferPipe()->DeviceAddress() == fRootHubAddress) 696 return fRootHub->ProcessTransfer(this, transfer); 697 698 TRACE("SubmitTransfer(%p)\n", transfer); 699 Pipe *pipe = transfer->TransferPipe(); 700 if ((pipe->Type() & USB_OBJECT_CONTROL_PIPE) != 0) 701 return SubmitControlRequest(transfer); 702 return SubmitNormalRequest(transfer); 703 } 704 705 706 status_t 707 XHCI::SubmitControlRequest(Transfer *transfer) 708 { 709 Pipe *pipe = transfer->TransferPipe(); 710 usb_request_data *requestData = transfer->RequestData(); 711 bool directionIn = (requestData->RequestType & USB_REQTYPE_DEVICE_IN) != 0; 712 713 TRACE("SubmitControlRequest() length %d\n", requestData->Length); 714 715 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 716 if (endpoint == NULL) { 717 TRACE_ERROR("invalid endpoint!\n"); 718 return B_BAD_VALUE; 719 } 720 status_t status = transfer->InitKernelAccess(); 721 if (status != B_OK) 722 return status; 723 724 xhci_td *descriptor = CreateDescriptor(3, 1, requestData->Length); 725 if (descriptor == NULL) 726 return B_NO_MEMORY; 727 descriptor->transfer = transfer; 728 729 // Setup Stage 730 uint8 index = 0; 731 memcpy(&descriptor->trbs[index].address, requestData, 732 sizeof(usb_request_data)); 733 descriptor->trbs[index].status = TRB_2_IRQ(0) | TRB_2_BYTES(8); 734 descriptor->trbs[index].flags 735 = TRB_3_TYPE(TRB_TYPE_SETUP_STAGE) | TRB_3_IDT_BIT | TRB_3_CYCLE_BIT; 736 if (requestData->Length > 0) { 737 descriptor->trbs[index].flags |= 738 directionIn ? TRB_3_TRT_IN : TRB_3_TRT_OUT; 739 } 740 741 index++; 742 743 // Data Stage (if any) 744 if (requestData->Length > 0) { 745 descriptor->trbs[index].address = descriptor->buffer_addrs[0]; 746 descriptor->trbs[index].status = TRB_2_IRQ(0) 747 | TRB_2_BYTES(requestData->Length) 748 | TRB_2_TD_SIZE(0); 749 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_DATA_STAGE) 750 | (directionIn ? TRB_3_DIR_IN : 0) 751 | TRB_3_CYCLE_BIT; 752 753 if (!directionIn) { 754 transfer->PrepareKernelAccess(); 755 memcpy(descriptor->buffers[0], 756 (uint8 *)transfer->Vector()[0].iov_base, requestData->Length); 757 } 758 759 index++; 760 } 761 762 // Status Stage 763 descriptor->trbs[index].address = 0; 764 descriptor->trbs[index].status = TRB_2_IRQ(0); 765 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_STATUS_STAGE) 766 | ((directionIn && requestData->Length > 0) ? 0 : TRB_3_DIR_IN) 767 | TRB_3_CHAIN_BIT | TRB_3_ENT_BIT | TRB_3_CYCLE_BIT; 768 // Status Stage is an OUT transfer when the device is sending data 769 // (XHCI 1.2 § 4.11.2.2 Table 4-7 p213), and the CHAIN bit must be 770 // set when using an Event Data TRB (as _LinkDescriptorForPipe does) 771 // (XHCI 1.2 § 6.4.1.2.3 Table 6-31 p472) 772 773 descriptor->trb_used = index + 1; 774 775 status = _LinkDescriptorForPipe(descriptor, endpoint); 776 if (status != B_OK) { 777 FreeDescriptor(descriptor); 778 return status; 779 } 780 781 return B_OK; 782 } 783 784 785 status_t 786 XHCI::SubmitNormalRequest(Transfer *transfer) 787 { 788 TRACE("SubmitNormalRequest() length %ld\n", transfer->DataLength()); 789 790 Pipe *pipe = transfer->TransferPipe(); 791 usb_isochronous_data *isochronousData = transfer->IsochronousData(); 792 bool directionIn = (pipe->Direction() == Pipe::In); 793 794 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 795 if (endpoint == NULL) 796 return B_BAD_VALUE; 797 798 status_t status = transfer->InitKernelAccess(); 799 if (status != B_OK) 800 return status; 801 802 // TRBs within a TD must be "grouped" into TD Fragments, which mostly means 803 // that a max_burst_payload boundary cannot be crossed within a TRB, but 804 // only between TRBs. More than one TRB can be in a TD Fragment, but we keep 805 // things simple by setting trbSize to the MBP. (XHCI 1.2 § 4.11.7.1 p235.) 806 size_t trbSize = endpoint->max_burst_payload; 807 808 if (isochronousData != NULL) { 809 if (isochronousData->packet_count == 0) 810 return B_BAD_VALUE; 811 812 // Isochronous transfers use more specifically sized packets. 813 trbSize = transfer->DataLength() / isochronousData->packet_count; 814 if (trbSize == 0 || trbSize > pipe->MaxPacketSize() || trbSize 815 != (size_t)isochronousData->packet_descriptors[0].request_length) 816 return B_BAD_VALUE; 817 } 818 819 // Now that we know trbSize, compute the count. 820 const int32 trbCount = (transfer->DataLength() + trbSize - 1) / trbSize; 821 822 xhci_td *td = CreateDescriptor(trbCount, trbCount, trbSize); 823 if (td == NULL) 824 return B_NO_MEMORY; 825 826 // Normal Stage 827 const size_t maxPacketSize = pipe->MaxPacketSize(); 828 size_t remaining = transfer->DataLength(); 829 for (int32 i = 0; i < trbCount; i++) { 830 int32 trbLength = (remaining < trbSize) ? remaining : trbSize; 831 remaining -= trbLength; 832 833 // The "TD Size" field of a transfer TRB indicates the number of 834 // remaining maximum-size *packets* in this TD, *not* including the 835 // packets in the current TRB, and capped at 31 if there are more 836 // than 31 packets remaining in the TD. (XHCI 1.2 § 4.11.2.4 p218.) 837 int32 tdSize = (remaining + maxPacketSize - 1) / maxPacketSize; 838 if (tdSize > 31) 839 tdSize = 31; 840 841 td->trbs[i].address = td->buffer_addrs[i]; 842 td->trbs[i].status = TRB_2_IRQ(0) 843 | TRB_2_BYTES(trbLength) 844 | TRB_2_TD_SIZE(tdSize); 845 td->trbs[i].flags = TRB_3_TYPE(TRB_TYPE_NORMAL) 846 | TRB_3_CYCLE_BIT | TRB_3_CHAIN_BIT; 847 848 td->trb_used++; 849 } 850 851 // Isochronous-specific 852 if (isochronousData != NULL) { 853 // This is an isochronous transfer; we need to make the first TRB 854 // an isochronous TRB. 855 td->trbs[0].flags &= ~(TRB_3_TYPE(TRB_TYPE_NORMAL)); 856 td->trbs[0].flags |= TRB_3_TYPE(TRB_TYPE_ISOCH); 857 858 // Isochronous pipes are scheduled by microframes, one of which 859 // is 125us for USB 2 and above. But for USB 1 it was 1ms, so 860 // we need to use a different frame delta for that case. 861 uint8 frameDelta = 1; 862 if (transfer->TransferPipe()->Speed() == USB_SPEED_FULLSPEED) 863 frameDelta = 8; 864 865 // TODO: We do not currently take Mult into account at all! 866 // How are we supposed to do that here? 867 868 // Determine the (starting) frame number: if ISO_ASAP is set, 869 // we are queueing this "right away", and so want to reset 870 // the starting_frame_number. Otherwise we use the passed one. 871 uint32 frame; 872 if ((isochronousData->flags & USB_ISO_ASAP) != 0 873 || isochronousData->starting_frame_number == NULL) { 874 frame = ReadRunReg32(XHCI_MFINDEX) + 1; 875 // TODO: The +1 comes from the XHCI spec; document that. 876 td->trbs[0].flags |= TRB_3_ISO_SIA_BIT; 877 } else { 878 frame = *isochronousData->starting_frame_number; 879 td->trbs[0].flags |= TRB_3_FRID(frame); 880 } 881 frame = (frame + frameDelta) % 2048; 882 if (isochronousData->starting_frame_number != NULL) 883 *isochronousData->starting_frame_number = frame; 884 885 // TODO: The OHCI bus driver seems to also do this for inbound 886 // isochronous transfers. Perhaps it should be moved into the stack? 887 if (directionIn) { 888 for (uint32 i = 0; i < isochronousData->packet_count; i++) { 889 isochronousData->packet_descriptors[i].actual_length = 0; 890 isochronousData->packet_descriptors[i].status = B_NO_INIT; 891 } 892 } 893 } 894 895 // Set the ENT (Evaluate Next TRB) bit, so that the HC will not switch 896 // contexts before evaluating the Link TRB that _LinkDescriptorForPipe 897 // will insert, as otherwise there would be a race between us freeing 898 // and unlinking the descriptor, and the controller evaluating the Link TRB 899 // and thus getting back onto the main ring and executing the Event Data 900 // TRB that generates the interrupt for this transfer. 901 // 902 // Note that we *do not* unset the CHAIN bit in this TRB, thus including 903 // the Link TRB in this TD formally, which is required when using the 904 // ENT bit. (XHCI 1.2 § 4.12.3 p250.) 905 td->trbs[td->trb_used - 1].flags |= TRB_3_ENT_BIT; 906 907 if (!directionIn) { 908 TRACE("copying out iov count %ld\n", transfer->VectorCount()); 909 status_t status = transfer->PrepareKernelAccess(); 910 if (status != B_OK) { 911 FreeDescriptor(td); 912 return status; 913 } 914 WriteDescriptor(td, transfer->Vector(), transfer->VectorCount()); 915 } 916 917 td->transfer = transfer; 918 status = _LinkDescriptorForPipe(td, endpoint); 919 if (status != B_OK) { 920 FreeDescriptor(td); 921 return status; 922 } 923 924 return B_OK; 925 } 926 927 928 status_t 929 XHCI::CancelQueuedTransfers(Pipe *pipe, bool force) 930 { 931 xhci_endpoint* endpoint = (xhci_endpoint*)pipe->ControllerCookie(); 932 if (endpoint == NULL || endpoint->trbs == NULL) { 933 // Someone's de-allocated this pipe or endpoint in the meantime. 934 // (Possibly AllocateDevice failed, and we were the temporary pipe.) 935 return B_NO_INIT; 936 } 937 938 TRACE_ALWAYS("cancel queued transfers (%" B_PRId8 ") for pipe %p (%d)\n", 939 endpoint->used, pipe, pipe->EndpointAddress()); 940 941 MutexLocker endpointLocker(endpoint->lock); 942 943 if (endpoint->td_head == NULL) { 944 // There aren't any currently pending transfers to cancel. 945 return B_OK; 946 } 947 948 // Calling the callbacks while holding the endpoint lock could potentially 949 // cause deadlocks, so we instead store them in a pointer array. We need 950 // to do this separately from freeing the TDs, for in the case we fail 951 // to stop the endpoint, we cancel the transfers but do not free the TDs. 952 Transfer* transfers[XHCI_MAX_TRANSFERS]; 953 int32 transfersCount = 0; 954 955 for (xhci_td* td = endpoint->td_head; td != NULL; td = td->next) { 956 if (td->transfer == NULL) 957 continue; 958 959 // We can't cancel or delete transfers under "force", as they probably 960 // are not safe to use anymore. 961 if (!force) { 962 transfers[transfersCount] = td->transfer; 963 transfersCount++; 964 } 965 td->transfer = NULL; 966 } 967 968 // It is possible that while waiting for the stop-endpoint command to 969 // complete, one of the queued transfers posts a completion event, so in 970 // order to avoid a deadlock, we must unlock the endpoint. 971 endpointLocker.Unlock(); 972 status_t status = StopEndpoint(false, endpoint); 973 if (status == B_NOT_ALLOWED) { 974 // XHCI 1.2, 4.8.3 Endpoint State Diagram 975 // Only exit from a HALTED state is a reset 976 TRACE_ERROR("cancel queued transfers: halted endpoint. reset!"); 977 status = ResetEndpoint(false, endpoint); 978 } 979 endpointLocker.Lock(); 980 981 // Detach the head TD from the endpoint. 982 xhci_td* td_head = endpoint->td_head; 983 endpoint->td_head = NULL; 984 985 if (status == B_OK) { 986 // Clear the endpoint's TRBs. 987 memset(endpoint->trbs, 0, sizeof(xhci_trb) * XHCI_ENDPOINT_RING_SIZE); 988 endpoint->used = 0; 989 endpoint->current = 0; 990 991 // Set dequeue pointer location to the beginning of the ring. 992 SetTRDequeue(endpoint->trb_addr, 0, endpoint->id + 1, 993 endpoint->device->slot); 994 995 // We don't need to do anything else to restart the ring, as it will resume 996 // operation as normal upon the next doorbell. (XHCI 1.2 § 4.6.9 p136.) 997 } else { 998 // We couldn't stop the endpoint. Most likely the device has been 999 // removed and the endpoint was stopped by the hardware, or is 1000 // for some reason busy and cannot be stopped. 1001 TRACE_ERROR("cancel queued transfers: could not stop endpoint: %s!\n", 1002 strerror(status)); 1003 1004 // Instead of freeing the TDs, we want to leave them in the endpoint 1005 // so that when/if the hardware returns, they can be properly unlinked, 1006 // as otherwise the endpoint could get "stuck" by having the "used" 1007 // slowly accumulate due to "dead" transfers. 1008 endpoint->td_head = td_head; 1009 td_head = NULL; 1010 } 1011 1012 endpointLocker.Unlock(); 1013 1014 for (int32 i = 0; i < transfersCount; i++) { 1015 transfers[i]->Finished(B_CANCELED, 0); 1016 delete transfers[i]; 1017 } 1018 1019 // This loop looks a bit strange because we need to store the "next" 1020 // pointer before freeing the descriptor. 1021 xhci_td* td; 1022 while ((td = td_head) != NULL) { 1023 td_head = td_head->next; 1024 FreeDescriptor(td); 1025 } 1026 1027 return B_OK; 1028 } 1029 1030 1031 status_t 1032 XHCI::StartDebugTransfer(Transfer *transfer) 1033 { 1034 Pipe *pipe = transfer->TransferPipe(); 1035 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 1036 if (endpoint == NULL) 1037 return B_BAD_VALUE; 1038 1039 // Check all locks that we are going to hit when running transfers. 1040 if (mutex_trylock(&endpoint->lock) != B_OK) 1041 return B_WOULD_BLOCK; 1042 if (mutex_trylock(&fFinishedLock) != B_OK) { 1043 mutex_unlock(&endpoint->lock); 1044 return B_WOULD_BLOCK; 1045 } 1046 if (mutex_trylock(&fEventLock) != B_OK) { 1047 mutex_unlock(&endpoint->lock); 1048 mutex_unlock(&fFinishedLock); 1049 return B_WOULD_BLOCK; 1050 } 1051 mutex_unlock(&endpoint->lock); 1052 mutex_unlock(&fFinishedLock); 1053 mutex_unlock(&fEventLock); 1054 1055 status_t status = SubmitTransfer(transfer); 1056 if (status != B_OK) 1057 return status; 1058 1059 // The endpoint's head TD is the TD of the just-submitted transfer. 1060 // Just like EHCI, abuse the callback cookie to hold the TD pointer. 1061 transfer->SetCallback(NULL, endpoint->td_head); 1062 1063 return B_OK; 1064 } 1065 1066 1067 status_t 1068 XHCI::CheckDebugTransfer(Transfer *transfer) 1069 { 1070 xhci_td *transfer_td = (xhci_td *)transfer->CallbackCookie(); 1071 if (transfer_td == NULL) 1072 return B_NO_INIT; 1073 1074 // Process events once, and then look for it in the finished list. 1075 ProcessEvents(); 1076 xhci_td *previous = NULL; 1077 for (xhci_td *td = fFinishedHead; td != NULL; td = td->next) { 1078 if (td != transfer_td) { 1079 previous = td; 1080 continue; 1081 } 1082 1083 // We've found it! 1084 if (previous == NULL) { 1085 fFinishedHead = fFinishedHead->next; 1086 } else { 1087 previous->next = td->next; 1088 } 1089 1090 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 1091 status_t status = (td->trb_completion_code == COMP_SUCCESS 1092 || td->trb_completion_code == COMP_SHORT_PACKET) ? B_OK : B_ERROR; 1093 1094 if (status == B_OK && directionIn) 1095 ReadDescriptor(td, transfer->Vector(), transfer->VectorCount()); 1096 1097 FreeDescriptor(td); 1098 transfer->SetCallback(NULL, NULL); 1099 return status; 1100 } 1101 1102 // We didn't find it. 1103 spin(75); 1104 return B_DEV_PENDING; 1105 } 1106 1107 1108 void 1109 XHCI::CancelDebugTransfer(Transfer *transfer) 1110 { 1111 while (CheckDebugTransfer(transfer) == B_DEV_PENDING) 1112 spin(100); 1113 } 1114 1115 1116 status_t 1117 XHCI::NotifyPipeChange(Pipe *pipe, usb_change change) 1118 { 1119 TRACE("pipe change %d for pipe %p (%d)\n", change, pipe, 1120 pipe->EndpointAddress()); 1121 1122 switch (change) { 1123 case USB_CHANGE_CREATED: 1124 return _InsertEndpointForPipe(pipe); 1125 case USB_CHANGE_DESTROYED: 1126 return _RemoveEndpointForPipe(pipe); 1127 1128 case USB_CHANGE_PIPE_POLICY_CHANGED: 1129 // We don't care about these, at least for now. 1130 return B_OK; 1131 } 1132 1133 TRACE_ERROR("unknown pipe change!\n"); 1134 return B_UNSUPPORTED; 1135 } 1136 1137 1138 xhci_td * 1139 XHCI::CreateDescriptor(uint32 trbCount, uint32 bufferCount, size_t bufferSize) 1140 { 1141 const bool inKDL = debug_debugger_running(); 1142 1143 xhci_td *result; 1144 if (!inKDL) { 1145 result = (xhci_td*)calloc(1, sizeof(xhci_td)); 1146 } else { 1147 // Just use the physical memory allocator while in KDL; it's less 1148 // secure than using the regular heap, but it's easier to deal with. 1149 phys_addr_t dummy; 1150 fStack->AllocateChunk((void **)&result, &dummy, sizeof(xhci_td)); 1151 } 1152 1153 if (result == NULL) { 1154 TRACE_ERROR("failed to allocate a transfer descriptor\n"); 1155 return NULL; 1156 } 1157 1158 // We always allocate 1 more TRB than requested, so that 1159 // _LinkDescriptorForPipe() has room to insert a link TRB. 1160 trbCount++; 1161 if (fStack->AllocateChunk((void **)&result->trbs, &result->trb_addr, 1162 (trbCount * sizeof(xhci_trb))) < B_OK) { 1163 TRACE_ERROR("failed to allocate TRBs\n"); 1164 FreeDescriptor(result); 1165 return NULL; 1166 } 1167 result->trb_count = trbCount; 1168 result->trb_used = 0; 1169 1170 if (bufferSize > 0) { 1171 // Due to how the USB stack allocates physical memory, we can't just 1172 // request one large chunk the size of the transfer, and so instead we 1173 // create a series of buffers as requested by our caller. 1174 1175 // We store the buffer pointers and addresses in one memory block. 1176 if (!inKDL) { 1177 result->buffers = (void**)calloc(bufferCount, 1178 (sizeof(void*) + sizeof(phys_addr_t))); 1179 } else { 1180 phys_addr_t dummy; 1181 fStack->AllocateChunk((void **)&result->buffers, &dummy, 1182 bufferCount * (sizeof(void*) + sizeof(phys_addr_t))); 1183 } 1184 if (result->buffers == NULL) { 1185 TRACE_ERROR("unable to allocate space for buffer infos\n"); 1186 FreeDescriptor(result); 1187 return NULL; 1188 } 1189 result->buffer_addrs = (phys_addr_t*)&result->buffers[bufferCount]; 1190 result->buffer_size = bufferSize; 1191 result->buffer_count = bufferCount; 1192 1193 // Optimization: If the requested total size of all buffers is less 1194 // than 32*B_PAGE_SIZE (the maximum size that the physical memory 1195 // allocator can handle), we allocate only one buffer and segment it. 1196 size_t totalSize = bufferSize * bufferCount; 1197 if (totalSize < (32 * B_PAGE_SIZE)) { 1198 if (fStack->AllocateChunk(&result->buffers[0], 1199 &result->buffer_addrs[0], totalSize) < B_OK) { 1200 TRACE_ERROR("unable to allocate space for large buffer (size %ld)\n", 1201 totalSize); 1202 FreeDescriptor(result); 1203 return NULL; 1204 } 1205 for (uint32 i = 1; i < bufferCount; i++) { 1206 result->buffers[i] = (void*)((addr_t)(result->buffers[i - 1]) 1207 + bufferSize); 1208 result->buffer_addrs[i] = result->buffer_addrs[i - 1] 1209 + bufferSize; 1210 } 1211 } else { 1212 // Otherwise, we allocate each buffer individually. 1213 for (uint32 i = 0; i < bufferCount; i++) { 1214 if (fStack->AllocateChunk(&result->buffers[i], 1215 &result->buffer_addrs[i], bufferSize) < B_OK) { 1216 TRACE_ERROR("unable to allocate space for a buffer (size " 1217 "%" B_PRIuSIZE ", count %" B_PRIu32 ")\n", 1218 bufferSize, bufferCount); 1219 FreeDescriptor(result); 1220 return NULL; 1221 } 1222 } 1223 } 1224 } else { 1225 result->buffers = NULL; 1226 result->buffer_addrs = NULL; 1227 } 1228 1229 // Initialize all other fields. 1230 result->transfer = NULL; 1231 result->trb_completion_code = 0; 1232 result->trb_left = 0; 1233 result->next = NULL; 1234 1235 TRACE("CreateDescriptor allocated %p, buffer_size %ld, buffer_count %" B_PRIu32 "\n", 1236 result, result->buffer_size, result->buffer_count); 1237 1238 return result; 1239 } 1240 1241 1242 void 1243 XHCI::FreeDescriptor(xhci_td *descriptor) 1244 { 1245 if (descriptor == NULL) 1246 return; 1247 1248 const bool inKDL = debug_debugger_running(); 1249 1250 if (descriptor->trbs != NULL) { 1251 fStack->FreeChunk(descriptor->trbs, descriptor->trb_addr, 1252 (descriptor->trb_count * sizeof(xhci_trb))); 1253 } 1254 if (descriptor->buffers != NULL) { 1255 size_t totalSize = descriptor->buffer_size * descriptor->buffer_count; 1256 if (totalSize < (32 * B_PAGE_SIZE)) { 1257 // This was allocated as one contiguous buffer. 1258 fStack->FreeChunk(descriptor->buffers[0], descriptor->buffer_addrs[0], 1259 totalSize); 1260 } else { 1261 for (uint32 i = 0; i < descriptor->buffer_count; i++) { 1262 if (descriptor->buffers[i] == NULL) 1263 continue; 1264 fStack->FreeChunk(descriptor->buffers[i], descriptor->buffer_addrs[i], 1265 descriptor->buffer_size); 1266 } 1267 } 1268 1269 if (!inKDL) { 1270 free(descriptor->buffers); 1271 } else { 1272 fStack->FreeChunk(descriptor->buffers, 0, 1273 descriptor->buffer_count * (sizeof(void*) + sizeof(phys_addr_t))); 1274 } 1275 } 1276 1277 if (!inKDL) 1278 free(descriptor); 1279 else 1280 fStack->FreeChunk(descriptor, 0, sizeof(xhci_td)); 1281 } 1282 1283 1284 size_t 1285 XHCI::WriteDescriptor(xhci_td *descriptor, iovec *vector, size_t vectorCount) 1286 { 1287 size_t written = 0; 1288 1289 size_t bufIdx = 0, bufUsed = 0; 1290 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1291 size_t length = vector[vecIdx].iov_len; 1292 1293 while (length > 0 && bufIdx < descriptor->buffer_count) { 1294 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1295 memcpy((uint8 *)descriptor->buffers[bufIdx] + bufUsed, 1296 (uint8 *)vector[vecIdx].iov_base + (vector[vecIdx].iov_len - length), 1297 toCopy); 1298 1299 written += toCopy; 1300 bufUsed += toCopy; 1301 length -= toCopy; 1302 if (bufUsed == descriptor->buffer_size) { 1303 bufIdx++; 1304 bufUsed = 0; 1305 } 1306 } 1307 } 1308 1309 TRACE("wrote descriptor (%" B_PRIuSIZE " bytes)\n", written); 1310 return written; 1311 } 1312 1313 1314 size_t 1315 XHCI::ReadDescriptor(xhci_td *descriptor, iovec *vector, size_t vectorCount) 1316 { 1317 size_t read = 0; 1318 1319 size_t bufIdx = 0, bufUsed = 0; 1320 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1321 size_t length = vector[vecIdx].iov_len; 1322 1323 while (length > 0 && bufIdx < descriptor->buffer_count) { 1324 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1325 memcpy((uint8 *)vector[vecIdx].iov_base + (vector[vecIdx].iov_len - length), 1326 (uint8 *)descriptor->buffers[bufIdx] + bufUsed, toCopy); 1327 1328 read += toCopy; 1329 bufUsed += toCopy; 1330 length -= toCopy; 1331 if (bufUsed == descriptor->buffer_size) { 1332 bufIdx++; 1333 bufUsed = 0; 1334 } 1335 } 1336 } 1337 1338 TRACE("read descriptor (%" B_PRIuSIZE " bytes)\n", read); 1339 return read; 1340 } 1341 1342 1343 Device * 1344 XHCI::AllocateDevice(Hub *parent, int8 hubAddress, uint8 hubPort, 1345 usb_speed speed) 1346 { 1347 TRACE("AllocateDevice hubAddress %d hubPort %d speed %d\n", hubAddress, 1348 hubPort, speed); 1349 1350 uint8 slot = XHCI_MAX_SLOTS; 1351 if (EnableSlot(&slot) != B_OK) { 1352 TRACE_ERROR("AllocateDevice() failed enable slot\n"); 1353 return NULL; 1354 } 1355 1356 if (slot == 0 || slot > fSlotCount) { 1357 TRACE_ERROR("AllocateDevice() bad slot\n"); 1358 return NULL; 1359 } 1360 1361 if (fDevices[slot].slot != 0) { 1362 TRACE_ERROR("AllocateDevice() slot already used\n"); 1363 return NULL; 1364 } 1365 1366 struct xhci_device *device = &fDevices[slot]; 1367 memset(device, 0, sizeof(struct xhci_device)); 1368 device->slot = slot; 1369 1370 device->input_ctx_area = fStack->AllocateArea((void **)&device->input_ctx, 1371 &device->input_ctx_addr, sizeof(*device->input_ctx) << fContextSizeShift, 1372 "XHCI input context"); 1373 if (device->input_ctx_area < B_OK) { 1374 TRACE_ERROR("unable to create a input context area\n"); 1375 return NULL; 1376 } 1377 1378 memset(device->input_ctx, 0, sizeof(*device->input_ctx) << fContextSizeShift); 1379 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1380 _WriteContext(&device->input_ctx->input.addFlags, 3); 1381 1382 uint32 route = 0; 1383 uint8 routePort = hubPort; 1384 uint8 rhPort = hubPort; 1385 for (Device *hubDevice = parent; hubDevice != RootObject(); 1386 hubDevice = (Device *)hubDevice->Parent()) { 1387 1388 rhPort = routePort; 1389 if (hubDevice->Parent() == RootObject()) 1390 break; 1391 route *= 16; 1392 if (hubPort > 15) 1393 route += 15; 1394 else 1395 route += routePort; 1396 1397 routePort = hubDevice->HubPort(); 1398 } 1399 1400 // Get speed of port, only if device connected to root hub port 1401 // else we have to rely on value reported by the Hub Explore thread 1402 if (route == 0) { 1403 GetPortSpeed(hubPort - 1, &speed); 1404 TRACE("speed updated %d\n", speed); 1405 } 1406 1407 uint32 dwslot0 = SLOT_0_NUM_ENTRIES(1) | SLOT_0_ROUTE(route); 1408 1409 // add the speed 1410 switch (speed) { 1411 case USB_SPEED_LOWSPEED: 1412 dwslot0 |= SLOT_0_SPEED(2); 1413 break; 1414 case USB_SPEED_HIGHSPEED: 1415 dwslot0 |= SLOT_0_SPEED(3); 1416 break; 1417 case USB_SPEED_FULLSPEED: 1418 dwslot0 |= SLOT_0_SPEED(1); 1419 break; 1420 case USB_SPEED_SUPERSPEED: 1421 dwslot0 |= SLOT_0_SPEED(4); 1422 break; 1423 default: 1424 TRACE_ERROR("unknown usb speed\n"); 1425 break; 1426 } 1427 1428 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1429 // TODO enable power save 1430 _WriteContext(&device->input_ctx->slot.dwslot1, SLOT_1_RH_PORT(rhPort)); 1431 uint32 dwslot2 = SLOT_2_IRQ_TARGET(0); 1432 1433 // If LS/FS device connected to non-root HS device 1434 if (route != 0 && parent->Speed() == USB_SPEED_HIGHSPEED 1435 && (speed == USB_SPEED_LOWSPEED || speed == USB_SPEED_FULLSPEED)) { 1436 struct xhci_device *parenthub = (struct xhci_device *) 1437 parent->ControllerCookie(); 1438 dwslot2 |= SLOT_2_PORT_NUM(hubPort); 1439 dwslot2 |= SLOT_2_TT_HUB_SLOT(parenthub->slot); 1440 } 1441 1442 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1443 1444 _WriteContext(&device->input_ctx->slot.dwslot3, SLOT_3_SLOT_STATE(0) 1445 | SLOT_3_DEVICE_ADDRESS(0)); 1446 1447 TRACE("slot 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 1448 "\n", _ReadContext(&device->input_ctx->slot.dwslot0), 1449 _ReadContext(&device->input_ctx->slot.dwslot1), 1450 _ReadContext(&device->input_ctx->slot.dwslot2), 1451 _ReadContext(&device->input_ctx->slot.dwslot3)); 1452 1453 device->device_ctx_area = fStack->AllocateArea((void **)&device->device_ctx, 1454 &device->device_ctx_addr, sizeof(*device->device_ctx) << fContextSizeShift, 1455 "XHCI device context"); 1456 if (device->device_ctx_area < B_OK) { 1457 TRACE_ERROR("unable to create a device context area\n"); 1458 delete_area(device->input_ctx_area); 1459 memset(device, 0, sizeof(xhci_device)); 1460 return NULL; 1461 } 1462 memset(device->device_ctx, 0, sizeof(*device->device_ctx) << fContextSizeShift); 1463 1464 device->trb_area = fStack->AllocateArea((void **)&device->trbs, 1465 &device->trb_addr, sizeof(xhci_trb) * (XHCI_MAX_ENDPOINTS - 1) 1466 * XHCI_ENDPOINT_RING_SIZE, "XHCI endpoint trbs"); 1467 if (device->trb_area < B_OK) { 1468 TRACE_ERROR("unable to create a device trbs area\n"); 1469 delete_area(device->input_ctx_area); 1470 delete_area(device->device_ctx_area); 1471 memset(device, 0, sizeof(xhci_device)); 1472 return NULL; 1473 } 1474 1475 // set up slot pointer to device context 1476 fDcba->baseAddress[slot] = device->device_ctx_addr; 1477 1478 size_t maxPacketSize; 1479 switch (speed) { 1480 case USB_SPEED_LOWSPEED: 1481 case USB_SPEED_FULLSPEED: 1482 maxPacketSize = 8; 1483 break; 1484 case USB_SPEED_HIGHSPEED: 1485 maxPacketSize = 64; 1486 break; 1487 default: 1488 maxPacketSize = 512; 1489 break; 1490 } 1491 1492 xhci_endpoint* endpoint0 = &device->endpoints[0]; 1493 mutex_init(&endpoint0->lock, "xhci endpoint lock"); 1494 endpoint0->device = device; 1495 endpoint0->id = 0; 1496 endpoint0->td_head = NULL; 1497 endpoint0->used = 0; 1498 endpoint0->current = 0; 1499 endpoint0->trbs = device->trbs; 1500 endpoint0->trb_addr = device->trb_addr; 1501 1502 // configure the Control endpoint 0 1503 if (ConfigureEndpoint(endpoint0, slot, 0, USB_OBJECT_CONTROL_PIPE, false, 1504 0, maxPacketSize, speed, 0, 0) != B_OK) { 1505 TRACE_ERROR("unable to configure default control endpoint\n"); 1506 delete_area(device->input_ctx_area); 1507 delete_area(device->device_ctx_area); 1508 delete_area(device->trb_area); 1509 memset(device, 0, sizeof(xhci_device)); 1510 return NULL; 1511 } 1512 1513 // device should get to addressed state (bsr = 0) 1514 if (SetAddress(device->input_ctx_addr, false, slot) != B_OK) { 1515 TRACE_ERROR("unable to set address\n"); 1516 delete_area(device->input_ctx_area); 1517 delete_area(device->device_ctx_area); 1518 delete_area(device->trb_area); 1519 memset(device, 0, sizeof(xhci_device)); 1520 return NULL; 1521 } 1522 1523 device->address = SLOT_3_DEVICE_ADDRESS_GET(_ReadContext( 1524 &device->device_ctx->slot.dwslot3)); 1525 1526 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", device->address, 1527 SLOT_3_SLOT_STATE_GET(_ReadContext( 1528 &device->device_ctx->slot.dwslot3))); 1529 TRACE("endpoint0 state 0x%08" B_PRIx32 "\n", 1530 ENDPOINT_0_STATE_GET(_ReadContext( 1531 &device->device_ctx->endpoints[0].dwendpoint0))); 1532 1533 // Create a temporary pipe with the new address 1534 ControlPipe pipe(parent); 1535 pipe.SetControllerCookie(endpoint0); 1536 pipe.InitCommon(device->address + 1, 0, speed, Pipe::Default, maxPacketSize, 0, 1537 hubAddress, hubPort); 1538 1539 // Get the device descriptor 1540 // Just retrieve the first 8 bytes of the descriptor -> minimum supported 1541 // size of any device. It is enough because it includes the device type. 1542 1543 size_t actualLength = 0; 1544 usb_device_descriptor deviceDescriptor; 1545 1546 TRACE("getting the device descriptor\n"); 1547 status_t status = pipe.SendRequest( 1548 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_STANDARD, // type 1549 USB_REQUEST_GET_DESCRIPTOR, // request 1550 USB_DESCRIPTOR_DEVICE << 8, // value 1551 0, // index 1552 8, // length 1553 (void *)&deviceDescriptor, // buffer 1554 8, // buffer length 1555 &actualLength); // actual length 1556 1557 if (actualLength != 8) { 1558 TRACE_ERROR("failed to get the device descriptor: %s\n", 1559 strerror(status)); 1560 delete_area(device->input_ctx_area); 1561 delete_area(device->device_ctx_area); 1562 delete_area(device->trb_area); 1563 memset(device, 0, sizeof(xhci_device)); 1564 return NULL; 1565 } 1566 1567 TRACE("device_class: %d device_subclass %d device_protocol %d\n", 1568 deviceDescriptor.device_class, deviceDescriptor.device_subclass, 1569 deviceDescriptor.device_protocol); 1570 1571 if (speed == USB_SPEED_FULLSPEED && deviceDescriptor.max_packet_size_0 != 8) { 1572 TRACE("Full speed device with different max packet size for Endpoint 0\n"); 1573 uint32 dwendpoint1 = _ReadContext( 1574 &device->input_ctx->endpoints[0].dwendpoint1); 1575 dwendpoint1 &= ~ENDPOINT_1_MAXPACKETSIZE(0xffff); 1576 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE( 1577 deviceDescriptor.max_packet_size_0); 1578 _WriteContext(&device->input_ctx->endpoints[0].dwendpoint1, 1579 dwendpoint1); 1580 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1581 _WriteContext(&device->input_ctx->input.addFlags, (1 << 1)); 1582 EvaluateContext(device->input_ctx_addr, device->slot); 1583 } 1584 1585 Device *deviceObject = NULL; 1586 if (deviceDescriptor.device_class == 0x09) { 1587 TRACE("creating new Hub\n"); 1588 TRACE("getting the hub descriptor\n"); 1589 size_t actualLength = 0; 1590 usb_hub_descriptor hubDescriptor; 1591 status = pipe.SendRequest( 1592 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_CLASS, // type 1593 USB_REQUEST_GET_DESCRIPTOR, // request 1594 USB_DESCRIPTOR_HUB << 8, // value 1595 0, // index 1596 sizeof(usb_hub_descriptor), // length 1597 (void *)&hubDescriptor, // buffer 1598 sizeof(usb_hub_descriptor), // buffer length 1599 &actualLength); 1600 1601 if (actualLength != sizeof(usb_hub_descriptor)) { 1602 TRACE_ERROR("error while getting the hub descriptor: %s\n", 1603 strerror(status)); 1604 delete_area(device->input_ctx_area); 1605 delete_area(device->device_ctx_area); 1606 delete_area(device->trb_area); 1607 memset(device, 0, sizeof(xhci_device)); 1608 return NULL; 1609 } 1610 1611 uint32 dwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1612 dwslot0 |= SLOT_0_HUB_BIT; 1613 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1614 uint32 dwslot1 = _ReadContext(&device->input_ctx->slot.dwslot1); 1615 dwslot1 |= SLOT_1_NUM_PORTS(hubDescriptor.num_ports); 1616 _WriteContext(&device->input_ctx->slot.dwslot1, dwslot1); 1617 if (speed == USB_SPEED_HIGHSPEED) { 1618 uint32 dwslot2 = _ReadContext(&device->input_ctx->slot.dwslot2); 1619 dwslot2 |= SLOT_2_TT_TIME(HUB_TTT_GET(hubDescriptor.characteristics)); 1620 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1621 } 1622 1623 deviceObject = new(std::nothrow) Hub(parent, hubAddress, hubPort, 1624 deviceDescriptor, device->address + 1, speed, false, device); 1625 } else { 1626 TRACE("creating new device\n"); 1627 deviceObject = new(std::nothrow) Device(parent, hubAddress, hubPort, 1628 deviceDescriptor, device->address + 1, speed, false, device); 1629 } 1630 if (deviceObject == NULL || deviceObject->InitCheck() != B_OK) { 1631 if (deviceObject == NULL) { 1632 TRACE_ERROR("no memory to allocate device\n"); 1633 } else { 1634 TRACE_ERROR("device object failed to initialize\n"); 1635 } 1636 delete_area(device->input_ctx_area); 1637 delete_area(device->device_ctx_area); 1638 delete_area(device->trb_area); 1639 memset(device, 0, sizeof(xhci_device)); 1640 return NULL; 1641 } 1642 1643 // We don't want to disable the default endpoint, naturally, which would 1644 // otherwise happen when this Pipe object is destroyed. 1645 pipe.SetControllerCookie(NULL); 1646 1647 fPortSlots[hubPort] = slot; 1648 TRACE("AllocateDevice() port %d slot %d\n", hubPort, slot); 1649 return deviceObject; 1650 } 1651 1652 1653 void 1654 XHCI::FreeDevice(Device *device) 1655 { 1656 uint8 hubPort = device->HubPort(); 1657 uint8 slot = fPortSlots[hubPort]; 1658 TRACE("FreeDevice() port %d slot %d\n", hubPort, slot); 1659 1660 // Delete the device first, so it cleans up its pipes and tells us 1661 // what we need to destroy before we tear down our internal state. 1662 delete device; 1663 1664 DisableSlot(slot); 1665 fDcba->baseAddress[slot] = 0; 1666 fPortSlots[hubPort] = 0; 1667 delete_area(fDevices[slot].trb_area); 1668 delete_area(fDevices[slot].input_ctx_area); 1669 delete_area(fDevices[slot].device_ctx_area); 1670 1671 memset(&fDevices[slot], 0, sizeof(xhci_device)); 1672 } 1673 1674 1675 uint8 1676 XHCI::_GetEndpointState(xhci_endpoint* endpoint) 1677 { 1678 struct xhci_device_ctx* device_ctx = endpoint->device->device_ctx; 1679 return ENDPOINT_0_STATE_GET( 1680 _ReadContext(&device_ctx->endpoints[endpoint->id].dwendpoint0)); 1681 1682 } 1683 1684 1685 status_t 1686 XHCI::_InsertEndpointForPipe(Pipe *pipe) 1687 { 1688 TRACE("insert endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1689 1690 if (pipe->ControllerCookie() != NULL 1691 || pipe->Parent()->Type() != USB_OBJECT_DEVICE) { 1692 // default pipe is already referenced 1693 return B_OK; 1694 } 1695 1696 Device* usbDevice = (Device *)pipe->Parent(); 1697 struct xhci_device *device = (struct xhci_device *) 1698 usbDevice->ControllerCookie(); 1699 if (usbDevice->Parent() == RootObject()) 1700 return B_OK; 1701 if (device == NULL) { 1702 panic("device is NULL\n"); 1703 return B_NO_INIT; 1704 } 1705 1706 const uint8 id = (2 * pipe->EndpointAddress() 1707 + (pipe->Direction() != Pipe::Out ? 1 : 0)) - 1; 1708 if (id >= XHCI_MAX_ENDPOINTS - 1) 1709 return B_BAD_VALUE; 1710 1711 if (id > 0) { 1712 uint32 devicedwslot0 = _ReadContext(&device->device_ctx->slot.dwslot0); 1713 if (SLOT_0_NUM_ENTRIES_GET(devicedwslot0) == 1) { 1714 uint32 inputdwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1715 inputdwslot0 &= ~(SLOT_0_NUM_ENTRIES(0x1f)); 1716 inputdwslot0 |= SLOT_0_NUM_ENTRIES(XHCI_MAX_ENDPOINTS - 1); 1717 _WriteContext(&device->input_ctx->slot.dwslot0, inputdwslot0); 1718 EvaluateContext(device->input_ctx_addr, device->slot); 1719 } 1720 1721 xhci_endpoint* endpoint = &device->endpoints[id]; 1722 mutex_init(&endpoint->lock, "xhci endpoint lock"); 1723 MutexLocker endpointLocker(endpoint->lock); 1724 1725 endpoint->device = device; 1726 endpoint->id = id; 1727 endpoint->td_head = NULL; 1728 endpoint->used = 0; 1729 endpoint->current = 0; 1730 1731 endpoint->trbs = device->trbs + id * XHCI_ENDPOINT_RING_SIZE; 1732 endpoint->trb_addr = device->trb_addr 1733 + id * XHCI_ENDPOINT_RING_SIZE * sizeof(xhci_trb); 1734 memset(endpoint->trbs, 0, 1735 sizeof(xhci_trb) * XHCI_ENDPOINT_RING_SIZE); 1736 1737 TRACE("insert endpoint for pipe: trbs, device %p endpoint %p\n", 1738 device->trbs, endpoint->trbs); 1739 TRACE("insert endpoint for pipe: trb_addr, device 0x%" B_PRIxPHYSADDR 1740 " endpoint 0x%" B_PRIxPHYSADDR "\n", device->trb_addr, 1741 endpoint->trb_addr); 1742 1743 const uint8 endpointNum = id + 1; 1744 1745 status_t status = ConfigureEndpoint(endpoint, device->slot, id, pipe->Type(), 1746 pipe->Direction() == Pipe::In, pipe->Interval(), pipe->MaxPacketSize(), 1747 usbDevice->Speed(), pipe->MaxBurst(), pipe->BytesPerInterval()); 1748 if (status != B_OK) { 1749 TRACE_ERROR("unable to configure endpoint: %s\n", strerror(status)); 1750 return status; 1751 } 1752 1753 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1754 _WriteContext(&device->input_ctx->input.addFlags, 1755 (1 << endpointNum) | (1 << 0)); 1756 1757 ConfigureEndpoint(device->input_ctx_addr, false, device->slot); 1758 1759 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", 1760 device->address, SLOT_3_SLOT_STATE_GET(_ReadContext( 1761 &device->device_ctx->slot.dwslot3))); 1762 TRACE("endpoint[0] state 0x%08" B_PRIx32 "\n", 1763 ENDPOINT_0_STATE_GET(_ReadContext( 1764 &device->device_ctx->endpoints[0].dwendpoint0))); 1765 TRACE("endpoint[%d] state 0x%08" B_PRIx32 "\n", id, 1766 ENDPOINT_0_STATE_GET(_ReadContext( 1767 &device->device_ctx->endpoints[id].dwendpoint0))); 1768 } 1769 pipe->SetControllerCookie(&device->endpoints[id]); 1770 1771 return B_OK; 1772 } 1773 1774 1775 status_t 1776 XHCI::_RemoveEndpointForPipe(Pipe *pipe) 1777 { 1778 TRACE("remove endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1779 1780 if (pipe->Parent()->Type() != USB_OBJECT_DEVICE) 1781 return B_OK; 1782 Device* usbDevice = (Device *)pipe->Parent(); 1783 if (usbDevice->Parent() == RootObject()) 1784 return B_BAD_VALUE; 1785 1786 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 1787 if (endpoint == NULL || endpoint->trbs == NULL) 1788 return B_NO_INIT; 1789 1790 pipe->SetControllerCookie(NULL); 1791 1792 if (endpoint->id > 0) { 1793 xhci_device *device = endpoint->device; 1794 uint8 epNumber = endpoint->id + 1; 1795 StopEndpoint(true, endpoint); 1796 1797 mutex_lock(&endpoint->lock); 1798 1799 // See comment in CancelQueuedTransfers. 1800 xhci_td* td; 1801 while ((td = endpoint->td_head) != NULL) { 1802 endpoint->td_head = endpoint->td_head->next; 1803 FreeDescriptor(td); 1804 } 1805 1806 mutex_destroy(&endpoint->lock); 1807 memset(endpoint, 0, sizeof(xhci_endpoint)); 1808 1809 _WriteContext(&device->input_ctx->input.dropFlags, (1 << epNumber)); 1810 _WriteContext(&device->input_ctx->input.addFlags, 0); 1811 1812 if (epNumber > 1) 1813 ConfigureEndpoint(device->input_ctx_addr, true, device->slot); 1814 else 1815 EvaluateContext(device->input_ctx_addr, device->slot); 1816 } 1817 1818 return B_OK; 1819 } 1820 1821 1822 status_t 1823 XHCI::_LinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 1824 { 1825 TRACE("link descriptor for pipe\n"); 1826 1827 // We must check this before we lock the endpoint, because if it is 1828 // NULL, the mutex is probably uninitialized, too. 1829 if (endpoint->device == NULL) { 1830 TRACE_ERROR("trying to submit a transfer to a non-existent endpoint!\n"); 1831 return B_NO_INIT; 1832 } 1833 1834 // Use mutex_trylock first, in case we are in KDL. 1835 if (mutex_trylock(&endpoint->lock) != B_OK) 1836 mutex_lock(&endpoint->lock); 1837 1838 // "used" refers to the number of currently linked TDs, not the number of 1839 // used TRBs on the ring (we use 2 TRBs on the ring per transfer.) 1840 if (endpoint->used >= (XHCI_MAX_TRANSFERS - 1)) { 1841 TRACE_ERROR("link descriptor for pipe: max transfers count exceeded\n"); 1842 mutex_unlock(&endpoint->lock); 1843 return B_BAD_VALUE; 1844 } 1845 1846 endpoint->used++; 1847 descriptor->next = endpoint->td_head; 1848 endpoint->td_head = descriptor; 1849 1850 const uint8 current = endpoint->current, 1851 eventdata = current + 1; 1852 uint8 next = eventdata + 1; 1853 1854 TRACE("link descriptor for pipe: current %d, next %d\n", current, next); 1855 1856 // Add a Link TRB to the end of the descriptor. 1857 phys_addr_t addr = endpoint->trb_addr + eventdata * sizeof(xhci_trb); 1858 descriptor->trbs[descriptor->trb_used].address = addr; 1859 descriptor->trbs[descriptor->trb_used].status = TRB_2_IRQ(0); 1860 descriptor->trbs[descriptor->trb_used].flags = TRB_3_TYPE(TRB_TYPE_LINK) 1861 | TRB_3_CHAIN_BIT | TRB_3_CYCLE_BIT; 1862 // It is specified that (XHCI 1.2 § 4.12.3 Note 2 p251) if the TRB 1863 // following one with the ENT bit set is a Link TRB, the Link TRB 1864 // shall be evaluated *and* the subsequent TRB shall be. Thus a 1865 // TRB_3_ENT_BIT is unnecessary here; and from testing seems to 1866 // break all transfers on a (very) small number of controllers. 1867 1868 #if !B_HOST_IS_LENDIAN 1869 // Convert endianness. 1870 for (uint32 i = 0; i <= descriptor->trb_used; i++) { 1871 descriptor->trbs[i].address = 1872 B_HOST_TO_LENDIAN_INT64(descriptor->trbs[i].address); 1873 descriptor->trbs[i].status = 1874 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].status); 1875 descriptor->trbs[i].flags = 1876 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].flags); 1877 } 1878 #endif 1879 1880 // Link the descriptor. 1881 endpoint->trbs[current].address = 1882 B_HOST_TO_LENDIAN_INT64(descriptor->trb_addr); 1883 endpoint->trbs[current].status = 1884 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 1885 endpoint->trbs[current].flags = 1886 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_LINK)); 1887 1888 // Set up the Event Data TRB (XHCI 1.2 § 4.11.5.2 p230.) 1889 // 1890 // We do this on the main ring for two reasons: first, to avoid a small 1891 // potential race between the interrupt and the controller evaluating 1892 // the link TRB to get back onto the ring; and second, because many 1893 // controllers throw errors if the target of a Link TRB is not valid 1894 // (i.e. does not have its Cycle Bit set.) 1895 // 1896 // We also set the "address" field, which the controller will copy 1897 // verbatim into the TRB it posts to the event ring, to be the last 1898 // "real" TRB in the TD; this will allow us to determine what transfer 1899 // the resulting Transfer Event TRB refers to. 1900 endpoint->trbs[eventdata].address = 1901 B_HOST_TO_LENDIAN_INT64(descriptor->trb_addr 1902 + (descriptor->trb_used - 1) * sizeof(xhci_trb)); 1903 endpoint->trbs[eventdata].status = 1904 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 1905 endpoint->trbs[eventdata].flags = 1906 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_EVENT_DATA) 1907 | TRB_3_IOC_BIT | TRB_3_CYCLE_BIT); 1908 1909 if (next == (XHCI_ENDPOINT_RING_SIZE - 1)) { 1910 // We always use 2 TRBs per _Link..() call, so if "next" is the last 1911 // TRB in the ring, we need to generate a link TRB at "next", and 1912 // then wrap it to 0. 1913 endpoint->trbs[next].address = 1914 B_HOST_TO_LENDIAN_INT64(endpoint->trb_addr); 1915 endpoint->trbs[next].status = 1916 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 1917 endpoint->trbs[next].flags = 1918 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_LINK) | TRB_3_CYCLE_BIT); 1919 1920 next = 0; 1921 } 1922 1923 endpoint->trbs[next].address = 0; 1924 endpoint->trbs[next].status = 0; 1925 endpoint->trbs[next].flags = 0; 1926 1927 // Everything is ready, so write the cycle bit. 1928 endpoint->trbs[current].flags |= B_HOST_TO_LENDIAN_INT32(TRB_3_CYCLE_BIT); 1929 1930 TRACE("_LinkDescriptorForPipe pCurrent %p phys 0x%" B_PRIxPHYSADDR 1931 " 0x%" B_PRIxPHYSADDR " 0x%08" B_PRIx32 "\n", &endpoint->trbs[current], 1932 endpoint->trb_addr + current * sizeof(struct xhci_trb), 1933 endpoint->trbs[current].address, 1934 B_LENDIAN_TO_HOST_INT32(endpoint->trbs[current].flags)); 1935 1936 endpoint->current = next; 1937 mutex_unlock(&endpoint->lock); 1938 1939 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 1940 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 1941 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 1942 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 1943 1944 Ring(endpoint->device->slot, endpoint->id + 1); 1945 1946 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 1947 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 1948 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 1949 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 1950 1951 return B_OK; 1952 } 1953 1954 1955 status_t 1956 XHCI::_UnlinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 1957 { 1958 TRACE("unlink descriptor for pipe\n"); 1959 // We presume that the caller has already locked or owns the endpoint. 1960 1961 endpoint->used--; 1962 if (descriptor == endpoint->td_head) { 1963 endpoint->td_head = descriptor->next; 1964 descriptor->next = NULL; 1965 return B_OK; 1966 } else { 1967 for (xhci_td *td = endpoint->td_head; td->next != NULL; td = td->next) { 1968 if (td->next == descriptor) { 1969 td->next = descriptor->next; 1970 descriptor->next = NULL; 1971 return B_OK; 1972 } 1973 } 1974 } 1975 1976 endpoint->used++; 1977 return B_ERROR; 1978 } 1979 1980 1981 status_t 1982 XHCI::ConfigureEndpoint(xhci_endpoint* ep, uint8 slot, uint8 number, uint8 type, 1983 bool directionIn, uint16 interval, uint16 maxPacketSize, usb_speed speed, 1984 uint8 maxBurst, uint16 bytesPerInterval) 1985 { 1986 struct xhci_device* device = &fDevices[slot]; 1987 1988 uint32 dwendpoint0 = 0; 1989 uint32 dwendpoint1 = 0; 1990 uint64 qwendpoint2 = 0; 1991 uint32 dwendpoint4 = 0; 1992 1993 // Compute and assign the endpoint type. (XHCI 1.2 § 6.2.3 Table 6-9 p452.) 1994 uint8 xhciType = 4; 1995 if ((type & USB_OBJECT_INTERRUPT_PIPE) != 0) 1996 xhciType = 3; 1997 if ((type & USB_OBJECT_BULK_PIPE) != 0) 1998 xhciType = 2; 1999 if ((type & USB_OBJECT_ISO_PIPE) != 0) 2000 xhciType = 1; 2001 xhciType |= directionIn ? (1 << 2) : 0; 2002 dwendpoint1 |= ENDPOINT_1_EPTYPE(xhciType); 2003 2004 // Compute and assign interval. (XHCI 1.2 § 6.2.3.6 p456.) 2005 uint16 calcInterval; 2006 if ((type & USB_OBJECT_BULK_PIPE) != 0 2007 || (type & USB_OBJECT_CONTROL_PIPE) != 0) { 2008 // Bulk and Control endpoints never issue NAKs. 2009 calcInterval = 0; 2010 } else { 2011 switch (speed) { 2012 case USB_SPEED_FULLSPEED: 2013 if ((type & USB_OBJECT_ISO_PIPE) != 0) { 2014 // Convert 1-16 into 3-18. 2015 calcInterval = min_c(max_c(interval, 1), 16) + 2; 2016 break; 2017 } 2018 2019 // fall through 2020 case USB_SPEED_LOWSPEED: { 2021 // Convert 1ms-255ms into 3-10. 2022 2023 // Find the index of the highest set bit in "interval". 2024 uint32 temp = min_c(max_c(interval, 1), 255); 2025 for (calcInterval = 0; temp != 1; calcInterval++) 2026 temp = temp >> 1; 2027 calcInterval += 3; 2028 break; 2029 } 2030 2031 case USB_SPEED_HIGHSPEED: 2032 case USB_SPEED_SUPERSPEED: 2033 default: 2034 // Convert 1-16 into 0-15. 2035 calcInterval = min_c(max_c(interval, 1), 16) - 1; 2036 break; 2037 } 2038 } 2039 dwendpoint0 |= ENDPOINT_0_INTERVAL(calcInterval); 2040 2041 // For non-isochronous endpoints, we want the controller to retry failed 2042 // transfers, if possible. (XHCI 1.2 § 4.10.2.3 p197.) 2043 if ((type & USB_OBJECT_ISO_PIPE) == 0) 2044 dwendpoint1 |= ENDPOINT_1_CERR(3); 2045 2046 // Assign maximum burst size. For USB3 devices this is passed in; for 2047 // all other devices we compute it. (XHCI 1.2 § 4.8.2 p161.) 2048 if (speed == USB_SPEED_HIGHSPEED && (type & (USB_OBJECT_INTERRUPT_PIPE 2049 | USB_OBJECT_ISO_PIPE)) != 0) { 2050 maxBurst = (maxPacketSize & 0x1800) >> 11; 2051 } else if (speed != USB_SPEED_SUPERSPEED) { 2052 maxBurst = 0; 2053 } 2054 dwendpoint1 |= ENDPOINT_1_MAXBURST(maxBurst); 2055 2056 // Assign maximum packet size, set the ring address, and set the 2057 // "Dequeue Cycle State" bit. (XHCI 1.2 § 6.2.3 Table 6-10 p453.) 2058 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE(maxPacketSize); 2059 qwendpoint2 |= ENDPOINT_2_DCS_BIT | ep->trb_addr; 2060 2061 // The Max Burst Payload is the number of bytes moved by a 2062 // maximum sized burst. (XHCI 1.2 § 4.11.7.1 p236.) 2063 ep->max_burst_payload = (maxBurst + 1) * maxPacketSize; 2064 if (ep->max_burst_payload == 0) { 2065 TRACE_ERROR("ConfigureEndpoint() failed invalid max_burst_payload\n"); 2066 return B_BAD_VALUE; 2067 } 2068 2069 // Assign average TRB length. 2070 if ((type & USB_OBJECT_CONTROL_PIPE) != 0) { 2071 // Control pipes are a special case, as they rarely have 2072 // outbound transfers of any substantial size. 2073 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(8); 2074 } else if ((type & USB_OBJECT_ISO_PIPE) != 0) { 2075 // Isochronous pipes are another special case: the TRB size will be 2076 // one packet (which is normally smaller than the max packet size, 2077 // but we don't know what it is here.) 2078 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(maxPacketSize); 2079 } else { 2080 // Under all other circumstances, we put max_burst_payload in a TRB. 2081 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(ep->max_burst_payload); 2082 } 2083 2084 // Assign maximum ESIT payload. (XHCI 1.2 § 4.14.2 p259.) 2085 if ((type & (USB_OBJECT_INTERRUPT_PIPE | USB_OBJECT_ISO_PIPE)) != 0) { 2086 // TODO: For SuperSpeedPlus endpoints, there is yet another descriptor 2087 // for isochronous endpoints that specifies the maximum ESIT payload. 2088 // We don't fetch this yet, so just fall back to the USB2 computation 2089 // method if bytesPerInterval is 0. 2090 if (speed == USB_SPEED_SUPERSPEED && bytesPerInterval != 0) 2091 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD(bytesPerInterval); 2092 else if (speed >= USB_SPEED_HIGHSPEED) 2093 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD((maxBurst + 1) * maxPacketSize); 2094 } 2095 2096 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint0, 2097 dwendpoint0); 2098 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint1, 2099 dwendpoint1); 2100 _WriteContext(&device->input_ctx->endpoints[number].qwendpoint2, 2101 qwendpoint2); 2102 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint4, 2103 dwendpoint4); 2104 2105 TRACE("endpoint 0x%" B_PRIx32 " 0x%" B_PRIx32 " 0x%" B_PRIx64 " 0x%" 2106 B_PRIx32 "\n", 2107 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint0), 2108 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint1), 2109 _ReadContext(&device->input_ctx->endpoints[number].qwendpoint2), 2110 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint4)); 2111 2112 return B_OK; 2113 } 2114 2115 2116 status_t 2117 XHCI::GetPortSpeed(uint8 index, usb_speed* speed) 2118 { 2119 if (index >= fPortCount) 2120 return B_BAD_INDEX; 2121 2122 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 2123 2124 switch (PS_SPEED_GET(portStatus)) { 2125 case 3: 2126 *speed = USB_SPEED_HIGHSPEED; 2127 break; 2128 case 2: 2129 *speed = USB_SPEED_LOWSPEED; 2130 break; 2131 case 1: 2132 *speed = USB_SPEED_FULLSPEED; 2133 break; 2134 case 4: 2135 *speed = USB_SPEED_SUPERSPEED; 2136 break; 2137 default: 2138 TRACE_ALWAYS("nonstandard port speed %" B_PRId32 ", assuming SuperSpeed\n", 2139 PS_SPEED_GET(portStatus)); 2140 *speed = USB_SPEED_SUPERSPEED; 2141 break; 2142 } 2143 2144 return B_OK; 2145 } 2146 2147 2148 status_t 2149 XHCI::GetPortStatus(uint8 index, usb_port_status* status) 2150 { 2151 if (index >= fPortCount) 2152 return B_BAD_INDEX; 2153 2154 status->status = status->change = 0; 2155 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 2156 TRACE("port %" B_PRId8 " status=0x%08" B_PRIx32 "\n", index, portStatus); 2157 2158 // build the status 2159 switch (PS_SPEED_GET(portStatus)) { 2160 case 3: 2161 status->status |= PORT_STATUS_HIGH_SPEED; 2162 break; 2163 case 2: 2164 status->status |= PORT_STATUS_LOW_SPEED; 2165 break; 2166 default: 2167 break; 2168 } 2169 2170 if (portStatus & PS_CCS) 2171 status->status |= PORT_STATUS_CONNECTION; 2172 if (portStatus & PS_PED) 2173 status->status |= PORT_STATUS_ENABLE; 2174 if (portStatus & PS_OCA) 2175 status->status |= PORT_STATUS_OVER_CURRENT; 2176 if (portStatus & PS_PR) 2177 status->status |= PORT_STATUS_RESET; 2178 if (portStatus & PS_PP) { 2179 if (fPortSpeeds[index] == USB_SPEED_SUPERSPEED) 2180 status->status |= PORT_STATUS_SS_POWER; 2181 else 2182 status->status |= PORT_STATUS_POWER; 2183 } 2184 2185 // build the change 2186 if (portStatus & PS_CSC) 2187 status->change |= PORT_STATUS_CONNECTION; 2188 if (portStatus & PS_PEC) 2189 status->change |= PORT_STATUS_ENABLE; 2190 if (portStatus & PS_OCC) 2191 status->change |= PORT_STATUS_OVER_CURRENT; 2192 if (portStatus & PS_PRC) 2193 status->change |= PORT_STATUS_RESET; 2194 2195 if (fPortSpeeds[index] == USB_SPEED_SUPERSPEED) { 2196 if (portStatus & PS_PLC) 2197 status->change |= PORT_CHANGE_LINK_STATE; 2198 if (portStatus & PS_WRC) 2199 status->change |= PORT_CHANGE_BH_PORT_RESET; 2200 } 2201 2202 return B_OK; 2203 } 2204 2205 2206 status_t 2207 XHCI::SetPortFeature(uint8 index, uint16 feature) 2208 { 2209 TRACE("set port feature index %u feature %u\n", index, feature); 2210 if (index >= fPortCount) 2211 return B_BAD_INDEX; 2212 2213 uint32 portRegister = XHCI_PORTSC(index); 2214 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2215 2216 switch (feature) { 2217 case PORT_SUSPEND: 2218 if ((portStatus & PS_PED) == 0 || (portStatus & PS_PR) 2219 || (portStatus & PS_PLS_MASK) >= PS_XDEV_U3) { 2220 TRACE_ERROR("USB core suspending device not in U0/U1/U2.\n"); 2221 return B_BAD_VALUE; 2222 } 2223 portStatus &= ~PS_PLS_MASK; 2224 WriteOpReg(portRegister, portStatus | PS_LWS | PS_XDEV_U3); 2225 break; 2226 2227 case PORT_RESET: 2228 WriteOpReg(portRegister, portStatus | PS_PR); 2229 break; 2230 2231 case PORT_POWER: 2232 WriteOpReg(portRegister, portStatus | PS_PP); 2233 break; 2234 default: 2235 return B_BAD_VALUE; 2236 } 2237 ReadOpReg(portRegister); 2238 return B_OK; 2239 } 2240 2241 2242 status_t 2243 XHCI::ClearPortFeature(uint8 index, uint16 feature) 2244 { 2245 TRACE("clear port feature index %u feature %u\n", index, feature); 2246 if (index >= fPortCount) 2247 return B_BAD_INDEX; 2248 2249 uint32 portRegister = XHCI_PORTSC(index); 2250 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2251 2252 switch (feature) { 2253 case PORT_SUSPEND: 2254 portStatus = ReadOpReg(portRegister); 2255 if (portStatus & PS_PR) 2256 return B_BAD_VALUE; 2257 if (portStatus & PS_XDEV_U3) { 2258 if ((portStatus & PS_PED) == 0) 2259 return B_BAD_VALUE; 2260 portStatus &= ~PS_PLS_MASK; 2261 WriteOpReg(portRegister, portStatus | PS_XDEV_U0 | PS_LWS); 2262 } 2263 break; 2264 case PORT_ENABLE: 2265 WriteOpReg(portRegister, portStatus | PS_PED); 2266 break; 2267 case PORT_POWER: 2268 WriteOpReg(portRegister, portStatus & ~PS_PP); 2269 break; 2270 case C_PORT_CONNECTION: 2271 WriteOpReg(portRegister, portStatus | PS_CSC); 2272 break; 2273 case C_PORT_ENABLE: 2274 WriteOpReg(portRegister, portStatus | PS_PEC); 2275 break; 2276 case C_PORT_OVER_CURRENT: 2277 WriteOpReg(portRegister, portStatus | PS_OCC); 2278 break; 2279 case C_PORT_RESET: 2280 WriteOpReg(portRegister, portStatus | PS_PRC); 2281 break; 2282 case C_PORT_BH_PORT_RESET: 2283 WriteOpReg(portRegister, portStatus | PS_WRC); 2284 break; 2285 case C_PORT_LINK_STATE: 2286 WriteOpReg(portRegister, portStatus | PS_PLC); 2287 break; 2288 default: 2289 return B_BAD_VALUE; 2290 } 2291 2292 ReadOpReg(portRegister); 2293 return B_OK; 2294 } 2295 2296 2297 status_t 2298 XHCI::ControllerHalt() 2299 { 2300 // Mask off run state 2301 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) & ~CMD_RUN); 2302 2303 // wait for shutdown state 2304 if (WaitOpBits(XHCI_STS, STS_HCH, STS_HCH) != B_OK) { 2305 TRACE_ERROR("HCH shutdown timeout\n"); 2306 return B_ERROR; 2307 } 2308 return B_OK; 2309 } 2310 2311 2312 status_t 2313 XHCI::ControllerReset() 2314 { 2315 TRACE("ControllerReset() cmd: 0x%" B_PRIx32 " sts: 0x%" B_PRIx32 "\n", 2316 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 2317 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) | CMD_HCRST); 2318 2319 if (WaitOpBits(XHCI_CMD, CMD_HCRST, 0) != B_OK) { 2320 TRACE_ERROR("ControllerReset() failed CMD_HCRST\n"); 2321 return B_ERROR; 2322 } 2323 2324 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 2325 TRACE_ERROR("ControllerReset() failed STS_CNR\n"); 2326 return B_ERROR; 2327 } 2328 2329 return B_OK; 2330 } 2331 2332 2333 int32 2334 XHCI::InterruptHandler(void* data) 2335 { 2336 return ((XHCI*)data)->Interrupt(); 2337 } 2338 2339 2340 int32 2341 XHCI::Interrupt() 2342 { 2343 SpinLocker _(&fSpinlock); 2344 2345 uint32 status = ReadOpReg(XHCI_STS); 2346 uint32 temp = ReadRunReg32(XHCI_IMAN(0)); 2347 WriteOpReg(XHCI_STS, status); 2348 WriteRunReg32(XHCI_IMAN(0), temp); 2349 2350 int32 result = B_HANDLED_INTERRUPT; 2351 2352 if ((status & STS_HCH) != 0) { 2353 TRACE_ERROR("Host Controller halted\n"); 2354 return result; 2355 } 2356 if ((status & STS_HSE) != 0) { 2357 TRACE_ERROR("Host System Error\n"); 2358 return result; 2359 } 2360 if ((status & STS_HCE) != 0) { 2361 TRACE_ERROR("Host Controller Error\n"); 2362 return result; 2363 } 2364 2365 if ((status & STS_EINT) == 0) { 2366 TRACE("STS: 0x%" B_PRIx32 " IRQ_PENDING: 0x%" B_PRIx32 "\n", 2367 status, temp); 2368 return B_UNHANDLED_INTERRUPT; 2369 } 2370 2371 TRACE("Event Interrupt\n"); 2372 release_sem_etc(fEventSem, 1, B_DO_NOT_RESCHEDULE); 2373 return B_INVOKE_SCHEDULER; 2374 } 2375 2376 2377 void 2378 XHCI::Ring(uint8 slot, uint8 endpoint) 2379 { 2380 TRACE("Ding Dong! slot:%d endpoint %d\n", slot, endpoint) 2381 if ((slot == 0 && endpoint > 0) || (slot > 0 && endpoint == 0)) 2382 panic("Ring() invalid slot/endpoint combination\n"); 2383 if (slot > fSlotCount || endpoint >= XHCI_MAX_ENDPOINTS) 2384 panic("Ring() invalid slot or endpoint\n"); 2385 2386 WriteDoorReg32(XHCI_DOORBELL(slot), XHCI_DOORBELL_TARGET(endpoint) 2387 | XHCI_DOORBELL_STREAMID(0)); 2388 ReadDoorReg32(XHCI_DOORBELL(slot)); 2389 // Flush PCI writes 2390 } 2391 2392 2393 void 2394 XHCI::QueueCommand(xhci_trb* trb) 2395 { 2396 uint8 i, j; 2397 uint32 temp; 2398 2399 i = fCmdIdx; 2400 j = fCmdCcs; 2401 2402 TRACE("command[%u] = %" B_PRId32 " (0x%016" B_PRIx64 ", 0x%08" B_PRIx32 2403 ", 0x%08" B_PRIx32 ")\n", i, TRB_3_TYPE_GET(trb->flags), trb->address, 2404 trb->status, trb->flags); 2405 2406 fCmdRing[i].address = trb->address; 2407 fCmdRing[i].status = trb->status; 2408 temp = trb->flags; 2409 2410 if (j) 2411 temp |= TRB_3_CYCLE_BIT; 2412 else 2413 temp &= ~TRB_3_CYCLE_BIT; 2414 temp &= ~TRB_3_TC_BIT; 2415 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2416 2417 fCmdAddr = fErst->rs_addr + (XHCI_MAX_EVENTS + i) * sizeof(xhci_trb); 2418 2419 i++; 2420 2421 if (i == (XHCI_MAX_COMMANDS - 1)) { 2422 temp = TRB_3_TYPE(TRB_TYPE_LINK) | TRB_3_TC_BIT; 2423 if (j) 2424 temp |= TRB_3_CYCLE_BIT; 2425 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2426 2427 i = 0; 2428 j ^= 1; 2429 } 2430 2431 fCmdIdx = i; 2432 fCmdCcs = j; 2433 } 2434 2435 2436 void 2437 XHCI::HandleCmdComplete(xhci_trb* trb) 2438 { 2439 if (fCmdAddr == trb->address) { 2440 TRACE("Received command event\n"); 2441 fCmdResult[0] = trb->status; 2442 fCmdResult[1] = B_LENDIAN_TO_HOST_INT32(trb->flags); 2443 release_sem_etc(fCmdCompSem, 1, B_DO_NOT_RESCHEDULE); 2444 } else 2445 TRACE_ERROR("received command event for unknown command!\n") 2446 } 2447 2448 2449 void 2450 XHCI::HandleTransferComplete(xhci_trb* trb) 2451 { 2452 const uint32 flags = B_LENDIAN_TO_HOST_INT32(trb->flags); 2453 const uint8 endpointNumber = TRB_3_ENDPOINT_GET(flags), 2454 slot = TRB_3_SLOT_GET(flags); 2455 2456 if (slot > fSlotCount) 2457 TRACE_ERROR("invalid slot\n"); 2458 if (endpointNumber == 0 || endpointNumber >= XHCI_MAX_ENDPOINTS) { 2459 TRACE_ERROR("invalid endpoint\n"); 2460 return; 2461 } 2462 2463 xhci_device *device = &fDevices[slot]; 2464 xhci_endpoint *endpoint = &device->endpoints[endpointNumber - 1]; 2465 2466 if (endpoint->trbs == NULL) { 2467 TRACE_ERROR("got TRB but endpoint is not allocated!\n"); 2468 return; 2469 } 2470 2471 // Use mutex_trylock first, in case we are in KDL. 2472 MutexLocker endpointLocker(endpoint->lock, 2473 mutex_trylock(&endpoint->lock) == B_OK); 2474 if (!endpointLocker.IsLocked()) { 2475 // We failed to get the lock. Most likely it was destroyed 2476 // while we were waiting for it. 2477 return; 2478 } 2479 2480 // In the case of an Event Data TRB, the "transferred" field refers 2481 // to the actual number of bytes transferred across the whole TD. 2482 // (XHCI 1.2 § 6.4.2.1 Table 6-38 p478.) 2483 const uint8 completionCode = TRB_2_COMP_CODE_GET(trb->status); 2484 int32 transferred = TRB_2_REM_GET(trb->status), remainder = -1; 2485 2486 TRACE("HandleTransferComplete: ed %d, code %d, transferred %d\n", 2487 (flags & TRB_3_EVENT_DATA_BIT), completionCode, transferred); 2488 2489 if ((flags & TRB_3_EVENT_DATA_BIT) == 0) { 2490 // This should only occur under error conditions. 2491 TRACE("got an interrupt for a non-Event Data TRB!\n"); 2492 remainder = transferred; 2493 transferred = -1; 2494 } 2495 2496 if (completionCode != COMP_SUCCESS && completionCode != COMP_SHORT_PACKET) { 2497 TRACE_ALWAYS("transfer error on slot %" B_PRId8 " endpoint %" B_PRId8 2498 ": %s\n", slot, endpointNumber, xhci_error_string(completionCode)); 2499 } 2500 2501 const phys_addr_t source = B_LENDIAN_TO_HOST_INT64(trb->address); 2502 for (xhci_td *td = endpoint->td_head; td != NULL; td = td->next) { 2503 int64 offset = (source - td->trb_addr) / sizeof(xhci_trb); 2504 if (offset < 0 || offset >= td->trb_count) 2505 continue; 2506 2507 TRACE("HandleTransferComplete td %p trb %" B_PRId64 " found\n", 2508 td, offset); 2509 2510 // The TRB at offset trb_used will be the link TRB, which we do not 2511 // care about (and should not generate an interrupt at all.) We really 2512 // care about the properly last TRB, at index "count - 1", which the 2513 // Event Data TRB that _LinkDescriptorForPipe creates points to. 2514 // 2515 // But if we have an unsuccessful completion code, the transfer 2516 // likely failed midway; so just accept it anyway. 2517 if (offset == (td->trb_used - 1) || completionCode != COMP_SUCCESS) { 2518 _UnlinkDescriptorForPipe(td, endpoint); 2519 endpointLocker.Unlock(); 2520 2521 td->trb_completion_code = completionCode; 2522 td->td_transferred = transferred; 2523 td->trb_left = remainder; 2524 2525 // add descriptor to finished list 2526 if (mutex_trylock(&fFinishedLock) != B_OK) 2527 mutex_lock(&fFinishedLock); 2528 td->next = fFinishedHead; 2529 fFinishedHead = td; 2530 mutex_unlock(&fFinishedLock); 2531 2532 release_sem_etc(fFinishTransfersSem, 1, B_DO_NOT_RESCHEDULE); 2533 TRACE("HandleTransferComplete td %p done\n", td); 2534 } else { 2535 TRACE_ERROR("successful TRB 0x%" B_PRIxPHYSADDR " was found, but it wasn't " 2536 "the last in the TD!\n", source); 2537 } 2538 return; 2539 } 2540 TRACE_ERROR("TRB 0x%" B_PRIxPHYSADDR " was not found in the endpoint!\n", source); 2541 } 2542 2543 2544 void 2545 XHCI::DumpRing(xhci_trb *trbs, uint32 size) 2546 { 2547 if (!Lock()) { 2548 TRACE("Unable to get lock!\n"); 2549 return; 2550 } 2551 2552 for (uint32 i = 0; i < size; i++) { 2553 TRACE("command[%" B_PRId32 "] = %" B_PRId32 " (0x%016" B_PRIx64 "," 2554 " 0x%08" B_PRIx32 ", 0x%08" B_PRIx32 ")\n", i, 2555 TRB_3_TYPE_GET(B_LENDIAN_TO_HOST_INT32(trbs[i].flags)), 2556 trbs[i].address, trbs[i].status, trbs[i].flags); 2557 } 2558 2559 Unlock(); 2560 } 2561 2562 2563 status_t 2564 XHCI::DoCommand(xhci_trb* trb) 2565 { 2566 if (!Lock()) { 2567 TRACE("Unable to get lock!\n"); 2568 return B_ERROR; 2569 } 2570 2571 QueueCommand(trb); 2572 Ring(0, 0); 2573 2574 // Begin with a 50ms timeout. 2575 if (acquire_sem_etc(fCmdCompSem, 1, B_RELATIVE_TIMEOUT, 50 * 1000) != B_OK) { 2576 // We've hit the timeout. In some error cases, interrupts are not 2577 // generated; so here we force the event ring to be polled once. 2578 release_sem(fEventSem); 2579 2580 // Now try again, this time with a 750ms timeout. 2581 if (acquire_sem_etc(fCmdCompSem, 1, B_RELATIVE_TIMEOUT, 2582 750 * 1000) != B_OK) { 2583 TRACE("Unable to obtain fCmdCompSem!\n"); 2584 fCmdAddr = 0; 2585 Unlock(); 2586 return B_TIMED_OUT; 2587 } 2588 } 2589 2590 // eat up sems that have been released by multiple interrupts 2591 int32 semCount = 0; 2592 get_sem_count(fCmdCompSem, &semCount); 2593 if (semCount > 0) 2594 acquire_sem_etc(fCmdCompSem, semCount, B_RELATIVE_TIMEOUT, 0); 2595 2596 status_t status = B_OK; 2597 uint32 completionCode = TRB_2_COMP_CODE_GET(fCmdResult[0]); 2598 TRACE("command complete\n"); 2599 if (completionCode != COMP_SUCCESS) { 2600 TRACE_ERROR("unsuccessful command %" B_PRId32 ", error %s (%" B_PRId32 ")\n", 2601 TRB_3_TYPE_GET(trb->flags), xhci_error_string(completionCode), 2602 completionCode); 2603 status = B_IO_ERROR; 2604 } 2605 2606 trb->status = fCmdResult[0]; 2607 trb->flags = fCmdResult[1]; 2608 2609 fCmdAddr = 0; 2610 Unlock(); 2611 return status; 2612 } 2613 2614 2615 status_t 2616 XHCI::Noop() 2617 { 2618 TRACE("Issue No-Op\n"); 2619 xhci_trb trb; 2620 trb.address = 0; 2621 trb.status = 0; 2622 trb.flags = TRB_3_TYPE(TRB_TYPE_CMD_NOOP); 2623 2624 return DoCommand(&trb); 2625 } 2626 2627 2628 status_t 2629 XHCI::EnableSlot(uint8* slot) 2630 { 2631 TRACE("Enable Slot\n"); 2632 xhci_trb trb; 2633 trb.address = 0; 2634 trb.status = 0; 2635 trb.flags = TRB_3_TYPE(TRB_TYPE_ENABLE_SLOT); 2636 2637 status_t status = DoCommand(&trb); 2638 if (status != B_OK) 2639 return status; 2640 2641 *slot = TRB_3_SLOT_GET(trb.flags); 2642 return *slot != 0 ? B_OK : B_BAD_VALUE; 2643 } 2644 2645 2646 status_t 2647 XHCI::DisableSlot(uint8 slot) 2648 { 2649 TRACE("Disable Slot\n"); 2650 xhci_trb trb; 2651 trb.address = 0; 2652 trb.status = 0; 2653 trb.flags = TRB_3_TYPE(TRB_TYPE_DISABLE_SLOT) | TRB_3_SLOT(slot); 2654 2655 return DoCommand(&trb); 2656 } 2657 2658 2659 status_t 2660 XHCI::SetAddress(uint64 inputContext, bool bsr, uint8 slot) 2661 { 2662 TRACE("Set Address\n"); 2663 xhci_trb trb; 2664 trb.address = inputContext; 2665 trb.status = 0; 2666 trb.flags = TRB_3_TYPE(TRB_TYPE_ADDRESS_DEVICE) | TRB_3_SLOT(slot); 2667 2668 if (bsr) 2669 trb.flags |= TRB_3_BSR_BIT; 2670 2671 return DoCommand(&trb); 2672 } 2673 2674 2675 status_t 2676 XHCI::ConfigureEndpoint(uint64 inputContext, bool deconfigure, uint8 slot) 2677 { 2678 TRACE("Configure Endpoint\n"); 2679 xhci_trb trb; 2680 trb.address = inputContext; 2681 trb.status = 0; 2682 trb.flags = TRB_3_TYPE(TRB_TYPE_CONFIGURE_ENDPOINT) | TRB_3_SLOT(slot); 2683 2684 if (deconfigure) 2685 trb.flags |= TRB_3_DCEP_BIT; 2686 2687 return DoCommand(&trb); 2688 } 2689 2690 2691 status_t 2692 XHCI::EvaluateContext(uint64 inputContext, uint8 slot) 2693 { 2694 TRACE("Evaluate Context\n"); 2695 xhci_trb trb; 2696 trb.address = inputContext; 2697 trb.status = 0; 2698 trb.flags = TRB_3_TYPE(TRB_TYPE_EVALUATE_CONTEXT) | TRB_3_SLOT(slot); 2699 2700 return DoCommand(&trb); 2701 } 2702 2703 2704 status_t 2705 XHCI::ResetEndpoint(bool preserve, xhci_endpoint* endpoint) 2706 { 2707 TRACE("Reset Endpoint\n"); 2708 2709 switch (_GetEndpointState(endpoint)) { 2710 case ENDPOINT_STATE_STOPPED: 2711 TRACE("Reset Endpoint: already stopped"); 2712 return B_OK; 2713 case ENDPOINT_STATE_HALTED: 2714 TRACE("Reset Endpoint: warning, weird state!"); 2715 default: 2716 break; 2717 } 2718 2719 xhci_trb trb; 2720 trb.address = 0; 2721 trb.status = 0; 2722 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_ENDPOINT) 2723 | TRB_3_SLOT(endpoint->device->slot) | TRB_3_ENDPOINT(endpoint->id + 1); 2724 if (preserve) 2725 trb.flags |= TRB_3_PRSV_BIT; 2726 2727 return DoCommand(&trb); 2728 } 2729 2730 2731 status_t 2732 XHCI::StopEndpoint(bool suspend, xhci_endpoint* endpoint) 2733 { 2734 TRACE("Stop Endpoint\n"); 2735 2736 switch (_GetEndpointState(endpoint)) { 2737 case ENDPOINT_STATE_HALTED: 2738 TRACE("Stop Endpoint: error, halted"); 2739 return B_NOT_ALLOWED; 2740 case ENDPOINT_STATE_STOPPED: 2741 TRACE("Stop Endpoint: already stopped"); 2742 return B_OK; 2743 default: 2744 break; 2745 } 2746 2747 xhci_trb trb; 2748 trb.address = 0; 2749 trb.status = 0; 2750 trb.flags = TRB_3_TYPE(TRB_TYPE_STOP_ENDPOINT) 2751 | TRB_3_SLOT(endpoint->device->slot) | TRB_3_ENDPOINT(endpoint->id + 1); 2752 if (suspend) 2753 trb.flags |= TRB_3_SUSPEND_ENDPOINT_BIT; 2754 2755 return DoCommand(&trb); 2756 } 2757 2758 2759 status_t 2760 XHCI::SetTRDequeue(uint64 dequeue, uint16 stream, uint8 endpoint, uint8 slot) 2761 { 2762 TRACE("Set TR Dequeue\n"); 2763 xhci_trb trb; 2764 trb.address = dequeue | ENDPOINT_2_DCS_BIT; 2765 // The DCS bit is copied from the address field as in ConfigureEndpoint. 2766 // (XHCI 1.2 § 4.6.10 p142.) 2767 trb.status = TRB_2_STREAM(stream); 2768 trb.flags = TRB_3_TYPE(TRB_TYPE_SET_TR_DEQUEUE) 2769 | TRB_3_SLOT(slot) | TRB_3_ENDPOINT(endpoint); 2770 2771 return DoCommand(&trb); 2772 } 2773 2774 2775 status_t 2776 XHCI::ResetDevice(uint8 slot) 2777 { 2778 TRACE("Reset Device\n"); 2779 xhci_trb trb; 2780 trb.address = 0; 2781 trb.status = 0; 2782 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_DEVICE) | TRB_3_SLOT(slot); 2783 2784 return DoCommand(&trb); 2785 } 2786 2787 2788 int32 2789 XHCI::EventThread(void* data) 2790 { 2791 ((XHCI *)data)->CompleteEvents(); 2792 return B_OK; 2793 } 2794 2795 2796 void 2797 XHCI::CompleteEvents() 2798 { 2799 while (!fStopThreads) { 2800 if (acquire_sem(fEventSem) < B_OK) 2801 continue; 2802 2803 // eat up sems that have been released by multiple interrupts 2804 int32 semCount = 0; 2805 get_sem_count(fEventSem, &semCount); 2806 if (semCount > 0) 2807 acquire_sem_etc(fEventSem, semCount, B_RELATIVE_TIMEOUT, 0); 2808 2809 ProcessEvents(); 2810 } 2811 } 2812 2813 2814 void 2815 XHCI::ProcessEvents() 2816 { 2817 // Use mutex_trylock first, in case we are in KDL. 2818 MutexLocker locker(fEventLock, mutex_trylock(&fEventLock) == B_OK); 2819 if (!locker.IsLocked()) { 2820 // We failed to get the lock. This really should not happen. 2821 TRACE_ERROR("failed to acquire event lock!\n"); 2822 return; 2823 } 2824 2825 uint16 i = fEventIdx; 2826 uint8 j = fEventCcs; 2827 uint8 t = 2; 2828 2829 while (1) { 2830 uint32 temp = B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags); 2831 uint8 event = TRB_3_TYPE_GET(temp); 2832 TRACE("event[%u] = %u (0x%016" B_PRIx64 " 0x%08" B_PRIx32 " 0x%08" 2833 B_PRIx32 ")\n", i, event, fEventRing[i].address, 2834 fEventRing[i].status, B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags)); 2835 uint8 k = (temp & TRB_3_CYCLE_BIT) ? 1 : 0; 2836 if (j != k) 2837 break; 2838 2839 switch (event) { 2840 case TRB_TYPE_COMMAND_COMPLETION: 2841 HandleCmdComplete(&fEventRing[i]); 2842 break; 2843 case TRB_TYPE_TRANSFER: 2844 HandleTransferComplete(&fEventRing[i]); 2845 break; 2846 case TRB_TYPE_PORT_STATUS_CHANGE: 2847 TRACE("port change detected\n"); 2848 break; 2849 default: 2850 TRACE_ERROR("Unhandled event = %u\n", event); 2851 break; 2852 } 2853 2854 i++; 2855 if (i == XHCI_MAX_EVENTS) { 2856 i = 0; 2857 j ^= 1; 2858 if (!--t) 2859 break; 2860 } 2861 } 2862 2863 fEventIdx = i; 2864 fEventCcs = j; 2865 2866 uint64 addr = fErst->rs_addr + i * sizeof(xhci_trb); 2867 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)addr | ERDP_BUSY); 2868 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(addr >> 32)); 2869 } 2870 2871 2872 int32 2873 XHCI::FinishThread(void* data) 2874 { 2875 ((XHCI *)data)->FinishTransfers(); 2876 return B_OK; 2877 } 2878 2879 2880 void 2881 XHCI::FinishTransfers() 2882 { 2883 while (!fStopThreads) { 2884 if (acquire_sem(fFinishTransfersSem) < B_OK) 2885 continue; 2886 2887 // eat up sems that have been released by multiple interrupts 2888 int32 semCount = 0; 2889 get_sem_count(fFinishTransfersSem, &semCount); 2890 if (semCount > 0) 2891 acquire_sem_etc(fFinishTransfersSem, semCount, B_RELATIVE_TIMEOUT, 0); 2892 2893 mutex_lock(&fFinishedLock); 2894 TRACE("finishing transfers\n"); 2895 while (fFinishedHead != NULL) { 2896 xhci_td* td = fFinishedHead; 2897 fFinishedHead = td->next; 2898 td->next = NULL; 2899 mutex_unlock(&fFinishedLock); 2900 2901 TRACE("finishing transfer td %p\n", td); 2902 2903 Transfer* transfer = td->transfer; 2904 if (transfer == NULL) { 2905 // No transfer? Quick way out. 2906 FreeDescriptor(td); 2907 mutex_lock(&fFinishedLock); 2908 continue; 2909 } 2910 2911 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 2912 2913 status_t callbackStatus = B_OK; 2914 switch (td->trb_completion_code) { 2915 case COMP_SHORT_PACKET: 2916 case COMP_SUCCESS: 2917 callbackStatus = B_OK; 2918 break; 2919 case COMP_DATA_BUFFER: 2920 callbackStatus = directionIn ? B_DEV_DATA_OVERRUN 2921 : B_DEV_DATA_UNDERRUN; 2922 break; 2923 case COMP_BABBLE: 2924 callbackStatus = directionIn ? B_DEV_FIFO_OVERRUN 2925 : B_DEV_FIFO_UNDERRUN; 2926 break; 2927 case COMP_USB_TRANSACTION: 2928 callbackStatus = B_DEV_CRC_ERROR; 2929 break; 2930 case COMP_STALL: 2931 callbackStatus = B_DEV_STALLED; 2932 break; 2933 default: 2934 callbackStatus = B_DEV_STALLED; 2935 break; 2936 } 2937 2938 size_t actualLength = transfer->DataLength(); 2939 if (td->trb_completion_code != COMP_SUCCESS) { 2940 actualLength = td->td_transferred; 2941 if (td->td_transferred == -1) 2942 actualLength = transfer->DataLength() - td->trb_left; 2943 TRACE("transfer not successful, actualLength=%" B_PRIuSIZE "\n", 2944 actualLength); 2945 } 2946 2947 usb_isochronous_data* isochronousData = transfer->IsochronousData(); 2948 if (isochronousData != NULL) { 2949 size_t packetSize = transfer->DataLength() / isochronousData->packet_count, 2950 left = actualLength; 2951 for (uint32 i = 0; i < isochronousData->packet_count; i++) { 2952 size_t size = min_c(packetSize, left); 2953 isochronousData->packet_descriptors[i].actual_length = size; 2954 isochronousData->packet_descriptors[i].status = (size > 0) 2955 ? B_OK : B_DEV_FIFO_UNDERRUN; 2956 left -= size; 2957 } 2958 } 2959 2960 if (callbackStatus == B_OK && directionIn && actualLength > 0) { 2961 TRACE("copying in iov count %ld\n", transfer->VectorCount()); 2962 status_t status = transfer->PrepareKernelAccess(); 2963 if (status == B_OK) { 2964 ReadDescriptor(td, transfer->Vector(), 2965 transfer->VectorCount()); 2966 } else { 2967 callbackStatus = status; 2968 } 2969 } 2970 transfer->Finished(callbackStatus, actualLength); 2971 delete transfer; 2972 FreeDescriptor(td); 2973 mutex_lock(&fFinishedLock); 2974 } 2975 mutex_unlock(&fFinishedLock); 2976 } 2977 } 2978 2979 2980 inline void 2981 XHCI::WriteOpReg(uint32 reg, uint32 value) 2982 { 2983 *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg) = value; 2984 } 2985 2986 2987 inline uint32 2988 XHCI::ReadOpReg(uint32 reg) 2989 { 2990 return *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg); 2991 } 2992 2993 2994 inline status_t 2995 XHCI::WaitOpBits(uint32 reg, uint32 mask, uint32 expected) 2996 { 2997 int loops = 0; 2998 uint32 value = ReadOpReg(reg); 2999 while ((value & mask) != expected) { 3000 snooze(1000); 3001 value = ReadOpReg(reg); 3002 if (loops == 100) { 3003 TRACE("delay waiting on reg 0x%" B_PRIX32 " match 0x%" B_PRIX32 3004 " (0x%" B_PRIX32 ")\n", reg, expected, mask); 3005 } else if (loops > 250) { 3006 TRACE_ERROR("timeout waiting on reg 0x%" B_PRIX32 3007 " match 0x%" B_PRIX32 " (0x%" B_PRIX32 ")\n", reg, expected, 3008 mask); 3009 return B_ERROR; 3010 } 3011 loops++; 3012 } 3013 return B_OK; 3014 } 3015 3016 3017 inline uint32 3018 XHCI::ReadCapReg32(uint32 reg) 3019 { 3020 return *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg); 3021 } 3022 3023 3024 inline void 3025 XHCI::WriteCapReg32(uint32 reg, uint32 value) 3026 { 3027 *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg) = value; 3028 } 3029 3030 3031 inline uint32 3032 XHCI::ReadRunReg32(uint32 reg) 3033 { 3034 return *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg); 3035 } 3036 3037 3038 inline void 3039 XHCI::WriteRunReg32(uint32 reg, uint32 value) 3040 { 3041 *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg) = value; 3042 } 3043 3044 3045 inline uint32 3046 XHCI::ReadDoorReg32(uint32 reg) 3047 { 3048 return *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg); 3049 } 3050 3051 3052 inline void 3053 XHCI::WriteDoorReg32(uint32 reg, uint32 value) 3054 { 3055 *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg) = value; 3056 } 3057 3058 3059 inline addr_t 3060 XHCI::_OffsetContextAddr(addr_t p) 3061 { 3062 if (fContextSizeShift == 1) { 3063 // each structure is page aligned, each pointer is 32 bits aligned 3064 uint32 offset = p & ((B_PAGE_SIZE - 1) & ~31U); 3065 p += offset; 3066 } 3067 return p; 3068 } 3069 3070 inline uint32 3071 XHCI::_ReadContext(uint32* p) 3072 { 3073 p = (uint32*)_OffsetContextAddr((addr_t)p); 3074 return *p; 3075 } 3076 3077 3078 inline void 3079 XHCI::_WriteContext(uint32* p, uint32 value) 3080 { 3081 p = (uint32*)_OffsetContextAddr((addr_t)p); 3082 *p = value; 3083 } 3084 3085 3086 inline uint64 3087 XHCI::_ReadContext(uint64* p) 3088 { 3089 p = (uint64*)_OffsetContextAddr((addr_t)p); 3090 return *p; 3091 } 3092 3093 3094 inline void 3095 XHCI::_WriteContext(uint64* p, uint64 value) 3096 { 3097 p = (uint64*)_OffsetContextAddr((addr_t)p); 3098 *p = value; 3099 } 3100