1 /* 2 * Copyright 2011-2019, Haiku, Inc. All rights reserved. 3 * Distributed under the terms of the MIT License. 4 * 5 * Authors: 6 * Augustin Cavalier <waddlesplash> 7 * Jian Chiang <j.jian.chiang@gmail.com> 8 * Jérôme Duval <jerome.duval@gmail.com> 9 * Akshay Jaggi <akshay1994.leo@gmail.com> 10 * Michael Lotz <mmlr@mlotz.ch> 11 */ 12 13 14 #include <module.h> 15 #include <PCI.h> 16 #include <PCI_x86.h> 17 #include <USB3.h> 18 #include <KernelExport.h> 19 20 #include <util/AutoLock.h> 21 22 #include "xhci.h" 23 24 #define USB_MODULE_NAME "xhci" 25 26 pci_module_info *XHCI::sPCIModule = NULL; 27 pci_x86_module_info *XHCI::sPCIx86Module = NULL; 28 29 30 static int32 31 xhci_std_ops(int32 op, ...) 32 { 33 switch (op) { 34 case B_MODULE_INIT: 35 TRACE_MODULE("xhci init module\n"); 36 return B_OK; 37 case B_MODULE_UNINIT: 38 TRACE_MODULE("xhci uninit module\n"); 39 return B_OK; 40 } 41 42 return EINVAL; 43 } 44 45 46 static const char* 47 xhci_error_string(uint32 error) 48 { 49 switch (error) { 50 case COMP_INVALID: return "Invalid"; 51 case COMP_SUCCESS: return "Success"; 52 case COMP_DATA_BUFFER: return "Data buffer"; 53 case COMP_BABBLE: return "Babble detected"; 54 case COMP_USB_TRANSACTION: return "USB transaction"; 55 case COMP_TRB: return "TRB"; 56 case COMP_STALL: return "Stall"; 57 case COMP_RESOURCE: return "Resource"; 58 case COMP_BANDWIDTH: return "Bandwidth"; 59 case COMP_NO_SLOTS: return "No slots"; 60 case COMP_INVALID_STREAM: return "Invalid stream"; 61 case COMP_SLOT_NOT_ENABLED: return "Slot not enabled"; 62 case COMP_ENDPOINT_NOT_ENABLED: return "Endpoint not enabled"; 63 case COMP_SHORT_PACKET: return "Short packet"; 64 case COMP_RING_UNDERRUN: return "Ring underrun"; 65 case COMP_RING_OVERRUN: return "Ring overrun"; 66 case COMP_VF_RING_FULL: return "VF Event Ring Full"; 67 case COMP_PARAMETER: return "Parameter"; 68 case COMP_BANDWIDTH_OVERRUN: return "Bandwidth overrun"; 69 case COMP_CONTEXT_STATE: return "Context state"; 70 case COMP_NO_PING_RESPONSE: return "No ping response"; 71 case COMP_EVENT_RING_FULL: return "Event ring full"; 72 case COMP_INCOMPATIBLE_DEVICE: return "Incompatible device"; 73 case COMP_MISSED_SERVICE: return "Missed service"; 74 case COMP_COMMAND_RING_STOPPED: return "Command ring stopped"; 75 case COMP_COMMAND_ABORTED: return "Command aborted"; 76 case COMP_STOPPED: return "Stopped"; 77 case COMP_LENGTH_INVALID: return "Length invalid"; 78 case COMP_MAX_EXIT_LATENCY: return "Max exit latency too large"; 79 case COMP_ISOC_OVERRUN: return "Isoch buffer overrun"; 80 case COMP_EVENT_LOST: return "Event lost"; 81 case COMP_UNDEFINED: return "Undefined"; 82 case COMP_INVALID_STREAM_ID: return "Invalid stream ID"; 83 case COMP_SECONDARY_BANDWIDTH: return "Secondary bandwidth"; 84 case COMP_SPLIT_TRANSACTION: return "Split transaction"; 85 86 default: return "Undefined"; 87 } 88 } 89 90 91 usb_host_controller_info xhci_module = { 92 { 93 "busses/usb/xhci", 94 0, 95 xhci_std_ops 96 }, 97 NULL, 98 XHCI::AddTo 99 }; 100 101 102 module_info *modules[] = { 103 (module_info *)&xhci_module, 104 NULL 105 }; 106 107 108 status_t 109 XHCI::AddTo(Stack *stack) 110 { 111 if (!sPCIModule) { 112 status_t status = get_module(B_PCI_MODULE_NAME, 113 (module_info **)&sPCIModule); 114 if (status < B_OK) { 115 TRACE_MODULE_ERROR("getting pci module failed! 0x%08" B_PRIx32 116 "\n", status); 117 return status; 118 } 119 } 120 121 TRACE_MODULE("searching devices\n"); 122 bool found = false; 123 pci_info *item = new(std::nothrow) pci_info; 124 if (item == NULL) { 125 sPCIModule = NULL; 126 put_module(B_PCI_MODULE_NAME); 127 return B_NO_MEMORY; 128 } 129 130 // Try to get the PCI x86 module as well so we can enable possible MSIs. 131 if (sPCIx86Module == NULL && get_module(B_PCI_X86_MODULE_NAME, 132 (module_info **)&sPCIx86Module) != B_OK) { 133 // If it isn't there, that's not critical though. 134 TRACE_MODULE_ERROR("failed to get pci x86 module\n"); 135 sPCIx86Module = NULL; 136 } 137 138 for (int32 i = 0; sPCIModule->get_nth_pci_info(i, item) >= B_OK; i++) { 139 if (item->class_base == PCI_serial_bus && item->class_sub == PCI_usb 140 && item->class_api == PCI_usb_xhci) { 141 TRACE_MODULE("found device at PCI:%d:%d:%d\n", 142 item->bus, item->device, item->function); 143 XHCI *bus = new(std::nothrow) XHCI(item, stack); 144 if (bus == NULL) { 145 delete item; 146 sPCIModule = NULL; 147 put_module(B_PCI_MODULE_NAME); 148 if (sPCIx86Module != NULL) 149 put_module(B_PCI_X86_MODULE_NAME); 150 return B_NO_MEMORY; 151 } 152 153 // The bus will put the PCI modules when it is destroyed, so get 154 // them again to increase their reference count. 155 get_module(B_PCI_MODULE_NAME, (module_info **)&sPCIModule); 156 if (sPCIx86Module != NULL) 157 get_module(B_PCI_X86_MODULE_NAME, (module_info **)&sPCIx86Module); 158 159 if (bus->InitCheck() < B_OK) { 160 TRACE_MODULE_ERROR("bus failed init check\n"); 161 delete bus; 162 continue; 163 } 164 165 // the bus took it away 166 item = new(std::nothrow) pci_info; 167 168 if (bus->Start() != B_OK) { 169 delete bus; 170 continue; 171 } 172 found = true; 173 } 174 } 175 176 // The modules will have been gotten again if we successfully 177 // initialized a bus, so we should put them here. 178 put_module(B_PCI_MODULE_NAME); 179 if (sPCIx86Module != NULL) 180 put_module(B_PCI_X86_MODULE_NAME); 181 182 if (!found) 183 TRACE_MODULE_ERROR("no devices found\n"); 184 delete item; 185 return found ? B_OK : ENODEV; 186 } 187 188 189 XHCI::XHCI(pci_info *info, Stack *stack) 190 : BusManager(stack), 191 fRegisterArea(-1), 192 fRegisters(NULL), 193 fPCIInfo(info), 194 fStack(stack), 195 fIRQ(0), 196 fUseMSI(false), 197 fErstArea(-1), 198 fDcbaArea(-1), 199 fCmdCompSem(-1), 200 fStopThreads(false), 201 fRootHub(NULL), 202 fRootHubAddress(0), 203 fPortCount(0), 204 fSlotCount(0), 205 fScratchpadCount(0), 206 fContextSizeShift(0), 207 fFinishedHead(NULL), 208 fFinishTransfersSem(-1), 209 fFinishThread(-1), 210 fEventSem(-1), 211 fEventThread(-1), 212 fEventIdx(0), 213 fCmdIdx(0), 214 fEventCcs(1), 215 fCmdCcs(1) 216 { 217 B_INITIALIZE_SPINLOCK(&fSpinlock); 218 mutex_init(&fFinishedLock, "XHCI finished transfers"); 219 mutex_init(&fEventLock, "XHCI event handler"); 220 221 if (BusManager::InitCheck() < B_OK) { 222 TRACE_ERROR("bus manager failed to init\n"); 223 return; 224 } 225 226 TRACE("constructing new XHCI host controller driver\n"); 227 fInitOK = false; 228 229 // enable busmaster and memory mapped access 230 uint16 command = sPCIModule->read_pci_config(fPCIInfo->bus, 231 fPCIInfo->device, fPCIInfo->function, PCI_command, 2); 232 command &= ~(PCI_command_io | PCI_command_int_disable); 233 command |= PCI_command_master | PCI_command_memory; 234 235 sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device, 236 fPCIInfo->function, PCI_command, 2, command); 237 238 // map the registers (low + high for 64-bit when requested) 239 phys_addr_t physicalAddress = fPCIInfo->u.h0.base_registers[0]; 240 physicalAddress &= PCI_address_memory_32_mask; 241 if ((fPCIInfo->u.h0.base_register_flags[0] & 0xC) == PCI_address_type_64) 242 physicalAddress += (phys_addr_t)fPCIInfo->u.h0.base_registers[1] << 32; 243 244 size_t mapSize = fPCIInfo->u.h0.base_register_sizes[0]; 245 246 TRACE("map registers %08" B_PRIxPHYSADDR ", size: %" B_PRIuSIZE "\n", 247 physicalAddress, mapSize); 248 249 fRegisterArea = map_physical_memory("XHCI memory mapped registers", 250 physicalAddress, mapSize, B_ANY_KERNEL_BLOCK_ADDRESS, 251 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 252 (void **)&fRegisters); 253 if (fRegisterArea < B_OK) { 254 TRACE_ERROR("failed to map register memory\n"); 255 return; 256 } 257 258 // determine the register offsets 259 fCapabilityRegisterOffset = 0; 260 fOperationalRegisterOffset = HCI_CAPLENGTH(ReadCapReg32(XHCI_HCI_CAPLENGTH)); 261 fRuntimeRegisterOffset = ReadCapReg32(XHCI_RTSOFF) & ~0x1F; 262 fDoorbellRegisterOffset = ReadCapReg32(XHCI_DBOFF) & ~0x3; 263 264 TRACE("mapped registers: %p\n", fRegisters); 265 TRACE("operational register offset: %" B_PRId32 "\n", fOperationalRegisterOffset); 266 TRACE("runtime register offset: %" B_PRId32 "\n", fRuntimeRegisterOffset); 267 TRACE("doorbell register offset: %" B_PRId32 "\n", fDoorbellRegisterOffset); 268 269 TRACE_ALWAYS("interface version: 0x%04" B_PRIx32 "\n", 270 HCI_VERSION(ReadCapReg32(XHCI_HCI_VERSION))); 271 TRACE_ALWAYS("structural parameters: 1:0x%08" B_PRIx32 " 2:0x%08" 272 B_PRIx32 " 3:0x%08" B_PRIx32 "\n", ReadCapReg32(XHCI_HCSPARAMS1), 273 ReadCapReg32(XHCI_HCSPARAMS2), ReadCapReg32(XHCI_HCSPARAMS3)); 274 275 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 276 if (cparams == 0xffffffff) 277 return; 278 TRACE_ALWAYS("capability parameters: 0x%08" B_PRIx32 "\n", cparams); 279 280 // if 64 bytes context structures, then 1 281 fContextSizeShift = HCC_CSZ(cparams); 282 283 // Assume ownership of the controller from the BIOS. 284 uint32 eec = 0xffffffff; 285 uint32 eecp = HCS0_XECP(cparams) << 2; 286 for (; eecp != 0 && XECP_NEXT(eec); eecp += XECP_NEXT(eec) << 2) { 287 TRACE("eecp register: 0x%08" B_PRIx32 "\n", eecp); 288 289 eec = ReadCapReg32(eecp); 290 if (XECP_ID(eec) != XHCI_LEGSUP_CAPID) 291 continue; 292 293 if (eec & XHCI_LEGSUP_BIOSOWNED) { 294 TRACE_ALWAYS("the host controller is bios owned, claiming" 295 " ownership\n"); 296 WriteCapReg32(eecp, eec | XHCI_LEGSUP_OSOWNED); 297 298 for (int32 i = 0; i < 20; i++) { 299 eec = ReadCapReg32(eecp); 300 301 if ((eec & XHCI_LEGSUP_BIOSOWNED) == 0) 302 break; 303 304 TRACE_ALWAYS("controller is still bios owned, waiting\n"); 305 snooze(50000); 306 } 307 308 if (eec & XHCI_LEGSUP_BIOSOWNED) { 309 TRACE_ERROR("bios won't give up control over the host " 310 "controller (ignoring)\n"); 311 } else if (eec & XHCI_LEGSUP_OSOWNED) { 312 TRACE_ALWAYS("successfully took ownership of the host " 313 "controller\n"); 314 } 315 316 // Force off the BIOS owned flag, and clear all SMIs. Some BIOSes 317 // do indicate a successful handover but do not remove their SMIs 318 // and then freeze the system when interrupts are generated. 319 WriteCapReg32(eecp, eec & ~XHCI_LEGSUP_BIOSOWNED); 320 } 321 break; 322 } 323 uint32 legctlsts = ReadCapReg32(eecp + XHCI_LEGCTLSTS); 324 legctlsts &= XHCI_LEGCTLSTS_DISABLE_SMI; 325 legctlsts |= XHCI_LEGCTLSTS_EVENTS_SMI; 326 WriteCapReg32(eecp + XHCI_LEGCTLSTS, legctlsts); 327 328 // We need to explicitly take ownership of EHCI ports on earlier Intel chipsets. 329 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL) { 330 switch (fPCIInfo->device_id) { 331 case PCI_DEVICE_INTEL_PANTHER_POINT_XHCI: 332 case PCI_DEVICE_INTEL_LYNX_POINT_XHCI: 333 case PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI: 334 case PCI_DEVICE_INTEL_BAYTRAIL_XHCI: 335 case PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI: 336 case PCI_DEVICE_INTEL_WILDCAT_POINT_LP_XHCI: 337 _SwitchIntelPorts(); 338 break; 339 } 340 } 341 342 // halt the host controller 343 if (ControllerHalt() < B_OK) { 344 return; 345 } 346 347 // reset the host controller 348 if (ControllerReset() < B_OK) { 349 TRACE_ERROR("host controller failed to reset\n"); 350 return; 351 } 352 353 fCmdCompSem = create_sem(0, "XHCI Command Complete"); 354 fFinishTransfersSem = create_sem(0, "XHCI Finish Transfers"); 355 fEventSem = create_sem(0, "XHCI Event"); 356 if (fFinishTransfersSem < B_OK || fCmdCompSem < B_OK || fEventSem < B_OK) { 357 TRACE_ERROR("failed to create semaphores\n"); 358 return; 359 } 360 361 // create event handler thread 362 fEventThread = spawn_kernel_thread(EventThread, "xhci event thread", 363 B_URGENT_PRIORITY, (void *)this); 364 resume_thread(fEventThread); 365 366 // create finisher service thread 367 fFinishThread = spawn_kernel_thread(FinishThread, "xhci finish thread", 368 B_URGENT_PRIORITY - 1, (void *)this); 369 resume_thread(fFinishThread); 370 371 // Find the right interrupt vector, using MSIs if available. 372 fIRQ = fPCIInfo->u.h0.interrupt_line; 373 if (sPCIx86Module != NULL && sPCIx86Module->get_msi_count(fPCIInfo->bus, 374 fPCIInfo->device, fPCIInfo->function) >= 1) { 375 uint8 msiVector = 0; 376 if (sPCIx86Module->configure_msi(fPCIInfo->bus, fPCIInfo->device, 377 fPCIInfo->function, 1, &msiVector) == B_OK 378 && sPCIx86Module->enable_msi(fPCIInfo->bus, fPCIInfo->device, 379 fPCIInfo->function) == B_OK) { 380 TRACE_ALWAYS("using message signaled interrupts\n"); 381 fIRQ = msiVector; 382 fUseMSI = true; 383 } 384 } 385 386 if (fIRQ == 0 || fIRQ == 0xFF) { 387 TRACE_MODULE_ERROR("device PCI:%d:%d:%d was assigned an invalid IRQ\n", 388 fPCIInfo->bus, fPCIInfo->device, fPCIInfo->function); 389 return; 390 } 391 392 // Install the interrupt handler 393 TRACE("installing interrupt handler\n"); 394 install_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this, 0); 395 396 memset(fPortSpeeds, 0, sizeof(fPortSpeeds)); 397 memset(fPortSlots, 0, sizeof(fPortSlots)); 398 memset(fDevices, 0, sizeof(fDevices)); 399 400 fInitOK = true; 401 TRACE("driver construction successful\n"); 402 } 403 404 405 XHCI::~XHCI() 406 { 407 TRACE("tear down XHCI host controller driver\n"); 408 409 WriteOpReg(XHCI_CMD, 0); 410 411 int32 result = 0; 412 fStopThreads = true; 413 delete_sem(fCmdCompSem); 414 delete_sem(fFinishTransfersSem); 415 delete_sem(fEventSem); 416 wait_for_thread(fFinishThread, &result); 417 wait_for_thread(fEventThread, &result); 418 419 mutex_destroy(&fFinishedLock); 420 mutex_destroy(&fEventLock); 421 422 remove_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this); 423 424 delete_area(fRegisterArea); 425 delete_area(fErstArea); 426 for (uint32 i = 0; i < fScratchpadCount; i++) 427 delete_area(fScratchpadArea[i]); 428 delete_area(fDcbaArea); 429 430 if (fUseMSI && sPCIx86Module != NULL) { 431 sPCIx86Module->disable_msi(fPCIInfo->bus, 432 fPCIInfo->device, fPCIInfo->function); 433 sPCIx86Module->unconfigure_msi(fPCIInfo->bus, 434 fPCIInfo->device, fPCIInfo->function); 435 } 436 put_module(B_PCI_MODULE_NAME); 437 if (sPCIx86Module != NULL) 438 put_module(B_PCI_X86_MODULE_NAME); 439 } 440 441 442 void 443 XHCI::_SwitchIntelPorts() 444 { 445 TRACE("Looking for EHCI owned ports\n"); 446 uint32 ports = sPCIModule->read_pci_config(fPCIInfo->bus, 447 fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_USB3PRM, 4); 448 TRACE("Superspeed Ports: 0x%" B_PRIx32 "\n", ports); 449 sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device, 450 fPCIInfo->function, XHCI_INTEL_USB3_PSSEN, 4, ports); 451 ports = sPCIModule->read_pci_config(fPCIInfo->bus, 452 fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_USB3_PSSEN, 4); 453 TRACE("Superspeed ports now under XHCI : 0x%" B_PRIx32 "\n", ports); 454 ports = sPCIModule->read_pci_config(fPCIInfo->bus, 455 fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_USB2PRM, 4); 456 TRACE("USB 2.0 Ports : 0x%" B_PRIx32 "\n", ports); 457 sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device, 458 fPCIInfo->function, XHCI_INTEL_XUSB2PR, 4, ports); 459 ports = sPCIModule->read_pci_config(fPCIInfo->bus, 460 fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_XUSB2PR, 4); 461 TRACE("USB 2.0 ports now under XHCI: 0x%" B_PRIx32 "\n", ports); 462 } 463 464 465 status_t 466 XHCI::Start() 467 { 468 TRACE_ALWAYS("starting XHCI host controller\n"); 469 TRACE("usbcmd: 0x%08" B_PRIx32 "; usbsts: 0x%08" B_PRIx32 "\n", 470 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 471 472 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 473 TRACE("Start() failed STS_CNR\n"); 474 } 475 476 if ((ReadOpReg(XHCI_CMD) & CMD_RUN) != 0) { 477 TRACE_ERROR("Start() warning, starting running XHCI controller!\n"); 478 } 479 480 if ((ReadOpReg(XHCI_PAGESIZE) & (1 << 0)) == 0) { 481 TRACE_ERROR("controller does not support 4K page size\n"); 482 return B_ERROR; 483 } 484 485 // read port count from capability register 486 uint32 capabilities = ReadCapReg32(XHCI_HCSPARAMS1); 487 fPortCount = HCS_MAX_PORTS(capabilities); 488 if (fPortCount == 0) { 489 TRACE_ERROR("invalid number of ports: %u\n", fPortCount); 490 return B_ERROR; 491 } 492 493 fSlotCount = HCS_MAX_SLOTS(capabilities); 494 if (fSlotCount > XHCI_MAX_DEVICES) 495 fSlotCount = XHCI_MAX_DEVICES; 496 WriteOpReg(XHCI_CONFIG, fSlotCount); 497 498 // find out which protocol is used for each port 499 uint8 portFound = 0; 500 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 501 uint32 eec = 0xffffffff; 502 uint32 eecp = HCS0_XECP(cparams) << 2; 503 for (; eecp != 0 && XECP_NEXT(eec) && portFound < fPortCount; 504 eecp += XECP_NEXT(eec) << 2) { 505 eec = ReadCapReg32(eecp); 506 if (XECP_ID(eec) != XHCI_SUPPORTED_PROTOCOLS_CAPID) 507 continue; 508 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) > 3) 509 continue; 510 uint32 temp = ReadCapReg32(eecp + 8); 511 uint32 offset = XHCI_SUPPORTED_PROTOCOLS_1_OFFSET(temp); 512 uint32 count = XHCI_SUPPORTED_PROTOCOLS_1_COUNT(temp); 513 if (offset == 0 || count == 0) 514 continue; 515 offset--; 516 for (uint32 i = offset; i < offset + count; i++) { 517 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) == 0x3) 518 fPortSpeeds[i] = USB_SPEED_SUPERSPEED; 519 else 520 fPortSpeeds[i] = USB_SPEED_HIGHSPEED; 521 522 TRACE("speed for port %" B_PRId32 " is %s\n", i, 523 fPortSpeeds[i] == USB_SPEED_SUPERSPEED ? "super" : "high"); 524 } 525 portFound += count; 526 } 527 528 uint32 params2 = ReadCapReg32(XHCI_HCSPARAMS2); 529 fScratchpadCount = HCS_MAX_SC_BUFFERS(params2); 530 if (fScratchpadCount > XHCI_MAX_SCRATCHPADS) { 531 TRACE_ERROR("invalid number of scratchpads: %" B_PRIu32 "\n", 532 fScratchpadCount); 533 return B_ERROR; 534 } 535 536 uint32 params3 = ReadCapReg32(XHCI_HCSPARAMS3); 537 fExitLatMax = HCS_U1_DEVICE_LATENCY(params3) 538 + HCS_U2_DEVICE_LATENCY(params3); 539 540 // clear interrupts & disable device notifications 541 WriteOpReg(XHCI_STS, ReadOpReg(XHCI_STS)); 542 WriteOpReg(XHCI_DNCTRL, 0); 543 544 // allocate Device Context Base Address array 545 phys_addr_t dmaAddress; 546 fDcbaArea = fStack->AllocateArea((void **)&fDcba, &dmaAddress, 547 sizeof(*fDcba), "DCBA Area"); 548 if (fDcbaArea < B_OK) { 549 TRACE_ERROR("unable to create the DCBA area\n"); 550 return B_ERROR; 551 } 552 memset(fDcba, 0, sizeof(*fDcba)); 553 memset(fScratchpadArea, 0, sizeof(fScratchpadArea)); 554 memset(fScratchpad, 0, sizeof(fScratchpad)); 555 556 // setting the first address to the scratchpad array address 557 fDcba->baseAddress[0] = dmaAddress 558 + offsetof(struct xhci_device_context_array, scratchpad); 559 560 // fill up the scratchpad array with scratchpad pages 561 for (uint32 i = 0; i < fScratchpadCount; i++) { 562 phys_addr_t scratchDmaAddress; 563 fScratchpadArea[i] = fStack->AllocateArea((void **)&fScratchpad[i], 564 &scratchDmaAddress, B_PAGE_SIZE, "Scratchpad Area"); 565 if (fScratchpadArea[i] < B_OK) { 566 TRACE_ERROR("unable to create the scratchpad area\n"); 567 return B_ERROR; 568 } 569 fDcba->scratchpad[i] = scratchDmaAddress; 570 } 571 572 TRACE("setting DCBAAP %" B_PRIxPHYSADDR "\n", dmaAddress); 573 WriteOpReg(XHCI_DCBAAP_LO, (uint32)dmaAddress); 574 WriteOpReg(XHCI_DCBAAP_HI, (uint32)(dmaAddress >> 32)); 575 576 // allocate Event Ring Segment Table 577 uint8 *addr; 578 fErstArea = fStack->AllocateArea((void **)&addr, &dmaAddress, 579 (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 580 + sizeof(xhci_erst_element), 581 "USB XHCI ERST CMD_RING and EVENT_RING Area"); 582 583 if (fErstArea < B_OK) { 584 TRACE_ERROR("unable to create the ERST AND RING area\n"); 585 delete_area(fDcbaArea); 586 return B_ERROR; 587 } 588 fErst = (xhci_erst_element *)addr; 589 memset(fErst, 0, (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 590 + sizeof(xhci_erst_element)); 591 592 // fill with Event Ring Segment Base Address and Event Ring Segment Size 593 fErst->rs_addr = dmaAddress + sizeof(xhci_erst_element); 594 fErst->rs_size = XHCI_MAX_EVENTS; 595 fErst->rsvdz = 0; 596 597 addr += sizeof(xhci_erst_element); 598 fEventRing = (xhci_trb *)addr; 599 addr += XHCI_MAX_EVENTS * sizeof(xhci_trb); 600 fCmdRing = (xhci_trb *)addr; 601 602 TRACE("setting ERST size\n"); 603 WriteRunReg32(XHCI_ERSTSZ(0), XHCI_ERSTS_SET(1)); 604 605 TRACE("setting ERDP addr = 0x%" B_PRIx64 "\n", fErst->rs_addr); 606 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)fErst->rs_addr); 607 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(fErst->rs_addr >> 32)); 608 609 TRACE("setting ERST base addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 610 WriteRunReg32(XHCI_ERSTBA_LO(0), (uint32)dmaAddress); 611 WriteRunReg32(XHCI_ERSTBA_HI(0), (uint32)(dmaAddress >> 32)); 612 613 dmaAddress += sizeof(xhci_erst_element) + XHCI_MAX_EVENTS 614 * sizeof(xhci_trb); 615 616 // Make sure the Command Ring is stopped 617 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 618 TRACE_ALWAYS("Command Ring is running, send stop/cancel\n"); 619 WriteOpReg(XHCI_CRCR_LO, CRCR_CS); 620 WriteOpReg(XHCI_CRCR_HI, 0); 621 WriteOpReg(XHCI_CRCR_LO, CRCR_CA); 622 WriteOpReg(XHCI_CRCR_HI, 0); 623 snooze(1000); 624 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 625 TRACE_ERROR("Command Ring still running after stop/cancel\n"); 626 } 627 } 628 TRACE("setting CRCR addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 629 WriteOpReg(XHCI_CRCR_LO, (uint32)dmaAddress | CRCR_RCS); 630 WriteOpReg(XHCI_CRCR_HI, (uint32)(dmaAddress >> 32)); 631 // link trb 632 fCmdRing[XHCI_MAX_COMMANDS - 1].address = dmaAddress; 633 634 TRACE("setting interrupt rate\n"); 635 636 // Setting IMOD below 0x3F8 on Intel Lynx Point can cause IRQ lockups 637 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL 638 && (fPCIInfo->device_id == PCI_DEVICE_INTEL_PANTHER_POINT_XHCI 639 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_XHCI 640 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI 641 || fPCIInfo->device_id == PCI_DEVICE_INTEL_BAYTRAIL_XHCI 642 || fPCIInfo->device_id == PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI)) { 643 WriteRunReg32(XHCI_IMOD(0), 0x000003f8); // 4000 irq/s 644 } else { 645 WriteRunReg32(XHCI_IMOD(0), 0x000001f4); // 8000 irq/s 646 } 647 648 TRACE("enabling interrupt\n"); 649 WriteRunReg32(XHCI_IMAN(0), ReadRunReg32(XHCI_IMAN(0)) | IMAN_INTR_ENA); 650 651 WriteOpReg(XHCI_CMD, CMD_RUN | CMD_INTE | CMD_HSEE); 652 653 // wait for start up state 654 if (WaitOpBits(XHCI_STS, STS_HCH, 0) != B_OK) { 655 TRACE_ERROR("HCH start up timeout\n"); 656 } 657 658 fRootHubAddress = AllocateAddress(); 659 fRootHub = new(std::nothrow) XHCIRootHub(RootObject(), fRootHubAddress); 660 if (!fRootHub) { 661 TRACE_ERROR("no memory to allocate root hub\n"); 662 return B_NO_MEMORY; 663 } 664 665 if (fRootHub->InitCheck() < B_OK) { 666 TRACE_ERROR("root hub failed init check\n"); 667 return fRootHub->InitCheck(); 668 } 669 670 SetRootHub(fRootHub); 671 672 TRACE_ALWAYS("successfully started the controller\n"); 673 674 #ifdef TRACE_USB 675 TRACE("No-Op test...\n"); 676 Noop(); 677 #endif 678 679 return BusManager::Start(); 680 } 681 682 683 status_t 684 XHCI::SubmitTransfer(Transfer *transfer) 685 { 686 // short circuit the root hub 687 if (transfer->TransferPipe()->DeviceAddress() == fRootHubAddress) 688 return fRootHub->ProcessTransfer(this, transfer); 689 690 TRACE("SubmitTransfer(%p)\n", transfer); 691 Pipe *pipe = transfer->TransferPipe(); 692 if ((pipe->Type() & USB_OBJECT_CONTROL_PIPE) != 0) 693 return SubmitControlRequest(transfer); 694 return SubmitNormalRequest(transfer); 695 } 696 697 698 status_t 699 XHCI::SubmitControlRequest(Transfer *transfer) 700 { 701 Pipe *pipe = transfer->TransferPipe(); 702 usb_request_data *requestData = transfer->RequestData(); 703 bool directionIn = (requestData->RequestType & USB_REQTYPE_DEVICE_IN) != 0; 704 705 TRACE("SubmitControlRequest() length %d\n", requestData->Length); 706 707 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 708 if (endpoint == NULL) { 709 TRACE_ERROR("invalid endpoint!\n"); 710 return B_BAD_VALUE; 711 } 712 status_t status = transfer->InitKernelAccess(); 713 if (status != B_OK) 714 return status; 715 716 xhci_td *descriptor = CreateDescriptor(3, 1, requestData->Length); 717 if (descriptor == NULL) 718 return B_NO_MEMORY; 719 descriptor->transfer = transfer; 720 721 // Setup Stage 722 uint8 index = 0; 723 memcpy(&descriptor->trbs[index].address, requestData, 724 sizeof(usb_request_data)); 725 descriptor->trbs[index].status = TRB_2_IRQ(0) | TRB_2_BYTES(8); 726 descriptor->trbs[index].flags 727 = TRB_3_TYPE(TRB_TYPE_SETUP_STAGE) | TRB_3_IDT_BIT | TRB_3_CYCLE_BIT; 728 if (requestData->Length > 0) { 729 descriptor->trbs[index].flags |= 730 directionIn ? TRB_3_TRT_IN : TRB_3_TRT_OUT; 731 } 732 733 index++; 734 735 // Data Stage (if any) 736 if (requestData->Length > 0) { 737 descriptor->trbs[index].address = descriptor->buffer_addrs[0]; 738 descriptor->trbs[index].status = TRB_2_IRQ(0) 739 | TRB_2_BYTES(requestData->Length) 740 | TRB_2_TD_SIZE(0); 741 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_DATA_STAGE) 742 | (directionIn ? (TRB_3_DIR_IN | TRB_3_ISP_BIT) : 0) 743 | TRB_3_CYCLE_BIT; 744 745 if (!directionIn) { 746 transfer->PrepareKernelAccess(); 747 memcpy(descriptor->buffers[0], 748 (uint8 *)transfer->Vector()[0].iov_base, requestData->Length); 749 } 750 751 index++; 752 } 753 754 // Status Stage 755 descriptor->trbs[index].address = 0; 756 descriptor->trbs[index].status = TRB_2_IRQ(0); 757 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_STATUS_STAGE) 758 | ((directionIn && requestData->Length > 0) ? 0 : TRB_3_DIR_IN) 759 | TRB_3_CHAIN_BIT | TRB_3_ENT_BIT | TRB_3_CYCLE_BIT; 760 // Status Stage is an OUT transfer when the device is sending data 761 // (XHCI 1.2 § 4.11.2.2 Table 4-7 p213), and the CHAIN bit must be 762 // set when using an Event Data TRB (as _LinkDescriptorForPipe does) 763 // (XHCI 1.2 § 6.4.1.2.3 Table 6-31 p472) 764 765 descriptor->trb_used = index + 1; 766 767 status = _LinkDescriptorForPipe(descriptor, endpoint); 768 if (status != B_OK) { 769 FreeDescriptor(descriptor); 770 return status; 771 } 772 773 return B_OK; 774 } 775 776 777 status_t 778 XHCI::SubmitNormalRequest(Transfer *transfer) 779 { 780 TRACE("SubmitNormalRequest() length %ld\n", transfer->DataLength()); 781 782 Pipe *pipe = transfer->TransferPipe(); 783 usb_isochronous_data *isochronousData = transfer->IsochronousData(); 784 bool directionIn = (pipe->Direction() == Pipe::In); 785 786 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 787 if (endpoint == NULL) 788 return B_BAD_VALUE; 789 790 status_t status = transfer->InitKernelAccess(); 791 if (status != B_OK) 792 return status; 793 794 // TRBs within a TD must be "grouped" into TD Fragments, which mostly means 795 // that a max_burst_payload boundary cannot be crossed within a TRB, but 796 // only between TRBs. More than one TRB can be in a TD Fragment, but we keep 797 // things simple by setting trbSize to the MBP. (XHCI 1.2 § 4.11.7.1 p235.) 798 size_t trbSize = endpoint->max_burst_payload; 799 800 if (isochronousData != NULL) { 801 if (isochronousData->packet_count == 0) 802 return B_BAD_VALUE; 803 804 // Isochronous transfers use more specifically sized packets. 805 trbSize = transfer->DataLength() / isochronousData->packet_count; 806 if (trbSize > pipe->MaxPacketSize() || trbSize 807 != (size_t)isochronousData->packet_descriptors[0].request_length) 808 return B_BAD_VALUE; 809 } 810 811 // Now that we know trbSize, compute the count. 812 const int32 trbCount = (transfer->DataLength() + trbSize - 1) / trbSize; 813 814 xhci_td *td = CreateDescriptor(trbCount, trbCount, trbSize); 815 if (td == NULL) 816 return B_NO_MEMORY; 817 818 // Normal Stage 819 size_t remaining = transfer->DataLength(); 820 for (int32 i = 0; i < trbCount; i++) { 821 // The "TD Size" field of a transfer TRB indicates the number of 822 // remaining maximum-size *packets* in this TD, *not* including the 823 // packets in the current TRB, and capped at 31 if there are more 824 // than 31 packets remaining in the TD. (XHCI 1.2 § 4.11.2.4 p218.) 825 int32 tdSize = remaining / pipe->MaxPacketSize(); 826 if (tdSize > 31) 827 tdSize = 31; 828 int32 trbLength = (remaining < trbSize) ? remaining : trbSize; 829 830 td->trbs[i].address = td->buffer_addrs[i]; 831 td->trbs[i].status = TRB_2_IRQ(0) 832 | TRB_2_BYTES(trbLength) 833 | TRB_2_TD_SIZE(tdSize); 834 td->trbs[i].flags = TRB_3_TYPE(TRB_TYPE_NORMAL) 835 | TRB_3_CYCLE_BIT | TRB_3_CHAIN_BIT 836 | (directionIn ? TRB_3_ISP_BIT : 0); 837 838 td->trb_used++; 839 remaining -= trbLength; 840 } 841 842 // Isochronous-specific 843 if (isochronousData != NULL) { 844 // This is an isochronous transfer; we need to make the first TRB 845 // an isochronous TRB. 846 td->trbs[0].flags &= ~(TRB_3_TYPE(TRB_TYPE_NORMAL)); 847 td->trbs[0].flags |= TRB_3_TYPE(TRB_TYPE_ISOCH); 848 849 // Isochronous pipes are scheduled by microframes, one of which 850 // is 125us for USB 2 and above. But for USB 1 it was 1ms, so 851 // we need to use a different frame delta for that case. 852 uint8 frameDelta = 1; 853 if (transfer->TransferPipe()->Speed() == USB_SPEED_FULLSPEED) 854 frameDelta = 8; 855 856 // TODO: We do not currently take Mult into account at all! 857 // How are we supposed to do that here? 858 859 // Determine the (starting) frame number: if ISO_ASAP is set, 860 // we are queueing this "right away", and so want to reset 861 // the starting_frame_number. Otherwise we use the passed one. 862 uint32 frame; 863 if ((isochronousData->flags & USB_ISO_ASAP) != 0 864 || isochronousData->starting_frame_number == NULL) { 865 frame = ReadRunReg32(XHCI_MFINDEX) + 1; 866 // TODO: The +1 comes from the XHCI spec; document that. 867 td->trbs[0].flags |= TRB_3_ISO_SIA_BIT; 868 } else { 869 frame = *isochronousData->starting_frame_number; 870 td->trbs[0].flags |= TRB_3_FRID(frame); 871 } 872 frame = (frame + frameDelta) % 2048; 873 if (isochronousData->starting_frame_number != NULL) 874 *isochronousData->starting_frame_number = frame; 875 876 // TODO: The OHCI bus driver seems to also do this for inbound 877 // isochronous transfers. Perhaps it should be moved into the stack? 878 if (directionIn) { 879 for (uint32 i = 0; i < isochronousData->packet_count; i++) { 880 isochronousData->packet_descriptors[i].actual_length = 0; 881 isochronousData->packet_descriptors[i].status = B_NO_INIT; 882 } 883 } 884 } 885 886 // Set the ENT (Evaluate Next TRB) bit, so that the HC will not switch 887 // contexts before evaluating the Link TRB that _LinkDescriptorForPipe 888 // will insert, as otherwise there would be a race between us freeing 889 // and unlinking the descriptor, and the controller evaluating the Link TRB 890 // and thus getting back onto the main ring and executing the Event Data 891 // TRB that generates the interrupt for this transfer. 892 // 893 // Note that we *do not* unset the CHAIN bit in this TRB, thus including 894 // the Link TRB in this TD formally, which is required when using the 895 // ENT bit. (XHCI 1.2 § 4.12.3 p250.) 896 td->trbs[td->trb_used - 1].flags |= TRB_3_ENT_BIT; 897 898 if (!directionIn) { 899 TRACE("copying out iov count %ld\n", transfer->VectorCount()); 900 status_t status = transfer->PrepareKernelAccess(); 901 if (status != B_OK) { 902 FreeDescriptor(td); 903 return status; 904 } 905 WriteDescriptor(td, transfer->Vector(), transfer->VectorCount()); 906 } 907 908 td->transfer = transfer; 909 status = _LinkDescriptorForPipe(td, endpoint); 910 if (status != B_OK) { 911 FreeDescriptor(td); 912 return status; 913 } 914 915 return B_OK; 916 } 917 918 919 status_t 920 XHCI::CancelQueuedTransfers(Pipe *pipe, bool force) 921 { 922 xhci_endpoint* endpoint = (xhci_endpoint*)pipe->ControllerCookie(); 923 if (endpoint == NULL || endpoint->trbs == NULL) { 924 // Someone's de-allocated this pipe or endpoint in the meantime. 925 // (Possibly AllocateDevice failed, and we were the temporary pipe.) 926 return B_NO_INIT; 927 } 928 929 TRACE_ALWAYS("cancel queued transfers (%" B_PRId8 ") for pipe %p (%d)\n", 930 endpoint->used, pipe, pipe->EndpointAddress()); 931 932 MutexLocker endpointLocker(endpoint->lock); 933 934 if (endpoint->td_head == NULL) { 935 // There aren't any currently pending transfers to cancel. 936 return B_OK; 937 } 938 939 // Get the head TD from the endpoint. 940 xhci_td* td_head = endpoint->td_head; 941 endpoint->td_head = NULL; 942 943 // We don't want to call the callbacks while holding the endpoint lock, 944 // as they could potentially cause deadlocks, so we instead store 945 // them in a pointer array. We need to do this separately from freeing 946 // the TDs, for in the case we fail to stop the endpoint, we cancel 947 // the transfers but do not free the TDs. 948 Transfer* transfers[XHCI_MAX_TRANSFERS]; 949 int32 transfersCount = 0; 950 951 // We can't cancel or delete transfers under "force", as they probably 952 // are not safe to use anymore. 953 for (xhci_td* td = td_head; td != NULL; td = td->next) { 954 if (td->transfer == NULL) 955 continue; 956 if (!force) { 957 transfers[transfersCount] = td->transfer; 958 transfersCount++; 959 } 960 td->transfer = NULL; 961 } 962 963 status_t status = StopEndpoint(false, endpoint->id + 1, 964 endpoint->device->slot); 965 if (status == B_OK) { 966 // Clear the endpoint's TRBs. 967 memset(endpoint->trbs, 0, sizeof(xhci_trb) * XHCI_ENDPOINT_RING_SIZE); 968 endpoint->used = 0; 969 endpoint->current = 0; 970 971 // Set dequeue pointer location to the beginning of the ring. 972 SetTRDequeue(endpoint->trb_addr, 0, endpoint->id + 1, 973 endpoint->device->slot); 974 975 // We don't need to do anything else to restart the ring, as it will resume 976 // operation as normal upon the next doorbell. (XHCI 1.2 § 4.6.9 p136.) 977 } else { 978 // We couldn't stop the endpoint. Most likely the device has been 979 // removed and the endpoint was stopped by the hardware, or is 980 // for some reason busy and cannot be stopped. 981 TRACE_ERROR("cancel queued transfers: could not stop endpoint: %s!\n", 982 strerror(status)); 983 984 // Instead of freeing the TDs, we want to leave them in the endpoint 985 // so that when/if the hardware returns, they can be properly unlinked, 986 // as otherwise the endpoint could get "stuck" by having the "used" 987 // slowly accumulate due to "dead" transfers. 988 endpoint->td_head = td_head; 989 td_head = NULL; 990 } 991 992 endpointLocker.Unlock(); 993 994 for (int32 i = 0; i < transfersCount; i++) { 995 transfers[i]->Finished(B_CANCELED, 0); 996 delete transfers[i]; 997 } 998 999 // This loop looks a bit strange because we need to store the "next" 1000 // pointer before freeing the descriptor. 1001 xhci_td* td; 1002 while ((td = td_head) != NULL) { 1003 td_head = td_head->next; 1004 FreeDescriptor(td); 1005 } 1006 1007 return B_OK; 1008 } 1009 1010 1011 status_t 1012 XHCI::StartDebugTransfer(Transfer *transfer) 1013 { 1014 Pipe *pipe = transfer->TransferPipe(); 1015 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 1016 if (endpoint == NULL) 1017 return B_BAD_VALUE; 1018 1019 // Check all locks that we are going to hit when running transfers. 1020 if (mutex_trylock(&endpoint->lock) != B_OK) 1021 return B_WOULD_BLOCK; 1022 if (mutex_trylock(&fFinishedLock) != B_OK) { 1023 mutex_unlock(&endpoint->lock); 1024 return B_WOULD_BLOCK; 1025 } 1026 if (mutex_trylock(&fEventLock) != B_OK) { 1027 mutex_unlock(&endpoint->lock); 1028 mutex_unlock(&fFinishedLock); 1029 return B_WOULD_BLOCK; 1030 } 1031 mutex_unlock(&endpoint->lock); 1032 mutex_unlock(&fFinishedLock); 1033 mutex_unlock(&fEventLock); 1034 1035 status_t status = SubmitTransfer(transfer); 1036 if (status != B_OK) 1037 return status; 1038 1039 // The endpoint's head TD is the TD of the just-submitted transfer. 1040 // Just like EHCI, abuse the callback cookie to hold the TD pointer. 1041 transfer->SetCallback(NULL, endpoint->td_head); 1042 1043 return B_OK; 1044 } 1045 1046 1047 status_t 1048 XHCI::CheckDebugTransfer(Transfer *transfer) 1049 { 1050 xhci_td *transfer_td = (xhci_td *)transfer->CallbackCookie(); 1051 if (transfer_td == NULL) 1052 return B_NO_INIT; 1053 1054 // Process events once, and then look for it in the finished list. 1055 ProcessEvents(); 1056 xhci_td *previous = NULL; 1057 for (xhci_td *td = fFinishedHead; td != NULL; td = td->next) { 1058 if (td != transfer_td) { 1059 previous = td; 1060 continue; 1061 } 1062 1063 // We've found it! 1064 if (previous == NULL) { 1065 fFinishedHead = fFinishedHead->next; 1066 } else { 1067 previous->next = td->next; 1068 } 1069 1070 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 1071 status_t status = (td->trb_completion_code == COMP_SUCCESS 1072 || td->trb_completion_code == COMP_SHORT_PACKET) ? B_OK : B_ERROR; 1073 1074 if (status == B_OK && directionIn) 1075 ReadDescriptor(td, transfer->Vector(), transfer->VectorCount()); 1076 1077 FreeDescriptor(td); 1078 transfer->SetCallback(NULL, NULL); 1079 return status; 1080 } 1081 1082 // We didn't find it. 1083 spin(75); 1084 return B_DEV_PENDING; 1085 } 1086 1087 1088 void 1089 XHCI::CancelDebugTransfer(Transfer *transfer) 1090 { 1091 while (CheckDebugTransfer(transfer) == B_DEV_PENDING) 1092 spin(100); 1093 } 1094 1095 1096 status_t 1097 XHCI::NotifyPipeChange(Pipe *pipe, usb_change change) 1098 { 1099 TRACE("pipe change %d for pipe %p (%d)\n", change, pipe, 1100 pipe->EndpointAddress()); 1101 1102 switch (change) { 1103 case USB_CHANGE_CREATED: 1104 return _InsertEndpointForPipe(pipe); 1105 case USB_CHANGE_DESTROYED: 1106 return _RemoveEndpointForPipe(pipe); 1107 1108 case USB_CHANGE_PIPE_POLICY_CHANGED: 1109 // We don't care about these, at least for now. 1110 return B_OK; 1111 } 1112 1113 TRACE_ERROR("unknown pipe change!\n"); 1114 return B_UNSUPPORTED; 1115 } 1116 1117 1118 xhci_td * 1119 XHCI::CreateDescriptor(uint32 trbCount, uint32 bufferCount, size_t bufferSize) 1120 { 1121 const bool inKDL = debug_debugger_running(); 1122 1123 xhci_td *result; 1124 if (!inKDL) { 1125 result = (xhci_td*)calloc(1, sizeof(xhci_td)); 1126 } else { 1127 // Just use the physical memory allocator while in KDL; it's less 1128 // secure than using the regular heap, but it's easier to deal with. 1129 phys_addr_t dummy; 1130 fStack->AllocateChunk((void **)&result, &dummy, sizeof(xhci_td)); 1131 } 1132 1133 if (result == NULL) { 1134 TRACE_ERROR("failed to allocate a transfer descriptor\n"); 1135 return NULL; 1136 } 1137 1138 // We always allocate 1 more TRB than requested, so that 1139 // _LinkDescriptorForPipe() has room to insert a link TRB. 1140 trbCount++; 1141 if (fStack->AllocateChunk((void **)&result->trbs, &result->trb_addr, 1142 (trbCount * sizeof(xhci_trb))) < B_OK) { 1143 TRACE_ERROR("failed to allocate TRBs\n"); 1144 FreeDescriptor(result); 1145 return NULL; 1146 } 1147 result->trb_count = trbCount; 1148 result->trb_used = 0; 1149 1150 if (bufferSize > 0) { 1151 // Due to how the USB stack allocates physical memory, we can't just 1152 // request one large chunk the size of the transfer, and so instead we 1153 // create a series of buffers as requested by our caller. 1154 1155 // We store the buffer pointers and addresses in one memory block. 1156 if (!inKDL) { 1157 result->buffers = (void**)calloc(bufferCount, 1158 (sizeof(void*) + sizeof(phys_addr_t))); 1159 } else { 1160 phys_addr_t dummy; 1161 fStack->AllocateChunk((void **)&result->buffers, &dummy, 1162 bufferCount * (sizeof(void*) + sizeof(phys_addr_t))); 1163 } 1164 if (result->buffers == NULL) { 1165 TRACE_ERROR("unable to allocate space for buffer infos\n"); 1166 FreeDescriptor(result); 1167 return NULL; 1168 } 1169 result->buffer_addrs = (phys_addr_t*)&result->buffers[bufferCount]; 1170 result->buffer_size = bufferSize; 1171 result->buffer_count = bufferCount; 1172 1173 // Optimization: If the requested total size of all buffers is less 1174 // than 32*B_PAGE_SIZE (the maximum size that the physical memory 1175 // allocator can handle), we allocate only one buffer and segment it. 1176 size_t totalSize = bufferSize * bufferCount; 1177 if (totalSize < (32 * B_PAGE_SIZE)) { 1178 if (fStack->AllocateChunk(&result->buffers[0], 1179 &result->buffer_addrs[0], totalSize) < B_OK) { 1180 TRACE_ERROR("unable to allocate space for large buffer (size %ld)\n", 1181 totalSize); 1182 FreeDescriptor(result); 1183 return NULL; 1184 } 1185 for (uint32 i = 1; i < bufferCount; i++) { 1186 result->buffers[i] = (void*)((addr_t)(result->buffers[i - 1]) 1187 + bufferSize); 1188 result->buffer_addrs[i] = result->buffer_addrs[i - 1] 1189 + bufferSize; 1190 } 1191 } else { 1192 // Otherwise, we allocate each buffer individually. 1193 for (uint32 i = 0; i < bufferCount; i++) { 1194 if (fStack->AllocateChunk(&result->buffers[i], 1195 &result->buffer_addrs[i], bufferSize) < B_OK) { 1196 TRACE_ERROR("unable to allocate space for a buffer (size " 1197 "%" B_PRIuSIZE ", count %" B_PRIu32 ")\n", 1198 bufferSize, bufferCount); 1199 FreeDescriptor(result); 1200 return NULL; 1201 } 1202 } 1203 } 1204 } else { 1205 result->buffers = NULL; 1206 result->buffer_addrs = NULL; 1207 } 1208 1209 // Initialize all other fields. 1210 result->transfer = NULL; 1211 result->trb_completion_code = 0; 1212 result->trb_left = 0; 1213 result->next = NULL; 1214 1215 TRACE("CreateDescriptor allocated %p, buffer_size %ld, buffer_count %" B_PRIu32 "\n", 1216 result, result->buffer_size, result->buffer_count); 1217 1218 return result; 1219 } 1220 1221 1222 void 1223 XHCI::FreeDescriptor(xhci_td *descriptor) 1224 { 1225 if (descriptor == NULL) 1226 return; 1227 1228 const bool inKDL = debug_debugger_running(); 1229 1230 if (descriptor->trbs != NULL) { 1231 fStack->FreeChunk(descriptor->trbs, descriptor->trb_addr, 1232 (descriptor->trb_count * sizeof(xhci_trb))); 1233 } 1234 if (descriptor->buffers != NULL) { 1235 size_t totalSize = descriptor->buffer_size * descriptor->buffer_count; 1236 if (totalSize < (32 * B_PAGE_SIZE)) { 1237 // This was allocated as one contiguous buffer. 1238 fStack->FreeChunk(descriptor->buffers[0], descriptor->buffer_addrs[0], 1239 totalSize); 1240 } else { 1241 for (uint32 i = 0; i < descriptor->buffer_count; i++) { 1242 if (descriptor->buffers[i] == NULL) 1243 continue; 1244 fStack->FreeChunk(descriptor->buffers[i], descriptor->buffer_addrs[i], 1245 descriptor->buffer_size); 1246 } 1247 } 1248 1249 if (!inKDL) { 1250 free(descriptor->buffers); 1251 } else { 1252 fStack->FreeChunk(descriptor->buffers, 0, 1253 descriptor->buffer_count * (sizeof(void*) + sizeof(phys_addr_t))); 1254 } 1255 } 1256 1257 if (!inKDL) 1258 free(descriptor); 1259 else 1260 fStack->FreeChunk(descriptor, 0, sizeof(xhci_td)); 1261 } 1262 1263 1264 size_t 1265 XHCI::WriteDescriptor(xhci_td *descriptor, iovec *vector, size_t vectorCount) 1266 { 1267 size_t written = 0; 1268 1269 size_t bufIdx = 0, bufUsed = 0; 1270 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1271 size_t length = vector[vecIdx].iov_len; 1272 1273 while (length > 0 && bufIdx < descriptor->buffer_count) { 1274 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1275 memcpy((uint8 *)descriptor->buffers[bufIdx] + bufUsed, 1276 (uint8 *)vector[vecIdx].iov_base + (vector[vecIdx].iov_len - length), 1277 toCopy); 1278 1279 written += toCopy; 1280 bufUsed += toCopy; 1281 length -= toCopy; 1282 if (bufUsed == descriptor->buffer_size) { 1283 bufIdx++; 1284 bufUsed = 0; 1285 } 1286 } 1287 } 1288 1289 TRACE("wrote descriptor (%" B_PRIuSIZE " bytes)\n", written); 1290 return written; 1291 } 1292 1293 1294 size_t 1295 XHCI::ReadDescriptor(xhci_td *descriptor, iovec *vector, size_t vectorCount) 1296 { 1297 size_t read = 0; 1298 1299 size_t bufIdx = 0, bufUsed = 0; 1300 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1301 size_t length = vector[vecIdx].iov_len; 1302 1303 while (length > 0 && bufIdx < descriptor->buffer_count) { 1304 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1305 memcpy((uint8 *)vector[vecIdx].iov_base + (vector[vecIdx].iov_len - length), 1306 (uint8 *)descriptor->buffers[bufIdx] + bufUsed, toCopy); 1307 1308 read += toCopy; 1309 bufUsed += toCopy; 1310 length -= toCopy; 1311 if (bufUsed == descriptor->buffer_size) { 1312 bufIdx++; 1313 bufUsed = 0; 1314 } 1315 } 1316 } 1317 1318 TRACE("read descriptor (%" B_PRIuSIZE " bytes)\n", read); 1319 return read; 1320 } 1321 1322 1323 Device * 1324 XHCI::AllocateDevice(Hub *parent, int8 hubAddress, uint8 hubPort, 1325 usb_speed speed) 1326 { 1327 TRACE("AllocateDevice hubAddress %d hubPort %d speed %d\n", hubAddress, 1328 hubPort, speed); 1329 1330 uint8 slot = XHCI_MAX_SLOTS; 1331 if (EnableSlot(&slot) != B_OK) { 1332 TRACE_ERROR("AllocateDevice() failed enable slot\n"); 1333 return NULL; 1334 } 1335 1336 if (slot == 0 || slot > fSlotCount) { 1337 TRACE_ERROR("AllocateDevice() bad slot\n"); 1338 return NULL; 1339 } 1340 1341 if (fDevices[slot].state != XHCI_STATE_DISABLED) { 1342 TRACE_ERROR("AllocateDevice() slot already used\n"); 1343 return NULL; 1344 } 1345 1346 struct xhci_device *device = &fDevices[slot]; 1347 memset(device, 0, sizeof(struct xhci_device)); 1348 device->state = XHCI_STATE_ENABLED; 1349 device->slot = slot; 1350 1351 device->input_ctx_area = fStack->AllocateArea((void **)&device->input_ctx, 1352 &device->input_ctx_addr, sizeof(*device->input_ctx) << fContextSizeShift, 1353 "XHCI input context"); 1354 if (device->input_ctx_area < B_OK) { 1355 TRACE_ERROR("unable to create a input context area\n"); 1356 device->state = XHCI_STATE_DISABLED; 1357 return NULL; 1358 } 1359 1360 memset(device->input_ctx, 0, sizeof(*device->input_ctx) << fContextSizeShift); 1361 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1362 _WriteContext(&device->input_ctx->input.addFlags, 3); 1363 1364 uint32 route = 0; 1365 uint8 routePort = hubPort; 1366 uint8 rhPort = hubPort; 1367 for (Device *hubDevice = parent; hubDevice != RootObject(); 1368 hubDevice = (Device *)hubDevice->Parent()) { 1369 1370 rhPort = routePort; 1371 if (hubDevice->Parent() == RootObject()) 1372 break; 1373 route *= 16; 1374 if (hubPort > 15) 1375 route += 15; 1376 else 1377 route += routePort; 1378 1379 routePort = hubDevice->HubPort(); 1380 } 1381 1382 // Get speed of port, only if device connected to root hub port 1383 // else we have to rely on value reported by the Hub Explore thread 1384 if (route == 0) { 1385 GetPortSpeed(hubPort - 1, &speed); 1386 TRACE("speed updated %d\n", speed); 1387 } 1388 1389 uint32 dwslot0 = SLOT_0_NUM_ENTRIES(1) | SLOT_0_ROUTE(route); 1390 1391 // add the speed 1392 switch (speed) { 1393 case USB_SPEED_LOWSPEED: 1394 dwslot0 |= SLOT_0_SPEED(2); 1395 break; 1396 case USB_SPEED_HIGHSPEED: 1397 dwslot0 |= SLOT_0_SPEED(3); 1398 break; 1399 case USB_SPEED_FULLSPEED: 1400 dwslot0 |= SLOT_0_SPEED(1); 1401 break; 1402 case USB_SPEED_SUPERSPEED: 1403 dwslot0 |= SLOT_0_SPEED(4); 1404 break; 1405 default: 1406 TRACE_ERROR("unknown usb speed\n"); 1407 break; 1408 } 1409 1410 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1411 // TODO enable power save 1412 _WriteContext(&device->input_ctx->slot.dwslot1, SLOT_1_RH_PORT(rhPort)); 1413 uint32 dwslot2 = SLOT_2_IRQ_TARGET(0); 1414 1415 // If LS/FS device connected to non-root HS device 1416 if (route != 0 && parent->Speed() == USB_SPEED_HIGHSPEED 1417 && (speed == USB_SPEED_LOWSPEED || speed == USB_SPEED_FULLSPEED)) { 1418 struct xhci_device *parenthub = (struct xhci_device *) 1419 parent->ControllerCookie(); 1420 dwslot2 |= SLOT_2_PORT_NUM(hubPort); 1421 dwslot2 |= SLOT_2_TT_HUB_SLOT(parenthub->slot); 1422 } 1423 1424 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1425 1426 _WriteContext(&device->input_ctx->slot.dwslot3, SLOT_3_SLOT_STATE(0) 1427 | SLOT_3_DEVICE_ADDRESS(0)); 1428 1429 TRACE("slot 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 1430 "\n", _ReadContext(&device->input_ctx->slot.dwslot0), 1431 _ReadContext(&device->input_ctx->slot.dwslot1), 1432 _ReadContext(&device->input_ctx->slot.dwslot2), 1433 _ReadContext(&device->input_ctx->slot.dwslot3)); 1434 1435 device->device_ctx_area = fStack->AllocateArea((void **)&device->device_ctx, 1436 &device->device_ctx_addr, sizeof(*device->device_ctx) << fContextSizeShift, 1437 "XHCI device context"); 1438 if (device->device_ctx_area < B_OK) { 1439 TRACE_ERROR("unable to create a device context area\n"); 1440 delete_area(device->input_ctx_area); 1441 memset(device, 0, sizeof(xhci_device)); 1442 device->state = XHCI_STATE_DISABLED; 1443 return NULL; 1444 } 1445 memset(device->device_ctx, 0, sizeof(*device->device_ctx) << fContextSizeShift); 1446 1447 device->trb_area = fStack->AllocateArea((void **)&device->trbs, 1448 &device->trb_addr, sizeof(xhci_trb) * (XHCI_MAX_ENDPOINTS - 1) 1449 * XHCI_ENDPOINT_RING_SIZE, "XHCI endpoint trbs"); 1450 if (device->trb_area < B_OK) { 1451 TRACE_ERROR("unable to create a device trbs area\n"); 1452 delete_area(device->input_ctx_area); 1453 delete_area(device->device_ctx_area); 1454 memset(device, 0, sizeof(xhci_device)); 1455 device->state = XHCI_STATE_DISABLED; 1456 return NULL; 1457 } 1458 1459 // set up slot pointer to device context 1460 fDcba->baseAddress[slot] = device->device_ctx_addr; 1461 1462 size_t maxPacketSize; 1463 switch (speed) { 1464 case USB_SPEED_LOWSPEED: 1465 case USB_SPEED_FULLSPEED: 1466 maxPacketSize = 8; 1467 break; 1468 case USB_SPEED_HIGHSPEED: 1469 maxPacketSize = 64; 1470 break; 1471 default: 1472 maxPacketSize = 512; 1473 break; 1474 } 1475 1476 xhci_endpoint* endpoint0 = &device->endpoints[0]; 1477 mutex_init(&endpoint0->lock, "xhci endpoint lock"); 1478 endpoint0->device = device; 1479 endpoint0->id = 0; 1480 endpoint0->td_head = NULL; 1481 endpoint0->used = 0; 1482 endpoint0->current = 0; 1483 endpoint0->trbs = device->trbs; 1484 endpoint0->trb_addr = device->trb_addr; 1485 1486 // configure the Control endpoint 0 1487 if (ConfigureEndpoint(endpoint0, slot, 0, USB_OBJECT_CONTROL_PIPE, false, 1488 0, maxPacketSize, speed, 0, 0) != B_OK) { 1489 TRACE_ERROR("unable to configure default control endpoint\n"); 1490 delete_area(device->input_ctx_area); 1491 delete_area(device->device_ctx_area); 1492 delete_area(device->trb_area); 1493 memset(device, 0, sizeof(xhci_device)); 1494 device->state = XHCI_STATE_DISABLED; 1495 return NULL; 1496 } 1497 1498 // device should get to addressed state (bsr = 0) 1499 if (SetAddress(device->input_ctx_addr, false, slot) != B_OK) { 1500 TRACE_ERROR("unable to set address\n"); 1501 delete_area(device->input_ctx_area); 1502 delete_area(device->device_ctx_area); 1503 delete_area(device->trb_area); 1504 memset(device, 0, sizeof(xhci_device)); 1505 device->state = XHCI_STATE_DISABLED; 1506 return NULL; 1507 } 1508 1509 device->state = XHCI_STATE_ADDRESSED; 1510 device->address = SLOT_3_DEVICE_ADDRESS_GET(_ReadContext( 1511 &device->device_ctx->slot.dwslot3)); 1512 1513 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", device->address, 1514 SLOT_3_SLOT_STATE_GET(_ReadContext( 1515 &device->device_ctx->slot.dwslot3))); 1516 TRACE("endpoint0 state 0x%08" B_PRIx32 "\n", 1517 ENDPOINT_0_STATE_GET(_ReadContext( 1518 &device->device_ctx->endpoints[0].dwendpoint0))); 1519 1520 // Create a temporary pipe with the new address 1521 ControlPipe pipe(parent); 1522 pipe.SetControllerCookie(endpoint0); 1523 pipe.InitCommon(device->address + 1, 0, speed, Pipe::Default, maxPacketSize, 0, 1524 hubAddress, hubPort); 1525 1526 // Get the device descriptor 1527 // Just retrieve the first 8 bytes of the descriptor -> minimum supported 1528 // size of any device. It is enough because it includes the device type. 1529 1530 size_t actualLength = 0; 1531 usb_device_descriptor deviceDescriptor; 1532 1533 TRACE("getting the device descriptor\n"); 1534 status_t status = pipe.SendRequest( 1535 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_STANDARD, // type 1536 USB_REQUEST_GET_DESCRIPTOR, // request 1537 USB_DESCRIPTOR_DEVICE << 8, // value 1538 0, // index 1539 8, // length 1540 (void *)&deviceDescriptor, // buffer 1541 8, // buffer length 1542 &actualLength); // actual length 1543 1544 if (actualLength != 8) { 1545 TRACE_ERROR("failed to get the device descriptor: %s\n", 1546 strerror(status)); 1547 delete_area(device->input_ctx_area); 1548 delete_area(device->device_ctx_area); 1549 delete_area(device->trb_area); 1550 memset(device, 0, sizeof(xhci_device)); 1551 device->state = XHCI_STATE_DISABLED; 1552 return NULL; 1553 } 1554 1555 TRACE("device_class: %d device_subclass %d device_protocol %d\n", 1556 deviceDescriptor.device_class, deviceDescriptor.device_subclass, 1557 deviceDescriptor.device_protocol); 1558 1559 if (speed == USB_SPEED_FULLSPEED && deviceDescriptor.max_packet_size_0 != 8) { 1560 TRACE("Full speed device with different max packet size for Endpoint 0\n"); 1561 uint32 dwendpoint1 = _ReadContext( 1562 &device->input_ctx->endpoints[0].dwendpoint1); 1563 dwendpoint1 &= ~ENDPOINT_1_MAXPACKETSIZE(0xffff); 1564 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE( 1565 deviceDescriptor.max_packet_size_0); 1566 _WriteContext(&device->input_ctx->endpoints[0].dwendpoint1, 1567 dwendpoint1); 1568 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1569 _WriteContext(&device->input_ctx->input.addFlags, (1 << 1)); 1570 EvaluateContext(device->input_ctx_addr, device->slot); 1571 } 1572 1573 Device *deviceObject = NULL; 1574 if (deviceDescriptor.device_class == 0x09) { 1575 TRACE("creating new Hub\n"); 1576 TRACE("getting the hub descriptor\n"); 1577 size_t actualLength = 0; 1578 usb_hub_descriptor hubDescriptor; 1579 status = pipe.SendRequest( 1580 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_CLASS, // type 1581 USB_REQUEST_GET_DESCRIPTOR, // request 1582 USB_DESCRIPTOR_HUB << 8, // value 1583 0, // index 1584 sizeof(usb_hub_descriptor), // length 1585 (void *)&hubDescriptor, // buffer 1586 sizeof(usb_hub_descriptor), // buffer length 1587 &actualLength); 1588 1589 if (actualLength != sizeof(usb_hub_descriptor)) { 1590 TRACE_ERROR("error while getting the hub descriptor: %s\n", 1591 strerror(status)); 1592 delete_area(device->input_ctx_area); 1593 delete_area(device->device_ctx_area); 1594 delete_area(device->trb_area); 1595 memset(device, 0, sizeof(xhci_device)); 1596 device->state = XHCI_STATE_DISABLED; 1597 return NULL; 1598 } 1599 1600 uint32 dwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1601 dwslot0 |= SLOT_0_HUB_BIT; 1602 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1603 uint32 dwslot1 = _ReadContext(&device->input_ctx->slot.dwslot1); 1604 dwslot1 |= SLOT_1_NUM_PORTS(hubDescriptor.num_ports); 1605 _WriteContext(&device->input_ctx->slot.dwslot1, dwslot1); 1606 if (speed == USB_SPEED_HIGHSPEED) { 1607 uint32 dwslot2 = _ReadContext(&device->input_ctx->slot.dwslot2); 1608 dwslot2 |= SLOT_2_TT_TIME(HUB_TTT_GET(hubDescriptor.characteristics)); 1609 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1610 } 1611 1612 deviceObject = new(std::nothrow) Hub(parent, hubAddress, hubPort, 1613 deviceDescriptor, device->address + 1, speed, false, device); 1614 } else { 1615 TRACE("creating new device\n"); 1616 deviceObject = new(std::nothrow) Device(parent, hubAddress, hubPort, 1617 deviceDescriptor, device->address + 1, speed, false, device); 1618 } 1619 if (deviceObject == NULL || deviceObject->InitCheck() != B_OK) { 1620 if (deviceObject == NULL) { 1621 TRACE_ERROR("no memory to allocate device\n"); 1622 } else { 1623 TRACE_ERROR("device object failed to initialize\n"); 1624 } 1625 delete_area(device->input_ctx_area); 1626 delete_area(device->device_ctx_area); 1627 delete_area(device->trb_area); 1628 memset(device, 0, sizeof(xhci_device)); 1629 device->state = XHCI_STATE_DISABLED; 1630 return NULL; 1631 } 1632 1633 // We don't want to disable the default endpoint, naturally, which would 1634 // otherwise happen when this Pipe object is destroyed. 1635 pipe.SetControllerCookie(NULL); 1636 1637 fPortSlots[hubPort] = slot; 1638 TRACE("AllocateDevice() port %d slot %d\n", hubPort, slot); 1639 return deviceObject; 1640 } 1641 1642 1643 void 1644 XHCI::FreeDevice(Device *device) 1645 { 1646 uint8 hubPort = device->HubPort(); 1647 uint8 slot = fPortSlots[hubPort]; 1648 TRACE("FreeDevice() port %d slot %d\n", hubPort, slot); 1649 1650 // Delete the device first, so it cleans up its pipes and tells us 1651 // what we need to destroy before we tear down our internal state. 1652 delete device; 1653 1654 DisableSlot(slot); 1655 fDcba->baseAddress[slot] = 0; 1656 fPortSlots[hubPort] = 0; 1657 delete_area(fDevices[slot].trb_area); 1658 delete_area(fDevices[slot].input_ctx_area); 1659 delete_area(fDevices[slot].device_ctx_area); 1660 1661 memset(&fDevices[slot], 0, sizeof(xhci_device)); 1662 fDevices[slot].state = XHCI_STATE_DISABLED; 1663 } 1664 1665 1666 status_t 1667 XHCI::_InsertEndpointForPipe(Pipe *pipe) 1668 { 1669 TRACE("insert endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1670 1671 if (pipe->ControllerCookie() != NULL 1672 || pipe->Parent()->Type() != USB_OBJECT_DEVICE) { 1673 // default pipe is already referenced 1674 return B_OK; 1675 } 1676 1677 Device* usbDevice = (Device *)pipe->Parent(); 1678 struct xhci_device *device = (struct xhci_device *) 1679 usbDevice->ControllerCookie(); 1680 if (usbDevice->Parent() == RootObject()) 1681 return B_OK; 1682 if (device == NULL) { 1683 panic("device is NULL\n"); 1684 return B_NO_INIT; 1685 } 1686 1687 const uint8 id = (2 * pipe->EndpointAddress() 1688 + (pipe->Direction() != Pipe::Out ? 1 : 0)) - 1; 1689 if (id >= XHCI_MAX_ENDPOINTS - 1) 1690 return B_BAD_VALUE; 1691 1692 if (id > 0) { 1693 uint32 devicedwslot0 = _ReadContext(&device->device_ctx->slot.dwslot0); 1694 if (SLOT_0_NUM_ENTRIES_GET(devicedwslot0) == 1) { 1695 uint32 inputdwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1696 inputdwslot0 &= ~(SLOT_0_NUM_ENTRIES(0x1f)); 1697 inputdwslot0 |= SLOT_0_NUM_ENTRIES(XHCI_MAX_ENDPOINTS - 1); 1698 _WriteContext(&device->input_ctx->slot.dwslot0, inputdwslot0); 1699 EvaluateContext(device->input_ctx_addr, device->slot); 1700 } 1701 1702 xhci_endpoint* endpoint = &device->endpoints[id]; 1703 mutex_init(&endpoint->lock, "xhci endpoint lock"); 1704 MutexLocker endpointLocker(endpoint->lock); 1705 1706 endpoint->device = device; 1707 endpoint->id = id; 1708 endpoint->td_head = NULL; 1709 endpoint->used = 0; 1710 endpoint->current = 0; 1711 1712 endpoint->trbs = device->trbs + id * XHCI_ENDPOINT_RING_SIZE; 1713 endpoint->trb_addr = device->trb_addr 1714 + id * XHCI_ENDPOINT_RING_SIZE * sizeof(xhci_trb); 1715 memset(endpoint->trbs, 0, 1716 sizeof(xhci_trb) * XHCI_ENDPOINT_RING_SIZE); 1717 1718 TRACE("insert endpoint for pipe: trbs, device %p endpoint %p\n", 1719 device->trbs, endpoint->trbs); 1720 TRACE("insert endpoint for pipe: trb_addr, device 0x%" B_PRIxPHYSADDR 1721 " endpoint 0x%" B_PRIxPHYSADDR "\n", device->trb_addr, 1722 endpoint->trb_addr); 1723 1724 const uint8 endpointNum = id + 1; 1725 1726 status_t status = ConfigureEndpoint(endpoint, device->slot, id, pipe->Type(), 1727 pipe->Direction() == Pipe::In, pipe->Interval(), pipe->MaxPacketSize(), 1728 usbDevice->Speed(), pipe->MaxBurst(), pipe->BytesPerInterval()); 1729 if (status != B_OK) { 1730 TRACE_ERROR("unable to configure endpoint: %s\n", strerror(status)); 1731 return status; 1732 } 1733 1734 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1735 _WriteContext(&device->input_ctx->input.addFlags, 1736 (1 << endpointNum) | (1 << 0)); 1737 1738 if (endpointNum > 1) 1739 ConfigureEndpoint(device->input_ctx_addr, false, device->slot); 1740 else 1741 EvaluateContext(device->input_ctx_addr, device->slot); 1742 1743 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", 1744 device->address, SLOT_3_SLOT_STATE_GET(_ReadContext( 1745 &device->device_ctx->slot.dwslot3))); 1746 TRACE("endpoint[0] state 0x%08" B_PRIx32 "\n", 1747 ENDPOINT_0_STATE_GET(_ReadContext( 1748 &device->device_ctx->endpoints[0].dwendpoint0))); 1749 TRACE("endpoint[%d] state 0x%08" B_PRIx32 "\n", id, 1750 ENDPOINT_0_STATE_GET(_ReadContext( 1751 &device->device_ctx->endpoints[id].dwendpoint0))); 1752 1753 device->state = XHCI_STATE_CONFIGURED; 1754 } 1755 pipe->SetControllerCookie(&device->endpoints[id]); 1756 1757 return B_OK; 1758 } 1759 1760 1761 status_t 1762 XHCI::_RemoveEndpointForPipe(Pipe *pipe) 1763 { 1764 TRACE("remove endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1765 1766 if (pipe->Parent()->Type() != USB_OBJECT_DEVICE) 1767 return B_OK; 1768 Device* usbDevice = (Device *)pipe->Parent(); 1769 if (usbDevice->Parent() == RootObject()) 1770 return B_BAD_VALUE; 1771 1772 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 1773 if (endpoint == NULL || endpoint->trbs == NULL) 1774 return B_NO_INIT; 1775 1776 xhci_device *device = endpoint->device; 1777 1778 if (endpoint->id > 0) { 1779 mutex_lock(&endpoint->lock); 1780 1781 uint8 epNumber = endpoint->id + 1; 1782 StopEndpoint(true, epNumber, device->slot); 1783 1784 // See comment in CancelQueuedTransfers. 1785 xhci_td* td; 1786 while ((td = endpoint->td_head) != NULL) { 1787 endpoint->td_head = endpoint->td_head->next; 1788 FreeDescriptor(td); 1789 } 1790 1791 mutex_destroy(&endpoint->lock); 1792 memset(endpoint, 0, sizeof(xhci_endpoint)); 1793 1794 _WriteContext(&device->input_ctx->input.dropFlags, (1 << epNumber)); 1795 _WriteContext(&device->input_ctx->input.addFlags, 0); 1796 1797 if (epNumber > 1) 1798 ConfigureEndpoint(device->input_ctx_addr, true, device->slot); 1799 else 1800 EvaluateContext(device->input_ctx_addr, device->slot); 1801 1802 device->state = XHCI_STATE_ADDRESSED; 1803 } 1804 pipe->SetControllerCookie(NULL); 1805 1806 return B_OK; 1807 } 1808 1809 1810 status_t 1811 XHCI::_LinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 1812 { 1813 TRACE("link descriptor for pipe\n"); 1814 1815 // We must check this before we lock the endpoint, because if it is 1816 // NULL, the mutex is probably uninitialized, too. 1817 if (endpoint->device == NULL) { 1818 TRACE_ERROR("trying to submit a transfer to a non-existent endpoint!\n"); 1819 return B_NO_INIT; 1820 } 1821 1822 // Use mutex_trylock first, in case we are in KDL. 1823 if (mutex_trylock(&endpoint->lock) != B_OK) 1824 mutex_lock(&endpoint->lock); 1825 1826 // "used" refers to the number of currently linked TDs, not the number of 1827 // used TRBs on the ring (we use 2 TRBs on the ring per transfer.) 1828 if (endpoint->used >= (XHCI_MAX_TRANSFERS - 1)) { 1829 TRACE_ERROR("link descriptor for pipe: max transfers count exceeded\n"); 1830 mutex_unlock(&endpoint->lock); 1831 return B_BAD_VALUE; 1832 } 1833 1834 endpoint->used++; 1835 descriptor->next = endpoint->td_head; 1836 endpoint->td_head = descriptor; 1837 1838 const uint8 current = endpoint->current, 1839 eventdata = current + 1; 1840 uint8 next = eventdata + 1; 1841 1842 TRACE("link descriptor for pipe: current %d, next %d\n", current, next); 1843 1844 // Add a Link TRB to the end of the descriptor. 1845 phys_addr_t addr = endpoint->trb_addr + eventdata * sizeof(xhci_trb); 1846 descriptor->trbs[descriptor->trb_used].address = addr; 1847 descriptor->trbs[descriptor->trb_used].status = TRB_2_IRQ(0); 1848 descriptor->trbs[descriptor->trb_used].flags = TRB_3_TYPE(TRB_TYPE_LINK) 1849 | TRB_3_CHAIN_BIT | TRB_3_CYCLE_BIT; 1850 // It is specified that (XHCI 1.2 § 4.12.3 Note 2 p251) if the TRB 1851 // following one with the ENT bit set is a Link TRB, the Link TRB 1852 // shall be evaluated *and* the subsequent TRB shall be. Thus a 1853 // TRB_3_ENT_BIT is unnecessary here; and from testing seems to 1854 // break all transfers on a (very) small number of controllers. 1855 1856 #if !B_HOST_IS_LENDIAN 1857 // Convert endianness. 1858 for (uint32 i = 0; i <= descriptor->trb_used; i++) { 1859 descriptor->trbs[i].address = 1860 B_HOST_TO_LENDIAN_INT64(descriptor->trbs[i].address); 1861 descriptor->trbs[i].status = 1862 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].status); 1863 descriptor->trbs[i].flags = 1864 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].flags); 1865 } 1866 #endif 1867 1868 // Link the descriptor. 1869 endpoint->trbs[current].address = 1870 B_HOST_TO_LENDIAN_INT64(descriptor->trb_addr); 1871 endpoint->trbs[current].status = 1872 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 1873 endpoint->trbs[current].flags = 1874 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_LINK)); 1875 1876 // Set up the Event Data TRB (XHCI 1.2 § 4.11.5.2 p230.) 1877 // 1878 // We do this on the main ring for two reasons: first, to avoid a small 1879 // potential race between the interrupt and the controller evaluating 1880 // the link TRB to get back onto the ring; and second, because many 1881 // controllers throw errors if the target of a Link TRB is not valid 1882 // (i.e. does not have its Cycle Bit set.) 1883 // 1884 // We also set the "address" field, which the controller will copy 1885 // verbatim into the TRB it posts to the event ring, to be the last 1886 // "real" TRB in the TD; this will allow us to determine what transfer 1887 // the resulting Transfer Event TRB refers to. 1888 endpoint->trbs[eventdata].address = 1889 B_HOST_TO_LENDIAN_INT64(descriptor->trb_addr 1890 + (descriptor->trb_used - 1) * sizeof(xhci_trb)); 1891 endpoint->trbs[eventdata].status = 1892 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 1893 endpoint->trbs[eventdata].flags = 1894 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_EVENT_DATA) 1895 | TRB_3_IOC_BIT | TRB_3_CYCLE_BIT); 1896 1897 if (next == (XHCI_ENDPOINT_RING_SIZE - 1)) { 1898 // We always use 2 TRBs per _Link..() call, so if "next" is the last 1899 // TRB in the ring, we need to generate a link TRB at "next", and 1900 // then wrap it to 0. 1901 endpoint->trbs[next].address = 1902 B_HOST_TO_LENDIAN_INT64(endpoint->trb_addr); 1903 endpoint->trbs[next].status = 1904 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 1905 endpoint->trbs[next].flags = 1906 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_LINK) | TRB_3_CYCLE_BIT); 1907 1908 next = 0; 1909 } 1910 1911 endpoint->trbs[next].address = 0; 1912 endpoint->trbs[next].status = 0; 1913 endpoint->trbs[next].flags = 0; 1914 1915 // Everything is ready, so write the cycle bit. 1916 endpoint->trbs[current].flags |= B_HOST_TO_LENDIAN_INT32(TRB_3_CYCLE_BIT); 1917 1918 TRACE("_LinkDescriptorForPipe pCurrent %p phys 0x%" B_PRIxPHYSADDR 1919 " 0x%" B_PRIxPHYSADDR " 0x%08" B_PRIx32 "\n", &endpoint->trbs[current], 1920 endpoint->trb_addr + current * sizeof(struct xhci_trb), 1921 endpoint->trbs[current].address, 1922 B_LENDIAN_TO_HOST_INT32(endpoint->trbs[current].flags)); 1923 1924 endpoint->current = next; 1925 mutex_unlock(&endpoint->lock); 1926 1927 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 1928 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 1929 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 1930 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 1931 1932 Ring(endpoint->device->slot, endpoint->id + 1); 1933 1934 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 1935 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 1936 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 1937 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 1938 1939 return B_OK; 1940 } 1941 1942 1943 status_t 1944 XHCI::_UnlinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 1945 { 1946 TRACE("unlink descriptor for pipe\n"); 1947 // We presume that the caller has already locked or owns the endpoint. 1948 1949 endpoint->used--; 1950 if (descriptor == endpoint->td_head) { 1951 endpoint->td_head = descriptor->next; 1952 descriptor->next = NULL; 1953 return B_OK; 1954 } else { 1955 for (xhci_td *td = endpoint->td_head; td->next != NULL; td = td->next) { 1956 if (td->next == descriptor) { 1957 td->next = descriptor->next; 1958 descriptor->next = NULL; 1959 return B_OK; 1960 } 1961 } 1962 } 1963 1964 endpoint->used++; 1965 return B_ERROR; 1966 } 1967 1968 1969 status_t 1970 XHCI::ConfigureEndpoint(xhci_endpoint* ep, uint8 slot, uint8 number, uint8 type, 1971 bool directionIn, uint16 interval, uint16 maxPacketSize, usb_speed speed, 1972 uint8 maxBurst, uint16 bytesPerInterval) 1973 { 1974 struct xhci_device* device = &fDevices[slot]; 1975 1976 uint32 dwendpoint0 = 0; 1977 uint32 dwendpoint1 = 0; 1978 uint64 qwendpoint2 = 0; 1979 uint32 dwendpoint4 = 0; 1980 1981 // Compute and assign the endpoint type. (XHCI 1.2 § 6.2.3 Table 6-9 p452.) 1982 uint8 xhciType = 4; 1983 if ((type & USB_OBJECT_INTERRUPT_PIPE) != 0) 1984 xhciType = 3; 1985 if ((type & USB_OBJECT_BULK_PIPE) != 0) 1986 xhciType = 2; 1987 if ((type & USB_OBJECT_ISO_PIPE) != 0) 1988 xhciType = 1; 1989 xhciType |= directionIn ? (1 << 2) : 0; 1990 dwendpoint1 |= ENDPOINT_1_EPTYPE(xhciType); 1991 1992 // Compute and assign interval. (XHCI 1.2 § 6.2.3.6 p456.) 1993 uint16 calcInterval; 1994 if ((type & USB_OBJECT_BULK_PIPE) != 0 1995 || (type & USB_OBJECT_CONTROL_PIPE) != 0) { 1996 // Bulk and Control endpoints never issue NAKs. 1997 calcInterval = 0; 1998 } else { 1999 switch (speed) { 2000 case USB_SPEED_FULLSPEED: 2001 if ((type & USB_OBJECT_ISO_PIPE) != 0) { 2002 // Convert 1-16 into 3-18. 2003 calcInterval = min_c(max_c(interval, 1), 16) + 2; 2004 break; 2005 } 2006 2007 // fall through 2008 case USB_SPEED_LOWSPEED: { 2009 // Convert 1ms-255ms into 3-10. 2010 2011 // Find the index of the highest set bit in "interval". 2012 uint32 temp = min_c(max_c(interval, 1), 255); 2013 for (calcInterval = 0; temp != 1; calcInterval++) 2014 temp = temp >> 1; 2015 calcInterval += 3; 2016 break; 2017 } 2018 2019 case USB_SPEED_HIGHSPEED: 2020 case USB_SPEED_SUPERSPEED: 2021 default: 2022 // Convert 1-16 into 0-15. 2023 calcInterval = min_c(max_c(interval, 1), 16) - 1; 2024 break; 2025 } 2026 } 2027 dwendpoint0 |= ENDPOINT_0_INTERVAL(calcInterval); 2028 2029 // For non-isochronous endpoints, we want the controller to retry failed 2030 // transfers, if possible. (XHCI 1.2 § 4.10.2.3 p197.) 2031 if ((type & USB_OBJECT_ISO_PIPE) == 0) 2032 dwendpoint1 |= ENDPOINT_1_CERR(3); 2033 2034 // Assign maximum burst size. For USB3 devices this is passed in; for 2035 // all other devices we compute it. (XHCI 1.2 § 4.8.2 p161.) 2036 if (speed == USB_SPEED_HIGHSPEED && (type & (USB_OBJECT_INTERRUPT_PIPE 2037 | USB_OBJECT_ISO_PIPE)) != 0) { 2038 maxBurst = (maxPacketSize & 0x1800) >> 11; 2039 } else if (speed != USB_SPEED_SUPERSPEED) { 2040 maxBurst = 0; 2041 } 2042 dwendpoint1 |= ENDPOINT_1_MAXBURST(maxBurst); 2043 2044 // Assign maximum packet size, set the ring address, and set the 2045 // "Dequeue Cycle State" bit. (XHCI 1.2 § 6.2.3 Table 6-10 p453.) 2046 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE(maxPacketSize); 2047 qwendpoint2 |= ENDPOINT_2_DCS_BIT | ep->trb_addr; 2048 2049 // The Max Burst Payload is the number of bytes moved by a 2050 // maximum sized burst. (XHCI 1.2 § 4.11.7.1 p236.) 2051 ep->max_burst_payload = (maxBurst + 1) * maxPacketSize; 2052 2053 // Assign average TRB length. 2054 if ((type & USB_OBJECT_CONTROL_PIPE) != 0) { 2055 // Control pipes are a special case, as they rarely have 2056 // outbound transfers of any substantial size. 2057 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(8); 2058 } else if ((type & USB_OBJECT_ISO_PIPE) != 0) { 2059 // Isochronous pipes are another special case: the TRB size will be 2060 // one packet (which is normally smaller than the max packet size, 2061 // but we don't know what it is here.) 2062 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(maxPacketSize); 2063 } else { 2064 // Under all other circumstances, we put max_burst_payload in a TRB. 2065 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(ep->max_burst_payload); 2066 } 2067 2068 // Assign maximum ESIT payload. (XHCI 1.2 § 4.14.2 p259.) 2069 if ((type & (USB_OBJECT_INTERRUPT_PIPE | USB_OBJECT_ISO_PIPE)) != 0) { 2070 // TODO: For SuperSpeedPlus endpoints, there is yet another descriptor 2071 // for isochronous endpoints that specifies the maximum ESIT payload. 2072 // We don't fetch this yet, so just fall back to the USB2 computation 2073 // method if bytesPerInterval is 0. 2074 if (speed == USB_SPEED_SUPERSPEED && bytesPerInterval != 0) 2075 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD(bytesPerInterval); 2076 else if (speed >= USB_SPEED_HIGHSPEED) 2077 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD((maxBurst + 1) * maxPacketSize); 2078 } 2079 2080 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint0, 2081 dwendpoint0); 2082 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint1, 2083 dwendpoint1); 2084 _WriteContext(&device->input_ctx->endpoints[number].qwendpoint2, 2085 qwendpoint2); 2086 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint4, 2087 dwendpoint4); 2088 2089 TRACE("endpoint 0x%" B_PRIx32 " 0x%" B_PRIx32 " 0x%" B_PRIx64 " 0x%" 2090 B_PRIx32 "\n", 2091 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint0), 2092 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint1), 2093 _ReadContext(&device->input_ctx->endpoints[number].qwendpoint2), 2094 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint4)); 2095 2096 return B_OK; 2097 } 2098 2099 2100 status_t 2101 XHCI::GetPortSpeed(uint8 index, usb_speed* speed) 2102 { 2103 if (index >= fPortCount) 2104 return B_BAD_INDEX; 2105 2106 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 2107 2108 switch (PS_SPEED_GET(portStatus)) { 2109 case 3: 2110 *speed = USB_SPEED_HIGHSPEED; 2111 break; 2112 case 2: 2113 *speed = USB_SPEED_LOWSPEED; 2114 break; 2115 case 1: 2116 *speed = USB_SPEED_FULLSPEED; 2117 break; 2118 case 4: 2119 *speed = USB_SPEED_SUPERSPEED; 2120 break; 2121 default: 2122 TRACE_ALWAYS("nonstandard port speed %" B_PRId32 ", assuming SuperSpeed\n", 2123 PS_SPEED_GET(portStatus)); 2124 *speed = USB_SPEED_SUPERSPEED; 2125 break; 2126 } 2127 2128 return B_OK; 2129 } 2130 2131 2132 status_t 2133 XHCI::GetPortStatus(uint8 index, usb_port_status* status) 2134 { 2135 if (index >= fPortCount) 2136 return B_BAD_INDEX; 2137 2138 status->status = status->change = 0; 2139 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 2140 TRACE("port %" B_PRId8 " status=0x%08" B_PRIx32 "\n", index, portStatus); 2141 2142 // build the status 2143 switch (PS_SPEED_GET(portStatus)) { 2144 case 3: 2145 status->status |= PORT_STATUS_HIGH_SPEED; 2146 break; 2147 case 2: 2148 status->status |= PORT_STATUS_LOW_SPEED; 2149 break; 2150 default: 2151 break; 2152 } 2153 2154 if (portStatus & PS_CCS) 2155 status->status |= PORT_STATUS_CONNECTION; 2156 if (portStatus & PS_PED) 2157 status->status |= PORT_STATUS_ENABLE; 2158 if (portStatus & PS_OCA) 2159 status->status |= PORT_STATUS_OVER_CURRENT; 2160 if (portStatus & PS_PR) 2161 status->status |= PORT_STATUS_RESET; 2162 if (portStatus & PS_PP) { 2163 if (fPortSpeeds[index] == USB_SPEED_SUPERSPEED) 2164 status->status |= PORT_STATUS_SS_POWER; 2165 else 2166 status->status |= PORT_STATUS_POWER; 2167 } 2168 2169 // build the change 2170 if (portStatus & PS_CSC) 2171 status->change |= PORT_STATUS_CONNECTION; 2172 if (portStatus & PS_PEC) 2173 status->change |= PORT_STATUS_ENABLE; 2174 if (portStatus & PS_OCC) 2175 status->change |= PORT_STATUS_OVER_CURRENT; 2176 if (portStatus & PS_PRC) 2177 status->change |= PORT_STATUS_RESET; 2178 2179 if (fPortSpeeds[index] == USB_SPEED_SUPERSPEED) { 2180 if (portStatus & PS_PLC) 2181 status->change |= PORT_CHANGE_LINK_STATE; 2182 if (portStatus & PS_WRC) 2183 status->change |= PORT_CHANGE_BH_PORT_RESET; 2184 } 2185 2186 return B_OK; 2187 } 2188 2189 2190 status_t 2191 XHCI::SetPortFeature(uint8 index, uint16 feature) 2192 { 2193 TRACE("set port feature index %u feature %u\n", index, feature); 2194 if (index >= fPortCount) 2195 return B_BAD_INDEX; 2196 2197 uint32 portRegister = XHCI_PORTSC(index); 2198 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2199 2200 switch (feature) { 2201 case PORT_SUSPEND: 2202 if ((portStatus & PS_PED) == 0 || (portStatus & PS_PR) 2203 || (portStatus & PS_PLS_MASK) >= PS_XDEV_U3) { 2204 TRACE_ERROR("USB core suspending device not in U0/U1/U2.\n"); 2205 return B_BAD_VALUE; 2206 } 2207 portStatus &= ~PS_PLS_MASK; 2208 WriteOpReg(portRegister, portStatus | PS_LWS | PS_XDEV_U3); 2209 break; 2210 2211 case PORT_RESET: 2212 WriteOpReg(portRegister, portStatus | PS_PR); 2213 break; 2214 2215 case PORT_POWER: 2216 WriteOpReg(portRegister, portStatus | PS_PP); 2217 break; 2218 default: 2219 return B_BAD_VALUE; 2220 } 2221 ReadOpReg(portRegister); 2222 return B_OK; 2223 } 2224 2225 2226 status_t 2227 XHCI::ClearPortFeature(uint8 index, uint16 feature) 2228 { 2229 TRACE("clear port feature index %u feature %u\n", index, feature); 2230 if (index >= fPortCount) 2231 return B_BAD_INDEX; 2232 2233 uint32 portRegister = XHCI_PORTSC(index); 2234 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2235 2236 switch (feature) { 2237 case PORT_SUSPEND: 2238 portStatus = ReadOpReg(portRegister); 2239 if (portStatus & PS_PR) 2240 return B_BAD_VALUE; 2241 if (portStatus & PS_XDEV_U3) { 2242 if ((portStatus & PS_PED) == 0) 2243 return B_BAD_VALUE; 2244 portStatus &= ~PS_PLS_MASK; 2245 WriteOpReg(portRegister, portStatus | PS_XDEV_U0 | PS_LWS); 2246 } 2247 break; 2248 case PORT_ENABLE: 2249 WriteOpReg(portRegister, portStatus | PS_PED); 2250 break; 2251 case PORT_POWER: 2252 WriteOpReg(portRegister, portStatus & ~PS_PP); 2253 break; 2254 case C_PORT_CONNECTION: 2255 WriteOpReg(portRegister, portStatus | PS_CSC); 2256 break; 2257 case C_PORT_ENABLE: 2258 WriteOpReg(portRegister, portStatus | PS_PEC); 2259 break; 2260 case C_PORT_OVER_CURRENT: 2261 WriteOpReg(portRegister, portStatus | PS_OCC); 2262 break; 2263 case C_PORT_RESET: 2264 WriteOpReg(portRegister, portStatus | PS_PRC); 2265 break; 2266 case C_PORT_BH_PORT_RESET: 2267 WriteOpReg(portRegister, portStatus | PS_WRC); 2268 break; 2269 case C_PORT_LINK_STATE: 2270 WriteOpReg(portRegister, portStatus | PS_PLC); 2271 break; 2272 default: 2273 return B_BAD_VALUE; 2274 } 2275 2276 ReadOpReg(portRegister); 2277 return B_OK; 2278 } 2279 2280 2281 status_t 2282 XHCI::ControllerHalt() 2283 { 2284 // Mask off run state 2285 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) & ~CMD_RUN); 2286 2287 // wait for shutdown state 2288 if (WaitOpBits(XHCI_STS, STS_HCH, STS_HCH) != B_OK) { 2289 TRACE_ERROR("HCH shutdown timeout\n"); 2290 return B_ERROR; 2291 } 2292 return B_OK; 2293 } 2294 2295 2296 status_t 2297 XHCI::ControllerReset() 2298 { 2299 TRACE("ControllerReset() cmd: 0x%" B_PRIx32 " sts: 0x%" B_PRIx32 "\n", 2300 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 2301 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) | CMD_HCRST); 2302 2303 if (WaitOpBits(XHCI_CMD, CMD_HCRST, 0) != B_OK) { 2304 TRACE_ERROR("ControllerReset() failed CMD_HCRST\n"); 2305 return B_ERROR; 2306 } 2307 2308 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 2309 TRACE_ERROR("ControllerReset() failed STS_CNR\n"); 2310 return B_ERROR; 2311 } 2312 2313 return B_OK; 2314 } 2315 2316 2317 int32 2318 XHCI::InterruptHandler(void* data) 2319 { 2320 return ((XHCI*)data)->Interrupt(); 2321 } 2322 2323 2324 int32 2325 XHCI::Interrupt() 2326 { 2327 SpinLocker _(&fSpinlock); 2328 2329 uint32 status = ReadOpReg(XHCI_STS); 2330 uint32 temp = ReadRunReg32(XHCI_IMAN(0)); 2331 WriteOpReg(XHCI_STS, status); 2332 WriteRunReg32(XHCI_IMAN(0), temp); 2333 2334 int32 result = B_HANDLED_INTERRUPT; 2335 2336 if ((status & STS_HCH) != 0) { 2337 TRACE_ERROR("Host Controller halted\n"); 2338 return result; 2339 } 2340 if ((status & STS_HSE) != 0) { 2341 TRACE_ERROR("Host System Error\n"); 2342 return result; 2343 } 2344 if ((status & STS_HCE) != 0) { 2345 TRACE_ERROR("Host Controller Error\n"); 2346 return result; 2347 } 2348 2349 if ((status & STS_EINT) == 0) { 2350 TRACE("STS: 0x%" B_PRIx32 " IRQ_PENDING: 0x%" B_PRIx32 "\n", 2351 status, temp); 2352 return B_UNHANDLED_INTERRUPT; 2353 } 2354 2355 TRACE("Event Interrupt\n"); 2356 release_sem_etc(fEventSem, 1, B_DO_NOT_RESCHEDULE); 2357 return B_INVOKE_SCHEDULER; 2358 } 2359 2360 2361 void 2362 XHCI::Ring(uint8 slot, uint8 endpoint) 2363 { 2364 TRACE("Ding Dong! slot:%d endpoint %d\n", slot, endpoint) 2365 if ((slot == 0 && endpoint > 0) || (slot > 0 && endpoint == 0)) 2366 panic("Ring() invalid slot/endpoint combination\n"); 2367 if (slot > fSlotCount || endpoint >= XHCI_MAX_ENDPOINTS) 2368 panic("Ring() invalid slot or endpoint\n"); 2369 2370 WriteDoorReg32(XHCI_DOORBELL(slot), XHCI_DOORBELL_TARGET(endpoint) 2371 | XHCI_DOORBELL_STREAMID(0)); 2372 ReadDoorReg32(XHCI_DOORBELL(slot)); 2373 // Flush PCI writes 2374 } 2375 2376 2377 void 2378 XHCI::QueueCommand(xhci_trb* trb) 2379 { 2380 uint8 i, j; 2381 uint32 temp; 2382 2383 i = fCmdIdx; 2384 j = fCmdCcs; 2385 2386 TRACE("command[%u] = %" B_PRId32 " (0x%016" B_PRIx64 ", 0x%08" B_PRIx32 2387 ", 0x%08" B_PRIx32 ")\n", i, TRB_3_TYPE_GET(trb->flags), trb->address, 2388 trb->status, trb->flags); 2389 2390 fCmdRing[i].address = trb->address; 2391 fCmdRing[i].status = trb->status; 2392 temp = trb->flags; 2393 2394 if (j) 2395 temp |= TRB_3_CYCLE_BIT; 2396 else 2397 temp &= ~TRB_3_CYCLE_BIT; 2398 temp &= ~TRB_3_TC_BIT; 2399 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2400 2401 fCmdAddr = fErst->rs_addr + (XHCI_MAX_EVENTS + i) * sizeof(xhci_trb); 2402 2403 i++; 2404 2405 if (i == (XHCI_MAX_COMMANDS - 1)) { 2406 temp = TRB_3_TYPE(TRB_TYPE_LINK) | TRB_3_TC_BIT; 2407 if (j) 2408 temp |= TRB_3_CYCLE_BIT; 2409 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2410 2411 i = 0; 2412 j ^= 1; 2413 } 2414 2415 fCmdIdx = i; 2416 fCmdCcs = j; 2417 } 2418 2419 2420 void 2421 XHCI::HandleCmdComplete(xhci_trb* trb) 2422 { 2423 if (fCmdAddr == trb->address) { 2424 TRACE("Received command event\n"); 2425 fCmdResult[0] = trb->status; 2426 fCmdResult[1] = B_LENDIAN_TO_HOST_INT32(trb->flags); 2427 release_sem_etc(fCmdCompSem, 1, B_DO_NOT_RESCHEDULE); 2428 } else 2429 TRACE_ERROR("received command event for unknown command!\n") 2430 } 2431 2432 2433 void 2434 XHCI::HandleTransferComplete(xhci_trb* trb) 2435 { 2436 const uint32 flags = B_LENDIAN_TO_HOST_INT32(trb->flags); 2437 const uint8 endpointNumber = TRB_3_ENDPOINT_GET(flags), 2438 slot = TRB_3_SLOT_GET(flags); 2439 2440 if (slot > fSlotCount) 2441 TRACE_ERROR("invalid slot\n"); 2442 if (endpointNumber == 0 || endpointNumber >= XHCI_MAX_ENDPOINTS) { 2443 TRACE_ERROR("invalid endpoint\n"); 2444 return; 2445 } 2446 2447 xhci_device *device = &fDevices[slot]; 2448 xhci_endpoint *endpoint = &device->endpoints[endpointNumber - 1]; 2449 2450 if (endpoint->trbs == NULL) { 2451 TRACE_ERROR("got TRB but endpoint is not allocated!\n"); 2452 return; 2453 } 2454 2455 // Use mutex_trylock first, in case we are in KDL. 2456 MutexLocker endpointLocker(endpoint->lock, 2457 mutex_trylock(&endpoint->lock) == B_OK); 2458 if (!endpointLocker.IsLocked()) { 2459 // We failed to get the lock. Most likely it was destroyed 2460 // while we were waiting for it. 2461 return; 2462 } 2463 2464 // In the case of an Event Data TRB, the "transferred" field refers 2465 // to the actual number of bytes transferred across the whole TD. 2466 // (XHCI 1.2 § 6.4.2.1 Table 6-38 p478.) 2467 const uint8 completionCode = TRB_2_COMP_CODE_GET(trb->status); 2468 int32 transferred = TRB_2_REM_GET(trb->status), remainder = -1; 2469 2470 TRACE("HandleTransferComplete: ed %d, code %d, transferred %d\n", 2471 (flags & TRB_3_EVENT_DATA_BIT), completionCode, transferred); 2472 2473 if ((flags & TRB_3_EVENT_DATA_BIT) == 0) { 2474 TRACE("got an interrupt for a non-Event Data TRB!\n"); 2475 remainder = transferred; 2476 transferred = -1; 2477 } 2478 2479 if (completionCode != COMP_SUCCESS && completionCode != COMP_SHORT_PACKET) { 2480 TRACE_ALWAYS("transfer error on slot %" B_PRId8 " endpoint %" B_PRId8 2481 ": %s\n", slot, endpointNumber, xhci_error_string(completionCode)); 2482 } 2483 2484 const phys_addr_t source = B_LENDIAN_TO_HOST_INT64(trb->address); 2485 for (xhci_td *td = endpoint->td_head; td != NULL; td = td->next) { 2486 int64 offset = (source - td->trb_addr) / sizeof(xhci_trb); 2487 if (offset < 0 || offset >= td->trb_count) 2488 continue; 2489 2490 TRACE("HandleTransferComplete td %p trb %" B_PRId64 " found\n", 2491 td, offset); 2492 2493 // The TRB at offset trb_used will be the link TRB, which we do not 2494 // care about (and should not generate an interrupt at all.) We really 2495 // care about the properly last TRB, at index "count - 1", which the 2496 // Event Data TRB that _LinkDescriptorForPipe creates points to. 2497 // 2498 // But if we have an unsuccessful completion code, the transfer 2499 // likely failed midway; so just accept it anyway. 2500 if (offset == (td->trb_used - 1) || completionCode != COMP_SUCCESS) { 2501 _UnlinkDescriptorForPipe(td, endpoint); 2502 endpointLocker.Unlock(); 2503 2504 td->trb_completion_code = completionCode; 2505 td->td_transferred = transferred; 2506 td->trb_left = remainder; 2507 2508 // add descriptor to finished list 2509 if (mutex_trylock(&fFinishedLock) != B_OK) 2510 mutex_lock(&fFinishedLock); 2511 td->next = fFinishedHead; 2512 fFinishedHead = td; 2513 mutex_unlock(&fFinishedLock); 2514 2515 release_sem_etc(fFinishTransfersSem, 1, B_DO_NOT_RESCHEDULE); 2516 TRACE("HandleTransferComplete td %p done\n", td); 2517 } else { 2518 TRACE_ERROR("successful TRB 0x%" B_PRIxPHYSADDR " was found, but it wasn't " 2519 "the last in the TD!\n", source); 2520 } 2521 return; 2522 } 2523 TRACE_ERROR("TRB 0x%" B_PRIxPHYSADDR " was not found in the endpoint!\n", source); 2524 } 2525 2526 2527 void 2528 XHCI::DumpRing(xhci_trb *trbs, uint32 size) 2529 { 2530 if (!Lock()) { 2531 TRACE("Unable to get lock!\n"); 2532 return; 2533 } 2534 2535 for (uint32 i = 0; i < size; i++) { 2536 TRACE("command[%" B_PRId32 "] = %" B_PRId32 " (0x%016" B_PRIx64 "," 2537 " 0x%08" B_PRIx32 ", 0x%08" B_PRIx32 ")\n", i, 2538 TRB_3_TYPE_GET(B_LENDIAN_TO_HOST_INT32(trbs[i].flags)), 2539 trbs[i].address, trbs[i].status, trbs[i].flags); 2540 } 2541 2542 Unlock(); 2543 } 2544 2545 2546 status_t 2547 XHCI::DoCommand(xhci_trb* trb) 2548 { 2549 if (!Lock()) { 2550 TRACE("Unable to get lock!\n"); 2551 return B_ERROR; 2552 } 2553 2554 QueueCommand(trb); 2555 Ring(0, 0); 2556 2557 // Begin with a 50ms timeout. 2558 if (acquire_sem_etc(fCmdCompSem, 1, B_RELATIVE_TIMEOUT, 50 * 1000) != B_OK) { 2559 // We've hit the timeout. In some error cases, interrupts are not 2560 // generated; so here we force the event ring to be polled once. 2561 release_sem(fEventSem); 2562 2563 // Now try again, this time with a 750ms timeout. 2564 if (acquire_sem_etc(fCmdCompSem, 1, B_RELATIVE_TIMEOUT, 2565 750 * 1000) != B_OK) { 2566 TRACE("Unable to obtain fCmdCompSem!\n"); 2567 fCmdAddr = 0; 2568 Unlock(); 2569 return B_TIMED_OUT; 2570 } 2571 } 2572 2573 // eat up sems that have been released by multiple interrupts 2574 int32 semCount = 0; 2575 get_sem_count(fCmdCompSem, &semCount); 2576 if (semCount > 0) 2577 acquire_sem_etc(fCmdCompSem, semCount, B_RELATIVE_TIMEOUT, 0); 2578 2579 status_t status = B_OK; 2580 uint32 completionCode = TRB_2_COMP_CODE_GET(fCmdResult[0]); 2581 TRACE("command complete\n"); 2582 if (completionCode != COMP_SUCCESS) { 2583 TRACE_ERROR("unsuccessful command %" B_PRId32 ", error %s (%" B_PRId32 ")\n", 2584 TRB_3_TYPE_GET(trb->flags), xhci_error_string(completionCode), 2585 completionCode); 2586 status = B_IO_ERROR; 2587 } 2588 2589 trb->status = fCmdResult[0]; 2590 trb->flags = fCmdResult[1]; 2591 2592 fCmdAddr = 0; 2593 Unlock(); 2594 return status; 2595 } 2596 2597 2598 status_t 2599 XHCI::Noop() 2600 { 2601 TRACE("Issue No-Op\n"); 2602 xhci_trb trb; 2603 trb.address = 0; 2604 trb.status = 0; 2605 trb.flags = TRB_3_TYPE(TRB_TYPE_CMD_NOOP); 2606 2607 return DoCommand(&trb); 2608 } 2609 2610 2611 status_t 2612 XHCI::EnableSlot(uint8* slot) 2613 { 2614 TRACE("Enable Slot\n"); 2615 xhci_trb trb; 2616 trb.address = 0; 2617 trb.status = 0; 2618 trb.flags = TRB_3_TYPE(TRB_TYPE_ENABLE_SLOT); 2619 2620 status_t status = DoCommand(&trb); 2621 if (status != B_OK) 2622 return status; 2623 2624 *slot = TRB_3_SLOT_GET(trb.flags); 2625 return *slot != 0 ? B_OK : B_BAD_VALUE; 2626 } 2627 2628 2629 status_t 2630 XHCI::DisableSlot(uint8 slot) 2631 { 2632 TRACE("Disable Slot\n"); 2633 xhci_trb trb; 2634 trb.address = 0; 2635 trb.status = 0; 2636 trb.flags = TRB_3_TYPE(TRB_TYPE_DISABLE_SLOT) | TRB_3_SLOT(slot); 2637 2638 return DoCommand(&trb); 2639 } 2640 2641 2642 status_t 2643 XHCI::SetAddress(uint64 inputContext, bool bsr, uint8 slot) 2644 { 2645 TRACE("Set Address\n"); 2646 xhci_trb trb; 2647 trb.address = inputContext; 2648 trb.status = 0; 2649 trb.flags = TRB_3_TYPE(TRB_TYPE_ADDRESS_DEVICE) | TRB_3_SLOT(slot); 2650 2651 if (bsr) 2652 trb.flags |= TRB_3_BSR_BIT; 2653 2654 return DoCommand(&trb); 2655 } 2656 2657 2658 status_t 2659 XHCI::ConfigureEndpoint(uint64 inputContext, bool deconfigure, uint8 slot) 2660 { 2661 TRACE("Configure Endpoint\n"); 2662 xhci_trb trb; 2663 trb.address = inputContext; 2664 trb.status = 0; 2665 trb.flags = TRB_3_TYPE(TRB_TYPE_CONFIGURE_ENDPOINT) | TRB_3_SLOT(slot); 2666 2667 if (deconfigure) 2668 trb.flags |= TRB_3_DCEP_BIT; 2669 2670 return DoCommand(&trb); 2671 } 2672 2673 2674 status_t 2675 XHCI::EvaluateContext(uint64 inputContext, uint8 slot) 2676 { 2677 TRACE("Evaluate Context\n"); 2678 xhci_trb trb; 2679 trb.address = inputContext; 2680 trb.status = 0; 2681 trb.flags = TRB_3_TYPE(TRB_TYPE_EVALUATE_CONTEXT) | TRB_3_SLOT(slot); 2682 2683 return DoCommand(&trb); 2684 } 2685 2686 2687 status_t 2688 XHCI::ResetEndpoint(bool preserve, uint8 endpoint, uint8 slot) 2689 { 2690 TRACE("Reset Endpoint\n"); 2691 xhci_trb trb; 2692 trb.address = 0; 2693 trb.status = 0; 2694 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_ENDPOINT) 2695 | TRB_3_SLOT(slot) | TRB_3_ENDPOINT(endpoint); 2696 if (preserve) 2697 trb.flags |= TRB_3_PRSV_BIT; 2698 2699 return DoCommand(&trb); 2700 } 2701 2702 2703 status_t 2704 XHCI::StopEndpoint(bool suspend, uint8 endpoint, uint8 slot) 2705 { 2706 TRACE("Stop Endpoint\n"); 2707 xhci_trb trb; 2708 trb.address = 0; 2709 trb.status = 0; 2710 trb.flags = TRB_3_TYPE(TRB_TYPE_STOP_ENDPOINT) 2711 | TRB_3_SLOT(slot) | TRB_3_ENDPOINT(endpoint); 2712 if (suspend) 2713 trb.flags |= TRB_3_SUSPEND_ENDPOINT_BIT; 2714 2715 return DoCommand(&trb); 2716 } 2717 2718 2719 status_t 2720 XHCI::SetTRDequeue(uint64 dequeue, uint16 stream, uint8 endpoint, uint8 slot) 2721 { 2722 TRACE("Set TR Dequeue\n"); 2723 xhci_trb trb; 2724 trb.address = dequeue | ENDPOINT_2_DCS_BIT; 2725 // The DCS bit is copied from the address field as in ConfigureEndpoint. 2726 // (XHCI 1.2 § 4.6.10 p142.) 2727 trb.status = TRB_2_STREAM(stream); 2728 trb.flags = TRB_3_TYPE(TRB_TYPE_SET_TR_DEQUEUE) 2729 | TRB_3_SLOT(slot) | TRB_3_ENDPOINT(endpoint); 2730 2731 return DoCommand(&trb); 2732 } 2733 2734 2735 status_t 2736 XHCI::ResetDevice(uint8 slot) 2737 { 2738 TRACE("Reset Device\n"); 2739 xhci_trb trb; 2740 trb.address = 0; 2741 trb.status = 0; 2742 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_DEVICE) | TRB_3_SLOT(slot); 2743 2744 return DoCommand(&trb); 2745 } 2746 2747 2748 int32 2749 XHCI::EventThread(void* data) 2750 { 2751 ((XHCI *)data)->CompleteEvents(); 2752 return B_OK; 2753 } 2754 2755 2756 void 2757 XHCI::CompleteEvents() 2758 { 2759 while (!fStopThreads) { 2760 if (acquire_sem(fEventSem) < B_OK) 2761 continue; 2762 2763 // eat up sems that have been released by multiple interrupts 2764 int32 semCount = 0; 2765 get_sem_count(fEventSem, &semCount); 2766 if (semCount > 0) 2767 acquire_sem_etc(fEventSem, semCount, B_RELATIVE_TIMEOUT, 0); 2768 2769 ProcessEvents(); 2770 } 2771 } 2772 2773 2774 void 2775 XHCI::ProcessEvents() 2776 { 2777 // Use mutex_trylock first, in case we are in KDL. 2778 MutexLocker locker(fEventLock, mutex_trylock(&fEventLock) == B_OK); 2779 if (!locker.IsLocked()) { 2780 // We failed to get the lock. This really should not happen. 2781 TRACE_ERROR("failed to acquire event lock!\n"); 2782 return; 2783 } 2784 2785 uint16 i = fEventIdx; 2786 uint8 j = fEventCcs; 2787 uint8 t = 2; 2788 2789 while (1) { 2790 uint32 temp = B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags); 2791 uint8 event = TRB_3_TYPE_GET(temp); 2792 TRACE("event[%u] = %u (0x%016" B_PRIx64 " 0x%08" B_PRIx32 " 0x%08" 2793 B_PRIx32 ")\n", i, event, fEventRing[i].address, 2794 fEventRing[i].status, B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags)); 2795 uint8 k = (temp & TRB_3_CYCLE_BIT) ? 1 : 0; 2796 if (j != k) 2797 break; 2798 2799 switch (event) { 2800 case TRB_TYPE_COMMAND_COMPLETION: 2801 HandleCmdComplete(&fEventRing[i]); 2802 break; 2803 case TRB_TYPE_TRANSFER: 2804 HandleTransferComplete(&fEventRing[i]); 2805 break; 2806 case TRB_TYPE_PORT_STATUS_CHANGE: 2807 TRACE("port change detected\n"); 2808 break; 2809 default: 2810 TRACE_ERROR("Unhandled event = %u\n", event); 2811 break; 2812 } 2813 2814 i++; 2815 if (i == XHCI_MAX_EVENTS) { 2816 i = 0; 2817 j ^= 1; 2818 if (!--t) 2819 break; 2820 } 2821 } 2822 2823 fEventIdx = i; 2824 fEventCcs = j; 2825 2826 uint64 addr = fErst->rs_addr + i * sizeof(xhci_trb); 2827 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)addr | ERDP_BUSY); 2828 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(addr >> 32)); 2829 } 2830 2831 2832 int32 2833 XHCI::FinishThread(void* data) 2834 { 2835 ((XHCI *)data)->FinishTransfers(); 2836 return B_OK; 2837 } 2838 2839 2840 void 2841 XHCI::FinishTransfers() 2842 { 2843 while (!fStopThreads) { 2844 if (acquire_sem(fFinishTransfersSem) < B_OK) 2845 continue; 2846 2847 // eat up sems that have been released by multiple interrupts 2848 int32 semCount = 0; 2849 get_sem_count(fFinishTransfersSem, &semCount); 2850 if (semCount > 0) 2851 acquire_sem_etc(fFinishTransfersSem, semCount, B_RELATIVE_TIMEOUT, 0); 2852 2853 mutex_lock(&fFinishedLock); 2854 TRACE("finishing transfers\n"); 2855 while (fFinishedHead != NULL) { 2856 xhci_td* td = fFinishedHead; 2857 fFinishedHead = td->next; 2858 td->next = NULL; 2859 mutex_unlock(&fFinishedLock); 2860 2861 TRACE("finishing transfer td %p\n", td); 2862 2863 Transfer* transfer = td->transfer; 2864 if (transfer == NULL) { 2865 // No transfer? Quick way out. 2866 FreeDescriptor(td); 2867 mutex_lock(&fFinishedLock); 2868 continue; 2869 } 2870 2871 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 2872 2873 status_t callbackStatus = B_OK; 2874 switch (td->trb_completion_code) { 2875 case COMP_SHORT_PACKET: 2876 case COMP_SUCCESS: 2877 callbackStatus = B_OK; 2878 break; 2879 case COMP_DATA_BUFFER: 2880 callbackStatus = directionIn ? B_DEV_DATA_OVERRUN 2881 : B_DEV_DATA_UNDERRUN; 2882 break; 2883 case COMP_BABBLE: 2884 callbackStatus = directionIn ? B_DEV_FIFO_OVERRUN 2885 : B_DEV_FIFO_UNDERRUN; 2886 break; 2887 case COMP_USB_TRANSACTION: 2888 callbackStatus = B_DEV_CRC_ERROR; 2889 break; 2890 case COMP_STALL: 2891 callbackStatus = B_DEV_STALLED; 2892 break; 2893 default: 2894 callbackStatus = B_DEV_STALLED; 2895 break; 2896 } 2897 2898 size_t actualLength = transfer->DataLength(); 2899 if (td->trb_completion_code != COMP_SUCCESS) { 2900 actualLength = td->td_transferred; 2901 if (td->td_transferred == -1) 2902 actualLength = transfer->DataLength() - td->trb_left; 2903 TRACE("transfer not successful, actualLength=%" B_PRIuSIZE "\n", 2904 actualLength); 2905 } 2906 2907 usb_isochronous_data* isochronousData = transfer->IsochronousData(); 2908 if (isochronousData != NULL) { 2909 size_t packetSize = transfer->DataLength() / isochronousData->packet_count, 2910 left = actualLength; 2911 for (uint32 i = 0; i < isochronousData->packet_count; i++) { 2912 size_t size = min_c(packetSize, left); 2913 isochronousData->packet_descriptors[i].actual_length = size; 2914 isochronousData->packet_descriptors[i].status = (size > 0) 2915 ? B_OK : B_DEV_FIFO_UNDERRUN; 2916 left -= size; 2917 } 2918 } 2919 2920 if (callbackStatus == B_OK && directionIn && actualLength > 0) { 2921 TRACE("copying in iov count %ld\n", transfer->VectorCount()); 2922 status_t status = transfer->PrepareKernelAccess(); 2923 if (status == B_OK) { 2924 ReadDescriptor(td, transfer->Vector(), 2925 transfer->VectorCount()); 2926 } else { 2927 callbackStatus = status; 2928 } 2929 } 2930 transfer->Finished(callbackStatus, actualLength); 2931 delete transfer; 2932 FreeDescriptor(td); 2933 mutex_lock(&fFinishedLock); 2934 } 2935 mutex_unlock(&fFinishedLock); 2936 } 2937 } 2938 2939 2940 inline void 2941 XHCI::WriteOpReg(uint32 reg, uint32 value) 2942 { 2943 *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg) = value; 2944 } 2945 2946 2947 inline uint32 2948 XHCI::ReadOpReg(uint32 reg) 2949 { 2950 return *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg); 2951 } 2952 2953 2954 inline status_t 2955 XHCI::WaitOpBits(uint32 reg, uint32 mask, uint32 expected) 2956 { 2957 int loops = 0; 2958 uint32 value = ReadOpReg(reg); 2959 while ((value & mask) != expected) { 2960 snooze(1000); 2961 value = ReadOpReg(reg); 2962 if (loops == 100) { 2963 TRACE("delay waiting on reg 0x%" B_PRIX32 " match 0x%" B_PRIX32 2964 " (0x%" B_PRIX32 ")\n", reg, expected, mask); 2965 } else if (loops > 250) { 2966 TRACE_ERROR("timeout waiting on reg 0x%" B_PRIX32 2967 " match 0x%" B_PRIX32 " (0x%" B_PRIX32 ")\n", reg, expected, 2968 mask); 2969 return B_ERROR; 2970 } 2971 loops++; 2972 } 2973 return B_OK; 2974 } 2975 2976 2977 inline uint32 2978 XHCI::ReadCapReg32(uint32 reg) 2979 { 2980 return *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg); 2981 } 2982 2983 2984 inline void 2985 XHCI::WriteCapReg32(uint32 reg, uint32 value) 2986 { 2987 *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg) = value; 2988 } 2989 2990 2991 inline uint32 2992 XHCI::ReadRunReg32(uint32 reg) 2993 { 2994 return *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg); 2995 } 2996 2997 2998 inline void 2999 XHCI::WriteRunReg32(uint32 reg, uint32 value) 3000 { 3001 *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg) = value; 3002 } 3003 3004 3005 inline uint32 3006 XHCI::ReadDoorReg32(uint32 reg) 3007 { 3008 return *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg); 3009 } 3010 3011 3012 inline void 3013 XHCI::WriteDoorReg32(uint32 reg, uint32 value) 3014 { 3015 *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg) = value; 3016 } 3017 3018 3019 inline addr_t 3020 XHCI::_OffsetContextAddr(addr_t p) 3021 { 3022 if (fContextSizeShift == 1) { 3023 // each structure is page aligned, each pointer is 32 bits aligned 3024 uint32 offset = p & ((B_PAGE_SIZE - 1) & ~31U); 3025 p += offset; 3026 } 3027 return p; 3028 } 3029 3030 inline uint32 3031 XHCI::_ReadContext(uint32* p) 3032 { 3033 p = (uint32*)_OffsetContextAddr((addr_t)p); 3034 return *p; 3035 } 3036 3037 3038 inline void 3039 XHCI::_WriteContext(uint32* p, uint32 value) 3040 { 3041 p = (uint32*)_OffsetContextAddr((addr_t)p); 3042 *p = value; 3043 } 3044 3045 3046 inline uint64 3047 XHCI::_ReadContext(uint64* p) 3048 { 3049 p = (uint64*)_OffsetContextAddr((addr_t)p); 3050 return *p; 3051 } 3052 3053 3054 inline void 3055 XHCI::_WriteContext(uint64* p, uint64 value) 3056 { 3057 p = (uint64*)_OffsetContextAddr((addr_t)p); 3058 *p = value; 3059 } 3060