1 /* 2 * Copyright 2011-2021, Haiku, Inc. All rights reserved. 3 * Distributed under the terms of the MIT License. 4 * 5 * Authors: 6 * Augustin Cavalier <waddlesplash> 7 * Jian Chiang <j.jian.chiang@gmail.com> 8 * Jérôme Duval <jerome.duval@gmail.com> 9 * Akshay Jaggi <akshay1994.leo@gmail.com> 10 * Michael Lotz <mmlr@mlotz.ch> 11 * Alexander von Gluck <kallisti5@unixzen.com> 12 */ 13 14 15 #include <stdio.h> 16 17 #include <PCI_x86.h> 18 #include <bus/PCI.h> 19 #include <USB3.h> 20 #include <KernelExport.h> 21 22 #include <ByteOrder.h> 23 #include <util/AutoLock.h> 24 25 #include "xhci.h" 26 27 28 #define CALLED(x...) TRACE_MODULE("CALLED %s\n", __PRETTY_FUNCTION__) 29 30 31 #define USB_MODULE_NAME "xhci" 32 33 static pci_x86_module_info* sPCIx86Module = NULL; 34 device_manager_info* gDeviceManager; 35 static usb_for_controller_interface* gUSB; 36 37 38 #define XHCI_PCI_DEVICE_MODULE_NAME "busses/usb/xhci/pci/driver_v1" 39 #define XHCI_PCI_USB_BUS_MODULE_NAME "busses/usb/xhci/device_v1" 40 41 42 typedef struct { 43 XHCI* xhci; 44 pci_device_module_info* pci; 45 pci_device* device; 46 47 pci_info pciinfo; 48 49 device_node* node; 50 device_node* driver_node; 51 } xhci_pci_sim_info; 52 53 54 // #pragma mark - 55 56 57 static status_t 58 init_bus(device_node* node, void** bus_cookie) 59 { 60 CALLED(); 61 62 driver_module_info* driver; 63 xhci_pci_sim_info* bus; 64 device_node* parent = gDeviceManager->get_parent_node(node); 65 gDeviceManager->get_driver(parent, &driver, (void**)&bus); 66 gDeviceManager->put_node(parent); 67 68 Stack *stack; 69 if (gUSB->get_stack((void**)&stack) != B_OK) 70 return B_ERROR; 71 72 XHCI *xhci = new(std::nothrow) XHCI(&bus->pciinfo, bus->pci, bus->device, stack, node); 73 if (xhci == NULL) { 74 return B_NO_MEMORY; 75 } 76 77 if (xhci->InitCheck() < B_OK) { 78 TRACE_MODULE_ERROR("bus failed init check\n"); 79 delete xhci; 80 return B_ERROR; 81 } 82 83 if (xhci->Start() != B_OK) { 84 delete xhci; 85 return B_ERROR; 86 } 87 88 *bus_cookie = xhci; 89 90 return B_OK; 91 } 92 93 94 static void 95 uninit_bus(void* bus_cookie) 96 { 97 CALLED(); 98 XHCI* xhci = (XHCI*)bus_cookie; 99 delete xhci; 100 } 101 102 103 static status_t 104 register_child_devices(void* cookie) 105 { 106 CALLED(); 107 xhci_pci_sim_info* bus = (xhci_pci_sim_info*)cookie; 108 device_node* node = bus->driver_node; 109 110 char prettyName[25]; 111 sprintf(prettyName, "XHCI Controller %" B_PRIu16, 0); 112 113 device_attr attrs[] = { 114 // properties of this controller for the usb bus manager 115 { B_DEVICE_PRETTY_NAME, B_STRING_TYPE, 116 { .string = prettyName }}, 117 { B_DEVICE_FIXED_CHILD, B_STRING_TYPE, 118 { .string = USB_FOR_CONTROLLER_MODULE_NAME }}, 119 120 // private data to identify the device 121 { NULL } 122 }; 123 124 return gDeviceManager->register_node(node, XHCI_PCI_USB_BUS_MODULE_NAME, 125 attrs, NULL, NULL); 126 } 127 128 129 static status_t 130 init_device(device_node* node, void** device_cookie) 131 { 132 CALLED(); 133 xhci_pci_sim_info* bus = (xhci_pci_sim_info*)calloc(1, 134 sizeof(xhci_pci_sim_info)); 135 if (bus == NULL) 136 return B_NO_MEMORY; 137 138 pci_device_module_info* pci; 139 pci_device* device; 140 { 141 device_node* pciParent = gDeviceManager->get_parent_node(node); 142 gDeviceManager->get_driver(pciParent, (driver_module_info**)&pci, 143 (void**)&device); 144 gDeviceManager->put_node(pciParent); 145 } 146 147 bus->pci = pci; 148 bus->device = device; 149 bus->driver_node = node; 150 151 pci_info *pciInfo = &bus->pciinfo; 152 pci->get_pci_info(device, pciInfo); 153 154 *device_cookie = bus; 155 return B_OK; 156 } 157 158 159 static void 160 uninit_device(void* device_cookie) 161 { 162 CALLED(); 163 xhci_pci_sim_info* bus = (xhci_pci_sim_info*)device_cookie; 164 free(bus); 165 } 166 167 168 static status_t 169 register_device(device_node* parent) 170 { 171 CALLED(); 172 device_attr attrs[] = { 173 {B_DEVICE_PRETTY_NAME, B_STRING_TYPE, {.string = "XHCI PCI"}}, 174 {} 175 }; 176 177 return gDeviceManager->register_node(parent, 178 XHCI_PCI_DEVICE_MODULE_NAME, attrs, NULL, NULL); 179 } 180 181 182 static float 183 supports_device(device_node* parent) 184 { 185 CALLED(); 186 const char* bus; 187 uint16 type, subType, api; 188 189 // make sure parent is a XHCI PCI device node 190 if (gDeviceManager->get_attr_string(parent, B_DEVICE_BUS, &bus, false) 191 < B_OK) { 192 return -1; 193 } 194 195 if (strcmp(bus, "pci") != 0) 196 return 0.0f; 197 198 if (gDeviceManager->get_attr_uint16(parent, B_DEVICE_SUB_TYPE, &subType, 199 false) < B_OK 200 || gDeviceManager->get_attr_uint16(parent, B_DEVICE_TYPE, &type, 201 false) < B_OK 202 || gDeviceManager->get_attr_uint16(parent, B_DEVICE_INTERFACE, &api, 203 false) < B_OK) { 204 TRACE_MODULE("Could not find type/subtype/interface attributes\n"); 205 return -1; 206 } 207 208 if (type == PCI_serial_bus && subType == PCI_usb && api == PCI_usb_xhci) { 209 pci_device_module_info* pci; 210 pci_device* device; 211 gDeviceManager->get_driver(parent, (driver_module_info**)&pci, 212 (void**)&device); 213 TRACE_MODULE("XHCI Device found!\n"); 214 215 return 0.8f; 216 } 217 218 return 0.0f; 219 } 220 221 222 static const char* 223 xhci_error_string(uint32 error) 224 { 225 switch (error) { 226 case COMP_INVALID: return "Invalid"; 227 case COMP_SUCCESS: return "Success"; 228 case COMP_DATA_BUFFER: return "Data buffer"; 229 case COMP_BABBLE: return "Babble detected"; 230 case COMP_USB_TRANSACTION: return "USB transaction"; 231 case COMP_TRB: return "TRB"; 232 case COMP_STALL: return "Stall"; 233 case COMP_RESOURCE: return "Resource"; 234 case COMP_BANDWIDTH: return "Bandwidth"; 235 case COMP_NO_SLOTS: return "No slots"; 236 case COMP_INVALID_STREAM: return "Invalid stream"; 237 case COMP_SLOT_NOT_ENABLED: return "Slot not enabled"; 238 case COMP_ENDPOINT_NOT_ENABLED: return "Endpoint not enabled"; 239 case COMP_SHORT_PACKET: return "Short packet"; 240 case COMP_RING_UNDERRUN: return "Ring underrun"; 241 case COMP_RING_OVERRUN: return "Ring overrun"; 242 case COMP_VF_RING_FULL: return "VF Event Ring Full"; 243 case COMP_PARAMETER: return "Parameter"; 244 case COMP_BANDWIDTH_OVERRUN: return "Bandwidth overrun"; 245 case COMP_CONTEXT_STATE: return "Context state"; 246 case COMP_NO_PING_RESPONSE: return "No ping response"; 247 case COMP_EVENT_RING_FULL: return "Event ring full"; 248 case COMP_INCOMPATIBLE_DEVICE: return "Incompatible device"; 249 case COMP_MISSED_SERVICE: return "Missed service"; 250 case COMP_COMMAND_RING_STOPPED: return "Command ring stopped"; 251 case COMP_COMMAND_ABORTED: return "Command aborted"; 252 case COMP_STOPPED: return "Stopped"; 253 case COMP_LENGTH_INVALID: return "Length invalid"; 254 case COMP_MAX_EXIT_LATENCY: return "Max exit latency too large"; 255 case COMP_ISOC_OVERRUN: return "Isoch buffer overrun"; 256 case COMP_EVENT_LOST: return "Event lost"; 257 case COMP_UNDEFINED: return "Undefined"; 258 case COMP_INVALID_STREAM_ID: return "Invalid stream ID"; 259 case COMP_SECONDARY_BANDWIDTH: return "Secondary bandwidth"; 260 case COMP_SPLIT_TRANSACTION: return "Split transaction"; 261 262 default: return "Undefined"; 263 } 264 } 265 266 267 module_dependency module_dependencies[] = { 268 { USB_FOR_CONTROLLER_MODULE_NAME, (module_info**)&gUSB }, 269 { B_DEVICE_MANAGER_MODULE_NAME, (module_info**)&gDeviceManager }, 270 {} 271 }; 272 273 274 static status_t 275 device_std_ops(int32 op, ...) 276 { 277 switch (op) { 278 case B_MODULE_INIT: 279 if (get_module(B_PCI_X86_MODULE_NAME, (module_info**)&sPCIx86Module) != B_OK) 280 sPCIx86Module = NULL; 281 return B_OK; 282 case B_MODULE_UNINIT: 283 if (sPCIx86Module != NULL) 284 put_module(B_PCI_X86_MODULE_NAME); 285 return B_OK; 286 default: 287 return B_ERROR; 288 } 289 } 290 291 292 static usb_bus_interface gXHCIPCIDeviceModule = { 293 { 294 { 295 XHCI_PCI_USB_BUS_MODULE_NAME, 296 0, 297 device_std_ops 298 }, 299 NULL, // supports device 300 NULL, // register device 301 init_bus, 302 uninit_bus, 303 NULL, // register child devices 304 NULL, // rescan 305 NULL, // device removed 306 }, 307 }; 308 309 // Root device that binds to the PCI bus. It will register an usb_bus_interface 310 // node for each device. 311 static driver_module_info sXHCIDevice = { 312 { 313 XHCI_PCI_DEVICE_MODULE_NAME, 314 0, 315 NULL 316 }, 317 supports_device, 318 register_device, 319 init_device, 320 uninit_device, 321 register_child_devices, 322 NULL, // rescan 323 NULL, // device removed 324 }; 325 326 module_info* modules[] = { 327 (module_info* )&sXHCIDevice, 328 (module_info* )&gXHCIPCIDeviceModule, 329 NULL 330 }; 331 332 333 XHCI::XHCI(pci_info *info, pci_device_module_info* pci, pci_device* device, Stack *stack, 334 device_node* node) 335 : BusManager(stack, node), 336 fRegisterArea(-1), 337 fRegisters(NULL), 338 fPCIInfo(info), 339 fPci(pci), 340 fDevice(device), 341 fStack(stack), 342 fIRQ(0), 343 fUseMSI(false), 344 fErstArea(-1), 345 fDcbaArea(-1), 346 fCmdCompSem(-1), 347 fStopThreads(false), 348 fRootHub(NULL), 349 fPortCount(0), 350 fSlotCount(0), 351 fScratchpadCount(0), 352 fContextSizeShift(0), 353 fFinishedHead(NULL), 354 fFinishTransfersSem(-1), 355 fFinishThread(-1), 356 fEventSem(-1), 357 fEventThread(-1), 358 fEventIdx(0), 359 fCmdIdx(0), 360 fEventCcs(1), 361 fCmdCcs(1) 362 { 363 B_INITIALIZE_SPINLOCK(&fSpinlock); 364 mutex_init(&fFinishedLock, "XHCI finished transfers"); 365 mutex_init(&fEventLock, "XHCI event handler"); 366 367 if (BusManager::InitCheck() < B_OK) { 368 TRACE_ERROR("bus manager failed to init\n"); 369 return; 370 } 371 372 TRACE("constructing new XHCI host controller driver\n"); 373 fInitOK = false; 374 375 // enable busmaster and memory mapped access 376 uint16 command = fPci->read_pci_config(fDevice, PCI_command, 2); 377 command &= ~(PCI_command_io | PCI_command_int_disable); 378 command |= PCI_command_master | PCI_command_memory; 379 380 fPci->write_pci_config(fDevice, PCI_command, 2, command); 381 382 // map the registers (low + high for 64-bit when requested) 383 phys_addr_t physicalAddress = fPCIInfo->u.h0.base_registers[0]; 384 if ((fPCIInfo->u.h0.base_register_flags[0] & PCI_address_type) 385 == PCI_address_type_64) { 386 physicalAddress |= (uint64)fPCIInfo->u.h0.base_registers[1] << 32; 387 } 388 389 size_t mapSize = fPCIInfo->u.h0.base_register_sizes[0]; 390 391 TRACE("map registers %08" B_PRIxPHYSADDR ", size: %" B_PRIuSIZE "\n", 392 physicalAddress, mapSize); 393 394 fRegisterArea = map_physical_memory("XHCI memory mapped registers", 395 physicalAddress, mapSize, B_ANY_KERNEL_BLOCK_ADDRESS, 396 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 397 (void **)&fRegisters); 398 if (fRegisterArea < B_OK) { 399 TRACE_ERROR("failed to map register memory\n"); 400 return; 401 } 402 403 // determine the register offsets 404 fCapabilityRegisterOffset = 0; 405 fOperationalRegisterOffset = HCI_CAPLENGTH(ReadCapReg32(XHCI_HCI_CAPLENGTH)); 406 fRuntimeRegisterOffset = ReadCapReg32(XHCI_RTSOFF) & ~0x1F; 407 fDoorbellRegisterOffset = ReadCapReg32(XHCI_DBOFF) & ~0x3; 408 409 TRACE("mapped registers: %p\n", fRegisters); 410 TRACE("operational register offset: %" B_PRId32 "\n", fOperationalRegisterOffset); 411 TRACE("runtime register offset: %" B_PRId32 "\n", fRuntimeRegisterOffset); 412 TRACE("doorbell register offset: %" B_PRId32 "\n", fDoorbellRegisterOffset); 413 414 int32 interfaceVersion = HCI_VERSION(ReadCapReg32(XHCI_HCI_VERSION)); 415 if (interfaceVersion < 0x0090 || interfaceVersion > 0x0120) { 416 TRACE_ERROR("unsupported interface version: 0x%04" B_PRIx32 "\n", 417 interfaceVersion); 418 return; 419 } 420 TRACE_ALWAYS("interface version: 0x%04" B_PRIx32 "\n", interfaceVersion); 421 422 TRACE_ALWAYS("structural parameters: 1:0x%08" B_PRIx32 " 2:0x%08" 423 B_PRIx32 " 3:0x%08" B_PRIx32 "\n", ReadCapReg32(XHCI_HCSPARAMS1), 424 ReadCapReg32(XHCI_HCSPARAMS2), ReadCapReg32(XHCI_HCSPARAMS3)); 425 426 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 427 if (cparams == 0xffffffff) 428 return; 429 TRACE_ALWAYS("capability parameters: 0x%08" B_PRIx32 "\n", cparams); 430 431 // if 64 bytes context structures, then 1 432 fContextSizeShift = HCC_CSZ(cparams); 433 434 // Assume ownership of the controller from the BIOS. 435 uint32 eec = 0xffffffff; 436 uint32 eecp = HCS0_XECP(cparams) << 2; 437 for (; eecp != 0 && XECP_NEXT(eec); eecp += XECP_NEXT(eec) << 2) { 438 TRACE("eecp register: 0x%08" B_PRIx32 "\n", eecp); 439 440 eec = ReadCapReg32(eecp); 441 if (XECP_ID(eec) != XHCI_LEGSUP_CAPID) 442 continue; 443 444 if (eec & XHCI_LEGSUP_BIOSOWNED) { 445 TRACE_ALWAYS("the host controller is bios owned, claiming" 446 " ownership\n"); 447 WriteCapReg32(eecp, eec | XHCI_LEGSUP_OSOWNED); 448 449 for (int32 i = 0; i < 20; i++) { 450 eec = ReadCapReg32(eecp); 451 452 if ((eec & XHCI_LEGSUP_BIOSOWNED) == 0) 453 break; 454 455 TRACE_ALWAYS("controller is still bios owned, waiting\n"); 456 snooze(50000); 457 } 458 459 if (eec & XHCI_LEGSUP_BIOSOWNED) { 460 TRACE_ERROR("bios won't give up control over the host " 461 "controller (ignoring)\n"); 462 } else if (eec & XHCI_LEGSUP_OSOWNED) { 463 TRACE_ALWAYS("successfully took ownership of the host " 464 "controller\n"); 465 } 466 467 // Force off the BIOS owned flag, and clear all SMIs. Some BIOSes 468 // do indicate a successful handover but do not remove their SMIs 469 // and then freeze the system when interrupts are generated. 470 WriteCapReg32(eecp, eec & ~XHCI_LEGSUP_BIOSOWNED); 471 } 472 break; 473 } 474 uint32 legctlsts = ReadCapReg32(eecp + XHCI_LEGCTLSTS); 475 legctlsts &= XHCI_LEGCTLSTS_DISABLE_SMI; 476 legctlsts |= XHCI_LEGCTLSTS_EVENTS_SMI; 477 WriteCapReg32(eecp + XHCI_LEGCTLSTS, legctlsts); 478 479 // We need to explicitly take ownership of EHCI ports on earlier Intel chipsets. 480 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL) { 481 switch (fPCIInfo->device_id) { 482 case PCI_DEVICE_INTEL_PANTHER_POINT_XHCI: 483 case PCI_DEVICE_INTEL_LYNX_POINT_XHCI: 484 case PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI: 485 case PCI_DEVICE_INTEL_BAYTRAIL_XHCI: 486 case PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI: 487 case PCI_DEVICE_INTEL_WILDCAT_POINT_LP_XHCI: 488 _SwitchIntelPorts(); 489 break; 490 } 491 } 492 493 // halt the host controller 494 if (ControllerHalt() < B_OK) { 495 return; 496 } 497 498 // reset the host controller 499 if (ControllerReset() < B_OK) { 500 TRACE_ERROR("host controller failed to reset\n"); 501 return; 502 } 503 504 fCmdCompSem = create_sem(0, "XHCI Command Complete"); 505 fFinishTransfersSem = create_sem(0, "XHCI Finish Transfers"); 506 fEventSem = create_sem(0, "XHCI Event"); 507 if (fFinishTransfersSem < B_OK || fCmdCompSem < B_OK || fEventSem < B_OK) { 508 TRACE_ERROR("failed to create semaphores\n"); 509 return; 510 } 511 512 // create event handler thread 513 fEventThread = spawn_kernel_thread(EventThread, "xhci event thread", 514 B_URGENT_PRIORITY, (void *)this); 515 resume_thread(fEventThread); 516 517 // create finisher service thread 518 fFinishThread = spawn_kernel_thread(FinishThread, "xhci finish thread", 519 B_URGENT_PRIORITY - 1, (void *)this); 520 resume_thread(fFinishThread); 521 522 // Find the right interrupt vector, using MSIs if available. 523 fIRQ = fPCIInfo->u.h0.interrupt_line; 524 if (sPCIx86Module != NULL && sPCIx86Module->get_msi_count(fPCIInfo->bus, 525 fPCIInfo->device, fPCIInfo->function) >= 1) { 526 uint8 msiVector = 0; 527 if (sPCIx86Module->configure_msi(fPCIInfo->bus, fPCIInfo->device, 528 fPCIInfo->function, 1, &msiVector) == B_OK 529 && sPCIx86Module->enable_msi(fPCIInfo->bus, fPCIInfo->device, 530 fPCIInfo->function) == B_OK) { 531 TRACE_ALWAYS("using message signaled interrupts\n"); 532 fIRQ = msiVector; 533 fUseMSI = true; 534 } 535 } 536 537 if (fIRQ == 0 || fIRQ == 0xFF) { 538 TRACE_MODULE_ERROR("device PCI:%d:%d:%d was assigned an invalid IRQ\n", 539 fPCIInfo->bus, fPCIInfo->device, fPCIInfo->function); 540 return; 541 } 542 543 // Install the interrupt handler 544 TRACE("installing interrupt handler\n"); 545 install_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this, 0); 546 547 memset(fPortSpeeds, 0, sizeof(fPortSpeeds)); 548 memset(fDevices, 0, sizeof(fDevices)); 549 550 fInitOK = true; 551 TRACE("driver construction successful\n"); 552 } 553 554 555 XHCI::~XHCI() 556 { 557 TRACE("tear down XHCI host controller driver\n"); 558 559 WriteOpReg(XHCI_CMD, 0); 560 561 int32 result = 0; 562 fStopThreads = true; 563 delete_sem(fCmdCompSem); 564 delete_sem(fFinishTransfersSem); 565 delete_sem(fEventSem); 566 wait_for_thread(fFinishThread, &result); 567 wait_for_thread(fEventThread, &result); 568 569 mutex_destroy(&fFinishedLock); 570 mutex_destroy(&fEventLock); 571 572 remove_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this); 573 574 delete_area(fRegisterArea); 575 delete_area(fErstArea); 576 for (uint32 i = 0; i < fScratchpadCount; i++) 577 delete_area(fScratchpadArea[i]); 578 delete_area(fDcbaArea); 579 580 if (fUseMSI && sPCIx86Module != NULL) { 581 sPCIx86Module->disable_msi(fPCIInfo->bus, 582 fPCIInfo->device, fPCIInfo->function); 583 sPCIx86Module->unconfigure_msi(fPCIInfo->bus, 584 fPCIInfo->device, fPCIInfo->function); 585 } 586 } 587 588 589 void 590 XHCI::_SwitchIntelPorts() 591 { 592 TRACE("Looking for EHCI owned ports\n"); 593 uint32 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_USB3PRM, 4); 594 TRACE("Superspeed Ports: 0x%" B_PRIx32 "\n", ports); 595 fPci->write_pci_config(fDevice, XHCI_INTEL_USB3_PSSEN, 4, ports); 596 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_USB3_PSSEN, 4); 597 TRACE("Superspeed ports now under XHCI : 0x%" B_PRIx32 "\n", ports); 598 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_USB2PRM, 4); 599 TRACE("USB 2.0 Ports : 0x%" B_PRIx32 "\n", ports); 600 fPci->write_pci_config(fDevice, XHCI_INTEL_XUSB2PR, 4, ports); 601 ports = fPci->read_pci_config(fDevice, XHCI_INTEL_XUSB2PR, 4); 602 TRACE("USB 2.0 ports now under XHCI: 0x%" B_PRIx32 "\n", ports); 603 } 604 605 606 status_t 607 XHCI::Start() 608 { 609 TRACE_ALWAYS("starting XHCI host controller\n"); 610 TRACE("usbcmd: 0x%08" B_PRIx32 "; usbsts: 0x%08" B_PRIx32 "\n", 611 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 612 613 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 614 TRACE("Start() failed STS_CNR\n"); 615 } 616 617 if ((ReadOpReg(XHCI_CMD) & CMD_RUN) != 0) { 618 TRACE_ERROR("Start() warning, starting running XHCI controller!\n"); 619 } 620 621 if ((ReadOpReg(XHCI_PAGESIZE) & (1 << 0)) == 0) { 622 TRACE_ERROR("controller does not support 4K page size\n"); 623 return B_ERROR; 624 } 625 626 // read port count from capability register 627 uint32 capabilities = ReadCapReg32(XHCI_HCSPARAMS1); 628 fPortCount = HCS_MAX_PORTS(capabilities); 629 if (fPortCount == 0) { 630 TRACE_ERROR("invalid number of ports: %u\n", fPortCount); 631 return B_ERROR; 632 } 633 634 fSlotCount = HCS_MAX_SLOTS(capabilities); 635 if (fSlotCount > XHCI_MAX_DEVICES) 636 fSlotCount = XHCI_MAX_DEVICES; 637 WriteOpReg(XHCI_CONFIG, fSlotCount); 638 639 // find out which protocol is used for each port 640 uint8 portFound = 0; 641 uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS); 642 uint32 eec = 0xffffffff; 643 uint32 eecp = HCS0_XECP(cparams) << 2; 644 for (; eecp != 0 && XECP_NEXT(eec) && portFound < fPortCount; 645 eecp += XECP_NEXT(eec) << 2) { 646 eec = ReadCapReg32(eecp); 647 if (XECP_ID(eec) != XHCI_SUPPORTED_PROTOCOLS_CAPID) 648 continue; 649 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) > 3) 650 continue; 651 uint32 temp = ReadCapReg32(eecp + 8); 652 uint32 offset = XHCI_SUPPORTED_PROTOCOLS_1_OFFSET(temp); 653 uint32 count = XHCI_SUPPORTED_PROTOCOLS_1_COUNT(temp); 654 if (offset == 0 || count == 0) 655 continue; 656 offset--; 657 for (uint32 i = offset; i < offset + count; i++) { 658 if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) == 0x3) 659 fPortSpeeds[i] = USB_SPEED_SUPERSPEED; 660 else 661 fPortSpeeds[i] = USB_SPEED_HIGHSPEED; 662 663 TRACE("speed for port %" B_PRId32 " is %s\n", i, 664 fPortSpeeds[i] == USB_SPEED_SUPERSPEED ? "super" : "high"); 665 } 666 portFound += count; 667 } 668 669 uint32 params2 = ReadCapReg32(XHCI_HCSPARAMS2); 670 fScratchpadCount = HCS_MAX_SC_BUFFERS(params2); 671 if (fScratchpadCount > XHCI_MAX_SCRATCHPADS) { 672 TRACE_ERROR("invalid number of scratchpads: %" B_PRIu32 "\n", 673 fScratchpadCount); 674 return B_ERROR; 675 } 676 677 uint32 params3 = ReadCapReg32(XHCI_HCSPARAMS3); 678 fExitLatMax = HCS_U1_DEVICE_LATENCY(params3) 679 + HCS_U2_DEVICE_LATENCY(params3); 680 681 // clear interrupts & disable device notifications 682 WriteOpReg(XHCI_STS, ReadOpReg(XHCI_STS)); 683 WriteOpReg(XHCI_DNCTRL, 0); 684 685 // allocate Device Context Base Address array 686 phys_addr_t dmaAddress; 687 fDcbaArea = fStack->AllocateArea((void **)&fDcba, &dmaAddress, 688 sizeof(*fDcba), "DCBA Area"); 689 if (fDcbaArea < B_OK) { 690 TRACE_ERROR("unable to create the DCBA area\n"); 691 return B_ERROR; 692 } 693 memset(fDcba, 0, sizeof(*fDcba)); 694 memset(fScratchpadArea, 0, sizeof(fScratchpadArea)); 695 memset(fScratchpad, 0, sizeof(fScratchpad)); 696 697 // setting the first address to the scratchpad array address 698 fDcba->baseAddress[0] = dmaAddress 699 + offsetof(struct xhci_device_context_array, scratchpad); 700 701 // fill up the scratchpad array with scratchpad pages 702 for (uint32 i = 0; i < fScratchpadCount; i++) { 703 phys_addr_t scratchDmaAddress; 704 fScratchpadArea[i] = fStack->AllocateArea((void **)&fScratchpad[i], 705 &scratchDmaAddress, B_PAGE_SIZE, "Scratchpad Area"); 706 if (fScratchpadArea[i] < B_OK) { 707 TRACE_ERROR("unable to create the scratchpad area\n"); 708 return B_ERROR; 709 } 710 fDcba->scratchpad[i] = scratchDmaAddress; 711 } 712 713 TRACE("setting DCBAAP %" B_PRIxPHYSADDR "\n", dmaAddress); 714 WriteOpReg(XHCI_DCBAAP_LO, (uint32)dmaAddress); 715 WriteOpReg(XHCI_DCBAAP_HI, (uint32)(dmaAddress >> 32)); 716 717 // allocate Event Ring Segment Table 718 uint8 *addr; 719 fErstArea = fStack->AllocateArea((void **)&addr, &dmaAddress, 720 (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 721 + sizeof(xhci_erst_element), 722 "USB XHCI ERST CMD_RING and EVENT_RING Area"); 723 724 if (fErstArea < B_OK) { 725 TRACE_ERROR("unable to create the ERST AND RING area\n"); 726 delete_area(fDcbaArea); 727 return B_ERROR; 728 } 729 fErst = (xhci_erst_element *)addr; 730 memset(fErst, 0, (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb) 731 + sizeof(xhci_erst_element)); 732 733 // fill with Event Ring Segment Base Address and Event Ring Segment Size 734 fErst->rs_addr = dmaAddress + sizeof(xhci_erst_element); 735 fErst->rs_size = XHCI_MAX_EVENTS; 736 fErst->rsvdz = 0; 737 738 addr += sizeof(xhci_erst_element); 739 fEventRing = (xhci_trb *)addr; 740 addr += XHCI_MAX_EVENTS * sizeof(xhci_trb); 741 fCmdRing = (xhci_trb *)addr; 742 743 TRACE("setting ERST size\n"); 744 WriteRunReg32(XHCI_ERSTSZ(0), XHCI_ERSTS_SET(1)); 745 746 TRACE("setting ERDP addr = 0x%" B_PRIx64 "\n", fErst->rs_addr); 747 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)fErst->rs_addr); 748 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(fErst->rs_addr >> 32)); 749 750 TRACE("setting ERST base addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 751 WriteRunReg32(XHCI_ERSTBA_LO(0), (uint32)dmaAddress); 752 WriteRunReg32(XHCI_ERSTBA_HI(0), (uint32)(dmaAddress >> 32)); 753 754 dmaAddress += sizeof(xhci_erst_element) + XHCI_MAX_EVENTS 755 * sizeof(xhci_trb); 756 757 // Make sure the Command Ring is stopped 758 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 759 TRACE_ALWAYS("Command Ring is running, send stop/cancel\n"); 760 WriteOpReg(XHCI_CRCR_LO, CRCR_CS); 761 WriteOpReg(XHCI_CRCR_HI, 0); 762 WriteOpReg(XHCI_CRCR_LO, CRCR_CA); 763 WriteOpReg(XHCI_CRCR_HI, 0); 764 snooze(1000); 765 if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) { 766 TRACE_ERROR("Command Ring still running after stop/cancel\n"); 767 } 768 } 769 TRACE("setting CRCR addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress); 770 WriteOpReg(XHCI_CRCR_LO, (uint32)dmaAddress | CRCR_RCS); 771 WriteOpReg(XHCI_CRCR_HI, (uint32)(dmaAddress >> 32)); 772 // link trb 773 fCmdRing[XHCI_MAX_COMMANDS - 1].address = dmaAddress; 774 775 TRACE("setting interrupt rate\n"); 776 777 // Setting IMOD below 0x3F8 on Intel Lynx Point can cause IRQ lockups 778 if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL 779 && (fPCIInfo->device_id == PCI_DEVICE_INTEL_PANTHER_POINT_XHCI 780 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_XHCI 781 || fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI 782 || fPCIInfo->device_id == PCI_DEVICE_INTEL_BAYTRAIL_XHCI 783 || fPCIInfo->device_id == PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI)) { 784 WriteRunReg32(XHCI_IMOD(0), 0x000003f8); // 4000 irq/s 785 } else { 786 WriteRunReg32(XHCI_IMOD(0), 0x000001f4); // 8000 irq/s 787 } 788 789 TRACE("enabling interrupt\n"); 790 WriteRunReg32(XHCI_IMAN(0), ReadRunReg32(XHCI_IMAN(0)) | IMAN_INTR_ENA); 791 792 WriteOpReg(XHCI_CMD, CMD_RUN | CMD_INTE | CMD_HSEE); 793 794 // wait for start up state 795 if (WaitOpBits(XHCI_STS, STS_HCH, 0) != B_OK) { 796 TRACE_ERROR("HCH start up timeout\n"); 797 } 798 799 fRootHub = new(std::nothrow) XHCIRootHub(RootObject(), 1); 800 if (!fRootHub) { 801 TRACE_ERROR("no memory to allocate root hub\n"); 802 return B_NO_MEMORY; 803 } 804 805 if (fRootHub->InitCheck() < B_OK) { 806 TRACE_ERROR("root hub failed init check\n"); 807 return fRootHub->InitCheck(); 808 } 809 810 SetRootHub(fRootHub); 811 812 fRootHub->RegisterNode(Node()); 813 814 TRACE_ALWAYS("successfully started the controller\n"); 815 816 #ifdef TRACE_USB 817 TRACE("No-Op test...\n"); 818 Noop(); 819 #endif 820 821 return BusManager::Start(); 822 } 823 824 825 status_t 826 XHCI::SubmitTransfer(Transfer *transfer) 827 { 828 // short circuit the root hub 829 if (transfer->TransferPipe()->DeviceAddress() == 1) 830 return fRootHub->ProcessTransfer(this, transfer); 831 832 TRACE("SubmitTransfer(%p)\n", transfer); 833 Pipe *pipe = transfer->TransferPipe(); 834 if ((pipe->Type() & USB_OBJECT_CONTROL_PIPE) != 0) 835 return SubmitControlRequest(transfer); 836 return SubmitNormalRequest(transfer); 837 } 838 839 840 status_t 841 XHCI::SubmitControlRequest(Transfer *transfer) 842 { 843 Pipe *pipe = transfer->TransferPipe(); 844 usb_request_data *requestData = transfer->RequestData(); 845 bool directionIn = (requestData->RequestType & USB_REQTYPE_DEVICE_IN) != 0; 846 847 TRACE("SubmitControlRequest() length %d\n", requestData->Length); 848 849 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 850 if (endpoint == NULL) { 851 TRACE_ERROR("control pipe has no endpoint!\n"); 852 return B_BAD_VALUE; 853 } 854 if (endpoint->device == NULL) { 855 panic("endpoint is not initialized!"); 856 return B_NO_INIT; 857 } 858 859 status_t status = transfer->InitKernelAccess(); 860 if (status != B_OK) 861 return status; 862 863 xhci_td *descriptor = CreateDescriptor(3, 1, requestData->Length); 864 if (descriptor == NULL) 865 return B_NO_MEMORY; 866 descriptor->transfer = transfer; 867 868 // Setup Stage 869 uint8 index = 0; 870 memcpy(&descriptor->trbs[index].address, requestData, 871 sizeof(usb_request_data)); 872 descriptor->trbs[index].status = TRB_2_IRQ(0) | TRB_2_BYTES(8); 873 descriptor->trbs[index].flags 874 = TRB_3_TYPE(TRB_TYPE_SETUP_STAGE) | TRB_3_IDT_BIT | TRB_3_CYCLE_BIT; 875 if (requestData->Length > 0) { 876 descriptor->trbs[index].flags |= 877 directionIn ? TRB_3_TRT_IN : TRB_3_TRT_OUT; 878 } 879 880 index++; 881 882 // Data Stage (if any) 883 if (requestData->Length > 0) { 884 descriptor->trbs[index].address = descriptor->buffer_addrs[0]; 885 descriptor->trbs[index].status = TRB_2_IRQ(0) 886 | TRB_2_BYTES(requestData->Length) 887 | TRB_2_TD_SIZE(0); 888 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_DATA_STAGE) 889 | (directionIn ? TRB_3_DIR_IN : 0) 890 | TRB_3_CYCLE_BIT; 891 892 if (!directionIn) { 893 transfer->PrepareKernelAccess(); 894 memcpy(descriptor->buffers[0], 895 (uint8 *)transfer->Vector()[0].iov_base, requestData->Length); 896 } 897 898 index++; 899 } 900 901 // Status Stage 902 descriptor->trbs[index].address = 0; 903 descriptor->trbs[index].status = TRB_2_IRQ(0); 904 descriptor->trbs[index].flags = TRB_3_TYPE(TRB_TYPE_STATUS_STAGE) 905 | TRB_3_CHAIN_BIT | TRB_3_ENT_BIT | TRB_3_CYCLE_BIT; 906 // The CHAIN bit must be set when using an Event Data TRB 907 // (XHCI 1.2 § 6.4.1.2.3 Table 6-31 p472). 908 909 // Status Stage is an OUT transfer when the device is sending data 910 // (XHCI 1.2 § 4.11.2.2 Table 4-7 p213), otherwise set the IN bit. 911 if (requestData->Length == 0 || !directionIn) 912 descriptor->trbs[index].flags |= TRB_3_DIR_IN; 913 914 descriptor->trb_used = index + 1; 915 916 status = _LinkDescriptorForPipe(descriptor, endpoint); 917 if (status != B_OK) { 918 FreeDescriptor(descriptor); 919 return status; 920 } 921 922 return B_OK; 923 } 924 925 926 status_t 927 XHCI::SubmitNormalRequest(Transfer *transfer) 928 { 929 TRACE("SubmitNormalRequest() length %" B_PRIuSIZE "\n", transfer->FragmentLength()); 930 931 Pipe *pipe = transfer->TransferPipe(); 932 usb_isochronous_data *isochronousData = transfer->IsochronousData(); 933 bool directionIn = (pipe->Direction() == Pipe::In); 934 935 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 936 if (endpoint == NULL) { 937 TRACE_ERROR("pipe has no endpoint!\n"); 938 return B_BAD_VALUE; 939 } 940 if (endpoint->device == NULL) { 941 panic("endpoint is not initialized!"); 942 return B_NO_INIT; 943 } 944 945 status_t status = transfer->InitKernelAccess(); 946 if (status != B_OK) 947 return status; 948 949 // TRBs within a TD must be "grouped" into TD Fragments, which mostly means 950 // that a max_burst_payload boundary cannot be crossed within a TRB, but 951 // only between TRBs. More than one TRB can be in a TD Fragment, but we keep 952 // things simple by setting trbSize to the MBP. (XHCI 1.2 § 4.11.7.1 p235.) 953 size_t trbSize = endpoint->max_burst_payload; 954 955 if (isochronousData != NULL) { 956 if (isochronousData->packet_count == 0) 957 return B_BAD_VALUE; 958 959 // Isochronous transfers use more specifically sized packets. 960 trbSize = transfer->DataLength() / isochronousData->packet_count; 961 if (trbSize == 0 || trbSize > pipe->MaxPacketSize() || trbSize 962 != (size_t)isochronousData->packet_descriptors[0].request_length) 963 return B_BAD_VALUE; 964 } 965 966 // Now that we know trbSize, compute the count. 967 const int32 trbCount = (transfer->FragmentLength() + trbSize - 1) / trbSize; 968 969 xhci_td *td = CreateDescriptor(trbCount, trbCount, trbSize); 970 if (td == NULL) 971 return B_NO_MEMORY; 972 973 // Normal Stage 974 const size_t maxPacketSize = pipe->MaxPacketSize(); 975 size_t remaining = transfer->FragmentLength(); 976 for (int32 i = 0; i < trbCount; i++) { 977 int32 trbLength = (remaining < trbSize) ? remaining : trbSize; 978 remaining -= trbLength; 979 980 // The "TD Size" field of a transfer TRB indicates the number of 981 // remaining maximum-size *packets* in this TD, *not* including the 982 // packets in the current TRB, and capped at 31 if there are more 983 // than 31 packets remaining in the TD. (XHCI 1.2 § 4.11.2.4 p218.) 984 int32 tdSize = (remaining + maxPacketSize - 1) / maxPacketSize; 985 if (tdSize > 31) 986 tdSize = 31; 987 988 td->trbs[i].address = td->buffer_addrs[i]; 989 td->trbs[i].status = TRB_2_IRQ(0) 990 | TRB_2_BYTES(trbLength) 991 | TRB_2_TD_SIZE(tdSize); 992 td->trbs[i].flags = TRB_3_TYPE(TRB_TYPE_NORMAL) 993 | TRB_3_CYCLE_BIT | TRB_3_CHAIN_BIT; 994 995 td->trb_used++; 996 } 997 998 // Isochronous-specific 999 if (isochronousData != NULL) { 1000 // This is an isochronous transfer; we need to make the first TRB 1001 // an isochronous TRB. 1002 td->trbs[0].flags &= ~(TRB_3_TYPE(TRB_TYPE_NORMAL)); 1003 td->trbs[0].flags |= TRB_3_TYPE(TRB_TYPE_ISOCH); 1004 1005 // Isochronous pipes are scheduled by microframes, one of which 1006 // is 125us for USB 2 and above. But for USB 1 it was 1ms, so 1007 // we need to use a different frame delta for that case. 1008 uint8 frameDelta = 1; 1009 if (transfer->TransferPipe()->Speed() == USB_SPEED_FULLSPEED) 1010 frameDelta = 8; 1011 1012 // TODO: We do not currently take Mult into account at all! 1013 // How are we supposed to do that here? 1014 1015 // Determine the (starting) frame number: if ISO_ASAP is set, 1016 // we are queueing this "right away", and so want to reset 1017 // the starting_frame_number. Otherwise we use the passed one. 1018 uint32 frame; 1019 if ((isochronousData->flags & USB_ISO_ASAP) != 0 1020 || isochronousData->starting_frame_number == NULL) { 1021 // All reads from the microframe index register must be 1022 // incremented by 1. (XHCI 1.2 § 4.14.2.1.4 p265.) 1023 frame = ReadRunReg32(XHCI_MFINDEX) + 1; 1024 td->trbs[0].flags |= TRB_3_ISO_SIA_BIT; 1025 } else { 1026 frame = *isochronousData->starting_frame_number; 1027 td->trbs[0].flags |= TRB_3_FRID(frame); 1028 } 1029 frame = (frame + frameDelta) % 2048; 1030 if (isochronousData->starting_frame_number != NULL) 1031 *isochronousData->starting_frame_number = frame; 1032 1033 // TODO: The OHCI bus driver seems to also do this for inbound 1034 // isochronous transfers. Perhaps it should be moved into the stack? 1035 if (directionIn) { 1036 for (uint32 i = 0; i < isochronousData->packet_count; i++) { 1037 isochronousData->packet_descriptors[i].actual_length = 0; 1038 isochronousData->packet_descriptors[i].status = B_NO_INIT; 1039 } 1040 } 1041 } 1042 1043 // Set the ENT (Evaluate Next TRB) bit, so that the HC will not switch 1044 // contexts before evaluating the Link TRB that _LinkDescriptorForPipe 1045 // will insert, as otherwise there would be a race between us freeing 1046 // and unlinking the descriptor, and the controller evaluating the Link TRB 1047 // and thus getting back onto the main ring and executing the Event Data 1048 // TRB that generates the interrupt for this transfer. 1049 // 1050 // Note that we *do not* unset the CHAIN bit in this TRB, thus including 1051 // the Link TRB in this TD formally, which is required when using the 1052 // ENT bit. (XHCI 1.2 § 4.12.3 p250.) 1053 td->trbs[td->trb_used - 1].flags |= TRB_3_ENT_BIT; 1054 1055 if (!directionIn) { 1056 TRACE("copying out iov count %ld\n", transfer->VectorCount()); 1057 status_t status = transfer->PrepareKernelAccess(); 1058 if (status != B_OK) { 1059 FreeDescriptor(td); 1060 return status; 1061 } 1062 WriteDescriptor(td, transfer->Vector(), transfer->VectorCount()); 1063 } 1064 1065 td->transfer = transfer; 1066 status = _LinkDescriptorForPipe(td, endpoint); 1067 if (status != B_OK) { 1068 FreeDescriptor(td); 1069 return status; 1070 } 1071 1072 return B_OK; 1073 } 1074 1075 1076 status_t 1077 XHCI::CancelQueuedTransfers(Pipe *pipe, bool force) 1078 { 1079 xhci_endpoint* endpoint = (xhci_endpoint*)pipe->ControllerCookie(); 1080 if (endpoint == NULL || endpoint->trbs == NULL) { 1081 // Someone's de-allocated this pipe or endpoint in the meantime. 1082 // (Possibly AllocateDevice failed, and we were the temporary pipe.) 1083 return B_NO_INIT; 1084 } 1085 1086 #ifndef TRACE_USB 1087 if (force) 1088 #endif 1089 { 1090 TRACE_ALWAYS("cancel queued transfers (%" B_PRId8 ") for pipe %p (%d)\n", 1091 endpoint->used, pipe, pipe->EndpointAddress()); 1092 } 1093 1094 MutexLocker endpointLocker(endpoint->lock); 1095 1096 if (endpoint->td_head == NULL) { 1097 // There aren't any currently pending transfers to cancel. 1098 return B_OK; 1099 } 1100 1101 // Calling the callbacks while holding the endpoint lock could potentially 1102 // cause deadlocks, so we instead store them in a pointer array. We need 1103 // to do this separately from freeing the TDs, for in the case we fail 1104 // to stop the endpoint, we cancel the transfers but do not free the TDs. 1105 Transfer* transfers[XHCI_MAX_TRANSFERS]; 1106 int32 transfersCount = 0; 1107 1108 for (xhci_td* td = endpoint->td_head; td != NULL; td = td->next) { 1109 if (td->transfer == NULL) 1110 continue; 1111 1112 // We can't cancel or delete transfers under "force", as they probably 1113 // are not safe to use anymore. 1114 if (!force) { 1115 transfers[transfersCount] = td->transfer; 1116 transfersCount++; 1117 } 1118 td->transfer = NULL; 1119 } 1120 1121 // It is possible that while waiting for the stop-endpoint command to 1122 // complete, one of the queued transfers posts a completion event, so in 1123 // order to avoid a deadlock, we must unlock the endpoint. 1124 endpointLocker.Unlock(); 1125 status_t status = StopEndpoint(false, endpoint); 1126 if (status == B_DEV_STALLED) { 1127 // Only exit from a Halted state is a reset. (XHCI 1.2 § 4.8.3 p163.) 1128 TRACE_ERROR("cancel queued transfers: halted endpoint, reset!\n"); 1129 status = ResetEndpoint(false, endpoint); 1130 } 1131 endpointLocker.Lock(); 1132 1133 // Detach the head TD from the endpoint. 1134 xhci_td* td_head = endpoint->td_head; 1135 endpoint->td_head = NULL; 1136 1137 if (status == B_OK) { 1138 // Clear the endpoint's TRBs. 1139 memset(endpoint->trbs, 0, sizeof(xhci_trb) * XHCI_ENDPOINT_RING_SIZE); 1140 endpoint->used = 0; 1141 endpoint->current = 0; 1142 1143 // Set dequeue pointer location to the beginning of the ring. 1144 SetTRDequeue(endpoint->trb_addr, 0, endpoint->id + 1, 1145 endpoint->device->slot); 1146 1147 // We don't need to do anything else to restart the ring, as it will resume 1148 // operation as normal upon the next doorbell. (XHCI 1.2 § 4.6.9 p136.) 1149 } else { 1150 // We couldn't stop the endpoint. Most likely the device has been 1151 // removed and the endpoint was stopped by the hardware, or is 1152 // for some reason busy and cannot be stopped. 1153 TRACE_ERROR("cancel queued transfers: could not stop endpoint: %s!\n", 1154 strerror(status)); 1155 1156 // Instead of freeing the TDs, we want to leave them in the endpoint 1157 // so that when/if the hardware returns, they can be properly unlinked, 1158 // as otherwise the endpoint could get "stuck" by having the "used" 1159 // slowly accumulate due to "dead" transfers. 1160 endpoint->td_head = td_head; 1161 td_head = NULL; 1162 } 1163 1164 endpointLocker.Unlock(); 1165 1166 for (int32 i = 0; i < transfersCount; i++) { 1167 transfers[i]->Finished(B_CANCELED, 0); 1168 delete transfers[i]; 1169 } 1170 1171 // This loop looks a bit strange because we need to store the "next" 1172 // pointer before freeing the descriptor. 1173 xhci_td* td; 1174 while ((td = td_head) != NULL) { 1175 td_head = td_head->next; 1176 FreeDescriptor(td); 1177 } 1178 1179 return B_OK; 1180 } 1181 1182 1183 status_t 1184 XHCI::StartDebugTransfer(Transfer *transfer) 1185 { 1186 Pipe *pipe = transfer->TransferPipe(); 1187 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 1188 if (endpoint == NULL) 1189 return B_BAD_VALUE; 1190 1191 // Check all locks that we are going to hit when running transfers. 1192 if (mutex_trylock(&endpoint->lock) != B_OK) 1193 return B_WOULD_BLOCK; 1194 if (mutex_trylock(&fFinishedLock) != B_OK) { 1195 mutex_unlock(&endpoint->lock); 1196 return B_WOULD_BLOCK; 1197 } 1198 if (mutex_trylock(&fEventLock) != B_OK) { 1199 mutex_unlock(&endpoint->lock); 1200 mutex_unlock(&fFinishedLock); 1201 return B_WOULD_BLOCK; 1202 } 1203 mutex_unlock(&endpoint->lock); 1204 mutex_unlock(&fFinishedLock); 1205 mutex_unlock(&fEventLock); 1206 1207 status_t status = SubmitTransfer(transfer); 1208 if (status != B_OK) 1209 return status; 1210 1211 // The endpoint's head TD is the TD of the just-submitted transfer. 1212 // Just like EHCI, abuse the callback cookie to hold the TD pointer. 1213 transfer->SetCallback(NULL, endpoint->td_head); 1214 1215 return B_OK; 1216 } 1217 1218 1219 status_t 1220 XHCI::CheckDebugTransfer(Transfer *transfer) 1221 { 1222 xhci_td *transfer_td = (xhci_td *)transfer->CallbackCookie(); 1223 if (transfer_td == NULL) 1224 return B_NO_INIT; 1225 1226 // Process events once, and then look for it in the finished list. 1227 ProcessEvents(); 1228 xhci_td *previous = NULL; 1229 for (xhci_td *td = fFinishedHead; td != NULL; td = td->next) { 1230 if (td != transfer_td) { 1231 previous = td; 1232 continue; 1233 } 1234 1235 // We've found it! 1236 if (previous == NULL) { 1237 fFinishedHead = fFinishedHead->next; 1238 } else { 1239 previous->next = td->next; 1240 } 1241 1242 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 1243 status_t status = (td->trb_completion_code == COMP_SUCCESS 1244 || td->trb_completion_code == COMP_SHORT_PACKET) ? B_OK : B_ERROR; 1245 1246 if (status == B_OK && directionIn) 1247 ReadDescriptor(td, transfer->Vector(), transfer->VectorCount()); 1248 1249 FreeDescriptor(td); 1250 transfer->SetCallback(NULL, NULL); 1251 return status; 1252 } 1253 1254 // We didn't find it. 1255 spin(75); 1256 return B_DEV_PENDING; 1257 } 1258 1259 1260 void 1261 XHCI::CancelDebugTransfer(Transfer *transfer) 1262 { 1263 while (CheckDebugTransfer(transfer) == B_DEV_PENDING) 1264 spin(100); 1265 } 1266 1267 1268 status_t 1269 XHCI::NotifyPipeChange(Pipe *pipe, usb_change change) 1270 { 1271 TRACE("pipe change %d for pipe %p (%d)\n", change, pipe, 1272 pipe->EndpointAddress()); 1273 1274 switch (change) { 1275 case USB_CHANGE_CREATED: 1276 return _InsertEndpointForPipe(pipe); 1277 case USB_CHANGE_DESTROYED: 1278 return _RemoveEndpointForPipe(pipe); 1279 1280 case USB_CHANGE_PIPE_POLICY_CHANGED: 1281 // We don't care about these, at least for now. 1282 return B_OK; 1283 } 1284 1285 TRACE_ERROR("unknown pipe change!\n"); 1286 return B_UNSUPPORTED; 1287 } 1288 1289 1290 xhci_td * 1291 XHCI::CreateDescriptor(uint32 trbCount, uint32 bufferCount, size_t bufferSize) 1292 { 1293 const bool inKDL = debug_debugger_running(); 1294 1295 xhci_td *result; 1296 if (!inKDL) { 1297 result = (xhci_td*)calloc(1, sizeof(xhci_td)); 1298 } else { 1299 // Just use the physical memory allocator while in KDL; it's less 1300 // secure than using the regular heap, but it's easier to deal with. 1301 phys_addr_t dummy; 1302 fStack->AllocateChunk((void **)&result, &dummy, sizeof(xhci_td)); 1303 } 1304 1305 if (result == NULL) { 1306 TRACE_ERROR("failed to allocate a transfer descriptor\n"); 1307 return NULL; 1308 } 1309 1310 // We always allocate 1 more TRB than requested, so that 1311 // _LinkDescriptorForPipe() has room to insert a link TRB. 1312 trbCount++; 1313 if (fStack->AllocateChunk((void **)&result->trbs, &result->trb_addr, 1314 (trbCount * sizeof(xhci_trb))) < B_OK) { 1315 TRACE_ERROR("failed to allocate TRBs\n"); 1316 FreeDescriptor(result); 1317 return NULL; 1318 } 1319 result->trb_count = trbCount; 1320 result->trb_used = 0; 1321 1322 if (bufferSize > 0) { 1323 // Due to how the USB stack allocates physical memory, we can't just 1324 // request one large chunk the size of the transfer, and so instead we 1325 // create a series of buffers as requested by our caller. 1326 1327 // We store the buffer pointers and addresses in one memory block. 1328 if (!inKDL) { 1329 result->buffers = (void**)calloc(bufferCount, 1330 (sizeof(void*) + sizeof(phys_addr_t))); 1331 } else { 1332 phys_addr_t dummy; 1333 fStack->AllocateChunk((void **)&result->buffers, &dummy, 1334 bufferCount * (sizeof(void*) + sizeof(phys_addr_t))); 1335 } 1336 if (result->buffers == NULL) { 1337 TRACE_ERROR("unable to allocate space for buffer infos\n"); 1338 FreeDescriptor(result); 1339 return NULL; 1340 } 1341 result->buffer_addrs = (phys_addr_t*)&result->buffers[bufferCount]; 1342 result->buffer_size = bufferSize; 1343 result->buffer_count = bufferCount; 1344 1345 // Optimization: If the requested total size of all buffers is less 1346 // than 32*B_PAGE_SIZE (the maximum size that the physical memory 1347 // allocator can handle), we allocate only one buffer and segment it. 1348 size_t totalSize = bufferSize * bufferCount; 1349 if (totalSize < (32 * B_PAGE_SIZE)) { 1350 if (fStack->AllocateChunk(&result->buffers[0], 1351 &result->buffer_addrs[0], totalSize) < B_OK) { 1352 TRACE_ERROR("unable to allocate space for large buffer (size %ld)\n", 1353 totalSize); 1354 FreeDescriptor(result); 1355 return NULL; 1356 } 1357 for (uint32 i = 1; i < bufferCount; i++) { 1358 result->buffers[i] = (void*)((addr_t)(result->buffers[i - 1]) 1359 + bufferSize); 1360 result->buffer_addrs[i] = result->buffer_addrs[i - 1] 1361 + bufferSize; 1362 } 1363 } else { 1364 // Otherwise, we allocate each buffer individually. 1365 for (uint32 i = 0; i < bufferCount; i++) { 1366 if (fStack->AllocateChunk(&result->buffers[i], 1367 &result->buffer_addrs[i], bufferSize) < B_OK) { 1368 TRACE_ERROR("unable to allocate space for a buffer (size " 1369 "%" B_PRIuSIZE ", count %" B_PRIu32 ")\n", 1370 bufferSize, bufferCount); 1371 FreeDescriptor(result); 1372 return NULL; 1373 } 1374 } 1375 } 1376 } else { 1377 result->buffers = NULL; 1378 result->buffer_addrs = NULL; 1379 } 1380 1381 // Initialize all other fields. 1382 result->transfer = NULL; 1383 result->trb_completion_code = 0; 1384 result->trb_left = 0; 1385 result->next = NULL; 1386 1387 TRACE("CreateDescriptor allocated %p, buffer_size %ld, buffer_count %" B_PRIu32 "\n", 1388 result, result->buffer_size, result->buffer_count); 1389 1390 return result; 1391 } 1392 1393 1394 void 1395 XHCI::FreeDescriptor(xhci_td *descriptor) 1396 { 1397 if (descriptor == NULL) 1398 return; 1399 1400 const bool inKDL = debug_debugger_running(); 1401 1402 if (descriptor->trbs != NULL) { 1403 fStack->FreeChunk(descriptor->trbs, descriptor->trb_addr, 1404 (descriptor->trb_count * sizeof(xhci_trb))); 1405 } 1406 if (descriptor->buffers != NULL) { 1407 size_t totalSize = descriptor->buffer_size * descriptor->buffer_count; 1408 if (totalSize < (32 * B_PAGE_SIZE)) { 1409 // This was allocated as one contiguous buffer. 1410 fStack->FreeChunk(descriptor->buffers[0], descriptor->buffer_addrs[0], 1411 totalSize); 1412 } else { 1413 for (uint32 i = 0; i < descriptor->buffer_count; i++) { 1414 if (descriptor->buffers[i] == NULL) 1415 continue; 1416 fStack->FreeChunk(descriptor->buffers[i], descriptor->buffer_addrs[i], 1417 descriptor->buffer_size); 1418 } 1419 } 1420 1421 if (!inKDL) { 1422 free(descriptor->buffers); 1423 } else { 1424 fStack->FreeChunk(descriptor->buffers, 0, 1425 descriptor->buffer_count * (sizeof(void*) + sizeof(phys_addr_t))); 1426 } 1427 } 1428 1429 if (!inKDL) 1430 free(descriptor); 1431 else 1432 fStack->FreeChunk(descriptor, 0, sizeof(xhci_td)); 1433 } 1434 1435 1436 size_t 1437 XHCI::WriteDescriptor(xhci_td *descriptor, iovec *vector, size_t vectorCount) 1438 { 1439 size_t written = 0; 1440 1441 size_t bufIdx = 0, bufUsed = 0; 1442 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1443 size_t length = vector[vecIdx].iov_len; 1444 1445 while (length > 0 && bufIdx < descriptor->buffer_count) { 1446 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1447 memcpy((uint8 *)descriptor->buffers[bufIdx] + bufUsed, 1448 (uint8 *)vector[vecIdx].iov_base + (vector[vecIdx].iov_len - length), 1449 toCopy); 1450 1451 written += toCopy; 1452 bufUsed += toCopy; 1453 length -= toCopy; 1454 if (bufUsed == descriptor->buffer_size) { 1455 bufIdx++; 1456 bufUsed = 0; 1457 } 1458 } 1459 } 1460 1461 TRACE("wrote descriptor (%" B_PRIuSIZE " bytes)\n", written); 1462 return written; 1463 } 1464 1465 1466 size_t 1467 XHCI::ReadDescriptor(xhci_td *descriptor, iovec *vector, size_t vectorCount) 1468 { 1469 size_t read = 0; 1470 1471 size_t bufIdx = 0, bufUsed = 0; 1472 for (size_t vecIdx = 0; vecIdx < vectorCount; vecIdx++) { 1473 size_t length = vector[vecIdx].iov_len; 1474 1475 while (length > 0 && bufIdx < descriptor->buffer_count) { 1476 size_t toCopy = min_c(length, descriptor->buffer_size - bufUsed); 1477 memcpy((uint8 *)vector[vecIdx].iov_base + (vector[vecIdx].iov_len - length), 1478 (uint8 *)descriptor->buffers[bufIdx] + bufUsed, toCopy); 1479 1480 read += toCopy; 1481 bufUsed += toCopy; 1482 length -= toCopy; 1483 if (bufUsed == descriptor->buffer_size) { 1484 bufIdx++; 1485 bufUsed = 0; 1486 } 1487 } 1488 } 1489 1490 TRACE("read descriptor (%" B_PRIuSIZE " bytes)\n", read); 1491 return read; 1492 } 1493 1494 1495 Device * 1496 XHCI::AllocateDevice(Hub *parent, int8 hubAddress, uint8 hubPort, 1497 usb_speed speed) 1498 { 1499 TRACE("AllocateDevice hubAddress %d hubPort %d speed %d\n", hubAddress, 1500 hubPort, speed); 1501 1502 uint8 slot = XHCI_MAX_SLOTS; 1503 status_t status = EnableSlot(&slot); 1504 if (status != B_OK) { 1505 TRACE_ERROR("failed to enable slot: %s\n", strerror(status)); 1506 return NULL; 1507 } 1508 1509 if (slot == 0 || slot > fSlotCount) { 1510 TRACE_ERROR("AllocateDevice: bad slot\n"); 1511 return NULL; 1512 } 1513 1514 if (fDevices[slot].slot != 0) { 1515 TRACE_ERROR("AllocateDevice: slot already used\n"); 1516 return NULL; 1517 } 1518 1519 struct xhci_device *device = &fDevices[slot]; 1520 device->slot = slot; 1521 1522 device->input_ctx_area = fStack->AllocateArea((void **)&device->input_ctx, 1523 &device->input_ctx_addr, sizeof(*device->input_ctx) << fContextSizeShift, 1524 "XHCI input context"); 1525 if (device->input_ctx_area < B_OK) { 1526 TRACE_ERROR("unable to create a input context area\n"); 1527 CleanupDevice(device); 1528 return NULL; 1529 } 1530 if (fContextSizeShift == 1) { 1531 // 64-byte contexts have to be page-aligned in order for 1532 // _OffsetContextAddr to function properly. 1533 ASSERT((((addr_t)device->input_ctx) % B_PAGE_SIZE) == 0); 1534 } 1535 1536 memset(device->input_ctx, 0, sizeof(*device->input_ctx) << fContextSizeShift); 1537 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1538 _WriteContext(&device->input_ctx->input.addFlags, 3); 1539 1540 uint8 rhPort = hubPort; 1541 uint32 route = 0; 1542 for (Device *hubDevice = parent; hubDevice != RootObject(); 1543 hubDevice = (Device *)hubDevice->Parent()) { 1544 if (hubDevice->Parent() == RootObject()) 1545 break; 1546 1547 if (rhPort > 15) 1548 rhPort = 15; 1549 route = route << 4; 1550 route |= rhPort; 1551 1552 rhPort = hubDevice->HubPort(); 1553 } 1554 1555 uint32 dwslot0 = SLOT_0_NUM_ENTRIES(1) | SLOT_0_ROUTE(route); 1556 1557 // Get speed of port, only if device connected to root hub port 1558 // else we have to rely on value reported by the Hub Explore thread 1559 if (route == 0) { 1560 GetPortSpeed(hubPort - 1, &speed); 1561 TRACE("speed updated %d\n", speed); 1562 } 1563 1564 // add the speed 1565 switch (speed) { 1566 case USB_SPEED_LOWSPEED: 1567 dwslot0 |= SLOT_0_SPEED(2); 1568 break; 1569 case USB_SPEED_FULLSPEED: 1570 dwslot0 |= SLOT_0_SPEED(1); 1571 break; 1572 case USB_SPEED_HIGHSPEED: 1573 dwslot0 |= SLOT_0_SPEED(3); 1574 break; 1575 case USB_SPEED_SUPERSPEED: 1576 dwslot0 |= SLOT_0_SPEED(4); 1577 break; 1578 default: 1579 TRACE_ERROR("unknown usb speed\n"); 1580 break; 1581 } 1582 1583 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1584 // TODO enable power save 1585 _WriteContext(&device->input_ctx->slot.dwslot1, SLOT_1_RH_PORT(rhPort)); 1586 uint32 dwslot2 = SLOT_2_IRQ_TARGET(0); 1587 1588 // If LS/FS device connected to non-root HS device 1589 if (route != 0 && parent->Speed() == USB_SPEED_HIGHSPEED 1590 && (speed == USB_SPEED_LOWSPEED || speed == USB_SPEED_FULLSPEED)) { 1591 struct xhci_device *parenthub = (struct xhci_device *) 1592 parent->ControllerCookie(); 1593 dwslot2 |= SLOT_2_PORT_NUM(hubPort); 1594 dwslot2 |= SLOT_2_TT_HUB_SLOT(parenthub->slot); 1595 } 1596 1597 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1598 1599 _WriteContext(&device->input_ctx->slot.dwslot3, SLOT_3_SLOT_STATE(0) 1600 | SLOT_3_DEVICE_ADDRESS(0)); 1601 1602 TRACE("slot 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 1603 "\n", _ReadContext(&device->input_ctx->slot.dwslot0), 1604 _ReadContext(&device->input_ctx->slot.dwslot1), 1605 _ReadContext(&device->input_ctx->slot.dwslot2), 1606 _ReadContext(&device->input_ctx->slot.dwslot3)); 1607 1608 device->device_ctx_area = fStack->AllocateArea((void **)&device->device_ctx, 1609 &device->device_ctx_addr, sizeof(*device->device_ctx) << fContextSizeShift, 1610 "XHCI device context"); 1611 if (device->device_ctx_area < B_OK) { 1612 TRACE_ERROR("unable to create a device context area\n"); 1613 CleanupDevice(device); 1614 return NULL; 1615 } 1616 memset(device->device_ctx, 0, sizeof(*device->device_ctx) << fContextSizeShift); 1617 1618 device->trb_area = fStack->AllocateArea((void **)&device->trbs, 1619 &device->trb_addr, sizeof(xhci_trb) * (XHCI_MAX_ENDPOINTS - 1) 1620 * XHCI_ENDPOINT_RING_SIZE, "XHCI endpoint trbs"); 1621 if (device->trb_area < B_OK) { 1622 TRACE_ERROR("unable to create a device trbs area\n"); 1623 CleanupDevice(device); 1624 return NULL; 1625 } 1626 1627 // set up slot pointer to device context 1628 fDcba->baseAddress[slot] = device->device_ctx_addr; 1629 1630 size_t maxPacketSize; 1631 switch (speed) { 1632 case USB_SPEED_LOWSPEED: 1633 case USB_SPEED_FULLSPEED: 1634 maxPacketSize = 8; 1635 break; 1636 case USB_SPEED_HIGHSPEED: 1637 maxPacketSize = 64; 1638 break; 1639 default: 1640 maxPacketSize = 512; 1641 break; 1642 } 1643 1644 xhci_endpoint* endpoint0 = &device->endpoints[0]; 1645 mutex_init(&endpoint0->lock, "xhci endpoint lock"); 1646 endpoint0->device = device; 1647 endpoint0->id = 0; 1648 endpoint0->td_head = NULL; 1649 endpoint0->used = 0; 1650 endpoint0->current = 0; 1651 endpoint0->trbs = device->trbs; 1652 endpoint0->trb_addr = device->trb_addr; 1653 1654 // configure the Control endpoint 0 1655 if (ConfigureEndpoint(endpoint0, slot, 0, USB_OBJECT_CONTROL_PIPE, false, 1656 0, maxPacketSize, speed, 0, 0) != B_OK) { 1657 TRACE_ERROR("unable to configure default control endpoint\n"); 1658 CleanupDevice(device); 1659 return NULL; 1660 } 1661 1662 // device should get to addressed state (bsr = 0) 1663 status = SetAddress(device->input_ctx_addr, false, slot); 1664 if (status != B_OK) { 1665 TRACE_ERROR("unable to set address: %s\n", strerror(status)); 1666 CleanupDevice(device); 1667 return NULL; 1668 } 1669 1670 device->address = SLOT_3_DEVICE_ADDRESS_GET(_ReadContext( 1671 &device->device_ctx->slot.dwslot3)); 1672 1673 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", device->address, 1674 SLOT_3_SLOT_STATE_GET(_ReadContext( 1675 &device->device_ctx->slot.dwslot3))); 1676 TRACE("endpoint0 state 0x%08" B_PRIx32 "\n", 1677 ENDPOINT_0_STATE_GET(_ReadContext( 1678 &device->device_ctx->endpoints[0].dwendpoint0))); 1679 1680 // Wait a bit for the device to complete addressing 1681 snooze(USB_DELAY_SET_ADDRESS); 1682 1683 // Create a temporary pipe with the new address 1684 ControlPipe pipe(parent); 1685 pipe.SetControllerCookie(endpoint0); 1686 pipe.InitCommon(device->address + 1, 0, speed, Pipe::Default, maxPacketSize, 0, 1687 hubAddress, hubPort); 1688 1689 // Get the device descriptor 1690 // Just retrieve the first 8 bytes of the descriptor -> minimum supported 1691 // size of any device. It is enough because it includes the device type. 1692 1693 size_t actualLength = 0; 1694 usb_device_descriptor deviceDescriptor; 1695 1696 TRACE("getting the device descriptor\n"); 1697 status = pipe.SendRequest( 1698 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_STANDARD, // type 1699 USB_REQUEST_GET_DESCRIPTOR, // request 1700 USB_DESCRIPTOR_DEVICE << 8, // value 1701 0, // index 1702 8, // length 1703 (void *)&deviceDescriptor, // buffer 1704 8, // buffer length 1705 &actualLength); // actual length 1706 1707 if (actualLength != 8) { 1708 TRACE_ERROR("failed to get the device descriptor: %s\n", 1709 strerror(status)); 1710 CleanupDevice(device); 1711 return NULL; 1712 } 1713 1714 TRACE("device_class: %d device_subclass %d device_protocol %d\n", 1715 deviceDescriptor.device_class, deviceDescriptor.device_subclass, 1716 deviceDescriptor.device_protocol); 1717 1718 if (speed == USB_SPEED_FULLSPEED && deviceDescriptor.max_packet_size_0 != 8) { 1719 TRACE("Full speed device with different max packet size for Endpoint 0\n"); 1720 uint32 dwendpoint1 = _ReadContext( 1721 &device->input_ctx->endpoints[0].dwendpoint1); 1722 dwendpoint1 &= ~ENDPOINT_1_MAXPACKETSIZE(0xffff); 1723 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE( 1724 deviceDescriptor.max_packet_size_0); 1725 _WriteContext(&device->input_ctx->endpoints[0].dwendpoint1, 1726 dwendpoint1); 1727 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1728 _WriteContext(&device->input_ctx->input.addFlags, (1 << 1)); 1729 EvaluateContext(device->input_ctx_addr, device->slot); 1730 } 1731 1732 Device *deviceObject = NULL; 1733 if (deviceDescriptor.device_class == 0x09) { 1734 TRACE("creating new Hub\n"); 1735 TRACE("getting the hub descriptor\n"); 1736 size_t actualLength = 0; 1737 usb_hub_descriptor hubDescriptor; 1738 status = pipe.SendRequest( 1739 USB_REQTYPE_DEVICE_IN | USB_REQTYPE_CLASS, // type 1740 USB_REQUEST_GET_DESCRIPTOR, // request 1741 USB_DESCRIPTOR_HUB << 8, // value 1742 0, // index 1743 sizeof(usb_hub_descriptor), // length 1744 (void *)&hubDescriptor, // buffer 1745 sizeof(usb_hub_descriptor), // buffer length 1746 &actualLength); 1747 1748 if (actualLength != sizeof(usb_hub_descriptor)) { 1749 TRACE_ERROR("error while getting the hub descriptor: %s\n", 1750 strerror(status)); 1751 CleanupDevice(device); 1752 return NULL; 1753 } 1754 1755 uint32 dwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1756 dwslot0 |= SLOT_0_HUB_BIT; 1757 _WriteContext(&device->input_ctx->slot.dwslot0, dwslot0); 1758 uint32 dwslot1 = _ReadContext(&device->input_ctx->slot.dwslot1); 1759 dwslot1 |= SLOT_1_NUM_PORTS(hubDescriptor.num_ports); 1760 _WriteContext(&device->input_ctx->slot.dwslot1, dwslot1); 1761 if (speed == USB_SPEED_HIGHSPEED) { 1762 uint32 dwslot2 = _ReadContext(&device->input_ctx->slot.dwslot2); 1763 dwslot2 |= SLOT_2_TT_TIME(HUB_TTT_GET(hubDescriptor.characteristics)); 1764 _WriteContext(&device->input_ctx->slot.dwslot2, dwslot2); 1765 } 1766 1767 deviceObject = new(std::nothrow) Hub(parent, hubAddress, hubPort, 1768 deviceDescriptor, device->address + 1, speed, false, device); 1769 } else { 1770 TRACE("creating new device\n"); 1771 deviceObject = new(std::nothrow) Device(parent, hubAddress, hubPort, 1772 deviceDescriptor, device->address + 1, speed, false, device); 1773 } 1774 if (deviceObject == NULL || deviceObject->InitCheck() != B_OK) { 1775 if (deviceObject == NULL) { 1776 TRACE_ERROR("no memory to allocate device\n"); 1777 } else { 1778 TRACE_ERROR("device object failed to initialize\n"); 1779 } 1780 CleanupDevice(device); 1781 return NULL; 1782 } 1783 1784 // We don't want to disable the default endpoint, naturally, which would 1785 // otherwise happen when this Pipe object is destroyed. 1786 pipe.SetControllerCookie(NULL); 1787 1788 deviceObject->RegisterNode(); 1789 1790 TRACE("AllocateDevice() port %d slot %d\n", hubPort, slot); 1791 return deviceObject; 1792 } 1793 1794 1795 void 1796 XHCI::FreeDevice(Device *usbDevice) 1797 { 1798 xhci_device* device = (xhci_device*)usbDevice->ControllerCookie(); 1799 TRACE("FreeDevice() slot %d\n", device->slot); 1800 1801 // Delete the device first, so it cleans up its pipes and tells us 1802 // what we need to destroy before we tear down our internal state. 1803 delete usbDevice; 1804 1805 CleanupDevice(device); 1806 } 1807 1808 1809 void 1810 XHCI::CleanupDevice(xhci_device *device) 1811 { 1812 if (device->slot != 0) { 1813 DisableSlot(device->slot); 1814 fDcba->baseAddress[device->slot] = 0; 1815 } 1816 1817 if (device->trb_addr != 0) 1818 delete_area(device->trb_area); 1819 if (device->input_ctx_addr != 0) 1820 delete_area(device->input_ctx_area); 1821 if (device->device_ctx_addr != 0) 1822 delete_area(device->device_ctx_area); 1823 1824 memset(device, 0, sizeof(xhci_device)); 1825 } 1826 1827 1828 uint8 1829 XHCI::_GetEndpointState(xhci_endpoint* endpoint) 1830 { 1831 struct xhci_device_ctx* device_ctx = endpoint->device->device_ctx; 1832 return ENDPOINT_0_STATE_GET( 1833 _ReadContext(&device_ctx->endpoints[endpoint->id].dwendpoint0)); 1834 } 1835 1836 1837 status_t 1838 XHCI::_InsertEndpointForPipe(Pipe *pipe) 1839 { 1840 TRACE("insert endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1841 1842 if (pipe->ControllerCookie() != NULL 1843 || pipe->Parent()->Type() != USB_OBJECT_DEVICE) { 1844 // default pipe is already referenced 1845 return B_OK; 1846 } 1847 1848 Device* usbDevice = (Device *)pipe->Parent(); 1849 if (usbDevice->Parent() == RootObject()) { 1850 // root hub needs no initialization 1851 return B_OK; 1852 } 1853 1854 struct xhci_device *device = (struct xhci_device *) 1855 usbDevice->ControllerCookie(); 1856 if (device == NULL) { 1857 panic("device is NULL\n"); 1858 return B_NO_INIT; 1859 } 1860 1861 const uint8 id = (2 * pipe->EndpointAddress() 1862 + (pipe->Direction() != Pipe::Out ? 1 : 0)) - 1; 1863 if (id >= XHCI_MAX_ENDPOINTS - 1) 1864 return B_BAD_VALUE; 1865 1866 if (id > 0) { 1867 uint32 devicedwslot0 = _ReadContext(&device->device_ctx->slot.dwslot0); 1868 if (SLOT_0_NUM_ENTRIES_GET(devicedwslot0) == 1) { 1869 uint32 inputdwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0); 1870 inputdwslot0 &= ~(SLOT_0_NUM_ENTRIES(0x1f)); 1871 inputdwslot0 |= SLOT_0_NUM_ENTRIES(XHCI_MAX_ENDPOINTS - 1); 1872 _WriteContext(&device->input_ctx->slot.dwslot0, inputdwslot0); 1873 EvaluateContext(device->input_ctx_addr, device->slot); 1874 } 1875 1876 xhci_endpoint* endpoint = &device->endpoints[id]; 1877 mutex_init(&endpoint->lock, "xhci endpoint lock"); 1878 MutexLocker endpointLocker(endpoint->lock); 1879 1880 endpoint->device = device; 1881 endpoint->id = id; 1882 endpoint->td_head = NULL; 1883 endpoint->used = 0; 1884 endpoint->current = 0; 1885 1886 endpoint->trbs = device->trbs + id * XHCI_ENDPOINT_RING_SIZE; 1887 endpoint->trb_addr = device->trb_addr 1888 + id * XHCI_ENDPOINT_RING_SIZE * sizeof(xhci_trb); 1889 memset(endpoint->trbs, 0, 1890 sizeof(xhci_trb) * XHCI_ENDPOINT_RING_SIZE); 1891 1892 TRACE("insert endpoint for pipe: trbs, device %p endpoint %p\n", 1893 device->trbs, endpoint->trbs); 1894 TRACE("insert endpoint for pipe: trb_addr, device 0x%" B_PRIxPHYSADDR 1895 " endpoint 0x%" B_PRIxPHYSADDR "\n", device->trb_addr, 1896 endpoint->trb_addr); 1897 1898 const uint8 endpointNum = id + 1; 1899 1900 status_t status = ConfigureEndpoint(endpoint, device->slot, id, pipe->Type(), 1901 pipe->Direction() == Pipe::In, pipe->Interval(), pipe->MaxPacketSize(), 1902 usbDevice->Speed(), pipe->MaxBurst(), pipe->BytesPerInterval()); 1903 if (status != B_OK) { 1904 TRACE_ERROR("unable to configure endpoint: %s\n", strerror(status)); 1905 return status; 1906 } 1907 1908 _WriteContext(&device->input_ctx->input.dropFlags, 0); 1909 _WriteContext(&device->input_ctx->input.addFlags, 1910 (1 << endpointNum) | (1 << 0)); 1911 1912 ConfigureEndpoint(device->input_ctx_addr, false, device->slot); 1913 1914 TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", 1915 device->address, SLOT_3_SLOT_STATE_GET(_ReadContext( 1916 &device->device_ctx->slot.dwslot3))); 1917 TRACE("endpoint[0] state 0x%08" B_PRIx32 "\n", 1918 ENDPOINT_0_STATE_GET(_ReadContext( 1919 &device->device_ctx->endpoints[0].dwendpoint0))); 1920 TRACE("endpoint[%d] state 0x%08" B_PRIx32 "\n", id, 1921 ENDPOINT_0_STATE_GET(_ReadContext( 1922 &device->device_ctx->endpoints[id].dwendpoint0))); 1923 } 1924 pipe->SetControllerCookie(&device->endpoints[id]); 1925 1926 return B_OK; 1927 } 1928 1929 1930 status_t 1931 XHCI::_RemoveEndpointForPipe(Pipe *pipe) 1932 { 1933 TRACE("remove endpoint for pipe %p (%d)\n", pipe, pipe->EndpointAddress()); 1934 1935 if (pipe->Parent()->Type() != USB_OBJECT_DEVICE) 1936 return B_OK; 1937 Device* usbDevice = (Device *)pipe->Parent(); 1938 if (usbDevice->Parent() == RootObject()) 1939 return B_BAD_VALUE; 1940 1941 xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie(); 1942 if (endpoint == NULL || endpoint->trbs == NULL) 1943 return B_NO_INIT; 1944 1945 pipe->SetControllerCookie(NULL); 1946 1947 if (endpoint->id > 0) { 1948 xhci_device *device = endpoint->device; 1949 uint8 epNumber = endpoint->id + 1; 1950 StopEndpoint(true, endpoint); 1951 1952 mutex_lock(&endpoint->lock); 1953 1954 // See comment in CancelQueuedTransfers. 1955 xhci_td* td; 1956 while ((td = endpoint->td_head) != NULL) { 1957 endpoint->td_head = endpoint->td_head->next; 1958 FreeDescriptor(td); 1959 } 1960 1961 mutex_destroy(&endpoint->lock); 1962 memset(endpoint, 0, sizeof(xhci_endpoint)); 1963 1964 _WriteContext(&device->input_ctx->input.dropFlags, (1 << epNumber)); 1965 _WriteContext(&device->input_ctx->input.addFlags, (1 << 0)); 1966 1967 // The Deconfigure bit in the Configure Endpoint command indicates 1968 // that *all* endpoints are to be deconfigured, and not just the ones 1969 // specified in the context flags. (XHCI 1.2 § 4.6.6 p115.) 1970 ConfigureEndpoint(device->input_ctx_addr, false, device->slot); 1971 } 1972 1973 return B_OK; 1974 } 1975 1976 1977 status_t 1978 XHCI::_LinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 1979 { 1980 TRACE("link descriptor for pipe\n"); 1981 1982 // Use mutex_trylock first, in case we are in KDL. 1983 MutexLocker endpointLocker(&endpoint->lock, mutex_trylock(&endpoint->lock) == B_OK); 1984 1985 // "used" refers to the number of currently linked TDs, not the number of 1986 // used TRBs on the ring (we use 2 TRBs on the ring per transfer.) 1987 if (endpoint->used >= (XHCI_MAX_TRANSFERS - 1)) { 1988 TRACE_ERROR("link descriptor for pipe: max transfers count exceeded\n"); 1989 return B_BAD_VALUE; 1990 } 1991 1992 // We do not support queuing other transfers in tandem with a fragmented one. 1993 if (endpoint->td_head != NULL && endpoint->td_head->transfer != NULL 1994 && endpoint->td_head->transfer->IsFragmented()) { 1995 TRACE_ERROR("cannot submit transfer: a fragmented transfer is queued\n"); 1996 return B_DEV_RESOURCE_CONFLICT; 1997 } 1998 1999 endpoint->used++; 2000 descriptor->next = endpoint->td_head; 2001 endpoint->td_head = descriptor; 2002 2003 const uint32 current = endpoint->current, 2004 eventdata = current + 1, 2005 last = XHCI_ENDPOINT_RING_SIZE - 1; 2006 uint32 next = eventdata + 1; 2007 2008 TRACE("link descriptor for pipe: current %d, next %d\n", current, next); 2009 2010 // Add a Link TRB to the end of the descriptor. 2011 phys_addr_t addr = endpoint->trb_addr + eventdata * sizeof(xhci_trb); 2012 descriptor->trbs[descriptor->trb_used].address = addr; 2013 descriptor->trbs[descriptor->trb_used].status = TRB_2_IRQ(0); 2014 descriptor->trbs[descriptor->trb_used].flags = TRB_3_TYPE(TRB_TYPE_LINK) 2015 | TRB_3_CHAIN_BIT | TRB_3_CYCLE_BIT; 2016 // It is specified that (XHCI 1.2 § 4.12.3 Note 2 p251) if the TRB 2017 // following one with the ENT bit set is a Link TRB, the Link TRB 2018 // shall be evaluated *and* the subsequent TRB shall be. Thus a 2019 // TRB_3_ENT_BIT is unnecessary here; and from testing seems to 2020 // break all transfers on a (very) small number of controllers. 2021 2022 #if !B_HOST_IS_LENDIAN 2023 // Convert endianness. 2024 for (uint32 i = 0; i <= descriptor->trb_used; i++) { 2025 descriptor->trbs[i].address = 2026 B_HOST_TO_LENDIAN_INT64(descriptor->trbs[i].address); 2027 descriptor->trbs[i].status = 2028 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].status); 2029 descriptor->trbs[i].flags = 2030 B_HOST_TO_LENDIAN_INT32(descriptor->trbs[i].flags); 2031 } 2032 #endif 2033 2034 // Link the descriptor. 2035 endpoint->trbs[current].address = 2036 B_HOST_TO_LENDIAN_INT64(descriptor->trb_addr); 2037 endpoint->trbs[current].status = 2038 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 2039 endpoint->trbs[current].flags = 2040 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_LINK)); 2041 2042 // Set up the Event Data TRB (XHCI 1.2 § 4.11.5.2 p230.) 2043 // 2044 // We do this on the main ring for two reasons: first, to avoid a small 2045 // potential race between the interrupt and the controller evaluating 2046 // the link TRB to get back onto the ring; and second, because many 2047 // controllers throw errors if the target of a Link TRB is not valid 2048 // (i.e. does not have its Cycle Bit set.) 2049 // 2050 // We also set the "address" field, which the controller will copy 2051 // verbatim into the TRB it posts to the event ring, to be the last 2052 // "real" TRB in the TD; this will allow us to determine what transfer 2053 // the resulting Transfer Event TRB refers to. 2054 endpoint->trbs[eventdata].address = 2055 B_HOST_TO_LENDIAN_INT64(descriptor->trb_addr 2056 + (descriptor->trb_used - 1) * sizeof(xhci_trb)); 2057 endpoint->trbs[eventdata].status = 2058 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 2059 endpoint->trbs[eventdata].flags = 2060 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_EVENT_DATA) 2061 | TRB_3_IOC_BIT | TRB_3_CYCLE_BIT); 2062 2063 if (next == last) { 2064 // We always use 2 TRBs per _Link..() call, so if "next" is the last 2065 // TRB in the ring, we need to generate a link TRB at "next", and 2066 // then wrap it to 0. (We write the cycle bit later, after wrapping, 2067 // for the reason noted in the previous comment.) 2068 endpoint->trbs[next].address = 2069 B_HOST_TO_LENDIAN_INT64(endpoint->trb_addr); 2070 endpoint->trbs[next].status = 2071 B_HOST_TO_LENDIAN_INT32(TRB_2_IRQ(0)); 2072 endpoint->trbs[next].flags = 2073 B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_LINK)); 2074 2075 next = 0; 2076 } 2077 2078 endpoint->trbs[next].address = 0; 2079 endpoint->trbs[next].status = 0; 2080 endpoint->trbs[next].flags = 0; 2081 2082 memory_write_barrier(); 2083 2084 // Everything is ready, so write the cycle bit(s). 2085 endpoint->trbs[current].flags |= B_HOST_TO_LENDIAN_INT32(TRB_3_CYCLE_BIT); 2086 if (current == 0 && endpoint->trbs[last].address != 0) 2087 endpoint->trbs[last].flags |= B_HOST_TO_LENDIAN_INT32(TRB_3_CYCLE_BIT); 2088 2089 TRACE("_LinkDescriptorForPipe pCurrent %p phys 0x%" B_PRIxPHYSADDR 2090 " 0x%" B_PRIxPHYSADDR " 0x%08" B_PRIx32 "\n", &endpoint->trbs[current], 2091 endpoint->trb_addr + current * sizeof(struct xhci_trb), 2092 endpoint->trbs[current].address, 2093 B_LENDIAN_TO_HOST_INT32(endpoint->trbs[current].flags)); 2094 2095 endpoint->current = next; 2096 endpointLocker.Unlock(); 2097 2098 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 2099 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 2100 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 2101 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 2102 2103 Ring(endpoint->device->slot, endpoint->id + 1); 2104 2105 TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n", 2106 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint0), 2107 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].dwendpoint1), 2108 _ReadContext(&endpoint->device->device_ctx->endpoints[endpoint->id].qwendpoint2)); 2109 2110 return B_OK; 2111 } 2112 2113 2114 status_t 2115 XHCI::_UnlinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint) 2116 { 2117 TRACE("unlink descriptor for pipe\n"); 2118 // We presume that the caller has already locked or owns the endpoint. 2119 2120 endpoint->used--; 2121 if (descriptor == endpoint->td_head) { 2122 endpoint->td_head = descriptor->next; 2123 descriptor->next = NULL; 2124 return B_OK; 2125 } else { 2126 for (xhci_td *td = endpoint->td_head; td->next != NULL; td = td->next) { 2127 if (td->next == descriptor) { 2128 td->next = descriptor->next; 2129 descriptor->next = NULL; 2130 return B_OK; 2131 } 2132 } 2133 } 2134 2135 endpoint->used++; 2136 return B_ERROR; 2137 } 2138 2139 2140 status_t 2141 XHCI::ConfigureEndpoint(xhci_endpoint* ep, uint8 slot, uint8 number, uint8 type, 2142 bool directionIn, uint16 interval, uint16 maxPacketSize, usb_speed speed, 2143 uint8 maxBurst, uint16 bytesPerInterval) 2144 { 2145 struct xhci_device* device = &fDevices[slot]; 2146 2147 uint32 dwendpoint0 = 0; 2148 uint32 dwendpoint1 = 0; 2149 uint64 qwendpoint2 = 0; 2150 uint32 dwendpoint4 = 0; 2151 2152 // Compute and assign the endpoint type. (XHCI 1.2 § 6.2.3 Table 6-9 p452.) 2153 uint8 xhciType = 4; 2154 if ((type & USB_OBJECT_INTERRUPT_PIPE) != 0) 2155 xhciType = 3; 2156 if ((type & USB_OBJECT_BULK_PIPE) != 0) 2157 xhciType = 2; 2158 if ((type & USB_OBJECT_ISO_PIPE) != 0) 2159 xhciType = 1; 2160 xhciType |= directionIn ? (1 << 2) : 0; 2161 dwendpoint1 |= ENDPOINT_1_EPTYPE(xhciType); 2162 2163 // Compute and assign interval. (XHCI 1.2 § 6.2.3.6 p456.) 2164 uint16 calcInterval; 2165 if ((type & USB_OBJECT_BULK_PIPE) != 0 2166 || (type & USB_OBJECT_CONTROL_PIPE) != 0) { 2167 // Bulk and Control endpoints never issue NAKs. 2168 calcInterval = 0; 2169 } else { 2170 switch (speed) { 2171 case USB_SPEED_FULLSPEED: 2172 if ((type & USB_OBJECT_ISO_PIPE) != 0) { 2173 // Convert 1-16 into 3-18. 2174 calcInterval = min_c(max_c(interval, 1), 16) + 2; 2175 break; 2176 } 2177 2178 // fall through 2179 case USB_SPEED_LOWSPEED: { 2180 // Convert 1ms-255ms into 3-10. 2181 2182 // Find the index of the highest set bit in "interval". 2183 uint32 temp = min_c(max_c(interval, 1), 255); 2184 for (calcInterval = 0; temp != 1; calcInterval++) 2185 temp = temp >> 1; 2186 calcInterval += 3; 2187 break; 2188 } 2189 2190 case USB_SPEED_HIGHSPEED: 2191 case USB_SPEED_SUPERSPEED: 2192 default: 2193 // Convert 1-16 into 0-15. 2194 calcInterval = min_c(max_c(interval, 1), 16) - 1; 2195 break; 2196 } 2197 } 2198 dwendpoint0 |= ENDPOINT_0_INTERVAL(calcInterval); 2199 2200 // For non-isochronous endpoints, we want the controller to retry failed 2201 // transfers, if possible. (XHCI 1.2 § 4.10.2.3 p197.) 2202 if ((type & USB_OBJECT_ISO_PIPE) == 0) 2203 dwendpoint1 |= ENDPOINT_1_CERR(3); 2204 2205 // Assign maximum burst size. For USB3 devices this is passed in; for 2206 // all other devices we compute it. (XHCI 1.2 § 4.8.2 p161.) 2207 if (speed == USB_SPEED_HIGHSPEED && (type & (USB_OBJECT_INTERRUPT_PIPE 2208 | USB_OBJECT_ISO_PIPE)) != 0) { 2209 maxBurst = (maxPacketSize & 0x1800) >> 11; 2210 } else if (speed != USB_SPEED_SUPERSPEED) { 2211 maxBurst = 0; 2212 } 2213 dwendpoint1 |= ENDPOINT_1_MAXBURST(maxBurst); 2214 2215 // Assign maximum packet size, set the ring address, and set the 2216 // "Dequeue Cycle State" bit. (XHCI 1.2 § 6.2.3 Table 6-10 p453.) 2217 dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE(maxPacketSize); 2218 qwendpoint2 |= ENDPOINT_2_DCS_BIT | ep->trb_addr; 2219 2220 // The Max Burst Payload is the number of bytes moved by a 2221 // maximum sized burst. (XHCI 1.2 § 4.11.7.1 p236.) 2222 ep->max_burst_payload = (maxBurst + 1) * maxPacketSize; 2223 if (ep->max_burst_payload == 0) { 2224 TRACE_ERROR("ConfigureEndpoint() failed invalid max_burst_payload\n"); 2225 return B_BAD_VALUE; 2226 } 2227 2228 // Assign average TRB length. 2229 if ((type & USB_OBJECT_CONTROL_PIPE) != 0) { 2230 // Control pipes are a special case, as they rarely have 2231 // outbound transfers of any substantial size. 2232 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(8); 2233 } else if ((type & USB_OBJECT_ISO_PIPE) != 0) { 2234 // Isochronous pipes are another special case: the TRB size will be 2235 // one packet (which is normally smaller than the max packet size, 2236 // but we don't know what it is here.) 2237 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(maxPacketSize); 2238 } else { 2239 // Under all other circumstances, we put max_burst_payload in a TRB. 2240 dwendpoint4 |= ENDPOINT_4_AVGTRBLENGTH(ep->max_burst_payload); 2241 } 2242 2243 // Assign maximum ESIT payload. (XHCI 1.2 § 4.14.2 p259.) 2244 if ((type & (USB_OBJECT_INTERRUPT_PIPE | USB_OBJECT_ISO_PIPE)) != 0) { 2245 // TODO: For SuperSpeedPlus endpoints, there is yet another descriptor 2246 // for isochronous endpoints that specifies the maximum ESIT payload. 2247 // We don't fetch this yet, so just fall back to the USB2 computation 2248 // method if bytesPerInterval is 0. 2249 if (speed == USB_SPEED_SUPERSPEED && bytesPerInterval != 0) 2250 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD(bytesPerInterval); 2251 else if (speed >= USB_SPEED_HIGHSPEED) 2252 dwendpoint4 |= ENDPOINT_4_MAXESITPAYLOAD((maxBurst + 1) * maxPacketSize); 2253 } 2254 2255 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint0, 2256 dwendpoint0); 2257 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint1, 2258 dwendpoint1); 2259 _WriteContext(&device->input_ctx->endpoints[number].qwendpoint2, 2260 qwendpoint2); 2261 _WriteContext(&device->input_ctx->endpoints[number].dwendpoint4, 2262 dwendpoint4); 2263 2264 TRACE("endpoint 0x%" B_PRIx32 " 0x%" B_PRIx32 " 0x%" B_PRIx64 " 0x%" 2265 B_PRIx32 "\n", 2266 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint0), 2267 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint1), 2268 _ReadContext(&device->input_ctx->endpoints[number].qwendpoint2), 2269 _ReadContext(&device->input_ctx->endpoints[number].dwendpoint4)); 2270 2271 return B_OK; 2272 } 2273 2274 2275 status_t 2276 XHCI::GetPortSpeed(uint8 index, usb_speed* speed) 2277 { 2278 if (index >= fPortCount) 2279 return B_BAD_INDEX; 2280 2281 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 2282 2283 switch (PS_SPEED_GET(portStatus)) { 2284 case 2: 2285 *speed = USB_SPEED_LOWSPEED; 2286 break; 2287 case 1: 2288 *speed = USB_SPEED_FULLSPEED; 2289 break; 2290 case 3: 2291 *speed = USB_SPEED_HIGHSPEED; 2292 break; 2293 case 4: 2294 *speed = USB_SPEED_SUPERSPEED; 2295 break; 2296 default: 2297 TRACE_ALWAYS("nonstandard port speed %" B_PRId32 ", assuming SuperSpeed\n", 2298 PS_SPEED_GET(portStatus)); 2299 *speed = USB_SPEED_SUPERSPEED; 2300 break; 2301 } 2302 2303 return B_OK; 2304 } 2305 2306 2307 status_t 2308 XHCI::GetPortStatus(uint8 index, usb_port_status* status) 2309 { 2310 if (index >= fPortCount) 2311 return B_BAD_INDEX; 2312 2313 status->status = status->change = 0; 2314 uint32 portStatus = ReadOpReg(XHCI_PORTSC(index)); 2315 TRACE("port %" B_PRId8 " status=0x%08" B_PRIx32 "\n", index, portStatus); 2316 2317 // build the status 2318 switch (PS_SPEED_GET(portStatus)) { 2319 case 3: 2320 status->status |= PORT_STATUS_HIGH_SPEED; 2321 break; 2322 case 2: 2323 status->status |= PORT_STATUS_LOW_SPEED; 2324 break; 2325 default: 2326 break; 2327 } 2328 2329 if (portStatus & PS_CCS) 2330 status->status |= PORT_STATUS_CONNECTION; 2331 if (portStatus & PS_PED) 2332 status->status |= PORT_STATUS_ENABLE; 2333 if (portStatus & PS_OCA) 2334 status->status |= PORT_STATUS_OVER_CURRENT; 2335 if (portStatus & PS_PR) 2336 status->status |= PORT_STATUS_RESET; 2337 if (portStatus & PS_PP) { 2338 if (fPortSpeeds[index] == USB_SPEED_SUPERSPEED) 2339 status->status |= PORT_STATUS_SS_POWER; 2340 else 2341 status->status |= PORT_STATUS_POWER; 2342 } 2343 2344 // build the change 2345 if (portStatus & PS_CSC) 2346 status->change |= PORT_STATUS_CONNECTION; 2347 if (portStatus & PS_PEC) 2348 status->change |= PORT_STATUS_ENABLE; 2349 if (portStatus & PS_OCC) 2350 status->change |= PORT_STATUS_OVER_CURRENT; 2351 if (portStatus & PS_PRC) 2352 status->change |= PORT_STATUS_RESET; 2353 2354 if (fPortSpeeds[index] == USB_SPEED_SUPERSPEED) { 2355 if (portStatus & PS_PLC) 2356 status->change |= PORT_CHANGE_LINK_STATE; 2357 if (portStatus & PS_WRC) 2358 status->change |= PORT_CHANGE_BH_PORT_RESET; 2359 } 2360 2361 return B_OK; 2362 } 2363 2364 2365 status_t 2366 XHCI::SetPortFeature(uint8 index, uint16 feature) 2367 { 2368 TRACE("set port feature index %u feature %u\n", index, feature); 2369 if (index >= fPortCount) 2370 return B_BAD_INDEX; 2371 2372 uint32 portRegister = XHCI_PORTSC(index); 2373 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2374 2375 switch (feature) { 2376 case PORT_SUSPEND: 2377 if ((portStatus & PS_PED) == 0 || (portStatus & PS_PR) 2378 || (portStatus & PS_PLS_MASK) >= PS_XDEV_U3) { 2379 TRACE_ERROR("USB core suspending device not in U0/U1/U2.\n"); 2380 return B_BAD_VALUE; 2381 } 2382 portStatus &= ~PS_PLS_MASK; 2383 WriteOpReg(portRegister, portStatus | PS_LWS | PS_XDEV_U3); 2384 break; 2385 2386 case PORT_RESET: 2387 WriteOpReg(portRegister, portStatus | PS_PR); 2388 break; 2389 2390 case PORT_POWER: 2391 WriteOpReg(portRegister, portStatus | PS_PP); 2392 break; 2393 default: 2394 return B_BAD_VALUE; 2395 } 2396 ReadOpReg(portRegister); 2397 return B_OK; 2398 } 2399 2400 2401 status_t 2402 XHCI::ClearPortFeature(uint8 index, uint16 feature) 2403 { 2404 TRACE("clear port feature index %u feature %u\n", index, feature); 2405 if (index >= fPortCount) 2406 return B_BAD_INDEX; 2407 2408 uint32 portRegister = XHCI_PORTSC(index); 2409 uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR; 2410 2411 switch (feature) { 2412 case PORT_SUSPEND: 2413 portStatus = ReadOpReg(portRegister); 2414 if (portStatus & PS_PR) 2415 return B_BAD_VALUE; 2416 if (portStatus & PS_XDEV_U3) { 2417 if ((portStatus & PS_PED) == 0) 2418 return B_BAD_VALUE; 2419 portStatus &= ~PS_PLS_MASK; 2420 WriteOpReg(portRegister, portStatus | PS_XDEV_U0 | PS_LWS); 2421 } 2422 break; 2423 case PORT_ENABLE: 2424 WriteOpReg(portRegister, portStatus | PS_PED); 2425 break; 2426 case PORT_POWER: 2427 WriteOpReg(portRegister, portStatus & ~PS_PP); 2428 break; 2429 case C_PORT_CONNECTION: 2430 WriteOpReg(portRegister, portStatus | PS_CSC); 2431 break; 2432 case C_PORT_ENABLE: 2433 WriteOpReg(portRegister, portStatus | PS_PEC); 2434 break; 2435 case C_PORT_OVER_CURRENT: 2436 WriteOpReg(portRegister, portStatus | PS_OCC); 2437 break; 2438 case C_PORT_RESET: 2439 WriteOpReg(portRegister, portStatus | PS_PRC); 2440 break; 2441 case C_PORT_BH_PORT_RESET: 2442 WriteOpReg(portRegister, portStatus | PS_WRC); 2443 break; 2444 case C_PORT_LINK_STATE: 2445 WriteOpReg(portRegister, portStatus | PS_PLC); 2446 break; 2447 default: 2448 return B_BAD_VALUE; 2449 } 2450 2451 ReadOpReg(portRegister); 2452 return B_OK; 2453 } 2454 2455 2456 status_t 2457 XHCI::ControllerHalt() 2458 { 2459 // Mask off run state 2460 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) & ~CMD_RUN); 2461 2462 // wait for shutdown state 2463 if (WaitOpBits(XHCI_STS, STS_HCH, STS_HCH) != B_OK) { 2464 TRACE_ERROR("HCH shutdown timeout\n"); 2465 return B_ERROR; 2466 } 2467 return B_OK; 2468 } 2469 2470 2471 status_t 2472 XHCI::ControllerReset() 2473 { 2474 TRACE("ControllerReset() cmd: 0x%" B_PRIx32 " sts: 0x%" B_PRIx32 "\n", 2475 ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS)); 2476 WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) | CMD_HCRST); 2477 2478 if (WaitOpBits(XHCI_CMD, CMD_HCRST, 0) != B_OK) { 2479 TRACE_ERROR("ControllerReset() failed CMD_HCRST\n"); 2480 return B_ERROR; 2481 } 2482 2483 if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) { 2484 TRACE_ERROR("ControllerReset() failed STS_CNR\n"); 2485 return B_ERROR; 2486 } 2487 2488 return B_OK; 2489 } 2490 2491 2492 int32 2493 XHCI::InterruptHandler(void* data) 2494 { 2495 return ((XHCI*)data)->Interrupt(); 2496 } 2497 2498 2499 int32 2500 XHCI::Interrupt() 2501 { 2502 SpinLocker _(&fSpinlock); 2503 2504 uint32 status = ReadOpReg(XHCI_STS); 2505 uint32 temp = ReadRunReg32(XHCI_IMAN(0)); 2506 WriteOpReg(XHCI_STS, status); 2507 WriteRunReg32(XHCI_IMAN(0), temp); 2508 2509 int32 result = B_HANDLED_INTERRUPT; 2510 2511 if ((status & STS_HCH) != 0) { 2512 TRACE_ERROR("Host Controller halted\n"); 2513 return result; 2514 } 2515 if ((status & STS_HSE) != 0) { 2516 TRACE_ERROR("Host System Error\n"); 2517 return result; 2518 } 2519 if ((status & STS_HCE) != 0) { 2520 TRACE_ERROR("Host Controller Error\n"); 2521 return result; 2522 } 2523 2524 if ((status & STS_EINT) == 0) { 2525 TRACE("STS: 0x%" B_PRIx32 " IRQ_PENDING: 0x%" B_PRIx32 "\n", 2526 status, temp); 2527 return B_UNHANDLED_INTERRUPT; 2528 } 2529 2530 TRACE("Event Interrupt\n"); 2531 release_sem_etc(fEventSem, 1, B_DO_NOT_RESCHEDULE); 2532 return B_INVOKE_SCHEDULER; 2533 } 2534 2535 2536 void 2537 XHCI::Ring(uint8 slot, uint8 endpoint) 2538 { 2539 TRACE("Ding Dong! slot:%d endpoint %d\n", slot, endpoint) 2540 if ((slot == 0 && endpoint > 0) || (slot > 0 && endpoint == 0)) 2541 panic("Ring() invalid slot/endpoint combination\n"); 2542 if (slot > fSlotCount || endpoint >= XHCI_MAX_ENDPOINTS) 2543 panic("Ring() invalid slot or endpoint\n"); 2544 2545 WriteDoorReg32(XHCI_DOORBELL(slot), XHCI_DOORBELL_TARGET(endpoint) 2546 | XHCI_DOORBELL_STREAMID(0)); 2547 ReadDoorReg32(XHCI_DOORBELL(slot)); 2548 // Flush PCI writes 2549 } 2550 2551 2552 void 2553 XHCI::QueueCommand(xhci_trb* trb) 2554 { 2555 uint8 i, j; 2556 uint32 temp; 2557 2558 i = fCmdIdx; 2559 j = fCmdCcs; 2560 2561 TRACE("command[%u] = %" B_PRId32 " (0x%016" B_PRIx64 ", 0x%08" B_PRIx32 2562 ", 0x%08" B_PRIx32 ")\n", i, TRB_3_TYPE_GET(trb->flags), trb->address, 2563 trb->status, trb->flags); 2564 2565 fCmdRing[i].address = trb->address; 2566 fCmdRing[i].status = trb->status; 2567 temp = trb->flags; 2568 2569 if (j) 2570 temp |= TRB_3_CYCLE_BIT; 2571 else 2572 temp &= ~TRB_3_CYCLE_BIT; 2573 temp &= ~TRB_3_TC_BIT; 2574 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2575 2576 fCmdAddr = fErst->rs_addr + (XHCI_MAX_EVENTS + i) * sizeof(xhci_trb); 2577 2578 i++; 2579 2580 if (i == (XHCI_MAX_COMMANDS - 1)) { 2581 temp = TRB_3_TYPE(TRB_TYPE_LINK) | TRB_3_TC_BIT; 2582 if (j) 2583 temp |= TRB_3_CYCLE_BIT; 2584 fCmdRing[i].flags = B_HOST_TO_LENDIAN_INT32(temp); 2585 2586 i = 0; 2587 j ^= 1; 2588 } 2589 2590 fCmdIdx = i; 2591 fCmdCcs = j; 2592 } 2593 2594 2595 void 2596 XHCI::HandleCmdComplete(xhci_trb* trb) 2597 { 2598 if (fCmdAddr == trb->address) { 2599 TRACE("Received command event\n"); 2600 fCmdResult[0] = trb->status; 2601 fCmdResult[1] = B_LENDIAN_TO_HOST_INT32(trb->flags); 2602 release_sem_etc(fCmdCompSem, 1, B_DO_NOT_RESCHEDULE); 2603 } else 2604 TRACE_ERROR("received command event for unknown command!\n") 2605 } 2606 2607 2608 void 2609 XHCI::HandleTransferComplete(xhci_trb* trb) 2610 { 2611 const uint32 flags = B_LENDIAN_TO_HOST_INT32(trb->flags); 2612 const uint8 endpointNumber = TRB_3_ENDPOINT_GET(flags), 2613 slot = TRB_3_SLOT_GET(flags); 2614 2615 if (slot > fSlotCount) 2616 TRACE_ERROR("invalid slot\n"); 2617 if (endpointNumber == 0 || endpointNumber >= XHCI_MAX_ENDPOINTS) { 2618 TRACE_ERROR("invalid endpoint\n"); 2619 return; 2620 } 2621 2622 xhci_device *device = &fDevices[slot]; 2623 xhci_endpoint *endpoint = &device->endpoints[endpointNumber - 1]; 2624 2625 if (endpoint->trbs == NULL) { 2626 TRACE_ERROR("got TRB but endpoint is not allocated!\n"); 2627 return; 2628 } 2629 2630 // Use mutex_trylock first, in case we are in KDL. 2631 MutexLocker endpointLocker(endpoint->lock, mutex_trylock(&endpoint->lock) == B_OK); 2632 if (!endpointLocker.IsLocked()) { 2633 // We failed to get the lock. Most likely it was destroyed 2634 // while we were waiting for it. 2635 return; 2636 } 2637 2638 // In the case of an Event Data TRB, the "transferred" field refers 2639 // to the actual number of bytes transferred across the whole TD. 2640 // (XHCI 1.2 § 6.4.2.1 Table 6-38 p478.) 2641 const uint8 completionCode = TRB_2_COMP_CODE_GET(trb->status); 2642 int32 transferred = TRB_2_REM_GET(trb->status), remainder = -1; 2643 2644 TRACE("HandleTransferComplete: ed %" B_PRIu32 ", code %" B_PRIu8 ", transferred %" B_PRId32 "\n", 2645 (flags & TRB_3_EVENT_DATA_BIT), completionCode, transferred); 2646 2647 if ((flags & TRB_3_EVENT_DATA_BIT) == 0) { 2648 // This should only occur under error conditions. 2649 TRACE("got an interrupt for a non-Event Data TRB!\n"); 2650 remainder = transferred; 2651 transferred = -1; 2652 } 2653 2654 if (completionCode != COMP_SUCCESS && completionCode != COMP_SHORT_PACKET 2655 && completionCode != COMP_STOPPED) { 2656 TRACE_ALWAYS("transfer error on slot %" B_PRId8 " endpoint %" B_PRId8 2657 ": %s\n", slot, endpointNumber, xhci_error_string(completionCode)); 2658 } 2659 2660 const phys_addr_t source = B_LENDIAN_TO_HOST_INT64(trb->address); 2661 for (xhci_td *td = endpoint->td_head; td != NULL; td = td->next) { 2662 int64 offset = (source - td->trb_addr) / sizeof(xhci_trb); 2663 if (offset < 0 || offset >= td->trb_count) 2664 continue; 2665 2666 TRACE("HandleTransferComplete td %p trb %" B_PRId64 " found\n", 2667 td, offset); 2668 2669 // The TRB at offset trb_used will be the link TRB, which we do not 2670 // care about (and should not generate an interrupt at all.) We really 2671 // care about the properly last TRB, at index "count - 1", which the 2672 // Event Data TRB that _LinkDescriptorForPipe creates points to. 2673 // 2674 // But if we have an unsuccessful completion code, the transfer 2675 // likely failed midway; so just accept it anyway. 2676 if (offset == (td->trb_used - 1) || completionCode != COMP_SUCCESS) { 2677 _UnlinkDescriptorForPipe(td, endpoint); 2678 endpointLocker.Unlock(); 2679 2680 td->trb_completion_code = completionCode; 2681 td->td_transferred = transferred; 2682 td->trb_left = remainder; 2683 2684 // add descriptor to finished list 2685 if (mutex_trylock(&fFinishedLock) != B_OK) 2686 mutex_lock(&fFinishedLock); 2687 td->next = fFinishedHead; 2688 fFinishedHead = td; 2689 mutex_unlock(&fFinishedLock); 2690 2691 release_sem_etc(fFinishTransfersSem, 1, B_DO_NOT_RESCHEDULE); 2692 TRACE("HandleTransferComplete td %p done\n", td); 2693 } else { 2694 TRACE_ERROR("successful TRB 0x%" B_PRIxPHYSADDR " was found, but it wasn't " 2695 "the last in the TD!\n", source); 2696 } 2697 return; 2698 } 2699 TRACE_ERROR("TRB 0x%" B_PRIxPHYSADDR " was not found in the endpoint!\n", source); 2700 } 2701 2702 2703 void 2704 XHCI::DumpRing(xhci_trb *trbs, uint32 size) 2705 { 2706 if (!Lock()) { 2707 TRACE("Unable to get lock!\n"); 2708 return; 2709 } 2710 2711 for (uint32 i = 0; i < size; i++) { 2712 TRACE("command[%" B_PRId32 "] = %" B_PRId32 " (0x%016" B_PRIx64 "," 2713 " 0x%08" B_PRIx32 ", 0x%08" B_PRIx32 ")\n", i, 2714 TRB_3_TYPE_GET(B_LENDIAN_TO_HOST_INT32(trbs[i].flags)), 2715 trbs[i].address, trbs[i].status, trbs[i].flags); 2716 } 2717 2718 Unlock(); 2719 } 2720 2721 2722 status_t 2723 XHCI::DoCommand(xhci_trb* trb) 2724 { 2725 if (!Lock()) { 2726 TRACE("Unable to get lock!\n"); 2727 return B_ERROR; 2728 } 2729 2730 QueueCommand(trb); 2731 Ring(0, 0); 2732 2733 // Begin with a 50ms timeout. 2734 if (acquire_sem_etc(fCmdCompSem, 1, B_RELATIVE_TIMEOUT, 50 * 1000) != B_OK) { 2735 // We've hit the timeout. In some error cases, interrupts are not 2736 // generated; so here we force the event ring to be polled once. 2737 release_sem(fEventSem); 2738 2739 // Now try again, this time with a 750ms timeout. 2740 if (acquire_sem_etc(fCmdCompSem, 1, B_RELATIVE_TIMEOUT, 2741 750 * 1000) != B_OK) { 2742 TRACE("Unable to obtain fCmdCompSem!\n"); 2743 fCmdAddr = 0; 2744 Unlock(); 2745 return B_TIMED_OUT; 2746 } 2747 } 2748 2749 // eat up sems that have been released by multiple interrupts 2750 int32 semCount = 0; 2751 get_sem_count(fCmdCompSem, &semCount); 2752 if (semCount > 0) 2753 acquire_sem_etc(fCmdCompSem, semCount, B_RELATIVE_TIMEOUT, 0); 2754 2755 status_t status = B_OK; 2756 uint32 completionCode = TRB_2_COMP_CODE_GET(fCmdResult[0]); 2757 TRACE("command complete\n"); 2758 if (completionCode != COMP_SUCCESS) { 2759 TRACE_ERROR("unsuccessful command %" B_PRId32 ", error %s (%" B_PRId32 ")\n", 2760 TRB_3_TYPE_GET(trb->flags), xhci_error_string(completionCode), 2761 completionCode); 2762 status = B_IO_ERROR; 2763 } 2764 2765 trb->status = fCmdResult[0]; 2766 trb->flags = fCmdResult[1]; 2767 2768 fCmdAddr = 0; 2769 Unlock(); 2770 return status; 2771 } 2772 2773 2774 status_t 2775 XHCI::Noop() 2776 { 2777 TRACE("Issue No-Op\n"); 2778 xhci_trb trb; 2779 trb.address = 0; 2780 trb.status = 0; 2781 trb.flags = TRB_3_TYPE(TRB_TYPE_CMD_NOOP); 2782 2783 return DoCommand(&trb); 2784 } 2785 2786 2787 status_t 2788 XHCI::EnableSlot(uint8* slot) 2789 { 2790 TRACE("Enable Slot\n"); 2791 xhci_trb trb; 2792 trb.address = 0; 2793 trb.status = 0; 2794 trb.flags = TRB_3_TYPE(TRB_TYPE_ENABLE_SLOT); 2795 2796 status_t status = DoCommand(&trb); 2797 if (status != B_OK) 2798 return status; 2799 2800 *slot = TRB_3_SLOT_GET(trb.flags); 2801 return *slot != 0 ? B_OK : B_BAD_VALUE; 2802 } 2803 2804 2805 status_t 2806 XHCI::DisableSlot(uint8 slot) 2807 { 2808 TRACE("Disable Slot\n"); 2809 xhci_trb trb; 2810 trb.address = 0; 2811 trb.status = 0; 2812 trb.flags = TRB_3_TYPE(TRB_TYPE_DISABLE_SLOT) | TRB_3_SLOT(slot); 2813 2814 return DoCommand(&trb); 2815 } 2816 2817 2818 status_t 2819 XHCI::SetAddress(uint64 inputContext, bool bsr, uint8 slot) 2820 { 2821 TRACE("Set Address\n"); 2822 xhci_trb trb; 2823 trb.address = inputContext; 2824 trb.status = 0; 2825 trb.flags = TRB_3_TYPE(TRB_TYPE_ADDRESS_DEVICE) | TRB_3_SLOT(slot); 2826 2827 if (bsr) 2828 trb.flags |= TRB_3_BSR_BIT; 2829 2830 return DoCommand(&trb); 2831 } 2832 2833 2834 status_t 2835 XHCI::ConfigureEndpoint(uint64 inputContext, bool deconfigure, uint8 slot) 2836 { 2837 TRACE("Configure Endpoint\n"); 2838 xhci_trb trb; 2839 trb.address = inputContext; 2840 trb.status = 0; 2841 trb.flags = TRB_3_TYPE(TRB_TYPE_CONFIGURE_ENDPOINT) | TRB_3_SLOT(slot); 2842 2843 if (deconfigure) 2844 trb.flags |= TRB_3_DCEP_BIT; 2845 2846 return DoCommand(&trb); 2847 } 2848 2849 2850 status_t 2851 XHCI::EvaluateContext(uint64 inputContext, uint8 slot) 2852 { 2853 TRACE("Evaluate Context\n"); 2854 xhci_trb trb; 2855 trb.address = inputContext; 2856 trb.status = 0; 2857 trb.flags = TRB_3_TYPE(TRB_TYPE_EVALUATE_CONTEXT) | TRB_3_SLOT(slot); 2858 2859 return DoCommand(&trb); 2860 } 2861 2862 2863 status_t 2864 XHCI::ResetEndpoint(bool preserve, xhci_endpoint* endpoint) 2865 { 2866 TRACE("Reset Endpoint\n"); 2867 2868 switch (_GetEndpointState(endpoint)) { 2869 case ENDPOINT_STATE_STOPPED: 2870 TRACE("Reset Endpoint: already stopped"); 2871 return B_OK; 2872 case ENDPOINT_STATE_HALTED: 2873 TRACE("Reset Endpoint: warning, weird state!"); 2874 default: 2875 break; 2876 } 2877 2878 xhci_trb trb; 2879 trb.address = 0; 2880 trb.status = 0; 2881 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_ENDPOINT) 2882 | TRB_3_SLOT(endpoint->device->slot) | TRB_3_ENDPOINT(endpoint->id + 1); 2883 if (preserve) 2884 trb.flags |= TRB_3_PRSV_BIT; 2885 2886 return DoCommand(&trb); 2887 } 2888 2889 2890 status_t 2891 XHCI::StopEndpoint(bool suspend, xhci_endpoint* endpoint) 2892 { 2893 TRACE("Stop Endpoint\n"); 2894 2895 switch (_GetEndpointState(endpoint)) { 2896 case ENDPOINT_STATE_HALTED: 2897 TRACE("Stop Endpoint: error, halted"); 2898 return B_DEV_STALLED; 2899 case ENDPOINT_STATE_STOPPED: 2900 TRACE("Stop Endpoint: already stopped"); 2901 return B_OK; 2902 default: 2903 break; 2904 } 2905 2906 xhci_trb trb; 2907 trb.address = 0; 2908 trb.status = 0; 2909 trb.flags = TRB_3_TYPE(TRB_TYPE_STOP_ENDPOINT) 2910 | TRB_3_SLOT(endpoint->device->slot) | TRB_3_ENDPOINT(endpoint->id + 1); 2911 if (suspend) 2912 trb.flags |= TRB_3_SUSPEND_ENDPOINT_BIT; 2913 2914 return DoCommand(&trb); 2915 } 2916 2917 2918 status_t 2919 XHCI::SetTRDequeue(uint64 dequeue, uint16 stream, uint8 endpoint, uint8 slot) 2920 { 2921 TRACE("Set TR Dequeue\n"); 2922 xhci_trb trb; 2923 trb.address = dequeue | ENDPOINT_2_DCS_BIT; 2924 // The DCS bit is copied from the address field as in ConfigureEndpoint. 2925 // (XHCI 1.2 § 4.6.10 p142.) 2926 trb.status = TRB_2_STREAM(stream); 2927 trb.flags = TRB_3_TYPE(TRB_TYPE_SET_TR_DEQUEUE) 2928 | TRB_3_SLOT(slot) | TRB_3_ENDPOINT(endpoint); 2929 2930 return DoCommand(&trb); 2931 } 2932 2933 2934 status_t 2935 XHCI::ResetDevice(uint8 slot) 2936 { 2937 TRACE("Reset Device\n"); 2938 xhci_trb trb; 2939 trb.address = 0; 2940 trb.status = 0; 2941 trb.flags = TRB_3_TYPE(TRB_TYPE_RESET_DEVICE) | TRB_3_SLOT(slot); 2942 2943 return DoCommand(&trb); 2944 } 2945 2946 2947 int32 2948 XHCI::EventThread(void* data) 2949 { 2950 ((XHCI *)data)->CompleteEvents(); 2951 return B_OK; 2952 } 2953 2954 2955 void 2956 XHCI::CompleteEvents() 2957 { 2958 while (!fStopThreads) { 2959 if (acquire_sem(fEventSem) < B_OK) 2960 continue; 2961 2962 // eat up sems that have been released by multiple interrupts 2963 int32 semCount = 0; 2964 get_sem_count(fEventSem, &semCount); 2965 if (semCount > 0) 2966 acquire_sem_etc(fEventSem, semCount, B_RELATIVE_TIMEOUT, 0); 2967 2968 ProcessEvents(); 2969 } 2970 } 2971 2972 2973 void 2974 XHCI::ProcessEvents() 2975 { 2976 // Use mutex_trylock first, in case we are in KDL. 2977 MutexLocker locker(fEventLock, mutex_trylock(&fEventLock) == B_OK); 2978 if (!locker.IsLocked()) { 2979 // We failed to get the lock. This really should not happen. 2980 TRACE_ERROR("failed to acquire event lock!\n"); 2981 return; 2982 } 2983 2984 uint16 i = fEventIdx; 2985 uint8 j = fEventCcs; 2986 uint8 t = 2; 2987 2988 while (1) { 2989 uint32 temp = B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags); 2990 uint8 event = TRB_3_TYPE_GET(temp); 2991 TRACE("event[%u] = %u (0x%016" B_PRIx64 " 0x%08" B_PRIx32 " 0x%08" 2992 B_PRIx32 ")\n", i, event, fEventRing[i].address, 2993 fEventRing[i].status, B_LENDIAN_TO_HOST_INT32(fEventRing[i].flags)); 2994 uint8 k = (temp & TRB_3_CYCLE_BIT) ? 1 : 0; 2995 if (j != k) 2996 break; 2997 2998 switch (event) { 2999 case TRB_TYPE_COMMAND_COMPLETION: 3000 HandleCmdComplete(&fEventRing[i]); 3001 break; 3002 case TRB_TYPE_TRANSFER: 3003 HandleTransferComplete(&fEventRing[i]); 3004 break; 3005 case TRB_TYPE_PORT_STATUS_CHANGE: 3006 TRACE("port change detected\n"); 3007 break; 3008 default: 3009 TRACE_ERROR("Unhandled event = %u\n", event); 3010 break; 3011 } 3012 3013 i++; 3014 if (i == XHCI_MAX_EVENTS) { 3015 i = 0; 3016 j ^= 1; 3017 if (!--t) 3018 break; 3019 } 3020 } 3021 3022 fEventIdx = i; 3023 fEventCcs = j; 3024 3025 uint64 addr = fErst->rs_addr + i * sizeof(xhci_trb); 3026 WriteRunReg32(XHCI_ERDP_LO(0), (uint32)addr | ERDP_BUSY); 3027 WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(addr >> 32)); 3028 } 3029 3030 3031 int32 3032 XHCI::FinishThread(void* data) 3033 { 3034 ((XHCI *)data)->FinishTransfers(); 3035 return B_OK; 3036 } 3037 3038 3039 void 3040 XHCI::FinishTransfers() 3041 { 3042 while (!fStopThreads) { 3043 if (acquire_sem(fFinishTransfersSem) < B_OK) 3044 continue; 3045 3046 // eat up sems that have been released by multiple interrupts 3047 int32 semCount = 0; 3048 get_sem_count(fFinishTransfersSem, &semCount); 3049 if (semCount > 0) 3050 acquire_sem_etc(fFinishTransfersSem, semCount, B_RELATIVE_TIMEOUT, 0); 3051 3052 mutex_lock(&fFinishedLock); 3053 TRACE("finishing transfers\n"); 3054 while (fFinishedHead != NULL) { 3055 xhci_td* td = fFinishedHead; 3056 fFinishedHead = td->next; 3057 td->next = NULL; 3058 mutex_unlock(&fFinishedLock); 3059 3060 TRACE("finishing transfer td %p\n", td); 3061 3062 Transfer* transfer = td->transfer; 3063 if (transfer == NULL) { 3064 // No transfer? Quick way out. 3065 FreeDescriptor(td); 3066 mutex_lock(&fFinishedLock); 3067 continue; 3068 } 3069 3070 bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out); 3071 3072 status_t callbackStatus = B_OK; 3073 const uint8 completionCode = td->trb_completion_code; 3074 switch (completionCode) { 3075 case COMP_SHORT_PACKET: 3076 case COMP_SUCCESS: 3077 callbackStatus = B_OK; 3078 break; 3079 case COMP_DATA_BUFFER: 3080 callbackStatus = directionIn ? B_DEV_DATA_OVERRUN 3081 : B_DEV_DATA_UNDERRUN; 3082 break; 3083 case COMP_BABBLE: 3084 callbackStatus = directionIn ? B_DEV_FIFO_OVERRUN 3085 : B_DEV_FIFO_UNDERRUN; 3086 break; 3087 case COMP_USB_TRANSACTION: 3088 callbackStatus = B_DEV_CRC_ERROR; 3089 break; 3090 case COMP_STALL: 3091 callbackStatus = B_DEV_STALLED; 3092 break; 3093 default: 3094 callbackStatus = B_DEV_STALLED; 3095 break; 3096 } 3097 3098 size_t actualLength = transfer->FragmentLength(); 3099 if (completionCode != COMP_SUCCESS) { 3100 actualLength = td->td_transferred; 3101 if (td->td_transferred == -1) 3102 actualLength = transfer->FragmentLength() - td->trb_left; 3103 TRACE("transfer not successful, actualLength=%" B_PRIuSIZE "\n", 3104 actualLength); 3105 } 3106 3107 usb_isochronous_data* isochronousData = transfer->IsochronousData(); 3108 if (isochronousData != NULL) { 3109 size_t packetSize = transfer->DataLength() 3110 / isochronousData->packet_count, 3111 left = actualLength; 3112 for (uint32 i = 0; i < isochronousData->packet_count; i++) { 3113 size_t size = min_c(packetSize, left); 3114 isochronousData->packet_descriptors[i].actual_length = size; 3115 isochronousData->packet_descriptors[i].status = (size > 0) 3116 ? B_OK : B_DEV_FIFO_UNDERRUN; 3117 left -= size; 3118 } 3119 } 3120 3121 if (callbackStatus == B_OK && directionIn && actualLength > 0) { 3122 TRACE("copying in iov count %ld\n", transfer->VectorCount()); 3123 status_t status = transfer->PrepareKernelAccess(); 3124 if (status == B_OK) { 3125 ReadDescriptor(td, transfer->Vector(), 3126 transfer->VectorCount()); 3127 } else { 3128 callbackStatus = status; 3129 } 3130 } 3131 3132 FreeDescriptor(td); 3133 3134 // this transfer may still have data left 3135 bool finished = true; 3136 transfer->AdvanceByFragment(actualLength); 3137 if (completionCode == COMP_SUCCESS 3138 && transfer->FragmentLength() > 0) { 3139 TRACE("still %" B_PRIuSIZE " bytes left on transfer\n", 3140 transfer->FragmentLength()); 3141 callbackStatus = SubmitTransfer(transfer); 3142 finished = (callbackStatus != B_OK); 3143 } 3144 if (finished) { 3145 // The actualLength was already handled in AdvanceByFragment. 3146 transfer->Finished(callbackStatus, 0); 3147 delete transfer; 3148 } 3149 3150 mutex_lock(&fFinishedLock); 3151 } 3152 mutex_unlock(&fFinishedLock); 3153 } 3154 } 3155 3156 3157 inline void 3158 XHCI::WriteOpReg(uint32 reg, uint32 value) 3159 { 3160 *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg) = value; 3161 } 3162 3163 3164 inline uint32 3165 XHCI::ReadOpReg(uint32 reg) 3166 { 3167 return *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg); 3168 } 3169 3170 3171 inline status_t 3172 XHCI::WaitOpBits(uint32 reg, uint32 mask, uint32 expected) 3173 { 3174 int loops = 0; 3175 uint32 value = ReadOpReg(reg); 3176 while ((value & mask) != expected) { 3177 snooze(1000); 3178 value = ReadOpReg(reg); 3179 if (loops == 100) { 3180 TRACE("delay waiting on reg 0x%" B_PRIX32 " match 0x%" B_PRIX32 3181 " (0x%" B_PRIX32 ")\n", reg, expected, mask); 3182 } else if (loops > 250) { 3183 TRACE_ERROR("timeout waiting on reg 0x%" B_PRIX32 3184 " match 0x%" B_PRIX32 " (0x%" B_PRIX32 ")\n", reg, expected, 3185 mask); 3186 return B_ERROR; 3187 } 3188 loops++; 3189 } 3190 return B_OK; 3191 } 3192 3193 3194 inline uint32 3195 XHCI::ReadCapReg32(uint32 reg) 3196 { 3197 return *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg); 3198 } 3199 3200 3201 inline void 3202 XHCI::WriteCapReg32(uint32 reg, uint32 value) 3203 { 3204 *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg) = value; 3205 } 3206 3207 3208 inline uint32 3209 XHCI::ReadRunReg32(uint32 reg) 3210 { 3211 return *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg); 3212 } 3213 3214 3215 inline void 3216 XHCI::WriteRunReg32(uint32 reg, uint32 value) 3217 { 3218 *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg) = value; 3219 } 3220 3221 3222 inline uint32 3223 XHCI::ReadDoorReg32(uint32 reg) 3224 { 3225 return *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg); 3226 } 3227 3228 3229 inline void 3230 XHCI::WriteDoorReg32(uint32 reg, uint32 value) 3231 { 3232 *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg) = value; 3233 } 3234 3235 3236 inline addr_t 3237 XHCI::_OffsetContextAddr(addr_t p) 3238 { 3239 if (fContextSizeShift == 1) { 3240 // each structure is page aligned, each pointer is 32 bits aligned 3241 uint32 offset = p & ((B_PAGE_SIZE - 1) & ~31U); 3242 p += offset; 3243 } 3244 return p; 3245 } 3246 3247 inline uint32 3248 XHCI::_ReadContext(uint32* p) 3249 { 3250 p = (uint32*)_OffsetContextAddr((addr_t)p); 3251 return *p; 3252 } 3253 3254 3255 inline void 3256 XHCI::_WriteContext(uint32* p, uint32 value) 3257 { 3258 p = (uint32*)_OffsetContextAddr((addr_t)p); 3259 *p = value; 3260 } 3261 3262 3263 inline uint64 3264 XHCI::_ReadContext(uint64* p) 3265 { 3266 p = (uint64*)_OffsetContextAddr((addr_t)p); 3267 return *p; 3268 } 3269 3270 3271 inline void 3272 XHCI::_WriteContext(uint64* p, uint64 value) 3273 { 3274 p = (uint64*)_OffsetContextAddr((addr_t)p); 3275 *p = value; 3276 } 3277