xref: /haiku/src/add-ons/kernel/busses/usb/xhci.cpp (revision 5a7156f4352715268eec645cb4c64c25884c64b1)
1 /*
2  * Copyright 2011-2019, Haiku Inc. All rights reserved.
3  * Distributed under the terms of the MIT License.
4  *
5  * Authors:
6  *		Augustin Cavalier <waddlesplash>
7  * 		Jian Chiang <j.jian.chiang@gmail.com>
8  *		Jérôme Duval <jerome.duval@gmail.com>
9  *		Akshay Jaggi <akshay1994.leo@gmail.com>
10  *		Michael Lotz <mmlr@mlotz.ch>
11  */
12 
13 
14 #include <module.h>
15 #include <PCI.h>
16 #include <PCI_x86.h>
17 #include <USB3.h>
18 #include <KernelExport.h>
19 
20 #include <util/AutoLock.h>
21 
22 #include "xhci.h"
23 
24 #define USB_MODULE_NAME	"xhci"
25 
26 pci_module_info *XHCI::sPCIModule = NULL;
27 pci_x86_module_info *XHCI::sPCIx86Module = NULL;
28 
29 
30 static int32
31 xhci_std_ops(int32 op, ...)
32 {
33 	switch (op) {
34 		case B_MODULE_INIT:
35 			TRACE_MODULE("xhci init module\n");
36 			return B_OK;
37 		case B_MODULE_UNINIT:
38 			TRACE_MODULE("xhci uninit module\n");
39 			return B_OK;
40 	}
41 
42 	return EINVAL;
43 }
44 
45 
46 static const char*
47 xhci_error_string(uint32 error)
48 {
49 	switch (error) {
50 		case COMP_INVALID: return "Invalid";
51 		case COMP_SUCCESS: return "Success";
52 		case COMP_DATA_BUFFER: return "Data buffer";
53 		case COMP_BABBLE: return "Babble detected";
54 		case COMP_USB_TRANSACTION: return "USB transaction";
55 		case COMP_TRB: return "TRB";
56 		case COMP_STALL: return "Stall";
57 		case COMP_RESOURCE: return "Resource";
58 		case COMP_BANDWIDTH: return "Bandwidth";
59 		case COMP_NO_SLOTS: return "No slots";
60 		case COMP_INVALID_STREAM: return "Invalid stream";
61 		case COMP_SLOT_NOT_ENABLED: return "Slot not enabled";
62 		case COMP_ENDPOINT_NOT_ENABLED: return "Endpoint not enabled";
63 		case COMP_SHORT_PACKET: return "Short packet";
64 		case COMP_RING_UNDERRUN: return "Ring underrun";
65 		case COMP_RING_OVERRUN: return "Ring overrun";
66 		case COMP_VF_RING_FULL: return "VF Event Ring Full";
67 		case COMP_PARAMETER: return "Parameter";
68 		case COMP_BANDWIDTH_OVERRUN: return "Bandwidth overrun";
69 		case COMP_CONTEXT_STATE: return "Context state";
70 		case COMP_NO_PING_RESPONSE: return "No ping response";
71 		case COMP_EVENT_RING_FULL: return "Event ring full";
72 		case COMP_INCOMPATIBLE_DEVICE: return "Incompatible device";
73 		case COMP_MISSED_SERVICE: return "Missed service";
74 		case COMP_COMMAND_RING_STOPPED: return "Command ring stopped";
75 		case COMP_COMMAND_ABORTED: return "Command aborted";
76 		case COMP_STOPPED: return "Stopped";
77 		case COMP_LENGTH_INVALID: return "Length invalid";
78 		case COMP_MAX_EXIT_LATENCY: return "Max exit latency too large";
79 		case COMP_ISOC_OVERRUN: return "Isoch buffer overrun";
80 		case COMP_EVENT_LOST: return "Event lost";
81 		case COMP_UNDEFINED: return "Undefined";
82 		case COMP_INVALID_STREAM_ID: return "Invalid stream ID";
83 		case COMP_SECONDARY_BANDWIDTH: return "Secondary bandwidth";
84 		case COMP_SPLIT_TRANSACTION: return "Split transaction";
85 
86 		default: return "Undefined";
87 	}
88 }
89 
90 
91 usb_host_controller_info xhci_module = {
92 	{
93 		"busses/usb/xhci",
94 		0,
95 		xhci_std_ops
96 	},
97 	NULL,
98 	XHCI::AddTo
99 };
100 
101 
102 module_info *modules[] = {
103 	(module_info *)&xhci_module,
104 	NULL
105 };
106 
107 
108 XHCI::XHCI(pci_info *info, Stack *stack)
109 	:	BusManager(stack),
110 		fRegisterArea(-1),
111 		fRegisters(NULL),
112 		fPCIInfo(info),
113 		fStack(stack),
114 		fIRQ(0),
115 		fUseMSI(false),
116 		fErstArea(-1),
117 		fDcbaArea(-1),
118 		fCmdCompSem(-1),
119 		fFinishTransfersSem(-1),
120 		fFinishThread(-1),
121 		fStopThreads(false),
122 		fFinishedHead(NULL),
123 		fRootHub(NULL),
124 		fRootHubAddress(0),
125 		fPortCount(0),
126 		fSlotCount(0),
127 		fScratchpadCount(0),
128 		fContextSizeShift(0),
129 		fEventSem(-1),
130 		fEventThread(-1),
131 		fEventIdx(0),
132 		fCmdIdx(0),
133 		fEventCcs(1),
134 		fCmdCcs(1)
135 {
136 	B_INITIALIZE_SPINLOCK(&fSpinlock);
137 
138 	if (BusManager::InitCheck() < B_OK) {
139 		TRACE_ERROR("bus manager failed to init\n");
140 		return;
141 	}
142 
143 	TRACE("constructing new XHCI host controller driver\n");
144 	fInitOK = false;
145 
146 	// enable busmaster and memory mapped access
147 	uint16 command = sPCIModule->read_pci_config(fPCIInfo->bus,
148 		fPCIInfo->device, fPCIInfo->function, PCI_command, 2);
149 	command &= ~(PCI_command_io | PCI_command_int_disable);
150 	command |= PCI_command_master | PCI_command_memory;
151 
152 	sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device,
153 		fPCIInfo->function, PCI_command, 2, command);
154 
155 	// map the registers (low + high for 64-bit when requested)
156 	phys_addr_t physicalAddress = fPCIInfo->u.h0.base_registers[0];
157 	physicalAddress &= PCI_address_memory_32_mask;
158 	if ((fPCIInfo->u.h0.base_register_flags[0] & 0xC) == PCI_address_type_64)
159 		physicalAddress += (phys_addr_t)fPCIInfo->u.h0.base_registers[1] << 32;
160 
161 	size_t mapSize = fPCIInfo->u.h0.base_register_sizes[0];
162 
163 	TRACE("map physical memory %08" B_PRIxPHYSADDR ", size: %" B_PRId32 "\n",
164 		physicalAddress, mapSize);
165 
166 	fRegisterArea = map_physical_memory("XHCI memory mapped registers",
167 		physicalAddress, mapSize, B_ANY_KERNEL_BLOCK_ADDRESS,
168 		B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA,
169 		(void **)&fRegisters);
170 	if (fRegisterArea < B_OK) {
171 		TRACE_ERROR("failed to map register memory\n");
172 		return;
173 	}
174 
175 	// determine the register offsets
176 	fCapabilityRegisterOffset = 0;
177 	fOperationalRegisterOffset = HCI_CAPLENGTH(ReadCapReg32(XHCI_HCI_CAPLENGTH));
178 	fRuntimeRegisterOffset = ReadCapReg32(XHCI_RTSOFF) & ~0x1F;
179 	fDoorbellRegisterOffset = ReadCapReg32(XHCI_DBOFF) & ~0x3;
180 
181 	TRACE("mapped registers: %p\n", fRegisters);
182 	TRACE("operational register offset: %" B_PRId32 "\n", fOperationalRegisterOffset);
183 	TRACE("runtime register offset: %" B_PRId32 "\n", fRuntimeRegisterOffset);
184 	TRACE("doorbell register offset: %" B_PRId32 "\n", fDoorbellRegisterOffset);
185 
186 	TRACE_ALWAYS("interface version: 0x%04" B_PRIx32 "\n",
187 		HCI_VERSION(ReadCapReg32(XHCI_HCI_VERSION)));
188 	TRACE_ALWAYS("structural parameters: 1:0x%08" B_PRIx32 " 2:0x%08"
189 		B_PRIx32 " 3:0x%08" B_PRIx32 "\n", ReadCapReg32(XHCI_HCSPARAMS1),
190 		ReadCapReg32(XHCI_HCSPARAMS2), ReadCapReg32(XHCI_HCSPARAMS3));
191 	uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS);
192 	TRACE_ALWAYS("capability params: 0x%08" B_PRIx32 "\n", cparams);
193 
194 	// if 64 bytes context structures, then 1
195 	fContextSizeShift = HCC_CSZ(cparams);
196 
197 	uint32 eec = 0xffffffff;
198 	uint32 eecp = HCS0_XECP(cparams) << 2;
199 	for (; eecp != 0 && XECP_NEXT(eec); eecp += XECP_NEXT(eec) << 2) {
200 		TRACE("eecp register: 0x%08" B_PRIx32 "\n", eecp);
201 
202 		eec = ReadCapReg32(eecp);
203 		if (XECP_ID(eec) != XHCI_LEGSUP_CAPID)
204 			continue;
205 
206 		if (eec & XHCI_LEGSUP_BIOSOWNED) {
207 			TRACE_ALWAYS("the host controller is bios owned, claiming"
208 				" ownership\n");
209 			WriteCapReg32(eecp, eec | XHCI_LEGSUP_OSOWNED);
210 
211 			for (int32 i = 0; i < 20; i++) {
212 				eec = ReadCapReg32(eecp);
213 
214 				if ((eec & XHCI_LEGSUP_BIOSOWNED) == 0)
215 					break;
216 
217 				TRACE_ALWAYS("controller is still bios owned, waiting\n");
218 				snooze(50000);
219 			}
220 
221 			if (eec & XHCI_LEGSUP_BIOSOWNED) {
222 				TRACE_ERROR("bios won't give up control over the host "
223 					"controller (ignoring)\n");
224 			} else if (eec & XHCI_LEGSUP_OSOWNED) {
225 				TRACE_ALWAYS("successfully took ownership of the host "
226 					"controller\n");
227 			}
228 
229 			// Force off the BIOS owned flag, and clear all SMIs. Some BIOSes
230 			// do indicate a successful handover but do not remove their SMIs
231 			// and then freeze the system when interrupts are generated.
232 			WriteCapReg32(eecp, eec & ~XHCI_LEGSUP_BIOSOWNED);
233 		}
234 		break;
235 	}
236 	uint32 legctlsts = ReadCapReg32(eecp + XHCI_LEGCTLSTS);
237 	legctlsts &= XHCI_LEGCTLSTS_DISABLE_SMI;
238 	legctlsts |= XHCI_LEGCTLSTS_EVENTS_SMI;
239 	WriteCapReg32(eecp + XHCI_LEGCTLSTS, legctlsts);
240 
241 	// On Intel's Panther Point and Lynx Point Chipset taking ownership
242 	// of EHCI owned ports, is what we do here.
243 	if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL) {
244 		switch (fPCIInfo->device_id) {
245 			case PCI_DEVICE_INTEL_PANTHER_POINT_XHCI:
246 			case PCI_DEVICE_INTEL_LYNX_POINT_XHCI:
247 			case PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI:
248 			case PCI_DEVICE_INTEL_BAYTRAIL_XHCI:
249 			case PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI:
250 			case PCI_DEVICE_INTEL_WILDCAT_POINT_LP_XHCI:
251 				_SwitchIntelPorts();
252 				break;
253 		}
254 	}
255 
256 	// halt the host controller
257 	if (ControllerHalt() < B_OK) {
258 		return;
259 	}
260 
261 	// reset the host controller
262 	if (ControllerReset() < B_OK) {
263 		TRACE_ERROR("host controller failed to reset\n");
264 		return;
265 	}
266 
267 	fCmdCompSem = create_sem(0, "XHCI Command Complete");
268 	fFinishTransfersSem = create_sem(0, "XHCI Finish Transfers");
269 	fEventSem = create_sem(0, "XHCI Event");
270 	if (fFinishTransfersSem < B_OK || fCmdCompSem < B_OK || fEventSem < B_OK) {
271 		TRACE_ERROR("failed to create semaphores\n");
272 		return;
273 	}
274 
275 	// create finisher service thread
276 	fFinishThread = spawn_kernel_thread(FinishThread, "xhci finish thread",
277 		B_NORMAL_PRIORITY, (void *)this);
278 	resume_thread(fFinishThread);
279 
280 	// create finisher service thread
281 	fEventThread = spawn_kernel_thread(EventThread, "xhci event thread",
282 		B_NORMAL_PRIORITY, (void *)this);
283 	resume_thread(fEventThread);
284 
285 	// Find the right interrupt vector, using MSIs if available.
286 	fIRQ = fPCIInfo->u.h0.interrupt_line;
287 	if (sPCIx86Module != NULL && sPCIx86Module->get_msi_count(fPCIInfo->bus,
288 			fPCIInfo->device, fPCIInfo->function) >= 1) {
289 		uint8 msiVector = 0;
290 		if (sPCIx86Module->configure_msi(fPCIInfo->bus, fPCIInfo->device,
291 				fPCIInfo->function, 1, &msiVector) == B_OK
292 			&& sPCIx86Module->enable_msi(fPCIInfo->bus, fPCIInfo->device,
293 				fPCIInfo->function) == B_OK) {
294 			TRACE_ALWAYS("using message signaled interrupts\n");
295 			fIRQ = msiVector;
296 			fUseMSI = true;
297 		}
298 	}
299 
300 	if (fIRQ == 0 || fIRQ == 0xFF) {
301 		TRACE_MODULE_ERROR("device was assigned an invalid IRQ\n");
302 		return;
303 	}
304 
305 	// Install the interrupt handler
306 	TRACE("installing interrupt handler\n");
307 	install_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this, 0);
308 
309 	memset(fPortSpeeds, 0, sizeof(fPortSpeeds));
310 	memset(fPortSlots, 0, sizeof(fPortSlots));
311 	memset(fDevices, 0, sizeof(fDevices));
312 
313 	fInitOK = true;
314 	TRACE("XHCI host controller driver constructed\n");
315 }
316 
317 
318 XHCI::~XHCI()
319 {
320 	TRACE("tear down XHCI host controller driver\n");
321 
322 	WriteOpReg(XHCI_CMD, 0);
323 
324 	int32 result = 0;
325 	fStopThreads = true;
326 	delete_sem(fCmdCompSem);
327 	delete_sem(fFinishTransfersSem);
328 	delete_sem(fEventSem);
329 	wait_for_thread(fFinishThread, &result);
330 	wait_for_thread(fEventThread, &result);
331 
332 	remove_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this);
333 
334 	delete_area(fRegisterArea);
335 	delete_area(fErstArea);
336 	for (uint32 i = 0; i < fScratchpadCount; i++)
337 		delete_area(fScratchpadArea[i]);
338 	delete_area(fDcbaArea);
339 
340 	if (fUseMSI && sPCIx86Module != NULL) {
341 		sPCIx86Module->disable_msi(fPCIInfo->bus,
342 			fPCIInfo->device, fPCIInfo->function);
343 		sPCIx86Module->unconfigure_msi(fPCIInfo->bus,
344 			fPCIInfo->device, fPCIInfo->function);
345 	}
346 	put_module(B_PCI_MODULE_NAME);
347 	if (sPCIx86Module != NULL) {
348 		sPCIx86Module = NULL;
349 		put_module(B_PCI_X86_MODULE_NAME);
350 	}
351 }
352 
353 
354 void
355 XHCI::_SwitchIntelPorts()
356 {
357 	TRACE("Intel xHC Controller\n");
358 	TRACE("Looking for EHCI owned ports\n");
359 	uint32 ports = sPCIModule->read_pci_config(fPCIInfo->bus,
360 		fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_USB3PRM, 4);
361 	TRACE("Superspeed Ports: 0x%" B_PRIx32 "\n", ports);
362 	sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device,
363 		fPCIInfo->function, XHCI_INTEL_USB3_PSSEN, 4, ports);
364 	ports = sPCIModule->read_pci_config(fPCIInfo->bus,
365 		fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_USB3_PSSEN, 4);
366 	TRACE("Superspeed ports now under XHCI : 0x%" B_PRIx32 "\n", ports);
367 	ports = sPCIModule->read_pci_config(fPCIInfo->bus,
368 		fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_USB2PRM, 4);
369 	TRACE("USB 2.0 Ports : 0x%" B_PRIx32 "\n", ports);
370 	sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device,
371 		fPCIInfo->function, XHCI_INTEL_XUSB2PR, 4, ports);
372 	ports = sPCIModule->read_pci_config(fPCIInfo->bus,
373 		fPCIInfo->device, fPCIInfo->function, XHCI_INTEL_XUSB2PR, 4);
374 	TRACE("USB 2.0 ports now under XHCI: 0x%" B_PRIx32 "\n", ports);
375 }
376 
377 
378 status_t
379 XHCI::Start()
380 {
381 	TRACE_ALWAYS("starting XHCI host controller\n");
382 	TRACE("usbcmd: 0x%08" B_PRIx32 "; usbsts: 0x%08" B_PRIx32 "\n",
383 		ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS));
384 
385 	if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) {
386 		TRACE("Start() failed STS_CNR\n");
387 	}
388 
389 	if ((ReadOpReg(XHCI_CMD) & CMD_RUN) != 0) {
390 		TRACE_ERROR("Start() warning, starting running XHCI controller!\n");
391 	}
392 
393 	if ((ReadOpReg(XHCI_PAGESIZE) & (1 << 0)) == 0) {
394 		TRACE_ERROR("Controller does not support 4K page size.\n");
395 		return B_ERROR;
396 	}
397 
398 	// read port count from capability register
399 	uint32 capabilities = ReadCapReg32(XHCI_HCSPARAMS1);
400 	fPortCount = HCS_MAX_PORTS(capabilities);
401 	if (fPortCount == 0) {
402 		TRACE_ERROR("Invalid number of ports: %u\n", fPortCount);
403 		return B_ERROR;
404 	}
405 
406 	fSlotCount = HCS_MAX_SLOTS(capabilities);
407 	if (fSlotCount > XHCI_MAX_DEVICES)
408 		fSlotCount = XHCI_MAX_DEVICES;
409 	WriteOpReg(XHCI_CONFIG, fSlotCount);
410 
411 	// find out which protocol is used for each port
412 	uint8 portFound = 0;
413 	uint32 cparams = ReadCapReg32(XHCI_HCCPARAMS);
414 	uint32 eec = 0xffffffff;
415 	uint32 eecp = HCS0_XECP(cparams) << 2;
416 	for (; eecp != 0 && XECP_NEXT(eec) && portFound < fPortCount;
417 		eecp += XECP_NEXT(eec) << 2) {
418 		eec = ReadCapReg32(eecp);
419 		if (XECP_ID(eec) != XHCI_SUPPORTED_PROTOCOLS_CAPID)
420 			continue;
421 		if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) > 3)
422 			continue;
423 		uint32 temp = ReadCapReg32(eecp + 8);
424 		uint32 offset = XHCI_SUPPORTED_PROTOCOLS_1_OFFSET(temp);
425 		uint32 count = XHCI_SUPPORTED_PROTOCOLS_1_COUNT(temp);
426 		if (offset == 0 || count == 0)
427 			continue;
428 		offset--;
429 		for (uint32 i = offset; i < offset + count; i++) {
430 			if (XHCI_SUPPORTED_PROTOCOLS_0_MAJOR(eec) == 0x3)
431 				fPortSpeeds[i] = USB_SPEED_SUPER;
432 			else
433 				fPortSpeeds[i] = USB_SPEED_HIGHSPEED;
434 			TRACE("speed for port %" B_PRId32 " is %s\n", i,
435 				fPortSpeeds[i] == USB_SPEED_SUPER ? "super" : "high");
436 		}
437 		portFound += count;
438 	}
439 
440 	uint32 params2 = ReadCapReg32(XHCI_HCSPARAMS2);
441 	fScratchpadCount = HCS_MAX_SC_BUFFERS(params2);
442 	if (fScratchpadCount > XHCI_MAX_SCRATCHPADS) {
443 		TRACE_ERROR("Invalid number of scratchpads: %" B_PRIu32 "\n",
444 			fScratchpadCount);
445 		return B_ERROR;
446 	}
447 
448 	uint32 params3 = ReadCapReg32(XHCI_HCSPARAMS3);
449 	fExitLatMax = HCS_U1_DEVICE_LATENCY(params3)
450 		+ HCS_U2_DEVICE_LATENCY(params3);
451 
452 	// clear interrupts & disable device notifications
453 	WriteOpReg(XHCI_STS, ReadOpReg(XHCI_STS));
454 	WriteOpReg(XHCI_DNCTRL, 0);
455 
456 	// allocate Device Context Base Address array
457 	phys_addr_t dmaAddress;
458 	fDcbaArea = fStack->AllocateArea((void **)&fDcba, &dmaAddress,
459 		sizeof(*fDcba), "DCBA Area");
460 	if (fDcbaArea < B_OK) {
461 		TRACE_ERROR("unable to create the DCBA area\n");
462 		return B_ERROR;
463 	}
464 	memset(fDcba, 0, sizeof(*fDcba));
465 	memset(fScratchpadArea, 0, sizeof(fScratchpadArea));
466 	memset(fScratchpad, 0, sizeof(fScratchpad));
467 
468 	// setting the first address to the scratchpad array address
469 	fDcba->baseAddress[0] = dmaAddress
470 		+ offsetof(struct xhci_device_context_array, scratchpad);
471 
472 	// fill up the scratchpad array with scratchpad pages
473 	for (uint32 i = 0; i < fScratchpadCount; i++) {
474 		phys_addr_t scratchDmaAddress;
475 		fScratchpadArea[i] = fStack->AllocateArea((void **)&fScratchpad[i],
476 			&scratchDmaAddress, B_PAGE_SIZE, "Scratchpad Area");
477 		if (fScratchpadArea[i] < B_OK) {
478 			TRACE_ERROR("unable to create the scratchpad area\n");
479 			return B_ERROR;
480 		}
481 		fDcba->scratchpad[i] = scratchDmaAddress;
482 	}
483 
484 	TRACE("setting DCBAAP %" B_PRIxPHYSADDR "\n", dmaAddress);
485 	WriteOpReg(XHCI_DCBAAP_LO, (uint32)dmaAddress);
486 	WriteOpReg(XHCI_DCBAAP_HI, (uint32)(dmaAddress >> 32));
487 
488 	// allocate Event Ring Segment Table
489 	uint8 *addr;
490 	fErstArea = fStack->AllocateArea((void **)&addr, &dmaAddress,
491 		(XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb)
492 		+ sizeof(xhci_erst_element),
493 		"USB XHCI ERST CMD_RING and EVENT_RING Area");
494 
495 	if (fErstArea < B_OK) {
496 		TRACE_ERROR("unable to create the ERST AND RING area\n");
497 		delete_area(fDcbaArea);
498 		return B_ERROR;
499 	}
500 	fErst = (xhci_erst_element *)addr;
501 	memset(fErst, 0, (XHCI_MAX_COMMANDS + XHCI_MAX_EVENTS) * sizeof(xhci_trb)
502 		+ sizeof(xhci_erst_element));
503 
504 	// fill with Event Ring Segment Base Address and Event Ring Segment Size
505 	fErst->rs_addr = dmaAddress + sizeof(xhci_erst_element);
506 	fErst->rs_size = XHCI_MAX_EVENTS;
507 	fErst->rsvdz = 0;
508 
509 	addr += sizeof(xhci_erst_element);
510 	fEventRing = (xhci_trb *)addr;
511 	addr += XHCI_MAX_EVENTS * sizeof(xhci_trb);
512 	fCmdRing = (xhci_trb *)addr;
513 
514 	TRACE("setting ERST size\n");
515 	WriteRunReg32(XHCI_ERSTSZ(0), XHCI_ERSTS_SET(1));
516 
517 	TRACE("setting ERDP addr = 0x%" B_PRIx64 "\n", fErst->rs_addr);
518 	WriteRunReg32(XHCI_ERDP_LO(0), (uint32)fErst->rs_addr);
519 	WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(fErst->rs_addr >> 32));
520 
521 	TRACE("setting ERST base addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress);
522 	WriteRunReg32(XHCI_ERSTBA_LO(0), (uint32)dmaAddress);
523 	WriteRunReg32(XHCI_ERSTBA_HI(0), (uint32)(dmaAddress >> 32));
524 
525 	dmaAddress += sizeof(xhci_erst_element) + XHCI_MAX_EVENTS
526 		* sizeof(xhci_trb);
527 
528 	// Make sure the Command Ring is stopped
529 	if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) {
530 		TRACE_ALWAYS("Command Ring is running, send stop/cancel\n");
531 		WriteOpReg(XHCI_CRCR_LO, CRCR_CS);
532 		WriteOpReg(XHCI_CRCR_HI, 0);
533 		WriteOpReg(XHCI_CRCR_LO, CRCR_CA);
534 		WriteOpReg(XHCI_CRCR_HI, 0);
535 		snooze(1000);
536 		if ((ReadOpReg(XHCI_CRCR_LO) & CRCR_CRR) != 0) {
537 			TRACE_ERROR("Command Ring still running after stop/cancel\n");
538 		}
539 	}
540 	TRACE("setting CRCR addr = 0x%" B_PRIxPHYSADDR "\n", dmaAddress);
541 	WriteOpReg(XHCI_CRCR_LO, (uint32)dmaAddress | CRCR_RCS);
542 	WriteOpReg(XHCI_CRCR_HI, (uint32)(dmaAddress >> 32));
543 	// link trb
544 	fCmdRing[XHCI_MAX_COMMANDS - 1].qwtrb0 = dmaAddress;
545 
546 	TRACE("setting interrupt rate\n");
547 
548 	// Setting IMOD below 0x3F8 on Intel Lynx Point can cause IRQ lockups
549 	if (fPCIInfo->vendor_id == PCI_VENDOR_INTEL
550 		&& (fPCIInfo->device_id == PCI_DEVICE_INTEL_PANTHER_POINT_XHCI
551 			|| fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_XHCI
552 			|| fPCIInfo->device_id == PCI_DEVICE_INTEL_LYNX_POINT_LP_XHCI
553 			|| fPCIInfo->device_id == PCI_DEVICE_INTEL_BAYTRAIL_XHCI
554 			|| fPCIInfo->device_id == PCI_DEVICE_INTEL_WILDCAT_POINT_XHCI)) {
555 		WriteRunReg32(XHCI_IMOD(0), 0x000003f8); // 4000 irq/s
556 	} else {
557 		WriteRunReg32(XHCI_IMOD(0), 0x000001f4); // 8000 irq/s
558 	}
559 
560 	TRACE("enabling interrupt\n");
561 	WriteRunReg32(XHCI_IMAN(0), ReadRunReg32(XHCI_IMAN(0)) | IMAN_INTR_ENA);
562 
563 	WriteOpReg(XHCI_CMD, CMD_RUN | CMD_INTE | CMD_HSEE);
564 
565 	// wait for start up state
566 	if (WaitOpBits(XHCI_STS, STS_HCH, 0) != B_OK) {
567 		TRACE_ERROR("HCH start up timeout\n");
568 	}
569 
570 	fRootHubAddress = AllocateAddress();
571 	fRootHub = new(std::nothrow) XHCIRootHub(RootObject(), fRootHubAddress);
572 	if (!fRootHub) {
573 		TRACE_ERROR("no memory to allocate root hub\n");
574 		return B_NO_MEMORY;
575 	}
576 
577 	if (fRootHub->InitCheck() < B_OK) {
578 		TRACE_ERROR("root hub failed init check\n");
579 		return fRootHub->InitCheck();
580 	}
581 
582 	SetRootHub(fRootHub);
583 
584 	TRACE_ALWAYS("successfully started the controller\n");
585 #ifdef TRACE_USB
586 	TRACE("No-Op test...\n");
587 	status_t noopResult = Noop();
588 	TRACE("No-Op %ssuccessful\n", noopResult < B_OK ? "un" : "");
589 #endif
590 
591 	//DumpRing(fCmdRing, (XHCI_MAX_COMMANDS - 1));
592 
593 	return BusManager::Start();
594 }
595 
596 
597 status_t
598 XHCI::SubmitTransfer(Transfer *transfer)
599 {
600 	// short circuit the root hub
601 	if (transfer->TransferPipe()->DeviceAddress() == fRootHubAddress)
602 		return fRootHub->ProcessTransfer(this, transfer);
603 
604 	TRACE("SubmitTransfer()\n");
605 	Pipe *pipe = transfer->TransferPipe();
606 	if ((pipe->Type() & USB_OBJECT_ISO_PIPE) != 0)
607 		return B_UNSUPPORTED;
608 	if ((pipe->Type() & USB_OBJECT_CONTROL_PIPE) != 0)
609 		return SubmitControlRequest(transfer);
610 	return SubmitNormalRequest(transfer);
611 }
612 
613 
614 status_t
615 XHCI::SubmitControlRequest(Transfer *transfer)
616 {
617 	Pipe *pipe = transfer->TransferPipe();
618 	usb_request_data *requestData = transfer->RequestData();
619 	bool directionIn = (requestData->RequestType & USB_REQTYPE_DEVICE_IN) != 0;
620 
621 	TRACE("SubmitControlRequest() length %d\n", requestData->Length);
622 
623 	xhci_td *setupDescriptor = CreateDescriptor(requestData->Length);
624 
625 	// set SetupStage
626 	uint8 index = 0;
627 	setupDescriptor->trbs[index].qwtrb0 = 0;
628 	setupDescriptor->trbs[index].dwtrb2 = TRB_2_IRQ(0) | TRB_2_BYTES(8);
629 	setupDescriptor->trbs[index].dwtrb3
630 		= B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_SETUP_STAGE)
631 			| TRB_3_IDT_BIT | TRB_3_CYCLE_BIT);
632 	if (requestData->Length > 0) {
633 		setupDescriptor->trbs[index].dwtrb3 |= B_HOST_TO_LENDIAN_INT32(
634 			directionIn ? TRB_3_TRT_IN : TRB_3_TRT_OUT);
635 	}
636 	memcpy(&setupDescriptor->trbs[index].qwtrb0, requestData,
637 		sizeof(usb_request_data));
638 
639 	index++;
640 
641 	if (requestData->Length > 0) {
642 		// set DataStage if any
643 		setupDescriptor->trbs[index].qwtrb0 = setupDescriptor->buffer_phy[0];
644 		setupDescriptor->trbs[index].dwtrb2 = TRB_2_IRQ(0)
645 			| TRB_2_BYTES(requestData->Length)
646 			| TRB_2_TD_SIZE(transfer->VectorCount());
647 		setupDescriptor->trbs[index].dwtrb3 = B_HOST_TO_LENDIAN_INT32(
648 			TRB_3_TYPE(TRB_TYPE_DATA_STAGE)
649 				| (directionIn ? (TRB_3_DIR_IN | TRB_3_ISP_BIT) : 0)
650 				| TRB_3_CYCLE_BIT);
651 
652 		// TODO copy data for out transfers
653 		index++;
654 	}
655 
656 	// set StatusStage
657 	setupDescriptor->trbs[index].dwtrb2 = TRB_2_IRQ(0);
658 	setupDescriptor->trbs[index].dwtrb3 = B_HOST_TO_LENDIAN_INT32(
659 		TRB_3_TYPE(TRB_TYPE_STATUS_STAGE)
660 			| ((directionIn && requestData->Length > 0) ? 0 : TRB_3_DIR_IN)
661 			| TRB_3_IOC_BIT | TRB_3_CYCLE_BIT);
662 
663 	setupDescriptor->trb_count = index + 1;
664 
665 	xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie();
666 	uint8 id = XHCI_ENDPOINT_ID(pipe);
667 	if (id >= XHCI_MAX_ENDPOINTS) {
668 		TRACE_ERROR("Invalid Endpoint");
669 		return B_BAD_VALUE;
670 	}
671 	status_t status = transfer->InitKernelAccess();
672 	if (status != B_OK)
673 		return status;
674 
675 	setupDescriptor->transfer = transfer;
676 	status = _LinkDescriptorForPipe(setupDescriptor, endpoint);
677 	if (status != B_OK)
678 		return status;
679 	TRACE("SubmitControlRequest() request linked\n");
680 
681 	TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n",
682 		_ReadContext(&endpoint->device->device_ctx->endpoints[id - 1].dwendpoint0),
683 		_ReadContext(&endpoint->device->device_ctx->endpoints[id - 1].dwendpoint1),
684 		_ReadContext(&endpoint->device->device_ctx->endpoints[id - 1].qwendpoint2));
685 	Ring(endpoint->device->slot, id);
686 	TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n",
687 		_ReadContext(&endpoint->device->device_ctx->endpoints[id - 1].dwendpoint0),
688 		_ReadContext(&endpoint->device->device_ctx->endpoints[id - 1].dwendpoint1),
689 		_ReadContext(&endpoint->device->device_ctx->endpoints[id - 1].qwendpoint2));
690 	return B_OK;
691 }
692 
693 
694 status_t
695 XHCI::SubmitNormalRequest(Transfer *transfer)
696 {
697 	TRACE("SubmitNormalRequest() length %ld\n", transfer->DataLength());
698 	Pipe *pipe = transfer->TransferPipe();
699 	uint8 id = XHCI_ENDPOINT_ID(pipe);
700 	if (id >= XHCI_MAX_ENDPOINTS)
701 		return B_BAD_VALUE;
702 	bool directionIn = (pipe->Direction() == Pipe::In);
703 
704 	status_t status = transfer->InitKernelAccess();
705 	if (status != B_OK)
706 		return status;
707 
708 	int32 trbCount = 0;
709 	xhci_td *descriptor = CreateDescriptorChain(transfer->DataLength(), trbCount);
710 	if (descriptor == NULL)
711 		return B_NO_MEMORY;
712 
713 	xhci_td *td_chain = descriptor;
714 	xhci_td *last = descriptor;
715 	int32 rest = trbCount - 1;
716 
717 	// set NormalStage
718 	while (td_chain != NULL) {
719 		td_chain->trb_count = td_chain->buffer_count;
720 		uint8 index;
721 		for (index = 0; index < td_chain->buffer_count; index++) {
722 			td_chain->trbs[index].qwtrb0 = descriptor->buffer_phy[index];
723 			td_chain->trbs[index].dwtrb2 = TRB_2_IRQ(0)
724 				| TRB_2_BYTES(descriptor->buffer_size[index])
725 				| TRB_2_TD_SIZE(rest);
726 			td_chain->trbs[index].dwtrb3 = B_HOST_TO_LENDIAN_INT32(
727 				TRB_3_TYPE(TRB_TYPE_NORMAL) | TRB_3_CYCLE_BIT | TRB_3_CHAIN_BIT
728 					| (directionIn ? TRB_3_ISP_BIT : 0));
729 			rest--;
730 		}
731 		// link next td, if any
732 		if (td_chain->next_chain != NULL) {
733 			td_chain->trbs[td_chain->trb_count].qwtrb0 = td_chain->next_chain->this_phy;
734 			td_chain->trbs[td_chain->trb_count].dwtrb2 = TRB_2_IRQ(0);
735 			td_chain->trbs[td_chain->trb_count].dwtrb3
736 				= B_HOST_TO_LENDIAN_INT32(TRB_3_TYPE(TRB_TYPE_LINK)
737 					| TRB_3_CYCLE_BIT | TRB_3_CHAIN_BIT);
738 		}
739 
740 		last = td_chain;
741 		td_chain = td_chain->next_chain;
742 	}
743 
744 	if (last->trb_count > 0) {
745 		last->trbs[last->trb_count - 1].dwtrb3
746 			|= B_HOST_TO_LENDIAN_INT32(TRB_3_IOC_BIT);
747 		last->trbs[last->trb_count - 1].dwtrb3
748 			&= B_HOST_TO_LENDIAN_INT32(~TRB_3_CHAIN_BIT);
749 	}
750 
751 	if (!directionIn) {
752 		TRACE("copying out iov count %ld\n", transfer->VectorCount());
753 		transfer->PrepareKernelAccess();
754 		WriteDescriptorChain(descriptor, transfer->Vector(),
755 			transfer->VectorCount());
756 	}
757 
758 	xhci_endpoint *endpoint = (xhci_endpoint *)pipe->ControllerCookie();
759 	descriptor->transfer = transfer;
760 	status = _LinkDescriptorForPipe(descriptor, endpoint);
761 	if (status != B_OK)
762 		return status;
763 	TRACE("SubmitNormalRequest() request linked\n");
764 
765 	TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n",
766 		_ReadContext(&endpoint->device->device_ctx->endpoints[id - 1].dwendpoint0),
767 		_ReadContext(&endpoint->device->device_ctx->endpoints[id - 1].dwendpoint1),
768 		_ReadContext(&endpoint->device->device_ctx->endpoints[id - 1].qwendpoint2));
769 	Ring(endpoint->device->slot, id);
770 	TRACE("Endpoint status 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%016" B_PRIx64 "\n",
771 		_ReadContext(&endpoint->device->device_ctx->endpoints[id - 1].dwendpoint0),
772 		_ReadContext(&endpoint->device->device_ctx->endpoints[id - 1].dwendpoint1),
773 		_ReadContext(&endpoint->device->device_ctx->endpoints[id - 1].qwendpoint2));
774 	return B_OK;
775 }
776 
777 
778 status_t
779 XHCI::CancelQueuedTransfers(Pipe *pipe, bool force)
780 {
781 	TRACE_ALWAYS("cancel queued transfers for pipe %p (%d)\n", pipe,
782 		pipe->EndpointAddress());
783 	return B_OK;
784 }
785 
786 
787 status_t
788 XHCI::NotifyPipeChange(Pipe *pipe, usb_change change)
789 {
790 	TRACE("pipe change %d for pipe %p (%d)\n", change, pipe,
791 		pipe->EndpointAddress());
792 	switch (change) {
793 		case USB_CHANGE_CREATED:
794 			_InsertEndpointForPipe(pipe);
795 			break;
796 		case USB_CHANGE_DESTROYED:
797 			_RemoveEndpointForPipe(pipe);
798 			break;
799 
800 		case USB_CHANGE_PIPE_POLICY_CHANGED: {
801 			// ToDo: for isochronous pipes we might need to adapt to new
802 			// pipe policy settings here
803 			break;
804 		}
805 	}
806 
807 	return B_OK;
808 }
809 
810 
811 status_t
812 XHCI::AddTo(Stack *stack)
813 {
814 #ifdef TRACE_USB
815 	set_dprintf_enabled(true);
816 #endif
817 
818 	if (!sPCIModule) {
819 		status_t status = get_module(B_PCI_MODULE_NAME,
820 			(module_info **)&sPCIModule);
821 		if (status < B_OK) {
822 			TRACE_MODULE_ERROR("getting pci module failed! 0x%08" B_PRIx32
823 				"\n", status);
824 			return status;
825 		}
826 	}
827 
828 	TRACE_MODULE("searching devices\n");
829 	bool found = false;
830 	pci_info *item = new(std::nothrow) pci_info;
831 	if (!item) {
832 		sPCIModule = NULL;
833 		put_module(B_PCI_MODULE_NAME);
834 		return B_NO_MEMORY;
835 	}
836 
837 	// Try to get the PCI x86 module as well so we can enable possible MSIs.
838 	if (sPCIx86Module == NULL && get_module(B_PCI_X86_MODULE_NAME,
839 			(module_info **)&sPCIx86Module) != B_OK) {
840 		// If it isn't there, that's not critical though.
841 		TRACE_MODULE_ERROR("failed to get pci x86 module\n");
842 		sPCIx86Module = NULL;
843 	}
844 
845 	for (int32 i = 0; sPCIModule->get_nth_pci_info(i, item) >= B_OK; i++) {
846 		if (item->class_base == PCI_serial_bus && item->class_sub == PCI_usb
847 			&& item->class_api == PCI_usb_xhci) {
848 			TRACE_MODULE("found device at PCI:%d:%d:%d\n",
849 				item->bus, item->device, item->function);
850 			XHCI *bus = new(std::nothrow) XHCI(item, stack);
851 			if (!bus) {
852 				delete item;
853 				sPCIModule = NULL;
854 				put_module(B_PCI_MODULE_NAME);
855 				return B_NO_MEMORY;
856 			}
857 
858 			if (bus->InitCheck() < B_OK) {
859 				TRACE_MODULE_ERROR("bus failed init check\n");
860 				delete bus;
861 				continue;
862 			}
863 
864 			// the bus took it away
865 			item = new(std::nothrow) pci_info;
866 
867 			if (bus->Start() != B_OK) {
868 				delete bus;
869 				continue;
870 			}
871 			found = true;
872 		}
873 	}
874 
875 	if (!found) {
876 		TRACE_MODULE_ERROR("no devices found\n");
877 		delete item;
878 		sPCIModule = NULL;
879 		put_module(B_PCI_MODULE_NAME);
880 		return ENODEV;
881 	}
882 
883 	delete item;
884 	return B_OK;
885 }
886 
887 
888 xhci_td *
889 XHCI::CreateDescriptorChain(size_t bufferSize, int32 &trbCount)
890 {
891 	size_t packetSize = B_PAGE_SIZE * 16;
892 	trbCount = (bufferSize + packetSize - 1) / packetSize;
893 	if (trbCount == 0)
894 		trbCount = 1;
895 
896 	// keep one trb for linking
897 	int32 tdCount = (trbCount + XHCI_MAX_TRBS_PER_TD - 2)
898 		/ (XHCI_MAX_TRBS_PER_TD - 1);
899 
900 	xhci_td *first = NULL;
901 	xhci_td *last = NULL;
902 	for (int32 i = 0; i < tdCount; i++) {
903 		xhci_td *descriptor = CreateDescriptor(0);
904 		if (!descriptor) {
905 			if (first != NULL)
906 				FreeDescriptor(first);
907 			return NULL;
908 		} else if (first == NULL)
909 			first = descriptor;
910 
911 		uint8 trbs = min_c(trbCount, XHCI_MAX_TRBS_PER_TD - 1);
912 		TRACE("CreateDescriptorChain trbs %d for td %" B_PRId32 "\n", trbs, i);
913 		for (int j = 0; j < trbs; j++) {
914 			if (fStack->AllocateChunk(&descriptor->buffer_log[j],
915 				&descriptor->buffer_phy[j],
916 				min_c(packetSize, bufferSize)) < B_OK) {
917 				TRACE_ERROR("unable to allocate space for the buffer (size %"
918 					B_PRIuSIZE ")\n", bufferSize);
919 				return NULL;
920 			}
921 
922 			descriptor->buffer_size[j] = min_c(packetSize, bufferSize);
923 			bufferSize -= descriptor->buffer_size[j];
924 			TRACE("CreateDescriptorChain allocated %ld for trb %d\n",
925 				descriptor->buffer_size[j], j);
926 		}
927 
928 		descriptor->buffer_count = trbs;
929 		trbCount -= trbs;
930 		if (last != NULL)
931 			last->next_chain = descriptor;
932 		last = descriptor;
933 	}
934 
935 	return first;
936 }
937 
938 
939 xhci_td *
940 XHCI::CreateDescriptor(size_t bufferSize)
941 {
942 	xhci_td *result;
943 	phys_addr_t physicalAddress;
944 
945 	if (fStack->AllocateChunk((void **)&result, &physicalAddress,
946 		sizeof(xhci_td)) < B_OK) {
947 		TRACE_ERROR("failed to allocate a transfer descriptor\n");
948 		return NULL;
949 	}
950 
951 	result->this_phy = physicalAddress;
952 	result->buffer_size[0] = bufferSize;
953 	result->trb_count = 0;
954 	result->buffer_count = 1;
955 	result->next = NULL;
956 	result->next_chain = NULL;
957 	if (bufferSize <= 0) {
958 		result->buffer_log[0] = NULL;
959 		result->buffer_phy[0] = 0;
960 		return result;
961 	}
962 
963 	if (fStack->AllocateChunk(&result->buffer_log[0],
964 		&result->buffer_phy[0], bufferSize) < B_OK) {
965 		TRACE_ERROR("unable to allocate space for the buffer (size %ld)\n",
966 			bufferSize);
967 		fStack->FreeChunk(result, result->this_phy, sizeof(xhci_td));
968 		return NULL;
969 	}
970 
971 	TRACE("CreateDescriptor allocated buffer_size %ld %p\n",
972 				result->buffer_size[0], result->buffer_log[0]);
973 
974 	return result;
975 }
976 
977 
978 void
979 XHCI::FreeDescriptor(xhci_td *descriptor)
980 {
981 	while (descriptor != NULL) {
982 
983 		for (int i = 0; i < descriptor->buffer_count; i++) {
984 			if (descriptor->buffer_size[i] == 0)
985 				continue;
986 			TRACE("FreeDescriptor buffer %d buffer_size %ld %p\n", i,
987 				descriptor->buffer_size[i], descriptor->buffer_log[i]);
988 			fStack->FreeChunk(descriptor->buffer_log[i],
989 				descriptor->buffer_phy[i], descriptor->buffer_size[i]);
990 		}
991 
992 		xhci_td *next = descriptor->next_chain;
993 		fStack->FreeChunk(descriptor, descriptor->this_phy,
994 			sizeof(xhci_td));
995 		descriptor = next;
996 	}
997 }
998 
999 
1000 size_t
1001 XHCI::WriteDescriptorChain(xhci_td *descriptor, iovec *vector,
1002 	size_t vectorCount)
1003 {
1004 	xhci_td *current = descriptor;
1005 	uint8 trbIndex = 0;
1006 	size_t actualLength = 0;
1007 	uint8 vectorIndex = 0;
1008 	size_t vectorOffset = 0;
1009 	size_t bufferOffset = 0;
1010 
1011 	while (current != NULL) {
1012 		if (current->buffer_log[0] == NULL)
1013 			break;
1014 
1015 		while (true) {
1016 			size_t length = min_c(current->buffer_size[trbIndex] - bufferOffset,
1017 				vector[vectorIndex].iov_len - vectorOffset);
1018 
1019 			TRACE("copying %ld bytes to bufferOffset %ld from"
1020 				" vectorOffset %ld at index %d of %ld\n", length, bufferOffset,
1021 				vectorOffset, vectorIndex, vectorCount);
1022 			memcpy((uint8 *)current->buffer_log[trbIndex] + bufferOffset,
1023 				(uint8 *)vector[vectorIndex].iov_base + vectorOffset, length);
1024 
1025 			actualLength += length;
1026 			vectorOffset += length;
1027 			bufferOffset += length;
1028 
1029 			if (vectorOffset >= vector[vectorIndex].iov_len) {
1030 				if (++vectorIndex >= vectorCount) {
1031 					TRACE("wrote descriptor chain (%ld bytes, no more vectors)\n",
1032 						actualLength);
1033 					return actualLength;
1034 				}
1035 
1036 				vectorOffset = 0;
1037 			}
1038 
1039 			if (bufferOffset >= current->buffer_size[trbIndex]) {
1040 				bufferOffset = 0;
1041 				if (++trbIndex >= current->buffer_count)
1042 					break;
1043 			}
1044 		}
1045 
1046 		current = current->next_chain;
1047 		trbIndex = 0;
1048 	}
1049 
1050 	TRACE("wrote descriptor chain (%ld bytes)\n", actualLength);
1051 	return actualLength;
1052 }
1053 
1054 
1055 size_t
1056 XHCI::ReadDescriptorChain(xhci_td *descriptor, iovec *vector,
1057 	size_t vectorCount)
1058 {
1059 	xhci_td *current = descriptor;
1060 	uint8 trbIndex = 0;
1061 	size_t actualLength = 0;
1062 	uint8 vectorIndex = 0;
1063 	size_t vectorOffset = 0;
1064 	size_t bufferOffset = 0;
1065 
1066 	while (current != NULL) {
1067 		if (current->buffer_log[0] == NULL)
1068 			break;
1069 
1070 		while (true) {
1071 			size_t length = min_c(current->buffer_size[trbIndex] - bufferOffset,
1072 				vector[vectorIndex].iov_len - vectorOffset);
1073 
1074 			TRACE("copying %ld bytes to vectorOffset %ld from"
1075 				" bufferOffset %ld at index %d of %ld\n", length, vectorOffset,
1076 				bufferOffset, vectorIndex, vectorCount);
1077 			memcpy((uint8 *)vector[vectorIndex].iov_base + vectorOffset,
1078 				(uint8 *)current->buffer_log[trbIndex] + bufferOffset, length);
1079 
1080 			actualLength += length;
1081 			vectorOffset += length;
1082 			bufferOffset += length;
1083 
1084 			if (vectorOffset >= vector[vectorIndex].iov_len) {
1085 				if (++vectorIndex >= vectorCount) {
1086 					TRACE("read descriptor chain (%ld bytes, no more vectors)\n",
1087 						actualLength);
1088 					return actualLength;
1089 				}
1090 				vectorOffset = 0;
1091 
1092 			}
1093 
1094 			if (bufferOffset >= current->buffer_size[trbIndex]) {
1095 				bufferOffset = 0;
1096 				if (++trbIndex >= current->buffer_count)
1097 					break;
1098 			}
1099 		}
1100 
1101 		current = current->next_chain;
1102 		trbIndex = 0;
1103 	}
1104 
1105 	TRACE("read descriptor chain (%ld bytes)\n", actualLength);
1106 	return actualLength;
1107 }
1108 
1109 
1110 Device *
1111 XHCI::AllocateDevice(Hub *parent, int8 hubAddress, uint8 hubPort,
1112 	usb_speed speed)
1113 {
1114 	TRACE("AllocateDevice hubAddress %d hubPort %d speed %d\n", hubAddress,
1115 		hubPort, speed);
1116 
1117 	uint8 slot = XHCI_MAX_SLOTS;
1118 	if (EnableSlot(&slot) != B_OK) {
1119 		TRACE_ERROR("AllocateDevice() failed enable slot\n");
1120 		return NULL;
1121 	}
1122 
1123 	if (slot == 0 || slot > fSlotCount) {
1124 		TRACE_ERROR("AllocateDevice() bad slot\n");
1125 		return NULL;
1126 	}
1127 
1128 	if (fDevices[slot].state != XHCI_STATE_DISABLED) {
1129 		TRACE_ERROR("AllocateDevice() slot already used\n");
1130 		return NULL;
1131 	}
1132 
1133 	struct xhci_device *device = &fDevices[slot];
1134 	memset(device, 0, sizeof(struct xhci_device));
1135 	device->state = XHCI_STATE_ENABLED;
1136 	device->slot = slot;
1137 
1138 	device->input_ctx_area = fStack->AllocateArea((void **)&device->input_ctx,
1139 		&device->input_ctx_addr, sizeof(*device->input_ctx) << fContextSizeShift,
1140 		"XHCI input context");
1141 	if (device->input_ctx_area < B_OK) {
1142 		TRACE_ERROR("unable to create a input context area\n");
1143 		device->state = XHCI_STATE_DISABLED;
1144 		return NULL;
1145 	}
1146 
1147 	memset(device->input_ctx, 0, sizeof(*device->input_ctx) << fContextSizeShift);
1148 	_WriteContext(&device->input_ctx->input.dropFlags, 0);
1149 	_WriteContext(&device->input_ctx->input.addFlags, 3);
1150 
1151 	uint32 route = 0;
1152 	uint8 routePort = hubPort;
1153 	uint8 rhPort = hubPort;
1154 	for (Device *hubDevice = parent; hubDevice != RootObject();
1155 		hubDevice = (Device *)hubDevice->Parent()) {
1156 
1157 		rhPort = routePort;
1158 		if (hubDevice->Parent() == RootObject())
1159 			break;
1160 		route *= 16;
1161 		if (hubPort > 15)
1162 			route += 15;
1163 		else
1164 			route += routePort;
1165 
1166 		routePort = hubDevice->HubPort();
1167 	}
1168 
1169 	// Get speed of port, only if device connected to root hub port
1170 	// else we have to rely on value reported by the Hub Explore thread
1171 	if (route == 0) {
1172 		GetPortSpeed(hubPort - 1, &speed);
1173 		TRACE("speed updated %d\n", speed);
1174 	}
1175 
1176 	uint32 dwslot0 = SLOT_0_NUM_ENTRIES(1) | SLOT_0_ROUTE(route);
1177 
1178 	// add the speed
1179 	switch (speed) {
1180 	case USB_SPEED_LOWSPEED:
1181 		dwslot0 |= SLOT_0_SPEED(2);
1182 		break;
1183 	case USB_SPEED_HIGHSPEED:
1184 		dwslot0 |= SLOT_0_SPEED(3);
1185 		break;
1186 	case USB_SPEED_FULLSPEED:
1187 		dwslot0 |= SLOT_0_SPEED(1);
1188 		break;
1189 	case USB_SPEED_SUPER:
1190 		dwslot0 |= SLOT_0_SPEED(4);
1191 		break;
1192 	default:
1193 		TRACE_ERROR("unknown usb speed\n");
1194 		break;
1195 	}
1196 
1197 	_WriteContext(&device->input_ctx->slot.dwslot0, dwslot0);
1198 	// TODO enable power save
1199 	_WriteContext(&device->input_ctx->slot.dwslot1, SLOT_1_RH_PORT(rhPort));
1200 	uint32 dwslot2 = SLOT_2_IRQ_TARGET(0);
1201 
1202 	// If LS/FS device connected to non-root HS device
1203 	if (route != 0 && parent->Speed() == USB_SPEED_HIGHSPEED
1204 		&& (speed == USB_SPEED_LOWSPEED || speed == USB_SPEED_FULLSPEED)) {
1205 		struct xhci_device *parenthub = (struct xhci_device *)
1206 			parent->ControllerCookie();
1207 		dwslot2 |= SLOT_2_PORT_NUM(hubPort);
1208 		dwslot2 |= SLOT_2_TT_HUB_SLOT(parenthub->slot);
1209 	}
1210 
1211 	_WriteContext(&device->input_ctx->slot.dwslot2, dwslot2);
1212 
1213 	_WriteContext(&device->input_ctx->slot.dwslot3, SLOT_3_SLOT_STATE(0)
1214 		| SLOT_3_DEVICE_ADDRESS(0));
1215 
1216 	TRACE("slot 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32 " 0x%08" B_PRIx32
1217 		"\n", _ReadContext(&device->input_ctx->slot.dwslot0),
1218 		_ReadContext(&device->input_ctx->slot.dwslot1),
1219 		_ReadContext(&device->input_ctx->slot.dwslot2),
1220 		_ReadContext(&device->input_ctx->slot.dwslot3));
1221 
1222 	device->device_ctx_area = fStack->AllocateArea((void **)&device->device_ctx,
1223 		&device->device_ctx_addr, sizeof(*device->device_ctx) << fContextSizeShift,
1224 		"XHCI device context");
1225 	if (device->device_ctx_area < B_OK) {
1226 		TRACE_ERROR("unable to create a device context area\n");
1227 		device->state = XHCI_STATE_DISABLED;
1228 		delete_area(device->input_ctx_area);
1229 		return NULL;
1230 	}
1231 	memset(device->device_ctx, 0, sizeof(*device->device_ctx) << fContextSizeShift);
1232 
1233 	device->trb_area = fStack->AllocateArea((void **)&device->trbs,
1234 		&device->trb_addr, sizeof(*device->trbs) * (XHCI_MAX_ENDPOINTS - 1)
1235 			* XHCI_MAX_TRANSFERS, "XHCI endpoint trbs");
1236 	if (device->trb_area < B_OK) {
1237 		TRACE_ERROR("unable to create a device trbs area\n");
1238 		device->state = XHCI_STATE_DISABLED;
1239 		delete_area(device->input_ctx_area);
1240 		delete_area(device->device_ctx_area);
1241 		return NULL;
1242 	}
1243 
1244 	// set up slot pointer to device context
1245 	fDcba->baseAddress[slot] = device->device_ctx_addr;
1246 
1247 	size_t maxPacketSize;
1248 	switch (speed) {
1249 	case USB_SPEED_LOWSPEED:
1250 	case USB_SPEED_FULLSPEED:
1251 		maxPacketSize = 8;
1252 		break;
1253 	case USB_SPEED_HIGHSPEED:
1254 		maxPacketSize = 64;
1255 		break;
1256 	default:
1257 		maxPacketSize = 512;
1258 		break;
1259 	}
1260 
1261 	// configure the Control endpoint 0 (type 4)
1262 	if (ConfigureEndpoint(slot, 0, 4, device->trb_addr, 0,
1263 			maxPacketSize, maxPacketSize & 0x7ff, speed) != B_OK) {
1264 		TRACE_ERROR("unable to configure default control endpoint\n");
1265 		device->state = XHCI_STATE_DISABLED;
1266 		delete_area(device->input_ctx_area);
1267 		delete_area(device->device_ctx_area);
1268 		delete_area(device->trb_area);
1269 		return NULL;
1270 	}
1271 
1272 	device->endpoints[0].device = device;
1273 	device->endpoints[0].td_head = NULL;
1274 	device->endpoints[0].trbs = device->trbs;
1275 	device->endpoints[0].used = 0;
1276 	device->endpoints[0].current = 0;
1277 	device->endpoints[0].trb_addr = device->trb_addr;
1278 	mutex_init(&device->endpoints[0].lock, "xhci endpoint lock");
1279 
1280 	// device should get to addressed state (bsr = 0)
1281 	if (SetAddress(device->input_ctx_addr, false, slot) != B_OK) {
1282 		TRACE_ERROR("unable to set address\n");
1283 		device->state = XHCI_STATE_DISABLED;
1284 		delete_area(device->input_ctx_area);
1285 		delete_area(device->device_ctx_area);
1286 		delete_area(device->trb_area);
1287 		return NULL;
1288 	}
1289 
1290 	device->state = XHCI_STATE_ADDRESSED;
1291 	device->address = SLOT_3_DEVICE_ADDRESS_GET(_ReadContext(
1292 		&device->device_ctx->slot.dwslot3));
1293 
1294 	TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n", device->address,
1295 		SLOT_3_SLOT_STATE_GET(_ReadContext(
1296 			&device->device_ctx->slot.dwslot3)));
1297 	TRACE("endpoint0 state 0x%08" B_PRIx32 "\n",
1298 		ENDPOINT_0_STATE_GET(_ReadContext(
1299 			&device->device_ctx->endpoints[0].dwendpoint0)));
1300 
1301 	// Create a temporary pipe with the new address
1302 	ControlPipe pipe(parent);
1303 	pipe.SetControllerCookie(&device->endpoints[0]);
1304 	pipe.InitCommon(device->address + 1, 0, speed, Pipe::Default, maxPacketSize, 0,
1305 		hubAddress, hubPort);
1306 
1307 	// Get the device descriptor
1308 	// Just retrieve the first 8 bytes of the descriptor -> minimum supported
1309 	// size of any device. It is enough because it includes the device type.
1310 
1311 	size_t actualLength = 0;
1312 	usb_device_descriptor deviceDescriptor;
1313 
1314 	TRACE("getting the device descriptor\n");
1315 	pipe.SendRequest(
1316 		USB_REQTYPE_DEVICE_IN | USB_REQTYPE_STANDARD,		// type
1317 		USB_REQUEST_GET_DESCRIPTOR,							// request
1318 		USB_DESCRIPTOR_DEVICE << 8,							// value
1319 		0,													// index
1320 		8,													// length
1321 		(void *)&deviceDescriptor,							// buffer
1322 		8,													// buffer length
1323 		&actualLength);										// actual length
1324 
1325 	if (actualLength != 8) {
1326 		TRACE_ERROR("error while getting the device descriptor\n");
1327 		device->state = XHCI_STATE_DISABLED;
1328 		delete_area(device->input_ctx_area);
1329 		delete_area(device->device_ctx_area);
1330 		delete_area(device->trb_area);
1331 		return NULL;
1332 	}
1333 
1334 	TRACE("device_class: %d device_subclass %d device_protocol %d\n",
1335 		deviceDescriptor.device_class, deviceDescriptor.device_subclass,
1336 		deviceDescriptor.device_protocol);
1337 
1338 	if (speed == USB_SPEED_FULLSPEED && deviceDescriptor.max_packet_size_0 != 8) {
1339 		TRACE("Full speed device with different max packet size for Endpoint 0\n");
1340 		uint32 dwendpoint1 = _ReadContext(
1341 			&device->input_ctx->endpoints[0].dwendpoint1);
1342 		dwendpoint1 &= ~ENDPOINT_1_MAXPACKETSIZE(0xffff);
1343 		dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE(
1344 			deviceDescriptor.max_packet_size_0);
1345 		_WriteContext(&device->input_ctx->endpoints[0].dwendpoint1,
1346 			dwendpoint1);
1347 		_WriteContext(&device->input_ctx->input.dropFlags, 0);
1348 		_WriteContext(&device->input_ctx->input.addFlags, (1 << 1));
1349 		EvaluateContext(device->input_ctx_addr, device->slot);
1350 	}
1351 
1352 	Device *deviceObject = NULL;
1353 	if (deviceDescriptor.device_class == 0x09) {
1354 		TRACE("creating new Hub\n");
1355 		TRACE("getting the hub descriptor\n");
1356 		size_t actualLength = 0;
1357 		usb_hub_descriptor hubDescriptor;
1358 		pipe.SendRequest(
1359 			USB_REQTYPE_DEVICE_IN | USB_REQTYPE_CLASS,			// type
1360 			USB_REQUEST_GET_DESCRIPTOR,							// request
1361 			USB_DESCRIPTOR_HUB << 8,							// value
1362 			0,													// index
1363 			sizeof(usb_hub_descriptor),							// length
1364 			(void *)&hubDescriptor,								// buffer
1365 			sizeof(usb_hub_descriptor),							// buffer length
1366 			&actualLength);
1367 
1368 		if (actualLength != sizeof(usb_hub_descriptor)) {
1369 			TRACE_ERROR("error while getting the hub descriptor\n");
1370 			device->state = XHCI_STATE_DISABLED;
1371 			delete_area(device->input_ctx_area);
1372 			delete_area(device->device_ctx_area);
1373 			delete_area(device->trb_area);
1374 			return NULL;
1375 		}
1376 
1377 		uint32 dwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0);
1378 		dwslot0 |= SLOT_0_HUB_BIT;
1379 		_WriteContext(&device->input_ctx->slot.dwslot0, dwslot0);
1380 		uint32 dwslot1 = _ReadContext(&device->input_ctx->slot.dwslot1);
1381 		dwslot1 |= SLOT_1_NUM_PORTS(hubDescriptor.num_ports);
1382 		_WriteContext(&device->input_ctx->slot.dwslot1, dwslot1);
1383 		if (speed == USB_SPEED_HIGHSPEED) {
1384 			uint32 dwslot2 = _ReadContext(&device->input_ctx->slot.dwslot2);
1385 			dwslot2 |= SLOT_2_TT_TIME(HUB_TTT_GET(hubDescriptor.characteristics));
1386 			_WriteContext(&device->input_ctx->slot.dwslot2, dwslot2);
1387 		}
1388 
1389 		deviceObject = new(std::nothrow) Hub(parent, hubAddress, hubPort,
1390 			deviceDescriptor, device->address + 1, speed, false, device);
1391 	} else {
1392 		TRACE("creating new device\n");
1393 		deviceObject = new(std::nothrow) Device(parent, hubAddress, hubPort,
1394 			deviceDescriptor, device->address + 1, speed, false, device);
1395 	}
1396 	if (deviceObject == NULL) {
1397 		TRACE_ERROR("no memory to allocate device\n");
1398 		device->state = XHCI_STATE_DISABLED;
1399 		delete_area(device->input_ctx_area);
1400 		delete_area(device->device_ctx_area);
1401 		delete_area(device->trb_area);
1402 		return NULL;
1403 	}
1404 	fPortSlots[hubPort] = slot;
1405 	TRACE("AllocateDevice() port %d slot %d\n", hubPort, slot);
1406 	return deviceObject;
1407 }
1408 
1409 
1410 void
1411 XHCI::FreeDevice(Device *device)
1412 {
1413 	uint8 slot = fPortSlots[device->HubPort()];
1414 	TRACE("FreeDevice() port %d slot %d\n", device->HubPort(), slot);
1415 	DisableSlot(slot);
1416 	fDcba->baseAddress[slot] = 0;
1417 	fPortSlots[device->HubPort()] = 0;
1418 	delete_area(fDevices[slot].trb_area);
1419 	delete_area(fDevices[slot].input_ctx_area);
1420 	delete_area(fDevices[slot].device_ctx_area);
1421 	fDevices[slot].state = XHCI_STATE_DISABLED;
1422 	delete device;
1423 }
1424 
1425 
1426 status_t
1427 XHCI::_InsertEndpointForPipe(Pipe *pipe)
1428 {
1429 	TRACE("_InsertEndpointForPipe endpoint address %" B_PRId8 "\n",
1430 		pipe->EndpointAddress());
1431 	if (pipe->ControllerCookie() != NULL
1432 		|| pipe->Parent()->Type() != USB_OBJECT_DEVICE) {
1433 		// default pipe is already referenced
1434 		return B_OK;
1435 	}
1436 
1437 	Device* usbDevice = (Device *)pipe->Parent();
1438 	struct xhci_device *device = (struct xhci_device *)
1439 		usbDevice->ControllerCookie();
1440 	if (usbDevice->Parent() == RootObject())
1441 		return B_OK;
1442 	if (device == NULL) {
1443 		panic("_InsertEndpointForPipe device is NULL\n");
1444 		return B_OK;
1445 	}
1446 
1447 	uint8 id = XHCI_ENDPOINT_ID(pipe) - 1;
1448 	if (id >= XHCI_MAX_ENDPOINTS - 1)
1449 		return B_BAD_VALUE;
1450 
1451 	if (id > 0) {
1452 		uint32 devicedwslot0 = _ReadContext(&device->device_ctx->slot.dwslot0);
1453 		if (SLOT_0_NUM_ENTRIES_GET(devicedwslot0) == 1) {
1454 			uint32 inputdwslot0 = _ReadContext(&device->input_ctx->slot.dwslot0);
1455 			inputdwslot0 &= ~(SLOT_0_NUM_ENTRIES(0x1f));
1456 			inputdwslot0 |= SLOT_0_NUM_ENTRIES(XHCI_MAX_ENDPOINTS - 1);
1457 			_WriteContext(&device->input_ctx->slot.dwslot0, inputdwslot0);
1458 			EvaluateContext(device->input_ctx_addr, device->slot);
1459 		}
1460 
1461 		device->endpoints[id].device = device;
1462 		device->endpoints[id].trbs = device->trbs
1463 			+ id * XHCI_MAX_TRANSFERS;
1464 		device->endpoints[id].td_head = NULL;
1465 		device->endpoints[id].used = 0;
1466 		device->endpoints[id].trb_addr = device->trb_addr
1467 			+ id * XHCI_MAX_TRANSFERS * sizeof(xhci_trb);
1468 		mutex_init(&device->endpoints[id].lock, "xhci endpoint lock");
1469 
1470 		TRACE("_InsertEndpointForPipe trbs device %p endpoint %p\n",
1471 			device->trbs, device->endpoints[id].trbs);
1472 		TRACE("_InsertEndpointForPipe trb_addr device 0x%" B_PRIxPHYSADDR
1473 			" endpoint 0x%" B_PRIxPHYSADDR "\n", device->trb_addr,
1474 			device->endpoints[id].trb_addr);
1475 
1476 		uint8 endpoint = id + 1;
1477 
1478 		// configure the Control endpoint 0 (type 4)
1479 		uint32 type = 4;
1480 		if ((pipe->Type() & USB_OBJECT_INTERRUPT_PIPE) != 0)
1481 			type = 3;
1482 		if ((pipe->Type() & USB_OBJECT_BULK_PIPE) != 0)
1483 			type = 2;
1484 		if ((pipe->Type() & USB_OBJECT_ISO_PIPE) != 0)
1485 			type = 1;
1486 		type |= (pipe->Direction() == Pipe::In) ? (1 << 2) : 0;
1487 
1488 		TRACE("trb_addr 0x%" B_PRIxPHYSADDR "\n", device->endpoints[id].trb_addr);
1489 
1490 		status_t status = ConfigureEndpoint(device->slot, id, type,
1491 			device->endpoints[id].trb_addr, pipe->Interval(),
1492 			pipe->MaxPacketSize(), pipe->MaxPacketSize() & 0x7ff,
1493 			usbDevice->Speed());
1494 		if (status != B_OK) {
1495 			TRACE_ERROR("unable to configure endpoint\n");
1496 			return status;
1497 		}
1498 
1499 #if 0
1500 		/* These commands error with "Context state" on some devices,
1501 		 * and on others break transfers altogether. So just disable them
1502 		 * for now. */
1503 		StopEndpoint(false, endpoint, device->slot);
1504 		SetTRDequeue(device->endpoints[id].trb_addr, 0, endpoint,
1505 			device->slot);
1506 		ResetEndpoint(false, endpoint, device->slot);
1507 #endif
1508 
1509 		_WriteContext(&device->input_ctx->input.dropFlags, 0);
1510 		_WriteContext(&device->input_ctx->input.addFlags,
1511 			(1 << endpoint) | (1 << 0));
1512 
1513 		if (endpoint > 1)
1514 			ConfigureEndpoint(device->input_ctx_addr, false, device->slot);
1515 		else
1516 			EvaluateContext(device->input_ctx_addr, device->slot);
1517 
1518 		TRACE("device: address 0x%x state 0x%08" B_PRIx32 "\n",
1519 			device->address, SLOT_3_SLOT_STATE_GET(_ReadContext(
1520 				&device->device_ctx->slot.dwslot3)));
1521 		TRACE("endpoint[0] state 0x%08" B_PRIx32 "\n",
1522 			ENDPOINT_0_STATE_GET(_ReadContext(
1523 				&device->device_ctx->endpoints[0].dwendpoint0)));
1524 		TRACE("endpoint[%d] state 0x%08" B_PRIx32 "\n", id,
1525 			ENDPOINT_0_STATE_GET(_ReadContext(
1526 				&device->device_ctx->endpoints[id].dwendpoint0)));
1527 
1528 		device->state = XHCI_STATE_CONFIGURED;
1529 	}
1530 	pipe->SetControllerCookie(&device->endpoints[id]);
1531 
1532 	TRACE("_InsertEndpointForPipe for pipe %p at id %d\n", pipe, id);
1533 
1534 	return B_OK;
1535 }
1536 
1537 
1538 status_t
1539 XHCI::_RemoveEndpointForPipe(Pipe *pipe)
1540 {
1541 	if (pipe->Parent()->Type() != USB_OBJECT_DEVICE)
1542 		return B_OK;
1543 	//Device* device = (Device *)pipe->Parent();
1544 
1545 	return B_OK;
1546 }
1547 
1548 
1549 status_t
1550 XHCI::_LinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint)
1551 {
1552 	TRACE("_LinkDescriptorForPipe\n");
1553 	MutexLocker endpointLocker(endpoint->lock);
1554 	if (endpoint->used >= XHCI_MAX_TRANSFERS) {
1555 		TRACE_ERROR("_LinkDescriptorForPipe max transfers count exceeded\n");
1556 		return B_BAD_VALUE;
1557 	}
1558 
1559 	endpoint->used++;
1560 	if (endpoint->td_head == NULL)
1561 		descriptor->next = NULL;
1562 	else
1563 		descriptor->next = endpoint->td_head;
1564 	endpoint->td_head = descriptor;
1565 
1566 	uint8 current = endpoint->current;
1567 	uint8 next = (current + 1) % (XHCI_MAX_TRANSFERS);
1568 
1569 	TRACE("_LinkDescriptorForPipe current %d, next %d\n", current, next);
1570 
1571 	xhci_td *last = descriptor;
1572 	while (last->next_chain != NULL)
1573 		last = last->next_chain;
1574 
1575 	// compute next link
1576 	addr_t addr = endpoint->trb_addr + next * sizeof(struct xhci_trb);
1577 	last->trbs[last->trb_count].qwtrb0 = addr;
1578 	last->trbs[last->trb_count].dwtrb2 = TRB_2_IRQ(0);
1579 	last->trbs[last->trb_count].dwtrb3 = B_HOST_TO_LENDIAN_INT32(
1580 		TRB_3_TYPE(TRB_TYPE_LINK) | TRB_3_IOC_BIT | TRB_3_CYCLE_BIT);
1581 
1582 	endpoint->trbs[next].qwtrb0 = 0;
1583 	endpoint->trbs[next].dwtrb2 = 0;
1584 	endpoint->trbs[next].dwtrb3 = 0;
1585 
1586 	// link the descriptor
1587 	endpoint->trbs[current].qwtrb0 = descriptor->this_phy;
1588 	endpoint->trbs[current].dwtrb2 = TRB_2_IRQ(0);
1589 	endpoint->trbs[current].dwtrb3 = B_HOST_TO_LENDIAN_INT32(
1590 		TRB_3_TYPE(TRB_TYPE_LINK) | TRB_3_CYCLE_BIT);
1591 
1592 	TRACE("_LinkDescriptorForPipe pCurrent %p phys 0x%" B_PRIxPHYSADDR
1593 		" 0x%" B_PRIxPHYSADDR " 0x%08" B_PRIx32 "\n", &endpoint->trbs[current],
1594 		endpoint->trb_addr + current * sizeof(struct xhci_trb),
1595 		endpoint->trbs[current].qwtrb0,
1596 		B_LENDIAN_TO_HOST_INT32(endpoint->trbs[current].dwtrb3));
1597 	endpoint->current = next;
1598 
1599 	return B_OK;
1600 }
1601 
1602 
1603 status_t
1604 XHCI::_UnlinkDescriptorForPipe(xhci_td *descriptor, xhci_endpoint *endpoint)
1605 {
1606 	TRACE("_UnlinkDescriptorForPipe\n");
1607 	MutexLocker endpointLocker(endpoint->lock);
1608 	endpoint->used--;
1609 	if (descriptor == endpoint->td_head) {
1610 		endpoint->td_head = descriptor->next;
1611 		descriptor->next = NULL;
1612 		return B_OK;
1613 	} else {
1614 		for (xhci_td *td = endpoint->td_head; td->next != NULL; td = td->next) {
1615 			if (td->next == descriptor) {
1616 				td->next = descriptor->next;
1617 				descriptor->next = NULL;
1618 				return B_OK;
1619 			}
1620 		}
1621 	}
1622 
1623 	endpoint->used++;
1624 	return B_ERROR;
1625 }
1626 
1627 
1628 status_t
1629 XHCI::ConfigureEndpoint(uint8 slot, uint8 number, uint8 type, uint64 ringAddr,
1630 	uint16 interval, uint16 maxPacketSize, uint16 maxFrameSize, usb_speed speed)
1631 {
1632 	struct xhci_device* device = &fDevices[slot];
1633 
1634 	uint32 dwendpoint0 = 0;
1635 	uint32 dwendpoint1 = 0;
1636 	uint64 qwendpoint2 = 0;
1637 	uint32 dwendpoint4 = 0;
1638 
1639 	// Compute interval
1640 	uint16 calcInterval = 0;
1641 	if (speed == USB_SPEED_HIGHSPEED && (type == 4 || type == 2)) {
1642 		if (interval != 0) {
1643 			while ((1 << calcInterval) <= interval)
1644 				calcInterval++;
1645 			calcInterval--;
1646 		}
1647 	}
1648 	if ((type & 0x3) == 3
1649 			&& (speed == USB_SPEED_FULLSPEED || speed == USB_SPEED_LOWSPEED)) {
1650 		while ((1 << calcInterval) <= interval * 8)
1651 			calcInterval++;
1652 		calcInterval--;
1653 	}
1654 	if ((type & 0x3) == 1 && speed == USB_SPEED_FULLSPEED) {
1655 		calcInterval = interval + 2;
1656 	}
1657 	if (((type & 0x3) == 1 || (type & 0x3) == 3)
1658 			&& (speed == USB_SPEED_HIGHSPEED || speed == USB_SPEED_SUPER)) {
1659 		calcInterval = interval - 1;
1660 	}
1661 	dwendpoint0 |= ENDPOINT_0_INTERVAL(calcInterval);
1662 
1663 	// Assigning CERR for non-isochronous endpoints
1664 	if ((type & 0x3) != 1)
1665 		dwendpoint1 |= ENDPOINT_1_CERR(3);
1666 
1667 	dwendpoint1 |= ENDPOINT_1_EPTYPE(type);
1668 
1669 	// Assigning MaxBurst for HighSpeed
1670 	uint8 maxBurst = (maxPacketSize & 0x1800) >> 11;
1671 	maxPacketSize = (maxPacketSize & 0x7ff);
1672 	if (speed == USB_SPEED_HIGHSPEED &&
1673 			((type & 0x3) == 1 || (type & 0x3) == 3)) {
1674 		dwendpoint1 |= ENDPOINT_1_MAXBURST(maxBurst);
1675 	}
1676 	// TODO Assign MaxBurst for SuperSpeed
1677 
1678 	dwendpoint1 |= ENDPOINT_1_MAXPACKETSIZE(maxPacketSize);
1679 	qwendpoint2 |= ENDPOINT_2_DCS_BIT | ringAddr;
1680 
1681 	// Assign MaxESITPayload
1682 	// Assign AvgTRBLength
1683 	switch (type) {
1684 		case 4:
1685 			dwendpoint4 = ENDPOINT_4_AVGTRBLENGTH(8);
1686 			break;
1687 		case 1:
1688 		case 3:
1689 		case 5:
1690 		case 7:
1691 			dwendpoint4 = ENDPOINT_4_AVGTRBLENGTH(min_c(maxFrameSize,
1692 				B_PAGE_SIZE)) | ENDPOINT_4_MAXESITPAYLOAD((
1693 					(maxBurst+1) * maxPacketSize));
1694 			break;
1695 		default:
1696 			dwendpoint4 = ENDPOINT_4_AVGTRBLENGTH(B_PAGE_SIZE);
1697 			break;
1698 	}
1699 
1700 	_WriteContext(&device->input_ctx->endpoints[number].dwendpoint0,
1701 		dwendpoint0);
1702 	_WriteContext(&device->input_ctx->endpoints[number].dwendpoint1,
1703 		dwendpoint1);
1704 	_WriteContext(&device->input_ctx->endpoints[number].qwendpoint2,
1705 		qwendpoint2);
1706 	_WriteContext(&device->input_ctx->endpoints[number].dwendpoint4,
1707 		dwendpoint4);
1708 
1709 	TRACE("endpoint 0x%" B_PRIx32 " 0x%" B_PRIx32 " 0x%" B_PRIx64 " 0x%"
1710 		B_PRIx32 "\n",
1711 		_ReadContext(&device->input_ctx->endpoints[number].dwendpoint0),
1712 		_ReadContext(&device->input_ctx->endpoints[number].dwendpoint1),
1713 		_ReadContext(&device->input_ctx->endpoints[number].qwendpoint2),
1714 		_ReadContext(&device->input_ctx->endpoints[number].dwendpoint4));
1715 
1716 	return B_OK;
1717 }
1718 
1719 
1720 status_t
1721 XHCI::GetPortSpeed(uint8 index, usb_speed* speed)
1722 {
1723 	uint32 portStatus = ReadOpReg(XHCI_PORTSC(index));
1724 
1725 	switch (PS_SPEED_GET(portStatus)) {
1726 	case 3:
1727 		*speed = USB_SPEED_HIGHSPEED;
1728 		break;
1729 	case 2:
1730 		*speed = USB_SPEED_LOWSPEED;
1731 		break;
1732 	case 1:
1733 		*speed = USB_SPEED_FULLSPEED;
1734 		break;
1735 	case 4:
1736 		*speed = USB_SPEED_SUPER;
1737 		break;
1738 	default:
1739 		TRACE("Non Standard Port Speed\n");
1740 		TRACE("Assuming Superspeed\n");
1741 		*speed = USB_SPEED_SUPER;
1742 		break;
1743 	}
1744 
1745 	return B_OK;
1746 }
1747 
1748 
1749 status_t
1750 XHCI::GetPortStatus(uint8 index, usb_port_status* status)
1751 {
1752 	if (index >= fPortCount)
1753 		return B_BAD_INDEX;
1754 
1755 	status->status = status->change = 0;
1756 	uint32 portStatus = ReadOpReg(XHCI_PORTSC(index));
1757 	TRACE("port %" B_PRId8 " status=0x%08" B_PRIx32 "\n", index, portStatus);
1758 
1759 	// build the status
1760 	switch (PS_SPEED_GET(portStatus)) {
1761 	case 3:
1762 		status->status |= PORT_STATUS_HIGH_SPEED;
1763 		break;
1764 	case 2:
1765 		status->status |= PORT_STATUS_LOW_SPEED;
1766 		break;
1767 	default:
1768 		break;
1769 	}
1770 
1771 	if (portStatus & PS_CCS)
1772 		status->status |= PORT_STATUS_CONNECTION;
1773 	if (portStatus & PS_PED)
1774 		status->status |= PORT_STATUS_ENABLE;
1775 	if (portStatus & PS_OCA)
1776 		status->status |= PORT_STATUS_OVER_CURRENT;
1777 	if (portStatus & PS_PR)
1778 		status->status |= PORT_STATUS_RESET;
1779 	if (portStatus & PS_PP) {
1780 		if (fPortSpeeds[index] == USB_SPEED_SUPER)
1781 			status->status |= PORT_STATUS_SS_POWER;
1782 		else
1783 			status->status |= PORT_STATUS_POWER;
1784 	}
1785 
1786 	// build the change
1787 	if (portStatus & PS_CSC)
1788 		status->change |= PORT_STATUS_CONNECTION;
1789 	if (portStatus & PS_PEC)
1790 		status->change |= PORT_STATUS_ENABLE;
1791 	if (portStatus & PS_OCC)
1792 		status->change |= PORT_STATUS_OVER_CURRENT;
1793 	if (portStatus & PS_PRC)
1794 		status->change |= PORT_STATUS_RESET;
1795 
1796 	if (fPortSpeeds[index] == USB_SPEED_SUPER) {
1797 		if (portStatus & PS_PLC)
1798 			status->change |= PORT_CHANGE_LINK_STATE;
1799 		if (portStatus & PS_WRC)
1800 			status->change |= PORT_CHANGE_BH_PORT_RESET;
1801 	}
1802 
1803 	return B_OK;
1804 }
1805 
1806 
1807 status_t
1808 XHCI::SetPortFeature(uint8 index, uint16 feature)
1809 {
1810 	TRACE("set port feature index %u feature %u\n", index, feature);
1811 	if (index >= fPortCount)
1812 		return B_BAD_INDEX;
1813 
1814 	uint32 portRegister = XHCI_PORTSC(index);
1815 	uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR;
1816 
1817 	switch (feature) {
1818 	case PORT_SUSPEND:
1819 		if ((portStatus & PS_PED) == 0 || (portStatus & PS_PR)
1820 			|| (portStatus & PS_PLS_MASK) >= PS_XDEV_U3) {
1821 			TRACE_ERROR("USB core suspending device not in U0/U1/U2.\n");
1822 			return B_BAD_VALUE;
1823 		}
1824 		portStatus &= ~PS_PLS_MASK;
1825 		WriteOpReg(portRegister, portStatus | PS_LWS | PS_XDEV_U3);
1826 		break;
1827 
1828 	case PORT_RESET:
1829 		WriteOpReg(portRegister, portStatus | PS_PR);
1830 		break;
1831 
1832 	case PORT_POWER:
1833 		WriteOpReg(portRegister, portStatus | PS_PP);
1834 		break;
1835 	default:
1836 		return B_BAD_VALUE;
1837 	}
1838 	ReadOpReg(portRegister);
1839 	return B_OK;
1840 }
1841 
1842 
1843 status_t
1844 XHCI::ClearPortFeature(uint8 index, uint16 feature)
1845 {
1846 	TRACE("clear port feature index %u feature %u\n", index, feature);
1847 	if (index >= fPortCount)
1848 		return B_BAD_INDEX;
1849 
1850 	uint32 portRegister = XHCI_PORTSC(index);
1851 	uint32 portStatus = ReadOpReg(portRegister) & ~PS_CLEAR;
1852 
1853 	switch (feature) {
1854 	case PORT_SUSPEND:
1855 		portStatus = ReadOpReg(portRegister);
1856 		if (portStatus & PS_PR)
1857 			return B_BAD_VALUE;
1858 		if (portStatus & PS_XDEV_U3) {
1859 			if ((portStatus & PS_PED) == 0)
1860 				return B_BAD_VALUE;
1861 			portStatus &= ~PS_PLS_MASK;
1862 			WriteOpReg(portRegister, portStatus | PS_XDEV_U0 | PS_LWS);
1863 		}
1864 		break;
1865 	case PORT_ENABLE:
1866 		WriteOpReg(portRegister, portStatus | PS_PED);
1867 		break;
1868 	case PORT_POWER:
1869 		WriteOpReg(portRegister, portStatus & ~PS_PP);
1870 		break;
1871 	case C_PORT_CONNECTION:
1872 		WriteOpReg(portRegister, portStatus | PS_CSC);
1873 		break;
1874 	case C_PORT_ENABLE:
1875 		WriteOpReg(portRegister, portStatus | PS_PEC);
1876 		break;
1877 	case C_PORT_OVER_CURRENT:
1878 		WriteOpReg(portRegister, portStatus | PS_OCC);
1879 		break;
1880 	case C_PORT_RESET:
1881 		WriteOpReg(portRegister, portStatus | PS_PRC);
1882 		break;
1883 	case C_PORT_BH_PORT_RESET:
1884 		WriteOpReg(portRegister, portStatus | PS_WRC);
1885 		break;
1886 	case C_PORT_LINK_STATE:
1887 		WriteOpReg(portRegister, portStatus | PS_PLC);
1888 		break;
1889 	default:
1890 		return B_BAD_VALUE;
1891 	}
1892 
1893 	ReadOpReg(portRegister);
1894 	return B_OK;
1895 }
1896 
1897 
1898 status_t
1899 XHCI::ControllerHalt()
1900 {
1901 	// Mask off run state
1902 	WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) & ~CMD_RUN);
1903 
1904 	// wait for shutdown state
1905 	if (WaitOpBits(XHCI_STS, STS_HCH, STS_HCH) != B_OK) {
1906 		TRACE_ERROR("HCH shutdown timeout\n");
1907 		return B_ERROR;
1908 	}
1909 	return B_OK;
1910 }
1911 
1912 
1913 status_t
1914 XHCI::ControllerReset()
1915 {
1916 	TRACE("ControllerReset() cmd: 0x%" B_PRIx32 " sts: 0x%" B_PRIx32 "\n",
1917 		ReadOpReg(XHCI_CMD), ReadOpReg(XHCI_STS));
1918 	WriteOpReg(XHCI_CMD, ReadOpReg(XHCI_CMD) | CMD_HCRST);
1919 
1920 	if (WaitOpBits(XHCI_CMD, CMD_HCRST, 0) != B_OK) {
1921 		TRACE_ERROR("ControllerReset() failed CMD_HCRST\n");
1922 		return B_ERROR;
1923 	}
1924 
1925 	if (WaitOpBits(XHCI_STS, STS_CNR, 0) != B_OK) {
1926 		TRACE_ERROR("ControllerReset() failed STS_CNR\n");
1927 		return B_ERROR;
1928 	}
1929 
1930 	return B_OK;
1931 }
1932 
1933 
1934 int32
1935 XHCI::InterruptHandler(void* data)
1936 {
1937 	return ((XHCI*)data)->Interrupt();
1938 }
1939 
1940 
1941 int32
1942 XHCI::Interrupt()
1943 {
1944 	SpinLocker _(&fSpinlock);
1945 
1946 	uint32 status = ReadOpReg(XHCI_STS);
1947 	uint32 temp = ReadRunReg32(XHCI_IMAN(0));
1948 	WriteOpReg(XHCI_STS, status);
1949 	WriteRunReg32(XHCI_IMAN(0), temp);
1950 
1951 	int32 result = B_HANDLED_INTERRUPT;
1952 
1953 	if ((status & STS_HCH) != 0) {
1954 		TRACE_ERROR("Host Controller halted\n");
1955 		return result;
1956 	}
1957 	if ((status & STS_HSE) != 0) {
1958 		TRACE_ERROR("Host System Error\n");
1959 		return result;
1960 	}
1961 	if ((status & STS_HCE) != 0) {
1962 		TRACE_ERROR("Host Controller Error\n");
1963 		return result;
1964 	}
1965 
1966 	if ((status & STS_EINT) == 0) {
1967 		TRACE("STS: 0x%" B_PRIx32 " IRQ_PENDING: 0x%" B_PRIx32 "\n",
1968 			status, temp);
1969 		return B_UNHANDLED_INTERRUPT;
1970 	}
1971 
1972 	TRACE("Event Interrupt\n");
1973 	release_sem_etc(fEventSem, 1, B_DO_NOT_RESCHEDULE);
1974 	return B_INVOKE_SCHEDULER;
1975 }
1976 
1977 
1978 void
1979 XHCI::Ring(uint8 slot, uint8 endpoint)
1980 {
1981 	TRACE("Ding Dong! slot:%d endpoint %d\n", slot, endpoint)
1982 	if ((slot == 0 && endpoint > 0) || (slot > 0 && endpoint == 0))
1983 		panic("Ring() invalid slot/endpoint combination\n");
1984 	if (slot > fSlotCount || endpoint >= XHCI_MAX_ENDPOINTS)
1985 		panic("Ring() invalid slot or endpoint\n");
1986 	WriteDoorReg32(XHCI_DOORBELL(slot), XHCI_DOORBELL_TARGET(endpoint)
1987 		| XHCI_DOORBELL_STREAMID(0));
1988 	/* Flush PCI posted writes */
1989 	ReadDoorReg32(XHCI_DOORBELL(slot));
1990 }
1991 
1992 
1993 void
1994 XHCI::QueueCommand(xhci_trb* trb)
1995 {
1996 	uint8 i, j;
1997 	uint32 temp;
1998 
1999 	i = fCmdIdx;
2000 	j = fCmdCcs;
2001 
2002 	TRACE("command[%u] = %" B_PRId32 " (0x%016" B_PRIx64 ", 0x%08" B_PRIx32
2003 		", 0x%08" B_PRIx32 ")\n", i, TRB_3_TYPE_GET(trb->dwtrb3), trb->qwtrb0,
2004 		trb->dwtrb2, trb->dwtrb3);
2005 
2006 	fCmdRing[i].qwtrb0 = trb->qwtrb0;
2007 	fCmdRing[i].dwtrb2 = trb->dwtrb2;
2008 	temp = trb->dwtrb3;
2009 
2010 	if (j)
2011 		temp |= TRB_3_CYCLE_BIT;
2012 	else
2013 		temp &= ~TRB_3_CYCLE_BIT;
2014 	temp &= ~TRB_3_TC_BIT;
2015 	fCmdRing[i].dwtrb3 = B_HOST_TO_LENDIAN_INT32(temp);
2016 
2017 	fCmdAddr = fErst->rs_addr + (XHCI_MAX_EVENTS + i) * sizeof(xhci_trb);
2018 
2019 	i++;
2020 
2021 	if (i == (XHCI_MAX_COMMANDS - 1)) {
2022 		temp = TRB_3_TYPE(TRB_TYPE_LINK) | TRB_3_TC_BIT;
2023 		if (j)
2024 			temp |= TRB_3_CYCLE_BIT;
2025 		fCmdRing[i].dwtrb3 = B_HOST_TO_LENDIAN_INT32(temp);
2026 
2027 		i = 0;
2028 		j ^= 1;
2029 	}
2030 
2031 	fCmdIdx = i;
2032 	fCmdCcs = j;
2033 }
2034 
2035 
2036 void
2037 XHCI::HandleCmdComplete(xhci_trb* trb)
2038 {
2039 	if (fCmdAddr == trb->qwtrb0) {
2040 		TRACE("Received command event\n");
2041 		fCmdResult[0] = trb->dwtrb2;
2042 		fCmdResult[1] = B_LENDIAN_TO_HOST_INT32(trb->dwtrb3);
2043 		release_sem_etc(fCmdCompSem, 1, B_DO_NOT_RESCHEDULE);
2044 	}
2045 
2046 }
2047 
2048 
2049 void
2050 XHCI::HandleTransferComplete(xhci_trb* trb)
2051 {
2052 	TRACE("HandleTransferComplete trb %p\n", trb);
2053 	addr_t source = trb->qwtrb0;
2054 	uint8 completionCode = TRB_2_COMP_CODE_GET(trb->dwtrb2);
2055 	uint32 remainder = TRB_2_REM_GET(trb->dwtrb2);
2056 	uint8 endpointNumber
2057 		= TRB_3_ENDPOINT_GET(B_LENDIAN_TO_HOST_INT32(trb->dwtrb3));
2058 	uint8 slot = TRB_3_SLOT_GET(B_LENDIAN_TO_HOST_INT32(trb->dwtrb3));
2059 
2060 	if (slot > fSlotCount)
2061 		TRACE_ERROR("invalid slot\n");
2062 	if (endpointNumber == 0 || endpointNumber >= XHCI_MAX_ENDPOINTS)
2063 		TRACE_ERROR("invalid endpoint\n");
2064 
2065 	xhci_device *device = &fDevices[slot];
2066 	xhci_endpoint *endpoint = &device->endpoints[endpointNumber - 1];
2067 	xhci_td *td = endpoint->td_head;
2068 	for (; td != NULL; td = td->next) {
2069 		xhci_td *td_chain = td;
2070 		for (; td_chain != NULL; td_chain = td_chain->next_chain) {
2071 			int64 offset = source - td_chain->this_phy;
2072 			TRACE("HandleTransferComplete td %p offset %" B_PRId64 " %"
2073 				B_PRIxADDR "\n", td_chain, offset, source);
2074 			offset = offset / sizeof(xhci_trb) + 1;
2075 			if (offset <= td_chain->trb_count && offset >= 1) {
2076 				TRACE("HandleTransferComplete td %p trb %" B_PRId64 " found "
2077 					"\n", td_chain, offset);
2078 				// is it the last trb?
2079 				if (offset == td_chain->trb_count) {
2080 					_UnlinkDescriptorForPipe(td, endpoint);
2081 					td->trb_completion_code = completionCode;
2082 					td->trb_left = remainder;
2083 					// add descriptor to finished list
2084 					Lock();
2085 					td->next = fFinishedHead;
2086 					fFinishedHead = td;
2087 					Unlock();
2088 					release_sem(fFinishTransfersSem);
2089 					TRACE("HandleTransferComplete td %p\n", td);
2090 				}
2091 				return;
2092 			}
2093 		}
2094 	}
2095 
2096 }
2097 
2098 
2099 void
2100 XHCI::DumpRing(xhci_trb *trbs, uint32 size)
2101 {
2102 	if (!Lock()) {
2103 		TRACE("Unable to get lock!\n");
2104 		return;
2105 	}
2106 
2107 	for (uint32 i = 0; i < size; i++) {
2108 		TRACE("command[%" B_PRId32 "] = %" B_PRId32 " (0x%016" B_PRIx64 ","
2109 			" 0x%08" B_PRIx32 ", 0x%08" B_PRIx32 ")\n", i,
2110 			TRB_3_TYPE_GET(B_LENDIAN_TO_HOST_INT32(trbs[i].dwtrb3)),
2111 			trbs[i].qwtrb0, trbs[i].dwtrb2, trbs[i].dwtrb3);
2112 	}
2113 
2114 	Unlock();
2115 }
2116 
2117 
2118 status_t
2119 XHCI::DoCommand(xhci_trb* trb)
2120 {
2121 	if (!Lock()) {
2122 		TRACE("Unable to get lock!\n");
2123 		return B_ERROR;
2124 	}
2125 
2126 	QueueCommand(trb);
2127 	Ring(0, 0);
2128 
2129 	if (acquire_sem(fCmdCompSem) < B_OK) {
2130 		TRACE("Unable to obtain fCmdCompSem semaphore!\n");
2131 		Unlock();
2132 		return B_ERROR;
2133 	}
2134 	// eat up sems that have been released by multiple interrupts
2135 	int32 semCount = 0;
2136 	get_sem_count(fCmdCompSem, &semCount);
2137 	if (semCount > 0)
2138 		acquire_sem_etc(fCmdCompSem, semCount, B_RELATIVE_TIMEOUT, 0);
2139 
2140 	status_t status = B_OK;
2141 	uint32 completionCode = TRB_2_COMP_CODE_GET(fCmdResult[0]);
2142 	TRACE("Command Complete. Result: %" B_PRId32 "\n", completionCode);
2143 	if (completionCode != COMP_SUCCESS) {
2144 		uint32 errorCode = TRB_2_COMP_CODE_GET(fCmdResult[0]);
2145 		TRACE_ERROR("unsuccessful command %" B_PRId32 ", error %s (%" B_PRId32 ")\n",
2146 			TRB_3_TYPE_GET(trb->dwtrb3), xhci_error_string(errorCode), errorCode);
2147 		status = B_IO_ERROR;
2148 	}
2149 
2150 	trb->dwtrb2 = fCmdResult[0];
2151 	trb->dwtrb3 = fCmdResult[1];
2152 	TRACE("Storing trb 0x%08" B_PRIx32 " 0x%08" B_PRIx32 "\n", trb->dwtrb2,
2153 		trb->dwtrb3);
2154 
2155 	Unlock();
2156 	return status;
2157 }
2158 
2159 
2160 status_t
2161 XHCI::Noop()
2162 {
2163 	TRACE("Issue No-Op\n");
2164 	xhci_trb trb;
2165 	trb.qwtrb0 = 0;
2166 	trb.dwtrb2 = 0;
2167 	trb.dwtrb3 = TRB_3_TYPE(TRB_TYPE_CMD_NOOP);
2168 
2169 	return DoCommand(&trb);
2170 }
2171 
2172 
2173 status_t
2174 XHCI::EnableSlot(uint8* slot)
2175 {
2176 	TRACE("Enable Slot\n");
2177 	xhci_trb trb;
2178 	trb.qwtrb0 = 0;
2179 	trb.dwtrb2 = 0;
2180 	trb.dwtrb3 = TRB_3_TYPE(TRB_TYPE_ENABLE_SLOT);
2181 
2182 	status_t status = DoCommand(&trb);
2183 	if (status != B_OK)
2184 		return status;
2185 
2186 	*slot = TRB_3_SLOT_GET(trb.dwtrb3);
2187 	return *slot != 0 ? B_OK : B_BAD_VALUE;
2188 }
2189 
2190 
2191 status_t
2192 XHCI::DisableSlot(uint8 slot)
2193 {
2194 	TRACE("Disable Slot\n");
2195 	xhci_trb trb;
2196 	trb.qwtrb0 = 0;
2197 	trb.dwtrb2 = 0;
2198 	trb.dwtrb3 = TRB_3_TYPE(TRB_TYPE_DISABLE_SLOT) | TRB_3_SLOT(slot);
2199 
2200 	return DoCommand(&trb);
2201 }
2202 
2203 
2204 status_t
2205 XHCI::SetAddress(uint64 inputContext, bool bsr, uint8 slot)
2206 {
2207 	TRACE("Set Address\n");
2208 	xhci_trb trb;
2209 	trb.qwtrb0 = inputContext;
2210 	trb.dwtrb2 = 0;
2211 	trb.dwtrb3 = TRB_3_TYPE(TRB_TYPE_ADDRESS_DEVICE) | TRB_3_SLOT(slot);
2212 
2213 	if (bsr)
2214 		trb.dwtrb3 |= TRB_3_BSR_BIT;
2215 
2216 	return DoCommand(&trb);
2217 }
2218 
2219 
2220 status_t
2221 XHCI::ConfigureEndpoint(uint64 inputContext, bool deconfigure, uint8 slot)
2222 {
2223 	TRACE("Configure Endpoint\n");
2224 	xhci_trb trb;
2225 	trb.qwtrb0 = inputContext;
2226 	trb.dwtrb2 = 0;
2227 	trb.dwtrb3 = TRB_3_TYPE(TRB_TYPE_CONFIGURE_ENDPOINT) | TRB_3_SLOT(slot);
2228 
2229 	if (deconfigure)
2230 		trb.dwtrb3 |= TRB_3_DCEP_BIT;
2231 
2232 	return DoCommand(&trb);
2233 }
2234 
2235 
2236 status_t
2237 XHCI::EvaluateContext(uint64 inputContext, uint8 slot)
2238 {
2239 	TRACE("Evaluate Context\n");
2240 	xhci_trb trb;
2241 	trb.qwtrb0 = inputContext;
2242 	trb.dwtrb2 = 0;
2243 	trb.dwtrb3 = TRB_3_TYPE(TRB_TYPE_EVALUATE_CONTEXT) | TRB_3_SLOT(slot);
2244 
2245 	return DoCommand(&trb);
2246 }
2247 
2248 
2249 status_t
2250 XHCI::ResetEndpoint(bool preserve, uint8 endpoint, uint8 slot)
2251 {
2252 	TRACE("Reset Endpoint\n");
2253 	xhci_trb trb;
2254 	trb.qwtrb0 = 0;
2255 	trb.dwtrb2 = 0;
2256 	trb.dwtrb3 = TRB_3_TYPE(TRB_TYPE_RESET_ENDPOINT)
2257 		| TRB_3_SLOT(slot) | TRB_3_ENDPOINT(endpoint);
2258 	if (preserve)
2259 		trb.dwtrb3 |= TRB_3_PRSV_BIT;
2260 
2261 	return DoCommand(&trb);
2262 }
2263 
2264 
2265 status_t
2266 XHCI::StopEndpoint(bool suspend, uint8 endpoint, uint8 slot)
2267 {
2268 	TRACE("Stop Endpoint\n");
2269 	xhci_trb trb;
2270 	trb.qwtrb0 = 0;
2271 	trb.dwtrb2 = 0;
2272 	trb.dwtrb3 = TRB_3_TYPE(TRB_TYPE_STOP_ENDPOINT)
2273 		| TRB_3_SLOT(slot) | TRB_3_ENDPOINT(endpoint);
2274 	if (suspend)
2275 		trb.dwtrb3 |= TRB_3_SUSPEND_ENDPOINT_BIT;
2276 
2277 	return DoCommand(&trb);
2278 }
2279 
2280 
2281 status_t
2282 XHCI::SetTRDequeue(uint64 dequeue, uint16 stream, uint8 endpoint, uint8 slot)
2283 {
2284 	TRACE("Set TR Dequeue\n");
2285 	xhci_trb trb;
2286 	trb.qwtrb0 = dequeue;
2287 	trb.dwtrb2 = TRB_2_STREAM(stream);
2288 	trb.dwtrb3 = TRB_3_TYPE(TRB_TYPE_SET_TR_DEQUEUE)
2289 		| TRB_3_SLOT(slot) | TRB_3_ENDPOINT(endpoint);
2290 
2291 	return DoCommand(&trb);
2292 }
2293 
2294 
2295 status_t
2296 XHCI::ResetDevice(uint8 slot)
2297 {
2298 	TRACE("Reset Device\n");
2299 	xhci_trb trb;
2300 	trb.qwtrb0 = 0;
2301 	trb.dwtrb2 = 0;
2302 	trb.dwtrb3 = TRB_3_TYPE(TRB_TYPE_RESET_DEVICE) | TRB_3_SLOT(slot);
2303 
2304 	return DoCommand(&trb);
2305 }
2306 
2307 
2308 int32
2309 XHCI::EventThread(void* data)
2310 {
2311 	((XHCI *)data)->CompleteEvents();
2312 	return B_OK;
2313 }
2314 
2315 
2316 void
2317 XHCI::CompleteEvents()
2318 {
2319 	while (!fStopThreads) {
2320 		if (acquire_sem(fEventSem) < B_OK)
2321 			continue;
2322 
2323 		// eat up sems that have been released by multiple interrupts
2324 		int32 semCount = 0;
2325 		get_sem_count(fEventSem, &semCount);
2326 		if (semCount > 0)
2327 			acquire_sem_etc(fEventSem, semCount, B_RELATIVE_TIMEOUT, 0);
2328 
2329 		uint16 i = fEventIdx;
2330 		uint8 j = fEventCcs;
2331 		uint8 t = 2;
2332 
2333 		while (1) {
2334 			uint32 temp = B_LENDIAN_TO_HOST_INT32(fEventRing[i].dwtrb3);
2335 			uint8 event = TRB_3_TYPE_GET(temp);
2336 			TRACE("event[%u] = %u (0x%016" B_PRIx64 " 0x%08" B_PRIx32 " 0x%08"
2337 				B_PRIx32 ")\n", i, event, fEventRing[i].qwtrb0,
2338 				fEventRing[i].dwtrb2, B_LENDIAN_TO_HOST_INT32(fEventRing[i].dwtrb3));
2339 			uint8 k = (temp & TRB_3_CYCLE_BIT) ? 1 : 0;
2340 			if (j != k)
2341 				break;
2342 
2343 
2344 			switch (event) {
2345 			case TRB_TYPE_COMMAND_COMPLETION:
2346 				HandleCmdComplete(&fEventRing[i]);
2347 				break;
2348 			case TRB_TYPE_TRANSFER:
2349 				HandleTransferComplete(&fEventRing[i]);
2350 				break;
2351 			case TRB_TYPE_PORT_STATUS_CHANGE:
2352 				TRACE("port change detected\n");
2353 				break;
2354 			default:
2355 				TRACE_ERROR("Unhandled event = %u\n", event);
2356 				break;
2357 			}
2358 
2359 			i++;
2360 			if (i == XHCI_MAX_EVENTS) {
2361 				i = 0;
2362 				j ^= 1;
2363 				if (!--t)
2364 					break;
2365 			}
2366 		}
2367 
2368 		fEventIdx = i;
2369 		fEventCcs = j;
2370 
2371 		uint64 addr = fErst->rs_addr + i * sizeof(xhci_trb);
2372 		addr |= ERST_EHB;
2373 		WriteRunReg32(XHCI_ERDP_LO(0), (uint32)addr);
2374 		WriteRunReg32(XHCI_ERDP_HI(0), (uint32)(addr >> 32));
2375 	}
2376 }
2377 
2378 
2379 int32
2380 XHCI::FinishThread(void* data)
2381 {
2382 	((XHCI *)data)->FinishTransfers();
2383 	return B_OK;
2384 }
2385 
2386 
2387 void
2388 XHCI::FinishTransfers()
2389 {
2390 	while (!fStopThreads) {
2391 		if (acquire_sem(fFinishTransfersSem) < B_OK)
2392 			continue;
2393 
2394 		// eat up sems that have been released by multiple interrupts
2395 		int32 semCount = 0;
2396 		get_sem_count(fFinishTransfersSem, &semCount);
2397 		if (semCount > 0)
2398 			acquire_sem_etc(fFinishTransfersSem, semCount, B_RELATIVE_TIMEOUT, 0);
2399 
2400 		Lock();
2401 		TRACE("finishing transfers\n");
2402 		while (fFinishedHead != NULL) {
2403 			xhci_td* td = fFinishedHead;
2404 			fFinishedHead = td->next;
2405 			td->next = NULL;
2406 			Unlock();
2407 
2408 			TRACE("finishing transfer td %p\n", td);
2409 
2410 			Transfer* transfer = td->transfer;
2411 			bool directionIn = (transfer->TransferPipe()->Direction() != Pipe::Out);
2412 			usb_request_data *requestData = transfer->RequestData();
2413 
2414 			status_t callbackStatus = B_OK;
2415 			switch (td->trb_completion_code) {
2416 				case COMP_SHORT_PACKET:
2417 				case COMP_SUCCESS:
2418 					callbackStatus = B_OK;
2419 					break;
2420 				case COMP_DATA_BUFFER:
2421 					callbackStatus = directionIn ? B_DEV_DATA_OVERRUN
2422 						: B_DEV_DATA_UNDERRUN;
2423 					break;
2424 				case COMP_BABBLE:
2425 					callbackStatus = directionIn ? B_DEV_FIFO_OVERRUN
2426 						: B_DEV_FIFO_UNDERRUN;
2427 					break;
2428 				case COMP_USB_TRANSACTION:
2429 					callbackStatus = B_DEV_CRC_ERROR;
2430 					break;
2431 				case COMP_STALL:
2432 					callbackStatus = B_DEV_STALLED;
2433 					break;
2434 				default:
2435 					callbackStatus = B_DEV_STALLED;
2436 					break;
2437 			}
2438 
2439 			size_t actualLength = 0;
2440 			if (callbackStatus == B_OK) {
2441 				actualLength = requestData ? requestData->Length
2442 					: transfer->DataLength();
2443 
2444 				if (td->trb_completion_code == COMP_SHORT_PACKET)
2445 					actualLength -= td->trb_left;
2446 
2447 				if (directionIn && actualLength > 0) {
2448 					if (requestData) {
2449 						TRACE("copying in data %d bytes\n", requestData->Length);
2450 						transfer->PrepareKernelAccess();
2451 						memcpy((uint8 *)transfer->Vector()[0].iov_base,
2452 							td->buffer_log[0], requestData->Length);
2453 					} else {
2454 						TRACE("copying in iov count %ld\n", transfer->VectorCount());
2455 						transfer->PrepareKernelAccess();
2456 						ReadDescriptorChain(td, transfer->Vector(),
2457 							transfer->VectorCount());
2458 					}
2459 				}
2460 			}
2461 			transfer->Finished(callbackStatus, actualLength);
2462 			delete transfer;
2463 			FreeDescriptor(td);
2464 			Lock();
2465 		}
2466 		Unlock();
2467 	}
2468 }
2469 
2470 
2471 inline void
2472 XHCI::WriteOpReg(uint32 reg, uint32 value)
2473 {
2474 	*(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg) = value;
2475 }
2476 
2477 
2478 inline uint32
2479 XHCI::ReadOpReg(uint32 reg)
2480 {
2481 	return *(volatile uint32 *)(fRegisters + fOperationalRegisterOffset + reg);
2482 }
2483 
2484 
2485 inline status_t
2486 XHCI::WaitOpBits(uint32 reg, uint32 mask, uint32 expected)
2487 {
2488 	int loops = 0;
2489 	uint32 value = ReadOpReg(reg);
2490 	while ((value & mask) != expected) {
2491 		snooze(1000);
2492 		value = ReadOpReg(reg);
2493 		if (loops == 25) {
2494 			TRACE("delay waiting on reg 0x%" B_PRIX32 " match 0x%" B_PRIX32
2495 				" (0x%" B_PRIX32 ")\n",	reg, expected, mask);
2496 		} else if (loops > 100) {
2497 			TRACE_ERROR("timeout waiting on reg 0x%" B_PRIX32
2498 				" match 0x%" B_PRIX32 " (0x%" B_PRIX32 ")\n", reg, expected,
2499 				mask);
2500 			return B_ERROR;
2501 		}
2502 		loops++;
2503 	}
2504 	return B_OK;
2505 }
2506 
2507 
2508 inline uint32
2509 XHCI::ReadCapReg32(uint32 reg)
2510 {
2511 	return *(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg);
2512 }
2513 
2514 
2515 inline void
2516 XHCI::WriteCapReg32(uint32 reg, uint32 value)
2517 {
2518 	*(volatile uint32 *)(fRegisters + fCapabilityRegisterOffset + reg) = value;
2519 }
2520 
2521 
2522 inline uint32
2523 XHCI::ReadRunReg32(uint32 reg)
2524 {
2525 	return *(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg);
2526 }
2527 
2528 
2529 inline void
2530 XHCI::WriteRunReg32(uint32 reg, uint32 value)
2531 {
2532 	*(volatile uint32 *)(fRegisters + fRuntimeRegisterOffset + reg) = value;
2533 }
2534 
2535 
2536 inline uint32
2537 XHCI::ReadDoorReg32(uint32 reg)
2538 {
2539 	return *(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg);
2540 }
2541 
2542 
2543 inline void
2544 XHCI::WriteDoorReg32(uint32 reg, uint32 value)
2545 {
2546 	*(volatile uint32 *)(fRegisters + fDoorbellRegisterOffset + reg) = value;
2547 }
2548 
2549 
2550 inline addr_t
2551 XHCI::_OffsetContextAddr(addr_t p)
2552 {
2553 	if (fContextSizeShift == 1) {
2554 		// each structure is page aligned, each pointer is 32 bits aligned
2555 		uint32 offset = p & ((B_PAGE_SIZE - 1) & ~31U);
2556 		p += offset;
2557 	}
2558 	return p;
2559 }
2560 
2561 inline uint32
2562 XHCI::_ReadContext(uint32* p)
2563 {
2564 	p = (uint32*)_OffsetContextAddr((addr_t)p);
2565 	return *p;
2566 }
2567 
2568 
2569 inline void
2570 XHCI::_WriteContext(uint32* p, uint32 value)
2571 {
2572 	p = (uint32*)_OffsetContextAddr((addr_t)p);
2573 	*p = value;
2574 }
2575 
2576 
2577 inline uint64
2578 XHCI::_ReadContext(uint64* p)
2579 {
2580 	p = (uint64*)_OffsetContextAddr((addr_t)p);
2581 	return *p;
2582 }
2583 
2584 
2585 inline void
2586 XHCI::_WriteContext(uint64* p, uint64 value)
2587 {
2588 	p = (uint64*)_OffsetContextAddr((addr_t)p);
2589 	*p = value;
2590 }
2591