1 /* 2 * Copyright 2013, 2018, Jérôme Duval, jerome.duval@gmail.com. 3 * Distributed under the terms of the MIT License. 4 */ 5 6 7 #include <new> 8 #include <stdio.h> 9 #include <string.h> 10 11 #include <bus/PCI.h> 12 #include <PCI_x86.h> 13 #include <virtio.h> 14 15 #include "virtio_pci.h" 16 17 18 //#define TRACE_VIRTIO 19 #ifdef TRACE_VIRTIO 20 # define TRACE(x...) dprintf("\33[33mvirtio_pci:\33[0m " x) 21 #else 22 # define TRACE(x...) ; 23 #endif 24 #define TRACE_ALWAYS(x...) dprintf("\33[33mvirtio_pci:\33[0m " x) 25 #define ERROR(x...) dprintf("\33[33mvirtio_pci:\33[0m " x) 26 #define CALLED(x...) TRACE("CALLED %s\n", __PRETTY_FUNCTION__) 27 28 29 #define VIRTIO_PCI_DEVICE_MODULE_NAME "busses/virtio/virtio_pci/driver_v1" 30 #define VIRTIO_PCI_SIM_MODULE_NAME "busses/virtio/virtio_pci/device/v1" 31 32 #define VIRTIO_PCI_CONTROLLER_TYPE_NAME "virtio pci controller" 33 34 typedef enum { 35 VIRTIO_IRQ_LEGACY, 36 VIRTIO_IRQ_MSI, 37 VIRTIO_IRQ_MSI_X_SHARED, 38 VIRTIO_IRQ_MSI_X, 39 } virtio_irq_type; 40 41 typedef struct { 42 virtio_sim sim; 43 uint16 queue; 44 } virtio_pci_queue_cookie; 45 46 typedef struct { 47 pci_device_module_info* pci; 48 pci_device* device; 49 addr_t base_addr; 50 uint8 irq; 51 virtio_irq_type irq_type; 52 virtio_sim sim; 53 uint16 queue_count; 54 55 device_node* node; 56 pci_info info; 57 58 virtio_pci_queue_cookie *cookies; 59 } virtio_pci_sim_info; 60 61 62 device_manager_info* gDeviceManager; 63 virtio_for_controller_interface* gVirtio; 64 static pci_x86_module_info* sPCIx86Module; 65 66 67 int32 68 virtio_pci_interrupt(void *data) 69 { 70 virtio_pci_sim_info* bus = (virtio_pci_sim_info*)data; 71 uint8 isr = bus->pci->read_io_8(bus->device, 72 bus->base_addr + VIRTIO_PCI_ISR); 73 if (isr == 0) 74 return B_UNHANDLED_INTERRUPT; 75 76 if (isr & VIRTIO_PCI_ISR_CONFIG) 77 gVirtio->config_interrupt_handler(bus->sim); 78 79 if (isr & VIRTIO_PCI_ISR_INTR) 80 gVirtio->queue_interrupt_handler(bus->sim, INT16_MAX); 81 82 return B_HANDLED_INTERRUPT; 83 } 84 85 86 int32 87 virtio_pci_config_interrupt(void *data) 88 { 89 virtio_pci_sim_info* bus = (virtio_pci_sim_info*)data; 90 gVirtio->config_interrupt_handler(bus->sim); 91 92 return B_HANDLED_INTERRUPT; 93 } 94 95 96 int32 97 virtio_pci_queue_interrupt(void *data) 98 { 99 virtio_pci_queue_cookie* cookie = (virtio_pci_queue_cookie*)data; 100 gVirtio->queue_interrupt_handler(cookie->sim, cookie->queue); 101 102 return B_HANDLED_INTERRUPT; 103 } 104 105 106 status_t 107 virtio_pci_setup_msix_interrupts(virtio_pci_sim_info* bus) 108 { 109 CALLED(); 110 uint8 irq = 0; // first irq slot 111 bus->pci->write_io_16(bus->device, bus->base_addr 112 + VIRTIO_MSI_CONFIG_VECTOR, irq); 113 if (bus->pci->read_io_16(bus->device, bus->base_addr 114 + VIRTIO_MSI_CONFIG_VECTOR) == VIRTIO_MSI_NO_VECTOR) { 115 ERROR("msix config vector incorrect\n"); 116 return B_BAD_VALUE; 117 } 118 if (bus->irq_type == VIRTIO_IRQ_MSI_X) 119 irq++; 120 121 for (uint16 queue = 0; queue < bus->queue_count; queue++) { 122 bus->pci->write_io_16(bus->device, bus->base_addr 123 + VIRTIO_PCI_QUEUE_SEL, queue); 124 bus->pci->write_io_16(bus->device, bus->base_addr 125 + VIRTIO_MSI_QUEUE_VECTOR, irq); 126 127 if (bus->pci->read_io_16(bus->device, bus->base_addr 128 + VIRTIO_MSI_QUEUE_VECTOR) == VIRTIO_MSI_NO_VECTOR) { 129 ERROR("msix queue vector incorrect\n"); 130 return B_BAD_VALUE; 131 } 132 133 if (bus->irq_type == VIRTIO_IRQ_MSI_X) 134 irq++; 135 } 136 137 return B_OK; 138 } 139 140 141 static void 142 set_sim(void* cookie, virtio_sim sim) 143 { 144 CALLED(); 145 virtio_pci_sim_info* bus = (virtio_pci_sim_info*)cookie; 146 bus->sim = sim; 147 } 148 149 150 static status_t 151 read_host_features(void* cookie, uint32 *features) 152 { 153 CALLED(); 154 virtio_pci_sim_info* bus = (virtio_pci_sim_info*)cookie; 155 156 TRACE("read_host_features() %p node %p pci %p device %p\n", bus, 157 bus->node, bus->pci, bus->device); 158 159 *features = bus->pci->read_io_32(bus->device, 160 bus->base_addr + VIRTIO_PCI_HOST_FEATURES); 161 return B_OK; 162 } 163 164 165 static status_t 166 write_guest_features(void* cookie, uint32 features) 167 { 168 CALLED(); 169 virtio_pci_sim_info* bus = (virtio_pci_sim_info*)cookie; 170 bus->pci->write_io_32(bus->device, bus->base_addr 171 + VIRTIO_PCI_GUEST_FEATURES, features); 172 return B_OK; 173 } 174 175 176 uint8 177 get_status(void* cookie) 178 { 179 CALLED(); 180 virtio_pci_sim_info* bus = (virtio_pci_sim_info*)cookie; 181 return bus->pci->read_io_8(bus->device, bus->base_addr 182 + VIRTIO_PCI_STATUS); 183 } 184 185 186 void 187 set_status(void* cookie, uint8 status) 188 { 189 CALLED(); 190 virtio_pci_sim_info* bus = (virtio_pci_sim_info*)cookie; 191 bus->pci->write_io_8(bus->device, bus->base_addr + VIRTIO_PCI_STATUS, 192 status); 193 } 194 195 196 status_t 197 read_device_config(void* cookie, uint8 _offset, void* _buffer, 198 size_t bufferSize) 199 { 200 CALLED(); 201 virtio_pci_sim_info* bus = (virtio_pci_sim_info*)cookie; 202 203 addr_t offset = bus->base_addr + VIRTIO_PCI_CONFIG(bus) + _offset; 204 uint8* buffer = (uint8*)_buffer; 205 while (bufferSize > 0) { 206 uint8 size = 4; 207 if (bufferSize == 1) { 208 size = 1; 209 *buffer = bus->pci->read_io_8(bus->device, 210 offset); 211 } else if (bufferSize <= 3) { 212 size = 2; 213 *(uint16*)buffer = bus->pci->read_io_16(bus->device, 214 offset); 215 } else { 216 *(uint32*)buffer = bus->pci->read_io_32(bus->device, 217 offset); 218 } 219 buffer += size; 220 bufferSize -= size; 221 offset += size; 222 } 223 224 return B_OK; 225 } 226 227 228 status_t 229 write_device_config(void* cookie, uint8 _offset, const void* _buffer, 230 size_t bufferSize) 231 { 232 CALLED(); 233 virtio_pci_sim_info* bus = (virtio_pci_sim_info*)cookie; 234 235 addr_t offset = bus->base_addr + VIRTIO_PCI_CONFIG(bus) + _offset; 236 const uint8* buffer = (const uint8*)_buffer; 237 while (bufferSize > 0) { 238 uint8 size = 4; 239 if (bufferSize == 1) { 240 size = 1; 241 bus->pci->write_io_8(bus->device, 242 offset, *buffer); 243 } else if (bufferSize <= 3) { 244 size = 2; 245 bus->pci->write_io_16(bus->device, 246 offset, *(const uint16*)buffer); 247 } else { 248 bus->pci->write_io_32(bus->device, 249 offset, *(const uint32*)buffer); 250 } 251 buffer += size; 252 bufferSize -= size; 253 offset += size; 254 } 255 return B_OK; 256 } 257 258 259 uint16 260 get_queue_ring_size(void* cookie, uint16 queue) 261 { 262 CALLED(); 263 virtio_pci_sim_info* bus = (virtio_pci_sim_info*)cookie; 264 bus->pci->write_io_16(bus->device, bus->base_addr + VIRTIO_PCI_QUEUE_SEL, 265 queue); 266 return bus->pci->read_io_16(bus->device, bus->base_addr 267 + VIRTIO_PCI_QUEUE_NUM); 268 } 269 270 271 status_t 272 setup_queue(void* cookie, uint16 queue, phys_addr_t phy) 273 { 274 CALLED(); 275 virtio_pci_sim_info* bus = (virtio_pci_sim_info*)cookie; 276 bus->pci->write_io_16(bus->device, bus->base_addr + VIRTIO_PCI_QUEUE_SEL, 277 queue); 278 bus->pci->write_io_32(bus->device, bus->base_addr + VIRTIO_PCI_QUEUE_PFN, 279 (uint32)phy >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); 280 return B_OK; 281 } 282 283 284 status_t 285 setup_interrupt(void* cookie, uint16 queueCount) 286 { 287 CALLED(); 288 virtio_pci_sim_info* bus = (virtio_pci_sim_info*)cookie; 289 pci_info *pciInfo = &bus->info; 290 291 bus->queue_count = queueCount; 292 293 if (sPCIx86Module != NULL) { 294 // try MSI-X 295 uint8 msixCount = sPCIx86Module->get_msix_count( 296 pciInfo->bus, pciInfo->device, pciInfo->function); 297 if (msixCount >= 1) { 298 if (msixCount >= (queueCount + 1)) { 299 uint8 vector; 300 bus->cookies = new(std::nothrow) 301 virtio_pci_queue_cookie[queueCount]; 302 if (bus->cookies != NULL 303 && sPCIx86Module->configure_msix(pciInfo->bus, 304 pciInfo->device, pciInfo->function, queueCount + 1, 305 &vector) == B_OK 306 && sPCIx86Module->enable_msix(pciInfo->bus, pciInfo->device, 307 pciInfo->function) == B_OK) { 308 TRACE_ALWAYS("using MSI-X count %u starting at %d\n", 309 queueCount + 1, vector); 310 bus->irq = vector; 311 bus->irq_type = VIRTIO_IRQ_MSI_X; 312 } else { 313 ERROR("couldn't use MSI-X\n"); 314 } 315 } else { 316 uint8 vector; 317 if (sPCIx86Module->configure_msix(pciInfo->bus, pciInfo->device, 318 pciInfo->function, queueCount + 1, &vector) == B_OK 319 && sPCIx86Module->enable_msix(pciInfo->bus, pciInfo->device, 320 pciInfo->function) == B_OK) { 321 TRACE_ALWAYS("using MSI-X vector shared %u\n", 1); 322 bus->irq = vector; 323 bus->irq_type = VIRTIO_IRQ_MSI_X_SHARED; 324 } else { 325 ERROR("couldn't use MSI-X SHARED\n"); 326 } 327 } 328 } else if (sPCIx86Module->get_msi_count( 329 pciInfo->bus, pciInfo->device, pciInfo->function) >= 1) { 330 // try MSI 331 uint8 vector; 332 if (sPCIx86Module->configure_msi(pciInfo->bus, pciInfo->device, 333 pciInfo->function, 1, &vector) == B_OK 334 && sPCIx86Module->enable_msi(pciInfo->bus, pciInfo->device, 335 pciInfo->function) == B_OK) { 336 TRACE_ALWAYS("using MSI vector %u\n", vector); 337 bus->irq = vector; 338 bus->irq_type = VIRTIO_IRQ_MSI; 339 } else { 340 ERROR("couldn't use MSI\n"); 341 } 342 } 343 } 344 if (bus->irq_type == VIRTIO_IRQ_LEGACY) { 345 bus->irq = pciInfo->u.h0.interrupt_line; 346 TRACE_ALWAYS("using legacy interrupt %u\n", bus->irq); 347 } 348 if (bus->irq == 0 || bus->irq == 0xff) { 349 ERROR("PCI IRQ not assigned\n"); 350 if (sPCIx86Module != NULL) { 351 put_module(B_PCI_X86_MODULE_NAME); 352 sPCIx86Module = NULL; 353 } 354 delete bus; 355 return B_ERROR; 356 } 357 358 if (bus->irq_type == VIRTIO_IRQ_MSI_X) { 359 status_t status = install_io_interrupt_handler(bus->irq, 360 virtio_pci_config_interrupt, bus, 0); 361 if (status != B_OK) { 362 ERROR("can't install interrupt handler\n"); 363 return status; 364 } 365 int32 irq = bus->irq + 1; 366 for (int32 queue = 0; queue < queueCount; queue++, irq++) { 367 bus->cookies[queue].sim = bus->sim; 368 bus->cookies[queue].queue = queue; 369 status_t status = install_io_interrupt_handler(irq, 370 virtio_pci_queue_interrupt, &bus->cookies[queue], 0); 371 if (status != B_OK) { 372 ERROR("can't install interrupt handler\n"); 373 return status; 374 } 375 } 376 TRACE("setup_interrupt() installed MSI-X interrupt handlers\n"); 377 virtio_pci_setup_msix_interrupts(bus); 378 } else { 379 // setup interrupt handler 380 status_t status = install_io_interrupt_handler(bus->irq, 381 virtio_pci_interrupt, bus, 0); 382 if (status != B_OK) { 383 ERROR("can't install interrupt handler\n"); 384 return status; 385 } 386 TRACE("setup_interrupt() installed legacy interrupt handler\n"); 387 } 388 389 return B_OK; 390 } 391 392 393 status_t 394 free_interrupt(void* cookie) 395 { 396 CALLED(); 397 virtio_pci_sim_info* bus = (virtio_pci_sim_info*)cookie; 398 pci_info *pciInfo = &bus->info; 399 400 if (bus->irq_type == VIRTIO_IRQ_MSI_X) { 401 remove_io_interrupt_handler(bus->irq, virtio_pci_config_interrupt, 402 bus); 403 int32 irq = bus->irq + 1; 404 for (int32 queue = 0; queue < bus->queue_count; queue++, irq++) { 405 remove_io_interrupt_handler(irq, virtio_pci_queue_interrupt, 406 &bus->cookies[queue]); 407 } 408 delete[] bus->cookies; 409 410 } else 411 remove_io_interrupt_handler(bus->irq, virtio_pci_interrupt, bus); 412 413 if (sPCIx86Module != NULL && bus->irq_type != VIRTIO_IRQ_LEGACY) { 414 sPCIx86Module->disable_msi(pciInfo->bus, pciInfo->device, 415 pciInfo->function); 416 sPCIx86Module->unconfigure_msi(pciInfo->bus, pciInfo->device, 417 pciInfo->function); 418 } 419 return B_OK; 420 } 421 422 423 void 424 notify_queue(void* cookie, uint16 queue) 425 { 426 CALLED(); 427 virtio_pci_sim_info* bus = (virtio_pci_sim_info*)cookie; 428 bus->pci->write_io_16(bus->device, bus->base_addr 429 + VIRTIO_PCI_QUEUE_NOTIFY, queue); 430 } 431 432 433 // #pragma mark - 434 435 436 static status_t 437 init_bus(device_node* node, void** bus_cookie) 438 { 439 CALLED(); 440 status_t status = B_OK; 441 442 virtio_pci_sim_info* bus = new(std::nothrow) virtio_pci_sim_info; 443 if (bus == NULL) { 444 return B_NO_MEMORY; 445 } 446 447 pci_device_module_info* pci; 448 pci_device* device; 449 { 450 device_node* parent = gDeviceManager->get_parent_node(node); 451 device_node* pciParent = gDeviceManager->get_parent_node(parent); 452 gDeviceManager->get_driver(pciParent, (driver_module_info**)&pci, 453 (void**)&device); 454 gDeviceManager->put_node(pciParent); 455 gDeviceManager->put_node(parent); 456 } 457 458 if (get_module(B_PCI_X86_MODULE_NAME, (module_info**)&sPCIx86Module) 459 != B_OK) { 460 sPCIx86Module = NULL; 461 } 462 463 bus->node = node; 464 bus->pci = pci; 465 bus->device = device; 466 bus->cookies = NULL; 467 bus->irq_type = VIRTIO_IRQ_LEGACY; 468 469 pci_info *pciInfo = &bus->info; 470 pci->get_pci_info(device, pciInfo); 471 472 // legacy interrupt 473 bus->base_addr = pciInfo->u.h0.base_registers[0]; 474 475 // enable bus master and io 476 uint16 pcicmd = pci->read_pci_config(device, PCI_command, 2); 477 pcicmd &= ~(PCI_command_memory | PCI_command_int_disable); 478 pcicmd |= PCI_command_master | PCI_command_io; 479 pci->write_pci_config(device, PCI_command, 2, pcicmd); 480 481 set_status(bus, VIRTIO_CONFIG_STATUS_RESET); 482 set_status(bus, VIRTIO_CONFIG_STATUS_ACK); 483 484 TRACE("init_bus() %p node %p pci %p device %p\n", bus, node, 485 bus->pci, bus->device); 486 487 *bus_cookie = bus; 488 return status; 489 } 490 491 492 static void 493 uninit_bus(void* bus_cookie) 494 { 495 virtio_pci_sim_info* bus = (virtio_pci_sim_info*)bus_cookie; 496 if (bus->irq_type != VIRTIO_IRQ_LEGACY) { 497 if (bus->irq_type == VIRTIO_IRQ_MSI) { 498 remove_io_interrupt_handler(bus->irq, virtio_pci_interrupt, bus); 499 500 } else { 501 int32 irq = bus->irq + 1; 502 for (uint16 queue = 0; queue < bus->queue_count; queue++, irq++) { 503 remove_io_interrupt_handler(irq, virtio_pci_queue_interrupt, 504 &bus->cookies[queue]); 505 } 506 remove_io_interrupt_handler(bus->irq, virtio_pci_config_interrupt, 507 bus); 508 } 509 510 if (sPCIx86Module != NULL) { 511 sPCIx86Module->disable_msi(bus->info.bus, 512 bus->info.device, bus->info.function); 513 sPCIx86Module->unconfigure_msi(bus->info.bus, 514 bus->info.device, bus->info.function); 515 } 516 } else 517 remove_io_interrupt_handler(bus->irq, virtio_pci_interrupt, bus); 518 if (sPCIx86Module != NULL) { 519 put_module(B_PCI_X86_MODULE_NAME); 520 sPCIx86Module = NULL; 521 } 522 delete[] bus->cookies; 523 delete bus; 524 } 525 526 527 static void 528 bus_removed(void* bus_cookie) 529 { 530 return; 531 } 532 533 534 // #pragma mark - 535 536 537 static status_t 538 register_child_devices(void* cookie) 539 { 540 CALLED(); 541 device_node* node = (device_node*)cookie; 542 device_node* parent = gDeviceManager->get_parent_node(node); 543 pci_device_module_info* pci; 544 pci_device* device; 545 gDeviceManager->get_driver(parent, (driver_module_info**)&pci, 546 (void**)&device); 547 548 uint16 pciSubDeviceId = pci->read_pci_config(device, PCI_subsystem_id, 549 2); 550 551 char prettyName[25]; 552 sprintf(prettyName, "Virtio Device %" B_PRIu16, pciSubDeviceId); 553 554 device_attr attrs[] = { 555 // properties of this controller for virtio bus manager 556 { B_DEVICE_PRETTY_NAME, B_STRING_TYPE, 557 { string: prettyName }}, 558 { B_DEVICE_FIXED_CHILD, B_STRING_TYPE, 559 { string: VIRTIO_FOR_CONTROLLER_MODULE_NAME }}, 560 561 // private data to identify the device 562 { VIRTIO_DEVICE_TYPE_ITEM, B_UINT16_TYPE, 563 { ui16: pciSubDeviceId }}, 564 { VIRTIO_VRING_ALIGNMENT_ITEM, B_UINT16_TYPE, 565 { ui16: VIRTIO_PCI_VRING_ALIGN }}, 566 { NULL } 567 }; 568 569 return gDeviceManager->register_node(node, VIRTIO_PCI_SIM_MODULE_NAME, 570 attrs, NULL, &node); 571 } 572 573 574 static status_t 575 init_device(device_node* node, void** device_cookie) 576 { 577 CALLED(); 578 *device_cookie = node; 579 return B_OK; 580 } 581 582 583 static status_t 584 register_device(device_node* parent) 585 { 586 device_attr attrs[] = { 587 {B_DEVICE_PRETTY_NAME, B_STRING_TYPE, {string: "Virtio PCI"}}, 588 {} 589 }; 590 591 return gDeviceManager->register_node(parent, VIRTIO_PCI_DEVICE_MODULE_NAME, 592 attrs, NULL, NULL); 593 } 594 595 596 static float 597 supports_device(device_node* parent) 598 { 599 CALLED(); 600 const char* bus; 601 uint16 vendorID, deviceID; 602 603 // make sure parent is a PCI Virtio device node 604 if (gDeviceManager->get_attr_string(parent, B_DEVICE_BUS, &bus, false) != B_OK 605 || gDeviceManager->get_attr_uint16(parent, B_DEVICE_VENDOR_ID, 606 &vendorID, false) < B_OK 607 || gDeviceManager->get_attr_uint16(parent, B_DEVICE_ID, &deviceID, 608 false) < B_OK) { 609 return -1; 610 } 611 612 if (strcmp(bus, "pci") != 0) 613 return 0.0f; 614 615 if (vendorID == VIRTIO_PCI_VENDORID) { 616 if (deviceID < VIRTIO_PCI_DEVICEID_MIN 617 || deviceID > VIRTIO_PCI_DEVICEID_MAX) { 618 return 0.0f; 619 } 620 621 pci_device_module_info* pci; 622 pci_device* device; 623 gDeviceManager->get_driver(parent, (driver_module_info**)&pci, 624 (void**)&device); 625 uint8 pciSubDeviceId = pci->read_pci_config(device, PCI_revision, 626 1); 627 if (pciSubDeviceId != VIRTIO_PCI_ABI_VERSION) 628 return 0.0f; 629 630 TRACE("Virtio device found! vendor 0x%04x, device 0x%04x\n", vendorID, 631 deviceID); 632 return 0.8f; 633 } 634 635 return 0.0f; 636 } 637 638 639 // #pragma mark - 640 641 642 module_dependency module_dependencies[] = { 643 { VIRTIO_FOR_CONTROLLER_MODULE_NAME, (module_info**)&gVirtio }, 644 { B_DEVICE_MANAGER_MODULE_NAME, (module_info**)&gDeviceManager }, 645 {} 646 }; 647 648 649 static virtio_sim_interface gVirtioPCIDeviceModule = { 650 { 651 { 652 VIRTIO_PCI_SIM_MODULE_NAME, 653 0, 654 NULL 655 }, 656 657 NULL, // supports device 658 NULL, // register device 659 init_bus, 660 uninit_bus, 661 NULL, // register child devices 662 NULL, // rescan 663 bus_removed, 664 }, 665 666 set_sim, 667 read_host_features, 668 write_guest_features, 669 get_status, 670 set_status, 671 read_device_config, 672 write_device_config, 673 get_queue_ring_size, 674 setup_queue, 675 setup_interrupt, 676 free_interrupt, 677 notify_queue 678 }; 679 680 681 static driver_module_info sVirtioDevice = { 682 { 683 VIRTIO_PCI_DEVICE_MODULE_NAME, 684 0, 685 NULL 686 }, 687 688 supports_device, 689 register_device, 690 init_device, 691 NULL, // uninit 692 register_child_devices, 693 NULL, // rescan 694 NULL, // device removed 695 }; 696 697 module_info* modules[] = { 698 (module_info* )&sVirtioDevice, 699 (module_info* )&gVirtioPCIDeviceModule, 700 NULL 701 }; 702