14b88e723SAugustin Cavalier /*
2138d4bbdSAugustin Cavalier * Copyright 2019-2022, Haiku, Inc. All rights reserved.
34b88e723SAugustin Cavalier * Distributed under the terms of the MIT License.
44b88e723SAugustin Cavalier *
54b88e723SAugustin Cavalier * Authors:
64b88e723SAugustin Cavalier * Augustin Cavalier <waddlesplash>
74b88e723SAugustin Cavalier */
84b88e723SAugustin Cavalier
94b88e723SAugustin Cavalier #include <debug.h>
104b88e723SAugustin Cavalier #include <kernel/vm/vm.h>
114b88e723SAugustin Cavalier #include <PCI.h>
124b88e723SAugustin Cavalier
134b88e723SAugustin Cavalier extern "C" {
144b88e723SAugustin Cavalier #include "nvme.h"
154b88e723SAugustin Cavalier #include "nvme_log.h"
164b88e723SAugustin Cavalier #include "nvme_mem.h"
174b88e723SAugustin Cavalier #include "nvme_pci.h"
184b88e723SAugustin Cavalier }
194b88e723SAugustin Cavalier
204b88e723SAugustin Cavalier
214b88e723SAugustin Cavalier static pci_module_info* sPCIModule = NULL;
224b88e723SAugustin Cavalier
234b88e723SAugustin Cavalier
244b88e723SAugustin Cavalier // #pragma mark - memory
254b88e723SAugustin Cavalier
264b88e723SAugustin Cavalier
274b88e723SAugustin Cavalier int
nvme_mem_init()284b88e723SAugustin Cavalier nvme_mem_init()
294b88e723SAugustin Cavalier {
304b88e723SAugustin Cavalier /* nothing to do */
314b88e723SAugustin Cavalier return 0;
324b88e723SAugustin Cavalier }
334b88e723SAugustin Cavalier
344b88e723SAugustin Cavalier
354b88e723SAugustin Cavalier void
nvme_mem_cleanup()364b88e723SAugustin Cavalier nvme_mem_cleanup()
374b88e723SAugustin Cavalier {
384b88e723SAugustin Cavalier /* nothing to do */
394b88e723SAugustin Cavalier }
404b88e723SAugustin Cavalier
414b88e723SAugustin Cavalier
424b88e723SAugustin Cavalier void*
nvme_mem_alloc_node(size_t size,size_t align,unsigned int node_id,phys_addr_t * paddr)434b88e723SAugustin Cavalier nvme_mem_alloc_node(size_t size, size_t align, unsigned int node_id,
444b88e723SAugustin Cavalier phys_addr_t* paddr)
454b88e723SAugustin Cavalier {
464b88e723SAugustin Cavalier size = ROUNDUP(size, B_PAGE_SIZE);
474b88e723SAugustin Cavalier
484b88e723SAugustin Cavalier virtual_address_restrictions virtualRestrictions = {};
494b88e723SAugustin Cavalier
504b88e723SAugustin Cavalier physical_address_restrictions physicalRestrictions = {};
514b88e723SAugustin Cavalier physicalRestrictions.alignment = align;
524b88e723SAugustin Cavalier
534b88e723SAugustin Cavalier void* address;
544b88e723SAugustin Cavalier area_id area = create_area_etc(B_SYSTEM_TEAM, "nvme physical buffer",
554b88e723SAugustin Cavalier size, B_CONTIGUOUS, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA,
564b88e723SAugustin Cavalier 0, 0, &virtualRestrictions, &physicalRestrictions, &address);
574b88e723SAugustin Cavalier if (area < 0)
584b88e723SAugustin Cavalier return NULL;
594b88e723SAugustin Cavalier
604b88e723SAugustin Cavalier if (paddr != NULL)
614b88e723SAugustin Cavalier *paddr = nvme_mem_vtophys(address);
624b88e723SAugustin Cavalier return address;
634b88e723SAugustin Cavalier }
644b88e723SAugustin Cavalier
654b88e723SAugustin Cavalier
664b88e723SAugustin Cavalier void*
nvme_malloc_node(size_t size,size_t align,unsigned int node_id)674b88e723SAugustin Cavalier nvme_malloc_node(size_t size, size_t align, unsigned int node_id)
684b88e723SAugustin Cavalier {
694b88e723SAugustin Cavalier return nvme_mem_alloc_node(size, align, node_id, NULL);
704b88e723SAugustin Cavalier }
714b88e723SAugustin Cavalier
724b88e723SAugustin Cavalier
734b88e723SAugustin Cavalier void
nvme_free(void * addr)744b88e723SAugustin Cavalier nvme_free(void* addr)
754b88e723SAugustin Cavalier {
764b88e723SAugustin Cavalier delete_area(area_for(addr));
774b88e723SAugustin Cavalier }
784b88e723SAugustin Cavalier
794b88e723SAugustin Cavalier
804b88e723SAugustin Cavalier phys_addr_t
nvme_mem_vtophys(void * vaddr)814b88e723SAugustin Cavalier nvme_mem_vtophys(void* vaddr)
824b88e723SAugustin Cavalier {
834b88e723SAugustin Cavalier physical_entry entry;
841b3ccbc5SAugustin Cavalier status_t status = get_memory_map((void*)vaddr, 1, &entry, 1);
854b88e723SAugustin Cavalier if (status != B_OK) {
86d4304f43SAugustin Cavalier panic("nvme: get_memory_map failed for %p: %s\n",
87d4304f43SAugustin Cavalier (void*)vaddr, strerror(status));
884b88e723SAugustin Cavalier return NVME_VTOPHYS_ERROR;
894b88e723SAugustin Cavalier }
904b88e723SAugustin Cavalier
914b88e723SAugustin Cavalier return entry.address;
924b88e723SAugustin Cavalier }
934b88e723SAugustin Cavalier
944b88e723SAugustin Cavalier
954b88e723SAugustin Cavalier // #pragma mark - PCI
964b88e723SAugustin Cavalier
974b88e723SAugustin Cavalier
984b88e723SAugustin Cavalier int
nvme_pci_init()994b88e723SAugustin Cavalier nvme_pci_init()
1004b88e723SAugustin Cavalier {
1014b88e723SAugustin Cavalier status_t status = get_module(B_PCI_MODULE_NAME,
1024b88e723SAugustin Cavalier (module_info**)&sPCIModule);
1034b88e723SAugustin Cavalier return status;
1044b88e723SAugustin Cavalier }
1054b88e723SAugustin Cavalier
1064b88e723SAugustin Cavalier
1074b88e723SAugustin Cavalier int
nvme_pcicfg_read32(struct pci_device * dev,uint32_t * value,uint32_t offset)1084b88e723SAugustin Cavalier nvme_pcicfg_read32(struct pci_device* dev, uint32_t* value, uint32_t offset)
1094b88e723SAugustin Cavalier {
1104b88e723SAugustin Cavalier *value = sPCIModule->read_pci_config(dev->bus, dev->dev, dev->func, offset,
1114b88e723SAugustin Cavalier sizeof(*value));
1124b88e723SAugustin Cavalier return 0;
1134b88e723SAugustin Cavalier }
1144b88e723SAugustin Cavalier
1154b88e723SAugustin Cavalier
1164b88e723SAugustin Cavalier int
nvme_pcicfg_write32(struct pci_device * dev,uint32_t value,uint32_t offset)1174b88e723SAugustin Cavalier nvme_pcicfg_write32(struct pci_device* dev, uint32_t value, uint32_t offset)
1184b88e723SAugustin Cavalier {
1194b88e723SAugustin Cavalier sPCIModule->write_pci_config(dev->bus, dev->dev, dev->func, offset,
1204b88e723SAugustin Cavalier sizeof(value), value);
1214b88e723SAugustin Cavalier return 0;
1224b88e723SAugustin Cavalier }
1234b88e723SAugustin Cavalier
1244b88e723SAugustin Cavalier
125138d4bbdSAugustin Cavalier void
nvme_pcicfg_get_bar_addr_len(void * devhandle,unsigned int bar,uint64_t * _addr,uint64_t * _size)126138d4bbdSAugustin Cavalier nvme_pcicfg_get_bar_addr_len(void* devhandle, unsigned int bar,
127138d4bbdSAugustin Cavalier uint64_t* _addr, uint64_t* _size)
1284b88e723SAugustin Cavalier {
1294b88e723SAugustin Cavalier struct pci_device* dev = (struct pci_device*)devhandle;
1304b88e723SAugustin Cavalier pci_info* info = (pci_info*)dev->pci_info;
1314b88e723SAugustin Cavalier
1326489b77dSAugustin Cavalier uint64 addr = info->u.h0.base_registers[bar];
1336489b77dSAugustin Cavalier uint64 size = info->u.h0.base_register_sizes[bar];
134f7179d9fSAugustin Cavalier if ((info->u.h0.base_register_flags[bar] & PCI_address_type) == PCI_address_type_64) {
135f7179d9fSAugustin Cavalier addr |= (uint64)info->u.h0.base_registers[bar + 1] << 32;
136f7179d9fSAugustin Cavalier size |= (uint64)info->u.h0.base_register_sizes[bar + 1] << 32;
1370877e4d7SX512 }
1384b88e723SAugustin Cavalier
139138d4bbdSAugustin Cavalier *_addr = addr;
140138d4bbdSAugustin Cavalier *_size = size;
141138d4bbdSAugustin Cavalier }
142138d4bbdSAugustin Cavalier
143138d4bbdSAugustin Cavalier
144138d4bbdSAugustin Cavalier int
nvme_pcicfg_map_bar(void * devhandle,unsigned int bar,bool read_only,void ** mapped_addr)145138d4bbdSAugustin Cavalier nvme_pcicfg_map_bar(void* devhandle, unsigned int bar, bool read_only,
146138d4bbdSAugustin Cavalier void** mapped_addr)
147138d4bbdSAugustin Cavalier {
1486489b77dSAugustin Cavalier uint64 addr, size;
149138d4bbdSAugustin Cavalier nvme_pcicfg_get_bar_addr_len(devhandle, bar, &addr, &size);
150138d4bbdSAugustin Cavalier
1516489b77dSAugustin Cavalier area_id area = map_physical_memory("nvme mapped bar", (phys_addr_t)addr, (size_t)size,
152f7179d9fSAugustin Cavalier B_ANY_KERNEL_ADDRESS, B_KERNEL_READ_AREA | (read_only ? 0 : B_KERNEL_WRITE_AREA),
1534b88e723SAugustin Cavalier mapped_addr);
1544b88e723SAugustin Cavalier if (area < B_OK)
1554b88e723SAugustin Cavalier return area;
1564b88e723SAugustin Cavalier
1574b88e723SAugustin Cavalier return 0;
1584b88e723SAugustin Cavalier }
1594b88e723SAugustin Cavalier
1604b88e723SAugustin Cavalier
1614b88e723SAugustin Cavalier int
nvme_pcicfg_map_bar_write_combine(void * devhandle,unsigned int bar,void ** mapped_addr)1624b88e723SAugustin Cavalier nvme_pcicfg_map_bar_write_combine(void* devhandle, unsigned int bar,
1634b88e723SAugustin Cavalier void** mapped_addr)
1644b88e723SAugustin Cavalier {
1654b88e723SAugustin Cavalier status_t status = nvme_pcicfg_map_bar(devhandle, bar, false, mapped_addr);
1664b88e723SAugustin Cavalier if (status != 0)
1674b88e723SAugustin Cavalier return status;
1684b88e723SAugustin Cavalier
1694b88e723SAugustin Cavalier // Turn on write combining for the area
1704b88e723SAugustin Cavalier status = vm_set_area_memory_type(area_for(*mapped_addr),
171*5c1f2319SAugustin Cavalier nvme_mem_vtophys(*mapped_addr), B_WRITE_COMBINING_MEMORY);
1724b88e723SAugustin Cavalier if (status != 0)
1734b88e723SAugustin Cavalier nvme_pcicfg_unmap_bar(devhandle, bar, *mapped_addr);
1744b88e723SAugustin Cavalier return status;
1754b88e723SAugustin Cavalier }
1764b88e723SAugustin Cavalier
1774b88e723SAugustin Cavalier
1784b88e723SAugustin Cavalier int
nvme_pcicfg_unmap_bar(void * devhandle,unsigned int bar,void * addr)1794b88e723SAugustin Cavalier nvme_pcicfg_unmap_bar(void* devhandle, unsigned int bar, void* addr)
1804b88e723SAugustin Cavalier {
1814b88e723SAugustin Cavalier return delete_area(area_for(addr));
1824b88e723SAugustin Cavalier }
1834b88e723SAugustin Cavalier
1844b88e723SAugustin Cavalier
1854b88e723SAugustin Cavalier // #pragma mark - logging
1864b88e723SAugustin Cavalier
1874b88e723SAugustin Cavalier
1884b88e723SAugustin Cavalier void
nvme_log(enum nvme_log_level level,const char * format,...)1894b88e723SAugustin Cavalier nvme_log(enum nvme_log_level level, const char *format, ...)
1904b88e723SAugustin Cavalier {
1914b88e723SAugustin Cavalier va_list ap;
1924b88e723SAugustin Cavalier
1934b88e723SAugustin Cavalier va_start(ap, format);
1944b88e723SAugustin Cavalier nvme_vlog(level, format, ap);
1954b88e723SAugustin Cavalier va_end(ap);
1964b88e723SAugustin Cavalier }
1974b88e723SAugustin Cavalier
1984b88e723SAugustin Cavalier
1994b88e723SAugustin Cavalier void
nvme_vlog(enum nvme_log_level level,const char * format,va_list ap)2004b88e723SAugustin Cavalier nvme_vlog(enum nvme_log_level level, const char *format, va_list ap)
2014b88e723SAugustin Cavalier {
2024b88e723SAugustin Cavalier dvprintf(format, ap);
2034b88e723SAugustin Cavalier }
204