Skip to content

Fixes for non-standard Arm SoC PCIe integrations #972

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mariobalanica wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Fixes for non-standard Arm SoC PCIe integrations #972

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ac7630d
Query OS for chipset I/O cache coherency
mariobalanica Nov 19, 2025
95bc81d
Disable WC iomaps by default for unknown Arm chipsets
mariobalanica Nov 19, 2025
9861907
Never skip cache flushing on dma_map_*() calls
mariobalanica Nov 19, 2025
492cb52
Rework DMA cache maintenance helper
mariobalanica Nov 19, 2025
1007273
Fix cached DMA allocations on non-coherent hardware
mariobalanica Nov 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion kernel-open/common/inc/nv.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -864,6 +864,9 @@ static inline NvBool IS_IMEM_OFFSET(nv_state_t *nv, NvU64 offset, NvU64 length)
#define NV_ALIGN_DOWN(v,g) ((v) & ~((g) - 1))
#endif

#define NV_OS_DMA_SYNC_TO_DEVICE NVBIT(0) // CPU flush
#define NV_OS_DMA_SYNC_FROM_DEVICE NVBIT(1) // CPU invalidate
#define NV_OS_DMA_SYNC_TO_FROM_DEVICE (NV_OS_DMA_SYNC_TO_DEVICE | NV_OS_DMA_SYNC_FROM_DEVICE) // CPU flush + invalidate

/*
* driver internal interfaces
Expand Down Expand Up @@ -921,9 +924,12 @@ void NV_API_CALL nv_dma_unmap_peer (nv_dma_device_t *, NvU64, NvU6
NV_STATUS NV_API_CALL nv_dma_map_mmio (nv_dma_device_t *, NvU64, NvU64 *);
void NV_API_CALL nv_dma_unmap_mmio (nv_dma_device_t *, NvU64, NvU64);

void NV_API_CALL nv_dma_cache_invalidate (nv_dma_device_t *, void *);
void NV_API_CALL nv_dma_sync (nv_dma_device_t *, void *, NvU32);

NvBool NV_API_CALL nv_grdma_pci_topology_supported(nv_state_t *, nv_dma_device_t *);

NvBool NV_API_CALL nv_dev_is_dma_coherent (nv_dma_device_t *);

NvS32 NV_API_CALL nv_start_rc_timer (nv_state_t *);
NvS32 NV_API_CALL nv_stop_rc_timer (nv_state_t *);

Expand Down
2 changes: 0 additions & 2 deletions kernel-open/common/inc/os-interface.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,6 @@ void NV_API_CALL os_unmap_kernel_space (void *, NvU64);
void* NV_API_CALL os_map_user_space (MemoryArea *, NvU32, NvU32, void **);
void NV_API_CALL os_unmap_user_space (void *, NvU64, void *);
#endif
NV_STATUS NV_API_CALL os_flush_cpu_cache_all (void);
NV_STATUS NV_API_CALL os_flush_user_cache (void);
void NV_API_CALL os_flush_cpu_write_combine_buffer(void);
NvU8 NV_API_CALL os_io_read_byte (NvU32);
NvU16 NV_API_CALL os_io_read_word (NvU32);
Expand Down
29 changes: 13 additions & 16 deletions kernel-open/conftest.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -625,22 +625,6 @@ compile_test() {
compile_check_conftest "$CODE" "NV_SET_PAGES_ARRAY_UC_PRESENT" "" "functions"
;;

flush_cache_all)
#
# Determine if flush_cache_all() function is present
#
# flush_cache_all() was removed by commit id
# 68234df4ea79 ("arm64: kill flush_cache_all()") in 4.2 (2015-04-20)
# for aarch64
#
CODE="
#include <asm/cacheflush.h>
int conftest_flush_cache_all(void) {
return flush_cache_all();
}"
compile_check_conftest "$CODE" "NV_FLUSH_CACHE_ALL_PRESENT" "" "functions"
;;

ioremap_cache)
#
# Determine if the ioremap_cache() function is present.
Expand Down Expand Up @@ -2467,6 +2451,19 @@ compile_test() {
compile_check_conftest "$CODE" "NV_DMA_IS_DIRECT_PRESENT" "" "functions"
;;

dev_is_dma_coherent)
#
# Determine whether dev_is_dma_coherent() exists.
#
CODE="
#include <linux/dma-map-ops.h>
void conftest_dev_is_dma_coherent(void) {
dev_is_dma_coherent();
}"

compile_check_conftest "$CODE" "NV_DEV_IS_DMA_COHERENT_PRESENT" "" "functions"
;;

cmd_uphy_display_port_init)
#
# Determine if CMD_UPHY_DISPLAY_PORT_INIT enum present in bpmp-abi header
Expand Down
72 changes: 54 additions & 18 deletions kernel-open/nvidia/nv-dma.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,11 +61,18 @@ static NV_STATUS nv_dma_map_contig(
NvU64 *va
)
{
*va = dma_map_page_attrs(dma_map->dev, dma_map->pages[0], 0,
/*
* Do not set DMA_ATTR_SKIP_CPU_SYNC here even if memory is "uncached".
* On Arm, we always allocate cacheable pages and then use aliased (vmap)
* uncached mappings when necessary. Without explicit flushing right after
* allocation, previous stale data in these backing pages could be evicted
* at any point and end up clobbering memory that was already written
* through the aliased mapping. Note that no flushing will be performed on
* cache-coherent hardware.
*/
*va = dma_map_page(dma_map->dev, dma_map->pages[0], 0,
dma_map->page_count * PAGE_SIZE,
DMA_BIDIRECTIONAL,
(dma_map->cache_type == NV_MEMORY_UNCACHED) ?
DMA_ATTR_SKIP_CPU_SYNC : 0);
DMA_BIDIRECTIONAL);
if (dma_mapping_error(dma_map->dev, *va))
{
return NV_ERR_OPERATING_SYSTEM;
Expand Down Expand Up @@ -93,7 +100,7 @@ static void nv_dma_unmap_contig(nv_dma_map_t *dma_map)
dma_unmap_page_attrs(dma_map->dev, dma_map->mapping.contig.dma_addr,
dma_map->page_count * PAGE_SIZE,
DMA_BIDIRECTIONAL,
(dma_map->cache_type == NV_MEMORY_UNCACHED) ?
(dma_map->cache_type != NV_MEMORY_CACHED) ?
DMA_ATTR_SKIP_CPU_SYNC : 0);
}

Expand Down Expand Up @@ -214,6 +221,7 @@ NV_STATUS nv_map_dma_map_scatterlist(nv_dma_map_t *dma_map)
nv_dma_submap_t *submap;
NvU64 i;

/* See the comment in nv_dma_map_contig() */
NV_FOR_EACH_DMA_SUBMAP(dma_map, submap, i)
{
/* Imported SGTs will have already been mapped by the exporter. */
Expand Down Expand Up @@ -256,9 +264,11 @@ void nv_unmap_dma_map_scatterlist(nv_dma_map_t *dma_map)
continue;
}

dma_unmap_sg(dma_map->dev, submap->sgt.sgl,
dma_unmap_sg_attrs(dma_map->dev, submap->sgt.sgl,
submap->sgt.orig_nents,
DMA_BIDIRECTIONAL);
DMA_BIDIRECTIONAL,
(dma_map->cache_type != NV_MEMORY_CACHED) ?
DMA_ATTR_SKIP_CPU_SYNC : 0);
}
}

Expand Down Expand Up @@ -870,28 +880,36 @@ void NV_API_CALL nv_dma_unmap_mmio
}

/*
* Invalidate DMA mapping in CPU caches by "syncing" to the device.
* Flush/invalidate DMA mapping in CPU caches by "syncing" to the device.
*
* This is only implemented for ARM platforms, since other supported
* platforms are cache coherent and have not required this (we
* explicitly haven't supported SWIOTLB bounce buffering either where
* this would be needed).
*/
void NV_API_CALL nv_dma_cache_invalidate
void NV_API_CALL nv_dma_sync
(
nv_dma_device_t *dma_dev,
void *priv
void *priv,
NvU32 dir
)
{
#if defined(NVCPU_AARCH64)
nv_dma_map_t *dma_map = priv;

if (dma_map->contiguous)
{
dma_sync_single_for_device(dma_dev->dev,
dma_map->mapping.contig.dma_addr,
(size_t) PAGE_SIZE * dma_map->page_count,
DMA_FROM_DEVICE);
if (dir & NV_OS_DMA_SYNC_TO_DEVICE)
dma_sync_single_for_device(dma_dev->dev,
dma_map->mapping.contig.dma_addr,
(size_t)PAGE_SIZE * dma_map->page_count,
DMA_TO_DEVICE);

if (dir & NV_OS_DMA_SYNC_FROM_DEVICE)
dma_sync_single_for_cpu(dma_dev->dev,
dma_map->mapping.contig.dma_addr,
(size_t)PAGE_SIZE * dma_map->page_count,
DMA_FROM_DEVICE);
}
else
{
Expand All @@ -900,15 +918,33 @@ void NV_API_CALL nv_dma_cache_invalidate

NV_FOR_EACH_DMA_SUBMAP(dma_map, submap, i)
{
dma_sync_sg_for_device(dma_dev->dev,
submap->sgt.sgl,
submap->sgt.orig_nents,
DMA_FROM_DEVICE);
if (dir & NV_OS_DMA_SYNC_TO_DEVICE)
dma_sync_sg_for_device(dma_dev->dev,
submap->sgt.sgl,
submap->sgt.orig_nents,
DMA_TO_DEVICE);

if (dir & NV_OS_DMA_SYNC_FROM_DEVICE)
dma_sync_sg_for_cpu(dma_dev->dev,
submap->sgt.sgl,
submap->sgt.orig_nents,
DMA_FROM_DEVICE);
}
}
#endif
}

NvBool NV_API_CALL nv_dev_is_dma_coherent
(
nv_dma_device_t *dma_dev
)
{
#if defined(NV_DEV_IS_DMA_COHERENT_PRESENT)
return dev_is_dma_coherent(dma_dev->dev);
#endif
return true;
}

#if defined(NV_DRM_AVAILABLE)

static inline void
Expand Down
2 changes: 1 addition & 1 deletion kernel-open/nvidia/nvidia.Kbuild
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ NV_CONFTEST_FUNCTION_COMPILE_TESTS += phys_to_dma
NV_CONFTEST_FUNCTION_COMPILE_TESTS += pci_rebar_get_possible_sizes
NV_CONFTEST_FUNCTION_COMPILE_TESTS += get_backlight_device_by_name
NV_CONFTEST_FUNCTION_COMPILE_TESTS += dma_direct_map_resource
NV_CONFTEST_FUNCTION_COMPILE_TESTS += flush_cache_all
NV_CONFTEST_FUNCTION_COMPILE_TESTS += dev_is_dma_coherent
NV_CONFTEST_FUNCTION_COMPILE_TESTS += vmf_insert_pfn
NV_CONFTEST_FUNCTION_COMPILE_TESTS += jiffies_to_timespec
NV_CONFTEST_FUNCTION_COMPILE_TESTS += ktime_get_raw_ts64
Expand Down
51 changes: 0 additions & 51 deletions kernel-open/nvidia/os-interface.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1034,57 +1034,6 @@ void NV_API_CALL os_unmap_kernel_space(
nv_iounmap(addr, size_bytes);
}

#if NVCPU_IS_AARCH64

static inline void nv_flush_cache_cpu(void *info)
{
if (!nvos_is_chipset_io_coherent())
{
#if defined(NV_FLUSH_CACHE_ALL_PRESENT)
flush_cache_all();
#else
WARN_ONCE(0, "kernel does not provide flush_cache_all()\n");
#endif
}
}

// flush the cache of all cpus
NV_STATUS NV_API_CALL os_flush_cpu_cache_all(void)
{
on_each_cpu(nv_flush_cache_cpu, NULL, 1);
return NV_OK;
}

NV_STATUS NV_API_CALL os_flush_user_cache(void)
{
if (!NV_MAY_SLEEP())
{
return NV_ERR_NOT_SUPPORTED;
}

//
// The Linux kernel does not export an interface for flushing a range,
// although it is possible. For now, just flush the entire cache to be
// safe.
//
on_each_cpu(nv_flush_cache_cpu, NULL, 1);
return NV_OK;
}

#else // NVCPU_IS_AARCH64

NV_STATUS NV_API_CALL os_flush_cpu_cache_all(void)
{
return NV_ERR_NOT_SUPPORTED;
}

NV_STATUS NV_API_CALL os_flush_user_cache(void)
{
return NV_ERR_NOT_SUPPORTED;
}

#endif

void NV_API_CALL os_flush_cpu_write_combine_buffer(void)
{
#if defined(NVCPU_X86_64)
Expand Down
8 changes: 4 additions & 4 deletions src/nvidia/arch/nvalloc/common/inc/nvcst.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ CHIPSET_SETUP_FUNC(PLDA_XpressRichAXI_setupFunc)
CHIPSET_SETUP_FUNC(Riscv_generic_setupFunc)
CHIPSET_SETUP_FUNC(Intel_A70D_setupFunc)
CHIPSET_SETUP_FUNC(AMD_14D8_setupFunc)

CHIPSET_SETUP_FUNC(Generic_setupFunc)

// Keep string length <=32 (including termination) to avoid string copy overflow
CSINFO chipsetInfo[] =
Expand Down Expand Up @@ -276,8 +276,8 @@ CSINFO chipsetInfo[] =
{PCI_VENDOR_ID_AMPERE, 0xE110, CS_AMPERE_ALTRA, "Ampere Altra", Ampere_Altra_setupFunc},
{PCI_VENDOR_ID_ARM, 0x0100, CS_ARM_NEOVERSEN1, "Arm Neoverse N1", Arm_NeoverseN1_setupFunc},
{PCI_VENDOR_ID_HYGON, 0x790E, CS_HYGON_C86, "Hygon-C86-7151", NULL},
{PCI_VENDOR_ID_MARVELL, 0xA02D, CS_MARVELL_OCTEON_CN96XX, "Marvell Octeon CN96xx", ARMV8_generic_setupFunc},
{PCI_VENDOR_ID_MARVELL, 0xA02D, CS_MARVELL_OCTEON_CN98XX, "Marvell Octeon CN98xx", ARMV8_generic_setupFunc},
{PCI_VENDOR_ID_MARVELL, 0xA02D, CS_MARVELL_OCTEON_CN96XX, "Marvell Octeon CN96xx", NULL},
{PCI_VENDOR_ID_MARVELL, 0xA02D, CS_MARVELL_OCTEON_CN98XX, "Marvell Octeon CN98xx", NULL},
{PCI_VENDOR_ID_SIFIVE, 0x0000, CS_SIFIVE_FU740_C000, "SiFive FU740-000", Riscv_generic_setupFunc},
{PCI_VENDOR_ID_PLDA, 0x1111, CS_PLDA_XPRESSRICH_AXI_REF, "XpressRich-AXI Ref Design", PLDA_XpressRichAXI_setupFunc},
{PCI_VENDOR_ID_AMPERE, 0xE200, CS_AMPERE_AMPEREONE160, "Ampere AmpereOne-160", Ampere_AmpereOne_setupFunc},
Expand All @@ -302,7 +302,7 @@ CSINFO chipsetInfo[] =
///////////////////////////////////////////////////////////////////////////////////////////////////

// last element must have chipset CS_UNKNOWN (zero)
{0, 0, CS_UNKNOWN, "Unknown", NULL}
{0, 0, CS_UNKNOWN, "Unknown", Generic_setupFunc}
};


Expand Down
8 changes: 7 additions & 1 deletion src/nvidia/arch/nvalloc/unix/include/nv.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -864,6 +864,9 @@ static inline NvBool IS_IMEM_OFFSET(nv_state_t *nv, NvU64 offset, NvU64 length)
#define NV_ALIGN_DOWN(v,g) ((v) & ~((g) - 1))
#endif

#define NV_OS_DMA_SYNC_TO_DEVICE NVBIT(0) // CPU flush
#define NV_OS_DMA_SYNC_FROM_DEVICE NVBIT(1) // CPU invalidate
#define NV_OS_DMA_SYNC_TO_FROM_DEVICE (NV_OS_DMA_SYNC_TO_DEVICE | NV_OS_DMA_SYNC_FROM_DEVICE) // CPU flush + invalidate

/*
* driver internal interfaces
Expand Down Expand Up @@ -921,9 +924,12 @@ void NV_API_CALL nv_dma_unmap_peer (nv_dma_device_t *, NvU64, NvU6
NV_STATUS NV_API_CALL nv_dma_map_mmio (nv_dma_device_t *, NvU64, NvU64 *);
void NV_API_CALL nv_dma_unmap_mmio (nv_dma_device_t *, NvU64, NvU64);

void NV_API_CALL nv_dma_cache_invalidate (nv_dma_device_t *, void *);
void NV_API_CALL nv_dma_sync (nv_dma_device_t *, void *, NvU32);

NvBool NV_API_CALL nv_grdma_pci_topology_supported(nv_state_t *, nv_dma_device_t *);

NvBool NV_API_CALL nv_dev_is_dma_coherent (nv_dma_device_t *);

NvS32 NV_API_CALL nv_start_rc_timer (nv_state_t *);
NvS32 NV_API_CALL nv_stop_rc_timer (nv_state_t *);

Expand Down
2 changes: 0 additions & 2 deletions src/nvidia/arch/nvalloc/unix/include/os-interface.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,8 +105,6 @@ NvBool NV_API_CALL os_pci_remove_supported (void);
void NV_API_CALL os_pci_remove (void *);
void* NV_API_CALL os_map_kernel_space (NvU64, NvU64, NvU32);
void NV_API_CALL os_unmap_kernel_space (void *, NvU64);
NV_STATUS NV_API_CALL os_flush_cpu_cache_all (void);
NV_STATUS NV_API_CALL os_flush_user_cache (void);
void NV_API_CALL os_flush_cpu_write_combine_buffer(void);
NvU8 NV_API_CALL os_io_read_byte (NvU32);
NvU16 NV_API_CALL os_io_read_word (NvU32);
Expand Down
Loading