Lustre Repository Browser

Viewing: lnet_rdma.c

// SPDX-License-Identifier: GPL-2.0

/* This file is part of Lustre, http://www.lustre.org/ */

#include <linux/lnet/lnet_rdma.h>
#include <linux/libcfs/libcfs.h>

/* MAX / MIN conflict */
#include <linux/lnet/lib-lnet.h>

#define NVFS_HOLD_TIME_MS 1000

#define ERROR_PRINT_DEADLINE 3600

static atomic_t nvfs_shutdown = ATOMIC_INIT(1);
static struct nvfs_dma_rw_ops *nvfs_ops;
static struct percpu_counter nvfs_n_ops;

static inline long nvfs_count_ops(void)
{
	return percpu_counter_sum(&nvfs_n_ops);
}

static struct nvfs_dma_rw_ops *nvfs_get_ops(void)
{
	if (!nvfs_ops || atomic_read(&nvfs_shutdown))
		return NULL;

	percpu_counter_inc(&nvfs_n_ops);

	return nvfs_ops;
}

static inline void nvfs_put_ops(void)
{
	percpu_counter_dec(&nvfs_n_ops);
}

static inline bool nvfs_check_feature_set(struct nvfs_dma_rw_ops *ops)
{
	bool supported = true;
	static time64_t last_printed;

	if (unlikely(!NVIDIA_FS_CHECK_FT_SGLIST_PREP(ops))) {
		if ((ktime_get_seconds() - last_printed) > ERROR_PRINT_DEADLINE)
			CDEBUG(D_CONSOLE,
			       "NVFS sg list preparation callback missing\n");
		supported = false;
	}
	if (unlikely(!NVIDIA_FS_CHECK_FT_SGLIST_DMA(ops))) {
		if ((ktime_get_seconds() - last_printed) > ERROR_PRINT_DEADLINE)
			CDEBUG(D_CONSOLE,
			       "NVFS DMA mapping callbacks missing\n");
		supported = false;
	}
	if (unlikely(!NVIDIA_FS_CHECK_FT_GPU_PAGE(ops))) {
		if ((ktime_get_seconds() - last_printed) > ERROR_PRINT_DEADLINE)
			CDEBUG(D_CONSOLE,
			       "NVFS page identification callback missing\n");
		supported = false;
	}
	if (unlikely(!NVIDIA_FS_CHECK_FT_DEVICE_PRIORITY(ops))) {
		if ((ktime_get_seconds() - last_printed) > ERROR_PRINT_DEADLINE)
			CDEBUG(D_CONSOLE,
			       "NVFS device priority callback not missing\n");
		supported = false;
	}

	if (unlikely(!supported &&
		     ((ktime_get_seconds() - last_printed) > ERROR_PRINT_DEADLINE)))
		last_printed = ktime_get_seconds();
	else if (supported)
		last_printed = 0;

	return supported;
}

int REGISTER_FUNC(struct nvfs_dma_rw_ops *ops)
{
	if (!ops || !nvfs_check_feature_set(ops))
		return -EINVAL;

	nvfs_ops = ops;
	(void)percpu_counter_init(&nvfs_n_ops, 0, GFP_KERNEL);
	atomic_set(&nvfs_shutdown, 0);
	CDEBUG(D_NET, "registering nvfs %p\n", ops);
	return 0;
}
EXPORT_SYMBOL_GPL(REGISTER_FUNC);

void UNREGISTER_FUNC(void)
{
	(void)atomic_cmpxchg(&nvfs_shutdown, 0, 1);
	do {
		CDEBUG(D_NET, "Attempting to de-register nvfs: %ld\n",
		       nvfs_count_ops());
		msleep(NVFS_HOLD_TIME_MS);
	} while (nvfs_count_ops());
	nvfs_ops = NULL;
	percpu_counter_destroy(&nvfs_n_ops);
}
EXPORT_SYMBOL_GPL(UNREGISTER_FUNC);

unsigned int
lnet_get_dev_prio(struct device *dev, unsigned int dev_idx)
{
	unsigned int dev_prio = UINT_MAX;
	struct nvfs_dma_rw_ops *nvfs_ops;

	if (!dev)
		return dev_prio;

	nvfs_ops = nvfs_get_ops();
	if (!nvfs_ops)
		return dev_prio;

	dev_prio = nvfs_ops->nvfs_device_priority (dev, dev_idx);

	nvfs_put_ops();
	return dev_prio;
}
EXPORT_SYMBOL(lnet_get_dev_prio);

unsigned int
lnet_get_dev_idx(struct page *page)
{
	unsigned int dev_idx = UINT_MAX;
	struct nvfs_dma_rw_ops *nvfs_ops;

	nvfs_ops = nvfs_get_ops();
	if (!nvfs_ops)
		return dev_idx;

	dev_idx = nvfs_ops->nvfs_gpu_index(page);

	nvfs_put_ops();
	return dev_idx;
}

int lnet_rdma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
			   int nents, enum dma_data_direction direction)
{
	struct nvfs_dma_rw_ops *nvfs_ops = nvfs_get_ops();

	if (nvfs_ops) {
		int count;

		count = nvfs_ops->nvfs_dma_map_sg_attrs(dev,
				sg, nents, direction,
				DMA_ATTR_NO_WARN);

		if (unlikely((count == NVFS_IO_ERR))) {
			nvfs_put_ops();
			return -EIO;
		}

		if (unlikely(count == NVFS_CPU_REQ))
			nvfs_put_ops();
		else
			return count;
	}

	return 0;
}
EXPORT_SYMBOL(lnet_rdma_map_sg_attrs);

int lnet_rdma_unmap_sg(struct device *dev,
		       struct scatterlist *sg, int nents,
		       enum dma_data_direction direction)
{
	struct nvfs_dma_rw_ops *nvfs_ops = nvfs_get_ops();

	if (nvfs_ops) {
		int count;

		count = nvfs_ops->nvfs_dma_unmap_sg(dev, sg,
						    nents, direction);

		/* drop the count we got by calling nvfs_get_ops() */
		nvfs_put_ops();

		if (count) {
			nvfs_put_ops();
			return count;
		}
	}

	return 0;
}
EXPORT_SYMBOL(lnet_rdma_unmap_sg);

bool
lnet_is_rdma_only_page(struct page *page)
{
	bool is_gpu_page = false;
	struct nvfs_dma_rw_ops *nvfs_ops;

	LASSERT(page != NULL);

	nvfs_ops = nvfs_get_ops();
	if (nvfs_ops != NULL) {
		is_gpu_page = nvfs_ops->nvfs_is_gpu_page(page);
		nvfs_put_ops();
	}
	return is_gpu_page;
}
EXPORT_SYMBOL(lnet_is_rdma_only_page);