| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20190319215401.6562-7-sonal.santan@xilinx.com>
Date: Tue, 19 Mar 2019 14:54:01 -0700
From: <sonal.santan@...inx.com>
To: <dri-devel@...ts.freedesktop.org>
CC: <linux-kernel@...r.kernel.org>, <gregkh@...uxfoundation.org>,
<airlied@...hat.com>, <cyrilc@...inx.com>, <michals@...inx.com>,
<lizhih@...inx.com>, <hyunk@...inx.com>,
Sonal Santan <sonal.santan@...inx.com>
Subject: [RFC PATCH Xilinx Alveo 6/6] Add user physical function driver
From: Sonal Santan <sonal.santan@...inx.com>
Signed-off-by: Sonal Santan <sonal.santan@...inx.com>
---
drivers/gpu/drm/xocl/userpf/common.h | 157 +++
drivers/gpu/drm/xocl/userpf/xocl_bo.c | 1255 ++++++++++++++++++++++
drivers/gpu/drm/xocl/userpf/xocl_bo.h | 119 ++
drivers/gpu/drm/xocl/userpf/xocl_drm.c | 640 +++++++++++
drivers/gpu/drm/xocl/userpf/xocl_drv.c | 743 +++++++++++++
drivers/gpu/drm/xocl/userpf/xocl_ioctl.c | 396 +++++++
drivers/gpu/drm/xocl/userpf/xocl_sysfs.c | 344 ++++++
7 files changed, 3654 insertions(+)
create mode 100644 drivers/gpu/drm/xocl/userpf/common.h
create mode 100644 drivers/gpu/drm/xocl/userpf/xocl_bo.c
create mode 100644 drivers/gpu/drm/xocl/userpf/xocl_bo.h
create mode 100644 drivers/gpu/drm/xocl/userpf/xocl_drm.c
create mode 100644 drivers/gpu/drm/xocl/userpf/xocl_drv.c
create mode 100644 drivers/gpu/drm/xocl/userpf/xocl_ioctl.c
create mode 100644 drivers/gpu/drm/xocl/userpf/xocl_sysfs.c
diff --git a/drivers/gpu/drm/xocl/userpf/common.h b/drivers/gpu/drm/xocl/userpf/common.h
new file mode 100644
index 000000000000..c7dd4a68441c
--- /dev/null
+++ b/drivers/gpu/drm/xocl/userpf/common.h
@@ -0,0 +1,157 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+/*
+ * Copyright (C) 2016-2019 Xilinx, Inc. All rights reserved.
+ *
+ * Authors:
+ * Lizhi Hou <lizhi.hou@...inx.com>
+ *
+ */
+
+#ifndef _USERPF_COMMON_H
+#define _USERPF_COMMON_H
+
+#include "../xocl_drv.h"
+#include "xocl_bo.h"
+#include "../xocl_drm.h"
+#include <drm/xocl_drm.h>
+#include <linux/hashtable.h>
+
+#define XOCL_DRIVER_DESC "Xilinx PCIe Accelerator Device Manager"
+#define XOCL_DRIVER_DATE "20180612"
+#define XOCL_DRIVER_MAJOR 2018
+#define XOCL_DRIVER_MINOR 2
+#define XOCL_DRIVER_PATCHLEVEL 8
+
+#define XOCL_MAX_CONCURRENT_CLIENTS 32
+
+#define XOCL_DRIVER_VERSION \
+ __stringify(XOCL_DRIVER_MAJOR) "." \
+ __stringify(XOCL_DRIVER_MINOR) "." \
+ __stringify(XOCL_DRIVER_PATCHLEVEL)
+
+#define XOCL_DRIVER_VERSION_NUMBER \
+ ((XOCL_DRIVER_MAJOR)*1000 + (XOCL_DRIVER_MINOR)*100 + \
+ XOCL_DRIVER_PATCHLEVEL)
+
+#define userpf_err(d, args...) \
+ xocl_err(&XDEV(d)->pdev->dev, ##args)
+#define userpf_info(d, args...) \
+ xocl_info(&XDEV(d)->pdev->dev, ##args)
+#define userpf_dbg(d, args...) \
+ xocl_dbg(&XDEV(d)->pdev->dev, ##args)
+
+#define xocl_get_root_dev(dev, root) \
+ for (root = dev; root->bus && root->bus->self; root = root->bus->self)
+
+#define XOCL_USER_PROC_HASH_SZ 256
+
+#define XOCL_U32_MASK 0xFFFFFFFF
+
+#define MAX_SLOTS 128
+#define MAX_CUS 128
+#define MAX_U32_SLOT_MASKS (((MAX_SLOTS-1)>>5) + 1)
+#define MAX_U32_CU_MASKS (((MAX_CUS-1)>>5) + 1)
+#define MAX_DEPS 8
+
+#define XOCL_DRM_FREE_MALLOC
+
+#define XOCL_PA_SECTION_SHIFT 28
+
+struct xocl_dev {
+ struct xocl_dev_core core;
+
+ bool offline;
+
+ /* health thread */
+ struct task_struct *health_thread;
+ struct xocl_health_thread_arg thread_arg;
+
+ u32 p2p_bar_idx;
+ resource_size_t p2p_bar_len;
+ void * __iomem p2p_bar_addr;
+
+ /*should be removed after mailbox is supported */
+ struct percpu_ref ref;
+ struct completion cmp;
+
+ struct dev_pagemap pgmap;
+ struct list_head ctx_list;
+ struct mutex ctx_list_lock;
+ unsigned int needs_reset; /* bool aligned */
+ atomic_t outstanding_execs;
+ atomic64_t total_execs;
+ void *p2p_res_grp;
+};
+
+/**
+ * struct client_ctx: Manage user space client attached to device
+ *
+ * @link: Client context is added to list in device
+ * @xclbin_id: UUID for xclbin loaded by client, or nullid if no xclbin loaded
+ * @xclbin_locked: Flag to denote that this context locked the xclbin
+ * @trigger: Poll wait counter for number of completed exec buffers
+ * @outstanding_execs: Counter for number outstanding exec buffers
+ * @abort: Flag to indicate that this context has detached from user space (ctrl-c)
+ * @num_cus: Number of resources (CUs) explcitly aquired
+ * @lock: Mutex lock for exclusive access
+ * @cu_bitmap: CUs reserved by this context, may contain implicit resources
+ */
+struct client_ctx {
+ struct list_head link;
+ uuid_t xclbin_id;
+ unsigned int xclbin_locked;
+ unsigned int abort;
+ unsigned int num_cus; /* number of resource locked explicitly by client */
+ atomic_t trigger; /* count of poll notification to acknowledge */
+ atomic_t outstanding_execs;
+ struct mutex lock;
+ struct xocl_dev *xdev;
+ DECLARE_BITMAP(cu_bitmap, MAX_CUS); /* may contain implicitly aquired resources such as CDMA */
+ struct pid *pid;
+};
+
+struct xocl_mm_wrapper {
+ struct drm_mm *mm;
+ struct drm_xocl_mm_stat *mm_usage_stat;
+ uint64_t start_addr;
+ uint64_t size;
+ uint32_t ddr;
+ struct hlist_node node;
+};
+
+/* ioctl functions */
+int xocl_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
+int xocl_execbuf_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp);
+int xocl_ctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
+int xocl_user_intr_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp);
+int xocl_read_axlf_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp);
+int xocl_hot_reset_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp);
+int xocl_reclock_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp);
+
+/* sysfs functions */
+int xocl_init_sysfs(struct device *dev);
+void xocl_fini_sysfs(struct device *dev);
+
+/* helper functions */
+int64_t xocl_hot_reset(struct xocl_dev *xdev, bool force);
+void xocl_p2p_mem_release(struct xocl_dev *xdev, bool recov_bar_sz);
+int xocl_p2p_mem_reserve(struct xocl_dev *xdev);
+int xocl_get_p2p_bar(struct xocl_dev *xdev, u64 *bar_size);
+int xocl_pci_resize_resource(struct pci_dev *dev, int resno, int size);
+void xocl_reset_notify(struct pci_dev *pdev, bool prepare);
+void user_pci_reset_prepare(struct pci_dev *pdev);
+void user_pci_reset_done(struct pci_dev *pdev);
+
+uint get_live_client_size(struct xocl_dev *xdev);
+void reset_notify_client_ctx(struct xocl_dev *xdev);
+
+void get_pcie_link_info(struct xocl_dev *xdev,
+ unsigned short *link_width, unsigned short *link_speed, bool is_cap);
+int xocl_reclock(struct xocl_dev *xdev, void *data);
+#endif
diff --git a/drivers/gpu/drm/xocl/userpf/xocl_bo.c b/drivers/gpu/drm/xocl/userpf/xocl_bo.c
new file mode 100644
index 000000000000..546ce5f7e428
--- /dev/null
+++ b/drivers/gpu/drm/xocl/userpf/xocl_bo.c
@@ -0,0 +1,1255 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * A GEM style device manager for PCIe based OpenCL accelerators.
+ *
+ * Copyright (C) 2016-2019 Xilinx, Inc. All rights reserved.
+ *
+ * Authors:
+ * Sonal Santan <sonal.santan@...inx.com>
+ * Sarabjeet Singh <sarabjeet.singh@...inx.com>
+ *
+ */
+
+#include <linux/bitops.h>
+#include <linux/swap.h>
+#include <linux/dma-buf.h>
+#include <linux/pagemap.h>
+#include <linux/version.h>
+#include <drm/drmP.h>
+#include "common.h"
+
+#ifdef _XOCL_BO_DEBUG
+#define BO_ENTER(fmt, args...) \
+ pr_info("[BO] Entering %s:"fmt"\n", __func__, ##args)
+#define BO_DEBUG(fmt, args...) \
+ pr_info("[BO] %s:%d:"fmt"\n", __func__, __LINE__, ##args)
+#else
+#define BO_ENTER(fmt, args...)
+#define BO_DEBUG(fmt, args...)
+#endif
+
+#if defined(XOCL_DRM_FREE_MALLOC)
+static inline void drm_free_large(void *ptr)
+{
+ kvfree(ptr);
+}
+
+static inline void *drm_malloc_ab(size_t nmemb, size_t size)
+{
+ return kvmalloc_array(nmemb, sizeof(struct page *), GFP_KERNEL);
+}
+#endif
+
+static inline void xocl_release_pages(struct page **pages, int nr, bool cold)
+{
+ release_pages(pages, nr);
+}
+
+
+static inline void __user *to_user_ptr(u64 address)
+{
+ return (void __user *)(uintptr_t)address;
+}
+
+static size_t xocl_bo_physical_addr(const struct drm_xocl_bo *xobj)
+{
+ uint64_t paddr = xobj->mm_node ? xobj->mm_node->start : 0xffffffffffffffffull;
+
+ //Sarab: Need to check for number of hops & size of DDRs
+ if (xobj->type & XOCL_BO_ARE)
+ paddr |= XOCL_ARE_HOP;
+ return paddr;
+}
+
+void xocl_describe(const struct drm_xocl_bo *xobj)
+{
+ size_t size_in_kb = xobj->base.size / 1024;
+ size_t physical_addr = xocl_bo_physical_addr(xobj);
+ unsigned int ddr = xocl_bo_ddr_idx(xobj->flags);
+ unsigned int userptr = xocl_bo_userptr(xobj) ? 1 : 0;
+
+ DRM_DEBUG("%p: H[%p] SIZE[0x%zxKB] D[0x%zx] DDR[%u] UPTR[%u] SGLCOUNT[%u]\n",
+ xobj, xobj->vmapping ? xobj->vmapping : xobj->bar_vmapping, size_in_kb,
+ physical_addr, ddr, userptr, xobj->sgt->orig_nents);
+}
+
+static void xocl_free_mm_node(struct drm_xocl_bo *xobj)
+{
+ struct drm_device *ddev = xobj->base.dev;
+ struct xocl_drm *drm_p = ddev->dev_private;
+ unsigned int ddr = xocl_bo_ddr_idx(xobj->flags);
+
+ mutex_lock(&drm_p->mm_lock);
+ BO_ENTER("xobj %p, mm_node %p", xobj, xobj->mm_node);
+ if (!xobj->mm_node)
+ goto end;
+
+ xocl_mm_update_usage_stat(drm_p, ddr, xobj->base.size, -1);
+ BO_DEBUG("remove mm_node:%p, start:%llx size: %llx", xobj->mm_node,
+ xobj->mm_node->start, xobj->mm_node->size);
+ drm_mm_remove_node(xobj->mm_node);
+ kfree(xobj->mm_node);
+ xobj->mm_node = NULL;
+end:
+ mutex_unlock(&drm_p->mm_lock);
+}
+
+static void xocl_free_bo(struct drm_gem_object *obj)
+{
+ struct drm_xocl_bo *xobj = to_xocl_bo(obj);
+ struct drm_device *ddev = xobj->base.dev;
+ struct xocl_drm *drm_p = ddev->dev_private;
+ struct xocl_dev *xdev = drm_p->xdev;
+ int npages = obj->size >> PAGE_SHIFT;
+
+ DRM_DEBUG("Freeing BO %p\n", xobj);
+
+ BO_ENTER("xobj %p pages %p", xobj, xobj->pages);
+ if (xobj->vmapping)
+ vunmap(xobj->vmapping);
+ xobj->vmapping = NULL;
+
+ if (xobj->dmabuf)
+ unmap_mapping_range(xobj->dmabuf->file->f_mapping, 0, 0, 1);
+
+ if (xobj->dma_nsg) {
+ pci_unmap_sg(xdev->core.pdev, xobj->sgt->sgl, xobj->dma_nsg,
+ PCI_DMA_BIDIRECTIONAL);
+ }
+
+ if (xobj->pages) {
+ if (xocl_bo_userptr(xobj)) {
+ xocl_release_pages(xobj->pages, npages, 0);
+ drm_free_large(xobj->pages);
+ } else if (xocl_bo_p2p(xobj)) {
+ drm_free_large(xobj->pages);
+ /*devm_* will release all the pages while unload xocl driver*/
+ xobj->bar_vmapping = NULL;
+ } else if (!xocl_bo_import(xobj)) {
+ drm_gem_put_pages(obj, xobj->pages, false, false);
+ }
+ }
+ xobj->pages = NULL;
+
+ if (!xocl_bo_import(xobj)) {
+ DRM_DEBUG("Freeing regular buffer\n");
+ if (xobj->sgt) {
+ sg_free_table(xobj->sgt);
+ kfree(xobj->sgt);
+ }
+ xobj->sgt = NULL;
+ xocl_free_mm_node(xobj);
+ } else {
+ DRM_DEBUG("Freeing imported buffer\n");
+ if (!(xobj->type & XOCL_BO_ARE))
+ xocl_free_mm_node(xobj);
+
+ if (obj->import_attach) {
+ DRM_DEBUG("Unnmapping attached dma buf\n");
+ dma_buf_unmap_attachment(obj->import_attach, xobj->sgt, DMA_TO_DEVICE);
+ drm_prime_gem_destroy(obj, NULL);
+ }
+ }
+
+ /* If it is imported BO then we do not delete SG Table
+ * And if is imported from ARE device then we do not free the mm_node
+ * as well
+ * Call detach here........
+ * to let the exporting device know that importing device do not need
+ * it anymore..
+ * else free_bo i.e this function is not called for exporting device
+ * as it assumes that the exported buffer is still being used
+ * dmabuf->ops->release(dmabuf);
+ * The drm_driver.gem_free_object callback is responsible for cleaning
+ * up the dma_buf attachment and references acquired at import time.
+ *
+ * This crashes machine.. Using above code instead
+ * drm_prime_gem_destroy calls detach function..
+ * struct dma_buf *imported_dma_buf = obj->dma_buf;
+ * if (imported_dma_buf->ops->detach)
+ * imported_dma_buf->ops->detach(imported_dma_buf, obj->import_attach);
+ */
+
+ drm_gem_object_release(obj);
+ kfree(xobj);
+}
+
+void xocl_drm_free_bo(struct drm_gem_object *obj)
+{
+ xocl_free_bo(obj);
+}
+
+static inline int check_bo_user_reqs(const struct drm_device *dev,
+ unsigned int flags, unsigned int type)
+{
+ struct xocl_drm *drm_p = dev->dev_private;
+ struct xocl_dev *xdev = drm_p->xdev;
+ u16 ddr_count;
+ unsigned int ddr;
+
+ if (type == DRM_XOCL_BO_EXECBUF)
+ return 0;
+ if (type == DRM_XOCL_BO_CMA)
+ return -EINVAL;
+
+ //From "mem_topology" or "feature rom" depending on
+ //unified or non-unified dsa
+ ddr_count = XOCL_DDR_COUNT(xdev);
+
+ if (ddr_count == 0)
+ return -EINVAL;
+ ddr = xocl_bo_ddr_idx(flags);
+ if (ddr >= ddr_count)
+ return -EINVAL;
+ if (XOCL_MEM_TOPOLOGY(xdev)->m_mem_data[ddr].m_type == MEM_STREAMING)
+ return -EINVAL;
+ if (!XOCL_IS_DDR_USED(xdev, ddr)) {
+ userpf_err(xdev, "Bank %d is marked as unused in axlf", ddr);
+ return -EINVAL;
+ }
+ return 0;
+}
+
+static struct drm_xocl_bo *xocl_create_bo(struct drm_device *dev,
+ uint64_t unaligned_size,
+ unsigned int user_flags,
+ unsigned int user_type)
+{
+ size_t size = PAGE_ALIGN(unaligned_size);
+ struct drm_xocl_bo *xobj;
+ struct xocl_drm *drm_p = dev->dev_private;
+ struct xocl_dev *xdev = drm_p->xdev;
+ unsigned int ddr = xocl_bo_ddr_idx(user_flags);
+ u16 ddr_count = 0;
+ bool xobj_inited = false;
+ int err = 0;
+
+ BO_DEBUG("New create bo flags:%u type:%u", user_flags, user_type);
+ if (!size)
+ return ERR_PTR(-EINVAL);
+
+ /* Either none or only one DDR should be specified */
+ /* Check the type */
+ if (check_bo_user_reqs(dev, user_flags, user_type))
+ return ERR_PTR(-EINVAL);
+
+ xobj = kzalloc(sizeof(*xobj), GFP_KERNEL);
+ if (!xobj)
+ return ERR_PTR(-ENOMEM);
+
+ BO_ENTER("xobj %p", xobj);
+ err = drm_gem_object_init(dev, &xobj->base, size);
+ if (err)
+ goto failed;
+ xobj_inited = true;
+
+ if (user_type == DRM_XOCL_BO_EXECBUF) {
+ xobj->type = XOCL_BO_EXECBUF;
+ xobj->metadata.state = DRM_XOCL_EXECBUF_STATE_ABORT;
+ return xobj;
+ }
+
+ if (user_type & DRM_XOCL_BO_P2P)
+ xobj->type = XOCL_BO_P2P;
+
+ xobj->mm_node = kzalloc(sizeof(*xobj->mm_node), GFP_KERNEL);
+ if (!xobj->mm_node) {
+ err = -ENOMEM;
+ goto failed;
+ }
+
+ ddr_count = XOCL_DDR_COUNT(xdev);
+
+ mutex_lock(&drm_p->mm_lock);
+ /* Attempt to allocate buffer on the requested DDR */
+ xocl_xdev_dbg(xdev, "alloc bo from bank%u", ddr);
+ err = xocl_mm_insert_node(drm_p, ddr, xobj->mm_node,
+ xobj->base.size);
+ BO_DEBUG("insert mm_node:%p, start:%llx size: %llx",
+ xobj->mm_node, xobj->mm_node->start,
+ xobj->mm_node->size);
+ if (err)
+ goto failed;
+
+ xocl_mm_update_usage_stat(drm_p, ddr, xobj->base.size, 1);
+ mutex_unlock(&drm_p->mm_lock);
+ /* Record the DDR we allocated the buffer on */
+ //xobj->flags |= (1 << ddr);
+ xobj->flags = ddr;
+
+ return xobj;
+failed:
+ mutex_unlock(&drm_p->mm_lock);
+ kfree(xobj->mm_node);
+
+ if (xobj_inited)
+ drm_gem_object_release(&xobj->base);
+
+ kfree(xobj);
+
+ return ERR_PTR(err);
+}
+
+struct drm_xocl_bo *xocl_drm_create_bo(struct xocl_drm *drm_p,
+ uint64_t unaligned_size,
+ unsigned int user_flags,
+ unsigned int user_type)
+{
+ return xocl_create_bo(drm_p->ddev, unaligned_size, user_flags,
+ user_type);
+}
+
+static struct page **xocl_p2p_get_pages(void *bar_vaddr, int npages)
+{
+ struct page *p, **pages;
+ int i;
+ uint64_t page_offset_enum = 0;
+
+ pages = drm_malloc_ab(npages, sizeof(struct page *));
+
+ if (pages == NULL)
+ return ERR_PTR(-ENOMEM);
+
+ for (i = 0; i < npages; i++) {
+ p = virt_to_page(bar_vaddr+page_offset_enum);
+ pages[i] = p;
+
+ if (IS_ERR(p))
+ goto fail;
+
+ page_offset_enum += PAGE_SIZE;
+ }
+
+ return pages;
+fail:
+ kvfree(pages);
+ return ERR_CAST(p);
+}
+
+/*
+ * For ARE device do not reserve DDR space
+ * In below import it will reuse the mm_node which is already created by other application
+ */
+
+static struct drm_xocl_bo *xocl_create_bo_forARE(struct drm_device *dev,
+ uint64_t unaligned_size,
+ struct drm_mm_node *exporting_mm_node)
+{
+ struct drm_xocl_bo *xobj;
+ size_t size = PAGE_ALIGN(unaligned_size);
+ int err = 0;
+
+ if (!size)
+ return ERR_PTR(-EINVAL);
+
+ xobj = kzalloc(sizeof(*xobj), GFP_KERNEL);
+ if (!xobj)
+ return ERR_PTR(-ENOMEM);
+
+ BO_ENTER("xobj %p", xobj);
+ err = drm_gem_object_init(dev, &xobj->base, size);
+ if (err)
+ goto out3;
+
+ xobj->mm_node = exporting_mm_node;
+ if (!xobj->mm_node) {
+ err = -ENOMEM;
+ goto out3;
+ }
+
+ /* Record that this buffer is on remote device to be access over ARE*/
+ //xobj->flags = XOCL_BO_ARE;
+ xobj->type |= XOCL_BO_ARE;
+ return xobj;
+out3:
+ kfree(xobj);
+ return ERR_PTR(err);
+}
+
+
+int xocl_create_bo_ioctl(struct drm_device *dev,
+ void *data,
+ struct drm_file *filp)
+{
+ int ret;
+ struct drm_xocl_bo *xobj;
+ struct xocl_drm *drm_p = dev->dev_private;
+ struct xocl_dev *xdev = drm_p->xdev;
+ struct drm_xocl_create_bo *args = data;
+ //unsigned ddr = args->flags & XOCL_MEM_BANK_MSK;
+ unsigned int ddr = args->flags;
+ //unsigned bar_mapped = (args->flags & DRM_XOCL_BO_P2P) ? 1 : 0;
+ unsigned int bar_mapped = (args->type & DRM_XOCL_BO_P2P) ? 1 : 0;
+
+// //Only one bit should be set in ddr. Other bits are now in "type"
+// if (hweight_long(ddr) > 1)
+// return -EINVAL;
+//// if (args->flags && (args->flags != DRM_XOCL_BO_EXECBUF)) {
+// if (hweight_long(ddr) > 1)
+// return -EINVAL;
+// }
+
+ if (bar_mapped) {
+ if (!xdev->p2p_bar_addr) {
+ xocl_xdev_err(xdev, "No P2P mem region available, Can't create p2p BO");
+ return -EINVAL;
+ }
+ }
+
+ xobj = xocl_create_bo(dev, args->size, args->flags, args->type);
+
+ BO_ENTER("xobj %p, mm_node %p", xobj, xobj->mm_node);
+ if (IS_ERR(xobj)) {
+ DRM_DEBUG("object creation failed\n");
+ return PTR_ERR(xobj);
+ }
+
+ if (bar_mapped) {
+ ddr = xocl_bo_ddr_idx(xobj->flags);
+ /*
+ * DRM allocate contiguous pages, shift the vmapping with
+ * bar address offset
+ */
+ xobj->bar_vmapping = xdev->p2p_bar_addr +
+ drm_p->mm_p2p_off[ddr] + xobj->mm_node->start -
+ XOCL_MEM_TOPOLOGY(xdev)->m_mem_data[ddr].m_base_address;
+ }
+
+ if (bar_mapped)
+ xobj->pages = xocl_p2p_get_pages(xobj->bar_vmapping, xobj->base.size >> PAGE_SHIFT);
+ else
+ xobj->pages = drm_gem_get_pages(&xobj->base);
+
+ if (IS_ERR(xobj->pages)) {
+ ret = PTR_ERR(xobj->pages);
+ goto out_free;
+ }
+
+ xobj->sgt = drm_prime_pages_to_sg(xobj->pages, xobj->base.size >> PAGE_SHIFT);
+ if (IS_ERR(xobj->sgt)) {
+ ret = PTR_ERR(xobj->sgt);
+ goto out_free;
+ }
+
+ if (!bar_mapped) {
+ xobj->vmapping = vmap(xobj->pages, xobj->base.size >> PAGE_SHIFT, VM_MAP, PAGE_KERNEL);
+ if (!xobj->vmapping) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ }
+
+ ret = drm_gem_create_mmap_offset(&xobj->base);
+ if (ret < 0)
+ goto out_free;
+ ret = drm_gem_handle_create(filp, &xobj->base, &args->handle);
+ if (ret < 0)
+ goto out_free;
+
+ xocl_describe(xobj);
+//PORT4_20
+// drm_gem_object_unreference_unlocked(&xobj->base);
+ drm_gem_object_put_unlocked(&xobj->base);
+ return ret;
+
+out_free:
+ xocl_free_bo(&xobj->base);
+ return ret;
+}
+
+int xocl_userptr_bo_ioctl(struct drm_device *dev,
+ void *data,
+ struct drm_file *filp)
+{
+ int ret;
+ struct drm_xocl_bo *xobj;
+ unsigned int page_count;
+ struct drm_xocl_userptr_bo *args = data;
+ //unsigned ddr = args->flags & XOCL_MEM_BANK_MSK;
+ //unsigned ddr = args->flags;
+
+ if (offset_in_page(args->addr))
+ return -EINVAL;
+
+ if (args->type & DRM_XOCL_BO_EXECBUF)
+ return -EINVAL;
+
+ if (args->type & DRM_XOCL_BO_CMA)
+ return -EINVAL;
+
+// if (args->flags && (hweight_long(ddr) > 1))
+// return -EINVAL;
+
+ xobj = xocl_create_bo(dev, args->size, args->flags, args->type);
+ BO_ENTER("xobj %p", xobj);
+
+ if (IS_ERR(xobj)) {
+ DRM_DEBUG("object creation failed\n");
+ return PTR_ERR(xobj);
+ }
+
+ /* Use the page rounded size so we can accurately account for number of pages */
+ page_count = xobj->base.size >> PAGE_SHIFT;
+
+ xobj->pages = drm_malloc_ab(page_count, sizeof(*xobj->pages));
+ if (!xobj->pages) {
+ ret = -ENOMEM;
+ goto out1;
+ }
+ ret = get_user_pages_fast(args->addr, page_count, 1, xobj->pages);
+
+ if (ret != page_count)
+ goto out0;
+
+ xobj->sgt = drm_prime_pages_to_sg(xobj->pages, page_count);
+ if (IS_ERR(xobj->sgt)) {
+ ret = PTR_ERR(xobj->sgt);
+ goto out0;
+ }
+
+ /* TODO: resolve the cache issue */
+ xobj->vmapping = vmap(xobj->pages, page_count, VM_MAP, PAGE_KERNEL);
+
+ if (!xobj->vmapping) {
+ ret = -ENOMEM;
+ goto out1;
+ }
+
+ ret = drm_gem_handle_create(filp, &xobj->base, &args->handle);
+ if (ret)
+ goto out1;
+
+ xobj->type |= XOCL_BO_USERPTR;
+ xocl_describe(xobj);
+//PORT4_20
+// drm_gem_object_unreference_unlocked(&xobj->base);
+ drm_gem_object_put_unlocked(&xobj->base);
+ return ret;
+
+out0:
+ drm_free_large(xobj->pages);
+ xobj->pages = NULL;
+out1:
+ xocl_free_bo(&xobj->base);
+ DRM_DEBUG("handle creation failed\n");
+ return ret;
+}
+
+
+int xocl_map_bo_ioctl(struct drm_device *dev,
+ void *data,
+ struct drm_file *filp)
+{
+ int ret = 0;
+ struct drm_xocl_map_bo *args = data;
+ struct drm_gem_object *obj;
+ struct drm_xocl_bo *xobj;
+
+ obj = xocl_gem_object_lookup(dev, filp, args->handle);
+ xobj = to_xocl_bo(obj);
+
+ if (!obj) {
+ DRM_ERROR("Failed to look up GEM BO %d\n", args->handle);
+ return -ENOENT;
+ }
+
+ BO_ENTER("xobj %p", xobj);
+ if (xocl_bo_userptr(xobj)) {
+ ret = -EPERM;
+ goto out;
+ }
+ /* The mmap offset was set up at BO allocation time. */
+ args->offset = drm_vma_node_offset_addr(&obj->vma_node);
+ xocl_describe(to_xocl_bo(obj));
+out:
+//PORT4_20
+// drm_gem_object_unreference_unlocked(obj);
+ drm_gem_object_put_unlocked(obj);
+ return ret;
+}
+
+static struct sg_table *alloc_onetime_sg_table(struct page **pages, uint64_t offset, uint64_t size)
+{
+ int ret;
+ unsigned int nr_pages;
+ struct sg_table *sgt = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
+
+ if (!sgt)
+ return ERR_PTR(-ENOMEM);
+
+ pages += (offset >> PAGE_SHIFT);
+ offset &= (~PAGE_MASK);
+ nr_pages = PAGE_ALIGN(size + offset) >> PAGE_SHIFT;
+
+ ret = sg_alloc_table_from_pages(sgt, pages, nr_pages, offset, size, GFP_KERNEL);
+ if (ret)
+ goto cleanup;
+ return sgt;
+
+cleanup:
+ kfree(sgt);
+ return ERR_PTR(-ENOMEM);
+}
+
+int xocl_sync_bo_ioctl(struct drm_device *dev,
+ void *data,
+ struct drm_file *filp)
+{
+ const struct drm_xocl_bo *xobj;
+ struct sg_table *sgt;
+ u64 paddr = 0;
+ int channel = 0;
+ ssize_t ret = 0;
+ const struct drm_xocl_sync_bo *args = data;
+ struct xocl_drm *drm_p = dev->dev_private;
+ struct xocl_dev *xdev = drm_p->xdev;
+
+ u32 dir = (args->dir == DRM_XOCL_SYNC_BO_TO_DEVICE) ? 1 : 0;
+ struct drm_gem_object *gem_obj = xocl_gem_object_lookup(dev, filp,
+ args->handle);
+ if (!gem_obj) {
+ DRM_ERROR("Failed to look up GEM BO %d\n", args->handle);
+ return -ENOENT;
+ }
+
+ xobj = to_xocl_bo(gem_obj);
+ BO_ENTER("xobj %p", xobj);
+ sgt = xobj->sgt;
+
+ if (xocl_bo_p2p(xobj)) {
+ DRM_DEBUG("P2P_BO doesn't support sync_bo\n");
+ ret = -EOPNOTSUPP;
+ goto out;
+ }
+
+ //Sarab: If it is a remote BO then why do sync over ARE.
+ //We should do sync directly using the other device which this bo locally.
+ //So that txfer is: HOST->PCIE->DDR; Else it will be HOST->PCIE->ARE->DDR
+ paddr = xocl_bo_physical_addr(xobj);
+
+ if (paddr == 0xffffffffffffffffull)
+ return -EINVAL;
+
+ /* If device is offline (due to error), reject all DMA requests */
+ if (xdev->offline)
+ return -ENODEV;
+
+
+ if ((args->offset + args->size) > gem_obj->size) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* only invalidate the range of addresses requested by the user */
+ paddr += args->offset;
+
+ if (args->offset || (args->size != xobj->base.size)) {
+ sgt = alloc_onetime_sg_table(xobj->pages, args->offset, args->size);
+ if (IS_ERR(sgt)) {
+ ret = PTR_ERR(sgt);
+ goto out;
+ }
+ }
+
+ //drm_clflush_sg(sgt);
+ channel = xocl_acquire_channel(xdev, dir);
+
+ if (channel < 0) {
+ ret = -EINVAL;
+ goto clear;
+ }
+ /* Now perform DMA */
+ ret = xocl_migrate_bo(xdev, sgt, dir, paddr, channel, args->size);
+ if (ret >= 0)
+ ret = (ret == args->size) ? 0 : -EIO;
+ xocl_release_channel(xdev, dir, channel);
+clear:
+ if (args->offset || (args->size != xobj->base.size)) {
+ sg_free_table(sgt);
+ kfree(sgt);
+ }
+out:
+//PORT4_20
+ drm_gem_object_put_unlocked(gem_obj);
+ return ret;
+}
+
+int xocl_info_bo_ioctl(struct drm_device *dev,
+ void *data,
+ struct drm_file *filp)
+{
+ const struct drm_xocl_bo *xobj;
+ struct drm_xocl_info_bo *args = data;
+ struct drm_gem_object *gem_obj = xocl_gem_object_lookup(dev, filp,
+ args->handle);
+
+ if (!gem_obj) {
+ DRM_ERROR("Failed to look up GEM BO %d\n", args->handle);
+ return -ENOENT;
+ }
+
+ xobj = to_xocl_bo(gem_obj);
+ BO_ENTER("xobj %p", xobj);
+
+ args->size = xobj->base.size;
+
+ args->paddr = xocl_bo_physical_addr(xobj);
+ xocl_describe(xobj);
+//PORT4_20
+ drm_gem_object_put_unlocked(gem_obj);
+
+ return 0;
+}
+
+int xocl_pwrite_bo_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp)
+{
+ struct drm_xocl_bo *xobj;
+ const struct drm_xocl_pwrite_bo *args = data;
+ struct drm_gem_object *gem_obj = xocl_gem_object_lookup(dev, filp,
+ args->handle);
+ char __user *user_data = to_user_ptr(args->data_ptr);
+ int ret = 0;
+ void *kaddr;
+
+ if (!gem_obj) {
+ DRM_ERROR("Failed to look up GEM BO %d\n", args->handle);
+ return -ENOENT;
+ }
+
+ if ((args->offset > gem_obj->size) || (args->size > gem_obj->size)
+ || ((args->offset + args->size) > gem_obj->size)) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ if (args->size == 0) {
+ ret = 0;
+ goto out;
+ }
+
+ if (!access_ok(user_data, args->size)) {
+ ret = -EFAULT;
+ goto out;
+ }
+
+ xobj = to_xocl_bo(gem_obj);
+ BO_ENTER("xobj %p", xobj);
+
+ if (xocl_bo_userptr(xobj)) {
+ ret = -EPERM;
+ goto out;
+ }
+
+ kaddr = xobj->vmapping ? xobj->vmapping : xobj->bar_vmapping;
+ kaddr += args->offset;
+
+ ret = copy_from_user(kaddr, user_data, args->size);
+out:
+//PORT4_20
+ drm_gem_object_put_unlocked(gem_obj);
+
+ return ret;
+}
+
+int xocl_pread_bo_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp)
+{
+ struct drm_xocl_bo *xobj;
+ const struct drm_xocl_pread_bo *args = data;
+ struct drm_gem_object *gem_obj = xocl_gem_object_lookup(dev, filp,
+ args->handle);
+ char __user *user_data = to_user_ptr(args->data_ptr);
+ int ret = 0;
+ void *kaddr;
+
+ if (!gem_obj) {
+ DRM_ERROR("Failed to look up GEM BO %d\n", args->handle);
+ return -ENOENT;
+ }
+
+ if (xocl_bo_userptr(to_xocl_bo(gem_obj))) {
+ ret = -EPERM;
+ goto out;
+ }
+
+ if ((args->offset > gem_obj->size) || (args->size > gem_obj->size)
+ || ((args->offset + args->size) > gem_obj->size)) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ if (args->size == 0) {
+ ret = 0;
+ goto out;
+ }
+
+ if (!access_ok(user_data, args->size)) {
+ ret = EFAULT;
+ goto out;
+ }
+
+ xobj = to_xocl_bo(gem_obj);
+ BO_ENTER("xobj %p", xobj);
+ kaddr = xobj->vmapping ? xobj->vmapping : xobj->bar_vmapping;
+ kaddr += args->offset;
+
+ ret = copy_to_user(user_data, kaddr, args->size);
+
+out:
+//PORT4_20
+ drm_gem_object_put_unlocked(gem_obj);
+
+ return ret;
+}
+
+int xocl_copy_bo_ioctl(struct drm_device *dev,
+ void *data,
+ struct drm_file *filp)
+{
+ const struct drm_xocl_bo *dst_xobj, *src_xobj;
+ struct sg_table *sgt;
+ u64 paddr = 0;
+ int channel = 0;
+ ssize_t ret = 0;
+ const struct drm_xocl_copy_bo *args = data;
+ struct xocl_drm *drm_p = dev->dev_private;
+ struct xocl_dev *xdev = drm_p->xdev;
+ u32 dir = 0; //always write data from source to destination
+ struct drm_gem_object *dst_gem_obj, *src_gem_obj;
+
+ dst_gem_obj = xocl_gem_object_lookup(dev, filp,
+ args->dst_handle);
+ if (!dst_gem_obj) {
+ DRM_ERROR("Failed to look up Destination GEM BO %d\n", args->dst_handle);
+ return -ENOENT;
+ }
+ src_gem_obj = xocl_gem_object_lookup(dev, filp,
+ args->src_handle);
+ if (!src_gem_obj) {
+ DRM_ERROR("Failed to look up Source GEM BO %d\n", args->src_handle);
+ ret = -ENOENT;
+ goto src_lookup_fail;
+ }
+
+ dst_xobj = to_xocl_bo(dst_gem_obj);
+ src_xobj = to_xocl_bo(src_gem_obj);
+
+ if (!xocl_bo_p2p(src_xobj)) {
+ DRM_ERROR("src_bo must be p2p bo, copy_bo aborted");
+ ret = -EINVAL;
+ goto out;
+ }
+
+ DRM_DEBUG("dst_xobj %p, src_xobj %p", dst_xobj, src_xobj);
+ DRM_DEBUG("dst_xobj->sgt %p, src_xobj->sgt %p", dst_xobj->sgt, src_xobj->sgt);
+ sgt = dst_xobj->sgt;
+
+ paddr = xocl_bo_physical_addr(src_xobj);
+
+ if (paddr == 0xffffffffffffffffull) {
+ ret = -EINVAL;
+ goto out;
+ }
+ /* If device is offline (due to error), reject all DMA requests */
+ if (xdev->offline) {
+ ret = -ENODEV;
+ goto out;
+ }
+
+ if (((args->src_offset + args->size) > src_gem_obj->size) ||
+ ((args->dst_offset + args->size) > dst_gem_obj->size)) {
+ DRM_ERROR("offsize + sizes out of boundary, copy_bo abort");
+ ret = -EINVAL;
+ goto out;
+ }
+ paddr += args->src_offset;
+
+ DRM_DEBUG("%s, xobj->pages = %p\n", __func__, dst_xobj->pages);
+
+
+ if (args->dst_offset || (args->size != dst_xobj->base.size)) {
+ sgt = alloc_onetime_sg_table(dst_xobj->pages, args->dst_offset, args->size);
+ if (IS_ERR(sgt)) {
+ ret = PTR_ERR(sgt);
+ goto out;
+ }
+ }
+
+ channel = xocl_acquire_channel(xdev, dir);
+
+ if (channel < 0) {
+ ret = -EINVAL;
+ goto clear;
+ }
+ /* Now perform DMA */
+ ret = xocl_migrate_bo(xdev, sgt, dir, paddr, channel,
+ args->size);
+
+ if (ret >= 0)
+ ret = (ret == args->size) ? 0 : -EIO;
+ xocl_release_channel(xdev, dir, channel);
+
+
+clear:
+ if (args->dst_offset || (args->size != dst_xobj->base.size)) {
+ sg_free_table(sgt);
+ kfree(sgt);
+ }
+out:
+//PORT4_20
+ drm_gem_object_put_unlocked(src_gem_obj);
+src_lookup_fail:
+//PORT4_20
+ drm_gem_object_put_unlocked(dst_gem_obj);
+ return ret;
+
+}
+
+
+struct sg_table *xocl_gem_prime_get_sg_table(struct drm_gem_object *obj)
+{
+ struct drm_xocl_bo *xobj = to_xocl_bo(obj);
+
+ BO_ENTER("xobj %p", xobj);
+ return drm_prime_pages_to_sg(xobj->pages, xobj->base.size >> PAGE_SHIFT);
+}
+
+
+static struct drm_xocl_bo *xocl_is_exporting_xare(struct drm_device *dev, struct dma_buf_attachment *attach)
+{
+ struct drm_gem_object *exporting_gem_obj;
+ struct drm_device *exporting_drm_dev;
+ struct xocl_drm *exporting_drmp;
+ struct xocl_dev *exporting_xdev;
+
+ struct device_driver *importing_dma_driver = dev->dev->driver;
+ struct dma_buf *exporting_dma_buf = attach->dmabuf;
+ struct device_driver *exporting_dma_driver = attach->dev->driver;
+ struct xocl_drm *drm_p = dev->dev_private;
+ struct xocl_dev *xdev = drm_p->xdev;
+
+ if (xocl_is_are(xdev))
+ return NULL;
+
+ //We don't know yet if the exporting device is Xilinx/XOCL or third party or USB device
+ //So checking it in below code
+ if (importing_dma_driver != exporting_dma_driver)
+ return NULL;
+
+ //Exporting devices have same driver as us. So this is Xilinx device
+ //So now we can get gem_object, drm_device & xocl_dev
+ exporting_gem_obj = exporting_dma_buf->priv;
+ exporting_drm_dev = exporting_gem_obj->dev;
+ exporting_drmp = exporting_drm_dev->dev_private;
+ exporting_xdev = exporting_drmp->xdev;
+ //exporting_xdev->header;//This has FeatureROM header
+ if (xocl_is_are(exporting_xdev))
+ return to_xocl_bo(exporting_gem_obj);
+
+ return NULL;
+}
+
+struct drm_gem_object *xocl_gem_prime_import_sg_table(struct drm_device *dev,
+ struct dma_buf_attachment *attach, struct sg_table *sgt)
+{
+ int ret = 0;
+ struct drm_xocl_bo *exporting_xobj;
+ struct drm_xocl_bo *importing_xobj;
+
+ /*
+ * For ARE device resue the mm node from exporting xobj
+ * For non ARE devices we need to create a full BO but share the SG
+ * table
+ * ???? add flags to create_bo.. for DDR bank??
+ */
+
+ exporting_xobj = xocl_is_exporting_xare(dev, attach);
+ importing_xobj = exporting_xobj ?
+ xocl_create_bo_forARE(dev, attach->dmabuf->size,
+ exporting_xobj->mm_node) :
+ xocl_create_bo(dev, attach->dmabuf->size, 0, 0);
+
+ BO_ENTER("xobj %p", importing_xobj);
+
+ if (IS_ERR(importing_xobj)) {
+ DRM_DEBUG("object creation failed\n");
+ return (struct drm_gem_object *)importing_xobj;
+ }
+
+ importing_xobj->type |= XOCL_BO_IMPORT;
+ importing_xobj->sgt = sgt;
+ importing_xobj->pages = drm_malloc_ab(attach->dmabuf->size >> PAGE_SHIFT, sizeof(*importing_xobj->pages));
+ if (!importing_xobj->pages) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ ret = drm_prime_sg_to_page_addr_arrays(sgt, importing_xobj->pages,
+ NULL, attach->dmabuf->size >> PAGE_SHIFT);
+ if (ret)
+ goto out_free;
+
+ importing_xobj->vmapping = vmap(importing_xobj->pages, importing_xobj->base.size >> PAGE_SHIFT, VM_MAP,
+ PAGE_KERNEL);
+
+ if (!importing_xobj->vmapping) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+
+ ret = drm_gem_create_mmap_offset(&importing_xobj->base);
+ if (ret < 0)
+ goto out_free;
+
+ xocl_describe(importing_xobj);
+ return &importing_xobj->base;
+
+out_free:
+ xocl_free_bo(&importing_xobj->base);
+ DRM_ERROR("Buffer import failed\n");
+ return ERR_PTR(ret);
+}
+
+void *xocl_gem_prime_vmap(struct drm_gem_object *obj)
+{
+ struct drm_xocl_bo *xobj = to_xocl_bo(obj);
+
+ BO_ENTER("xobj %p", xobj);
+ return xobj->vmapping;
+}
+
+void xocl_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr)
+{
+
+}
+
+int xocl_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
+{
+ struct drm_xocl_bo *xobj = to_xocl_bo(obj);
+ int ret;
+
+ BO_ENTER("obj %p", obj);
+ if (obj->size < vma->vm_end - vma->vm_start)
+ return -EINVAL;
+
+ if (!obj->filp)
+ return -ENODEV;
+
+ ret = obj->filp->f_op->mmap(obj->filp, vma);
+ if (ret)
+ return ret;
+
+ fput(vma->vm_file);
+ if (!IS_ERR(xobj->dmabuf)) {
+ vma->vm_file = get_file(xobj->dmabuf->file);
+ vma->vm_ops = xobj->dmabuf_vm_ops;
+ vma->vm_private_data = obj;
+ vma->vm_flags |= VM_MIXEDMAP;
+ }
+
+ return 0;
+}
+
+int xocl_init_unmgd(struct drm_xocl_unmgd *unmgd, uint64_t data_ptr,
+ uint64_t size, u32 write)
+{
+ int ret;
+ char __user *user_data = to_user_ptr(data_ptr);
+
+ if (!access_ok(user_data, size))
+ return -EFAULT;
+
+ memset(unmgd, 0, sizeof(struct drm_xocl_unmgd));
+
+ unmgd->npages = (((unsigned long)user_data + size + PAGE_SIZE - 1) -
+ ((unsigned long)user_data & PAGE_MASK)) >> PAGE_SHIFT;
+
+ unmgd->pages = drm_malloc_ab(unmgd->npages, sizeof(*unmgd->pages));
+ if (!unmgd->pages)
+ return -ENOMEM;
+
+ ret = get_user_pages_fast(data_ptr, unmgd->npages, (write == 0) ? 1 : 0, unmgd->pages);
+
+ if (ret != unmgd->npages)
+ goto clear_pages;
+
+ unmgd->sgt = alloc_onetime_sg_table(unmgd->pages, data_ptr & ~PAGE_MASK, size);
+ if (IS_ERR(unmgd->sgt)) {
+ ret = PTR_ERR(unmgd->sgt);
+ goto clear_release;
+ }
+
+ return 0;
+
+clear_release:
+ xocl_release_pages(unmgd->pages, unmgd->npages, 0);
+clear_pages:
+ drm_free_large(unmgd->pages);
+ unmgd->pages = NULL;
+ return ret;
+}
+
+void xocl_finish_unmgd(struct drm_xocl_unmgd *unmgd)
+{
+ if (!unmgd->pages)
+ return;
+ sg_free_table(unmgd->sgt);
+ kfree(unmgd->sgt);
+ xocl_release_pages(unmgd->pages, unmgd->npages, 0);
+ drm_free_large(unmgd->pages);
+ unmgd->pages = NULL;
+}
+
+static bool xocl_validate_paddr(struct xocl_dev *xdev, u64 paddr, u64 size)
+{
+ struct mem_data *mem_data;
+ int i;
+ uint64_t addr;
+ bool start_check = false;
+ bool end_check = false;
+
+ for (i = 0; i < XOCL_MEM_TOPOLOGY(xdev)->m_count; i++) {
+ mem_data = &XOCL_MEM_TOPOLOGY(xdev)->m_mem_data[i];
+ addr = mem_data->m_base_address;
+ start_check = (paddr >= addr);
+ end_check = (paddr + size <= addr + mem_data->m_size * 1024);
+ if (mem_data->m_used && start_check && end_check)
+ return true;
+ }
+
+ return false;
+}
+
+int xocl_pwrite_unmgd_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp)
+{
+ int channel;
+ struct drm_xocl_unmgd unmgd;
+ const struct drm_xocl_pwrite_unmgd *args = data;
+ struct xocl_drm *drm_p = dev->dev_private;
+ struct xocl_dev *xdev = drm_p->xdev;
+ u32 dir = 1;
+ ssize_t ret = 0;
+
+ if (args->address_space != 0) {
+ userpf_err(xdev, "invalid addr space");
+ return -EFAULT;
+ }
+
+ if (args->size == 0)
+ return 0;
+
+ if (!xocl_validate_paddr(xdev, args->paddr, args->size)) {
+ userpf_err(xdev, "invalid paddr: 0x%llx, size:0x%llx",
+ args->paddr, args->size);
+ /* currently we are not able to return error because
+ * it is unclear that what addresses are valid other than
+ * ddr area. we should revisit this sometime.
+ * return -EINVAL;
+ */
+ }
+
+ ret = xocl_init_unmgd(&unmgd, args->data_ptr, args->size, dir);
+ if (ret) {
+ userpf_err(xdev, "init unmgd failed %ld", ret);
+ return ret;
+ }
+
+ channel = xocl_acquire_channel(xdev, dir);
+ if (channel < 0) {
+ userpf_err(xdev, "acquire channel failed");
+ ret = -EINVAL;
+ goto clear;
+ }
+ /* Now perform DMA */
+ ret = xocl_migrate_bo(xdev, unmgd.sgt, dir, args->paddr, channel,
+ args->size);
+ if (ret >= 0)
+ ret = (ret == args->size) ? 0 : -EIO;
+ xocl_release_channel(xdev, dir, channel);
+clear:
+ xocl_finish_unmgd(&unmgd);
+ return ret;
+}
+
+int xocl_pread_unmgd_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp)
+{
+ int channel;
+ struct drm_xocl_unmgd unmgd;
+ const struct drm_xocl_pwrite_unmgd *args = data;
+ struct xocl_drm *drm_p = dev->dev_private;
+ struct xocl_dev *xdev = drm_p->xdev;
+ u32 dir = 0; /* read */
+ ssize_t ret = 0;
+
+ if (args->address_space != 0) {
+ userpf_err(xdev, "invalid addr space");
+ return -EFAULT;
+ }
+
+ if (args->size == 0)
+ return 0;
+
+ if (!xocl_validate_paddr(xdev, args->paddr, args->size)) {
+ userpf_err(xdev, "invalid paddr: 0x%llx, size:0x%llx",
+ args->paddr, args->size);
+ /* currently we are not able to return error because
+ * it is unclear that what addresses are valid other than
+ * ddr area. we should revisit this sometime.
+ * return -EINVAL;
+ */
+ }
+
+ ret = xocl_init_unmgd(&unmgd, args->data_ptr, args->size, dir);
+ if (ret) {
+ userpf_err(xdev, "init unmgd failed %ld", ret);
+ return ret;
+ }
+
+ channel = xocl_acquire_channel(xdev, dir);
+
+ if (channel < 0) {
+ userpf_err(xdev, "acquire channel failed");
+ ret = -EINVAL;
+ goto clear;
+ }
+ /* Now perform DMA */
+ ret = xocl_migrate_bo(xdev, unmgd.sgt, dir, args->paddr, channel,
+ args->size);
+ if (ret >= 0)
+ ret = (ret == args->size) ? 0 : -EIO;
+
+ xocl_release_channel(xdev, dir, channel);
+clear:
+ xocl_finish_unmgd(&unmgd);
+ return ret;
+}
+
+int xocl_usage_stat_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp)
+{
+ struct xocl_drm *drm_p = dev->dev_private;
+ struct xocl_dev *xdev = drm_p->xdev;
+ struct drm_xocl_usage_stat *args = data;
+ int i;
+
+ args->mm_channel_count = XOCL_DDR_COUNT(xdev);
+ if (args->mm_channel_count > 8)
+ args->mm_channel_count = 8;
+ for (i = 0; i < args->mm_channel_count; i++)
+ xocl_mm_get_usage_stat(drm_p, i, args->mm + i);
+
+ args->dma_channel_count = xocl_get_chan_count(xdev);
+ if (args->dma_channel_count > 8)
+ args->dma_channel_count = 8;
+
+ for (i = 0; i < args->dma_channel_count; i++) {
+ args->h2c[i] = xocl_get_chan_stat(xdev, i, 1);
+ args->c2h[i] = xocl_get_chan_stat(xdev, i, 0);
+ }
+
+ return 0;
+}
diff --git a/drivers/gpu/drm/xocl/userpf/xocl_bo.h b/drivers/gpu/drm/xocl/userpf/xocl_bo.h
new file mode 100644
index 000000000000..38ac78cd59f6
--- /dev/null
+++ b/drivers/gpu/drm/xocl/userpf/xocl_bo.h
@@ -0,0 +1,119 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+/*
+ * A GEM style device manager for PCIe based OpenCL accelerators.
+ *
+ * Copyright (C) 2016-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Authors:
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _XOCL_BO_H
+#define _XOCL_BO_H
+
+#include <drm/xocl_drm.h>
+#include "../xocl_drm.h"
+
+#define XOCL_BO_USERPTR (1 << 31)
+#define XOCL_BO_IMPORT (1 << 30)
+#define XOCL_BO_EXECBUF (1 << 29)
+#define XOCL_BO_CMA (1 << 28)
+#define XOCL_BO_P2P (1 << 27)
+
+#define XOCL_BO_DDR0 (1 << 0)
+#define XOCL_BO_DDR1 (1 << 1)
+#define XOCL_BO_DDR2 (1 << 2)
+#define XOCL_BO_DDR3 (1 << 3)
+
+
+
+//#define XOCL_MEM_BANK_MSK (0xFFFFFF)
+/*
+ * When the BO is imported from an ARE device. This is remote BO to
+ * be accessed over ARE
+ */
+#define XOCL_BO_ARE (1 << 26)
+
+static inline bool xocl_bo_userptr(const struct drm_xocl_bo *bo)
+{
+ return (bo->type & XOCL_BO_USERPTR);
+}
+
+static inline bool xocl_bo_import(const struct drm_xocl_bo *bo)
+{
+ return (bo->type & XOCL_BO_IMPORT);
+}
+
+static inline bool xocl_bo_execbuf(const struct drm_xocl_bo *bo)
+{
+ return (bo->type & XOCL_BO_EXECBUF);
+}
+
+static inline bool xocl_bo_cma(const struct drm_xocl_bo *bo)
+{
+ return (bo->type & XOCL_BO_CMA);
+}
+static inline bool xocl_bo_p2p(const struct drm_xocl_bo *bo)
+{
+ return (bo->type & XOCL_BO_P2P);
+}
+
+static inline struct drm_gem_object *xocl_gem_object_lookup(struct drm_device *dev,
+ struct drm_file *filp,
+ u32 handle)
+{
+ return drm_gem_object_lookup(filp, handle);
+}
+
+static inline struct drm_xocl_dev *bo_xocl_dev(const struct drm_xocl_bo *bo)
+{
+ return bo->base.dev->dev_private;
+}
+
+static inline unsigned int xocl_bo_ddr_idx(unsigned int flags)
+{
+ return flags;
+}
+
+int xocl_create_bo_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp);
+int xocl_userptr_bo_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp);
+int xocl_sync_bo_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp);
+int xocl_copy_bo_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp);
+int xocl_map_bo_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp);
+int xocl_info_bo_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp);
+int xocl_pwrite_bo_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp);
+int xocl_pread_bo_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp);
+int xocl_ctx_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp);
+int xocl_pwrite_unmgd_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp);
+int xocl_pread_unmgd_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp);
+int xocl_usage_stat_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp);
+
+struct sg_table *xocl_gem_prime_get_sg_table(struct drm_gem_object *obj);
+struct drm_gem_object *xocl_gem_prime_import_sg_table(struct drm_device *dev,
+ struct dma_buf_attachment *attach, struct sg_table *sgt);
+void *xocl_gem_prime_vmap(struct drm_gem_object *obj);
+void xocl_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);
+int xocl_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma);
+
+#endif
diff --git a/drivers/gpu/drm/xocl/userpf/xocl_drm.c b/drivers/gpu/drm/xocl/userpf/xocl_drm.c
new file mode 100644
index 000000000000..e07f5f8a054a
--- /dev/null
+++ b/drivers/gpu/drm/xocl/userpf/xocl_drm.c
@@ -0,0 +1,640 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * A GEM style device manager for PCIe based OpenCL accelerators.
+ *
+ * Copyright (C) 2016-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Authors:
+ *
+ */
+
+#include <linux/version.h>
+#include <drm/drmP.h>
+#include <drm/drm_gem.h>
+#include <drm/drm_mm.h>
+#include "../version.h"
+#include "../lib/libxdma_api.h"
+#include "common.h"
+
+#if defined(__PPC64__)
+#define XOCL_FILE_PAGE_OFFSET 0x10000
+#else
+#define XOCL_FILE_PAGE_OFFSET 0x100000
+#endif
+
+#ifndef VM_RESERVED
+#define VM_RESERVED (VM_DONTEXPAND | VM_DONTDUMP)
+#endif
+
+#ifdef _XOCL_DRM_DEBUG
+#define DRM_ENTER(fmt, args...) \
+ printk(KERN_INFO "[DRM] Entering %s:"fmt"\n", __func__, ##args)
+#define DRM_DBG(fmt, args...) \
+ printk(KERN_INFO "[DRM] %s:%d:"fmt"\n", __func__, __LINE__, ##args)
+#else
+#define DRM_ENTER(fmt, args...)
+#define DRM_DBG(fmt, args...)
+#endif
+
+static char driver_date[9];
+
+static void xocl_free_object(struct drm_gem_object *obj)
+{
+ DRM_ENTER("");
+ xocl_drm_free_bo(obj);
+}
+
+static int xocl_open(struct inode *inode, struct file *filp)
+{
+ struct xocl_drm *drm_p;
+ struct drm_file *priv;
+ struct drm_device *ddev;
+ int ret;
+
+ ret = drm_open(inode, filp);
+ if (ret)
+ return ret;
+
+ priv = filp->private_data;
+ ddev = priv->minor->dev;
+ drm_p = xocl_drvinst_open(ddev);
+ if (!drm_p)
+ return -ENXIO;
+
+ return 0;
+}
+
+static int xocl_release(struct inode *inode, struct file *filp)
+{
+ struct drm_file *priv = filp->private_data;
+ struct drm_device *ddev = priv->minor->dev;
+ struct xocl_drm *drm_p = ddev->dev_private;
+ int ret;
+
+ ret = drm_release(inode, filp);
+ xocl_drvinst_close(drm_p);
+
+ return ret;
+}
+
+static int xocl_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+ int ret;
+ struct drm_file *priv = filp->private_data;
+ struct drm_device *dev = priv->minor->dev;
+ struct mm_struct *mm = current->mm;
+ struct xocl_drm *drm_p = dev->dev_private;
+ xdev_handle_t xdev = drm_p->xdev;
+ unsigned long vsize;
+ phys_addr_t res_start;
+
+ DRM_ENTER("vm pgoff %lx", vma->vm_pgoff);
+
+ /*
+ * If the page offset is > than 4G, then let GEM handle that and do what
+ * it thinks is best,we will only handle page offsets less than 4G.
+ */
+ if (likely(vma->vm_pgoff >= XOCL_FILE_PAGE_OFFSET)) {
+ ret = drm_gem_mmap(filp, vma);
+ if (ret)
+ return ret;
+ /* Clear VM_PFNMAP flag set by drm_gem_mmap()
+ * we have "struct page" for all backing pages for bo
+ */
+ vma->vm_flags &= ~VM_PFNMAP;
+ /* Clear VM_IO flag set by drm_gem_mmap()
+ * it prevents gdb from accessing mapped buffers
+ */
+ vma->vm_flags &= ~VM_IO;
+ vma->vm_flags |= VM_MIXEDMAP;
+ vma->vm_flags |= mm->def_flags;
+ vma->vm_pgoff = 0;
+
+ /* Override pgprot_writecombine() mapping setup by
+ * drm_gem_mmap()
+ * which results in very poor read performance
+ */
+ if (vma->vm_flags & (VM_READ | VM_MAYREAD))
+ vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+ else
+ vma->vm_page_prot = pgprot_writecombine(
+ vm_get_page_prot(vma->vm_flags));
+ return ret;
+ }
+
+ if (vma->vm_pgoff != 0)
+ return -EINVAL;
+
+ vsize = vma->vm_end - vma->vm_start;
+ if (vsize > XDEV(xdev)->bar_size)
+ return -EINVAL;
+
+ DRM_DBG("MAP size %ld", vsize);
+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+ vma->vm_flags |= VM_IO;
+ vma->vm_flags |= VM_RESERVED;
+
+ res_start = pci_resource_start(XDEV(xdev)->pdev, XDEV(xdev)->bar_idx);
+ ret = io_remap_pfn_range(vma, vma->vm_start,
+ res_start >> PAGE_SHIFT,
+ vsize, vma->vm_page_prot);
+ userpf_info(xdev, "io_remap_pfn_range ret code: %d", ret);
+
+ return ret;
+}
+
+int xocl_gem_fault(struct vm_fault *vmf)
+{
+ loff_t num_pages;
+ unsigned int page_offset;
+ struct vm_area_struct *vma = vmf->vma;
+ struct drm_xocl_bo *xobj = to_xocl_bo(vma->vm_private_data);
+ int ret = 0;
+ unsigned long vmf_address = vmf->address;
+
+ page_offset = (vmf_address - vma->vm_start) >> PAGE_SHIFT;
+
+
+ if (!xobj->pages)
+ return VM_FAULT_SIGBUS;
+
+ num_pages = DIV_ROUND_UP(xobj->base.size, PAGE_SIZE);
+ if (page_offset > num_pages)
+ return VM_FAULT_SIGBUS;
+
+ if (xobj->type & XOCL_BO_P2P)
+ ret = vm_insert_page(vma, vmf_address, xobj->pages[page_offset]);
+ else
+ ret = vm_insert_page(vma, vmf_address, xobj->pages[page_offset]);
+
+ switch (ret) {
+ case -EAGAIN:
+ case 0:
+ case -ERESTARTSYS:
+ return VM_FAULT_NOPAGE;
+ case -ENOMEM:
+ return VM_FAULT_OOM;
+ default:
+ return VM_FAULT_SIGBUS;
+ }
+}
+
+static int xocl_client_open(struct drm_device *dev, struct drm_file *filp)
+{
+ struct xocl_drm *drm_p = dev->dev_private;
+ int ret = 0;
+
+ DRM_ENTER("");
+
+ /* We do not allow users to open PRIMARY node, /dev/dri/cardX node.
+ * Users should only open RENDER, /dev/dri/renderX node
+ */
+ if (drm_is_primary_client(filp))
+ return -EPERM;
+
+ if (get_live_client_size(drm_p->xdev) > XOCL_MAX_CONCURRENT_CLIENTS)
+ return -EBUSY;
+
+ ret = xocl_exec_create_client(drm_p->xdev, &filp->driver_priv);
+ if (ret)
+ goto failed;
+
+ return 0;
+
+failed:
+ return ret;
+}
+
+static void xocl_client_release(struct drm_device *dev, struct drm_file *filp)
+{
+ struct xocl_drm *drm_p = dev->dev_private;
+
+ xocl_exec_destroy_client(drm_p->xdev, &filp->driver_priv);
+}
+
+static uint xocl_poll(struct file *filp, poll_table *wait)
+{
+ struct drm_file *priv = filp->private_data;
+ struct drm_device *dev = priv->minor->dev;
+ struct xocl_drm *drm_p = dev->dev_private;
+
+ BUG_ON(!priv->driver_priv);
+
+ DRM_ENTER("");
+ return xocl_exec_poll_client(drm_p->xdev, filp, wait, priv->driver_priv);
+}
+
+static const struct drm_ioctl_desc xocl_ioctls[] = {
+ DRM_IOCTL_DEF_DRV(XOCL_CREATE_BO, xocl_create_bo_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(XOCL_USERPTR_BO, xocl_userptr_bo_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(XOCL_MAP_BO, xocl_map_bo_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(XOCL_SYNC_BO, xocl_sync_bo_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(XOCL_INFO_BO, xocl_info_bo_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(XOCL_PWRITE_BO, xocl_pwrite_bo_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(XOCL_PREAD_BO, xocl_pread_bo_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(XOCL_CTX, xocl_ctx_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(XOCL_INFO, xocl_info_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(XOCL_READ_AXLF, xocl_read_axlf_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(XOCL_PWRITE_UNMGD, xocl_pwrite_unmgd_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(XOCL_PREAD_UNMGD, xocl_pread_unmgd_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(XOCL_USAGE_STAT, xocl_usage_stat_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(XOCL_USER_INTR, xocl_user_intr_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(XOCL_EXECBUF, xocl_execbuf_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(XOCL_COPY_BO, xocl_copy_bo_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(XOCL_HOT_RESET, xocl_hot_reset_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(XOCL_RECLOCK, xocl_reclock_ioctl,
+ DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+};
+
+static long xocl_drm_ioctl(struct file *filp,
+ unsigned int cmd, unsigned long arg)
+{
+ return drm_ioctl(filp, cmd, arg);
+}
+
+static const struct file_operations xocl_driver_fops = {
+ .owner = THIS_MODULE,
+ .open = xocl_open,
+ .mmap = xocl_mmap,
+ .poll = xocl_poll,
+ .read = drm_read,
+ .unlocked_ioctl = xocl_drm_ioctl,
+ .release = xocl_release,
+};
+
+static const struct vm_operations_struct xocl_vm_ops = {
+ .fault = xocl_gem_fault,
+ .open = drm_gem_vm_open,
+ .close = drm_gem_vm_close,
+};
+
+static struct drm_driver mm_drm_driver = {
+ .driver_features = DRIVER_GEM | DRIVER_PRIME |
+ DRIVER_RENDER,
+
+ .postclose = xocl_client_release,
+ .open = xocl_client_open,
+
+ .gem_free_object = xocl_free_object,
+ .gem_vm_ops = &xocl_vm_ops,
+
+ .ioctls = xocl_ioctls,
+ .num_ioctls = ARRAY_SIZE(xocl_ioctls),
+ .fops = &xocl_driver_fops,
+
+ .gem_prime_get_sg_table = xocl_gem_prime_get_sg_table,
+ .gem_prime_import_sg_table = xocl_gem_prime_import_sg_table,
+ .gem_prime_vmap = xocl_gem_prime_vmap,
+ .gem_prime_vunmap = xocl_gem_prime_vunmap,
+ .gem_prime_mmap = xocl_gem_prime_mmap,
+
+ .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
+ .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
+ .gem_prime_import = drm_gem_prime_import,
+ .gem_prime_export = drm_gem_prime_export,
+ .name = XOCL_MODULE_NAME,
+ .desc = XOCL_DRIVER_DESC,
+ .date = driver_date,
+};
+
+void *xocl_drm_init(xdev_handle_t xdev_hdl)
+{
+ struct xocl_drm *drm_p = NULL;
+ struct drm_device *ddev = NULL;
+ int year, mon, day;
+ int ret = 0;
+ bool drm_registered = false;
+
+ sscanf(XRT_DRIVER_VERSION, "%d.%d.%d",
+ &mm_drm_driver.major,
+ &mm_drm_driver.minor,
+ &mm_drm_driver.patchlevel);
+
+ sscanf(xrt_build_version_date, "%d-%d-%d ", &year, &mon, &day);
+ snprintf(driver_date, sizeof(driver_date),
+ "%d%02d%02d", year, mon, day);
+
+ ddev = drm_dev_alloc(&mm_drm_driver, &XDEV(xdev_hdl)->pdev->dev);
+ if (!ddev) {
+ xocl_xdev_err(xdev_hdl, "alloc drm dev failed");
+ ret = -ENOMEM;
+ goto failed;
+ }
+
+ drm_p = xocl_drvinst_alloc(ddev->dev, sizeof(*drm_p));
+ if (!drm_p) {
+ xocl_xdev_err(xdev_hdl, "alloc drm inst failed");
+ ret = -ENOMEM;
+ goto failed;
+ }
+ drm_p->xdev = xdev_hdl;
+
+ ddev->pdev = XDEV(xdev_hdl)->pdev;
+
+ ret = drm_dev_register(ddev, 0);
+ if (ret) {
+ xocl_xdev_err(xdev_hdl, "register drm dev failed 0x%x", ret);
+ goto failed;
+ }
+ drm_registered = true;
+
+ drm_p->ddev = ddev;
+
+ mutex_init(&drm_p->mm_lock);
+ ddev->dev_private = drm_p;
+ hash_init(drm_p->mm_range);
+
+ xocl_drvinst_set_filedev(drm_p, ddev);
+ return drm_p;
+
+failed:
+ if (drm_registered)
+ drm_dev_unregister(ddev);
+//PORT4_20
+ if (ddev)
+ drm_dev_put(ddev);
+ if (drm_p)
+ xocl_drvinst_free(drm_p);
+
+ return NULL;
+}
+
+void xocl_drm_fini(struct xocl_drm *drm_p)
+{
+ xocl_cleanup_mem(drm_p);
+ drm_put_dev(drm_p->ddev);
+ mutex_destroy(&drm_p->mm_lock);
+
+ xocl_drvinst_free(drm_p);
+}
+
+void xocl_mm_get_usage_stat(struct xocl_drm *drm_p, u32 ddr,
+ struct drm_xocl_mm_stat *pstat)
+{
+ pstat->memory_usage = drm_p->mm_usage_stat[ddr] ?
+ drm_p->mm_usage_stat[ddr]->memory_usage : 0;
+ pstat->bo_count = drm_p->mm_usage_stat[ddr] ?
+ drm_p->mm_usage_stat[ddr]->bo_count : 0;
+}
+
+void xocl_mm_update_usage_stat(struct xocl_drm *drm_p, u32 ddr,
+ u64 size, int count)
+{
+ BUG_ON(!drm_p->mm_usage_stat[ddr]);
+
+ drm_p->mm_usage_stat[ddr]->memory_usage += (count > 0) ? size : -size;
+ drm_p->mm_usage_stat[ddr]->bo_count += count;
+}
+
+int xocl_mm_insert_node(struct xocl_drm *drm_p, u32 ddr,
+ struct drm_mm_node *node, u64 size)
+{
+ return drm_mm_insert_node_generic(drm_p->mm[ddr], node, size, PAGE_SIZE,
+#if defined(XOCL_DRM_FREE_MALLOC)
+ 0, 0);
+#else
+ 0, 0, 0);
+#endif
+}
+
+int xocl_check_topology(struct xocl_drm *drm_p)
+{
+ struct mem_topology *topology;
+ u16 i;
+ int err = 0;
+
+ topology = XOCL_MEM_TOPOLOGY(drm_p->xdev);
+ if (topology == NULL)
+ return 0;
+
+ for (i = 0; i < topology->m_count; i++) {
+ if (!topology->m_mem_data[i].m_used)
+ continue;
+
+ if (topology->m_mem_data[i].m_type == MEM_STREAMING)
+ continue;
+
+ if (drm_p->mm_usage_stat[i]->bo_count != 0) {
+ err = -EPERM;
+ xocl_err(drm_p->ddev->dev,
+ "The ddr %d has pre-existing buffer allocations, please exit and re-run.",
+ i);
+ }
+ }
+
+ return err;
+}
+
+uint32_t xocl_get_shared_ddr(struct xocl_drm *drm_p, struct mem_data *m_data)
+{
+ struct xocl_mm_wrapper *wrapper;
+ uint64_t start_addr = m_data->m_base_address;
+ uint64_t sz = m_data->m_size*1024;
+
+ hash_for_each_possible(drm_p->mm_range, wrapper, node, start_addr) {
+ if (!wrapper)
+ continue;
+
+ if (wrapper->start_addr == start_addr) {
+ if (wrapper->size == sz)
+ return wrapper->ddr;
+ else
+ return 0xffffffff;
+ }
+ }
+ return 0xffffffff;
+}
+
+void xocl_cleanup_mem(struct xocl_drm *drm_p)
+{
+ struct mem_topology *topology;
+ u16 i, ddr;
+ uint64_t addr;
+ struct xocl_mm_wrapper *wrapper;
+ struct hlist_node *tmp;
+
+ topology = XOCL_MEM_TOPOLOGY(drm_p->xdev);
+ if (topology) {
+ ddr = topology->m_count;
+ for (i = 0; i < ddr; i++) {
+ if (!topology->m_mem_data[i].m_used)
+ continue;
+
+ if (topology->m_mem_data[i].m_type == MEM_STREAMING)
+ continue;
+
+ xocl_info(drm_p->ddev->dev, "Taking down DDR : %d", i);
+ addr = topology->m_mem_data[i].m_base_address;
+
+ hash_for_each_possible_safe(drm_p->mm_range, wrapper,
+ tmp, node, addr) {
+ if (wrapper->ddr == i) {
+ hash_del(&wrapper->node);
+ vfree(wrapper);
+ drm_mm_takedown(drm_p->mm[i]);
+ vfree(drm_p->mm[i]);
+ vfree(drm_p->mm_usage_stat[i]);
+ }
+ }
+
+ drm_p->mm[i] = NULL;
+ drm_p->mm_usage_stat[i] = NULL;
+ }
+ }
+ vfree(drm_p->mm);
+ drm_p->mm = NULL;
+ vfree(drm_p->mm_usage_stat);
+ drm_p->mm_usage_stat = NULL;
+ vfree(drm_p->mm_p2p_off);
+ drm_p->mm_p2p_off = NULL;
+}
+
+int xocl_init_mem(struct xocl_drm *drm_p)
+{
+ size_t length = 0;
+ size_t mm_size = 0, mm_stat_size = 0;
+ size_t size = 0, wrapper_size = 0;
+ size_t ddr_bank_size;
+ struct mem_topology *topo;
+ struct mem_data *mem_data;
+ uint32_t shared;
+ struct xocl_mm_wrapper *wrapper = NULL;
+ uint64_t reserved1 = 0;
+ uint64_t reserved2 = 0;
+ uint64_t reserved_start;
+ uint64_t reserved_end;
+ int err = 0;
+ int i = -1;
+
+ if (XOCL_DSA_IS_MPSOC(drm_p->xdev)) {
+ /* TODO: This is still hardcoding.. */
+ reserved1 = 0x80000000;
+ reserved2 = 0x1000000;
+ }
+
+ topo = XOCL_MEM_TOPOLOGY(drm_p->xdev);
+ if (topo == NULL)
+ return 0;
+
+ length = topo->m_count * sizeof(struct mem_data);
+ size = topo->m_count * sizeof(void *);
+ wrapper_size = sizeof(struct xocl_mm_wrapper);
+ mm_size = sizeof(struct drm_mm);
+ mm_stat_size = sizeof(struct drm_xocl_mm_stat);
+
+ xocl_info(drm_p->ddev->dev, "Topology count = %d, data_length = %ld",
+ topo->m_count, length);
+
+ drm_p->mm = vzalloc(size);
+ drm_p->mm_usage_stat = vzalloc(size);
+ drm_p->mm_p2p_off = vzalloc(topo->m_count * sizeof(u64));
+ if (!drm_p->mm || !drm_p->mm_usage_stat || !drm_p->mm_p2p_off) {
+ err = -ENOMEM;
+ goto failed;
+ }
+
+ for (i = 0; i < topo->m_count; i++) {
+ mem_data = &topo->m_mem_data[i];
+ ddr_bank_size = mem_data->m_size * 1024;
+
+ xocl_info(drm_p->ddev->dev, " Mem Index %d", i);
+ xocl_info(drm_p->ddev->dev, " Base Address:0x%llx",
+ mem_data->m_base_address);
+ xocl_info(drm_p->ddev->dev, " Size:0x%lx", ddr_bank_size);
+ xocl_info(drm_p->ddev->dev, " Type:%d", mem_data->m_type);
+ xocl_info(drm_p->ddev->dev, " Used:%d", mem_data->m_used);
+ }
+
+ /* Initialize the used banks and their sizes */
+ /* Currently only fixed sizes are supported */
+ for (i = 0; i < topo->m_count; i++) {
+ mem_data = &topo->m_mem_data[i];
+ if (!mem_data->m_used)
+ continue;
+
+ if (mem_data->m_type == MEM_STREAMING ||
+ mem_data->m_type == MEM_STREAMING_CONNECTION)
+ continue;
+
+ ddr_bank_size = mem_data->m_size * 1024;
+ xocl_info(drm_p->ddev->dev, "Allocating DDR bank%d", i);
+ xocl_info(drm_p->ddev->dev, " base_addr:0x%llx, total size:0x%lx",
+ mem_data->m_base_address, ddr_bank_size);
+
+ if (XOCL_DSA_IS_MPSOC(drm_p->xdev)) {
+ reserved_end = mem_data->m_base_address + ddr_bank_size;
+ reserved_start = reserved_end - reserved1 - reserved2;
+ xocl_info(drm_p->ddev->dev, " reserved region:0x%llx - 0x%llx",
+ reserved_start, reserved_end - 1);
+ }
+
+ shared = xocl_get_shared_ddr(drm_p, mem_data);
+ if (shared != 0xffffffff) {
+ xocl_info(drm_p->ddev->dev, "Found duplicated memory region!");
+ drm_p->mm[i] = drm_p->mm[shared];
+ drm_p->mm_usage_stat[i] = drm_p->mm_usage_stat[shared];
+ continue;
+ }
+
+ xocl_info(drm_p->ddev->dev, "Found a new memory region");
+ wrapper = vzalloc(wrapper_size);
+ drm_p->mm[i] = vzalloc(mm_size);
+ drm_p->mm_usage_stat[i] = vzalloc(mm_stat_size);
+
+ if (!drm_p->mm[i] || !drm_p->mm_usage_stat[i] || !wrapper) {
+ err = -ENOMEM;
+ goto failed;
+ }
+
+ wrapper->start_addr = mem_data->m_base_address;
+ wrapper->size = mem_data->m_size*1024;
+ wrapper->mm = drm_p->mm[i];
+ wrapper->mm_usage_stat = drm_p->mm_usage_stat[i];
+ wrapper->ddr = i;
+ hash_add(drm_p->mm_range, &wrapper->node, wrapper->start_addr);
+
+ drm_mm_init(drm_p->mm[i], mem_data->m_base_address,
+ ddr_bank_size - reserved1 - reserved2);
+ drm_p->mm_p2p_off[i] = ddr_bank_size * i;
+
+ xocl_info(drm_p->ddev->dev, "drm_mm_init called");
+ }
+
+ return 0;
+
+failed:
+ vfree(wrapper);
+ if (drm_p->mm) {
+ for (; i >= 0; i--) {
+ drm_mm_takedown(drm_p->mm[i]);
+ vfree(drm_p->mm[i]);
+ vfree(drm_p->mm_usage_stat[i]);
+ }
+ vfree(drm_p->mm);
+ drm_p->mm = NULL;
+ }
+ vfree(drm_p->mm_usage_stat);
+ drm_p->mm_usage_stat = NULL;
+ vfree(drm_p->mm_p2p_off);
+ drm_p->mm_p2p_off = NULL;
+
+ return err;
+}
diff --git a/drivers/gpu/drm/xocl/userpf/xocl_drv.c b/drivers/gpu/drm/xocl/userpf/xocl_drv.c
new file mode 100644
index 000000000000..6fc57da3deab
--- /dev/null
+++ b/drivers/gpu/drm/xocl/userpf/xocl_drv.c
@@ -0,0 +1,743 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright (C) 2016-2019 Xilinx, Inc. All rights reserved.
+ *
+ * Authors: Lizhi.Hou@...inx.com
+ *
+ */
+
+#include <linux/pci.h>
+#include <linux/aer.h>
+#include <linux/version.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include "../xocl_drv.h"
+#include "common.h"
+#include "../version.h"
+#include <linux/memremap.h>
+
+#ifndef PCI_EXT_CAP_ID_REBAR
+#define PCI_EXT_CAP_ID_REBAR 0x15
+#endif
+
+#ifndef PCI_REBAR_CTRL
+#define PCI_REBAR_CTRL 8 /* control register */
+#endif
+
+#ifndef PCI_REBAR_CTRL_BAR_SIZE
+#define PCI_REBAR_CTRL_BAR_SIZE 0x00001F00 /* BAR size */
+#endif
+
+#ifndef PCI_REBAR_CTRL_BAR_SHIFT
+#define PCI_REBAR_CTRL_BAR_SHIFT 8 /* shift for BAR size */
+#endif
+
+#define REBAR_FIRST_CAP 4
+
+static const struct pci_device_id pciidlist[] = XOCL_USER_PCI_IDS;
+
+struct class *xrt_class;
+
+MODULE_DEVICE_TABLE(pci, pciidlist);
+
+static int userpf_intr_config(xdev_handle_t xdev_hdl, u32 intr, bool en)
+{
+ return xocl_dma_intr_config(xdev_hdl, intr, en);
+}
+
+static int userpf_intr_register(xdev_handle_t xdev_hdl, u32 intr,
+ irq_handler_t handler, void *arg)
+{
+ return handler ?
+ xocl_dma_intr_register(xdev_hdl, intr, handler, arg, -1) :
+ xocl_dma_intr_unreg(xdev_hdl, intr);
+}
+
+struct xocl_pci_funcs userpf_pci_ops = {
+ .intr_config = userpf_intr_config,
+ .intr_register = userpf_intr_register,
+};
+
+void xocl_reset_notify(struct pci_dev *pdev, bool prepare)
+{
+ struct xocl_dev *xdev = pci_get_drvdata(pdev);
+
+ xocl_info(&pdev->dev, "PCI reset NOTIFY, prepare %d", prepare);
+
+ if (prepare) {
+ xocl_mailbox_reset(xdev, false);
+ xocl_subdev_destroy_by_id(xdev, XOCL_SUBDEV_DMA);
+ } else {
+ reset_notify_client_ctx(xdev);
+ xocl_subdev_create_by_id(xdev, XOCL_SUBDEV_DMA);
+ xocl_mailbox_reset(xdev, true);
+ xocl_exec_reset(xdev);
+ }
+}
+
+static void kill_all_clients(struct xocl_dev *xdev)
+{
+ struct list_head *ptr;
+ struct client_ctx *entry;
+ int ret;
+ int total_wait_secs = 10; // sec
+ int wait_interval = 100; // millisec
+ int retry = total_wait_secs * 1000 / wait_interval;
+
+ mutex_lock(&xdev->ctx_list_lock);
+
+ list_for_each(ptr, &xdev->ctx_list) {
+ entry = list_entry(ptr, struct client_ctx, link);
+ ret = kill_pid(entry->pid, SIGBUS, 1);
+ if (ret) {
+ userpf_err(xdev, "killing pid: %d failed. err: %d",
+ pid_nr(entry->pid), ret);
+ }
+ }
+
+ mutex_unlock(&xdev->ctx_list_lock);
+
+ while (!list_empty(&xdev->ctx_list) && retry--)
+ msleep(wait_interval);
+
+ if (!list_empty(&xdev->ctx_list))
+ userpf_err(xdev, "failed to kill all clients");
+}
+
+int64_t xocl_hot_reset(struct xocl_dev *xdev, bool force)
+{
+ bool skip = false;
+ int64_t ret = 0, mbret = 0;
+ struct mailbox_req mbreq = { MAILBOX_REQ_HOT_RESET, };
+ size_t resplen = sizeof(ret);
+
+ mutex_lock(&xdev->ctx_list_lock);
+ if (xdev->offline) {
+ skip = true;
+ } else if (!force && !list_is_singular(&xdev->ctx_list)) {
+ /* We should have one context for ourselves. */
+ BUG_ON(list_empty(&xdev->ctx_list));
+ userpf_err(xdev, "device is in use, can't reset");
+ ret = -EBUSY;
+ } else {
+ xdev->offline = true;
+ }
+ mutex_unlock(&xdev->ctx_list_lock);
+ if (ret < 0 || skip)
+ return ret;
+
+ userpf_info(xdev, "resetting device...");
+
+ if (force)
+ kill_all_clients(xdev);
+
+ xocl_reset_notify(xdev->core.pdev, true);
+ mbret = xocl_peer_request(xdev, &mbreq, sizeof(struct mailbox_req), &ret, &resplen, NULL, NULL);
+ if (mbret)
+ ret = mbret;
+ xocl_reset_notify(xdev->core.pdev, false);
+
+ mutex_lock(&xdev->ctx_list_lock);
+ xdev->offline = false;
+ mutex_unlock(&xdev->ctx_list_lock);
+
+ return ret;
+}
+
+
+int xocl_reclock(struct xocl_dev *xdev, void *data)
+{
+ int err = 0;
+ int64_t msg = -ENODEV;
+ struct mailbox_req *req = NULL;
+ size_t resplen = sizeof(msg);
+ size_t reqlen = sizeof(struct mailbox_req)+sizeof(struct drm_xocl_reclock_info);
+
+ req = kzalloc(reqlen, GFP_KERNEL);
+ req->req = MAILBOX_REQ_RECLOCK;
+ req->data_total_len = sizeof(struct drm_xocl_reclock_info);
+ memcpy(req->data, data, sizeof(struct drm_xocl_reclock_info));
+
+ err = xocl_peer_request(xdev, req, reqlen,
+ &msg, &resplen, NULL, NULL);
+
+ if (msg != 0)
+ err = -ENODEV;
+
+ kfree(req);
+ return err;
+}
+
+static void xocl_mailbox_srv(void *arg, void *data, size_t len,
+ u64 msgid, int err)
+{
+ struct xocl_dev *xdev = (struct xocl_dev *)arg;
+ struct mailbox_req *req = (struct mailbox_req *)data;
+
+ if (err != 0)
+ return;
+
+ userpf_info(xdev, "received request (%d) from peer\n", req->req);
+
+ switch (req->req) {
+ case MAILBOX_REQ_FIREWALL:
+ (void) xocl_hot_reset(xdev, true);
+ break;
+ default:
+ userpf_err(xdev, "dropped bad request (%d)\n", req->req);
+ break;
+ }
+}
+
+void get_pcie_link_info(struct xocl_dev *xdev,
+ unsigned short *link_width, unsigned short *link_speed, bool is_cap)
+{
+ u16 stat;
+ long result;
+ int pos = is_cap ? PCI_EXP_LNKCAP : PCI_EXP_LNKSTA;
+
+ result = pcie_capability_read_word(xdev->core.pdev, pos, &stat);
+ if (result) {
+ *link_width = *link_speed = 0;
+ xocl_info(&xdev->core.pdev->dev, "Read pcie capability failed");
+ return;
+ }
+ *link_width = (stat & PCI_EXP_LNKSTA_NLW) >> PCI_EXP_LNKSTA_NLW_SHIFT;
+ *link_speed = stat & PCI_EXP_LNKSTA_CLS;
+}
+
+void user_pci_reset_prepare(struct pci_dev *pdev)
+{
+ xocl_reset_notify(pdev, true);
+}
+
+void user_pci_reset_done(struct pci_dev *pdev)
+{
+ xocl_reset_notify(pdev, false);
+}
+
+static void xocl_dev_percpu_release(struct percpu_ref *ref)
+{
+ struct xocl_dev *xdev = container_of(ref, struct xocl_dev, ref);
+
+ complete(&xdev->cmp);
+}
+
+static void xocl_dev_percpu_exit(void *data)
+{
+ struct percpu_ref *ref = data;
+ struct xocl_dev *xdev = container_of(ref, struct xocl_dev, ref);
+
+ wait_for_completion(&xdev->cmp);
+ percpu_ref_exit(ref);
+}
+
+
+static void xocl_dev_percpu_kill(void *data)
+{
+ struct percpu_ref *ref = data;
+
+ percpu_ref_kill(ref);
+}
+
+void xocl_p2p_mem_release(struct xocl_dev *xdev, bool recov_bar_sz)
+{
+ struct pci_dev *pdev = xdev->core.pdev;
+ int p2p_bar = -1;
+
+ if (xdev->p2p_bar_addr) {
+ devres_release_group(&pdev->dev, xdev->p2p_res_grp);
+ xdev->p2p_bar_addr = NULL;
+ xdev->p2p_res_grp = NULL;
+ }
+ if (xdev->p2p_res_grp) {
+ devres_remove_group(&pdev->dev, xdev->p2p_res_grp);
+ xdev->p2p_res_grp = NULL;
+ }
+
+ if (recov_bar_sz) {
+ p2p_bar = xocl_get_p2p_bar(xdev, NULL);
+ if (p2p_bar < 0)
+ return;
+
+ xocl_pci_resize_resource(pdev, p2p_bar,
+ (XOCL_PA_SECTION_SHIFT - 20));
+
+ xocl_info(&pdev->dev, "Resize p2p bar %d to %d M ", p2p_bar,
+ (1 << XOCL_PA_SECTION_SHIFT));
+ }
+}
+
+int xocl_p2p_mem_reserve(struct xocl_dev *xdev)
+{
+ resource_size_t p2p_bar_addr;
+ resource_size_t p2p_bar_len;
+ struct resource res;
+ uint32_t p2p_bar_idx;
+ struct pci_dev *pdev = xdev->core.pdev;
+ int32_t ret;
+
+ xocl_info(&pdev->dev, "reserve p2p mem, bar %d, len %lld",
+ xdev->p2p_bar_idx, xdev->p2p_bar_len);
+
+ if (xdev->p2p_bar_idx < 0 ||
+ xdev->p2p_bar_len <= (1<<XOCL_PA_SECTION_SHIFT)) {
+ /* only p2p_bar_len > SECTION (256MB) */
+ xocl_info(&pdev->dev, "Did not find p2p BAR");
+ return 0;
+ }
+
+ p2p_bar_len = xdev->p2p_bar_len;
+ p2p_bar_idx = xdev->p2p_bar_idx;
+
+ xdev->p2p_res_grp = devres_open_group(&pdev->dev, NULL, GFP_KERNEL);
+ if (!xdev->p2p_res_grp) {
+ xocl_err(&pdev->dev, "open p2p resource group failed");
+ ret = -ENOMEM;
+ goto failed;
+ }
+
+ p2p_bar_addr = pci_resource_start(pdev, p2p_bar_idx);
+
+ res.start = p2p_bar_addr;
+ res.end = p2p_bar_addr+p2p_bar_len-1;
+ res.name = NULL;
+ res.flags = IORESOURCE_MEM;
+
+ init_completion(&xdev->cmp);
+
+ ret = percpu_ref_init(&xdev->ref, xocl_dev_percpu_release, 0,
+ GFP_KERNEL);
+ if (ret)
+ goto failed;
+
+ ret = devm_add_action_or_reset(&(pdev->dev), xocl_dev_percpu_exit,
+ &xdev->ref);
+ if (ret)
+ goto failed;
+
+ xdev->pgmap.ref = &xdev->ref;
+ memcpy(&xdev->pgmap.res, &res, sizeof(struct resource));
+ xdev->pgmap.altmap_valid = false;
+ xdev->p2p_bar_addr = devm_memremap_pages(&(pdev->dev), &xdev->pgmap);
+
+ if (!xdev->p2p_bar_addr) {
+ ret = -ENOMEM;
+ percpu_ref_kill(&xdev->ref);
+ devres_close_group(&pdev->dev, xdev->p2p_res_grp);
+ goto failed;
+ }
+
+ ret = devm_add_action_or_reset(&(pdev->dev), xocl_dev_percpu_kill,
+ &xdev->ref);
+ if (ret) {
+ percpu_ref_kill(&xdev->ref);
+ devres_close_group(&pdev->dev, xdev->p2p_res_grp);
+ goto failed;
+ }
+
+ devres_close_group(&pdev->dev, xdev->p2p_res_grp);
+
+ return 0;
+
+failed:
+ xocl_p2p_mem_release(xdev, false);
+
+ return ret;
+}
+
+static inline u64 xocl_pci_rebar_size_to_bytes(int size)
+{
+ return 1ULL << (size + 20);
+}
+
+int xocl_get_p2p_bar(struct xocl_dev *xdev, u64 *bar_size)
+{
+ struct pci_dev *dev = xdev->core.pdev;
+ int i, pos;
+ u32 cap, ctrl, size;
+
+ pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_REBAR);
+ if (!pos) {
+ xocl_err(&dev->dev, "did not find rebar cap");
+ return -ENOTSUPP;
+ }
+
+ pos += REBAR_FIRST_CAP;
+ for (i = PCI_STD_RESOURCES; i <= PCI_STD_RESOURCE_END; i++) {
+ pci_read_config_dword(dev, pos, &cap);
+ pci_read_config_dword(dev, pos + 4, &ctrl);
+ size = (ctrl & PCI_REBAR_CTRL_BAR_SIZE) >>
+ PCI_REBAR_CTRL_BAR_SHIFT;
+ if (xocl_pci_rebar_size_to_bytes(size) >=
+ (1 << XOCL_PA_SECTION_SHIFT) &&
+ cap >= 0x1000) {
+ if (bar_size)
+ *bar_size = xocl_pci_rebar_size_to_bytes(size);
+ return i;
+ }
+ pos += 8;
+ }
+
+ if (bar_size)
+ *bar_size = 0;
+
+ return -1;
+}
+
+static int xocl_reassign_resources(struct pci_dev *dev, int resno)
+{
+ pci_assign_unassigned_bus_resources(dev->bus);
+
+ return 0;
+}
+
+int xocl_pci_resize_resource(struct pci_dev *dev, int resno, int size)
+{
+ struct resource *res = dev->resource + resno;
+ struct pci_dev *root;
+ struct resource *root_res;
+ u64 bar_size, req_size;
+ unsigned long flags;
+ u16 cmd;
+ int pos, ret = 0;
+ u32 ctrl, i;
+
+ pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_REBAR);
+ if (!pos) {
+ xocl_err(&dev->dev, "did not find rebar cap");
+ return -ENOTSUPP;
+ }
+
+ pos += resno * PCI_REBAR_CTRL;
+ pci_read_config_dword(dev, pos + PCI_REBAR_CTRL, &ctrl);
+
+ bar_size = xocl_pci_rebar_size_to_bytes(
+ (ctrl & PCI_REBAR_CTRL_BAR_SIZE) >>
+ PCI_REBAR_CTRL_BAR_SHIFT);
+ req_size = xocl_pci_rebar_size_to_bytes(size);
+
+ xocl_info(&dev->dev, "req_size %lld, bar size %lld\n",
+ req_size, bar_size);
+ if (req_size == bar_size) {
+ xocl_info(&dev->dev, "same size, return success");
+ return -EALREADY;
+ }
+
+ xocl_get_root_dev(dev, root);
+
+ for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) {
+ root_res = root->subordinate->resource[i];
+ root_res = (root_res) ? root_res->parent : NULL;
+ if (root_res && (root_res->flags & IORESOURCE_MEM)
+ && resource_size(root_res) > req_size)
+ break;
+ }
+
+ if (i == PCI_BRIDGE_RESOURCE_NUM) {
+ xocl_err(&dev->dev, "Not enough IO Mem space, Please check BIOS settings. ");
+ return -ENOSPC;
+ }
+ pci_release_selected_regions(dev, (1 << resno));
+ pci_read_config_word(dev, PCI_COMMAND, &cmd);
+ pci_write_config_word(dev, PCI_COMMAND,
+ cmd & ~PCI_COMMAND_MEMORY);
+
+ flags = res->flags;
+ if (res->parent)
+ release_resource(res);
+
+ ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
+ ctrl |= size << PCI_REBAR_CTRL_BAR_SHIFT;
+ pci_write_config_dword(dev, pos + PCI_REBAR_CTRL, ctrl);
+
+
+ res->start = 0;
+ res->end = req_size - 1;
+
+ xocl_info(&dev->dev, "new size %lld", resource_size(res));
+ xocl_reassign_resources(dev, resno);
+ res->flags = flags;
+
+ pci_write_config_word(dev, PCI_COMMAND, cmd | PCI_COMMAND_MEMORY);
+ pci_request_selected_regions(dev, (1 << resno),
+ XOCL_MODULE_NAME);
+
+ return ret;
+}
+
+static int identify_bar(struct xocl_dev *xdev)
+{
+ struct pci_dev *pdev = xdev->core.pdev;
+ resource_size_t bar_len;
+ int i;
+
+ for (i = PCI_STD_RESOURCES; i <= PCI_STD_RESOURCE_END; i++) {
+ bar_len = pci_resource_len(pdev, i);
+ if (bar_len >= (1 << XOCL_PA_SECTION_SHIFT)) {
+ xdev->p2p_bar_idx = i;
+ xdev->p2p_bar_len = bar_len;
+ pci_request_selected_regions(pdev, 1 << i,
+ XOCL_MODULE_NAME);
+ } else if (bar_len >= 32 * 1024 * 1024) {
+ xdev->core.bar_addr = ioremap_nocache(
+ pci_resource_start(pdev, i), bar_len);
+ if (!xdev->core.bar_addr)
+ return -EIO;
+ xdev->core.bar_idx = i;
+ xdev->core.bar_size = bar_len;
+ }
+ }
+
+ return 0;
+}
+
+static void unmap_bar(struct xocl_dev *xdev)
+{
+ if (xdev->core.bar_addr) {
+ iounmap(xdev->core.bar_addr);
+ xdev->core.bar_addr = NULL;
+ }
+
+ if (xdev->p2p_bar_len)
+ pci_release_selected_regions(xdev->core.pdev,
+ 1 << xdev->p2p_bar_idx);
+}
+
+/* pci driver callbacks */
+int xocl_userpf_probe(struct pci_dev *pdev,
+ const struct pci_device_id *ent)
+{
+ struct xocl_dev *xdev;
+ struct xocl_board_private *dev_info;
+ int ret;
+
+ xdev = devm_kzalloc(&pdev->dev, sizeof(*xdev), GFP_KERNEL);
+ if (!xdev) {
+ xocl_err(&pdev->dev, "failed to alloc xocl_dev");
+ return -ENOMEM;
+ }
+
+ /* this is used for all subdevs, bind it to device earlier */
+ pci_set_drvdata(pdev, xdev);
+ dev_info = (struct xocl_board_private *)ent->driver_data;
+
+ xdev->core.pci_ops = &userpf_pci_ops;
+ xdev->core.pdev = pdev;
+ xocl_fill_dsa_priv(xdev, dev_info);
+
+ ret = identify_bar(xdev);
+ if (ret) {
+ xocl_err(&pdev->dev, "failed to identify bar");
+ goto failed_to_bar;
+ }
+
+ ret = pci_enable_device(pdev);
+ if (ret) {
+ xocl_err(&pdev->dev, "failed to enable device.");
+ goto failed_to_enable;
+ }
+
+ ret = xocl_alloc_dev_minor(xdev);
+ if (ret)
+ goto failed_alloc_minor;
+
+ ret = xocl_subdev_create_all(xdev, dev_info->subdev_info,
+ dev_info->subdev_num);
+ if (ret) {
+ xocl_err(&pdev->dev, "failed to register subdevs");
+ goto failed_create_subdev;
+ }
+
+ ret = xocl_p2p_mem_reserve(xdev);
+ if (ret)
+ xocl_err(&pdev->dev, "failed to reserve p2p memory region");
+
+ ret = xocl_init_sysfs(&pdev->dev);
+ if (ret) {
+ xocl_err(&pdev->dev, "failed to init sysfs");
+ goto failed_init_sysfs;
+ }
+
+ mutex_init(&xdev->ctx_list_lock);
+ xdev->needs_reset = false;
+ atomic64_set(&xdev->total_execs, 0);
+ atomic_set(&xdev->outstanding_execs, 0);
+ INIT_LIST_HEAD(&xdev->ctx_list);
+
+ /* Launch the mailbox server. */
+ (void) xocl_peer_listen(xdev, xocl_mailbox_srv, (void *)xdev);
+
+ return 0;
+
+failed_init_sysfs:
+ xocl_p2p_mem_release(xdev, false);
+ xocl_subdev_destroy_all(xdev);
+
+failed_create_subdev:
+ xocl_free_dev_minor(xdev);
+
+failed_alloc_minor:
+ pci_disable_device(pdev);
+failed_to_enable:
+ unmap_bar(xdev);
+failed_to_bar:
+ devm_kfree(&pdev->dev, xdev);
+ pci_set_drvdata(pdev, NULL);
+
+ return ret;
+}
+
+void xocl_userpf_remove(struct pci_dev *pdev)
+{
+ struct xocl_dev *xdev;
+
+ xdev = pci_get_drvdata(pdev);
+ if (!xdev) {
+ xocl_err(&pdev->dev, "driver data is NULL");
+ return;
+ }
+
+ xocl_p2p_mem_release(xdev, false);
+ xocl_subdev_destroy_all(xdev);
+
+ xocl_fini_sysfs(&pdev->dev);
+ xocl_free_dev_minor(xdev);
+
+ pci_disable_device(pdev);
+
+ unmap_bar(xdev);
+
+ mutex_destroy(&xdev->ctx_list_lock);
+
+ pci_set_drvdata(pdev, NULL);
+ devm_kfree(&pdev->dev, xdev);
+}
+
+static pci_ers_result_t user_pci_error_detected(struct pci_dev *pdev,
+ pci_channel_state_t state)
+{
+ switch (state) {
+ case pci_channel_io_normal:
+ xocl_info(&pdev->dev, "PCI normal state error\n");
+ return PCI_ERS_RESULT_CAN_RECOVER;
+ case pci_channel_io_frozen:
+ xocl_info(&pdev->dev, "PCI frozen state error\n");
+ return PCI_ERS_RESULT_NEED_RESET;
+ case pci_channel_io_perm_failure:
+ xocl_info(&pdev->dev, "PCI failure state error\n");
+ return PCI_ERS_RESULT_DISCONNECT;
+ default:
+ xocl_info(&pdev->dev, "PCI unknown state (%d) error\n", state);
+ break;
+ }
+
+ return PCI_ERS_RESULT_NEED_RESET;
+}
+
+static pci_ers_result_t user_pci_slot_reset(struct pci_dev *pdev)
+{
+ xocl_info(&pdev->dev, "PCI reset slot");
+ pci_restore_state(pdev);
+
+ return PCI_ERS_RESULT_RECOVERED;
+}
+
+static void user_pci_error_resume(struct pci_dev *pdev)
+{
+ xocl_info(&pdev->dev, "PCI error resume");
+ pci_cleanup_aer_uncorrect_error_status(pdev);
+}
+
+static const struct pci_error_handlers xocl_err_handler = {
+ .error_detected = user_pci_error_detected,
+ .slot_reset = user_pci_slot_reset,
+ .resume = user_pci_error_resume,
+ .reset_prepare = user_pci_reset_prepare,
+ .reset_done = user_pci_reset_done,
+};
+
+static struct pci_driver userpf_driver = {
+ .name = XOCL_MODULE_NAME,
+ .id_table = pciidlist,
+ .probe = xocl_userpf_probe,
+ .remove = xocl_userpf_remove,
+ .err_handler = &xocl_err_handler,
+};
+
+/* INIT */
+static int (*xocl_drv_reg_funcs[])(void) __initdata = {
+ xocl_init_feature_rom,
+ xocl_init_xdma,
+ xocl_init_mb_scheduler,
+ xocl_init_mailbox,
+ xocl_init_xmc,
+ xocl_init_icap,
+ xocl_init_xvc,
+};
+
+static void (*xocl_drv_unreg_funcs[])(void) = {
+ xocl_fini_feature_rom,
+ xocl_fini_xdma,
+ xocl_fini_mb_scheduler,
+ xocl_fini_mailbox,
+ xocl_fini_xmc,
+ xocl_fini_icap,
+ xocl_fini_xvc,
+};
+
+static int __init xocl_init(void)
+{
+ int ret, i;
+
+ xrt_class = class_create(THIS_MODULE, "xrt_user");
+ if (IS_ERR(xrt_class)) {
+ ret = PTR_ERR(xrt_class);
+ goto err_class_create;
+ }
+
+ for (i = 0; i < ARRAY_SIZE(xocl_drv_reg_funcs); ++i) {
+ ret = xocl_drv_reg_funcs[i]();
+ if (ret)
+ goto failed;
+ }
+
+ ret = pci_register_driver(&userpf_driver);
+ if (ret)
+ goto failed;
+
+ return 0;
+
+failed:
+ for (i--; i >= 0; i--)
+ xocl_drv_unreg_funcs[i]();
+
+ class_destroy(xrt_class);
+ xrt_class = NULL;
+
+err_class_create:
+ return ret;
+}
+
+static void __exit xocl_exit(void)
+{
+ int i;
+
+ pci_unregister_driver(&userpf_driver);
+
+ for (i = ARRAY_SIZE(xocl_drv_unreg_funcs) - 1; i >= 0; i--)
+ xocl_drv_unreg_funcs[i]();
+
+ class_destroy(xrt_class);
+ xrt_class = NULL;
+}
+
+module_init(xocl_init);
+module_exit(xocl_exit);
+
+MODULE_VERSION(XRT_DRIVER_VERSION);
+
+MODULE_DESCRIPTION(XOCL_DRIVER_DESC);
+MODULE_AUTHOR("Lizhi Hou <lizhi.hou@...inx.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/xocl/userpf/xocl_ioctl.c b/drivers/gpu/drm/xocl/userpf/xocl_ioctl.c
new file mode 100644
index 000000000000..665ecb0e27ac
--- /dev/null
+++ b/drivers/gpu/drm/xocl/userpf/xocl_ioctl.c
@@ -0,0 +1,396 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * A GEM style device manager for PCIe based OpenCL accelerators.
+ *
+ * Copyright (C) 2016-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Authors: Sonal Santan
+ *
+ */
+
+#include <linux/version.h>
+#include <drm/drmP.h>
+#include <drm/drm_gem.h>
+#include <drm/drm_mm.h>
+#include <linux/eventfd.h>
+#include <linux/uuid.h>
+#include <linux/hashtable.h>
+#include "../version.h"
+#include "common.h"
+
+int xocl_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
+{
+ struct drm_xocl_info *obj = data;
+ struct xocl_drm *drm_p = dev->dev_private;
+ struct xocl_dev *xdev = drm_p->xdev;
+ struct pci_dev *pdev = xdev->core.pdev;
+ u32 major, minor, patch;
+
+ userpf_info(xdev, "INFO IOCTL");
+
+ if (sscanf(XRT_DRIVER_VERSION, "%d.%d.%d", &major, &minor, &patch) != 3)
+ return -ENODEV;
+
+ obj->vendor = pdev->vendor;
+ obj->device = pdev->device;
+ obj->subsystem_vendor = pdev->subsystem_vendor;
+ obj->subsystem_device = pdev->subsystem_device;
+ obj->driver_version = XOCL_DRV_VER_NUM(major, minor, patch);
+ obj->pci_slot = PCI_SLOT(pdev->devfn);
+
+ return 0;
+}
+
+int xocl_execbuf_ioctl(struct drm_device *dev,
+ void *data, struct drm_file *filp)
+{
+ struct xocl_drm *drm_p = dev->dev_private;
+ int ret = 0;
+
+ ret = xocl_exec_client_ioctl(drm_p->xdev,
+ DRM_XOCL_EXECBUF, data, filp);
+
+ return ret;
+}
+
+/*
+ * Create a context (only shared supported today) on a CU. Take a lock on xclbin if
+ * it has not been acquired before. Shared the same lock for all context requests
+ * for that process
+ */
+int xocl_ctx_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp)
+{
+ struct xocl_drm *drm_p = dev->dev_private;
+ int ret = 0;
+
+ ret = xocl_exec_client_ioctl(drm_p->xdev,
+ DRM_XOCL_CTX, data, filp);
+
+ return ret;
+}
+
+int xocl_user_intr_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp)
+{
+ struct xocl_drm *drm_p = dev->dev_private;
+ struct xocl_dev *xdev = drm_p->xdev;
+ struct drm_xocl_user_intr *args = data;
+ int ret = 0;
+
+ xocl_info(dev->dev, "USER INTR ioctl");
+
+ if (args->fd < 0)
+ return -EINVAL;
+
+ xocl_dma_intr_register(xdev, args->msix, NULL, NULL, args->fd);
+ xocl_dma_intr_config(xdev, args->msix, true);
+
+ return ret;
+}
+
+char *kind_to_string(enum axlf_section_kind kind)
+{
+ switch (kind) {
+ case 0: return "BITSTREAM";
+ case 1: return "CLEARING_BITSTREAM";
+ case 2: return "EMBEDDED_METADATA";
+ case 3: return "FIRMWARE";
+ case 4: return "DEBUG_DATA";
+ case 5: return "SCHED_FIRMWARE";
+ case 6: return "MEM_TOPOLOGY";
+ case 7: return "CONNECTIVITY";
+ case 8: return "IP_LAYOUT";
+ case 9: return "DEBUG_IP_LAYOUT";
+ case 10: return "DESIGN_CHECK_POINT";
+ case 11: return "CLOCK_FREQ_TOPOLOGY";
+ default: return "UNKNOWN";
+ }
+}
+
+/* should be obsoleted after mailbox implememted */
+static const struct axlf_section_header *
+get_axlf_section(const struct axlf *top, enum axlf_section_kind kind)
+{
+ int i = 0;
+
+ DRM_INFO("Finding %s section header", kind_to_string(kind));
+ for (i = 0; i < top->m_header.m_numSections; i++) {
+ if (top->m_sections[i].m_sectionKind == kind)
+ return &top->m_sections[i];
+ }
+ DRM_INFO("Did not find AXLF section %s", kind_to_string(kind));
+ return NULL;
+}
+
+static int
+xocl_check_section(const struct axlf_section_header *header, uint64_t len,
+ enum axlf_section_kind kind)
+{
+ uint64_t offset;
+ uint64_t size;
+
+ DRM_INFO("Section %s details:", kind_to_string(kind));
+ DRM_INFO(" offset = 0x%llx", header->m_sectionOffset);
+ DRM_INFO(" size = 0x%llx", header->m_sectionSize);
+
+ offset = header->m_sectionOffset;
+ size = header->m_sectionSize;
+ if (offset + size <= len)
+ return 0;
+
+ DRM_INFO("Section %s extends beyond xclbin boundary 0x%llx\n",
+ kind_to_string(kind), len);
+ return -EINVAL;
+}
+
+/* Return value: Negative for error, or the size in bytes has been copied */
+static int
+xocl_read_sect(enum axlf_section_kind kind, void **sect, struct axlf *axlf_full)
+{
+ const struct axlf_section_header *memHeader;
+ uint64_t xclbin_len;
+ uint64_t offset;
+ uint64_t size;
+ int err = 0;
+
+ memHeader = get_axlf_section(axlf_full, kind);
+ if (!memHeader)
+ return 0;
+
+ xclbin_len = axlf_full->m_header.m_length;
+ err = xocl_check_section(memHeader, xclbin_len, kind);
+ if (err)
+ return err;
+
+ offset = memHeader->m_sectionOffset;
+ size = memHeader->m_sectionSize;
+ *sect = &((char *)axlf_full)[offset];
+
+ return size;
+}
+
+/*
+ * Should be called with xdev->ctx_list_lock held
+ */
+static uint live_client_size(struct xocl_dev *xdev)
+{
+ const struct list_head *ptr;
+ const struct client_ctx *entry;
+ uint count = 0;
+
+ BUG_ON(!mutex_is_locked(&xdev->ctx_list_lock));
+
+ list_for_each(ptr, &xdev->ctx_list) {
+ entry = list_entry(ptr, struct client_ctx, link);
+ count++;
+ }
+ return count;
+}
+
+static int
+xocl_read_axlf_helper(struct xocl_drm *drm_p, struct drm_xocl_axlf *axlf_ptr)
+{
+ long err = 0;
+ struct axlf *axlf = 0;
+ struct axlf bin_obj;
+ size_t size;
+ int preserve_mem = 0;
+ struct mem_topology *new_topology = NULL, *topology;
+ struct xocl_dev *xdev = drm_p->xdev;
+ uuid_t *xclbin_id;
+
+ userpf_info(xdev, "READ_AXLF IOCTL\n");
+
+ if (!xocl_is_unified(xdev)) {
+ userpf_info(xdev, "XOCL: not unified dsa");
+ return err;
+ }
+
+ if (copy_from_user(&bin_obj, axlf_ptr->xclbin, sizeof(struct axlf)))
+ return -EFAULT;
+
+ if (memcmp(bin_obj.m_magic, "xclbin2", 8))
+ return -EINVAL;
+
+ if (xocl_xrt_version_check(xdev, &bin_obj, true))
+ return -EINVAL;
+
+ if (uuid_is_null(&bin_obj.m_header.uuid)) {
+ // Legacy xclbin, convert legacy id to new id
+ memcpy(&bin_obj.m_header.uuid, &bin_obj.m_header.m_timeStamp, 8);
+ }
+
+ xclbin_id = (uuid_t *)xocl_icap_get_data(xdev, XCLBIN_UUID);
+ if (!xclbin_id)
+ return -EINVAL;
+ /*
+ * Support for multiple processes
+ * 1. We lock &xdev->ctx_list_lock so no new contexts can be opened and no live contexts
+ * can be closed
+ * 2. If more than one context exists -- more than one clients are connected -- we cannot
+ * swap the xclbin return -EPERM
+ * 3. If no live contexts exist there may still be sumbitted exec BOs from a
+ * previous context (which was subsequently closed), hence we check for exec BO count.
+ * If exec BO are outstanding we return -EBUSY
+ */
+ if (!uuid_equal(xclbin_id, &bin_obj.m_header.uuid)) {
+ if (atomic_read(&xdev->outstanding_execs)) {
+ userpf_err(xdev, "Current xclbin is busy, can't change\n");
+ return -EBUSY;
+ }
+ }
+
+ //Ignore timestamp matching for AWS platform
+ if (!xocl_is_aws(xdev) && !xocl_verify_timestamp(xdev,
+ bin_obj.m_header.m_featureRomTimeStamp)) {
+ userpf_err(xdev, "TimeStamp of ROM did not match Xclbin\n");
+ return -EINVAL;
+ }
+
+ userpf_info(xdev, "XOCL: VBNV and TimeStamps matched\n");
+
+ if (uuid_equal(xclbin_id, &bin_obj.m_header.uuid)) {
+ userpf_info(xdev, "Skipping repopulating topology, connectivity,ip_layout data\n");
+ goto done;
+ }
+
+ //Copy from user space and proceed.
+ axlf = vmalloc(bin_obj.m_header.m_length);
+ if (!axlf) {
+ userpf_err(xdev, "Unable to create axlf\n");
+ err = -ENOMEM;
+ goto done;
+ }
+
+ if (copy_from_user(axlf, axlf_ptr->xclbin, bin_obj.m_header.m_length)) {
+ err = -EFAULT;
+ goto done;
+ }
+
+ /* Populating MEM_TOPOLOGY sections. */
+ size = xocl_read_sect(MEM_TOPOLOGY, (void **)&new_topology, axlf);
+ if (size <= 0) {
+ if (size != 0)
+ goto done;
+ } else if (sizeof_sect(new_topology, m_mem_data) != size) {
+ err = -EINVAL;
+ goto done;
+ }
+
+ topology = XOCL_MEM_TOPOLOGY(xdev);
+
+ /*
+ * Compare MEM_TOPOLOGY previous vs new.
+ * Ignore this and keep disable preserve_mem if not for aws.
+ */
+ if (xocl_is_aws(xdev) && (topology != NULL)) {
+ if ((size == sizeof_sect(topology, m_mem_data)) &&
+ !memcmp(new_topology, topology, size)) {
+ xocl_xdev_info(xdev, "MEM_TOPOLOGY match, preserve mem_topology.");
+ preserve_mem = 1;
+ } else {
+ xocl_xdev_info(xdev, "MEM_TOPOLOGY mismatch, do not preserve mem_topology.");
+ }
+ }
+
+ /* Switching the xclbin, make sure none of the buffers are used. */
+ if (!preserve_mem) {
+ err = xocl_check_topology(drm_p);
+ if (err)
+ goto done;
+ xocl_cleanup_mem(drm_p);
+ }
+
+ err = xocl_icap_download_axlf(xdev, axlf);
+ if (err) {
+ userpf_err(xdev, "%s Fail to download\n", __func__);
+ /*
+ * Don't just bail out here, always recreate drm mem
+ * since we have cleaned it up before download.
+ */
+ }
+
+ if (!preserve_mem) {
+ int rc = xocl_init_mem(drm_p);
+
+ if (err == 0)
+ err = rc;
+ }
+
+done:
+ if (size < 0)
+ err = size;
+ if (err)
+ userpf_err(xdev, "err: %ld\n", err);
+ else
+ userpf_info(xdev, "Loaded xclbin %pUb", xclbin_id);
+ vfree(axlf);
+ return err;
+}
+
+int xocl_read_axlf_ioctl(struct drm_device *dev,
+ void *data,
+ struct drm_file *filp)
+{
+ struct drm_xocl_axlf *axlf_obj_ptr = data;
+ struct xocl_drm *drm_p = dev->dev_private;
+ struct xocl_dev *xdev = drm_p->xdev;
+ struct client_ctx *client = filp->driver_priv;
+ int err = 0;
+ uuid_t *xclbin_id;
+
+ mutex_lock(&xdev->ctx_list_lock);
+ err = xocl_read_axlf_helper(drm_p, axlf_obj_ptr);
+ /*
+ * Record that user land configured this context for current device xclbin
+ * It doesn't mean that the context has a lock on the xclbin, only that
+ * when a lock is eventually acquired it can be verified to be against to
+ * be a lock on expected xclbin
+ */
+ xclbin_id = (uuid_t *)xocl_icap_get_data(xdev, XCLBIN_UUID);
+ uuid_copy(&client->xclbin_id,
+ ((err || !xclbin_id) ? &uuid_null : xclbin_id));
+ mutex_unlock(&xdev->ctx_list_lock);
+ return err;
+}
+
+uint get_live_client_size(struct xocl_dev *xdev)
+{
+ uint count;
+
+ mutex_lock(&xdev->ctx_list_lock);
+ count = live_client_size(xdev);
+ mutex_unlock(&xdev->ctx_list_lock);
+ return count;
+}
+
+void reset_notify_client_ctx(struct xocl_dev *xdev)
+{
+ xdev->needs_reset = false;
+// wmb();
+}
+
+int xocl_hot_reset_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp)
+{
+ struct xocl_drm *drm_p = dev->dev_private;
+ struct xocl_dev *xdev = drm_p->xdev;
+
+ int err = xocl_hot_reset(xdev, false);
+
+ userpf_info(xdev, "%s err: %d\n", __func__, err);
+ return err;
+}
+
+int xocl_reclock_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *filp)
+{
+ struct xocl_drm *drm_p = dev->dev_private;
+ struct xocl_dev *xdev = drm_p->xdev;
+ int err = xocl_reclock(xdev, data);
+
+ userpf_info(xdev, "%s err: %d\n", __func__, err);
+ return err;
+}
diff --git a/drivers/gpu/drm/xocl/userpf/xocl_sysfs.c b/drivers/gpu/drm/xocl/userpf/xocl_sysfs.c
new file mode 100644
index 000000000000..fccb27906897
--- /dev/null
+++ b/drivers/gpu/drm/xocl/userpf/xocl_sysfs.c
@@ -0,0 +1,344 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * A GEM style device manager for PCIe based OpenCL accelerators.
+ *
+ * Copyright (C) 2016-2019 Xilinx, Inc. All rights reserved.
+ *
+ * Authors: Lizhi.Hou@...inx.com
+ *
+ */
+#include "common.h"
+
+//Attributes followed by bin_attributes.
+//
+/* -Attributes -- */
+
+/* -xclbinuuid-- (supersedes xclbinid) */
+static ssize_t xclbinuuid_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_dev *xdev = dev_get_drvdata(dev);
+ uuid_t *xclbin_id;
+
+ xclbin_id = (uuid_t *)xocl_icap_get_data(xdev, XCLBIN_UUID);
+ return sprintf(buf, "%pUb\n", xclbin_id ? xclbin_id : 0);
+}
+
+static DEVICE_ATTR_RO(xclbinuuid);
+
+/* -userbar-- */
+static ssize_t userbar_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_dev *xdev = dev_get_drvdata(dev);
+
+ return sprintf(buf, "%d\n", xdev->core.bar_idx);
+}
+
+static DEVICE_ATTR_RO(userbar);
+
+static ssize_t user_pf_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ // The existence of entry indicates user function.
+ return sprintf(buf, "%s", "");
+}
+static DEVICE_ATTR_RO(user_pf);
+
+/* -live client contects-- */
+static ssize_t kdsstat_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_dev *xdev = dev_get_drvdata(dev);
+ int size;
+ uuid_t *xclbin_id;
+
+ xclbin_id = (uuid_t *)xocl_icap_get_data(xdev, XCLBIN_UUID);
+ size = sprintf(buf,
+ "xclbin:\t\t\t%pUb\noutstanding execs:\t%d\ntotal execs:\t\t%lld\ncontexts:\t\t%d\n",
+ xclbin_id ? xclbin_id : 0,
+ atomic_read(&xdev->outstanding_execs),
+ atomic64_read(&xdev->total_execs),
+ get_live_client_size(xdev));
+ return size;
+}
+static DEVICE_ATTR_RO(kdsstat);
+
+static ssize_t xocl_mm_stat(struct xocl_dev *xdev, char *buf, bool raw)
+{
+ int i;
+ ssize_t count = 0;
+ ssize_t size = 0;
+ size_t memory_usage = 0;
+ unsigned int bo_count = 0;
+ const char *txt_fmt = "[%s] %s@...012llx (%lluMB): %lluKB %dBOs\n";
+ const char *raw_fmt = "%llu %d\n";
+ struct mem_topology *topo = NULL;
+ struct drm_xocl_mm_stat stat;
+ void *drm_hdl;
+
+ drm_hdl = xocl_dma_get_drm_handle(xdev);
+ if (!drm_hdl)
+ return -EINVAL;
+
+ mutex_lock(&xdev->ctx_list_lock);
+
+ topo = XOCL_MEM_TOPOLOGY(xdev);
+ if (!topo) {
+ mutex_unlock(&xdev->ctx_list_lock);
+ return -EINVAL;
+ }
+
+ for (i = 0; i < topo->m_count; i++) {
+ xocl_mm_get_usage_stat(drm_hdl, i, &stat);
+
+ if (raw) {
+ memory_usage = 0;
+ bo_count = 0;
+ memory_usage = stat.memory_usage;
+ bo_count = stat.bo_count;
+
+ count = sprintf(buf, raw_fmt,
+ memory_usage,
+ bo_count);
+ } else {
+ count = sprintf(buf, txt_fmt,
+ topo->m_mem_data[i].m_used ?
+ "IN-USE" : "UNUSED",
+ topo->m_mem_data[i].m_tag,
+ topo->m_mem_data[i].m_base_address,
+ topo->m_mem_data[i].m_size / 1024,
+ stat.memory_usage / 1024,
+ stat.bo_count);
+ }
+ buf += count;
+ size += count;
+ }
+ mutex_unlock(&xdev->ctx_list_lock);
+ return size;
+}
+
+/* -live memory usage-- */
+static ssize_t memstat_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_dev *xdev = dev_get_drvdata(dev);
+
+ return xocl_mm_stat(xdev, buf, false);
+}
+static DEVICE_ATTR_RO(memstat);
+
+static ssize_t memstat_raw_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_dev *xdev = dev_get_drvdata(dev);
+
+ return xocl_mm_stat(xdev, buf, true);
+}
+static DEVICE_ATTR_RO(memstat_raw);
+
+static ssize_t p2p_enable_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_dev *xdev = dev_get_drvdata(dev);
+ u64 size;
+
+ if (xdev->p2p_bar_addr)
+ return sprintf(buf, "1\n");
+ else if (xocl_get_p2p_bar(xdev, &size) >= 0 &&
+ size > (1 << XOCL_PA_SECTION_SHIFT))
+ return sprintf(buf, "2\n");
+
+ return sprintf(buf, "0\n");
+}
+
+static ssize_t p2p_enable_store(struct device *dev,
+ struct device_attribute *da, const char *buf, size_t count)
+{
+ struct xocl_dev *xdev = dev_get_drvdata(dev);
+ struct pci_dev *pdev = xdev->core.pdev;
+ int ret, p2p_bar;
+ u32 enable;
+ u64 size;
+
+
+ if (kstrtou32(buf, 10, &enable) == -EINVAL || enable > 1)
+ return -EINVAL;
+
+ p2p_bar = xocl_get_p2p_bar(xdev, NULL);
+ if (p2p_bar < 0) {
+ xocl_err(&pdev->dev, "p2p bar is not configurable");
+ return -EACCES;
+ }
+
+ size = xocl_get_ddr_channel_size(xdev) *
+ xocl_get_ddr_channel_count(xdev); /* GB */
+ size = (ffs(size) == fls(size)) ? (fls(size) - 1) : fls(size);
+ size = enable ? (size + 10) : (XOCL_PA_SECTION_SHIFT - 20);
+ xocl_info(&pdev->dev, "Resize p2p bar %d to %d M ", p2p_bar,
+ (1 << size));
+ xocl_p2p_mem_release(xdev, false);
+
+ ret = xocl_pci_resize_resource(pdev, p2p_bar, size);
+ if (ret) {
+ xocl_err(&pdev->dev, "Failed to resize p2p BAR %d", ret);
+ goto failed;
+ }
+
+ xdev->p2p_bar_idx = p2p_bar;
+ xdev->p2p_bar_len = pci_resource_len(pdev, p2p_bar);
+
+ if (enable) {
+ ret = xocl_p2p_mem_reserve(xdev);
+ if (ret) {
+ xocl_err(&pdev->dev, "Failed to reserve p2p memory %d",
+ ret);
+ }
+ }
+
+ return count;
+
+failed:
+ return ret;
+
+}
+
+static DEVICE_ATTR(p2p_enable, 0644, p2p_enable_show, p2p_enable_store);
+
+static ssize_t dev_offline_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_dev *xdev = dev_get_drvdata(dev);
+ int val = xdev->core.offline ? 1 : 0;
+
+ return sprintf(buf, "%d\n", val);
+}
+static ssize_t dev_offline_store(struct device *dev,
+ struct device_attribute *da, const char *buf, size_t count)
+{
+ struct xocl_dev *xdev = dev_get_drvdata(dev);
+ int ret;
+ u32 offline;
+
+
+ if (kstrtou32(buf, 10, &offline) == -EINVAL || offline > 1)
+ return -EINVAL;
+
+ device_lock(dev);
+ if (offline) {
+ xocl_subdev_destroy_all(xdev);
+ xdev->core.offline = true;
+ } else {
+ ret = xocl_subdev_create_all(xdev, xdev->core.priv.subdev_info,
+ xdev->core.priv.subdev_num);
+ if (ret) {
+ xocl_err(dev, "Online subdevices failed");
+ return -EIO;
+ }
+ xdev->core.offline = false;
+ }
+ device_unlock(dev);
+
+ return count;
+}
+
+static DEVICE_ATTR(dev_offline, 0644, dev_offline_show, dev_offline_store);
+
+static ssize_t mig_calibration_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ return sprintf(buf, "0\n");
+}
+
+static DEVICE_ATTR_RO(mig_calibration);
+
+static ssize_t link_width_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ unsigned short speed, width;
+ struct xocl_dev *xdev = dev_get_drvdata(dev);
+
+ get_pcie_link_info(xdev, &width, &speed, false);
+ return sprintf(buf, "%d\n", width);
+}
+static DEVICE_ATTR_RO(link_width);
+
+static ssize_t link_speed_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ unsigned short speed, width;
+ struct xocl_dev *xdev = dev_get_drvdata(dev);
+
+ get_pcie_link_info(xdev, &width, &speed, false);
+ return sprintf(buf, "%d\n", speed);
+}
+static DEVICE_ATTR_RO(link_speed);
+
+static ssize_t link_width_max_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ unsigned short speed, width;
+ struct xocl_dev *xdev = dev_get_drvdata(dev);
+
+ get_pcie_link_info(xdev, &width, &speed, true);
+ return sprintf(buf, "%d\n", width);
+}
+static DEVICE_ATTR_RO(link_width_max);
+
+static ssize_t link_speed_max_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ unsigned short speed, width;
+ struct xocl_dev *xdev = dev_get_drvdata(dev);
+
+ get_pcie_link_info(xdev, &width, &speed, true);
+ return sprintf(buf, "%d\n", speed);
+}
+static DEVICE_ATTR_RO(link_speed_max);
+/* - End attributes-- */
+
+static struct attribute *xocl_attrs[] = {
+ &dev_attr_xclbinuuid.attr,
+ &dev_attr_userbar.attr,
+ &dev_attr_kdsstat.attr,
+ &dev_attr_memstat.attr,
+ &dev_attr_memstat_raw.attr,
+ &dev_attr_user_pf.attr,
+ &dev_attr_p2p_enable.attr,
+ &dev_attr_dev_offline.attr,
+ &dev_attr_mig_calibration.attr,
+ &dev_attr_link_width.attr,
+ &dev_attr_link_speed.attr,
+ &dev_attr_link_speed_max.attr,
+ &dev_attr_link_width_max.attr,
+ NULL,
+};
+
+static struct attribute_group xocl_attr_group = {
+ .attrs = xocl_attrs,
+};
+
+//---
+int xocl_init_sysfs(struct device *dev)
+{
+ int ret;
+ struct pci_dev *rdev;
+
+ ret = sysfs_create_group(&dev->kobj, &xocl_attr_group);
+ if (ret)
+ xocl_err(dev, "create xocl attrs failed: %d", ret);
+
+ xocl_get_root_dev(to_pci_dev(dev), rdev);
+ ret = sysfs_create_link(&dev->kobj, &rdev->dev.kobj, "root_dev");
+ if (ret)
+ xocl_err(dev, "create root device link failed: %d", ret);
+
+ return ret;
+}
+
+void xocl_fini_sysfs(struct device *dev)
+{
+ sysfs_remove_link(&dev->kobj, "root_dev");
+ sysfs_remove_group(&dev->kobj, &xocl_attr_group);
+}
--
2.17.0
Powered by blists - more mailing lists