| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Date: Sat, 17 Sep 2022 16:23:51 +0900
From: Jiho Chu <jiho.chu@...sung.com>
To: gregkh@...uxfoundation.org, arnd@...db.de, ogabbay@...nel.org,
krzysztof.kozlowski@...aro.org, broonie@...nel.org
Cc: linux-kernel@...r.kernel.org, yelini.jeong@...sung.com,
myungjoo.ham@...sung.com, jiho.chu@...sung.com
Subject: [PATCH v2 08/13] trinity: Add ioctl feature
This patch implements ioctl operations.
The ioctl routines are added to give controls to the user
library. TRINITY_IOCTL_HWMEM_ALLOC/DEALLOC is for memory
allocation for the model load, RUN/STOP operations are
provided to control NPU works. And several STAT controls can
provide statistics of the NPU.
Signed-off-by: Jiho Chu <jiho.chu@...sung.com>
Signed-off-by: Yelin Jeong <yelini.jeong@...sung.com>
Signed-off-by: Dongju Chae <dongju.chae@...sung.com>
Signed-off-by: MyungJoo Ham <myungjoo.ham@...sung.com>
---
drivers/misc/trinity/trinity.c | 629 +++++++++++++++++++++++++++++++++
1 file changed, 629 insertions(+)
diff --git a/drivers/misc/trinity/trinity.c b/drivers/misc/trinity/trinity.c
index a785a5dca4d9..0fb5ccf9f035 100644
--- a/drivers/misc/trinity/trinity.c
+++ b/drivers/misc/trinity/trinity.c
@@ -22,6 +22,635 @@
static DEFINE_IDA(dev_nrs);
static DEFINE_IDA(model_ids);
+static uint64_t trinity_gen_model_id(int32_t dbuf_fd)
+{
+ uint64_t mid;
+
+ mid = ida_alloc_max(&model_ids, ((1 << TRINITY_SHIFT_MODEL_ID) - 1), GFP_KERNEL);
+ if (mid < 0)
+ return mid;
+
+ mid |= (dbuf_fd << TRINITY_SHIFT_MODEL_ID);
+
+ return mid;
+}
+
+static void trinity_release_model_id(uint64_t mid)
+{
+ ida_free(&model_ids, mid & ((1 << TRINITY_SHIFT_MODEL_ID) - 1));
+}
+
+static int32_t trinity_model_id_to_dbuf_fd(uint64_t id)
+{
+ return (id >> TRINITY_SHIFT_MODEL_ID) & UINT_MAX;
+}
+
+static void trinity_model_htable_init(const struct device *dev)
+{
+ int i;
+ struct trinity_driver *drv;
+
+ drv = dev_get_drvdata(dev);
+ for (i = 0; i < TRINITY_MODEL_HASH_SIZE; ++i)
+ INIT_HLIST_BL_HEAD(drv->model_htable + i);
+}
+
+static struct trinity_model *
+trinity_get_model_by_id(const struct trinity_driver *drv, const uint64_t id)
+{
+ struct hlist_bl_head *model_hlist;
+ struct hlist_bl_node *hn;
+ struct trinity_model *hm;
+ unsigned long key;
+ int32_t dbuf_fd;
+
+ dbuf_fd = trinity_model_id_to_dbuf_fd(id);
+ key = hash_long(dbuf_fd, TRINITY_MODEL_HASH_BITS);
+ model_hlist = (struct hlist_bl_head *)(drv->model_htable + key);
+
+ hlist_bl_lock(model_hlist);
+ hlist_bl_for_each_entry(hm, hn, model_hlist, hnode) {
+ if (hm->config.id != id)
+ continue;
+
+ hlist_bl_unlock(model_hlist);
+ return hm;
+ }
+ hlist_bl_unlock(model_hlist);
+
+ return NULL;
+}
+
+/**
+ * trinity_register_model() - Registers a model to the internal hashtable. Note
+ * that the device is responsible for the hashtable maintenance.
+ *
+ * @drv: An instance of the trinity driver
+ * @model: Model information to be registered
+ *
+ * Returns 0 and sets model->id with a valid value, which is unique system-wide,
+ * on success. Otherwise, returns negative error.
+ */
+int32_t trinity_register_model(struct trinity_driver *drv,
+ struct trinity_model *model)
+{
+ struct hlist_bl_head *model_hlist;
+ unsigned long key;
+ int32_t ret;
+
+ ret = trinity_hwmem_import_dmabuf_begin(drv_to_dev_ptr(drv),
+ model->config.dbuf_fd,
+ &model->import_info);
+ if (ret)
+ return ret;
+
+#ifdef ARM
+ /* sync model program data */
+ __cpuc_flush_dcache_area(model->import_info.addr,
+ model->import_info.buf->size);
+#endif
+
+ model->config.id = trinity_gen_model_id(model->config.dbuf_fd);
+ model->owner_id = trinity_get_app_id();
+
+ INIT_HLIST_BL_NODE(&model->hnode);
+
+ key = hash_long(model->config.dbuf_fd, TRINITY_MODEL_HASH_BITS);
+ model_hlist = (struct hlist_bl_head *)(drv->model_htable + key);
+
+ hlist_bl_lock(model_hlist);
+ hlist_bl_add_head(&model->hnode, model_hlist);
+ hlist_bl_unlock(model_hlist);
+
+ kref_init(&model->refcnt);
+
+ return 0;
+}
+
+static void trinity_destroy_model(struct kref *refcnt)
+{
+ struct trinity_model *model =
+ container_of(refcnt, struct trinity_model, refcnt);
+
+ trinity_release_model_id(model->config.id);
+ trinity_hwmem_import_dmabuf_end(&model->import_info);
+ kfree(model);
+}
+
+static void trinity_model_get(struct trinity_model *model)
+{
+ if (!model)
+ return;
+
+ kref_get(&model->refcnt);
+}
+
+static void trinity_model_put(struct trinity_model *model)
+{
+ if (!model)
+ return;
+
+ kref_put(&model->refcnt, trinity_destroy_model);
+}
+
+phys_addr_t trinity_get_paddr(struct iommu_domain *domain, dma_addr_t daddr)
+{
+ if (domain)
+ return iommu_iova_to_phys(domain, daddr);
+
+ return TRINITY_PADDR_BASE + daddr;
+}
+
+void trinity_finish_req(struct trinity_driver *drv, struct trinity_req *req)
+{
+ if (drv->desc->check_profile(drv, req) < 0)
+ dev_warn(drv_to_dev_ptr(drv),
+ "Unable to get profile data from NPU\n");
+ trinity_hwmem_import_dmabuf_end(&req->input.import_info);
+ trinity_stat_finish_req(drv, req);
+ trinity_model_put(req->model);
+}
+
+/**
+ * trinity_deregister_model() - Deregisters the model with a given id from the
+ * table
+ *
+ * @drv: An instance of the trinity driver
+ * @id: An id of the model to be deregistered
+ *
+ * Returns 0 on success. Otherwise, returns negative error.
+ */
+int32_t trinity_deregister_model(const struct trinity_driver *drv,
+ const uint64_t id)
+{
+ struct hlist_bl_head *model_hlist;
+ int32_t dbuf_fd;
+ unsigned long key;
+ struct hlist_bl_node *hn;
+ struct trinity_model *hm = NULL;
+
+ dbuf_fd = trinity_model_id_to_dbuf_fd(id);
+ key = hash_long(dbuf_fd, TRINITY_MODEL_HASH_BITS);
+ model_hlist = (struct hlist_bl_head *)(drv->model_htable + key);
+
+ hlist_bl_lock(model_hlist);
+ hlist_bl_for_each_entry(hm, hn, model_hlist, hnode) {
+ if (hm->config.id == id) {
+ hlist_bl_del_init(&hm->hnode);
+ break;
+ }
+ }
+ hlist_bl_unlock(model_hlist);
+
+ if (!hm)
+ return -ENOENT;
+
+ trinity_model_put(hm);
+
+ return 0;
+}
+
+/**
+ * trinity_deregister_models_owned() - Deregisters models owned
+ *
+ * @drv: An instance of the trinity driver
+ */
+void trinity_deregister_models_owned(struct trinity_driver *drv)
+{
+ struct hlist_bl_head *model_htable;
+ struct trinity_model *hm;
+ struct hlist_bl_node *hn;
+ int i, app_id;
+
+ app_id = trinity_get_app_id();
+ model_htable = drv->model_htable;
+retry:
+ for (i = 0; i < TRINITY_MODEL_HASH_SIZE; i++) {
+ hlist_bl_lock(model_htable + i);
+ hlist_bl_for_each_entry(hm, hn, model_htable + i, hnode) {
+ if (hm->owner_id == app_id) {
+ hlist_bl_del_init(&hm->hnode);
+ hlist_bl_unlock(model_htable + i);
+
+ trinity_model_put(hm);
+
+ goto retry;
+ }
+ }
+ hlist_bl_unlock(model_htable + i);
+ }
+}
+
+static int32_t trinity_submit_req(struct trinity_driver *drv,
+ struct trinity_req *req)
+{
+ struct device *dev;
+ wait_queue_head_t wq;
+ unsigned long timeout, timeout_ms;
+ unsigned long retry;
+ int ret = 0;
+
+ dev = drv_to_dev_ptr(drv);
+
+ /* optional req setup before submission */
+ if (drv->desc->prepare_req) {
+ ret = drv->desc->prepare_req(drv, req);
+ if (ret < 0) {
+ dev_err(dev, "Unable to prepare req submission: %d",
+ ret);
+ return ret;
+ }
+ }
+
+ req->submit_retry = 0;
+ timeout_ms = req->input.config.timeout_ms;
+ /* use the default timeout if a user didn't set */
+ if (timeout_ms == 0)
+ timeout_ms = TRINITY_RUN_TIMEOUT_MSEC;
+
+ retry = 0;
+ init_waitqueue_head(&wq);
+ init_completion(&req->complete);
+
+ timeout = msecs_to_jiffies(timeout_ms);
+ while (wait_event_interruptible_timeout(wq, trinity_sched_ready(drv),
+ timeout / 10) == 0) {
+ if (retry == 10) {
+ ret = -ETIMEDOUT;
+ break;
+ }
+ retry++;
+ }
+
+ if (ret == 0) {
+ ret = trinity_stat_append_req(drv, req);
+ if (ret < 0) {
+ dev_err(dev, "Unable to append request stat: %d", ret);
+ return ret;
+ }
+
+ ret = trinity_sched_submit(drv, req);
+ if (ret < 0)
+ trinity_stat_remove_req(drv, req, true);
+ }
+
+ if (ret < 0) {
+ dev_err(dev, "Unable to submit req to scheduler: %d", ret);
+ return ret;
+ }
+
+ if (req->input.config.output_mode != TRINITY_OUTPUT_HW) {
+ timeout = wait_for_completion_timeout(&req->complete, timeout);
+ /* Check and handle the timeout if its handler exists */
+ if (timeout == 0) {
+ drv->desc->handle_timeout(drv, req);
+
+ req->stat->status = TRINITY_REQ_STATUS_ERROR;
+ ret = -ECANCELED;
+ } else if (req->stat->status == TRINITY_REQ_STATUS_ERROR) {
+ ret = -ECANCELED;
+ }
+
+ trinity_finish_req(drv, req);
+ }
+
+ return ret;
+}
+
+static int32_t trinity_run_input(struct trinity_driver *drv,
+ struct trinity_input *input,
+ struct trinity_req *req)
+{
+ struct trinity_model *model;
+ int32_t err;
+
+ model = trinity_get_model_by_id(drv, input->config.model_id);
+ if (!model) {
+ dev_err(drv_to_dev_ptr(drv), "Unable to find the model");
+ return -EINVAL;
+ }
+
+ /* skip to submit this req */
+ if (model->config.program_size == 0 &&
+ input->config.output_mode != TRINITY_OUTPUT_HW)
+ return 0;
+
+ trinity_model_get(model);
+
+ err = trinity_hwmem_import_dmabuf_begin(drv_to_dev_ptr(drv),
+ input->config.dbuf_fd,
+ &input->import_info);
+ if (err < 0)
+ return err;
+
+ req->model = model;
+ err = trinity_submit_req(drv, req);
+ if (err == 0)
+ return 0;
+
+ if (err != -ECANCELED)
+ trinity_hwmem_import_dmabuf_end(&input->import_info);
+ return err;
+}
+
+/**
+ * trinity_ioctl() - A common callback for unlocked_ioctl() in file_operations for
+ * a Trinity device node.
+ *
+ * @f: A file instance of the opened device node
+ * @cmd: The target IOCTL command to be handled
+ * @arg: A user argument
+ *
+ * Returns 0 on success. Otherwise, returns negative error.
+ */
+long trinity_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
+{
+ struct trinity_driver *drv = f->private_data;
+ const struct trinity_desc *desc = drv->desc;
+ ssize_t err = 0L;
+
+ switch (cmd) {
+ case TRINITY_IOCTL_GET_VERSION: {
+ if (copy_to_user((uint32_t __user *)arg, &(desc->ver),
+ sizeof((desc->ver))))
+ return -EFAULT;
+
+ break;
+ }
+ case TRINITY_IOCTL_GET_API_LEVEL: {
+ uint32_t api_level = TRINITY_API_LEVEL;
+
+ if (copy_to_user((uint32_t __user *)arg, &api_level,
+ sizeof(api_level)))
+ return -EFAULT;
+
+ break;
+ }
+ case TRINITY_IOCTL_GET_STATE: {
+ enum trinity_state ready;
+
+ ready = drv->desc->get_state(drv);
+ if (copy_to_user((enum trinity_state __user *)arg, &ready,
+ sizeof(ready)))
+ return -EFAULT;
+
+ break;
+ }
+ case TRINITY_IOCTL_GET_TOPS: {
+ if (copy_to_user((uint32_t __user *)arg, &(drv->tops),
+ sizeof((drv->tops))))
+ return -EFAULT;
+
+ break;
+ }
+ case TRINITY_IOCTL_GET_DSPM: {
+ if (copy_to_user((uint32_t __user *)arg, &(drv->dspm),
+ sizeof((drv->dspm))))
+ return -EFAULT;
+
+ break;
+ }
+ case TRINITY_IOCTL_GET_NEXT_REQUEST: {
+ int32_t req_id = atomic_inc_return(&drv->global_req_id);
+
+ if (copy_to_user((int32_t __user *)arg, &req_id,
+ sizeof(req_id)))
+ return -EFAULT;
+
+ break;
+ }
+ case TRINITY_IOCTL_HWMEM_ALLOC: {
+ struct trinity_ioctl_hwmem hwmem;
+
+ if (copy_from_user(&hwmem, (size_t __user *)arg, sizeof(hwmem)))
+ return -EFAULT;
+
+ err = trinity_hwmem_alloc(drv_to_dev_ptr(drv), hwmem.size,
+ hwmem.type);
+ if (err >= 0)
+ trinity_stat_app_total_alloc(drv, hwmem.size);
+
+ break;
+ }
+ case TRINITY_IOCTL_HWMEM_DEALLOC: {
+ struct trinity_ioctl_hwmem hwmem;
+ struct dma_buf *dbuf;
+
+ if (copy_from_user(&hwmem, (size_t __user *)arg, sizeof(hwmem)))
+ return -EFAULT;
+
+ dbuf = dma_buf_get(hwmem.dbuf_fd);
+ if (IS_ERR(dbuf))
+ return PTR_ERR(dbuf);
+
+ err = trinity_hwmem_free(drv_to_dev_ptr(drv), hwmem.dbuf_fd);
+ if (err == 0)
+ trinity_stat_app_total_freed(drv, dbuf->size);
+
+ break;
+ }
+ case TRINITY_IOCTL_REGISTER_MODEL: {
+ struct trinity_model *model =
+ kzalloc(sizeof(struct trinity_model), GFP_KERNEL);
+
+ if (IS_ERR_OR_NULL(model))
+ return -ENOMEM;
+
+ if (copy_from_user(&model->config,
+ (struct trinity_model __user *)arg,
+ sizeof(model->config))) {
+ kfree(model);
+ return -EFAULT;
+ }
+
+ err = trinity_register_model(drv, model);
+ if (err < 0)
+ break;
+
+ if (copy_to_user((struct trinity_model __user *)arg,
+ &model->config, sizeof(model->config)))
+ return -EFAULT;
+
+ break;
+ }
+ case TRINITY_IOCTL_DEREGISTER_MODEL: {
+ uint64_t id;
+
+ if (copy_from_user(&id, (uint64_t __user *)arg, sizeof(id)))
+ return -EFAULT;
+
+ err = trinity_deregister_model(drv, id);
+
+ break;
+ }
+ case TRINITY_IOCTL_RUN_INPUT: {
+ struct trinity_req *req;
+ struct trinity_input *input;
+
+ if (!IDU_LOADED(drv))
+ return -EFAULT;
+
+ req = drv->desc->alloc_req(drv);
+ if (!req)
+ return -ENOMEM;
+ req->drv = drv;
+ req->time_started = ktime_get();
+
+ input = &(req->input);
+ /** run input based on config received from the user */
+ if (copy_from_user(&input->config,
+ (struct trinity_input __user *)arg,
+ sizeof(input->config))) {
+ drv->desc->dealloc_req(drv, req);
+ return -EACCES;
+ }
+
+ err = trinity_run_input(drv, input, req);
+ if (err < 0) {
+ drv->desc->dealloc_req(drv, req);
+ return err;
+ }
+
+ if (copy_to_user((struct trinity_input __user *)arg,
+ &input->config, sizeof(input->config))) {
+ drv->desc->dealloc_req(drv, req);
+ return -EACCES;
+ }
+
+ /* this will be freed when stop request is called */
+ if (!req->is_kernel)
+ drv->desc->dealloc_req(drv, req);
+
+ break;
+ }
+ case TRINITY_IOCTL_STOP_REQUESTS: {
+ if (!IDU_LOADED(drv))
+ return -EFAULT;
+
+ if (drv->desc->stop_reqs)
+ schedule_work(&drv->work_stop);
+
+ break;
+ }
+ case TRINITY_IOCTL_STAT_CURRENT_APP: {
+ struct trinity_ioctl_stat_app ioctl_stat_app;
+
+ if (copy_from_user(&ioctl_stat_app,
+ (struct trinity_ioctl_stat_app __user *)arg,
+ sizeof(ioctl_stat_app)))
+ return -EACCES;
+
+ trinity_stat_app_copy_ioctl(drv, &ioctl_stat_app);
+
+ if (copy_to_user((struct trinity_ioctl_stat_app __user *)arg,
+ &ioctl_stat_app, sizeof(ioctl_stat_app)))
+ return -EACCES;
+
+ break;
+ }
+ case TRINITY_IOCTL_STAT_APPS: {
+ struct trinity_ioctl_stat_apps ioctl_stat_apps;
+
+ if (copy_from_user(&ioctl_stat_apps,
+ (struct trinity_ioctl_stat_apps __user *)arg,
+ sizeof(ioctl_stat_apps)))
+ return -EACCES;
+
+ trinity_stat_apps_copy_ioctl(drv, &ioctl_stat_apps);
+
+ if (copy_to_user((struct trinity_ioctl_stat_apps __user *)arg,
+ &ioctl_stat_apps, sizeof(ioctl_stat_apps)))
+ return -EACCES;
+
+ break;
+ }
+ case TRINITY_IOCTL_STAT_REQS: {
+ struct trinity_ioctl_stat_reqs ioctl_stat_reqs;
+
+ if (copy_from_user(&ioctl_stat_reqs,
+ (struct trinity_ioctl_stat_reqs __user *)arg,
+ sizeof(ioctl_stat_reqs)))
+ return -EACCES;
+
+ if (ioctl_stat_reqs.app_id == 0)
+ ioctl_stat_reqs.app_id = trinity_get_app_id();
+
+ trinity_stat_reqs_copy_ioctl(drv, &ioctl_stat_reqs);
+
+ if (copy_to_user((struct trinity_ioctl_stat_reqs __user *)arg,
+ &ioctl_stat_reqs, sizeof(ioctl_stat_reqs)))
+ return -EACCES;
+
+ break;
+ }
+ case TRINITY_IOCTL_GET_PROFILE_META: {
+ struct trinity_ioctl_profile_meta profile;
+
+ if (copy_from_user(
+ &profile,
+ (struct trinity_ioctl_profile_meta __user *)arg,
+ sizeof(profile)))
+ return -EACCES;
+
+ if (drv->desc->get_profile_meta) {
+ err = drv->desc->get_profile_meta(drv, &profile);
+ } else {
+ profile.total_cycles = -1;
+ profile.total_ops = 0;
+ profile.profile_size = 0;
+ profile.input_footprint = -1;
+ profile.output_footprint = -1;
+ }
+
+ if (copy_to_user((struct trinity_ioctl_profile_meta __user *)arg,
+ &profile, sizeof(profile)))
+ return -EACCES;
+
+ break;
+ }
+ case TRINITY_IOCTL_GET_PROFILE_BUFF: {
+ struct trinity_ioctl_profile_buff profile;
+
+ if (copy_from_user(
+ &profile,
+ (struct trinity_ioctl_profile_buff __user *)arg,
+ sizeof(profile)))
+ return -EACCES;
+
+ if (drv->desc->get_profile_buff)
+ err = drv->desc->get_profile_buff(drv, &profile);
+
+ if (copy_to_user((struct trinity_ioctl_profile_buff __user *)arg,
+ &profile, sizeof(profile)))
+ return -EACCES;
+
+ break;
+ }
+ case TRINITY_IOCTL_IDU_SET: {
+ struct trinity_ioctl_idu idu;
+
+ if ((struct trinity_ioctl_idu __user *)arg == NULL) {
+ drv->desc->idu_unset(drv);
+ break;
+ }
+
+ if (copy_from_user(&idu,
+ (struct trinity_ioctl_idu __user *)arg,
+ sizeof(idu))) {
+ return -EACCES;
+ }
+
+ err = drv->desc->idu_set(drv, &idu);
+
+ break;
+ }
+ default:
+ return -ENOTTY;
+ }
+
+ return err;
+}
+
/**
* trinity_release() - A common callback for close() in file_operations for a
* Trinity device node. If there are device-specific data to be
--
2.25.1
Powered by blists - more mailing lists