diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c index 1080f9acf70a3..e60a776b80778 100644 --- a/drivers/misc/fastrpc.c +++ b/drivers/misc/fastrpc.c @@ -23,6 +23,8 @@ #include #include #include +#include +#include #define ADSP_DOMAIN_ID (0) #define MDSP_DOMAIN_ID (1) @@ -37,7 +39,8 @@ #define FASTRPC_CTX_MAX (256) #define FASTRPC_INIT_HANDLE 1 #define FASTRPC_DSP_UTILITIES_HANDLE 2 -#define FASTRPC_CTXID_MASK (0xFF0) +#define FASTRPC_MAX_STATIC_HANDLE (20) +#define FASTRPC_CTXID_MASK GENMASK(15, 8) #define INIT_FILELEN_MAX (2 * 1024 * 1024) #define INIT_FILE_NAMELEN_MAX (128) #define FASTRPC_DEVICE_NAME "fastrpc" @@ -105,6 +108,12 @@ #define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev) +/* Poll response number from remote processor for call completion */ +#define FASTRPC_POLL_RESPONSE (0xdecaf) + +/* Polling mode timeout limit */ +#define FASTRPC_POLL_MAX_TIMEOUT_US (10000) + struct fastrpc_phy_page { dma_addr_t addr; /* dma address */ u64 size; /* size of contiguous region */ @@ -233,9 +242,16 @@ struct fastrpc_invoke_ctx { int pid; int client_id; u32 sc; + u64 *fdlist; u32 *crc; + /* Poll memory that DSP updates */ + u32 *poll; u64 ctxid; u64 msg_sz; + /* work done status flag */ + bool is_work_done; + /* process updates poll memory instead of glink response */ + bool is_polled; struct kref refcount; struct list_head node; /* list of ctxs */ struct completion work; @@ -255,6 +271,8 @@ struct fastrpc_session_ctx { int sid; bool used; bool valid; + bool allocated; + struct mutex mutex; }; struct fastrpc_soc_data { @@ -276,11 +294,12 @@ struct fastrpc_channel_ctx { struct kref refcount; /* Flag if dsp attributes are cached */ bool valid_attributes; + /* Flag if audio PD init mem was allocated */ + bool audio_init_mem; u32 dsp_attributes[FASTRPC_MAX_DSP_ATTRIBUTES]; struct fastrpc_device *secure_fdevice; struct fastrpc_device *fdevice; struct fastrpc_buf *remote_heap; - struct list_head invoke_interrupted_mmaps; bool secure; bool unsigned_support; u64 dma_mask; @@ -297,6 +316,7 @@ struct fastrpc_user { struct list_head user; struct list_head maps; struct list_head pending; + struct list_head interrupted; struct list_head mmaps; struct fastrpc_channel_ctx *cctx; @@ -306,10 +326,14 @@ struct fastrpc_user { int client_id; int pd; bool is_secure_dev; + /* Flags poll mode state */ + bool poll_mode; /* Lock for lists */ spinlock_t lock; /* lock for allocations */ struct mutex mutex; + /* Reference count */ + struct kref refcount; }; /* Extract SMMU PA from consolidated IOVA */ @@ -333,9 +357,14 @@ static inline u64 fastrpc_sid_offset(struct fastrpc_channel_ctx *cctx, static void fastrpc_free_map(struct kref *ref) { struct fastrpc_map *map; + struct fastrpc_user *fl; map = container_of(ref, struct fastrpc_map, refcount); + fl = map->fl; + if (!fl) + return; + if (map->table) { if (map->attr & FASTRPC_ATTR_SECUREMAP) { struct qcom_scm_vmperm perm; @@ -354,10 +383,16 @@ static void fastrpc_free_map(struct kref *ref) return; } } + mutex_lock(&fl->sctx->mutex); + if (!fl->sctx->dev) { + mutex_unlock(&fl->sctx->mutex); + return; + } dma_buf_unmap_attachment_unlocked(map->attach, map->table, DMA_BIDIRECTIONAL); dma_buf_detach(map->buf, map->attach); dma_buf_put(map->buf); + mutex_unlock(&fl->sctx->mutex); } if (map->fl) { @@ -414,9 +449,21 @@ static int fastrpc_map_lookup(struct fastrpc_user *fl, int fd, static void fastrpc_buf_free(struct fastrpc_buf *buf) { - dma_free_coherent(buf->dev, buf->size, buf->virt, - fastrpc_ipa_to_dma_addr(buf->fl->cctx, buf->dma_addr)); - kfree(buf); + if (!buf) + return; + + struct fastrpc_user *fl = buf->fl; + + if (!fl) + return; + mutex_lock(&fl->sctx->mutex); + if (fl->sctx->dev) { + dma_free_coherent(buf->dev, buf->size, buf->virt, + fastrpc_ipa_to_dma_addr(buf->fl->cctx, + buf->dma_addr)); + kfree(buf); + } + mutex_unlock(&fl->sctx->mutex); } static int __fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev, @@ -439,8 +486,11 @@ static int __fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev, buf->dev = dev; buf->raddr = 0; - buf->virt = dma_alloc_coherent(dev, buf->size, &buf->dma_addr, - GFP_KERNEL); + mutex_lock(&fl->sctx->mutex); + if (fl->sctx->dev) + buf->virt = dma_alloc_coherent(dev, buf->size, &buf->dma_addr, + GFP_KERNEL); + mutex_unlock(&fl->sctx->mutex); if (!buf->virt) { mutex_destroy(&buf->lock); kfree(buf); @@ -483,6 +533,10 @@ static void fastrpc_channel_ctx_free(struct kref *ref) struct fastrpc_channel_ctx *cctx; cctx = container_of(ref, struct fastrpc_channel_ctx, refcount); + for (int i = 0; i < FASTRPC_MAX_SESSIONS; i++) { + if (cctx->session[i].allocated) + mutex_destroy(&cctx->session[i].mutex); + } kfree(cctx); } @@ -497,30 +551,52 @@ static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx *cctx) kref_put(&cctx->refcount, fastrpc_channel_ctx_free); } +static void fastrpc_user_free(struct kref *ref) +{ + struct fastrpc_user *fl = container_of(ref, struct fastrpc_user, refcount); + + fastrpc_channel_ctx_put(fl->cctx); + mutex_destroy(&fl->mutex); + kfree(fl); +} + +static void fastrpc_user_get(struct fastrpc_user *fl) +{ + kref_get(&fl->refcount); +} + +static void fastrpc_user_put(struct fastrpc_user *fl) +{ + kref_put(&fl->refcount, fastrpc_user_free); +} + static void fastrpc_context_free(struct kref *ref) { struct fastrpc_invoke_ctx *ctx; struct fastrpc_channel_ctx *cctx; + struct fastrpc_user *fl; unsigned long flags; int i; ctx = container_of(ref, struct fastrpc_invoke_ctx, refcount); cctx = ctx->cctx; + fl = ctx->fl; for (i = 0; i < ctx->nbufs; i++) fastrpc_map_put(ctx->maps[i]); - if (ctx->buf) - fastrpc_buf_free(ctx->buf); + fastrpc_buf_free(ctx->buf); spin_lock_irqsave(&cctx->lock, flags); - idr_remove(&cctx->ctx_idr, ctx->ctxid >> 4); + idr_remove(&cctx->ctx_idr, FIELD_GET(FASTRPC_CTXID_MASK, ctx->ctxid)); spin_unlock_irqrestore(&cctx->lock, flags); kfree(ctx->maps); kfree(ctx->olaps); kfree(ctx); + /* Release the reference taken in fastrpc_context_alloc() */ + fastrpc_user_put(fl); fastrpc_channel_ctx_put(cctx); } @@ -542,6 +618,36 @@ static void fastrpc_context_put_wq(struct work_struct *work) fastrpc_context_put(ctx); } +static void fastrpc_context_save_interrupted(struct fastrpc_invoke_ctx *ctx) +{ + spin_lock(&ctx->fl->lock); + list_del(&ctx->node); + list_add_tail(&ctx->node, &ctx->fl->interrupted); + spin_unlock(&ctx->fl->lock); +} + +static struct fastrpc_invoke_ctx *fastrpc_context_restore_interrupted( + struct fastrpc_user *fl, u32 sc) +{ + struct fastrpc_invoke_ctx *ctx = NULL, *ictx, *n; + + spin_lock(&fl->lock); + list_for_each_entry_safe(ictx, n, &fl->interrupted, node) { + if (ictx->pid != current->pid) + continue; + if (ictx->sc != sc || ictx->fl != fl) { + spin_unlock(&fl->lock); + return ERR_PTR(-EINVAL); + } + ctx = ictx; + list_del(&ctx->node); + list_add_tail(&ctx->node, &fl->pending); + break; + } + spin_unlock(&fl->lock); + return ctx; +} + #define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1) static int olaps_cmp(const void *a, const void *b) { @@ -628,6 +734,8 @@ static struct fastrpc_invoke_ctx *fastrpc_context_alloc( /* Released in fastrpc_context_put() */ fastrpc_channel_ctx_get(cctx); + /* Take a reference to user, released in fastrpc_context_free() */ + fastrpc_user_get(user); ctx->sc = sc; ctx->retval = -1; @@ -648,7 +756,7 @@ static struct fastrpc_invoke_ctx *fastrpc_context_alloc( spin_unlock_irqrestore(&cctx->lock, flags); goto err_idr; } - ctx->ctxid = ret << 4; + ctx->ctxid = FIELD_PREP(FASTRPC_CTXID_MASK, ret); spin_unlock_irqrestore(&cctx->lock, flags); kref_init(&ctx->refcount); @@ -658,6 +766,7 @@ static struct fastrpc_invoke_ctx *fastrpc_context_alloc( spin_lock(&user->lock); list_del(&ctx->node); spin_unlock(&user->lock); + fastrpc_user_put(user); fastrpc_channel_ctx_put(cctx); kfree(ctx->maps); kfree(ctx->olaps); @@ -800,19 +909,29 @@ static int fastrpc_map_attach(struct fastrpc_user *fl, int fd, goto get_err; } + mutex_lock(&fl->sctx->mutex); + if (!fl->sctx->dev) { + err = -ENODEV; + mutex_unlock(&fl->sctx->mutex); + goto attach_err; + } + map->attach = dma_buf_attach(map->buf, sess->dev); if (IS_ERR(map->attach)) { dev_err(sess->dev, "Failed to attach dmabuf\n"); err = PTR_ERR(map->attach); + mutex_unlock(&fl->sctx->mutex); goto attach_err; } table = dma_buf_map_attachment_unlocked(map->attach, DMA_BIDIRECTIONAL); if (IS_ERR(table)) { err = PTR_ERR(table); + mutex_unlock(&fl->sctx->mutex); goto map_err; } map->table = table; + mutex_unlock(&fl->sctx->mutex); if (attr & FASTRPC_ATTR_SECUREMAP) map->dma_addr = sg_phys(map->table->sgl); @@ -921,7 +1040,8 @@ static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx *ctx) sizeof(struct fastrpc_invoke_buf) + sizeof(struct fastrpc_phy_page)) * ctx->nscalars + sizeof(u64) * FASTRPC_MAX_FDLIST + - sizeof(u32) * FASTRPC_MAX_CRCLIST; + sizeof(u32) * FASTRPC_MAX_CRCLIST + + sizeof(u32); return size; } @@ -1016,6 +1136,10 @@ static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx) rpra = ctx->buf->virt; list = fastrpc_invoke_buf_start(rpra, ctx->nscalars); pages = fastrpc_phy_page_start(list, ctx->nscalars); + ctx->fdlist = (u64 *)(pages + ctx->nscalars); + ctx->poll = (u32 *)((uintptr_t)ctx->fdlist + sizeof(u64) * FASTRPC_MAX_FDLIST + + sizeof(u32) * FASTRPC_MAX_CRCLIST); + args = (uintptr_t)ctx->buf->virt + metalen; rlen = pkt_size - metalen; ctx->rpra = rpra; @@ -1118,18 +1242,10 @@ static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx, union fastrpc_remote_arg *rpra = ctx->rpra; struct fastrpc_user *fl = ctx->fl; struct fastrpc_map *mmap = NULL; - struct fastrpc_invoke_buf *list; - struct fastrpc_phy_page *pages; - u64 *fdlist; - int i, inbufs, outbufs, handles; + int i, inbufs; int ret = 0; inbufs = REMOTE_SCALARS_INBUFS(ctx->sc); - outbufs = REMOTE_SCALARS_OUTBUFS(ctx->sc); - handles = REMOTE_SCALARS_INHANDLES(ctx->sc) + REMOTE_SCALARS_OUTHANDLES(ctx->sc); - list = fastrpc_invoke_buf_start(rpra, ctx->nscalars); - pages = fastrpc_phy_page_start(list, ctx->nscalars); - fdlist = (uint64_t *)(pages + inbufs + outbufs + handles); for (i = inbufs; i < ctx->nbufs; ++i) { if (!ctx->maps[i]) { @@ -1151,9 +1267,9 @@ static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx, cleanup_fdlist: /* Clean up fdlist which is updated by DSP */ for (i = 0; i < FASTRPC_MAX_FDLIST; i++) { - if (!fdlist[i]) + if (!ctx->fdlist[i]) break; - if (!fastrpc_map_lookup(fl, (int)fdlist[i], &mmap)) + if (!fastrpc_map_lookup(fl, (int)ctx->fdlist[i], &mmap)) fastrpc_map_put(mmap); } @@ -1192,13 +1308,80 @@ static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx, } +static inline u32 fastrpc_poll_op(void *p) +{ + struct fastrpc_invoke_ctx *ctx = p; + + dma_rmb(); + return READ_ONCE(*ctx->poll); +} + +static int poll_for_remote_response(struct fastrpc_invoke_ctx *ctx) +{ + u32 val; + int ret; + + /* + * Poll until DSP writes FASTRPC_POLL_RESPONSE into *ctx->poll + * or until another path marks the work done. + */ + ret = read_poll_timeout_atomic(fastrpc_poll_op, val, + (val == FASTRPC_POLL_RESPONSE) || + ctx->is_work_done, 1, + FASTRPC_POLL_MAX_TIMEOUT_US, false, ctx); + + if (!ret && val == FASTRPC_POLL_RESPONSE) { + ctx->is_work_done = true; + ctx->retval = 0; + } + + if (ret == -ETIMEDOUT) + ret = -EIO; + + return ret; +} + +static inline int fastrpc_wait_for_response(struct fastrpc_invoke_ctx *ctx, + u32 kernel) +{ + int err = 0; + + if (kernel) { + if (!wait_for_completion_timeout(&ctx->work, 10 * HZ)) + err = -ETIMEDOUT; + } else { + err = wait_for_completion_interruptible(&ctx->work); + } + + return err; +} + +static int fastrpc_wait_for_completion(struct fastrpc_invoke_ctx *ctx, + u32 kernel) +{ + int err; + + do { + if (ctx->is_polled) { + err = poll_for_remote_response(ctx); + /* If polling timed out, move to normal response mode */ + if (err) + ctx->is_polled = false; + } else { + err = fastrpc_wait_for_response(ctx, kernel); + if (err) + return err; + } + } while (!ctx->is_work_done); + + return err; +} + static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel, u32 handle, u32 sc, struct fastrpc_invoke_args *args) { struct fastrpc_invoke_ctx *ctx = NULL; - struct fastrpc_buf *buf, *b; - int err = 0; if (!fl->sctx) @@ -1212,6 +1395,14 @@ static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel, return -EPERM; } + if (!kernel) { + ctx = fastrpc_context_restore_interrupted(fl, sc); + if (IS_ERR(ctx)) + return PTR_ERR(ctx); + if (ctx) + goto wait; + } + ctx = fastrpc_context_alloc(fl, kernel, sc, args); if (IS_ERR(ctx)) return PTR_ERR(ctx); @@ -1227,16 +1418,27 @@ static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel, if (err) goto bail; - if (kernel) { - if (!wait_for_completion_timeout(&ctx->work, 10 * HZ)) - err = -ETIMEDOUT; - } else { - err = wait_for_completion_interruptible(&ctx->work); - } + /* + * Set message context as polled if the call is for a user PD + * dynamic module and user has enabled poll mode. + */ + if (handle > FASTRPC_MAX_STATIC_HANDLE && fl->pd == USER_PD && + fl->poll_mode) + ctx->is_polled = true; + +wait: + err = fastrpc_wait_for_completion(ctx, kernel); if (err) goto bail; + if (!ctx->is_work_done) { + err = -ETIMEDOUT; + dev_dbg(fl->sctx->dev, "Invalid workdone state for handle 0x%x, sc 0x%x\n", + handle, sc); + goto bail; + } + /* make sure that all memory writes by DSP are seen by CPU */ dma_rmb(); /* populate all the output buffers with results */ @@ -1250,7 +1452,9 @@ static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel, goto bail; bail: - if (err != -ERESTARTSYS && err != -ETIMEDOUT) { + if (ctx && err == -ERESTARTSYS) { + fastrpc_context_save_interrupted(ctx); + } else if (ctx && err != -ETIMEDOUT) { /* We are done with this compute context */ spin_lock(&fl->lock); list_del(&ctx->node); @@ -1258,13 +1462,6 @@ static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel, fastrpc_context_put(ctx); } - if (err == -ERESTARTSYS) { - list_for_each_entry_safe(buf, b, &fl->mmaps, node) { - list_del(&buf->node); - list_add_tail(&buf->node, &fl->cctx->invoke_interrupted_mmaps); - } - } - if (err) dev_dbg(fl->sctx->dev, "Error: Invoke Failed %d\n", err); @@ -1295,15 +1492,16 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl, struct fastrpc_init_create_static init; struct fastrpc_invoke_args *args; struct fastrpc_phy_page pages[1]; + struct fastrpc_channel_ctx *cctx = fl->cctx; char *name; int err; - bool scm_done = false; struct { int client_id; u32 namelen; u32 pageslen; } inbuf; u32 sc; + unsigned long flags; args = kzalloc_objs(*args, FASTRPC_CREATE_STATIC_PROCESS_NARGS); if (!args) @@ -1324,35 +1522,10 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl, err = PTR_ERR(name); goto err; } - - if (!fl->cctx->remote_heap) { - err = fastrpc_remote_heap_alloc(fl, fl->sctx->dev, init.memlen, - &fl->cctx->remote_heap); - if (err) - goto err_name; - - /* Map if we have any heap VMIDs associated with this ADSP Static Process. */ - if (fl->cctx->vmcount) { - u64 src_perms = BIT(QCOM_SCM_VMID_HLOS); - - err = qcom_scm_assign_mem(fl->cctx->remote_heap->dma_addr, - (u64)fl->cctx->remote_heap->size, - &src_perms, - fl->cctx->vmperms, fl->cctx->vmcount); - if (err) { - dev_err(fl->sctx->dev, - "Failed to assign memory with dma_addr %pad size 0x%llx err %d\n", - &fl->cctx->remote_heap->dma_addr, - fl->cctx->remote_heap->size, err); - goto err_map; - } - scm_done = true; - } - } - inbuf.client_id = fl->client_id; inbuf.namelen = init.namelen; inbuf.pageslen = 0; + fl->pd = USER_PD; args[0].ptr = (u64)(uintptr_t)&inbuf; @@ -1363,8 +1536,25 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl, args[1].length = inbuf.namelen; args[1].fd = -1; - pages[0].addr = fl->cctx->remote_heap->dma_addr; - pages[0].size = fl->cctx->remote_heap->size; + spin_lock_irqsave(&cctx->lock, flags); + if (!fl->cctx->audio_init_mem) { + if (!fl->cctx->remote_heap || + !fl->cctx->remote_heap->dma_addr || + !fl->cctx->remote_heap->size) { + spin_unlock_irqrestore(&cctx->lock, flags); + err = -ENOMEM; + goto err; + } + + pages[0].addr = fl->cctx->remote_heap->dma_addr; + pages[0].size = fl->cctx->remote_heap->size; + fl->cctx->audio_init_mem = true; + inbuf.pageslen = 1; + } else { + pages[0].addr = 0; + pages[0].size = 0; + } + spin_unlock_irqrestore(&cctx->lock, flags); args[2].ptr = (u64)(uintptr_t) pages; args[2].length = sizeof(*pages); @@ -1382,27 +1572,7 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl, return 0; err_invoke: - if (fl->cctx->vmcount && scm_done) { - u64 src_perms = 0; - struct qcom_scm_vmperm dst_perms; - u32 i; - - for (i = 0; i < fl->cctx->vmcount; i++) - src_perms |= BIT(fl->cctx->vmperms[i].vmid); - - dst_perms.vmid = QCOM_SCM_VMID_HLOS; - dst_perms.perm = QCOM_SCM_PERM_RWX; - err = qcom_scm_assign_mem(fl->cctx->remote_heap->dma_addr, - (u64)fl->cctx->remote_heap->size, - &src_perms, &dst_perms, 1); - if (err) - dev_err(fl->sctx->dev, "Failed to assign memory dma_addr %pad size 0x%llx err %d\n", - &fl->cctx->remote_heap->dma_addr, fl->cctx->remote_heap->size, err); - } -err_map: - fastrpc_buf_free(fl->cctx->remote_heap); - fl->cctx->remote_heap = NULL; -err_name: + fl->cctx->audio_init_mem = false; kfree(name); err: kfree(args); @@ -1590,14 +1760,18 @@ static int fastrpc_device_release(struct inode *inode, struct file *file) list_del(&fl->user); spin_unlock_irqrestore(&cctx->lock, flags); - if (fl->init_mem) - fastrpc_buf_free(fl->init_mem); + fastrpc_buf_free(fl->init_mem); list_for_each_entry_safe(ctx, n, &fl->pending, node) { list_del(&ctx->node); fastrpc_context_put(ctx); } + list_for_each_entry_safe(ctx, n, &fl->interrupted, node) { + list_del(&ctx->node); + fastrpc_context_put(ctx); + } + list_for_each_entry_safe(map, m, &fl->maps, node) fastrpc_map_put(map); @@ -1607,11 +1781,9 @@ static int fastrpc_device_release(struct inode *inode, struct file *file) } fastrpc_session_free(cctx, fl->sctx); - fastrpc_channel_ctx_put(cctx); - - mutex_destroy(&fl->mutex); - kfree(fl); file->private_data = NULL; + /* Release the reference taken in fastrpc_device_open */ + fastrpc_user_put(fl); return 0; } @@ -1637,6 +1809,7 @@ static int fastrpc_device_open(struct inode *inode, struct file *filp) spin_lock_init(&fl->lock); mutex_init(&fl->mutex); INIT_LIST_HEAD(&fl->pending); + INIT_LIST_HEAD(&fl->interrupted); INIT_LIST_HEAD(&fl->maps); INIT_LIST_HEAD(&fl->mmaps); INIT_LIST_HEAD(&fl->user); @@ -1655,6 +1828,7 @@ static int fastrpc_device_open(struct inode *inode, struct file *filp) spin_lock_irqsave(&cctx->lock, flags); list_add_tail(&fl->user, &cctx->users); spin_unlock_irqrestore(&cctx->lock, flags); + kref_init(&fl->refcount); return 0; } @@ -1802,7 +1976,7 @@ static int fastrpc_get_info_from_kernel(struct fastrpc_ioctl_capability *cap, kfree(dsp_attributes); return -EOPNOTSUPP; } else if (err) { - dev_err(&cctx->rpdev->dev, "Error: dsp information is incorrect err: %d\n", err); + dev_dbg(&cctx->rpdev->dev, "Error: dsp information is incorrect err: %d\n", err); kfree(dsp_attributes); return err; } @@ -1817,6 +1991,30 @@ static int fastrpc_get_info_from_kernel(struct fastrpc_ioctl_capability *cap, return 0; } +static int fastrpc_set_option(struct fastrpc_user *fl, char __user *argp) +{ + struct fastrpc_ioctl_set_option opt = {0}; + int i; + + if (copy_from_user(&opt, argp, sizeof(opt))) + return -EFAULT; + + for (i = 0; i < ARRAY_SIZE(opt.reserved); i++) { + if (opt.reserved[i] != 0) + return -EINVAL; + } + + if (opt.req != FASTRPC_POLL_MODE) + return -EINVAL; + + if (opt.value) + fl->poll_mode = true; + else + fl->poll_mode = false; + + return 0; +} + static int fastrpc_get_dsp_info(struct fastrpc_user *fl, char __user *argp) { struct fastrpc_ioctl_capability cap = {0}; @@ -1863,9 +2061,6 @@ static int fastrpc_req_munmap_impl(struct fastrpc_user *fl, struct fastrpc_buf * &args[0]); if (!err) { dev_dbg(dev, "unmmap\tpt 0x%09lx OK\n", buf->raddr); - spin_lock(&fl->lock); - list_del(&buf->node); - spin_unlock(&fl->lock); fastrpc_buf_free(buf); } else { dev_err(dev, "unmmap\tpt 0x%09lx ERROR\n", buf->raddr); @@ -1879,6 +2074,7 @@ static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp) struct fastrpc_buf *buf = NULL, *iter, *b; struct fastrpc_req_munmap req; struct device *dev = fl->sctx->dev; + int err; if (copy_from_user(&req, argp, sizeof(req))) return -EFAULT; @@ -1886,6 +2082,7 @@ static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp) spin_lock(&fl->lock); list_for_each_entry_safe(iter, b, &fl->mmaps, node) { if ((iter->raddr == req.vaddrout) && (iter->size == req.size)) { + list_del(&iter->node); buf = iter; break; } @@ -1898,7 +2095,14 @@ static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp) return -EINVAL; } - return fastrpc_req_munmap_impl(fl, buf); + err = fastrpc_req_munmap_impl(fl, buf); + if (err) { + spin_lock(&fl->lock); + list_add_tail(&buf->node, &fl->mmaps); + spin_unlock(&fl->lock); + } + + return err; } static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp) @@ -1989,14 +2193,17 @@ static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp) if (copy_to_user((void __user *)argp, &req, sizeof(req))) { err = -EFAULT; - goto err_assign; + goto err_copy; } dev_dbg(dev, "mmap\t\tpt 0x%09lx OK [len 0x%08llx]\n", buf->raddr, buf->size); return 0; - +err_copy: + spin_lock(&fl->lock); + list_del(&buf->node); + spin_unlock(&fl->lock); err_assign: fastrpc_req_munmap_impl(fl, buf); @@ -2172,6 +2379,9 @@ static long fastrpc_device_ioctl(struct file *file, unsigned int cmd, case FASTRPC_IOCTL_MEM_UNMAP: err = fastrpc_req_mem_unmap(fl, argp); break; + case FASTRPC_IOCTL_SET_OPTION: + err = fastrpc_set_option(fl, argp); + break; case FASTRPC_IOCTL_GET_DSP_INFO: err = fastrpc_get_dsp_info(fl, argp); break; @@ -2217,6 +2427,8 @@ static int fastrpc_cb_probe(struct platform_device *pdev) sess->used = false; sess->valid = true; sess->dev = dev; + mutex_init(&sess->mutex); + sess->allocated = true; dev_set_drvdata(dev, sess); if (cctx->domain_id == CDSP_DOMAIN_ID) @@ -2233,6 +2445,8 @@ static int fastrpc_cb_probe(struct platform_device *pdev) break; dup_sess = &cctx->session[cctx->sesscount++]; memcpy(dup_sess, sess, sizeof(*dup_sess)); + mutex_init(&dup_sess->mutex); + dup_sess->allocated = true; } } spin_unlock_irqrestore(&cctx->lock, flags); @@ -2255,6 +2469,11 @@ static void fastrpc_cb_remove(struct platform_device *pdev) spin_lock_irqsave(&cctx->lock, flags); for (i = 0; i < FASTRPC_MAX_SESSIONS; i++) { if (cctx->session[i].sid == sess->sid) { + spin_unlock_irqrestore(&cctx->lock, flags); + mutex_lock(&cctx->session[i].mutex); + cctx->session[i].dev = NULL; + mutex_unlock(&cctx->session[i].mutex); + spin_lock_irqsave(&cctx->lock, flags); cctx->session[i].valid = false; cctx->sesscount--; } @@ -2361,7 +2580,7 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) } if (of_reserved_mem_device_init_by_idx(rdev, rdev->of_node, 0)) - dev_info(rdev, "no reserved DMA memory for FASTRPC\n"); + dev_dbg(rdev, "no reserved DMA memory for FASTRPC\n"); vmcount = of_property_read_variable_u32_array(rdev->of_node, "qcom,vmids", &vmids[0], 0, FASTRPC_MAX_VMIDS); @@ -2382,7 +2601,7 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) } } - if (domain_id == SDSP_DOMAIN_ID) { + if (domain_id == SDSP_DOMAIN_ID || domain_id == ADSP_DOMAIN_ID) { struct resource res; u64 src_perms; @@ -2396,6 +2615,15 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) goto err_free_data; } + if (domain_id == ADSP_DOMAIN_ID) { + data->remote_heap = + kzalloc_obj(*data->remote_heap, GFP_KERNEL); + if (!data->remote_heap) + return -ENOMEM; + + data->remote_heap->dma_addr = res.start; + data->remote_heap->size = resource_size(&res); + } } secure_dsp = !(of_property_read_bool(rdev->of_node, "qcom,non-secure-domain")); @@ -2435,7 +2663,6 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) rdev->dma_mask = &data->dma_mask; dma_set_mask_and_coherent(rdev, DMA_BIT_MASK(32)); INIT_LIST_HEAD(&data->users); - INIT_LIST_HEAD(&data->invoke_interrupted_mmaps); spin_lock_init(&data->lock); idr_init(&data->ctx_idr); data->domain_id = domain_id; @@ -2467,15 +2694,19 @@ static void fastrpc_notify_users(struct fastrpc_user *user) ctx->retval = -EPIPE; complete(&ctx->work); } + list_for_each_entry(ctx, &user->interrupted, node) { + ctx->retval = -EPIPE; + complete(&ctx->work); + } spin_unlock(&user->lock); } static void fastrpc_rpmsg_remove(struct rpmsg_device *rpdev) { struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev); - struct fastrpc_buf *buf, *b; struct fastrpc_user *user; unsigned long flags; + int err; /* No invocations past this point */ spin_lock_irqsave(&cctx->lock, flags); @@ -2490,11 +2721,22 @@ static void fastrpc_rpmsg_remove(struct rpmsg_device *rpdev) if (cctx->secure_fdevice) misc_deregister(&cctx->secure_fdevice->miscdev); - list_for_each_entry_safe(buf, b, &cctx->invoke_interrupted_mmaps, node) - list_del(&buf->node); + if (cctx->remote_heap && cctx->vmcount) { + u64 src_perms = 0; + struct qcom_scm_vmperm dst_perms; + + for (u32 i = 0; i < cctx->vmcount; i++) + src_perms |= BIT(cctx->vmperms[i].vmid); + + dst_perms.vmid = QCOM_SCM_VMID_HLOS; + dst_perms.perm = QCOM_SCM_PERM_RWX; - if (cctx->remote_heap) - fastrpc_buf_free(cctx->remote_heap); + err = qcom_scm_assign_mem(cctx->remote_heap->dma_addr, + cctx->remote_heap->size, &src_perms, + &dst_perms, 1); + if (!err) + fastrpc_buf_free(cctx->remote_heap); + } of_platform_depopulate(&rpdev->dev); @@ -2513,7 +2755,7 @@ static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data, if (len < sizeof(*rsp)) return -EINVAL; - ctxid = ((rsp->ctx & FASTRPC_CTXID_MASK) >> 4); + ctxid = FIELD_GET(FASTRPC_CTXID_MASK, rsp->ctx); spin_lock_irqsave(&cctx->lock, flags); ctx = idr_find(&cctx->ctx_idr, ctxid); @@ -2525,6 +2767,7 @@ static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data, } ctx->retval = rsp->retval; + ctx->is_work_done = true; complete(&ctx->work); /* diff --git a/include/uapi/misc/fastrpc.h b/include/uapi/misc/fastrpc.h index c6e2925f47e69..c37e24a764ae6 100644 --- a/include/uapi/misc/fastrpc.h +++ b/include/uapi/misc/fastrpc.h @@ -16,6 +16,7 @@ #define FASTRPC_IOCTL_INIT_CREATE_STATIC _IOWR('R', 9, struct fastrpc_init_create_static) #define FASTRPC_IOCTL_MEM_MAP _IOWR('R', 10, struct fastrpc_mem_map) #define FASTRPC_IOCTL_MEM_UNMAP _IOWR('R', 11, struct fastrpc_mem_unmap) +#define FASTRPC_IOCTL_SET_OPTION _IOWR('R', 12, struct fastrpc_ioctl_set_option) #define FASTRPC_IOCTL_GET_DSP_INFO _IOWR('R', 13, struct fastrpc_ioctl_capability) /** @@ -67,6 +68,9 @@ enum fastrpc_proc_attr { /* Fastrpc attribute for memory protection of buffers */ #define FASTRPC_ATTR_SECUREMAP (1) +/* Set option request ID to enable poll mode */ +#define FASTRPC_POLL_MODE (1) + struct fastrpc_invoke_args { __u64 ptr; __u64 length; @@ -133,6 +137,12 @@ struct fastrpc_mem_unmap { __s32 reserved[5]; }; +struct fastrpc_ioctl_set_option { + __u32 req; /* request id */ + __u32 value; /* value */ + __s32 reserved[6]; +}; + struct fastrpc_ioctl_capability { __u32 unused; /* deprecated, ignored by the kernel */ __u32 attribute_id;