linux-kernel - [PATCH v2 08/12] staging: media: rkvdec: Add a valid SPS check as a callback

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]

Message-Id: <20230101-patch-series-v2-6-2-rc1-v2-8-fa1897efac14@collabora.com>
Date:   Thu, 12 Jan 2023 13:56:23 +0100
From:   Sebastian Fricke <sebastian.fricke@...labora.com>
To:     Mauro Carvalho Chehab <mchehab@...nel.org>,
        Ezequiel Garcia <ezequiel@...guardiasur.com.ar>,
        Greg Kroah-Hartman <gregkh@...uxfoundation.org>
Cc:     linux-media@...r.kernel.org, linux-kernel@...r.kernel.org,
        linux-rockchip@...ts.infradead.org, linux-staging@...ts.linux.dev,
        Jonas Karlman <jonas@...boo.se>,
        Alex Bee <knaerzche@...il.com>,
        Nicolas Dufresne <nicolas.dufresne@...labora.com>,
        Collabora Kernel-domain <kernel@...labora.com>,
        Robert Beckett <bob.beckett@...labora.com>,
        Laurent Pinchart <laurent.pinchart@...asonboard.com>,
        Andrzej Pietrasiewicz <andrzej.p@...labora.com>,
        Benjamin Gaignard <benjamin.gaignard@...labora.com>,
        Sebastian Fricke <sebastian.fricke@...labora.com>
Subject: [PATCH v2 08/12] staging: media: rkvdec: Add a valid SPS check as a callback

Add a callback for each codec variant, that ensures that a given format
is compatible with the SPS that is stored in the context of the
instance.
This is used for two scenarios:
- New output queue format is set, which is incompatible with the
  currently stored SPS
- Start streaming is called but the SPS doesn't match the current format
  anymore

Signed-off-by: Sebastian Fricke <sebastian.fricke@...labora.com>
---
 drivers/staging/media/rkvdec/rkvdec.c | 35 +++++++++++++++++++++++++++++++++++
 drivers/staging/media/rkvdec/rkvdec.h |  1 +
 2 files changed, 36 insertions(+)

diff --git a/drivers/staging/media/rkvdec/rkvdec.c b/drivers/staging/media/rkvdec/rkvdec.c
index e8c750a7343a..8d948bcc4e46 100644
--- a/drivers/staging/media/rkvdec/rkvdec.c
+++ b/drivers/staging/media/rkvdec/rkvdec.c
@@ -52,6 +52,26 @@ static int rkvdec_get_valid_fmt(struct rkvdec_ctx *ctx)
 	return ctx->coded_fmt_desc->decoded_fmts[0];
 }
 
+static int rkvdec_sps_check(struct rkvdec_ctx *ctx, struct v4l2_pix_format_mplane *pix_mp)
+{
+	const struct rkvdec_coded_fmt_desc *coded_desc = ctx->coded_fmt_desc;
+
+	/*
+	 * When a codec doesn't implement the SPS check, handle it as if no
+	 * SPS exists for the codec.
+	 */
+	if (coded_desc->ops->sps_check) {
+		if (!ctx->sps)
+			return -EINVAL;
+
+		if (!pix_mp)
+			pix_mp = &ctx->decoded_fmt.fmt.pix_mp;
+		return coded_desc->ops->sps_check(ctx, ctx->sps, pix_mp);
+	}
+
+	return 0;
+}
+
 static int rkvdec_get_sps_attributes(struct rkvdec_ctx *ctx, void *sps,
 				     struct sps_attributes *attributes)
 {
@@ -375,6 +395,13 @@ static int rkvdec_s_output_fmt(struct file *file, void *priv,
 	struct vb2_queue *vq;
 	int ret;
 
+	/*
+	 * When the new output format doesn't match the existing SPS stored in
+	 * the context, then the SPS needs to be reset by user-space.
+	 */
+	if (rkvdec_sps_check(ctx, &f->fmt.pix_mp) < 0)
+		ctx->sps = NULL;
+
 	/*
 	 * In order to support dynamic resolution change, the decoder admits
 	 * a resolution change, as long as the pixelformat remains. Can't be
@@ -603,6 +630,14 @@ static int rkvdec_start_streaming(struct vb2_queue *q, unsigned int count)
 	if (V4L2_TYPE_IS_CAPTURE(q->type))
 		return 0;
 
+	/*
+	 * An incompatible SPS at this point is invalid, abort the process.
+	 */
+	if (rkvdec_sps_check(ctx, NULL) < 0) {
+		ctx->sps = NULL;
+		return -EINVAL;
+	}
+
 	/*
 	 * Make sure that both the output and the capture queue are running
 	 */
diff --git a/drivers/staging/media/rkvdec/rkvdec.h b/drivers/staging/media/rkvdec/rkvdec.h
index 7b6702c360fd..d74821f9fd0b 100644
--- a/drivers/staging/media/rkvdec/rkvdec.h
+++ b/drivers/staging/media/rkvdec/rkvdec.h
@@ -75,6 +75,7 @@ struct rkvdec_coded_fmt_ops {
 	int (*adjust_fmt)(struct rkvdec_ctx *ctx,
 			  struct v4l2_format *f);
 	u32 (*valid_fmt)(struct rkvdec_ctx *ctx);
+	int (*sps_check)(struct rkvdec_ctx *ctx, void *sps, struct v4l2_pix_format_mplane *pix_mp);
 	int (*get_sps_attributes)(struct rkvdec_ctx *ctx, void *sps,
 				  struct sps_attributes *attributes);
 	int (*start)(struct rkvdec_ctx *ctx);

-- 
2.25.1