linux-kernel - [PATCH v3 1/5] drm: move native byte order quirk to new drm_mode_legacy_fb

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [thread-next>] [day] [month] [year] [list]

Date:   Wed, 19 Sep 2018 13:12:48 +0200
From:   Gerd Hoffmann <kraxel@...hat.com>
To:     dri-devel@...ts.freedesktop.org
Cc:     michel@...nzer.net, imirkin@...m.mit.edu,
        Gerd Hoffmann <kraxel@...hat.com>,
        Gustavo Padovan <gustavo@...ovan.org>,
        Maarten Lankhorst <maarten.lankhorst@...ux.intel.com>,
        Sean Paul <sean@...rly.run>, David Airlie <airlied@...ux.ie>,
        linux-kernel@...r.kernel.org (open list)
Subject: [PATCH v3 1/5] drm: move native byte order quirk to new drm_mode_legacy_fb_format2 function

Turns out we need the pixel format fixup not only for the addfb ioctl,
but also for fbdev emulation code.

Ideally we would place it in drm_mode_legacy_fb_format().  That would
create alot of churn though, and most drivers don't care because they
never ever run on a big endian platform.  So add a new
drm_mode_legacy_fb_format2() function instead.

Signed-off-by: Gerd Hoffmann <kraxel@...hat.com>
---
 include/drm/drm_fourcc.h          |  2 ++
 drivers/gpu/drm/drm_fourcc.c      | 28 ++++++++++++++++++++++++++++
 drivers/gpu/drm/drm_framebuffer.c | 15 +++------------
 3 files changed, 33 insertions(+), 12 deletions(-)

diff --git a/include/drm/drm_fourcc.h b/include/drm/drm_fourcc.h
index fac831c401..4b82bb6576 100644
--- a/include/drm/drm_fourcc.h
+++ b/include/drm/drm_fourcc.h
@@ -88,6 +88,8 @@ const struct drm_format_info *
 drm_get_format_info(struct drm_device *dev,
 		    const struct drm_mode_fb_cmd2 *mode_cmd);
 uint32_t drm_mode_legacy_fb_format(uint32_t bpp, uint32_t depth);
+uint32_t drm_mode_legacy_fb_format2(uint32_t bpp, uint32_t depth,
+				    bool native);
 int drm_format_num_planes(uint32_t format);
 int drm_format_plane_cpp(uint32_t format, int plane);
 int drm_format_horz_chroma_subsampling(uint32_t format);
diff --git a/drivers/gpu/drm/drm_fourcc.c b/drivers/gpu/drm/drm_fourcc.c
index be1d6aaef6..1c1aaa8b23 100644
--- a/drivers/gpu/drm/drm_fourcc.c
+++ b/drivers/gpu/drm/drm_fourcc.c
@@ -96,6 +96,34 @@ uint32_t drm_mode_legacy_fb_format(uint32_t bpp, uint32_t depth)
 EXPORT_SYMBOL(drm_mode_legacy_fb_format);
 
 /**
+ * drm_mode_legacy_fb_format2 - compute drm fourcc code from legacy description
+ * @bpp: bits per pixels
+ * @depth: bit depth per pixel
+ * @native: use host native byte order
+ *
+ * Computes a drm fourcc pixel format code for the given @bpp/@...th values.
+ * Useful in fbdev emulation code, since that deals in those values.
+ */
+uint32_t drm_mode_legacy_fb_format2(uint32_t bpp, uint32_t depth,
+				    bool native)
+{
+	uint32_t fmt = drm_mode_legacy_fb_format(bpp, depth);
+
+	if (native) {
+		if (fmt == DRM_FORMAT_XRGB8888)
+			fmt = DRM_FORMAT_HOST_XRGB8888;
+		if (fmt == DRM_FORMAT_ARGB8888)
+			fmt = DRM_FORMAT_HOST_ARGB8888;
+		if (fmt == DRM_FORMAT_RGB565)
+			fmt = DRM_FORMAT_HOST_RGB565;
+		if (fmt == DRM_FORMAT_XRGB1555)
+			fmt = DRM_FORMAT_HOST_XRGB1555;
+	}
+	return fmt;
+}
+EXPORT_SYMBOL(drm_mode_legacy_fb_format2);
+
+/**
  * drm_get_format_name - fill a string with a drm fourcc format's name
  * @format: format to compute name of
  * @buf: caller-supplied buffer
diff --git a/drivers/gpu/drm/drm_framebuffer.c b/drivers/gpu/drm/drm_framebuffer.c
index 1ee3d6b442..f72e3dffc7 100644
--- a/drivers/gpu/drm/drm_framebuffer.c
+++ b/drivers/gpu/drm/drm_framebuffer.c
@@ -111,12 +111,14 @@ int drm_mode_addfb(struct drm_device *dev, struct drm_mode_fb_cmd *or,
 		   struct drm_file *file_priv)
 {
 	struct drm_mode_fb_cmd2 r = {};
+	bool native = dev->mode_config.quirk_addfb_prefer_host_byte_order;
 	int ret;
 
 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
 		return -EOPNOTSUPP;
 
-	r.pixel_format = drm_mode_legacy_fb_format(or->bpp, or->depth);
+	r.pixel_format = drm_mode_legacy_fb_format2(or->bpp, or->depth,
+						    native);
 	if (r.pixel_format == DRM_FORMAT_INVALID) {
 		DRM_DEBUG("bad {bpp:%d, depth:%d}\n", or->bpp, or->depth);
 		return -EINVAL;
@@ -133,17 +135,6 @@ int drm_mode_addfb(struct drm_device *dev, struct drm_mode_fb_cmd *or,
 	    r.pixel_format == DRM_FORMAT_XRGB2101010)
 		r.pixel_format = DRM_FORMAT_XBGR2101010;
 
-	if (dev->mode_config.quirk_addfb_prefer_host_byte_order) {
-		if (r.pixel_format == DRM_FORMAT_XRGB8888)
-			r.pixel_format = DRM_FORMAT_HOST_XRGB8888;
-		if (r.pixel_format == DRM_FORMAT_ARGB8888)
-			r.pixel_format = DRM_FORMAT_HOST_ARGB8888;
-		if (r.pixel_format == DRM_FORMAT_RGB565)
-			r.pixel_format = DRM_FORMAT_HOST_RGB565;
-		if (r.pixel_format == DRM_FORMAT_XRGB1555)
-			r.pixel_format = DRM_FORMAT_HOST_XRGB1555;
-	}
-
 	ret = drm_mode_addfb2(dev, &r, file_priv);
 	if (ret)
 		return ret;
-- 
2.9.3