qemu/include/ui/qemu-pixman.h
Daniel P. Berrangé 7009744285 ui/vnc: take account of client byte order in pixman format
The set_pixel_conversion() method is responsible for determining whether
the VNC client pixel format matches the server format, and thus whether
we can use the fast path "copy" impl for sending pixels, or must use
the generic impl with bit swizzling.

The VNC server format is set at build time to VNC_SERVER_FB_FORMAT,
which corresponds to PIXMAN_x8r8g8b8.

The qemu_pixman_get_format() method is then responsible for converting
the VNC pixel format into a pixman format.

The VNC client pixel shifts are relative to the associated endianness.

The pixman formats are always relative to the host native endianness.

The qemu_pixman_get_format() method does not take into account the
VNC client endianness, and is thus returning a pixman format that is
only valid with the host endianness matches that of the VNC client.

This has been broken since pixman was introduced to the VNC server:

  commit 9f64916da2
  Author: Gerd Hoffmann <kraxel@redhat.com>
  Date:   Wed Oct 10 13:29:43 2012 +0200

    pixman/vnc: use pixman images in vnc.

The flaw can be demonstrated using the Tigervnc client by using

   vncviewer -AutoSelect=0 -PreferredEncoding=raw server:display

connecting from a LE client to a QEMU on a BE server, or the
reverse.

The bug was masked, however, because almost all VNC clients will
advertize support for the "tight" encoding and the QEMU VNC server
will prefer "tight" if advertized.

The tight_pack24 method is responsible for taking a set of pixels
which have already been converted into client endianness and then
repacking them into the TPIXEL format which the RFB spec defines
as

  "TPIXEL is only 3 bytes long, where the first byte is the
   red component, the second byte is the green component,
   and the third byte is the blue component of the pixel
   color value"

IOW, the TPIXEL format is fixed on the wire, regardless of what
the VNC client declare as its endianness.

Since the VNC pixel encoding code was failing to honour the endian
flag of the client, the tight_pack24 method was always operating
on data in native endianness. Its impl cancelled out the VNC pixel
encoding bug.

With the VNC pixel encoding code now fixed, the tight_pack24 method
needs to take into account that it is operating on data in client
endianness, not native endianness. It thus may need to invert the
pixel shifts.

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Daniel P. Berrangé <berrange@redhat.com>
2025-05-21 19:28:01 +01:00

126 lines
4.5 KiB
C

/*
* This work is licensed under the terms of the GNU GPL, version 2 or later.
* See the COPYING file in the top-level directory.
*/
#ifndef QEMU_PIXMAN_H
#define QEMU_PIXMAN_H
#ifdef CONFIG_PIXMAN
#include <pixman.h>
#else
#include "pixman-minimal.h"
#endif
#include "qapi/error.h"
/*
* pixman image formats are defined to be native endian,
* that means host byte order on qemu. So we go define
* fixed formats here for cases where it is needed, like
* feeding libjpeg / libpng and writing screenshots.
*/
#if HOST_BIG_ENDIAN
# define PIXMAN_BE_r8g8b8 PIXMAN_r8g8b8
# define PIXMAN_BE_x8r8g8b8 PIXMAN_x8r8g8b8
# define PIXMAN_BE_a8r8g8b8 PIXMAN_a8r8g8b8
# define PIXMAN_BE_b8g8r8x8 PIXMAN_b8g8r8x8
# define PIXMAN_BE_b8g8r8a8 PIXMAN_b8g8r8a8
# define PIXMAN_BE_r8g8b8x8 PIXMAN_r8g8b8x8
# define PIXMAN_BE_r8g8b8a8 PIXMAN_r8g8b8a8
# define PIXMAN_BE_x8b8g8r8 PIXMAN_x8b8g8r8
# define PIXMAN_BE_a8b8g8r8 PIXMAN_a8b8g8r8
# define PIXMAN_LE_r8g8b8 PIXMAN_b8g8r8
# define PIXMAN_LE_a8r8g8b8 PIXMAN_b8g8r8a8
# define PIXMAN_LE_x8r8g8b8 PIXMAN_b8g8r8x8
# define PIXMAN_LE_a8b8g8r8 PIXMAN_r8g8b8a8
# define PIXMAN_LE_x8b8g8r8 PIXMAN_r8g8b8x8
#else
# define PIXMAN_BE_r8g8b8 PIXMAN_b8g8r8
# define PIXMAN_BE_x8r8g8b8 PIXMAN_b8g8r8x8
# define PIXMAN_BE_a8r8g8b8 PIXMAN_b8g8r8a8
# define PIXMAN_BE_b8g8r8x8 PIXMAN_x8r8g8b8
# define PIXMAN_BE_b8g8r8a8 PIXMAN_a8r8g8b8
# define PIXMAN_BE_r8g8b8x8 PIXMAN_x8b8g8r8
# define PIXMAN_BE_r8g8b8a8 PIXMAN_a8b8g8r8
# define PIXMAN_BE_x8b8g8r8 PIXMAN_r8g8b8x8
# define PIXMAN_BE_a8b8g8r8 PIXMAN_r8g8b8a8
# define PIXMAN_LE_r8g8b8 PIXMAN_r8g8b8
# define PIXMAN_LE_a8r8g8b8 PIXMAN_a8r8g8b8
# define PIXMAN_LE_x8r8g8b8 PIXMAN_x8r8g8b8
# define PIXMAN_LE_a8b8g8r8 PIXMAN_a8b8g8r8
# define PIXMAN_LE_x8b8g8r8 PIXMAN_x8b8g8r8
#endif
#define QEMU_PIXMAN_COLOR(r, g, b) \
{ .red = r << 8, .green = g << 8, .blue = b << 8, .alpha = 0xffff }
#define QEMU_PIXMAN_COLOR_BLACK QEMU_PIXMAN_COLOR(0x00, 0x00, 0x00)
#define QEMU_PIXMAN_COLOR_GRAY QEMU_PIXMAN_COLOR(0xaa, 0xaa, 0xaa)
/* -------------------------------------------------------------------- */
typedef struct PixelFormat {
uint8_t bits_per_pixel;
uint8_t bytes_per_pixel;
uint8_t depth; /* color depth in bits */
uint32_t rmask, gmask, bmask, amask;
uint8_t rshift, gshift, bshift, ashift;
uint8_t rmax, gmax, bmax, amax;
uint8_t rbits, gbits, bbits, abits;
} PixelFormat;
PixelFormat qemu_pixelformat_from_pixman(pixman_format_code_t format);
pixman_format_code_t qemu_default_pixman_format(int bpp, bool native_endian);
pixman_format_code_t qemu_drm_format_to_pixman(uint32_t drm_format);
uint32_t qemu_pixman_to_drm_format(pixman_format_code_t pixman);
int qemu_pixman_get_type(int rshift, int gshift, int bshift, int endian);
bool qemu_pixman_check_format(DisplayChangeListener *dcl,
pixman_format_code_t format);
#ifdef CONFIG_PIXMAN
pixman_format_code_t qemu_pixman_get_format(PixelFormat *pf, int endian);
pixman_image_t *qemu_pixman_linebuf_create(pixman_format_code_t format,
int width);
void qemu_pixman_linebuf_fill(pixman_image_t *linebuf, pixman_image_t *fb,
int width, int x, int y);
pixman_image_t *qemu_pixman_mirror_create(pixman_format_code_t format,
pixman_image_t *image);
pixman_image_t *qemu_pixman_glyph_from_vgafont(int height, const uint8_t *font,
unsigned int ch);
void qemu_pixman_glyph_render(pixman_image_t *glyph,
pixman_image_t *surface,
pixman_color_t *fgcol,
pixman_color_t *bgcol,
int x, int y, int cw, int ch);
#endif
void qemu_pixman_image_unref(pixman_image_t *image);
#ifdef WIN32
typedef HANDLE qemu_pixman_shareable;
#define SHAREABLE_NONE (NULL)
#define SHAREABLE_TO_PTR(handle) (handle)
#define PTR_TO_SHAREABLE(ptr) (ptr)
#else
typedef int qemu_pixman_shareable;
#define SHAREABLE_NONE (-1)
#define SHAREABLE_TO_PTR(handle) GINT_TO_POINTER(handle)
#define PTR_TO_SHAREABLE(ptr) GPOINTER_TO_INT(ptr)
#endif
bool qemu_pixman_image_new_shareable(
pixman_image_t **image,
qemu_pixman_shareable *handle,
const char *name,
pixman_format_code_t format,
int width,
int height,
int rowstride_bytes,
Error **errp);
G_DEFINE_AUTOPTR_CLEANUP_FUNC(pixman_image_t, qemu_pixman_image_unref)
#endif /* QEMU_PIXMAN_H */