File gtk-vnc-0.3.4-endianess.patch of Package gtk-vnc

File gtk-vnc-0.3.4-endianess.patch of Package gtk-vnc (Revision fa40370c2fca9b76a3db8f43c005c791)

Currently displaying revision fa40370c2fca9b76a3db8f43c005c791, show latest

diff -rup gtk-vnc-0.3.4.orig/src/blt.h gtk-vnc-0.3.4.new/src/blt.h
--- gtk-vnc-0.3.4.orig/src/blt.h	2008-02-22 09:14:48.000000000 -0500
+++ gtk-vnc-0.3.4.new/src/blt.h	2008-04-03 10:15:01.000000000 -0400
@@ -16,8 +16,9 @@
 #define RGB24_BLIT SPLICE(gvnc_rgb24_blt_, SUFFIX())
 #define TIGHT_COMPUTE_PREDICTED SPLICE(gvnc_tight_compute_predicted_, SUFFIX())
 #define TIGHT_SUM_PIXEL SPLICE(gvnc_tight_sum_pixel_, SUFFIX())
-#define SWAP(gvnc, pixel) SPLICE(gvnc_swap_, DST)(gvnc, pixel)
-#define COMPONENT(color, pixel) ((SWAP(gvnc, pixel) >> gvnc->fmt.SPLICE(color, _shift) & gvnc->fmt.SPLICE(color, _max)))
+#define SWAP_RFB(gvnc, pixel) SPLICE(gvnc_swap_rfb_, SRC)(gvnc, pixel)
+#define SWAP_IMG(gvnc, pixel) SPLICE(gvnc_swap_img_, DST)(gvnc, pixel)
+#define COMPONENT(color, pixel) ((SWAP_RFB(gvnc, pixel) >> gvnc->fmt.SPLICE(color, _shift) & gvnc->fmt.SPLICE(color, _max)))
 
 static void FAST_FILL(struct gvnc *gvnc, src_pixel_t *sp,
 		      int x, int y, int width, int height)
@@ -41,18 +42,18 @@ static void FAST_FILL(struct gvnc *gvnc,
 	}
 }
 
-static void SET_PIXEL(struct gvnc *gvnc, dst_pixel_t *dp, src_pixel_t *sp)
+static void SET_PIXEL(struct gvnc *gvnc, dst_pixel_t *dp, src_pixel_t sp)
 {
-	*dp = SWAP(gvnc, ((*sp >> gvnc->rrs) & gvnc->rm) << gvnc->rls
-		| ((*sp >> gvnc->grs) & gvnc->gm) << gvnc->gls
-		| ((*sp >> gvnc->brs) & gvnc->bm) << gvnc->bls);
+	*dp = SWAP_IMG(gvnc, ((sp >> gvnc->rrs) & gvnc->rm) << gvnc->rls
+		       | ((sp >> gvnc->grs) & gvnc->gm) << gvnc->gls
+		       | ((sp >> gvnc->brs) & gvnc->bm) << gvnc->bls);
 }
 
 static void SET_PIXEL_AT(struct gvnc *gvnc, int x, int y, src_pixel_t *sp)
 {
 	dst_pixel_t *dp = (dst_pixel_t *)gvnc_get_local(gvnc, x, y);
 
-	SET_PIXEL(gvnc, dp, sp);
+	SET_PIXEL(gvnc, dp, SWAP_RFB(gvnc, *sp));
 }
 
 static void FILL(struct gvnc *gvnc, src_pixel_t *sp,
@@ -66,7 +67,7 @@ static void FILL(struct gvnc *gvnc, src_
 		int j;
 
 		for (j = 0; j < width; j++) {
-			SET_PIXEL(gvnc, dp, sp);
+			SET_PIXEL(gvnc, dp, SWAP_RFB(gvnc, *sp));
 			dp++;
 		}
 		dst += gvnc->local.linesize;
@@ -88,7 +89,7 @@ static void BLIT(struct gvnc *gvnc, uint
 		int j;
 
 		for (j = 0; j < w; j++) {
-			SET_PIXEL(gvnc, dp, sp);
+			SET_PIXEL(gvnc, dp, SWAP_RFB(gvnc, *sp));
 			dp++;
 			sp++;
 		}
@@ -171,11 +172,36 @@ static void RICH_CURSOR_BLIT(struct gvnc
 	uint32_t *dst = (uint32_t *)pixbuf;
 	uint8_t *src = image;
 	uint8_t *alpha = mask;
-	int rs, gs, bs;
+	int as, rs, gs, bs, n;
 
-	rs = 24 - ((sizeof(src_pixel_t) * 8) - gvnc->fmt.red_shift);
-	gs = 16 - (gvnc->fmt.red_shift - gvnc->fmt.green_shift);
-	bs = 8 - (gvnc->fmt.green_shift - gvnc->fmt.blue_shift);
+	/*
+	 * GdkPixbuf is always 32-bit RGB, so we can't use the precomputed
+	 * left / right shift data from gvnc->{r,g,b}{r,l}s. The latter
+	 * is set for the local display depth, which may be different
+	 * to GdkPixbuf's fixed 32-bit RGBA
+	 *
+	 * This function isn't called often, so just re-compute them now
+	 */
+
+#if G_BYTE_ORDER == G_BIG_ENDIAN
+	as = 0;
+	rs = 8;
+	gs = 16;
+	bs = 24;
+#else
+	as = 24;
+	rs = 16;
+	gs = 8;
+	bs = 0;
+#endif
+
+	/* Then this adjusts for remote having less bpp than 32 */
+	for (n = 255 ; n > gvnc->fmt.red_max ; n>>= 1)
+		rs++;
+	for (n = 255 ; n > gvnc->fmt.green_max ; n>>= 1)
+		gs++;
+	for (n = 255 ; n > gvnc->fmt.blue_max ; n>>= 1)
+		bs++;
 
 	for (y1 = 0; y1 < height; y1++) {
 		src_pixel_t *sp = (src_pixel_t *)src;
@@ -186,7 +212,7 @@ static void RICH_CURSOR_BLIT(struct gvnc
 				| (COMPONENT(blue, *sp) << bs);
 
 			if ((mp[x1 / 8] >> (7 - (x1 % 8))) & 1)
-				*dst |= 0xFF000000;
+				*dst |= (0xFF << as);
 
 			dst++;
 			sp++;
@@ -209,6 +235,11 @@ static void RGB24_BLIT(struct gvnc *gvnc
 		uint8_t *sp = data;
 
 		for (i = 0; i < width; i++) {
+			/*
+			 * We use gvnc->fmt.XXX_shift instead of usual gvnc->Xls
+			 * because the source pixel component is a full 8 bits in
+			 * size, and so doesn't need the adjusted shift
+			 */
 			*dp = (((sp[0] * gvnc->fmt.red_max) / 255) << gvnc->fmt.red_shift) |
 				(((sp[1] * gvnc->fmt.green_max) / 255) << gvnc->fmt.green_shift) |
 				(((sp[2] * gvnc->fmt.blue_max) / 255) << gvnc->fmt.blue_shift);
@@ -242,7 +273,7 @@ static void TIGHT_COMPUTE_PREDICTED(stru
 	blue = MAX(blue, 0);
 	blue = MIN(blue, gvnc->fmt.blue_max);
 
-	*ppixel = SWAP(gvnc,
+	*ppixel = SWAP_RFB(gvnc,
 		       (red << gvnc->fmt.red_shift) |
 		       (green << gvnc->fmt.green_shift) |
 		       (blue << gvnc->fmt.blue_shift));
@@ -257,7 +288,7 @@ static void TIGHT_SUM_PIXEL(struct gvnc 
 	green = COMPONENT(green, *lhs) + COMPONENT(green, *rhs);
 	blue = COMPONENT(blue, *lhs) + COMPONENT(blue, *rhs);
 
-	*lhs = SWAP(gvnc,
+	*lhs = SWAP_RFB(gvnc,
 		    ((red & gvnc->fmt.red_max) << gvnc->fmt.red_shift) |
 		    ((green & gvnc->fmt.green_max) << gvnc->fmt.green_shift) |
 		    ((blue & gvnc->fmt.blue_max) << gvnc->fmt.blue_shift));
diff -rup gtk-vnc-0.3.4.orig/src/gvnc.c gtk-vnc-0.3.4.new/src/gvnc.c
--- gtk-vnc-0.3.4.orig/src/gvnc.c	2008-02-22 09:33:46.000000000 -0500
+++ gtk-vnc-0.3.4.new/src/gvnc.c	2008-04-03 10:16:21.000000000 -0400
@@ -826,14 +826,39 @@ gboolean gvnc_set_pixel_format(struct gv
 gboolean gvnc_set_encodings(struct gvnc *gvnc, int n_encoding, int32_t *encoding)
 {
 	uint8_t pad[1] = {0};
-	int i;
+	int i, skip_zrle=0;
+
+	/*
+	 * RealVNC server is broken for ZRLE in some pixel formats.
+	 * Specifically if you have a format with either R, G or B
+	 * components with a max value > 255, it still uses a CPIXEL
+	 * of 3 bytes, even though the colour requirs 4 bytes. It
+	 * thus messes up the colours of the server in a way we can't
+	 * recover from on the client. Most VNC clients don't see this
+	 * problem since they send a 'set pixel format' message instead
+	 * of running with the server's default format.
+	 *
+	 * So we kill off ZRLE encoding for problematic pixel formats
+	 */
+	for (i = 0; i < n_encoding; i++)
+		if (gvnc->fmt.depth == 32 &&
+		    (gvnc->fmt.red_max > 255 ||
+		     gvnc->fmt.blue_max > 255 ||
+		     gvnc->fmt.green_max > 255) &&
+		    encoding[i] == GVNC_ENCODING_ZRLE) {
+			GVNC_DEBUG("Dropping ZRLE encoding for broken pixel format\n");
+			skip_zrle++;
+		}
 
 	gvnc->has_ext_key_event = FALSE;
 	gvnc_write_u8(gvnc, 2);
 	gvnc_write(gvnc, pad, 1);
-	gvnc_write_u16(gvnc, n_encoding);
-	for (i = 0; i < n_encoding; i++)
+	gvnc_write_u16(gvnc, n_encoding - skip_zrle);
+	for (i = 0; i < n_encoding; i++) {
+		if (skip_zrle && encoding[i] == GVNC_ENCODING_ZRLE)
+			continue;
 		gvnc_write_s32(gvnc, encoding[i]);
+	}
 	gvnc_flush(gvnc);
 	return !gvnc_has_error(gvnc);
 }
@@ -951,23 +976,52 @@ static inline uint8_t *gvnc_get_local(st
 		(x * gvnc->local.bpp);
 }
 
-static uint8_t gvnc_swap_8(struct gvnc *gvnc G_GNUC_UNUSED, uint8_t pixel)
+static uint8_t gvnc_swap_img_8(struct gvnc *gvnc G_GNUC_UNUSED, uint8_t pixel)
 {
 	return pixel;
 }
 
-static uint16_t gvnc_swap_16(struct gvnc *gvnc, uint16_t pixel)
+static uint8_t gvnc_swap_rfb_8(struct gvnc *gvnc G_GNUC_UNUSED, uint8_t pixel)
+{
+	return pixel;
+}
+
+/* local host native format -> X server image format */
+static uint16_t gvnc_swap_img_16(struct gvnc *gvnc, uint16_t pixel)
+{
+	if (G_BYTE_ORDER != gvnc->local.byte_order)
+		return  (((pixel >> 8) & 0xFF) << 0) |
+			(((pixel >> 0) & 0xFF) << 8);
+	else
+		return pixel;
+}
+
+/* VNC server RFB  format ->  local host native format */
+static uint16_t gvnc_swap_rfb_16(struct gvnc *gvnc, uint16_t pixel)
 {
-	if (gvnc->fmt.byte_order != gvnc->local.byte_order)
+	if (gvnc->fmt.byte_order != G_BYTE_ORDER)
 		return  (((pixel >> 8) & 0xFF) << 0) |
 			(((pixel >> 0) & 0xFF) << 8);
 	else
 		return pixel;
 }
 
-static uint32_t gvnc_swap_32(struct gvnc *gvnc, uint32_t pixel)
+/* local host native format -> X server image format */
+static uint32_t gvnc_swap_img_32(struct gvnc *gvnc, uint32_t pixel)
+{
+	if (G_BYTE_ORDER != gvnc->local.byte_order)
+		return  (((pixel >> 24) & 0xFF) <<  0) |
+			(((pixel >> 16) & 0xFF) <<  8) |
+			(((pixel >>  8) & 0xFF) << 16) |
+			(((pixel >>  0) & 0xFF) << 24);			
+	else
+		return pixel;
+}
+
+/* VNC server RFB  format ->  local host native format */
+static uint32_t gvnc_swap_rfb_32(struct gvnc *gvnc, uint32_t pixel)
 {
-	if (gvnc->fmt.byte_order != gvnc->local.byte_order)
+	if (gvnc->fmt.byte_order != G_BYTE_ORDER)
 		return  (((pixel >> 24) & 0xFF) <<  0) |
 			(((pixel >> 16) & 0xFF) <<  8) |
 			(((pixel >>  8) & 0xFF) << 16) |
@@ -1206,10 +1260,26 @@ static void gvnc_read_cpixel(struct gvnc
 
 	memset(pixel, 0, bpp);
 
-	if (bpp == 4 && gvnc->fmt.true_color_flag && gvnc->fmt.depth == 24) {
-		bpp = 3;
-		if (gvnc->fmt.byte_order == __BIG_ENDIAN)
-			pixel += 1;
+	if (bpp == 4 && gvnc->fmt.true_color_flag) {
+		int fitsInMSB = ((gvnc->fmt.red_shift > 7) &&
+				 (gvnc->fmt.green_shift > 7) &&
+				 (gvnc->fmt.blue_shift > 7));
+		int fitsInLSB = (((gvnc->fmt.red_max << gvnc->fmt.red_shift) < (1 << 24)) &&
+				 ((gvnc->fmt.green_max << gvnc->fmt.green_shift) < (1 << 24)) &&
+				 ((gvnc->fmt.blue_max << gvnc->fmt.blue_shift) < (1 << 24)));
+
+		/* 
+		 * We need to analyse the shifts to see if they fit in 3 bytes,
+		 * rather than looking at the declared  'depth' for the format
+		 * because despite what the RFB spec says, this is what RealVNC
+		 * server actually does in practice.
+		 */
+		if (fitsInMSB || fitsInLSB) {
+			bpp = 3;
+			if (gvnc->fmt.depth == 24 &&
+			    gvnc->fmt.byte_order == G_BIG_ENDIAN)
+				pixel++;
+		}
 	}
 
 	gvnc_read(gvnc, pixel, bpp);
@@ -2995,7 +3065,8 @@ gboolean gvnc_set_local(struct gvnc *gvn
 	    fb->red_shift == gvnc->fmt.red_shift &&
 	    fb->green_shift == gvnc->fmt.green_shift &&
 	    fb->blue_shift == gvnc->fmt.blue_shift &&
-	    fb->byte_order == gvnc->fmt.byte_order)
+	    fb->byte_order == G_BYTE_ORDER &&
+	    gvnc->fmt.byte_order == G_BYTE_ORDER)
 		gvnc->perfect_match = TRUE;
 	else
 		gvnc->perfect_match = FALSE;
@@ -3024,17 +3095,6 @@ gboolean gvnc_set_local(struct gvnc *gvn
 	gvnc->bls = gvnc->local.blue_shift;
 
 
-	/* This adjusts for server/client endianness mismatch */
-	if (__BYTE_ORDER != gvnc->fmt.byte_order) {
-		gvnc->rrs = gvnc->fmt.bits_per_pixel - gvnc->rrs - (gvnc->fmt.bits_per_pixel - gvnc->fmt.depth);
-		gvnc->grs = gvnc->fmt.bits_per_pixel - gvnc->grs - (gvnc->fmt.bits_per_pixel - gvnc->fmt.depth);
-		gvnc->brs = gvnc->fmt.bits_per_pixel - gvnc->brs - (gvnc->fmt.bits_per_pixel - gvnc->fmt.depth);
-
-		GVNC_DEBUG("Flipped shifts red: %3d, green: %3d, blue: %3d\n",
-			   gvnc->rrs, gvnc->grs, gvnc->brs);
-	}
-
-
 	/* This adjusts for remote having more bpp than local */
 	for (n = gvnc->fmt.red_max; n > gvnc->local.red_mask ; n>>= 1)
 		gvnc->rrs++;
diff -rup gtk-vnc-0.3.4.orig/src/vncdisplay.c gtk-vnc-0.3.4.new/src/vncdisplay.c
--- gtk-vnc-0.3.4.orig/src/vncdisplay.c	2008-04-03 10:10:51.000000000 -0400
+++ gtk-vnc-0.3.4.new/src/vncdisplay.c	2008-04-03 10:13:53.000000000 -0400
@@ -750,9 +750,17 @@ static void setup_gdk_image(VncDisplay *
 	VncDisplayPrivate *priv = obj->priv;
 	GdkVisual *visual;
 
-	visual = gdk_drawable_get_visual(GTK_WIDGET(obj)->window);
+	visual = gdk_screen_get_system_visual(gdk_screen_get_default());
 	
 	priv->image = gdk_image_new(GDK_IMAGE_FASTEST, visual, width, height);
+	GVNC_DEBUG("Visual mask: %3d %3d %3d\n      shift: %3d %3d %3d\n",
+		   visual->red_mask,
+		   visual->green_mask,
+		   visual->blue_mask,
+		   visual->red_shift,
+		   visual->green_shift,
+		   visual->blue_shift);
+
 	priv->fb.red_mask = visual->red_mask >> visual->red_shift;
 	priv->fb.green_mask = visual->green_mask >> visual->green_shift;
 	priv->fb.blue_mask = visual->blue_mask >> visual->blue_shift;