qemu: fix VMware VGA depth calculation error

VMware SVGA presents to the guest with the depth of the host surface it renders
to, and rejects to work if the two sides are mismatched. One problem is that
current VMware VGA may calculate a wrong host depth, and then memcpy from virtual
framebuffer to host surface may trigger segmentation fault. For example, when
launching Qemu in a VNC connection, VMware SVGA thinks depth as '32', however the
actual depth of VNC is '16'. The fault also happens when the host depth is not
32 bit.

Qemu <4b5db3749c5fdba93e1ac0e8748c9a9a1064319f> tempts to fix a similar issue, by
changing from hard-coded 24bit depth to instead query the surface allocator
(e.g. sdl). However it doesn't really work, because the point where query
is invoked is earlier than the point where sdl is initialized. At query time,
qemu uses a default surface allocator which, again, provides another hard-coded
depth value - 32bit. So it happens to make VMware SVGA working on some hosts,
but still fails in others.

To solve this issue, this commit introduces a postcall interface to display
surface, which is walked after surface allocators are actually initialized.
At that point it's then safe to query host depth and present to the guest.

Signed-off-by Kevin Tian <kevin.tian@intel.com>

3 files changed, 121 insertions, 4 deletions

diff --git a/meta/packages/qemu/qemu-0.12.4/qemu-vmware-vga-depth.patch b/meta/packages/qemu/qemu-0.12.4/qemu-vmware-vga-depth.patch
new file mode 100644
index 0000000000..43071868f3
--- /dev/null
+++ b/meta/packages/qemu/qemu-0.12.4/qemu-vmware-vga-depth.patch
@@ -0,0 +1,115 @@
+# fix VMware VGA driver depth calculation error, which may cause segmentation fault
+#
+# ktian1, 06/29/2010
+diff --git a/console.h b/console.h
+index dfc8ae4..05fbf17 100644
+--- a/console.h
++++ b/console.h
+@@ -122,6 +122,12 @@ struct DisplayAllocator {
+     void (*free_displaysurface)(DisplaySurface *surface);
+ };
+ 
++struct DisplayPostCallback {
++    void (*postcall) (void *);
++    void *parm;
++    struct DisplayPostCallback *next;
++};
++
+ struct DisplayState {
+     struct DisplaySurface *surface;
+     void *opaque;
+@@ -129,6 +135,7 @@ struct DisplayState {
+ 
+     struct DisplayAllocator* allocator;
+     struct DisplayChangeListener* listeners;
++    struct DisplayPostCallback* postcalls;
+ 
+     void (*mouse_set)(int x, int y, int on);
+     void (*cursor_define)(int width, int height, int bpp, int hot_x, int hot_y,
+@@ -185,6 +192,12 @@ static inline void register_displaychangelistener(DisplayState *ds, DisplayChang
+     ds->listeners = dcl;
+ }
+ 
++static inline void register_displaypostcallback(DisplayState *ds, DisplayPostCallback *dpc)
++{
++    dpc->next = ds->postcalls;
++    ds->postcalls = dpc;
++}
++
+ static inline void dpy_update(DisplayState *s, int x, int y, int w, int h)
+ {
+     struct DisplayChangeListener *dcl = s->listeners;
+diff --git a/hw/vmware_vga.c b/hw/vmware_vga.c
+index 01bb85b..d73cca6 100644
+--- a/hw/vmware_vga.c
++++ b/hw/vmware_vga.c
+@@ -927,8 +927,9 @@ static void vmsvga_update_display(void *opaque)
+     }
+ }
+ 
+-static void vmsvga_reset(struct vmsvga_state_s *s)
++static void vmsvga_reset(void *parm)
+ {
++    struct vmsvga_state_s *s = (struct vmsvga_state_s *)parm;
+     s->index = 0;
+     s->enable = 0;
+     s->config = 0;
+@@ -1133,6 +1134,8 @@ static const VMStateDescription vmstate_vmware_vga = {
+ 
+ static void vmsvga_init(struct vmsvga_state_s *s, int vga_ram_size)
+ {
++    DisplayPostCallback *dpc;
++
+     s->scratch_size = SVGA_SCRATCH_SIZE;
+     s->scratch = qemu_malloc(s->scratch_size * 4);
+ 
+@@ -1160,7 +1163,10 @@ static void vmsvga_init(struct vmsvga_state_s *s, int vga_ram_size)
+ 
+     rom_add_vga(VGABIOS_FILENAME);
+ 
+-    vmsvga_reset(s);
++    dpc = qemu_mallocz(sizeof(DisplayPostCallback));
++    dpc->postcall = vmsvga_reset;
++    dpc->parm = s;
++    register_displaypostcallback(s->vga.ds, dpc);
+ }
+ 
+ static void pci_vmsvga_map_ioport(PCIDevice *pci_dev, int region_num,
+diff --git a/qemu-common.h b/qemu-common.h
+index a23afbc..19f107a 100644
+--- a/qemu-common.h
++++ b/qemu-common.h
+@@ -198,6 +198,7 @@ typedef struct DisplayState DisplayState;
+ typedef struct DisplayChangeListener DisplayChangeListener;
+ typedef struct DisplaySurface DisplaySurface;
+ typedef struct DisplayAllocator DisplayAllocator;
++typedef struct DisplayPostCallback DisplayPostCallback;
+ typedef struct PixelFormat PixelFormat;
+ typedef struct TextConsole TextConsole;
+ typedef TextConsole QEMUConsole;
+diff --git a/vl.c b/vl.c
+index 39182ea..9a3e9fd 100644
+--- a/vl.c
++++ b/vl.c
+@@ -4863,6 +4863,7 @@ int main(int argc, char **argv, char **envp)
+     char boot_devices[33] = "cad"; /* default to HD->floppy->CD-ROM */
+     DisplayState *ds;
+     DisplayChangeListener *dcl;
++    DisplayPostCallback *dpc;
+     int cyls, heads, secs, translation;
+     QemuOpts *hda_opts = NULL, *opts;
+     int optind;
+@@ -6053,6 +6053,13 @@ int main(int argc, char **argv, char **envp)
+     }
+     dpy_resize(ds);
+ 
++    dpc = ds->postcalls;
++    while (dpc != NULL) {
++        if (dpc->postcall != NULL)
++            dpc->postcall(dpc->parm);
++        dpc = dpc->next;
++    }
++
+     dcl = ds->listeners;
+     while (dcl != NULL) {
+         if (dcl->dpy_refresh != NULL) {
diff --git a/meta/packages/qemu/qemu_0.12.4.bb b/meta/packages/qemu/qemu_0.12.4.bb
index 20cc8e8a5a..7d158bf863 100644
--- a/meta/packages/qemu/qemu_0.12.4.bb
+++ b/meta/packages/qemu/qemu_0.12.4.bb
@@ -1,6 +1,6 @@
 require qemu.inc
 
-PR = "r15"
+PR = "r16"
 
 FILESPATH = "${FILE_DIRNAME}/qemu-${PV}"
 FILESDIR = "${WORKDIR}"
@@ -14,6 +14,7 @@ SRC_URI = "\
     file://fix-nogl.patch \
     file://qemugl-allow-glxcontext-release.patch \
     file://linker-flags.patch \
-    file://init-info.patch"
+    file://init-info.patch \
+    file://qemu-vmware-vga-depth.patch"
 
 S = "${WORKDIR}/qemu-${PV}"
diff --git a/meta/packages/qemu/qemu_git.bb b/meta/packages/qemu/qemu_git.bb
index b1468704ac..00b361063a 100644
--- a/meta/packages/qemu/qemu_git.bb
+++ b/meta/packages/qemu/qemu_git.bb
@@ -1,7 +1,7 @@
 require qemu.inc
 
 PV = "0.12.4"
-PR = "r8"
+PR = "r9"
 
 FILESPATH = "${FILE_DIRNAME}/qemu-${PV}/:${FILE_DIRNAME}/qemu-git/"
 FILESDIR = "${WORKDIR}"
@@ -14,7 +14,8 @@ SRC_URI = "\
     file://fix-dirent.patch \
     file://fix-nogl.patch \
     file://qemugl-allow-glxcontext-release.patch \
-    file://linker-flags.patch"
+    file://linker-flags.patch \
+    file://qemu-vmware-vga-depth.patch"
 
 S = "${WORKDIR}/git"