Bug 1590832 - fix Wayland dmabuf texture crash and rendering on Radeo…

…n r=stransky

1. Use GBM map/unmap API correctly

The documentation of gbm_bo_unmap describes the second argument as
"map_data opaque ptr returned from prior gbm_bo_map",
which refers to the pointer written to the map_data location by gbm_bo_map,
not the pointer *returned* by gbm_bo_map.

This fixes crashes with widget.wayland-dmabuf-textures.enabled
on the Mesa RadeonSI driver.

2. Set modifier to invalid on the fallback code path

This fixes fCreateImage failure, allowing actual rendering
to happen.

(mBufferModifier was not set to DRM_FORMAT_MOD_INVALID,
so unsupported attributes were added to the CreateImage call)

Differential Revision: https://phabricator.services.mozilla.com/D65239

widget/gtk/WaylandDMABufSurface.cpp

@@ -245,6 +245,8 @@ bool WaylandDMABufSurfaceRGBA::Create(int aWidth, int aHeight,
     mGbmBufferObject =
         nsGbmLib::Create(display->GetGbmDevice(), mWidth, mHeight,
                          mGmbFormat->mFormat, mGbmBufferFlags);
+
+    mBufferModifier = DRM_FORMAT_MOD_INVALID;
   }
 
   if (!mGbmBufferObject) {
@@ -532,10 +534,10 @@ void* WaylandDMABufSurfaceRGBA::MapInternal(uint32_t aX, uint32_t aY,
     NS_WARNING("We should not map dmabuf surfaces with modifiers!");
   }
 
-  void* map_data = nullptr;
   mMappedRegionStride = 0;
-  mMappedRegion = nsGbmLib::Map(mGbmBufferObject, aX, aY, aWidth, aHeight,
-                                aGbmFlags, &mMappedRegionStride, &map_data);
+  mMappedRegion =
+      nsGbmLib::Map(mGbmBufferObject, aX, aY, aWidth, aHeight, aGbmFlags,
+                    &mMappedRegionStride, &mMappedRegionData);
   if (aStride) {
     *aStride = mMappedRegionStride;
   }
@@ -565,8 +567,9 @@ void* WaylandDMABufSurfaceRGBA::Map(uint32_t* aStride) {
 
 void WaylandDMABufSurfaceRGBA::Unmap() {
   if (mMappedRegion) {
-    nsGbmLib::Unmap(mGbmBufferObject, mMappedRegion);
+    nsGbmLib::Unmap(mGbmBufferObject, mMappedRegionData);
     mMappedRegion = nullptr;
+    mMappedRegionData = nullptr;
     mMappedRegionStride = 0;
   }
 }

widget/gtk/WaylandDMABufSurface.h

@@ -184,6 +184,7 @@ class WaylandDMABufSurfaceRGBA : public WaylandDMABufSurface {
 
   wl_buffer* mWLBuffer;
   void* mMappedRegion;
+  void* mMappedRegionData;
   uint32_t mMappedRegionStride;
 
   struct gbm_bo* mGbmBufferObject;