Add GLFW_TRANSPARENT and X11 implementation

This is a squashed extract of several commits, minimally edited to
ensure it compiles.

Related to #197.
Related to #715.

examples/gears.c

@@ -172,6 +172,7 @@ static GLfloat angle = 0.f;
 /* OpenGL draw function & timing */
 static void draw(void)
 {
+  glClearColor(0., 0., 0., 0.);
   glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
 
   glPushMatrix();
@@ -311,6 +312,8 @@ int main(int argc, char *argv[])
     }
 
     glfwWindowHint(GLFW_DEPTH_BITS, 16);
+    glfwWindowHint(GLFW_ALPHA_BITS, 8);
+    glfwWindowHint(GLFW_TRANSPARENT, GLFW_TRUE);
 
     window = glfwCreateWindow( 300, 300, "Gears", NULL, NULL );
     if (!window)

include/GLFW/glfw3.h

@@ -787,6 +787,7 @@ extern "C" {
  *  Cursor centering [window hint](@ref GLFW_CENTER_CURSOR_hint).
  */
 #define GLFW_CENTER_CURSOR          0x00020009
+#define GLFW_TRANSPARENT            0x0002000A
 
 /*! @brief Framebuffer bit depth hint.
  *

src/egl_context.c

@@ -87,13 +87,21 @@ static int getEGLConfigAttrib(EGLConfig config, int attrib)
 //
 static GLFWbool chooseEGLConfig(const _GLFWctxconfig* ctxconfig,
                                 const _GLFWfbconfig* desired,
-                                EGLConfig* result)
+                                EGLConfig* result,
+                GLFWbool findTransparent)
 {
     EGLConfig* nativeConfigs;
     _GLFWfbconfig* usableConfigs;
     const _GLFWfbconfig* closest;
     int i, nativeCount, usableCount;
 
+#if defined(_GLFW_X11)
+    XVisualInfo visualTemplate = {0};
+    if ( !(_glfw.xrender.major || _glfw.xrender.minor) ) {
+        findTransparent = GLFW_FALSE;
+    }
+#endif // _GLFW_X11
+
     eglGetConfigs(_glfw.egl.display, NULL, 0, &nativeCount);
     if (!nativeCount)
     {
@@ -107,6 +115,7 @@ static GLFWbool chooseEGLConfig(const _GLFWctxconfig* ctxconfig,
     usableConfigs = calloc(nativeCount, sizeof(_GLFWfbconfig));
     usableCount = 0;
 
+selectionloop:
     for (i = 0;  i < nativeCount;  i++)
     {
         const EGLConfig n = nativeConfigs[i];
@@ -122,8 +131,31 @@ static GLFWbool chooseEGLConfig(const _GLFWctxconfig* ctxconfig,
 
 #if defined(_GLFW_X11)
         // Only consider EGLConfigs with associated Visuals
-        if (!getEGLConfigAttrib(n, EGL_NATIVE_VISUAL_ID))
+    visualTemplate.visualid = getEGLConfigAttrib(n, EGL_NATIVE_VISUAL_ID);
+        if (!visualTemplate.visualid)
+            continue;
+
+    if( findTransparent ) {
+        int n_vi;
+            XVisualInfo *visualinfo;
+            XRenderPictFormat *pictFormat;
+
+        visualinfo = XGetVisualInfo(_glfw.x11.display, VisualIDMask, &visualTemplate, &n_vi);
+        if (!visualinfo)
+            continue;
+
+            pictFormat = XRenderFindVisualFormat(_glfw.x11.display, visualinfo->visual);
+            if( !pictFormat ) {
+            XFree( visualinfo );
+            continue;
+        }
+
+            if( !pictFormat->direct.alphaMask ) {
+            XFree( visualinfo );
             continue;
+        }
+        XFree( visualinfo );
+        }
 #endif // _GLFW_X11
 
         if (ctxconfig->client == GLFW_OPENGL_ES_API)
@@ -159,6 +191,12 @@ static GLFWbool chooseEGLConfig(const _GLFWctxconfig* ctxconfig,
         u->handle = (uintptr_t) n;
         usableCount++;
     }
+    // reiterate the selection loop without looking for transparency supporting
+    // formats if no matchig FB configs for a transparent window were found. 
+    if( findTransparent && !usableCount ) {
+        findTransparent = GLFW_FALSE;
+    goto selectionloop;
+    }
 
     closest = _glfwChooseFBConfig(desired, usableConfigs, usableCount);
     if (closest)
@@ -455,7 +493,7 @@ GLFWbool _glfwCreateContextEGL(_GLFWwindow* window,
     if (ctxconfig->share)
         share = ctxconfig->share->context.egl.handle;
 
-    if (!chooseEGLConfig(ctxconfig, fbconfig, &config))
+    if (!chooseEGLConfig(ctxconfig, fbconfig, &config, window->transparent))
     {
         _glfwInputError(GLFW_FORMAT_UNAVAILABLE,
                         "EGL: Failed to find a suitable EGLConfig");
@@ -689,7 +727,8 @@ GLFWbool _glfwCreateContextEGL(_GLFWwindow* window,
 // Returns the Visual and depth of the chosen EGLConfig
 //
 #if defined(_GLFW_X11)
-GLFWbool _glfwChooseVisualEGL(const _GLFWctxconfig* ctxconfig,
+GLFWbool _glfwChooseVisualEGL(const _GLFWwndconfig* wndconfig,
+                              const _GLFWctxconfig* ctxconfig,
                               const _GLFWfbconfig* fbconfig,
                               Visual** visual, int* depth)
 {
@@ -699,7 +738,7 @@ GLFWbool _glfwChooseVisualEGL(const _GLFWctxconfig* ctxconfig,
     EGLint visualID = 0, count = 0;
     const long vimask = VisualScreenMask | VisualIDMask;
 
-    if (!chooseEGLConfig(ctxconfig, fbconfig, &native))
+    if (!chooseEGLConfig(ctxconfig, fbconfig, &native, wndconfig->transparent))
     {
         _glfwInputError(GLFW_FORMAT_UNAVAILABLE,
                         "EGL: Failed to find a suitable EGLConfig");

src/egl_context.h

@@ -211,7 +211,8 @@ GLFWbool _glfwCreateContextEGL(_GLFWwindow* window,
                                const _GLFWctxconfig* ctxconfig,
                                const _GLFWfbconfig* fbconfig);
 #if defined(_GLFW_X11)
-GLFWbool _glfwChooseVisualEGL(const _GLFWctxconfig* ctxconfig,
+GLFWbool _glfwChooseVisualEGL(const _GLFWwndconfig* wndconfig,
+                              const _GLFWctxconfig* ctxconfig,
                               const _GLFWfbconfig* fbconfig,
                               Visual** visual, int* depth);
 #endif /*_GLFW_X11*/

src/glx_context.c

@@ -47,7 +47,10 @@ static int getGLXFBConfigAttrib(GLXFBConfig fbconfig, int attrib)
 
 // Return the GLXFBConfig most closely matching the specified hints
 //
-static GLFWbool chooseGLXFBConfig(const _GLFWfbconfig* desired, GLXFBConfig* result)
+static GLFWbool chooseGLXFBConfig(
+    const _GLFWfbconfig* desired,
+    GLXFBConfig* result,
+    GLFWbool findTransparent)
 {
     GLXFBConfig* nativeConfigs;
     _GLFWfbconfig* usableConfigs;
@@ -56,6 +59,10 @@ static GLFWbool chooseGLXFBConfig(const _GLFWfbconfig* desired, GLXFBConfig* res
     const char* vendor;
     GLFWbool trustWindowBit = GLFW_TRUE;
 
+    if ( !(_glfw.xrender.major || _glfw.xrender.minor) ) {
+        findTransparent = GLFW_FALSE;
+    }
+
     // HACK: This is a (hopefully temporary) workaround for Chromium
     //       (VirtualBox GL) not setting the window bit on any GLXFBConfigs
     vendor = glXGetClientString(_glfw.x11.display, GLX_VENDOR);
@@ -73,6 +80,7 @@ static GLFWbool chooseGLXFBConfig(const _GLFWfbconfig* desired, GLXFBConfig* res
     usableConfigs = calloc(nativeCount, sizeof(_GLFWfbconfig));
     usableCount = 0;
 
+selectionloop:
     for (i = 0;  i < nativeCount;  i++)
     {
         const GLXFBConfig n = nativeConfigs[i];
@@ -89,6 +97,27 @@ static GLFWbool chooseGLXFBConfig(const _GLFWfbconfig* desired, GLXFBConfig* res
                 continue;
         }
 
+    if( findTransparent ) {
+            XVisualInfo *visualinfo;
+            XRenderPictFormat *pictFormat;
+
+        visualinfo = glXGetVisualFromFBConfig(_glfw.x11.display, n);
+        if (!visualinfo)
+            continue;
+
+            pictFormat = XRenderFindVisualFormat(_glfw.x11.display, visualinfo->visual);
+            if( !pictFormat ) {
+            XFree( visualinfo );
+            continue;
+        }
+
+            if( !pictFormat->direct.alphaMask ) {
+            XFree( visualinfo );
+            continue;
+        }
+        XFree( visualinfo );
+        }
+
         u->redBits = getGLXFBConfigAttrib(n, GLX_RED_SIZE);
         u->greenBits = getGLXFBConfigAttrib(n, GLX_GREEN_SIZE);
         u->blueBits = getGLXFBConfigAttrib(n, GLX_BLUE_SIZE);
@@ -118,6 +147,12 @@ static GLFWbool chooseGLXFBConfig(const _GLFWfbconfig* desired, GLXFBConfig* res
         u->handle = (uintptr_t) n;
         usableCount++;
     }
+    // reiterate the selection loop without looking for transparency supporting
+    // formats if no matchig FB configs for a transparent window were found. 
+    if( findTransparent && !usableCount ) {
+        findTransparent = GLFW_FALSE;
+    goto selectionloop;
+    }
 
     closest = _glfwChooseFBConfig(desired, usableConfigs, usableCount);
     if (closest)
@@ -442,7 +477,7 @@ GLFWbool _glfwCreateContextGLX(_GLFWwindow* window,
     if (ctxconfig->share)
         share = ctxconfig->share->context.glx.handle;
 
-    if (!chooseGLXFBConfig(fbconfig, &native))
+    if (!chooseGLXFBConfig(fbconfig, &native, window->transparent))
     {
         _glfwInputError(GLFW_FORMAT_UNAVAILABLE,
                         "GLX: Failed to find a suitable GLXFBConfig");
@@ -622,14 +657,15 @@ GLFWbool _glfwCreateContextGLX(_GLFWwindow* window,
 
 // Returns the Visual and depth of the chosen GLXFBConfig
 //
-GLFWbool _glfwChooseVisualGLX(const _GLFWctxconfig* ctxconfig,
+GLFWbool _glfwChooseVisualGLX(const _GLFWwndconfig* wndconfig,
+                  const _GLFWctxconfig* ctxconfig,
                               const _GLFWfbconfig* fbconfig,
                               Visual** visual, int* depth)
 {
     GLXFBConfig native;
     XVisualInfo* result;
 
-    if (!chooseGLXFBConfig(fbconfig, &native))
+    if (!chooseGLXFBConfig(fbconfig, &native, wndconfig->transparent))
     {
         _glfwInputError(GLFW_FORMAT_UNAVAILABLE,
                         "GLX: Failed to find a suitable GLXFBConfig");

src/glx_context.h

@@ -168,14 +168,14 @@ typedef struct _GLFWlibraryGLX
 
 } _GLFWlibraryGLX;
 
-
 GLFWbool _glfwInitGLX(void);
 void _glfwTerminateGLX(void);
 GLFWbool _glfwCreateContextGLX(_GLFWwindow* window,
                                const _GLFWctxconfig* ctxconfig,
                                const _GLFWfbconfig* fbconfig);
 void _glfwDestroyContextGLX(_GLFWwindow* window);
-GLFWbool _glfwChooseVisualGLX(const _GLFWctxconfig* ctxconfig,
+GLFWbool _glfwChooseVisualGLX(const _GLFWwndconfig* wndconfig,
+                              const _GLFWctxconfig* ctxconfig,
                               const _GLFWfbconfig* fbconfig,
                               Visual** visual, int* depth);
 

src/internal.h

@@ -299,6 +299,7 @@ struct _GLFWwndconfig
     GLFWbool      resizable;
     GLFWbool      visible;
     GLFWbool      decorated;
+    GLFWbool      transparent;
     GLFWbool      focused;
     GLFWbool      autoIconify;
     GLFWbool      floating;
@@ -402,6 +403,7 @@ struct _GLFWwindow
     // Window settings and state
     GLFWbool            resizable;
     GLFWbool            decorated;
+    GLFWbool            transparent;
     GLFWbool            autoIconify;
     GLFWbool            floating;
     GLFWbool            shouldClose;

src/window.c

@@ -180,6 +180,7 @@ GLFWAPI GLFWwindow* glfwCreateWindow(int width, int height,
     window->monitor     = (_GLFWmonitor*) monitor;
     window->resizable   = wndconfig.resizable;
     window->decorated   = wndconfig.decorated;
+    window->transparent = wndconfig.transparent;
     window->autoIconify = wndconfig.autoIconify;
     window->floating    = wndconfig.floating;
     window->cursorMode  = GLFW_CURSOR_NORMAL;
@@ -249,6 +250,7 @@ void glfwDefaultWindowHints(void)
     _glfw.hints.window.resizable   = GLFW_TRUE;
     _glfw.hints.window.visible     = GLFW_TRUE;
     _glfw.hints.window.decorated   = GLFW_TRUE;
+    _glfw.hints.window.transparent = GLFW_FALSE;
     _glfw.hints.window.focused     = GLFW_TRUE;
     _glfw.hints.window.autoIconify = GLFW_TRUE;
 
@@ -327,6 +329,9 @@ GLFWAPI void glfwWindowHint(int hint, int value)
         case GLFW_DECORATED:
             _glfw.hints.window.decorated = value ? GLFW_TRUE : GLFW_FALSE;
             return;
+        case GLFW_TRANSPARENT:
+            _glfw.hints.window.transparent = value ? GLFW_TRUE : GLFW_FALSE;
+            return;
         case GLFW_FOCUSED:
             _glfw.hints.window.focused = value ? GLFW_TRUE : GLFW_FALSE;
             return;
@@ -728,6 +733,8 @@ GLFWAPI int glfwGetWindowAttrib(GLFWwindow* handle, int attrib)
             return window->resizable;
         case GLFW_DECORATED:
             return window->decorated;
+        case GLFW_TRANSPARENT:
+            return window->transparent;
         case GLFW_FLOATING:
             return window->floating;
         case GLFW_AUTO_ICONIFY:

src/x11_init.c

@@ -717,6 +717,55 @@ static GLFWbool initExtensions(void)
     _glfw.x11.MOTIF_WM_HINTS =
         XInternAtom(_glfw.x11.display, "_MOTIF_WM_HINTS", False);
 
+    int i;
+    const char* sonames_xrender[] =
+    {
+#if defined(__CYGWIN__)
+        "libXrender-1.so",
+#else
+        "libXrender.so.1",
+        "libXrender.so",
+#endif
+        NULL
+    };
+
+    // Xrender support is optional and not a requirement for GLX/EGL
+    // to work. Xrender is required for selecting a FB config that
+    // supports a picture format with an alpha mask, which in turn
+    // is required for transparent windows. I Xrender is not supported
+    // the GLFW_TRANSPARENT window hint is ignored.
+    for (i = 0;  sonames_xrender[i];  i++)
+    {
+        _glfw.xrender.handle = dlopen(sonames_xrender[i], RTLD_LAZY | RTLD_GLOBAL);
+        if (_glfw.xrender.handle)
+            break;
+    }
+    _glfw.xrender.errorBase = 0;
+    _glfw.xrender.eventBase = 0;
+    _glfw.xrender.major = 0;
+    _glfw.xrender.minor = 0;
+    if (_glfw.xrender.handle) do {
+        int errorBase, eventBase, major, minor;
+        _glfw.xrender.QueryExtension =
+            dlsym(_glfw.xrender.handle, "XRenderQueryExtension");
+        _glfw.xrender.QueryVersion =
+            dlsym(_glfw.xrender.handle, "XRenderQueryVersion");
+        _glfw.xrender.FindVisualFormat =
+            dlsym(_glfw.xrender.handle, "XRenderFindVisualFormat");
+
+        if ( !XRenderQueryExtension(_glfw.x11.display, &errorBase, &eventBase)) {
+        break;
+    }
+        if ( !XRenderQueryVersion(_glfw.x11.display, &major, &minor)) {
+        break;
+    }
+
+        _glfw.xrender.errorBase = errorBase;
+        _glfw.xrender.eventBase = eventBase;
+        _glfw.xrender.major = major;
+        _glfw.xrender.minor = minor;
+    } while(0);
+
     return GLFW_TRUE;
 }
 

src/x11_platform.h

@@ -162,10 +162,20 @@ typedef VkBool32 (APIENTRY *PFN_vkGetPhysicalDeviceXcbPresentationSupportKHR)(Vk
 #define _GLFW_EGL_NATIVE_DISPLAY ((EGLNativeDisplayType) _glfw.x11.display)
 
 #define _GLFW_PLATFORM_WINDOW_STATE         _GLFWwindowX11  x11
-#define _GLFW_PLATFORM_LIBRARY_WINDOW_STATE _GLFWlibraryX11 x11
+#define _GLFW_PLATFORM_LIBRARY_WINDOW_STATE _GLFWlibraryX11 x11 ; _GLFWlibraryXrender xrender
 #define _GLFW_PLATFORM_MONITOR_STATE        _GLFWmonitorX11 x11
 #define _GLFW_PLATFORM_CURSOR_STATE         _GLFWcursorX11  x11
 
+// libXrender.so function pointer typedefs
+typedef Bool (*PFNXRENDERQUERYEXTENSIONPROC)(Display*,int*,int*);
+typedef Status (*PFNXRENDERQUERYVERSIONPROC)(Display*dpy,int*,int*);
+typedef XRenderPictFormat* (*PFNXRENDERFINDVISUALFORMATPROC)(Display*,Visual const *);
+
+// libXrender.so function identifier overlays
+#define XRenderQueryExtension   _glfw.xrender.QueryExtension
+#define XRenderQueryVersion     _glfw.xrender.QueryVersion
+#define XRenderFindVisualFormat _glfw.xrender.FindVisualFormat
+
 
 // X11-specific per-window data
 //
@@ -375,6 +385,22 @@ typedef struct _GLFWlibraryX11
 
 } _GLFWlibraryX11;
 
+// Xrender-specific global data
+typedef struct _GLFWlibraryXrender
+{
+    int             major, minor;
+    int             eventBase;
+    int             errorBase;
+
+    // dlopen handle for libGL.so.1
+    void*           handle;
+
+    // Xrender functions (subset required for transparent window)
+    PFNXRENDERQUERYEXTENSIONPROC    QueryExtension;
+    PFNXRENDERQUERYVERSIONPROC      QueryVersion;
+    PFNXRENDERFINDVISUALFORMATPROC  FindVisualFormat;
+} _GLFWlibraryXrender;
+
 // X11-specific per-monitor data
 //
 typedef struct _GLFWmonitorX11

src/x11_window.c

@@ -1905,14 +1905,14 @@ int _glfwPlatformCreateWindow(_GLFWwindow* window,
         {
             if (!_glfwInitGLX())
                 return GLFW_FALSE;
-            if (!_glfwChooseVisualGLX(ctxconfig, fbconfig, &visual, &depth))
+            if (!_glfwChooseVisualGLX(wndconfig, ctxconfig, fbconfig, &visual, &depth))
                 return GLFW_FALSE;
         }
         else if (ctxconfig->source == GLFW_EGL_CONTEXT_API)
         {
             if (!_glfwInitEGL())
                 return GLFW_FALSE;
-            if (!_glfwChooseVisualEGL(ctxconfig, fbconfig, &visual, &depth))
+            if (!_glfwChooseVisualEGL(wndconfig, ctxconfig, fbconfig, &visual, &depth))
                 return GLFW_FALSE;
         }
         else if (ctxconfig->source == GLFW_OSMESA_CONTEXT_API)