Bug fix chjj#302: Use more EWMH-compliant way to determine frame extents

Bug fix: Select the larger of the X window border width and
_NET_FRAME_EXTENTS, instead of adding them together, when determining
the frame extents. (Supposedly) more EWMH compliant, as EWMH
_NET_FRAME_EXTENTS should include the X window border width already. We
did not choose to completely rely on _NET_FRAME_EXTENTS to remove the
need to keep track of whether there is a _NET_FRAME_EXTENTS, and
(hopefully) take care of some cases when it is wrong. The commit should
fix awesomeWM/awesome#425. Thanks to psychon for the information. (chjj#302)

The commit fixes the bug in win_get_region_noframe() that the X window
border width is not considered when determining the width/height of the
window without frame, as well.

src/compton.c

@@ -901,12 +901,13 @@ win_get_region(session_t *ps, win *w, bool use_offset) {
  */
 static XserverRegion
 win_get_region_noframe(session_t *ps, win *w, bool use_offset) {
+  const margin_t extents = win_calc_frame_extents(ps, w);
   XRectangle r;
 
-  r.x = (use_offset ? w->a.x: 0) + w->a.border_width + w->frame_extents.left;
-  r.y = (use_offset ? w->a.y: 0) + w->a.border_width + w->frame_extents.top;
-  r.width = max_i(w->a.width - w->frame_extents.left - w->frame_extents.right, 0);
-  r.height = max_i(w->a.height - w->frame_extents.top - w->frame_extents.bottom, 0);
+  r.x = (use_offset ? w->a.x: 0) + extents.left;
+  r.y = (use_offset ? w->a.y: 0) + extents.top;
+  r.width = max_i(w->a.width - extents.left - extents.right, 0);
+  r.height = max_i(w->a.height - extents.top - extents.bottom, 0);
 
   if (r.width > 0 && r.height > 0)
     return XFixesCreateRegion(ps->dpy, &r, 1);
@@ -1632,10 +1633,11 @@ win_paint_win(session_t *ps, win *w, XserverRegion reg_paint,
   }
   else {
     // Painting parameters
-    const int t = w->a.border_width + w->frame_extents.top;
-    const int l = w->a.border_width + w->frame_extents.left;
-    const int b = w->a.border_width + w->frame_extents.bottom;
-    const int r = w->a.border_width + w->frame_extents.right;
+    const margin_t extents = win_calc_frame_extents(ps, w);
+    const int t = extents.top;
+    const int l = extents.left;
+    const int b = extents.bottom;
+    const int r = extents.right;
 
 #define COMP_BDR(cx, cy, cwid, chei) \
     win_render(ps, w, (cx), (cy), (cwid), (chei), w->frame_opacity, \

src/compton.h

@@ -507,6 +507,20 @@ win_has_frame(const win *w) {
     || w->frame_extents.right || w->frame_extents.bottom;
 }
 
+/**
+ * Calculate the extents of the frame of the given window based on EWMH
+ * _NET_FRAME_EXTENTS and the X window border width.
+ */
+static inline margin_t __attribute__((pure))
+win_calc_frame_extents(session_t *ps, const win *w) {
+  margin_t result = w->frame_extents;
+  result.top = max_i(result.top, w->a.border_width);
+  result.left = max_i(result.left, w->a.border_width);
+  result.bottom = max_i(result.bottom, w->a.border_width);
+  result.right = max_i(result.right, w->a.border_width);
+  return result;
+}
+
 static inline void
 wid_set_opacity_prop(session_t *ps, Window wid, opacity_t val) {
   const unsigned long v = val;