Fixed drop-shadow not being damaged when the reference buffer is damaged

Helps when the shadow is far from the reference buffer where the
original buffer damage wouldn't reach the shadow node

Author: ErikReider

include/scenefx/types/wlr_scene.h

@@ -170,8 +170,10 @@ struct wlr_scene_shadow {
 	float blur_sigma;
 
 	enum wlr_scene_shadow_type type;
-	// Used to draw a drop-shadow. If NULL, falls-back to a box-shadow
-	struct wlr_scene_buffer *reference_buffer;
+	struct {
+		// Used to draw a drop-shadow. If NULL, falls-back to a box-shadow
+		struct wlr_scene_buffer *reference_buffer;
+	} WLR_PRIVATE;
 
 	struct clipped_region clipped_region;
 };
@@ -253,6 +255,9 @@ struct wlr_scene_buffer {
 		// If is_single_pixel_buffer is set, contains the color of the buffer
 		// as {R, G, B, A} where the max value of each component is UINT32_MAX
 		uint32_t single_pixel_buffer_color[4];
+
+		/** May be NULL. Used to know which shadow's linked to this scene_buffer */
+		struct wlr_scene_shadow *linked_drop_shadow;
 	} WLR_PRIVATE;
 };
 

render/fx_renderer/fx_pass.c

@@ -1163,12 +1163,11 @@ void fx_render_pass_add_drop_shadow(struct fx_gles_render_pass *pass,
 			dst_box.width + blur_sample_size * 2, dst_box.height + blur_sample_size * 2);
 	pixman_region32_intersect(&clip, &clip, tex_options->base.clip);
 
-	pixman_region32_t clip_scaled;
-	pixman_region32_init(&clip_scaled);
-	wlr_region_scale(&clip_scaled, &clip, drop_shadow_downscale);
 	pixman_region32_t clip_extended;
 	pixman_region32_init(&clip_extended);
-	wlr_region_expand(&clip_extended, &clip_scaled, blur_sample_size);
+	// Downsample and extend the region to increase the sample size
+	wlr_region_scale(&clip_extended, &clip, drop_shadow_downscale);
+	wlr_region_expand(&clip_extended, &clip_extended, blur_sample_size);
 
 	fx_framebuffer_bind(effects_buffer);
 
@@ -1186,7 +1185,7 @@ void fx_render_pass_add_drop_shadow(struct fx_gles_render_pass *pass,
 	glDisable(GL_SCISSOR_TEST);
 
 	// Initially Render to the effects buffer
-	tex_options->base.clip = &clip_scaled;
+	tex_options->base.clip = &clip_extended;
 	tex_options->clip_box = NULL;
 
 	scale_box(&tex_options->base.dst_box, drop_shadow_downscale);
@@ -1235,7 +1234,6 @@ void fx_render_pass_add_drop_shadow(struct fx_gles_render_pass *pass,
 	fx_render_pass_add_drop_shadow_final(pass, tex_options, color);
 
 	pixman_region32_fini(&clip_extended);
-	pixman_region32_fini(&clip_scaled);
 	pixman_region32_fini(&clip);
 }
 

types/scene/wlr_scene.c

@@ -46,6 +46,10 @@
 #define SCENE_RECT_SHOULD_BLUR(scene_rect, blur_data) \
 	(scene_rect->backdrop_blur && is_scene_blur_enabled(blur_data) && \
 		scene_rect->color[3] < 1.0f)
+#define SCENE_SHADOW_IS_DROP(scene_shadow) \
+	(scene_shadow->type == WLR_SCENE_SHADOW_TYPE_DROP \
+		&& scene_shadow->reference_buffer \
+		&& scene_shadow->blur_sigma >= 0)
 
 struct wlr_scene_tree *wlr_scene_tree_from_node(struct wlr_scene_node *node) {
 	assert(node->type == WLR_SCENE_NODE_TREE);
@@ -186,6 +190,13 @@ void wlr_scene_node_destroy(struct wlr_scene_node *node) {
 				&scene_tree->children, link) {
 			wlr_scene_node_destroy(child);
 		}
+	} else if (node->type == WLR_SCENE_NODE_SHADOW) {
+		// Remove the drop-shadows referenced buffers link to this node
+		struct wlr_scene_shadow *scene_shadow = wlr_scene_shadow_from_node(node);
+		if (scene_shadow->reference_buffer
+				&& scene_shadow->reference_buffer->linked_drop_shadow == scene_shadow) {
+			scene_shadow->reference_buffer->linked_drop_shadow = NULL;
+		}
 	}
 
 	assert(wl_list_empty(&node->events.destroy.listener_list));
@@ -1039,6 +1050,15 @@ void wlr_scene_shadow_set_reference_buffer(struct wlr_scene_shadow *shadow,
 				"Please change its type before calling this function");
 		return;
 	}
+
+	// Unlink the other drop shadow from the reference buffer if it exists
+	if (shadow->reference_buffer) {
+		shadow->reference_buffer->linked_drop_shadow = NULL;
+	}
+	if (ref_buffer) {
+		ref_buffer->linked_drop_shadow = shadow;
+	}
+
 	if (ref_buffer == shadow->reference_buffer) {
 		return;
 	}
@@ -1244,6 +1264,7 @@ struct wlr_scene_buffer *wlr_scene_buffer_create(struct wlr_scene_tree *parent,
 	scene_buffer->backdrop_blur_optimized = false;
 	scene_buffer->backdrop_blur_ignore_transparent = true;
 	scene_buffer->corners = CORNER_LOCATION_NONE;
+	scene_buffer->linked_drop_shadow = NULL;
 
 	scene_buffer_set_buffer(scene_buffer, buffer);
 	scene_node_update(&scene_buffer->node, NULL);
@@ -1392,6 +1413,28 @@ void wlr_scene_buffer_set_buffer_with_options(struct wlr_scene_buffer *scene_buf
 		pixman_region32_intersect(&output_damage, &output_damage, &cull_region);
 		pixman_region32_fini(&cull_region);
 
+		// Also damage the linked drop-shadow
+		struct wlr_scene_shadow *scene_shadow = scene_buffer->linked_drop_shadow;
+		if (scene_shadow && SCENE_SHADOW_IS_DROP(scene_shadow)) {
+			int shadow_lx, shadow_ly;
+			if (wlr_scene_node_coords(&scene_shadow->node, &shadow_lx, &shadow_ly)) {
+				const int shadow_size = drop_shadow_calc_size(scene_shadow->blur_sigma);
+
+				// Copy the damaged region, translate it to the shadow nodes
+				// position, and add it to the original damage
+				pixman_region32_t shadow_damage;
+				pixman_region32_init(&shadow_damage);
+				pixman_region32_copy(&shadow_damage, &output_damage);
+				pixman_region32_translate(&shadow_damage,
+						((shadow_lx + shadow_size) - lx) * output_scale,
+						((shadow_ly + shadow_size) - ly) * output_scale);
+				wlr_region_expand(&shadow_damage, &shadow_damage,
+						drop_shadow_calc_size(scene_shadow->blur_sigma));
+
+				pixman_region32_union(&output_damage, &output_damage, &shadow_damage);
+			}
+		}
+
 		// Expand the damage when committed to, fixes blur artifacts
 		if (SCENE_BUFFER_SHOULD_BLUR(scene_buffer, &scene->blur_data)) {
 			wlr_region_expand(&output_damage, &output_damage,
@@ -2814,18 +2857,11 @@ static bool scene_output_has_blur(int list_len,
 			break;
 		case WLR_SCENE_NODE_SHADOW:;
 			struct wlr_scene_shadow *scene_shadow = wlr_scene_shadow_from_node(node);
-			if (scene_shadow->type != WLR_SCENE_SHADOW_TYPE_DROP
-					|| !scene_shadow->reference_buffer
-					|| scene_shadow->blur_sigma < 0) {
+			if (!SCENE_SHADOW_IS_DROP(scene_shadow)) {
 				goto fini;
 			}
 			node_blur_region(node, scene_output, &node_region);
-			const int shadow_size = drop_shadow_calc_size(scene_shadow->blur_sigma);
-			// Make sure to re-render the shadow due to it extending past
-			// the buffer
-			wlr_region_expand(&node_region, &node_region, shadow_size);
-
-			blur_size = shadow_size;
+			blur_size = drop_shadow_calc_size(scene_shadow->blur_sigma);
 			break;
 		default:
 			goto fini;