DO NOT MERGE: drm_hwcomposer: Fix protected layer hole punch

The old code didn't dereference the layer index from the
bitmask which caused it to subtract layers it shouldn't
have. In order to do this right, we need to look at the
layer index corresponding to the bit in the id_set to
ensure we're excluding the correct layers.

BUG=None
TEST=Precomposition doesn't draw over dedicated layers

Change-Id: I8531e1ef3b2beb4674041145e2b38ce4b3dbe346
Signed-off-by: Sean Paul <seanpaul@chromium.org>

2 files changed, 6 insertions, 12 deletions

diff --git a/drmdisplaycomposition.cpp b/drmdisplaycomposition.cpp
index d5e97b4..2c5ef09 100644
--- a/drmdisplaycomposition.cpp
+++ b/drmdisplaycomposition.cpp
@@ -238,15 +238,17 @@ static void SeparateLayers(DrmHwcLayer *layers, size_t *used_layers,
       if (!(protected_intersect & (1 << (i + num_exclude_rects))))
         continue;
 
-      region.id_set.subtract(layer_offset, layer_offset + protected_layers[i]);
+      for (size_t j = 0; j < num_used_layers; ++j) {
+        if (used_layers[j] < protected_layers[i])
+          region.id_set.subtract(j + layer_offset);
+      }
     }
-    if (region.id_set.isEmpty())
+    if (!(region.id_set.getBits() >> layer_offset))
       continue;
 
     regions.emplace_back(DrmCompositionRegion{
         region.rect,
-        SetBitsToVector(region.id_set.getBits() >> layer_offset,
-                        used_layers)});
+        SetBitsToVector(region.id_set.getBits() >> layer_offset, used_layers)});
   }
 }
 
diff --git a/separate_rects.h b/separate_rects.h
index cb46ecb..de8b660 100644
--- a/separate_rects.h
+++ b/separate_rects.h
@@ -105,14 +105,6 @@ struct IdSet {
     bitset &= ~(((TUInt)1) << id);
   }
 
-  void subtract(TId start, TId end) {
-    if (start > end)
-      return;
-    TId start_mask = (1 << start) - 1;
-    TId end_mask = (1 << end) - 1;
-    bitset &= ~(start_mask ^ end_mask);
-  }
-
   bool isEmpty() const {
     return bitset == 0;
   }