Remove bit-packing for leaf flags: Use the fact that nodes with primi…

…tive_count > 0 are leaves

include/bvh/binned_sah_builder.hpp

@@ -125,7 +125,6 @@ class BinnedSahBuildTask : public TopDownBuildTask {
         auto make_leaf = [] (typename Bvh::Node& node, size_t begin, size_t end) {
             node.first_child_or_primitive = begin;
             node.primitive_count          = end - begin;
-            node.is_leaf                  = true;
         };
 
         if (item.work_size() <= 1 || item.depth >= builder.max_depth) {
@@ -210,7 +209,6 @@ class BinnedSahBuildTask : public TopDownBuildTask {
             auto& right = bvh.nodes[first_child + 1];
             node.first_child_or_primitive = first_child;
             node.primitive_count          = 0;
-            node.is_leaf                  = false;
 
             // Compute the bounding boxes of each node
             auto& bins = bins_per_axis[best_axis];

include/bvh/bottom_up_algorithm.hpp

@@ -36,7 +36,7 @@ class BottomUpAlgorithm {
         #pragma omp parallel for
         for (size_t i = 0; i < bvh.node_count; i++) {
             auto& node = bvh.nodes[i];
-            if (node.is_leaf)
+            if (node.is_leaf())
                 continue;
             auto first_child = node.first_child_or_primitive;
             parents[first_child + 0] = i;
@@ -65,7 +65,7 @@ class BottomUpAlgorithm {
         #pragma omp for
         for (size_t i = 1; i < bvh.node_count; ++i) {
             // Only process leaves
-            if (MaintainChildIndices ? children[i] != 0 : !bvh.nodes[i].is_leaf)
+            if (MaintainChildIndices ? children[i] != 0 : !bvh.nodes[i].is_leaf())
                 continue;
 
             process_leaf(i);

include/bvh/bvh.hpp

@@ -24,10 +24,11 @@ struct Bvh {
     // single precision and 64 bytes in double precision.
     struct Node {
         Scalar bounds[6];
-        bool is_leaf : 1;
-        IndexType primitive_count : sizeof(IndexType) * CHAR_BIT - 1;
+        IndexType primitive_count;
         IndexType first_child_or_primitive;
 
+        bool is_leaf() const { return primitive_count != 0; }
+
         /// Accessor to simplify the manipulation of the bounding box of a node.
         /// This type is convertible to a `BoundingBox`.
         struct BoundingBoxProxy {

include/bvh/heuristic_primitive_splitter.hpp

@@ -140,7 +140,7 @@ class HeuristicPrimitiveSplitter {
         #pragma omp parallel for
         for (size_t i = 0; i < bvh.node_count; ++i) {
             auto& node = bvh.nodes[i];
-            if (node.is_leaf) {
+            if (node.is_leaf()) {
                 auto begin = bvh.primitive_indices.get() + node.first_child_or_primitive;
                 auto end   = begin + node.primitive_count;
                 std::transform(begin, end, begin, [&] (size_t i) { return original_indices[i]; });

include/bvh/leaf_collapser.hpp

@@ -34,7 +34,7 @@ class LeafCollapser : public SahBasedAlgorithm<Bvh>, public BottomUpAlgorithm<Bv
     {}
 
     void collapse() {
-        if (bvh__unlikely(bvh.nodes[0].is_leaf))
+        if (bvh__unlikely(bvh.nodes[0].is_leaf()))
             return;
 
         std::unique_ptr<size_t[]> primitive_indices_copy;
@@ -52,7 +52,7 @@ class LeafCollapser : public SahBasedAlgorithm<Bvh>, public BottomUpAlgorithm<Bv
                 [&] (size_t i) { primitive_counts[i] = bvh.nodes[i].primitive_count; },
                 [&] (size_t i) {
                     auto& node = bvh.nodes[i];
-                    assert(!node.is_leaf);
+                    assert(!node.is_leaf());
                     auto first_child  = node.first_child_or_primitive;
                     auto& left_child  = bvh.nodes[first_child + 0];
                     auto& right_child = bvh.nodes[first_child + 1];
@@ -63,15 +63,15 @@ class LeafCollapser : public SahBasedAlgorithm<Bvh>, public BottomUpAlgorithm<Bv
                     primitive_counts[i] = total_primitive_count;
 
                     // Compute the cost of collapsing this node when both children are leaves
-                    if (left_child.is_leaf && right_child.is_leaf) {
+                    if (left_child.is_leaf() && right_child.is_leaf()) {
                         auto collapse_cost =
                             node.bounding_box_proxy().to_bounding_box().half_area() * (Scalar(total_primitive_count) - traversal_cost);
                         auto base_cost =
                             left_child .bounding_box_proxy().to_bounding_box().half_area() * left_primitive_count +
                             right_child.bounding_box_proxy().to_bounding_box().half_area() * right_primitive_count;
                         if (collapse_cost <= base_cost) {
                             node_index[(first_child + 1) / 2] = 0;
-                            node.is_leaf = true;
+                            node.primitive_count = total_primitive_count;
                             return;
                         }
                     }
@@ -92,7 +92,7 @@ class LeafCollapser : public SahBasedAlgorithm<Bvh>, public BottomUpAlgorithm<Bv
 
             #pragma omp for
             for (size_t i = 1; i < bvh.node_count; i++) {
-                if (!bvh.nodes[i].is_leaf || node_index[(i - 1) / 2] == node_index[(i + 1) / 2])
+                if (!bvh.nodes[i].is_leaf() || node_index[(i - 1) / 2] == node_index[(i + 1) / 2])
                     continue;
 
                 // Find the index of the first primitive in this subtree
@@ -135,10 +135,10 @@ class LeafCollapser : public SahBasedAlgorithm<Bvh>, public BottomUpAlgorithm<Bv
                     continue;
                 nodes_copy[j + 0] = bvh.nodes[i + 0];
                 nodes_copy[j + 1] = bvh.nodes[i + 1];
-                if (!bvh.nodes[i + 0].is_leaf)
+                if (!bvh.nodes[i + 0].is_leaf())
                     nodes_copy[j + 0].first_child_or_primitive =
                         node_index[(bvh.nodes[i + 0].first_child_or_primitive - 1) / 2];
-                if (!bvh.nodes[i + 1].is_leaf)
+                if (!bvh.nodes[i + 1].is_leaf())
                     nodes_copy[j + 1].first_child_or_primitive =
                         node_index[(bvh.nodes[i + 1].first_child_or_primitive - 1) / 2];
             }

include/bvh/linear_bvh_builder.hpp

@@ -87,7 +87,7 @@ class LinearBvhBuilder : public MortonCodeBasedBuilder<Bvh, Morton> {
                         .bounding_box_proxy()
                         .to_bounding_box()
                         .extend(input_nodes[i + 1].bounding_box_proxy());
-                    unmerged_node.is_leaf = false;
+                    unmerged_node.primitive_count = 0;
                     unmerged_node.first_child_or_primitive = first_child;
                     output_nodes[first_child + 0] = input_nodes[i + 0];
                     output_nodes[first_child + 1] = input_nodes[i + 1];
@@ -150,7 +150,6 @@ class LinearBvhBuilder : public MortonCodeBasedBuilder<Bvh, Morton> {
             for (size_t i = 0; i < primitive_count; ++i) {
                 auto& node = nodes[begin + i];
                 node.bounding_box_proxy()     = bboxes[primitive_indices[i]];
-                node.is_leaf                  = true;
                 node.primitive_count          = 1;
                 node.first_child_or_primitive = i;
             }
@@ -180,9 +179,6 @@ class LinearBvhBuilder : public MortonCodeBasedBuilder<Bvh, Morton> {
             end          = next_end;
         }
 
-        nodes[0].first_child_or_primitive = 1;
-        nodes[0].is_leaf = false;
-
         std::swap(bvh.nodes, nodes);
         std::swap(bvh.primitive_indices, primitive_indices);
         bvh.node_count = node_count;

include/bvh/locally_ordered_clustering_builder.hpp

@@ -153,7 +153,7 @@ class LocallyOrderedClusteringBuilder : public MortonCodeBasedBuilder<Bvh, Morto
                             .bounding_box_proxy()
                             .to_bounding_box()
                             .extend(input[i].bounding_box_proxy());
-                        unmerged_node.is_leaf = false;
+                        unmerged_node.primitive_count = 0;
                         unmerged_node.first_child_or_primitive = first_child;
                         output[first_child + 0] = input[i];
                         output[first_child + 1] = input[j];
@@ -206,7 +206,6 @@ class LocallyOrderedClusteringBuilder : public MortonCodeBasedBuilder<Bvh, Morto
         for (size_t i = 0; i < primitive_count; ++i) {
             auto& node = nodes[begin + i];
             node.bounding_box_proxy()     = bboxes[primitive_indices[i]];
-            node.is_leaf                  = true;
             node.primitive_count          = 1;
             node.first_child_or_primitive = i;
         }

include/bvh/node_layout_optimizer.hpp

@@ -81,7 +81,7 @@ class NodeLayoutOptimizer {
             // Remap children indices to the new layout
             #pragma omp for
             for (size_t i = 0; i < bvh.node_count; ++i) {
-                if (nodes_copy[i].is_leaf)
+                if (nodes_copy[i].is_leaf())
                     continue;
                 nodes_copy[i].first_child_or_primitive =
                     unsorted_indices[(nodes_copy[i].first_child_or_primitive - 1) / 2];

include/bvh/parallel_reinsertion_optimizer.hpp

@@ -54,16 +54,16 @@ class ParallelReinsertionOptimizer :
         // Re-insert it into the destination
         bvh.nodes[out].bounding_box_proxy().extend(bvh.nodes[in].bounding_box_proxy());
         bvh.nodes[out].first_child_or_primitive = std::min(in, sibling_in);
-        bvh.nodes[out].is_leaf = false;
+        bvh.nodes[out].primitive_count = 0;
         bvh.nodes[sibling_in] = out_node;
         bvh.nodes[parent_in] = sibling_node;
 
         // Update parent-child indices
-        if (!out_node.is_leaf) {
+        if (!out_node.is_leaf()) {
             parents[out_node.first_child_or_primitive + 0] = sibling_in;
             parents[out_node.first_child_or_primitive + 1] = sibling_in;
         }
-        if (!sibling_node.is_leaf) {
+        if (!sibling_node.is_leaf()) {
             parents[sibling_node.first_child_or_primitive + 0] = parent_in;
             parents[sibling_node.first_child_or_primitive + 1] = parent_in;
         }
@@ -96,7 +96,7 @@ class ParallelReinsertionOptimizer :
                     out_best = out;
                 }
                 d = d + bbox_out.half_area() - bbox_merged.half_area();
-                if (bvh.nodes[out].is_leaf || d_bound + d <= d_best)
+                if (bvh.nodes[out].is_leaf() || d_bound + d <= d_best)
                     down = false;
                 else
                     out = bvh.nodes[out].first_child_or_primitive;

include/bvh/sah_based_algorithm.hpp

@@ -23,7 +23,7 @@ class SahBasedAlgorithm {
         Scalar cost(0);
         #pragma omp parallel for reduction(+: cost)
         for (size_t i = 0; i < bvh.node_count; ++i) {
-            if (bvh.nodes[i].is_leaf)
+            if (bvh.nodes[i].is_leaf())
                 cost += bvh.nodes[i].bounding_box_proxy().half_area() * bvh.nodes[i].primitive_count;
             else
                 cost += traversal_cost * bvh.nodes[i].bounding_box_proxy().half_area();

include/bvh/single_ray_traverser.hpp

@@ -47,7 +47,7 @@ class SingleRayTraverser {
         PrimitiveIntersector& primitive_intersector,
         Statistics& statistics) const
     {
-        assert(node.is_leaf);
+        assert(node.is_leaf());
         size_t begin = node.first_child_or_primitive;
         size_t end   = begin + node.primitive_count;
         statistics.intersections += end - begin;
@@ -69,7 +69,7 @@ class SingleRayTraverser {
         auto best_hit = std::optional<typename PrimitiveIntersector::Result>(std::nullopt);
 
         // If the root is a leaf, intersect it and return
-        if (bvh__unlikely(bvh.nodes[0].is_leaf))
+        if (bvh__unlikely(bvh.nodes[0].is_leaf()))
             return intersect_leaf(bvh.nodes[0], ray, best_hit, primitive_intersector, statistics);
 
         NodeIntersector node_intersector(ray);
@@ -89,7 +89,7 @@ class SingleRayTraverser {
             auto distance_right = node_intersector.intersect(*right_child, ray);
 
             if (distance_left.first <= distance_left.second) {
-                if (bvh__unlikely(left_child->is_leaf)) {
+                if (bvh__unlikely(left_child->is_leaf())) {
                     if (intersect_leaf(*left_child, ray, best_hit, primitive_intersector, statistics) &&
                         primitive_intersector.any_hit)
                         break;
@@ -99,7 +99,7 @@ class SingleRayTraverser {
                 left_child = nullptr;
 
             if (distance_right.first <= distance_right.second) {
-                if (bvh__unlikely(right_child->is_leaf)) {
+                if (bvh__unlikely(right_child->is_leaf())) {
                     if (intersect_leaf(*right_child, ray, best_hit, primitive_intersector, statistics) &&
                         primitive_intersector.any_hit)
                         break;

include/bvh/spatial_split_bvh_builder.hpp

@@ -237,8 +237,7 @@ class SpatialSplitBvhBuildTask : public TopDownBuildTask {
         auto& right = bvh.nodes[first_child + 1];
         parent.first_child_or_primitive = first_child;
         parent.primitive_count          = 0;
-        parent.is_leaf                  = false;
-
+
         left.bounding_box_proxy()  = left_bbox;
         right.bounding_box_proxy() = right_bbox;
 
@@ -512,7 +511,6 @@ class SpatialSplitBvhBuildTask : public TopDownBuildTask {
                 bvh.primitive_indices[first_primitive + i] = references[0][begin + i].primitive_index;
             node.first_child_or_primitive = first_primitive;
             node.primitive_count          = primitive_count;
-            node.is_leaf                  = true;
         };
 
         if (item.work_size() <= 1 || item.depth >= builder.max_depth) {

include/bvh/sweep_sah_builder.hpp

@@ -168,7 +168,6 @@ class SweepSahBuildTask : public TopDownBuildTask {
         auto make_leaf = [] (typename Bvh::Node& node, size_t begin, size_t end) {
             node.first_child_or_primitive = begin;
             node.primitive_count          = end - begin;
-            node.is_leaf                  = true;
         };
 
         if (item.work_size() <= 1 || item.depth >= builder.max_depth) {
@@ -242,8 +241,7 @@ class SweepSahBuildTask : public TopDownBuildTask {
         auto& right = bvh.nodes[first_child + 1];
         node.first_child_or_primitive = first_child;
         node.primitive_count          = 0;
-        node.is_leaf                  = false;
-
+
         left.bounding_box_proxy()  = left_bbox;
         right.bounding_box_proxy() = right_bbox;
         WorkItem first_item (first_child + 0, item.begin, split_index, item.depth + 1);

test/refit_bvh.cpp

@@ -38,7 +38,7 @@ bool check_bvh(const Bvh& bvh, const std::vector<Primitive>& primitives) {
     return std::all_of(
         bvh.nodes.get(), bvh.nodes.get() + bvh.node_count,
         [&] (const Bvh::Node& node) {
-            if (node.is_leaf) {
+            if (node.is_leaf()) {
                 return std::all_of(
                     bvh.primitive_indices.get() + node.first_child_or_primitive,
                     bvh.primitive_indices.get() + node.first_child_or_primitive + node.primitive_count,
@@ -78,7 +78,7 @@ static bool create_and_refit_bvh(size_t primitive_count) {
     // Refit the BVH
     bvh::HierarchyRefitter<Bvh> refitter(bvh);
     refitter.refit([&] (Bvh::Node& leaf) {
-        assert(leaf.is_leaf);
+        assert(leaf.is_leaf());
         auto bbox = BoundingBox::empty();
         for (size_t i = 0; i < leaf.primitive_count; ++i) {
             auto& triangle = triangles[bvh.primitive_indices[leaf.first_child_or_primitive + i]];