Move storage to new BlobSidecar (Consensys#7737)

sicw · Nov 21, 2023 · 5fbbf6a · 5fbbf6a
1 parent 6f83c43
commit 5fbbf6a
Show file tree

Hide file tree

Showing 56 changed files with 441 additions and 572 deletions.
diff --git a/...nc/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java b/...nc/src/test/java/tech/pegasys/teku/beacon/sync/historical/HistoricalBatchFetcherTest.java
@@ -30,7 +30,6 @@
 import java.util.Optional;
 import java.util.stream.Collectors;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
@@ -185,7 +184,6 @@ public void run_returnAllBlocksOnFirstRequest() {
   }
 
   @Test
-  @Disabled("TODO for new BlobSidecars")
   public void run_returnAllBlocksAndBlobSidecarsOnFirstRequest() {
     when(blobSidecarManager.isAvailabilityRequiredAtSlot(any())).thenReturn(true);
 
@@ -208,7 +206,6 @@ public void run_returnAllBlocksAndBlobSidecarsOnFirstRequest() {
   }
 
   @Test
-  @Disabled("TODO for new BlobSidecars")
   public void run_failsOnBlobSidecarsValidationFailure() {
     when(blobSidecarManager.isAvailabilityRequiredAtSlot(any())).thenReturn(true);
     when(blobSidecarManager.createAvailabilityCheckerAndValidateImmediately(any(), anyList()))

diff --git a/...ech/pegasys/teku/beaconrestapi/tekuv1/beacon/GetAllBlobSidecarsAtSlotIntegrationTest.java b/...ech/pegasys/teku/beaconrestapi/tekuv1/beacon/GetAllBlobSidecarsAtSlotIntegrationTest.java
@@ -28,6 +28,7 @@
 import okhttp3.Response;
 import org.apache.tuweni.bytes.Bytes;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import tech.pegasys.teku.beaconrestapi.AbstractDataBackedRestAPIIntegrationTest;
 import tech.pegasys.teku.beaconrestapi.handlers.tekuv1.beacon.GetAllBlobSidecarsAtSlot;
@@ -39,12 +40,14 @@
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.spec.SpecMilestone;
 import tech.pegasys.teku.spec.config.SpecConfigDeneb;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarOld;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarSchemaOld;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockAndState;
 import tech.pegasys.teku.spec.generator.ChainBuilder;
 import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb;
 
+@Disabled("Fix for new BlobSidecars")
 public class GetAllBlobSidecarsAtSlotIntegrationTest
     extends AbstractDataBackedRestAPIIntegrationTest {
 
@@ -64,14 +67,14 @@ public void shouldGetNonCanonicalBlobSidecars() throws IOException {
     final ChainBuilder fork = chainBuilder.fork();
     final SignedBlockAndState nonCanonicalBlock = fork.generateNextBlock(chainUpdater.blockOptions);
 
-    final List<BlobSidecarOld> nonCanonicalBlobSidecars =
-        fork.getBlobSidecarsOld(nonCanonicalBlock.getRoot());
+    final List<BlobSidecar> nonCanonicalBlobSidecars =
+        fork.getBlobSidecars(nonCanonicalBlock.getRoot());
     chainUpdater.saveBlock(nonCanonicalBlock, nonCanonicalBlobSidecars);
 
     final SignedBlockAndState canonicalBlock =
         chainBuilder.generateNextBlock(1, chainUpdater.blockOptions);
-    final List<BlobSidecarOld> canonicalBlobSidecars =
-        chainBuilder.getBlobSidecarsOld(canonicalBlock.getRoot());
+    final List<BlobSidecar> canonicalBlobSidecars =
+        chainBuilder.getBlobSidecars(canonicalBlock.getRoot());
     chainUpdater.saveBlock(canonicalBlock, canonicalBlobSidecars);
     chainUpdater.updateBestBlock(canonicalBlock);
     chainUpdater.finalizeEpoch(targetSlot.plus(1));
@@ -106,14 +109,14 @@ public void shouldGetNonCanonicalBlobSidecarsAsSsz() throws Exception {
     final ChainBuilder fork = chainBuilder.fork();
     final SignedBlockAndState nonCanonicalBlock = fork.generateNextBlock(chainUpdater.blockOptions);
 
-    final List<BlobSidecarOld> nonCanonicalBlobSidecars =
-        fork.getBlobSidecarsOld(nonCanonicalBlock.getRoot());
+    final List<BlobSidecar> nonCanonicalBlobSidecars =
+        fork.getBlobSidecars(nonCanonicalBlock.getRoot());
     chainUpdater.saveBlock(nonCanonicalBlock, nonCanonicalBlobSidecars);
 
     final SignedBlockAndState canonicalBlock =
         chainBuilder.generateNextBlock(1, chainUpdater.blockOptions);
-    final List<BlobSidecarOld> canonicalBlobSidecars =
-        chainBuilder.getBlobSidecarsOld(canonicalBlock.getRoot());
+    final List<BlobSidecar> canonicalBlobSidecars =
+        chainBuilder.getBlobSidecars(canonicalBlock.getRoot());
     chainUpdater.saveBlock(canonicalBlock, canonicalBlobSidecars);
     chainUpdater.updateBestBlock(canonicalBlock);
     chainUpdater.finalizeEpoch(targetSlot.plus(1));

diff --git a/...n-test/java/tech/pegasys/teku/beaconrestapi/v1/beacon/GetBlobSidecarsIntegrationTest.java b/...n-test/java/tech/pegasys/teku/beaconrestapi/v1/beacon/GetBlobSidecarsIntegrationTest.java
@@ -24,10 +24,13 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.function.Function;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import okhttp3.Response;
 import org.apache.tuweni.bytes.Bytes;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import tech.pegasys.teku.beaconrestapi.AbstractDataBackedRestAPIIntegrationTest;
 import tech.pegasys.teku.beaconrestapi.handlers.v1.beacon.GetBlobSidecars;
@@ -39,14 +42,25 @@
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.spec.SpecMilestone;
 import tech.pegasys.teku.spec.config.SpecConfigDeneb;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarOld;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarSchemaOld;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockAndState;
 import tech.pegasys.teku.spec.generator.ChainBuilder;
 import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb;
 
+@Disabled("Remove blobSidecarRepacker and fix for new BlobSidecars")
 public class GetBlobSidecarsIntegrationTest extends AbstractDataBackedRestAPIIntegrationTest {
 
+  @Deprecated
+  private final Supplier<Function<BlobSidecar, BlobSidecarOld>> blobSidecarRepacker =
+      () ->
+          blobSidecar ->
+              SchemaDefinitionsDeneb.required(
+                      spec.forMilestone(SpecMilestone.DENEB).getSchemaDefinitions())
+                  .getBlobSidecarOldSchema()
+                  .create(blobSidecar);
+
   @BeforeEach
   public void beforeEach() {
     startRestApiAtGenesisStoringNonCanonicalBlocks(SpecMilestone.DENEB);
@@ -63,7 +77,9 @@ public void shouldGetBlobSidecars() throws Exception {
     final SignedBlockAndState lastBlock = chainUpdater.advanceChainUntil(targetSlot);
     chainUpdater.updateBestBlock(lastBlock);
     final List<BlobSidecarOld> expected =
-        recentChainData.getBlobSidecars(lastBlock.getSlotAndBlockRoot()).get();
+        recentChainData.getBlobSidecars(lastBlock.getSlotAndBlockRoot()).get().stream()
+            .map(blobSidecar -> blobSidecarRepacker.get().apply(blobSidecar))
+            .toList();
 
     final Response responseAll = get("head");
     assertThat(responseAll.code()).isEqualTo(SC_OK);
@@ -116,7 +132,9 @@ public void shouldGetBlobSidecarsAsSsz() throws Exception {
     final SignedBlockAndState lastBlock = chainUpdater.advanceChainUntil(targetSlot);
     chainUpdater.updateBestBlock(lastBlock);
     final List<BlobSidecarOld> expected =
-        recentChainData.getBlobSidecars(lastBlock.getSlotAndBlockRoot()).get();
+        recentChainData.getBlobSidecars(lastBlock.getSlotAndBlockRoot()).get().stream()
+            .map(blobSidecar -> blobSidecarRepacker.get().apply(blobSidecar))
+            .toList();
 
     final Response response = get("head", OCTET_STREAM);
     assertThat(response.code()).isEqualTo(SC_OK);
@@ -149,14 +167,13 @@ public void shouldGetNonCanonicalBlobSidecarsByRoot() throws IOException {
     final ChainBuilder fork = chainBuilder.fork();
     final SignedBlockAndState nonCanonicalBlock = fork.generateNextBlock(chainUpdater.blockOptions);
 
-    final List<BlobSidecarOld> nonCanonicalBlobSidecars =
-        fork.getBlobSidecarsOld(nonCanonicalBlock.getRoot());
+    final List<BlobSidecar> nonCanonicalBlobSidecars =
+        fork.getBlobSidecars(nonCanonicalBlock.getRoot());
     chainUpdater.saveBlock(nonCanonicalBlock, nonCanonicalBlobSidecars);
 
     final SignedBlockAndState canonicalBlock =
         chainBuilder.generateNextBlock(1, chainUpdater.blockOptions);
-    chainUpdater.saveBlock(
-        canonicalBlock, chainBuilder.getBlobSidecarsOld(canonicalBlock.getRoot()));
+    chainUpdater.saveBlock(canonicalBlock, chainBuilder.getBlobSidecars(canonicalBlock.getRoot()));
     chainUpdater.updateBestBlock(canonicalBlock);
 
     final Response byRootResponse =

diff --git a/...java/tech/pegasys/teku/beaconrestapi/handlers/tekuv1/beacon/GetAllBlobSidecarsAtSlot.java b/...java/tech/pegasys/teku/beaconrestapi/handlers/tekuv1/beacon/GetAllBlobSidecarsAtSlot.java
@@ -75,7 +75,7 @@ private static EndpointMetadata createEndpointMetadata(final SchemaDefinitionCac
   public void handleRequest(RestApiRequest request) throws JsonProcessingException {
     final List<UInt64> indices = request.getQueryParameterList(BLOB_INDICES_PARAMETER);
     final SafeFuture<Optional<List<BlobSidecarOld>>> future =
-        chainDataProvider.getAllBlobSidecarsAtSlot(
+        chainDataProvider.getAllBlobSidecarsAtSlotOld(
             request.getPathParameter(SLOT_PARAMETER), indices);
 
     request.respondAsync(

diff --git a/...api/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/beacon/GetBlobSidecars.java b/...api/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/beacon/GetBlobSidecars.java
@@ -76,7 +76,7 @@ private static EndpointMetadata createEndpointMetadata(final SchemaDefinitionCac
   public void handleRequest(RestApiRequest request) throws JsonProcessingException {
     final List<UInt64> indices = request.getQueryParameterList(BLOB_INDICES_PARAMETER);
     final SafeFuture<Optional<List<BlobSidecarOld>>> future =
-        chainDataProvider.getBlobSidecars(request.getPathParameter(PARAMETER_BLOCK_ID), indices);
+        chainDataProvider.getBlobSidecarsOld(request.getPathParameter(PARAMETER_BLOCK_ID), indices);
 
     request.respondAsync(
         future.thenApply(

diff --git a/.../tech/pegasys/teku/beaconrestapi/handlers/tekuv1/beacon/GetAllBlobSidecarsAtSlotTest.java b/.../tech/pegasys/teku/beaconrestapi/handlers/tekuv1/beacon/GetAllBlobSidecarsAtSlotTest.java
@@ -59,7 +59,7 @@ void setup() {
   void shouldReturnAllBlobSidecarsAtSlot() throws JsonProcessingException {
     final List<BlobSidecarOld> nonCanonicalBlobSidecars =
         dataStructureUtil.randomBlobSidecarsOld(4);
-    when(chainDataProvider.getAllBlobSidecarsAtSlot(eq(UInt64.ONE), eq(indices)))
+    when(chainDataProvider.getAllBlobSidecarsAtSlotOld(eq(UInt64.ONE), eq(indices)))
         .thenReturn(SafeFuture.completedFuture(Optional.of(nonCanonicalBlobSidecars)));
 
     handler.handleRequest(request);

diff --git a/...src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/beacon/GetBlobSidecarsTest.java b/...src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/beacon/GetBlobSidecarsTest.java
@@ -28,13 +28,17 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
+import java.util.function.Function;
+import java.util.function.Supplier;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import tech.pegasys.teku.beaconrestapi.AbstractMigratedBeaconHandlerWithChainDataProviderTest;
 import tech.pegasys.teku.spec.SpecMilestone;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarOld;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
 import tech.pegasys.teku.spec.datastructures.metadata.ObjectAndMetaData;
+import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb;
 
 class GetBlobSidecarsTest extends AbstractMigratedBeaconHandlerWithChainDataProviderTest {
 
@@ -47,6 +51,15 @@ void setup() {
     request.setPathParameter("block_id", "head");
   }
 
+  @Deprecated
+  private final Supplier<Function<BlobSidecar, BlobSidecarOld>> blobSidecarRepacker =
+      () ->
+          blobSidecar ->
+              SchemaDefinitionsDeneb.required(
+                      spec.forMilestone(SpecMilestone.DENEB).getSchemaDefinitions())
+                  .getBlobSidecarOldSchema()
+                  .create(blobSidecar);
+
   @Test
   void shouldReturnBlobSidecars()
       throws JsonProcessingException, ExecutionException, InterruptedException {
@@ -56,7 +69,10 @@ void shouldReturnBlobSidecars()
         combinedChainDataClient
             .getBlobSidecars(
                 blockAndMetaData.getData().getSlotAndBlockRoot(), Collections.emptyList())
-            .get();
+            .get()
+            .stream()
+            .map(blobSidecar -> blobSidecarRepacker.get().apply(blobSidecar))
+            .toList();
 
     handler.handleRequest(request);
 

diff --git a/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java b/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java
@@ -28,6 +28,7 @@
 import java.util.function.Function;
 import java.util.function.IntPredicate;
 import java.util.function.Predicate;
+import java.util.function.Supplier;
 import java.util.stream.IntStream;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -58,6 +59,7 @@
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.spec.Spec;
 import tech.pegasys.teku.spec.SpecMilestone;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarOld;
 import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
@@ -74,6 +76,7 @@
 import tech.pegasys.teku.spec.logic.common.statetransition.epoch.status.ValidatorStatuses;
 import tech.pegasys.teku.spec.logic.common.statetransition.exceptions.EpochProcessingException;
 import tech.pegasys.teku.spec.logic.common.statetransition.exceptions.SlotProcessingException;
+import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb;
 import tech.pegasys.teku.storage.client.ChainDataUnavailableException;
 import tech.pegasys.teku.storage.client.CombinedChainDataClient;
 import tech.pegasys.teku.storage.client.RecentChainData;
@@ -179,18 +182,61 @@ public SafeFuture<Optional<ObjectAndMetaData<SignedBeaconBlock>>> getBlock(
     return fromBlock(blockIdParam, Function.identity());
   }
 
-  public SafeFuture<Optional<List<BlobSidecarOld>>> getBlobSidecars(
+  public SafeFuture<Optional<List<BlobSidecar>>> getBlobSidecars(
       final String blockIdParam, final List<UInt64> indices) {
     return blobSidecarSelectorFactory
         .createSelectorForBlockId(blockIdParam)
         .getBlobSidecars(indices);
   }
 
-  public SafeFuture<Optional<List<BlobSidecarOld>>> getAllBlobSidecarsAtSlot(
+  @Deprecated
+  private final Supplier<Function<BlobSidecar, BlobSidecarOld>> blobSidecarRepacker =
+      new Supplier<>() {
+        @Override
+        public Function<BlobSidecar, BlobSidecarOld> get() {
+          return blobSidecar ->
+              SchemaDefinitionsDeneb.required(
+                      spec.forMilestone(SpecMilestone.DENEB).getSchemaDefinitions())
+                  .getBlobSidecarOldSchema()
+                  .create(blobSidecar);
+        }
+      };
+
+  @Deprecated
+  public SafeFuture<Optional<List<BlobSidecarOld>>> getBlobSidecarsOld(
+      final String blockIdParam, final List<UInt64> indices) {
+    return blobSidecarSelectorFactory
+        .createSelectorForBlockId(blockIdParam)
+        .getBlobSidecars(indices)
+        .thenApply(
+            maybeBlobSidecars ->
+                maybeBlobSidecars.map(
+                    blobSidecars ->
+                        blobSidecars.stream()
+                            .map(blobSidecar -> blobSidecarRepacker.get().apply(blobSidecar))
+                            .toList()));
+  }
+
+  public SafeFuture<Optional<List<BlobSidecar>>> getAllBlobSidecarsAtSlot(
       final UInt64 slot, final List<UInt64> indices) {
     return blobSidecarSelectorFactory.slotSelectorForAll(slot).getBlobSidecars(indices);
   }
 
+  @Deprecated
+  public SafeFuture<Optional<List<BlobSidecarOld>>> getAllBlobSidecarsAtSlotOld(
+      final UInt64 slot, final List<UInt64> indices) {
+    return blobSidecarSelectorFactory
+        .slotSelectorForAll(slot)
+        .getBlobSidecars(indices)
+        .thenApply(
+            maybeBlobSidecars ->
+                maybeBlobSidecars.map(
+                    blobSidecars ->
+                        blobSidecars.stream()
+                            .map(blobSidecar -> blobSidecarRepacker.get().apply(blobSidecar))
+                            .toList()));
+  }
+
   public SafeFuture<Optional<ObjectAndMetaData<Bytes32>>> getBlockRoot(final String blockIdParam) {
     return fromBlock(blockIdParam, SignedBeaconBlock::getRoot);
   }

diff --git a/data/provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelector.java b/data/provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelector.java
@@ -17,8 +17,8 @@
 import java.util.Optional;
 import tech.pegasys.teku.infrastructure.async.SafeFuture;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
-import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarOld;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 
 public interface BlobSidecarSelector {
-  SafeFuture<Optional<List<BlobSidecarOld>>> getBlobSidecars(List<UInt64> indices);
+  SafeFuture<Optional<List<BlobSidecar>>> getBlobSidecars(List<UInt64> indices);
 }
diff --git a/...provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactory.java b/...provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactory.java
@@ -22,7 +22,7 @@
 import tech.pegasys.teku.api.AbstractSelectorFactory;
 import tech.pegasys.teku.infrastructure.async.SafeFuture;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
-import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecarOld;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
 import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
 import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot;
 import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.deneb.BeaconBlockBodyDeneb;
@@ -99,7 +99,7 @@ public BlobSidecarSelector slotSelectorForAll(final UInt64 slot) {
                     blobSidecars.isEmpty() ? Optional.empty() : Optional.of(blobSidecars));
   }
 
-  private SafeFuture<Optional<List<BlobSidecarOld>>> getBlobSidecarsForBlock(
+  private SafeFuture<Optional<List<BlobSidecar>>> getBlobSidecarsForBlock(
       final Optional<SignedBeaconBlock> maybeBlock, final List<UInt64> indices) {
     if (maybeBlock.isEmpty()) {
       return SafeFuture.completedFuture(Optional.empty());
@@ -116,12 +116,12 @@ private SafeFuture<Optional<List<BlobSidecarOld>>> getBlobSidecarsForBlock(
     return getBlobSidecars(block.getSlotAndBlockRoot(), indices);
   }
 
-  private SafeFuture<Optional<List<BlobSidecarOld>>> getBlobSidecars(
+  private SafeFuture<Optional<List<BlobSidecar>>> getBlobSidecars(
       final SlotAndBlockRoot slotAndBlockRoot, final List<UInt64> indices) {
     return client.getBlobSidecars(slotAndBlockRoot, indices).thenApply(Optional::of);
   }
 
-  private SafeFuture<Optional<List<BlobSidecarOld>>> getBlobSidecars(
+  private SafeFuture<Optional<List<BlobSidecar>>> getBlobSidecars(
       final UInt64 slot, final List<UInt64> indices) {
     return client.getBlobSidecars(slot, indices).thenApply(Optional::of);
   }