go uses arbbrotli

Makefile

@@ -51,6 +51,10 @@ replay_deps=arbos wavmio arbstate arbcompress solgen/go/node-interfacegen blsSig
 
 replay_wasm=$(output_latest)/replay.wasm
 
+arb_brotli_lib = $(output_root)/lib/libbrotli.a
+brotli_cgo_header = $(output_root)/include/arb_brotli.h
+arb_brotli_files = $(wildcard arbitrator/brotli/src/*.* arbitrator/brotli/src/*/*.* arbitrator/brotli/*.toml arbitrator/brotli/*.rs) .make/cbrotli-lib
+
 arbitrator_generated_header=$(output_root)/include/arbitrator.h
 arbitrator_wasm_libs=$(patsubst %, $(output_root)/machines/latest/%.wasm, forward wasi_stub host_io soft-float arbcompress user_host program_exec)
 arbitrator_stylus_lib=$(output_root)/lib/libstylus.a
@@ -157,12 +161,13 @@ build-node-deps: $(go_source) build-prover-header build-prover-lib build-jit .ma
 test-go-deps: \
  build-replay-env \
  $(stylus_test_wasms) \
+ $(arb_brotli_lib) \
  $(arbitrator_stylus_lib) \
  $(patsubst %,$(arbitrator_cases)/%.wasm, global-state read-inboxmsg-10 global-state-wrapper const)
 
-build-prover-header: $(arbitrator_generated_header)
+build-prover-header: $(arbitrator_generated_header) $(brotli_cgo_header)
 
-build-prover-lib: $(arbitrator_stylus_lib)
+build-prover-lib: $(arbitrator_stylus_lib) $(arb_brotli_lib)
 
 build-prover-bin: $(prover_bin)
 
@@ -269,6 +274,11 @@ $(prover_bin): $(DEP_PREDICATE) $(rust_prover_files)
  cargo build --manifest-path arbitrator/Cargo.toml --release --bin prover ${CARGOFLAGS}
  install arbitrator/target/release/prover $@
 
+$(arb_brotli_lib): $(DEP_PREDICATE) $(arb_brotli_files)
+ mkdir -p `dirname $(arb_brotli_lib)`
+ cargo build --manifest-path arbitrator/Cargo.toml --release --lib -p brotli ${CARGOFLAGS}
+ install arbitrator/target/release/libbrotli.a $@
+
 $(arbitrator_stylus_lib): $(DEP_PREDICATE) $(stylus_files)
  mkdir -p `dirname $(arbitrator_stylus_lib)`
  cargo build --manifest-path arbitrator/Cargo.toml --release --lib -p stylus ${CARGOFLAGS}
@@ -291,6 +301,12 @@ $(arbitrator_generated_header): $(DEP_PREDICATE) $(stylus_files)
  cd arbitrator/stylus && cbindgen --config cbindgen.toml --crate stylus --output ../../$(arbitrator_generated_header)
  @touch -c $@ # cargo might decide to not rebuild the header
 
+$(brotli_cgo_header): $(DEP_PREDICATE) $(arb_brotli_files)
+ @echo creating ${PWD}/$(brotli_cgo_header)
+ mkdir -p `dirname $(brotli_cgo_header)`
+ cd arbitrator/brotli && cbindgen --config cbindgen.toml --crate brotli --output ../../$(brotli_cgo_header)
+ @touch -c $@ # cargo might decide to not rebuild the header
+
 $(output_latest)/wasi_stub.wasm: $(DEP_PREDICATE) $(call wasm_lib_deps,wasi-stub)
  cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-unknown-unknown --package wasi-stub
  install arbitrator/wasm-libraries/$(wasm32_unknown)/wasi_stub.wasm $@

arbcompress/compress_cgo.go

@@ -8,103 +8,69 @@ package arbcompress
 
 /*
 #cgo CFLAGS: -g -Wall -I${SRCDIR}/../target/include/
-#cgo LDFLAGS: ${SRCDIR}/../target/lib/libbrotlidec-static.a ${SRCDIR}/../target/lib/libbrotlienc-static.a ${SRCDIR}/../target/lib/libbrotlicommon-static.a -lm
-#include "brotli/encode.h"
-#include "brotli/decode.h"
+#cgo LDFLAGS: ${SRCDIR}/../target/lib/libbrotli.a -lm
+#include "arb_brotli.h"
 */
 import "C"
-import (
- "fmt"
-)
+import "fmt"
 
 type u8 = C.uint8_t
+type u32 = C.uint32_t
 type usize = C.size_t
 
 type brotliBool = uint32
+type brotliBuffer = C.BrotliBuffer
 
 const (
  brotliFalse brotliBool = iota
  brotliTrue
 )
 
-const (
- rawSharedDictionary C.BrotliSharedDictionaryType = iota // LZ77 prefix dictionary
- serializedSharedDictionary // Serialized dictionary
-)
-
-func (d Dictionary) data() []byte {
- return []byte{}
-}
-
 func Decompress(input []byte, maxSize int) ([]byte, error) {
  return DecompressWithDictionary(input, maxSize, EmptyDictionary)
 }
 
 func DecompressWithDictionary(input []byte, maxSize int, dictionary Dictionary) ([]byte, error) {
- state := C.BrotliDecoderCreateInstance(nil, nil, nil)
- defer C.BrotliDecoderDestroyInstance(state)
-
- if dictionary != EmptyDictionary {
- data := dictionary.data()
- attached := C.BrotliDecoderAttachDictionary(
- state,
- rawSharedDictionary,
- usize(len(data)),
- sliceToPointer(data),
- )
- if uint32(attached) != brotliTrue {
- return nil, fmt.Errorf("failed decompression: failed to attach dictionary")
- }
- }
-
- inLen := usize(len(input))
- inPtr := sliceToPointer(input)
  output := make([]byte, maxSize)
- outLen := usize(maxSize)
- outLeft := usize(len(output))
- outPtr := sliceToPointer(output)
+ outbuf := sliceToBuffer(output)
+ inbuf := sliceToBuffer(input)
 
- status := C.BrotliDecoderDecompressStream(
- state,
- &inLen,
- &inPtr,
- &outLeft,
- &outPtr,
- &outLen, //nolint:gocritic
- )
- if uint32(status) != brotliSuccess {
- return nil, fmt.Errorf("failed decompression: failed streaming: %d", status)
+ status := C.brotli_decompress(inbuf, outbuf, C.Dictionary(dictionary))
+ if status != C.BrotliStatus_Success {
+ return nil, fmt.Errorf("failed decompression: %d", status)
  }
- if int(outLen) > maxSize {
- return nil, fmt.Errorf("failed decompression: result too large: %d", outLen)
+ if *outbuf.len > usize(maxSize) {
+ return nil, fmt.Errorf("failed decompression: result too large: %d", *outbuf.len)
  }
- return output[:outLen], nil
+ output = output[:*outbuf.len]
+ return output, nil
 }
 
-func compressLevel(input []byte, level int) ([]byte, error) {
- maxOutSize := compressedBufferSizeFor(len(input))
- outbuf := make([]byte, maxOutSize)
- outSize := C.size_t(maxOutSize)
- inputPtr := sliceToPointer(input)
- outPtr := sliceToPointer(outbuf)
-
- res := C.BrotliEncoderCompress(
- C.int(level), C.BROTLI_DEFAULT_WINDOW, C.BROTLI_MODE_GENERIC,
- C.size_t(len(input)), inputPtr, &outSize, outPtr,
- )
- if uint32(res) != brotliSuccess {
- return nil, fmt.Errorf("failed compression: %d", res)
- }
- return outbuf[:outSize], nil
+func CompressWell(input []byte) ([]byte, error) {
+ return compressLevel(input, EmptyDictionary, LEVEL_WELL)
 }
 
-func CompressWell(input []byte) ([]byte, error) {
- return compressLevel(input, LEVEL_WELL)
+func compressLevel(input []byte, dictionary Dictionary, level int) ([]byte, error) {
+ maxSize := compressedBufferSizeFor(len(input))
+ output := make([]byte, maxSize)
+ outbuf := sliceToBuffer(output)
+ inbuf := sliceToBuffer(input)
+
+ status := C.brotli_compress(inbuf, outbuf, C.Dictionary(dictionary), u32(level))
+ if status != C.BrotliStatus_Success {
+ return nil, fmt.Errorf("failed decompression: %d", status)
+ }
+ output = output[:*outbuf.len]
+ return output, nil
 }
 
-func sliceToPointer(slice []byte) *u8 {
- if len(slice) == 0 {
+func sliceToBuffer(slice []byte) brotliBuffer {
+ count := usize(len(slice))
+ if count == 0 {
  slice = []byte{0x00} // ensures pointer is not null (shouldn't be necessary, but brotli docs are picky about NULL)
  }
- return (*u8)(&slice[0])
+ return brotliBuffer{
+ ptr: (*u8)(&slice[0]),
+ len: &count,
+ }
 }

arbcompress/compress_common.go

@@ -3,13 +3,6 @@
 
 package arbcompress
 
-type brotliStatus = uint32
-
-const (
- brotliFailure brotliStatus = iota
- brotliSuccess
-)
-
 type Dictionary uint32
 
 const (
@@ -26,5 +19,5 @@ func compressedBufferSizeFor(length int) int {
 }
 
 func CompressFast(input []byte) ([]byte, error) {
- return compressLevel(input, LEVEL_FAST)
+ return compressLevel(input, EmptyDictionary, LEVEL_FAST)
 }

arbcompress/compress_wasm.go

@@ -13,6 +13,13 @@ import (
  "github.com/offchainlabs/nitro/arbutil"
 )
 
+type brotliStatus = uint32
+
+const (
+ brotliFailure brotliStatus = iota
+ brotliSuccess
+)
+
 //go:wasmimport arbcompress brotli_compress
 func brotliCompress(inBuf unsafe.Pointer, inBufLen uint32, outBuf unsafe.Pointer, outBufLen unsafe.Pointer, level, windowSize uint32) brotliStatus
 

arbitrator/brotli/Cargo.toml

@@ -12,5 +12,8 @@ rust-version.workspace = true
 num_enum.workspace = true
 wasmer = { path = "../tools/wasmer/lib/api", optional = true }
 
+[lib]
+crate-type = ["lib", "staticlib"]
+
 [features]
 wasmer_traits = ["dep:wasmer"]

arbitrator/brotli/build.rs

@@ -1,13 +1,22 @@
 // Copyright 2021-2024, Offchain Labs, Inc.
 // For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE
 
+use std::{env, path::Path};
+
 fn main() {
- let target_arch = std::env::var("TARGET").unwrap();
+ let target_arch = env::var("TARGET").unwrap();
+ let manifest = env::var("CARGO_MANIFEST_DIR").unwrap();
+ let manifest = Path::new(&manifest);
 
  if target_arch.contains("wasm32") {
  println!("cargo:rustc-link-search=../../target/lib-wasm/");
  } else {
- println!("cargo:rustc-link-search=../../target/lib/");
+ // search for brotli libs depending on where cargo is invoked
+ let arbitrator = Some(Path::new("arbitrator").file_name());
+ match arbitrator == manifest.parent().map(Path::file_name) {
+ true => println!("cargo:rustc-link-search=../target/lib/"),
+ false => println!("cargo:rustc-link-search=../../target/lib/"),
+ }
  }
  println!("cargo:rustc-link-lib=static=brotlienc-static");
  println!("cargo:rustc-link-lib=static=brotlidec-static");

arbitrator/brotli/cbindgen.toml

@@ -0,0 +1,10 @@
+language = "C"
+include_guard = "brotli_bindings"
+
+[parse]
+parse_deps = false
+
+[enum]
+prefix_with_name = true
+
+[export]

arbitrator/brotli/src/cgo.rs

@@ -0,0 +1,60 @@
+// Copyright 2021-2024, Offchain Labs, Inc.
+// For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE
+
+use crate::{BrotliStatus, Dictionary, DEFAULT_WINDOW_SIZE};
+
+#[derive(Clone, Copy)]
+#[repr(C)]
+pub struct BrotliBuffer {
+ /// Points to data owned by Go.
+ ptr: *mut u8,
+ /// The length in bytes. 
+ len: *mut usize,
+}
+
+impl BrotliBuffer {
+ fn as_slice(&self) -> &[u8] {
+ let len = unsafe { *self.len };
+ if len == 0 {
+ return &[];
+ }
+ unsafe { std::slice::from_raw_parts(self.ptr, len) }
+ }
+
+ fn as_mut_slice(&mut self) -> &mut [u8] {
+ let len = unsafe { *self.len };
+ if len == 0 {
+ return &mut [];
+ }
+ unsafe { std::slice::from_raw_parts_mut(self.ptr, len) }
+ }
+}
+
+#[no_mangle]
+pub extern "C" fn brotli_compress(
+ input: BrotliBuffer,
+ mut output: BrotliBuffer,
+ dictionary: Dictionary,
+ level: u32,
+) -> BrotliStatus {
+ let window = DEFAULT_WINDOW_SIZE;
+ let buffer = output.as_mut_slice();
+ match crate::compress_fixed(input.as_slice(), buffer, level, window, dictionary) {
+ Ok(written) => unsafe { *output.len = written },
+ Err(status) => return status,
+ }
+ BrotliStatus::Success
+}
+
+#[no_mangle]
+pub extern "C" fn brotli_decompress(
+ input: BrotliBuffer,
+ mut output: BrotliBuffer,
+ dictionary: Dictionary,
+) -> BrotliStatus {
+ match crate::decompress_fixed(input.as_slice(), output.as_mut_slice(), dictionary) {
+ Ok(written) => unsafe { *output.len = written },
+ Err(status) => return status,
+ }
+ BrotliStatus::Success
+}