Skip to content

Commit

Permalink
[data ingestion] http based remote reader (MystenLabs#15977)
Browse files Browse the repository at this point in the history
add support for http based remote reader. 
Workload can be enabled by just specifying following setting in config:
```remote_store_url: https://checkpoints.testnet.sui.io```
  • Loading branch information
phoenix-o authored Jan 29, 2024
1 parent 2f4573c commit df9cef7
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 7 deletions.
2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -376,7 +376,7 @@ ntest = "0.9.0"
num-bigint = "0.4.4"
num_cpus = "1.15.0"
num_enum = "0.6.1"
object_store = { version = "0.7", features = ["aws", "gcp", "azure"] }
object_store = { version = "0.7", features = ["aws", "gcp", "azure", "http"] }
once_cell = "1.18.0"
ouroboros = "0.17"
parking_lot = "0.12.1"
Expand Down
21 changes: 15 additions & 6 deletions crates/sui-data-ingestion/src/reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -164,12 +164,21 @@ impl CheckpointReader {
let (processed_sender, processed_receiver) = mpsc::channel(MAX_CHECKPOINTS_IN_PROGRESS);
let (exit_sender, exit_receiver) = oneshot::channel();
let remote_store = remote_store_url.map(|url| {
parse_url_opts(
&Url::parse(&url).expect("failed to parse remote store url"),
remote_store_options,
)
.expect("failed to parse remote store config")
.0
if remote_store_options.is_empty() {
let builder = object_store::http::HttpBuilder::new().with_url(url);
Box::new(
builder
.build()
.expect("failed to parse remote store config"),
)
} else {
parse_url_opts(
&Url::parse(&url).expect("failed to parse remote store url"),
remote_store_options,
)
.expect("failed to parse remote store config")
.0
}
});
let reader = Self {
path,
Expand Down

0 comments on commit df9cef7

Please sign in to comment.