Skip to content

Commit

Permalink
indexer-alt: generate example config
Browse files Browse the repository at this point in the history
## Description

Using the default impl for `IndexerConfig` to generate the output for
`generate-config` lead to sub-par results because most of its fields are
optional, and so will not show up.

This change introduces an explicit `example` function that is
responsible for generating a non-default output that does a better job
of documenting what fields and pipelines are available to configure.

## Test plan

Generate a config:

```
sui$ cargo run -p sui-indexer-alt -- generate-config

[ingestion]
checkpoint-buffer-size = 5000
ingest-concurrency = 200
retry-interval-ms = 200

[consistency]
consistent-pruning-interval-ms = 300000
pruner-delay-ms = 120000

[committer]
write-concurrency = 5
collect-interval-ms = 500
watermark-interval-ms = 500

[pruner]
interval-ms = 300000
delay-ms = 120000
retention = 4000000
max-chunk-size = 2000

[pipeline.sum_coin_balances]

[pipeline.wal_coin_balances]

[pipeline.sum_obj_types]

[pipeline.wal_obj_types]

[pipeline.sum_displays]

[pipeline.sum_packages]

[pipeline.ev_emit_mod]

[pipeline.ev_struct_inst]

[pipeline.kv_checkpoints]

[pipeline.kv_epoch_ends]

[pipeline.kv_epoch_starts]

[pipeline.kv_feature_flags]

[pipeline.kv_objects]

[pipeline.kv_protocol_configs]

[pipeline.kv_transactions]

[pipeline.obj_versions]

[pipeline.tx_affected_addresses]

[pipeline.tx_affected_objects]

[pipeline.tx_balance_changes]

[pipeline.tx_calls]

[pipeline.tx_digests]

[pipeline.tx_kinds]

```
  • Loading branch information
amnn committed Nov 29, 2024
1 parent d695cc8 commit b6d0428
Show file tree
Hide file tree
Showing 2 changed files with 110 additions and 2 deletions.
110 changes: 109 additions & 1 deletion crates/sui-indexer-alt/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ pub struct IndexerConfig {
pub extra: toml::Table,
}

#[DefaultConfig]
#[derive(Clone)]
pub struct ConsistencyConfig {
/// How often to check whether write-ahead logs related to the consistent range can be
Expand Down Expand Up @@ -177,6 +176,20 @@ macro_rules! merge_recursive {
}

impl IndexerConfig {
/// Generate an example configuration, suitable for demonstrating the fields available to
/// configure.
pub fn example() -> Self {
let mut example: Self = Default::default();

example.ingestion = IngestionConfig::default().into();
example.consistency = ConsistencyConfig::default().into();
example.committer = CommitterConfig::default().into();
example.pruner = PrunerConfig::default().into();
example.pipeline = PipelineLayer::example();

example
}

pub fn merge(self, other: IndexerConfig) -> IndexerConfig {
check_extra("top-level", self.extra);
check_extra("top-level", other.extra);
Expand Down Expand Up @@ -354,6 +367,36 @@ impl PrunerLayer {
}

impl PipelineLayer {
/// Generate an example configuration, suitable for demonstrating the fields available to
/// configure.
pub fn example() -> Self {
PipelineLayer {
sum_coin_balances: Some(Default::default()),
wal_coin_balances: Some(Default::default()),
sum_obj_types: Some(Default::default()),
wal_obj_types: Some(Default::default()),
sum_displays: Some(Default::default()),
sum_packages: Some(Default::default()),
ev_emit_mod: Some(Default::default()),
ev_struct_inst: Some(Default::default()),
kv_checkpoints: Some(Default::default()),
kv_epoch_ends: Some(Default::default()),
kv_epoch_starts: Some(Default::default()),
kv_feature_flags: Some(Default::default()),
kv_objects: Some(Default::default()),
kv_protocol_configs: Some(Default::default()),
kv_transactions: Some(Default::default()),
obj_versions: Some(Default::default()),
tx_affected_addresses: Some(Default::default()),
tx_affected_objects: Some(Default::default()),
tx_balance_changes: Some(Default::default()),
tx_calls: Some(Default::default()),
tx_digests: Some(Default::default()),
tx_kinds: Some(Default::default()),
extra: Default::default(),
}
}

pub fn merge(self, other: PipelineLayer) -> PipelineLayer {
check_extra("pipeline", self.extra);
check_extra("pipeline", other.extra);
Expand Down Expand Up @@ -409,6 +452,71 @@ impl Default for ConsistencyConfig {
}
}

impl From<IngestionConfig> for IngestionLayer {
fn from(config: IngestionConfig) -> Self {
Self {
checkpoint_buffer_size: Some(config.checkpoint_buffer_size),
ingest_concurrency: Some(config.ingest_concurrency),
retry_interval_ms: Some(config.retry_interval_ms),
extra: Default::default(),
}
}
}

impl From<ConsistencyConfig> for ConsistencyLayer {
fn from(config: ConsistencyConfig) -> Self {
Self {
consistent_pruning_interval_ms: Some(config.consistent_pruning_interval_ms),
pruner_delay_ms: Some(config.pruner_delay_ms),
consistent_range: config.consistent_range,
extra: Default::default(),
}
}
}

impl From<SequentialConfig> for SequentialLayer {
fn from(config: SequentialConfig) -> Self {
Self {
committer: Some(config.committer.into()),
checkpoint_lag: Some(config.checkpoint_lag),
extra: Default::default(),
}
}
}

impl From<ConcurrentConfig> for ConcurrentLayer {
fn from(config: ConcurrentConfig) -> Self {
Self {
committer: Some(config.committer.into()),
pruner: config.pruner.map(Into::into),
extra: Default::default(),
}
}
}

impl From<CommitterConfig> for CommitterLayer {
fn from(config: CommitterConfig) -> Self {
Self {
write_concurrency: Some(config.write_concurrency),
collect_interval_ms: Some(config.collect_interval_ms),
watermark_interval_ms: Some(config.watermark_interval_ms),
extra: Default::default(),
}
}
}

impl From<PrunerConfig> for PrunerLayer {
fn from(config: PrunerConfig) -> Self {
Self {
interval_ms: Some(config.interval_ms),
delay_ms: Some(config.delay_ms),
retention: Some(config.retention),
max_chunk_size: Some(config.max_chunk_size),
extra: Default::default(),
}
}
}

/// Check whether there are any unrecognized extra fields and if so, warn about them.
fn check_extra(pos: &str, extra: toml::Table) {
if !extra.is_empty() {
Expand Down
2 changes: 1 addition & 1 deletion crates/sui-indexer-alt/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ async fn main() -> Result<()> {
}

Command::GenerateConfig => {
let config = IndexerConfig::default();
let config = IndexerConfig::example();
let config_toml = toml::to_string_pretty(&config)
.context("Failed to serialize default configuration to TOML.")?;

Expand Down

0 comments on commit b6d0428

Please sign in to comment.