[sui move] generate layouts for Move structs

Add build option to SuiCompiledPackage that generate a `serde-reflection` `Registry` containing the layout of relevant structs in the package. For now, we define "relevant" as the structs declared by each module in the package, plus the struct types passed to entry functions of each module (either directly or by reference). A `Registry` lets you do a bunch of useful things: - Write logic in our BCS Typescript SDK that consumes a `Registry` YAML file and automatically calls `bcs.registerStructType` on every entry in the registry. This will hopefully make writing Sui web apps much faster--if you've written Move types already, you get the TS type definitions + BCS serializers and deserializers for free. - Use `serde-generate` https://github.com/zefchain/serde-reflection/tree/main/serde-generate to generate type definitions from a `Registry` in a variety of languages (e.g. Python, Go, Rust, Java, Swift, C++). This makes it much easier to write an SDK that talks to an on-chain Move package in these langs because you can generate all the boilerplate wrapper types. - Autogenerate a `Registry` for your types that should not change, serialize the result to a YAML file, check it in, and then use this to detect breaking changes to those types. We do this for our key Rust types used in txes etc. Check out https://github.com/MystenLabs/sui/blob/main/crates/sui-core/tests/README.md--https://github.com/MystenLabs/sui/blob/main/crates/sui-core/tests/staged/sui.yaml is a serialized `Registry` autogenerated in the manner described above. Here's an example of what the `Registry` for the `games` package looks like: ``` --- "0000000000000000000000000000000000000000::hero::Boar": STRUCT: - id: TYPENAME: "0000000000000000000000000000000000000002::object::UID" - hp: U64 - strength: U64 - game_id: TYPENAME: "0000000000000000000000000000000000000002::object::ID" "0000000000000000000000000000000000000000::hero::BoarSlainEvent": STRUCT: - slayer_address: TYPENAME: AccountAddress - hero: TYPENAME: "0000000000000000000000000000000000000002::object::ID" - boar: TYPENAME: "0000000000000000000000000000000000000002::object::ID" - game_id: TYPENAME: "0000000000000000000000000000000000000002::object::ID" "0000000000000000000000000000000000000000::hero::GameAdmin": STRUCT: - id: TYPENAME: "0000000000000000000000000000000000000002::object::UID" - boars_created: U64 - potions_created: U64 - game_id: TYPENAME: "0000000000000000000000000000000000000002::object::ID" "0000000000000000000000000000000000000000::hero::GameInfo": STRUCT: - id: TYPENAME: "0000000000000000000000000000000000000002::object::UID" - admin: TYPENAME: AccountAddress "0000000000000000000000000000000000000000::hero::Hero": STRUCT: - id: TYPENAME: "0000000000000000000000000000000000000002::object::UID" - hp: U64 - experience: U64 - sword: TYPENAME: "0000000000000000000000000000000000000001::option::Option<0000000000000000000000000000000000000000::hero::Sword>" - game_id: TYPENAME: "0000000000000000000000000000000000000002::object::ID" "0000000000000000000000000000000000000000::hero::Potion": STRUCT: - id: TYPENAME: "0000000000000000000000000000000000000002::object::UID" - potency: U64 - game_id: TYPENAME: "0000000000000000000000000000000000000002::object::ID" "0000000000000000000000000000000000000000::hero::Sword": STRUCT: - id: TYPENAME: "0000000000000000000000000000000000000002::object::UID" - magic: U64 - strength: U64 - game_id: TYPENAME: "0000000000000000000000000000000000000002::object::ID" "0000000000000000000000000000000000000000::rock_paper_scissors::Game": STRUCT: - id: TYPENAME: "0000000000000000000000000000000000000002::object::UID" - prize: TYPENAME: "0000000000000000000000000000000000000000::rock_paper_scissors::ThePrize" - player_one: TYPENAME: AccountAddress - player_two: TYPENAME: AccountAddress - hash_one: BYTES - hash_two: BYTES - gesture_one: U8 - gesture_two: U8 "0000000000000000000000000000000000000000::rock_paper_scissors::PlayerTurn": STRUCT: - id: TYPENAME: "0000000000000000000000000000000000000002::object::UID" - hash: BYTES - player: TYPENAME: AccountAddress "0000000000000000000000000000000000000000::rock_paper_scissors::Secret": STRUCT: - id: TYPENAME: "0000000000000000000000000000000000000002::object::UID" - salt: BYTES - player: TYPENAME: AccountAddress "0000000000000000000000000000000000000000::rock_paper_scissors::ThePrize": STRUCT: - id: TYPENAME: "0000000000000000000000000000000000000002::object::UID" "0000000000000000000000000000000000000000::sea_hero::RUM": STRUCT: - dummy_field: BOOL "0000000000000000000000000000000000000000::sea_hero::SeaHeroAdmin": STRUCT: - id: TYPENAME: "0000000000000000000000000000000000000002::object::UID" - supply: TYPENAME: "0000000000000000000000000000000000000002::balance::Supply<0000000000000000000000000000000000000000::sea_hero::RUM>" - monsters_created: U64 - token_supply_max: U64 - monster_max: U64 "0000000000000000000000000000000000000000::sea_hero::SeaMonster": STRUCT: - id: TYPENAME: "0000000000000000000000000000000000000002::object::UID" - reward: TYPENAME: "0000000000000000000000000000000000000002::balance::Balance<0000000000000000000000000000000000000000::sea_hero::RUM>" "0000000000000000000000000000000000000000::sea_hero_helper::HelpMeSlayThisMonster": STRUCT: - id: TYPENAME: "0000000000000000000000000000000000000002::object::UID" - monster: TYPENAME: "0000000000000000000000000000000000000000::sea_hero::SeaMonster" - monster_owner: TYPENAME: AccountAddress - helper_reward: U64 "0000000000000000000000000000000000000000::shared_tic_tac_toe::GameEndEvent": STRUCT: - game_id: TYPENAME: "0000000000000000000000000000000000000002::object::ID" "0000000000000000000000000000000000000000::shared_tic_tac_toe::TicTacToe": STRUCT: - id: TYPENAME: "0000000000000000000000000000000000000002::object::UID" - gameboard: SEQ: BYTES - cur_turn: U8 - game_status: U8 - x_address: TYPENAME: AccountAddress - o_address: TYPENAME: AccountAddress "0000000000000000000000000000000000000000::shared_tic_tac_toe::Trophy": STRUCT: - id: TYPENAME: "0000000000000000000000000000000000000002::object::UID" "0000000000000000000000000000000000000000::tic_tac_toe::GameEndEvent": STRUCT: - game_id: TYPENAME: "0000000000000000000000000000000000000002::object::ID" "0000000000000000000000000000000000000000::tic_tac_toe::Mark": STRUCT: - id: TYPENAME: "0000000000000000000000000000000000000002::object::UID" - player: TYPENAME: AccountAddress - row: U64 - col: U64 "0000000000000000000000000000000000000000::tic_tac_toe::MarkMintCap": STRUCT: - id: TYPENAME: "0000000000000000000000000000000000000002::object::UID" - game_id: TYPENAME: "0000000000000000000000000000000000000002::object::ID" - remaining_supply: U8 "0000000000000000000000000000000000000000::tic_tac_toe::MarkSentEvent": STRUCT: - game_id: TYPENAME: "0000000000000000000000000000000000000002::object::ID" - mark_id: TYPENAME: "0000000000000000000000000000000000000002::object::ID" "0000000000000000000000000000000000000000::tic_tac_toe::TicTacToe": STRUCT: - id: TYPENAME: "0000000000000000000000000000000000000002::object::UID" - gameboard: SEQ: SEQ: TYPENAME: "0000000000000000000000000000000000000001::option::Option<0000000000000000000000000000000000000000::tic_tac_toe::Mark>" - cur_turn: U8 - game_status: U8 - x_address: TYPENAME: AccountAddress - o_address: TYPENAME: AccountAddress "0000000000000000000000000000000000000000::tic_tac_toe::Trophy": STRUCT: - id: TYPENAME: "0000000000000000000000000000000000000002::object::UID" "0000000000000000000000000000000000000001::option::Option<0000000000000000000000000000000000000000::hero::Sword>": STRUCT: - vec: SEQ: TYPENAME: "0000000000000000000000000000000000000000::hero::Sword" "0000000000000000000000000000000000000001::option::Option<0000000000000000000000000000000000000000::tic_tac_toe::Mark>": STRUCT: - vec: SEQ: TYPENAME: "0000000000000000000000000000000000000000::tic_tac_toe::Mark" "0000000000000000000000000000000000000002::balance::Balance<0000000000000000000000000000000000000000::sea_hero::RUM>": STRUCT: - value: U64 "0000000000000000000000000000000000000002::balance::Balance<0000000000000000000000000000000000000002::sui::SUI>": STRUCT: - value: U64 "0000000000000000000000000000000000000002::balance::Supply<0000000000000000000000000000000000000000::sea_hero::RUM>": STRUCT: - value: U64 "0000000000000000000000000000000000000002::coin::Coin<0000000000000000000000000000000000000002::sui::SUI>": STRUCT: - id: TYPENAME: "0000000000000000000000000000000000000002::object::UID" - balance: TYPENAME: "0000000000000000000000000000000000000002::balance::Balance<0000000000000000000000000000000000000002::sui::SUI>" "0000000000000000000000000000000000000002::object::ID": STRUCT: - bytes: TYPENAME: AccountAddress "0000000000000000000000000000000000000002::object::UID": STRUCT: - id: TYPENAME: "0000000000000000000000000000000000000002::object::ID" "0000000000000000000000000000000000000002::sui::SUI": STRUCT: - dummy_field: BOOL "0000000000000000000000000000000000000002::tx_context::TxContext": STRUCT: - signer: TYPENAME: Signer - tx_hash: BYTES - epoch: U64 - ids_created: U64 AccountAddress: NEWTYPESTRUCT: TUPLEARRAY: CONTENT: U8 SIZE: 20 Signer: NEWTYPESTRUCT: TUPLEARRAY: CONTENT: U8 SIZE: 20 ``` This contains all declared structs, all structs passed as arguments to `entry` functions, and all of the types they (transitively) depend on. Note: there are a number of fixes we need to make to `SerdeLayoutGenerator` to really get this working. The biggest problem now is that the generator will not create layout files for open types (i.e., struct declarations with unbound type parameters). Note: this PR also upgrades the Move version to pull in a fix to `SerdeLayoutGenerator` that I needed. This required a few updates to the gas logic--please take a look @oxade.
JonathanH4H · Nov 8, 2022 · 672c5ec · 672c5ec
1 parent 25265ef
commit 672c5ec
Show file tree

Hide file tree

Showing 8 changed files with 148 additions and 6 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/sui-framework-build/Cargo.toml b/crates/sui-framework-build/Cargo.toml
@@ -12,6 +12,7 @@ anyhow = { version = "1.0.64", features = ["backtrace"] }
 fastcrypto = { workspace = true }
 once_cell = "1.16"
 
+serde-reflection = "0.3.6"
 sui-types = { path = "../sui-types" }
 sui-verifier = { path = "../../crates/sui-verifier" }
 

diff --git a/crates/sui-framework-build/src/compiled_package.rs b/crates/sui-framework-build/src/compiled_package.rs
@@ -1,17 +1,28 @@
 // Copyright (c) Mysten Labs, Inc.
 // SPDX-License-Identifier: Apache-2.0
 
-use std::{collections::HashSet, path::PathBuf};
+use std::{
+    collections::{BTreeSet, HashSet},
+    path::PathBuf,
+};
 
 use fastcrypto::encoding::Base64;
-use move_binary_format::CompiledModule;
-use move_bytecode_utils::{module_cache::GetModule, Modules};
+use move_binary_format::{
+    access::ModuleAccess,
+    normalized::{self, Type},
+    CompiledModule,
+};
+use move_bytecode_utils::{layout::SerdeLayoutBuilder, module_cache::GetModule, Modules};
 use move_compiler::compiled_unit::CompiledUnitEnum;
-use move_core_types::{account_address::AccountAddress, language_storage::ModuleId};
+use move_core_types::{
+    account_address::AccountAddress,
+    language_storage::{ModuleId, StructTag, TypeTag},
+};
 use move_package::{
     compilation::compiled_package::CompiledPackage as MoveCompiledPackage,
     BuildConfig as MoveBuildConfig,
 };
+use serde_reflection::Registry;
 use sui_types::{
     error::{SuiError, SuiResult},
     MOVE_STDLIB_ADDRESS, SUI_FRAMEWORK_ADDRESS,
@@ -249,6 +260,72 @@ impl CompiledPackage {
 
         Ok(())
     }
+
+    /// Generate layout schemas for all types declared by this package, as well as
+    /// all struct types passed into `entry` functions declared by modules in this package
+    /// (either directly or by reference).
+    /// These layout schemas can be consumed by clients (e.g., the TypeScript SDK) to enable
+    /// BCS serialization/deserialization of the package's objects, tx arguments, and events.
+    pub fn generate_struct_layouts(&self) -> Registry {
+        let mut package_types = BTreeSet::new();
+        for m in self.get_modules() {
+            let normalized_m = normalized::Module::new(m);
+            // 1. generate struct layouts for all declared types
+            'structs: for (name, s) in normalized_m.structs {
+                let mut dummy_type_parameters = Vec::new();
+                for t in &s.type_parameters {
+                    if t.is_phantom {
+                        // if all of t's type parameters are phantom, we can generate a type layout
+                        // we make this happen by creating a StructTag with dummy `type_params`, since the layout generator won't look at them.
+                        // we need to do this because SerdeLayoutBuilder will refuse to generate a layout for any open StructTag, but phantom types
+                        // cannot affect the layout of a struct, so we just use dummy values
+                        dummy_type_parameters.push(TypeTag::Signer)
+                    } else {
+                        // open type--do not attempt to generate a layout
+                        // TODO: handle generating layouts for open types?
+                        continue 'structs;
+                    }
+                }
+                debug_assert!(dummy_type_parameters.len() == s.type_parameters.len());
+                package_types.insert(StructTag {
+                    address: *m.address(),
+                    module: m.name().to_owned(),
+                    name,
+                    type_params: dummy_type_parameters,
+                });
+            }
+            // 2. generate struct layouts for all parameters of `entry` funs
+            for (_name, f) in normalized_m.exposed_functions {
+                if f.is_entry {
+                    for t in f.parameters {
+                        let tag_opt = match t.clone() {
+                            Type::Address
+                            | Type::Bool
+                            | Type::Signer
+                            | Type::TypeParameter(_)
+                            | Type::U8
+                            | Type::U16
+                            | Type::U32
+                            | Type::U64
+                            | Type::U128
+                            | Type::U256
+                            | Type::Vector(_) => continue,
+                            Type::Reference(t) | Type::MutableReference(t) => t.into_struct_tag(),
+                            s @ Type::Struct { .. } => s.into_struct_tag(),
+                        };
+                        if let Some(tag) = tag_opt {
+                            package_types.insert(tag);
+                        }
+                    }
+                }
+            }
+        }
+        let mut layout_builder = SerdeLayoutBuilder::new(self);
+        for typ in &package_types {
+            layout_builder.build_struct_layout(typ).unwrap();
+        }
+        layout_builder.into_registry()
+    }
 }
 
 impl Default for BuildConfig {

diff --git a/crates/sui-framework-build/src/lib.rs b/crates/sui-framework-build/src/lib.rs
@@ -3,5 +3,9 @@
 
 pub mod compiled_package;
 
+#[cfg(test)]
+#[path = "unit_tests/build_tests.rs"]
+mod build_tests;
+
 const SUI_PACKAGE_NAME: &str = "Sui";
 const MOVE_STDLIB_PACKAGE_NAME: &str = "MoveStdlib";
diff --git a/crates/sui-framework-build/src/unit_tests/build_tests.rs b/crates/sui-framework-build/src/unit_tests/build_tests.rs
@@ -0,0 +1,24 @@
+// Copyright (c) Mysten Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::path::Path;
+
+use crate::compiled_package::BuildConfig;
+
+#[test]
+fn generate_struct_layouts() {
+    // build the Sui framework and generate struct layouts to make sure nothing crashes
+    let mut path = Path::new(env!("CARGO_MANIFEST_DIR"))
+        .parent()
+        .unwrap()
+        .to_path_buf();
+    path.push("sui-framework");
+    let pkg = BuildConfig::default().build(path).unwrap();
+    let registry = pkg.generate_struct_layouts();
+    // check for a couple of types that aren't likely to go away
+    assert!(registry.contains_key("0000000000000000000000000000000000000001::string::String"));
+    assert!(registry.contains_key("0000000000000000000000000000000000000002::object::UID"));
+    assert!(
+        registry.contains_key("0000000000000000000000000000000000000002::tx_context::TxContext")
+    );
+}
diff --git a/crates/sui/Cargo.toml b/crates/sui/Cargo.toml
@@ -10,6 +10,7 @@ edition = "2021"
 anyhow = { version = "1.0.64", features = ["backtrace"] }
 serde = { version = "1.0.144", features = ["derive"] }
 serde_json = "1.0.83"
+serde_yaml = "0.8.26"
 signature = "1.6.0"
 camino = "1.1.1"
 tokio = { version = "1.20.1", features = ["full"] }

diff --git a/crates/sui/src/sui_move/build.rs b/crates/sui/src/sui_move/build.rs
@@ -5,16 +5,29 @@ use clap::Parser;
 use move_cli::base::{self, build};
 use move_package::BuildConfig as MoveBuildConfig;
 use serde_json::json;
-use std::path::{Path, PathBuf};
+use std::{
+    fs,
+    path::{Path, PathBuf},
+};
 use sui_framework_build::compiled_package::BuildConfig;
 
+const LAYOUTS_DIR: &str = "layouts";
+const STRUCT_LAYOUTS_FILENAME: &str = "struct_layouts.yaml";
+
 #[derive(Parser)]
 pub struct Build {
     #[clap(flatten)]
     pub build: build::Build,
     /// Whether we are printing in base64.
     #[clap(long, global = true)]
     pub dump_bytecode_as_base64: bool,
+    /// If true, generate struct layout schemas for
+    /// all struct types passed into `entry` functions declared by modules in this package
+    /// These layout schemas can be consumed by clients (e.g.,
+    /// the TypeScript SDK) to enable serialization/deserialization of transaction arguments
+    /// and events.
+    #[clap(long, global = true)]
+    pub generate_struct_layouts: bool,
 }
 
 impl Build {
@@ -24,13 +37,19 @@ impl Build {
         build_config: MoveBuildConfig,
     ) -> anyhow::Result<()> {
         let rerooted_path = base::reroot_path(path)?;
-        Self::execute_internal(&rerooted_path, build_config, self.dump_bytecode_as_base64)
+        Self::execute_internal(
+            &rerooted_path,
+            build_config,
+            self.dump_bytecode_as_base64,
+            self.generate_struct_layouts,
+        )
     }
 
     pub fn execute_internal(
         rerooted_path: &Path,
         config: MoveBuildConfig,
         dump_bytecode_as_base64: bool,
+        generate_struct_layouts: bool,
     ) -> anyhow::Result<()> {
         let pkg = sui_framework::build_move_package(
             rerooted_path,
@@ -43,6 +62,18 @@ impl Build {
         if dump_bytecode_as_base64 {
             println!("{}", json!(pkg.get_package_base64()))
         }
+
+        if generate_struct_layouts {
+            let layout_str = serde_yaml::to_string(&pkg.generate_struct_layouts()).unwrap();
+            // store under <package_path>/build/<package_name>/layouts/struct_layouts.yaml
+            let mut layout_filename = pkg.path;
+            layout_filename.push("build");
+            layout_filename.push(pkg.package.compiled_package_info.package_name.as_str());
+            layout_filename.push(LAYOUTS_DIR);
+            layout_filename.push(STRUCT_LAYOUTS_FILENAME);
+            fs::write(layout_filename, layout_str)?
+        }
+
         Ok(())
     }
 }
diff --git a/crates/sui/src/sui_move/unit_test.rs b/crates/sui/src/sui_move/unit_test.rs
@@ -27,13 +27,15 @@ impl Test {
         let rerooted_path = base::reroot_path(path)?;
         // pre build for Sui-specific verifications
         let dump_bytecode_as_base64 = false;
+        let generate_struct_layouts: bool = false;
         build::Build::execute_internal(
             &rerooted_path,
             BuildConfig {
                 test_mode: true, // make sure to verify tests
                 ..build_config.clone()
             },
             dump_bytecode_as_base64,
+            generate_struct_layouts,
         )?;
         sui_framework::run_move_unit_tests(
             &rerooted_path,