split spec_compliance system tests

icwizardmonke · Jul 3, 2023 · a985cae · a985cae
1 parent 7742d96
commit a985cae
Show file tree

Hide file tree

Showing 7 changed files with 151 additions and 16 deletions.
diff --git a/rs/tests/Cargo.toml b/rs/tests/Cargo.toml
@@ -190,12 +190,20 @@ name = "ic-systest-canister-http-correctness"
 path = "networking/canister_http_correctness_test.rs"
 
 [[bin]]
-name = "ic-systest-spec-compliance-application"
-path = "testing_verification/spec_compliance_application_test.rs"
+name = "ic-systest-spec-compliance-application-subnet"
+path = "testing_verification/spec_compliance_application_subnet_test.rs"
 
 [[bin]]
-name = "ic-systest-spec-compliance-system"
-path = "testing_verification/spec_compliance_system_test.rs"
+name = "ic-systest-spec-compliance-system-subnet"
+path = "testing_verification/spec_compliance_system_subnet_test.rs"
+
+[[bin]]
+name = "ic-systest-spec-compliance-system-api-application-subnet"
+path = "testing_verification/spec_compliance_system_api_application_subnet_test.rs"
+
+[[bin]]
+name = "ic-systest-spec-compliance-system-api-system-subnet"
+path = "testing_verification/spec_compliance_system_api_system_subnet_test.rs"
 
 [[bin]]
 name = "ic-systest-tecdsa-add-nodes"

diff --git a/rs/tests/src/spec_compliance.rs b/rs/tests/src/spec_compliance.rs
@@ -129,6 +129,8 @@ pub fn test_subnet(
     env: TestEnv,
     test_subnet_type: Option<SubnetType>,
     peer_subnet_type: Option<SubnetType>,
+    excluded_tests: Vec<&str>,
+    included_tests: Vec<&str>,
 ) {
     let log = env.logger();
     let topology_snapshot = &env.topology_snapshot();
@@ -145,14 +147,17 @@ pub fn test_subnet(
         .into_os_string()
         .into_string()
         .unwrap();
+    let mut all_excluded_tests = excluded_tests;
+    all_excluded_tests.append(&mut EXCLUDED.to_vec());
     with_endpoint(
         env,
         test_subnet,
         peer_subnet,
         httpbin,
         ic_ref_test_path,
         log,
-        EXCLUDED.to_vec(),
+        all_excluded_tests,
+        included_tests,
     );
 }
 
@@ -187,6 +192,7 @@ pub fn with_endpoint(
     ic_ref_test_path: String,
     log: Logger,
     excluded_tests: Vec<&str>,
+    included_tests: Vec<&str>,
 ) {
     let node = test_subnet.nodes().next().unwrap();
     let test_subnet_config = subnet_config(&test_subnet);
@@ -200,7 +206,7 @@ pub fn with_endpoint(
         )
         .arg("-j20")
         .arg("--pattern")
-        .arg(tests_to_pattern(excluded_tests))
+        .arg(tests_to_pattern(excluded_tests, included_tests))
         .arg("--endpoint")
         .arg(node.get_public_url().to_string())
         .arg("--httpbin")
@@ -217,6 +223,9 @@ pub fn with_endpoint(
     assert!(status.success());
 }
 
-fn tests_to_pattern(tests: Vec<&str>) -> String {
-    format!("!({})", tests.join(" || "))
+fn tests_to_pattern(excluded_tests: Vec<&str>, included_tests: Vec<&str>) -> String {
+    let inner = format!("!({})", excluded_tests.join(" || "));
+    let mut patterns = vec![inner.as_str()];
+    patterns.append(&mut included_tests.clone());
+    patterns.join(" && ")
 }
diff --git a/rs/tests/testing_verification/BUILD.bazel b/rs/tests/testing_verification/BUILD.bazel
@@ -22,7 +22,7 @@ rust_binary(
 )
 
 system_test(
-    name = "spec_compliance_application_test",
+    name = "spec_compliance_application_subnet_test",
     proc_macro_deps = MACRO_DEPENDENCIES,
     target_compatible_with = ["@platforms//os:linux"],  # requires libssh that does not build on Mac OS
     runtime_deps = GUESTOS_RUNTIME_DEPS + UNIVERSAL_VM_RUNTIME_DEPS + CANISTER_HTTP_RUNTIME_DEPS + IC_HS_RUNTIME_DEPS + [
@@ -32,7 +32,27 @@ system_test(
 )
 
 system_test(
-    name = "spec_compliance_system_test",
+    name = "spec_compliance_system_subnet_test",
+    proc_macro_deps = MACRO_DEPENDENCIES,
+    target_compatible_with = ["@platforms//os:linux"],  # requires libssh that does not build on Mac OS
+    runtime_deps = GUESTOS_RUNTIME_DEPS + UNIVERSAL_VM_RUNTIME_DEPS + CANISTER_HTTP_RUNTIME_DEPS + IC_HS_RUNTIME_DEPS + [
+        "//ic-os/guestos:rootfs/dev-certs/canister_http_test_ca.cert",
+    ],
+    deps = DEPENDENCIES + ["//rs/tests"],
+)
+
+system_test(
+    name = "spec_compliance_system_api_application_subnet_test",
+    proc_macro_deps = MACRO_DEPENDENCIES,
+    target_compatible_with = ["@platforms//os:linux"],  # requires libssh that does not build on Mac OS
+    runtime_deps = GUESTOS_RUNTIME_DEPS + UNIVERSAL_VM_RUNTIME_DEPS + CANISTER_HTTP_RUNTIME_DEPS + IC_HS_RUNTIME_DEPS + [
+        "//ic-os/guestos:rootfs/dev-certs/canister_http_test_ca.cert",
+    ],
+    deps = DEPENDENCIES + ["//rs/tests"],
+)
+
+system_test(
+    name = "spec_compliance_system_api_system_subnet_test",
     proc_macro_deps = MACRO_DEPENDENCIES,
     target_compatible_with = ["@platforms//os:linux"],  # requires libssh that does not build on Mac OS
     runtime_deps = GUESTOS_RUNTIME_DEPS + UNIVERSAL_VM_RUNTIME_DEPS + CANISTER_HTTP_RUNTIME_DEPS + IC_HS_RUNTIME_DEPS + [

diff --git a/...ation/spec_compliance_application_test.rs → ...pec_compliance_application_subnet_test.rs b/...ation/spec_compliance_application_test.rs → ...pec_compliance_application_subnet_test.rs
@@ -4,8 +4,8 @@ Title:: Specification compliance test
 Goal:: Ensure that the replica implementation is compliant with the formal specification.
 
 Runbook::
-. Set up application subnet containing one node
-. Run ic-ref-test
+. Set up system and application subnet containing two nodes each
+. Run ic-ref-test against system subnet
 
 Success:: The ic-ref-test binary does not return an error.
 
@@ -24,7 +24,13 @@ pub fn config(env: TestEnv) {
 }
 
 pub fn test(env: TestEnv) {
-    test_subnet(env, Some(SubnetType::Application), None);
+    test_subnet(
+        env,
+        Some(SubnetType::Application),
+        None,
+        vec!["($0 ~ /API availability/)"],
+        vec![],
+    );
 }
 
 fn main() -> Result<()> {

diff --git a/...rification/spec_compliance_system_test.rs → ...nce_system_api_application_subnet_test.rs b/...rification/spec_compliance_system_test.rs → ...nce_system_api_application_subnet_test.rs
@@ -4,8 +4,8 @@ Title:: Specification compliance test
 Goal:: Ensure that the replica implementation is compliant with the formal specification.
 
 Runbook::
-. Set up system subnet containing one node
-. Run ic-ref-test
+. Set up system and application subnet containing two nodes each
+. Run ic-ref-test against system subnet
 
 Success:: The ic-ref-test binary does not return an error.
 
@@ -24,7 +24,13 @@ pub fn config(env: TestEnv) {
 }
 
 pub fn test(env: TestEnv) {
-    test_subnet(env, None, Some(SubnetType::Application));
+    test_subnet(
+        env,
+        Some(SubnetType::Application),
+        None,
+        vec![],
+        vec!["($0 ~ /API availability/)"],
+    );
 }
 
 fn main() -> Result<()> {

diff --git a/rs/tests/testing_verification/spec_compliance_system_api_system_subnet_test.rs b/rs/tests/testing_verification/spec_compliance_system_api_system_subnet_test.rs
@@ -0,0 +1,43 @@
+/* tag::catalog[]
+Title:: Specification compliance test
+
+Goal:: Ensure that the replica implementation is compliant with the formal specification.
+
+Runbook::
+. Set up system system and application subnet containing two nodes each
+. Run ic-ref-test against application subnet
+
+Success:: The ic-ref-test binary does not return an error.
+
+end::catalog[] */
+
+use anyhow::Result;
+
+use ic_registry_subnet_type::SubnetType;
+use ic_tests::driver::group::SystemTestGroup;
+use ic_tests::driver::test_env::TestEnv;
+use ic_tests::spec_compliance::{config_impl, test_subnet};
+use ic_tests::systest;
+
+pub fn config(env: TestEnv) {
+    config_impl(env);
+}
+
+pub fn test(env: TestEnv) {
+    test_subnet(
+        env,
+        None,
+        Some(SubnetType::Application),
+        vec![],
+        vec!["($0 ~ /API availability/)"],
+    );
+}
+
+fn main() -> Result<()> {
+    SystemTestGroup::new()
+        .with_setup(config)
+        .add_test(systest!(test))
+        .execute_from_args()?;
+
+    Ok(())
+}
diff --git a/rs/tests/testing_verification/spec_compliance_system_subnet_test.rs b/rs/tests/testing_verification/spec_compliance_system_subnet_test.rs
@@ -0,0 +1,43 @@
+/* tag::catalog[]
+Title:: Specification compliance test
+
+Goal:: Ensure that the replica implementation is compliant with the formal specification.
+
+Runbook::
+. Set up system system and application subnet containing two nodes each
+. Run ic-ref-test against application subnet
+
+Success:: The ic-ref-test binary does not return an error.
+
+end::catalog[] */
+
+use anyhow::Result;
+
+use ic_registry_subnet_type::SubnetType;
+use ic_tests::driver::group::SystemTestGroup;
+use ic_tests::driver::test_env::TestEnv;
+use ic_tests::spec_compliance::{config_impl, test_subnet};
+use ic_tests::systest;
+
+pub fn config(env: TestEnv) {
+    config_impl(env);
+}
+
+pub fn test(env: TestEnv) {
+    test_subnet(
+        env,
+        None,
+        Some(SubnetType::Application),
+        vec!["($0 ~ /API availability/)"],
+        vec![],
+    );
+}
+
+fn main() -> Result<()> {
+    SystemTestGroup::new()
+        .with_setup(config)
+        .add_test(systest!(test))
+        .execute_from_args()?;
+
+    Ok(())
+}