From 3c30cf97c3ccbba3e7a18b6c0097e981b673c79f Mon Sep 17 00:00:00 2001 From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com> Date: Sat, 4 Dec 2021 17:16:55 +0100 Subject: [PATCH 01/11] remove unhygenic imports, reorder trait --- turbolift_macros/src/lib.rs | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs index a097319b..32f9ffb0 100644 --- a/turbolift_macros/src/lib.rs +++ b/turbolift_macros/src/lib.rs @@ -74,28 +74,27 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS let sanitized_file = extract_function::get_sanitized_file(&function); // todo make code below hygienic in case sanitized_file also imports from actix_web let main_file = q! { - use turbolift::actix_web::{self, get, web, HttpResponse, HttpRequest, Result, Responder}; + #sanitized_file use turbolift::tokio_compat_02::FutureExt; - #sanitized_file #dummy_function #target_function - #[get("/health-probe")] - async fn health_probe(_req: HttpRequest) -> impl Responder { - HttpResponse::Ok() + #[turbolift::actix_web::get("/health-probe")] + async fn health_probe(_req: turbolift::actix_web::HttpRequest) -> impl turbolift::actix_web::Responder { + turbolift::actix_web::HttpResponse::Ok() } - #[get(#wrapper_route)] + #[turbolift::actix_web::get(#wrapper_route)] #[turbolift::tracing::instrument] - async fn turbolift_wrapper(web::Path((#untyped_params_tokens_with_run_id)): web::Path<(#param_types)>) -> Result { + async fn turbolift_wrapper(turbolift::actix_web::web::Path((#untyped_params_tokens_with_run_id)): web::Path<(#param_types)>) -> Result { Ok( - HttpResponse::Ok() + turbolift::actix_web::HttpResponse::Ok() .json(#function_name(#untyped_params_tokens)) ) } - #[actix_web::main] + #[turbolift::actix_web::main] #[turbolift::tracing::instrument] async fn main() -> std::io::Result<()> { use actix_web::{App, HttpServer, HttpRequest, web}; @@ -103,9 +102,9 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS let args: Vec = std::env::args().collect(); let ip_and_port = &args[1]; turbolift::tracing::info!("service main() started. ip_and_port parsed."); - HttpServer::new( + turbolift::actix_web::HttpServer::new( || - App::new() + turbolift::actix_web::App::new() .service( turbolift_wrapper ) @@ -113,10 +112,10 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS health_probe ) .default_service( - web::get() + turbolift::actix_web::web::get() .to( - |req: HttpRequest| - HttpResponse::NotFound().body( + |req: turbolift::actix_web::HttpRequest| + turbolift::actix_web::HttpResponse::NotFound().body( format!("endpoint not found: {}", req.uri()) ) ) From ef9283ce1fd5ca9c8285964f4ecf07e3bd4974bc Mon Sep 17 00:00:00 2001 From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com> Date: Sat, 4 Dec 2021 19:16:09 +0100 Subject: [PATCH 02/11] don't use get macro, it expands unhygenically --- turbolift_macros/src/lib.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs index 32f9ffb0..55ffcf28 100644 --- a/turbolift_macros/src/lib.rs +++ b/turbolift_macros/src/lib.rs @@ -80,14 +80,12 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS #dummy_function #target_function - #[turbolift::actix_web::get("/health-probe")] async fn health_probe(_req: turbolift::actix_web::HttpRequest) -> impl turbolift::actix_web::Responder { turbolift::actix_web::HttpResponse::Ok() } - #[turbolift::actix_web::get(#wrapper_route)] #[turbolift::tracing::instrument] - async fn turbolift_wrapper(turbolift::actix_web::web::Path((#untyped_params_tokens_with_run_id)): web::Path<(#param_types)>) -> Result { + async fn turbolift_wrapper(turbolift::actix_web::web::Path((#untyped_params_tokens_with_run_id)): web::Path<(#param_types)>) -> impl turbolift::actix_web::Responder { Ok( turbolift::actix_web::HttpResponse::Ok() .json(#function_name(#untyped_params_tokens)) @@ -105,11 +103,11 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS turbolift::actix_web::HttpServer::new( || turbolift::actix_web::App::new() - .service( - turbolift_wrapper + .route( + #wrapper_route, turbolift::actix_web::web::get().to(turbolift_wrapper) ) - .service( - health_probe + .route( + "/health-probe", turbolift::actix_web::web::get().to(health_probe) ) .default_service( turbolift::actix_web::web::get() From 6f64c351a6a5250b98c661cf12129a58b5744f96 Mon Sep 17 00:00:00 2001 From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com> Date: Sat, 4 Dec 2021 23:20:16 +0100 Subject: [PATCH 03/11] fix remaining compilation errors --- turbolift_macros/src/lib.rs | 60 ++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs index 55ffcf28..f0cf2345 100644 --- a/turbolift_macros/src/lib.rs +++ b/turbolift_macros/src/lib.rs @@ -85,45 +85,45 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS } #[turbolift::tracing::instrument] - async fn turbolift_wrapper(turbolift::actix_web::web::Path((#untyped_params_tokens_with_run_id)): web::Path<(#param_types)>) -> impl turbolift::actix_web::Responder { + async fn turbolift_wrapper(turbolift::actix_web::web::Path((#untyped_params_tokens_with_run_id)): turbolift::actix_web::web::Path<(#param_types)>) -> impl turbolift::actix_web::Responder { Ok( turbolift::actix_web::HttpResponse::Ok() .json(#function_name(#untyped_params_tokens)) ) } - #[turbolift::actix_web::main] #[turbolift::tracing::instrument] - async fn main() -> std::io::Result<()> { - use actix_web::{App, HttpServer, HttpRequest, web}; - - let args: Vec = std::env::args().collect(); - let ip_and_port = &args[1]; - turbolift::tracing::info!("service main() started. ip_and_port parsed."); - turbolift::actix_web::HttpServer::new( - || - turbolift::actix_web::App::new() - .route( - #wrapper_route, turbolift::actix_web::web::get().to(turbolift_wrapper) - ) - .route( - "/health-probe", turbolift::actix_web::web::get().to(health_probe) - ) - .default_service( - turbolift::actix_web::web::get() - .to( - |req: turbolift::actix_web::HttpRequest| - turbolift::actix_web::HttpResponse::NotFound().body( - format!("endpoint not found: {}", req.uri()) + fn main() { + turbolift::actix_web::rt::System::new("main".to_string()) + .block_on(async move { + let args: Vec = std::env::args().collect(); + let ip_and_port = &args[1]; + turbolift::tracing::info!("service main() started. ip_and_port parsed."); + turbolift::actix_web::HttpServer::new( + || + turbolift::actix_web::App::new() + .route( + "#wrapper_route", turbolift::actix_web::web::get().to(turbolift_wrapper) + ) + .route( + "/health-probe", turbolift::actix_web::web::get().to(health_probe) + ) + .default_service( + turbolift::actix_web::web::get() + .to( + |req: turbolift::actix_web::HttpRequest| + turbolift::actix_web::HttpResponse::NotFound().body( + format!("endpoint not found: {}", req.uri()) + ) ) ) - ) - ) - .bind(ip_and_port)? - .run() - .compat() - .await - } + ) + .bind(ip_and_port)? + .run() + .compat() + .await + }); + } }; // copy all files in repo into cache From f28351449ec790e5e0366a83ef54b20c8e2eb32b Mon Sep 17 00:00:00 2001 From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com> Date: Sun, 5 Dec 2021 09:17:08 +0100 Subject: [PATCH 04/11] removed unused response wrapper --- turbolift_macros/src/lib.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs index f0cf2345..4265cd56 100644 --- a/turbolift_macros/src/lib.rs +++ b/turbolift_macros/src/lib.rs @@ -86,10 +86,8 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS #[turbolift::tracing::instrument] async fn turbolift_wrapper(turbolift::actix_web::web::Path((#untyped_params_tokens_with_run_id)): turbolift::actix_web::web::Path<(#param_types)>) -> impl turbolift::actix_web::Responder { - Ok( - turbolift::actix_web::HttpResponse::Ok() - .json(#function_name(#untyped_params_tokens)) - ) + turbolift::actix_web::HttpResponse::Ok() + .json(#function_name(#untyped_params_tokens)) } #[turbolift::tracing::instrument] From cb55c246cd810a3400d94e1acd3984a33b41412c Mon Sep 17 00:00:00 2001 From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com> Date: Sun, 5 Dec 2021 09:31:25 +0100 Subject: [PATCH 05/11] consume result --- turbolift_macros/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs index 4265cd56..e396a049 100644 --- a/turbolift_macros/src/lib.rs +++ b/turbolift_macros/src/lib.rs @@ -120,7 +120,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS .run() .compat() .await - }); + }).unwrap(); } }; From 79050a79f259495a797c180768a2983697b37982 Mon Sep 17 00:00:00 2001 From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com> Date: Sun, 5 Dec 2021 10:33:41 +0100 Subject: [PATCH 06/11] add debug statements --- turbolift_macros/src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs index e396a049..4558f8ea 100644 --- a/turbolift_macros/src/lib.rs +++ b/turbolift_macros/src/lib.rs @@ -215,13 +215,16 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS let mut platform = #distribution_platform.lock().await; if !platform.has_declared(#original_target_function_name) { + println!("declaring target function"); platform .declare(#original_target_function_name, #project_source_binary) .compat() .await?; + println!("target function declared"); } let params = #params_vec.join("/"); + println!("dispatching request"); let resp_string = platform .dispatch( #original_target_function_name, @@ -229,6 +232,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS ) .compat() .await?; + println!("response: {}", resp_string); Ok(turbolift::serde_json::from_str(&resp_string)?) } }; From 60b6a9c9ccbc038ec776900b895fc2c0642a424c Mon Sep 17 00:00:00 2001 From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com> Date: Sun, 5 Dec 2021 10:49:40 +0100 Subject: [PATCH 07/11] fix macro definition --- turbolift_macros/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs index 4558f8ea..4d1684ab 100644 --- a/turbolift_macros/src/lib.rs +++ b/turbolift_macros/src/lib.rs @@ -101,7 +101,7 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS || turbolift::actix_web::App::new() .route( - "#wrapper_route", turbolift::actix_web::web::get().to(turbolift_wrapper) + #wrapper_route, turbolift::actix_web::web::get().to(turbolift_wrapper) ) .route( "/health-probe", turbolift::actix_web::web::get().to(health_probe) From 40689aa676890c5da78d28304225593ed0843fe8 Mon Sep 17 00:00:00 2001 From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com> Date: Sun, 5 Dec 2021 12:30:32 +0100 Subject: [PATCH 08/11] Revert "add debug statements" This reverts commit 79050a79f259495a797c180768a2983697b37982. --- turbolift_macros/src/lib.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/turbolift_macros/src/lib.rs b/turbolift_macros/src/lib.rs index 4d1684ab..b2436768 100644 --- a/turbolift_macros/src/lib.rs +++ b/turbolift_macros/src/lib.rs @@ -215,16 +215,13 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS let mut platform = #distribution_platform.lock().await; if !platform.has_declared(#original_target_function_name) { - println!("declaring target function"); platform .declare(#original_target_function_name, #project_source_binary) .compat() .await?; - println!("target function declared"); } let params = #params_vec.join("/"); - println!("dispatching request"); let resp_string = platform .dispatch( #original_target_function_name, @@ -232,7 +229,6 @@ pub fn on(distribution_platform_: TokenStream, function_: TokenStream) -> TokenS ) .compat() .await?; - println!("response: {}", resp_string); Ok(turbolift::serde_json::from_str(&resp_string)?) } }; From aa15ae366f95cffbc50fce2b7c88160ffc8597ac Mon Sep 17 00:00:00 2001 From: Dominic Burkart <1351120+DominicBurkart@users.noreply.github.com> Date: Sun, 5 Dec 2021 12:36:29 +0100 Subject: [PATCH 09/11] add feature definition in tests --- examples/kubernetes_example/src/main.rs | 1 + examples/local_queue_example/src/main.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/examples/kubernetes_example/src/main.rs b/examples/kubernetes_example/src/main.rs index 8deb4572..8ebbef1b 100644 --- a/examples/kubernetes_example/src/main.rs +++ b/examples/kubernetes_example/src/main.rs @@ -1,3 +1,4 @@ +#![feature(async_closure)] // not actually necessary, just added to test that feature defs work. #[macro_use] extern crate lazy_static; #[macro_use(c)] diff --git a/examples/local_queue_example/src/main.rs b/examples/local_queue_example/src/main.rs index 6b09adc9..b305d888 100644 --- a/examples/local_queue_example/src/main.rs +++ b/examples/local_queue_example/src/main.rs @@ -1,3 +1,4 @@ +#![feature(async_closure)] // not actually necessary, just added to test that feature defs work. extern crate proc_macro; use futures::future::try_join_all; use rand; From 26f6abec38ac73a21deafcb9cd6ff49ce4efa383 Mon Sep 17 00:00:00 2001 From: Dominic Burkart Date: Sun, 5 Dec 2021 13:08:36 +0100 Subject: [PATCH 10/11] Update readme --- README.md | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index fa5e44f0..0e2e9c5d 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![last commit](https://img.shields.io/github/last-commit/dominicburkart/turbolift)](https://github.com/DominicBurkart/turbolift) [![website](https://img.shields.io/badge/-website-blue)](https://dominic.computer/turbolift) -Turbolift is a distribution interface for rust. It's designed to make +Turbolift is a prototype distribution interface for rust. It's designed to make distribution easier and more maintainable by extracting and distributing specific functions and their dependencies from a larger rust application. Turbolift then acts as the glue between these extracted mini-apps and @@ -16,20 +16,9 @@ Look in the [examples](https://github.com/DominicBurkart/turbolift/tree/main/exa directory for full projects with working syntax examples. An [external example](https://github.com/DominicBurkart/turbolift_example) is maintained that can be used as a template repo. -## Distribution as an afterthought - -Turbolift allows developers to turn normal rust functions into distributed services -just by tagging them with a macro. Right now, Turbolift only works with K8s, though -it's designed to be extended to other cluster management utilities. - -## Orchestration with a feature flag - -Distribution is feature-gated in Turbolift, so it's easy to activate distribution -for some builds and deactivate it for others (while developing locally, for -example). - ## Important implementation notes +- Distribution is feature-gated in Turbolift to facilitate development / conditional distribution. The feature is called "distributed." - implemented over http using `reqwest` and `actix-web` (no current plans to refactor to use a lower level network protocol). - assumes a secure network– function parameters are sent in plaintext to the @@ -42,14 +31,12 @@ More information is available on the [project homepage](https://dominic.computer ## Current Limitations -- DO NOT RUN TURBOLIFT ON A PUBLIC-FACING CLUSTER. - *Because of reliance on unstable proc_macro::Span features, all programs using turbolift need to be built with an unstable nightly compiler flag (e.g. `RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo build`)* ([tracking issue](https://github.com/rust-lang/rust/issues/54725)). - Functions are assumed to be pure (lacking side-effects such as writing to the file system or mutation of a global variable). -*Today, this is not enforced by the code.* - For a function to be distributed, its inputs and outputs have to be (de)serializable with [Serde](https://github.com/serde-rs/serde). - Distributed functions cannot be nested in other functions. @@ -82,11 +69,3 @@ readiness check instead of just sleeping ([code location](https://github.com/Dom tagged function into an async function (to provide an identical API), but don't build any microservices or alter any code. - [ ] build cross-architecture compilation tests into the CI. - -## Current tech debt todo - -- [ ] start reducing ginormous API, right now basically everything is public -- [ ] refactor split between turbolift_internals and turbolift_macros -- [ ] improve names -- [ ] send params in json as payload instead of directly in the url -- [ ] we need to do a better job of cleaning up docker images, locally and in the cluster. From 15824c5b2d6b577999aa2da3cd9ad1e08a9b69f2 Mon Sep 17 00:00:00 2001 From: Dominic Burkart Date: Sun, 5 Dec 2021 13:10:48 +0100 Subject: [PATCH 11/11] Update README.md --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0e2e9c5d..492a9cbe 100644 --- a/README.md +++ b/README.md @@ -16,14 +16,15 @@ Look in the [examples](https://github.com/DominicBurkart/turbolift/tree/main/exa directory for full projects with working syntax examples. An [external example](https://github.com/DominicBurkart/turbolift_example) is maintained that can be used as a template repo. -## Important implementation notes +## Notes +- Turbolift works as a proof-of-concept, but has not been optimized to shrink compilation time/requirements. - Distribution is feature-gated in Turbolift to facilitate development / conditional distribution. The feature is called "distributed." -- implemented over http using `reqwest` and `actix-web` (no current plans to +- Turbolift is implemented over http using `reqwest` and `actix-web` (no current plans to refactor to use a lower level network protocol). -- assumes a secure network– function parameters are sent in plaintext to the +- Turbolift assumes a secure network– function parameters are sent in plaintext to the microservice. -- source vulnerability: when building, anything in the project directory or in +- When building, anything in the project directory or in local dependencies declared in the project manifest could be bundled and sent over the network to workers.