Merge pull request spinframework#381 from lann/host-components

Add support for "host components"

Cargo.toml

@@ -16,6 +16,7 @@ env_logger = "0.9"
 futures = "0.3"
 hippo-openapi = "0.4"
 lazy_static = "1.4.0"
+outbound-redis = { path = "crates/outbound-redis" }
 path-absolutize = "3.0.11"
 reqwest = { version = "0.11", features = ["stream"] }
 semver = "1.0"

crates/engine/Cargo.toml

@@ -8,7 +8,6 @@ authors = ["Fermyon Engineering <[email protected]>"]
 anyhow = "1.0.44"
 bytes = "1.1.0"
 dirs = "4.0"
-outbound-redis = { path = "../outbound-redis" }
 sanitize-filename = "0.3.0"
 spin-config = { path = "../config" }
 spin-manifest = { path = "../manifest" }

crates/engine/src/host_component.rs

@@ -0,0 +1,89 @@
+use std::{any::Any, marker::PhantomData};
+
+use spin_manifest::CoreComponent;
+use wasmtime::Linker;
+
+use crate::RuntimeContext;
+
+/// Represents a host implementation of a Wasm interface.
+pub trait HostComponent: Send + Sync {
+    /// Host component runtime state.
+    type Data: Any + Send;
+
+    /// Add this component to the given Linker, using the given runtime state-getting closure.
+    fn add_to_linker<T>(
+        linker: &mut Linker<RuntimeContext<T>>,
+        data_handle: HostComponentsDataHandle<Self::Data>,
+    ) -> anyhow::Result<()>;
+
+    /// Build a new runtime state object for the given component.
+    fn build_data(&self, component: &CoreComponent) -> anyhow::Result<Self::Data>;
+}
+type HostComponentData = Box<dyn Any + Send>;
+
+type DataBuilder = Box<dyn Fn(&CoreComponent) -> anyhow::Result<HostComponentData> + Send + Sync>;
+
+#[derive(Default)]
+pub(crate) struct HostComponents {
+    data_builders: Vec<DataBuilder>,
+}
+
+impl HostComponents {
+    pub(crate) fn insert<'a, T: 'static, Component: HostComponent + 'static>(
+        &mut self,
+        linker: &'a mut Linker<RuntimeContext<T>>,
+        host_component: Component,
+    ) -> anyhow::Result<()> {
+        let handle = HostComponentsDataHandle {
+            idx: self.data_builders.len(),
+            _phantom: PhantomData,
+        };
+        Component::add_to_linker(linker, handle)?;
+        self.data_builders.push(Box::new(move |c| {
+            Ok(Box::new(host_component.build_data(c)?))
+        }));
+        Ok(())
+    }
+
+    pub(crate) fn build_data(&self, c: &CoreComponent) -> anyhow::Result<HostComponentsData> {
+        Ok(HostComponentsData(
+            self.data_builders
+                .iter()
+                .map(|build_data| build_data(c))
+                .collect::<anyhow::Result<_>>()?,
+        ))
+    }
+}
+
+/// A collection of host component data.
+#[derive(Default)]
+pub struct HostComponentsData(Vec<HostComponentData>);
+
+/// A handle to component data, used in HostComponent::add_to_linker.
+pub struct HostComponentsDataHandle<T> {
+    idx: usize,
+    _phantom: PhantomData<fn(T) -> T>,
+}
+
+impl<T: 'static> HostComponentsDataHandle<T> {
+    /// Get the component data associated with this handle from the RuntimeContext.
+    pub fn get_mut<'a, U>(&self, ctx: &'a mut RuntimeContext<U>) -> &'a mut T {
+        ctx.host_components_data
+            .0
+            .get_mut(self.idx)
+            .unwrap()
+            .downcast_mut()
+            .unwrap()
+    }
+}
+
+impl<T> Clone for HostComponentsDataHandle<T> {
+    fn clone(&self) -> Self {
+        Self {
+            idx: self.idx,
+            _phantom: PhantomData,
+        }
+    }
+}
+
+impl<T> Copy for HostComponentsDataHandle<T> {}

crates/engine/src/lib.rs

@@ -2,10 +2,13 @@
 
 #![deny(missing_docs)]
 
+/// Host components.
+pub mod host_component;
 /// Input / Output redirects.
 pub mod io;
 
 use anyhow::{bail, Context, Result};
+use host_component::{HostComponent, HostComponents, HostComponentsData};
 use io::IoStreamRedirects;
 use spin_config::{host_component::ComponentConfig, Resolver};
 use spin_manifest::{Application, CoreComponent, DirectoryMount, ModuleSource};
@@ -56,8 +59,8 @@ pub struct RuntimeContext<T> {
     pub outbound_http: Option<wasi_outbound_http::OutboundHttp>,
     /// Component configuration.
     pub component_config: Option<spin_config::host_component::ComponentConfig>,
-    /// Outbound Redis configuration.
-    pub outbound_redis: Option<outbound_redis::OutboundRedis>,
+    /// Host components data.
+    pub host_components_data: HostComponentsData,
     /// Generic runtime data that can be configured by specialized engines.
     pub data: Option<T>,
 }
@@ -68,9 +71,10 @@ pub struct Builder<T: Default> {
     linker: Linker<RuntimeContext<T>>,
     store: Store<RuntimeContext<T>>,
     engine: Engine,
+    host_components: HostComponents,
 }
 
-impl<T: Default> Builder<T> {
+impl<T: Default + 'static> Builder<T> {
     /// Creates a new instance of the execution builder.
     pub fn new(config: ExecutionContextConfiguration) -> Result<Builder<T>> {
         Self::with_wasmtime_config(config, Default::default())
@@ -91,12 +95,14 @@ impl<T: Default> Builder<T> {
         let engine = Engine::new(&wasmtime)?;
         let store = Store::new(&engine, data);
         let linker = Linker::new(&engine);
+        let host_components = Default::default();
 
         Ok(Self {
             config,
             linker,
             store,
             engine,
+            host_components,
         })
     }
 
@@ -128,17 +134,19 @@ impl<T: Default> Builder<T> {
         Ok(self)
     }
 
-    /// Configures the ability to execute outbound Redis commands.
-    pub fn link_redis(&mut self) -> Result<&mut Self> {
-        outbound_redis::add_to_linker(&mut self.linker, |ctx| {
-            ctx.outbound_redis.as_mut().unwrap()
-        })?;
+    /// Adds a HostComponent to the execution context.
+    pub fn add_host_component(
+        &mut self,
+        host_component: impl HostComponent + 'static,
+    ) -> Result<&mut Self> {
+        self.host_components
+            .insert(&mut self.linker, host_component)?;
         Ok(self)
     }
 
     /// Builds a new instance of the execution context.
     #[instrument(skip(self))]
-    pub async fn build(&mut self) -> Result<ExecutionContext<T>> {
+    pub async fn build(mut self) -> Result<ExecutionContext<T>> {
         let _sloth_warning = warn_if_slothful();
         let mut components = HashMap::new();
         for c in &self.config.components {
@@ -175,15 +183,13 @@ impl<T: Default> Builder<T> {
             components.insert(c.id.clone(), Component { core, pre });
         }
 
-        let config = self.config.clone();
-        let engine = self.engine.clone();
-
         log::trace!("Execution context initialized.");
 
         Ok(ExecutionContext {
-            config,
-            engine,
+            config: self.config,
+            engine: self.engine,
             components,
+            host_components: Arc::new(self.host_components),
         })
     }
 
@@ -196,7 +202,9 @@ impl<T: Default> Builder<T> {
     pub async fn build_default(
         config: ExecutionContextConfiguration,
     ) -> Result<ExecutionContext<T>> {
-        Self::new(config)?.link_defaults()?.build().await
+        let mut builder = Self::new(config)?;
+        builder.link_defaults()?;
+        builder.build().await
     }
 }
 
@@ -218,6 +226,8 @@ pub struct ExecutionContext<T: Default> {
     pub engine: Engine,
     /// Collection of pre-initialized (and already linked) components.
     pub components: HashMap<String, Component<T>>,
+
+    host_components: Arc<HostComponents>,
 }
 
 impl<T: Default> ExecutionContext<T> {
@@ -344,10 +354,11 @@ impl<T: Default> ExecutionContext<T> {
                 Some(ComponentConfig::new(&component.core.id, resolver.clone())?);
         }
 
+        ctx.host_components_data = self.host_components.build_data(&component.core)?;
+
         ctx.wasi = Some(wasi_ctx.build());
         ctx.experimental_http = Some(experimental_http);
         ctx.outbound_http = Some(outbound_http);
-        ctx.outbound_redis = Some(outbound_redis::OutboundRedis);
         ctx.data = data;
 
         let store = Store::new(&self.engine, ctx);

crates/http/src/lib.rs

@@ -62,7 +62,7 @@ pub struct HttpTrigger {
 impl HttpTrigger {
     /// Creates a new Spin HTTP trigger.
     pub async fn new(
-        mut builder: Builder<SpinHttpData>,
+        builder: Builder<SpinHttpData>,
         app: Application<CoreComponent>,
         address: String,
         tls: Option<TlsConfig>,

crates/object-store/Cargo.toml

@@ -12,6 +12,8 @@ anyhow = "1.0"
 async-trait = "0.1"
 cap-std = "0.24.1"
 serde = { version = "1.0", features = ["derive"] }
+spin-engine = { path = "../engine" }
+spin-manifest = { path = "../manifest" }
 tracing = { version = "0.1", features = ["log"] }
 wasmtime = "0.34"
 wit-bindgen-wasmtime = { git = "https://github.com/bytecodealliance/wit-bindgen", rev = "2f46ce4cc072107153da0cefe15bdc69aa5b84d0" }

crates/object-store/src/file.rs

@@ -7,11 +7,8 @@ use std::{
 use crate::wit::spin_object_store;
 use anyhow::Context;
 use cap_std::{ambient_authority, fs::OpenOptions};
-
-pub type FileObjectStoreData = (
-    FileObjectStore,
-    spin_object_store::SpinObjectStoreTables<FileObjectStore>,
-);
+use spin_engine::host_component::{HostComponent, HostComponentsDataHandle};
+use spin_manifest::CoreComponent;
 
 pub struct FileObjectStore {
     root: cap_std::fs::Dir,
@@ -25,9 +22,29 @@ impl FileObjectStore {
     }
 }
 
-impl From<FileObjectStore> for FileObjectStoreData {
-    fn from(fos: FileObjectStore) -> Self {
-        (fos, Default::default())
+pub struct FileObjectStoreComponent {
+    pub root: Path,
+}
+
+impl HostComponent for FileObjectStoreComponent {
+    type Data = (
+        FileObjectStore,
+        spin_object_store::SpinObjectStoreTables<FileObjectStore>,
+    );
+
+    fn add_to_linker<T>(
+        linker: &mut wasmtime::Linker<spin_engine::RuntimeContext<T>>,
+        data_handle: HostComponentsDataHandle<Self::Data>,
+    ) -> anyhow::Result<()> {
+        crate::add_to_linker(linker, move |ctx| {
+            let (data, table) = data_handle.get_mut(ctx);
+            (data, table)
+        })
+    }
+
+    fn build_data(&self, _component: &CoreComponent) -> anyhow::Result<Self::Data> {
+        let store = FileObjectStore::new(&self.root)?;
+        Ok((store, Default::default()))
     }
 }
 

crates/outbound-redis/Cargo.toml

@@ -9,4 +9,6 @@ doctest = false
 [dependencies]
 anyhow = "1.0"
 redis = { version = "0.21", features = [ "tokio-comp" ] }
+spin-engine = { path = "../engine" }
+spin-manifest = { path = "../manifest" }
 wit-bindgen-wasmtime = { git = "https://github.com/bytecodealliance/wit-bindgen", rev = "2f46ce4cc072107153da0cefe15bdc69aa5b84d0" }

crates/outbound-redis/src/lib.rs

@@ -2,13 +2,29 @@ use outbound_redis::*;
 use redis::Commands;
 
 pub use outbound_redis::add_to_linker;
+use spin_engine::host_component::HostComponent;
 
 wit_bindgen_wasmtime::export!("../../wit/ephemeral/outbound-redis.wit");
 
 /// A simple implementation to support outbound Redis commands.
 #[derive(Default, Clone)]
 pub struct OutboundRedis;
 
+impl HostComponent for OutboundRedis {
+    type Data = Self;
+
+    fn add_to_linker<T>(
+        linker: &mut wit_bindgen_wasmtime::wasmtime::Linker<spin_engine::RuntimeContext<T>>,
+        data_handle: spin_engine::host_component::HostComponentsDataHandle<Self::Data>,
+    ) -> anyhow::Result<()> {
+        add_to_linker(linker, move |ctx| data_handle.get_mut(ctx))
+    }
+
+    fn build_data(&self, _component: &spin_manifest::CoreComponent) -> anyhow::Result<Self::Data> {
+        Ok(Self)
+    }
+}
+
 impl outbound_redis::OutboundRedis for OutboundRedis {
     fn publish(&mut self, address: &str, channel: &str, payload: &[u8]) -> Result<(), Error> {
         let client = redis::Client::open(address).map_err(|_| Error::Error)?;

crates/redis/src/lib.rs

@@ -35,7 +35,7 @@ pub struct RedisTrigger {
 impl RedisTrigger {
     /// Create a new Spin Redis trigger.
     pub async fn new(
-        mut builder: Builder<SpinRedisData>,
+        builder: Builder<SpinRedisData>,
         app: Application<CoreComponent>,
     ) -> Result<Self> {
         let trigger_config = app

crates/testing/src/lib.rs

@@ -96,7 +96,10 @@ impl TestConfig {
         }
     }
 
-    pub async fn prepare_builder<T: Default>(&self, app: Application<CoreComponent>) -> Builder<T> {
+    pub async fn prepare_builder<T: Default + 'static>(
+        &self,
+        app: Application<CoreComponent>,
+    ) -> Builder<T> {
         let mut builder = Builder::new(app.into()).expect("Builder::new failed");
         builder.link_defaults().expect("link_defaults failed");
         builder