Merge pull request #3361 from itowlson/oci-bare-wasm-packages

Enable running `wkg oci` packages as bare Wasm

Author: itowlson

.github/workflows/build.yml

@@ -217,6 +217,8 @@ jobs:
           chmod +x grain-linux-x64
           mv grain-linux-x64 grain
           echo "$PWD" >> $GITHUB_PATH
+      - name: Install wkg
+        run: cargo install wkg
 
       - name: Run Full Integration Tests
         run: make test-integration-full

crates/oci/src/client.rs

@@ -35,6 +35,8 @@ pub const DATA_MEDIATYPE: &str = "application/vnd.wasm.content.layer.v1+data";
 pub const ARCHIVE_MEDIATYPE: &str = "application/vnd.wasm.content.bundle.v1.tar+gzip";
 // Note: this will be updated with a canonical value once defined upstream
 const WASM_LAYER_MEDIA_TYPE: &str = "application/vnd.wasm.content.layer.v1+wasm";
+// Media type for a Wasm binary pushed by wkg
+const WASM_LAYER_MEDIA_TYPE_WKG: &str = "application/wasm";
 
 const CONFIG_FILE: &str = "config.json";
 const LATEST_TAG: &str = "latest";
@@ -499,7 +501,7 @@ impl Client {
     }
 
     /// Pull a Spin application from an OCI registry.
-    pub async fn pull(&mut self, reference: &str) -> Result<()> {
+    pub async fn pull(&mut self, reference: &str) -> Result<OciImageManifest> {
         let reference: Reference = reference.parse().context("cannot parse reference")?;
         let auth = Self::auth(&reference).await?;
 
@@ -526,7 +528,7 @@ impl Client {
 
         // If a layer is a Wasm module, write it in the Wasm directory.
         // Otherwise, write it in the data directory (after unpacking if archive layer)
-        stream::iter(manifest.layers)
+        stream::iter(&manifest.layers)
             .map(|layer| {
                 let this = &self;
                 let reference = reference.clone();
@@ -541,14 +543,14 @@ impl Client {
 
                     tracing::debug!("Pulling layer {}", &layer.digest);
                     let mut bytes = Vec::with_capacity(layer.size.try_into()?);
-                    this.oci.pull_blob(&reference, &layer, &mut bytes).await?;
+                    this.oci.pull_blob(&reference, layer, &mut bytes).await?;
                     match layer.media_type.as_str() {
                         SPIN_APPLICATION_MEDIA_TYPE => {
                             this.write_locked_app_config(&reference.to_string(), &bytes)
                                 .await
                                 .with_context(|| "unable to write locked app config to cache")?;
                         }
-                        WASM_LAYER_MEDIA_TYPE => {
+                        WASM_LAYER_MEDIA_TYPE | WASM_LAYER_MEDIA_TYPE_WKG => {
                             this.cache.write_wasm(&bytes, &layer.digest).await?;
                         }
                         ARCHIVE_MEDIATYPE => {
@@ -566,7 +568,7 @@ impl Client {
             .await?;
         tracing::info!("Pulled {}@{}", reference, digest);
 
-        Ok(())
+        Ok(manifest)
     }
 
     /// Get the file path to an OCI manifest given a reference.

crates/oci/src/lib.rs

@@ -8,7 +8,7 @@ pub mod utils;
 mod validate;
 
 pub use client::{Client, ComposeMode};
-pub use loader::OciLoader;
+pub use loader::{ExecutableArtifact, OciLoader};
 
 /// URL scheme used for the locked app "origin" metadata field for OCI-sourced apps.
 pub const ORIGIN_URL_SCHEME: &str = "vnd.fermyon.origin-oci";

crates/oci/src/loader.rs

@@ -14,6 +14,14 @@ pub struct OciLoader {
     working_dir: PathBuf,
 }
 
+/// The artifact loaded by the `OciLoader`
+pub enum ExecutableArtifact {
+    /// The OCI reference contained a Spin application
+    Application(LockedApp),
+    /// The OCI reference contained a Wasm package
+    Package(PathBuf),
+}
+
 impl OciLoader {
     /// Creates a new OciLoader which builds temporary mount directory(s) in
     /// the given working_dir.
@@ -23,9 +31,13 @@ impl OciLoader {
     }
 
     /// Pulls and loads an OCI Artifact and returns a LockedApp with the given OCI client and reference
-    pub async fn load_app(&self, client: &mut Client, reference: &str) -> Result<LockedApp> {
+    pub async fn load_app(
+        &self,
+        client: &mut Client,
+        reference: &str,
+    ) -> Result<ExecutableArtifact> {
         // Fetch app
-        client.pull(reference).await.with_context(|| {
+        let manifest = client.pull(reference).await.with_context(|| {
             format!("cannot pull Spin application from registry reference {reference:?}")
         })?;
 
@@ -34,42 +46,58 @@ impl OciLoader {
             .lockfile_path(&reference)
             .await
             .context("cannot get path to spin.lock")?;
-        self.load_from_cache(lockfile_path, reference, &client.cache)
+        self.load_from_cache(manifest, lockfile_path, reference, &client.cache)
             .await
     }
 
     /// Loads an OCI Artifact from the given cache and returns a LockedApp with the given reference
     pub async fn load_from_cache(
         &self,
+        manifest: oci_distribution::manifest::OciImageManifest,
         lockfile_path: PathBuf,
         reference: &str,
         cache: &Cache,
-    ) -> std::result::Result<LockedApp, anyhow::Error> {
+    ) -> std::result::Result<ExecutableArtifact, anyhow::Error> {
         let locked_content = tokio::fs::read(&lockfile_path)
             .await
             .with_context(|| format!("failed to read from {}", quoted_path(&lockfile_path)))?;
-        let mut locked_app = LockedApp::from_json(&locked_content).with_context(|| {
-            format!(
-                "failed to decode locked app from {}",
-                quoted_path(&lockfile_path)
-            )
-        })?;
-
-        // Update origin metadata
-        let resolved_reference = Reference::try_from(reference).context("invalid reference")?;
-        let origin_uri = format!("{ORIGIN_URL_SCHEME}:{resolved_reference}");
-        locked_app
-            .metadata
-            .insert("origin".to_string(), origin_uri.into());
-
-        for component in &mut locked_app.components {
-            self.resolve_component_content_refs(component, cache)
-                .await
-                .with_context(|| {
-                    format!("failed to resolve content for component {:?}", component.id)
-                })?;
+        let locked_json: serde_json::Value = serde_json::from_slice(&locked_content)
+            .with_context(|| format!("OCI config {} is not JSON", quoted_path(&lockfile_path)))?;
+
+        if locked_json.get("spin_lock_version").is_some() {
+            let mut locked_app = LockedApp::from_json(&locked_content).with_context(|| {
+                format!(
+                    "failed to decode locked app from {}",
+                    quoted_path(&lockfile_path)
+                )
+            })?;
+
+            // Update origin metadata
+            let resolved_reference = Reference::try_from(reference).context("invalid reference")?;
+            let origin_uri = format!("{ORIGIN_URL_SCHEME}:{resolved_reference}");
+            locked_app
+                .metadata
+                .insert("origin".to_string(), origin_uri.into());
+
+            for component in &mut locked_app.components {
+                self.resolve_component_content_refs(component, cache)
+                    .await
+                    .with_context(|| {
+                        format!("failed to resolve content for component {:?}", component.id)
+                    })?;
+            }
+            Ok(ExecutableArtifact::Application(locked_app))
+        } else {
+            if manifest.layers.len() != 1 {
+                anyhow::bail!(
+                    "expected single layer in OCI package, found {} layers",
+                    manifest.layers.len()
+                );
+            }
+            let layer = &manifest.layers[0]; // guaranteed safe by previous check
+            let wasm_path = cache.wasm_path(&layer.digest);
+            Ok(ExecutableArtifact::Package(wasm_path))
         }
-        Ok(locked_app)
     }
 
     async fn resolve_component_content_refs(

src/commands/up.rs

@@ -15,7 +15,7 @@ use spin_app::locked::LockedApp;
 use spin_common::ui::quoted_path;
 use spin_factor_outbound_networking::validate_service_chaining_for_components;
 use spin_loader::FilesMountStrategy;
-use spin_oci::OciLoader;
+use spin_oci::{ExecutableArtifact, OciLoader};
 use spin_trigger::cli::{LaunchMetadata, SPIN_LOCAL_APP_DIR, SPIN_LOCKED_URL, SPIN_WORKING_DIR};
 use tempfile::TempDir;
 
@@ -467,10 +467,18 @@ impl UpCommand {
                     .await
                     .context("cannot create registry client")?;
 
-                let locked_app = OciLoader::new(working_dir)
+                let artifact = OciLoader::new(working_dir)
                     .load_app(&mut client, reference)
                     .await?;
-                ResolvedAppSource::OciRegistry { locked_app }
+
+                match artifact {
+                    ExecutableArtifact::Application(locked_app) => {
+                        ResolvedAppSource::OciRegistry { locked_app }
+                    }
+                    ExecutableArtifact::Package(wasm_path) => {
+                        ResolvedAppSource::BareWasm { wasm_path }
+                    }
+                }
             }
             AppSource::BareWasm(path) => ResolvedAppSource::BareWasm {
                 wasm_path: path.clone(),

tests/integration.rs

@@ -872,6 +872,47 @@ mod integration_tests {
         Ok(())
     }
 
+    #[test]
+    #[cfg(feature = "extern-dependencies-tests")]
+    fn registry_bare_wasm_works() -> anyhow::Result<()> {
+        let services = ServicesConfig::new(vec!["registry"])?;
+        let spin_up_args = |env: &mut test_environment::TestEnvironment<()>| {
+            let port = env
+                .get_port(5000)?
+                .context("no registry port was exposed by test services")?;
+            let registry_url =
+                format!("localhost:{port}/spin-e2e-tests/registry-base-wasm-works/v1");
+            let mut registry_push = std::process::Command::new("wkg");
+            registry_push.args([
+                "oci",
+                "push",
+                &registry_url,
+                "./target/wasm32-wasip1/release/test.wasm",
+                "--insecure",
+            ]);
+            registry_push.arg(format!("localhost:{port}"));
+            env.run_in(&mut registry_push)?;
+            Ok(vec!["-f".into(), registry_url, "--insecure".into()])
+        };
+        let mut env = super::testcases::bootstrap_smoke_test(
+            services,
+            None,
+            &[],
+            "http-rust",
+            |_| Ok(Vec::new()),
+            |_| Ok(()),
+            HashMap::default(),
+            spin_up_args,
+            SpinAppType::Http,
+        )?;
+        assert_spin_request(
+            env.runtime_mut(),
+            Request::new(Method::Get, "/"),
+            Response::new_with_body(200, "Hello World!"),
+        )?;
+        Ok(())
+    }
+
     #[test]
     fn test_wasi_http_rc_11_10() -> anyhow::Result<()> {
         test_wasi_http_rc("wasi-http-0.2.0-rc-2023-11-10")