Development merge myfs

rfs/README.md

@@ -1,4 +1,3 @@
-
 # Introduction
 
 `rfs` is the main tool to create, mount and extract FungiStore lists (FungiList)`fl` for short. An `fl` is a simple format
@@ -41,6 +40,7 @@ The simplest form of `<store-specs>` is a `url`. the store `url` defines the sto
 - `s3`: aws-s3 is used for storing and retrieving large amounts of data (blobs) in buckets (directories). An example `s3://<username>:<password>@<host>:<port>/<bucket-name>`
 
   `region` is an optional param for s3 stores, if you want to provide one you can add it as a query to the url `?region=<region-name>`
+
 - `http`: http is a store mostly used for wrapping a dir store to fetch data through http requests. It does not support uploading, just fetching the data.
   It can be set in the FL file as the store to fetch the data with `rfs config`. Example: `http://localhost:9000/store` (https works too).
 
@@ -144,6 +144,41 @@ Options:
 
 By default when unpacking the `-p` flag is not set. which means downloaded files will be `owned` by the current user/group. If `-p` flag is set, the files ownership will be same as the original files used to create the fl (preserve `uid` and `gid` of the files and directories) this normally requires `sudo` while unpacking.
 
+# Merge an `fl`
+
+rfs provides a `merge` subcommand that combines multiple file lists (FL files) into a single unified file list.
+
+```bash
+rfs merge merged.fl flist1.fl flist2.fl --token token
+```
+
+This tells rfs to create an `fl` named `merged.fl` by combining the file lists `flist1.fl` and `flist2.fl`. A token for the server must be specified with the `--token` option.
+
+## Requirements for Merge
+
+- At least 2 input file lists must be specified
+- A token for the server must be specified with the `--token` option
+
+## Full Command Help
+
+```bash
+# rfs merge --help
+
+merge 2 or more FLs into a new one
+
+Usage: rfs merge [OPTIONS] --token <TOKEN> <META> <TARGET_FLISTS>...
+
+Arguments:
+  <META>              path to metadata file (flist)
+  <TARGET_FLISTS>...  
+
+Options:
+  -s, --server <SERVER>  server URL (e.g., http://localhost:8080) [default: http://localhost:8080]
+      --token <TOKEN>    authentication token for the server [default: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE3NTAwOTAyNTUsImlhdCI6MTc1MDA3MjI1NSwidXNlcm5hbWUiOiJyYXdkYSJ9.PieIEa-O4R2G1_H1Dq8lmpQihNyGun1qLMF9B4ToxEY]
+  -c, --cache <CACHE>    [default: /tmp/cache]
+  -h, --help             Print help
+```
+
 # Specifications
 
 Please check [docs](../docs)

rfs/src/lib.rs

@@ -12,6 +12,8 @@ pub use unpack::unpack;
 mod clone;
 pub use clone::clone;
 pub mod config;
+mod merge;
+pub use merge::merge;
 mod docker;
 pub use docker::DockerImageToFlist;
 mod upload;
@@ -118,4 +120,35 @@ mod test {
 
         assert!(status.success());
     }
+
+    async fn create_test_files<P: AsRef<std::path::Path>>(dir: P, name: &str, size: usize) {
+        let mut urandom = fs::OpenOptions::default()
+            .read(true)
+            .open("/dev/urandom")
+            .await
+            .unwrap()
+            .take(size as u64);
+
+        let p = dir.as_ref().join(name);
+        let mut file = fs::OpenOptions::default()
+            .create(true)
+            .write(true)
+            .open(p)
+            .await
+            .unwrap();
+
+        tokio::io::copy(&mut urandom, &mut file).await.unwrap();
+    }
+
+    async fn verify_file_content<P: AsRef<std::path::Path>>(path: P, expected_size: usize) {
+        let mut file = fs::OpenOptions::default()
+            .read(true)
+            .open(path)
+            .await
+            .unwrap();
+
+        let mut buffer = vec![0; expected_size];
+        let size = file.read(&mut buffer).await.unwrap();
+        assert_eq!(size, expected_size);
+    }
 }

rfs/src/main.rs

@@ -41,6 +41,8 @@ enum Commands {
     Clone(CloneOptions),
     /// list or modify FL metadata and stores
     Config(ConfigOptions),
+    /// merge 2 or more FLs into a new one
+    Merge(MergeOptions),
     /// convert a docker image to an FL
     Docker(DockerOptions),
     /// run the fl-server
@@ -208,6 +210,37 @@ struct CloneOptions {
     cache: String,
 }
 
+#[derive(Args, Debug)]
+struct MergeOptions {
+    /// path to metadata file (flist)
+    meta: String,
+
+    /// server URL (e.g., http://localhost:8080)
+    #[clap(short, long, default_value_t = String::from("http://localhost:8080"))]
+    server: String,
+
+    /// authentication token for the server
+    #[clap(long, default_value_t = std::env::var("RFS_TOKEN").unwrap_or_default(), required = true)]
+    token: String,
+
+    #[clap(action=ArgAction::Append, required = true)]
+    target_flists: Vec<String>,
+
+    #[clap(short, long, default_value_t = String::from("/tmp/cache"))]
+    cache: String,
+}
+
+impl MergeOptions {
+    fn validate(&self) -> Result<()> {
+        if self.target_flists.len() < 2 {
+            return Err(anyhow::anyhow!(
+                "At least 2 target file lists are required for merge operation"
+            ));
+        }
+        Ok(())
+    }
+}
+
 #[derive(Args, Debug)]
 struct ConfigOptions {
     /// path to metadata file (flist)
@@ -487,6 +520,7 @@ fn main() -> Result<()> {
         Commands::Unpack(opts) => unpack(opts),
         Commands::Clone(opts) => clone(opts),
         Commands::Config(opts) => config(opts),
+        Commands::Merge(opts) => merge(opts),
         Commands::Docker(opts) => docker(opts),
         Commands::Server(opts) => server(opts),
         Commands::Upload(opts) => upload_file(opts),
@@ -779,6 +813,24 @@ fn config(opts: ConfigOptions) -> Result<()> {
     })
 }
 
+fn merge(opts: MergeOptions) -> Result<()> {
+    opts.validate()?;
+
+    let rt = tokio::runtime::Runtime::new()?;
+
+    rt.block_on(async move {
+        rfs::merge(
+            opts.meta,
+            opts.server,
+            &opts.token,
+            opts.target_flists,
+            opts.cache,
+        )
+        .await?;
+        Ok(())
+    })
+}
+
 fn docker(opts: DockerOptions) -> Result<()> {
     use bollard::auth::DockerCredentials;
     use uuid::Uuid;