feat: implement simple abstraction for storage (#64)

Author: Fleeym

src/config.rs

@@ -2,6 +2,7 @@ use std::time::Duration;
 
 use moka::future::Cache;
 
+use crate::storage::{PrivateStorage, StaticStorage};
 use crate::{
     endpoints::mods::IndexQueryParams,
     types::{
@@ -17,6 +18,8 @@ pub struct AppData {
     front_url: String,
     github: GitHubClientData,
     webhook_url: String,
+    static_storage: StaticStorage,
+    private_storage: PrivateStorage,
     disable_downloads: bool,
     max_download_mb: u32,
     port: u16,
@@ -34,7 +37,8 @@ pub struct GitHubClientData {
 pub async fn build_config() -> anyhow::Result<AppData> {
     let env_url = dotenvy::var("DATABASE_URL")?;
 
-    let pg_connections = dotenvy::var("DATABASE_CONNECTIONS").map_or(10, |x: String| x.parse::<u32>().unwrap_or(10));
+    let pg_connections =
+        dotenvy::var("DATABASE_CONNECTIONS").map_or(10, |x: String| x.parse::<u32>().unwrap_or(10));
 
     let pool = sqlx::postgres::PgPoolOptions::default()
         .max_connections(pg_connections)
@@ -68,6 +72,8 @@ pub async fn build_config() -> anyhow::Result<AppData> {
             client_secret: github_secret,
         },
         webhook_url,
+        static_storage: StaticStorage::new(),
+        private_storage: PrivateStorage::new(),
         disable_downloads,
         max_download_mb,
         port,
@@ -123,6 +129,14 @@ impl AppData {
         self.debug
     }
 
+    pub fn static_storage(&self) -> &StaticStorage {
+        &self.static_storage
+    }
+
+    pub fn private_storage(&self) -> &PrivateStorage {
+        &self.private_storage
+    }
+
     pub fn mods_cache(&self) -> &Cache<IndexQueryParams, ApiResponse<PaginatedData<Mod>>> {
         &self.mods_cache
     }

src/main.rs

@@ -21,6 +21,7 @@ mod mod_zip;
 mod openapi;
 mod types;
 mod webhook;
+mod storage;
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {

src/storage.rs

@@ -0,0 +1,106 @@
+use std::path::{Path, PathBuf};
+
+#[derive(Clone, Debug)]
+pub struct StaticStorage {
+    base_path: PathBuf,
+}
+
+impl StaticStorage {
+    pub fn new() -> Self {
+        Self {
+            base_path: PathBuf::from("storage/static"),
+        }
+    }
+}
+
+impl StorageDisk for StaticStorage {
+    fn base_path(&self) -> &Path {
+        &self.base_path
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct PrivateStorage {
+    base_path: PathBuf,
+}
+
+impl PrivateStorage {
+    pub fn new() -> Self {
+        Self {
+            base_path: PathBuf::from("storage/private"),
+        }
+    }
+}
+
+impl StorageDisk for PrivateStorage {
+    fn base_path(&self) -> &Path {
+        &self.base_path
+    }
+}
+
+trait StorageDisk {
+    async fn init(&self) -> std::io::Result<()> {
+        tokio::fs::create_dir_all(self.base_path()).await?;
+        Ok(())
+    }
+    fn base_path(&self) -> &Path;
+    fn path(&self, relative_path: &str) -> PathBuf {
+        self.base_path().join(relative_path)
+    }
+    async fn store(&self, relative_path: &str, data: &[u8]) -> std::io::Result<()> {
+        let path = self.path(relative_path);
+        if let Some(parent) = path.parent() {
+            tokio::fs::create_dir_all(parent).await?;
+        }
+
+        tokio::fs::write(path, data).await
+    }
+    /// Store data at a path calculated from the hash of the data. Uses content-addressable storage with 2 levels
+    async fn store_hashed(&self, relative_path: &str, data: &[u8]) -> std::io::Result<()> {
+        self.store_hashed_with_extension(relative_path, data, None)
+            .await
+    }
+    /// Store data at a path calculated from the hash of the data. Uses content-addressable storage with 2 levels.
+    /// Extension should not include the dot, and will be added to the end of the filename if provided.
+    async fn store_hashed_with_extension(
+        &self,
+        relative_path: &str,
+        data: &[u8],
+        extension: Option<&str>,
+    ) -> std::io::Result<()> {
+        let hash = sha256::digest(data);
+
+        let hashed_path = format!(
+            "{}/{}/{}{}",
+            relative_path,
+            &hash[0..2],
+            hash,
+            extension.map_or("".to_string(), |ext| format!(
+                ".{}",
+                ext.trim_start_matches('.')
+            ))
+        );
+        self.store(&hashed_path, data).await
+    }
+    async fn read(&self, relative_path: &str) -> std::io::Result<Vec<u8>> {
+        match tokio::fs::read(self.path(relative_path)).await {
+            Ok(data) => Ok(data),
+            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(vec![]),
+            Err(e) => Err(e),
+        }
+    }
+    async fn read_stream(&self, relative_path: &str) -> std::io::Result<tokio::fs::File> {
+        tokio::fs::File::open(self.path(relative_path)).await
+    }
+    async fn exists(&self, relative_path: &str) -> std::io::Result<bool> {
+        let path = self.path(relative_path);
+        Ok(tokio::fs::metadata(path).await.is_ok())
+    }
+    async fn delete(&self, relative_path: &str) -> std::io::Result<()> {
+        match tokio::fs::remove_file(self.path(relative_path)).await {
+            Ok(()) => Ok(()),
+            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
+            Err(e) => Err(e),
+        }
+    }
+}

storage/.gitignore

@@ -0,0 +1,4 @@
+!
+!.gitignore
+!public
+!private

storage/private/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

storage/public/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore