From e1a136ff51f1cfb29428644caf1b680c5521cde2 Mon Sep 17 00:00:00 2001 From: Radical Date: Mon, 26 May 2025 13:22:56 +0000 Subject: [PATCH] feat: allow usage of local folder for file storage --- Cargo.toml | 1 + Dockerfile | 11 ++-- compose.dev.yml | 11 ++-- compose.yml | 11 ++-- entrypoint.sh | 6 ++- src/api/v1/servers/uuid/icon.rs | 3 +- src/api/v1/users/me.rs | 3 +- src/config.rs | 72 +++++++++++++------------ src/error.rs | 2 + src/main.rs | 24 ++++----- src/structs.rs | 95 +++++++++++++++++++++++++++------ 11 files changed, 158 insertions(+), 81 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7a062d8..18aa043 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ diesel-async = { version = "0.5", features = ["deadpool", "postgres", "async-con diesel_migrations = { version = "2.2.0", features = ["postgres"] } thiserror = "2.0.12" actix-multipart = "0.7.2" +actix-files = "0.6.6" [dependencies.tokio] version = "1.44" diff --git a/Dockerfile b/Dockerfile index 0f07fcb..e556fe6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,16 +18,17 @@ RUN useradd --create-home --home-dir /gorb gorb USER gorb -ENV DATABASE_USERNAME=gorb \ +ENV URL=http://localhost:8080 \ +DATABASE_USERNAME=gorb \ DATABASE_PASSWORD=gorb \ DATABASE=gorb \ DATABASE_HOST=database \ DATABASE_PORT=5432 \ CACHE_DB_HOST=valkey \ CACHE_DB_PORT=6379 \ -BUNNY_API_KEY=your_storage_zone_password_here \ -BUNNY_ENDPOINT=Frankfurt \ -BUNNY_ZONE=gorb \ -BUNNY_CDN_URL=https://cdn.gorb.app +BUNNY_API_KEY= \ +BUNNY_ENDPOINT= \ +BUNNY_ZONE= \ +BUNNY_CDN_URL= ENTRYPOINT ["/usr/bin/entrypoint.sh"] diff --git a/compose.dev.yml b/compose.dev.yml index 3da7c89..6b1ecda 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -18,15 +18,18 @@ services: - gorb-backend:/gorb environment: #- RUST_LOG=debug + # This should be changed to the public URL of the server! + - URL=http://localhost:8080 - DATABASE_USERNAME=gorb - DATABASE_PASSWORD=gorb - DATABASE=gorb - DATABASE_HOST=database - DATABASE_PORT=5432 - - BUNNY_API_KEY=your_storage_zone_password_here - - BUNNY_ENDPOINT=Frankfurt - - BUNNY_ZONE=gorb - - BUNNY_CDN_URL=https://cdn.gorb.app + # These can be set to use a CDN, if they are not set then files will be stored locally + #- BUNNY_API_KEY=your_storage_zone_password_here + #- BUNNY_ENDPOINT=Frankfurt + #- BUNNY_ZONE=gorb + #- BUNNY_CDN_URL=https://cdn.gorb.app database: image: postgres:16 restart: always diff --git a/compose.yml b/compose.yml index f87411a..438b4ba 100644 --- a/compose.yml +++ b/compose.yml @@ -16,15 +16,18 @@ services: - gorb-backend:/gorb environment: #- RUST_LOG=debug + # This should be changed to the public URL of the server! + - URL=http://localhost:8080 - DATABASE_USERNAME=gorb - DATABASE_PASSWORD=gorb - DATABASE=gorb - DATABASE_HOST=database - DATABASE_PORT=5432 - - BUNNY_API_KEY=your_storage_zone_password_here - - BUNNY_ENDPOINT=Frankfurt - - BUNNY_ZONE=gorb - - BUNNY_CDN_URL=https://cdn.gorb.app + # These can be set to use a CDN, if they are not set then files will be stored locally + #- BUNNY_API_KEY=your_storage_zone_password_here + #- BUNNY_ENDPOINT=Frankfurt + #- BUNNY_ZONE=gorb + #- BUNNY_CDN_URL=https://cdn.gorb.app database: image: postgres:16 restart: always diff --git a/entrypoint.sh b/entrypoint.sh index a29e6bb..82b8271 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -8,6 +8,10 @@ if [ ! -d "/gorb/logs" ]; then mkdir /gorb/logs fi +if [ ! -d "/gorb/data" ]; then + mkdir /gorb/data +fi + if [ ! -f "/gorb/config/config.toml" ]; then cat > /gorb/config/config.toml <&1 | tee /gorb/logs/backend.log +/usr/bin/gorb-backend --config /gorb/config/config.toml --data-dir /gorb/data 2>&1 | tee /gorb/logs/backend.log diff --git a/src/api/v1/servers/uuid/icon.rs b/src/api/v1/servers/uuid/icon.rs index 2155f55..66f2e8a 100644 --- a/src/api/v1/servers/uuid/icon.rs +++ b/src/api/v1/servers/uuid/icon.rs @@ -38,9 +38,8 @@ pub async fn upload( guild .set_icon( - &data.bunny_cdn, + &data.storage, &mut conn, - data.config.bunny.cdn_url.clone(), bytes, ) .await?; diff --git a/src/api/v1/users/me.rs b/src/api/v1/users/me.rs index 83d02db..fab34dd 100644 --- a/src/api/v1/users/me.rs +++ b/src/api/v1/users/me.rs @@ -58,9 +58,8 @@ pub async fn update( let byte_slice: &[u8] = &bytes; me.set_avatar( - &data.bunny_cdn, + &data.storage, &mut conn, - data.config.bunny.cdn_url.clone(), byte_slice.into(), ) .await?; diff --git a/src/config.rs b/src/config.rs index 102318f..9cefbe1 100644 --- a/src/config.rs +++ b/src/config.rs @@ -9,8 +9,8 @@ use url::Url; pub struct ConfigBuilder { database: Database, cache_database: CacheDatabase, - web: Option, - bunny: BunnyBuilder, + web: WebBuilder, + bunny: Option, } #[derive(Debug, Deserialize, Clone)] @@ -33,8 +33,9 @@ pub struct CacheDatabase { #[derive(Debug, Deserialize)] struct WebBuilder { - url: Option, + ip: Option, port: Option, + url: Url, _ssl: Option, } @@ -57,37 +58,37 @@ impl ConfigBuilder { } pub fn build(self) -> Config { - let web = if let Some(web) = self.web { - Web { - url: web.url.unwrap_or(String::from("0.0.0.0")), - port: web.port.unwrap_or(8080), - } + let web = Web { + ip: self.web.ip.unwrap_or(String::from("0.0.0.0")), + port: self.web.port.unwrap_or(8080), + url: self.web.url + }; + + let bunny; + + if let Some(bunny_builder) = self.bunny { + let endpoint = match &*bunny_builder.endpoint { + "Frankfurt" => Endpoint::Frankfurt, + "London" => Endpoint::London, + "New York" => Endpoint::NewYork, + "Los Angeles" => Endpoint::LosAngeles, + "Singapore" => Endpoint::Singapore, + "Stockholm" => Endpoint::Stockholm, + "Sao Paulo" => Endpoint::SaoPaulo, + "Johannesburg" => Endpoint::Johannesburg, + "Sydney" => Endpoint::Sydney, + url => Endpoint::Custom(url.to_string()), + }; + + bunny = Some(Bunny { + api_key: bunny_builder.api_key, + endpoint, + storage_zone: bunny_builder.storage_zone, + cdn_url: bunny_builder.cdn_url, + }); } else { - Web { - url: String::from("0.0.0.0"), - port: 8080, - } - }; - - let endpoint = match &*self.bunny.endpoint { - "Frankfurt" => Endpoint::Frankfurt, - "London" => Endpoint::London, - "New York" => Endpoint::NewYork, - "Los Angeles" => Endpoint::LosAngeles, - "Singapore" => Endpoint::Singapore, - "Stockholm" => Endpoint::Stockholm, - "Sao Paulo" => Endpoint::SaoPaulo, - "Johannesburg" => Endpoint::Johannesburg, - "Sydney" => Endpoint::Sydney, - url => Endpoint::Custom(url.to_string()), - }; - - let bunny = Bunny { - api_key: self.bunny.api_key, - endpoint, - storage_zone: self.bunny.storage_zone, - cdn_url: self.bunny.cdn_url, - }; + bunny = None; + } Config { database: self.database, @@ -103,13 +104,14 @@ pub struct Config { pub database: Database, pub cache_database: CacheDatabase, pub web: Web, - pub bunny: Bunny, + pub bunny: Option, } #[derive(Debug, Clone)] pub struct Web { - pub url: String, + pub ip: String, pub port: u16, + pub url: Url, } #[derive(Debug, Clone)] diff --git a/src/error.rs b/src/error.rs index ce586ac..135982d 100644 --- a/src/error.rs +++ b/src/error.rs @@ -55,6 +55,8 @@ pub enum Error { #[error("{0}")] PasswordHashError(String), #[error("{0}")] + PathError(String), + #[error("{0}")] BadRequest(String), #[error("{0}")] Unauthorized(String), diff --git a/src/main.rs b/src/main.rs index 5ad1dc8..74b1066 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ use actix_cors::Cors; +use actix_files::Files; use actix_web::{App, HttpServer, web}; use argon2::Argon2; use clap::Parser; @@ -6,6 +7,7 @@ use diesel_async::pooled_connection::AsyncDieselConnectionManager; use diesel_async::pooled_connection::deadpool::Pool; use error::Error; use simple_logger::SimpleLogger; +use structs::Storage; use std::time::SystemTime; mod config; use config::{Config, ConfigBuilder}; @@ -22,11 +24,13 @@ pub mod schema; pub mod structs; pub mod utils; -#[derive(Parser, Debug)] +#[derive(Parser, Debug, Clone)] #[command(version, about, long_about = None)] struct Args { #[arg(short, long, default_value_t = String::from("/etc/gorb/config.toml"))] config: String, + #[arg(short, long, default_value_t = String::from("/var/lib/gorb/"))] + data_dir: String, } #[derive(Clone)] @@ -39,7 +43,7 @@ pub struct Data { pub config: Config, pub argon2: Argon2<'static>, pub start_time: SystemTime, - pub bunny_cdn: bunny_api_tokio::Client, + pub storage: Storage, } #[tokio::main] @@ -63,14 +67,7 @@ async fn main() -> Result<(), Error> { let cache_pool = redis::Client::open(config.cache_database.url())?; - let mut bunny_cdn = bunny_api_tokio::Client::new("").await?; - - let bunny = config.bunny.clone(); - - bunny_cdn - .storage - .init(bunny.api_key, bunny.endpoint, bunny.storage_zone) - .await?; + let storage = Storage::new(config.clone(), args.data_dir.clone()).await?; let database_url = config.database.url(); @@ -111,9 +108,11 @@ async fn main() -> Result<(), Error> { // TODO: Possibly implement "pepper" into this (thinking it could generate one if it doesnt exist and store it on disk) argon2: Argon2::default(), start_time: SystemTime::now(), - bunny_cdn, + storage, }; + let data_dir = args.data_dir.clone(); + HttpServer::new(move || { // Set CORS headers let cors = Cors::default() @@ -143,9 +142,10 @@ async fn main() -> Result<(), Error> { App::new() .app_data(web::Data::new(data.clone())) .wrap(cors) + .service(Files::new("/api/assets", &data_dir)) .service(api::web()) }) - .bind((web.url, web.port))? + .bind((web.ip, web.port))? .run() .await?; diff --git a/src/structs.rs b/src/structs.rs index c86bc5b..92bc599 100644 --- a/src/structs.rs +++ b/src/structs.rs @@ -1,3 +1,5 @@ +use std::path::Path; + use actix_web::web::BytesMut; use diesel::{ ExpressionMethods, QueryDsl, Selectable, SelectableHelper, delete, insert_into, @@ -6,11 +8,11 @@ use diesel::{ }; use diesel_async::{RunQueryDsl, pooled_connection::AsyncDieselConnectionManager}; use serde::{Deserialize, Serialize}; -use tokio::task; +use tokio::{fs::{create_dir_all, remove_file, File}, io::AsyncWriteExt, task}; use url::Url; use uuid::Uuid; -use crate::{Conn, Data, error::Error, schema::*, utils::image_check}; +use crate::{config::Config, error::Error, schema::*, utils::image_check, Conn, Data}; fn load_or_empty( query_result: Result, diesel::result::Error>, @@ -22,6 +24,74 @@ fn load_or_empty( } } +#[derive(Clone)] +pub struct Storage { + bunny_cdn: Option, + cdn_url: Url, + data_dir: String, +} + +impl Storage { + pub async fn new(config: Config, data_dir: String) -> Result { + let mut bunny_cdn; + let cdn_url; + + if let Some(bunny) = config.bunny { + bunny_cdn = Some(bunny_api_tokio::Client::new("").await?); + + bunny_cdn + .as_mut() + .unwrap() + .storage + .init(bunny.api_key, bunny.endpoint, bunny.storage_zone) + .await?; + + cdn_url = bunny.cdn_url; + } else { + bunny_cdn = None; + cdn_url = config.web.url.join("api/assets/")?; + } + + Ok(Self { + bunny_cdn, + data_dir, + cdn_url + }) + } + + pub async fn write>(&self, path: T, bytes: BytesMut) -> Result { + if let Some(bunny_cdn) = &self.bunny_cdn { + bunny_cdn.storage.upload(&path, bytes.into()).await?; + Ok(self.cdn_url.join(&path.as_ref())?) + } else { + let file_path = Path::new(&self.data_dir); + + let file_path = file_path.join(path.as_ref()); + + create_dir_all(file_path.parent().ok_or(Error::PathError("Unable to get parent directory".to_string()))?).await?; + + let mut file = File::create(file_path).await?; + + file.write_all(&bytes).await?; + + Ok(self.cdn_url.join(path.as_ref())?) + } + } + + pub async fn delete>(&self, path: T) -> Result<(), Error> { + if let Some(bunny_cdn) = &self.bunny_cdn { + Ok(bunny_cdn.storage.delete(&path).await?) + } else { + let file_path = Path::new(&self.data_dir); + + let file_path = file_path.join(path.as_ref()); + + remove_file(file_path).await?; + Ok(()) + } + } +} + #[derive(Queryable, Selectable, Insertable, Clone)] #[diesel(table_name = channels)] #[diesel(check_for_backend(diesel::pg::Pg))] @@ -419,9 +489,8 @@ impl Guild { // FIXME: Horrible security pub async fn set_icon( &mut self, - bunny_cdn: &bunny_api_tokio::Client, + storage: &Storage, conn: &mut Conn, - cdn_url: Url, icon: BytesMut, ) -> Result<(), Error> { let icon_clone = icon.clone(); @@ -430,14 +499,12 @@ impl Guild { if let Some(icon) = &self.icon { let relative_url = icon.path().trim_start_matches('/'); - bunny_cdn.storage.delete(relative_url).await?; + storage.delete(relative_url).await?; } let path = format!("icons/{}/icon.{}", self.uuid, image_type); - bunny_cdn.storage.upload(path.clone(), icon.into()).await?; - - let icon_url = cdn_url.join(&path)?; + let icon_url = storage.write(path.clone(), icon.into()).await?; use guilds::dsl; update(guilds::table) @@ -673,9 +740,8 @@ impl Me { pub async fn set_avatar( &mut self, - bunny_cdn: &bunny_api_tokio::Client, + storage: &Storage, conn: &mut Conn, - cdn_url: Url, avatar: BytesMut, ) -> Result<(), Error> { let avatar_clone = avatar.clone(); @@ -686,18 +752,15 @@ impl Me { let relative_url = avatar_url.path().trim_start_matches('/'); - bunny_cdn.storage.delete(relative_url).await?; + storage.delete(relative_url).await?; } let path = format!("avatar/{}/avatar.{}", self.uuid, image_type); - bunny_cdn - .storage - .upload(path.clone(), avatar.into()) + let avatar_url = storage + .write(path.clone(), avatar.into()) .await?; - let avatar_url = cdn_url.join(&path)?; - use users::dsl; update(users::table) .filter(dsl::uuid.eq(self.uuid))