gorb/backend
4
0
Fork 1
Code Issues 1 Pull requests 1 Projects Releases Packages 1 Wiki Activity Actions

feat: allow usage of local folder for file storage

Browse source
This commit is contained in:
Radical 2025-05-26 13:22:56 +00:00
parent 6640d03b70
commit e1a136ff51
11 changed files with 158 additions and 81 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Cargo.toml
View file

@ -37,6 +37,7 @@ diesel-async = { version = "0.5", features = ["deadpool", "postgres", "async-con
diesel_migrations = { version = "2.2.0", features = ["postgres"] }
thiserror = "2.0.12"
actix-multipart = "0.7.2"
actix-files = "0.6.6"
[dependencies.tokio]
version = "1.44"

11
Dockerfile
View file

@ -18,16 +18,17 @@ RUN useradd --create-home --home-dir /gorb gorb
USER gorb
ENV DATABASE_USERNAME=gorb \
ENV URL=http://localhost:8080 \
DATABASE_USERNAME=gorb \
DATABASE_PASSWORD=gorb \
DATABASE=gorb \
DATABASE_HOST=database \
DATABASE_PORT=5432 \
CACHE_DB_HOST=valkey \
CACHE_DB_PORT=6379 \
BUNNY_API_KEY=your_storage_zone_password_here \
BUNNY_ENDPOINT=Frankfurt \
BUNNY_ZONE=gorb \
BUNNY_CDN_URL=https://cdn.gorb.app
BUNNY_API_KEY= \
BUNNY_ENDPOINT= \
BUNNY_ZONE= \
BUNNY_CDN_URL=
ENTRYPOINT ["/usr/bin/entrypoint.sh"]

11
compose.dev.yml
View file

@ -18,15 +18,18 @@ services:
- gorb-backend:/gorb
environment:
#- RUST_LOG=debug
# This should be changed to the public URL of the server!
- URL=http://localhost:8080
- DATABASE_USERNAME=gorb
- DATABASE_PASSWORD=gorb
- DATABASE=gorb
- DATABASE_HOST=database
- DATABASE_PORT=5432
- BUNNY_API_KEY=your_storage_zone_password_here
- BUNNY_ENDPOINT=Frankfurt
- BUNNY_ZONE=gorb
- BUNNY_CDN_URL=https://cdn.gorb.app
# These can be set to use a CDN, if they are not set then files will be stored locally
#- BUNNY_API_KEY=your_storage_zone_password_here
#- BUNNY_ENDPOINT=Frankfurt
#- BUNNY_ZONE=gorb
#- BUNNY_CDN_URL=https://cdn.gorb.app
database:
image: postgres:16
restart: always

11
compose.yml
View file

@ -16,15 +16,18 @@ services:
- gorb-backend:/gorb
environment:
#- RUST_LOG=debug
# This should be changed to the public URL of the server!
- URL=http://localhost:8080
- DATABASE_USERNAME=gorb
- DATABASE_PASSWORD=gorb
- DATABASE=gorb
- DATABASE_HOST=database
- DATABASE_PORT=5432
- BUNNY_API_KEY=your_storage_zone_password_here
- BUNNY_ENDPOINT=Frankfurt
- BUNNY_ZONE=gorb
- BUNNY_CDN_URL=https://cdn.gorb.app
# These can be set to use a CDN, if they are not set then files will be stored locally
#- BUNNY_API_KEY=your_storage_zone_password_here
#- BUNNY_ENDPOINT=Frankfurt
#- BUNNY_ZONE=gorb
#- BUNNY_CDN_URL=https://cdn.gorb.app
database:
image: postgres:16
restart: always

6
entrypoint.sh
View file

@ -8,6 +8,10 @@ if [ ! -d "/gorb/logs" ]; then
mkdir /gorb/logs
fi
if [ ! -d "/gorb/data" ]; then
mkdir /gorb/data
fi
if [ ! -f "/gorb/config/config.toml" ]; then
cat > /gorb/config/config.toml <<EOF
[database]
@ -52,4 +56,4 @@ rotate_log "/gorb/logs/backend.log"
# Give the DB time to start up before connecting
sleep 5
/usr/bin/gorb-backend --config /gorb/config/config.toml 2>&1 | tee /gorb/logs/backend.log
/usr/bin/gorb-backend --config /gorb/config/config.toml --data-dir /gorb/data 2>&1 | tee /gorb/logs/backend.log

3
src/api/v1/servers/uuid/icon.rs
View file

@ -38,9 +38,8 @@ pub async fn upload(
guild
.set_icon(
&data.bunny_cdn,
&data.storage,
&mut conn,
data.config.bunny.cdn_url.clone(),
bytes,
)
.await?;

3
src/api/v1/users/me.rs
View file

@ -58,9 +58,8 @@ pub async fn update(
let byte_slice: &[u8] = &bytes;
me.set_avatar(
&data.bunny_cdn,
&data.storage,
&mut conn,
data.config.bunny.cdn_url.clone(),
byte_slice.into(),
)
.await?;

72
src/config.rs
View file

@ -9,8 +9,8 @@ use url::Url;
pub struct ConfigBuilder {
database: Database,
cache_database: CacheDatabase,
web: Option<WebBuilder>,
bunny: BunnyBuilder,
web: WebBuilder,
bunny: Option<BunnyBuilder>,
}
#[derive(Debug, Deserialize, Clone)]
@ -33,8 +33,9 @@ pub struct CacheDatabase {
#[derive(Debug, Deserialize)]
struct WebBuilder {
url: Option<String>,
ip: Option<String>,
port: Option<u16>,
url: Url,
_ssl: Option<bool>,
}
@ -57,37 +58,37 @@ impl ConfigBuilder {
}
pub fn build(self) -> Config {
let web = if let Some(web) = self.web {
Web {
url: web.url.unwrap_or(String::from("0.0.0.0")),
port: web.port.unwrap_or(8080),
}
let web = Web {
ip: self.web.ip.unwrap_or(String::from("0.0.0.0")),
port: self.web.port.unwrap_or(8080),
url: self.web.url
};
let bunny;
if let Some(bunny_builder) = self.bunny {
let endpoint = match &*bunny_builder.endpoint {
"Frankfurt" => Endpoint::Frankfurt,
"London" => Endpoint::London,
"New York" => Endpoint::NewYork,
"Los Angeles" => Endpoint::LosAngeles,
"Singapore" => Endpoint::Singapore,
"Stockholm" => Endpoint::Stockholm,
"Sao Paulo" => Endpoint::SaoPaulo,
"Johannesburg" => Endpoint::Johannesburg,
"Sydney" => Endpoint::Sydney,
url => Endpoint::Custom(url.to_string()),
};
bunny = Some(Bunny {
api_key: bunny_builder.api_key,
endpoint,
storage_zone: bunny_builder.storage_zone,
cdn_url: bunny_builder.cdn_url,
});
} else {
Web {
url: String::from("0.0.0.0"),
port: 8080,
}
};
let endpoint = match &*self.bunny.endpoint {
"Frankfurt" => Endpoint::Frankfurt,
"London" => Endpoint::London,
"New York" => Endpoint::NewYork,
"Los Angeles" => Endpoint::LosAngeles,
"Singapore" => Endpoint::Singapore,
"Stockholm" => Endpoint::Stockholm,
"Sao Paulo" => Endpoint::SaoPaulo,
"Johannesburg" => Endpoint::Johannesburg,
"Sydney" => Endpoint::Sydney,
url => Endpoint::Custom(url.to_string()),
};
let bunny = Bunny {
api_key: self.bunny.api_key,
endpoint,
storage_zone: self.bunny.storage_zone,
cdn_url: self.bunny.cdn_url,
};
bunny = None;
}
Config {
database: self.database,
@ -103,13 +104,14 @@ pub struct Config {
pub database: Database,
pub cache_database: CacheDatabase,
pub web: Web,
pub bunny: Bunny,
pub bunny: Option<Bunny>,
}
#[derive(Debug, Clone)]
pub struct Web {
pub url: String,
pub ip: String,
pub port: u16,
pub url: Url,
}
#[derive(Debug, Clone)]

2
src/error.rs
View file

@ -55,6 +55,8 @@ pub enum Error {
#[error("{0}")]
PasswordHashError(String),
#[error("{0}")]
PathError(String),
#[error("{0}")]
BadRequest(String),
#[error("{0}")]
Unauthorized(String),

24
src/main.rs
View file

@ -1,4 +1,5 @@
use actix_cors::Cors;
use actix_files::Files;
use actix_web::{App, HttpServer, web};
use argon2::Argon2;
use clap::Parser;
@ -6,6 +7,7 @@ use diesel_async::pooled_connection::AsyncDieselConnectionManager;
use diesel_async::pooled_connection::deadpool::Pool;
use error::Error;
use simple_logger::SimpleLogger;
use structs::Storage;
use std::time::SystemTime;
mod config;
use config::{Config, ConfigBuilder};
@ -22,11 +24,13 @@ pub mod schema;
pub mod structs;
pub mod utils;
#[derive(Parser, Debug)]
#[derive(Parser, Debug, Clone)]
#[command(version, about, long_about = None)]
struct Args {
#[arg(short, long, default_value_t = String::from("/etc/gorb/config.toml"))]
config: String,
#[arg(short, long, default_value_t = String::from("/var/lib/gorb/"))]
data_dir: String,
}
#[derive(Clone)]
@ -39,7 +43,7 @@ pub struct Data {
pub config: Config,
pub argon2: Argon2<'static>,
pub start_time: SystemTime,
pub bunny_cdn: bunny_api_tokio::Client,
pub storage: Storage,
}
#[tokio::main]
@ -63,14 +67,7 @@ async fn main() -> Result<(), Error> {
let cache_pool = redis::Client::open(config.cache_database.url())?;
let mut bunny_cdn = bunny_api_tokio::Client::new("").await?;
let bunny = config.bunny.clone();
bunny_cdn
.storage
.init(bunny.api_key, bunny.endpoint, bunny.storage_zone)
.await?;
let storage = Storage::new(config.clone(), args.data_dir.clone()).await?;
let database_url = config.database.url();
@ -111,9 +108,11 @@ async fn main() -> Result<(), Error> {
// TODO: Possibly implement "pepper" into this (thinking it could generate one if it doesnt exist and store it on disk)
argon2: Argon2::default(),
start_time: SystemTime::now(),
bunny_cdn,
storage,
};
let data_dir = args.data_dir.clone();
HttpServer::new(move || {
// Set CORS headers
let cors = Cors::default()
@ -143,9 +142,10 @@ async fn main() -> Result<(), Error> {
App::new()
.app_data(web::Data::new(data.clone()))
.wrap(cors)
.service(Files::new("/api/assets", &data_dir))
.service(api::web())
})
.bind((web.url, web.port))?
.bind((web.ip, web.port))?
.run()
.await?;

95
src/structs.rs
View file

@ -1,3 +1,5 @@
use std::path::Path;
use actix_web::web::BytesMut;
use diesel::{
ExpressionMethods, QueryDsl, Selectable, SelectableHelper, delete, insert_into,
@ -6,11 +8,11 @@ use diesel::{
};
use diesel_async::{RunQueryDsl, pooled_connection::AsyncDieselConnectionManager};
use serde::{Deserialize, Serialize};
use tokio::task;
use tokio::{fs::{create_dir_all, remove_file, File}, io::AsyncWriteExt, task};
use url::Url;
use uuid::Uuid;
use crate::{Conn, Data, error::Error, schema::*, utils::image_check};
use crate::{config::Config, error::Error, schema::*, utils::image_check, Conn, Data};
fn load_or_empty<T>(
query_result: Result<Vec<T>, diesel::result::Error>,
@ -22,6 +24,74 @@ fn load_or_empty<T>(
}
}
#[derive(Clone)]
pub struct Storage {
bunny_cdn: Option<bunny_api_tokio::Client>,
cdn_url: Url,
data_dir: String,
}
impl Storage {
pub async fn new(config: Config, data_dir: String) -> Result<Self, Error> {
let mut bunny_cdn;
let cdn_url;
if let Some(bunny) = config.bunny {
bunny_cdn = Some(bunny_api_tokio::Client::new("").await?);
bunny_cdn
.as_mut()
.unwrap()
.storage
.init(bunny.api_key, bunny.endpoint, bunny.storage_zone)
.await?;
cdn_url = bunny.cdn_url;
} else {
bunny_cdn = None;
cdn_url = config.web.url.join("api/assets/")?;
}
Ok(Self {
bunny_cdn,
data_dir,
cdn_url
})
}
pub async fn write<T: AsRef<str>>(&self, path: T, bytes: BytesMut) -> Result<Url, Error> {
if let Some(bunny_cdn) = &self.bunny_cdn {
bunny_cdn.storage.upload(&path, bytes.into()).await?;
Ok(self.cdn_url.join(&path.as_ref())?)
} else {
let file_path = Path::new(&self.data_dir);
let file_path = file_path.join(path.as_ref());
create_dir_all(file_path.parent().ok_or(Error::PathError("Unable to get parent directory".to_string()))?).await?;
let mut file = File::create(file_path).await?;
file.write_all(&bytes).await?;
Ok(self.cdn_url.join(path.as_ref())?)
}
}
pub async fn delete<T: AsRef<str>>(&self, path: T) -> Result<(), Error> {
if let Some(bunny_cdn) = &self.bunny_cdn {
Ok(bunny_cdn.storage.delete(&path).await?)
} else {
let file_path = Path::new(&self.data_dir);
let file_path = file_path.join(path.as_ref());
remove_file(file_path).await?;
Ok(())
}
}
}
#[derive(Queryable, Selectable, Insertable, Clone)]
#[diesel(table_name = channels)]
#[diesel(check_for_backend(diesel::pg::Pg))]
@ -419,9 +489,8 @@ impl Guild {
// FIXME: Horrible security
pub async fn set_icon(
&mut self,
bunny_cdn: &bunny_api_tokio::Client,
storage: &Storage,
conn: &mut Conn,
cdn_url: Url,
icon: BytesMut,
) -> Result<(), Error> {
let icon_clone = icon.clone();
@ -430,14 +499,12 @@ impl Guild {
if let Some(icon) = &self.icon {
let relative_url = icon.path().trim_start_matches('/');
bunny_cdn.storage.delete(relative_url).await?;
storage.delete(relative_url).await?;
}
let path = format!("icons/{}/icon.{}", self.uuid, image_type);
bunny_cdn.storage.upload(path.clone(), icon.into()).await?;
let icon_url = cdn_url.join(&path)?;
let icon_url = storage.write(path.clone(), icon.into()).await?;
use guilds::dsl;
update(guilds::table)
@ -673,9 +740,8 @@ impl Me {
pub async fn set_avatar(
&mut self,
bunny_cdn: &bunny_api_tokio::Client,
storage: &Storage,
conn: &mut Conn,
cdn_url: Url,
avatar: BytesMut,
) -> Result<(), Error> {
let avatar_clone = avatar.clone();
@ -686,18 +752,15 @@ impl Me {
let relative_url = avatar_url.path().trim_start_matches('/');
bunny_cdn.storage.delete(relative_url).await?;
storage.delete(relative_url).await?;
}
let path = format!("avatar/{}/avatar.{}", self.uuid, image_type);
bunny_cdn
.storage
.upload(path.clone(), avatar.into())
let avatar_url = storage
.write(path.clone(), avatar.into())
.await?;
let avatar_url = cdn_url.join(&path)?;
use users::dsl;
update(users::table)
.filter(dsl::uuid.eq(self.uuid))