diff --git a/Cargo.lock b/Cargo.lock
index 2e219cbb076ab36a043f7386e6149ecba70704ee..c4ffd95b00a1bf83af151fe6f22281abd56d75da 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -227,6 +227,12 @@ version = "0.21.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
+[[package]]
+name = "base64"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51"
+
[[package]]
name = "base64ct"
version = "1.6.0"
@@ -412,9 +418,10 @@ dependencies = [
"axum",
"axum-server",
"axum-server-dual-protocol",
- "base64",
+ "base64 0.22.0",
"bytes",
"clap",
+ "cyborgtime",
"either",
"figment",
"futures-util",
@@ -562,6 +569,12 @@ dependencies = [
"syn 2.0.50",
]
+[[package]]
+name = "cyborgtime"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "817fa642fb0ee7fe42e95783e00e0969927b96091bdd4b9b1af082acd943913b"
+
[[package]]
name = "data-encoding"
version = "2.5.0"
@@ -894,7 +907,7 @@ version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270"
dependencies = [
- "base64",
+ "base64 0.21.7",
"bytes",
"headers-core",
"http",
@@ -1206,7 +1219,7 @@ version = "9.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c7ea04a7c5c055c175f189b6dc6ba036fd62306b58c66c9f6389036c503a3f4"
dependencies = [
- "base64",
+ "base64 0.21.7",
"js-sys",
"pem",
"ring",
@@ -1746,7 +1759,7 @@ version = "3.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b8fcc794035347fb64beda2d3b462595dd2753e3f268d89c5aae77e8cf2c310"
dependencies = [
- "base64",
+ "base64 0.21.7",
"serde",
]
@@ -2015,7 +2028,7 @@ version = "0.11.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251"
dependencies = [
- "base64",
+ "base64 0.21.7",
"bytes",
"encoding_rs",
"futures-core",
@@ -2089,7 +2102,7 @@ dependencies = [
[[package]]
name = "ruma"
version = "0.9.4"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"assign",
"js_int",
@@ -2108,7 +2121,7 @@ dependencies = [
[[package]]
name = "ruma-appservice-api"
version = "0.9.0"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"js_int",
"ruma-common",
@@ -2120,7 +2133,7 @@ dependencies = [
[[package]]
name = "ruma-client-api"
version = "0.17.4"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"as_variant",
"assign",
@@ -2139,10 +2152,10 @@ dependencies = [
[[package]]
name = "ruma-common"
version = "0.12.1"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"as_variant",
- "base64",
+ "base64 0.21.7",
"bytes",
"form_urlencoded",
"http",
@@ -2167,7 +2180,7 @@ dependencies = [
[[package]]
name = "ruma-events"
version = "0.27.11"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"as_variant",
"indexmap",
@@ -2189,7 +2202,7 @@ dependencies = [
[[package]]
name = "ruma-federation-api"
version = "0.8.0"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"js_int",
"ruma-common",
@@ -2201,7 +2214,7 @@ dependencies = [
[[package]]
name = "ruma-identifiers-validation"
version = "0.9.3"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"js_int",
"thiserror",
@@ -2210,7 +2223,7 @@ dependencies = [
[[package]]
name = "ruma-identity-service-api"
version = "0.8.0"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"js_int",
"ruma-common",
@@ -2220,7 +2233,7 @@ dependencies = [
[[package]]
name = "ruma-macros"
version = "0.12.0"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"once_cell",
"proc-macro-crate",
@@ -2235,7 +2248,7 @@ dependencies = [
[[package]]
name = "ruma-push-gateway-api"
version = "0.8.0"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"js_int",
"ruma-common",
@@ -2247,9 +2260,9 @@ dependencies = [
[[package]]
name = "ruma-signatures"
version = "0.14.0"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
- "base64",
+ "base64 0.21.7",
"ed25519-dalek",
"pkcs8",
"rand",
@@ -2263,7 +2276,7 @@ dependencies = [
[[package]]
name = "ruma-state-res"
version = "0.10.0"
-source = "git+https://github.com/girlbossceo/ruma?rev=9f243f1e89bd2ef52dde521c34a791fee7b36d5a#9f243f1e89bd2ef52dde521c34a791fee7b36d5a"
+source = "git+https://github.com/girlbossceo/ruma?rev=1623fffe150356ad6a7388a9df2cfed80aae1a9e#1623fffe150356ad6a7388a9df2cfed80aae1a9e"
dependencies = [
"itertools 0.11.0",
"js_int",
@@ -2339,7 +2352,7 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c"
dependencies = [
- "base64",
+ "base64 0.21.7",
]
[[package]]
diff --git a/Cargo.toml b/Cargo.toml
index d03dfa948d36bcd24ff253e5a643da62920da5e3..d5a20770ca18eaa08088c2b836e258354117b1ea 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -115,6 +115,8 @@ axum-server-dual-protocol = { version = "0.5.2", optional = true }
# to get the client IP address of requests
#axum-client-ip = "0.4.2"
+# to parse user-friendly time durations in admin commands
+cyborgtime = "2.1.1"
[target.'cfg(unix)'.dependencies]
nix = { version = "0.28.0", features = ["resource"] }
diff --git a/src/database/key_value/media.rs b/src/database/key_value/media.rs
index dcafc74e3f5a4cb4f33d7a302e48ab9645162956..f901bd55e97b98863de0527665ca54da489b3da5 100644
--- a/src/database/key_value/media.rs
+++ b/src/database/key_value/media.rs
@@ -52,12 +52,11 @@ fn delete_file_mxc(&self, mxc: String) -> Result<()> {
debug!("Deleting key: {:?}", key);
self.mediaid_file.remove(&key)?;
}
- //return Err(Error::bad_database("Media not found."));
Ok(())
}
- /// Searches for all files with the given MXC (e.g. thumbnail and original image)
+ /// Searches for all files with the given MXC
fn search_mxc_metadata_prefix(&self, mxc: String) -> Result<Vec<Vec<u8>>> {
debug!("MXC URI: {:?}", mxc);
@@ -126,6 +125,17 @@ fn search_file_metadata(
Ok((content_disposition, content_type, key))
}
+ /// Gets all the media keys in our database (this includes all the metadata associated with it such as width, height, content-type, etc)
+ fn get_all_media_keys(&self) -> Result<Vec<Vec<u8>>> {
+ let mut keys: Vec<Vec<u8>> = vec![];
+
+ for (key, _) in self.mediaid_file.iter() {
+ keys.push(key);
+ }
+
+ Ok(keys)
+ }
+
fn remove_url_preview(&self, url: &str) -> Result<()> {
self.url_previews.remove(url.as_bytes())
}
diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs
index 0ea67abfd98df576735c8266b9364e51d10d8863..eb61de3f283d2b22e12ba9af2f403016f6a1da68 100644
--- a/src/service/admin/mod.rs
+++ b/src/service/admin/mod.rs
@@ -96,6 +96,12 @@ enum MediaCommand {
/// - Deletes a codeblock list of MXC URLs from our database and on the filesystem
DeleteList,
+
+ // - Deletes all remote media in the last <x> amount of time using filesystem metadata first created at date.
+ DeletePastRemoteMedia {
+ /// - The duration (at or after), e.g. "5m" to delete all media in the past 5 minutes
+ duration: String,
+ },
}
#[cfg_attr(test, derive(Debug))]
@@ -785,6 +791,17 @@ async fn process_admin_command(
));
}
}
+ MediaCommand::DeletePastRemoteMedia { duration } => {
+ let deleted_count = services()
+ .media
+ .delete_all_remote_media_at_after_time(duration)
+ .await?;
+
+ return Ok(RoomMessageEventContent::text_plain(format!(
+ "Deleted {} total files.",
+ deleted_count
+ )));
+ }
},
AdminCommand::Users(command) => match command {
UserCommand::List => match services().users.list_local_users() {
diff --git a/src/service/media/data.rs b/src/service/media/data.rs
index 0404b5489d878e38edb30921d441cd852f9e4e40..bb44de80c3dc69de2b111a4d033a0e66e5684efb 100644
--- a/src/service/media/data.rs
+++ b/src/service/media/data.rs
@@ -22,6 +22,8 @@ fn search_file_metadata(
fn search_mxc_metadata_prefix(&self, mxc: String) -> Result<Vec<Vec<u8>>>;
+ fn get_all_media_keys(&self) -> Result<Vec<Vec<u8>>>;
+
fn remove_url_preview(&self, url: &str) -> Result<()>;
fn set_url_preview(
diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs
index 7f8311c53f75d5464751105b93ac0e5b87bfd46c..7d021e52845eb79d5af162bdb5afde6a22de94b4 100644
--- a/src/service/media/mod.rs
+++ b/src/service/media/mod.rs
@@ -7,14 +7,15 @@
};
pub(crate) use data::Data;
+use ruma::OwnedMxcUri;
use serde::Serialize;
use tracing::{debug, error};
-use crate::{services, Error, Result};
+use crate::{services, utils, Error, Result};
use image::imageops::FilterType;
use tokio::{
- fs::File,
+ fs::{self, File},
io::{AsyncReadExt, AsyncWriteExt, BufReader},
sync::Mutex,
};
@@ -174,6 +175,112 @@ pub async fn get(&self, mxc: String) -> Result<Option<FileMeta>> {
}
}
+ /// Deletes all remote only media files in the given at or after time/duration. Returns a u32
+ /// with the amount of media files deleted.
+ pub async fn delete_all_remote_media_at_after_time(&self, time: String) -> Result<u32> {
+ if let Ok(all_keys) = self.db.get_all_media_keys() {
+ let user_duration: SystemTime = match cyborgtime::parse_duration(&time) {
+ Ok(duration) => {
+ debug!("Parsed duration: {:?}", duration);
+ debug!("System time now: {:?}", SystemTime::now());
+ SystemTime::now() - duration
+ }
+ Err(e) => {
+ error!("Failed to parse user-specified time duration: {}", e);
+ return Err(Error::bad_database(
+ "Failed to parse user-specified time duration.",
+ ));
+ }
+ };
+
+ let mut remote_mxcs: Vec<String> = vec![];
+
+ for key in all_keys {
+ debug!("Full MXC key from database: {:?}", key);
+
+ // we need to get the MXC URL from the first part of the key (the first 0xff / 255 push)
+ // this code does look kinda crazy but blame conduit for using magic keys
+ let mut parts = key.split(|&b| b == 0xff);
+ let mxc = parts
+ .next()
+ .map(|bytes| {
+ utils::string_from_bytes(bytes).map_err(|e| {
+ error!("Failed to parse MXC unicode bytes from our database: {}", e);
+ Error::bad_database(
+ "Failed to parse MXC unicode bytes from our database",
+ )
+ })
+ })
+ .transpose()?;
+
+ let mxc_s = match mxc {
+ Some(mxc) => mxc,
+ None => {
+ return Err(Error::bad_database(
+ "Parsed MXC URL unicode bytes from database but still is None",
+ ));
+ }
+ };
+
+ debug!("Parsed MXC key to URL: {}", mxc_s);
+
+ let mxc = OwnedMxcUri::from(mxc_s);
+ if mxc.server_name() == Ok(services().globals.server_name()) {
+ debug!("Ignoring local media MXC: {}", mxc);
+ // ignore our own MXC URLs as this would be local media.
+ continue;
+ }
+
+ let path = if cfg!(feature = "sha256_media") {
+ services().globals.get_media_file_new(&key)
+ } else {
+ #[allow(deprecated)]
+ services().globals.get_media_file(&key)
+ };
+
+ debug!("MXC path: {:?}", path);
+
+ let file_metadata = fs::metadata(path.clone()).await?;
+ debug!("File metadata: {:?}", file_metadata);
+
+ let file_created_at = file_metadata.created()?;
+ debug!("File created at: {:?}", file_created_at);
+
+ if file_created_at >= user_duration {
+ debug!("File is within user duration, pushing to list of file paths and keys to delete.");
+ remote_mxcs.push(mxc.to_string());
+ } else {
+ // don't need to log this even in debug as it would be noisy
+ continue;
+ }
+ }
+
+ debug!("Finished going through all our media in database for eligible keys to delete, checking if these are empty");
+
+ if remote_mxcs.is_empty() {
+ return Err(Error::bad_database(
+ "Did not found any eligible MXCs to delete.",
+ ));
+ }
+
+ debug!("Deleting media now in the past \"{:?}\".", user_duration);
+
+ let mut deletion_count = 0;
+
+ for mxc in remote_mxcs {
+ debug!("Deleting MXC {mxc} from database and filesystem");
+ self.delete(mxc).await?;
+ deletion_count += 1;
+ }
+
+ Ok(deletion_count)
+ } else {
+ Err(Error::bad_database(
+ "Failed to get all our media keys (filesystem or database issue?).",
+ ))
+ }
+ }
+
/// Returns width, height of the thumbnail and whether it should be cropped. Returns None when
/// the server should send the original file.
pub fn thumbnail_properties(&self, width: u32, height: u32) -> Option<(u32, u32, bool)> {