diff --git a/Cargo.lock b/Cargo.lock index 1922a25f8..e62755c72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1136,6 +1136,7 @@ dependencies = [ "log", "loole", "lru-cache", + "mime", "nonzero_ext", "openidconnect", "opentelemetry", diff --git a/Cargo.toml b/Cargo.toml index 1d97d0be4..cafbaebc2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -578,6 +578,9 @@ version = "0.10.4" default-features = false features = ["std"] +[workspace.dependencies.mime] +version = "0.3.18" + [workspace.dependencies.nonzero_ext] version = "0.3.0" diff --git a/changelog.d/2448.bugfix.md b/changelog.d/2448.bugfix.md new file mode 100644 index 000000000..557e68c4a --- /dev/null +++ b/changelog.d/2448.bugfix.md @@ -0,0 +1 @@ +URL previews no longer completely fail if downloading auxiliary content (images/videos/audio) fails. Contributed by @nex. diff --git a/src/api/client/media.rs b/src/api/client/media.rs index 669cda35d..a078bb968 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -2,7 +2,7 @@ use std::time::Duration; use axum::extract::State; use conduwuit::{ - Err, Result, err, + Err, Result, debug, err, utils::{self, content_disposition::make_content_disposition, math::ruma_from_usize}, }; use conduwuit_core::error; @@ -219,10 +219,12 @@ pub(crate) async fn get_media_preview_route( ))); } + debug!("Fetching a URL preview for {url} on behalf of {sender_user}"); let preview = services .media .get_url_preview(&url) .await + .inspect(|_| debug!(user=%sender_user, %url, "Finished fetching a URL preview")) .map_err(|error| { err!(Request(Unknown( debug_error!(%sender_user, %url, "Failed to fetch URL preview: {error}") diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index 0e3670134..fb9240529 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -96,6 +96,7 @@ itertools.workspace = true log.workspace = true loole.workspace = true lru-cache.workspace = true +mime.workspace = true nonzero_ext.workspace = true p256.workspace = true regex.workspace = true diff --git a/src/service/client/mod.rs b/src/service/client/mod.rs index 2cd157f84..5959c0a8a 100644 --- a/src/service/client/mod.rs +++ b/src/service/client/mod.rs @@ -111,7 +111,7 @@ impl crate::Service for Service { .local_address(url_preview_bind_addr) .timeout(Duration::from_secs(config.url_preview_timeout)) .redirect(redirect::Policy::limited(3)) - .user_agent(url_preview_user_agent) + .user_agent(url_preview_user_agent.replace("", conduwuit::version_ua())) .build()?, external_resource: base(config, reqwest_dns_resolver.clone())? diff --git a/src/service/media/data.rs b/src/service/media/data.rs index d4a1c15cd..bc20ee0b8 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -223,6 +223,8 @@ impl Data { x => x, };*/ + // TODO(nex): Oh my god this is awful when can we kill it + let title = match values .next() .and_then(|b| String::from_utf8(b.to_vec()).ok()) @@ -321,6 +323,11 @@ impl Data { video_height, audio, audio_size, + ..Default::default() /* TODO(nex,silversys): This weird manual field + * de/serialisation just needs replacing with JSON storage to + * avoid this mess. This is consequently intentionally + * dropping information added later (like image alt text). + * I'll do it later. */ }) } } diff --git a/src/service/media/preview.rs b/src/service/media/preview.rs index 084aa9a1b..de649b709 100644 --- a/src/service/media/preview.rs +++ b/src/service/media/preview.rs @@ -7,29 +7,79 @@ use std::time::SystemTime; +#[cfg(feature = "url_preview")] +use conduwuit::info; use conduwuit::{Err, Result, debug, err, utils::response::LimitReadExt}; use ipaddress::IPAddress; +use mime::Mime; use ruma::OwnedMxcUri; -use serde::Serialize; +use serde::{Serialize, Serializer}; use url::Url; use super::Service; use crate::media::mxc::Mxc; -#[derive(Serialize, Default)] +// serde `serialize_with` passes `&Option` for `Option` fields. +#[allow(clippy::ref_option)] +fn serialize_mime(mime: &Option, serializer: S) -> Result { + match mime { + | Some(m) => serializer.serialize_str(m.as_ref()), + | None => serializer.serialize_none(), + } +} + +#[derive(Serialize, Default, Debug)] pub struct UrlPreviewData { + // From https://ogp.me/ + /// The title of your object as it should appear within the graph + /// "The Rock". #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:title"))] pub title: Option, + + /// The type of your object, e.g., "video.movie". + #[serde(skip_serializing_if = "String::is_empty", rename(serialize = "og:type"))] + pub site_type: String, + /// The canonical URL of your object that will be used as its permanent ID + /// in the graph. + #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:url"))] + pub url: Option, + /// A one to two sentence description of your object. #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:description"))] pub description: Option, + /// The word that appears before this object's title in a sentence. + #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:determiner"))] + pub determiner: Option, + /// The locale these tags are marked up in + #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:locale"))] + pub locale: Option, + /// If your object is part of a larger web site, the name which should be + /// displayed for the overall site. + #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:site_name"))] + pub site_name: Option, + + /// An image URL which should represent your object within the graph. #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image"))] pub image: Option, + /// A MIME type for this image. + #[serde( + skip_serializing_if = "Option::is_none", + serialize_with = "serialize_mime", + rename(serialize = "og:image:type") + )] + pub image_type: Option, #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "matrix:image:size"))] pub image_size: Option, + /// The number of pixels wide. #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image:width"))] pub image_width: Option, + /// The number of pixels high. #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image:height"))] pub image_height: Option, + /// A description of what is in the image (not a caption). + #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image:alt"))] + pub image_alt: Option, + + /// A URL to a video file that complements this object. #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:video"))] pub video: Option, #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "matrix:video:size"))] @@ -38,6 +88,15 @@ pub struct UrlPreviewData { pub video_width: Option, #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:video:height"))] pub video_height: Option, + #[serde( + skip_serializing_if = "Option::is_none", + serialize_with = "serialize_mime", + rename(serialize = "og:video:type") + )] + pub video_type: Option, + #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:video:alt"))] + pub video_alt: Option, + /// A URL to an audio file to accompany this object. #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:audio"))] pub audio: Option, #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "matrix:audio:size"))] @@ -78,43 +137,39 @@ impl Service { } async fn request_url_preview(&self, url: &Url) -> Result { - if let Ok(ip) = IPAddress::parse(url.host_str().expect("URL previously validated")) { - if !self.services.client.valid_cidr_range(&ip) { - return Err!(Request(Forbidden("Requesting from this address is forbidden"))); - } - } + self.check_url_ip(url.as_str())?; let client = &self.services.client.url_preview; let response = client.head(url.as_str()).send().await?; debug!(%url, "URL preview response headers: {:?}", response.headers()); - if let Some(remote_addr) = response.remote_addr() { - debug!(%url, "URL preview response remote address: {:?}", remote_addr); - - if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) { - if !self.services.client.valid_cidr_range(&ip) { - return Err!(Request(Forbidden("Requesting from this address is forbidden"))); - } - } + self.check_response_ip(&response)?; + if !response.status().is_success() { + return Err!(Request(Unknown( + "URL preview request returned HTTP {}", + response.status() + ))); } let Some(content_type) = response.headers().get(reqwest::header::CONTENT_TYPE) else { - return Err!(Request(Unknown("Unknown or invalid Content-Type header"))); + return Err!(Request(Unknown( + "Response contains unknown, invalid, or missing Content-Type header" + ))); }; - let content_type = content_type + let content_type: Mime = content_type .to_str() + .map_err(|e| err!(Request(Unknown("Unknown or invalid Content-Type header: {e}"))))? + .parse() .map_err(|e| err!(Request(Unknown("Unknown or invalid Content-Type header: {e}"))))?; - let data = match content_type { - | html if html.starts_with("text/html") => self.download_html(url.as_str()).await?, - | img if img.starts_with("image/") => self.download_image(url.as_str(), None).await?, - | video if video.starts_with("video/") => - self.download_video(url.as_str(), None).await?, - | audio if audio.starts_with("audio/") => - self.download_audio(url.as_str(), None).await?, - | _ => return Err!(Request(Unknown("Unsupported Content-Type"))), + let data = match (content_type.type_(), content_type.subtype()) { + | (mime::TEXT, mime::HTML) => self.download_html(url.as_str()).await?, + | (mime::IMAGE, _) => self.download_image(url.as_str()).await?, + | (mime::VIDEO, _) => self.download_video(url.as_str()).await?, + | (mime::AUDIO, _) => self.download_audio(url.as_str()).await?, + | _ => return Err!(Request(Unknown("Unsupported Content-Type: {content_type}"))), }; self.set_url_preview(url.as_str(), &data).await?; @@ -123,19 +178,16 @@ impl Service { } #[cfg(feature = "url_preview")] - pub async fn download_image( - &self, - url: &str, - preview_data: Option, - ) -> Result { + pub async fn download_image(&self, url: &str) -> Result { use conduwuit::utils::random_string; use image::ImageReader; - use crate::media::mxc::Mxc; + let mut preview_data = UrlPreviewData::default(); - let mut preview_data = preview_data.unwrap_or_default(); + self.check_url_ip(url)?; let response = self.services.client.url_preview.get(url).send().await?; + self.check_response_ip(&response)?; if !response.status().is_success() { return Err!(Request(Unknown("Image download returned HTTP {}", response.status()))); } @@ -157,13 +209,13 @@ impl Service { .ok_or_else(|| err!(Request(Unknown("Downloaded file is not a supported image"))))? .to_mime_type(); - if preview_data.image_height.is_none() || preview_data.image_width.is_none() { - let (width, height) = reader.into_dimensions().map_err(|error| { - err!(Request(Unknown("Failed to read preview image dimensions: {error}"))) - })?; - preview_data.image_width.get_or_insert(width); - preview_data.image_height.get_or_insert(height); - } + let (width, height) = reader.into_dimensions().map_err(|error| { + err!(Request(Unknown("Failed to read preview image dimensions: {error}"))) + })?; + preview_data.image_width = Some(width); + preview_data.image_height = Some(height); + preview_data.image_size = Some(image.len()); + preview_data.image_type = content_type.parse().ok(); let mxc = Mxc { server_name: self.services.globals.server_name(), @@ -179,12 +231,9 @@ impl Service { } #[cfg(feature = "url_preview")] - pub async fn download_video( - &self, - url: &str, - preview_data: Option, - ) -> Result { - let mut preview_data = preview_data.unwrap_or_default(); + #[tracing::instrument(skip(self))] + pub async fn download_video(&self, url: &str) -> Result { + let mut preview_data = UrlPreviewData::default(); if self.services.globals.url_preview_allow_audio_video() { let (url, size) = self.download_media(url).await?; @@ -196,12 +245,9 @@ impl Service { } #[cfg(feature = "url_preview")] - pub async fn download_audio( - &self, - url: &str, - preview_data: Option, - ) -> Result { - let mut preview_data = preview_data.unwrap_or_default(); + #[tracing::instrument(skip(self))] + pub async fn download_audio(&self, url: &str) -> Result { + let mut preview_data = UrlPreviewData::default(); if self.services.globals.url_preview_allow_audio_video() { let (url, size) = self.download_media(url).await?; @@ -212,11 +258,18 @@ impl Service { Ok(preview_data) } + #[tracing::instrument(skip(self))] pub async fn download_media(&self, url: &str) -> Result<(OwnedMxcUri, usize)> { use conduwuit::utils::random_string; use http::header::CONTENT_TYPE; + self.check_url_ip(url)?; + let response = self.services.client.url_preview.get(url).send().await?; + self.check_response_ip(&response)?; + if !response.status().is_success() { + return Err!(Request(Unknown("Media download returned HTTP {}", response.status()))); + } let content_type = response.headers().get(CONTENT_TYPE).cloned(); let media = response .limit_read( @@ -242,41 +295,32 @@ impl Service { } #[cfg(not(feature = "url_preview"))] - pub async fn download_image( - &self, - _url: &str, - _preview_data: Option, - ) -> Result { - Err!(FeatureDisabled("url_preview")) + pub fn download_image(&self, _url: &str) -> impl Future> { + std::future::ready(Err!(FeatureDisabled("url_preview"))) } #[cfg(not(feature = "url_preview"))] - pub async fn download_video( - &self, - _url: &str, - _preview_data: Option, - ) -> Result { - Err!(FeatureDisabled("url_preview")) + pub fn download_video(&self, _url: &str) -> impl Future> { + std::future::ready(Err!(FeatureDisabled("url_preview"))) } #[cfg(not(feature = "url_preview"))] - pub async fn download_audio( - &self, - _url: &str, - _preview_data: Option, - ) -> Result { - Err!(FeatureDisabled("url_preview")) + pub fn download_audio(&self, _url: &str) -> impl Future> { + std::future::ready(Err!(FeatureDisabled("url_preview"))) } #[cfg(feature = "url_preview")] + #[tracing::instrument(skip(self))] async fn download_html(&self, url: &str) -> Result { use webpage::HTML; let client = &self.services.client.url_preview; - let body = client - .get(url) - .send() - .await? + let response = client.get(url).send().await?; + self.check_response_ip(&response)?; + if !response.status().is_success() { + return Err!(Request(Unknown("HTML download returned HTTP {}", response.status()))); + } + let body = response .limit_read_text( self.services .server @@ -286,39 +330,138 @@ impl Service { .expect("u64 should fit in usize"), ) .await?; - let Ok(html) = HTML::from_string(body.clone(), Some(url.to_owned())) else { + let Ok(html) = HTML::from_string(body, Some(url.to_owned())) + .inspect_err(|e| debug!(error=?e, "Failed to parse HTML data")) + else { return Err!(Request(Unknown("Failed to parse HTML"))); }; - let mut preview_data = UrlPreviewData::default(); + let mut preview_data = UrlPreviewData::from(&html.opengraph); + debug!(parsed=?preview_data, deserialised=?html.opengraph, "Parsed opengraph data"); - if let Some(obj) = html.opengraph.images.first() { - preview_data = self.download_image(&obj.url, Some(preview_data)).await?; + let base_url = Url::parse(url).ok(); + let resolve_asset_url = |asset_url: &str| -> String { + base_url + .as_ref() + .and_then(|base| base.join(asset_url).ok()) + .map_or_else(|| asset_url.to_owned(), Into::into) + }; + + if let Some(obj) = html + .opengraph + .images + .first() + .filter(|obj| !obj.url.is_empty()) + { + let asset_url = resolve_asset_url(&obj.url); + info!(%asset_url, "Downloading preview image"); + if let Ok(image_data) = self.download_image(&asset_url).await.inspect_err( + |e| info!(error=?e, %asset_url, "Could not download URL preview image"), + ) { + preview_data.image = image_data.image; + preview_data.image_size = image_data.image_size; + // Prioritise what the top-level site specified, falling back to + // detected params. + preview_data.image_width = preview_data.image_width.or(image_data.image_width); + preview_data.image_height = preview_data.image_height.or(image_data.image_height); + preview_data.image_type = preview_data.image_type.or(image_data.image_type); + } } - if let Some(obj) = html.opengraph.videos.first() { - preview_data = self.download_video(&obj.url, Some(preview_data)).await?; - preview_data.video_width = obj.properties.get("width").and_then(|v| v.parse().ok()); - preview_data.video_height = obj.properties.get("height").and_then(|v| v.parse().ok()); + if let Some(obj) = html + .opengraph + .videos + .first() + .filter(|obj| !obj.url.is_empty()) + { + let asset_url = resolve_asset_url(&obj.url); + info!(%asset_url, "Downloading preview video"); + if let Ok(video_data) = self.download_video(&asset_url).await.inspect_err( + |e| info!(error=?e, %asset_url, "Could not download URL preview video"), + ) { + preview_data.video = video_data.video; + preview_data.video_size = video_data.video_size; + } } - if let Some(obj) = html.opengraph.audios.first() { - preview_data = self.download_audio(&obj.url, Some(preview_data)).await?; + if let Some(obj) = html + .opengraph + .audios + .first() + .filter(|obj| !obj.url.is_empty()) + { + let asset_url = resolve_asset_url(&obj.url); + info!(%asset_url, "Downloading URL preview audio"); + if let Ok(audio_data) = self.download_audio(&asset_url).await.inspect_err( + |e| info!(error=?e, %asset_url, "Could not download URL preview audio"), + ) { + preview_data.audio = audio_data.audio; + preview_data.audio_size = audio_data.audio_size; + } } - let props = html.opengraph.properties; + // Provide defaults for some params + preview_data.title = preview_data.title.or(html.title); + preview_data.description = preview_data.description.or(html.description); + preview_data.url = preview_data.url.or_else(|| Some(url.to_owned())); - /* use OpenGraph title/description, but fall back to HTML if not - * available */ - preview_data.title = props.get("title").cloned().or(html.title); - preview_data.description = props.get("description").cloned().or(html.description); + // We remove og:image:*, og:video:*, and og:audio:* if their respective + // prior-level properties are not present. + // We do this here instead of in the From<> impl because we can't know + // if we'll be able to fetch the media to convert it to an mxc that + // point. + if preview_data.image.is_none() { + preview_data.image_type = None; + preview_data.image_alt = None; + preview_data.image_height = None; + preview_data.image_width = None; + preview_data.image_size = None; + } + if preview_data.video.is_none() { + preview_data.video_type = None; + preview_data.video_alt = None; + preview_data.video_height = None; + preview_data.video_width = None; + preview_data.video_size = None; + } + if preview_data.audio.is_none() { + preview_data.audio_size = None; + } Ok(preview_data) } #[cfg(not(feature = "url_preview"))] - async fn download_html(&self, _url: &str) -> Result { - Err!(FeatureDisabled("url_preview")) + #[allow(clippy::unused_self)] + fn download_html(&self, _url: &str) -> impl Future> { + std::future::ready(Err!(FeatureDisabled("url_preview"))) + } + + fn check_url_ip(&self, url: &str) -> Result<()> { + let parsed = Url::parse(url) + .map_err(|e| err!(Request(InvalidParam("Requested URL is not valid: {e}"))))?; + let Some(host) = parsed.host_str() else { + return Err!(Request(InvalidParam("Requested URL does not have a host"))); + }; + if let Ok(ip) = IPAddress::parse(host) { + if !self.services.client.valid_cidr_range(&ip) { + return Err!(Request(Forbidden("Requesting from this address is forbidden"))); + } + } + Ok(()) + } + + fn check_response_ip(&self, response: &reqwest::Response) -> Result<()> { + if let Some(remote_addr) = response.remote_addr() { + debug!(url = %response.url(), "URL preview response remote address: {:?}", remote_addr); + + if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) { + if !self.services.client.valid_cidr_range(&ip) { + return Err!(Request(Forbidden("Requesting from this address is forbidden"))); + } + } + } + Ok(()) } pub fn url_preview_allowed(&self, url: &Url) -> bool { @@ -443,3 +586,115 @@ impl Service { false } } + +#[cfg(feature = "url_preview")] +impl From<&webpage::Opengraph> for UrlPreviewData { + fn from(og: &webpage::Opengraph) -> Self { + Self { + title: og.properties.get("title").cloned(), + description: og.properties.get("description").cloned(), + site_type: og.og_type.clone(), + url: og.properties.get("url").cloned(), + determiner: og.properties.get("determiner").cloned(), + locale: og.properties.get("locale").cloned(), + site_name: og.properties.get("site_name").cloned(), + image_type: og + .images + .first() + .and_then(|img| img.properties.get("type").and_then(|t| t.parse().ok())), + image_width: og + .images + .first() + .and_then(|img| img.properties.get("width").and_then(|w| w.parse().ok())), + image_height: og + .images + .first() + .and_then(|img| img.properties.get("height").and_then(|h| h.parse().ok())), + image_alt: og + .images + .first() + .and_then(|img| img.properties.get("alt").cloned()), + video_type: og + .videos + .first() + .and_then(|vid| vid.properties.get("type")) + .and_then(|t| t.parse().ok()), + video_width: og + .videos + .first() + .and_then(|vid| vid.properties.get("width")) + .and_then(|w| w.parse().ok()), + video_height: og + .videos + .first() + .and_then(|vid| vid.properties.get("height")) + .and_then(|h| h.parse().ok()), + video_alt: og + .videos + .first() + .and_then(|vid| vid.properties.get("alt").cloned()), + ..Default::default() + } + } +} + +#[cfg(test)] +mod tests { + #[cfg(feature = "url_preview")] + #[test] + fn test_urlpreviewdata_parses_correctly() { + use webpage::HTML; + + use super::UrlPreviewData; + let html = r#" + + + + + + + + + + + + + + + + + + "#; + + let parsed_html = + HTML::from_string(html.to_owned(), Some("https://example.com".to_owned())) + .expect("Failed to parse HTML"); + + let data = UrlPreviewData::from(&parsed_html.opengraph); + + assert_eq!(data.title, Some("Test Title".to_owned())); + assert_eq!(data.description, Some("Test Description".to_owned())); + assert_eq!( + data.image, None, + "UrlPreviewData should not inherit HTTP/S URL from URL preview" + ); + assert_eq!( + data.video, None, + "UrlPreviewData should not inherit HTTP/S URL from URL preview" + ); + assert_eq!( + data.audio, None, + "UrlPreviewData should not inherit HTTP/S URL from URL preview" + ); + assert_eq!(data.url, Some("https://example.com/index.html".to_owned())); + assert_eq!(data.site_type, "org.continuwuity.other".to_owned()); + assert_eq!(data.site_name, Some("Example Site".to_owned())); + assert_eq!(data.image_alt, Some("An example image".to_owned())); + assert_eq!(data.image_type, Some("image/png".parse().unwrap())); + assert_eq!(data.image_height, Some(69)); + assert_eq!(data.image_width, Some(420)); + assert_eq!(data.image_size, None, "*_size must be None when pulling from opengraph"); + assert_eq!(data.video_size, None, "*_size must be None when pulling from opengraph"); + assert_eq!(data.audio_size, None, "*_size must be None when pulling from opengraph"); + } +}