feat: Upgrade URL Previews To The Moon Mk 2

This commit is contained in:
Jacob Taylor committed 2026-11-21 17:02:26 -08:00
1 parent 0b0f25bab1
commit 396b754fd1
8 files changed
+362 -92

No files matched your search

Generated
+1
View File
@@ -1136,6 +1136,7 @@ dependencies = [
"log",
"loole",
"lru-cache",
"mime",
"nonzero_ext",
"openidconnect",
"opentelemetry",
+3
View File
@@ -578,6 +578,9 @@ version = "0.10.4"
default-features = false
features = ["std"]
[workspace.dependencies.mime]
version = "0.3.18"
[workspace.dependencies.nonzero_ext]
version = "0.3.0"
+1
View File
@@ -0,0 +1 @@
URL previews no longer completely fail if downloading auxiliary content (images/videos/audio) fails. Contributed by @nex.
+3 -1
View File
@@ -2,7 +2,7 @@ use std::time::Duration;
use axum::extract::State;
use conduwuit::{
Err, Result, err,
Err, Result, debug, err,
utils::{self, content_disposition::make_content_disposition, math::ruma_from_usize},
};
use conduwuit_core::error;
@@ -219,10 +219,12 @@ pub(crate) async fn get_media_preview_route(
)));
}
debug!("Fetching a URL preview for {url} on behalf of {sender_user}");
let preview = services
.media
.get_url_preview(&url)
.await
.inspect(|_| debug!(user=%sender_user, %url, "Finished fetching a URL preview"))
.map_err(|error| {
err!(Request(Unknown(
debug_error!(%sender_user, %url, "Failed to fetch URL preview: {error}")
+1
View File
@@ -96,6 +96,7 @@ itertools.workspace = true
log.workspace = true
loole.workspace = true
lru-cache.workspace = true
mime.workspace = true
nonzero_ext.workspace = true
p256.workspace = true
regex.workspace = true
+1 -1
View File
@@ -111,7 +111,7 @@ impl crate::Service for Service {
.local_address(url_preview_bind_addr)
.timeout(Duration::from_secs(config.url_preview_timeout))
.redirect(redirect::Policy::limited(3))
.user_agent(url_preview_user_agent)
.user_agent(url_preview_user_agent.replace("<version>", conduwuit::version_ua()))
.build()?,
external_resource: base(config, reqwest_dns_resolver.clone())?
+7
View File
@@ -223,6 +223,8 @@ impl Data {
x => x,
};*/
// TODO(nex): Oh my god this is awful when can we kill it
let title = match values
.next()
.and_then(|b| String::from_utf8(b.to_vec()).ok())
@@ -321,6 +323,11 @@ impl Data {
video_height,
audio,
audio_size,
..Default::default() /* TODO(nex,silversys): This weird manual field
* de/serialisation just needs replacing with JSON storage to
* avoid this mess. This is consequently intentionally
* dropping information added later (like image alt text).
* I'll do it later. */
})
}
}
+345 -90
View File
@@ -7,29 +7,79 @@
use std::time::SystemTime;
#[cfg(feature = "url_preview")]
use conduwuit::info;
use conduwuit::{Err, Result, debug, err, utils::response::LimitReadExt};
use ipaddress::IPAddress;
use mime::Mime;
use ruma::OwnedMxcUri;
use serde::Serialize;
use serde::{Serialize, Serializer};
use url::Url;
use super::Service;
use crate::media::mxc::Mxc;
#[derive(Serialize, Default)]
// serde `serialize_with` passes `&Option<T>` for `Option<T>` fields.
#[allow(clippy::ref_option)]
fn serialize_mime<S: Serializer>(mime: &Option<Mime>, serializer: S) -> Result<S::Ok, S::Error> {
match mime {
| Some(m) => serializer.serialize_str(m.as_ref()),
| None => serializer.serialize_none(),
}
}
#[derive(Serialize, Default, Debug)]
pub struct UrlPreviewData {
// From https://ogp.me/
/// The title of your object as it should appear within the graph
/// "The Rock".
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:title"))]
pub title: Option<String>,
/// The type of your object, e.g., "video.movie".
#[serde(skip_serializing_if = "String::is_empty", rename(serialize = "og:type"))]
pub site_type: String,
/// The canonical URL of your object that will be used as its permanent ID
/// in the graph.
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:url"))]
pub url: Option<String>,
/// A one to two sentence description of your object.
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:description"))]
pub description: Option<String>,
/// The word that appears before this object's title in a sentence.
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:determiner"))]
pub determiner: Option<String>,
/// The locale these tags are marked up in
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:locale"))]
pub locale: Option<String>,
/// If your object is part of a larger web site, the name which should be
/// displayed for the overall site.
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:site_name"))]
pub site_name: Option<String>,
/// An image URL which should represent your object within the graph.
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image"))]
pub image: Option<String>,
/// A MIME type for this image.
#[serde(
skip_serializing_if = "Option::is_none",
serialize_with = "serialize_mime",
rename(serialize = "og:image:type")
)]
pub image_type: Option<Mime>,
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "matrix:image:size"))]
pub image_size: Option<usize>,
/// The number of pixels wide.
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image:width"))]
pub image_width: Option<u32>,
/// The number of pixels high.
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image:height"))]
pub image_height: Option<u32>,
/// A description of what is in the image (not a caption).
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image:alt"))]
pub image_alt: Option<String>,
/// A URL to a video file that complements this object.
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:video"))]
pub video: Option<String>,
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "matrix:video:size"))]
@@ -38,6 +88,15 @@ pub struct UrlPreviewData {
pub video_width: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:video:height"))]
pub video_height: Option<u32>,
#[serde(
skip_serializing_if = "Option::is_none",
serialize_with = "serialize_mime",
rename(serialize = "og:video:type")
)]
pub video_type: Option<Mime>,
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:video:alt"))]
pub video_alt: Option<String>,
/// A URL to an audio file to accompany this object.
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:audio"))]
pub audio: Option<String>,
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "matrix:audio:size"))]
@@ -78,43 +137,39 @@ impl Service {
}
async fn request_url_preview(&self, url: &Url) -> Result<UrlPreviewData> {
if let Ok(ip) = IPAddress::parse(url.host_str().expect("URL previously validated")) {
if !self.services.client.valid_cidr_range(&ip) {
return Err!(Request(Forbidden("Requesting from this address is forbidden")));
}
}
self.check_url_ip(url.as_str())?;
let client = &self.services.client.url_preview;
let response = client.head(url.as_str()).send().await?;
debug!(%url, "URL preview response headers: {:?}", response.headers());
if let Some(remote_addr) = response.remote_addr() {
debug!(%url, "URL preview response remote address: {:?}", remote_addr);
if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) {
if !self.services.client.valid_cidr_range(&ip) {
return Err!(Request(Forbidden("Requesting from this address is forbidden")));
}
}
self.check_response_ip(&response)?;
if !response.status().is_success() {
return Err!(Request(Unknown(
"URL preview request returned HTTP {}",
response.status()
)));
}
let Some(content_type) = response.headers().get(reqwest::header::CONTENT_TYPE) else {
return Err!(Request(Unknown("Unknown or invalid Content-Type header")));
return Err!(Request(Unknown(
"Response contains unknown, invalid, or missing Content-Type header"
)));
};
let content_type = content_type
let content_type: Mime = content_type
.to_str()
.map_err(|e| err!(Request(Unknown("Unknown or invalid Content-Type header: {e}"))))?
.parse()
.map_err(|e| err!(Request(Unknown("Unknown or invalid Content-Type header: {e}"))))?;
let data = match content_type {
| html if html.starts_with("text/html") => self.download_html(url.as_str()).await?,
| img if img.starts_with("image/") => self.download_image(url.as_str(), None).await?,
| video if video.starts_with("video/") =>
self.download_video(url.as_str(), None).await?,
| audio if audio.starts_with("audio/") =>
self.download_audio(url.as_str(), None).await?,
| _ => return Err!(Request(Unknown("Unsupported Content-Type"))),
let data = match (content_type.type_(), content_type.subtype()) {
| (mime::TEXT, mime::HTML) => self.download_html(url.as_str()).await?,
| (mime::IMAGE, _) => self.download_image(url.as_str()).await?,
| (mime::VIDEO, _) => self.download_video(url.as_str()).await?,
| (mime::AUDIO, _) => self.download_audio(url.as_str()).await?,
| _ => return Err!(Request(Unknown("Unsupported Content-Type: {content_type}"))),
};
self.set_url_preview(url.as_str(), &data).await?;
@@ -123,19 +178,16 @@ impl Service {
}
#[cfg(feature = "url_preview")]
pub async fn download_image(
&self,
url: &str,
preview_data: Option<UrlPreviewData>,
) -> Result<UrlPreviewData> {
pub async fn download_image(&self, url: &str) -> Result<UrlPreviewData> {
use conduwuit::utils::random_string;
use image::ImageReader;
use crate::media::mxc::Mxc;
let mut preview_data = UrlPreviewData::default();
let mut preview_data = preview_data.unwrap_or_default();
self.check_url_ip(url)?;
let response = self.services.client.url_preview.get(url).send().await?;
self.check_response_ip(&response)?;
if !response.status().is_success() {
return Err!(Request(Unknown("Image download returned HTTP {}", response.status())));
}
@@ -157,13 +209,13 @@ impl Service {
.ok_or_else(|| err!(Request(Unknown("Downloaded file is not a supported image"))))?
.to_mime_type();
if preview_data.image_height.is_none() || preview_data.image_width.is_none() {
let (width, height) = reader.into_dimensions().map_err(|error| {
err!(Request(Unknown("Failed to read preview image dimensions: {error}")))
})?;
preview_data.image_width.get_or_insert(width);
preview_data.image_height.get_or_insert(height);
}
let (width, height) = reader.into_dimensions().map_err(|error| {
err!(Request(Unknown("Failed to read preview image dimensions: {error}")))
})?;
preview_data.image_width = Some(width);
preview_data.image_height = Some(height);
preview_data.image_size = Some(image.len());
preview_data.image_type = content_type.parse().ok();
let mxc = Mxc {
server_name: self.services.globals.server_name(),
@@ -179,12 +231,9 @@ impl Service {
}
#[cfg(feature = "url_preview")]
pub async fn download_video(
&self,
url: &str,
preview_data: Option<UrlPreviewData>,
) -> Result<UrlPreviewData> {
let mut preview_data = preview_data.unwrap_or_default();
#[tracing::instrument(skip(self))]
pub async fn download_video(&self, url: &str) -> Result<UrlPreviewData> {
let mut preview_data = UrlPreviewData::default();
if self.services.globals.url_preview_allow_audio_video() {
let (url, size) = self.download_media(url).await?;
@@ -196,12 +245,9 @@ impl Service {
}
#[cfg(feature = "url_preview")]
pub async fn download_audio(
&self,
url: &str,
preview_data: Option<UrlPreviewData>,
) -> Result<UrlPreviewData> {
let mut preview_data = preview_data.unwrap_or_default();
#[tracing::instrument(skip(self))]
pub async fn download_audio(&self, url: &str) -> Result<UrlPreviewData> {
let mut preview_data = UrlPreviewData::default();
if self.services.globals.url_preview_allow_audio_video() {
let (url, size) = self.download_media(url).await?;
@@ -212,11 +258,18 @@ impl Service {
Ok(preview_data)
}
#[tracing::instrument(skip(self))]
pub async fn download_media(&self, url: &str) -> Result<(OwnedMxcUri, usize)> {
use conduwuit::utils::random_string;
use http::header::CONTENT_TYPE;
self.check_url_ip(url)?;
let response = self.services.client.url_preview.get(url).send().await?;
self.check_response_ip(&response)?;
if !response.status().is_success() {
return Err!(Request(Unknown("Media download returned HTTP {}", response.status())));
}
let content_type = response.headers().get(CONTENT_TYPE).cloned();
let media = response
.limit_read(
@@ -242,41 +295,32 @@ impl Service {
}
#[cfg(not(feature = "url_preview"))]
pub async fn download_image(
&self,
_url: &str,
_preview_data: Option<UrlPreviewData>,
) -> Result<UrlPreviewData> {
Err!(FeatureDisabled("url_preview"))
pub fn download_image(&self, _url: &str) -> impl Future<Output = Result<UrlPreviewData>> {
std::future::ready(Err!(FeatureDisabled("url_preview")))
}
#[cfg(not(feature = "url_preview"))]
pub async fn download_video(
&self,
_url: &str,
_preview_data: Option<UrlPreviewData>,
) -> Result<UrlPreviewData> {
Err!(FeatureDisabled("url_preview"))
pub fn download_video(&self, _url: &str) -> impl Future<Output = Result<UrlPreviewData>> {
std::future::ready(Err!(FeatureDisabled("url_preview")))
}
#[cfg(not(feature = "url_preview"))]
pub async fn download_audio(
&self,
_url: &str,
_preview_data: Option<UrlPreviewData>,
) -> Result<UrlPreviewData> {
Err!(FeatureDisabled("url_preview"))
pub fn download_audio(&self, _url: &str) -> impl Future<Output = Result<UrlPreviewData>> {
std::future::ready(Err!(FeatureDisabled("url_preview")))
}
#[cfg(feature = "url_preview")]
#[tracing::instrument(skip(self))]
async fn download_html(&self, url: &str) -> Result<UrlPreviewData> {
use webpage::HTML;
let client = &self.services.client.url_preview;
let body = client
.get(url)
.send()
.await?
let response = client.get(url).send().await?;
self.check_response_ip(&response)?;
if !response.status().is_success() {
return Err!(Request(Unknown("HTML download returned HTTP {}", response.status())));
}
let body = response
.limit_read_text(
self.services
.server
@@ -286,39 +330,138 @@ impl Service {
.expect("u64 should fit in usize"),
)
.await?;
let Ok(html) = HTML::from_string(body.clone(), Some(url.to_owned())) else {
let Ok(html) = HTML::from_string(body, Some(url.to_owned()))
.inspect_err(|e| debug!(error=?e, "Failed to parse HTML data"))
else {
return Err!(Request(Unknown("Failed to parse HTML")));
};
let mut preview_data = UrlPreviewData::default();
let mut preview_data = UrlPreviewData::from(&html.opengraph);
debug!(parsed=?preview_data, deserialised=?html.opengraph, "Parsed opengraph data");
if let Some(obj) = html.opengraph.images.first() {
preview_data = self.download_image(&obj.url, Some(preview_data)).await?;
let base_url = Url::parse(url).ok();
let resolve_asset_url = |asset_url: &str| -> String {
base_url
.as_ref()
.and_then(|base| base.join(asset_url).ok())
.map_or_else(|| asset_url.to_owned(), Into::into)
};
if let Some(obj) = html
.opengraph
.images
.first()
.filter(|obj| !obj.url.is_empty())
{
let asset_url = resolve_asset_url(&obj.url);
info!(%asset_url, "Downloading preview image");
if let Ok(image_data) = self.download_image(&asset_url).await.inspect_err(
|e| info!(error=?e, %asset_url, "Could not download URL preview image"),
) {
preview_data.image = image_data.image;
preview_data.image_size = image_data.image_size;
// Prioritise what the top-level site specified, falling back to
// detected params.
preview_data.image_width = preview_data.image_width.or(image_data.image_width);
preview_data.image_height = preview_data.image_height.or(image_data.image_height);
preview_data.image_type = preview_data.image_type.or(image_data.image_type);
}
}
if let Some(obj) = html.opengraph.videos.first() {
preview_data = self.download_video(&obj.url, Some(preview_data)).await?;
preview_data.video_width = obj.properties.get("width").and_then(|v| v.parse().ok());
preview_data.video_height = obj.properties.get("height").and_then(|v| v.parse().ok());
if let Some(obj) = html
.opengraph
.videos
.first()
.filter(|obj| !obj.url.is_empty())
{
let asset_url = resolve_asset_url(&obj.url);
info!(%asset_url, "Downloading preview video");
if let Ok(video_data) = self.download_video(&asset_url).await.inspect_err(
|e| info!(error=?e, %asset_url, "Could not download URL preview video"),
) {
preview_data.video = video_data.video;
preview_data.video_size = video_data.video_size;
}
}
if let Some(obj) = html.opengraph.audios.first() {
preview_data = self.download_audio(&obj.url, Some(preview_data)).await?;
if let Some(obj) = html
.opengraph
.audios
.first()
.filter(|obj| !obj.url.is_empty())
{
let asset_url = resolve_asset_url(&obj.url);
info!(%asset_url, "Downloading URL preview audio");
if let Ok(audio_data) = self.download_audio(&asset_url).await.inspect_err(
|e| info!(error=?e, %asset_url, "Could not download URL preview audio"),
) {
preview_data.audio = audio_data.audio;
preview_data.audio_size = audio_data.audio_size;
}
}
let props = html.opengraph.properties;
// Provide defaults for some params
preview_data.title = preview_data.title.or(html.title);
preview_data.description = preview_data.description.or(html.description);
preview_data.url = preview_data.url.or_else(|| Some(url.to_owned()));
/* use OpenGraph title/description, but fall back to HTML if not
* available */
preview_data.title = props.get("title").cloned().or(html.title);
preview_data.description = props.get("description").cloned().or(html.description);
// We remove og:image:*, og:video:*, and og:audio:* if their respective
// prior-level properties are not present.
// We do this here instead of in the From<> impl because we can't know
// if we'll be able to fetch the media to convert it to an mxc that
// point.
if preview_data.image.is_none() {
preview_data.image_type = None;
preview_data.image_alt = None;
preview_data.image_height = None;
preview_data.image_width = None;
preview_data.image_size = None;
}
if preview_data.video.is_none() {
preview_data.video_type = None;
preview_data.video_alt = None;
preview_data.video_height = None;
preview_data.video_width = None;
preview_data.video_size = None;
}
if preview_data.audio.is_none() {
preview_data.audio_size = None;
}
Ok(preview_data)
}
#[cfg(not(feature = "url_preview"))]
async fn download_html(&self, _url: &str) -> Result<UrlPreviewData> {
Err!(FeatureDisabled("url_preview"))
#[allow(clippy::unused_self)]
fn download_html(&self, _url: &str) -> impl Future<Output = Result<UrlPreviewData>> {
std::future::ready(Err!(FeatureDisabled("url_preview")))
}
fn check_url_ip(&self, url: &str) -> Result<()> {
let parsed = Url::parse(url)
.map_err(|e| err!(Request(InvalidParam("Requested URL is not valid: {e}"))))?;
let Some(host) = parsed.host_str() else {
return Err!(Request(InvalidParam("Requested URL does not have a host")));
};
if let Ok(ip) = IPAddress::parse(host) {
if !self.services.client.valid_cidr_range(&ip) {
return Err!(Request(Forbidden("Requesting from this address is forbidden")));
}
}
Ok(())
}
fn check_response_ip(&self, response: &reqwest::Response) -> Result<()> {
if let Some(remote_addr) = response.remote_addr() {
debug!(url = %response.url(), "URL preview response remote address: {:?}", remote_addr);
if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) {
if !self.services.client.valid_cidr_range(&ip) {
return Err!(Request(Forbidden("Requesting from this address is forbidden")));
}
}
}
Ok(())
}
pub fn url_preview_allowed(&self, url: &Url) -> bool {
@@ -443,3 +586,115 @@ impl Service {
false
}
}
#[cfg(feature = "url_preview")]
impl From<&webpage::Opengraph> for UrlPreviewData {
fn from(og: &webpage::Opengraph) -> Self {
Self {
title: og.properties.get("title").cloned(),
description: og.properties.get("description").cloned(),
site_type: og.og_type.clone(),
url: og.properties.get("url").cloned(),
determiner: og.properties.get("determiner").cloned(),
locale: og.properties.get("locale").cloned(),
site_name: og.properties.get("site_name").cloned(),
image_type: og
.images
.first()
.and_then(|img| img.properties.get("type").and_then(|t| t.parse().ok())),
image_width: og
.images
.first()
.and_then(|img| img.properties.get("width").and_then(|w| w.parse().ok())),
image_height: og
.images
.first()
.and_then(|img| img.properties.get("height").and_then(|h| h.parse().ok())),
image_alt: og
.images
.first()
.and_then(|img| img.properties.get("alt").cloned()),
video_type: og
.videos
.first()
.and_then(|vid| vid.properties.get("type"))
.and_then(|t| t.parse().ok()),
video_width: og
.videos
.first()
.and_then(|vid| vid.properties.get("width"))
.and_then(|w| w.parse().ok()),
video_height: og
.videos
.first()
.and_then(|vid| vid.properties.get("height"))
.and_then(|h| h.parse().ok()),
video_alt: og
.videos
.first()
.and_then(|vid| vid.properties.get("alt").cloned()),
..Default::default()
}
}
}
#[cfg(test)]
mod tests {
#[cfg(feature = "url_preview")]
#[test]
fn test_urlpreviewdata_parses_correctly() {
use webpage::HTML;
use super::UrlPreviewData;
let html = r#"
<html>
<head>
<meta property="og:title" content="Test Title" />
<meta property="og:description" content="Test Description" />
<meta property="og:image" content="https://example.com/image.png" />
<meta property="og:url" content="https://example.com/index.html" />
<meta property="og:type" content="org.continuwuity.other" />
<meta property="og:site_name" content="Example Site" />
<meta property="og:image:alt" content="An example image" />
<meta property="og:image:type" content="image/png" />
<meta property="og:image:height" content="69" />
<meta property="og:image:width" content="420" />
<meta property="og:video" content="https://example.com/video.mp4" />
<meta property="og:audio" content="https://example.com/audio.ogg" />
</head>
<body></body>
</html>
"#;
let parsed_html =
HTML::from_string(html.to_owned(), Some("https://example.com".to_owned()))
.expect("Failed to parse HTML");
let data = UrlPreviewData::from(&parsed_html.opengraph);
assert_eq!(data.title, Some("Test Title".to_owned()));
assert_eq!(data.description, Some("Test Description".to_owned()));
assert_eq!(
data.image, None,
"UrlPreviewData should not inherit HTTP/S URL from URL preview"
);
assert_eq!(
data.video, None,
"UrlPreviewData should not inherit HTTP/S URL from URL preview"
);
assert_eq!(
data.audio, None,
"UrlPreviewData should not inherit HTTP/S URL from URL preview"
);
assert_eq!(data.url, Some("https://example.com/index.html".to_owned()));
assert_eq!(data.site_type, "org.continuwuity.other".to_owned());
assert_eq!(data.site_name, Some("Example Site".to_owned()));
assert_eq!(data.image_alt, Some("An example image".to_owned()));
assert_eq!(data.image_type, Some("image/png".parse().unwrap()));
assert_eq!(data.image_height, Some(69));
assert_eq!(data.image_width, Some(420));
assert_eq!(data.image_size, None, "*_size must be None when pulling from opengraph");
assert_eq!(data.video_size, None, "*_size must be None when pulling from opengraph");
assert_eq!(data.audio_size, None, "*_size must be None when pulling from opengraph");
}
}