diff --git a/crates/core/models/src/v0/embeds.rs b/crates/core/models/src/v0/embeds.rs index 4944afae..d1ec9eb8 100644 --- a/crates/core/models/src/v0/embeds.rs +++ b/crates/core/models/src/v0/embeds.rs @@ -61,6 +61,8 @@ auto_derived!( /// YouTube video YouTube { id: String, + creator_name: Option, + creator_url: Option, #[serde(skip_serializing_if = "Option::is_none")] timestamp: Option, diff --git a/crates/services/january/src/main.rs b/crates/services/january/src/main.rs index e203ce28..1d287a94 100644 --- a/crates/services/january/src/main.rs +++ b/crates/services/january/src/main.rs @@ -11,6 +11,7 @@ use utoipa_scalar::{Scalar, Servable as ScalarServable}; mod api; pub mod requests; +pub mod specialty; pub mod website_embed; #[tokio::main] diff --git a/crates/services/january/src/requests.rs b/crates/services/january/src/requests.rs index 5566e326..7ee44cc6 100644 --- a/crates/services/january/src/requests.rs +++ b/crates/services/january/src/requests.rs @@ -20,6 +20,8 @@ use std::{ }; use url::{Host, Url}; +use crate::specialty; + lazy_static! { /// Request client static ref CLIENT: Client = reqwest::Client::builder() @@ -37,7 +39,13 @@ lazy_static! { static ref RE_URL_NEW_REDDIT: Regex = Regex::new("^(?:(?:new\\.|www\\.)?reddit).com").expect("valid regex"); /// Regex for matching YouTube Shorts URLs - static ref RE_URL_YOUTUBE_SHORTS: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:www\\.)?youtube\\.com)/shorts/([a-zA-Z0-9_-]+)").expect("valid regex"); + pub static ref RE_URL_YOUTUBE_SHORTS: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:www\\.)?youtube\\.com)/shorts/([a-zA-Z0-9_-]+)").expect("valid regex"); + + /// Regex for matching YouTube URLs + pub static ref RE_URL_YOUTUBE: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:www|m)\\.)?(?:(?:youtube\\.com|youtu\\.be))(?:/(?:[\\w\\-]+\\?v=|embed/|v/|shorts/)?)([\\w\\-]+)(?:(?:&t|&start)=([\\d]+))?(?:\\S+)?$").unwrap(); + + /// Url for YouTube oembed + pub static ref OEMBED_URL: Url = Url::parse("https://www.youtube.com/oembed").unwrap(); /// Cache for proxy results static ref PROXY_CACHE: moka::future::Cache)>> = moka::future::Cache::builder() @@ -130,8 +138,8 @@ impl reqwest::dns::Resolve for CachedDnsResolver { /// Information about a successful request pub struct Request { - response: Response, - mime: Mime, + pub response: Response, + pub mime: Mime, } impl Request { @@ -285,6 +293,16 @@ impl Request { // Generate the actual embed if let Some(hit) = EMBED_CACHE.get(&url).await { Ok(hit) + } else if RE_URL_YOUTUBE.is_match(&url) { + let mut yt_url = OEMBED_URL.clone(); + yt_url.set_query(Some(&format!("url={url}"))); + + let request = Request::new(yt_url).await?; + let embed = specialty::SpecialtySitesGenerator::youtube(&url, request).await?; + + EMBED_CACHE.insert(url.to_owned(), embed.clone()).await; + + Ok(embed) } else { let request = Request::new_from_str(&url).await?; let embed = match (request.mime.type_(), request.mime.subtype()) { diff --git a/crates/services/january/src/specialty.rs b/crates/services/january/src/specialty.rs new file mode 100644 index 00000000..6f830301 --- /dev/null +++ b/crates/services/january/src/specialty.rs @@ -0,0 +1,62 @@ +use revolt_models::v0::{Embed, Special, WebsiteMetadata}; +use revolt_result::{create_error, Result}; +use serde::Deserialize; + +use crate::requests::{Request, RE_URL_YOUTUBE}; +pub struct SpecialtySitesGenerator {} + +#[allow(unused)] +#[derive(Debug, Deserialize)] +struct YoutubeOEmbed { + pub title: String, + pub author_name: String, + pub author_url: String, + // #[serde(rename = "type")] + // pub kind: String, + // pub height: u32, + // pub width: u32, + // pub version: String, + pub provider_name: String, + // pub provider_url: String, + // pub thumbnail_height: u32, + // pub thumbnail_width: u32, + pub thumbnail_url: String, + // pub html: String, +} + +impl SpecialtySitesGenerator { + pub async fn youtube(url: &str, request: Request) -> Result { + let json: YoutubeOEmbed = request + .response + .json() + .await + .map_err(|_| create_error!(ProxyError))?; + + let captures = RE_URL_YOUTUBE + .captures(url) + .ok_or_else(|| create_error!(ProxyError))?; + let id = captures[1].to_string(); + let timestamp = captures + .get(2) + .map(|e| Some(e.as_str().to_string())) + .unwrap_or(None); + + Ok(Embed::Website(WebsiteMetadata { + url: Some(url.to_string()), + original_url: None, + special: Some(Special::YouTube { + id, + timestamp, + creator_name: Some(json.author_name), + creator_url: Some(json.author_url), + }), + title: Some(json.title), + description: None, + image: None, + video: None, + site_name: Some(json.provider_name), + icon_url: Some(json.thumbnail_url), + colour: None, + })) + } +} diff --git a/crates/services/january/src/website_embed.rs b/crates/services/january/src/website_embed.rs index 0e025b73..b4c51694 100644 --- a/crates/services/january/src/website_embed.rs +++ b/crates/services/january/src/website_embed.rs @@ -219,45 +219,6 @@ pub async fn populate_special(original_url: String, metadata: &mut WebsiteMetada Some(Special::Streamable { id: captures[1].to_string(), }) - } else if let Some(captures) = RE_YOUTUBE.captures_iter(url).next() { - let id = captures[1].to_string(); - - lazy_static! { - static ref RE_TIMESTAMP: Regex = Regex::new("(?:\\?|&)(?:t|start)=([\\w]+)").unwrap(); - } - - // YouTube now blocks datacentre IPs from fetching information - // This is a fallback to prevent the embed from looking weird - if metadata.video.is_none() { - metadata.title.replace("YouTube".to_owned()); - metadata.description.take(); - metadata.colour.take(); - metadata.icon_url.take(); - metadata.site_name.take(); - - // Verify the video exists - if !crate::requests::Request::exists_from_str(&format!( - "http://img.youtube.com/vi/{}/sddefault.jpg", - id - )) - .await - .unwrap_or(false) - { - return; - } - } - - if let Some(timestamp_captures) = RE_TIMESTAMP.captures_iter(url).next() { - Some(Special::YouTube { - id, - timestamp: Some(timestamp_captures[1].to_string()), - }) - } else { - Some(Special::YouTube { - id, - timestamp: None, - }) - } } else if let Some(captures) = RE_LIGHTSPEED.captures_iter(url).next() { Some(Special::Lightspeed { id: captures[1].to_string(),