fix: rewrite youtube embedder to use youtube oembed (#878)

* fix: rewrite youtube embedder to use youtube oembed

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

* chore: remove extra fields from oembed

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

---------

Signed-off-by: IAmTomahawkx <iamtomahawkx@gmail.com>

crates/core/models/src/v0/embeds.rs

@@ -61,6 +61,8 @@ auto_derived!(
         /// YouTube video
         YouTube {
             id: String,
+            creator_name: Option<String>,
+            creator_url: Option<String>,
 
             #[serde(skip_serializing_if = "Option::is_none")]
             timestamp: Option<String>,

crates/services/january/src/main.rs

@@ -11,6 +11,7 @@ use utoipa_scalar::{Scalar, Servable as ScalarServable};
 
 mod api;
 pub mod requests;
+pub mod specialty;
 pub mod website_embed;
 
 #[tokio::main]

crates/services/january/src/requests.rs

@@ -20,6 +20,8 @@ use std::{
 };
 use url::{Host, Url};
 
+use crate::specialty;
+
 lazy_static! {
     /// Request client
     static ref CLIENT: Client = reqwest::Client::builder()
@@ -37,7 +39,13 @@ lazy_static! {
     static ref RE_URL_NEW_REDDIT: Regex = Regex::new("^(?:(?:new\\.|www\\.)?reddit).com").expect("valid regex");
 
     /// Regex for matching YouTube Shorts URLs
-    static ref RE_URL_YOUTUBE_SHORTS: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:www\\.)?youtube\\.com)/shorts/([a-zA-Z0-9_-]+)").expect("valid regex");
+    pub static ref RE_URL_YOUTUBE_SHORTS: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:www\\.)?youtube\\.com)/shorts/([a-zA-Z0-9_-]+)").expect("valid regex");
+
+    /// Regex for matching YouTube URLs
+    pub static ref RE_URL_YOUTUBE: Regex = Regex::new("^(?:(?:https?:)?//)?(?:(?:www|m)\\.)?(?:(?:youtube\\.com|youtu\\.be))(?:/(?:[\\w\\-]+\\?v=|embed/|v/|shorts/)?)([\\w\\-]+)(?:(?:&t|&start)=([\\d]+))?(?:\\S+)?$").unwrap();
+
+    /// Url for YouTube oembed
+    pub static ref OEMBED_URL: Url = Url::parse("https://www.youtube.com/oembed").unwrap();
 
     /// Cache for proxy results
     static ref PROXY_CACHE: moka::future::Cache<String, Result<(String, Vec<u8>)>> = moka::future::Cache::builder()
@@ -130,8 +138,8 @@ impl reqwest::dns::Resolve for CachedDnsResolver {
 
 /// Information about a successful request
 pub struct Request {
-    response: Response,
+    pub response: Response,
-    mime: Mime,
+    pub mime: Mime,
 }
 
 impl Request {
@@ -285,6 +293,16 @@ impl Request {
         // Generate the actual embed
         if let Some(hit) = EMBED_CACHE.get(&url).await {
             Ok(hit)
+        } else if RE_URL_YOUTUBE.is_match(&url) {
+            let mut yt_url = OEMBED_URL.clone();
+            yt_url.set_query(Some(&format!("url={url}")));
+
+            let request = Request::new(yt_url).await?;
+            let embed = specialty::SpecialtySitesGenerator::youtube(&url, request).await?;
+
+            EMBED_CACHE.insert(url.to_owned(), embed.clone()).await;
+
+            Ok(embed)
         } else {
             let request = Request::new_from_str(&url).await?;
             let embed = match (request.mime.type_(), request.mime.subtype()) {

crates/services/january/src/specialty.rs

@@ -0,0 +1,62 @@
+use revolt_models::v0::{Embed, Special, WebsiteMetadata};
+use revolt_result::{create_error, Result};
+use serde::Deserialize;
+
+use crate::requests::{Request, RE_URL_YOUTUBE};
+pub struct SpecialtySitesGenerator {}
+
+#[allow(unused)]
+#[derive(Debug, Deserialize)]
+struct YoutubeOEmbed {
+    pub title: String,
+    pub author_name: String,
+    pub author_url: String,
+    // #[serde(rename = "type")]
+    // pub kind: String,
+    // pub height: u32,
+    // pub width: u32,
+    // pub version: String,
+    pub provider_name: String,
+    // pub provider_url: String,
+    // pub thumbnail_height: u32,
+    // pub thumbnail_width: u32,
+    pub thumbnail_url: String,
+    // pub html: String,
+}
+
+impl SpecialtySitesGenerator {
+    pub async fn youtube(url: &str, request: Request) -> Result<Embed> {
+        let json: YoutubeOEmbed = request
+            .response
+            .json()
+            .await
+            .map_err(|_| create_error!(ProxyError))?;
+
+        let captures = RE_URL_YOUTUBE
+            .captures(url)
+            .ok_or_else(|| create_error!(ProxyError))?;
+        let id = captures[1].to_string();
+        let timestamp = captures
+            .get(2)
+            .map(|e| Some(e.as_str().to_string()))
+            .unwrap_or(None);
+
+        Ok(Embed::Website(WebsiteMetadata {
+            url: Some(url.to_string()),
+            original_url: None,
+            special: Some(Special::YouTube {
+                id,
+                timestamp,
+                creator_name: Some(json.author_name),
+                creator_url: Some(json.author_url),
+            }),
+            title: Some(json.title),
+            description: None,
+            image: None,
+            video: None,
+            site_name: Some(json.provider_name),
+            icon_url: Some(json.thumbnail_url),
+            colour: None,
+        }))
+    }
+}

crates/services/january/src/website_embed.rs

@@ -219,45 +219,6 @@ pub async fn populate_special(original_url: String, metadata: &mut WebsiteMetada
         Some(Special::Streamable {
             id: captures[1].to_string(),
         })
-    } else if let Some(captures) = RE_YOUTUBE.captures_iter(url).next() {
-        let id = captures[1].to_string();
-
-        lazy_static! {
-            static ref RE_TIMESTAMP: Regex = Regex::new("(?:\\?|&)(?:t|start)=([\\w]+)").unwrap();
-        }
-
-        // YouTube now blocks datacentre IPs from fetching information
-        // This is a fallback to prevent the embed from looking weird
-        if metadata.video.is_none() {
-            metadata.title.replace("YouTube".to_owned());
-            metadata.description.take();
-            metadata.colour.take();
-            metadata.icon_url.take();
-            metadata.site_name.take();
-
-            // Verify the video exists
-            if !crate::requests::Request::exists_from_str(&format!(
-                "http://img.youtube.com/vi/{}/sddefault.jpg",
-                id
-            ))
-            .await
-            .unwrap_or(false)
-            {
-                return;
-            }
-        }
-
-        if let Some(timestamp_captures) = RE_TIMESTAMP.captures_iter(url).next() {
-            Some(Special::YouTube {
-                id,
-                timestamp: Some(timestamp_captures[1].to_string()),
-            })
-        } else {
-            Some(Special::YouTube {
-                id,
-                timestamp: None,
-            })
-        }
     } else if let Some(captures) = RE_LIGHTSPEED.captures_iter(url).next() {
         Some(Special::Lightspeed {
             id: captures[1].to_string(),