From 09fe23c4dd5e3fd3672b646b5a9dcfa85b63bcbf Mon Sep 17 00:00:00 2001 From: zhanglun Date: Thu, 14 Sep 2023 15:09:45 +0800 Subject: [PATCH] improvement: :zap::children_crossing: optimize sync articles --- src-tauri/src/cmd.rs | 164 ++-------------------- src-tauri/src/core/scraper.rs | 10 +- src-tauri/src/feed/channel.rs | 56 +++++++- src-tauri/src/feed/mod.rs | 81 +++++++++++ src-tauri/src/main.rs | 3 - src-tauri/src/server/handlers/article.rs | 24 +++- src/components/ChannelList/useRefresh.tsx | 28 +++- src/containers/Article/index.tsx | 7 +- src/helpers/dataAgent.ts | 13 +- src/stores/createFeedSlice.ts | 1 + 10 files changed, 209 insertions(+), 178 deletions(-) diff --git a/src-tauri/src/cmd.rs b/src-tauri/src/cmd.rs index b382c0e21..86c7f05d9 100644 --- a/src-tauri/src/cmd.rs +++ b/src-tauri/src/cmd.rs @@ -1,7 +1,5 @@ use std::collections::HashMap; -use feed_rs::parser; -use reqwest; use serde::Serialize; use tauri::{command, Window}; use tokio::sync::{mpsc, Mutex}; @@ -18,81 +16,6 @@ pub struct AsyncProcInputTx { pub sender: Mutex>, } -pub fn create_client() -> reqwest::Client { - let user_config = config::get_user_config(); - let client = match user_config { - Some(user_config) => match user_config.local_proxy { - Some(proxy) => { - let mut scheme = String::from("socks5h://"); - - scheme.push_str(&proxy.ip.to_string()); - scheme.push_str(":"); - scheme.push_str(&proxy.port.to_string()); - - reqwest::Client::builder() - .proxy(reqwest::Proxy::all(scheme).unwrap()) - .build() - .unwrap() - } - None => reqwest::Client::builder().build().unwrap(), - }, - None => reqwest::Client::builder().build().unwrap(), - }; - - client -} - -/// request feed, parse Feeds -/// -/// # Examples -/// ``` -/// let url = "https://sspai.com/feed".to_string(); -/// let res = parse_feed(&url).await; -/// ``` -pub async fn parse_feed(url: &str) -> Result { - let client = create_client(); - let result = client.get(url).send().await; - - let a = match result { - Ok(response) => match response.status() { - reqwest::StatusCode::OK => { - let content = response.text().await; - - match content { - Ok(content) => { - let res = parser::parse(content.as_bytes()); - - match res { - Ok(res) => Ok(res), - Err(error) => { - println!("content parse error{:?}", error); - Err(error.to_string()) - } - } - } - Err(error) => { - println!("response not OK {:?}", error); - Err(error.to_string()) - } - } - } - reqwest::StatusCode::NOT_FOUND => Err(String::from("Could not find a feed at the location.")), - _ => { - println!("o {:?}", response); - Err("Not 200 OK".to_string()) - } - }, - Err(error) => { - println!("ERROR: {:?}", error); - println!("URL: {:?}", url); - - Err(error.to_string()) - } - }; - - a -} - #[derive(Debug, Serialize)] pub struct FeedFetchResponse { feed: models::NewFeed, @@ -101,7 +24,7 @@ pub struct FeedFetchResponse { #[command] pub async fn fetch_feed(url: String) -> (Option, String) { - let res = parse_feed(&url).await; + let res = feed::parse_feed(&url).await; match res { Ok(res) => { @@ -273,7 +196,7 @@ pub fn create_article_models( pub async fn add_feed(url: String) -> (usize, String) { println!("request channel {}", &url); - let res = parse_feed(&url).await; + let res = feed::parse_feed(&url).await; match res { Ok(res) => { @@ -325,59 +248,6 @@ pub fn delete_feed(uuid: String) -> usize { result } -pub async fn sync_articles(uuid: String) -> Vec<(usize, String, String)> { - let channel = match feed::channel::get_feed_by_uuid(&uuid) { - Some(channel) => channel, - None => return vec![(0, uuid, "feed not found".to_string())], - }; - - let res = match parse_feed(&channel.feed_url).await { - Ok(res) => { - feed::channel::update_health_status(&uuid, 0, "".to_string()); - res - }, - Err(err) => { - feed::channel::update_health_status(&uuid, 1, err.to_string()); - return vec![(0, uuid, err.to_string())]; - }, - }; - - let articles = create_article_models(&channel.uuid, &channel.feed_url, &res); - let result = feed::article::Article::add_articles(channel.uuid, articles); - - vec![(result, uuid, "".to_string())] -} - -pub async fn sync_article_in_folder(uuid: String) -> Vec<(usize, String, String)> { - let connection = db::establish_connection(); - let channels = feed::folder::get_channels_in_folders(connection, vec![uuid]); - - println!("{:?}", channels); - let mut res = vec![]; - - for channel in channels { - res.extend(sync_articles(channel.child_uuid).await); - } - - res -} - -#[command] -pub async fn sync_articles_with_channel_uuid( - feed_type: String, - uuid: String, -) -> Vec<(usize, String, String)> { - if feed_type == "folder" { - let res = sync_article_in_folder(uuid).await; - println!("res folder {:?}", res); - res - } else { - let res = sync_articles(uuid).await; - println!("res {:?}", res); - res - } -} - #[command] pub async fn import_channels(list: Vec) -> usize { println!("{:?}", &list); @@ -512,20 +382,20 @@ pub async fn update_icon(uuid: String, url: String) -> usize { favicon } -#[command] -pub async fn get_web_best_image(url: String) -> Option { - let res = core::scraper::PageScraper::get_first_image_or_og_image(&url) - .await - .unwrap_or("".to_string()); +// #[command] +// pub async fn get_web_best_image(url: String) -> Option { +// let res = core::scraper::PageScraper::get_first_image_or_og_image(&url) +// .await +// .unwrap_or("".to_string()); - Some(res) -} +// Some(res) +// } -#[command] -pub async fn get_web_source(url: String) -> Option { - let res = core::scraper::PageScraper::fetch_page(&url).await; - res -} +// #[command] +// pub async fn get_web_source(url: String) -> Option { +// let res = core::scraper::PageScraper::fetch_page(&url).await; +// res +// } #[cfg(test)] mod tests { @@ -562,7 +432,7 @@ mod tests { println!("{:?}", url); - let res = parse_feed(&url).await; + let res = feed::parse_feed(&url).await; match res { Ok(res) => { @@ -589,8 +459,4 @@ mod tests { println!("result: {:?}", result); } - #[tokio::test] - async fn test_sync_() { - sync_article_in_folder(String::from("52f4b910-2551-4bce-84cb-5ceae1f3773c")).await; - } } diff --git a/src-tauri/src/core/scraper.rs b/src-tauri/src/core/scraper.rs index 4169a50d7..2d9faf3fa 100644 --- a/src-tauri/src/core/scraper.rs +++ b/src-tauri/src/core/scraper.rs @@ -1,4 +1,3 @@ -use crate::cmd::{self, create_client}; use reqwest::{self}; use scraper::{self, Selector}; use serde::Serialize; @@ -6,6 +5,9 @@ use tokio::sync::{Mutex}; use std::sync::Arc; use tokio::sync::mpsc::{channel}; +use crate::feed; +use crate::cmd; + #[derive(Debug, Default, Serialize)] pub struct PageScraper { pub title: String, @@ -49,7 +51,7 @@ impl PageScraper { } pub async fn fetch_page(url: &str) -> Option { - let client = cmd::create_client(); + let client = feed::create_client(); let response = client.get(url).send().await; let result = match response { @@ -68,7 +70,7 @@ impl PageScraper { } pub async fn get_first_image_or_og_image(url: &str) -> Option { - let client = create_client(); + let client = feed::create_client(); let res = client.get(url).send().await.ok()?; let body = res.text().await.ok()?; let document = scraper::Html::parse_document(&body); @@ -141,7 +143,7 @@ use super::*; // let url = "https://anyway.fm/rss.xml"; println!("request channel {}", &url); - let res =cmd::parse_feed(&url).await; + let res = feed::parse_feed(&url).await; match res { Ok(res) => { diff --git a/src-tauri/src/feed/channel.rs b/src-tauri/src/feed/channel.rs index 6cad5a784..1f9052130 100644 --- a/src-tauri/src/feed/channel.rs +++ b/src-tauri/src/feed/channel.rs @@ -6,14 +6,12 @@ use diesel::sql_types::*; use scraper::{Html, Selector}; use serde::{Deserialize, Serialize}; -use crate::cmd::create_client; use crate::db; use crate::models; +use crate::cmd::create_article_models; use crate::feed; use crate::schema; - - pub fn get_feed_by_uuid(channel_uuid: &str) -> Option { let mut connection = db::establish_connection(); let mut channel = schema::feeds::dsl::feeds @@ -628,7 +626,7 @@ pub async fn update_icon(uuid: &str, url: &str) -> usize { } pub async fn fetch_site_favicon(url: &str) -> Option { - let client = create_client(); + let client = feed::create_client(); let response = client.get(url).send().await.unwrap(); let html = response.text().await.unwrap(); let url = String::from(url); @@ -652,6 +650,51 @@ pub async fn fetch_site_favicon(url: &str) -> Option { favicon_url } +pub async fn sync_articles(uuid: String) -> Vec<(usize, String, String)> { + let channel = match feed::channel::get_feed_by_uuid(&uuid) { + Some(channel) => channel, + None => return vec![(0, uuid, "feed not found".to_string())], + }; + + let res = match feed::parse_feed(&channel.feed_url).await { + Ok(res) => { + feed::channel::update_health_status(&uuid, 0, "".to_string()); + res + }, + Err(err) => { + feed::channel::update_health_status(&uuid, 1, err.to_string()); + return vec![(0, uuid, err.to_string())]; + }, + }; + + let articles = create_article_models(&channel.uuid, &channel.feed_url, &res); + let result = feed::article::Article::add_articles(channel.uuid, articles); + + vec![(result, uuid, "".to_string())] +} + +pub async fn sync_article_in_folder(uuid: String) -> Vec<(usize, String, String)> { + let connection = db::establish_connection(); + let channels = feed::folder::get_channels_in_folders(connection, vec![uuid]); + + println!("{:?}", channels); + let mut res = vec![]; + + for channel in channels { + res.extend(sync_articles(channel.child_uuid).await); + } + + res +} + +pub async fn sync_feed(uuid: String, feed_type: String) -> Vec<(usize, String, String)> { + if feed_type == "folder" { + return feed::channel::sync_article_in_folder(uuid.to_string()).await; + } else { + return feed::channel::sync_articles(uuid.to_string()).await; + } +} + #[cfg(test)] mod tests { use super::*; @@ -692,4 +735,9 @@ mod tests { println!("sort {:?}", sort); } + + #[tokio::test] + async fn test_sync_feed() { + sync_feed("1e2dbbcd-5f44-4811-a907-0b6320b3ee9e".to_string(), "feed".to_string()).await; + } } diff --git a/src-tauri/src/feed/mod.rs b/src-tauri/src/feed/mod.rs index 6bcd86792..5962a260a 100644 --- a/src-tauri/src/feed/mod.rs +++ b/src-tauri/src/feed/mod.rs @@ -1,3 +1,84 @@ +use feed_rs::parser; +use reqwest; + +use crate::core::config; + pub mod article; pub mod channel; pub mod folder; + +pub fn create_client() -> reqwest::Client { + let user_config = config::get_user_config(); + let client = match user_config { + Some(user_config) => match user_config.local_proxy { + Some(proxy) => { + let mut scheme = String::from("socks5h://"); + + scheme.push_str(&proxy.ip.to_string()); + scheme.push_str(":"); + scheme.push_str(&proxy.port.to_string()); + + reqwest::Client::builder() + .proxy(reqwest::Proxy::all(scheme).unwrap()) + .build() + .unwrap() + } + None => reqwest::Client::builder().build().unwrap(), + }, + None => reqwest::Client::builder().build().unwrap(), + }; + + client +} + +/// request feed, parse Feeds +/// +/// # Examples +/// ``` +/// let url = "https://sspai.com/feed".to_string(); +/// let res = parse_feed(&url).await; +/// ``` +pub async fn parse_feed(url: &str) -> Result { + let client = create_client(); + let result = client.get(url).send().await; + + let a = match result { + Ok(response) => match response.status() { + reqwest::StatusCode::OK => { + let content = response.text().await; + + match content { + Ok(content) => { + let res = parser::parse(content.as_bytes()); + + match res { + Ok(res) => Ok(res), + Err(error) => { + println!("content parse error{:?}", error); + Err(error.to_string()) + } + } + } + Err(error) => { + println!("response not OK {:?}", error); + Err(error.to_string()) + } + } + } + reqwest::StatusCode::NOT_FOUND => Err(String::from("Could not find a feed at the location.")), + _ => { + println!("o {:?}", response); + Err("Not 200 OK".to_string()) + } + }, + Err(error) => { + println!("ERROR: {:?}", error); + println!("URL: {:?}", url); + + Err(error.to_string()) + } + }; + + a +} + diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs index b40ea5b6e..cbcaafccc 100644 --- a/src-tauri/src/main.rs +++ b/src-tauri/src/main.rs @@ -178,7 +178,6 @@ async fn main() { cmd::get_articles, cmd::get_today_articles, cmd::get_all_articles, - cmd::sync_articles_with_channel_uuid, cmd::import_channels, cmd::get_unread_total, cmd::update_article_read_status, @@ -196,8 +195,6 @@ async fn main() { cmd::init_process, cmd::get_article_detail, cmd::update_icon, - cmd::get_web_best_image, - cmd::get_web_source, ]) .build(context) .expect("error while running tauri Application") diff --git a/src-tauri/src/server/handlers/article.rs b/src-tauri/src/server/handlers/article.rs index 736783fbd..50d8565bd 100644 --- a/src-tauri/src/server/handlers/article.rs +++ b/src-tauri/src/server/handlers/article.rs @@ -2,12 +2,10 @@ use actix_web::{get, post, web, App, HttpRequest, HttpResponse, HttpServer, Resp use serde::{Deserialize, Serialize}; use crate::feed; - #[derive(Serialize)] struct MyObj { name: String, } - #[get("/api/articles/{name}")] pub async fn handle_test(name: web::Path) -> Result { let obj = MyObj { @@ -24,15 +22,30 @@ pub async fn handle_collection_metas() -> Result { Ok(web::Json(obj)) } +#[derive(Debug, Deserialize, Serialize)] +pub struct SyncFeedQuery { + feed_type: String, +} + +#[get("/api/feeds/{uuid}/sync")] +pub async fn handle_sync_feed( + uuid: web::Path, + query: web::Query, +) -> Result { + let res = feed::channel::sync_feed(uuid.to_string(), query.feed_type.to_string()).await; + + Ok(web::Json(res)) +} + #[post("/api/mark-all-as-read")] pub async fn handle_mark_as_read( body: web::Json, ) -> Result { - let res = feed::article::Article::mark_as_read(feed::article::MarkAllUnreadParam{ + let res = feed::article::Article::mark_as_read(feed::article::MarkAllUnreadParam { uuid: body.uuid.clone(), is_today: body.is_today, - is_all: body.is_all - }); + is_all: body.is_all, + }); println!("{:?}", body); @@ -70,6 +83,7 @@ pub fn config(cfg: &mut web::ServiceConfig) { cfg .service(handle_test) .service(handle_collection_metas) + .service(handle_sync_feed) .service(handle_mark_as_read) .service(handle_articles); } diff --git a/src/components/ChannelList/useRefresh.tsx b/src/components/ChannelList/useRefresh.tsx index 87981b371..9c6a8340e 100644 --- a/src/components/ChannelList/useRefresh.tsx +++ b/src/components/ChannelList/useRefresh.tsx @@ -7,11 +7,14 @@ import { useBearStore } from "@/stores"; export const useRefresh = () => { const store = useBearStore((state) => ({ userConfig: state.userConfig, + feedList: state.feedList, getFeedList: state.getFeedList, updateFeed: state.updateFeed, + + collectionMeta: state.collectionMeta, + updateCollectionMeta: state.updateCollectionMeta, })); - // const [feedList, setFeedList] = useState([]); const [refreshing, setRefreshing] = useState(false); const [done, setDone] = useState(0); @@ -23,16 +26,29 @@ export const useRefresh = () => { const loadAndUpdate = (type: string, uuid: string, unread: number) => { return dataAgent - .syncArticlesWithChannelUuid(type, uuid) + .syncFeed(type, uuid) .then((res) => { - console.log("%c Line:222 🍬 res", "color:#7f2b82", res); - res.forEach((item) => { + const { data } = res; + console.log("%c Line:29 🍌 data", "color:#ffdd4d", data); + + data.forEach((item) => { const [count, uuid, _msg] = item; - count > 0 && store.updateFeed(uuid, { unread: unread + count }); + if (count) { + store.updateCollectionMeta({ + total: { + unread: store.collectionMeta.total.unread + count, + }, + today: { + unread: store.collectionMeta.today.unread + count, + }, + }); + + store.updateFeed(uuid, { unread: unread + count }); + } }); - return res; + return data; }) .catch((err) => { console.log("%c Line:239 🍬 err", "color:#2eafb0", err); diff --git a/src/containers/Article/index.tsx b/src/containers/Article/index.tsx index 71e84096c..2d0ec1985 100644 --- a/src/containers/Article/index.tsx +++ b/src/containers/Article/index.tsx @@ -153,12 +153,13 @@ export const ArticleContainer = (): JSX.Element => { setSyncing(true); dataAgent - .syncArticlesWithChannelUuid( + .syncFeed( store.feed?.item_type as string, store.feed?.uuid as string ) - .then((res) => { - const [num, , message] = res[0]; + .then(({ data }) => { + console.log("%c Line:161 🍞 data", "color:#33a5ff", data); + const [num, , message] = data[0]; if (message) { toast({ diff --git a/src/helpers/dataAgent.ts b/src/helpers/dataAgent.ts index b9db97d07..338c935ec 100644 --- a/src/helpers/dataAgent.ts +++ b/src/helpers/dataAgent.ts @@ -108,11 +108,16 @@ export const addChannel = async (url: string): Promise<[number, string]> => { return invoke("add_feed", { url }); }; -export const syncArticlesWithChannelUuid = async ( - feedType: string, +export const syncFeed = async ( + feed_type: string, uuid: string -): Promise<[[number, string, string]]> => { - return invoke("sync_articles_with_channel_uuid", { feedType, uuid }); +): Promise> => { + return request.get(`/feeds/${uuid}/sync`, { + params: { + feed_type, + } + }) + // return invoke("sync_articles_with_channel_uuid", { feedType, uuid }); }; export const getUnreadTotal = async (): Promise<{ [key: string]: number }> => { diff --git a/src/stores/createFeedSlice.ts b/src/stores/createFeedSlice.ts index 4db6dbf25..50b6de13e 100644 --- a/src/stores/createFeedSlice.ts +++ b/src/stores/createFeedSlice.ts @@ -121,6 +121,7 @@ export const createFeedSlice: StateCreator = ( }, updateCollectionMeta(meta: any) { + console.log("%c Line:124 🥖 meta", "color:#e41a6a", meta); set({ collectionMeta: meta, });