From 7277856788116d3d6bbbb47f1ec02046fca0c86e Mon Sep 17 00:00:00 2001 From: 0xMimir Date: Sun, 1 Oct 2023 20:59:48 +0200 Subject: [PATCH] added cron to scan for issues --- README.md | 2 - api/src/jobs/github_issues/contract.rs | 27 ++++ api/src/jobs/github_issues/domain.rs | 115 ++++++++++++++++++ .../jobs/github_issues/infrastructure/mod.rs | 5 + .../infrastructure/repository.rs | 35 ++++++ .../github_issues/infrastructure/service.rs | 48 ++++++++ api/src/jobs/github_issues/mod.rs | 19 +++ api/src/jobs/mod.rs | 8 +- libs/sdks/src/github/data.rs | 10 +- libs/sdks/src/github/domain.rs | 2 +- libs/sdks/src/github/mod.rs | 4 +- libs/store/src/github_repositories.rs | 8 ++ libs/store/src/issues.rs | 38 ++++++ libs/store/src/lib.rs | 1 + libs/store/src/prelude.rs | 1 + migrations/2023-10-01-145428_issues/up.sql | 4 +- 16 files changed, 318 insertions(+), 9 deletions(-) create mode 100644 api/src/jobs/github_issues/contract.rs create mode 100644 api/src/jobs/github_issues/domain.rs create mode 100644 api/src/jobs/github_issues/infrastructure/mod.rs create mode 100644 api/src/jobs/github_issues/infrastructure/repository.rs create mode 100644 api/src/jobs/github_issues/infrastructure/service.rs create mode 100644 api/src/jobs/github_issues/mod.rs create mode 100644 libs/store/src/issues.rs diff --git a/README.md b/README.md index 7f932e8..c4403c7 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,5 @@ sea generate entity --lib -o libs/store/src * Create api for coingecko * fetch all assets * fetch all assets info - * Create cron to scan over repositories for issues. - * Create cron to check if issue has been closed. * Create api routes * Create openapi docs \ No newline at end of file diff --git a/api/src/jobs/github_issues/contract.rs b/api/src/jobs/github_issues/contract.rs new file mode 100644 index 0000000..9906ba0 --- /dev/null +++ b/api/src/jobs/github_issues/contract.rs @@ -0,0 +1,27 @@ +use error::Result; +use sdks::github::data::GithubIssue; +use sea_orm::prelude::Uuid; +use store::{ + github_projects::Model as GithubProject, github_repositories::Model as GithubRepository, +}; + +#[async_trait] +pub trait DbRepositoryContract { + /// + /// Returns all github projects in db, might needed to be refactored to paginate + /// + async fn get_projects(&self) -> Result>; + + /// + /// Return all repositories for project, might needed to be refactored to paginate + /// + async fn get_project_repositories(&self, project_id: Uuid) -> Result>; +} + +#[async_trait] +pub trait DbServiceContract { + /// + /// Create entities in `issues` table + /// + async fn create_issues(&self, repository_id: Uuid, issues: Vec) -> Result<()>; +} diff --git a/api/src/jobs/github_issues/domain.rs b/api/src/jobs/github_issues/domain.rs new file mode 100644 index 0000000..ac15f78 --- /dev/null +++ b/api/src/jobs/github_issues/domain.rs @@ -0,0 +1,115 @@ +use error::{Error, Result}; +use sdks::github::GithubContract; +use std::time::Duration; +use store::{ + github_projects::Model as GithubProject, github_repositories::Model as GithubRepository, +}; +use tokio::{ + task::JoinHandle, + time::{interval, sleep}, +}; + +use super::contract::{DbRepositoryContract, DbServiceContract}; + +pub struct GithubIssueCron< + Repository: DbRepositoryContract, + Service: DbServiceContract, + Github: GithubContract, +> { + repository: Repository, + service: Service, + github: Github, +} + +impl< + Repository: DbRepositoryContract + Send + Sync + 'static, + Service: DbServiceContract + Send + Sync + 'static, + Github: GithubContract + Send + Sync + 'static, + > GithubIssueCron +{ + /// + /// Creates `GithubIssueCron` + /// + pub fn new(repository: Repository, service: Service, github: Github) -> Self { + Self { + repository, + service, + github, + } + } + + /// + /// Get all github projects from db and call `handle_project` for every project + /// + async fn cron_job(&self) -> Result<()> { + let projects = self.repository.get_projects().await?; + for project in projects { + if let Err(error) = self.handle_project(project).await { + error!("{}", error); + } + } + Ok(()) + } + + /// + /// Get all repositories for project then call `handle_issues` for every repository + /// + async fn handle_project(&self, project: GithubProject) -> Result<()> { + let repositories = self.repository.get_project_repositories(project.id).await?; + + for repository in repositories { + if let Err(error) = self.handle_issues(&project.name, repository).await { + error!("{}", error); + } + } + Ok(()) + } + + /// + /// Scan first 500 issues + /// + async fn handle_issues(&self, project: &str, repository: GithubRepository) -> Result<()> { + let mut page = 1; + + while page <= 5 { + let issues = match self + .github + .get_issues(project, &repository.repository_name, page) + .await + { + Ok(issues) => issues, + Err(Error::RateLimitExceeded) => { + warn!("Rate limit exceeded sleeping for 10 minutes"); + sleep(Duration::from_secs(6000)).await; + continue; + } + Err(error) => return Err(error), + }; + + if issues.is_empty() { + break; + } + + self.service.create_issues(repository.id, issues).await?; + page += 1; + } + + Ok(()) + } + + /// + /// Spawns tokio task, that runs every 3 hours + /// + pub fn spawn_cron(self) -> JoinHandle<()> { + tokio::spawn(async move { + let mut interval = interval(Duration::from_secs(21600)); + + loop { + interval.tick().await; + if let Err(error) = self.cron_job().await { + error!("{}", error); + } + } + }) + } +} diff --git a/api/src/jobs/github_issues/infrastructure/mod.rs b/api/src/jobs/github_issues/infrastructure/mod.rs new file mode 100644 index 0000000..09b8cf3 --- /dev/null +++ b/api/src/jobs/github_issues/infrastructure/mod.rs @@ -0,0 +1,5 @@ +mod repository; +mod service; + +pub use repository::PgRepository; +pub use service::PgService; diff --git a/api/src/jobs/github_issues/infrastructure/repository.rs b/api/src/jobs/github_issues/infrastructure/repository.rs new file mode 100644 index 0000000..ecfb651 --- /dev/null +++ b/api/src/jobs/github_issues/infrastructure/repository.rs @@ -0,0 +1,35 @@ +use error::Result; +use sea_orm::{prelude::Uuid, DatabaseConnection, EntityTrait, QueryFilter, ColumnTrait}; +use std::sync::Arc; + +use super::super::contract::DbRepositoryContract; +use store::{ + github_projects::Model as GithubProject, + github_repositories::{Column, Model as GithubRepository}, + prelude::{GithubProjects, GithubRepositories}, +}; +pub struct PgRepository { + conn: Arc, +} + +impl PgRepository { + pub fn new(conn: Arc) -> Self { + Self { conn } + } +} + +#[async_trait] +impl DbRepositoryContract for PgRepository { + async fn get_projects(&self) -> Result> { + let projects = GithubProjects::find().all(self.conn.as_ref()).await?; + Ok(projects) + } + async fn get_project_repositories(&self, project_id: Uuid) -> Result> { + let repositories = GithubRepositories::find() + .filter(Column::Project.eq(project_id)) + .all(self.conn.as_ref()) + .await?; + + Ok(repositories) + } +} diff --git a/api/src/jobs/github_issues/infrastructure/service.rs b/api/src/jobs/github_issues/infrastructure/service.rs new file mode 100644 index 0000000..f04fc13 --- /dev/null +++ b/api/src/jobs/github_issues/infrastructure/service.rs @@ -0,0 +1,48 @@ +use error::Result; +use sdks::github::data::{GithubIssue, State}; +use sea_orm::{ + prelude::Uuid, sea_query::OnConflict, ActiveValue::Set, DatabaseConnection, EntityTrait, +}; +use std::sync::Arc; + +use super::super::contract::DbServiceContract; +use store::issues::{ActiveModel, Column, Entity}; + +pub struct PgService { + conn: Arc, +} + +impl PgService { + pub fn new(conn: Arc) -> Self { + Self { conn } + } +} + +#[async_trait] +impl DbServiceContract for PgService { + async fn create_issues(&self, repository_id: Uuid, issues: Vec) -> Result<()> { + let models = issues + .into_iter() + .map(|issue| ActiveModel { + repository: Set(repository_id), + issue: Set(issue.id), + title: Set(issue.title), + description: Set(Some(issue.description)), + created_at: Set(issue.created_at), + closed: Set(issue.state == State::Closed), + ..Default::default() + }) + .collect::>(); + + Entity::insert_many(models) + .on_conflict( + OnConflict::columns([Column::Repository, Column::Issue]) + .update_columns([Column::Title, Column::Description, Column::Closed]) + .to_owned(), + ) + .exec(self.conn.as_ref()) + .await?; + + Ok(()) + } +} diff --git a/api/src/jobs/github_issues/mod.rs b/api/src/jobs/github_issues/mod.rs new file mode 100644 index 0000000..038ef61 --- /dev/null +++ b/api/src/jobs/github_issues/mod.rs @@ -0,0 +1,19 @@ +mod contract; +mod domain; +mod infrastructure; + +use std::sync::Arc; + +use domain::GithubIssueCron; +use infrastructure::{PgRepository, PgService}; +use sdks::github::Github; +use sea_orm::DatabaseConnection; + +pub fn setup(sea_pool: Arc) -> tokio::task::JoinHandle<()> { + let repository = PgRepository::new(sea_pool.clone()); + let service = PgService::new(sea_pool); + let coingecko = Github::default(); + + let cron = GithubIssueCron::new(repository, service, coingecko); + cron.spawn_cron() +} diff --git a/api/src/jobs/mod.rs b/api/src/jobs/mod.rs index a709849..97bc11a 100644 --- a/api/src/jobs/mod.rs +++ b/api/src/jobs/mod.rs @@ -2,12 +2,16 @@ use sea_orm::DatabaseConnection; use std::sync::Arc; use tokio::task::JoinHandle; +/// Public because of init pub mod github_repositories; pub mod info; -pub fn setup(sea_pool: Arc) -> [JoinHandle<()>; 2] { +mod github_issues; + +pub fn setup(sea_pool: Arc) -> [JoinHandle<()>; 3] { [ info::setup(sea_pool.clone()), - github_repositories::setup(sea_pool), + github_repositories::setup(sea_pool.clone()), + github_issues::setup(sea_pool) ] } diff --git a/libs/sdks/src/github/data.rs b/libs/sdks/src/github/data.rs index 0dcec20..dffad97 100644 --- a/libs/sdks/src/github/data.rs +++ b/libs/sdks/src/github/data.rs @@ -28,7 +28,7 @@ impl From for Error { } } -#[derive(Deserialize, Debug)] +#[derive(Deserialize)] pub struct GithubIssue { pub id: i64, pub title: String, @@ -36,6 +36,14 @@ pub struct GithubIssue { pub description: String, #[serde(deserialize_with = "deserialize_datetime")] pub created_at: NaiveDateTime, + pub state: State, +} + +#[derive(Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum State { + Closed, + Open, } pub fn deserialize_datetime<'de, D: Deserializer<'de>>( diff --git a/libs/sdks/src/github/domain.rs b/libs/sdks/src/github/domain.rs index c83ab66..943f541 100644 --- a/libs/sdks/src/github/domain.rs +++ b/libs/sdks/src/github/domain.rs @@ -51,7 +51,7 @@ impl GithubContract for Github { page: u64, ) -> Result> { let url = format!( - "https://api.github.com/repos/{project}/{repository}/issues?page={page}&per_page=100" + "https://api.github.com/repos/{project}/{repository}/issues?state=all&page={page}&per_page=100" ); self.get(url).await } diff --git a/libs/sdks/src/github/mod.rs b/libs/sdks/src/github/mod.rs index e9b69d4..5bac527 100644 --- a/libs/sdks/src/github/mod.rs +++ b/libs/sdks/src/github/mod.rs @@ -1,6 +1,6 @@ +pub mod data; mod contract; -mod data; mod domain; pub use contract::GithubContract; -pub use domain::Github; +pub use domain::Github; \ No newline at end of file diff --git a/libs/store/src/github_repositories.rs b/libs/store/src/github_repositories.rs index d2dd06a..615de03 100644 --- a/libs/store/src/github_repositories.rs +++ b/libs/store/src/github_repositories.rs @@ -21,6 +21,8 @@ pub enum Relation { on_delete = "NoAction" )] GithubProjects, + #[sea_orm(has_many = "super::issues::Entity")] + Issues, } impl Related for Entity { @@ -29,4 +31,10 @@ impl Related for Entity { } } +impl Related for Entity { + fn to() -> RelationDef { + Relation::Issues.def() + } +} + impl ActiveModelBehavior for ActiveModel {} diff --git a/libs/store/src/issues.rs b/libs/store/src/issues.rs new file mode 100644 index 0000000..50b6529 --- /dev/null +++ b/libs/store/src/issues.rs @@ -0,0 +1,38 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.3 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "issues")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub id: Uuid, + pub repository: Uuid, + pub issue: i64, + #[sea_orm(column_type = "Text")] + pub title: String, + #[sea_orm(column_type = "Text", nullable)] + pub description: Option, + pub created_at: DateTime, + pub closed: bool, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::github_repositories::Entity", + from = "Column::Repository", + to = "super::github_repositories::Column::Id", + on_update = "NoAction", + on_delete = "NoAction" + )] + GithubRepositories, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::GithubRepositories.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/libs/store/src/lib.rs b/libs/store/src/lib.rs index 9768814..34a38f3 100644 --- a/libs/store/src/lib.rs +++ b/libs/store/src/lib.rs @@ -5,3 +5,4 @@ pub mod prelude; pub mod cryptocurrencies; pub mod github_projects; pub mod github_repositories; +pub mod issues; diff --git a/libs/store/src/prelude.rs b/libs/store/src/prelude.rs index 8348470..6bd6bd9 100644 --- a/libs/store/src/prelude.rs +++ b/libs/store/src/prelude.rs @@ -3,3 +3,4 @@ pub use super::cryptocurrencies::Entity as Cryptocurrencies; pub use super::github_projects::Entity as GithubProjects; pub use super::github_repositories::Entity as GithubRepositories; +pub use super::issues::Entity as Issues; diff --git a/migrations/2023-10-01-145428_issues/up.sql b/migrations/2023-10-01-145428_issues/up.sql index 3f82e28..2938395 100644 --- a/migrations/2023-10-01-145428_issues/up.sql +++ b/migrations/2023-10-01-145428_issues/up.sql @@ -4,5 +4,7 @@ create table issues( issue bigint not null, title text not null, description text, - created_at timestamp not null + created_at timestamp not null, + closed boolean not null, + unique(repository, issue) ) \ No newline at end of file