added cron to scan for issues

README.md

@@ -11,7 +11,5 @@ sea generate entity --lib -o libs/store/src
  * Create api for coingecko
   * fetch all assets
   * fetch all assets info
- * Create cron to scan over repositories for issues.
- * Create cron to check if issue has been closed.
  * Create api routes
  * Create openapi docs

api/src/jobs/github_issues/contract.rs

@@ -0,0 +1,27 @@
+use error::Result;
+use sdks::github::data::GithubIssue;
+use sea_orm::prelude::Uuid;
+use store::{
+ github_projects::Model as GithubProject, github_repositories::Model as GithubRepository,
+};
+
+#[async_trait]
+pub trait DbRepositoryContract {
+ ///
+ /// Returns all github projects in db, might needed to be refactored to paginate
+ ///
+ async fn get_projects(&self) -> Result<Vec<GithubProject>>;
+
+ ///
+ /// Return all repositories for project, might needed to be refactored to paginate
+ ///
+ async fn get_project_repositories(&self, project_id: Uuid) -> Result<Vec<GithubRepository>>;
+}
+
+#[async_trait]
+pub trait DbServiceContract {
+ ///
+ /// Create entities in `issues` table
+ ///
+ async fn create_issues(&self, repository_id: Uuid, issues: Vec<GithubIssue>) -> Result<()>;
+}

api/src/jobs/github_issues/domain.rs

@@ -0,0 +1,115 @@
+use error::{Error, Result};
+use sdks::github::GithubContract;
+use std::time::Duration;
+use store::{
+ github_projects::Model as GithubProject, github_repositories::Model as GithubRepository,
+};
+use tokio::{
+ task::JoinHandle,
+ time::{interval, sleep},
+};
+
+use super::contract::{DbRepositoryContract, DbServiceContract};
+
+pub struct GithubIssueCron<
+ Repository: DbRepositoryContract,
+ Service: DbServiceContract,
+ Github: GithubContract,
+> {
+ repository: Repository,
+ service: Service,
+ github: Github,
+}
+
+impl<
+ Repository: DbRepositoryContract + Send + Sync + 'static,
+ Service: DbServiceContract + Send + Sync + 'static,
+ Github: GithubContract + Send + Sync + 'static,
+ > GithubIssueCron<Repository, Service, Github>
+{
+ ///
+ /// Creates `GithubIssueCron`
+ ///
+ pub fn new(repository: Repository, service: Service, github: Github) -> Self {
+ Self {
+ repository,
+ service,
+ github,
+ }
+ }
+
+ ///
+ /// Get all github projects from db and call `handle_project` for every project
+ /// 
+ async fn cron_job(&self) -> Result<()> {
+ let projects = self.repository.get_projects().await?;
+ for project in projects {
+ if let Err(error) = self.handle_project(project).await {
+ error!("{}", error);
+ }
+ }
+ Ok(())
+ }
+
+ ///
+ /// Get all repositories for project then call `handle_issues` for every repository
+ /// 
+ async fn handle_project(&self, project: GithubProject) -> Result<()> {
+ let repositories = self.repository.get_project_repositories(project.id).await?;
+
+ for repository in repositories {
+ if let Err(error) = self.handle_issues(&project.name, repository).await {
+ error!("{}", error);
+ }
+ }
+ Ok(())
+ }
+
+ ///
+ /// Scan first 500 issues
+ /// 
+ async fn handle_issues(&self, project: &str, repository: GithubRepository) -> Result<()> {
+ let mut page = 1;
+
+ while page <= 5 {
+ let issues = match self
+ .github
+ .get_issues(project, &repository.repository_name, page)
+ .await
+ {
+ Ok(issues) => issues,
+ Err(Error::RateLimitExceeded) => {
+ warn!("Rate limit exceeded sleeping for 10 minutes");
+ sleep(Duration::from_secs(6000)).await;
+ continue;
+ }
+ Err(error) => return Err(error),
+ };
+
+ if issues.is_empty() {
+ break;
+ }
+
+ self.service.create_issues(repository.id, issues).await?;
+ page += 1;
+ }
+
+ Ok(())
+ }
+
+ ///
+ /// Spawns tokio task, that runs every 3 hours
+ ///
+ pub fn spawn_cron(self) -> JoinHandle<()> {
+ tokio::spawn(async move {
+ let mut interval = interval(Duration::from_secs(21600));
+
+ loop {
+ interval.tick().await;
+ if let Err(error) = self.cron_job().await {
+ error!("{}", error);
+ }
+ }
+ })
+ }
+}

api/src/jobs/github_issues/infrastructure/mod.rs

@@ -0,0 +1,5 @@
+mod repository;
+mod service;
+
+pub use repository::PgRepository;
+pub use service::PgService;

api/src/jobs/github_issues/infrastructure/repository.rs

@@ -0,0 +1,35 @@
+use error::Result;
+use sea_orm::{prelude::Uuid, DatabaseConnection, EntityTrait, QueryFilter, ColumnTrait};
+use std::sync::Arc;
+
+use super::super::contract::DbRepositoryContract;
+use store::{
+ github_projects::Model as GithubProject,
+ github_repositories::{Column, Model as GithubRepository},
+ prelude::{GithubProjects, GithubRepositories},
+};
+pub struct PgRepository {
+ conn: Arc<DatabaseConnection>,
+}
+
+impl PgRepository {
+ pub fn new(conn: Arc<DatabaseConnection>) -> Self {
+ Self { conn }
+ }
+}
+
+#[async_trait]
+impl DbRepositoryContract for PgRepository {
+ async fn get_projects(&self) -> Result<Vec<GithubProject>> {
+ let projects = GithubProjects::find().all(self.conn.as_ref()).await?;
+ Ok(projects)
+ }
+ async fn get_project_repositories(&self, project_id: Uuid) -> Result<Vec<GithubRepository>> {
+ let repositories = GithubRepositories::find()
+ .filter(Column::Project.eq(project_id))
+ .all(self.conn.as_ref())
+ .await?;
+
+ Ok(repositories)
+ }
+}

api/src/jobs/github_issues/infrastructure/service.rs

@@ -0,0 +1,48 @@
+use error::Result;
+use sdks::github::data::{GithubIssue, State};
+use sea_orm::{
+ prelude::Uuid, sea_query::OnConflict, ActiveValue::Set, DatabaseConnection, EntityTrait,
+};
+use std::sync::Arc;
+
+use super::super::contract::DbServiceContract;
+use store::issues::{ActiveModel, Column, Entity};
+
+pub struct PgService {
+ conn: Arc<DatabaseConnection>,
+}
+
+impl PgService {
+ pub fn new(conn: Arc<DatabaseConnection>) -> Self {
+ Self { conn }
+ }
+}
+
+#[async_trait]
+impl DbServiceContract for PgService {
+ async fn create_issues(&self, repository_id: Uuid, issues: Vec<GithubIssue>) -> Result<()> {
+ let models = issues
+ .into_iter()
+ .map(|issue| ActiveModel {
+ repository: Set(repository_id),
+ issue: Set(issue.id),
+ title: Set(issue.title),
+ description: Set(Some(issue.description)),
+ created_at: Set(issue.created_at),
+ closed: Set(issue.state == State::Closed),
+ ..Default::default()
+ })
+ .collect::<Vec<_>>();
+
+ Entity::insert_many(models)
+ .on_conflict(
+ OnConflict::columns([Column::Repository, Column::Issue])
+ .update_columns([Column::Title, Column::Description, Column::Closed])
+ .to_owned(),
+ )
+ .exec(self.conn.as_ref())
+ .await?;
+
+ Ok(())
+ }
+}

api/src/jobs/github_issues/mod.rs

@@ -0,0 +1,19 @@
+mod contract;
+mod domain;
+mod infrastructure;
+
+use std::sync::Arc;
+
+use domain::GithubIssueCron;
+use infrastructure::{PgRepository, PgService};
+use sdks::github::Github;
+use sea_orm::DatabaseConnection;
+
+pub fn setup(sea_pool: Arc<DatabaseConnection>) -> tokio::task::JoinHandle<()> {
+ let repository = PgRepository::new(sea_pool.clone());
+ let service = PgService::new(sea_pool);
+ let coingecko = Github::default();
+
+ let cron = GithubIssueCron::new(repository, service, coingecko);
+ cron.spawn_cron()
+}

api/src/jobs/mod.rs

@@ -2,12 +2,16 @@ use sea_orm::DatabaseConnection;
 use std::sync::Arc;
 use tokio::task::JoinHandle;
 
+/// Public because of init
 pub mod github_repositories;
 pub mod info;
 
-pub fn setup(sea_pool: Arc<DatabaseConnection>) -> [JoinHandle<()>; 2] {
+mod github_issues;
+
+pub fn setup(sea_pool: Arc<DatabaseConnection>) -> [JoinHandle<()>; 3] {
  [
  info::setup(sea_pool.clone()),
- github_repositories::setup(sea_pool),
+ github_repositories::setup(sea_pool.clone()),
+ github_issues::setup(sea_pool)
  ]
 }

libs/sdks/src/github/data.rs

@@ -28,14 +28,22 @@ impl From<ErrorResponse> for Error {
  }
 }
 
-#[derive(Deserialize, Debug)]
+#[derive(Deserialize)]
 pub struct GithubIssue {
  pub id: i64,
  pub title: String,
  #[serde(rename = "body")]
  pub description: String,
  #[serde(deserialize_with = "deserialize_datetime")]
  pub created_at: NaiveDateTime,
+ pub state: State,
+}
+
+#[derive(Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "lowercase")]
+pub enum State {
+ Closed,
+ Open,
 }
 
 pub fn deserialize_datetime<'de, D: Deserializer<'de>>(

libs/sdks/src/github/domain.rs

@@ -51,7 +51,7 @@ impl GithubContract for Github {
  page: u64,
  ) -> Result<Vec<GithubIssue>> {
  let url = format!(
- "https://api.github.com/repos/{project}/{repository}/issues?page={page}&per_page=100"
+ "https://api.github.com/repos/{project}/{repository}/issues?state=all&page={page}&per_page=100"
  );
  self.get(url).await
  }

libs/sdks/src/github/mod.rs

@@ -1,6 +1,6 @@
+pub mod data;
 mod contract;
-mod data;
 mod domain;
 
 pub use contract::GithubContract;
-pub use domain::Github;
+pub use domain::Github;

libs/store/src/github_repositories.rs

@@ -21,6 +21,8 @@ pub enum Relation {
  on_delete = "NoAction"
  )]
  GithubProjects,
+ #[sea_orm(has_many = "super::issues::Entity")]
+ Issues,
 }
 
 impl Related<super::github_projects::Entity> for Entity {
@@ -29,4 +31,10 @@ impl Related<super::github_projects::Entity> for Entity {
  }
 }
 
+impl Related<super::issues::Entity> for Entity {
+ fn to() -> RelationDef {
+ Relation::Issues.def()
+ }
+}
+
 impl ActiveModelBehavior for ActiveModel {}

libs/store/src/issues.rs

@@ -0,0 +1,38 @@
+//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.3
+
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[sea_orm(table_name = "issues")]
+pub struct Model {
+ #[sea_orm(primary_key, auto_increment = false)]
+ pub id: Uuid,
+ pub repository: Uuid,
+ pub issue: i64,
+ #[sea_orm(column_type = "Text")]
+ pub title: String,
+ #[sea_orm(column_type = "Text", nullable)]
+ pub description: Option<String>,
+ pub created_at: DateTime,
+ pub closed: bool,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {
+ #[sea_orm(
+ belongs_to = "super::github_repositories::Entity",
+ from = "Column::Repository",
+ to = "super::github_repositories::Column::Id",
+ on_update = "NoAction",
+ on_delete = "NoAction"
+ )]
+ GithubRepositories,
+}
+
+impl Related<super::github_repositories::Entity> for Entity {
+ fn to() -> RelationDef {
+ Relation::GithubRepositories.def()
+ }
+}
+
+impl ActiveModelBehavior for ActiveModel {}

libs/store/src/lib.rs

@@ -5,3 +5,4 @@ pub mod prelude;
 pub mod cryptocurrencies;
 pub mod github_projects;
 pub mod github_repositories;
+pub mod issues;

libs/store/src/prelude.rs

@@ -3,3 +3,4 @@
 pub use super::cryptocurrencies::Entity as Cryptocurrencies;
 pub use super::github_projects::Entity as GithubProjects;
 pub use super::github_repositories::Entity as GithubRepositories;
+pub use super::issues::Entity as Issues;