diff --git a/backend/api/src/api/cryptocurrencies/repository.rs b/backend/api/src/api/cryptocurrencies/repository.rs index 8de5ef1..e6ed3fa 100644 --- a/backend/api/src/api/cryptocurrencies/repository.rs +++ b/backend/api/src/api/cryptocurrencies/repository.rs @@ -115,6 +115,9 @@ impl DbRepositoryContract for PgRepository { .columns([ github_repositories::Column::Id, github_repositories::Column::RepositoryName, + github_repositories::Column::Language, + github_repositories::Column::StargazersCount, + github_repositories::Column::ForksCount, ]) .into_model::() .all(self.conn.as_ref()) diff --git a/backend/api/src/api/repository/repository.rs b/backend/api/src/api/repository/repository.rs index 5bc84ee..10159a5 100644 --- a/backend/api/src/api/repository/repository.rs +++ b/backend/api/src/api/repository/repository.rs @@ -75,6 +75,9 @@ impl DbRepositoryContract for PgRepository { .select_only() .column(github_repositories::Column::Id) .column_as(github_repositories::Column::RepositoryName, "name") + .column(github_repositories::Column::Language) + .column(github_repositories::Column::ForksCount) + .column_as(github_repositories::Column::StargazersCount, "stars_count") .column_as(github_repositories::Column::Project, "project_id") .column_as(github_projects::Column::Name, "project") .into_model::() diff --git a/backend/api/src/jobs/github_repositories/contract.rs b/backend/api/src/jobs/github_repositories/contract.rs index dae2e1c..b266f91 100644 --- a/backend/api/src/jobs/github_repositories/contract.rs +++ b/backend/api/src/jobs/github_repositories/contract.rs @@ -1,4 +1,5 @@ use error::Result; +use sdks::github::data::GithubRepository; use sea_orm::prelude::Uuid; use store::github_projects::Model; @@ -15,5 +16,5 @@ pub trait DbServiceContract { /// /// Create entries in `github_repositories` table /// - async fn create_repository(&self, project_id: Uuid, repositories: Vec) -> Result<()>; + async fn create_repository(&self, project_id: Uuid, repositories: Vec) -> Result<()>; } diff --git a/backend/api/src/jobs/github_repositories/domain.rs b/backend/api/src/jobs/github_repositories/domain.rs index 224e5c7..99f05ef 100644 --- a/backend/api/src/jobs/github_repositories/domain.rs +++ b/backend/api/src/jobs/github_repositories/domain.rs @@ -74,6 +74,7 @@ impl< self.service .create_repository(github.id, repositories) .await?; + page += 1; } @@ -84,6 +85,8 @@ impl< /// Spawns tokio task, that waits for a day, then runs once a week /// pub fn spawn_cron(self) -> JoinHandle<()> { + info!("Spawning github repositories task"); + tokio::spawn(async move { let mut interval = interval_at( (Instant::now() + Duration::from_secs(86_400)).into(), diff --git a/backend/api/src/jobs/github_repositories/infrastructure/service.rs b/backend/api/src/jobs/github_repositories/infrastructure/service.rs index 2e646e2..c212769 100644 --- a/backend/api/src/jobs/github_repositories/infrastructure/service.rs +++ b/backend/api/src/jobs/github_repositories/infrastructure/service.rs @@ -1,4 +1,5 @@ use error::Result; +use sdks::github::data::GithubRepository; use sea_orm::{ prelude::Uuid, sea_query::OnConflict, ActiveValue::Set, DatabaseConnection, EntityTrait, }; @@ -19,22 +20,33 @@ impl PgService { #[async_trait] impl DbServiceContract for PgService { - async fn create_repository(&self, project: Uuid, repositories: Vec) -> Result<()> { + async fn create_repository( + &self, + project: Uuid, + repositories: Vec, + ) -> Result<()> { let models = repositories .into_iter() - .map(|repository_name| ActiveModel { + .map(|repository| ActiveModel { + id: Default::default(), project: Set(project.to_owned()), - repository_name: Set(repository_name), - ..Default::default() + repository_name: Set(repository.name), + language: Set(repository.language), + stargazers_count: Set(repository.stargazers_count), + forks_count: Set(repository.forks_count), }) .collect::>(); + let mut on_conflict = OnConflict::columns([Column::Project, Column::RepositoryName]); + + on_conflict.update_columns([ + Column::ForksCount, + Column::StargazersCount, + Column::Language, + ]); + Entity::insert_many(models) - .on_conflict( - OnConflict::columns([Column::Project, Column::RepositoryName]) - .do_nothing() - .to_owned(), - ) + .on_conflict(on_conflict) .exec(self.conn.as_ref()) .await?; diff --git a/backend/libs/sdks/Cargo.toml b/backend/libs/sdks/Cargo.toml index becf48a..1f12dce 100644 --- a/backend/libs/sdks/Cargo.toml +++ b/backend/libs/sdks/Cargo.toml @@ -14,4 +14,5 @@ serde.workspace = true serde_json.workspace = true [dev-dependencies] +config.workspace = true tokio.workspace = true diff --git a/backend/libs/sdks/src/github/contract.rs b/backend/libs/sdks/src/github/contract.rs index 9404f6f..d32b6ac 100644 --- a/backend/libs/sdks/src/github/contract.rs +++ b/backend/libs/sdks/src/github/contract.rs @@ -1,13 +1,13 @@ use error::Result; -use super::data::GithubIssue; +use super::data::{GithubIssue, GithubRepository}; #[async_trait] pub trait GithubContract { /// /// Returns repos for github username for page /// - async fn get_repos(&self, username: &str, page: u64) -> Result>; + async fn get_repos(&self, username: &str, page: u64) -> Result>; /// /// Get issues for repository diff --git a/backend/libs/sdks/src/github/data.rs b/backend/libs/sdks/src/github/data.rs index 5949691..d65368d 100644 --- a/backend/libs/sdks/src/github/data.rs +++ b/backend/libs/sdks/src/github/data.rs @@ -2,15 +2,12 @@ use chrono::NaiveDateTime; use error::Error; use serde::{de::Error as DeError, Deserialize, Deserializer}; -#[derive(Deserialize)] -pub(super) struct GithubRepository { +#[derive(Deserialize, Debug)] +pub struct GithubRepository { pub name: String, -} - -impl GithubRepository { - pub fn into(response: Vec) -> Vec { - response.into_iter().map(|gr| gr.name).collect() - } + pub language: Option, + pub stargazers_count: i64, + pub forks_count: i64 } #[derive(Deserialize)] diff --git a/backend/libs/sdks/src/github/domain.rs b/backend/libs/sdks/src/github/domain.rs index f62b242..1d92d47 100644 --- a/backend/libs/sdks/src/github/domain.rs +++ b/backend/libs/sdks/src/github/domain.rs @@ -60,11 +60,9 @@ impl Github { #[async_trait] impl GithubContract for Github { - async fn get_repos(&self, username: &str, page: u64) -> Result> { + async fn get_repos(&self, username: &str, page: u64) -> Result> { let url = format!("https://api.github.com/users/{username}/repos?page={page}&per_page=100"); - let response = self.get(url).await.map_err(|e| e.add_cause(username))?; - - Ok(GithubRepository::into(response)) + self.get(url).await.map_err(|e| e.add_cause(username)) } async fn get_issues( diff --git a/backend/libs/sdks/tests/github.rs b/backend/libs/sdks/tests/github.rs index 5dc08bf..d3cf249 100644 --- a/backend/libs/sdks/tests/github.rs +++ b/backend/libs/sdks/tests/github.rs @@ -1,11 +1,26 @@ use error::{Error, Result}; use sdks::github::{Github, GithubContract}; +fn client() -> Github { + let key = config::get("GITHUB_KEY"); + + match key.ok() { + Some(api_key) => Github::new_with_auth(api_key), + None => Github::default(), + } +} + #[tokio::test] async fn test_repos() -> Result<()> { - let github = Github::default(); - - let mut repos = github.get_repos("bitcoin", 1).await?; + let github = client(); + + let mut repos = github + .get_repos("bitcoin", 1) + .await? + .into_iter() + .map(|repo| repo.name) + .collect::>(); + repos.sort(); assert_eq!(repos, vec!["bips", "bitcoin", "libbase58", "libblkmaker"]); @@ -14,7 +29,7 @@ async fn test_repos() -> Result<()> { #[tokio::test] async fn test_issues() -> Result<()> { - let github = Github::default(); + let github = client(); let issues = github.get_issues("bitcoin", "bitcoin", 1).await?; diff --git a/backend/libs/store/src/migrations/github_repositories.rs b/backend/libs/store/src/migrations/github_repositories.rs index a44d028..a876739 100644 --- a/backend/libs/store/src/migrations/github_repositories.rs +++ b/backend/libs/store/src/migrations/github_repositories.rs @@ -10,6 +10,9 @@ pub struct Model { pub id: Uuid, pub project: Uuid, pub repository_name: String, + pub language: Option, + pub stargazers_count: i64, + pub forks_count: i64, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/backend/libs/store/src/objects/cryptocurrency_with_repositories.rs b/backend/libs/store/src/objects/cryptocurrency_with_repositories.rs index 873fdd5..96d9575 100644 --- a/backend/libs/store/src/objects/cryptocurrency_with_repositories.rs +++ b/backend/libs/store/src/objects/cryptocurrency_with_repositories.rs @@ -17,4 +17,7 @@ pub struct CryptoCurrencyWithRepositories { pub struct Repository { pub id: Uuid, pub repository_name: String, + pub language: Option, + pub stargazers_count: i64, + pub forks_count: i64, } diff --git a/backend/libs/store/src/objects/repository_view.rs b/backend/libs/store/src/objects/repository_view.rs index 907cc87..a0c9dbd 100644 --- a/backend/libs/store/src/objects/repository_view.rs +++ b/backend/libs/store/src/objects/repository_view.rs @@ -5,6 +5,9 @@ use sea_orm::{prelude::Uuid, FromQueryResult}; pub struct RepositoryView { pub id: Uuid, pub name: String, + pub language: Option, + pub forks_count: i64, + pub stars_count: i64, pub project_id: Uuid, pub project: String, } diff --git a/backend/migrations/2023-10-01-102605_github-repositories/up.sql b/backend/migrations/2023-10-01-102605_github-repositories/up.sql index fbdbb57..30f1bfe 100644 --- a/backend/migrations/2023-10-01-102605_github-repositories/up.sql +++ b/backend/migrations/2023-10-01-102605_github-repositories/up.sql @@ -2,5 +2,8 @@ create table github_repositories( id uuid primary key default uuid_generate_v4() not null, project uuid not null references github_projects(id) on delete cascade, repository_name varchar(255) not null, + language varchar(255), + stargazers_count bigint not null, + forks_count bigint not null, unique(project, repository_name) ) \ No newline at end of file