extract repository-stats logic into subcrate

Author: syphar

.github/workflows/ci.yml

@@ -22,7 +22,7 @@ jobs:
 
       - uses: taiki-e/install-action@v2
         with:
-          tool: just,sqlx-cli
+          tool: just,sqlx-cli,fd-find
 
       - name: restore build & cargo cache
         uses: Swatinem/rust-cache@v2

.sqlx/query-0107ab57a47a423721cc6257cf1572348bf76ecf16632fe625ebafa17f45738a.json

@@ -56,6 +56,7 @@ docs_rs_headers = { path = "crates/lib/docs_rs_headers" }
 docs_rs_logging = { path = "crates/lib/docs_rs_logging" }
 docs_rs_opentelemetry = { path = "crates/lib/docs_rs_opentelemetry" }
 docs_rs_registry_api = { path = "crates/lib/docs_rs_registry_api" }
+docs_rs_repository_stats = { path = "crates/lib/docs_rs_repository_stats" }
 docs_rs_types = { path = "crates/lib/docs_rs_types" }
 docs_rs_uri = { path = "crates/lib/docs_rs_uri" }
 docs_rs_utils = { path = "crates/lib/docs_rs_utils" }
@@ -120,7 +121,6 @@ serde_json = { workspace = true }
 bincode = { workspace = true }
 
 # axum dependencies
-async-trait = "0.1.83"
 axum = { version = "0.8.1", features = ["macros"] }
 axum-extra = { workspace = true }
 tower = "0.5.1"

.sqlx/query-f550ed904fdb5d3ee6581fe1ad036c9b5b8db8765d5665042deb9ade67394d3c.json

@@ -0,0 +1,25 @@
+[package]
+name = "docs_rs_repository_stats"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+anyhow = { workspace = true }
+async-trait  = "0.1.89"
+chrono = { workspace = true }
+docs_rs_cargo_metadata = { path = "../docs_rs_cargo_metadata" }
+docs_rs_database = { path = "../docs_rs_database" }
+docs_rs_env_vars = { path = "../docs_rs_env_vars" }
+docs_rs_utils = { path = "../docs_rs_utils" }
+futures-util = { workspace = true }
+regex = { workspace = true }
+reqwest = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+sqlx = { workspace = true }
+thiserror = { workspace = true }
+tracing = { workspace = true }
+
+[dev-dependencies]
+mockito = { workspace = true }
+tokio = { workspace = true }

Cargo.lock

@@ -0,0 +1,21 @@
+use docs_rs_env_vars::{env, maybe_env};
+
+#[derive(Debug)]
+pub struct Config {
+    // Github authentication
+    pub(crate) github_accesstoken: Option<String>,
+    pub(crate) github_updater_min_rate_limit: u32,
+
+    // GitLab authentication
+    pub(crate) gitlab_accesstoken: Option<String>,
+}
+
+impl Config {
+    pub fn from_environment() -> anyhow::Result<Self> {
+        Ok(Self {
+            github_accesstoken: maybe_env("DOCSRS_GITHUB_ACCESSTOKEN")?,
+            github_updater_min_rate_limit: env("DOCSRS_GITHUB_UPDATER_MIN_RATE_LIMIT", 2500u32)?,
+            gitlab_accesstoken: maybe_env("DOCSRS_GITLAB_ACCESSTOKEN")?,
+        })
+    }
+}

Cargo.toml

@@ -0,0 +1,3 @@
+#[derive(Debug, thiserror::Error)]
+#[error("rate limit reached")]
+pub struct RateLimitReached;

crates/lib/docs_rs_repository_stats/Cargo.toml

@@ -1,21 +1,19 @@
-use crate::Config;
-use crate::error::Result;
+use crate::{
+    RateLimitReached,
+    config::Config,
+    updater::{FetchRepositoriesResult, Repository, RepositoryForge, RepositoryName},
+};
+use anyhow::Result;
 use async_trait::async_trait;
 use chrono::{DateTime, Utc};
+use docs_rs_utils::APP_USER_AGENT;
 use reqwest::{
     Client as HttpClient,
     header::{ACCEPT, AUTHORIZATION, HeaderMap, HeaderValue, USER_AGENT},
 };
 use serde::Deserialize;
 use tracing::{trace, warn};
 
-use crate::{
-    APP_USER_AGENT,
-    repositories::{
-        FetchRepositoriesResult, RateLimitReached, Repository, RepositoryForge, RepositoryName,
-    },
-};
-
 const GRAPHQL_UPDATE: &str = "query($ids: [ID!]!) {
     nodes(ids: $ids) {
         ... on Repository {
@@ -92,10 +90,6 @@ impl RepositoryForge for GitHub {
         "github.com"
     }
 
-    fn icon(&self) -> &'static str {
-        "github"
-    }
-
     /// How many repositories to update in a single chunk. Values over 100 are probably going to be
     /// rejected by the GraphQL API.
     fn chunk_size(&self) -> usize {
@@ -268,19 +262,18 @@ struct GraphIssues {
 
 #[cfg(test)]
 mod tests {
-    use super::{Config, GitHub};
-    use crate::repositories::RateLimitReached;
-    use crate::repositories::updater::{RepositoryForge, repository_name};
-    use crate::test::TestEnvironment;
+    use crate::{
+        Config, GitHub, RateLimitReached,
+        updater::{RepositoryForge, repository_name},
+    };
     use anyhow::Result;
 
     const TEST_TOKEN: &str = "qsjdnfqdq";
 
     fn github_config() -> anyhow::Result<Config> {
-        TestEnvironment::base_config()
-            .github_accesstoken(Some(TEST_TOKEN.to_owned()))
-            .build()
-            .map_err(Into::into)
+        let mut cfg = Config::from_environment()?;
+        cfg.github_accesstoken = Some(TEST_TOKEN.to_owned());
+        Ok(cfg)
     }
 
     async fn mock_server_and_github(config: &Config) -> (mockito::ServerGuard, GitHub) {

crates/lib/docs_rs_repository_stats/src/config.rs

@@ -1,6 +1,7 @@
-use crate::error::Result;
+use anyhow::Result;
 use async_trait::async_trait;
 use chrono::{DateTime, Utc};
+use docs_rs_utils::APP_USER_AGENT;
 use reqwest::{
     Client as HttpClient,
     header::{ACCEPT, AUTHORIZATION, HeaderMap, HeaderValue, USER_AGENT},
@@ -11,10 +12,8 @@ use std::str::FromStr;
 use tracing::warn;
 
 use crate::{
-    APP_USER_AGENT,
-    repositories::{
-        FetchRepositoriesResult, RateLimitReached, Repository, RepositoryForge, RepositoryName,
-    },
+    RateLimitReached,
+    updater::{FetchRepositoriesResult, Repository, RepositoryForge, RepositoryName},
 };
 
 const GRAPHQL_UPDATE: &str = "query($ids: [ID!]!) {
@@ -90,10 +89,6 @@ impl RepositoryForge for GitLab {
         self.host
     }
 
-    fn icon(&self) -> &'static str {
-        "gitlab"
-    }
-
     fn chunk_size(&self) -> usize {
         100
     }
@@ -266,9 +261,10 @@ struct GraphProject {
 
 #[cfg(test)]
 mod tests {
-    use super::GitLab;
-    use crate::repositories::RateLimitReached;
-    use crate::repositories::updater::{RepositoryForge, repository_name};
+    use crate::{
+        GitLab, RateLimitReached,
+        updater::{RepositoryForge, repository_name},
+    };
     use anyhow::Result;
 
     async fn mock_server_and_gitlab() -> (mockito::ServerGuard, GitLab) {