pub mod retry; pub mod repo; use reqwest::Error; use std::time::Duration; use std::path::Path; use std::fs; use std::process::Command; use retry::{RetryConfig, RetryableError, retry_with_backoff, check_response_status}; use repo::Repo; /// Validate that a repository name contains only safe characters /// and cannot be interpreted as a command-line flag by git. fn is_valid_repo_name(name: &str) -> bool { !name.is_empty() && !name.starts_with('-') && name.chars().all(|c| c.is_alphanumeric() || c == '-' || c == '_' || c == '.') } /// Validate that a branch name contains only safe characters /// and cannot be interpreted as a command-line flag by git. fn is_valid_branch_name(name: &str) -> bool { !name.is_empty() && !name.starts_with('-') && name .chars() .all(|c| c.is_alphanumeric() || c == '-' || c == '_' || c == '.' || c == '/') } /// Shared context passed through the clone/archive pipeline. struct CloneContext<'a> { client: &'a reqwest::Client, retry_config: RetryConfig, base_url: &'a str, org: &'a str, clone_folder: &'a str, days_last_updated: i64, use_archive: bool, clone_depth: u32, } /// Create a configured reqwest client with proper user agent. /// /// If `token` is provided, all requests will include an `Authorization: Bearer ` /// header, which raises the GitHub API rate limit from 60 to 5,000 requests per hour. pub fn create_client(token: Option<&str>) -> Result { let mut headers = reqwest::header::HeaderMap::new(); if let Some(t) = token { let mut auth_value = reqwest::header::HeaderValue::from_str(&format!("Bearer {t}")) .unwrap_or_else(|_| reqwest::header::HeaderValue::from_static("")); auth_value.set_sensitive(true); headers.insert(reqwest::header::AUTHORIZATION, auth_value); } reqwest::Client::builder() .user_agent("richardsondev/githubrepocloner") .default_headers(headers) .build() } /// Clone or archive repositories from a GitHub organization. /// /// Fetches the repository list from `{base_url}/orgs/{org}/repos`, filters by /// fork status and last update date, then either git-clones or downloads archives /// depending on `use_archive`. pub async fn clone_repos_with_client_and_url( client: &reqwest::Client, base_url: &str, org: &str, clone_folder: &str, days_last_updated: i64, use_archive: bool, clone_depth: u32, ) -> Result<(), Error> { let ctx = CloneContext { client, retry_config: RetryConfig::default(), base_url, org, clone_folder, days_last_updated, use_archive, clone_depth, }; let mut page = 1; let mut total_repos_found: usize = 0; loop { let resp = fetch_repo_page(&ctx, page, total_repos_found).await?; let repo_data: Vec = resp.json().await?; if repo_data.is_empty() { break; } total_repos_found += repo_data.len(); for repo in repo_data { process_repo(&ctx, &repo).await; } page += 1; } Ok(()) } /// Fetch a single page of repositories from the GitHub API with retry logic. async fn fetch_repo_page( ctx: &CloneContext<'_>, page: u32, total_repos_found: usize, ) -> Result { let base_url = ctx.base_url; let org = ctx.org; match retry_with_backoff(&ctx.retry_config, || async { let response = ctx.client .get(format!("{base_url}/orgs/{org}/repos?per_page=100&page={page}")) .send() .await?; check_response_status(response) }) .await { Ok(resp) => Ok(resp), Err(RetryableError::RequestError(e)) => Err(e), Err(e) => { eprintln!("Failed listing repos for {org} on page {page}. Found {total_repos_found} repos before failure."); eprintln!("Error: {e}"); std::process::exit(1); } } } /// Process a single repository — either clone it or download its archive. async fn process_repo(ctx: &CloneContext<'_>, repo: &Repo) { if !is_valid_repo_name(&repo.name) { eprintln!("Skipping repo with invalid name: {}", repo.name); return; } if !is_valid_branch_name(&repo.default_branch) { eprintln!( "Skipping repo {} with invalid default branch: {}", repo.name, repo.default_branch ); return; } let repo_path = format!("{}/{}", ctx.clone_folder, repo.name); if Path::new(&repo_path).exists() { println!("Skipping {}, folder already exists.", repo.name); return; } let updated_at = match chrono::DateTime::parse_from_rfc3339(&repo.updated_at) { Ok(dt) => dt.naive_utc(), Err(e) => { println!("Warning: Failed to parse date for {}: {e}. Skipping.", repo.name); return; } }; let days_since_update = (chrono::Utc::now().naive_utc() - updated_at).num_days(); if !repo.fork && days_since_update < ctx.days_last_updated { if ctx.use_archive { download_archive(ctx, repo).await; } else { clone_repo(ctx.org, repo, &repo_path, ctx.clone_depth); } } println!("Done with repo! Waiting 5 seconds"); tokio::time::sleep(Duration::from_secs(5)).await; } /// Download a repository archive via the GitHub Archive API. async fn download_archive(ctx: &CloneContext<'_>, repo: &Repo) { println!("Downloading archive for {}...", repo.name); let base_url = ctx.base_url; let org = ctx.org; let archive_url = format!( "{base_url}/repos/{org}/{}/tarball/{}", repo.name, repo.default_branch ); match retry_with_backoff(&ctx.retry_config, || async { let response = ctx.client.get(&archive_url).send().await?; check_response_status(response) }) .await { Ok(archive_resp) => match archive_resp.bytes().await { Ok(bytes) => { let archive_path = format!("{}/{}.tar.gz", ctx.clone_folder, repo.name); match fs::write(&archive_path, bytes) { Ok(()) => println!("Successfully downloaded archive for {}!", repo.name), Err(e) => println!("Failed to write archive for {}: {e}", repo.name), } } Err(e) => println!("Failed to download archive bytes for {}: {e}", repo.name), }, Err(RetryableError::RequestError(e)) => { println!("Failed to download archive for {}: {e}", repo.name); } Err(e) => { println!("Failed to download archive for {} after retries: {e}", repo.name); } } } /// Clone a repository using git. fn clone_repo(org: &str, repo: &Repo, repo_path: &str, clone_depth: u32) { println!("Cloning {}...", repo.name); match Command::new("git") .arg("clone") .arg(format!("--depth={clone_depth}")) .arg(format!("--branch={}", repo.default_branch)) .arg(format!("https://github.com/{org}/{}.git", repo.name)) .arg(repo_path) .status() { Ok(status) => { if status.success() { println!("Successfully cloned {}!", repo.name); } else { println!("Failed to clone {} (exit code: {:?}).", repo.name, status.code()); } } Err(e) => { println!("Error executing git command for {}: {e}", repo.name); } } } #[cfg(test)] #[allow(clippy::unwrap_used, clippy::expect_used)] mod tests { use super::*; #[test] fn test_create_client_no_token() { let client = create_client(None); assert!(client.is_ok()); } #[test] fn test_create_client_with_token() { let client = create_client(Some("ghp_testtoken123")); assert!(client.is_ok()); } #[test] fn test_client_has_user_agent() { let client = create_client(None).unwrap(); assert_eq!(std::mem::size_of_val(&client), std::mem::size_of::()); } #[test] fn test_valid_repo_names() { assert!(is_valid_repo_name("my-repo")); assert!(is_valid_repo_name("repo_name")); assert!(is_valid_repo_name("repo.name")); assert!(is_valid_repo_name("Repo123")); } #[test] fn test_invalid_repo_names() { assert!(!is_valid_repo_name("")); assert!(!is_valid_repo_name("-flag")); assert!(!is_valid_repo_name("--upload-pack=evil")); assert!(!is_valid_repo_name("repo name")); assert!(!is_valid_repo_name("repo;rm -rf /")); assert!(!is_valid_repo_name("repo$(cmd)")); } #[test] fn test_valid_branch_names() { assert!(is_valid_branch_name("main")); assert!(is_valid_branch_name("feature/my-branch")); assert!(is_valid_branch_name("release-1.0")); assert!(is_valid_branch_name("v2.0.1")); } #[test] fn test_invalid_branch_names() { assert!(!is_valid_branch_name("")); assert!(!is_valid_branch_name("-flag")); assert!(!is_valid_branch_name("--upload-pack=evil")); assert!(!is_valid_branch_name("branch name")); assert!(!is_valid_branch_name("branch;cmd")); } }