diff --git a/src/config.rs b/src/config.rs index 936018b6a..d84f12872 100644 --- a/src/config.rs +++ b/src/config.rs @@ -30,6 +30,8 @@ pub struct Config { pub(crate) max_file_size_html: usize, // The most memory that can be used to parse an HTML file pub(crate) max_parse_memory: usize, + // Time between 'git gc --auto' calls in seconds + pub(crate) registry_gc_interval: u64, } impl Config { @@ -61,6 +63,7 @@ impl Config { // LOL HTML only uses as much memory as the size of the start tag! // https://github.com/rust-lang/docs.rs/pull/930#issuecomment-667729380 max_parse_memory: env("DOCSRS_MAX_PARSE_MEMORY", 5 * 1024 * 1024)?, + registry_gc_interval: env("DOCSRS_REGISTRY_GC_INTERVAL", 60 * 60)?, }) } diff --git a/src/docbuilder/queue.rs b/src/docbuilder/queue.rs index ae3bfc76e..d357fd2d6 100644 --- a/src/docbuilder/queue.rs +++ b/src/docbuilder/queue.rs @@ -84,4 +84,8 @@ impl DocBuilder { Ok(processed) } + + pub fn run_git_gc(&self) { + self.index.run_git_gc(); + } } diff --git a/src/index/mod.rs b/src/index/mod.rs index 68e806bcd..c6ca75869 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -1,4 +1,7 @@ -use std::path::{Path, PathBuf}; +use std::{ + path::{Path, PathBuf}, + process::Command, +}; use url::Url; @@ -71,6 +74,22 @@ impl Index { pub fn api(&self) -> &Api { &self.api } + + pub fn run_git_gc(&self) { + let gc = Command::new("git") + .arg("-C") + .arg(&self.path) + .args(&["gc", "--auto"]) + .output(); + + if let Err(err) = gc { + log::error!( + "failed to run `git gc --auto`\npath: {:#?}\nerror: {:#?}", + &self.path, + err + ); + } + } } impl Clone for Index { diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 683b9a64b..61a9fa062 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -10,16 +10,20 @@ use chrono::{Timelike, Utc}; use failure::Error; use log::{debug, error, info}; use std::thread; -use std::time::Duration; +use std::time::{Duration, Instant}; fn start_registry_watcher(opts: DocBuilderOptions, context: &dyn Context) -> Result<(), Error> { let pool = context.pool()?; let build_queue = context.build_queue()?; + let config = context.config()?; + thread::Builder::new() .name("registry index reader".to_string()) .spawn(move || { // space this out to prevent it from clashing against the queue-builder thread on launch thread::sleep(Duration::from_secs(30)); + + let mut last_gc = Instant::now(); loop { let mut doc_builder = DocBuilder::new(opts.clone(), pool.clone(), build_queue.clone()); @@ -34,6 +38,10 @@ fn start_registry_watcher(opts: DocBuilderOptions, context: &dyn Context) -> Res } } + if last_gc.elapsed().as_secs() >= config.registry_gc_interval { + doc_builder.run_git_gc(); + last_gc = Instant::now(); + } thread::sleep(Duration::from_secs(60)); } })?;