rust
/
cargo
mirror of https://github.com/rust-lang/cargo
0
0
Fork 0
Code Issues Releases Wiki Activity

Add a global cache garbage collector. 

This adds a garbage collector which will remove old files from cargo's
global cache.

A general overview of the changes here:

- `cargo::core::global_cache_tracker` contains the `GlobalCacheTracker`
  which handles the interface to a sqlite database which stores
  timestamps of the last time a file was used.
- `DeferredGlobalLastUse` is a type that implements an optimization for
  collecting last-use timestamps so that they can be flushed to disk all
  at once.
- `cargo::core::gc` contains the `Gc` type which is the interface for
  performing garbage collection. It coordinates with the
  `GlobalCacheTracker` for determining what to delete.
- Garbage collection can either be automatic or manual. The automatic
  garbage collection supports some config options for defining when
  it runs and how much it deletes.
- Manual garbage collection can be performed via options to `cargo
  clean`.
- `cargo clean` uses the new package cache locking system to coordinate
  access to the package cache to prevent interference with other cargo
  commands running concurrently.
This commit is contained in:
Eric Huss 2023-09-06 21:34:05 -07:00
parent 34fdf5fb03
commit da3ca05677
23 changed files with 4652 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
Cargo.lock generated
View File

@ -287,6 +287,7 @@ dependencies = [
"pathdiff",
"pulldown-cmark",
"rand",
"regex",
"rusqlite",
"rustfix",
"same-file",
@ -407,6 +408,7 @@ dependencies = [
"time",
"toml",
"url",
"walkdir",
"windows-sys",
]
@ -2669,7 +2671,7 @@ dependencies = [
"rand",
"rand_chacha",
"rand_xorshift",
"regex-syntax 0.7.2",
"regex-syntax 0.7.5",
"rusty-fork",
"tempfile",
"unarray",
@ -2797,13 +2799,14 @@ dependencies = [
[[package]]
name = "regex"
version = "1.8.4"
version = "1.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f"
checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax 0.7.2",
"regex-automata 0.3.8",
"regex-syntax 0.7.5",
]
[[package]]
@ -2820,6 +2823,11 @@ name = "regex-automata"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax 0.7.5",
]
[[package]]
name = "regex-syntax"
@ -2829,9 +2837,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
[[package]]
name = "regex-syntax"
version = "0.7.2"
version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78"
checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
[[package]]
name = "resolver-tests"

2
Cargo.toml
View File

@ -73,6 +73,7 @@ pretty_assertions = "1.4.0"
proptest = "1.3.1"
pulldown-cmark = { version = "0.9.3", default-features = false }
rand = "0.8.5"
regex = "1.9.3"
rusqlite = { version = "0.29.0", features = ["bundled"] }
rustfix = "0.6.1"
same-file = "1.0.6"
@ -163,6 +164,7 @@ pasetors.workspace = true
pathdiff.workspace = true
pulldown-cmark.workspace = true
rand.workspace = true
regex.workspace = true
rusqlite.workspace = true
rustfix.workspace = true
semver.workspace = true

1
crates/cargo-test-support/Cargo.toml
View File

@ -29,6 +29,7 @@ tar.workspace = true
time.workspace = true
toml.workspace = true
url.workspace = true
walkdir.workspace = true
[target.'cfg(windows)'.dependencies]
windows-sys = { workspace = true, features = ["Win32_Storage_FileSystem"] }

13
crates/cargo-test-support/src/paths.rs
View File

@ -114,6 +114,10 @@ pub trait CargoPathExt {
fn rm_rf(&self);
fn mkdir_p(&self);
/// Returns a list of all files and directories underneath the given
/// directory, recursively, including the starting path.
fn ls_r(&self) -> Vec<PathBuf>;
fn move_into_the_past(&self) {
self.move_in_time(|sec, nsec| (sec - 3600, nsec))
}
@ -155,6 +159,15 @@ impl CargoPathExt for Path {
.unwrap_or_else(|e| panic!("failed to mkdir_p {}: {}", self.display(), e))
}
fn ls_r(&self) -> Vec<PathBuf> {
let mut file_list: Vec<_> = walkdir::WalkDir::new(self)
.into_iter()
.filter_map(|e| e.map(|e| e.path().to_owned()).ok())
.collect();
file_list.sort();
file_list
}
fn move_in_time<F>(&self, travel_amount: F)
where
F: Fn(i64, u32) -> (i64, u32),

222
src/bin/cargo/commands/clean.rs
View File

@ -1,7 +1,11 @@
use crate::command_prelude::*;
use cargo::core::gc::{parse_human_size, parse_time_span};
use cargo::core::gc::{AutoGcKind, GcOpts};
use cargo::ops::{self, CleanOptions};
use cargo::util::print_available_packages;
use cargo::CargoResult;
use clap::builder::{PossibleValuesParser, TypedValueParser};
use std::time::Duration;
pub fn cli() -> Command {
subcommand("clean")
@ -15,18 +19,227 @@ pub fn cli() -> Command {
.arg_target_dir()
.arg_manifest_path()
.arg_dry_run("Display what would be deleted without deleting anything")
// NOTE: Not all of these options may get stabilized. Some of them are
// very low-level details, and may not be something typical users need.
.arg(
optional_opt(
"gc",
"Delete old and unused files (unstable) (comma separated): all, download, target, shared-target",
)
.hide(true)
.value_name("KINDS")
.value_parser(
PossibleValuesParser::new(["all", "download", "target", "shared-target"]).map(|x|
match x.as_str() {
"all" => AutoGcKind::All,
"download" => AutoGcKind::Download,
"target" => panic!("target is not yet implemented"),
"shared-target" => panic!("shared-target is not yet implemented"),
x => panic!("possible value out of sync with `{x}`"),
}
))
.require_equals(true),
)
.arg(
opt(
"max-src-age",
"Deletes source cache files that have not been used since the given age (unstable)",
)
.hide(true)
.value_name("DURATION"),
)
.arg(
opt(
"max-crate-age",
"Deletes crate cache files that have not been used since the given age (unstable)",
)
.hide(true)
.value_name("DURATION"),
)
.arg(
opt(
"max-index-age",
"Deletes registry indexes that have not been used since then given age (unstable)",
)
.hide(true)
.value_name("DURATION"),
)
.arg(
opt(
"max-git-co-age",
"Deletes git dependency checkouts that have not been used since then given age (unstable)",
)
.hide(true)
.value_name("DURATION"),
)
.arg(
opt(
"max-git-db-age",
"Deletes git dependency clones that have not been used since then given age (unstable)",
)
.hide(true)
.value_name("DURATION"),
)
.arg(
opt(
"max-download-age",
"Deletes any downloaded cache data that has not been used since then given age (unstable)",
)
.hide(true)
.value_name("DURATION"),
)
.arg(
opt(
"max-src-size",
"Deletes source cache files until the cache is under the given size (unstable)",
)
.hide(true)
.value_name("SIZE"),
)
.arg(
opt(
"max-crate-size",
"Deletes crate cache files until the cache is under the given size (unstable)",
)
.hide(true)
.value_name("SIZE"),
)
.arg(
opt("max-git-size",
"Deletes git dependency caches until the cache is under the given size (unstable")
.hide(true)
.value_name("SIZE"))
.arg(
opt(
"max-download-size",
"Deletes downloaded cache data until the cache is under the given size (unstable)",
)
.hide(true)
.value_name("DURATION"),
)
// These are unimplemented. Leaving here as a guide for how this is
// intended to evolve. These will likely change, this is just a sketch
// of ideas.
.arg(
opt(
"max-target-age",
"Deletes any build artifact files that have not been used since then given age (unstable) (UNIMPLEMENTED)",
)
.hide(true)
.value_name("DURATION"),
)
.arg(
// TODO: come up with something less wordy?
opt(
"max-shared-target-age",
"Deletes any shared build artifact files that have not been used since then given age (unstable) (UNIMPLEMENTED)",
)
.hide(true)
.value_name("DURATION"),
)
.arg(
opt(
"max-target-size",
"Deletes build artifact files until the cache is under the given size (unstable) (UNIMPLEMENTED)",
)
.hide(true)
.value_name("SIZE"),
)
.arg(
// TODO: come up with something less wordy?
opt(
"max-shared-target-size",
"Deletes shared build artifact files until the cache is under the given size (unstable) (UNIMPLEMENTED)",
)
.hide(true)
.value_name("DURATION"),
)
.after_help(color_print::cstr!(
"Run `<cyan,bold>cargo help clean</>` for more detailed information.\n"
))
}
pub fn exec(config: &mut Config, args: &ArgMatches) -> CliResult {
let ws = args.workspace(config)?;
let ws = args.workspace(config);
if args.is_present_with_zero_values("package") {
print_available_packages(&ws)?;
print_available_packages(&ws?)?;
return Ok(());
}
let unstable_gc = |opt| {
// TODO: issue number
config
.cli_unstable()
.fail_if_stable_opt_custom_z(opt, 0, "gc", config.cli_unstable().gc)
};
let unstable_cache_opt = |opt| -> CargoResult<Option<&str>> {
let arg = args.get_one::<String>(opt).map(String::as_str);
if arg.is_some() {
unstable_gc(opt)?;
}
Ok(arg)
};
let unstable_size_opt = |opt| -> CargoResult<Option<u64>> {
unstable_cache_opt(opt)?
.map(|s| parse_human_size(s))
.transpose()
};
let unstable_duration_opt = |opt| -> CargoResult<Option<Duration>> {
unstable_cache_opt(opt)?
.map(|s| parse_time_span(s))
.transpose()
};
let unimplemented_opt = |opt| -> CargoResult<Option<&str>> {
let arg = args.get_one::<String>(opt).map(String::as_str);
if arg.is_some() {
anyhow::bail!("option --{opt} is not yet implemented");
}
Ok(None)
};
let unimplemented_size_opt = |opt| -> CargoResult<Option<u64>> {
unimplemented_opt(opt)?;
Ok(None)
};
let unimplemented_duration_opt = |opt| -> CargoResult<Option<Duration>> {
unimplemented_opt(opt)?;
Ok(None)
};
let mut gc: Vec<_> = args
.get_many::<AutoGcKind>("gc")
.unwrap_or_default()
.cloned()
.collect();
if gc.is_empty() && args.contains_id("gc") {
gc.push(AutoGcKind::All);
}
if !gc.is_empty() {
unstable_gc("gc")?;
}
let mut gc_opts = GcOpts {
max_src_age: unstable_duration_opt("max-src-age")?,
max_crate_age: unstable_duration_opt("max-crate-age")?,
max_index_age: unstable_duration_opt("max-index-age")?,
max_git_co_age: unstable_duration_opt("max-git-co-age")?,
max_git_db_age: unstable_duration_opt("max-git-db-age")?,
max_src_size: unstable_size_opt("max-src-size")?,
max_crate_size: unstable_size_opt("max-crate-size")?,
max_git_size: unstable_size_opt("max-git-size")?,
max_download_size: unstable_size_opt("max-download-size")?,
max_target_age: unimplemented_duration_opt("max-target-age")?,
max_shared_target_age: unimplemented_duration_opt("max-shared-target-age")?,
max_target_size: unimplemented_size_opt("max-target-size")?,
max_shared_target_size: unimplemented_size_opt("max-shared-target-size")?,
};
let max_download_age = unstable_duration_opt("max-download-age")?;
gc_opts.update_for_auto_gc(config, &gc, max_download_age)?;
let opts = CleanOptions {
config,
spec: values(args, "package"),
@ -35,7 +248,8 @@ pub fn exec(config: &mut Config, args: &ArgMatches) -> CliResult {
profile_specified: args.contains_id("profile") || args.flag("release"),
doc: args.flag("doc"),
dry_run: args.dry_run(),
gc_opts,
};
ops::clean(&ws, &opts)?;
ops::clean(ws, &opts)?;
Ok(())
}

550
src/cargo/core/gc.rs Normal file
View File

@ -0,0 +1,550 @@
//! Support for garbage collecting unused files from downloaded files or
//! artifacts from the target directory.
//!
//! Garbage collection can be done "automatically" by cargo, which it does by
//! default once a day when running any command that does a lot of work (like
//! `cargo build`).
//!
//! Garbage collection can also be done manually via the `cargo clean` command
//! by passing any option that requests deleting unused files.
//!
//! Garbage collection is guided by the last-use tracking implemented in the
//! [`crate::core::global_cache_tracker`] module.
use crate::core::global_cache_tracker::{self, GlobalCacheTracker};
use crate::core::Verbosity;
use crate::ops::CleanContext;
use crate::util::cache_lock::{CacheLock, CacheLockMode};
use crate::{CargoResult, Config};
use anyhow::format_err;
use anyhow::{bail, Context};
use serde::Deserialize;
use std::time::Duration;
/// Garbage collector.
pub struct Gc<'a, 'config> {
config: &'config Config,
global_cache_tracker: &'a mut GlobalCacheTracker,
/// A lock on the package cache.
///
/// This is important to be held, since we don't want multiple cargos to
/// be allowed to write to the cache at the same time, or for others to
/// read while we are modifying the cache.
#[allow(dead_code)] // Held for drop.
lock: CacheLock<'config>,
}
/// Automatic garbage collection settings from the `gc.auto` config table.
///
/// NOTE: Not all of these options may get stabilized. Some of them are very
/// low-level details, and may not be something typical users need.
///
/// If any of these options are `None`, the built-in default is used.
#[derive(Deserialize, Default)]
#[serde(rename_all = "kebab-case")]
struct AutoConfig {
/// The maximum frequency that automatic garbage collection happens.
frequency: Option<String>,
/// Anything older than this duration will be deleted in the source cache.
max_src_age: Option<String>,
/// Anything older than this duration will be deleted in the compressed crate cache.
max_crate_age: Option<String>,
/// Any index older than this duration will be deleted from the index cache.
max_index_age: Option<String>,
/// Any git checkout older than this duration will be deleted from the checkout cache.
max_git_co_age: Option<String>,
/// Any git clone older than this duration will be deleted from the git cache.
max_git_db_age: Option<String>,
}
/// Options to use for garbage collection.
#[derive(Clone, Debug, Default)]
pub struct GcOpts {
/// The `--max-src-age` CLI option.
pub max_src_age: Option<Duration>,
// The `--max-crate-age` CLI option.
pub max_crate_age: Option<Duration>,
/// The `--max-index-age` CLI option.
pub max_index_age: Option<Duration>,
/// The `--max-git-co-age` CLI option.
pub max_git_co_age: Option<Duration>,
/// The `--max-git-db-age` CLI option.
pub max_git_db_age: Option<Duration>,
/// The `--max-src-size` CLI option.
pub max_src_size: Option<u64>,
/// The `--max-crate-size` CLI option.
pub max_crate_size: Option<u64>,
/// The `--max-git-size` CLI option.
pub max_git_size: Option<u64>,
/// The `--max-download-size` CLI option.
pub max_download_size: Option<u64>,
/// The `--max-target-age` CLI option (UNIMPLEMENTED).
pub max_target_age: Option<Duration>,
/// The `--max-shared-target-age CLI option (UNIMPLEMENTED).
pub max_shared_target_age: Option<Duration>,
/// The `--max-target-size` CLI option (UNIMPLEMENTED).
pub max_target_size: Option<u64>,
/// The `--max-shared-target-size` CLI option (UNIMPLEMENTED).
pub max_shared_target_size: Option<u64>,
}
impl GcOpts {
/// Returns whether any download cache cleaning options are set.
pub fn is_download_cache_opt_set(&self) -> bool {
self.max_src_age.is_some()
|| self.max_crate_age.is_some()
|| self.max_index_age.is_some()
|| self.max_git_co_age.is_some()
|| self.max_git_db_age.is_some()
|| self.max_src_size.is_some()
|| self.max_crate_size.is_some()
|| self.max_git_size.is_some()
|| self.max_download_size.is_some()
}
/// Returns whether any download cache cleaning options based on size are set.
pub fn is_download_cache_size_set(&self) -> bool {
self.max_src_size.is_some()
|| self.max_crate_size.is_some()
|| self.max_git_size.is_some()
|| self.max_download_size.is_some()
}
/// Returns whether any target directory cleaning options are set.
pub fn is_target_opt_set(&self) -> bool {
self.max_target_size.is_some()
|| self.max_target_age.is_some()
|| self.max_shared_target_age.is_some()
|| self.max_shared_target_size.is_some()
}
/// Updates the configuration of this [`GcOpts`] to incorporate the
/// settings from config and the given CLI options.
///
/// * `kinds` is a list of [`AutoGcKind`] that is being requested to
/// perform. This corresponds to the `cargo clean --gc` flag. If empty,
/// no config options are incorporated.
/// * `max_download_age` is the `--max-download-age` CLI option which
/// requires special handling since it implicitly overlaps two options.
/// It will use the newer value of either this or the explicit value.
///
/// The `kinds` list is used in a few different ways:
///
/// * If empty, uses only the options the user specified on the
/// command-line, like `cargo clean --max-crate-size=…`.
/// * If the user specified a `cargo clean --gc` option, then the `kinds`
/// list is filled in with whatever `--gc` option the user picked, and
/// then this function *merges* the settings between the requested
/// `--gc` option and any options that were explicitly specified.
/// * [`AutoGcKind::All`] is used in `cargo clean` when no options are
/// specified.
pub fn update_for_auto_gc(
&mut self,
config: &Config,
kinds: &[AutoGcKind],
max_download_age: Option<Duration>,
) -> CargoResult<()> {
let auto_config = config
.get::<Option<AutoConfig>>("gc.auto")?
.unwrap_or_default();
self.update_for_auto_gc_config(&auto_config, kinds, max_download_age)
}
fn update_for_auto_gc_config(
&mut self,
auto_config: &AutoConfig,
kinds: &[AutoGcKind],
max_download_age: Option<Duration>,
) -> CargoResult<()> {
for kind in kinds {
if matches!(kind, AutoGcKind::All | AutoGcKind::Download) {
self.max_src_age = newer_time_span_for_config(
self.max_src_age,
"gc.auto.max-src-age",
auto_config.max_src_age.as_deref().unwrap_or("1 month"),
)?;
self.max_crate_age = newer_time_span_for_config(
self.max_crate_age,
"gc.auto.max-crate-age",
auto_config.max_crate_age.as_deref().unwrap_or("3 months"),
)?;
self.max_index_age = newer_time_span_for_config(
self.max_index_age,
"gc.auto.max-index-age",
auto_config.max_index_age.as_deref().unwrap_or("3 months"),
)?;
self.max_git_co_age = newer_time_span_for_config(
self.max_git_co_age,
"gc.auto.max-git-co-age",
auto_config.max_git_co_age.as_deref().unwrap_or("1 month"),
)?;
self.max_git_db_age = newer_time_span_for_config(
self.max_git_db_age,
"gc.auto.max-git-db-age",
auto_config.max_git_db_age.as_deref().unwrap_or("3 months"),
)?;
}
if matches!(kind, AutoGcKind::Target | AutoGcKind::SharedTarget) {
bail!("target is unimplemented");
}
}
if let Some(max_download_age) = max_download_age {
self.max_src_age = Some(maybe_newer_span(max_download_age, self.max_src_age));
self.max_crate_age = Some(maybe_newer_span(max_download_age, self.max_crate_age));
self.max_index_age = Some(maybe_newer_span(max_download_age, self.max_index_age));
self.max_git_co_age = Some(maybe_newer_span(max_download_age, self.max_git_co_age));
self.max_git_db_age = Some(maybe_newer_span(max_download_age, self.max_git_db_age));
}
Ok(())
}
}
/// The kind of automatic garbage collection to perform.
///
/// "Automatic" is the kind of gc performed automatically by Cargo in any
/// command that is already doing a bunch of work. See [`auto_gc`] for more.
#[derive(Clone, Debug)]
pub enum AutoGcKind {
/// Automatically clean up the downloaded files *and* the target directory.
///
/// This is the mode used by default.
All,
/// Automatically clean only downloaded files.
///
/// This corresponds to `cargo clean --gc=download`.
Download,
/// Automatically clean only the target directory.
///
/// THIS IS NOT IMPLEMENTED.
///
/// This corresponds to `cargo clean --gc=target`.
Target,
/// Automatically clean only the shared target directory.
///
/// THIS IS NOT IMPLEMENTED.
///
/// This corresponds to `cargo clean --gc=shared-target`.
SharedTarget,
}
impl<'a, 'config> Gc<'a, 'config> {
pub fn new(
config: &'config Config,
global_cache_tracker: &'a mut GlobalCacheTracker,
) -> CargoResult<Gc<'a, 'config>> {
let lock = config.acquire_package_cache_lock(CacheLockMode::MutateExclusive)?;
Ok(Gc {
config,
global_cache_tracker,
lock,
})
}
/// Performs automatic garbage cleaning.
///
/// This returns immediately without doing work if garbage collection has
/// been performed recently (since `gc.auto.frequency`).
fn auto(&mut self, clean_ctx: &mut CleanContext<'config>) -> CargoResult<()> {
if !self.config.cli_unstable().gc {
return Ok(());
}
let auto_config = self
.config
.get::<Option<AutoConfig>>("gc.auto")?
.unwrap_or_default();
let Some(freq) = parse_frequency(auto_config.frequency.as_deref().unwrap_or("1 day"))?
else {
tracing::trace!("auto gc disabled");
return Ok(());
};
if !self.global_cache_tracker.should_run_auto_gc(freq)? {
return Ok(());
}
let mut gc_opts = GcOpts::default();
gc_opts.update_for_auto_gc_config(&auto_config, &[AutoGcKind::All], None)?;
self.gc(clean_ctx, &gc_opts)?;
if !clean_ctx.dry_run {
self.global_cache_tracker.set_last_auto_gc()?;
}
Ok(())
}
/// Performs garbage collection based on the given options.
pub fn gc(
&mut self,
clean_ctx: &mut CleanContext<'config>,
gc_opts: &GcOpts,
) -> CargoResult<()> {
self.global_cache_tracker.clean(clean_ctx, gc_opts)?;
// In the future, other gc operations go here, such as target cleaning.
Ok(())
}
}
/// Returns the shorter duration from `cur_span` versus `config_span`.
///
/// This is used because the user may specify multiple options which overlap,
/// and this will pick whichever one is shorter.
///
/// * `cur_span` is the span we are comparing against (the value from the CLI
/// option). If None, just returns the config duration.
/// * `config_name` is the name of the config option the span is loaded from.
/// * `config_span` is the span value loaded from config.
fn newer_time_span_for_config(
cur_span: Option<Duration>,
config_name: &str,
config_span: &str,
) -> CargoResult<Option<Duration>> {
let config_span = parse_time_span_for_config(config_name, config_span)?;
Ok(Some(maybe_newer_span(config_span, cur_span)))
}
/// Returns whichever [`Duration`] is shorter.
fn maybe_newer_span(a: Duration, b: Option<Duration>) -> Duration {
match b {
Some(b) => {
if b < a {
b
} else {
a
}
}
None => a,
}
}
/// Parses a frequency string.
///
/// Returns `Ok(None)` if the frequency is "never".
fn parse_frequency(frequency: &str) -> CargoResult<Option<Duration>> {
if frequency == "always" {
return Ok(Some(Duration::new(0, 0)));
} else if frequency == "never" {
return Ok(None);
}
let duration = maybe_parse_time_span(frequency).ok_or_else(|| {
format_err!(
"config option `gc.auto.frequency` expected a value of \"always\", \"never\", \
or \"N seconds/minutes/days/weeks/months\", got: {frequency:?}"
)
})?;
Ok(Some(duration))
}
/// Parses a time span value fetched from config.
///
/// This is here to provide better error messages specific to reading from
/// config.
fn parse_time_span_for_config(config_name: &str, span: &str) -> CargoResult<Duration> {
maybe_parse_time_span(span).ok_or_else(|| {
format_err!(
"config option `{config_name}` expected a value of the form \
\"N seconds/minutes/days/weeks/months\", got: {span:?}"
)
})
}
/// Parses a time span string.
///
/// Returns None if the value is not valid. See [`parse_time_span`] if you
/// need a variant that generates an error message.
fn maybe_parse_time_span(span: &str) -> Option<Duration> {
let Some(right_i) = span.find(|c: char| !c.is_ascii_digit()) else {
return None;
};
let left = &span[..right_i];
let mut right = &span[right_i..];
if right.starts_with(' ') {
right = &right[1..];
}
let count: u64 = left.parse().ok()?;
let factor = match right {
"second" | "seconds" => 1,
"minute" | "minutes" => 60,
"hour" | "hours" => 60 * 60,
"day" | "days" => 24 * 60 * 60,
"week" | "weeks" => 7 * 24 * 60 * 60,
"month" | "months" => 30 * 24 * 60 * 60,
_ => return None,
};
Some(Duration::from_secs(factor * count))
}
/// Parses a time span string.
pub fn parse_time_span(span: &str) -> CargoResult<Duration> {
maybe_parse_time_span(span).ok_or_else(|| {
format_err!(
"expected a value of the form \
\"N seconds/minutes/days/weeks/months\", got: {span:?}"
)
})
}
/// Parses a file size using metric or IEC units.
pub fn parse_human_size(input: &str) -> CargoResult<u64> {
let re = regex::Regex::new(r"(?i)^([0-9]+(\.[0-9])?) ?(b|kb|mb|gb|kib|mib|gib)?$").unwrap();
let cap = re.captures(input).ok_or_else(|| {
format_err!(
"invalid size `{input}`, \
expected a number with an optional B, kB, MB, GB, kiB, MiB, or GiB suffix"
)
})?;
let factor = match cap.get(3) {
Some(suffix) => match suffix.as_str().to_lowercase().as_str() {
"b" => 1.0,
"kb" => 1_000.0,
"mb" => 1_000_000.0,
"gb" => 1_000_000_000.0,
"kib" => 1024.0,
"mib" => 1024.0 * 1024.0,
"gib" => 1024.0 * 1024.0 * 1024.0,
s => panic!("suffix `{s}` out of sync with regex"),
},
None => {
return cap[1]
.parse()
.with_context(|| format!("expected an integer size, got `{}`", &cap[1]))
}
};
let num = cap[1]
.parse::<f64>()
.with_context(|| format!("expected an integer or float, found `{}`", &cap[1]))?;
Ok((num * factor) as u64)
}
/// Performs automatic garbage collection.
///
/// This is called in various places in Cargo where garbage collection should
/// be performed automatically based on the config settings. The default
/// behavior is to only clean once a day.
///
/// This should only be called in code paths for commands that are already
/// doing a lot of work. It should only be called *after* crates are
/// downloaded so that the last-use data is updated first.
///
/// It should be cheap to call this multiple times (subsequent calls are
/// ignored), but try not to abuse that.
pub fn auto_gc(config: &Config) {
if !config.cli_unstable().gc {
return;
}
if !config.network_allowed() {
// As a conservative choice, auto-gc is disabled when offline. If the
// user is indefinitely offline, we don't want to delete things they
// may later depend on.
return;
}
if let Err(e) = auto_gc_inner(config) {
if global_cache_tracker::is_silent_error(&e)
&& config.shell().verbosity() != Verbosity::Verbose
{
tracing::warn!("failed to auto-clean cache data: {e:?}");
} else {
crate::display_warning_with_error(
"failed to auto-clean cache data",
&e,
&mut config.shell(),
);
}
}
}
fn auto_gc_inner(config: &Config) -> CargoResult<()> {
let _lock = match config.try_acquire_package_cache_lock(CacheLockMode::MutateExclusive)? {
Some(lock) => lock,
None => {
tracing::debug!("unable to acquire mutate lock, auto gc disabled");
return Ok(());
}
};
// This should not be called when there are pending deferred entries, so check that.
let deferred = config.deferred_global_last_use()?;
debug_assert!(deferred.is_empty());
let mut global_cache_tracker = config.global_cache_tracker()?;
let mut gc = Gc::new(config, &mut global_cache_tracker)?;
let mut clean_ctx = CleanContext::new(config);
gc.auto(&mut clean_ctx)?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn time_spans() {
let d = |x| Some(Duration::from_secs(x));
assert_eq!(maybe_parse_time_span("0 seconds"), d(0));
assert_eq!(maybe_parse_time_span("1second"), d(1));
assert_eq!(maybe_parse_time_span("23 seconds"), d(23));
assert_eq!(maybe_parse_time_span("5 minutes"), d(60 * 5));
assert_eq!(maybe_parse_time_span("2 hours"), d(60 * 60 * 2));
assert_eq!(maybe_parse_time_span("1 day"), d(60 * 60 * 24));
assert_eq!(maybe_parse_time_span("2 weeks"), d(60 * 60 * 24 * 14));
assert_eq!(maybe_parse_time_span("6 months"), d(60 * 60 * 24 * 30 * 6));
assert_eq!(parse_frequency("5 seconds").unwrap(), d(5));
assert_eq!(parse_frequency("always").unwrap(), d(0));
assert_eq!(parse_frequency("never").unwrap(), None);
}
#[test]
fn time_span_errors() {
assert_eq!(maybe_parse_time_span(""), None);
assert_eq!(maybe_parse_time_span("1"), None);
assert_eq!(maybe_parse_time_span("second"), None);
assert_eq!(maybe_parse_time_span("+2 seconds"), None);
assert_eq!(maybe_parse_time_span("day"), None);
assert_eq!(maybe_parse_time_span("-1 days"), None);
assert_eq!(maybe_parse_time_span("1.5 days"), None);
assert_eq!(maybe_parse_time_span("1 dayz"), None);
assert_eq!(maybe_parse_time_span("always"), None);
assert_eq!(maybe_parse_time_span("never"), None);
assert_eq!(maybe_parse_time_span("1 day "), None);
assert_eq!(maybe_parse_time_span(" 1 day"), None);
assert_eq!(maybe_parse_time_span("1 second"), None);
let e = parse_time_span_for_config("gc.auto.max-src-age", "-1 days").unwrap_err();
assert_eq!(
e.to_string(),
"config option `gc.auto.max-src-age` \
expected a value of the form \"N seconds/minutes/days/weeks/months\", \
got: \"-1 days\""
);
let e = parse_frequency("abc").unwrap_err();
assert_eq!(
e.to_string(),
"config option `gc.auto.frequency` \
expected a value of \"always\", \"never\", or \"N seconds/minutes/days/weeks/months\", \
got: \"abc\""
);
}
#[test]
fn human_sizes() {
assert_eq!(parse_human_size("0").unwrap(), 0);
assert_eq!(parse_human_size("123").unwrap(), 123);
assert_eq!(parse_human_size("123b").unwrap(), 123);
assert_eq!(parse_human_size("123B").unwrap(), 123);
assert_eq!(parse_human_size("123 b").unwrap(), 123);
assert_eq!(parse_human_size("123 B").unwrap(), 123);
assert_eq!(parse_human_size("1kb").unwrap(), 1_000);
assert_eq!(parse_human_size("5kb").unwrap(), 5_000);
assert_eq!(parse_human_size("1mb").unwrap(), 1_000_000);
assert_eq!(parse_human_size("1gb").unwrap(), 1_000_000_000);
assert_eq!(parse_human_size("1kib").unwrap(), 1_024);
assert_eq!(parse_human_size("1mib").unwrap(), 1_048_576);
assert_eq!(parse_human_size("1gib").unwrap(), 1_073_741_824);
assert_eq!(parse_human_size("1.5kb").unwrap(), 1_500);
assert_eq!(parse_human_size("1.7b").unwrap(), 1);
assert!(parse_human_size("").is_err());
assert!(parse_human_size("x").is_err());
assert!(parse_human_size("1x").is_err());
assert!(parse_human_size("1 2").is_err());
assert!(parse_human_size("1.5").is_err());
assert!(parse_human_size("+1").is_err());
assert!(parse_human_size("123 b").is_err());
}
}

1754
src/cargo/core/global_cache_tracker.rs Normal file
View File

File diff suppressed because it is too large Load Diff

2
src/cargo/core/mod.rs
View File

@ -19,6 +19,8 @@ pub use crate::util::toml::schema::InheritableFields;
pub mod compiler;
pub mod dependency;
pub mod features;
pub mod gc;
pub mod global_cache_tracker;
pub mod manifest;
pub mod package;
pub mod package_id;

4
src/cargo/core/package.rs
View File

@ -491,6 +491,10 @@ impl<'cfg> PackageSet<'cfg> {
pkgs.push(downloads.wait()?);
}
downloads.success = true;
drop(downloads);
let mut deferred = self.config.deferred_global_last_use()?;
deferred.save_no_error(self.config);
Ok(pkgs)
}

97
src/cargo/ops/cargo_clean.rs
View File

@ -1,7 +1,10 @@
use crate::core::compiler::{CompileKind, CompileMode, Layout, RustcTargetData};
use crate::core::gc::{AutoGcKind, Gc, GcOpts};
use crate::core::global_cache_tracker::GlobalCacheTracker;
use crate::core::profiles::Profiles;
use crate::core::{PackageIdSpec, TargetKind, Workspace};
use crate::ops;
use crate::util::cache_lock::CacheLockMode;
use crate::util::edit_distance;
use crate::util::errors::CargoResult;
use crate::util::interning::InternedString;
@ -25,6 +28,7 @@ pub struct CleanOptions<'cfg> {
pub doc: bool,
/// If set, doesn't delete anything.
pub dry_run: bool,
pub gc_opts: GcOpts,
}
pub struct CleanContext<'cfg> {
@ -37,45 +41,76 @@ pub struct CleanContext<'cfg> {
}
/// Cleans various caches.
pub fn clean(ws: &Workspace<'_>, opts: &CleanOptions<'_>) -> CargoResult<()> {
let mut target_dir = ws.target_dir();
pub fn clean(ws: CargoResult<Workspace<'_>>, opts: &CleanOptions<'_>) -> CargoResult<()> {
let config = opts.config;
let mut ctx = CleanContext::new(config);
ctx.dry_run = opts.dry_run;
if opts.doc {
if !opts.spec.is_empty() {
// FIXME: https://github.com/rust-lang/cargo/issues/8790
// This should support the ability to clean specific packages
// within the doc directory. It's a little tricky since it
// needs to find all documentable targets, but also consider
// the fact that target names might overlap with dependency
// names and such.
bail!("--doc cannot be used with -p");
}
// If the doc option is set, we just want to delete the doc directory.
target_dir = target_dir.join("doc");
ctx.remove_paths(&[target_dir.into_path_unlocked()])?;
} else {
let profiles = Profiles::new(&ws, opts.requested_profile)?;
let any_download_cache_opts = opts.gc_opts.is_download_cache_opt_set();
if opts.profile_specified {
// After parsing profiles we know the dir-name of the profile, if a profile
// was passed from the command line. If so, delete only the directory of
// that profile.
let dir_name = profiles.get_dir_name();
target_dir = target_dir.join(dir_name);
}
// The following options need a workspace.
let any_ws_opts = !opts.spec.is_empty()
|| !opts.targets.is_empty()
|| opts.profile_specified
|| opts.doc
|| opts.gc_opts.is_target_opt_set();
// If we have a spec, then we need to delete some packages, otherwise, just
// remove the whole target directory and be done with it!
//
// Note that we don't bother grabbing a lock here as we're just going to
// blow it all away anyway.
if opts.spec.is_empty() {
// When no options are specified, do the default action.
let no_opts_specified = !any_download_cache_opts && !any_ws_opts;
if any_ws_opts || no_opts_specified {
let ws = ws?;
let mut target_dir = ws.target_dir();
if opts.doc {
if !opts.spec.is_empty() {
// FIXME: https://github.com/rust-lang/cargo/issues/8790
// This should support the ability to clean specific packages
// within the doc directory. It's a little tricky since it
// needs to find all documentable targets, but also consider
// the fact that target names might overlap with dependency
// names and such.
bail!("--doc cannot be used with -p");
}
// If the doc option is set, we just want to delete the doc directory.
target_dir = target_dir.join("doc");
ctx.remove_paths(&[target_dir.into_path_unlocked()])?;
} else {
clean_specs(&mut ctx, &ws, &profiles, &opts.targets, &opts.spec)?;
let profiles = Profiles::new(&ws, opts.requested_profile)?;
if opts.profile_specified {
// After parsing profiles we know the dir-name of the profile, if a profile
// was passed from the command line. If so, delete only the directory of
// that profile.
let dir_name = profiles.get_dir_name();
target_dir = target_dir.join(dir_name);
}
// If we have a spec, then we need to delete some packages, otherwise, just
// remove the whole target directory and be done with it!
//
// Note that we don't bother grabbing a lock here as we're just going to
// blow it all away anyway.
if opts.spec.is_empty() {
ctx.remove_paths(&[target_dir.into_path_unlocked()])?;
} else {
clean_specs(&mut ctx, &ws, &profiles, &opts.targets, &opts.spec)?;
}
}
}
if config.cli_unstable().gc {
let _lock = config.acquire_package_cache_lock(CacheLockMode::MutateExclusive)?;
let mut cache_track = GlobalCacheTracker::new(&config)?;
let mut gc = Gc::new(config, &mut cache_track)?;
if no_opts_specified {
// This is the behavior for `cargo clean` without *any* options.
// It uses the defaults from config to determine what is cleaned.
let mut gc_opts = opts.gc_opts.clone();
gc_opts.update_for_auto_gc(config, &[AutoGcKind::All], None)?;
gc.gc(&mut ctx, &gc_opts)?;
} else {
gc.gc(&mut ctx, &opts.gc_opts)?;
}
}

1
src/cargo/ops/cargo_compile/mod.rs
View File

@ -153,6 +153,7 @@ pub fn compile_ws<'a>(
unit_graph::emit_serialized_unit_graph(&bcx.roots, &bcx.unit_graph, ws.config())?;
return Compilation::new(&bcx);
}
crate::core::gc::auto_gc(bcx.config);
let _p = profile::start("compiling");
let cx = Context::new(&bcx)?;
cx.compile(exec)

1
src/cargo/ops/cargo_fetch.rs
View File

@ -76,6 +76,7 @@ pub fn fetch<'a>(
}
packages.get_many(to_download)?;
crate::core::gc::auto_gc(config);
Ok((resolve, packages))
}

2
src/cargo/ops/mod.rs
View File

@ -1,6 +1,6 @@
use crate::sources::CRATES_IO_DOMAIN;
pub use self::cargo_clean::{clean, CleanOptions};
pub use self::cargo_clean::{clean, CleanContext, CleanOptions};
pub use self::cargo_compile::{
compile, compile_with_exec, compile_ws, create_bcx, print, resolve_all_features, CompileOptions,
};

3
src/cargo/ops/resolve.rs
View File

@ -530,6 +530,9 @@ pub fn resolve_with_previous<'cfg>(
if let Some(previous) = previous {
resolved.merge_from(previous)?;
}
let config = ws.config();
let mut deferred = config.deferred_global_last_use()?;
deferred.save_no_error(config);
Ok(resolved)
}

34
src/cargo/sources/git/source.rs
View File

@ -1,5 +1,6 @@
//! See [GitSource].
use crate::core::global_cache_tracker;
use crate::core::GitReference;
use crate::core::SourceId;
use crate::core::{Dependency, Package, PackageId, Summary};
@ -11,6 +12,7 @@ use crate::sources::PathSource;
use crate::util::cache_lock::CacheLockMode;
use crate::util::errors::CargoResult;
use crate::util::hex::short_hash;
use crate::util::interning::InternedString;
use crate::util::Config;
use anyhow::Context;
use cargo_util::paths::exclude_from_backups_and_indexing;
@ -74,9 +76,10 @@ pub struct GitSource<'cfg> {
source_id: SourceId,
/// The underlying path source to discover packages inside the Git repository.
path_source: Option<PathSource<'cfg>>,
short_id: Option<InternedString>,
/// The identifier of this source for Cargo's Git cache directory.
/// See [`ident`] for more.
ident: String,
ident: InternedString,
config: &'cfg Config,
/// Disables status messages.
quiet: bool,
@ -104,7 +107,8 @@ impl<'cfg> GitSource<'cfg> {
locked_rev,
source_id,
path_source: None,
ident,
short_id: None,
ident: ident.into(),
config,
quiet: false,
};
@ -127,6 +131,17 @@ impl<'cfg> GitSource<'cfg> {
}
self.path_source.as_mut().unwrap().read_packages()
}
fn mark_used(&self, size: Option<u64>) -> CargoResult<()> {
self.config
.deferred_global_last_use()?
.mark_git_checkout_used(global_cache_tracker::GitCheckout {
encoded_git_name: self.ident,
short_name: self.short_id.expect("update before download"),
size,
});
Ok(())
}
}
/// Create an identifier from a URL,
@ -200,6 +215,7 @@ impl<'cfg> Source for GitSource<'cfg> {
fn block_until_ready(&mut self) -> CargoResult<()> {
if self.path_source.is_some() {
self.mark_used(None)?;
return Ok(());
}
@ -290,8 +306,19 @@ impl<'cfg> Source for GitSource<'cfg> {
let path_source = PathSource::new_recursive(&checkout_path, source_id, self.config);
self.path_source = Some(path_source);
self.short_id = Some(short_id.as_str().into());
self.locked_rev = Some(actual_rev);
self.path_source.as_mut().unwrap().update()
self.path_source.as_mut().unwrap().update()?;
// Hopefully this shouldn't incur too much of a performance hit since
// most of this should already be in cache since it was just
// extracted.
//
// !.git is used because clones typically use hardlinks for the git
// contents. TODO: Verify behavior on Windows.
let size = cargo_util::du(&checkout_path, &["!.git"])?;
self.mark_used(Some(size))?;
Ok(())
}
fn download(&mut self, id: PackageId) -> CargoResult<MaybePackage> {
@ -300,6 +327,7 @@ impl<'cfg> Source for GitSource<'cfg> {
id,
self.remote
);
self.mark_used(None)?;
self.path_source
.as_mut()
.expect("BUG: `update()` must be called before `get()`")

18
src/cargo/sources/registry/download.rs
View File

@ -3,11 +3,13 @@
//! [`HttpRegistry`]: super::http_remote::HttpRegistry
//! [`RemoteRegistry`]: super::remote::RemoteRegistry
use crate::util::interning::InternedString;
use anyhow::Context;
use cargo_credential::Operation;
use cargo_util::registry::make_dep_path;
use cargo_util::Sha256;
use crate::core::global_cache_tracker;
use crate::core::PackageId;
use crate::sources::registry::MaybeLock;
use crate::sources::registry::RegistryConfig;
@ -34,6 +36,7 @@ const CHECKSUM_TEMPLATE: &str = "{sha256-checksum}";
pub(super) fn download(
cache_path: &Filesystem,
config: &Config,
encoded_registry_name: InternedString,
pkg: PackageId,
checksum: &str,
registry_config: RegistryConfig,
@ -50,6 +53,13 @@ pub(super) fn download(
if let Ok(dst) = File::open(path) {
let meta = dst.metadata()?;
if meta.len() > 0 {
config.deferred_global_last_use()?.mark_registry_crate_used(
global_cache_tracker::RegistryCrate {
encoded_registry_name,
crate_filename: pkg.tarball_name().into(),
size: meta.len(),
},
);
return Ok(MaybeLock::Ready(dst));
}
}
@ -106,6 +116,7 @@ pub(super) fn download(
pub(super) fn finish_download(
cache_path: &Filesystem,
config: &Config,
encoded_registry_name: InternedString,
pkg: PackageId,
checksum: &str,
data: &[u8],
@ -115,6 +126,13 @@ pub(super) fn finish_download(
if actual != checksum {
anyhow::bail!("failed to verify the checksum of `{}`", pkg)
}
config.deferred_global_last_use()?.mark_registry_crate_used(
global_cache_tracker::RegistryCrate {
encoded_registry_name,
crate_filename: pkg.tarball_name().into(),
size: data.len() as u64,
},
);
cache_path.create_dir()?;
let path = cache_path.join(&pkg.tarball_name());

19
src/cargo/sources/registry/http_remote.rs
View File

@ -1,11 +1,13 @@
//! Access to a HTTP-based crate registry. See [`HttpRegistry`] for details.
use crate::core::global_cache_tracker;
use crate::core::{PackageId, SourceId};
use crate::sources::registry::download;
use crate::sources::registry::MaybeLock;
use crate::sources::registry::{LoadResponse, RegistryConfig, RegistryData};
use crate::util::cache_lock::CacheLockMode;
use crate::util::errors::{CargoResult, HttpNotSuccessful};
use crate::util::interning::InternedString;
use crate::util::network::http::http_handle;
use crate::util::network::retry::{Retry, RetryResult};
use crate::util::network::sleep::SleepTracker;
@ -52,6 +54,7 @@ const UNKNOWN: &'static str = "Unknown";
///
/// [RFC 2789]: https://github.com/rust-lang/rfcs/pull/2789
pub struct HttpRegistry<'cfg> {
name: InternedString,
/// Path to the registry index (`$CARGO_HOME/registry/index/$REG-HASH`).
///
/// To be fair, `HttpRegistry` doesn't store the registry index it
@ -199,6 +202,7 @@ impl<'cfg> HttpRegistry<'cfg> {
.expect("a url with the sparse+ stripped should still be valid");
Ok(HttpRegistry {
name: name.into(),
index_path: config.registry_index_path().join(name),
cache_path: config.registry_cache_path().join(name),
source_id,
@ -454,6 +458,11 @@ impl<'cfg> HttpRegistry<'cfg> {
impl<'cfg> RegistryData for HttpRegistry<'cfg> {
fn prepare(&self) -> CargoResult<()> {
self.config
.deferred_global_last_use()?
.mark_registry_index_used(global_cache_tracker::RegistryIndex {
encoded_registry_name: self.name,
});
Ok(())
}
@ -750,6 +759,7 @@ impl<'cfg> RegistryData for HttpRegistry<'cfg> {
download::download(
&self.cache_path,
&self.config,
self.name.clone(),
pkg,
checksum,
registry_config,
@ -762,7 +772,14 @@ impl<'cfg> RegistryData for HttpRegistry<'cfg> {
checksum: &str,
data: &[u8],
) -> CargoResult<File> {
download::finish_download(&self.cache_path, &self.config, pkg, checksum, data)
download::finish_download(
&self.cache_path,
&self.config,
self.name.clone(),
pkg,
checksum,
data,
)
}
fn is_crate_downloaded(&self, pkg: PackageId) -> bool {

20
src/cargo/sources/registry/mod.rs
View File

@ -201,6 +201,7 @@ use tar::Archive;
use tracing::debug;
use crate::core::dependency::Dependency;
use crate::core::global_cache_tracker;
use crate::core::{Package, PackageId, SourceId, Summary};
use crate::sources::source::MaybePackage;
use crate::sources::source::QueryKind;
@ -239,6 +240,7 @@ struct LockMetadata {
///
/// For general concepts of registries, see the [module-level documentation](crate::sources::registry).
pub struct RegistrySource<'cfg> {
name: InternedString,
/// The unique identifier of this source.
source_id: SourceId,
/// The path where crate files are extracted (`$CARGO_HOME/registry/src/$REG-HASH`).
@ -514,6 +516,7 @@ impl<'cfg> RegistrySource<'cfg> {
yanked_whitelist: &HashSet<PackageId>,
) -> RegistrySource<'cfg> {
RegistrySource {
name: name.into(),
src_path: config.registry_source_path().join(name),
config,
source_id,
@ -589,6 +592,13 @@ impl<'cfg> RegistrySource<'cfg> {
match fs::read_to_string(path) {
Ok(ok) => match serde_json::from_str::<LockMetadata>(&ok) {
Ok(lock_meta) if lock_meta.v == 1 => {
self.config
.deferred_global_last_use()?
.mark_registry_src_used(global_cache_tracker::RegistrySrc {
encoded_registry_name: self.name,
package_dir: package_dir.into(),
size: None,
});
return Ok(unpack_dir.to_path_buf());
}
_ => {
@ -613,6 +623,7 @@ impl<'cfg> RegistrySource<'cfg> {
set_mask(&mut tar);
tar
};
let mut bytes_written = 0;
let prefix = unpack_dir.file_name().unwrap();
let parent = unpack_dir.parent().unwrap();
for entry in tar.entries()? {
@ -644,6 +655,7 @@ impl<'cfg> RegistrySource<'cfg> {
continue;
}
// Unpacking failed
bytes_written += entry.size();
let mut result = entry.unpack_in(parent).map_err(anyhow::Error::from);
if cfg!(windows) && restricted_names::is_windows_reserved_path(&entry_path) {
result = result.with_context(|| {
@ -670,6 +682,14 @@ impl<'cfg> RegistrySource<'cfg> {
let lock_meta = LockMetadata { v: 1 };
write!(ok, "{}", serde_json::to_string(&lock_meta).unwrap())?;
self.config
.deferred_global_last_use()?
.mark_registry_src_used(global_cache_tracker::RegistrySrc {
encoded_registry_name: self.name,
package_dir: package_dir.into(),
size: Some(bytes_written),
});
Ok(unpack_dir.to_path_buf())
}

18
src/cargo/sources/registry/remote.rs
View File

@ -1,5 +1,6 @@
//! Access to a Git index based registry. See [`RemoteRegistry`] for details.
use crate::core::global_cache_tracker;
use crate::core::{GitReference, PackageId, SourceId};
use crate::sources::git;
use crate::sources::git::fetch::RemoteKind;
@ -47,6 +48,7 @@ use tracing::{debug, trace};
///
/// [`HttpRegistry`]: super::http_remote::HttpRegistry
pub struct RemoteRegistry<'cfg> {
name: InternedString,
/// Path to the registry index (`$CARGO_HOME/registry/index/$REG-HASH`).
index_path: Filesystem,
/// Path to the cache of `.crate` files (`$CARGO_HOME/registry/cache/$REG-HASH`).
@ -87,6 +89,7 @@ impl<'cfg> RemoteRegistry<'cfg> {
/// registry index are stored. Expect to be unique.
pub fn new(source_id: SourceId, config: &'cfg Config, name: &str) -> RemoteRegistry<'cfg> {
RemoteRegistry {
name: name.into(),
index_path: config.registry_index_path().join(name),
cache_path: config.registry_cache_path().join(name),
source_id,
@ -211,6 +214,11 @@ impl<'cfg> RemoteRegistry<'cfg> {
impl<'cfg> RegistryData for RemoteRegistry<'cfg> {
fn prepare(&self) -> CargoResult<()> {
self.repo()?;
self.config
.deferred_global_last_use()?
.mark_registry_index_used(global_cache_tracker::RegistryIndex {
encoded_registry_name: self.name,
});
Ok(())
}
@ -403,6 +411,7 @@ impl<'cfg> RegistryData for RemoteRegistry<'cfg> {
download::download(
&self.cache_path,
&self.config,
self.name,
pkg,
checksum,
registry_config,
@ -415,7 +424,14 @@ impl<'cfg> RegistryData for RemoteRegistry<'cfg> {
checksum: &str,
data: &[u8],
) -> CargoResult<File> {
download::finish_download(&self.cache_path, &self.config, pkg, checksum, data)
download::finish_download(
&self.cache_path,
&self.config,
self.name.clone(),
pkg,
checksum,
data,
)
}
fn is_crate_downloaded(&self, pkg: PackageId) -> bool {

24
src/cargo/util/config/mod.rs
View File

@ -68,6 +68,7 @@ use std::time::Instant;
use self::ConfigValue as CV;
use crate::core::compiler::rustdoc::RustdocExternMap;
use crate::core::global_cache_tracker::{DeferredGlobalLastUse, GlobalCacheTracker};
use crate::core::shell::Verbosity;
use crate::core::{features, CliUnstable, Shell, SourceId, Workspace, WorkspaceRootConfig};
use crate::ops::RegistryCredentialConfig;
@ -244,6 +245,8 @@ pub struct Config {
pub nightly_features_allowed: bool,
/// WorkspaceRootConfigs that have been found
pub ws_roots: RefCell<HashMap<PathBuf, WorkspaceRootConfig>>,
global_cache_tracker: LazyCell<RefCell<GlobalCacheTracker>>,
deferred_global_last_use: LazyCell<RefCell<DeferredGlobalLastUse>>,
}
impl Config {
@ -317,6 +320,8 @@ impl Config {
env_config: LazyCell::new(),
nightly_features_allowed: matches!(&*features::channel(), "nightly" | "dev"),
ws_roots: RefCell::new(HashMap::new()),
global_cache_tracker: LazyCell::new(),
deferred_global_last_use: LazyCell::new(),
}
}
@ -1919,6 +1924,25 @@ impl Config {
) -> CargoResult<Option<CacheLock<'_>>> {
self.package_cache_lock.try_lock(self, mode)
}
/// Returns a reference to the shared [`GlobalCacheTracker`].
///
/// The package cache lock must be held to call this function (and to use
/// it in general).
pub fn global_cache_tracker(&self) -> CargoResult<RefMut<'_, GlobalCacheTracker>> {
let tracker = self.global_cache_tracker.try_borrow_with(|| {
Ok::<_, anyhow::Error>(RefCell::new(GlobalCacheTracker::new(self)?))
})?;
Ok(tracker.borrow_mut())
}
/// Returns a reference to the shared [`DeferredGlobalLastUse`].
pub fn deferred_global_last_use(&self) -> CargoResult<RefMut<'_, DeferredGlobalLastUse>> {
let deferred = self.deferred_global_last_use.try_borrow_with(|| {
Ok::<_, anyhow::Error>(RefCell::new(DeferredGlobalLastUse::new()))
})?;
Ok(deferred.borrow_mut())
}
}
/// Internal error for serde errors.

14
tests/testsuite/clean.rs
View File

@ -1,5 +1,6 @@
//! Tests for the `cargo clean` command.
use cargo_test_support::paths::CargoPathExt;
use cargo_test_support::registry::Package;
use cargo_test_support::{
basic_bin_manifest, basic_manifest, git, main_file, project, project_in, rustc_host,
@ -805,15 +806,6 @@ fn clean_dry_run() {
.file("src/lib.rs", "")
.build();
let ls_r = || -> Vec<_> {
let mut file_list: Vec<_> = walkdir::WalkDir::new(p.build_dir())
.into_iter()
.filter_map(|e| e.map(|e| e.path().to_owned()).ok())
.collect();
file_list.sort();
file_list
};
// Start with no files.
p.cargo("clean --dry-run")
.with_stdout("")
@ -823,7 +815,7 @@ fn clean_dry_run() {
)
.run();
p.cargo("check").run();
let before = ls_r();
let before = p.build_dir().ls_r();
p.cargo("clean --dry-run")
.with_stderr(
"[SUMMARY] [..] files, [..] total\n\
@ -831,7 +823,7 @@ fn clean_dry_run() {
)
.run();
// Verify it didn't delete anything.
let after = ls_r();
let after = p.build_dir().ls_r();
assert_eq!(before, after);
let expected = cargo::util::iter_join(before.iter().map(|p| p.to_str().unwrap()), "\n");
eprintln!("{expected}");

1890
tests/testsuite/global_cache_tracker.rs Normal file
View File

File diff suppressed because it is too large Load Diff

1
tests/testsuite/main.rs
View File

@ -98,6 +98,7 @@ mod git_auth;
mod git_gc;
mod git_shallow;
mod glob_targets;
mod global_cache_tracker;
mod help;
mod https;
mod inheritable_workspace_fields;