-
Notifications
You must be signed in to change notification settings - Fork 8
/
Copy pathcrates.rs
187 lines (167 loc) · 7.44 KB
/
crates.rs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
//! **crates** subcommand focus on the sync crates index and crate files from upstream. The core
//! function implemented in the `src/crates/index`.
//!
//! **crates** subcommand provide major functions include:
//!
//! Arguments:
//! - __domain__: you can choose your own upstream by adding this argument in command,
//! this param can be changed in the configuration file or pass it here
//! - __download-threads__: specify the download threads to parallel download,
//! this param can be changed in the configuration file or pass it here
//! - __no-progressbar__: Whether to hide progress bar when start downloading
//!
//! # pull subcommand
//!
//! sync crates index from upstream to local:
//!
//! - The crates index is a git repository, and **cargo** clone and update from [GitHub](https://github.com/rust-lang/crates.io-index).
//! - The clone use `bare` mode, more details in the [cargo guide](https://github.com/rust-lang/cargo/blob/6b6b0b486d73c03ed952591d880debec1d47c534/src/doc/src/guide/cargo-home.md#directories)
//!
//! # download subcommand
//! sync crate file from upstream to local:
//!
//! - The crate file of upstream location detail from [crates.io-index](https://github.com/rust-lang/crates.io-index/blob/master/.github/workflows/update-dl-url.yml)
//! ```YAML
//! env:
//! URL_api: "https://crates.io/api/v1/crates"
//! URL_cdn: "https://static.crates.io/crates/sync{crate}/{crate}-{version}.crate"
//! URL_s3_primary: "https://crates-io.s3-us-west-1.amazonaws.com/crates/{crate}/{crate}-{version}.crate"
//! URL_s3_fallback: "https://crates-io-fallback.s3-eu-west-1.amazonaws.com/crates/{crate}/{crate}-{version}.crate"
//! ```
//!
//! Arguments:
//! - __init__: Whether to download all the crates files for initialization.
//! - __upload__: Whether to upload single file to s3 after download success.
//! - __bucket__: set the s3 bucket you want to upload files to, you must provide this param before upload.
//! - __delete-after-upload__: This optional parameter will be used to delete files after upload.
//!
//! # upload subcommand
//!
//! - Sync crate file to Object Storage Service compatible with [AWS S3](https://aws.amazon.com/s3/)
//! - Digitalocean Spaces
//! - Huawei Cloud OBS
//! - Alibaba Cloud OSS
//! - Tencent Cloud COS
//! - AWS S3
//! - minio
//! - Ceph
//! Arguments:
//! - __bucket__: set the s3 bucket you want to upload files to, you must provide this param before upload.
//!
use std::sync::Arc;
use clap::{arg, ArgMatches};
use rayon::ThreadPoolBuilder;
use crate::commands::command_prelude::*;
use crate::config::Config;
use crate::errors::FreightResult;
use crate::handler::crates_file::{download, upload_to_s3, CratesOptions};
use crate::handler::index::{pull, CrateIndex};
use crate::handler::DownloadMode;
/// The __crates__ subcommand
///
pub fn cli() -> clap::Command {
clap::Command::new("crates")
.arg(flag("no-progressbar", "Hide progressbar when start sync"))
.arg(arg!(-t --"download-threads" <VALUE> "specify the download threads to parallel download,
this param can be changed in the configuration file or pass it here")
.value_parser(value_parser!(usize))
)
.arg(arg!(-d --"domain" <VALUE> "specify the source you want to sync from,
this param can be changed in the configuration file or pass it here"))
.subcommand(subcommand("pull"))
.subcommand(subcommand("upload")
.arg(arg!(-b --"bucket" <VALUE> "set the s3 bucket name you want to upload files").required(true))
.arg(arg!(--"name" <VALUE> "only upload specify crates"))
)
.subcommand(subcommand("download")
.arg(flag("init", "Start init download of crates file, this will traverse all index for full download"))
.arg(flag("fix", "Handle the crates file that download failed, this opetion will traverse error log"))
.arg(arg!(--"name" <VALUE> "only fix the crates you specified, this command will try to re-download the crates"))
.arg(flag("upload", "upload every crate file after download"))
.arg(arg!(-b --"bucket" <VALUE> "set the s3 bucket name you want to upload files"))
.arg(flag("delete-after-upload", "this will delete file after upload"))
)
.subcommand_required(true)
.arg_required_else_help(true)
.about("Sync the crates from the upstream(crates.io) to the local registry")
.help_template(
"\
Sync the crates index and crate files from the upstream(crates.io) to the local filesystem, other cloud
storage services, or other registries.
USAGE:
{usage}
OPTIONS:
{options}
EXAMPLES
1. Sync the crates index with specify config path
freighter -c /mnt/freighter/config.toml crates pull
2. Download all crates file and unload:
freighter crates download --init --upload --bucket crates
3. Download crates file with multi-thread:
freighter crates -t 32 download --init
\n")
}
///
///
///
pub fn exec(config: &mut Config, args: &ArgMatches) -> FreightResult {
crate::cli::init_log(&config.log, &config.log_path, "crates").unwrap();
let opts = &mut CratesOptions {
config: config.crates.to_owned(),
proxy: config.proxy.to_owned(),
index: CrateIndex::new(&config.crates.index_domain, config.index_path.to_owned()),
no_progressbar: args.get_flag("no-progressbar"),
crates_path: config.crates_path.to_owned(),
log_path: config.log_path.to_owned(),
..Default::default()
};
let domain = args.get_one::<String>("domain").cloned();
match args.get_one::<usize>("download-threads").cloned() {
Some(download_threads) => opts.config.download_threads = download_threads,
None => tracing::info!("use default thread count: {}", opts.config.download_threads),
};
opts.thread_pool = Arc::new(
ThreadPoolBuilder::new()
.num_threads(opts.config.download_threads)
.build()
.unwrap(),
);
tracing::info!("CratesOptions info : {:#?}", opts);
match args.subcommand() {
Some(("pull", _args)) => {
if let Some(source) = domain {
config.crates.index_domain = source;
}
pull(opts)?
}
Some(("download", args)) => {
opts.upload = args.get_flag("upload");
opts.download_mode = DownloadMode::new(args.get_flag("init"), args.get_flag("fix"));
opts.delete_after_upload = args.get_flag("delete-after-upload");
opts.crates_name = args.get_one::<String>("name").cloned();
if opts.upload {
if let Some(bucket_name) = args.get_one::<String>("bucket").cloned() {
opts.bucket_name = bucket_name
} else {
unreachable!("can not upload with empty bucket name")
}
}
if let Some(source) = domain {
config.crates.domain = source;
}
download(opts)?
}
Some(("upload", args)) => {
opts.bucket_name = args.get_one::<String>("bucket").cloned().unwrap();
opts.crates_name = args.get_one::<String>("name").cloned();
upload_to_s3(opts)?
}
Some((cmd, _)) => {
unreachable!("unexpected command {}", cmd)
}
None => {
unreachable!("unexpected command")
}
};
Ok(())
}