ErrorNoInternet/r34-scraper
1
0
Fork 0
forked from danmax/r34-scraper
Code Pull Requests Activity

feat: misc improvements

Browse Source
This commit is contained in:
javalsai 2024-10-19 21:37:31 +02:00
parent ace4ac2811
commit 022352b554
Signed by: javalsai
SSH Key Fingerprint: SHA256:3G83yKhBUWVABVX/vPWH88xnK4+ptMtHkZGCRXD4Mk8
2 changed files with 18 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/args/mod.rs
View File

@ -1,6 +1,6 @@
use clap::Parser;
#[derive(Parser)]
#[derive(Debug, Parser)]
#[command(version)]
pub struct Args {
/// User Agent to use for requests
@ -17,4 +17,13 @@ pub struct Args {
/// Async jobs to use for fetching
#[arg(short, long, default_value = "4")]
pub jobs: usize,
/// Delay for rate-limits (ms)
#[arg(short, long, default_value = "1000", value_parser = parse_duration)]
pub delay: std::time::Duration,
}
fn parse_duration(arg: &str) -> Result<std::time::Duration, std::num::ParseIntError> {
let seconds = arg.parse()?;
Ok(std::time::Duration::from_millis(seconds))
}

13
src/main.rs
View File

@ -89,25 +89,28 @@ async fn main() -> ExitCode {
async move {
// "thread"
loop {
this_prog.set_message(format!("\x1b[30m[{i}/{urls_ammount}] \x1b[36mscraping {url:?}\x1b[0m"));
this_prog.set_message(format!("\x1b[30m[{i: >4}/{urls_ammount}] \x1b[36mscraping {url:?}\x1b[0m"));
let resp = client.get(&url).send().await.unwrap();
match extract_img_url(&resp.text().await.unwrap()) {
Ok(img_url) => {
if img_url.is_empty() {
this_prog.abandon_with_message(format!(
"\x1b[30m[{i}/{urls_ammount}] \x1b[1;31mimage url not found\x1b[0m"
"\x1b[30m[{i: >4}/{urls_ammount}] \x1b[1;31mimage url not found\x1b[0m"
));
} else {
this_prog.finish_with_message(format!(
"\x1b[30m[{i}/{urls_ammount}] \x1b[32mfound image url: {img_url}\x1b[0m"
"\x1b[30m[{i: >4}/{urls_ammount}] \x1b[32mfound image url: {img_url}\x1b[0m"
));
}
break img_url;
}
Err(_) => {
this_prog
.set_message(format!("\x1b[30m[{i}/{urls_ammount}] \x1b[31mratelimited, retrying after 1 second\x1b[0m"));
tokio::time::sleep(std::time::Duration::from_millis(1000)).await;
.set_message(format!(
"\x1b[30m[{i: >4}/{urls_ammount}] \x1b[31mratelimited, retrying after {}ms\x1b[0m",
args.delay.as_millis())
);
tokio::time::sleep(args.delay).await;
continue;
}
}