danmax/r34-scraper
1
3
Fork 3
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

javalsai-changes #6

Merged
danmax merged 9 commits from javalsai-changes into main 2024-10-19 21:39:41 +02:00
Conversation 8 Commits 9 Files Changed 5 +18 -6
2 changed files with 18 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 022352b554 - Show all commits

11
src/args/mod.rs
View File

@ -1,6 +1,6 @@
use clap::Parser; use clap::Parser;
#[derive(Parser)] #[derive(Debug, Parser)]
#[command(version)] #[command(version)]
pub struct Args { pub struct Args {
/// User Agent to use for requests /// User Agent to use for requests
@ -17,4 +17,13 @@ pub struct Args {
/// Async jobs to use for fetching /// Async jobs to use for fetching
#[arg(short, long, default_value = "4")] #[arg(short, long, default_value = "4")]
pub jobs: usize, pub jobs: usize,
/// Delay for rate-limits (ms)
#[arg(short, long, default_value = "1000", value_parser = parse_duration)]
pub delay: std::time::Duration,
}
fn parse_duration(arg: &str) -> Result<std::time::Duration, std::num::ParseIntError> {
let seconds = arg.parse()?;
Ok(std::time::Duration::from_millis(seconds))
} }

13
src/main.rs
View File

@ -89,25 +89,28 @@ async fn main() -> ExitCode {
async move { async move {
// "thread" // "thread"
loop { loop {
this_prog.set_message(format!("\x1b[30m[{i}/{urls_ammount}] \x1b[36mscraping {url:?}\x1b[0m")); this_prog.set_message(format!("\x1b[30m[{i: >4}/{urls_ammount}] \x1b[36mscraping {url:?}\x1b[0m"));
let resp = client.get(&url).send().await.unwrap(); let resp = client.get(&url).send().await.unwrap();
match extract_img_url(&resp.text().await.unwrap()) { match extract_img_url(&resp.text().await.unwrap()) {
Ok(img_url) => { Ok(img_url) => {
if img_url.is_empty() { if img_url.is_empty() {
this_prog.abandon_with_message(format!( this_prog.abandon_with_message(format!(
"\x1b[30m[{i}/{urls_ammount}] \x1b[1;31mimage url not found\x1b[0m" "\x1b[30m[{i: >4}/{urls_ammount}] \x1b[1;31mimage url not found\x1b[0m"
)); ));
} else { } else {
this_prog.finish_with_message(format!( this_prog.finish_with_message(format!(
"\x1b[30m[{i}/{urls_ammount}] \x1b[32mfound image url: {img_url}\x1b[0m" "\x1b[30m[{i: >4}/{urls_ammount}] \x1b[32mfound image url: {img_url}\x1b[0m"
)); ));
} }
break img_url; break img_url;
} }
Err(_) => { Err(_) => {
this_prog this_prog
.set_message(format!("\x1b[30m[{i}/{urls_ammount}] \x1b[31mratelimited, retrying after 1 second\x1b[0m")); .set_message(format!(
tokio::time::sleep(std::time::Duration::from_millis(1000)).await; "\x1b[30m[{i: >4}/{urls_ammount}] \x1b[31mratelimited, retrying after {}ms\x1b[0m",
args.delay.as_millis())
);
tokio::time::sleep(args.delay).await;
continue; continue;
} }
} }