README.md

@@ -4,7 +4,7 @@ a scraper that well scrapes r34
 
 ## note 
 
-this program is pretty much complete, although i might add a few extra features.
+this program is pretty much complete, although i am planning to add a few extra features.
 
 ## example usage image
 ![example image](./image.png)

src/args.rs

@@ -22,9 +22,6 @@ pub struct Args {
     #[arg(short, long, default_value_t = 4)]
     pub jobs: usize,
 
-    #[arg(long, default_value = "downloads")]
-    pub dir: String,
-
     /// Delay for rate-limits (ms)
     #[arg(short, long, default_value = "1000", value_parser = parse_duration)]
     pub delay: std::time::Duration,

src/main.rs

@@ -1,3 +1,4 @@
+#![feature(async_closure)]
 pub mod args;
 
 use clap::Parser;
@@ -19,8 +20,7 @@ async fn main() -> ExitCode {
     let args = args::Args::parse();
 
     let uri_tags = &args.tags.join("+");
-    let dir = &args.dir;
-    let _ = std::fs::create_dir(dir);
+    let _ = std::fs::create_dir(uri_tags);
 
     let running = Arc::new(AtomicBool::new(true));
     let running_t = running.clone();
@@ -100,14 +100,14 @@ async fn main() -> ExitCode {
                         }
 
                         this_bar.set_message(format!("\x1b[37m[{i: >4}/{urls_amount}] \x1b[36mscraping {url}\x1b[0m"));
-                        let resp = client.get(&url).send().await.unwrap().text().await.unwrap();
-                        if let Ok(img_url) = extract_img_url(&resp) {
+                        let resp = client.get(&url).send().await.unwrap();
+                        if let Ok(img_url) = extract_img_url(&resp.text().await.unwrap()) {
                             if img_url.is_empty() {
                                 this_bar.abandon_with_message(format!(
                                     "\x1b[37m[{i: >4}/{urls_amount}] \x1b[1;31mimage url not found\x1b[0m"
                                 ));
                             } else {
-                                download_file(running_t, &img_url, this_bar, i, urls_amount, &resp, dir.as_str()).await;
+                                download_file(running_t, &img_url, this_bar, i, urls_amount, uri_tags).await;
                             }
                             break;
                         }
@@ -122,7 +122,7 @@ async fn main() -> ExitCode {
                 }
             })
             .buffered(args.jobs);
-        let () = responses.for_each(|()| async {}).await;
+        let _ = responses.for_each(|()| async {}).await;
     }
 
     ExitCode::SUCCESS
@@ -136,23 +136,6 @@ fn extract_urls(html: &str) -> Vec<String> {
         .collect()
 }
 
-fn write_to_database(html: &str, file_path: &str) {
-    let strings = html.split('\n');
-    for line in strings {
-        if line.contains("<title>") {
-            let line = line.trim_start();
-            let mut file = std::fs::OpenOptions::new()
-                .create(true)                
-                .append(true)
-                .open("database")
-                .unwrap();
-
-            writeln!(file, "{file_path}: {line}").unwrap();
-            break;
-        }
-    }
-}
-
 fn extract_img_url(html: &str) -> Result<String, &'static str> {
     if let Some(img_url) =
         Regex::new(r"https://us\.rule34\.xxx//images/([A-Za-z0-9]+(/[A-Za-z0-9]+)+)\.[A-Za-z0-9]+")
@@ -173,8 +156,7 @@ async fn download_file(
     this_bar: ProgressBar,
     i: usize,
     urls_amount: usize,
-    html: &str,
-    dir: &str,
+    uri_tags: &str,
 ) {
     let args = args::Args::parse();
 
@@ -184,7 +166,7 @@ async fn download_file(
         .map(|m| m.as_str())
         .unwrap();
 
-    let file_path = dir.to_owned() + "/" + file_name;
+    let file_path = uri_tags.to_owned() + "/" + file_name;
 
     let mut file = if std::fs::File::open(&file_path).is_ok() {
         this_bar.finish_with_message(format!(
@@ -216,7 +198,6 @@ async fn download_file(
         written += img_chunk.len();
         this_bar.set_position((written as f64 / file_length as f64 * BAR_LENGTH as f64) as u64);
     }
-    write_to_database(html, &file_path);
     this_bar.finish_with_message(format!(
         "\x1b[37m[{i: >4}/{urls_amount}] \x1b[32mdownloaded {img_url}\x1b[0m"
     ));