Compare commits
	
		
			6 Commits
		
	
	
		
			08ed5e51f2
			...
			javalsai-c
		
	
	| Author | SHA1 | Date | |
|---|---|---|---|
| 
						
						
							
						
						a852c8bcc5
	
				 | 
					
					
						|||
| 
						
						
							
						
						235e13230b
	
				 | 
					
					
						|||
| 
						 | 
					bd517ed0b5 | ||
| 5f848be434 | |||
| 
						
						
							
						
						8723769429
	
				 | 
					
					
						|||
| 
						
						
							
						
						bcd349e36f
	
				 | 
					
					
						
@@ -1,6 +1,6 @@
 | 
				
			|||||||
[package]
 | 
					[package]
 | 
				
			||||||
name = "r34-scraper"
 | 
					name = "r34-scraper"
 | 
				
			||||||
version = "0.1.0"
 | 
					version = "1.0.0"
 | 
				
			||||||
edition = "2021"
 | 
					edition = "2021"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
[dependencies]
 | 
					[dependencies]
 | 
				
			||||||
 
 | 
				
			|||||||
							
								
								
									
										2
									
								
								rust-toolchain.toml
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										2
									
								
								rust-toolchain.toml
									
									
									
									
									
										Normal file
									
								
							@@ -0,0 +1,2 @@
 | 
				
			|||||||
 | 
					[toolchain]
 | 
				
			||||||
 | 
					channel = "nightly"
 | 
				
			||||||
@@ -11,8 +11,8 @@ pub struct Args {
 | 
				
			|||||||
    pub user_agent: String,
 | 
					    pub user_agent: String,
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    /// Tags to search for
 | 
					    /// Tags to search for
 | 
				
			||||||
    #[arg(short, long)]
 | 
					    #[arg(short, long, required = true)]
 | 
				
			||||||
    pub tags: Option<Vec<String>>,
 | 
					    pub tags: Vec<String>,
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    /// Page to start scraping from
 | 
					    /// Page to start scraping from
 | 
				
			||||||
    #[arg(short, long, default_value_t = 1)]
 | 
					    #[arg(short, long, default_value_t = 1)]
 | 
				
			||||||
 
 | 
				
			|||||||
							
								
								
									
										26
									
								
								src/main.rs
									
									
									
									
									
								
							
							
						
						
									
										26
									
								
								src/main.rs
									
									
									
									
									
								
							@@ -1,4 +1,4 @@
 | 
				
			|||||||
#![feature(async_closure, iter_intersperse)]
 | 
					#![feature(async_closure)]
 | 
				
			||||||
pub mod args;
 | 
					pub mod args;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
use clap::Parser;
 | 
					use clap::Parser;
 | 
				
			||||||
@@ -17,22 +17,8 @@ const BAR_LENGTH: u64 = 8;
 | 
				
			|||||||
async fn main() -> ExitCode {
 | 
					async fn main() -> ExitCode {
 | 
				
			||||||
    let args = args::Args::parse();
 | 
					    let args = args::Args::parse();
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    let tags = args.tags.unwrap_or_else(|| {
 | 
					    let uri_tags = &args.tags.join("+");
 | 
				
			||||||
        println!("which tags do you want to scrape? ex: 1girls 1boys yomama");
 | 
					    let _ = std::fs::create_dir(uri_tags);
 | 
				
			||||||
        let tags_binding = std::io::stdin().lines().next().unwrap().unwrap();
 | 
					 | 
				
			||||||
        tags_binding
 | 
					 | 
				
			||||||
            .split(' ')
 | 
					 | 
				
			||||||
            .filter(|item| !item.is_empty())
 | 
					 | 
				
			||||||
            .map(std::borrow::ToOwned::to_owned)
 | 
					 | 
				
			||||||
            .collect()
 | 
					 | 
				
			||||||
    });
 | 
					 | 
				
			||||||
    let tags_folder = &tags.join("+");
 | 
					 | 
				
			||||||
    let uri_tags = tags
 | 
					 | 
				
			||||||
        .into_iter()
 | 
					 | 
				
			||||||
        .intersperse(String::from("+"))
 | 
					 | 
				
			||||||
        .collect::<String>();
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
    let _ = std::fs::create_dir(tags_folder);
 | 
					 | 
				
			||||||
 | 
					
 | 
				
			||||||
    let client = Client::builder()
 | 
					    let client = Client::builder()
 | 
				
			||||||
        .user_agent(&args.user_agent)
 | 
					        .user_agent(&args.user_agent)
 | 
				
			||||||
@@ -103,7 +89,7 @@ async fn main() -> ExitCode {
 | 
				
			|||||||
                                    "\x1b[37m[{i: >4}/{urls_amount}] \x1b[1;31mimage url not found\x1b[0m"
 | 
					                                    "\x1b[37m[{i: >4}/{urls_amount}] \x1b[1;31mimage url not found\x1b[0m"
 | 
				
			||||||
                                ));
 | 
					                                ));
 | 
				
			||||||
                            } else {
 | 
					                            } else {
 | 
				
			||||||
                                download_file(&img_url, this_bar, i, urls_amount, tags_folder).await;
 | 
					                                download_file(&img_url, this_bar, i, urls_amount, uri_tags).await;
 | 
				
			||||||
                            }
 | 
					                            }
 | 
				
			||||||
                            break;
 | 
					                            break;
 | 
				
			||||||
                        }
 | 
					                        }
 | 
				
			||||||
@@ -151,7 +137,7 @@ async fn download_file(
 | 
				
			|||||||
    this_bar: ProgressBar,
 | 
					    this_bar: ProgressBar,
 | 
				
			||||||
    i: usize,
 | 
					    i: usize,
 | 
				
			||||||
    urls_amount: usize,
 | 
					    urls_amount: usize,
 | 
				
			||||||
    tags_folder: &str,
 | 
					    uri_tags: &str,
 | 
				
			||||||
) {
 | 
					) {
 | 
				
			||||||
    let args = args::Args::parse();
 | 
					    let args = args::Args::parse();
 | 
				
			||||||
 | 
					
 | 
				
			||||||
@@ -161,7 +147,7 @@ async fn download_file(
 | 
				
			|||||||
        .map(|m| m.as_str())
 | 
					        .map(|m| m.as_str())
 | 
				
			||||||
        .unwrap();
 | 
					        .unwrap();
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    let file_path = tags_folder.to_owned() + "/" + file_name;
 | 
					    let file_path = uri_tags.to_owned() + "/" + file_name;
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    let mut file = if std::fs::File::open(&file_path).is_ok() {
 | 
					    let mut file = if std::fs::File::open(&file_path).is_ok() {
 | 
				
			||||||
        this_bar.finish_with_message(format!(
 | 
					        this_bar.finish_with_message(format!(
 | 
				
			||||||
 
 | 
				
			|||||||
		Reference in New Issue
	
	Block a user