Revert "Merge pull request 'Update gitignore, add taap, luv gaylord' (#5) from UwU/not-r34:main into main"

This reverts commit 9744c81ef58d3d81215a2a70a3daab79b021b9e6, reversing
changes made to eeee4f50b36a14a5a11b1204b031909ded3a9918.
This commit is contained in:
danmax 2024-10-19 13:24:44 -04:00
parent 9744c81ef5
commit 6831aaf6ff
5 changed files with 1357 additions and 47 deletions

1
.gitignore vendored
View File

@ -1,2 +1 @@
/target /target
/Cargo.lock

1344
Cargo.lock generated Normal file

File diff suppressed because it is too large Load Diff

View File

@ -6,5 +6,4 @@ edition = "2021"
[dependencies] [dependencies]
regex = "1.11.0" regex = "1.11.0"
reqwest = { version = "0.12.8", features = ["blocking"] } reqwest = { version = "0.12.8", features = ["blocking"] }
taap = "0.1.4"
tokio = { version = "1", features = ["full"] } tokio = { version = "1", features = ["full"] }

View File

@ -1,2 +0,0 @@
[toolchain]
channel = "nightly"

View File

@ -2,32 +2,20 @@
use regex::Regex; use regex::Regex;
use reqwest::Client; use reqwest::Client;
use std::process::ExitCode; use std::process::ExitCode;
use taap::Argument;
use tokio::time::{sleep, Duration}; use tokio::time::{sleep, Duration};
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"; const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36";
#[tokio::main] #[tokio::main]
async fn main() -> ExitCode { async fn main() -> ExitCode {
// Taap setup println!("which tags do you want to scrape? ex: 1girls+1boys+yomama");
let mut arguments = Argument::new( let tags = std::io::stdin()
"r34-scrape", .lines()
"A scraper for r34.xxx", .next()
"Users love this tool! Hear our reviews down below:\n\"It has never been easier to find what I love!\" - penguinlover\n\"This has made my life way easier!\" - FurryUser69\n\"Best tool I've ever used\" - Sean Combs\n", .unwrap()
"Danmax and authors 2024", .unwrap()
); .trim()
.to_string();
arguments.add_arg("TAGS", "+", Some("the tags you want to search for"));
let parsed_arguments = arguments.parse_args(None);
let tags = parsed_arguments.get("TAGS").unwrap();
// End of taap setup
// Check if empty and warn
// Can't use tags.0 because taap is not buggy at all :3
if tags.1.is_empty() {
println!("[warning] No tags were used, use --help for help")
}
let mut thread_counter = 0; let mut thread_counter = 0;
let client = Client::builder().user_agent(USER_AGENT).build().unwrap(); let client = Client::builder().user_agent(USER_AGENT).build().unwrap();
@ -40,12 +28,7 @@ async fn main() -> ExitCode {
extract_urls( extract_urls(
&client &client
.get(format!( .get(format!(
"https://rule34.xxx/index.php?page=post&s=list{}&pid={}", "https://rule34.xxx/index.php?page=post&s=list&tags={tags}&pid={}",
if tags.0 {
format!("&tags={}", tags.1.join("+"))
} else {
"".to_owned()
},
page * 42 page * 42
)) ))
.send() .send()
@ -87,23 +70,14 @@ async fn main() -> ExitCode {
let thread_id = format!("[{thread_counter: >4}]"); let thread_id = format!("[{thread_counter: >4}]");
loop { loop {
let client = Client::builder().user_agent(USER_AGENT).build().unwrap(); let client = Client::builder().user_agent(USER_AGENT).build().unwrap();
match extract_img_url( match extract_img_url(&client.get(url.clone()).send().await.unwrap().text().await.unwrap()) {
&client
.get(url.clone())
.send()
.await
.unwrap()
.text()
.await
.unwrap(),
) {
Ok(img_url) => { Ok(img_url) => {
if img_url.is_empty() { if img_url.is_empty() {
println!("{thread_id} image url not found"); println!("{thread_id} image url not found");
} else { } else {
println!("{thread_id} found image url: {img_url}"); println!("{thread_id} found image url: {img_url}");
} }
} },
Err(_) => { Err(_) => {
println!("{thread_id} ratelimited, retrying after 1 second"); println!("{thread_id} ratelimited, retrying after 1 second");
std::thread::sleep(std::time::Duration::from_millis(1000)); std::thread::sleep(std::time::Duration::from_millis(1000));
@ -117,11 +91,7 @@ async fn main() -> ExitCode {
if thread_counter > 9999 { if thread_counter > 9999 {
thread_counter = 0; thread_counter = 0;
} }
while tokio::runtime::Handle::current() while tokio::runtime::Handle::current().metrics().num_alive_tasks() > 4 {
.metrics()
.num_alive_tasks()
> 4
{
std::thread::sleep(std::time::Duration::from_millis(100)); std::thread::sleep(std::time::Duration::from_millis(100));
} }
} }