Revert "Merge pull request 'Update gitignore, add taap, luv gaylord' (#5) from UwU/not-r34:main into main"
This reverts commit 9744c81ef58d3d81215a2a70a3daab79b021b9e6, reversing changes made to eeee4f50b36a14a5a11b1204b031909ded3a9918.
This commit is contained in:
parent
9744c81ef5
commit
6831aaf6ff
1
.gitignore
vendored
1
.gitignore
vendored
@ -1,2 +1 @@
|
|||||||
/target
|
/target
|
||||||
/Cargo.lock
|
|
||||||
|
1344
Cargo.lock
generated
Normal file
1344
Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
@ -6,5 +6,4 @@ edition = "2021"
|
|||||||
[dependencies]
|
[dependencies]
|
||||||
regex = "1.11.0"
|
regex = "1.11.0"
|
||||||
reqwest = { version = "0.12.8", features = ["blocking"] }
|
reqwest = { version = "0.12.8", features = ["blocking"] }
|
||||||
taap = "0.1.4"
|
|
||||||
tokio = { version = "1", features = ["full"] }
|
tokio = { version = "1", features = ["full"] }
|
||||||
|
@ -1,2 +0,0 @@
|
|||||||
[toolchain]
|
|
||||||
channel = "nightly"
|
|
54
src/main.rs
54
src/main.rs
@ -2,32 +2,20 @@
|
|||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use reqwest::Client;
|
use reqwest::Client;
|
||||||
use std::process::ExitCode;
|
use std::process::ExitCode;
|
||||||
use taap::Argument;
|
|
||||||
use tokio::time::{sleep, Duration};
|
use tokio::time::{sleep, Duration};
|
||||||
|
|
||||||
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36";
|
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36";
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> ExitCode {
|
async fn main() -> ExitCode {
|
||||||
// Taap setup
|
println!("which tags do you want to scrape? ex: 1girls+1boys+yomama");
|
||||||
let mut arguments = Argument::new(
|
let tags = std::io::stdin()
|
||||||
"r34-scrape",
|
.lines()
|
||||||
"A scraper for r34.xxx",
|
.next()
|
||||||
"Users love this tool! Hear our reviews down below:\n\"It has never been easier to find what I love!\" - penguinlover\n\"This has made my life way easier!\" - FurryUser69\n\"Best tool I've ever used\" - Sean Combs\n",
|
.unwrap()
|
||||||
"Danmax and authors 2024",
|
.unwrap()
|
||||||
);
|
.trim()
|
||||||
|
.to_string();
|
||||||
arguments.add_arg("TAGS", "+", Some("the tags you want to search for"));
|
|
||||||
let parsed_arguments = arguments.parse_args(None);
|
|
||||||
|
|
||||||
let tags = parsed_arguments.get("TAGS").unwrap();
|
|
||||||
|
|
||||||
// End of taap setup
|
|
||||||
// Check if empty and warn
|
|
||||||
// Can't use tags.0 because taap is not buggy at all :3
|
|
||||||
if tags.1.is_empty() {
|
|
||||||
println!("[warning] No tags were used, use --help for help")
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut thread_counter = 0;
|
let mut thread_counter = 0;
|
||||||
let client = Client::builder().user_agent(USER_AGENT).build().unwrap();
|
let client = Client::builder().user_agent(USER_AGENT).build().unwrap();
|
||||||
@ -40,12 +28,7 @@ async fn main() -> ExitCode {
|
|||||||
extract_urls(
|
extract_urls(
|
||||||
&client
|
&client
|
||||||
.get(format!(
|
.get(format!(
|
||||||
"https://rule34.xxx/index.php?page=post&s=list{}&pid={}",
|
"https://rule34.xxx/index.php?page=post&s=list&tags={tags}&pid={}",
|
||||||
if tags.0 {
|
|
||||||
format!("&tags={}", tags.1.join("+"))
|
|
||||||
} else {
|
|
||||||
"".to_owned()
|
|
||||||
},
|
|
||||||
page * 42
|
page * 42
|
||||||
))
|
))
|
||||||
.send()
|
.send()
|
||||||
@ -87,23 +70,14 @@ async fn main() -> ExitCode {
|
|||||||
let thread_id = format!("[{thread_counter: >4}]");
|
let thread_id = format!("[{thread_counter: >4}]");
|
||||||
loop {
|
loop {
|
||||||
let client = Client::builder().user_agent(USER_AGENT).build().unwrap();
|
let client = Client::builder().user_agent(USER_AGENT).build().unwrap();
|
||||||
match extract_img_url(
|
match extract_img_url(&client.get(url.clone()).send().await.unwrap().text().await.unwrap()) {
|
||||||
&client
|
|
||||||
.get(url.clone())
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.text()
|
|
||||||
.await
|
|
||||||
.unwrap(),
|
|
||||||
) {
|
|
||||||
Ok(img_url) => {
|
Ok(img_url) => {
|
||||||
if img_url.is_empty() {
|
if img_url.is_empty() {
|
||||||
println!("{thread_id} image url not found");
|
println!("{thread_id} image url not found");
|
||||||
} else {
|
} else {
|
||||||
println!("{thread_id} found image url: {img_url}");
|
println!("{thread_id} found image url: {img_url}");
|
||||||
}
|
}
|
||||||
}
|
},
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
println!("{thread_id} ratelimited, retrying after 1 second");
|
println!("{thread_id} ratelimited, retrying after 1 second");
|
||||||
std::thread::sleep(std::time::Duration::from_millis(1000));
|
std::thread::sleep(std::time::Duration::from_millis(1000));
|
||||||
@ -117,11 +91,7 @@ async fn main() -> ExitCode {
|
|||||||
if thread_counter > 9999 {
|
if thread_counter > 9999 {
|
||||||
thread_counter = 0;
|
thread_counter = 0;
|
||||||
}
|
}
|
||||||
while tokio::runtime::Handle::current()
|
while tokio::runtime::Handle::current().metrics().num_alive_tasks() > 4 {
|
||||||
.metrics()
|
|
||||||
.num_alive_tasks()
|
|
||||||
> 4
|
|
||||||
{
|
|
||||||
std::thread::sleep(std::time::Duration::from_millis(100));
|
std::thread::sleep(std::time::Duration::from_millis(100));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user