Compare commits
No commits in common. "722df4305d4d611df561154717890f21b742c826" and "815c08cc361119f0e0186c86b58d26acf691e347" have entirely different histories.
722df4305d
...
815c08cc36
@ -21,6 +21,7 @@ async fn main() -> ExitCode {
|
||||
let mut page = 0;
|
||||
|
||||
loop {
|
||||
|
||||
println!("now scraping page {}", page + 1);
|
||||
|
||||
let post_html = async || {
|
||||
@ -44,6 +45,7 @@ async fn main() -> ExitCode {
|
||||
let mut wait_time = 5000;
|
||||
|
||||
if urls.is_empty() {
|
||||
|
||||
for reconnection_attempts in 0..4 {
|
||||
println!("no urls found, retrying in {} seconds...", wait_time / 1000);
|
||||
sleep(Duration::from_millis(wait_time)).await;
|
||||
@ -65,6 +67,7 @@ async fn main() -> ExitCode {
|
||||
}
|
||||
|
||||
for url in urls {
|
||||
|
||||
let img_url =
|
||||
extract_img_url(&client.get(url).send().await.unwrap().text().await.unwrap());
|
||||
if img_url.is_empty() {
|
||||
@ -88,7 +91,7 @@ fn extract_urls(html: &str) -> Vec<String> {
|
||||
|
||||
fn extract_img_url(html: &str) -> String {
|
||||
if let Some(img_url) =
|
||||
Regex::new(r"https://us\.rule34\.xxx/images/([A-Za-z0-9]+(/[A-Za-z0-9]+)+)\.[A-Za-z0-9]+")
|
||||
Regex::new(r"https://us\.rule34\.xxx//([A-Za-z0-9]+(/[A-Za-z0-9]+)+)\.[A-Za-z0-9]+")
|
||||
.unwrap()
|
||||
.find(html)
|
||||
{
|
||||
|
Loading…
x
Reference in New Issue
Block a user