fix: possibly fix safebooru retry
This commit is contained in:
parent
96dd41c42b
commit
a8ea85f971
SSH Key Fingerprint:
SHA256:Abmvag+juovVufZTxyWY8KcVgrznxvBjQpJesv071Aw
56
Cargo.lock
generated
56
Cargo.lock
generated
@ -211,6 +211,21 @@ dependencies = [
|
||||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-executor",
|
||||
"futures-io",
|
||||
"futures-sink",
|
||||
"futures-task",
|
||||
"futures-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.30"
|
||||
@ -218,6 +233,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -226,6 +242,23 @@ version = "0.3.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"
|
||||
|
||||
[[package]]
|
||||
name = "futures-executor"
|
||||
version = "0.3.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-task",
|
||||
"futures-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-io"
|
||||
version = "0.3.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
|
||||
|
||||
[[package]]
|
||||
name = "futures-macro"
|
||||
version = "0.3.30"
|
||||
@ -237,6 +270,23 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-retry"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fde5a672a61f96552aa5ed9fd9c81c3fbdae4be9b1e205d6eaf17c83705adc0f"
|
||||
dependencies = [
|
||||
"futures",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-sink"
|
||||
version = "0.3.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5"
|
||||
|
||||
[[package]]
|
||||
name = "futures-task"
|
||||
version = "0.3.30"
|
||||
@ -249,9 +299,13 @@ version = "0.3.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
"futures-macro",
|
||||
"futures-sink",
|
||||
"futures-task",
|
||||
"memchr",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
"slab",
|
||||
@ -560,7 +614,7 @@ dependencies = [
|
||||
"dashmap",
|
||||
"fastrand",
|
||||
"form_urlencoded",
|
||||
"futures-util",
|
||||
"futures-retry",
|
||||
"http",
|
||||
"maud",
|
||||
"reqwest",
|
||||
|
||||
@ -14,4 +14,4 @@ maud = "0.26"
|
||||
signal-hook = "0.3"
|
||||
serde_json = "1"
|
||||
form_urlencoded = "1"
|
||||
futures-util = "0.3"
|
||||
futures-retry = "0.6"
|
||||
38
src/main.rs
38
src/main.rs
@ -7,7 +7,6 @@ use axum::{
|
||||
use dashmap::DashMap;
|
||||
use data::{Art, ArtKind, Data};
|
||||
use error::AppResult;
|
||||
use futures_util::TryFutureExt;
|
||||
use http::Uri;
|
||||
use maud::PreEscaped;
|
||||
use std::{
|
||||
@ -123,20 +122,31 @@ async fn fetch_safebooru_image_link(http: &reqwest::Client, url: &Uri) -> AppRes
|
||||
|
||||
let url = format!("https://safebooru.org/index.php?page=dapi&s=post&q=index&json=1&id={id}");
|
||||
type Data = Vec<serde_json::Map<String, serde_json::Value>>;
|
||||
async fn try_request(count: usize, url: &str, http: &reqwest::Client) -> AppResult<Data> {
|
||||
println!("[safebooru] trying to fetch url (count {count}): {url}");
|
||||
let req = http.get(url).build()?;
|
||||
let resp = http.execute(req).await?.error_for_status()?;
|
||||
let data = resp.json::<Data>().await?;
|
||||
AppResult::Ok(data)
|
||||
}
|
||||
let try_request = || {
|
||||
let url = url.clone();
|
||||
let http = http.clone();
|
||||
async move {
|
||||
println!("[safebooru] trying to fetch url (count 0): {url}");
|
||||
let req = http.get(url).build()?;
|
||||
let resp = http.execute(req).await?.error_for_status()?;
|
||||
let data = resp.json::<Data>().await?;
|
||||
AppResult::Ok(data)
|
||||
}
|
||||
};
|
||||
|
||||
let data = try_request(0, &url, http)
|
||||
.or_else(|_| try_request(1, &url, http))
|
||||
.or_else(|_| try_request(2, &url, http))
|
||||
.or_else(|_| try_request(3, &url, http))
|
||||
.or_else(|_| try_request(4, &url, http))
|
||||
.await?;
|
||||
let mut attempts: usize = 0;
|
||||
let (data, _) = futures_retry::FutureRetry::new(try_request, |e| {
|
||||
if attempts > 4 {
|
||||
futures_retry::RetryPolicy::<error::AppError>::ForwardError(e)
|
||||
} else {
|
||||
attempts += 1;
|
||||
futures_retry::RetryPolicy::<error::AppError>::WaitRetry(
|
||||
std::time::Duration::from_secs(1),
|
||||
)
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|(e, _)| e)?;
|
||||
|
||||
let image_filename = data[0].get("image").unwrap().as_str().unwrap();
|
||||
let image_directory = data[0].get("directory").unwrap().as_str().unwrap();
|
||||
|
||||
Loading…
Reference in New Issue
Block a user