fix: possibly fix safebooru retry

This commit is contained in:
dusk 2024-06-11 05:04:28 +03:00
parent 96dd41c42b
commit a8ea85f971
Signed by: dusk
SSH Key Fingerprint: SHA256:Abmvag+juovVufZTxyWY8KcVgrznxvBjQpJesv071Aw
3 changed files with 80 additions and 16 deletions

56
Cargo.lock generated
View File

@ -211,6 +211,21 @@ dependencies = [
"percent-encoding", "percent-encoding",
] ]
[[package]]
name = "futures"
version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0"
dependencies = [
"futures-channel",
"futures-core",
"futures-executor",
"futures-io",
"futures-sink",
"futures-task",
"futures-util",
]
[[package]] [[package]]
name = "futures-channel" name = "futures-channel"
version = "0.3.30" version = "0.3.30"
@ -218,6 +233,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-sink",
] ]
[[package]] [[package]]
@ -226,6 +242,23 @@ version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"
[[package]]
name = "futures-executor"
version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d"
dependencies = [
"futures-core",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-io"
version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
[[package]] [[package]]
name = "futures-macro" name = "futures-macro"
version = "0.3.30" version = "0.3.30"
@ -237,6 +270,23 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "futures-retry"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fde5a672a61f96552aa5ed9fd9c81c3fbdae4be9b1e205d6eaf17c83705adc0f"
dependencies = [
"futures",
"pin-project-lite",
"tokio",
]
[[package]]
name = "futures-sink"
version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5"
[[package]] [[package]]
name = "futures-task" name = "futures-task"
version = "0.3.30" version = "0.3.30"
@ -249,9 +299,13 @@ version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"
dependencies = [ dependencies = [
"futures-channel",
"futures-core", "futures-core",
"futures-io",
"futures-macro", "futures-macro",
"futures-sink",
"futures-task", "futures-task",
"memchr",
"pin-project-lite", "pin-project-lite",
"pin-utils", "pin-utils",
"slab", "slab",
@ -560,7 +614,7 @@ dependencies = [
"dashmap", "dashmap",
"fastrand", "fastrand",
"form_urlencoded", "form_urlencoded",
"futures-util", "futures-retry",
"http", "http",
"maud", "maud",
"reqwest", "reqwest",

View File

@ -14,4 +14,4 @@ maud = "0.26"
signal-hook = "0.3" signal-hook = "0.3"
serde_json = "1" serde_json = "1"
form_urlencoded = "1" form_urlencoded = "1"
futures-util = "0.3" futures-retry = "0.6"

View File

@ -7,7 +7,6 @@ use axum::{
use dashmap::DashMap; use dashmap::DashMap;
use data::{Art, ArtKind, Data}; use data::{Art, ArtKind, Data};
use error::AppResult; use error::AppResult;
use futures_util::TryFutureExt;
use http::Uri; use http::Uri;
use maud::PreEscaped; use maud::PreEscaped;
use std::{ use std::{
@ -123,20 +122,31 @@ async fn fetch_safebooru_image_link(http: &reqwest::Client, url: &Uri) -> AppRes
let url = format!("https://safebooru.org/index.php?page=dapi&s=post&q=index&json=1&id={id}"); let url = format!("https://safebooru.org/index.php?page=dapi&s=post&q=index&json=1&id={id}");
type Data = Vec<serde_json::Map<String, serde_json::Value>>; type Data = Vec<serde_json::Map<String, serde_json::Value>>;
async fn try_request(count: usize, url: &str, http: &reqwest::Client) -> AppResult<Data> { let try_request = || {
println!("[safebooru] trying to fetch url (count {count}): {url}"); let url = url.clone();
let req = http.get(url).build()?; let http = http.clone();
let resp = http.execute(req).await?.error_for_status()?; async move {
let data = resp.json::<Data>().await?; println!("[safebooru] trying to fetch url (count 0): {url}");
AppResult::Ok(data) let req = http.get(url).build()?;
} let resp = http.execute(req).await?.error_for_status()?;
let data = resp.json::<Data>().await?;
AppResult::Ok(data)
}
};
let data = try_request(0, &url, http) let mut attempts: usize = 0;
.or_else(|_| try_request(1, &url, http)) let (data, _) = futures_retry::FutureRetry::new(try_request, |e| {
.or_else(|_| try_request(2, &url, http)) if attempts > 4 {
.or_else(|_| try_request(3, &url, http)) futures_retry::RetryPolicy::<error::AppError>::ForwardError(e)
.or_else(|_| try_request(4, &url, http)) } else {
.await?; attempts += 1;
futures_retry::RetryPolicy::<error::AppError>::WaitRetry(
std::time::Duration::from_secs(1),
)
}
})
.await
.map_err(|(e, _)| e)?;
let image_filename = data[0].get("image").unwrap().as_str().unwrap(); let image_filename = data[0].get("image").unwrap().as_str().unwrap();
let image_directory = data[0].get("directory").unwrap().as_str().unwrap(); let image_directory = data[0].get("directory").unwrap().as_str().unwrap();