add safebooru support

This commit is contained in:
dusk 2023-07-17 13:58:47 +03:00
parent 099c02c5ef
commit 13e7465c88
Signed by: dusk
SSH Key Fingerprint: SHA256:Abmvag+juovVufZTxyWY8KcVgrznxvBjQpJesv071Aw
5 changed files with 43 additions and 2 deletions

2
Cargo.lock generated
View File

@ -445,9 +445,11 @@ dependencies = [
"axum",
"dashmap",
"fastrand",
"form_urlencoded",
"http",
"maud",
"reqwest",
"serde_json",
"signal-hook",
"tokio",
]

View File

@ -8,7 +8,9 @@ axum = {git = "https://github.com/tokio-rs/axum.git", version = "0.6"}
tokio = {version = "1", features = ["rt-multi-thread", "macros"]}
http = "0.2"
fastrand = {version = "2", features = ["std"]}
reqwest = {version = "0.11", default-features = false, features = ["rustls-tls-native-roots"]}
reqwest = {version = "0.11", default-features = false, features = ["rustls-tls-native-roots", "json"]}
dashmap = "5"
maud = "0.25"
signal-hook = "0.3"
serde_json = "1"
form_urlencoded = "1"

View File

@ -21,6 +21,7 @@ type AppResult<T> = Result<T, AppError>;
#[derive(Clone)]
enum ArtKind {
Twitter,
Safebooru,
}
impl FromStr for ArtKind {
@ -29,6 +30,7 @@ impl FromStr for ArtKind {
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"twitter.com" => Ok(Self::Twitter),
"safebooru.org" => Ok(Self::Safebooru),
_ => Err("not support website".into()),
}
}
@ -161,6 +163,7 @@ async fn show_art(state: State<AppState>) -> AppResult<axum::response::Response>
} else {
let image_link = match art.kind {
ArtKind::Twitter => fetch_twitter_image_link(&state.http, &art.url).await?,
ArtKind::Safebooru => fetch_safebooru_image_link(&state.http, &art.url).await?,
};
state
.direct_links
@ -199,6 +202,30 @@ fn render_page(art: &Art, image_link: &str) -> Html<String> {
Html(content.into_string())
}
async fn fetch_safebooru_image_link(http: &reqwest::Client, url: &Uri) -> AppResult<String> {
let mut id = String::new();
for (name, value) in form_urlencoded::parse(url.query().unwrap().as_bytes()) {
if name == "id" {
id = value.into_owned();
}
}
if id.is_empty() {
return Err("no id?".into());
}
let url = format!("https://safebooru.org/index.php?page=dapi&s=post&q=index&json=1&id={id}");
let req = http.get(url).build()?;
let resp = http.execute(req).await?.error_for_status()?;
let data: Vec<serde_json::Map<String, serde_json::Value>> = resp.json().await?;
let image_filename = data[0].get("image").unwrap().as_str().unwrap();
let image_directory = data[0].get("directory").unwrap().as_str().unwrap();
Ok(format!(
"http://safebooru.org/images/{image_directory}/{image_filename}"
))
}
async fn fetch_twitter_image_link(http: &reqwest::Client, url: &Uri) -> AppResult<String> {
let fxurl = Uri::builder()
.scheme("https")

9
utils/safebooru-download.sh Executable file
View File

@ -0,0 +1,9 @@
#!/usr/bin/env bash
touch all.json
for i in $(seq 0 38); do
curl "https://safebooru.org/index.php?page=dapi&s=post&q=index&tags=project_moon&limit=100&pid=$i&json=1" > temp.json
jq -sr '. | add' temp.json all.json > all.json.temp
mv all.json.temp all.json
rm temp.json
done

1
utils/twitter-filter.sh Normal file
View File

@ -0,0 +1 @@
jq -r '.messages[] | select(.embeds[0].thumbnail.url | if type == "string" then test("video") else false end | not) | .content | select(test("//twitter.com")) | select(test("@") | not) | select(test("\\|\\|") | not) | select(test("^(https?|ftp):\\/\\/[a-z0-9-]+(\\.[a-z0-9-]+)*(\\.[a-z]{2,})(:[0-9]+)?(\\/[^ \\t\\r\\n]*)?$")) | split("/") | .[:6] | join("/") | split("?") | .[0]' limb.json > test.txt