feat: add robots txt
All checks were successful
create archive with lfs / tag (push) Successful in 9s
All checks were successful
create archive with lfs / tag (push) Successful in 9s
This commit is contained in:
parent
8ede972994
commit
6041a544d6
@ -22,6 +22,12 @@ export async function load({ request, cookies, url, setHeaders, fetch }) {
|
||||
})
|
||||
}).catch((why) => {
|
||||
console.log("failed sending dark visitors analytics:", why)
|
||||
return null
|
||||
}).then(async (resp) => {
|
||||
if (resp !== null) {
|
||||
const msg = await resp.json()
|
||||
console.log(`sent visitor analytic to dark visitors: ${resp.statusText}; ${msg.message}`)
|
||||
}
|
||||
})
|
||||
|
||||
let currentVisitCount = get(visitCount)
|
||||
|
39
src/routes/robots.txt/+server.ts
Normal file
39
src/routes/robots.txt/+server.ts
Normal file
@ -0,0 +1,39 @@
|
||||
import { env } from '$env/dynamic/private';
|
||||
import { get, writable } from 'svelte/store';
|
||||
|
||||
const cachedRobots = writable<string>("")
|
||||
const lastFetched = writable<number>(Date.now())
|
||||
|
||||
const fetchRobotsTxt = async () => {
|
||||
const robotsTxtResp = await fetch(
|
||||
"https://api.darkvisitors.com/robots-txts",
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Authorization": `Bearer ${env.DARK_VISITORS_TOKEN}`,
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
body: JSON.stringify({
|
||||
agent_types: [
|
||||
"AI Assistant",
|
||||
"AI Data Scraper",
|
||||
"AI Search Crawler",
|
||||
"Undocumented AI Agent",
|
||||
],
|
||||
disallow: "/"
|
||||
})
|
||||
}
|
||||
)
|
||||
const robotsTxt = await robotsTxtResp.text()
|
||||
lastFetched.set(Date.now())
|
||||
return robotsTxt
|
||||
}
|
||||
|
||||
export const GET = async ({ }) => {
|
||||
let robotsTxt = get(cachedRobots)
|
||||
if (robotsTxt.length === 0 || Date.now() - get(lastFetched) > 1000 * 60 * 60 * 24) {
|
||||
robotsTxt = await fetchRobotsTxt()
|
||||
cachedRobots.set(robotsTxt)
|
||||
}
|
||||
return new Response(robotsTxt)
|
||||
}
|
Loading…
Reference in New Issue
Block a user