commit 34e08cc3735f02ba8d42e1476b3d23623d1ad5df
from: Aleksey Ryndin <continue@to.any-key.press>
date: Thu Aug 01 07:55:25 2024 UTC

Hot fix: ban facebook*/meta* crawlers

commit - bfd26159040a19eefdaace1d78429e4e208060e3
commit + 34e08cc3735f02ba8d42e1476b3d23623d1ad5df
blob - 5532c1d6830d929ed7671b742d49de219f8a342d
blob + 8de977b6fb8791fa480c8443e0ede31d6c9cb938
--- yah2g.py
+++ yah2g.py
@@ -107,6 +107,12 @@ class _RequestHandler(BaseHTTPRequestHandler):
         return path, parse_qs(query) if query else {}
 
     def do_GET(self):
+        user_agent = self.headers.get("User-Agent", "")
+        for prefix in ("facebook", "meta", ):
+            if user_agent.startswith(prefix):
+                self.send_error(HTTPStatus.FORBIDDEN, "Crawlers are not allowed (see robots.txt)")
+                return
+
         path, query = self._parse_path()
         if path in {"/index.html", "/index.htm", "/index", "/"}:
             url = query.get("url", [None])[0]