commit bfd26159040a19eefdaace1d78429e4e208060e3
from: Aleksey Ryndin <continue@to.any-key.press>
date: Wed Jul 31 19:41:48 2024 UTC

Add: robots.txt as argument

commit - 918775082bad843cff6efef7994454cc97767ddb
commit + bfd26159040a19eefdaace1d78429e4e208060e3
blob - 8c87e9f97e0c2a478ebecff850954564c353630e
blob + 7b7c51ca53e37e8e1775d573877725ed468bddf6
--- Makefile
+++ Makefile
@@ -1,3 +1,4 @@
 .PHONY: run
 run:
-	python3 yah2g.py --header static/index.html.header --footer static/index.html.footer --icon static/favicon.ico --css static/style.css
+	python3 yah2g.py --header static/index.html.header --footer static/index.html.footer \
+			 --icon static/favicon.ico --css static/style.css --robots static/robots.txt
blob - /dev/null
blob + 1f53798bb4fe33c86020be7f10c44f29486fd190 (mode 644)
--- /dev/null
+++ static/robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /
blob - 4301e4885ba5445cceb56b0ebebb958f5de4a74f
blob + 5532c1d6830d929ed7671b742d49de219f8a342d
--- yah2g.py
+++ yah2g.py
@@ -43,6 +43,7 @@ class _HTTPServer(HTTPServer):
         footer_file_path=None,
         icon_file_path=None,
         css_file_path=None,
+        robots_file_path=None,
         **kwargs
     ):
         super().__init__(*args, **kwargs)
@@ -54,6 +55,8 @@ class _HTTPServer(HTTPServer):
             self.icon_file_bytes = f.read()
         with open(css_file_path, "rb") as f:
             self.css_file_bytes = f.read()
+        with open(robots_file_path, "rb") as f:
+            self.robots_file_bytes = f.read()
 
 
 class _Elem:
@@ -183,10 +186,9 @@ class _RequestHandler(BaseHTTPRequestHandler):
             self.send_response(HTTPStatus.OK)
             self.send_header("Content-type", "text/plain")
             self.end_headers()
-            self.wfile.write(b"User-agent: *\nDisallow: /")
+            self.wfile.write(self.server.robots_file_bytes)
             return
 
-
         self.send_error(HTTPStatus.NOT_FOUND, "File not found")
 
     def _convert_gemini_to_html(self, url, body, mime):
@@ -252,6 +254,7 @@ def _main():
     parser.add_argument("--footer", required=True, help="path to `index.html.footer`")
     parser.add_argument("--icon", required=True , help="path to `favicon.ico`")
     parser.add_argument("--css", required=True, help="path to `style.css`")
+    parser.add_argument("--robots", required=True, help="path to `robots.txt`")
     args = parser.parse_args()
 
     with _HTTPServer(
@@ -261,6 +264,7 @@ def _main():
         footer_file_path=args.footer,
         icon_file_path=args.icon,
         css_file_path=args.css,
+        robots_file_path=args.robots,
     ) as http_server:
         sock_host, sock_port = http_server.socket.getsockname()[:2]
         print(f"HTTP server started ({sock_host}:{sock_port})...")