commit bfd26159040a19eefdaace1d78429e4e208060e3 from: Aleksey Ryndin date: Wed Jul 31 19:41:48 2024 UTC Add: robots.txt as argument commit - 918775082bad843cff6efef7994454cc97767ddb commit + bfd26159040a19eefdaace1d78429e4e208060e3 blob - 8c87e9f97e0c2a478ebecff850954564c353630e blob + 7b7c51ca53e37e8e1775d573877725ed468bddf6 --- Makefile +++ Makefile @@ -1,3 +1,4 @@ .PHONY: run run: - python3 yah2g.py --header static/index.html.header --footer static/index.html.footer --icon static/favicon.ico --css static/style.css + python3 yah2g.py --header static/index.html.header --footer static/index.html.footer \ + --icon static/favicon.ico --css static/style.css --robots static/robots.txt blob - /dev/null blob + 1f53798bb4fe33c86020be7f10c44f29486fd190 (mode 644) --- /dev/null +++ static/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / blob - 4301e4885ba5445cceb56b0ebebb958f5de4a74f blob + 5532c1d6830d929ed7671b742d49de219f8a342d --- yah2g.py +++ yah2g.py @@ -43,6 +43,7 @@ class _HTTPServer(HTTPServer): footer_file_path=None, icon_file_path=None, css_file_path=None, + robots_file_path=None, **kwargs ): super().__init__(*args, **kwargs) @@ -54,6 +55,8 @@ class _HTTPServer(HTTPServer): self.icon_file_bytes = f.read() with open(css_file_path, "rb") as f: self.css_file_bytes = f.read() + with open(robots_file_path, "rb") as f: + self.robots_file_bytes = f.read() class _Elem: @@ -183,10 +186,9 @@ class _RequestHandler(BaseHTTPRequestHandler): self.send_response(HTTPStatus.OK) self.send_header("Content-type", "text/plain") self.end_headers() - self.wfile.write(b"User-agent: *\nDisallow: /") + self.wfile.write(self.server.robots_file_bytes) return - self.send_error(HTTPStatus.NOT_FOUND, "File not found") def _convert_gemini_to_html(self, url, body, mime): @@ -252,6 +254,7 @@ def _main(): parser.add_argument("--footer", required=True, help="path to `index.html.footer`") parser.add_argument("--icon", required=True , help="path to `favicon.ico`") parser.add_argument("--css", required=True, help="path to `style.css`") + parser.add_argument("--robots", required=True, help="path to `robots.txt`") args = parser.parse_args() with _HTTPServer( @@ -261,6 +264,7 @@ def _main(): footer_file_path=args.footer, icon_file_path=args.icon, css_file_path=args.css, + robots_file_path=args.robots, ) as http_server: sock_host, sock_port = http_server.socket.getsockname()[:2] print(f"HTTP server started ({sock_host}:{sock_port})...")