commit 4acbf994ae02d25a0bf1246afa8a910382a03bff from: Aleksey Ryndin date: Wed Sep 25 17:09:29 2024 UTC Add: work-in-porgress commit - 0fc414424b202ec7078b7198e6420dab05116cb9 commit + 4acbf994ae02d25a0bf1246afa8a910382a03bff blob - 094a44912c71f2bb2f0a53bce6107146911397a9 blob + 61b7bd6bc9df99a108fca291d18a79aeacf0bfb4 --- lonk.py +++ lonk.py @@ -4,8 +4,9 @@ from json import loads as json_loads from mimetypes import guess_type from os import environ from pathlib import Path -from sys import stdout +from sys import stdout, stderr from sqlite3 import connect as sqlite3_connect +from time import clock_gettime, CLOCK_MONOTONIC from urllib.error import HTTPError, URLError from urllib.parse import urlencode, urlunsplit, urljoin, urlsplit, parse_qsl, unquote, quote from urllib.request import urlopen @@ -239,11 +240,12 @@ def _proxy_url_factory(proxy_media_enabled, lonk_url): def demo(lonk_url, honk_url): collected = {} + banned = set() home = honk_url.do_get(action="gethonks", page="home") for honk in reversed(home["honks"]): convoy = honk["Convoy"] - if convoy in collected: + if convoy in collected or convoy in banned: continue if honk.get("RID"): for honk_in_convoy in honk_url.do_get(action="gethonks", page="convoy", c=convoy)["honks"]: @@ -252,6 +254,8 @@ def demo(lonk_url, honk_url): header = f'## From: {from_}, {honk_in_convoy["Date"]} (๐Ÿงต {honk["Handle"]})' collected[convoy] = (honk_in_convoy, header) break + else: + banned.add(convoy) else: oondle = honk.get("Oondle") if oondle: @@ -280,13 +284,41 @@ def demo(lonk_url, honk_url): def _create_schema(db_con, cert_hash): - db_con.execute(""" + db_con.execute( + """ CREATE TABLE client ( client_id INTEGER PRIMARY KEY AUTOINCREMENT, cert_hash TEXT UNIQUE, honk_url TEXT NOT NULL, token TEXT NOT NULL + ) + """ + ) + db_con.execute( + """ + CREATE TABLE + convoy ( + convoy_id TEXT, + client_id INTEGER, + header TEXT, + FOREIGN KEY (client_id) REFERENCES client(client_id), + PRIMARY KEY (convoy_id, client_id) + ) + """ + ) + db_con.execute( + """ + CREATE TABLE + honk ( + honk_id INTEGER, + client_id INTEGER, + convoy_id TEXT, + url TEXT NOT NULL, + html TEXT NOT NULL, + FOREIGN KEY (client_id) REFERENCES client(client_id), + FOREIGN KEY (convoy_id) REFERENCES convoy(convoy_id), + PRIMARY KEY (honk_id, client_id) ) """ ) @@ -311,17 +343,84 @@ def db_connect(cert_hash): with db_con: _create_schema(db_con, cert_hash) return db_con + + +def _lonk_impl(db_con, client_id, lonk_url, honk_url): + row = db_con.execute( + "SELECT MAX(honk_id) FROM honk WHERE client_id=?", + (client_id, ) + ) + after, = row + + home = honk_url.do_get(action="gethonks", page="home", after=after) + for honk in reversed(home["honks"]): + convoy_id = honk["Convoy"] + raise RuntimeError(convoy_id) + row = db_con.execute( + "SELECT convoy.header, honk.url, honk.html FROM convoy JOIN honk USING (convoy_id) WHERE convoy.client_id=? AND convoy_id=?", + (client_id, convoy_id) + ).fetchone() + if row: + continue + + def _save_convoy_and_honk(header, honk): + db_con.execute( + """ + INSERT INTO + convoy(convoy_id, client_id, header) + VALUES + (?, ?, ?) + """, + (convoy_id, client_id, header) + ) + db_con.execute( + """ + INSERT INTO + honk (honk_id, client_id, convoy_id, url, html) + VALUES + (?, ?, ?, ?, ?) + """, + (honk["ID"], client_id, convoy_id, honk["XID"], honk["HTML"]) + ) + + if honk.get("RID"): + for honk_in_convoy in honk_url.do_get(action="gethonks", page="convoy", c=convoy_id)["honks"]: + if not honk_in_convoy.get("RID"): + from_ = honk_in_convoy.get("Oondle") or honk_in_convoy["Handle"] + header = f'## From: {from_}, {honk_in_convoy["Date"]} (๐Ÿงต {honk["Handle"]})' + _save_convoy_and_honk(header, honk_in_convoy) + break + else: + db_con.execute( + """ + INSERT INTO + convoy(convoy_id, client_id) + VALUES + (?, ?) + """, + (convoy_id, client_id) + ) + else: + honk_from = honk.get("Oondle") or honk["Handle"] + oondle = honk.get("Oondle") + if oondle: + header = f'From: {oondle}, {honk["Date"]} (๐Ÿ” {honk["Handle"]})' + else: + header = f'From: {honk["Handle"]}, {honk["Date"]}' + _save_convoy_and_honk(header, honk) def lonk(cert_hash, lonk_url): db_con = db_connect(cert_hash) - row = db_con.execute("SELECT client_id, honk_url, token FROM client WHERE cert_hash = ?", (cert_hash, )).fetchone() + row = db_con.execute("SELECT client_id, honk_url, token FROM client WHERE cert_hash=?", (cert_hash, )).fetchone() if not row: print(f'30 {lonk_url.build("ask_server")}\r') return client_id, honk_url, token = row - demo(lonk_url, HonkUrl(honk_url, token)) + with db_con: + _lonk_impl(db_con, client_id, lonk_url, HonkUrl(honk_url, token)) + # demo(lonk_url, HonkUrl(honk_url, token)) def new_client_stage_1_ask_server(lonk_url): @@ -382,8 +481,8 @@ def proxy(mime, url): stdout.buffer.write(content) -def vgi(cert_hash): - lonk_url = LonkUrl(input().strip()) +def vgi(cert_hash, raw_url): + lonk_url = LonkUrl(raw_url) if lonk_url.page == "lonk": lonk(cert_hash, lonk_url) elif lonk_url.page == "ask_server": @@ -406,7 +505,12 @@ if __name__ == '__main__': cert_hash = environ.get("VGI_CERT_HASH") if cert_hash: try: - vgi(cert_hash) + start_time = clock_gettime(CLOCK_MONOTONIC) + try: + raw_url = input().strip() + vgi(cert_hash, raw_url) + finally: + stderr.write(f"{raw_url}|{clock_gettime(CLOCK_MONOTONIC) - start_time:.3f}sec.\n") except HTTPError as error: print(f"43 Remote server return {error.code}: {error.reason}\r") except URLError as error: