diff options
| -rwxr-xr-x | app.py | 25 | ||||
| -rw-r--r-- | curiouscat.py | 47 | ||||
| -rw-r--r-- | database.py | 127 | ||||
| -rwxr-xr-x | services.py | 91 | ||||
| -rw-r--r-- | static/images/GcyexeCW0AAYssz.jpg | bin | 0 -> 992976 bytes | |||
| -rw-r--r-- | static/images/Startech.jpg | bin | 0 -> 456042 bytes | |||
| -rwxr-xr-x | static/index.md | 3 | ||||
| -rwxr-xr-x | templates/index.html.j2 | 2 | ||||
| -rw-r--r-- | templates/questions.html.j2 | 2 | ||||
| -rwxr-xr-x | templates/template.html.j2 | 4 |
10 files changed, 26 insertions, 275 deletions
@@ -70,7 +70,6 @@ def index(): **get_template_items("eden's site :3", db), markdown = parser.parse_text(f.read())[0], featured_thoughts = db.get_featured_thoughts(), - tweets = db.get_cached_tweets(7) + [("view all tweets...", db.get_my_twitter())], commits = db.get_cached_commits()[:7], sidebar_img = get_sidebar_img(db) ) @@ -79,16 +78,6 @@ def index(): def robots(): return flask.send_from_directory("static", "robots.txt") -@app.route("/diary") -def diary(): - with database.Database() as db: - return flask.render_template( - "diary.html.j2", - **get_template_items("diary", db), - diary = db.get_diary(), - diary_account = db.config.get("twitter", "diary_account") - ) - @app.route("/discord") def discord(): with database.Database() as db: @@ -98,17 +87,6 @@ def discord(): discord = CONFIG["discord"]["username"] ) -@app.route("/services") -def serve_services(): - with database.Database() as db: - return flask.render_template( - "services.html.j2", - **get_template_items("services", db), - docker = services.get_docker_stats(), - trans = services.get_trans_stats(), - pihole = services.get_pihole_stats() - ) - @app.route("/thought") def get_thought(): thought_id = flask.request.args.get("id", type=int) @@ -247,8 +225,7 @@ def serve_questions(): return flask.render_template( "questions.html.j2", **get_template_items("questions and answers", db), - curiouscat_username = db.get_curiouscat_username(), - qnas = db.get_curiouscat_qnas() + qnas = db.get_qnas() ) if __name__ == "__main__": diff --git a/curiouscat.py b/curiouscat.py deleted file mode 100644 index 531f08d..0000000 --- a/curiouscat.py +++ /dev/null @@ -1,47 +0,0 @@ -import datetime -import requests -import json - -def get_curiouscat_qnas_after(name, last_timestamp = None): - if last_timestamp is None: - url = "https://curiouscat.live/api/v2/profile?username=%s&count=100" % (name) - else: - url = "https://curiouscat.live/api/v2/profile?username=%s&count=100&max_timestamp=%d" % (name, last_timestamp) - - req = requests.get(url) - return req.json()["posts"] - -def get_all_curiouscat_qnas(name): - out = [] - period = get_curiouscat_qnas_after(name) - out += period - while len(period) == 100: - oldest = min([i["timestamp"] for i in period]) - period = get_curiouscat_qnas_after("jwnskanzkwk", last_timestamp = oldest - 1) - - out += period - - return post_process(out, name) - -def get_all_curiouscat_qnas_before(name, min_dt): - url = "https://curiouscat.live/api/v2/profile?username=%s&count=100&min_timestamp=%d" % (name, int(min_dt.timestamp()) + 1) - req = requests.get(url) - return post_process(req.json()["posts"], name) - -def post_process(cc, name): - return [ - { - "id": i["id"], - "link": "https://curiouscat.me/%s/post/%d" % (name, i["id"]), - "datetime": datetime.datetime.fromtimestamp(i["timestamp"]), - "question": i["comment"], - "answer": i["reply"] - } - for i in cc - ] - -if __name__ == "__main__": - import database - - with database.Database() as db: - print(db.append_curiouscat_qnas(get_all_curiouscat_qnas_before("jwnskanzkwk", db.get_biggest_curiouscat_timestamp()))) diff --git a/database.py b/database.py index d21ae37..e68e085 100644 --- a/database.py +++ b/database.py @@ -2,7 +2,6 @@ from urllib.parse import urlparse from dataclasses import dataclass from lxml import html import configparser -import curiouscat import threading import services import operator @@ -147,15 +146,24 @@ class Database: return [(i[0], "https://%s/%s/status/%d" % (self.config.get("nitter", "outsideurl"), i[2], i[1])) for i in cursor.fetchall()] def get_cached_commits(self, since = None, recurse = True): + threading.Thread(target = update_cache).start() with self.__connection.cursor() as cursor: if since is not None: cursor.execute("SELECT message, url, commitTime, additions, deletions, total FROM commitCache WHERE commitTime > %s ORDER BY commitTime DESC;", (since, )) else: cursor.execute("SELECT message, url, commitTime, additions, deletions, total FROM commitCache ORDER BY commitTime DESC;") + # i think i might have spent too long doing functional programming return [{ "repo": urlparse(i[1]).path.split("/")[2], + "github_repo_url": "https://github.com" + "/".join(urlparse(i[1]).path.split("/")[:3]), + "git_repo_url": "https://%s/%s.git/about" % (self.config.get("github", "personal_domain"), urlparse(i[1]).path.split("/")[2]), "message": i[0], - "url": i[1], + "github_commit_url": i[1], + "git_commit_url": "https://%s/%s.git/commit/?id=%s" % ( + self.config.get("github", "personal_domain"), + urlparse(i[1]).path.split("/")[2], + urlparse(i[1]).path.split("/")[-1] + ), "datetime": i[2], "stats": { "additions": i[3], @@ -209,86 +217,7 @@ class Database: self.__connection.commit() return id_ - def append_diary(self, tweet_id, tweeted_at, replying_to, tweet, account): - if tweet is None: - tweet = "(Image only)" - with self.__connection.cursor() as cursor: - cursor.execute("INSERT INTO diary VALUES (%s, %s, %s, %s, %s);", (tweet_id, tweeted_at, replying_to, tweet, account)) - self.__connection.commit() - - print("Appended diary with tweet " + tweet + " @ " + str(tweeted_at)) - - def append_diary_images(self, tweet_id, imurl): - with self.__connection.cursor() as cursor: - cursor.execute("INSERT INTO diaryimages (tweet_id, link) VALUES (%s, %s);", (tweet_id, imurl)) - self.__connection.commit() - - - def get_diary(self, account = None): - threading.Thread(target = update_cache).start() - out = {} - if account is None: - account = self.get_my_diary_twitter() - - with self.__connection.cursor() as cursor: - # cursor.execute("SELECT tweet_id, tweeted_at, tweet FROM diary WHERE replying_to IS NULL ORDER BY tweeted_at DESC;") - # attempt to ignore curiouscat automatic tweets by comparing with the q&a table - cursor.execute("SELECT tweet_id, tweeted_at, tweet FROM diary WHERE replying_to IS NULL AND tweet_id NOT IN (SELECT tweet_id FROM diary INNER JOIN qnas ON SUBSTRING(tweet, 1, 16) = SUBSTRING(question, 1, 16)) AND account IN %s ORDER BY tweeted_at DESC;", ([account, "HONMISGENDERER"], )) - for tweet_id, tweeted_at, tweet_text in cursor.fetchall(): - # print(tweet_id, tweeted_at, tweet_text) - out[tweeted_at] = [{ - "text": tweet_text, - "images": self.get_diary_image(tweet_id), - "link": "https://%s/%s/status/%d" % ( - self.config.get("nitter", "outsideurl"), - self.get_my_diary_twitter(), - tweet_id - ) - }] - - next_tweet = self.get_child_tweets(tweet_id) - while next_tweet is not None: - tweet, id_ = next_tweet - out[tweeted_at].append(tweet) - next_tweet = self.get_child_tweets(id_) - - return out - - def get_diary_image(self, tweet_id): - with self.__connection.cursor() as cursor: - cursor.execute("SELECT link FROM diaryimages WHERE tweet_id = %s;", (tweet_id, )) - return [i[0] for i in cursor.fetchall()] - - def get_child_tweets(self, parent_id): - with self.__connection.cursor() as cursor: - cursor.execute("SELECT tweet_id, tweet FROM diary WHERE replying_to = %s;", (parent_id, )) - out = cursor.fetchall() - if out == (): - return None - - out = out[0] - id_ = out[0] - return { - "text": out[1], - "images": self.get_diary_image(id_), - "link": "https://%s/%s/status/%d" % ( - self.config.get("nitter", "outsideurl"), self.get_my_diary_twitter(), id_ - ) - }, id_ - - def get_newest_diary_tweet_id(self, account = None): - if account is None: - account = self.get_my_diary_twitter() - with self.__connection.cursor() as cursor: - cursor.execute("SELECT MAX(tweet_id) FROM diary WHERE account = %s;", (account, )) - return cursor.fetchone()[0] - - def get_curiouscat_username(self): - with self.__connection.cursor() as cursor: - cursor.execute("SELECT link FROM headerLinks WHERE name = 'curiouscat';") - return urlparse(cursor.fetchone()[0]).path.split("/")[1] - - def append_curiouscat_qnas(self, qnas): + def append_qnas(self, qnas): with self.__connection.cursor() as cursor: for qna in qnas: cursor.execute("SELECT curiouscat_id FROM qnas WHERE curiouscat_id = %s;", (qna["id"], )) @@ -303,12 +232,7 @@ class Database: print("Skipped question with timestamp %s" % datetime.datetime.fromtimestamp(qna["id"]).isoformat()) self.__connection.commit() - def get_biggest_curiouscat_timestamp(self): - with self.__connection.cursor() as cursor: - cursor.execute("SELECT MAX(`timestamp`) FROM `qnas`;") - return cursor.fetchone()[0] - - def get_curiouscat_qnas(self): + def get_qnas(self): with self.__connection.cursor() as cursor: cursor.execute("SELECT * FROM qnas;") return sorted(cursor.fetchall(), key = operator.itemgetter(2), reverse = True) @@ -316,35 +240,14 @@ class Database: def update_cache(): print("Updating cache...") with Database() as db: - db.append_curiouscat_qnas( - curiouscat.get_all_curiouscat_qnas_before( - db.get_curiouscat_username(), - db.get_biggest_curiouscat_timestamp() - ) - ) - print("Finished adding curiouscat...") db.update_commit_cache(services.request_recent_commits(since = db.get_last_commit_time())) print("Finished adding github commits...") - for id_, dt, replying_to, text, username, images in services.scrape_nitter(db.get_my_diary_twitter(), db.get_newest_diary_tweet_id()): - db.append_diary(id_, dt, replying_to, text, username) - for image in images: - db.append_diary_images(id_, image) - print("Finished getting diary tweets...") - main_account = db.config.get("twitter", "main_account") - oldest_tweet = db.get_newest_diary_tweet_id(main_account) - print("Fetching tweets from account '%s' older than %d" % (main_account, oldest_tweet)) - for id_, dt, replying_to, text, username, images in services.scrape_nitter( - main_account, - oldest_tweet - ): - db.append_diary(id_, dt, replying_to, text, username) - for image in images: - db.append_diary_images(id_, image) - print("Done updating commit cache...") if __name__ == "__main__": + # update_cache() + import json with Database() as db: - print(db.get_diary()) + print(db.get_cached_commits()[0]) diff --git a/services.py b/services.py index 9bbac3f..075d533 100755 --- a/services.py +++ b/services.py @@ -10,7 +10,6 @@ import requests import datetime import urllib import docker -import clutch import random import queue import json @@ -40,90 +39,6 @@ def humanbytes(B): elif TB <= B: return '{0:.2f} TB'.format(B/TB) -def timeout(func): - # cant get this to work with queue.Queue() for some reason? - # this works but Manager() uses an extra thread than Queue() - manager = multiprocessing.Manager() - returnVan = manager.list() - # ti = time.time() - - def runFunc(q, func): - q.append(func()) - - def beginTimeout(): - t = multiprocessing.Process(target = runFunc, args = (returnVan, func)) - t.start() - - t.join(timeout = CONFIG["servicetimeout"].getint("seconds")) - - # print("Request took:", time.time() - ti) - try: - return returnVan[0] - except IndexError: - if t.is_alive(): - t.terminate() - - return beginTimeout - -@timeout -def get_docker_stats(): - client = docker.DockerClient(base_url = "tcp://%s:%s" % (CONFIG["docker"]["url"], CONFIG["docker"]["port"])) - return { - container.name: container.status - for container in client.containers.list(all = True) - } - -# currently unused -@timeout -def get_qbit_stats(): - numtorrents = 0 - bytes_dl = 0 - bytes_up = 0 - qb = qbittorrent.Client('http://%s:%s/' % (CONFIG["qbittorrent"]["url"], CONFIG["qbittorrent"]["port"])) - qb.login(username = CONFIG["qbittorrent"]["user"], password = CONFIG["qbittorrent"]["passwd"]) - - for torrent in qb.torrents(): - numtorrents += 1 - bytes_up += torrent["uploaded"] - bytes_dl += torrent["downloaded"] - - return { - "bytes_dl": humanbytes(bytes_dl), - "bytes_up": humanbytes(bytes_up), - "num": numtorrents, - "ratio": "%.3f" % (float(bytes_up) / float(bytes_dl)) - } - -@timeout -def get_trans_stats(): - client = clutch.client.Client( - address = "http://%s:%s/transmission/rpc" % (CONFIG["transmission"]["url"], CONFIG["transmission"]["port"]), - # username = CONFIG["transmission"]["username"], - # password = CONFIG["transmission"]["password"] - ) - stats = json.loads(client.session.stats().json()) - active_for = datetime.timedelta(seconds = stats["arguments"]["cumulative_stats"]["seconds_active"]) - return { - "bytes_dl": humanbytes(stats["arguments"]["cumulative_stats"]["downloaded_bytes"]), - "bytes_up": humanbytes(stats["arguments"]["cumulative_stats"]["uploaded_bytes"]), - "num": stats["arguments"]["torrent_count"], - "ratio": "%.3f" % (float(stats["arguments"]["cumulative_stats"]["uploaded_bytes"]) / float(stats["arguments"]["cumulative_stats"]["downloaded_bytes"])), - "active_for": str(active_for) - } - -@timeout -def get_pihole_stats(): - pihole = ph.PiHole(CONFIG["pihole"]["url"]) - return { - "status": pihole.status, - "queries": pihole.total_queries, - "clients": pihole.unique_clients, - "percentage": pihole.ads_percentage, - "blocked": pihole.blocked, - "domains": pihole.domain_count, - "last_updated": str(datetime.datetime.fromtimestamp(pihole.gravity_last_updated["absolute"])) - } - @dataclass class SafebooruImage: id_: int @@ -252,7 +167,7 @@ def request_recent_commits(since = datetime.datetime.now() - datetime.timedelta( } }) except Exception as e: - print(e) + print(repo, e) return sorted(out, key = lambda a: a["datetime"], reverse = True) @@ -331,6 +246,8 @@ if __name__ == "__main__": # print(get_trans_stats()) #print(scrape_nitter(CONFIG.get("twitter", "diary_account"), 1697430888617840909)) - print(scrape_nitter("estrogenizedboy", 1698107440489734640)) + # print(scrape_nitter("estrogenizedboy", 1698107440489734640)) # print(parse_tweet("https://nitter.net/HONMISGENDERER/status/1694231618443981161#m")) + + print(request_recent_commits(since = datetime.datetime.now() - datetime.timedelta(days=30))) diff --git a/static/images/GcyexeCW0AAYssz.jpg b/static/images/GcyexeCW0AAYssz.jpg Binary files differnew file mode 100644 index 0000000..c0762a8 --- /dev/null +++ b/static/images/GcyexeCW0AAYssz.jpg diff --git a/static/images/Startech.jpg b/static/images/Startech.jpg Binary files differnew file mode 100644 index 0000000..ba7f02f --- /dev/null +++ b/static/images/Startech.jpg diff --git a/static/index.md b/static/index.md index 7aafcd1..99bfa0c 100755 --- a/static/index.md +++ b/static/index.md @@ -18,7 +18,8 @@ i'll post my thoughts on here sometimes, and use this site to link to other stuf these sites are hosted on my [homelab system](https://wiki.eda.gay) - + + ## nice websites - [wiby.me](http://wiby.me/) - search engine for old style websites with limited javascript (my site used to be on here but it got blacklisted for some reason?) diff --git a/templates/index.html.j2 b/templates/index.html.j2 index 49c2794..53ca77e 100755 --- a/templates/index.html.j2 +++ b/templates/index.html.j2 @@ -16,7 +16,7 @@ <h3>recent git commits:</h3> <ul> {% for commit in commits %} - <li>[<a href="{{'https://git.eda.gay/' + commit['repo']}}">{{commit["repo"]}}</a>] {+{{commit["stats"]["additions"]}}; -{{commit["stats"]["deletions"]}}} <a href="{{commit['url'].replace('github.com/jwansek', 'git.eda.gay')}}">{{commit["message"]}}</a> - <a href="{{commit['url']}}">github mirror</a></li> + <li>[<a href="{{commit['git_repo_url']}}">{{commit["repo"]}}</a>] {+{{commit["stats"]["additions"]}}; -{{commit["stats"]["deletions"]}}} <a href="{{commit['git_commit_url']}}">{{commit["message"]}}</a> - <a href="{{commit['github_commit_url']}}">github mirror</a></li> {% endfor %} </ul> </section> diff --git a/templates/questions.html.j2 b/templates/questions.html.j2 index bf6b278..97edd38 100644 --- a/templates/questions.html.j2 +++ b/templates/questions.html.j2 @@ -1,6 +1,6 @@ {% extends "template.html.j2" %} {% block content %} - <h4><a href="https://tellonym.me/boymoderology">ask a question!</a></h4> + <h4><a href="https://whispa.sh/@boymoderology">ask a question!</a></h4> <dl> {% for id_, link, dt, question, answer in qnas %} <dt><a href="{{ link }}">{{ dt.isoformat() }}</a></dt> diff --git a/templates/template.html.j2 b/templates/template.html.j2 index b0ce156..1c9e99e 100755 --- a/templates/template.html.j2 +++ b/templates/template.html.j2 @@ -8,7 +8,7 @@ <meta content="{{title}}" property="og:title" /> <meta content="Formerly chuck's" property="og:description" /> - <meta content="https://eda.gay" property="og:url" /> + <meta content="https://boymoder.blog" property="og:url" /> <meta content="/img/greenboi.jpg?h=512&w=512" property="og:image" /> </head> <body> @@ -49,7 +49,7 @@ </div> <footer> <p>this site is <a href="/thought?id=3">javascript free</a></p> - <a href="https://git.eda.gay/eda.gay/file/LICENSE.html">sauce code released under gplv3</a> <a href="https://github.com/jwansek/eda.gay">alternate git link</a> + <a href="https://git.eda.gay/eda.gay.git/tree/LICENSE">sauce code released under gplv3</a> <a href="https://github.com/jwansek/eda.gay">alternate git link</a> <div id="footer_banners"> <p> <a href="http://jigsaw.w3.org/css-validator/check/referer"> |
