From 4314d46fd9d14076c98c12d43f16861adda9bc2d Mon Sep 17 00:00:00 2001 From: jwansek Date: Sat, 21 Feb 2026 17:10:52 +0000 Subject: Added cow --- edaweb/app.py | 516 +++++++++++++++---------------- edaweb/cache.py | 2 + edaweb/database.py | 502 +++++++++++++++--------------- edaweb/services.py | 818 ++++++++++++++++++++++++------------------------- edaweb/static/cow.txt | 58 ++++ edaweb/static/index.md | 68 ++-- 6 files changed, 1014 insertions(+), 950 deletions(-) create mode 100644 edaweb/static/cow.txt (limited to 'edaweb') diff --git a/edaweb/app.py b/edaweb/app.py index 36fe8bb..923b792 100644 --- a/edaweb/app.py +++ b/edaweb/app.py @@ -1,256 +1,260 @@ -from paste.translogger import TransLogger -from waitress import serve -from PIL import Image -import configparser -import transmission_rpc -import downloader -import datetime -import database -import services -import urllib -import random -import parser -import flask -import sys -import os -import io - -app = flask.Flask(__name__) -CONFIG = configparser.ConfigParser(interpolation = None) -CONFIG.read(os.path.join(os.path.dirname(__file__), "edaweb.conf")) -shown_images = set() -shown_sidebar_images = set() - -def get_pfp_img(db:database.Database): - global shown_images - dbimg = db.get_pfp_images() - if len(shown_images) == len(dbimg): - shown_images = set() - folder = set(dbimg).difference(shown_images) - choice = random.choice(list(folder)) - shown_images.add(choice) - return choice - -def get_sidebar_img(db:database.Database): - global shown_sidebar_images - dbimg = db.get_sidebar_images() - if len(shown_sidebar_images) == len(dbimg): - shown_sidebar_images = set() - folder = set(dbimg).difference(shown_sidebar_images) - choice = random.choice(list(folder)) - shown_sidebar_images.add(choice) - return choice - -def get_correct_article_headers(db:database.Database, title): - db_headers = list(db.get_header_articles()) - if title in [i[0] for i in db_headers]: - out = [] - for i in db_headers: - if i[0] != title: - out.append(i) - return out + [("index", "/~")] - else: - return db_headers + [("index", "/~")] - -def get_template_items(title, db): - return { - "links": db.get_header_links(), - "image": get_pfp_img(db), - "title": title, - "articles": get_correct_article_headers(db, title) - } - -@app.route("/") -@app.route("/~") -def index(): - with database.Database() as db: - with open(os.path.join(os.path.dirname(__file__), "static", "index.md"), "r") as f: - return flask.render_template( - "index.html.j2", - **get_template_items("eden's site :3", db), - days_till_ffs = datetime.datetime(2025, 11, 8) - datetime.datetime.now(), - markdown = parser.parse_text(f.read())[0], - featured_thoughts = db.get_featured_thoughts(), - commits = services.get_recent_commits(db)[:15], - sidebar_img = get_sidebar_img(db) - ) - -@app.route("/robots.txt") -def robots(): - return flask.send_from_directory("static", "robots.txt") - -@app.route("/services") -def serve_services(): - with database.Database() as db: - return flask.render_template( - "services.html.j2", - **get_template_items("services", db), - docker = services.get_all_docker_containers(), - trans = services.get_torrent_stats(), - pihole = services.get_pihole_stats() - ) - -@app.route("/discord") -def discord(): - with database.Database() as db: - return flask.render_template( - "discord.html.j2", - **get_template_items("discord", db), - discord = CONFIG["discord"]["username"] - ) - -@app.route("/thought") -def get_thought(): - thought_id = flask.request.args.get("id", type=int) - with database.Database() as db: - try: - category_name, title, dt, parsed, headers, redirect = parser.get_thought_from_id(db, thought_id) - # print(headers) - except TypeError: - flask.abort(404) - return - - if redirect is not None: - return flask.redirect(redirect, code = 301) - - return flask.render_template( - "thought.html.j2", - **get_template_items(title, db), - md_html = parsed, - contents_html = headers, - dt = "published: " + str(dt), - category = category_name, - othercategories = db.get_categories_not(category_name), - related = db.get_similar_thoughts(category_name, thought_id) - ) - -@app.route("/thoughts") -def get_thoughts(): - with database.Database() as db: - all_ = db.get_all_thoughts() - tree = {} - for id_, title, dt, category in all_: - if category not in tree.keys(): - tree[category] = [(id_, title, dt)] - else: - tree[category].append((id_, title, str(dt))) - - return flask.render_template( - "thoughts.html.j2", - **get_template_items("thoughts", db), - tree = tree - ) - -@app.route("/img/") -def serve_image(filename): - imdirpath = os.path.join(os.path.dirname(__file__), "static", "images") - if filename in os.listdir(imdirpath): - try: - w = int(flask.request.args['w']) - h = int(flask.request.args['h']) - except (KeyError, ValueError): - return flask.send_from_directory(imdirpath, filename) - - img = Image.open(os.path.join(imdirpath, filename)) - img.thumbnail((w, h), Image.LANCZOS) - io_ = io.BytesIO() - img.save(io_, format='JPEG') - return flask.Response(io_.getvalue(), mimetype='image/jpeg') - else: - flask.abort(404) - -@app.route("/nhdl") -def serve_nhdl(): - with database.Database() as db: - try: - nhentai_id = int(flask.request.args["id"]) - with downloader.CompressedImages(nhentai_id) as zippath: - # return app.send_static_file(os.path.split(zippath)[-1]) - return flask.redirect("/zip/%s" % os.path.split(zippath)[-1]) - - except (KeyError, ValueError): - return flask.render_template( - "nhdl.html.j2", - **get_template_items("Hentai Downloader", db) - ) - -@app.route("/isocd") -def serve_iso_form(): - with database.Database() as db: - return flask.render_template( - "isocd.html.j2", - **get_template_items("Get a GNU/Linux install CD", db), - iso_options = db.get_iso_cd_options() - ) - -@app.route("/zip/") -def serve_zip(zipfile): - return flask.send_from_directory(os.path.join(os.path.dirname(__file__), "static", "zips"), zipfile) - -@app.route("/pdf/") -def serve_pdf(pdfname): - return flask.send_from_directory(os.path.join(os.path.dirname(__file__), "static", "papers"), pdfname) - -@app.route("/nhdlredirect", methods = ["POST"]) -def redirect_nhdl(): - req = dict(flask.request.form) - try: - return flask.redirect("/nhdl?id=%i" % int(req["number_input"])) - except (TypeError, ValueError, KeyError): - flask.abort(400) - -@app.route("/getisocd", methods = ["POST"]) -def get_iso_cd(): - req = dict(flask.request.form) - print(req) - with database.Database() as db: - id_ = db.append_cd_orders(**req) - print(id_) - return flask.render_template( - "isocd_confirmation.html.j2", - **get_template_items("Get a GNU/Linux install CD", db), - email = req["email"], - req = req, - id_ = id_ - ) - -#@app.route("/random") -#def serve_random(): -# try: -# tags = flask.request.args['tags'].split(" ") -# except KeyError: -# flask.abort(400) -# -# sbi = services.get_random_image(tags) -# req = urllib.request.Request(sbi.imurl) -# mediaContent = urllib.request.urlopen(req).read() -# with open(os.path.join(os.path.dirname(__file__), "static", "images", "random.jpg"), "wb") as f: -# f.write(mediaContent) -# -# with database.Database() as db: -# return flask.render_template( -# "random.html.j2", -# **get_template_items("random image", db), -# sbi = sbi, -# localimg = "/img/random.jpg?seed=%i" % random.randint(0, 9999) -# ) - -@app.route("/questions") -def serve_questions(): - with database.Database() as db: - return flask.render_template( - "questions.html.j2", - **get_template_items("questions and answers", db), - qnas_link = CONFIG.get("qnas", "url"), - qnas = db.get_qnas() - ) - -if __name__ == "__main__": - try: - if sys.argv[1] == "--production": - #serve(TransLogger(app), host='127.0.0.1', port = 6969) - serve(TransLogger(app), host='0.0.0.0', port = 6969, threads = 32) - else: - app.run(host = "0.0.0.0", port = 5001, debug = True) - except IndexError: - app.run(host = "0.0.0.0", port = 5001, debug = True) +from paste.translogger import TransLogger +from waitress import serve +from PIL import Image +import configparser +import transmission_rpc +import downloader +import datetime +import database +import services +import urllib +import random +import parser +import flask +import sys +import os +import io + +app = flask.Flask(__name__) +CONFIG = configparser.ConfigParser(interpolation = None) +CONFIG.read(os.path.join(os.path.dirname(__file__), "edaweb.conf")) +shown_images = set() +shown_sidebar_images = set() + +def get_pfp_img(db:database.Database): + global shown_images + dbimg = db.get_pfp_images() + if len(shown_images) == len(dbimg): + shown_images = set() + folder = set(dbimg).difference(shown_images) + choice = random.choice(list(folder)) + shown_images.add(choice) + return choice + +def get_sidebar_img(db:database.Database): + global shown_sidebar_images + dbimg = db.get_sidebar_images() + if len(shown_sidebar_images) == len(dbimg): + shown_sidebar_images = set() + folder = set(dbimg).difference(shown_sidebar_images) + choice = random.choice(list(folder)) + shown_sidebar_images.add(choice) + return choice + +def get_correct_article_headers(db:database.Database, title): + db_headers = list(db.get_header_articles()) + if title in [i[0] for i in db_headers]: + out = [] + for i in db_headers: + if i[0] != title: + out.append(i) + return out + [("index", "/~")] + else: + return db_headers + [("index", "/~")] + +def get_template_items(title, db): + return { + "links": db.get_header_links(), + "image": get_pfp_img(db), + "title": title, + "articles": get_correct_article_headers(db, title) + } + +@app.route("/") +@app.route("/~") +def index(): + with database.Database() as db: + with open(os.path.join(os.path.dirname(__file__), "static", "index.md"), "r") as f: + return flask.render_template( + "index.html.j2", + **get_template_items("eden's site :3", db), + days_till_ffs = datetime.datetime(2025, 11, 8) - datetime.datetime.now(), + markdown = parser.parse_text(f.read())[0], + featured_thoughts = db.get_featured_thoughts(), + commits = services.get_recent_commits(db)[:15], + sidebar_img = get_sidebar_img(db) + ) + +@app.route("/robots.txt") +def robots(): + return flask.send_from_directory("static", "robots.txt") + +@app.route("/cow.txt") +def moo(): + return flask.send_from_directory("static", "cow.txt") + +@app.route("/services") +def serve_services(): + with database.Database() as db: + return flask.render_template( + "services.html.j2", + **get_template_items("services", db), + docker = services.get_all_docker_containers(), + trans = services.get_torrent_stats(), + pihole = services.get_pihole_stats() + ) + +@app.route("/discord") +def discord(): + with database.Database() as db: + return flask.render_template( + "discord.html.j2", + **get_template_items("discord", db), + discord = CONFIG["discord"]["username"] + ) + +@app.route("/thought") +def get_thought(): + thought_id = flask.request.args.get("id", type=int) + with database.Database() as db: + try: + category_name, title, dt, parsed, headers, redirect = parser.get_thought_from_id(db, thought_id) + # print(headers) + except TypeError: + flask.abort(404) + return + + if redirect is not None: + return flask.redirect(redirect, code = 301) + + return flask.render_template( + "thought.html.j2", + **get_template_items(title, db), + md_html = parsed, + contents_html = headers, + dt = "published: " + str(dt), + category = category_name, + othercategories = db.get_categories_not(category_name), + related = db.get_similar_thoughts(category_name, thought_id) + ) + +@app.route("/thoughts") +def get_thoughts(): + with database.Database() as db: + all_ = db.get_all_thoughts() + tree = {} + for id_, title, dt, category in all_: + if category not in tree.keys(): + tree[category] = [(id_, title, dt)] + else: + tree[category].append((id_, title, str(dt))) + + return flask.render_template( + "thoughts.html.j2", + **get_template_items("thoughts", db), + tree = tree + ) + +@app.route("/img/") +def serve_image(filename): + imdirpath = os.path.join(os.path.dirname(__file__), "static", "images") + if filename in os.listdir(imdirpath): + try: + w = int(flask.request.args['w']) + h = int(flask.request.args['h']) + except (KeyError, ValueError): + return flask.send_from_directory(imdirpath, filename) + + img = Image.open(os.path.join(imdirpath, filename)) + img.thumbnail((w, h), Image.LANCZOS) + io_ = io.BytesIO() + img.save(io_, format='JPEG') + return flask.Response(io_.getvalue(), mimetype='image/jpeg') + else: + flask.abort(404) + +@app.route("/nhdl") +def serve_nhdl(): + with database.Database() as db: + try: + nhentai_id = int(flask.request.args["id"]) + with downloader.CompressedImages(nhentai_id) as zippath: + # return app.send_static_file(os.path.split(zippath)[-1]) + return flask.redirect("/zip/%s" % os.path.split(zippath)[-1]) + + except (KeyError, ValueError): + return flask.render_template( + "nhdl.html.j2", + **get_template_items("Hentai Downloader", db) + ) + +@app.route("/isocd") +def serve_iso_form(): + with database.Database() as db: + return flask.render_template( + "isocd.html.j2", + **get_template_items("Get a GNU/Linux install CD", db), + iso_options = db.get_iso_cd_options() + ) + +@app.route("/zip/") +def serve_zip(zipfile): + return flask.send_from_directory(os.path.join(os.path.dirname(__file__), "static", "zips"), zipfile) + +@app.route("/pdf/") +def serve_pdf(pdfname): + return flask.send_from_directory(os.path.join(os.path.dirname(__file__), "static", "papers"), pdfname) + +@app.route("/nhdlredirect", methods = ["POST"]) +def redirect_nhdl(): + req = dict(flask.request.form) + try: + return flask.redirect("/nhdl?id=%i" % int(req["number_input"])) + except (TypeError, ValueError, KeyError): + flask.abort(400) + +@app.route("/getisocd", methods = ["POST"]) +def get_iso_cd(): + req = dict(flask.request.form) + print(req) + with database.Database() as db: + id_ = db.append_cd_orders(**req) + print(id_) + return flask.render_template( + "isocd_confirmation.html.j2", + **get_template_items("Get a GNU/Linux install CD", db), + email = req["email"], + req = req, + id_ = id_ + ) + +#@app.route("/random") +#def serve_random(): +# try: +# tags = flask.request.args['tags'].split(" ") +# except KeyError: +# flask.abort(400) +# +# sbi = services.get_random_image(tags) +# req = urllib.request.Request(sbi.imurl) +# mediaContent = urllib.request.urlopen(req).read() +# with open(os.path.join(os.path.dirname(__file__), "static", "images", "random.jpg"), "wb") as f: +# f.write(mediaContent) +# +# with database.Database() as db: +# return flask.render_template( +# "random.html.j2", +# **get_template_items("random image", db), +# sbi = sbi, +# localimg = "/img/random.jpg?seed=%i" % random.randint(0, 9999) +# ) + +@app.route("/questions") +def serve_questions(): + with database.Database() as db: + return flask.render_template( + "questions.html.j2", + **get_template_items("questions and answers", db), + qnas_link = CONFIG.get("qnas", "url"), + qnas = db.get_qnas() + ) + +if __name__ == "__main__": + try: + if sys.argv[1] == "--production": + #serve(TransLogger(app), host='127.0.0.1', port = 6969) + serve(TransLogger(app), host='0.0.0.0', port = 6969, threads = 32) + else: + app.run(host = "0.0.0.0", port = 5001, debug = True) + except IndexError: + app.run(host = "0.0.0.0", port = 5001, debug = True) diff --git a/edaweb/cache.py b/edaweb/cache.py index 5b66e43..8694666 100644 --- a/edaweb/cache.py +++ b/edaweb/cache.py @@ -1,5 +1,6 @@ import database import services +import json def update_cache(): print("Updating cache...") @@ -7,6 +8,7 @@ def update_cache(): db.update_commit_cache(services.request_recent_commits(since = db.get_last_commit_time())) print("Finished adding github commits...") db.append_qnas(services.scrape_whispa(db.config.get("qnas", "url"), since = db.get_oldest_qna())) + # print(json.dumps(services.scrape_whispa(db.config.get("qnas", "url"), since = db.get_oldest_qna()), indent = 4)) print("Finished parsing Q&As...") print("Started getting docker information with SSH...") diff --git a/edaweb/database.py b/edaweb/database.py index c6553a6..8bb9a60 100644 --- a/edaweb/database.py +++ b/edaweb/database.py @@ -1,251 +1,251 @@ -from urllib.parse import urlparse -from dataclasses import dataclass -from lxml import html -import configparser -import threading -import services -import operator -import datetime -import requests -import twython -import pymysql -import random -import os -import re - -@dataclass -class Database: - safeLogin:bool = True #automatically login with the user in the config file, who is read only - user:str = None #otherwise, login with the given username and passwd - passwd:str = None - - def __enter__(self): - config_path = os.path.join(os.path.dirname(__file__), "..", "edaweb.conf") - if not os.path.exists(config_path): - raise FileNotFoundError("Could not find edaweb.conf config file") - self.config = configparser.ConfigParser(interpolation = None) - self.config.read(config_path) - - if self.safeLogin: - self.__connection = pymysql.connect( - **self.config["mysql"], - charset = "utf8mb4" - ) - else: - self.__connection = pymysql.connect( - user = self.user, - passwd = self.passwd, - host = self.config["mysql"]["host"], - db = self.config["mysql"]["db"], - charset = "utf8mb4" - ) - return self - - def __exit__(self, type, value, traceback): - self.__connection.commit() - self.__connection.close() - - def get_header_links(self): - with self.__connection.cursor() as cursor: - cursor.execute("SELECT name, link FROM headerLinks WHERE display = true ORDER BY name;") - return cursor.fetchall() - - def get_image(self, imageName): - with self.__connection.cursor() as cursor: - cursor.execute("SELECT alt, url FROM images WHERE imageName = %s;", (imageName, )) - return cursor.fetchone() - - def get_pfp_images(self): - with self.__connection.cursor() as cursor: - cursor.execute("SELECT alt, url FROM images WHERE pfp_img = 1;") - return cursor.fetchall() - - def get_sidebar_images(self): - with self.__connection.cursor() as cursor: - cursor.execute("SELECT alt, url FROM images WHERE sidebar_image = 1;") - return cursor.fetchall() - - def get_header_articles(self): - with self.__connection.cursor() as cursor: - cursor.execute("SELECT articleName, link FROM headerArticles WHERE display = true;") - return cursor.fetchall() - - def get_all_categories(self): - with self.__connection.cursor() as cursor: - cursor.execute("SELECT category_name FROM categories;") - return [i[0] for i in cursor.fetchall()] - - def add_category(self, category): - if not category in self.get_all_categories(): - with self.__connection.cursor() as cursor: - cursor.execute("INSERT INTO categories (category_name) VALUES (%s);", (category, )) - - self.__connection.commit() - return True - - return False - - def add_thought(self, category, title, markdown): - with self.__connection.cursor() as cursor: - cursor.execute(""" - INSERT INTO thoughts (category_id, title, markdown_text) - VALUES (( - SELECT category_id FROM categories WHERE category_name = %s - ), %s, %s);""", (category, title, markdown)) - self.__connection.commit() - - def get_thought(self, id_): - with self.__connection.cursor() as cursor: - cursor.execute(""" - SELECT categories.category_name, thoughts.title, thoughts.dt, thoughts.markdown_text, thoughts.redirect - FROM thoughts INNER JOIN categories - ON thoughts.category_id = categories.category_id - WHERE thought_id = %s;""", (id_, )) - return cursor.fetchone() - - def get_similar_thoughts(self, category, id_): - with self.__connection.cursor() as cursor: - cursor.execute(""" - SELECT thought_id, title, dt, category_name FROM thoughts - INNER JOIN categories ON thoughts.category_id = categories.category_id - WHERE category_name = %s AND thought_id != %s;""", - (category, id_)) - return cursor.fetchall() - - def get_featured_thoughts(self): - with self.__connection.cursor() as cursor: - cursor.execute("SELECT thought_id, title FROM thoughts WHERE featured = 1;") - return cursor.fetchall() - - def update_thought_markdown(self, id_, markdown): - with self.__connection.cursor() as cursor: - cursor.execute("UPDATE thoughts SET markdown_text = %s WHERE thought_id = %s;", (markdown, id_)) - self.__connection.commit() - - def get_categories_not(self, category_name): - with self.__connection.cursor() as cursor: - cursor.execute("SELECT category_name FROM categories WHERE category_name != %s;", (category_name, )) - return [i[0] for i in cursor.fetchall()] - - def get_all_thoughts(self): - with self.__connection.cursor() as cursor: - cursor.execute(""" - SELECT thought_id, title, dt, category_name FROM thoughts - INNER JOIN categories ON thoughts.category_id = categories.category_id; - """) - return cursor.fetchall() - - def get_cached_tweets(self, numToGet = None): - with self.__connection.cursor() as cursor: - sql = "SELECT tweet, tweet_id, account FROM diary WHERE account = %s ORDER BY tweeted_at DESC" - args = (self.config.get("twitter", "main_account"), ) - if numToGet is not None: - sql += " LIMIT %s;" - args = (self.config.get("twitter", "main_account"), numToGet) - else: - sql += ";" - cursor.execute(sql, args) - - return [(i[0], "https://%s/%s/status/%d" % (self.config.get("nitter", "outsideurl"), i[2], i[1])) for i in cursor.fetchall()] - - def get_cached_commits(self, since = None, recurse = True): - with self.__connection.cursor() as cursor: - if since is not None: - cursor.execute("SELECT DISTINCT message, url, commitTime, additions, deletions, total FROM commitCache WHERE commitTime > %s ORDER BY commitTime DESC;", (since, )) - else: - cursor.execute("SELECT DISTINCT message, url, commitTime, additions, deletions, total FROM commitCache ORDER BY commitTime DESC;") - # i think i might have spent too long doing functional programming - return [{ - "repo": urlparse(i[1]).path.split("/")[2], - "github_repo_url": "https://github.com" + "/".join(urlparse(i[1]).path.split("/")[:3]), - "git_repo_url": "https://%s/%s.git/about" % (self.config.get("github", "personal_domain"), urlparse(i[1]).path.split("/")[2]), - "message": i[0], - "github_commit_url": i[1], - "git_commit_url": "https://%s/%s.git/commit/?id=%s" % ( - self.config.get("github", "personal_domain"), - urlparse(i[1]).path.split("/")[2], - urlparse(i[1]).path.split("/")[-1] - ), - "datetime": i[2].timestamp(), - "stats": { - "additions": i[3], - "deletions": i[4], - "total": i[5] - } - } for i in cursor.fetchall()] - - def update_commit_cache(self, requested): - with self.__connection.cursor() as cursor: - for commit in requested: - cursor.execute("SELECT DISTINCT url FROM commitCache;") - urls = [i[0] for i in cursor.fetchall()] - - if commit["url"] not in urls: - cursor.execute(""" - INSERT INTO commitCache (message, url, commitTime, additions, deletions, total) - VALUES (%s, %s, %s, %s, %s, %s)""", - (commit["message"], commit["url"], commit["datetime"], commit["stats"]["additions"], commit["stats"]["deletions"], commit["stats"]["total"]) - ) - self.__connection.commit() - - def get_last_commit_time(self): - with self.__connection.cursor() as cursor: - cursor.execute("SELECT MAX(commitTime) FROM commitCache;") - return cursor.fetchone()[0] - - def get_my_twitter(self): - with self.__connection.cursor() as cursor: - cursor.execute("SELECT link FROM headerLinks WHERE name = 'twitter';") - return cursor.fetchone()[0] - - def get_my_diary_twitter(self): - return self.config.get("twitter", "diary_account") - - def get_iso_cd_options(self): - iso_dir = self.config.get("cds", "location") - return [ - i - for i in os.listdir(iso_dir) - if os.path.splitext(i)[-1].lower() in [".iso"] - and os.path.getsize(os.path.join(iso_dir, i)) < self.config.getint("cds", "maxsize") - ] - - def append_cd_orders(self, iso, email, house, street, city, county, postcode, name): - with self.__connection.cursor() as cursor: - cursor.execute(""" - INSERT INTO cd_orders_2 (iso, email, house, street, city, county, postcode, name) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s); - """, (iso, email, house, street, city, county, postcode, name)) - id_ = cursor.lastrowid - self.__connection.commit() - return id_ - - def append_qnas(self, qnas): - with self.__connection.cursor() as cursor: - for qna in qnas: - cursor.execute("SELECT curiouscat_id FROM qnas WHERE curiouscat_id = %s;", (qna["id"], )) - if cursor.fetchone() is None: - - cursor.execute("INSERT INTO `qnas` VALUES (%s, %s, %s, %s, %s, %s);", ( - qna["id"], qna["link"], qna["datetime"], qna["question"], qna["answer"], qna["host"] - )) - print("Appended question with timestamp %s" % qna["datetime"].isoformat()) - - else: - print("Skipped question with timestamp %s" % qna["datetime"].isoformat()) - self.__connection.commit() - - def get_oldest_qna(self): - with self.__connection.cursor() as cursor: - cursor.execute("SELECT MAX(timestamp) FROM qnas;") - return cursor.fetchone()[0] - - def get_qnas(self): - with self.__connection.cursor() as cursor: - cursor.execute("SELECT * FROM qnas;") - return sorted(cursor.fetchall(), key = operator.itemgetter(2), reverse = True) - - - - - +from urllib.parse import urlparse +from dataclasses import dataclass +from lxml import html +import configparser +import threading +import services +import operator +import datetime +import requests +import twython +import pymysql +import random +import os +import re + +@dataclass +class Database: + safeLogin:bool = True #automatically login with the user in the config file, who is read only + user:str = None #otherwise, login with the given username and passwd + passwd:str = None + + def __enter__(self): + config_path = os.path.join(os.path.dirname(__file__), "..", "edaweb.conf") + if not os.path.exists(config_path): + raise FileNotFoundError("Could not find edaweb.conf config file") + self.config = configparser.ConfigParser(interpolation = None) + self.config.read(config_path) + + if self.safeLogin: + self.__connection = pymysql.connect( + **self.config["mysql"], + charset = "utf8mb4" + ) + else: + self.__connection = pymysql.connect( + user = self.user, + passwd = self.passwd, + host = self.config["mysql"]["host"], + db = self.config["mysql"]["db"], + charset = "utf8mb4" + ) + return self + + def __exit__(self, type, value, traceback): + self.__connection.commit() + self.__connection.close() + + def get_header_links(self): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT name, link FROM headerLinks WHERE display = true ORDER BY name;") + return cursor.fetchall() + + def get_image(self, imageName): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT alt, url FROM images WHERE imageName = %s;", (imageName, )) + return cursor.fetchone() + + def get_pfp_images(self): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT alt, url FROM images WHERE pfp_img = 1;") + return cursor.fetchall() + + def get_sidebar_images(self): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT alt, url FROM images WHERE sidebar_image = 1;") + return cursor.fetchall() + + def get_header_articles(self): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT articleName, link FROM headerArticles WHERE display = true;") + return cursor.fetchall() + + def get_all_categories(self): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT category_name FROM categories;") + return [i[0] for i in cursor.fetchall()] + + def add_category(self, category): + if not category in self.get_all_categories(): + with self.__connection.cursor() as cursor: + cursor.execute("INSERT INTO categories (category_name) VALUES (%s);", (category, )) + + self.__connection.commit() + return True + + return False + + def add_thought(self, category, title, markdown): + with self.__connection.cursor() as cursor: + cursor.execute(""" + INSERT INTO thoughts (category_id, title, markdown_text) + VALUES (( + SELECT category_id FROM categories WHERE category_name = %s + ), %s, %s);""", (category, title, markdown)) + self.__connection.commit() + + def get_thought(self, id_): + with self.__connection.cursor() as cursor: + cursor.execute(""" + SELECT categories.category_name, thoughts.title, thoughts.dt, thoughts.markdown_text, thoughts.redirect + FROM thoughts INNER JOIN categories + ON thoughts.category_id = categories.category_id + WHERE thought_id = %s;""", (id_, )) + return cursor.fetchone() + + def get_similar_thoughts(self, category, id_): + with self.__connection.cursor() as cursor: + cursor.execute(""" + SELECT thought_id, title, dt, category_name FROM thoughts + INNER JOIN categories ON thoughts.category_id = categories.category_id + WHERE category_name = %s AND thought_id != %s;""", + (category, id_)) + return cursor.fetchall() + + def get_featured_thoughts(self): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT thought_id, title FROM thoughts WHERE featured = 1;") + return cursor.fetchall() + + def update_thought_markdown(self, id_, markdown): + with self.__connection.cursor() as cursor: + cursor.execute("UPDATE thoughts SET markdown_text = %s WHERE thought_id = %s;", (markdown, id_)) + self.__connection.commit() + + def get_categories_not(self, category_name): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT category_name FROM categories WHERE category_name != %s;", (category_name, )) + return [i[0] for i in cursor.fetchall()] + + def get_all_thoughts(self): + with self.__connection.cursor() as cursor: + cursor.execute(""" + SELECT thought_id, title, dt, category_name FROM thoughts + INNER JOIN categories ON thoughts.category_id = categories.category_id; + """) + return cursor.fetchall() + + def get_cached_tweets(self, numToGet = None): + with self.__connection.cursor() as cursor: + sql = "SELECT tweet, tweet_id, account FROM diary WHERE account = %s ORDER BY tweeted_at DESC" + args = (self.config.get("twitter", "main_account"), ) + if numToGet is not None: + sql += " LIMIT %s;" + args = (self.config.get("twitter", "main_account"), numToGet) + else: + sql += ";" + cursor.execute(sql, args) + + return [(i[0], "https://%s/%s/status/%d" % (self.config.get("nitter", "outsideurl"), i[2], i[1])) for i in cursor.fetchall()] + + def get_cached_commits(self, since = None, recurse = True): + with self.__connection.cursor() as cursor: + if since is not None: + cursor.execute("SELECT DISTINCT message, url, commitTime, additions, deletions, total FROM commitCache WHERE commitTime > %s ORDER BY commitTime DESC;", (since, )) + else: + cursor.execute("SELECT DISTINCT message, url, commitTime, additions, deletions, total FROM commitCache ORDER BY commitTime DESC;") + # i think i might have spent too long doing functional programming + return [{ + "repo": urlparse(i[1]).path.split("/")[2], + "github_repo_url": "https://github.com" + "/".join(urlparse(i[1]).path.split("/")[:3]), + "git_repo_url": "https://%s/%s.git/about" % (self.config.get("github", "personal_domain"), urlparse(i[1]).path.split("/")[2]), + "message": i[0], + "github_commit_url": i[1], + "git_commit_url": "https://%s/%s.git/commit/?id=%s" % ( + self.config.get("github", "personal_domain"), + urlparse(i[1]).path.split("/")[2], + urlparse(i[1]).path.split("/")[-1] + ), + "datetime": i[2].timestamp(), + "stats": { + "additions": i[3], + "deletions": i[4], + "total": i[5] + } + } for i in cursor.fetchall()] + + def update_commit_cache(self, requested): + with self.__connection.cursor() as cursor: + for commit in requested: + cursor.execute("SELECT DISTINCT url FROM commitCache;") + urls = [i[0] for i in cursor.fetchall()] + + if commit["url"] not in urls: + cursor.execute(""" + INSERT INTO commitCache (message, url, commitTime, additions, deletions, total) + VALUES (%s, %s, %s, %s, %s, %s)""", + (commit["message"], commit["url"], commit["datetime"], commit["stats"]["additions"], commit["stats"]["deletions"], commit["stats"]["total"]) + ) + self.__connection.commit() + + def get_last_commit_time(self): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT MAX(commitTime) FROM commitCache;") + return cursor.fetchone()[0] + + def get_my_twitter(self): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT link FROM headerLinks WHERE name = 'twitter';") + return cursor.fetchone()[0] + + def get_my_diary_twitter(self): + return self.config.get("twitter", "diary_account") + + def get_iso_cd_options(self): + iso_dir = self.config.get("cds", "location") + return [ + i + for i in os.listdir(iso_dir) + if os.path.splitext(i)[-1].lower() in [".iso"] + and os.path.getsize(os.path.join(iso_dir, i)) < self.config.getint("cds", "maxsize") + ] + + def append_cd_orders(self, iso, email, house, street, city, county, postcode, name): + with self.__connection.cursor() as cursor: + cursor.execute(""" + INSERT INTO cd_orders_2 (iso, email, house, street, city, county, postcode, name) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s); + """, (iso, email, house, street, city, county, postcode, name)) + id_ = cursor.lastrowid + self.__connection.commit() + return id_ + + def append_qnas(self, qnas): + with self.__connection.cursor() as cursor: + for qna in qnas: + cursor.execute("SELECT curiouscat_id FROM qnas WHERE curiouscat_id = %s;", (qna["id"], )) + if cursor.fetchone() is None: + + cursor.execute("INSERT INTO `qnas` VALUES (%s, %s, %s, %s, %s, %s);", ( + qna["id"], qna["link"], qna["datetime"], qna["question"], qna["answer"], qna["host"] + )) + print("Appended question with timestamp %s" % qna["datetime"].isoformat()) + + else: + print("Skipped question with timestamp %s" % qna["datetime"].isoformat()) + self.__connection.commit() + + def get_oldest_qna(self): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT MAX(timestamp) FROM qnas;") + return cursor.fetchone()[0] + + def get_qnas(self): + with self.__connection.cursor() as cursor: + cursor.execute("SELECT * FROM qnas;") + return sorted(cursor.fetchall(), key = operator.itemgetter(2), reverse = True) + + + + + diff --git a/edaweb/services.py b/edaweb/services.py index 50eed45..11f21fc 100644 --- a/edaweb/services.py +++ b/edaweb/services.py @@ -1,409 +1,409 @@ -from dataclasses import dataclass -from io import StringIO -from lxml import html, etree -from github import Github -import multiprocessing -import paramiko.client -from APiHole import PiHole -import transmission_rpc -import configparser -import math as maths -import requests -import datetime -import urllib -import docker -import random -import subprocess -import fabric -import pickle -import queue -import json -import time -import os - -theLastId = 0 -config_path = os.path.join(os.path.dirname(__file__), "..", "edaweb.conf") -if not os.path.exists(config_path): - raise FileNotFoundError("Could not find edaweb.conf config file") -CONFIG = configparser.ConfigParser(interpolation = None) -CONFIG.read(config_path) - -def humanbytes(B): - 'Return the given bytes as a human friendly KB, MB, GB, or TB string' - B = float(B) - KB = float(1024) - MB = float(KB ** 2) # 1,048,576 - GB = float(KB ** 3) # 1,073,741,824 - TB = float(KB ** 4) # 1,099,511,627,776 - - if B < KB: - return '{0} {1}'.format(B,'Bytes' if 0 == B > 1 else 'Byte') - elif KB <= B < MB: - return '{0:.2f} KB'.format(B/KB) - elif MB <= B < GB: - return '{0:.2f} MB'.format(B/MB) - elif GB <= B < TB: - return '{0:.2f} GB'.format(B/GB) - elif TB <= B: - return '{0:.2f} TB'.format(B/TB) - -@dataclass -class SafebooruImage: - id_: int - url: str - searchTags: list - tags: list - source: str - imurl: str - - def remove_tag(self, tag): - return list(set(self.searchTags).difference(set([tag]))) - -@dataclass -class DownloadedImage: - imurl: str - - def __enter__(self): - self.filename = os.path.join("static", "images", "random.jpg") - - req = urllib.request.Request(self.imurl, headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_5_8) AppleWebKit/534.50.2 (KHTML, like Gecko) Version/5.0.6 Safari/533.22.3'}) - mediaContent = urllib.request.urlopen(req).read() - with open(self.filename, "wb") as f: - f.write(mediaContent) - return self.filename - - def __exit__(self, type, value, traceback): - os.remove(self.filename) - -def get_num_pages(tags): - pages_url = "https://safebooru.org/index.php?page=post&s=list&tags=%s" % "+".join(tags) - tree = html.fromstring(requests.get(pages_url).content) - try: - finalpage_element = tree.xpath("/html/body/div[6]/div/div[2]/div[2]/div/a[12]")[0] - except IndexError: - return 1 - else: - return int(int(urllib.parse.parse_qs(finalpage_element.get("href"))["pid"][0]) / (5*8)) - -def get_id_from_url(url): - return int(urllib.parse.parse_qs(url)["id"][0]) - -def get_random_image(tags): - global theLastId - searchPage = random.randint(1, get_num_pages(tags)) * 5 * 8 - url = "https://safebooru.org/index.php?page=post&s=list&tags=%s&pid=%i" % ("+".join(tags), searchPage) - tree = html.fromstring(requests.get(url).content) - - imageElements = [e for e in tree.xpath("/html/body/div[6]/div/div[2]/div[1]")[0].iter(tag = "a")] - try: - element = random.choice(imageElements) - except IndexError: - # raise ConnectionError("Couldn't find any images") - return get_random_image(tags) - - url = "https://safebooru.org/" + element.get("href") - if get_id_from_url(url) == theLastId: - return get_random_image(tags) - theLastId = get_id_from_url(url) - - try: - sbi = SafebooruImage( - id_ = get_id_from_url(url), - url = url, - tags = element.find("img").get("alt").split(), - searchTags = tags, - source = fix_source_url(get_source(url)), - imurl = get_imurl(url) - ) - except (ConnectionError, KeyError) as e: - print("[ERROR]", e) - return get_random_image(tags) - - if link_deleted(sbi.url): - print("Retried since the source was deleted...") - return get_random_image(tags) - - return sbi - -def get_source(url): - tree = html.fromstring(requests.get(url).content) - for element in tree.xpath('//*[@id="stats"]')[0].iter("li"): - if element.text.startswith("Source: h"): - return element.text[8:] - elif element.text.startswith("Source:"): - for child in element.iter(): - if child.get("href") is not None: - return child.get("href") - raise ConnectionError("Couldn't find source image for id %i" % get_id_from_url(url)) - -def fix_source_url(url): - parsed = urllib.parse.urlparse(url) - if parsed.netloc == "www.pixiv.net": - return "https://www.pixiv.net/en/artworks/" + urllib.parse.parse_qs(parsed.query)["illust_id"][0] - elif parsed.netloc in ["bishie.booru.org", "www.secchan.net"]: - return ConnectionError("Couldn't get source") - elif "pximg.net" in parsed.netloc or "pixiv.net" in parsed.netloc: - return "https://www.pixiv.net/en/artworks/" + parsed.path.split("/")[-1][:8] - elif parsed.netloc == "twitter.com": - return url.replace("twitter.com", "nitter.eda.gay") - return url - -def get_imurl(url): - tree = html.fromstring(requests.get(url).content) - return tree.xpath('//*[@id="image"]')[0].get("src") - -def link_deleted(url): - text = requests.get(url).text - return text[text.find("") + 7 : text.find("")] in ["Error | nitter", "イラストコミュニケーションサービス[pixiv]"] - -def request_recent_commits(since = datetime.datetime.now() - datetime.timedelta(days=7)): - g = Github(CONFIG.get("github", "access_code")) - out = [] - for repo in g.get_user().get_repos(): - # print(repo.name, list(repo.get_branches())) - try: - for commit in repo.get_commits(since = since): - out.append({ - "repo": repo.name, - "message": commit.commit.message, - "url": commit.html_url, - "datetime": commit.commit.author.date, - "stats": { - "additions": commit.stats.additions, - "deletions": commit.stats.deletions, - "total": commit.stats.total - } - }) - except Exception as e: - print(repo, e) - - return sorted(out, key = lambda a: a["datetime"], reverse = True) - -def scrape_nitter(username, get_until:int): - new_tweets = [] - nitter_url = CONFIG.get("nitter", "internalurl") - nitter_port = CONFIG.getint("nitter", "internalport") - scrape_new_pages = True - url = "http://%s:%d/%s" % (nitter_url, nitter_port, username) - - while scrape_new_pages: - tree = html.fromstring(requests.get(url).content) - for i, tweetUrlElement in enumerate(tree.xpath('//*[@class="tweet-link"]'), 0): - if i > 0 and tweetUrlElement.get("href").split("/")[1] == username: - id_ = int(urllib.parse.urlparse(tweetUrlElement.get("href")).path.split("/")[-1]) - tweet_link = "http://%s:%d%s" % (nitter_url, nitter_port, tweetUrlElement.get("href")) - - if id_ == get_until: - scrape_new_pages = False - break - - try: - dt, replying_to, text, images = parse_tweet(tweet_link) - new_tweets.append((id_, dt, replying_to, text, username, images)) - print(dt, "'%s'" % text) - except IndexError: - print("Couldn't get any more tweets") - scrape_new_pages = False - break - except ConnectionError: - print("Rate limited, try again later") - return [] - - - try: - cursor = tree.xpath('//*[@class="show-more"]/a')[0].get("href") - except IndexError: - # no more elements - break - url = "http://%s:%d/%s%s" % (nitter_url, nitter_port, username, cursor) - - return new_tweets - -def parse_tweet(tweet_url): - # print(tweet_url) - tree = html.fromstring(requests.get(tweet_url).content) - # with open("2images.html", "r") as f: - # tree = html.fromstring(f.read()) - - rate_limited_elem = tree.xpath("/html/body/div/div/div/span") - if rate_limited_elem != []: - if rate_limited_elem[0].text == "Instance has been rate limited.": - raise ConnectionError("Instance has been rate limited.") - - main_tweet_elem = tree.xpath('//*[@class="main-tweet"]')[0] - - dt_str = main_tweet_elem.xpath('//*[@class="tweet-published"]')[0].text - dt = datetime.datetime.strptime(dt_str.replace("Â", ""), "%b %d, %Y · %I:%M %p UTC") - text = tree.xpath('//*[@class="main-tweet"]/div/div/div[2]')[0].text_content() - if text == "": - text = "[Image only]" - replying_to_elems = tree.xpath('//*[@class="before-tweet thread-line"]/div/a') - if replying_to_elems != []: - replying_to = int(urllib.parse.urlparse(replying_to_elems[-1].get("href")).path.split("/")[-1]) - else: - replying_to = None - - images = [] - images_elems = tree.xpath('//*[@class="main-tweet"]/div/div/div[3]/div/div/a/img') - for image_elem in images_elems: - images.append("https://" + CONFIG.get("nitter", "outsideurl") + urllib.parse.urlparse(image_elem.get("src")).path) - - return dt, replying_to, text, images - -def scrape_whispa(whispa_url, since = None): - def query_answer(answer_url, max_retries = 10): - for i in range(max_retries): - try: - return requests.get(answer_url) - except requests.exceptions.ConnectionError: - s = 5.05 * (i + 1) - print("Connection timed out, retrying in %.2fs" % s) - time.sleep(s) - continue - - # add a bit of wiggle room in case i don't answer the questions in order (i often do this) - if since is None: - stop_at = datetime.datetime(year = 2001, month = 8, day = 12) - else: - stop_at = since - datetime.timedelta(days = 14) - print("The newest Q&A timestamp in the database was %s, we will stop looking at %s." % (since.astimezone().isoformat(), stop_at.astimezone().isoformat())) - - html_ = requests.get(whispa_url).content.decode() - # with open("temp.html", "w") as f: - # f.write(html_) - - tree = html.fromstring(html_) - qnas = [] - # we're not doing proper HTML scraping here really... since the site uses client side rendering - # we rather parse the JS scripts to get the JSON payload of useful information... sadly this looks horrible - for i, script in enumerate(tree.xpath("/html/body/script"), 0): - js = str(script.text) - if "receivedFeedback" in js: - # my god this is horrible... - parsed_json = json.loads(json.loads(js[19:-1])[1][2:])[0][3]["loadedUser"]["receivedFeedback"] - # print(json.dumps(parsed_json, indent = 4)) - # with open("whispas_%i.json" % i, "w") as f: - # json.dump(parsed_json, f, indent = 4) - for j in parsed_json: - if j["_count"]["childFeedback"] < 0: - continue - - answer_url = "https://apiv4.whispa.sh/feedbacks/%s/children/public" % j["id"] - req = query_answer(answer_url) - try: - firstanswer = req.json()["data"][0] - except IndexError: - continue - dt = datetime.datetime.fromisoformat(firstanswer["createdAt"][:-1]) - - qna = { - # "id": int(j["id"], base = 16), - "id": int(dt.timestamp()), - "link": answer_url, - "datetime": dt, - "question": j["content"], - "answer": firstanswer["content"], - "host": "whispa.sh" - } - print(qna) - qnas.append(qna) - time.sleep(2.03) - if dt <= stop_at: - print("Met the threshold for oldest Q&A, so stopped looking.") - break - return qnas - -def get_docker_containers(host, ssh_key_path): - result = fabric.Connection( - host = host, - user = "root", - connect_kwargs = { - "key_filename": ssh_key_path, - "look_for_keys": False - } - ).run('docker ps -a -s --format "table {{.Names}};{{.Status}};{{.Image}}"', hide = True) - return [line.split(";") for line in result.stdout.split("\n")[1:-1]] - -def cache_all_docker_containers(ssh_key_path): - containers = {} - containers["containers"] = {} - for host, name in CONFIG["docker_hosts"].items(): - print(host) - containers["containers"][(host, name)] = get_docker_containers(host, ssh_key_path) - - containers["cachetime"] = "Docker information last updated at %s" % str(datetime.datetime.now()) - with open("/tmp/docker-cache.json", "wb") as f: - pickle.dump(containers, f) - -def get_all_docker_containers(): - if not os.path.exists("/tmp/docker-cache.json"): - return {"containers": {}, "cachetime": "No cached docker information"} - - with open("/tmp/docker-cache.json", "rb") as f: - return pickle.load(f) - -def timeout(func): - # cant get this to work with queue.Queue() for some reason? - # this works but Manager() uses an extra thread than Queue() - manager = multiprocessing.Manager() - returnVan = manager.list() - # ti = time.time() - - def runFunc(q, func): - q.append(func()) - - def beginTimeout(): - t = multiprocessing.Process(target = runFunc, args = (returnVan, func)) - t.start() - - t.join(timeout = CONFIG["servicetimeout"].getint("seconds")) - - # print("Request took:", time.time() - ti) - try: - return returnVan[0] - except IndexError: - if t.is_alive(): - t.terminate() - - return beginTimeout - -@timeout -def get_torrent_stats(): - client = transmission_rpc.client.Client( - host = CONFIG.get("transmission", "host") - ) - s = vars(client.session_stats())["fields"] - return { - "Active torrents:": s["activeTorrentCount"], - "Downloaded:": humanbytes(s["cumulative-stats"]["downloadedBytes"]), - "Uploaded:": humanbytes(s["cumulative-stats"]["uploadedBytes"]), - "Active time:": str(datetime.timedelta(seconds = s["cumulative-stats"]["secondsActive"])), - "Files added:": s["cumulative-stats"]["filesAdded"], - "Current upload speed:": humanbytes(s["uploadSpeed"]) + "s/S", - "Current download speed:": humanbytes(s["downloadSpeed"]) + "s/S" - } - -@timeout -def get_pihole_stats(): - return PiHole.GetSummary(CONFIG.get("pihole", "url"), CONFIG.get("pihole", "key"), True) - -def get_recent_commits(db, max_per_repo = 3): - cache = db.get_cached_commits() - num_per_repo = {} - out = [] - for commit in cache: - if commit["repo"] not in num_per_repo.keys(): - num_per_repo[commit["repo"]] = 0 - - num_per_repo[commit["repo"]] += 1 - if num_per_repo[commit["repo"]] <= max_per_repo: - out.append(commit) - - return sorted(out, key = lambda a: a["datetime"], reverse = True) - -if __name__ == "__main__": - print(scrape_whispa(CONFIG.get("qnas", "url"))) - # import database - - # with database.Database() as db: - # print(json.dumps(get_recent_commits(db), indent=4)) +from dataclasses import dataclass +from io import StringIO +from lxml import html, etree +from github import Github +import multiprocessing +import paramiko.client +from APiHole import PiHole +import transmission_rpc +import configparser +import math as maths +import requests +import datetime +import urllib +import docker +import random +import subprocess +import fabric +import pickle +import queue +import json +import time +import os + +theLastId = 0 +config_path = os.path.join(os.path.dirname(__file__), "..", "edaweb.conf") +if not os.path.exists(config_path): + raise FileNotFoundError("Could not find edaweb.conf config file") +CONFIG = configparser.ConfigParser(interpolation = None) +CONFIG.read(config_path) + +def humanbytes(B): + 'Return the given bytes as a human friendly KB, MB, GB, or TB string' + B = float(B) + KB = float(1024) + MB = float(KB ** 2) # 1,048,576 + GB = float(KB ** 3) # 1,073,741,824 + TB = float(KB ** 4) # 1,099,511,627,776 + + if B < KB: + return '{0} {1}'.format(B,'Bytes' if 0 == B > 1 else 'Byte') + elif KB <= B < MB: + return '{0:.2f} KB'.format(B/KB) + elif MB <= B < GB: + return '{0:.2f} MB'.format(B/MB) + elif GB <= B < TB: + return '{0:.2f} GB'.format(B/GB) + elif TB <= B: + return '{0:.2f} TB'.format(B/TB) + +@dataclass +class SafebooruImage: + id_: int + url: str + searchTags: list + tags: list + source: str + imurl: str + + def remove_tag(self, tag): + return list(set(self.searchTags).difference(set([tag]))) + +@dataclass +class DownloadedImage: + imurl: str + + def __enter__(self): + self.filename = os.path.join("static", "images", "random.jpg") + + req = urllib.request.Request(self.imurl, headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_5_8) AppleWebKit/534.50.2 (KHTML, like Gecko) Version/5.0.6 Safari/533.22.3'}) + mediaContent = urllib.request.urlopen(req).read() + with open(self.filename, "wb") as f: + f.write(mediaContent) + return self.filename + + def __exit__(self, type, value, traceback): + os.remove(self.filename) + +def get_num_pages(tags): + pages_url = "https://safebooru.org/index.php?page=post&s=list&tags=%s" % "+".join(tags) + tree = html.fromstring(requests.get(pages_url).content) + try: + finalpage_element = tree.xpath("/html/body/div[6]/div/div[2]/div[2]/div/a[12]")[0] + except IndexError: + return 1 + else: + return int(int(urllib.parse.parse_qs(finalpage_element.get("href"))["pid"][0]) / (5*8)) + +def get_id_from_url(url): + return int(urllib.parse.parse_qs(url)["id"][0]) + +def get_random_image(tags): + global theLastId + searchPage = random.randint(1, get_num_pages(tags)) * 5 * 8 + url = "https://safebooru.org/index.php?page=post&s=list&tags=%s&pid=%i" % ("+".join(tags), searchPage) + tree = html.fromstring(requests.get(url).content) + + imageElements = [e for e in tree.xpath("/html/body/div[6]/div/div[2]/div[1]")[0].iter(tag = "a")] + try: + element = random.choice(imageElements) + except IndexError: + # raise ConnectionError("Couldn't find any images") + return get_random_image(tags) + + url = "https://safebooru.org/" + element.get("href") + if get_id_from_url(url) == theLastId: + return get_random_image(tags) + theLastId = get_id_from_url(url) + + try: + sbi = SafebooruImage( + id_ = get_id_from_url(url), + url = url, + tags = element.find("img").get("alt").split(), + searchTags = tags, + source = fix_source_url(get_source(url)), + imurl = get_imurl(url) + ) + except (ConnectionError, KeyError) as e: + print("[ERROR]", e) + return get_random_image(tags) + + if link_deleted(sbi.url): + print("Retried since the source was deleted...") + return get_random_image(tags) + + return sbi + +def get_source(url): + tree = html.fromstring(requests.get(url).content) + for element in tree.xpath('//*[@id="stats"]')[0].iter("li"): + if element.text.startswith("Source: h"): + return element.text[8:] + elif element.text.startswith("Source:"): + for child in element.iter(): + if child.get("href") is not None: + return child.get("href") + raise ConnectionError("Couldn't find source image for id %i" % get_id_from_url(url)) + +def fix_source_url(url): + parsed = urllib.parse.urlparse(url) + if parsed.netloc == "www.pixiv.net": + return "https://www.pixiv.net/en/artworks/" + urllib.parse.parse_qs(parsed.query)["illust_id"][0] + elif parsed.netloc in ["bishie.booru.org", "www.secchan.net"]: + return ConnectionError("Couldn't get source") + elif "pximg.net" in parsed.netloc or "pixiv.net" in parsed.netloc: + return "https://www.pixiv.net/en/artworks/" + parsed.path.split("/")[-1][:8] + elif parsed.netloc == "twitter.com": + return url.replace("twitter.com", "nitter.eda.gay") + return url + +def get_imurl(url): + tree = html.fromstring(requests.get(url).content) + return tree.xpath('//*[@id="image"]')[0].get("src") + +def link_deleted(url): + text = requests.get(url).text + return text[text.find("") + 7 : text.find("")] in ["Error | nitter", "イラストコミュニケーションサービス[pixiv]"] + +def request_recent_commits(since = datetime.datetime.now() - datetime.timedelta(days=7)): + g = Github(CONFIG.get("github", "access_code")) + out = [] + for repo in g.get_user().get_repos(): + # print(repo.name, list(repo.get_branches())) + try: + for commit in repo.get_commits(since = since): + out.append({ + "repo": repo.name, + "message": commit.commit.message, + "url": commit.html_url, + "datetime": commit.commit.author.date, + "stats": { + "additions": commit.stats.additions, + "deletions": commit.stats.deletions, + "total": commit.stats.total + } + }) + except Exception as e: + print(repo, e) + + return sorted(out, key = lambda a: a["datetime"], reverse = True) + +def scrape_nitter(username, get_until:int): + new_tweets = [] + nitter_url = CONFIG.get("nitter", "internalurl") + nitter_port = CONFIG.getint("nitter", "internalport") + scrape_new_pages = True + url = "http://%s:%d/%s" % (nitter_url, nitter_port, username) + + while scrape_new_pages: + tree = html.fromstring(requests.get(url).content) + for i, tweetUrlElement in enumerate(tree.xpath('//*[@class="tweet-link"]'), 0): + if i > 0 and tweetUrlElement.get("href").split("/")[1] == username: + id_ = int(urllib.parse.urlparse(tweetUrlElement.get("href")).path.split("/")[-1]) + tweet_link = "http://%s:%d%s" % (nitter_url, nitter_port, tweetUrlElement.get("href")) + + if id_ == get_until: + scrape_new_pages = False + break + + try: + dt, replying_to, text, images = parse_tweet(tweet_link) + new_tweets.append((id_, dt, replying_to, text, username, images)) + print(dt, "'%s'" % text) + except IndexError: + print("Couldn't get any more tweets") + scrape_new_pages = False + break + except ConnectionError: + print("Rate limited, try again later") + return [] + + + try: + cursor = tree.xpath('//*[@class="show-more"]/a')[0].get("href") + except IndexError: + # no more elements + break + url = "http://%s:%d/%s%s" % (nitter_url, nitter_port, username, cursor) + + return new_tweets + +def parse_tweet(tweet_url): + # print(tweet_url) + tree = html.fromstring(requests.get(tweet_url).content) + # with open("2images.html", "r") as f: + # tree = html.fromstring(f.read()) + + rate_limited_elem = tree.xpath("/html/body/div/div/div/span") + if rate_limited_elem != []: + if rate_limited_elem[0].text == "Instance has been rate limited.": + raise ConnectionError("Instance has been rate limited.") + + main_tweet_elem = tree.xpath('//*[@class="main-tweet"]')[0] + + dt_str = main_tweet_elem.xpath('//*[@class="tweet-published"]')[0].text + dt = datetime.datetime.strptime(dt_str.replace("Â", ""), "%b %d, %Y · %I:%M %p UTC") + text = tree.xpath('//*[@class="main-tweet"]/div/div/div[2]')[0].text_content() + if text == "": + text = "[Image only]" + replying_to_elems = tree.xpath('//*[@class="before-tweet thread-line"]/div/a') + if replying_to_elems != []: + replying_to = int(urllib.parse.urlparse(replying_to_elems[-1].get("href")).path.split("/")[-1]) + else: + replying_to = None + + images = [] + images_elems = tree.xpath('//*[@class="main-tweet"]/div/div/div[3]/div/div/a/img') + for image_elem in images_elems: + images.append("https://" + CONFIG.get("nitter", "outsideurl") + urllib.parse.urlparse(image_elem.get("src")).path) + + return dt, replying_to, text, images + +def scrape_whispa(whispa_url, since = None): + def query_answer(answer_url, max_retries = 10): + for i in range(max_retries): + try: + return requests.get(answer_url) + except requests.exceptions.ConnectionError: + s = 5.05 * (i + 1) + print("Connection timed out, retrying in %.2fs" % s) + time.sleep(s) + continue + + # add a bit of wiggle room in case i don't answer the questions in order (i often do this) + if since is None: + stop_at = datetime.datetime(year = 2001, month = 8, day = 12) + else: + stop_at = since - datetime.timedelta(days = 14) + print("The newest Q&A timestamp in the database was %s, we will stop looking at %s." % (since.astimezone().isoformat(), stop_at.astimezone().isoformat())) + + html_ = requests.get(whispa_url).content.decode() + # with open("temp.html", "w") as f: + # f.write(html_) + + tree = html.fromstring(html_) + qnas = [] + # we're not doing proper HTML scraping here really... since the site uses client side rendering + # we rather parse the JS scripts to get the JSON payload of useful information... sadly this looks horrible + for i, script in enumerate(tree.xpath("/html/body/script"), 0): + js = str(script.text) + if "receivedFeedback" in js: + # my god this is horrible... + parsed_json = json.loads(json.loads(js[19:-1])[1][2:])[0][3]["loadedUser"]["receivedFeedback"] + # print(json.dumps(parsed_json, indent = 4)) + # with open("whispas_%i.json" % i, "w") as f: + # json.dump(parsed_json, f, indent = 4) + for j in parsed_json: + if j["_count"]["childFeedback"] < 0: + continue + + answer_url = "https://apiv4.whispa.sh/feedbacks/%s/children/public" % j["id"] + req = query_answer(answer_url) + try: + firstanswer = req.json()["data"][0] + except IndexError: + continue + dt = datetime.datetime.fromisoformat(firstanswer["createdAt"][:-1]) + + qna = { + # "id": int(j["id"], base = 16), + "id": int(dt.timestamp()), + "link": answer_url, + "datetime": dt, + "question": j["content"], + "answer": firstanswer["content"], + "host": "whispa.sh" + } + print(qna) + qnas.append(qna) + time.sleep(2.03) + if dt <= stop_at: + print("Met the threshold for oldest Q&A, so stopped looking.") + break + return qnas + +def get_docker_containers(host, ssh_key_path): + result = fabric.Connection( + host = host, + user = "root", + connect_kwargs = { + "key_filename": ssh_key_path, + "look_for_keys": False + } + ).run('docker ps -a -s --format "table {{.Names}};{{.Status}};{{.Image}}"', hide = True) + return [line.split(";") for line in result.stdout.split("\n")[1:-1]] + +def cache_all_docker_containers(ssh_key_path): + containers = {} + containers["containers"] = {} + for host, name in CONFIG["docker_hosts"].items(): + print(host) + containers["containers"][(host, name)] = get_docker_containers(host, ssh_key_path) + + containers["cachetime"] = "Docker information last updated at %s" % str(datetime.datetime.now()) + with open("/tmp/docker-cache.json", "wb") as f: + pickle.dump(containers, f) + +def get_all_docker_containers(): + if not os.path.exists("/tmp/docker-cache.json"): + return {"containers": {}, "cachetime": "No cached docker information"} + + with open("/tmp/docker-cache.json", "rb") as f: + return pickle.load(f) + +def timeout(func): + # cant get this to work with queue.Queue() for some reason? + # this works but Manager() uses an extra thread than Queue() + manager = multiprocessing.Manager() + returnVan = manager.list() + # ti = time.time() + + def runFunc(q, func): + q.append(func()) + + def beginTimeout(): + t = multiprocessing.Process(target = runFunc, args = (returnVan, func)) + t.start() + + t.join(timeout = CONFIG["servicetimeout"].getint("seconds")) + + # print("Request took:", time.time() - ti) + try: + return returnVan[0] + except IndexError: + if t.is_alive(): + t.terminate() + + return beginTimeout + +@timeout +def get_torrent_stats(): + client = transmission_rpc.client.Client( + host = CONFIG.get("transmission", "host") + ) + s = vars(client.session_stats())["fields"] + return { + "Active torrents:": s["activeTorrentCount"], + "Downloaded:": humanbytes(s["cumulative-stats"]["downloadedBytes"]), + "Uploaded:": humanbytes(s["cumulative-stats"]["uploadedBytes"]), + "Active time:": str(datetime.timedelta(seconds = s["cumulative-stats"]["secondsActive"])), + "Files added:": s["cumulative-stats"]["filesAdded"], + "Current upload speed:": humanbytes(s["uploadSpeed"]) + "s/S", + "Current download speed:": humanbytes(s["downloadSpeed"]) + "s/S" + } + +@timeout +def get_pihole_stats(): + return PiHole.GetSummary(CONFIG.get("pihole", "url"), CONFIG.get("pihole", "key"), True) + +def get_recent_commits(db, max_per_repo = 3): + cache = db.get_cached_commits() + num_per_repo = {} + out = [] + for commit in cache: + if commit["repo"] not in num_per_repo.keys(): + num_per_repo[commit["repo"]] = 0 + + num_per_repo[commit["repo"]] += 1 + if num_per_repo[commit["repo"]] <= max_per_repo: + out.append(commit) + + return sorted(out, key = lambda a: a["datetime"], reverse = True) + +if __name__ == "__main__": + print(scrape_whispa(CONFIG.get("qnas", "url"))) + # import database + + # with database.Database() as db: + # print(json.dumps(get_recent_commits(db), indent=4)) diff --git a/edaweb/static/cow.txt b/edaweb/static/cow.txt new file mode 100644 index 0000000..41aa536 --- /dev/null +++ b/edaweb/static/cow.txt @@ -0,0 +1,58 @@ + + + + + + + .......................... + ....*o|||||||8#@@@@@@@@@@@@@@@@@@@@@@@###&|o:_.. + ..*:o|||&8##@###8888888######@#@###########################|*... + .:o|||8#####8888|:::**. *&########################@@################&o_ + .*o&8###@#8&o*_. :###@##############@########################@@##&o_ + .*o8########& :##@#@##############@############################@###|_ + .*o|8##########8o .#######################################################&o_ + *&##|_ ..*&##8&o*|88888|_ _#######################################@##################|. + *#####& *&######&o_..*o|o:_ .&##o _###########################################################&_ + _##8*##8 .|88|:::|#######8###8|*:_ .&#@@8 _##@@@########################################################&_ + _#@8_##8_ *8#8|*_ _:|#####&&####8 .&##############################################################| + _#@8.|##8_ _::o###8&##8 .|##@############################8###########################@@#|_ + *###o.|88o ..*&####|..##& _|##########################8|_ .|#############################8 + *|###|_ ._&####8|*_ _*_ _::&8888888888888888|::*_ .|##@####@@@##################| + *&###|_ _:_ .&88###8|*_ ..... .|#####@@@##################8 + .##@#& _##& .|##o _#@@#@#| .|#######&:_ _|###@####################8 + .:8##8o _o:*&##| *##8_.&@@##@#| _::o8#8|::|#####|_ _|#################88###8 + .&##&*_ *###o_###| .|##8*&##|*###o _###8####8|_ _:|###|_ .*o|||o:_ _:::&8888888888|_ _##8 + .###|_. _###o *###|*&#######8 *##8 .##8_ _:|###|_ _|###|_ .&########o _##& + _|####8|&##8:_ _|#########88o .##8 *##& _|###o .|###o .#########| .o##o + o#8|*:#@@###o _:::*__*_ _##8_ _##8_ _&##|_ *##8_ *8#####8|_ .*oo:_ o##| + *###o.&#####& _oo* .8##& .8##8_ .|##& o##& _::::_.*o|8######|_ .##8. + _###&o&##8_:*_ .###& .###|_&#####| _##8 :###o *ooo&#########@#@#& ....:##& + .|8||###&. _**_ .###88##|*&###|*._&##& *|##8_ *o&####@@####@@######& .*o||||||&#######8_ + *&###o _|88##8_ _:8######|*:###|_ _##################88|_ *&#################& + *#####o *&8o *##& _:::*_.&##|_ _#@##############8_ :##################8* + .###&##8_.|88o *&8o _@@& .###| .&####@#########|_ .####@###@@########8* + _##&.|###|_.... .|88o _##8* *&###|_ *###o _:&88######8|_ .*o|||##################o + _##8_ _|########|_ .*o8####&#@@#@##o *#@8 _*:*. .&###@###################| + .|##& _:::::&##& .*&##############@#8_ .###o _#@####################|_ + .&##|_ .&##8_ *o&####################8_ *##& .&#####@@############8* + .|##8_.&###&####8_ _########################8**##& _##################|_ + .&###&##888888##8_ .|88######@###########|*######o _|8###############|_ + .&#####o *###|_ _::::::*o##8**o##8 .|###8o .&##@#############&* + .|####o _|###|_ _##8.*&##& _*_ .#################| + _*_ _|###|_.. .|#####8|_ *&#@#########8###& + *#######&|o:_... ..*:::*. ......._:o&8####888&o:#####8_ + _###|&888#####@#####&|||o:_........................._:o||||8##@@@@####8|:*_ _:::*_ + .|#@###o _:::o#@#888######@@@@@@@@@@@@@@@@@@@@@@@@#####888|::::::**_ + _::*_ :##& *&8|_:::::::::::::::::::::::::**_ + .###8||&##8o + _|888888|_ + + + + + + + + + + diff --git a/edaweb/static/index.md b/edaweb/static/index.md index 0d45f12..6259ba6 100644 --- a/edaweb/static/index.md +++ b/edaweb/static/index.md @@ -1,34 +1,34 @@ -site now also avaliable under the domain [boymoder.blog](https://boymoder.blog)! - -![yaoi](/img/shun-hashimoto-mio-chibana.gif) - -## haiiiiiii -my name is eden and im a 23yo (boymoder/[fujoshi](https://www.urbandictionary.com/define.php?term=fujoshi)) computer science/robotics PhD student. i made my own website to encourage others to do so too. -i'll post my thoughts on here sometimes, and use this site to link to other stuff i host [more about me](/thought?id=2). - -## FOSS alternative services - -- [nextcloud - dropbox (+ much more!) alternative](https://nc.eda.gay) -- [git server - github alternative](https://git.eda.gay/) -- [jellyfin - web player for ~~legally downloaded~~ TV and films](https://jellyfin.eda.gay) - RIP emby! - -[see the services im running right now](/services) (takes a couple seconds to load) - -these sites are hosted on my [homelab system](https://wiki.eda.gay) - -![startech 8u rack cropped](/img/GcyexeCW0AAYssz.jpg?w=300&h=5000) -![startech 12u rack cropped](/img/Startech.jpg?h=250&w=5000) - -## nice websites -- [wiby.me](http://wiby.me/) - search engine for old style websites with limited javascript (my site used to be on here but it got blacklisted for some reason?) -- [dysmorph.nekoweb.org](https://dysmorph.nekoweb.org/) - a site that is very based because it looks similar -- [transsexual.org](https://web.archive.org/web/20010802032136/http://transsexual.org/Toon.html) - awesome and relatable transsexual comics from a website that's slightly older than me -- [norfolkchurches.co.uk](http://www.norfolkchurches.co.uk/norwichintro.htm) - site about all the churches in norwich (and norfolk!), the city that has far too many medieval churches than it knows what to do with. this site is preciesly what the internet should be, the muted ramblings of someone with an expert knowledge on his preferred niche interest. without any javascript. nice if, like me, you have a middling interest in theology -- [boymoder.network](https://boymoder.network/) - website for boymoder awareness -- [4chan.org/lgbt/](https://boards.4channel.org/lgbt/) - but dont blame me if u catch brainworms -- [https://www.math.uni-bielefeld.de/~sillke/Twister/fun/elevator-fun90.html](https://www.math.uni-bielefeld.de/~sillke/Twister/fun/elevator-fun90.html) any website with a URL like this is gonna be good -- [boymoder.moe](https://nyaomidev.github.io/boymoder.moe/) -- [boymoders.com](https://boymoders.com) -- [john.citrons.xyz](https://john.citrons.xyz/) - for the nice 'ads' featured at the bottom of my page - - +site now also avaliable under the domain [boymoder.blog](https://boymoder.blog)! + +![yaoi](/img/shun-hashimoto-mio-chibana.gif) + +## haiiiiiii +my name is eden and im a 23yo (boymoder/[fujoshi](https://www.urbandictionary.com/define.php?term=fujoshi)) computer science/robotics PhD student. i made my own website to encourage others to do so too. +i'll post my thoughts on here sometimes, and use this site to link to other stuff i host [more about me](/thought?id=2). + +## FOSS alternative services + +- [nextcloud - dropbox (+ much more!) alternative](https://nc.eda.gay) +- [git server - github alternative](https://git.eda.gay/) +- [jellyfin - web player for ~~legally downloaded~~ TV and films](https://jellyfin.eda.gay) - RIP emby! + +[see the services im running right now](/services) (takes a couple seconds to load) + +these sites are hosted on my [homelab system](https://wiki.eda.gay) + +![startech 8u rack cropped](/img/GcyexeCW0AAYssz.jpg?w=300&h=5000) +![startech 12u rack cropped](/img/Startech.jpg?h=250&w=5000) + +## nice websites +- [wiby.me](http://wiby.me/) - search engine for old style websites with limited javascript (my site used to be on here but it got blacklisted for some reason?) +- [dysmorph.nekoweb.org](https://dysmorph.nekoweb.org/) - a site that is very based because it looks similar +- [transsexual.org](https://web.archive.org/web/20010802032136/http://transsexual.org/Toon.html) - awesome and relatable transsexual comics from a website that's slightly older than me +- [norfolkchurches.co.uk](http://www.norfolkchurches.co.uk/norwichintro.htm) - site about all the churches in norwich (and norfolk!), the city that has far too many medieval churches than it knows what to do with. this site is preciesly what the internet should be, the muted ramblings of someone with an expert knowledge on his preferred niche interest. without any javascript. nice if, like me, you have a middling interest in theology +- [boymoder.network](https://boymoder.network/) - website for boymoder awareness +- [4chan.org/lgbt/](https://boards.4channel.org/lgbt/) - but dont blame me if u catch brainworms +- [https://www.math.uni-bielefeld.de/~sillke/Twister/fun/elevator-fun90.html](https://www.math.uni-bielefeld.de/~sillke/Twister/fun/elevator-fun90.html) any website with a URL like this is gonna be good +- [boymoder.moe](https://nyaomidev.github.io/boymoder.moe/) +- [boymoders.com](https://boymoders.com) +- [john.citrons.xyz](https://john.citrons.xyz/) - for the nice 'ads' featured at the bottom of my page + + -- cgit v1.2.3