added caching tweets, added git commits

5 files changed, 145 insertions, 31 deletions

diff --git a/app.py b/app.py
index 60b0f03..c7d8756 100644
--- a/app.py
+++ b/app.py
@@ -54,7 +54,8 @@ def index():
                 **get_template_items("eden's site :3", db),
                 markdown = parser.parse_text(f.read()),
                 featured_thoughts = db.get_featured_thoughts(),
-                tweets = services.get_recent_tweets(6)
+                tweets = db.get_cached_tweets(7) + [("view all tweets...", db.get_my_twitter())],
+                commits = db.get_cached_commits(since = datetime.datetime.now() - datetime.timedelta(days = 7))
             )
 
 @app.route("/discord")
@@ -131,6 +132,21 @@ def serve_image(filename):
     else:
         flask.abort(404)
 
+@app.route("/api/<infoRequest>")
+def serve_api_request(infoRequest):
+    if infoRequest == "commits":
+        try:
+            return flask.jsonify(services.request_recent_commits(since = datetime.datetime.fromtimestamp(int(flask.request.args['since']))))
+        except (ValueError, KeyError):
+            flask.abort(400)
+    elif infoRequest == "tweets":
+        try:
+            return flask.jsonify(services.request_recent_tweets(int(flask.request.args['toGet'])))
+        except (ValueError, KeyError):
+            flask.abort(400)
+    else:
+        flask.abort(404)
+
 
 @app.route("/preview")
 def preview():
diff --git a/database.py b/database.py
index 5e50f82..0edcad7 100644
--- a/database.py
+++ b/database.py
@@ -1,7 +1,14 @@
+from urllib.parse import urlparse
 from dataclasses import dataclass
+from github import Github
+from lxml import html
 import configparser
+import threading
+import datetime
+import requests
 import pymysql
 import random
+import os
 
 @dataclass
 class Database:
@@ -107,6 +114,115 @@ class Database:
             """)
             return cursor.fetchall()
 
+    def get_cached_tweets(self, numToGet = None, recurse = True):
+        with self.__connection.cursor() as cursor:
+            if numToGet is not None:
+                cursor.execute("SELECT text, url FROM twitterCache ORDER BY appended DESC LIMIT %s;", (numToGet, ))
+            else:
+                cursor.execute("SELECT text, url FROM twitterCache ORDER BY appended DESC;")
+            if recurse:
+                threading.Thread(target = update_cache).start()
+            return list(cursor.fetchall())
+
+    def update_twitter_cache(self, requested):
+        urls = [i[1] for i in self.get_cached_tweets(recurse = False)]
+        with self.__connection.cursor() as cursor:
+            for url, text in requested:
+                if url not in urls:
+                    cursor.execute("INSERT INTO twitterCache (text, url) VALUES (%s, %s);", (text, url))
+        self.__connection.commit()
+
+    def get_cached_commits(self, since = None, recurse = True):
+        with self.__connection.cursor() as cursor:
+            if since is not None:
+                cursor.execute("SELECT message, url, commitTime, additions, deletions, total FROM commitCache WHERE commitTime > %s ORDER BY commitTime DESC;", (since, ))
+            else:
+                cursor.execute("SELECT message, url, commitTime, additions, deletions, total FROM commitCache ORDER BY commitTime DESC;")
+            return [{
+                "repo": urlparse(i[1]).path.split("/")[2],
+                "message": i[0],
+                "url": i[1],
+                "datetime": i[2],
+                "stats": {
+                    "additions": i[3],
+                    "deletions": i[4],
+                    "total": i[5]
+                }
+            } for i in cursor.fetchall()]
+
+    def update_commit_cache(self, requested):
+        urls = [i["url"] for i in self.get_cached_commits(recurse = False)]
+        with self.__connection.cursor() as cursor:
+            for commit in requested:
+                if commit["url"] not in urls:
+                    cursor.execute("""
+                    INSERT INTO commitCache (message, url, commitTime, additions, deletions, total)
+                    VALUES (%s, %s, %s, %s, %s, %s)""",
+                    (commit["message"], commit["url"], commit["datetime"], commit["stats"]["additions"], commit["stats"]["deletions"], commit["stats"]["total"])
+                )
+        self.__connection.commit()
+
+    def get_last_commit_time(self):
+        with self.__connection.cursor() as cursor:
+            cursor.execute("SELECT MAX(commitTime) FROM commitCache;")
+            return cursor.fetchone()[0]
+
+    def get_my_twitter(self):
+        with self.__connection.cursor() as cursor:
+            cursor.execute("SELECT link FROM headerLinks WHERE name = 'twitter';")
+            return cursor.fetchone()[0]
+
+def update_cache():
+    # print("updating cache...")
+    with Database() as db:
+        db.update_twitter_cache(request_recent_tweets(10000))
+        # print("Done updating twitter cache...")
+        db.update_commit_cache(request_recent_commits(since = db.get_last_commit_time()))
+        # print("Done updating commit cache...")
+
+CONFIG = configparser.ConfigParser()
+CONFIG.read("edaweb.conf")
+
+def request_recent_tweets(numToGet):
+    tweets = []
+    domain = "http://" + CONFIG.get("nitter", "domain")
+    with Database() as db:
+        for title, url in db.get_header_links():
+            if title == "twitter":
+                break
+    tree = html.fromstring(requests.get(url).content)
+    for i, tweetUrlElement in enumerate(tree.xpath('//*[@class="tweet-link"]'), 0):
+        if i > 0:
+            tweets.append((
+                domain + tweetUrlElement.get("href"),
+                tweetUrlElement.getparent().find_class("tweet-content media-body")[0].text
+            ))
+        if len(tweets) >= numToGet:
+            break
+    return tweets
+            
+def request_recent_commits(since = datetime.datetime.now() - datetime.timedelta(days=7)):
+    g = Github(CONFIG.get("github", "access_code"))
+    out = []
+    for repo in g.get_user().get_repos():
+        # print(repo.name, list(repo.get_branches()))
+        for commit in repo.get_commits(since = since):
+            out.append({
+                "repo": repo.name,
+                "message": commit.commit.message,
+                "url": commit.html_url,
+                "datetime": commit.commit.author.date,
+                "stats": {
+                    "additions": commit.stats.additions,
+                    "deletions": commit.stats.deletions,
+                    "total": commit.stats.total
+                }
+            })
+    return sorted(out, key = lambda a: a["datetime"], reverse = True) 
+
 if __name__ == "__main__":
+    import datetime
+    start = datetime.datetime.now()
     with Database() as db:
-        print(db.get_header_articles())
\ No newline at end of file
+        print(db.get_cached_tweets())
+        print("Took: ", (datetime.datetime.now() - start))
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 32044b1..3644402 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,3 +8,4 @@ docker
 PiHole-api
 Pillow==8.1.0
 python-qbittorrent==0.4.2
+PyGithub
diff --git a/services.py b/services.py
index 2cfe394..4f014d8 100644
--- a/services.py
+++ b/services.py
@@ -1,6 +1,5 @@
 from dataclasses import dataclass
 from io import StringIO
-from lxml import html
 import multiprocessing
 import pihole as ph
 import qbittorrent
@@ -112,30 +111,4 @@ def get_pihole_stats():
         "domains": pihole.domain_count,
         "last_updated": str(datetime.datetime.fromtimestamp(pihole.gravity_last_updated["absolute"]))
     }
-
-# @timeout
-def get_recent_tweets(numToGet):
-    tweets = []
-    domain = "http://" + app.CONFIG.get("nitter", "domain")
-    with app.database.Database() as db:
-        for title, url in db.get_header_links():
-            if title == "twitter":
-                break
-    tree = html.fromstring(requests.get(url).content)
-    for i, tweetUrlElement in enumerate(tree.xpath('//*[@class="tweet-link"]'), 0):
-        if i > 0:
-            tweets.append((
-                domain + tweetUrlElement.get("href"),
-                tweetUrlElement.getparent().find_class("tweet-content media-body")[0].text
-            ))
-        if len(tweets) >= numToGet:
-            break
-    return tweets + [(url, "view all tweets...")]
-            
-
-
-if __name__ == "__main__":
-    for tweet in get_recent_tweets():
-        print(tweet.get_url())
-        print(tweet.get_text())
-        print()
\ No newline at end of file
+ 
diff --git a/templates/index.html b/templates/index.html
index d79743b..fd0bae1 100644
--- a/templates/index.html
+++ b/templates/index.html
@@ -15,7 +15,7 @@
         <section id="recent_tweets">
             <h1>recent tweets</h1>
             <ul>
-                {% for url, text in tweets %}
+                {% for text, url in tweets %}
                     {% if text == None %}
                         <li><a href={{url}}>[image only]</a></li>
                     {% else %}
@@ -25,4 +25,12 @@
             </ul>
         </section>
     {% endif %}
+    <section id="recent commits">
+        <h1>recent git commits:</h4>
+        <ul>
+            {% for commit in commits %}
+                <li><a href={{commit["url"]}}>{{"[%s] %s {+%i;-%i}" % (commit["repo"], commit["message"], commit["stats"]["additions"], commit["stats"]["deletions"])}}</a></li>
+            {% endfor %}
+        </ul>
+    </section>
 {% endblock %}
\ No newline at end of file