diff options
| -rwxr-xr-x | .gitignore | 6 | ||||
| -rwxr-xr-x | Dockerfile | 4 | ||||
| -rwxr-xr-x | Dockerfile_cron | 11 | ||||
| -rw-r--r-- | cron_entrypoint.sh | 6 | ||||
| -rwxr-xr-x | docker-compose.yml | 26 | ||||
| -rw-r--r-- | edaweb/database.py | 4 | ||||
| -rw-r--r-- | edaweb/services.py | 12 | ||||
| -rw-r--r-- | edaweb/static/images/random.jpg | bin | 202536 -> 0 bytes | |||
| -rw-r--r-- | edaweb/static/robots.txt | 4 | ||||
| -rw-r--r-- | entrypoint.sh | 6 | ||||
| -rw-r--r-- | papers.md | 25 |
11 files changed, 61 insertions, 43 deletions
@@ -2,10 +2,10 @@ *.pub homelab-wiki/wiki.env homelab-wiki/images/* -edaweb.conf +*edaweb.conf markdowns/ -static/images/random.jpg -static/zips/*.zip +*random.jpg +*.zip .nfs* static/images/Thumbs.db nitter/nitter.conf @@ -3,12 +3,10 @@ MAINTAINER Eden Attenborough "eden.attenborough@outlook.com" ENV TZ=Europe/London RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone RUN apt-get update -y -RUN apt-get install -y python3-pip python3-dev build-essential clang libffi-dev libxml2-dev libxslt-dev libjpeg-dev zlib1g-dev tmux cron +RUN apt-get install -y python3-pip python3-dev build-essential clang libffi-dev libxml2-dev libxslt-dev libjpeg-dev zlib1g-dev COPY . /app WORKDIR /app RUN pip3 install -r requirements.txt -RUN echo "*/30 * * * * root python3 /app/edaweb/cache.py > /proc/1/fd/1 2>/proc/1/fd/2" > /etc/crontab - ENTRYPOINT ["bash"] CMD ["entrypoint.sh"] diff --git a/Dockerfile_cron b/Dockerfile_cron new file mode 100755 index 0000000..dea3208 --- /dev/null +++ b/Dockerfile_cron @@ -0,0 +1,11 @@ +FROM reg.reaweb.uk/edaweb +MAINTAINER Eden Attenborough "eden.attenborough@outlook.com" +ENV TZ=Europe/London +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +RUN apt-get update -y +RUN apt-get install -y tmux cron + +RUN echo "*/30 * * * * root python3 /app/edaweb/cache.py > /proc/1/fd/1 2>/proc/1/fd/2" > /etc/crontab + +ENTRYPOINT ["bash"] +CMD ["cron_entrypoint.sh"] diff --git a/cron_entrypoint.sh b/cron_entrypoint.sh new file mode 100644 index 0000000..2d98169 --- /dev/null +++ b/cron_entrypoint.sh @@ -0,0 +1,6 @@ +rm -rvf /app/edaweb/edaweb.conf/ +ln -s /app/edaweb.conf /app/edaweb/edaweb.conf +printenv | grep -v "no_proxy" >> /etc/environment +# tmux new-session -d -s "cron" 'cron -f || bash && bash'; +cron -f + diff --git a/docker-compose.yml b/docker-compose.yml index 67976bf..57cd71e 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3' services: edaweb: build: @@ -8,8 +7,8 @@ services: volumes: - /tmp/:/media/ISOs/ - ./edaweb/static/:/app/edaweb/static/ - - ./edaweb.conf:/app/edaweb/edaweb.conf - - ./edaweb-docker.pem:/keys/docker-key.pem + - ./edaweb.conf:/app/edaweb.conf + - edaweb-tmp:/tmp/ ports: - "6969:6969" networks: @@ -19,6 +18,24 @@ services: - mariadb:mysql restart: unless-stopped + edaweb_cron: + build: + context: . + dockerfile: Dockerfile_cron + image: reg.reaweb.uk/edaweb_cron + volumes: + - /tmp/:/media/ISOs/ + - ./edaweb/static/:/app/edaweb/static/ + - ./edaweb.conf:/app/edaweb.conf + - ./edaweb-docker.pem:/keys/docker-key.pem + - edaweb-tmp:/tmp/ + networks: + - db-network + - edaweb-net + external_links: + - mariadb:mysql + restart: unless-stopped + homelab-wiki: build: context: ./homelab-wiki @@ -38,6 +55,9 @@ services: - mariadb:mysql restart: unless-stopped +volumes: + edaweb-tmp: + networks: edaweb-net: external: diff --git a/edaweb/database.py b/edaweb/database.py index dab56e7..49ec33c 100644 --- a/edaweb/database.py +++ b/edaweb/database.py @@ -44,7 +44,7 @@ class Database: def get_header_links(self): with self.__connection.cursor() as cursor: - cursor.execute("SELECT name, link FROM headerLinks ORDER BY name;") + cursor.execute("SELECT name, link FROM headerLinks WHERE display = true ORDER BY name;") return cursor.fetchall() def get_image(self, imageName): @@ -64,7 +64,7 @@ class Database: def get_header_articles(self): with self.__connection.cursor() as cursor: - cursor.execute("SELECT articleName, link FROM headerArticles;") + cursor.execute("SELECT articleName, link FROM headerArticles WHERE display = true;") return cursor.fetchall() def get_all_categories(self): diff --git a/edaweb/services.py b/edaweb/services.py index 87af050..eca2bde 100644 --- a/edaweb/services.py +++ b/edaweb/services.py @@ -247,7 +247,7 @@ def parse_tweet(tweet_url): return dt, replying_to, text, images -def scrape_whispa(whispa_url, since): +def scrape_whispa(whispa_url, since = None): tree = html.fromstring(requests.get(whispa_url).content.decode()) qnas = [] # we're not doing proper HTML scraping here really... since the site uses client side rendering @@ -257,6 +257,9 @@ def scrape_whispa(whispa_url, since): if "receivedFeedback" in js: # my god this is horrible... for j in json.loads(json.loads(js[19:-1])[1][2:])[0][3]["loadedUser"]["receivedFeedback"]: + if j["childFeedback"] == []: + continue + dt = datetime.datetime.fromisoformat(j["childFeedback"][0]["createdAt"][:-1]) qnas.append({ @@ -359,7 +362,8 @@ def get_recent_commits(db, max_per_repo = 3): return sorted(out, key = lambda a: a["datetime"], reverse = True) if __name__ == "__main__": - import database + print(scrape_whispa(CONFIG.get("qnas", "url"))) + # import database - with database.Database() as db: - print(json.dumps(get_recent_commits(db), indent=4)) + # with database.Database() as db: + # print(json.dumps(get_recent_commits(db), indent=4)) diff --git a/edaweb/static/images/random.jpg b/edaweb/static/images/random.jpg Binary files differdeleted file mode 100644 index 4e23beb..0000000 --- a/edaweb/static/images/random.jpg +++ /dev/null diff --git a/edaweb/static/robots.txt b/edaweb/static/robots.txt index c2aab7e..7aa9c7e 100644 --- a/edaweb/static/robots.txt +++ b/edaweb/static/robots.txt @@ -1,2 +1,4 @@ User-agent: *
-Disallow: /
\ No newline at end of file +Allow: /
+User-agent: Googlebot-Image
+Disallow: *
diff --git a/entrypoint.sh b/entrypoint.sh index 335ad66..f8664f0 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,3 +1,5 @@ +rm -rvf /app/edaweb/edaweb.conf/ +ln -s /app/edaweb.conf /app/edaweb/edaweb.conf printenv | grep -v "no_proxy" >> /etc/environment -tmux new-session -d -s "cron" 'cron -f || bash && bash'; -python3 /app/edaweb/app.py --production
\ No newline at end of file +#tmux new-session -d -s "cron" 'cron -f || bash && bash'; +python3 /app/edaweb/app.py --production diff --git a/papers.md b/papers.md deleted file mode 100644 index 8da5119..0000000 --- a/papers.md +++ /dev/null @@ -1,25 +0,0 @@ -this page lists some of the papers i have written! it's not an exhaustive list, just mostly the ones i'm most proud of the latex of - -## published papers - -i pinky promise i've had academic papers published, im just going to add them when they've actually been published not just accepted - -## coursework - -### level 7 - -#### advanced ai - -<iframe class="pdf" src="/pdf/aai.pdf" width="65%" height="500pt"></iframe> - -this was part of the assessment for an 'advanced ai' module, it mostly pertains to bayesian networks - -#### advanced robotics - -<iframe class="pdf" src="/pdf/ar2.pdf" width="65%" height="500pt"></iframe> - -this assessment pertains to learning from demonstration with manipulators in robotics, and mostly talks about and does some experiments with dynamic movement primitives (DMPs) and stable estimator of dynamic systems (SEDs) - - - - |
