diff options
Diffstat (limited to 'edaweb')
| -rw-r--r-- | edaweb/services.py | 12 | ||||
| -rw-r--r-- | edaweb/static/images/random.jpg | bin | 202536 -> 0 bytes | |||
| -rw-r--r-- | edaweb/static/robots.txt | 4 |
3 files changed, 11 insertions, 5 deletions
diff --git a/edaweb/services.py b/edaweb/services.py index 87af050..eca2bde 100644 --- a/edaweb/services.py +++ b/edaweb/services.py @@ -247,7 +247,7 @@ def parse_tweet(tweet_url): return dt, replying_to, text, images -def scrape_whispa(whispa_url, since): +def scrape_whispa(whispa_url, since = None): tree = html.fromstring(requests.get(whispa_url).content.decode()) qnas = [] # we're not doing proper HTML scraping here really... since the site uses client side rendering @@ -257,6 +257,9 @@ def scrape_whispa(whispa_url, since): if "receivedFeedback" in js: # my god this is horrible... for j in json.loads(json.loads(js[19:-1])[1][2:])[0][3]["loadedUser"]["receivedFeedback"]: + if j["childFeedback"] == []: + continue + dt = datetime.datetime.fromisoformat(j["childFeedback"][0]["createdAt"][:-1]) qnas.append({ @@ -359,7 +362,8 @@ def get_recent_commits(db, max_per_repo = 3): return sorted(out, key = lambda a: a["datetime"], reverse = True) if __name__ == "__main__": - import database + print(scrape_whispa(CONFIG.get("qnas", "url"))) + # import database - with database.Database() as db: - print(json.dumps(get_recent_commits(db), indent=4)) + # with database.Database() as db: + # print(json.dumps(get_recent_commits(db), indent=4)) diff --git a/edaweb/static/images/random.jpg b/edaweb/static/images/random.jpg Binary files differdeleted file mode 100644 index 4e23beb..0000000 --- a/edaweb/static/images/random.jpg +++ /dev/null diff --git a/edaweb/static/robots.txt b/edaweb/static/robots.txt index c2aab7e..7aa9c7e 100644 --- a/edaweb/static/robots.txt +++ b/edaweb/static/robots.txt @@ -1,2 +1,4 @@ User-agent: *
-Disallow: /
\ No newline at end of file +Allow: /
+User-agent: Googlebot-Image
+Disallow: *
|
