From f2f734194c03dfff2024cf417c502515ddb7a855 Mon Sep 17 00:00:00 2001
From: jwansek <eddie.atten.ea29@gmail.com>
Date: Sat, 21 May 2022 22:38:52 +0100
Subject: Added running as an API
---
.gitignore | 3 +-
API/Dockerfile | 7 +++
API/api.conf | 13 +++++
API/app.py | 109 +++++++++++++++++++++++++++++++++++++++
API/docker-compose.yml | 23 +++++++++
API/helpers.py | 40 ++++++++++++++
API/requirements.txt | 5 ++
Dockerfile | 1 +
ExampleAssessments/CMP-4009B.yml | 71 -------------------------
ExampleAssessments/example.yml | 3 +-
Smarker/database.py | 17 ++++--
Smarker/requirements.txt | 2 -
Smarker/smarker.conf | 24 +++++++++
docs/source/api.rst | 26 ++++++++++
docs/source/conf.py | 1 +
docs/source/index.rst | 27 +++++++---
16 files changed, 284 insertions(+), 88 deletions(-)
create mode 100644 API/Dockerfile
create mode 100644 API/api.conf
create mode 100644 API/app.py
create mode 100644 API/docker-compose.yml
create mode 100644 API/helpers.py
create mode 100644 API/requirements.txt
delete mode 100644 ExampleAssessments/CMP-4009B.yml
create mode 100644 Smarker/smarker.conf
create mode 100644 docs/source/api.rst
diff --git a/.gitignore b/.gitignore
index db2ce2b..dade9b8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,8 @@
+API/.uploads/*.*
out/
*_report.*
*.zip
-smarker.conf
+#smarker.conf
*.aux
*.pickle
diff --git a/API/Dockerfile b/API/Dockerfile
new file mode 100644
index 0000000..5d482c7
--- /dev/null
+++ b/API/Dockerfile
@@ -0,0 +1,7 @@
+FROM smarker
+MAINTAINER Eden Attenborough "gae19jtu@uea.ac.uk"
+COPY . /API
+WORKDIR /API
+RUN pip3 install -r requirements.txt
+ENTRYPOINT ["python3"]
+CMD ["app.py", "--production"]
\ No newline at end of file
diff --git a/API/api.conf b/API/api.conf
new file mode 100644
index 0000000..b8143f3
--- /dev/null
+++ b/API/api.conf
@@ -0,0 +1,13 @@
+[production]
+port = 6970
+host = 0.0.0.0
+
+[testing]
+port = 5002
+host = 0.0.0.0
+
+[mysql]
+host = vps.eda.gay
+port = 3307
+user = root
+passwd = **********
\ No newline at end of file
diff --git a/API/app.py b/API/app.py
new file mode 100644
index 0000000..e4cb769
--- /dev/null
+++ b/API/app.py
@@ -0,0 +1,109 @@
+from paste.translogger import TransLogger
+from waitress import serve
+import configparser
+import werkzeug
+import helpers
+import flask
+import sys
+import os
+
+# os.environ["UPLOADS_DIR"] = "/media/veracrypt1/Edencloud/UniStuff/3.0 - CMP 3rd Year Project/Smarker/API/.uploads"
+
+sys.path.insert(1, os.path.join("..", "Smarker"))
+import database
+
+app = flask.Flask(__name__)
+app.config['UPLOAD_FOLDER'] = ".uploads"
+API_CONF = configparser.ConfigParser()
+API_CONF.read("api.conf")
+
+
+@app.route("/api/helloworld")
+def helloworld():
+ """GET ``/api/helloworld``
+
+ Returns a friendly message to check the server is working
+ """
+ return flask.jsonify({"hello": "world!"})
+
+@app.route("/api/mark", methods = ["post"])
+def mark():
+ """POST ``/api/mark``
+
+ Expects to be a POST request of ``Content-Type: multipart/form-data``.
+
+ * Expects a valid API key under the key ``key``
+
+ * Expects an assessment name under the key ``assessment``
+
+ * Expects a submission zip file under the key ``zip``
+
+ * File dependences can be added with keys prefixed with ``filedep``, but a corrisponding key must also be present with the location of this file in the sandboxed environment: e.g. ``-F "filedep1=@../../aDependency.txt" -F "aDependency.txt=/aDependency.txt"``
+
+ Returns a report in the JSON format.
+ """
+ # try:
+ assessment_name = flask.request.form.get('assessment')
+ api_key = flask.request.form.get('key')
+ files = []
+
+ with database.SmarkerDatabase(
+ host = API_CONF.get("mysql", "host"),
+ user = API_CONF.get("mysql", "user"),
+ passwd = API_CONF.get("mysql", "passwd"),
+ db = "Smarker",
+ port = API_CONF.getint("mysql", "port")) as db:
+
+ if db.valid_apikey(api_key):
+ f = flask.request.files["zip"]
+ zip_name = f.filename
+ f.save(os.path.join(".uploads/", f.filename))
+ # even though this is inside docker, we are accessing the HOST docker daemon
+ # so we have to pass through the HOST location for volumes... very annoying I know
+ # so we set this environment variable
+ # https://serverfault.com/questions/819369/mounting-a-volume-with-docker-in-docker
+ files.append("%s:/tmp/%s" % (os.path.join(os.environ["UPLOADS_DIR"], zip_name), zip_name))
+
+ for file_dep in flask.request.files.keys():
+ if file_dep.startswith("filedep"):
+ f = flask.request.files[file_dep]
+ f.save(os.path.join(".uploads/", f.filename))
+ dep_name = os.path.split(f.filename)[-1]
+ client_loc = flask.request.form.get(dep_name)
+ if client_loc is None:
+ return flask.abort(flask.Response("You need to specify a location to put file dependency '%s' e.g. '%s=/%s'" % (dep_name, dep_name, dep_name), status=500))
+
+ files.append("%s:%s" % (os.path.join(os.environ["UPLOADS_DIR"], dep_name), client_loc))
+
+
+ try:
+ return flask.jsonify(helpers.run_smarker_simple(db, zip_name, assessment_name, files))
+ except Exception as e:
+ flask.abort(flask.Response(str(e), status=500))
+ else:
+ flask.abort(403)
+ # except (KeyError, TypeError, ValueError):
+ # flask.abort(400)
+
+
+if __name__ == "__main__":
+ try:
+ if sys.argv[1] == "--production":
+ serve(
+ TransLogger(app),
+ host = API_CONF.get("production", "host"),
+ port = API_CONF.getint("production", "port"),
+ threads = 32
+ )
+ else:
+ app.run(
+ host = API_CONF.get("testing", "host"),
+ port = API_CONF.getint("testing", "port"),
+ debug = True
+ )
+ except IndexError:
+ app.run(
+ host = API_CONF.get("testing", "host"),
+ port = API_CONF.getint("testing", "port"),
+ debug = True
+ )
\ No newline at end of file
diff --git a/API/docker-compose.yml b/API/docker-compose.yml
new file mode 100644
index 0000000..19d9d3c
--- /dev/null
+++ b/API/docker-compose.yml
@@ -0,0 +1,23 @@
+version: '3'
+
+services:
+ smarker:
+ build:
+ context: ..
+ dockerfile: Dockerfile
+ image: smarker
+ smarker-api:
+ build:
+ context: .
+ dockerfile: Dockerfile
+ image: smarker-api
+ ports:
+ - "6970:6970"
+ volumes:
+ - /var/run/docker.sock:/var/run/docker.sock
+ - ./.uploads/:/API/.uploads/
+ - /tmp/:/tmp/
+ environment:
+ - UPLOADS_DIR=<your full uploads directory path here>
+ depends_on:
+ - smarker
\ No newline at end of file
diff --git a/API/helpers.py b/API/helpers.py
new file mode 100644
index 0000000..692e566
--- /dev/null
+++ b/API/helpers.py
@@ -0,0 +1,40 @@
+import tempfile
+import docker
+import json
+import os
+
+CLIENT = docker.from_env()
+
+def run_smarker_simple(db, zip_name, assessment_name, volumes):
+ with tempfile.TemporaryDirectory() as td: # remember to passthru /tmp/ as a volume
+ env = [ # probably need to find a better way tbh
+ "submission=/tmp/%s" % zip_name,
+ "assessment=%s" % assessment_name,
+ "format=json",
+ "output=/out/report.json"
+ ]
+ outjson = os.path.join(td, "report.json")
+ volumes.append("%s:/out/report.json" % (outjson))
+ # print("file_deps:", volumes)
+
+ try:
+ pypideps = db.get_assessment_yaml(assessment_name)["dependencies"]["libraries"]
+ env.append("SMARKERDEPS=" + ",".join(pypideps))
+ except KeyError:
+ pass
+ # print("env: ", env)
+
+ open(outjson, 'a').close() # make a blank file so docker doesnt make it as a directory
+ log = CLIENT.containers.run(
+ "smarker",
+ remove = True,
+ volumes = volumes,
+ environment = env
+ )
+ print("log: ", log)
+
+ for f in os.listdir(".uploads"):
+ os.remove(os.path.join(".uploads", f))
+
+ with open(outjson, "r") as f:
+ return json.load(f)
diff --git a/API/requirements.txt b/API/requirements.txt
new file mode 100644
index 0000000..32e26e6
--- /dev/null
+++ b/API/requirements.txt
@@ -0,0 +1,5 @@
+flask
+PasteScript==3.2.0
+waitress
+docker
+werkzeug
diff --git a/Dockerfile b/Dockerfile
index 8f69849..529c75f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -5,6 +5,7 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
RUN apt-get update -y
RUN apt-get install -y python3-pip python3-dev build-essential texlive-latex-base texlive-pictures texlive-science texlive-latex-extra wkhtmltopdf
RUN mkdir /out
+RUN mkdir /.uploads
COPY ./Smarker /Smarker
WORKDIR /Smarker
RUN pip3 install -r requirements.txt
diff --git a/ExampleAssessments/CMP-4009B.yml b/ExampleAssessments/CMP-4009B.yml
deleted file mode 100644
index c945907..0000000
--- a/ExampleAssessments/CMP-4009B.yml
+++ /dev/null
@@ -1,71 +0,0 @@
-name: CMP-4009B-2020-A2
-files:
- - pjtool.py:
- classes:
- - Date:
- methods:
- - __init__(4)
- - __str__(1)
- - __eq__(2)
- - __lt__(2)
- - numYears(2)
- - numMonths(2)
- tests:
- - |
- d1 = pjtool.Date(2001, 8, 12)
- d2 = pjtool.Date(2001, 8, 12)
- assert d1 == d2
- - |
- d1 = pjtool.Date(2001, 8, 12)
- d2 = pjtool.Date(1999, 4, 5)
- assert d1 != d2
- - |
- d1 = pjtool.Date(2001, 8, 12)
- d2 = pjtool.Date(1999, 4, 5)
- assert d1 > d2
- - |
- import random
- d1 = pjtool.Date(random.randint(2000, 2050), 8, 12)
- d2 = pjtool.Date(random.randint(1900, 2050), 4, 5)
- assert d1.numYears(d2) == abs(d1.year - d2.year)
- - |
- d1 = pjtool.Date(2020, 5, 1)
- d2 = pjtool.Date(2020, 6, 1)
- assert d1.numMonths(d2) == 0
- - |
- d1 = pjtool.Date(2020, 5, 1)
- d2 = pjtool.Date(2020, 8, 1)
- assert d1.numMonths(d2) == 2
- - tester.py:
- functions:
- - dateTester(2)
- run:
- - python tester.py:
- regexes:
- - True\nFalse
- - turbine.py:
- classes:
- - Turbine:
- methods:
- - __init__(5)
- - __str__(1)
- - numYearsInst(2)
- - serviceDue(2)
- - serviceAt(2)
- - powerAt(2)
- - AdvTurbine:
- methods:
- - __init__(5)
- - __str__(1)
- - numYearsInst(2)
- - serviceDue(2)
- - serviceAt(2)
- - powerAt(2)
-produced_files:
- - pdLine.png
- - pdResult.txt
-dependencies:
- libraries:
- - matplotlib
- files:
- - ../wsData.txt
diff --git a/ExampleAssessments/example.yml b/ExampleAssessments/example.yml
index 154c733..f998048 100644
--- a/ExampleAssessments/example.yml
+++ b/ExampleAssessments/example.yml
@@ -57,4 +57,5 @@ dependencies:
files:
- ../wsData.txt
libraries:
- - matplotlib
\ No newline at end of file
+ - matplotlib
+ - opencv-python-headless
\ No newline at end of file
diff --git a/Smarker/database.py b/Smarker/database.py
index 37a44db..a3f77af 100644
--- a/Smarker/database.py
+++ b/Smarker/database.py
@@ -1,5 +1,6 @@
from dataclasses import dataclass
import pymysql
+import secrets
import yaml
@dataclass
@@ -54,7 +55,8 @@ class SmarkerDatabase:
CREATE TABLE students(
student_no VARCHAR(10) PRIMARY KEY NOT NULL,
name TEXT NOT NULL,
- email VARCHAR(50) NOT NULL
+ email VARCHAR(50) NOT NULL,
+ apikey VARCHAR(64) NOT NULL
);
""")
cursor.execute("""
@@ -175,8 +177,8 @@ class SmarkerDatabase:
email (str): Student's email
"""
with self.__connection.cursor() as cursor:
- cursor.execute("INSERT INTO students VALUES (%s, %s, %s);",
- (student_id, name, email))
+ cursor.execute("INSERT INTO students VALUES (%s, %s, %s, %s);",
+ (student_id, name, email, secrets.token_urlsafe(32)))
self.__connection.commit()
def add_submission(self, student_id, assessment_name, report_yaml, files):
@@ -198,9 +200,9 @@ class SmarkerDatabase:
cursor.execute("""
INSERT INTO submitted_files
(submission_id, file_id, file_text)
- VALUES (%s, (SELECT file_id FROM assessment_file WHERE file_name = %s), %s);
+ VALUES (%s, (SELECT file_id FROM assessment_file WHERE file_name = %s AND assessment_name = %s), %s);
""", (
- submission_id, file_name, file_contents
+ submission_id, file_name, assessment_name, file_contents
))
self.__connection.commit()
@@ -240,6 +242,11 @@ class SmarkerDatabase:
cursor.execute("SELECT file_name FROM assessment_file WHERE assessment_name = %s;", (assessment_name, ))
return [i[0] for i in cursor.fetchall()]
+ def valid_apikey(self, key):
+ with self.__connection.cursor() as cursor:
+ cursor.execute("SELECT apikey FROM students WHERE apikey = %s", (key, ))
+ return key in [i[0] for i in cursor.fetchall()]
+
if __name__ == "__main__":
with SmarkerDatabase(host = "vps.eda.gay", user="root", passwd=input("Input password: "), db="Smarker", port=3307) as db:
# print(db.get_assessments_required_files("example"))
diff --git a/Smarker/requirements.txt b/Smarker/requirements.txt
index 3be9c36..af89c27 100644
--- a/Smarker/requirements.txt
+++ b/Smarker/requirements.txt
@@ -1,5 +1,3 @@
-# sudo apt-get install wkhtmltopdf
-# https://github.com/olivierverdier/python-latex-highlighting
Jinja2==3.0.3
misaka==2.1.1
Pygments==2.10.0
diff --git a/Smarker/smarker.conf b/Smarker/smarker.conf
new file mode 100644
index 0000000..3416564
--- /dev/null
+++ b/Smarker/smarker.conf
@@ -0,0 +1,24 @@
+[mysql]
+host = vps.eda.gay
+port = 3307
+user = root
+passwd = ************
+
+[tex]
+columns = 1
+show_full_docs = True
+show_source = True
+show_all_regex_occurrences = True
+show_all_run_output = True
+
+[md]
+show_full_docs = False
+show_source = False
+show_all_regex_occurrences = True
+show_all_run_output = False
+
+[txt]
+show_full_docs = True
+show_source = True
+show_all_regex_occurrences = True
+show_all_run_output = True
\ No newline at end of file
diff --git a/docs/source/api.rst b/docs/source/api.rst
new file mode 100644
index 0000000..61469f9
--- /dev/null
+++ b/docs/source/api.rst
@@ -0,0 +1,26 @@
+.. _api:
+
+Running as an API
+=================
+
+*Smarker* can be hosted on a server and accessed through an API. A valid docker-compose
+file is in the ``API/`` directory. Since the API docker container accesses the host docker
+daemon, you must pass set the host location of the ``.uploads/`` directory as the ``$UPLOADS_DIR``
+environment variable.
+
+.. autofunction:: app.helloworld
+
+.. autofunction:: app.mark
+
+An example CURL request could be:
+
+.. code-block:: bash
+
+ curl -X POST -H "Content-Type: multipart/form-data" \
+ -F "zip=@../100301654.zip" \
+ -F "key=2g_yU7n1SqTODGQmpuViIAwbdbownmVDpjUl9NKkRqz" \
+ -F "assessment=example" \
+ -F "filedep1=@../../dependency.txt" \
+ -F "dependency.txt=/dependency.txt" \
+ "localhost:6970/api/mark"
+
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 5dd644b..a168856 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -13,6 +13,7 @@
import os
import sys
sys.path.insert(0, os.path.abspath(os.path.join("..", "..", "Smarker")))
+sys.path.insert(0, os.path.abspath(os.path.join("..", "..", "API")))
sys.path.insert(0, os.path.abspath(os.path.join("..", "..", "ExampleAssessments")))
# print(os.listdir(os.path.abspath(os.path.join("..", "..", "Smarker"))))
diff --git a/docs/source/index.rst b/docs/source/index.rst
index f2d7426..80fefaf 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -11,6 +11,9 @@ Setting up
* Add an assessment yaml- :ref:`assessmentyaml`.
* Enroll students: ``python3 Smarker/assessments.py -s 123456789,Eden,Attenborough,E.Attenborough@uea.ac.uk``
+*Smarker* can be used inside docker, see :ref:`docker` (recommended for sandboxing client code)
+and through an API- see :ref:`api`.
+
``smarker.py`` usage
********************
@@ -37,20 +40,28 @@ Also see :ref:`assessments`
.. toctree::
:maxdepth: 2
+ :caption: Setting up:
+
+ quickstart.rst
+ configfile.rst
+
+.. toctree::
+ :maxdepth: 3
+ :caption: Docker:
+
+ docker.rst
+ assessmentyaml.rst
+ api.rst
+
+.. toctree::
+ :maxdepth: 3
:caption: Modules:
reflect.rst
database.rst
assessments.rst
+ api.rst
-.. toctree::
- :maxdepth: 2
- :caption: Other Pages:
-
- quickstart.rst
- configfile.rst
- docker.rst
- assessmentyaml.rst
Indices and tables
==================
--
cgit v1.2.3