Added running as an API

16 files changed, 284 insertions, 88 deletions

diff --git a/.gitignore b/.gitignore
index db2ce2b..dade9b8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,8 @@
+API/.uploads/*.*
 out/
 *_report.*
 *.zip
-smarker.conf
+#smarker.conf
 *.aux
 *.pickle
 
diff --git a/API/Dockerfile b/API/Dockerfile
new file mode 100644
index 0000000..5d482c7
--- /dev/null
+++ b/API/Dockerfile
@@ -0,0 +1,7 @@
+FROM smarker

+MAINTAINER Eden Attenborough "gae19jtu@uea.ac.uk"

+COPY . /API

+WORKDIR /API

+RUN pip3 install -r requirements.txt

+ENTRYPOINT ["python3"]

+CMD ["app.py", "--production"]
\ No newline at end of file
diff --git a/API/api.conf b/API/api.conf
new file mode 100644
index 0000000..b8143f3
--- /dev/null
+++ b/API/api.conf
@@ -0,0 +1,13 @@
+[production]

+port = 6970

+host = 0.0.0.0

+

+[testing]

+port = 5002

+host = 0.0.0.0

+

+[mysql]

+host = vps.eda.gay

+port = 3307

+user = root

+passwd = **********
\ No newline at end of file
diff --git a/API/app.py b/API/app.py
new file mode 100644
index 0000000..e4cb769
--- /dev/null
+++ b/API/app.py
@@ -0,0 +1,109 @@
+from paste.translogger import TransLogger

+from waitress import serve

+import configparser

+import werkzeug

+import helpers

+import flask

+import sys

+import os

+

+# os.environ["UPLOADS_DIR"] = "/media/veracrypt1/Edencloud/UniStuff/3.0 - CMP 3rd Year Project/Smarker/API/.uploads"

+

+sys.path.insert(1, os.path.join("..", "Smarker"))

+import database

+

+app = flask.Flask(__name__)

+app.config['UPLOAD_FOLDER'] = ".uploads"

+API_CONF = configparser.ConfigParser()

+API_CONF.read("api.conf")

+

+

+@app.route("/api/helloworld")

+def helloworld():

+    """GET ``/api/helloworld``

+

+    Returns a friendly message to check the server is working

+    """

+    return flask.jsonify({"hello": "world!"})

+

+@app.route("/api/mark", methods = ["post"])

+def mark():

+    """POST ``/api/mark``

+

+    Expects to be a POST request of ``Content-Type: multipart/form-data``.

+

+    * Expects a valid API key under the key ``key``

+

+    * Expects an assessment name under the key ``assessment``

+

+    * Expects a submission zip file under the key ``zip``

+

+    * File dependences can be added with keys prefixed with ``filedep``, but a corrisponding key must also be present with the location of this file in the sandboxed environment: e.g. ``-F "filedep1=@../../aDependency.txt" -F "aDependency.txt=/aDependency.txt"``

+

+    Returns a report in the JSON format.

+    """

+    # try:

+    assessment_name = flask.request.form.get('assessment')

+    api_key = flask.request.form.get('key')

+    files = []

+

+    with database.SmarkerDatabase(

+        host = API_CONF.get("mysql", "host"), 

+        user = API_CONF.get("mysql", "user"), 

+        passwd = API_CONF.get("mysql", "passwd"), 

+        db = "Smarker",

+        port = API_CONF.getint("mysql", "port")) as db:

+

+        if db.valid_apikey(api_key):

+            f = flask.request.files["zip"]

+            zip_name = f.filename

+            f.save(os.path.join(".uploads/", f.filename))

+            # even though this is inside docker, we are accessing the HOST docker daemon

+            # so we have to pass through the HOST location for volumes... very annoying I know

+            # so we set this environment variable

+            #       https://serverfault.com/questions/819369/mounting-a-volume-with-docker-in-docker

+            files.append("%s:/tmp/%s" % (os.path.join(os.environ["UPLOADS_DIR"], zip_name), zip_name))

+   

+            for file_dep in flask.request.files.keys():

+                if file_dep.startswith("filedep"):

+                    f = flask.request.files[file_dep]

+                    f.save(os.path.join(".uploads/", f.filename))

+                    dep_name = os.path.split(f.filename)[-1]

+                    client_loc = flask.request.form.get(dep_name)

+                    if client_loc is None:

+                        return flask.abort(flask.Response("You need to specify a location to put file dependency '%s' e.g.  '%s=/%s'" % (dep_name, dep_name, dep_name), status=500))

+                    

+                    files.append("%s:%s" % (os.path.join(os.environ["UPLOADS_DIR"], dep_name), client_loc))

+            

+            

+            try:

+                return flask.jsonify(helpers.run_smarker_simple(db, zip_name, assessment_name, files))

+            except Exception as e:

+                flask.abort(flask.Response(str(e), status=500))

+        else:

+            flask.abort(403)

+    # except (KeyError, TypeError, ValueError):

+    #     flask.abort(400)

+

+

+if __name__ == "__main__":

+    try:

+        if sys.argv[1] == "--production":

+            serve(

+                TransLogger(app), 

+                host = API_CONF.get("production", "host"), 

+                port = API_CONF.getint("production", "port"), 

+                threads = 32

+            )

+        else:

+            app.run(

+                host = API_CONF.get("testing", "host"), 

+                port = API_CONF.getint("testing", "port"), 

+                debug = True

+            )

+    except IndexError:

+        app.run(

+            host = API_CONF.get("testing", "host"), 

+            port = API_CONF.getint("testing", "port"), 

+            debug = True

+        )
\ No newline at end of file
diff --git a/API/docker-compose.yml b/API/docker-compose.yml
new file mode 100644
index 0000000..19d9d3c
--- /dev/null
+++ b/API/docker-compose.yml
@@ -0,0 +1,23 @@
+version: '3'

+

+services:

+    smarker:

+        build:

+            context: ..

+            dockerfile: Dockerfile

+        image: smarker

+    smarker-api:

+        build:

+            context: .

+            dockerfile: Dockerfile

+        image: smarker-api

+        ports:

+            - "6970:6970"

+        volumes:

+            - /var/run/docker.sock:/var/run/docker.sock

+            - ./.uploads/:/API/.uploads/

+            - /tmp/:/tmp/

+        environment:

+            - UPLOADS_DIR=<your full uploads directory path here>

+        depends_on:

+            - smarker
\ No newline at end of file
diff --git a/API/helpers.py b/API/helpers.py
new file mode 100644
index 0000000..692e566
--- /dev/null
+++ b/API/helpers.py
@@ -0,0 +1,40 @@
+import tempfile
+import docker
+import json
+import os
+
+CLIENT = docker.from_env()
+
+def run_smarker_simple(db, zip_name, assessment_name, volumes):
+    with tempfile.TemporaryDirectory() as td:   # remember to passthru /tmp/ as a volume
+        env = [                                 # probably need to find a better way tbh
+            "submission=/tmp/%s" % zip_name,
+            "assessment=%s" % assessment_name,
+            "format=json",
+            "output=/out/report.json"
+        ]
+        outjson = os.path.join(td, "report.json")
+        volumes.append("%s:/out/report.json" % (outjson))
+        # print("file_deps:", volumes)
+        
+        try:
+            pypideps = db.get_assessment_yaml(assessment_name)["dependencies"]["libraries"]
+            env.append("SMARKERDEPS=" + ",".join(pypideps))
+        except KeyError:
+            pass
+        # print("env: ", env)
+
+        open(outjson, 'a').close()  # make a blank file so docker doesnt make it as a directory
+        log = CLIENT.containers.run(
+            "smarker",
+            remove = True,
+            volumes = volumes,
+            environment = env
+        )
+        print("log: ", log)
+
+        for f in os.listdir(".uploads"):
+            os.remove(os.path.join(".uploads", f))
+
+        with open(outjson, "r") as f:
+            return json.load(f)
diff --git a/API/requirements.txt b/API/requirements.txt
new file mode 100644
index 0000000..32e26e6
--- /dev/null
+++ b/API/requirements.txt
@@ -0,0 +1,5 @@
+flask

+PasteScript==3.2.0

+waitress

+docker

+werkzeug

diff --git a/Dockerfile b/Dockerfile
index 8f69849..529c75f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -5,6 +5,7 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 RUN apt-get update -y
 RUN apt-get install -y python3-pip python3-dev build-essential texlive-latex-base texlive-pictures texlive-science texlive-latex-extra wkhtmltopdf
 RUN mkdir /out
+RUN mkdir /.uploads
 COPY ./Smarker /Smarker
 WORKDIR /Smarker
 RUN pip3 install -r requirements.txt
diff --git a/ExampleAssessments/CMP-4009B.yml b/ExampleAssessments/CMP-4009B.yml
deleted file mode 100644
index c945907..0000000
--- a/ExampleAssessments/CMP-4009B.yml
+++ /dev/null
@@ -1,71 +0,0 @@
-name: CMP-4009B-2020-A2
-files:
-    - pjtool.py:
-        classes:
-            - Date:
-                methods:
-                    - __init__(4)
-                    - __str__(1)
-                    - __eq__(2)
-                    - __lt__(2)
-                    - numYears(2)
-                    - numMonths(2)
-        tests:
-            - |
-                d1 = pjtool.Date(2001, 8, 12)
-                d2 = pjtool.Date(2001, 8, 12)
-                assert d1 == d2
-            - |
-                d1 = pjtool.Date(2001, 8, 12)
-                d2 = pjtool.Date(1999, 4, 5)
-                assert d1 != d2
-            - |
-                d1 = pjtool.Date(2001, 8, 12)
-                d2 = pjtool.Date(1999, 4, 5)
-                assert d1 > d2
-            - |
-                import random
-                d1 = pjtool.Date(random.randint(2000, 2050), 8, 12)
-                d2 = pjtool.Date(random.randint(1900, 2050), 4, 5)
-                assert d1.numYears(d2) == abs(d1.year - d2.year)
-            - |
-                d1 = pjtool.Date(2020, 5, 1)
-                d2 = pjtool.Date(2020, 6, 1)
-                assert d1.numMonths(d2) == 0
-            - |
-                d1 = pjtool.Date(2020, 5, 1)
-                d2 = pjtool.Date(2020, 8, 1)
-                assert d1.numMonths(d2) == 2
-    - tester.py:
-        functions:
-            - dateTester(2)
-        run:
-            - python tester.py:
-                regexes:
-                    - True\nFalse
-    - turbine.py:
-        classes:
-            - Turbine:
-                methods:
-                    - __init__(5)
-                    - __str__(1)
-                    - numYearsInst(2)
-                    - serviceDue(2)
-                    - serviceAt(2)
-                    - powerAt(2)
-            - AdvTurbine:
-                methods:
-                    - __init__(5)
-                    - __str__(1)
-                    - numYearsInst(2)
-                    - serviceDue(2)
-                    - serviceAt(2)
-                    - powerAt(2)
-produced_files:
-    - pdLine.png
-    - pdResult.txt
-dependencies:
-    libraries:
-        - matplotlib
-    files:
-        - ../wsData.txt  
diff --git a/ExampleAssessments/example.yml b/ExampleAssessments/example.yml
index 154c733..f998048 100644
--- a/ExampleAssessments/example.yml
+++ b/ExampleAssessments/example.yml
@@ -57,4 +57,5 @@ dependencies:
     files:

         - ../wsData.txt

     libraries:

-        - matplotlib
\ No newline at end of file
+        - matplotlib

+        - opencv-python-headless
\ No newline at end of file
diff --git a/Smarker/database.py b/Smarker/database.py
index 37a44db..a3f77af 100644
--- a/Smarker/database.py
+++ b/Smarker/database.py
@@ -1,5 +1,6 @@
 from dataclasses import dataclass
 import pymysql
+import secrets
 import yaml
 
 @dataclass
@@ -54,7 +55,8 @@ class SmarkerDatabase:
             CREATE TABLE students(
                 student_no VARCHAR(10) PRIMARY KEY NOT NULL,
                 name TEXT NOT NULL,
-                email VARCHAR(50) NOT NULL
+                email VARCHAR(50) NOT NULL,
+                apikey VARCHAR(64) NOT NULL
             );
             """)
             cursor.execute("""
@@ -175,8 +177,8 @@ class SmarkerDatabase:
             email (str): Student's email
         """
         with self.__connection.cursor() as cursor:
-            cursor.execute("INSERT INTO students VALUES (%s, %s, %s);",
-            (student_id, name, email))
+            cursor.execute("INSERT INTO students VALUES (%s, %s, %s, %s);",
+            (student_id, name, email, secrets.token_urlsafe(32)))
         self.__connection.commit()
 
     def add_submission(self, student_id, assessment_name, report_yaml, files):
@@ -198,9 +200,9 @@ class SmarkerDatabase:
                 cursor.execute("""
                 INSERT INTO submitted_files
                 (submission_id, file_id, file_text)
-                VALUES (%s, (SELECT file_id FROM assessment_file WHERE file_name = %s), %s);
+                VALUES (%s, (SELECT file_id FROM assessment_file WHERE file_name = %s AND assessment_name = %s), %s);
                 """, (
-                    submission_id, file_name, file_contents
+                    submission_id, file_name, assessment_name, file_contents
                 ))
         self.__connection.commit()
 
@@ -240,6 +242,11 @@ class SmarkerDatabase:
             cursor.execute("SELECT file_name FROM assessment_file WHERE assessment_name = %s;", (assessment_name, ))   
             return [i[0] for i in cursor.fetchall()]
 
+    def valid_apikey(self, key):
+        with self.__connection.cursor() as cursor:
+            cursor.execute("SELECT apikey FROM students WHERE apikey = %s", (key, ))
+            return key in [i[0] for i in cursor.fetchall()]
+
 if __name__ == "__main__":
     with SmarkerDatabase(host = "vps.eda.gay", user="root", passwd=input("Input password: "), db="Smarker", port=3307) as db:
         # print(db.get_assessments_required_files("example"))
diff --git a/Smarker/requirements.txt b/Smarker/requirements.txt
index 3be9c36..af89c27 100644
--- a/Smarker/requirements.txt
+++ b/Smarker/requirements.txt
@@ -1,5 +1,3 @@
-# sudo apt-get install wkhtmltopdf

-# https://github.com/olivierverdier/python-latex-highlighting

 Jinja2==3.0.3

 misaka==2.1.1

 Pygments==2.10.0

diff --git a/Smarker/smarker.conf b/Smarker/smarker.conf
new file mode 100644
index 0000000..3416564
--- /dev/null
+++ b/Smarker/smarker.conf
@@ -0,0 +1,24 @@
+[mysql]

+host = vps.eda.gay

+port = 3307

+user = root

+passwd = ************

+

+[tex]

+columns = 1

+show_full_docs = True

+show_source = True

+show_all_regex_occurrences = True

+show_all_run_output = True

+

+[md]

+show_full_docs = False

+show_source = False

+show_all_regex_occurrences = True

+show_all_run_output = False

+

+[txt]

+show_full_docs = True

+show_source = True

+show_all_regex_occurrences = True

+show_all_run_output = True
\ No newline at end of file
diff --git a/docs/source/api.rst b/docs/source/api.rst
new file mode 100644
index 0000000..61469f9
--- /dev/null
+++ b/docs/source/api.rst
@@ -0,0 +1,26 @@
+.. _api:
+
+Running as an API
+=================
+
+*Smarker* can be hosted on a server and accessed through an API. A valid docker-compose
+file is in the ``API/`` directory. Since the API docker container accesses the host docker
+daemon, you must pass set the host location of the ``.uploads/`` directory as the ``$UPLOADS_DIR``
+environment variable.
+
+.. autofunction:: app.helloworld
+
+.. autofunction:: app.mark
+
+An example CURL request could be:
+
+.. code-block:: bash
+    
+    curl -X POST -H "Content-Type: multipart/form-data" \
+        -F "zip=@../100301654.zip" \
+        -F "key=2g_yU7n1SqTODGQmpuViIAwbdbownmVDpjUl9NKkRqz" \
+        -F "assessment=example" \
+        -F "filedep1=@../../dependency.txt" \
+        -F "dependency.txt=/dependency.txt" \
+        "localhost:6970/api/mark"
+
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 5dd644b..a168856 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -13,6 +13,7 @@
 import os
 import sys
 sys.path.insert(0, os.path.abspath(os.path.join("..", "..", "Smarker")))
+sys.path.insert(0, os.path.abspath(os.path.join("..", "..", "API")))
 sys.path.insert(0, os.path.abspath(os.path.join("..", "..", "ExampleAssessments")))
 # print(os.listdir(os.path.abspath(os.path.join("..", "..", "Smarker"))))
 
diff --git a/docs/source/index.rst b/docs/source/index.rst
index f2d7426..80fefaf 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -11,6 +11,9 @@ Setting up
 * Add an assessment yaml- :ref:`assessmentyaml`.
 * Enroll students: ``python3 Smarker/assessments.py -s 123456789,Eden,Attenborough,E.Attenborough@uea.ac.uk``
 
+*Smarker* can be used inside docker, see :ref:`docker` (recommended for sandboxing client code)
+and through an API- see :ref:`api`.
+
 ``smarker.py`` usage
 ********************
 
@@ -37,20 +40,28 @@ Also see :ref:`assessments`
 
 .. toctree::
    :maxdepth: 2
+   :caption: Setting up:
+
+   quickstart.rst
+   configfile.rst
+
+.. toctree::
+   :maxdepth: 3
+   :caption: Docker:
+
+   docker.rst
+   assessmentyaml.rst
+   api.rst
+
+.. toctree::
+   :maxdepth: 3
    :caption: Modules:
    
    reflect.rst
    database.rst
    assessments.rst
+   api.rst
 
-.. toctree::
-   :maxdepth: 2
-   :caption: Other Pages:
-
-   quickstart.rst
-   configfile.rst
-   docker.rst
-   assessmentyaml.rst
 
 Indices and tables
 ==================