Started writing documentation

15 files changed, 279 insertions, 15 deletions

diff --git a/README.md b/README.md
index 0253245..57b939d 100644..120000
--- a/README.md
+++ b/README.md
@@ -1,8 +1 @@
-# Smarker
-An automated marking system for UEA programming assessments
-
-## Running in docker
-
-`sudo docker run -v "$(pwd)/../wsData.txt":/wsData.txt -v "$(pwd)/100301654.zip":/tmp/100301654.zip -v "$(pwd)/out/":/out/ -e submission=/tmp/100301654.zip -e assessment=example -e SMARKERDEPS=matplotlib -e format=pdf -e output=/out/100301654.pdf --rm smarker`
-
-`sudo docker run -it --entrypoint python --rm smarker assessments.py --list yes`
\ No newline at end of file
+docs/source/index.rst
\ No newline at end of file
diff --git a/Smarker/assessments.py b/Smarker/assessments.py
index d5c8daf..a6c4df8 100644
--- a/Smarker/assessments.py
+++ b/Smarker/assessments.py
@@ -1,11 +1,29 @@
 import misc_classes
 import configparser
 import jinja_helpers
+import pycode_similar
+import operator
 import database
 import argparse
+import tempfile
 import yaml
 import os
 
+def generate_plagarism_report(codes):
+    for file_name, codes in codes.items():
+        with tempfile.TemporaryDirectory() as td:
+            un_added_student_nos = {i[0] for i in codes.keys()}
+            # print(un_added_student_nos)
+            for k, v in sorted(codes.keys(), key=operator.itemgetter(0, 1), reverse=True):
+                if k in un_added_student_nos:
+                    with open(os.path.join(td, "%i.py" % k), "w") as f:
+                        f.write(codes[(k, v)])
+                    
+                    # print("Written %s at %s" % (k, v))
+                    un_added_student_nos.remove(k)
+            input("%s..." % td)
+            print(pycode_similar.detect(os.listdir(td)))
+
 if __name__ == "__main__":
     config = configparser.ConfigParser()
     config.read(os.path.join(os.path.split(__file__)[0], "smarker.conf"))
@@ -47,6 +65,13 @@ if __name__ == "__main__":
         help = "Add a student in the form e.g. 123456789,Eden,Attenborough,E.Attenborough@uea.ac.uk",
         required = False
     )
+    parser.add_argument(
+        "-p", "--plagarism_report",
+        action = misc_classes.EnvDefault,
+        envvar = "plagarism_report",
+        help = "Generate a plagarism report for the given assessment",
+        required = False
+    )
     
     for option in config.options("mysql"):
         parser.add_argument(
@@ -91,6 +116,9 @@ if __name__ == "__main__":
 
             print("Added student %s" % name)
 
+        if args["plagarism_report"] is not None:
+            generate_plagarism_report(db.get_submission_codes(args["plagarism_report"]))
+
         
         # print(db.get_assessment_yaml("CMP-4009B-2020-A2"))
 
diff --git a/Smarker/database.py b/Smarker/database.py
index 38af6a7..c1b1df0 100644
--- a/Smarker/database.py
+++ b/Smarker/database.py
@@ -111,6 +111,8 @@ class SmarkerDatabase:
 
     def remove_assessment(self, name):
         with self.__connection.cursor() as cursor:
+            cursor.execute("DELETE FROM submitted_files WHERE submission_id IN (SELECT submission_id FROM submissions WHERE assessment_name = %s);", (name, ))
+            cursor.execute("DELETE FROM submissions WHERE assessment_name = %s;", (name, ))
             cursor.execute("DELETE FROM assessment_file WHERE assessment_name = %s;", (name, ))
             cursor.execute("DELETE FROM assessment WHERE assessment_name = %s;", (name, ))
         self.__connection.commit()
@@ -153,3 +155,35 @@ class SmarkerDatabase:
                     submission_id, file_name, file_contents
                 ))
         self.__connection.commit()
+
+    def get_submission_codes(self, assessment_name):
+        out = {}
+        with self.__connection.cursor() as cursor:
+            cursor.execute("SELECT file_id, file_name FROM assessment_file WHERE assessment_name = %s;", (assessment_name, ))
+            for file_id, file_name in cursor.fetchall():
+                out[file_name] = {}
+
+                cursor.execute("""
+                SELECT 
+                    submitted_files.file_text, 
+                    submissions.student_no, 
+                    submissions.submission_dt 
+                FROM submitted_files 
+                INNER JOIN submissions 
+                ON submissions.submission_id = submitted_files.submission_id 
+                WHERE submitted_files.file_id = %s;
+                """, (file_id, ))
+
+                for code, student_no, dt in cursor.fetchall():
+                    out[file_name][(int(student_no), dt)] = code
+        return out
+
+    def get_most_recent_submission_report(self, assessment_name):
+        with self.__connection.cursor() as cursor:
+            cursor.execute("SELECT MAX(submission_id), student_no FROM submissions WHERE assessment_name = %s GROUP BY student_no;", (assessment_name, ))
+            return [(int(i[0]), int(i[1]), yaml.safe_load(i[2])) for i in cursor.fetchall()]
+                
+
+if __name__ == "__main__":
+    with SmarkerDatabase(host = "vps.eda.gay", user="root", passwd=input("Input password: "), db="Smarker", port=3307) as db:
+        print(db.get_most_recent_submission_report("simple_assessment"))
diff --git a/Smarker/reflect.py b/Smarker/reflect.py
index 2a1f552..44d69de 100644
--- a/Smarker/reflect.py
+++ b/Smarker/reflect.py
@@ -376,7 +376,7 @@ def gen_reflection_report(client_code_path, assessment_struct, student_no, confi
                                 lines = lines.replace("\r", "")
                                 matches = {}
                                 for regex_ in contents["regexes"]:
-                                    matches[regex_] = re.findall(regex_, lines)
+                                    matches[regex_] = re.findall(str(regex_), lines)
                                 required_files_features["run"][j][cmd]["regexes"] = matches
                                 required_files_features["run"][j][cmd]["full_output"] = lines
 
diff --git a/Smarker/requirements.txt b/Smarker/requirements.txt
index 944b77a..a8fef17 100644
--- a/Smarker/requirements.txt
+++ b/Smarker/requirements.txt
@@ -9,3 +9,4 @@ junit2html
 pdfkit

 lxml

 pymysql

+pycode_similar

diff --git a/Smarker/templates/tex.jinja2 b/Smarker/templates/tex.jinja2
index eaa7db7..5985875 100644
--- a/Smarker/templates/tex.jinja2
+++ b/Smarker/templates/tex.jinja2
@@ -57,7 +57,7 @@ breaklines=true
 \newcommand{\errortext}[1]{\textcolor{red}{\textbf{#1}}}

 

 \author{((( student_no )))}

-\title{((( name ))) - Automatic marking report}

+\title{((( tex_escape(name) ))) - Automatic marking report}

 

 \begin{document}

 

@@ -70,16 +70,19 @@ breaklines=true
 

 \begin{figure}[H]

     \centering

-    \begin{forest}

+    ((# \begin{forest}

         ((( recurse_class_tree_forest(class_tree)|indent(8, False) )))

-    \end{forest}

+    \end{forest} #))

+    \begin{lstlisting}

+((( recurse_class_tree_text(class_tree) )))

+    \end{lstlisting}

     \caption{Class inheritance tree}

 \end{figure}

 

 \section{File Analysis}

 ((* set flat_files = flatten_struct(files) *))

 ((* for filename, files_contents in flat_files.items() *))

-    \subsection{\texttt{((( filename )))}}

+    \subsection{\texttt{((( tex_escape(filename) )))}}

     ((* if files_contents["present"] *))

         ((* if files_contents["has_exception"] *))

             \errortext{File cannot be run - has compile time exception.}

@@ -87,9 +90,9 @@ breaklines=true
             Please note that this file cannot be analysed or have tests preformed upon it-

             this can lead to the whole test suite failing if another module imports this.

                     

-            \textbf{Exception Type:} \texttt{((( files_contents["exception"]["type"] )))}

+            \textbf{Exception Type:} \texttt{((( tex_escape(files_contents["exception"]["type"]) )))}

             

-            \textbf{Exception String:} \texttt{((( files_contents["exception"]["str"] )))}

+            \textbf{Exception String:} \texttt{((( tex_escape(files_contents["exception"]["str"]) )))}

 

             \textbf{Full Traceback:}

 

diff --git a/Smarker/templates/text.jinja2 b/Smarker/templates/text.jinja2
deleted file mode 100644
index eca6ebd..0000000
--- a/Smarker/templates/text.jinja2
+++ /dev/null
diff --git a/docs/Makefile b/docs/Makefile
new file mode 100644
index 0000000..d0c3cbf
--- /dev/null
+++ b/docs/Makefile
@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/docs/deploy.sh b/docs/deploy.sh
new file mode 100644
index 0000000..d43e998
--- /dev/null
+++ b/docs/deploy.sh
@@ -0,0 +1,6 @@
+tar czvf build.tar.gz build/
+scp build.tar.gz eden@vps.eda.gay:/home/eden/SmarkerDocs/build.tar.gz
+rm build.tar.gz
+ssh eden@vps.eda.gay "rm -rf /home/eden/SmarkerDocs/build/"
+ssh eden@vps.eda.gay "tar xvf /home/eden/SmarkerDocs/build.tar.gz -C /home/eden/SmarkerDocs"
+ssh eden@vps.eda.gay "rm /home/eden/SmarkerDocs/build.tar.gz"
\ No newline at end of file
diff --git a/docs/source/conf.py b/docs/source/conf.py
new file mode 100644
index 0000000..fd85d58
--- /dev/null
+++ b/docs/source/conf.py
@@ -0,0 +1,52 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+sys.path.insert(0, os.path.abspath(os.path.join("..", "..", "Smarker")))
+# print(os.listdir(os.path.abspath(os.path.join("..", "..", "Smarker"))))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'Smarker'
+copyright = '2022, Eden Attenborough'
+author = 'Eden Attenborough'
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.coverage', 'sphinx.ext.napoleon']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
\ No newline at end of file
diff --git a/docs/source/configfile.rst b/docs/source/configfile.rst
new file mode 100644
index 0000000..646230f
--- /dev/null
+++ b/docs/source/configfile.rst
@@ -0,0 +1,39 @@
+.. _configfile:
+
+``smarker.conf``: Configuration File
+====================================
+
+Here is what it should look like. It needs to be in the same directory as the 
+source code. Its a standard .ini format.
+
+Once it's been created, you can safely forget about it since all of the options 
+can be over-ridden in command line arguments.::
+    
+    [mysql]
+    host = vps.eda.gay
+    port = 3307
+    user = root
+    passwd = ************
+
+    [tex]
+    columns = 1
+    show_full_docs = True
+    show_source = True
+    show_all_regex_occurrences = True
+    show_all_run_output = True
+
+    [md]
+    show_full_docs = False
+    show_source = False
+    show_all_regex_occurrences = True
+    show_all_run_output = False
+
+    [txt]
+    show_full_docs = False
+    show_source = False
+    show_all_regex_occurrences = True
+    show_all_run_output = False
+
+The first block configures where your SQL server is. The other options are
+the default options when generating reports. ``[tex]`` options are inherited
+for rendering to PDF.
\ No newline at end of file
diff --git a/docs/source/database.rst b/docs/source/database.rst
new file mode 100644
index 0000000..663c596
--- /dev/null
+++ b/docs/source/database.rst
@@ -0,0 +1,5 @@
+``database.py``: Interfacing with the database
+==============================================
+
+.. automodule:: database
+    :members:
\ No newline at end of file
diff --git a/docs/source/docker.rst b/docs/source/docker.rst
new file mode 100644
index 0000000..7c3237a
--- /dev/null
+++ b/docs/source/docker.rst
@@ -0,0 +1,44 @@
+.. _docker:
+
+Running in docker
+=================
+
+Running the system in docker has many advantages:
+
+* Don't have to worry about getting dependencies- the system requires many libraries to be avaliable on the PATH
+
+* Isolation: we are running remote code supplied by anyone, which can potentially be very dangerous. Docker isolates the dangerous code, a significant security concern
+
+* Makes the system be able to be used in Windows- Smarker has been tested in docker for windows using WSL for the backend
+
+Using docker
+------------
+
+We have a ``Dockerfile`` ready for use: ``sudo docker build -t smarker .``
+
+To input files and get the output report we use docker volumes ``-v``. Unfortunately
+docker seems to be rather buggy passing through single files, so we recommend making
+a directory to pass through instead. 
+
+Command line arguments can be replaced with environment variables, using the ``-e``
+flag in a ``key=value`` format. There is the additional environment variable ``SMARKERDEPS``
+which will pip install pypi modules at the start, deliminated with commas.
+
+.. code-block:: bash
+    
+    sudo docker run \
+        -v "$(pwd)/../wsData.txt":/wsData.txt \
+        -v "$(pwd)/100301654.zip":/tmp/100301654.zip \
+        -v "$(pwd)/out/":/out/ \
+        -e submission=/tmp/100301654.zip \
+        -e assessment=example \
+        -e SMARKERDEPS=matplotlib \
+        -e format=pdf \
+        -e output=/out/100301654.pdf \
+        --rm smarker
+
+To list assessments in the database using docker:
+
+.. code-block:: bash
+    
+    sudo docker run -it --entrypoint python --rm smarker assessments.py --list yes
\ No newline at end of file
diff --git a/docs/source/index.rst b/docs/source/index.rst
new file mode 100644
index 0000000..c238253
--- /dev/null
+++ b/docs/source/index.rst
@@ -0,0 +1,34 @@
+Welcome to Smarker's documentation!
+===================================
+
+Setting up
+----------
+
+* Set up a MySQL/MariaDB server. The database will be generated automatically.
+* Add ``smarker.conf`` to the root directory of the code. See :ref:`configfile` for the structure of this file.
+* Decide how you want to run the program: with or without docker.
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Modules:
+   
+   reflect.rst
+   database.rst
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Other Pages:
+
+   configfile.rst
+   docker.rst
+
+
+
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/docs/source/reflect.rst b/docs/source/reflect.rst
new file mode 100644
index 0000000..c059206
--- /dev/null
+++ b/docs/source/reflect.rst
@@ -0,0 +1,5 @@
+``reflect.py``: Getting information about code
+==============================================
+
+.. automodule:: reflect
+    :members:
\ No newline at end of file