Add sample file upload for "load_data" blueprint

Also add a success page that redirects to the Dashboard

9 files changed, 195 insertions, 11 deletions

diff --git a/ustayml/__init__.py b/ustayml/__init__.py
index 135ef57..fbe1b3a 100644
--- a/ustayml/__init__.py
+++ b/ustayml/__init__.py
@@ -11,6 +11,8 @@ def create_app(test_config=None):
     app.config.from_mapping(
         SECRET_KEY='dev',
         DATABASE=os.path.join(app.instance_path, 'ustayml.sqlite'),
+        DATASET_PATH=os.path.join(app.instance_path, 'dataset'),
+        STUDENT_DATA_PATH=os.path.join(app.instance_path, 'student'),
     )
 
     if test_config is None:
@@ -32,14 +34,13 @@ def create_app(test_config=None):
     from . import db
     db.init_app(app)
 
-    from .views import auth
-    app.register_blueprint(auth.bp)
-
     # from .views import blog
     # app.register_blueprint(blog.bp)
 
-    from .views import students
+    from .views import auth, students, load_data
+    app.register_blueprint(auth.bp)
     app.register_blueprint(students.bp)
+    app.register_blueprint(load_data.bp)
 
     # Extra
     app.add_url_rule('/', endpoint='index')
diff --git a/ustayml/db.py b/ustayml/db.py
index 04587cc..5fb5ea8 100644
--- a/ustayml/db.py
+++ b/ustayml/db.py
@@ -35,6 +35,16 @@ def close_db(e=None):
 # CLI:
 # https://flask.palletsprojects.com/en/3.0.x/cli/
 
+def init_fs():
+    """Init file system directories"""
+    import os
+    dirs = [
+        current_app.config['DATASET_PATH'],
+        current_app.config['STUDENT_DATA_PATH']
+    ]
+    for d in dirs:
+        os.makedirs(d, exist_ok=True)
+
 def init_db():
     db = get_db()
 
@@ -45,8 +55,9 @@ def init_db():
 @click.command('init-db')
 def init_db_command():
     """Clear the existing data and create new tables."""
-    init_db()
-    click.echo('Initialized the database.')
+    init_fs()
+    # init_db()
+    # click.echo('Initialized the database.')
 
 # Register function with application
 
diff --git a/ustayml/static/style.css b/ustayml/static/style.css
index 0357f46..3d52059 100644
--- a/ustayml/static/style.css
+++ b/ustayml/static/style.css
@@ -117,7 +117,7 @@ header .action {
 
 .flex-container-horizontal {
   display: flex;
-  align-items: flex-end;
+  align-items: center;
 }
 
 /* tooltip */
@@ -257,6 +257,39 @@ header .action {
   flex: auto;
 }
 
+/* load_data */
+
+ul.data-validation-list {
+  list-style: none;
+  margin-left: 2em;
+  padding-left: 0;
+}
+
+ul.data-validation-list li:before {
+  content: '☑️';
+  padding-right: 1em;
+}
+
+ul.data-validation-list li {
+  /* padding-left: 0.5em; */
+  text-indent: -2em;
+}
+
+.load-data-step {
+  margin-top: 1em;
+  padding: 0 .5em;
+  border: 1px solid black;
+}
+
+.load-data-step .left {
+  /* background-color: aqua; */
+  padding-right: 1em;
+}
+
+.load-data-step .right {
+  width: 40%;
+}
+
 /* post */
 
 .post > header {
@@ -314,7 +347,7 @@ input.danger {
 
 input[type="submit"] {
   align-self: start;
-  min-width: 10em;
+  /* min-width: 10em; */
 }
 
 /* text status styles */
diff --git a/ustayml/templates/base.html b/ustayml/templates/base.html
index b8d243c..eefbc06 100644
--- a/ustayml/templates/base.html
+++ b/ustayml/templates/base.html
@@ -5,10 +5,10 @@
 <nav>
   <h1><a href="{{ url_for('index') }}">u-stayML</a></h1>
   <ul>
-    <li><a href="{{ url_for('students.index') }}">Dashboard</a></li>
+    <li><a href="#">Dashboard</a></li>
     <li><a href="{{ url_for('students.index') }}">Estudiantes</a></li>
     <li><a href="#">Historial</a>       </li>
-    <li><a href="#">Cargar datos</a>    </li>
+    <li><a href="{{ url_for('load_data.index') }}">Cargar datos</a>    </li>
   </ul>
   <ul>
     {% if g.user %}
diff --git a/ustayml/templates/load_data/index.html b/ustayml/templates/load_data/index.html
new file mode 100644
index 0000000..3400de3
--- /dev/null
+++ b/ustayml/templates/load_data/index.html
@@ -0,0 +1,71 @@
+{% extends 'base.html' %}
+
+
+{% block header %}
+  <h1>{% block title %}Cargar datos{% endblock %}</h1>
+{% endblock %}
+
+{% block content %}
+  <span>Siga los siguientes pasos para actualizar los datos (dataset) sobre el
+  cual se desarrollarán las posteriores predicciones de riesgo de deserción
+  estudiantil.</span>
+  <form method=post enctype=multipart/form-data>
+    <div class="load-data-step">
+      <h2>1. Diccionario de Variables</h2>
+      <div class="flex-container-horizontal">
+        <div class="left">
+          <p>El diccionario de variables lista información sobre las variables del dataset.</p>
+          <input type=file name=diccionario>
+          {# <input type=submit value="Subir"> #}
+          <p></p>
+        </div>
+        <div class="right">
+          <ul class="data-validation-list">
+            <li>Tipo de archivo debe ser CSV</li>
+            <li>Tamaño debe ser &lt;1 MiB</li>
+            <li>Debe existir la variable "target"</li>
+          </ul>
+        </div>
+      </div>
+    </div>
+    <div class="load-data-step">
+      <h2>2. Dataset</h2>
+      <div class="flex-container-horizontal">
+        <div class="left">
+          <p>El diccionario de variables lista información sobre las variables del dataset.</p>
+          <input type=file name=dataset>
+          {# <input type=submit value="Subir"> #}
+          <p></p>
+        </div>
+        <div class="right">
+          <ul class="data-validation-list">
+            <li>Tipo de archivo debe ser CSV</li>
+            <li>Tamaño debe ser &lt;2 GiB</li>
+            <li>Las variables deben coincidir con el diccionario de variables</li>
+          </ul>
+        </div>
+      </div>
+    </div>
+    <div class="load-data-step">
+      <h2>3. Añadir identificadores y guardar</h2>
+      <div class="flex-container-horizontal">
+        <div class="left">
+          <p>El diccionario de variables lista información sobre las variables del dataset.</p>
+            <label for="dataset-label">Etiqueta: </label>
+            <input name="dataset-label" id="dataset-label" required>
+            <p></p>
+            <label for="dataset-date">Fecha del conjunto de datos: </label>
+            <input name="dataset-date" id="dataset-date" required type="date">
+
+            <div></div>
+            <p></p>
+            <input type="submit" value="💾 Guardar">
+            <p></p>
+        </div>
+        <div class="right">
+          <p>(*) Campos Obligatorios</p>
+        </div>
+      </div>
+    </div>
+  </form>
+{% endblock %}
\ No newline at end of file
diff --git a/ustayml/templates/load_data/success.html b/ustayml/templates/load_data/success.html
new file mode 100644
index 0000000..442ef74
--- /dev/null
+++ b/ustayml/templates/load_data/success.html
@@ -0,0 +1,16 @@
+
+{% extends 'base.html' %}
+
+
+{% block header %}
+  <h1>{% block title %}Procesando datos ⏳{% endblock %}</h1>
+{% endblock %}
+
+{% block content %}
+  <p>La carga de datos fue exitosa.  Por favor espere unos minutos hasta que el
+  nuevo conjunto de datos termine de procesar y se generen nuevas predicciones 
+  de riesgo de deserción.</p>
+  <p>Podrá validar que el procesamiento terminó verificando la fecha del
+  reporte.</p>
+  <a href="{{ url_for('index') }}">Regresar al dashboard</a>
+{% endblock %}
\ No newline at end of file
diff --git a/ustayml/templates/students/details.html b/ustayml/templates/students/details.html
index a9c0cca..663124c 100644
--- a/ustayml/templates/students/details.html
+++ b/ustayml/templates/students/details.html
@@ -47,7 +47,6 @@
           </div>
         </li>
         <li>Mérito: {{ student['current_merit'] }}</li>
-
         <li>
           <div class="tooltip">
             Long. de estudios est. (semestres):
diff --git a/ustayml/views/load_data.py b/ustayml/views/load_data.py
new file mode 100644
index 0000000..e57f40c
--- /dev/null
+++ b/ustayml/views/load_data.py
@@ -0,0 +1,50 @@
+import os
+from flask import (
+    Blueprint, flash, g, redirect, render_template, request, url_for, current_app
+)
+from werkzeug.exceptions import abort
+from werkzeug.utils import secure_filename
+
+from ustayml.views.auth import login_required
+from ustayml.db import get_db, get_paginated_rows, get_row
+
+bp = Blueprint('load_data', __name__, url_prefix='/load_data')
+
+ALLOWED_EXTENSIONS = ['txt', 'csv']
+
+
[email protected]('/', methods=('GET', 'POST'))
+@login_required
+def index():
+    if request.method == 'POST':
+        # check if the post request has the file part
+        if 'diccionario' not in request.files:
+            flash('No file part')
+            return redirect(request.url)
+        file = request.files['diccionario']
+        # If the user does not select a file, the browser submits an
+        # empty file without a filename.
+        if file.filename == '':
+            flash('No selected file')
+            return redirect(request.url)
+        if file and allowed_file(file.filename):
+            filename = secure_filename(file.filename)
+            file.save(os.path.join(current_app.config['DATASET_PATH'], filename))
+            return redirect(url_for('load_data.success', name=filename))
+    return render_template(
+        'load_data/index.html'
+    )
+
[email protected]('/success', methods=('GET', 'POST'))
+@login_required
+def success():
+    return render_template(
+        'load_data/success.html'
+    )
+
+
+# Helper functions
+
+def allowed_file(filename):
+    return '.' in filename and \
+           filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
\ No newline at end of file
diff --git a/ustayml/views/students.py b/ustayml/views/students.py
index 6132746..3b31790 100644
--- a/ustayml/views/students.py
+++ b/ustayml/views/students.py
@@ -68,6 +68,9 @@ def details(student_id):
         [student_id]
     )
 
+    if s is None:
+        abort(404, "El ID del estudiante es inválido.")
+
     s['fullname'] = f"{s['first_name']} {s['last_name']}"
     s['current_attendance'] = f"{s['current_attendance'] * 100:.2f}%"
     s['current_merit'] = f"{s['current_merit'] * 100:.2f}%"