Importer page 2.0

.github/copilot-instructions.md

@@ -3,6 +3,10 @@
 ## Repository Summary
 SISmanager is a Python-based project for managing Student Information Systems, focused on data import/export, deduplication, and backup of student records. It is designed for use in Dockerized environments and uses Poetry for dependency management. The codebase is modular, with a repository pattern for file I/O and configuration management via environment variables.
 
+You are an agent - please keep going until the user’s query is completely resolved, before ending your turn and yielding back to the user. Only terminate your turn when you are sure that the problem is solved.
+If you are not sure about file content or codebase structure pertaining to the user’s request, use your tools to read files and gather the relevant information: do NOT guess or make up an answer.
+You MUST plan extensively before each function call, and reflect extensively on the outcomes of the previous function calls. DO NOT do this entire process by making function calls only, as this can impair your ability to solve the problem and think insightfully.
+
 ## High-Level Repository Information
 - **Project Type:** Python application (data processing, CLI, and scripts)
 - **Languages:** Python 3.10+
@@ -79,3 +83,30 @@ SISmanager is a Python-based project for managing Student Information Systems, f
 ## Trust These Instructions
 - Trust these instructions for build, test, and validation steps.
 - Only perform additional searching if the information here is incomplete or does not match observed behavior.
+
+
+## General Prompt
+
+You are an expert coding assistant.
+Your goals are:
+
+Keep solutions as simple as possible.
+Double check all steps, outputs, and changes.
+Always verify before finalizing.
+If you create temporary files/scripts, clean them up at the end.
+If a task is unclear, ask for clarification before proceeding.
+When I ask for help:
+
+Be explicit and clear in your instructions and explanations.
+Add context or reasoning for your choices.
+If you use tools, reflect on their results before taking the next step.
+For multi-step tasks, break them down and plan before acting.
+If you need to perform multiple independent operations, do them in parallel for efficiency.
+Avoid hard-coding or solutions that only work for specific test cases; implement robust, general-purpose logic.
+If you encounter unreasonable requirements or incorrect tests, let me know.
+Format your responses as follows:
+
+State your plan and reasoning.
+List steps or actions you will take.
+After each step, reflect and verify correctness.
+Summarize results and next steps.

sismanager/blueprints/importer/routes.py

@@ -2,14 +2,108 @@
 Routes for importer blueprint in SISmanager.
 """
 
-from flask import Blueprint, render_template
+import os
+import uuid
+from flask import (
+    Blueprint,
+    render_template,
+    request,
+    redirect,
+    url_for,
+    send_from_directory,
+    flash,
+)
+from sismanager.services.inout.xlsx_importer_service import XLSXImporter
+from sismanager.services.inout.backup_service import BackupManager
 
 importer_bp = Blueprint(
     "importer", __name__, template_folder="../../templates/importer"
 )
 
+ALLOWED_EXTENSIONS = {"xlsx", "xls"}
+
+
+def allowed_file(filename):
+    """Check if file extension is allowed."""
+    return "." in filename and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS
+
 
 @importer_bp.route("/importer")
-def importer():
+def importer_page():
     """Render the importer page."""
     return render_template("importer/importer.html")
+
+
+@importer_bp.route("/importer/upload", methods=["POST"])
+def upload_and_process():
+    """
+    Handle file upload and complete processing workflow.
+    This single endpoint handles the entire workflow:
+    1. Upload and validate files
+    2. Process through XLSXImporter
+    3. Optionally remove duplicates
+    4. Provide download links
+    This eliminates the need for complex client-side orchestration.
+    """
+    if "file" not in request.files:
+        flash("No file part")
+        return redirect(request.url)
+    file = request.files["file"]
+    if file.filename == "":
+        flash("No selected file")
+        return redirect(request.url)
+    if not allowed_file(file.filename):
+        flash("File type not allowed")
+        return redirect(request.url)
+
+    # Save uploaded file
+    uploads_dir = os.path.abspath(
+        os.path.join(os.path.dirname(__file__), "../../..", "data", "uploads")
+    )
+    os.makedirs(uploads_dir, exist_ok=True)
+    unique_id = str(uuid.uuid4())
+    filename = f"{unique_id}_{file.filename}"
+    file_path = os.path.join(uploads_dir, filename)
+    file.save(file_path)
+
+    # Process XLSX, pass original filename for orderCode
+    importer = XLSXImporter(file_path, original_filename=file.filename)
+    importer.process()
+
+    # Delete backups older than 30 days
+    backup_manager = BackupManager()
+    backup_manager.delete_old_backups(days=30)
+
+    # Remove duplicates if checkbox checked
+    remove_duplicates = request.form.get("remove_duplicates") == "yes"
+    if remove_duplicates:
+        importer.remove_duplicates(mode="forceful")
+
+    # Export processed file
+    processed_dir = os.path.abspath(
+        os.path.join(os.path.dirname(__file__), "../../..", "data", "processed")
+    )
+    os.makedirs(processed_dir, exist_ok=True)
+    output_filename = f"processed_{unique_id}.xlsx"
+    output_path = os.path.join(processed_dir, output_filename)
+    importer.export_to_xlsx(output_path)
+
+    # Generate preview HTML (full table, scrollable in frontend) using in-memory DataFrame
+    df = importer.repository.read()
+    output_preview = df.to_html(classes="table table-bordered", index=False)
+
+    # Provide download link and preview
+    return render_template(
+        "importer/importer.html",
+        download_link=url_for("importer.download_file", file_id=output_filename),
+        output_preview=output_preview,
+    )
+
+
+@importer_bp.route("/api/download/<file_id>")
+def download_file(file_id: str):
+    """Download a processed file."""
+    processed_dir = os.path.abspath(
+        os.path.join(os.path.dirname(__file__), "../../..", "data", "processed")
+    )
+    return send_from_directory(processed_dir, file_id, as_attachment=True)

sismanager/services/inout/xlsx_importer_service.py

@@ -22,10 +22,12 @@ def __init__(
         xlsx_path: str,
         columns_to_keep: Optional[List[str]] = None,
         repository: Optional[CentralDBRepository] = None,
+        original_filename: Optional[str] = None,
     ):
         """Initialize with XLSX file path, optional columns to keep, and repository (DI)."""
         self.xlsx_path = xlsx_path
         self.file_name = os.path.basename(xlsx_path)
+        self.original_filename = original_filename
         self.rows = []
         self.columns_to_keep = columns_to_keep
         self.backup_manager = BackupManager()
@@ -37,8 +39,8 @@ def read_xlsx(self):
             df = pd.read_excel(self.xlsx_path)
             if self.columns_to_keep:
                 df = df[self.columns_to_keep]
-            # Add orderCode column with the file name (without extension) as the first column
-            order_code = os.path.splitext(self.file_name)[0]
+            # Add orderCode col with the original file name (without extension) as the first column
+            order_code = os.path.splitext(self.original_filename or self.file_name)[0]
             df.insert(0, "orderCode", order_code)
             # Progress bar for converting to dict
             self.rows = []

sismanager/templates/importer/importer.html

@@ -2,4 +2,25 @@
 {% block content %}
 <h1>Importer</h1>
 <p>Import your data here.</p>
+<form action="/importer/upload" method="post" enctype="multipart/form-data">
+	<label for="file">Select XLSX file:</label>
+	<input type="file" id="file" name="file" accept=".xlsx,.xls" required><br><br>
+	<input type="checkbox" id="remove_duplicates" name="remove_duplicates" value="yes">
+	<label for="remove_duplicates">Remove duplicates</label><br><br>
+	<button type="submit">Import</button>
+</form>
+
+<br>
+<form action="{{ download_link if download_link else '#' }}" method="get">
+	<button type="submit"
+		{% if not download_link %}disabled title="No processed file available. Please import a file first."{% else %}title="Download the processed file."{% endif %}
+	>Download processed file</button>
+</form>
+
+{% if output_preview %}
+	<h2>Preview of processed file</h2>
+	<div style="max-height:400px; overflow:auto; border:1px solid #ccc; margin-top:10px;">
+		{{ output_preview|safe }}
+	</div>
+{% endif %}
 {% endblock %}

test.sh

@@ -4,4 +4,4 @@
 set -e
 
 # Run all tests with pytest
-poetry run pytest -v --cov=src --cov-report=term-missing
+poetry run pytest -v --cov=sismanager --cov-report=term-missing