diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..647f5f8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +Scanners/PE-Sieve/Analysis \ No newline at end of file diff --git a/Config/config.yaml b/Config/config.yaml index 462681b..8203f19 100644 --- a/Config/config.yaml +++ b/Config/config.yaml @@ -16,6 +16,7 @@ upload: - sys max_file_size: 16777216 # 16MB in bytes upload_folder: "Uploads" + result_folder: "Results" analysis: static: diff --git a/README.md b/README.md index 3f4adf4..a772d35 100644 --- a/README.md +++ b/README.md @@ -76,10 +76,15 @@ Features include: ### File Operations ```http -POST /upload # Upload files for analysis -GET /analyze/static/ # Static file analysis -POST /analyze/dynamic/ # Dynamic file analysis -POST /analyze/dynamic/ # Process analysis +POST /upload # Upload files for analysis +GET /analyze/static/ # Static file analysis +POST /analyze/dynamic/ # Dynamic file analysis +POST /analyze/dynamic/ # Process analysis +GET /files # Get list of processed files +GET /file//info # Get file info +GET /file//static # Get results for file static analysis +GET /file//dynamic # Get results for file dynamic analysis +DELETE /file/ # Delete single analysis ``` ### System Management @@ -152,4 +157,4 @@ This project incorporates the following open-source components and acknowledges ![dynamic](https://github.com/user-attachments/assets/c4251116-ebe3-45eb-9a22-0254a3945e5a) - +![summary](./app/static/images/summary.jpg) diff --git a/app/routes.py b/app/routes.py index c9a8dd3..f3d9dac 100644 --- a/app/routes.py +++ b/app/routes.py @@ -8,6 +8,7 @@ import os import shutil import psutil import pefile +import json from .analyzers.manager import AnalysisManager from flask import render_template, request, jsonify from werkzeug.utils import secure_filename @@ -216,7 +217,7 @@ def get_office_info(filepath): print(f"Error analyzing Office file: {e}") return {'office_info': None} -def save_uploaded_file(file, upload_folder): +def save_uploaded_file(file, upload_folder, result_folder): file_content = file.read() file.close() md5_hash = hashlib.md5(file_content).hexdigest() @@ -228,6 +229,8 @@ def save_uploaded_file(file, upload_folder): os.makedirs(upload_folder, exist_ok=True) filepath = os.path.join(upload_folder, filename) + os.makedirs(result_folder, exist_ok=True) + os.makedirs(os.path.join(result_folder, filename), exist_ok=True) with open(filepath, 'wb') as f: f.write(file_content) @@ -270,6 +273,10 @@ def save_uploaded_file(file, upload_folder): print(f"Warning: {office_result['error']}") file_info.update(office_result) + # save file info to result folder + with open(os.path.join(result_folder, filename, 'file_info.json'), 'w') as f: + json.dump(file_info, f) + return file_info def find_file_by_hash(file_hash, upload_folder): @@ -335,7 +342,7 @@ def register_routes(app): if file and allowed_file(file.filename, app.config): try: - file_info = save_uploaded_file(file, app.config['upload']['upload_folder']) + file_info = save_uploaded_file(file, app.config['upload']['upload_folder'], app.config['upload']['result_folder']) return jsonify({ 'message': 'File uploaded successfully', 'file_info': file_info @@ -369,6 +376,7 @@ def register_routes(app): else: # Look for file as before file_path = find_file_by_hash(target, app.config['upload']['upload_folder']) + result_path = find_file_by_hash(target, app.config['upload']['result_folder']) if not file_path: return jsonify({'error': 'File not found'}), 404 if request.method == 'GET': @@ -381,9 +389,15 @@ def register_routes(app): if is_pid: return jsonify({'error': 'Cannot perform static analysis on PID'}), 400 results = analysis_manager.run_static_analysis(file_path) + # save results to result folder + with open(os.path.join(result_path, 'static_analysis_results.json'), 'w') as f: + json.dump(results, f) elif analysis_type == 'dynamic': target_for_analysis = target if is_pid else file_path results = analysis_manager.run_dynamic_analysis(target_for_analysis, is_pid) + # save results to result folder + with open(os.path.join(result_path, 'dynamic_analysis_results.json'), 'w') as f: + json.dump(results, f) else: return jsonify({'error': 'Invalid analysis type'}), 400 @@ -395,12 +409,55 @@ def register_routes(app): except Exception as e: return jsonify({'error': str(e)}), 500 + @app.route('/file//', methods=['GET']) + def get_analysis_results(target, analysis_type): + try: + # Find result folder for the given hash + result_path = find_file_by_hash(target, app.config['upload']['result_folder']) + if not result_path: + return jsonify({'error': 'Results not found'}), 404 + + # Handle different types of requests + if analysis_type == 'info': + # Read and return the file info + file_info_path = os.path.join(result_path, 'file_info.json') + if not os.path.exists(file_info_path): + return jsonify({'error': 'File info not found'}), 404 + + with open(file_info_path, 'r') as f: + results = json.load(f) + + elif analysis_type in ['static', 'dynamic']: + # Read and return the analysis results + results_file = f'{analysis_type}_analysis_results.json' + results_path = os.path.join(result_path, results_file) + if not os.path.exists(results_path): + return jsonify({'error': 'Analysis results not found'}), 404 + + with open(results_path, 'r') as f: + results = json.load(f) + + else: + return jsonify({'error': 'Invalid analysis type'}), 400 + + return jsonify({ + 'status': 'success', + 'results': results + }) + + except Exception as e: + return jsonify({ + 'status': 'error', + 'error': str(e) + }), 500 + @app.route('/cleanup', methods=['POST']) def cleanup(): try: results = { 'uploads_cleaned': 0, 'analysis_cleaned': 0, + 'result_cleaned': 0, 'errors': [] } @@ -420,6 +477,22 @@ def register_routes(app): except Exception as e: results['errors'].append(f"Error accessing uploads folder: {str(e)}") + # delete all folders in result folder + result_folder = app.config['upload']['result_folder'] + if os.path.exists(result_folder): + try: + folders = os.listdir(result_folder) + for folder in folders: + folder_path = os.path.join(result_folder, folder) + try: + if os.path.isdir(folder_path): + shutil.rmtree(folder_path) + results['result_cleaned'] += 1 + except Exception as e: + results['errors'].append(f"Error deleting {folder}: {str(e)}") + except Exception as e: + results['errors'].append(f"Error accessing result folder: {str(e)}") + # Clean analysis folders analysis_path = os.path.join('.', 'Scanners', 'PE-Sieve', 'Analysis') if os.path.exists(analysis_path): @@ -530,4 +603,123 @@ def register_routes(app): 'issues': [str(e)] }), 500 + @app.route('/summary') + @app.route('/summary.html') + def summary_page(): + """Route for the summary page""" + return render_template('summary.html') + + @app.route('/files', methods=['GET']) + def get_files_summary(): + try: + results_dir = app.config['upload']['result_folder'] + summary = {} + + # Iterate through all subdirectories in the results folder + for md5_dir in os.listdir(results_dir): + dir_path = os.path.join(results_dir, md5_dir) + if not os.path.isdir(dir_path): + continue + + # Read file_info.json + file_info_path = os.path.join(dir_path, 'file_info.json') + if not os.path.exists(file_info_path): + continue + + with open(file_info_path, 'r') as f: + file_info = json.load(f) + + # Check for existence of analysis results + static_exists = os.path.exists(os.path.join(dir_path, 'static_analysis_results.json')) + dynamic_exists = os.path.exists(os.path.join(dir_path, 'dynamic_analysis_results.json')) + + # Create summary for this file + summary[md5_dir] = { + 'md5': file_info.get('md5', 'unknown'), + 'sha256': file_info.get('sha256', 'unknown'), + 'filename': file_info.get('original_name', 'unknown'), + 'file_size': file_info.get('size', 0), + 'upload_time': file_info.get('upload_time', 'unknown'), + 'result_dir_full_path': os.path.abspath(dir_path), + 'entropy_value': file_info.get('entropy_analysis', {}).get('value', 0), + 'detection_risk': file_info.get('entropy_analysis', {}).get('detection_risk', 'Unknown'), + 'has_static_analysis': static_exists, + 'has_dynamic_analysis': dynamic_exists + } + + return jsonify({ + 'status': 'success', + 'count': len(summary), + 'files': summary + }) + + except Exception as e: + return jsonify({ + 'status': 'error', + 'error': str(e) + }), 500 + + @app.route('/file/', methods=['DELETE']) + def delete_file(target): + try: + # Find file in uploads folder + upload_path = find_file_by_hash(target, app.config['upload']['upload_folder']) + result_path = find_file_by_hash(target, app.config['upload']['result_folder']) + analysis_path = os.path.join('.', 'Scanners', 'PE-Sieve', 'Analysis') + + deleted = { + 'upload': False, + 'result': False, + 'analysis': False + } + + # Delete from uploads if exists + if upload_path: + try: + if os.path.isfile(upload_path): + os.unlink(upload_path) + deleted['upload'] = True + except Exception as e: + app.logger.error(f"Error deleting upload file: {str(e)}") + + # Delete result folder if exists + if result_path: + try: + if os.path.isdir(result_path): + shutil.rmtree(result_path) + deleted['result'] = True + except Exception as e: + app.logger.error(f"Error deleting result folder: {str(e)}") + + # Delete analysis folders if they exist + if os.path.exists(analysis_path): + try: + # Find all process_* folders related to this file + process_folders = glob.glob(os.path.join(analysis_path, f'*_{target}_*')) + for folder in process_folders: + if os.path.isdir(folder): + shutil.rmtree(folder) + deleted['analysis'] = True + except Exception as e: + app.logger.error(f"Error deleting analysis folders: {str(e)}") + + # Check if anything was deleted + if not any(deleted.values()): + return jsonify({ + 'status': 'error', + 'message': 'File not found' + }), 404 + + return jsonify({ + 'status': 'success', + 'message': 'File deleted successfully', + 'details': deleted + }) + + except Exception as e: + return jsonify({ + 'status': 'error', + 'message': str(e) + }), 500 + return app diff --git a/app/static/css/style.css b/app/static/css/style.css index 9e49428..ec0a614 100644 --- a/app/static/css/style.css +++ b/app/static/css/style.css @@ -73,4 +73,31 @@ body { to { transform: rotate(360deg); } +} + +/* Form Controls Dark Theme Override */ +select, +input { + background-color: rgb(17 24 39 / 0.3) !important; + -webkit-appearance: none; + -moz-appearance: none; + appearance: none; +} + +/* Custom Select Dropdown Arrow */ +select { + background-image: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='none' viewBox='0 0 20 20'%3e%3cpath stroke='%236b7280' stroke-linecap='round' stroke-linejoin='round' stroke-width='1.5' d='M6 8l4 4 4-4'/%3e%3c/svg%3e"); + background-position: right 0.5rem center; + background-repeat: no-repeat; + background-size: 1.5em 1.5em; + padding-right: 2.5rem; +} + +/* Autofill Style Override */ +input:-webkit-autofill, +input:-webkit-autofill:hover, +input:-webkit-autofill:focus, +input:-webkit-autofill:active { + -webkit-box-shadow: 0 0 0 30px rgb(17 24 39 / 0.3) inset !important; + -webkit-text-fill-color: rgb(209 213 219) !important; } \ No newline at end of file diff --git a/app/static/images/summary.jpg b/app/static/images/summary.jpg new file mode 100644 index 0000000..51a98d8 Binary files /dev/null and b/app/static/images/summary.jpg differ diff --git a/app/static/js/status.js b/app/static/js/status.js index 0a08756..6c3bf3e 100644 --- a/app/static/js/status.js +++ b/app/static/js/status.js @@ -194,6 +194,11 @@ class StatusManager { } } +// Show Summary +function showSummary() { + window.location.href = '/summary'; +} + // Notification System const NotificationSystem = { show(message, className, duration = CONFIG.notificationDuration) { @@ -385,12 +390,12 @@ const CleanupSystem = { formatResponse(data) { if (data.status === 'success') { return { - message: `Cleanup successful:\n- ${data.details.uploads_cleaned} files removed\n- ${data.details.analysis_cleaned} PE-Sieve folders cleaned`, + message: `Cleanup successful:\n- ${data.details.uploads_cleaned} files removed\n- ${data.details.analysis_cleaned} PE-Sieve folders cleaned\n- ${data.details.result_cleaned} result folders cleaned`, className: 'bg-green-500' }; } else if (data.status === 'warning') { return { - message: `Cleanup completed with warnings:\n- ${data.details.uploads_cleaned} files removed\n- ${data.details.analysis_cleaned} PE-Sieve folders cleaned\n\nErrors:\n${data.details.errors.join('\n')}`, + message: `Cleanup completed with warnings:\n- ${data.details.uploads_cleaned} files removed\n- ${data.details.analysis_cleaned} PE-Sieve folders cleaned\n- ${data.details.result_cleaned} result folders cleaned\n\nErrors:\n${data.details.errors.join('\n')}`, className: 'bg-yellow-500' }; } else { diff --git a/app/static/js/summary.js b/app/static/js/summary.js new file mode 100644 index 0000000..f9cadfa --- /dev/null +++ b/app/static/js/summary.js @@ -0,0 +1,310 @@ +// DOM Elements +const elements = { + fileList: document.getElementById('fileList'), + fileRowTemplate: document.getElementById('fileRowTemplate'), + emptyState: document.getElementById('emptyState'), + searchFiles: document.getElementById('searchFiles'), + filterType: document.getElementById('filterType'), + filterRisk: document.getElementById('filterRisk'), + sortBy: document.getElementById('sortBy'), + totalFiles: document.getElementById('totalFiles'), + storageUsed: document.getElementById('storageUsed'), + averageRisk: document.getElementById('averageRisk'), + averageEntropy: document.getElementById('averageEntropy') +}; + +// State +let files = []; + +// Fetch and render files +async function loadFiles() { + try { + const response = await fetch('/files'); + const data = await response.json(); + + if (data.status === 'success') { + files = Object.entries(data.files).map(([md5, file]) => ({ + md5, + ...file + })); + updateStats(); + renderFiles(); + } + } catch (error) { + console.error('Error loading files:', error); + } +} + +// Update statistics +function updateStats() { + elements.totalFiles.textContent = files.length; + + // Calculate total storage + const totalBytes = files.reduce((sum, file) => sum + (file.file_size || 0), 0); + elements.storageUsed.textContent = formatFileSize(totalBytes); + + // Calculate average entropy and determine risk + const filesWithEntropy = files.filter(f => f.entropy_value); + + if (filesWithEntropy.length > 0) { + const avgEntropy = filesWithEntropy.reduce((sum, file) => sum + file.entropy_value, 0) / filesWithEntropy.length; + + // Determine risk level based on entropy value + let riskText; + let riskClass; + + if (avgEntropy >= 7.2) { + riskText = 'High'; + riskClass = 'bg-red-500 text-white'; + } else if (avgEntropy >= 6.8) { + riskText = 'Medium'; + riskClass = 'bg-yellow-500 text-black'; + } else { + riskText = 'Low'; + riskClass = 'bg-green-500 text-white'; + } + // console.log(avgEntropy); + + elements.averageRisk.textContent = riskText; + elements.averageRisk.className = 'px-2 py-1 text-sm rounded-lg inline-flex items-center justify-center font-medium ' + riskClass; + elements.averageEntropy.textContent = `Entropy: ${avgEntropy.toFixed(3)}`; + } else { + elements.averageRisk.textContent = '-'; + elements.averageRisk.className = 'px-2 py-1 text-sm rounded-lg inline-flex items-center justify-center font-medium bg-gray-500 text-white'; + elements.averageEntropy.textContent = 'Entropy: -'; + } +} + +// Render file list +function renderFiles() { + const filteredFiles = filterFiles(files); + const sortedFiles = sortFiles(filteredFiles); + + elements.fileList.innerHTML = ''; + elements.emptyState.classList.toggle('hidden', sortedFiles.length > 0); + + sortedFiles.forEach(file => { + const row = elements.fileRowTemplate.content.cloneNode(true); + + // File name and hash + row.querySelector('[data-field="fileName"]').textContent = file.filename; + row.querySelector('[data-field="fileHash"]').textContent = file.md5; + + // Entropy and Risk + const entropyEl = row.querySelector('[data-field="fileEntropy"]'); + const riskEl = row.querySelector('[data-field="fileRisk"]'); + + if (file.entropy_value) { + entropyEl.textContent = `Entropy: ${file.entropy_value.toFixed(2)}`; + } + + if (file.detection_risk) { + riskEl.textContent = file.detection_risk; + riskEl.className = 'px-3 py-1 text-xs rounded-lg inline-flex items-center justify-center font-medium'; + switch(file.detection_risk.toLowerCase()) { + case 'high': + // riskEl.className += ' bg-red-500/10 text-red-400 border border-red-900/20'; + riskEl.className += ' bg-red-500 text-white'; + break; + case 'medium': + // riskEl.className += ' bg-yellow-500/10 text-yellow-400 border border-yellow-900/20'; + riskEl.className += ' bg-yellow-500 text-black'; + break; + case 'low': + // riskEl.className += ' bg-green-500/10 text-green-400 border border-green-900/20'; + riskEl.className += ' bg-green-500 text-white'; + break; + default: + // riskEl.className += ' bg-gray-500/10 text-gray-400 border border-gray-900/20'; + riskEl.className += ' bg-gray-500 text-white'; + } + } + // File type + // const typeCell = row.querySelector('#fileType'); + // const fileExt = file.filename.split('.').pop().toLowerCase(); + // typeCell.textContent = fileExt; + + // File size + row.querySelector('[data-field="fileSize"]').textContent = formatFileSize(file.file_size); + + // Upload time + row.querySelector('[data-field="fileUploadDate"]').textContent = file.upload_time; + + // Analysis status + const statusCell = row.querySelector('[data-field="fileAnalysisStatus"]'); + const status = getAnalysisStatus(file); + statusCell.className = `px-2 py-1 text-sm rounded-lg ${status.class}`; + statusCell.textContent = status.text; + + // Action buttons + const viewButton = row.querySelector('[data-action="view"]'); + const deleteButton = row.querySelector('[data-action="delete"]'); + + viewButton.onclick = () => viewFile(file.md5); + deleteButton.onclick = () => showFileDeleteWarning(file.md5); + + elements.fileList.appendChild(row); + }); +} + +// Filter files based on search and type +function filterFiles(files) { + const searchTerm = elements.searchFiles.value.toLowerCase(); + const fileType = elements.filterType.value; + const riskLevel = elements.filterRisk.value; + + return files.filter(file => { + const matchesSearch = file.filename.toLowerCase().includes(searchTerm) || + file.md5.toLowerCase().includes(searchTerm); + const matchesType = fileType === 'all' || file.filename.toLowerCase().endsWith(fileType); + const matchesRisk = riskLevel === 'all' || + (file.detection_risk && file.detection_risk.toLowerCase() === riskLevel); + return matchesSearch && matchesType && matchesRisk; + }); +} + +// Sort files based on selected criteria +function sortFiles(files) { + const sortBy = elements.sortBy.value; + + return [...files].sort((a, b) => { + switch (sortBy) { + case 'name': + return a.filename.localeCompare(b.filename); + case 'newest': + return new Date(b.upload_time).getTime() - new Date(a.upload_time).getTime(); + case 'oldest': + return new Date(a.upload_time).getTime() - new Date(b.upload_time).getTime(); + case 'size': + return (b.file_size || 0) - (a.file_size || 0); + case 'entropy': + return (b.entropy_value || 0) - (a.entropy_value || 0); + default: + return 0; + } + }); +} + +// Get analysis status display properties +function getAnalysisStatus(file) { + if (file.has_static_analysis && file.has_dynamic_analysis) { + return { + text: 'Complete', + class: 'bg-green-500/10 text-green-400 border border-green-900/20' + }; + } else if (file.has_static_analysis || file.has_dynamic_analysis) { + return { + text: 'Partial', + class: 'bg-yellow-500/10 text-yellow-400 border border-yellow-900/20' + }; + } + return { + text: 'Pending', + class: 'bg-gray-500/10 text-gray-400 border border-gray-900/20' + }; +} + +// View file details +function viewFile(md5) { + window.location.href = `/file/${md5}/info`; +} + +// Show/hide file delete warning +function showFileDeleteWarning(md5) { + const modal = document.getElementById('fileDeleteWarningModal'); + const confirmButton = document.getElementById('confirmDeleteButton'); + + // Set up the confirm button to call deleteFile with the correct md5 + confirmButton.onclick = () => deleteFile(md5); + modal?.classList.remove('hidden'); +} + +function hideFileDeleteWarning() { + const modal = document.getElementById('fileDeleteWarningModal'); + modal?.classList.add('hidden'); +} + +// Delete file +async function deleteFile(md5) { + try { + const response = await fetch(`/file/${md5}`, { + method: 'DELETE' + }); + + if (response.ok) { + // Hide modal first + hideFileDeleteWarning(); + + // Wait a brief moment for the modal to hide + await new Promise(resolve => setTimeout(resolve, 300)); + + // Remove from local array and update UI + files = files.filter(file => file.md5 !== md5); + updateStats(); + renderFiles(); + } + } catch (error) { + console.error('Error deleting file:', error); + } +} + +// Show cleanup warning for summary page +function showSummaryCleanupWarning() { + const modal = document.getElementById('summaryCleanupWarningModal'); + modal?.classList.remove('hidden'); +} + +// Hide cleanup warning for summary page +function hideSummaryCleanupWarning() { + const modal = document.getElementById('summaryCleanupWarningModal'); + modal?.classList.add('hidden'); +} + +// Cleanup all files +async function cleanupFiles() { + try { + const response = await fetch('/cleanup', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json' + } + }); + + if (response.ok) { + // Hide modal first + hideSummaryCleanupWarning(); + // Wait a brief moment for the modal to hide + await new Promise(resolve => setTimeout(resolve, 300)); + // Then reload + window.location.reload(true); + } + } catch (error) { + console.error('Error cleaning files:', error); + } +} + +// Make functions available globally +window.showSummaryCleanupWarning = showSummaryCleanupWarning; +window.hideSummaryCleanupWarning = hideSummaryCleanupWarning; +window.cleanupFiles = cleanupFiles; +window.showFileDeleteWarning = showFileDeleteWarning; +window.hideFileDeleteWarning = hideFileDeleteWarning; + +// Utility: Format file size +function formatFileSize(bytes) { + if (bytes === 0) return '0 B'; + const k = 1024; + const sizes = ['B', 'KB', 'MB', 'GB', 'TB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; +} + +// Event listeners +elements.searchFiles.addEventListener('input', () => renderFiles()); +elements.filterType.addEventListener('change', () => renderFiles()); +elements.sortBy.addEventListener('change', () => renderFiles()); +elements.filterRisk.addEventListener('change', () => renderFiles()); + +// Initialize +loadFiles(); \ No newline at end of file diff --git a/app/templates/base.html b/app/templates/base.html index 9ddece0..50bbe7f 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -66,6 +66,14 @@ Cleanup Garbage + + + diff --git a/app/templates/summary.html b/app/templates/summary.html new file mode 100644 index 0000000..63c6674 --- /dev/null +++ b/app/templates/summary.html @@ -0,0 +1,238 @@ +{% extends "base.html" %} + +{% block page_title %}File Management{% endblock %} + +{% block content %} +
+ +
+
+

File Management

+

Manage uploaded files and analysis results

+
+ + +
+ + + +
+
+ + +
+ +
+ +
+
+
Total Files
+
0
+
+
+
Storage Used
+
0 MB
+
+
+
Average Risk
+
+ - + Entropy: - +
+
+
+ + +
+
+ + + + +
+
+ + + +
+
+ + +
+ + + + + + + + + + + + + + + +
File NameRiskSizeUpload DateAnalysis StatusActions
+
+ + + +
+
+
+ + + + + + + + + +{% endblock %} + +{% block scripts %} + +{% endblock %} \ No newline at end of file