diff --git a/backend/cases/serializers.py b/backend/cases/serializers.py index d280680a..5ad34c3f 100644 --- a/backend/cases/serializers.py +++ b/backend/cases/serializers.py @@ -2,6 +2,7 @@ from .models import Case from django.contrib.auth.models import User from core.serializers import UserSerializer +from core.validators import validate_upload_filename # class UserSerializer(serializers.ModelSerializer): # class Meta: @@ -18,7 +19,9 @@ class Meta: class InitiateUploadSerializer(serializers.Serializer): - filename = serializers.CharField(max_length=255) + filename = serializers.CharField( + max_length=255, validators=[validate_upload_filename] + ) os = serializers.CharField(max_length=255) case_id = serializers.IntegerField() diff --git a/backend/cases/views.py b/backend/cases/views.py index 410b9eb5..27d6791a 100644 --- a/backend/cases/views.py +++ b/backend/cases/views.py @@ -9,6 +9,7 @@ from evidences.models import Evidence from .serializers import CaseSerializer, InitiateUploadSerializer, UploadChunkSerializer, CompleteUploadSerializer from core.permissions import get_accessible_cases, check_case_access +from core.validators import safe_media_path import os import shutil @@ -69,8 +70,11 @@ def post(self, request): except ValueError: return Response({'error': 'Invalid chunk filenames.'}, status=status.HTTP_400_BAD_REQUEST) - final_filename = upload_session.filename - final_file_path = os.path.join(settings.MEDIA_ROOT, 'evidences', final_filename) + try: + final_file_path = safe_media_path('evidences', upload_session.filename) + except ValueError: + return Response({'error': 'Invalid filename.'}, status=status.HTTP_400_BAD_REQUEST) + final_filename = os.path.basename(final_file_path) os.makedirs(os.path.dirname(final_file_path), exist_ok=True) # Assemble the chunks into the final file @@ -85,7 +89,7 @@ def post(self, request): # Create the Evidence record evidence = Evidence.objects.create( name=final_filename, - url=f"file://{os.path.join(settings.MEDIA_ROOT, 'evidences', final_filename)}", + url=f"file://{final_file_path}", linked_case=upload_session.case, os=upload_session.os, etag=upload_session.upload_id, diff --git a/backend/core/validators.py b/backend/core/validators.py new file mode 100644 index 00000000..2c6306d8 --- /dev/null +++ b/backend/core/validators.py @@ -0,0 +1,44 @@ +import os + +from django.conf import settings +from rest_framework import serializers + + +def is_safe_upload_filename(value: str) -> bool: + """ + Return True only if ``value`` is a plain filename that cannot escape its + directory: no path separators (POSIX or Windows), no parent-directory + references, no null bytes, and equal to its own basename. + """ + if not value or value in (".", ".."): + return False + if "/" in value or "\\" in value or "\x00" in value: + return False + return value == os.path.basename(value) + + +def validate_upload_filename(value: str) -> str: + """DRF field validator wrapper around :func:`is_safe_upload_filename`.""" + if not is_safe_upload_filename(value): + raise serializers.ValidationError("Invalid filename.") + return value + + +def safe_media_path(subdir: str, filename: str) -> str: + """ + Resolve ``MEDIA_ROOT//`` and guarantee the result stays + inside ``MEDIA_ROOT/``. Raises ``ValueError`` on any unsafe filename + or escape attempt. + + The filename is validated here so callers cannot forget to; this is the + single safe sink for assembling user-named uploads. ``subdir`` MUST be a + trusted, hardcoded value (never user-controlled) — the guarantee only covers + the filename, not the subdir. + """ + if not is_safe_upload_filename(filename): + raise ValueError("Invalid upload filename.") + base_dir = os.path.realpath(os.path.join(settings.MEDIA_ROOT, subdir)) + target = os.path.realpath(os.path.join(base_dir, filename)) + if target == base_dir or os.path.commonpath([base_dir, target]) != base_dir: + raise ValueError("Resolved upload path escapes the target directory.") + return target diff --git a/backend/entrypoint.sh b/backend/entrypoint.sh index 9615b20a..f344e181 100755 --- a/backend/entrypoint.sh +++ b/backend/entrypoint.sh @@ -16,38 +16,44 @@ if [ "$DATABASE" = "postgresql" ] || [ "$DATABASE" = "postgres" ]; then echo "PostgreSQL is ready!" fi -echo "=== STARTING MIGRATIONS ===" -echo "Current working directory: $(pwd)" -echo "Current user: $(whoami)" -echo "Python version: $(python --version)" - -echo "Checking migration status..." -if ! python manage.py showmigrations --list > /dev/null 2>&1; then - echo "Migration table appears to be corrupted. Attempting to fix..." - - python manage.py migrate --run-syncdb --verbosity=2 || { - echo "Syncdb failed, trying fake-initial migration..." - python manage.py migrate --fake-initial --verbosity=2 - } +# Only one service runs migrations/static/admin; running them in every worker +# would make concurrent migrate runs race on the same DDL. +if [ "${RUN_MIGRATIONS:-false}" = "true" ]; then + echo "=== STARTING MIGRATIONS ===" + echo "Current working directory: $(pwd)" + echo "Current user: $(whoami)" + echo "Python version: $(python --version)" + + echo "Checking migration status..." + if ! python manage.py showmigrations --list > /dev/null 2>&1; then + echo "Migration table appears to be corrupted. Attempting to fix..." + + python manage.py migrate --run-syncdb --verbosity=2 || { + echo "Syncdb failed, trying fake-initial migration..." + python manage.py migrate --fake-initial --verbosity=2 + } + else + echo "Migration table exists, proceeding normally..." + echo "Django apps: $(python manage.py showmigrations --list)" + fi + + echo "Making migrations..." + python manage.py makemigrations cases evidences yararulesets yararules volatility_engine symbols --noinput --verbosity=2 + + echo "Applying migrations..." + python manage.py migrate --noinput --verbosity=2 + + echo "Migrations completed successfully!" + + echo "Collecting static files..." + python manage.py collectstatic --noinput --verbosity=2 + + echo "Creating admin user if not exists..." + python manage.py initadmin else - echo "Migration table exists, proceeding normally..." - echo "Django apps: $(python manage.py showmigrations --list)" + echo "=== SKIPPING MIGRATIONS (RUN_MIGRATIONS not set) ===" fi -echo "Making migrations..." -python manage.py makemigrations cases evidences yararulesets yararules volatility_engine symbols --noinput --verbosity=2 - -echo "Applying migrations..." -python manage.py migrate --noinput --verbosity=2 - -echo "Migrations completed successfully!" - -echo "Collecting static files..." -python manage.py collectstatic --noinput --verbosity=2 - -echo "Creating admin user if not exists..." -python manage.py initadmin - echo "=== ENTRYPOINT COMPLETED ===" exec "$@" \ No newline at end of file diff --git a/backend/evidences/signals.py b/backend/evidences/signals.py index 5c48d2b4..23252352 100644 --- a/backend/evidences/signals.py +++ b/backend/evidences/signals.py @@ -1,3 +1,8 @@ +import logging +import os +import shutil + +from django.conf import settings from django.db.models.signals import post_save, post_delete from django.dispatch import receiver from evidences.models import Evidence @@ -5,10 +10,24 @@ from asgiref.sync import async_to_sync from channels.layers import get_channel_layer +logger = logging.getLogger(__name__) + @receiver(post_save, sender=Evidence) def send_evidence_created(sender, instance, created, **kwargs): - + + # Kick off automatic Linux ISF resolution so the analyst doesn't have to + # upload kernel symbols manually before running plugins. + if created and instance.os == "linux": + try: + from volatility_engine.tasks import generate_linux_symbols + generate_linux_symbols.delay(instance.id) + except Exception: + import logging + logging.getLogger(__name__).exception( + "Failed to enqueue Linux ISF resolution for evidence %s", instance.id + ) + channel_layer = get_channel_layer() serializer = EvidenceSerializer(instance) @@ -26,6 +45,22 @@ def send_evidence_created(sender, instance, created, **kwargs): ) +@receiver(post_delete, sender=Evidence) +def cleanup_evidence_media(sender, instance, **kwargs): + """ + Remove the evidence's dumped-artefact directory (media/{id}/) from disk. + """ + artefact_dir = os.path.join(settings.MEDIA_ROOT, str(instance.id)) + if os.path.isdir(artefact_dir): + try: + shutil.rmtree(artefact_dir, ignore_errors=True) + logger.info("Removed media directory for deleted evidence %s", instance.id) + except Exception: + logger.exception( + "Failed to remove media directory for deleted evidence %s", instance.id + ) + + @receiver(post_delete, sender=Evidence) def send_evidence_deleted(sender, instance, **kwargs): channel_layer = get_channel_layer() diff --git a/backend/symbols/signals.py b/backend/symbols/signals.py index fc8775b4..740c5d3b 100644 --- a/backend/symbols/signals.py +++ b/backend/symbols/signals.py @@ -15,6 +15,23 @@ def send_symbol_created(sender, instance, created, **kwargs): {"type": "send_notification", "status": "created", "message": serializer.data}, ) + # A newly uploaded Linux ISF may satisfy evidences still waiting for symbols: + # re-verify the ones not yet ready so the extraction gate can open. + if created and instance.os == "Linux": + try: + from volatility_engine.models import LinuxSymbolResolution + from volatility_engine.tasks import reverify_linux_symbols + pending = LinuxSymbolResolution.objects.exclude(status="ready").values_list( + "evidence_id", flat=True + ) + for evidence_id in pending: + reverify_linux_symbols.delay(evidence_id) + except Exception: + import logging + logging.getLogger(__name__).exception( + "Failed to trigger ISF re-verification after symbol upload" + ) + @receiver(post_delete, sender=Symbol) def send_symbol_deleted(sender, instance, **kwargs): @@ -24,3 +41,20 @@ def send_symbol_deleted(sender, instance, **kwargs): "symbols", {"type": "send_notification", "status": "deleted", "message": serializer.data}, ) + + # Deleting a Linux ISF may break evidences that were relying on it: + # re-validate the currently-ready ones so the extraction gate re-closes. + if instance.os == "Linux": + try: + from volatility_engine.models import LinuxSymbolResolution + from volatility_engine.tasks import reverify_linux_symbols + ready = LinuxSymbolResolution.objects.filter(status="ready").values_list( + "evidence_id", flat=True + ) + for evidence_id in ready: + reverify_linux_symbols.delay(evidence_id) + except Exception: + import logging + logging.getLogger(__name__).exception( + "Failed to trigger ISF re-validation after symbol deletion" + ) diff --git a/backend/volatility_engine/admin.py b/backend/volatility_engine/admin.py index 731bbbe8..f36f8ac7 100644 --- a/backend/volatility_engine/admin.py +++ b/backend/volatility_engine/admin.py @@ -1,7 +1,125 @@ +import json +import logging + from django.contrib import admin +from django.template.response import TemplateResponse +from django.urls import path -# Register your models here. from .models import VolatilityPlugin, EnrichedProcess -admin.site.register(VolatilityPlugin) +logger = logging.getLogger(__name__) + +_CURATED_FILES = ( + "volatility_engine/volweb_plugins.json", + "volatility_engine/volweb_misc.json", +) + + +def _curated_names(os_name): + """Set of curated plugin names (full dotted paths) for an OS.""" + names = set() + for fname in _CURATED_FILES: + try: + with open(fname) as fh: + data = json.load(fh) + names.update((data.get("plugins", {}).get(os_name, {}) or {}).keys()) + except Exception as e: + logger.warning(f"Could not read curated plugins from {fname}: {e}") + return names + + +def _installed_names_by_os(): + """ + Plugin names actually exposed by the installed volatility3, grouped by OS + and normalized to the same full dotted path used in the curated JSON. + """ + from volatility3 import framework + import volatility3.plugins + + framework.import_files(volatility3.plugins, True) + result = {"windows": set(), "linux": set()} + for short_name in framework.list_plugins().keys(): + # YARA scanning plugins (yarascan / vadyarascan / vmayarascan) have their + # own dedicated flow in VolWeb, not the curated selector — skip them. + if "yarascan" in short_name.lower(): + continue + for os_name in result: + if short_name.startswith(f"{os_name}."): + result[os_name].add(f"volatility3.plugins.{short_name}") + return result + + +def _deprecated_aliases(available): + """ + Top-level plugins that are just deprecated aliases of a plugin moved into a + subpackage (``.`` when ``.malware.`` or + ``.registry.`` also exists). These are noise in the diff. + """ + sub_modules = set() + for name in available: + parts = name[len("volatility3.plugins."):].split(".") + if len(parts) >= 3 and parts[1] in ("malware", "registry"): + sub_modules.add(f"{parts[0]}.{parts[1]}.{parts[2]}") + + deprecated = set() + for name in available: + parts = name[len("volatility3.plugins."):].split(".") + if len(parts) == 3: # os.module.Class (top-level) + if any(f"{parts[0]}.{sub}.{parts[1]}" in sub_modules for sub in ("malware", "registry")): + deprecated.add(name) + return deprecated + + +class VolatilityPluginAdmin(admin.ModelAdmin): + change_list_template = "admin/volatility_engine/volatilityplugin/change_list.html" + + def get_urls(self): + custom = [ + path( + "plugins-diff/", + self.admin_site.admin_view(self.plugins_diff_view), + name="volatility_engine_plugins_diff", + ), + ] + return custom + super().get_urls() + + def plugins_diff_view(self, request): + """ + Maintenance helper: diff the curated plugin list against the plugins + actually exposed by the *installed* volatility3, so the curated JSON can + be updated deliberately when the pinned volatility3 version is bumped. + Read-only; does not affect extraction. + """ + error = None + report = {} + try: + installed = _installed_names_by_os() + for os_name in ("windows", "linux"): + curated = _curated_names(os_name) + available = installed.get(os_name, set()) + deprecated = _deprecated_aliases(available) + report[os_name] = { + # in vol3, not curated, and not a deprecated alias -> real candidates + "missing": sorted((available - curated) - deprecated), + "stale": sorted(curated - available), # curated, gone from vol3 -> fix/remove + "deprecated_hidden": len(deprecated & (available - curated)), + "curated_count": len(curated), + "available_count": len(available), + } + except Exception as e: + logger.exception("Plugin diff failed") + error = str(e) + + context = { + **self.admin_site.each_context(request), + "title": "Volatility plugins — curated vs installed", + "report": report, + "error": error, + } + return TemplateResponse( + request, "admin/volatility_engine/plugins_diff.html", context + ) + + +admin.site.register(VolatilityPlugin, VolatilityPluginAdmin) admin.site.register(EnrichedProcess) diff --git a/backend/volatility_engine/engine.py b/backend/volatility_engine/engine.py index 93c9f287..fd7566e3 100644 --- a/backend/volatility_engine/engine.py +++ b/backend/volatility_engine/engine.py @@ -6,6 +6,7 @@ import volatility3 import traceback import os +import re import json import shutil from volatility3.cli import MuteProgress @@ -37,6 +38,116 @@ volatility3.framework.require_interface_version(2, 0, 0) logger = logging.getLogger(__name__) + +def _match_yara_braces(source, start): + """ + Given the index of an opening ``{`` in ``source``, return the index just + past its matching ``}``. String literals and comments are skipped so that + braces appearing inside them are ignored. Hex strings and regex + quantifiers ({n,m}) keep their braces balanced, so plain depth counting + locates the real end of a rule body. + """ + depth = 0 + k, n = start, len(source) + while k < n: + ch = source[k] + if ch == '"': + k += 1 + while k < n: + if source[k] == '\\': + k += 2 + continue + if source[k] == '"': + k += 1 + break + k += 1 + continue + if source.startswith("//", k): + nl = source.find("\n", k) + k = n if nl == -1 else nl + continue + if source.startswith("/*", k): + end = source.find("*/", k) + k = n if end == -1 else end + 2 + continue + if ch == '{': + depth += 1 + elif ch == '}': + depth -= 1 + if depth == 0: + return k + 1 + k += 1 + return n + + +def _dedupe_yara_rules(source): + """ + Drop duplicate top-level rule definitions (keeping the first of each + identifier) and collapse duplicate import/include statements, so a + duplicated identifier doesn't make yara.compile abort the whole scan. + + Returns ``(deduped_source, dropped_identifiers)``. + """ + imports = [] + seen_imports = set() + rules_out = [] + seen_rules = set() + dropped = [] + + rule_decl = re.compile(r'((?:(?:private|global)\s+)*)rule\s+([A-Za-z_]\w*)') + import_decl = re.compile(r'(import|include)\s+("[^"]*")') + + i, n = 0, len(source) + while i < n: + ch = source[i] + + if ch.isspace(): + i += 1 + continue + if source.startswith("//", i): + nl = source.find("\n", i) + i = n if nl == -1 else nl + continue + if source.startswith("/*", i): + end = source.find("*/", i) + i = n if end == -1 else end + 2 + continue + + m = import_decl.match(source, i) + if m: + stmt = f"{m.group(1)} {m.group(2)}" + if stmt not in seen_imports: + seen_imports.add(stmt) + imports.append(stmt) + i = m.end() + continue + + m = rule_decl.match(source, i) + if m: + identifier = m.group(2) + brace_start = source.find("{", m.end()) + if brace_start == -1: + break # malformed input — stop parsing defensively + end = _match_yara_braces(source, brace_start) + rule_text = source[i:end].strip() + if identifier in seen_rules: + dropped.append(identifier) + else: + seen_rules.add(identifier) + rules_out.append(rule_text) + i = end + continue + + # Unrecognised token — advance one char to stay robust. + i += 1 + + parts = [] + if imports: + parts.append("\n".join(imports)) + parts.extend(rules_out) + return ("\n\n".join(parts) + "\n", dropped) + + class VolatilityEngine: """ The Volatility3 Engine is a modular class to enable the execution multiple volatility3 plugins. @@ -182,6 +293,58 @@ def run_plugin(self, constructed): return result return None + def _collect_grid(self, plugin, column=None): + """ + Run a plugin and collect rows in-memory (no DB persistence). + Returns a list of the values of ``column`` (or full value tuples). + """ + self.build_context(plugin) + constructed = self.construct_plugin() + if not constructed: + return [] + grid = constructed.run() + col_names = [c.name for c in grid.columns] + idx = col_names.index(column) if column in col_names else None + rows = [] + + def _visit(node, acc): + try: + values = list(node.values) + rows.append(values[idx] if idx is not None else values) + except Exception: + pass + return acc + + if not grid.populated: + grid.populate(_visit, None) + else: + grid.visit(node=None, function=_visit, initial_accumulator=None) + return rows + + def detect_linux_banner(self): + """Scan the image for the Linux kernel banner; return it or None.""" + import volatility3.plugins.banners + + banners = self._collect_grid( + {volatility3.plugins.banners.Banners: {"name": "banners"}}, column="Banner" + ) + banners = [str(b).strip().strip("\x00").strip() for b in banners if b] + for banner in banners: + if "Linux version" in banner: + return banner + return banners[0] if banners else None + + def verify_linux_symbols(self): + """Return True if Linux symbols resolve for this image (runs linux.pslist).""" + try: + rows = self._collect_grid({PsList: {"name": "verify"}}) + return len(rows) > 0 + except UnsatisfiedException: + return False + except Exception as e: + logger.warning(f"Linux symbol verification failed: {e}") + return False + def start_timeliner(self): timeliner_plugin = { volatility3.plugins.timeliner.Timeliner: { @@ -387,13 +550,13 @@ def dump_file_windows(self, offset): } } self.build_context(dumpfiles_plugin) - self.context.config["plugins.DumpFiles.virtaddr"] = int(offset) + self.context.config["plugins.DumpFiles.virtaddr"] = [int(offset)] builted_plugin = self.construct_plugin() try: result = self.run_plugin(builted_plugin) if not result: del self.context.config["plugins.DumpFiles.virtaddr"] - self.context.config["plugins.DumpFiles.physaddr"] = int(offset) + self.context.config["plugins.DumpFiles.physaddr"] = [int(offset)] result = self.run_plugin(builted_plugin) fix_permissions(f"media/{self.obj.id}") @@ -414,14 +577,14 @@ def dump_file_linux(self, offset): } } self.build_context(dumpfiles_plugin) - self.context.config["plugins.DumpFiles.virtaddr"] = int(offset) + self.context.config["plugins.DumpFiles.virtaddr"] = [int(offset)] builted_plugin = self.construct_plugin() try: result = self.run_plugin(builted_plugin) fix_permissions(f"media/{self.evidence.id}") if not result: del self.context.config["plugins.DumpFiles.virtaddr"] - self.context.config["plugins.DumpFiles.physaddr"] = int(offset) + self.context.config["plugins.DumpFiles.physaddr"] = [int(offset)] result = self.run_plugin(builted_plugin) fix_permissions(f"media/{self.obj.id}") @@ -670,9 +833,15 @@ def start_ruleset_validation(self, skip_rule_validation=False): self.obj.save() return self.obj.status - def run_yara_scan(self, yara_ruleset=None, yara_rules=None, yara_rulesets=None): + def run_yara_scan(self, yara_ruleset=None, yara_rules=None, yara_rulesets=None, scan_scope="vad"): """ Run YARA scan on evidence with selected ruleset(s) or rules. + + scan_scope: + - "vad" -> scan process memory via VadYaraScan (default). + Best for malware/ransomware artefacts in user space. + - "kernel" -> scan the primary (kernel) layer via plain YaraScan. + Use for rootkits or kernel-mode threats. """ from yararules.models import YaraRule import traceback @@ -779,6 +948,15 @@ def run_yara_scan(self, yara_ruleset=None, yara_rules=None, yara_rulesets=None): return None logger.info(f"Combined {active_rules.count()} rules for scanning") + + # Drop duplicate identifiers so a collision doesn't abort the scan. + combined_rules, dropped_rules = _dedupe_yara_rules(combined_rules) + if dropped_rules: + logger.warning( + f"Dropped {len(dropped_rules)} duplicate rule identifier(s) " + f"before compilation: {', '.join(sorted(set(dropped_rules)))}" + ) + logger.debug(f"Combined rules content length: {len(combined_rules)} characters") # === FILE CREATION PHASE === @@ -811,9 +989,25 @@ def run_yara_scan(self, yara_ruleset=None, yara_rules=None, yara_rulesets=None): formatted_timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") scan_description = f"YARA scan using {' + '.join(scan_description_parts)} - processing at {formatted_timestamp}" - # Configure YARA scan plugin for Volatility + # "vad" -> per-process VadYaraScan (user space); "kernel" -> generic + # YaraScan on the primary layer (kernel space). + target_os = getattr(self.obj, "os", "windows") + normalized_scope = (scan_scope or "vad").lower() + if normalized_scope == "kernel": + yarascan_cls = volatility3.plugins.yarascan.YaraScan + plugin_config_prefix = "plugins.YaraScan" + else: + if target_os == "linux": + import volatility3.plugins.linux.vmayarascan as _vad_mod + yarascan_cls = _vad_mod.VmaYaraScan + plugin_config_prefix = "plugins.VmaYaraScan" + else: + import volatility3.plugins.windows.vadyarascan as _vad_mod + yarascan_cls = _vad_mod.VadYaraScan + plugin_config_prefix = "plugins.VadYaraScan" + yara_plugin = { - volatility3.plugins.yarascan.YaraScan: { + yarascan_cls: { "icon": "🔍", "description": scan_description, "category": "Malware", @@ -821,37 +1015,41 @@ def run_yara_scan(self, yara_ruleset=None, yara_rules=None, yara_rulesets=None): "name": f"volatility3.plugins.yarascan.{scan_id}", } } - + # Build context and configure plugin self.build_context(yara_plugin) - - # Try different path formats for the file - # Option 1: file:// URL with absolute path + + # Configure both the chosen plugin and the underlying YaraScan it delegates to. file_url = f"file://{os.path.abspath(temp_file_path)}" - self.context.config["plugins.YaraScan.yara_file"] = file_url - - logger.info(f"Context config after YARA file: {self.context.config.get('plugins.YaraScan.yara_file')}") + for prefix in (plugin_config_prefix, "plugins.YaraScan"): + self.context.config[f"{prefix}.yara_file"] = file_url + + logger.info(f"YARA plugin selected: {yarascan_cls.__module__}.{yarascan_cls.__name__}") + logger.info(f"Context config after YARA file: {self.context.config.get(f'{plugin_config_prefix}.yara_file')}") logger.info(f"Layer stacker location: {self.context.config.get('automagic.LayerStacker.single_location')}") - + # Build and run the plugin builted_plugin = self.construct_plugin() if not builted_plugin: - logger.error("Failed to construct YaraScan plugin") + logger.error("Failed to construct VadYaraScan plugin") - # If it fails with file://, try with direct absolute path + # Retry with the absolute path (some builds reject the file:// form). logger.info("Retrying with absolute path...") - self.context.config["plugins.YaraScan.yara_file"] = os.path.abspath(temp_file_path) + abs_path = os.path.abspath(temp_file_path) + for prefix in (plugin_config_prefix, "plugins.YaraScan"): + self.context.config[f"{prefix}.yara_file"] = abs_path builted_plugin = self.construct_plugin() if not builted_plugin: # Last attempt: relative path logger.info("Retrying with relative path...") - self.context.config["plugins.YaraScan.yara_file"] = temp_file_name + for prefix in (plugin_config_prefix, "plugins.YaraScan"): + self.context.config[f"{prefix}.yara_file"] = temp_file_name builted_plugin = self.construct_plugin() if not builted_plugin: - logger.error("All attempts to construct YaraScan plugin failed") + logger.error("All attempts to construct VadYaraScan plugin failed") return None # Stream each match directly to a JSONL file — no in-memory accumulation. @@ -943,7 +1141,8 @@ def _stream_visitor(node, accumulator): except Exception as e: logger.error(f"Failed to run YARA scan on evidence '{self.obj.name}': {str(e)}") logger.error(traceback.format_exc()) - return None + # Propagate so the calling task reports an error, not a fake success. + raise finally: # Cleanup of temporary file diff --git a/backend/volatility_engine/isf.py b/backend/volatility_engine/isf.py new file mode 100644 index 00000000..db91480c --- /dev/null +++ b/backend/volatility_engine/isf.py @@ -0,0 +1,227 @@ +""" +Linux ISF (Intermediate Symbol File) resolution. + +Given a kernel banner extracted from a memory image, look it up in the +community remote index (Abyss-W4tcher/volatility3-symbols) and download the +matching ISF into the VolWeb symbols directory so Volatility can use it. + +When no ISF can be imported, build human-readable guidance telling the analyst +exactly which debug package to fetch and how to produce the ISF manually. +""" +import json +import logging +import os +import re +import time + +import requests +from django.conf import settings + +logger = logging.getLogger(__name__) + +_REPO_RAW_BASE = "https://github.com/Abyss-W4tcher/volatility3-symbols/raw/master/" +# banners_plain.json maps the *plain* banner string -> [ISF paths]. (banners.json +# is the internal --remote-isf-url format, keyed by Python bytes-repr, and must +# NOT be used for plain-string lookups.) +_BANNERS_INDEX_URL = _REPO_RAW_BASE + "banners/banners_plain.json" + +# Where downloaded ISFs land — already on Volatility's symbol path (see +# volatility_engine/utils.py). Kept under a "linux" subdir by convention. +_SYMBOLS_SUBDIR = os.path.join("symbols", "linux") +_INDEX_CACHE = os.path.join("symbols", "_banners_plain_index.json") +_INDEX_TTL_SECONDS = 24 * 3600 + + +def _abs(media_relpath): + return os.path.join(settings.MEDIA_ROOT, media_relpath) + + +def _fetch_index(): + """Return the banner->ISF-paths mapping, cached locally with a TTL.""" + cache_path = _abs(_INDEX_CACHE) + fresh = ( + os.path.exists(cache_path) + and (time.time() - os.path.getmtime(cache_path)) < _INDEX_TTL_SECONDS + ) + if not fresh: + try: + os.makedirs(os.path.dirname(cache_path), exist_ok=True) + resp = requests.get(_BANNERS_INDEX_URL, timeout=120) + resp.raise_for_status() + with open(cache_path, "wb") as fh: + fh.write(resp.content) + except Exception as e: + logger.warning(f"Failed to refresh remote ISF index: {e}") + if not os.path.exists(cache_path): + return {} + try: + with open(cache_path, "r", encoding="utf-8") as fh: + return json.load(fh) + except Exception as e: + logger.warning(f"Failed to read cached ISF index: {e}") + return {} + + +def _normalize(banner): + return (banner or "").strip().strip("\x00").strip() + + +def _match_index(index, banner): + """ + Return the best ISF path for ``banner`` from the index, or None. + + Tries an exact (normalized) match first, then a looser match on the kernel + version + build id — our extracted banner may differ slightly from the + index key, but Volatility re-checks the banner exactly during verification, + so a version/build match is safe for *selecting* the ISF to download. + """ + # 1. Exact match (fast path). + if index.get(banner): + return index[banner][0] + target = _normalize(banner) + for key, paths in index.items(): + if paths and _normalize(key) == target: + return paths[0] + + # 2. Version + build fallback. + vm = re.search(r"Linux version (\S+)", banner or "") + if not vm: + return None + version = vm.group(1) # e.g. 6.5.0-41-generic + bm = re.search(r"(#\S+)", banner or "") + build = bm.group(1) if bm else None # e.g. #41~22.04.2-Ubuntu + for key, paths in index.items(): + if paths and version in key and (build is None or build in key): + return paths[0] + return None + + +def resolve_isf_remote(banner): + """ + Look up ``banner`` in the remote index and download the matching ISF. + + Returns the saved path relative to MEDIA_ROOT, or None if not found. + """ + index = _fetch_index() + if not index: + return None + + rel = _match_index(index, banner) + if not rel: + return None + + url = _REPO_RAW_BASE + rel + filename = os.path.basename(rel) + dest_rel = os.path.join(_SYMBOLS_SUBDIR, filename) + dest_abs = _abs(dest_rel) + os.makedirs(os.path.dirname(dest_abs), exist_ok=True) + + try: + with requests.get(url, stream=True, timeout=300) as resp: + resp.raise_for_status() + with open(dest_abs, "wb") as fh: + for chunk in resp.iter_content(chunk_size=1 << 16): + fh.write(chunk) + except Exception as e: + logger.warning(f"Failed to download ISF {url}: {e}") + return None + + logger.info(f"Downloaded ISF for banner to {dest_rel}") + return dest_rel + + +def parse_banner(banner): + """Extract (kernel_version, distro) best-effort from a banner string.""" + kernel = None + m = re.search(r"Linux version (\S+)", banner or "") + if m: + kernel = m.group(1) + + distro = "unknown" + low = (banner or "").lower() + if "ubuntu" in low: + distro = "ubuntu" + elif "debian" in low: + distro = "debian" + elif "kali" in low: + distro = "kali" + elif "red hat" in low or "redhat" in low or ".el" in low: + distro = "rhel" + elif "almalinux" in low or ".alma" in low: + distro = "almalinux" + elif "rocky" in low: + distro = "rocky" + elif "arch" in low: + distro = "arch" + elif "suse" in low: + distro = "suse" + return kernel, distro + + +def _arch_from_banner(banner): + low = (banner or "").lower() + if "x86_64" in low or "amd64" in low: + return "amd64" + if "aarch64" in low or "arm64" in low: + return "arm64" + return "i386" + + +def build_manual_guidance(banner): + """ + Produce structured guidance for building the ISF by hand when it could not + be imported automatically. Returned to the frontend for display. + + Ubuntu/Kali guidance is made precise using the same banner parsing as + Abyss-W4tcher/volatility3-symbols' ubuntu_symbols_finder: it derives the + exact dbgsym package name and .ddeb filename from the banner. + """ + kernel, distro = parse_banner(banner) + arch = _arch_from_banner(banner) + package = None + where = None + + ubuntu_m = re.search( + r"Linux version (\d+\.\d+\.\d+-\d+-\S+).+\(Ubuntu (\d+\.\d+\.\d+-\d+\.\d+)", + banner or "", + ) + if distro in ("ubuntu", "kali") and ubuntu_m: + short, extended = ubuntu_m.group(1), ubuntu_m.group(2) # 5.15.0-79-generic / 5.15.0-79.86 + package = f"linux-image-unsigned-{short}-dbgsym (version {extended})" + ddeb = f"linux-image-unsigned-{short}-dbgsym_{extended}_{arch}.ddeb" + where = ( + f"https://launchpad.net/ubuntu/+source/linux " + f"(package linux-image-unsigned-{short}-dbgsym, version {extended}) " + f"— or http://ddebs.ubuntu.com/pool/main/l/linux/{ddeb}" + ) + elif distro in ("ubuntu", "kali"): + package = f"linux-image-{kernel}-dbgsym (or linux-image-unsigned-{kernel}-dbgsym)" + where = "http://ddebs.ubuntu.com/pool/main/l/linux/" + elif distro == "debian": + package = f"linux-image-{kernel}-dbg" + where = "debian-debug archive (https://deb.debian.org/debian-debug/)" + elif distro in ("rhel", "almalinux", "rocky"): + package = f"kernel-debuginfo matching {kernel}" + where = "the distribution debuginfo repo (e.g. debuginfo.centos.org, vault, or subscription)" + else: + package = f"the kernel debug symbols (vmlinux with DWARF) for {kernel or 'this kernel'}" + where = "the debug-symbols repository of the distribution" + + steps = [ + f"Download the debug package: {package} from {where}.", + "Extract the unstripped vmlinux: `dpkg-deb -x .ddeb out/` → out/usr/lib/debug/boot/vmlinux-*.", + "Build the ISF with dwarf2json (github.com/volatilityfoundation/dwarf2json): " + "`dwarf2json linux --elf out/usr/lib/debug/boot/vmlinux-... > isf.json`.", + "Optionally compress: `xz -9e isf.json`.", + "Upload isf.json(.xz) in the Symbols page. VolWeb re-verifies it against this banner and, if it matches, unlocks the analysis.", + ] + + return { + "banner": banner, + "kernel": kernel, + "distro": distro, + "arch": arch, + "package": package, + "where": where, + "steps": steps, + } diff --git a/backend/volatility_engine/migrations/0002_volatilityplugin_error_message.py b/backend/volatility_engine/migrations/0002_volatilityplugin_error_message.py new file mode 100644 index 00000000..8dc837d3 --- /dev/null +++ b/backend/volatility_engine/migrations/0002_volatilityplugin_error_message.py @@ -0,0 +1,16 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("volatility_engine", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="volatilityplugin", + name="error_message", + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/backend/volatility_engine/migrations/0003_linuxsymbolresolution.py b/backend/volatility_engine/migrations/0003_linuxsymbolresolution.py new file mode 100644 index 00000000..bb7ff60f --- /dev/null +++ b/backend/volatility_engine/migrations/0003_linuxsymbolresolution.py @@ -0,0 +1,76 @@ +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("evidences", "0001_initial"), + ("symbols", "0001_initial"), + ("volatility_engine", "0002_volatilityplugin_error_message"), + ] + + operations = [ + migrations.CreateModel( + name="LinuxSymbolResolution", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "status", + models.CharField( + choices=[ + ("detecting", "Detecting banner"), + ("resolving", "Resolving ISF"), + ("verifying", "Verifying ISF"), + ("ready", "Ready"), + ("failed_banner", "Banner not found"), + ("failed_isf", "ISF not found"), + ], + default="detecting", + max_length=20, + ), + ), + ("banner", models.TextField(blank=True, null=True)), + ( + "method", + models.CharField( + blank=True, + choices=[ + ("remote", "Remote index"), + ("manual", "Manual upload"), + ], + max_length=20, + null=True, + ), + ), + ("guidance", models.JSONField(blank=True, null=True)), + ("message", models.TextField(blank=True, null=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "evidence", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="isf_resolution", + to="evidences.evidence", + ), + ), + ( + "linked_symbol", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="symbols.symbol", + ), + ), + ], + ), + ] diff --git a/backend/volatility_engine/models.py b/backend/volatility_engine/models.py index d31b0c98..0e26afaf 100644 --- a/backend/volatility_engine/models.py +++ b/backend/volatility_engine/models.py @@ -34,3 +34,40 @@ class EnrichedProcess(models.Model): def __str__(self): return str(self.pid) + + +class LinuxSymbolResolution(models.Model): + """ + Tracks automatic Linux ISF (symbol table) resolution for a Linux evidence. + Gates plugin execution: extraction is only allowed once status == "ready". + """ + + STATUS = ( + ("detecting", "Detecting banner"), + ("resolving", "Resolving ISF"), + ("verifying", "Verifying ISF"), + ("ready", "Ready"), + ("failed_banner", "Banner not found"), + ("failed_isf", "ISF not found"), + ) + METHODS = ( + ("remote", "Remote index"), + ("manual", "Manual upload"), + ) + + evidence = models.OneToOneField( + Evidence, on_delete=models.CASCADE, related_name="isf_resolution" + ) + status = models.CharField(max_length=20, choices=STATUS, default="detecting") + banner = models.TextField(null=True, blank=True) + method = models.CharField(max_length=20, choices=METHODS, null=True, blank=True) + # Manual-build guidance shown to the analyst when no ISF could be imported. + guidance = models.JSONField(null=True, blank=True) + message = models.TextField(null=True, blank=True) + linked_symbol = models.ForeignKey( + "symbols.Symbol", null=True, blank=True, on_delete=models.SET_NULL + ) + updated_at = models.DateTimeField(auto_now=True) + + def __str__(self): + return f"{self.evidence_id}:{self.status}" diff --git a/backend/volatility_engine/plugins/linux/volweb_main.py b/backend/volatility_engine/plugins/linux/volweb_main.py index 45bc5b9a..f32ee89c 100644 --- a/backend/volatility_engine/plugins/linux/volweb_main.py +++ b/backend/volatility_engine/plugins/linux/volweb_main.py @@ -7,6 +7,7 @@ from volatility3.framework.configuration import requirements from volatility3.framework.renderers import TreeGrid from volatility_engine.utils import DjangoRenderer, file_handler +from volatility_engine.models import VolatilityPlugin from evidences.models import Evidence vollog = logging.getLogger(__name__) @@ -50,8 +51,16 @@ def run_all(self): # Read optional PID filter pid_filter = self.context.config.get("VolWeb.PidFilter", None) + try: + kernel_module = self.context.modules[self.config["kernel"]] + self.context.config[f"{self.config_path}.primary"] = kernel_module.layer_name + except Exception as e: + vollog.warning(f"Could not expose 'primary' layer for layer-scanning plugins: {e}") + instances = {} + failed_plugins = [] for plugin, details in volweb_plugins.items(): + plugin_class = None try: plugin_class = self.dynamic_import(plugin) instances[plugin] = { @@ -61,10 +70,38 @@ def run_all(self): instances[plugin]["details"]["name"] = plugin except ImportError as e: vollog.error(f"Could not import {plugin}: {e}") + except Exception as e: + try: + unmet = plugin_class.unsatisfied(self.context, self.config_path) + unmet_desc = ", ".join(sorted(unmet.keys())) if unmet else "unknown" + except Exception: + unmet_desc = "unknown" + vollog.error( + f"Skipping plugin {plugin}: failed to validate; " + f"unsatisfied requirements: [{unmet_desc}] ({e})" + ) + failed_plugins.append( + (plugin, details, f"Unsatisfied requirements: {unmet_desc}") + ) evidence_id = self.context.config["VolWeb.Evidence"] evidence = Evidence.objects.get(id=evidence_id) + for name, details, err in failed_plugins: + VolatilityPlugin.objects.update_or_create( + name=name, + evidence=evidence, + defaults={ + "icon": details.get("icon"), + "description": details.get("description"), + "category": details.get("category", "Other"), + "display": details.get("display", "True"), + "artefacts": None, + "results": False, + "error_message": f"Plugin failed to construct/validate: {err}", + }, + ) + # Read optional per-plugin timeout (in seconds) plugin_timeout = self.context.config.get("VolWeb.PluginTimeout", None) diff --git a/backend/volatility_engine/plugins/windows/volweb_main.py b/backend/volatility_engine/plugins/windows/volweb_main.py index f53eee32..cb635aa6 100644 --- a/backend/volatility_engine/plugins/windows/volweb_main.py +++ b/backend/volatility_engine/plugins/windows/volweb_main.py @@ -53,8 +53,16 @@ def run_all(self): # Read optional PID filter pid_filter = self.context.config.get("VolWeb.PidFilter", None) + try: + kernel_module = self.context.modules[self.config["kernel"]] + self.context.config[f"{self.config_path}.primary"] = kernel_module.layer_name + except Exception as e: + vollog.warning(f"Could not expose 'primary' layer for layer-scanning plugins: {e}") + instances = {} + failed_plugins = [] for plugin, details in volweb_plugins.items(): + plugin_class = None try: plugin_class = self.dynamic_import(plugin) instances[plugin] = { @@ -64,10 +72,38 @@ def run_all(self): instances[plugin]["details"]["name"] = plugin except ImportError as e: vollog.error(f"Could not import {plugin}: {e}") + except Exception as e: + try: + unmet = plugin_class.unsatisfied(self.context, self.config_path) + unmet_desc = ", ".join(sorted(unmet.keys())) if unmet else "unknown" + except Exception: + unmet_desc = "unknown" + vollog.error( + f"Skipping plugin {plugin}: failed to validate; " + f"unsatisfied requirements: [{unmet_desc}] ({e})" + ) + failed_plugins.append( + (plugin, details, f"Unsatisfied requirements: {unmet_desc}") + ) evidence_id = self.context.config["VolWeb.Evidence"] evidence = Evidence.objects.get(id=evidence_id) + for name, details, err in failed_plugins: + VolatilityPlugin.objects.update_or_create( + name=name, + evidence=evidence, + defaults={ + "icon": details.get("icon"), + "description": details.get("description"), + "category": details.get("category", "Other"), + "display": details.get("display", "True"), + "artefacts": None, + "results": False, + "error_message": f"Plugin failed to construct/validate: {err}", + }, + ) + # Read optional per-plugin timeout (in seconds) plugin_timeout = self.context.config.get("VolWeb.PluginTimeout", None) diff --git a/backend/volatility_engine/tasks.py b/backend/volatility_engine/tasks.py index 3b35f335..fa99aa15 100644 --- a/backend/volatility_engine/tasks.py +++ b/backend/volatility_engine/tasks.py @@ -6,6 +6,8 @@ from channels.layers import get_channel_layer from asgiref.sync import async_to_sync from yararules.utils import is_batch_upload_active +from django.conf import settings +import os import logging logger = logging.getLogger(__name__) @@ -274,14 +276,17 @@ def start_ruleset_validation(yara_ruleset_id, skip_rule_validation=False): @shared_task -def start_yarascan(evidence_id, rulesets=None, rules=None): +def start_yarascan(evidence_id, rulesets=None, rules=None, scan_scope="vad"): """ Run YARA scan on evidence with selected rulesets and/or individual rules. - + Args: evidence_id: ID of the evidence to scan rulesets: List of ruleset IDs to use rules: List of individual rule IDs to use + scan_scope: "vad" (per-process memory, default) or "kernel" + (kernel layer). Selects which Volatility plugin + drives the scan; see VolatilityEngine.run_yara_scan. """ import traceback from datetime import datetime @@ -306,15 +311,15 @@ def start_yarascan(evidence_id, rulesets=None, rules=None): }, ) - # Initialize scan_executed to track if any scan was performed + # run_yara_scan: True=matches, False=no matches, None=could not run; raises on failure. + scan_result = None scan_executed = False - scan_results = [] - + # If specific rulesets are selected, combine them in a single scan if rulesets: from yararulesets.models import YaraRuleSet selected_rulesets = [] - + for ruleset_id in rulesets: try: # Try to fetch the ruleset regardless of its status. We will @@ -328,54 +333,45 @@ def start_yarascan(evidence_id, rulesets=None, rules=None): logger.warning(f"Ruleset {ruleset_id} found but not compiled yet; skipping") except YaraRuleSet.DoesNotExist: logger.warning(f"Ruleset {ruleset_id} not found") - - if selected_rulesets: - logger.info(f"Running YARA scan with {len(selected_rulesets)} rulesets combined") - scan_result = engine.run_yara_scan(yara_rulesets=selected_rulesets) - - # Mark that a scan was executed - scan_executed = True - - # Collect results if any matches found - if scan_result is not None and scan_result != []: - scan_results.extend(scan_result if isinstance(scan_result, list) else [scan_result]) - + + if not selected_rulesets: + raise RuntimeError( + "None of the selected rulesets are compiled and ready to scan." + ) + + logger.info(f"Running YARA scan with {len(selected_rulesets)} rulesets combined (scope={scan_scope})") + scan_result = engine.run_yara_scan(yara_rulesets=selected_rulesets, scan_scope=scan_scope) + scan_executed = True + # If specific rules are selected (without ruleset) elif rules: - logger.info(f"Running YARA scan with individual rules: {rules}") - - scan_result = engine.run_yara_scan(yara_rules=rules) - - # Mark that a scan was executed + logger.info(f"Running YARA scan with individual rules: {rules} (scope={scan_scope})") + scan_result = engine.run_yara_scan(yara_rules=rules, scan_scope=scan_scope) scan_executed = True - - # Collect results if any matches found - if scan_result is not None and scan_result != []: - scan_results.extend(scan_result if isinstance(scan_result, list) else [scan_result]) - + # If no specific selections, run with all active rules else: - logger.info("Running YARA scan with all active rules") - scan_result = engine.run_yara_scan() - - # Mark that a scan was executed + logger.info(f"Running YARA scan with all active rules (scope={scan_scope})") + scan_result = engine.run_yara_scan(scan_scope=scan_scope) scan_executed = True - - # Collect results if any matches found - if scan_result is not None and scan_result != []: - scan_results.extend(scan_result if isinstance(scan_result, list) else [scan_result]) - - # Determine the result based on whether scan was executed successfully - # A scan is successful if it was executed, regardless of whether matches were found - result = scan_executed - + + # None means nothing was scanned (inactive/uncompiled rules): treat as failure. + if not scan_executed or scan_result is None: + raise RuntimeError( + "YARA scan did not run: no compiled, active rules were available " + "for the selected rulesets/rules." + ) + + matches_found = bool(scan_result) + result = matches_found + # Generate a unique scan ID with timestamp for logging scan_timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") scan_id = f"scan_{scan_timestamp}" - - logger.info(f"YARA scan completed for evidence {evidence_id}. Found {len(scan_results)} total matches. Scan ID: {scan_id}") - - # Send finished notification + + logger.info(f"YARA scan completed for evidence {evidence_id}. Matches found: {matches_found}. Scan ID: {scan_id}") + + # result="true" => matches found; "false" => completed with no matches. async_to_sync(channel_layer.group_send)( f"volatility_tasks_{evidence_id}", { @@ -383,9 +379,8 @@ def start_yarascan(evidence_id, rulesets=None, rules=None): "message": { "name": "yarascan", "status": "finished", - "result": str(result).lower(), - "scan_id": scan_id, # Include scan ID in notification - "matches_count": len(scan_results), # Include match count + "result": str(matches_found).lower(), + "scan_id": scan_id, }, }, ) @@ -426,5 +421,150 @@ def start_yarascan(evidence_id, rulesets=None, rules=None): }, }, ) - - return result \ No newline at end of file + + return result + + +@shared_task +def generate_linux_symbols(evidence_id): + """ + Auto-resolve the Linux kernel ISF for a Linux evidence: detect the banner, + download a matching ISF from the community remote index, verify it against + the image, and register it as a Symbol. Gates Linux plugin execution until + status == "ready". On failure, records guidance for building it manually. + """ + from volatility_engine.models import LinuxSymbolResolution + from volatility_engine import isf as isf_mod + from symbols.models import Symbol + + instance = Evidence.objects.get(id=evidence_id) + if instance.os != "linux": + return + + channel_layer = get_channel_layer() + resolution, _ = LinuxSymbolResolution.objects.get_or_create(evidence=instance) + + def _set(status, **fields): + resolution.status = status + for key, value in fields.items(): + setattr(resolution, key, value) + resolution.save() + async_to_sync(channel_layer.group_send)( + f"volatility_tasks_{evidence_id}", + { + "type": "send_notification", + "message": { + "name": "isf", + "status": resolution.status, + "banner": resolution.banner, + "method": resolution.method, + "message": resolution.message, + "guidance": resolution.guidance, + }, + }, + ) + + engine = VolatilityEngine(instance) + + # Reset any stale banner/guidance from a previous run so intermediate + # states don't carry the old "manual build" guidance. + _set("detecting", banner=None, method=None, guidance=None, + message="Detecting kernel banner…") + try: + banner = engine.detect_linux_banner() + except Exception as e: + logger.error(f"Banner detection failed for evidence {evidence_id}: {e}") + banner = None + + if not banner: + _set("failed_banner", message="Could not find a Linux kernel banner in the image.") + return + + _set("resolving", banner=banner, message="Looking up matching ISF…") + try: + isf_rel = isf_mod.resolve_isf_remote(banner) + except Exception as e: + logger.error(f"Remote ISF resolution failed for evidence {evidence_id}: {e}") + isf_rel = None + + if not isf_rel: + _set( + "failed_isf", + method="remote", + guidance=isf_mod.build_manual_guidance(banner), + message="No matching ISF in the remote index. Build it manually and upload it.", + ) + return + + _set("verifying", method="remote", message="Verifying ISF against the image…") + if not engine.verify_linux_symbols(): + # Drop the non-working ISF so it doesn't pollute the symbol path. + try: + os.remove(os.path.join(settings.MEDIA_ROOT, isf_rel)) + except OSError: + pass + _set( + "failed_isf", + method="remote", + guidance=isf_mod.build_manual_guidance(banner), + message="An ISF was found but did not resolve against this image. Build it manually.", + ) + return + + symbol = Symbol.objects.create( + name=(banner.split("(")[0].strip()[:100] or "Linux ISF"), + os="Linux", + description=banner[:500], + symbols_file=isf_rel, + ) + _set("ready", method="remote", linked_symbol=symbol, message="Kernel symbols ready.") + + +@shared_task +def reverify_linux_symbols(evidence_id): + """ + Re-validate Linux kernel symbols for an evidence and update the extraction + gate accordingly. Runs after a manual ISF upload (may *open* the gate) or + after an ISF deletion (may *close* it), so it re-checks regardless of the + current status. + """ + from volatility_engine.models import LinuxSymbolResolution + from volatility_engine import isf as isf_mod + + instance = Evidence.objects.get(id=evidence_id) + if instance.os != "linux": + return + resolution = LinuxSymbolResolution.objects.filter(evidence=instance).first() + if not resolution: + return + + channel_layer = get_channel_layer() + + def _set(status, **fields): + resolution.status = status + for key, value in fields.items(): + setattr(resolution, key, value) + resolution.save() + async_to_sync(channel_layer.group_send)( + f"volatility_tasks_{evidence_id}", + { + "type": "send_notification", + "message": { + "name": "isf", + "status": resolution.status, + "banner": resolution.banner, + "method": resolution.method, + "message": resolution.message, + "guidance": resolution.guidance, + }, + }, + ) + + _set("verifying", message="Re-checking kernel symbols…") + engine = VolatilityEngine(instance) + if engine.verify_linux_symbols(): + _set("ready", guidance=None, message="Kernel symbols ready.") + else: + guidance = isf_mod.build_manual_guidance(resolution.banner) if resolution.banner else None + _set("failed_isf", guidance=guidance, + message="No usable ISF for this evidence. Re-fetch automatically or upload a matching ISF.") \ No newline at end of file diff --git a/backend/volatility_engine/templates/admin/volatility_engine/plugins_diff.html b/backend/volatility_engine/templates/admin/volatility_engine/plugins_diff.html new file mode 100644 index 00000000..7e3f987b --- /dev/null +++ b/backend/volatility_engine/templates/admin/volatility_engine/plugins_diff.html @@ -0,0 +1,50 @@ +{% extends "admin/base_site.html" %} + +{% block breadcrumbs %} + +{% endblock %} + +{% block content %} +

+ Compares the curated plugin list (volweb_plugins.json + + volweb_misc.json) against the plugins exposed by the + installed volatility3. Use it when bumping the pinned + volatility3 version to update the curated list deliberately. Read-only. +
Note: deprecated top-level aliases (modules moved under + malware/ or registry/) are hidden from "Missing" + automatically. Remaining entries can still be infra/utility plugins or ones + used only programmatically by the engine, so cross-check before adding. +

+ +{% if error %} +

Error: {{ error }}

+{% endif %} + +{% for os_name, data in report.items %} +

{{ os_name }} + (curated: {{ data.curated_count }}, installed: {{ data.available_count }}) +

+ +

Missing from curated list, candidates to add ({{ data.missing|length }}) + {% if data.deprecated_hidden %}({{ data.deprecated_hidden }} deprecated alias(es) hidden){% endif %} +

+ {% if data.missing %} +
    {% for name in data.missing %}
  • {{ name }}
  • {% endfor %}
+ {% else %} +

None, curated list covers everything installed.

+ {% endif %} + +

Curated but not in installed volatility3, fix / remove ({{ data.stale|length }})

+ {% if data.stale %} +
    {% for name in data.stale %}
  • {{ name }}
  • {% endfor %}
+ {% else %} +

None, no stale entries.

+ {% endif %} + +
+{% endfor %} +{% endblock %} diff --git a/backend/volatility_engine/templates/admin/volatility_engine/volatilityplugin/change_list.html b/backend/volatility_engine/templates/admin/volatility_engine/volatilityplugin/change_list.html new file mode 100644 index 00000000..8d033c23 --- /dev/null +++ b/backend/volatility_engine/templates/admin/volatility_engine/volatilityplugin/change_list.html @@ -0,0 +1,8 @@ +{% extends "admin/change_list.html" %} + +{% block object-tools-items %} +
  • + Plugin catalog diff +
  • + {{ block.super }} +{% endblock %} diff --git a/backend/volatility_engine/urls.py b/backend/volatility_engine/urls.py index cd27d41c..ec2936aa 100644 --- a/backend/volatility_engine/urls.py +++ b/backend/volatility_engine/urls.py @@ -22,6 +22,8 @@ YaraScanResultsView, AvailablePluginsView, SelectiveExtractionTask, + IsfResolutionView, + EvidenceLootView, ) urlpatterns = [ @@ -61,4 +63,6 @@ path("evidence/tasks/yarascan/stop/", StopYaraScanTask.as_view(), name="yarascan-stop"), path("evidence//available-plugins/", AvailablePluginsView.as_view(), name="available-plugins"), path("evidence/tasks/selective-extraction/", SelectiveExtractionTask.as_view(), name="selective-extraction"), + path("evidence//isf/", IsfResolutionView.as_view(), name="isf-resolution"), + path("evidence//loot/", EvidenceLootView.as_view(), name="evidence-loot"), ] diff --git a/backend/volatility_engine/views.py b/backend/volatility_engine/views.py index 75be93d4..d950cc80 100644 --- a/backend/volatility_engine/views.py +++ b/backend/volatility_engine/views.py @@ -3,6 +3,8 @@ from rest_framework import status from .models import VolatilityPlugin, EnrichedProcess from evidences.models import Evidence +from django.conf import settings +from datetime import datetime, timezone import os import json from yararules.models import YaraRule @@ -336,12 +338,28 @@ def post(self, request): evidence, err = _get_evidence_or_403(evidence_id, request.user) if err: return err + # Linux YARA scanning needs the kernel symbols too. + if evidence.os == "linux": + from .models import LinuxSymbolResolution + resolution = LinuxSymbolResolution.objects.filter(evidence=evidence).first() + if not resolution or resolution.status != "ready": + return Response( + {"error": "Kernel symbols (ISF) are not ready for this Linux evidence. " + "Resolve them from the plugin selection screen or upload a matching ISF."}, + status=status.HTTP_409_CONFLICT, + ) try: rulesets = request.data.get("rulesets", []) rules = request.data.get("rules", []) + scan_scope = request.data.get("scan_scope", "vad") + if scan_scope not in ("vad", "kernel"): + return Response( + {"error": "scan_scope must be either 'vad' or 'kernel'"}, + status=status.HTTP_400_BAD_REQUEST, + ) task = start_yarascan.apply_async( args=[evidence.id], - kwargs={"rulesets": rulesets, "rules": rules}, + kwargs={"rulesets": rulesets, "rules": rules, "scan_scope": scan_scope}, queue="yarascan", ) evidence.celery_task_id = task.id @@ -670,6 +688,18 @@ def post(self, request): if err: return err + # Gate Linux extraction on kernel-symbol readiness: without a matching + # ISF the Linux plugins cannot run. + if evidence.os == "linux": + from .models import LinuxSymbolResolution + resolution = LinuxSymbolResolution.objects.filter(evidence=evidence).first() + if not resolution or resolution.status != "ready": + return Response( + {"error": "Kernel symbols (ISF) are not ready for this Linux evidence. " + "Wait for automatic resolution to finish or upload a matching ISF."}, + status=status.HTTP_409_CONFLICT, + ) + try: run_timeliner = request.data.get("run_timeliner", False) pid_filter = request.data.get("pid", None) @@ -698,4 +728,116 @@ def post(self, request): return Response({"message": "Selective extraction started"}, status=status.HTTP_200_OK) except Exception as e: - return Response({"error": str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) \ No newline at end of file + return Response({"error": str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + +class EvidenceLootView(APIView): + """ + Inventory of the artefact files a plugin run or an on-demand dump wrote to + disk for an evidence (media/{id}/) + """ + + permission_classes = (IsAuthenticated,) + + # Files that live in the evidence media dir but are internal result stores, + # not user-facing forensic artefacts. + EXCLUDED_NAMES = {"yarascan_results.jsonl"} + + def _evidence_dir(self, evidence_id): + return os.path.realpath(os.path.join(settings.MEDIA_ROOT, str(evidence_id))) + + def get(self, request, evidence_id): + evidence, err = _get_evidence_or_403(evidence_id, request.user) + if err: + return err + + base_dir = self._evidence_dir(evidence_id) + artefacts = [] + if os.path.isdir(base_dir): + with os.scandir(base_dir) as entries: + for entry in entries: + if not entry.is_file() or entry.name in self.EXCLUDED_NAMES: + continue + stat = entry.stat() + artefacts.append({ + "name": entry.name, + "size": stat.st_size, + "modified": datetime.fromtimestamp( + stat.st_mtime, tz=timezone.utc + ).isoformat(), + "url": f"/media/{evidence_id}/{entry.name}", + }) + + artefacts.sort(key=lambda a: a["modified"], reverse=True) + total_size = sum(a["size"] for a in artefacts) + return Response( + {"count": len(artefacts), "total_size": total_size, "artefacts": artefacts}, + status=status.HTTP_200_OK, + ) + + def delete(self, request, evidence_id): + evidence, err = _get_evidence_or_403(evidence_id, request.user) + if err: + return err + + name = request.data.get("name") + if not name: + return Response({"error": "name is required"}, status=status.HTTP_400_BAD_REQUEST) + + base_dir = self._evidence_dir(evidence_id) + target = os.path.realpath(os.path.join(base_dir, name)) + + # Reject anything that escapes the evidence directory (path traversal) + # or targets the directory itself / an excluded internal file. + if ( + os.path.commonpath([base_dir, target]) != base_dir + or target == base_dir + or os.path.basename(target) in self.EXCLUDED_NAMES + ): + return Response({"error": "Invalid file name"}, status=status.HTTP_400_BAD_REQUEST) + + if not os.path.isfile(target): + return Response({"error": "Artefact not found"}, status=status.HTTP_404_NOT_FOUND) + + try: + os.remove(target) + except OSError as e: + return Response({"error": str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + return Response({"message": "Artefact deleted"}, status=status.HTTP_200_OK) + + +class IsfResolutionView(APIView): + """ + Linux kernel-symbol (ISF) resolution status for an evidence, and a retry + trigger. Windows evidences are always reported as ready (symbols bundled). + """ + permission_classes = (IsAuthenticated,) + + def get(self, request, evidence_id): + evidence, err = _get_evidence_or_403(evidence_id, request.user) + if err: + return err + if evidence.os != "linux": + return Response({"status": "ready", "banner": None, "guidance": None, "message": None}) + from .models import LinuxSymbolResolution + resolution = LinuxSymbolResolution.objects.filter(evidence=evidence).first() + if not resolution: + return Response({"status": "pending", "banner": None, "guidance": None, "message": None}) + return Response({ + "status": resolution.status, + "banner": resolution.banner, + "method": resolution.method, + "guidance": resolution.guidance, + "message": resolution.message, + }) + + def post(self, request, evidence_id): + evidence, err = _get_evidence_or_403(evidence_id, request.user) + if err: + return err + if evidence.os != "linux": + return Response({"error": "Not a Linux evidence"}, status=status.HTTP_400_BAD_REQUEST) + from .tasks import generate_linux_symbols + generate_linux_symbols.delay(evidence.id) + return Response({"message": "ISF resolution restarted"}, status=status.HTTP_200_OK) \ No newline at end of file diff --git a/backend/volatility_engine/volweb_plugins.json b/backend/volatility_engine/volweb_plugins.json index a919fc42..46b7b1d9 100644 --- a/backend/volatility_engine/volweb_plugins.json +++ b/backend/volatility_engine/volweb_plugins.json @@ -25,7 +25,7 @@ "category": "Filesystem", "display": "True" }, - "volatility3.plugins.windows.svcdiff.SvcDiff": { + "volatility3.plugins.windows.malware.svcdiff.SvcDiff": { "icon": "SyncProblem", "description": "Compares services found through list walking versus scanning to find rootkits.", "category": "Kernel", @@ -43,6 +43,12 @@ "category": "Registry", "display": "True" }, + "volatility3.plugins.windows.registry.printkey.PrintKey": { + "icon": "Key", + "description": "Lists and prints registry keys and values (root keys when no target key is given).", + "category": "Registry", + "display": "True" + }, "volatility3.plugins.windows.registry.hivescan.HiveScan": { "icon": "Search", "description": "Scans for registry hives present in a particular windows memory image.", @@ -91,12 +97,42 @@ "category": "Processes", "display": "True" }, - "volatility3.plugins.windows.malfind.Malfind": { + "volatility3.plugins.windows.malware.malfind.Malfind": { "icon": "BugReport", "description": "Lists process memory ranges that potentially contain injected code.", "category": "Malware", "display": "True" }, + "volatility3.plugins.windows.malware.pebmasquerade.PebMasquerade": { + "icon": "TheaterComedy", + "description": "Detects PEB masquerading (mismatch between PEB and on-disk image paths).", + "category": "Malware", + "display": "True" + }, + "volatility3.plugins.windows.vadwalk.VadWalk": { + "icon": "AccountTree", + "description": "Walks the VAD tree for each process.", + "category": "Processes", + "display": "True" + }, + "volatility3.plugins.windows.vadinfo.VadInfo": { + "icon": "Memory", + "description": "Lists process memory ranges (VADs) with detailed protection and mapping info.", + "category": "Processes", + "display": "True" + }, + "volatility3.plugins.windows.mftscan.MFTScan": { + "icon": "Storage", + "description": "Scans for Master File Table (MFT) entries recoverable from memory.", + "category": "Filesystem", + "display": "True" + }, + "volatility3.plugins.windows.etwpatch.EtwPatch": { + "icon": "EditOff", + "description": "Detects ETW patching used by malware to evade event logging.", + "category": "Malware", + "display": "True" + }, "volatility3.plugins.windows.cachedump.Cachedump": { "icon": "VpnKey", "description": "Dumps lsa secrets from memory.", @@ -121,7 +157,7 @@ "category": "Kernel", "display": "True" }, - "volatility3.plugins.windows.drivermodule.DriverModule": { + "volatility3.plugins.windows.malware.drivermodule.DriverModule": { "icon": "Build", "description": "Determines if any loaded drivers were hidden by a rootkit.", "category": "Kernel", @@ -163,7 +199,7 @@ "category": "Security", "display": "True" }, - "volatility3.plugins.windows.hollowprocesses.HollowProcesses": { + "volatility3.plugins.windows.malware.hollowprocesses.HollowProcesses": { "icon": "VisibilityOff", "description": "Lists hollowed processes.", "category": "Malware", @@ -175,7 +211,7 @@ "category": "Processes", "display": "True" }, - "volatility3.plugins.windows.ldrmodules.LdrModules": { + "volatility3.plugins.windows.malware.ldrmodules.LdrModules": { "icon": "Layers", "description": "Lists the loaded modules in a particular windows memory image.", "category": "Processes", @@ -229,7 +265,7 @@ "category": "Processes", "display": "True" }, - "volatility3.plugins.windows.processghosting.ProcessGhosting": { + "volatility3.plugins.windows.malware.processghosting.ProcessGhosting": { "icon": "Ghost", "description": "Lists processes whose DeletePending bit is set or whose FILE_OBJECT is set to 0.", "category": "Malware", @@ -241,19 +277,19 @@ "category": "Processes", "display": "True" }, - "volatility3.plugins.windows.psxview.PsXView": { + "volatility3.plugins.windows.malware.psxview.PsXView": { "icon": "Visibility", "description": "Lists all processes found via four methods described in 'The Art of Memory Forensics' to help identify hidden processes.", "category": "Malware", "display": "True" }, - "volatility3.plugins.windows.skeleton_key_check.Skeleton_Key_Check": { + "volatility3.plugins.windows.malware.skeleton_key_check.Skeleton_Key_Check": { "icon": "Warning", "description": "Looks for signs of Skeleton Key malware.", "category": "Security", "display": "True" }, - "volatility3.plugins.windows.suspicious_threads.SuspiciousThreads": { + "volatility3.plugins.windows.malware.suspicious_threads.SuspiciousThreads": { "icon": "VisibilityOff", "description": "Lists suspicious userland process threads.", "category": "Malware", @@ -331,7 +367,7 @@ "category": "Registry", "display": "True" }, - "volatility3.plugins.windows.unhooked_system_calls.unhooked_system_calls": { + "volatility3.plugins.windows.malware.unhooked_system_calls.UnhookedSystemCalls": { "icon": "Phishing", "description": "Looks for signs of Skeleton Key malware", "category": "Malware", @@ -355,13 +391,13 @@ "category": "GUI", "display": "True" }, - "volatility3.plugins.windows.direct_system_calls.DirectSystemCalls": { + "volatility3.plugins.windows.malware.direct_system_calls.DirectSystemCalls": { "icon": "CallMade", "description": "Detects the Direct System Call technique used to bypass EDRs.", "category": "Malware", "display": "True" }, - "volatility3.plugins.windows.indirect_system_calls.IndirectSystemCalls": { + "volatility3.plugins.windows.malware.indirect_system_calls.IndirectSystemCalls": { "icon": "CallMissed", "description": "Detects Indirect System Call technique used to evade detection.", "category": "Malware", @@ -418,31 +454,49 @@ "category": "Security", "display": "True" }, - "volatility3.plugins.linux.check_afinfo.Check_afinfo": { + "volatility3.plugins.linux.malware.check_afinfo.Check_afinfo": { "icon": "CheckBox", "description": "Verifies the operation function pointers of network protocols.", "category": "Network", "display": "True" }, - "volatility3.plugins.linux.check_creds.Check_creds": { + "volatility3.plugins.linux.malware.process_spoofing.ProcessSpoofing": { + "icon": "TheaterComedy", + "description": "Detects potential process spoofing via mismatched process metadata.", + "category": "Malware", + "display": "True" + }, + "volatility3.plugins.linux.sockscan.Sockscan": { + "icon": "Lan", + "description": "Scans for network socket objects in memory.", + "category": "Network", + "display": "True" + }, + "volatility3.plugins.linux.vmcoreinfo.VMCoreInfo": { + "icon": "Info", + "description": "Extracts the VMCoreInfo structure from the memory image.", + "category": "Kernel", + "display": "True" + }, + "volatility3.plugins.linux.malware.check_creds.Check_creds": { "icon": "AccountCircle", "description": "Checks if any processes are sharing credential structures.", "category": "Security", "display": "True" }, - "volatility3.plugins.linux.check_idt.Check_idt": { + "volatility3.plugins.linux.malware.check_idt.Check_idt": { "icon": "Compare", "description": "Checks if the IDT has been altered.", "category": "Kernel", "display": "True" }, - "volatility3.plugins.linux.check_modules.Check_modules": { + "volatility3.plugins.linux.malware.check_modules.Check_modules": { "icon": "InsertChart", "description": "Compares module list to sysfs info, if available.", "category": "Kernel", "display": "True" }, - "volatility3.plugins.linux.check_syscall.Check_syscall": { + "volatility3.plugins.linux.malware.check_syscall.Check_syscall": { "icon": "SettingsEthernet", "description": "Check system call table for hooks.", "category": "Kernel", @@ -466,7 +520,7 @@ "category": "Kernel", "display": "True" }, - "volatility3.plugins.linux.keyboard_notifiers.Keyboard_notifiers": { + "volatility3.plugins.linux.malware.keyboard_notifiers.Keyboard_notifiers": { "icon": "Keyboard", "description": "Parses the keyboard notifier call chain.", "category": "Kernel", @@ -490,7 +544,7 @@ "category": "Filesystem", "display": "True" }, - "volatility3.plugins.linux.malfind.Malfind": { + "volatility3.plugins.linux.malware.malfind.Malfind": { "icon": "BugReport", "description": "Lists process memory ranges that potentially contain injected code.", "category": "Malware", @@ -502,7 +556,7 @@ "category": "Filesystem", "display": "True" }, - "volatility3.plugins.linux.netfilter.Netfilter": { + "volatility3.plugins.linux.malware.netfilter.Netfilter": { "icon": "FilterList", "description": "Lists Netfilter hooks.", "category": "Network", @@ -532,7 +586,7 @@ "category": "Network", "display": "True" }, - "volatility3.plugins.linux.tty_check.tty_check": { + "volatility3.plugins.linux.malware.tty_check.Tty_Check": { "icon": "Call", "description": "Checks tty devices for hooks.", "category": "Kernel", @@ -550,7 +604,7 @@ "category": "Kernel", "display": "True" }, - "volatility3.plugins.linux.hidden_modules.Hidden_modules": { + "volatility3.plugins.linux.malware.hidden_modules.Hidden_modules": { "icon": "ViewModule", "description": "Carves memory to find hidden kernel modules", "category": "Malware", @@ -604,7 +658,7 @@ "category": "Processes", "display": "True" }, - "volatility3.plugins.linux.modxview.Modxview": { + "volatility3.plugins.linux.malware.modxview.Modxview": { "icon": "Visibility", "description": "Centralizes module check results to efficiently spot modules presence and taints.", "category": "Kernel", diff --git a/backend/yararules/migrations/0001_initial.py b/backend/yararules/migrations/0001_initial.py new file mode 100644 index 00000000..3c82daa6 --- /dev/null +++ b/backend/yararules/migrations/0001_initial.py @@ -0,0 +1,43 @@ +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("yararulesets", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="YaraRule", + fields=[ + ("id", models.AutoField(primary_key=True, serialize=False)), + ("name", models.CharField(max_length=250)), + ("etag", models.CharField(max_length=256, unique=True)), + ("rule_content", models.TextField()), + ("description", models.TextField(blank=True, null=True)), + ("status", models.IntegerField(default=0)), + ("url", models.TextField(null=True)), + ( + "source", + models.CharField( + choices=[("custom", "manual"), ("github", "github")], + max_length=10, + null=True, + ), + ), + ("is_active", models.BooleanField(default=True)), + ( + "linked_yararuleset", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="yararulesets.yararuleset", + ), + ), + ], + ), + ] diff --git a/backend/yararules/migrations/0002_yararule_content_hash.py b/backend/yararules/migrations/0002_yararule_content_hash.py new file mode 100644 index 00000000..9131a3f4 --- /dev/null +++ b/backend/yararules/migrations/0002_yararule_content_hash.py @@ -0,0 +1,18 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("yararules", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="yararule", + name="content_hash", + field=models.CharField( + blank=True, db_index=True, max_length=64, null=True + ), + ), + ] diff --git a/backend/yararules/migrations/__init__.py b/backend/yararules/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/yararules/models.py b/backend/yararules/models.py index 2767ef1c..47dbfea0 100644 --- a/backend/yararules/models.py +++ b/backend/yararules/models.py @@ -1,6 +1,7 @@ # yararule/models.py from django.db import models from yararulesets.models import YaraRuleSet +from yararules.utils import compute_content_hash RULE_SOURCES = ( ("custom", "manual"), @@ -18,6 +19,8 @@ class YaraRule(models.Model): name = models.CharField(max_length=250) etag = models.CharField(max_length=256, unique=True) rule_content = models.TextField() + # SHA-256 of rule_content, for content-based de-duplication. + content_hash = models.CharField(max_length=64, db_index=True, null=True, blank=True) description = models.TextField(null=True, blank=True) linked_yararuleset = models.ForeignKey(YaraRuleSet, on_delete=models.CASCADE, null=True) status = models.IntegerField(default=0) @@ -25,5 +28,16 @@ class YaraRule(models.Model): source = models.CharField(max_length=10, choices=RULE_SOURCES, null=True) is_active = models.BooleanField(default=True) + def save(self, *args, **kwargs): + # Keep content_hash in sync; add it to update_fields only when content changes. + self.content_hash = compute_content_hash(self.rule_content) + update_fields = kwargs.get("update_fields") + if update_fields is not None: + update_fields = set(update_fields) + if "rule_content" in update_fields: + update_fields.add("content_hash") + kwargs["update_fields"] = update_fields + super().save(*args, **kwargs) + def __str__(self): return str(self.name) \ No newline at end of file diff --git a/backend/yararules/serializers.py b/backend/yararules/serializers.py index b88ea81b..f96296ef 100644 --- a/backend/yararules/serializers.py +++ b/backend/yararules/serializers.py @@ -36,7 +36,33 @@ class YaraRuleSerializer(serializers.ModelSerializer): class Meta: model = YaraRule fields = "__all__" - + + def validate(self, attrs): + """Reject a manual create/update whose content duplicates an existing rule.""" + from yararules.utils import compute_content_hash, backfill_content_hashes + + rule_content = attrs.get("rule_content") + if rule_content is None and self.instance is not None: + # Partial update not touching the content. + return attrs + + backfill_content_hashes() + + content_hash = compute_content_hash(rule_content) + qs = YaraRule.objects.filter(content_hash=content_hash) + if self.instance is not None: + qs = qs.exclude(pk=self.instance.pk) + + existing = qs.first() + if existing is not None: + raise serializers.ValidationError({ + "rule_content": ( + f"A YARA rule with identical content already exists " + f"(name: '{existing.name}', id: {existing.id})." + ) + }) + return attrs + def to_representation(self, instance): """ Override to add linked_yararuleset information safely diff --git a/backend/yararules/utils.py b/backend/yararules/utils.py index 342f100f..2453c9c0 100644 --- a/backend/yararules/utils.py +++ b/backend/yararules/utils.py @@ -6,6 +6,7 @@ from threading import local import logging import os +import hashlib import logging from django.conf import settings @@ -14,6 +15,40 @@ # Thread-local storage for batch operation state _local = local() + +def compute_content_hash(rule_content): + """SHA-256 of the rule content, normalized for line endings and whitespace.""" + if not rule_content: + rule_content = "" + text = rule_content.replace("\r\n", "\n").replace("\r", "\n") + normalized = "\n".join(line.rstrip() for line in text.split("\n")).strip() + return hashlib.sha256(normalized.encode("utf-8")).hexdigest() + + +def backfill_content_hashes(): + """Populate content_hash for legacy rows where it is still NULL (idempotent).""" + # local import: models imports this module + from yararules.models import YaraRule + + missing = YaraRule.objects.filter(content_hash__isnull=True).only("id", "rule_content") + for rule in missing.iterator(): + YaraRule.objects.filter(id=rule.id).update( + content_hash=compute_content_hash(rule.rule_content) + ) + + +def get_existing_content_hashes(): + """Return the set of content hashes of all existing rules (backfilling legacy rows).""" + # local import: models imports this module + from yararules.models import YaraRule + + backfill_content_hashes() + + return set( + YaraRule.objects.exclude(content_hash__isnull=True) + .values_list("content_hash", flat=True) + ) + def create_yara_rule_file(rule): """ Create a physical .yar file for a YARA rule. diff --git a/backend/yararulesets/migrations/0001_initial.py b/backend/yararulesets/migrations/0001_initial.py new file mode 100644 index 00000000..be9b0922 --- /dev/null +++ b/backend/yararulesets/migrations/0001_initial.py @@ -0,0 +1,60 @@ +import django.db.models.deletion +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="YaraRuleSet", + fields=[ + ("id", models.AutoField(primary_key=True, serialize=False)), + ("name", models.CharField(max_length=200, unique=True)), + ("description", models.TextField(blank=True, null=True)), + ("is_default", models.BooleanField(default=False)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("compiled_rules", models.BinaryField(blank=True, null=True)), + ("status", models.IntegerField(default=0)), + ], + options={ + "ordering": ["-created_at"], + }, + ), + migrations.CreateModel( + name="UploadSession", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "upload_id", + models.UUIDField(default=uuid.uuid4, editable=False, unique=True), + ), + ("filename", models.CharField(max_length=255)), + ("description", models.TextField(blank=True, null=True)), + ("source", models.CharField(max_length=255, null=True)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ( + "yararuleset", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="yararulesets.yararuleset", + ), + ), + ], + ), + ] diff --git a/backend/yararulesets/migrations/__init__.py b/backend/yararulesets/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/yararulesets/serializers.py b/backend/yararulesets/serializers.py index d55abb7b..034f63a5 100644 --- a/backend/yararulesets/serializers.py +++ b/backend/yararulesets/serializers.py @@ -2,6 +2,7 @@ from rest_framework import serializers from .models import YaraRuleSet from yararules.models import YaraRule +from core.validators import validate_upload_filename class YaraRuleSetSerializer(serializers.ModelSerializer): @@ -21,7 +22,9 @@ def get_rules(self, obj): class InitiateUploadSerializer(serializers.Serializer): - filename = serializers.CharField(max_length=255) + filename = serializers.CharField( + max_length=255, validators=[validate_upload_filename] + ) description = serializers.CharField(max_length=255,required=False, allow_blank=True, default="") yara_ruleset_id = serializers.IntegerField(required=False, allow_null=True, default=None) source = serializers.CharField(max_length=255) diff --git a/backend/yararulesets/views.py b/backend/yararulesets/views.py index 703059b4..0fb4aff7 100644 --- a/backend/yararulesets/views.py +++ b/backend/yararulesets/views.py @@ -12,7 +12,8 @@ from yararules.models import YaraRule from .serializers import YaraRuleSetSerializer, InitiateUploadSerializer, UploadChunkSerializer, CompleteUploadSerializer from yararules.serializers import YaraRuleSerializer -from yararules.utils import BatchUploadManager +from yararules.utils import BatchUploadManager, compute_content_hash, get_existing_content_hashes +from core.validators import safe_media_path import os import shutil from asgiref.sync import async_to_sync @@ -188,8 +189,10 @@ def post(self, request): except ValueError: return Response({'error': 'Invalid chunk filenames.'}, status=status.HTTP_400_BAD_REQUEST) - final_filename = upload_session.filename - final_file_path = os.path.join(settings.MEDIA_ROOT, 'yara_files', final_filename) + try: + final_file_path = safe_media_path('yara_files', upload_session.filename) + except ValueError: + return Response({'error': 'Invalid filename.'}, status=status.HTTP_400_BAD_REQUEST) os.makedirs(os.path.dirname(final_file_path), exist_ok=True) # Assemble the chunks into the final file @@ -204,16 +207,16 @@ def post(self, request): # Now process the uploaded file with batch optimization try: - created_rules = self._process_uploaded_file_with_batch( + created_rules, skipped_duplicates = self._process_uploaded_file_with_batch( final_file_path, upload_session.yararuleset, upload_session.description, upload_session.source ) - + # Clean up the temporary file os.remove(final_file_path) - + # Delete the upload session upload_session.delete() @@ -230,8 +233,9 @@ def post(self, request): ) return Response({ - 'status': 'upload complete', + 'status': 'upload complete', 'rules_created': len(created_rules), + 'skipped_duplicates': skipped_duplicates, 'rules': YaraRuleSerializer(created_rules, many=True).data }, status=status.HTTP_200_OK) @@ -266,56 +270,68 @@ def _process_uploaded_file_with_batch(self, file_path, yara_ruleset, description batch_manager = BatchUploadManager( ruleset_id=yara_ruleset.id if yara_ruleset else None ) - + created_rules = [] - + skipped_duplicates = 0 + # Skip rules whose content already exists in the DB or earlier in this batch. + seen_hashes = get_existing_content_hashes() + with batch_manager.batch_context(): # Read the file content with open(file_path, 'r', encoding='utf-8', errors='ignore') as f: content = f.read() - + # Check if it's a single rule or multiple rules if self._is_single_rule_file(content): # Single rule file rule = self._create_rule_from_content( - content, - yara_ruleset, - description, - source, - os.path.basename(file_path) + content, + yara_ruleset, + description, + source, + os.path.basename(file_path), + seen_hashes ) if rule: batch_manager.add_created_rule(rule) created_rules.append(rule) logger.debug(f"Created single rule {rule.name}") + else: + skipped_duplicates += 1 else: # Multiple rules in one file - split them individual_rules = self._split_yara_rules(content) - + for i, rule_content in enumerate(individual_rules): if rule_content.strip(): try: rule = self._create_rule_from_content( - rule_content, - yara_ruleset, - description, + rule_content, + yara_ruleset, + description, source, - f"{os.path.basename(file_path)}_rule_{i+1}" + f"{os.path.basename(file_path)}_rule_{i+1}", + seen_hashes ) if rule: batch_manager.add_created_rule(rule) created_rules.append(rule) logger.debug(f"Created rule {rule.name} ({i+1}/{len(individual_rules)})") - + else: + skipped_duplicates += 1 + except Exception as e: logger.error(f"Failed to create rule {i+1}: {e}") continue - + # At this point, we're outside the batch context # The BatchUploadManager automatically triggered ruleset validation - - logger.info(f"Successfully processed {batch_manager.get_created_count()} rules from uploaded file") - return created_rules + + logger.info( + f"Successfully processed {batch_manager.get_created_count()} rules from " + f"uploaded file ({skipped_duplicates} duplicate(s) skipped)" + ) + return created_rules, skipped_duplicates def _is_single_rule_file(self, content): """Check if the file contains a single YARA rule""" @@ -340,11 +356,19 @@ def _split_yara_rules(self, content): return rules - def _create_rule_from_content(self, content, yara_ruleset, description, source, filename): - """Create a single YARA rule from content""" + def _create_rule_from_content(self, content, yara_ruleset, description, source, filename, seen_hashes): + """ + Create a YARA rule from content, or return None if it duplicates an + existing rule. Updates ``seen_hashes`` in place on creation. + """ import re import hashlib - + + content_hash = compute_content_hash(content) + if content_hash in seen_hashes: + logger.info(f"Skipping duplicate rule from {filename} (content hash {content_hash[:12]})") + return None + # Extract rule name from content rule_match = re.search(r'rule\s+(\w+)', content) if rule_match: @@ -352,22 +376,29 @@ def _create_rule_from_content(self, content, yara_ruleset, description, source, else: # Fallback to filename rule_name = os.path.splitext(filename)[0] - + # Generate unique etag etag_content = f"{rule_name}_{content}_{source}" etag = hashlib.md5(etag_content.encode()).hexdigest() - + # Create the rule (this will NOT trigger ruleset validation due to batch context) - rule = YaraRule.objects.create( - name=rule_name, - etag=etag, - rule_content=content, - description=description or f"Uploaded from file: {filename}", - linked_yararuleset=yara_ruleset, - source=source, - is_active=True - ) - + try: + rule = YaraRule.objects.create( + name=rule_name, + etag=etag, + rule_content=content, + description=description or f"Uploaded from file: {filename}", + linked_yararuleset=yara_ruleset, + source=source, + is_active=True + ) + except IntegrityError: + # An identical rule (same etag) already exists — treat as duplicate. + logger.info(f"Skipping duplicate rule from {filename} (etag collision)") + seen_hashes.add(content_hash) + return None + + seen_hashes.add(content_hash) return rule @@ -411,15 +442,16 @@ def post(self, request): download_url = self._convert_to_download_url(github_url) # Process the repository with batch optimization - imported_rules = self._process_github_repo_optimized( - download_url, - yara_ruleset, + imported_rules, skipped_duplicates = self._process_github_repo_optimized( + download_url, + yara_ruleset, description ) - + return Response({ 'success': True, 'imported_count': len(imported_rules), + 'skipped_duplicates': skipped_duplicates, 'rules': YaraRuleSerializer(imported_rules, many=True).data }, status=status.HTTP_200_OK) @@ -445,10 +477,14 @@ def _process_github_repo_optimized(self, download_url, yara_ruleset, description batch_manager = BatchUploadManager( ruleset_id=yara_ruleset.id if yara_ruleset else None ) - + + skipped_duplicates = 0 + # Skip rules whose content already exists in the DB or earlier in this batch. + seen_hashes = get_existing_content_hashes() + with batch_manager.batch_context(): imported_rules = [] - + with tempfile.TemporaryDirectory() as temp_dir: zip_path = os.path.join(temp_dir, 'repo.zip') @@ -477,55 +513,75 @@ def _process_github_repo_optimized(self, download_url, yara_ruleset, description for i, yara_file in enumerate(yara_files): try: rule = self._import_yara_file_optimized( - yara_file, - yara_ruleset, + yara_file, + yara_ruleset, description, - download_url + download_url, + seen_hashes ) if rule: batch_manager.add_created_rule(rule) imported_rules.append(rule) logger.debug(f"Created rule {rule.name} ({i+1}/{len(yara_files)})") - + else: + skipped_duplicates += 1 + except Exception as e: logger.error(f"Failed to create rule from {yara_file}: {e}") continue - + # At this point, we're outside the batch context # The BatchUploadManager automatically triggered ruleset validation - - logger.info(f"Successfully imported {batch_manager.get_created_count()} rules from GitHub") - return imported_rules - - def _import_yara_file_optimized(self, file_path, yara_ruleset, description, source_url): - """Import a single YARA file with optimized etag generation""" + + logger.info( + f"Successfully imported {batch_manager.get_created_count()} rules from GitHub " + f"({skipped_duplicates} duplicate(s) skipped)" + ) + return imported_rules, skipped_duplicates + + def _import_yara_file_optimized(self, file_path, yara_ruleset, description, source_url, seen_hashes): + """ + Import a single YARA file, or return None if it duplicates an existing + rule. Updates ``seen_hashes`` in place on creation. + """ with open(file_path, 'r', encoding='utf-8', errors='ignore') as f: content = f.read() - + + content_hash = compute_content_hash(content) + if content_hash in seen_hashes: + logger.info(f"Skipping duplicate rule from {os.path.basename(file_path)} (content hash {content_hash[:12]})") + return None + # Extract rule name from file or content filename = os.path.basename(file_path) rule_name = os.path.splitext(filename)[0] - + # Try to extract rule name from content rule_match = re.search(r'rule\s+(\w+)', content) if rule_match: rule_name = rule_match.group(1) - + # Generate unique etag etag = self._generate_etag(rule_name, content, source_url) - + # Create the rule (this will NOT trigger ruleset validation due to batch context) - rule = YaraRule.objects.create( - name=rule_name, - etag=etag, - rule_content=content, - description=description or f"Imported from GitHub: {filename}", - linked_yararuleset=yara_ruleset, - source="github", - url=source_url, - is_active=True - ) - + try: + rule = YaraRule.objects.create( + name=rule_name, + etag=etag, + rule_content=content, + description=description or f"Imported from GitHub: {filename}", + linked_yararuleset=yara_ruleset, + source="github", + url=source_url, + is_active=True + ) + except IntegrityError: + logger.info(f"Skipping duplicate rule from {filename} (etag collision)") + seen_hashes.add(content_hash) + return None + + seen_hashes.add(content_hash) return rule def _generate_etag(self, rule_name, rule_content, source_url): diff --git a/docker-compose-prod.yaml b/docker-compose-prod.yaml index 38f10ea2..4277a975 100644 --- a/docker-compose-prod.yaml +++ b/docker-compose-prod.yaml @@ -35,6 +35,7 @@ services: - DJANGO_SECRET=${DJANGO_SECRET} - BROKER_HOST=${BROKER_HOST} - BROKER_PORT=${BROKER_PORT} + - RUN_MIGRATIONS=true command: daphne -b 0.0.0.0 -p 8000 backend.asgi:application ports: - "8000:8000" @@ -64,6 +65,7 @@ services: depends_on: - volweb-redis - volweb-postgresdb + - volweb-backend restart: always volumes: - media:/home/app/web/media @@ -89,6 +91,7 @@ services: depends_on: - volweb-redis - volweb-postgresdb + - volweb-backend restart: always volumes: - media:/home/app/web/media diff --git a/docker-compose.yaml b/docker-compose.yaml index e34ce306..6f41d096 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -23,7 +23,9 @@ services: - "6379:6379" volweb-backend: - image: "forensicxlab/volweb-backend:latest" + build: + context: ./backend + dockerfile: Dockerfile environment: - DATABASE=${DATABASE} - DATABASE_HOST=${DATABASE_HOST} @@ -35,6 +37,7 @@ services: - DJANGO_SECRET=${DJANGO_SECRET} - BROKER_HOST=${BROKER_HOST} - BROKER_PORT=${BROKER_PORT} + - RUN_MIGRATIONS=true command: daphne -b 0.0.0.0 -p 8000 backend.asgi:application ports: - "8000:8000" @@ -47,7 +50,9 @@ services: - media:/home/app/web/media volweb-workers: - image: "forensicxlab/volweb-backend:latest" + build: + context: ./backend + dockerfile: Dockerfile container_name: volweb-workers environment: - DATABASE=${DATABASE} @@ -64,6 +69,7 @@ services: depends_on: - volweb-redis - volweb-postgresdb + - volweb-backend restart: always volumes: - media:/home/app/web/media @@ -72,7 +78,9 @@ services: replicas: 1 volweb-workers-yarascan: - image: "forensicxlab/volweb-backend:latest" + build: + context: ./backend + dockerfile: Dockerfile container_name: volweb-workers-yarascan environment: - DATABASE=${DATABASE} @@ -89,6 +97,7 @@ services: depends_on: - volweb-redis - volweb-postgresdb + - volweb-backend restart: always volumes: - media:/home/app/web/media @@ -99,7 +108,9 @@ services: memory: "${YARASCAN_MEM_LIMIT:-2g}" volweb-frontend: - image: "forensicxlab/volweb-frontend:latest" + build: + context: ./frontend + dockerfile: Dockerfile container_name: volweb-frontend ports: - "3000:3000" @@ -114,4 +125,4 @@ volumes: postgres-data: redis-data: staticfiles: - media: + media: \ No newline at end of file diff --git a/frontend/src/components/Dialogs/YaraRuleCreationDialog.tsx b/frontend/src/components/Dialogs/YaraRuleCreationDialog.tsx index a5b2a5ac..4e621241 100644 --- a/frontend/src/components/Dialogs/YaraRuleCreationDialog.tsx +++ b/frontend/src/components/Dialogs/YaraRuleCreationDialog.tsx @@ -25,13 +25,18 @@ import { CloudDownload as CloudDownloadIcon, } from "@mui/icons-material"; import axiosInstance from "../../utils/axiosInstance"; -import { YaraRule, YaraRuleSet } from "../../types"; +import { YaraRuleSet } from "../../types"; + +export interface ImportSummary { + created: number; + skipped: number; +} interface YaraRuleCreationDialogProps { open: boolean; onClose: () => void; - onCreateSuccess: (newYaraRule: YaraRule) => void; - onImportSuccess: () => void; + onCreateSuccess: (summary: ImportSummary) => void; + onImportSuccess: (summary: ImportSummary) => void; onCreateFailed?: (error: unknown) => void; onImportFailed?: (error: string) => void; yara_ruleset?: YaraRuleSet; @@ -172,7 +177,10 @@ const YaraRuleCreationDialog: React.FC = ({ upload_id: uploadId, }); - onCreateSuccess(completeRes.data); + onCreateSuccess({ + created: completeRes.data?.rules_created ?? 0, + skipped: completeRes.data?.skipped_duplicates ?? 0, + }); handleClose(); } catch (err) { console.error("Upload failed", err); @@ -224,7 +232,10 @@ const YaraRuleCreationDialog: React.FC = ({ }); if (response.data.success) { - onImportSuccess(); + onImportSuccess({ + created: response.data?.imported_count ?? 0, + skipped: response.data?.skipped_duplicates ?? 0, + }); handleClose(); } else { setGithubError(response.data.error || "Import failed."); diff --git a/frontend/src/components/Dialogs/YaraRuleEditDialog.tsx b/frontend/src/components/Dialogs/YaraRuleEditDialog.tsx index 1b93e93a..0e89f8f8 100644 --- a/frontend/src/components/Dialogs/YaraRuleEditDialog.tsx +++ b/frontend/src/components/Dialogs/YaraRuleEditDialog.tsx @@ -171,10 +171,18 @@ const YaraRuleEditDialog: React.FC = ({ handleClose(); } catch (error: any) { - display_message( - "error", - `Failed to save YARA rule: ${error.response?.data?.error || error.message}` - ); + const data = error.response?.data; + let detail: string = error.message; + if (typeof data === "string") { + detail = data; + } else if (data?.error) { + detail = data.error; + } else if (data && typeof data === "object") { + // Surface DRF field errors (e.g. duplicate content on rule_content) + const first = data.rule_content ?? Object.values(data)[0]; + detail = Array.isArray(first) ? first[0] : String(first); + } + display_message("error", `Failed to save YARA rule: ${detail}`); } finally { setSaving(false); } diff --git a/frontend/src/components/Investigate/Loot.tsx b/frontend/src/components/Investigate/Loot.tsx index 97fca6da..c71c6c67 100644 --- a/frontend/src/components/Investigate/Loot.tsx +++ b/frontend/src/components/Investigate/Loot.tsx @@ -1 +1,193 @@ -// This should display the list of dumped artefacts (registry, process, files, ...) +import React, { useCallback, useEffect, useState } from "react"; +import { + Box, + Button, + CircularProgress, + IconButton, + Tooltip, + Typography, +} from "@mui/material"; +import { DataGrid, GridColDef, GridToolbar } from "@mui/x-data-grid"; +import { Download, Delete } from "@mui/icons-material"; +import { useParams } from "react-router-dom"; +import axiosInstance from "../../utils/axiosInstance"; +import { downloadFile } from "../../utils/downloadFile"; +import { useSnackbar } from "../SnackbarProvider"; + +interface LootArtefact { + name: string; + size: number; + modified: string; + url: string; +} + +const formatBytes = (bytes: number): string => { + if (!bytes) { + return "0 B"; + } + const units = ["B", "KB", "MB", "GB", "TB"]; + const i = Math.min( + Math.floor(Math.log(bytes) / Math.log(1024)), + units.length - 1, + ); + return `${(bytes / Math.pow(1024, i)).toFixed(i === 0 ? 0 : 1)} ${units[i]}`; +}; + +const Loot: React.FC = () => { + const { id: evidenceId } = useParams<{ id: string }>(); + const { display_message } = useSnackbar(); + + const [artefacts, setArtefacts] = useState([]); + const [totalSize, setTotalSize] = useState(0); + const [loading, setLoading] = useState(true); + const [busy, setBusy] = useState<{ [name: string]: boolean }>({}); + + const fetchLoot = useCallback(async () => { + try { + const response = await axiosInstance.get( + `/api/evidence/${evidenceId}/loot/`, + ); + setArtefacts(response.data.artefacts || []); + setTotalSize(response.data.total_size || 0); + } catch (error) { + console.error("Error fetching loot", error); + display_message("error", `Error fetching loot: ${error}`); + } finally { + setLoading(false); + } + }, [evidenceId, display_message]); + + useEffect(() => { + fetchLoot(); + }, [fetchLoot]); + + const handleDelete = async (name: string) => { + setBusy((prev) => ({ ...prev, [name]: true })); + try { + await axiosInstance.delete(`/api/evidence/${evidenceId}/loot/`, { + data: { name }, + }); + display_message("success", `${name} deleted.`); + await fetchLoot(); + } catch (error) { + console.error("Error deleting artefact", error); + display_message("error", `Error deleting ${name}: ${error}`); + } finally { + setBusy((prev) => ({ ...prev, [name]: false })); + } + }; + + const columns: GridColDef[] = [ + { field: "name", headerName: "Name", flex: 2 }, + { + field: "size", + headerName: "Size", + flex: 1, + renderCell: (params) => formatBytes(params.value as number), + }, + { + field: "modified", + headerName: "Extracted", + flex: 1, + renderCell: (params) => + params.value ? new Date(params.value as string).toLocaleString() : "", + }, + { + field: "actions", + headerName: "Actions", + sortable: false, + filterable: false, + flex: 1, + renderCell: (params) => { + const row = params.row as LootArtefact; + const isBusy = busy[row.name]; + return ( + + + + downloadFile(row.url, row.name)} + > + + + + + + + handleDelete(row.name)} + > + {isBusy ? ( + + ) : ( + + )} + + + + + ); + }, + }, + ]; + + if (loading) { + return ( + + + + ); + } + + return ( + + + + {artefacts.length} artefact{artefacts.length === 1 ? "" : "s"} on disk + {" · "} + {formatBytes(totalSize)} + + + + {artefacts.length === 0 ? ( + + No artefacts have been dumped for this evidence yet. + + ) : ( + row.name} + pagination + showToolbar + getRowHeight={() => "auto"} + /> + )} + + ); +}; + +export default Loot; diff --git a/frontend/src/components/Investigate/PluginDashboard.tsx b/frontend/src/components/Investigate/PluginDashboard.tsx index 24311e9d..ecbb9318 100644 --- a/frontend/src/components/Investigate/PluginDashboard.tsx +++ b/frontend/src/components/Investigate/PluginDashboard.tsx @@ -14,10 +14,11 @@ import { Tooltip, } from "@mui/material"; import * as Icons from "@mui/icons-material"; -import { HomeRepairService } from "@mui/icons-material"; +import { HomeRepairService, Inventory2 } from "@mui/icons-material"; import { useParams } from "react-router-dom"; import axiosInstance from "../../utils/axiosInstance"; import PluginDataGrid from "./PluginDataGrid"; +import Loot from "./Loot"; import { Plugin, Evidence } from "../../types"; import WindowsFileScanButton from "./Windows/Buttons/FileScanButton"; import LinuxFileScanButton from "./Linux/Buttons/FileScanButton"; @@ -31,6 +32,7 @@ const PluginDashboard: React.FC = ({ evidence }) => { const [plugins, setPlugins] = useState(null); const [open, setOpen] = useState(false); const [currentPlugin, setCurrentPlugin] = useState(null); + const [lootOpen, setLootOpen] = useState(false); const [loading, setLoading] = useState(true); // State to track loading const handleClickOpen = (plugin: Plugin) => { @@ -103,18 +105,35 @@ const PluginDashboard: React.FC = ({ evidence }) => { ) : ( - - - Tools - + + + Tools + + + {Object.keys(groupedPlugins).map((category) => ( @@ -197,6 +216,25 @@ const PluginDashboard: React.FC = ({ evidence }) => { /> + setLootOpen(false)} + sx={{ + "& .MuiBackdrop-root": { + backgroundColor: "transparent", + }, + }} + > + + + Loot + + {lootOpen && } + ); }; diff --git a/frontend/src/components/Investigate/PluginSelector.tsx b/frontend/src/components/Investigate/PluginSelector.tsx index 8af51453..8d9f413c 100644 --- a/frontend/src/components/Investigate/PluginSelector.tsx +++ b/frontend/src/components/Investigate/PluginSelector.tsx @@ -95,6 +95,15 @@ const PluginSelector: React.FC = ({ const [pluginTimeout, setPluginTimeout] = useState("600"); const [searchQuery, setSearchQuery] = useState(""); + // Linux kernel-symbol (ISF) resolution state — gates extraction. + const isLinux = evidenceOs === "linux"; + const [isfStatus, setIsfStatus] = useState(isLinux ? "pending" : "ready"); + const [isfBanner, setIsfBanner] = useState(null); + const [isfMessage, setIsfMessage] = useState(null); + const [isfGuidance, setIsfGuidance] = useState(null); + const [isfRetrying, setIsfRetrying] = useState(false); + const isfReady = !isLinux || isfStatus === "ready"; + const requiredPlugins = REQUIRED_PLUGINS[evidenceOs] || []; const fetchAvailablePlugins = useCallback(async () => { @@ -139,6 +148,36 @@ const PluginSelector: React.FC = ({ fetchAvailablePlugins(); }, [fetchAvailablePlugins]); + const fetchIsfStatus = useCallback(async () => { + if (!isLinux) return; + try { + const res = await axiosInstance.get(`/api/evidence/${evidenceId}/isf/`); + setIsfStatus(res.data.status); + setIsfBanner(res.data.banner ?? null); + setIsfMessage(res.data.message ?? null); + setIsfGuidance(res.data.guidance ?? null); + } catch { + /* keep pending; WS will update */ + } + }, [evidenceId, isLinux]); + + useEffect(() => { + fetchIsfStatus(); + }, [fetchIsfStatus]); + + const handleRetryIsf = async () => { + try { + setIsfRetrying(true); + await axiosInstance.post(`/api/evidence/${evidenceId}/isf/`); + setIsfStatus("detecting"); + setIsfGuidance(null); + display_message("info", "Kernel symbol resolution restarted"); + } catch (error) { + setIsfRetrying(false); + display_message("error", `Failed to restart ISF resolution: ${error}`); + } + }; + // WebSocket for extraction progress useEffect(() => { const protocol = window.location.protocol === "https:" ? "wss" : "ws"; @@ -160,6 +199,21 @@ const PluginSelector: React.FC = ({ } } } + if (message.name === "isf") { + setIsfStatus(message.status); + if (message.banner !== undefined) setIsfBanner(message.banner); + if (message.message !== undefined) setIsfMessage(message.message); + if (message.guidance !== undefined) setIsfGuidance(message.guidance); + if (message.status === "ready") { + setIsfRetrying(false); + display_message( + "success", + "Kernel symbols (ISF) retrieved successfully and added to your Symbols database." + ); + } else if (message.status.startsWith("failed")) { + setIsfRetrying(false); + } + } }; return () => { @@ -320,6 +374,68 @@ const PluginSelector: React.FC = ({ /> + {/* Linux kernel-symbol (ISF) status — extraction is gated until ready */} + {isLinux && isfStatus !== "ready" && ( + } + sx={{ mb: 2 }} + action={ + isfStatus.startsWith("failed") ? ( + + ) : undefined + } + > + + Kernel symbols (ISF):{" "} + {( + { + pending: "waiting…", + detecting: "detecting kernel banner…", + resolving: "looking up matching ISF…", + verifying: "verifying ISF against the image…", + failed_banner: "banner not found", + failed_isf: "no matching ISF", + } as Record + )[isfStatus] || isfStatus} + + {isfMessage && {isfMessage}} + {isfBanner && ( + + {isfBanner} + + )} + {isfGuidance && isfStatus.startsWith("failed") && ( + + + Build it manually, then upload it in the Symbols page: + + {isfGuidance.package && ( + Package: {isfGuidance.package} + )} + {isfGuidance.where && ( + + Where: {isfGuidance.where} + + )} +
      + {(isfGuidance.steps || []).map((step: string, i: number) => ( +
    1. + {step} +
    2. + ))} +
    +
    + )} +
    + )} + {/* Global controls */} @@ -505,16 +621,22 @@ const PluginSelector: React.FC = ({ {/* Action bar */} - + + + + diff --git a/frontend/src/components/Investigate/YaraScan.tsx b/frontend/src/components/Investigate/YaraScan.tsx index 35d0a470..f5b01b86 100644 --- a/frontend/src/components/Investigate/YaraScan.tsx +++ b/frontend/src/components/Investigate/YaraScan.tsx @@ -11,6 +11,10 @@ import { ListItemText, Checkbox, FormControlLabel, + FormControl, + FormLabel, + Radio, + RadioGroup, Divider, Alert, Chip, @@ -35,6 +39,7 @@ import { useSnackbar } from "../SnackbarProvider"; interface YaraScanProps { evidenceId: string; + evidenceOs?: string; } interface ScanHistory { @@ -48,7 +53,14 @@ interface ScanHistory { count?: number; } -const YaraScan: React.FC = ({ evidenceId }) => { +const YaraScan: React.FC = ({ evidenceId, evidenceOs }) => { + // Per-process user-space scan uses VadYaraScan on Windows (VADs) and + // VmaYaraScan on Linux (VMAs) — surface the right plugin/region per OS. + const isLinuxScan = evidenceOs === "linux"; + const vadPluginId = isLinuxScan + ? "linux.vmayarascan.VmaYaraScan" + : "windows.vadyarascan.VadYaraScan"; + const vadRegion = isLinuxScan ? "VMA" : "VAD"; const { display_message } = useSnackbar(); const ws = useRef(null); const [rulesets, setRulesets] = useState([]); @@ -71,6 +83,7 @@ const YaraScan: React.FC = ({ evidenceId }) => { const [scanResultsPagination, setScanResultsPagination] = useState({ page: 0, pageSize: 100 }); const [scanResultsLoading, setScanResultsLoading] = useState(false); const [exportLoading, setExportLoading] = useState(false); + const [scanScope, setScanScope] = useState<"vad" | "kernel">("vad"); const fetchScanResults = useCallback((page: number, pageSize: number) => { setScanResultsLoading(true); @@ -286,6 +299,14 @@ const YaraScan: React.FC = ({ evidenceId }) => { } else if (message.status === "stopped") { setProcessing(false); display_message("info", "YARA scan was stopped"); + } else if (message.status === "error") { + setProcessing(false); + display_message( + "error", + message.error + ? `YaraScan failed: ${message.error}` + : "YaraScan failed" + ); } } }; @@ -349,6 +370,7 @@ const YaraScan: React.FC = ({ evidenceId }) => { id: evidenceId, rulesets: selectedRulesets, rules: effectiveRules, + scan_scope: scanScope, }); display_message("info", "YaraScan task started"); } catch (error) { @@ -562,7 +584,63 @@ const YaraScan: React.FC = ({ evidenceId }) => { /> )} - + + + + Scan scope + + setScanScope(e.target.value as "vad" | "kernel")} + > + } + label={ + + + Process memory ({vadRegion}) + + + {vadPluginId} + + + Recommended for malware. Scans every process {vadRegion}. + + + } + sx={{ mr: 4, alignItems: "flex-start" }} + /> + } + label={ + + + Kernel layer + + + yarascan.YaraScan + + + For rootkits and kernel-mode threats. Faster, narrower coverage. + + + } + sx={{ alignItems: "flex-start" }} + /> + + + diff --git a/frontend/src/components/Lists/PluginList.tsx b/frontend/src/components/Lists/PluginList.tsx index ebaabf23..2605e55c 100644 --- a/frontend/src/components/Lists/PluginList.tsx +++ b/frontend/src/components/Lists/PluginList.tsx @@ -48,7 +48,19 @@ const PluginList: React.FC = ({ evidenceId }) => { {getIcon(plugin.icon)} + {plugin.name.replace(/^volatility3\.plugins\./, "")} + + } secondary={ { setOpenCreationDialog(false); }} - onCreateSuccess={(rule) => { - display_message("success", "YARA rule created successfully"); - handleUpdateSuccess(rule); + onCreateSuccess={({ created, skipped }) => { + display_message( + "success", + `Upload complete: ${created} rule(s) created` + + (skipped ? `, ${skipped} duplicate(s) skipped` : "") + ); + handleUpdateSuccess({} as YaraRule); }} - onImportSuccess={() => { - display_message("success", "GitHub import completed"); + onImportSuccess={({ created, skipped }) => { + display_message( + "success", + `GitHub import completed: ${created} rule(s) imported` + + (skipped ? `, ${skipped} duplicate(s) skipped` : "") + ); handleUpdateSuccess({} as YaraRule); }} yara_ruleset={yararuleset} diff --git a/frontend/src/pages/evidences/EvidenceDetails.tsx b/frontend/src/pages/evidences/EvidenceDetails.tsx index 2a119969..8322af80 100644 --- a/frontend/src/pages/evidences/EvidenceDetails.tsx +++ b/frontend/src/pages/evidences/EvidenceDetails.tsx @@ -86,6 +86,7 @@ const EvidenceDetail: React.FC = () => { return ( { @@ -195,7 +196,7 @@ const EvidenceDetail: React.FC = () => { - {id && } + {id && }