Run extraction modules in parallel

README.md

@@ -82,6 +82,11 @@ Module-running `check-*` commands can load custom Python modules with
 [development documentation](https://docs.mvt.re/en/latest/development/) for
 details.
 
+Extraction commands run independent modules concurrently with four workers by
+default. Use `--jobs 1` for sequential execution or another positive value to
+set the worker limit. Dependencies retain their declared ordering, and parallel
+console logs are printed as grouped module blocks.
+
 
 ## License
 

docs/development.md

@@ -39,6 +39,33 @@ Selecting a single module also runs its transitive dependencies. If a dependency
 is unavailable or the dependency graph contains a cycle, the command logs a
 warning and does not run any modules.
 
+## Parallel module execution
+
+Extraction modules run concurrently, using four worker threads by default. Use
+`--jobs INTEGER` to change the worker limit, or `--jobs 1` for sequential
+execution and live per-line logging:
+
+```bash
+mvt-ios check-backup --jobs 8 --output ./out ./backup
+```
+
+The scheduler starts a module only after all of its declared dependencies have
+completed. Results, alerts, and timeline entries are still aggregated in stable
+topological order. During parallel execution, console records from each module
+are buffered and printed together as a labeled block when that module finishes;
+`command.log` is written immediately and retains the complete log stream.
+
+Parallel-safe modules must log through `self.log`, must not write directly to
+stdout or stderr, and must not mutate shared global state. A module that uses a
+non-thread-safe resource or direct terminal output can opt out:
+
+```python
+class TerminalModule(MVTModule):
+    parallel_safe = False
+```
+
+Such modules run synchronously and exclusively after active workers finish.
+
 ## Custom modules
 
 Module-running `check-*` commands can load custom modules from Python files that
@@ -82,6 +109,9 @@ class ExampleCustomModule(MVTModule):
         return None
 ```
 
+Custom modules are considered parallel-safe by default. Follow the parallel
+module requirements above or set `parallel_safe = False`.
+
 Use `supported_commands` to restrict a module to specific platform/command
 pairs. Missing or empty `supported_commands` means the module is available to
 all commands, which keeps older modules compatible. Supported pairs are:
@@ -120,7 +150,9 @@ class DependentCustomModule(MVTModule):
 Some MVT modules extract and process significant amounts of data during the analysis process or while checking results against known indicators. Care must be
 take to avoid inefficient code paths as we add new modules.
 
-MVT modules can be profiled with Python built-in `cProfile` by setting the `MVT_PROFILE` environment variable.
+MVT modules can be profiled with Python built-in `cProfile` by setting the
+`MVT_PROFILE` environment variable. Profiling forces sequential execution even
+when `--jobs` is greater than one.
 
 ```bash
 MVT_PROFILE=1 dev/mvt-ios check-backup test_backup

src/mvt/android/cli.py

@@ -29,6 +29,7 @@
     HELP_MSG_DISABLE_UPDATE_CHECK,
     HELP_MSG_HASHES,
     HELP_MSG_IOC,
+    HELP_MSG_JOBS,
     HELP_MSG_LIST_MODULES,
     HELP_MSG_LOAD_MODULE,
     HELP_MSG_MODULE,
@@ -180,6 +181,9 @@ def check_adb(ctx):
     help=HELP_MSG_LOAD_MODULE,
 )
 @click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE)
+@click.option(
+    "--jobs", type=click.IntRange(min=1), default=4, show_default=True, help=HELP_MSG_JOBS
+)
 @click.argument("BUGREPORT_PATH", type=click.Path(exists=True))
 @click.pass_context
 def check_bugreport(
@@ -190,6 +194,7 @@ def check_bugreport(
     module,
     load_module,
     verbose,
+    jobs,
     bugreport_path,
 ):
     set_verbose_logging(verbose)
@@ -204,6 +209,7 @@ def check_bugreport(
         disable_version_check=_get_disable_flags(ctx)[0],
         disable_indicator_check=_get_disable_flags(ctx)[1],
         custom_modules=custom_modules,
+        jobs=jobs,
     )
 
     if list_modules:
@@ -245,6 +251,9 @@ def check_bugreport(
 @click.option("--non-interactive", "-n", is_flag=True, help=HELP_MSG_NONINTERACTIVE)
 @click.option("--backup-password", "-p", help=HELP_MSG_ANDROID_BACKUP_PASSWORD)
 @click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE)
+@click.option(
+    "--jobs", type=click.IntRange(min=1), default=4, show_default=True, help=HELP_MSG_JOBS
+)
 @click.argument("BACKUP_PATH", type=click.Path(exists=True))
 @click.pass_context
 def check_backup(
@@ -256,6 +265,7 @@ def check_backup(
     non_interactive,
     backup_password,
     verbose,
+    jobs,
     backup_path,
 ):
     set_verbose_logging(verbose)
@@ -274,6 +284,7 @@ def check_backup(
         disable_version_check=_get_disable_flags(ctx)[0],
         disable_indicator_check=_get_disable_flags(ctx)[1],
         custom_modules=custom_modules,
+        jobs=jobs,
     )
 
     if list_modules:
@@ -319,6 +330,9 @@ def check_backup(
 @click.option("--non-interactive", "-n", is_flag=True, help=HELP_MSG_NONINTERACTIVE)
 @click.option("--backup-password", "-p", help=HELP_MSG_ANDROID_BACKUP_PASSWORD)
 @click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE)
+@click.option(
+    "--jobs", type=click.IntRange(min=1), default=4, show_default=True, help=HELP_MSG_JOBS
+)
 @click.argument("ANDROIDQF_PATH", type=click.Path(exists=True))
 @click.pass_context
 def check_androidqf(
@@ -334,6 +348,7 @@ def check_androidqf(
     non_interactive,
     backup_password,
     verbose,
+    jobs,
     androidqf_path,
 ):
     set_verbose_logging(verbose)
@@ -354,6 +369,7 @@ def check_androidqf(
         disable_version_check=_get_disable_flags(ctx)[0],
         disable_indicator_check=_get_disable_flags(ctx)[1],
         custom_modules=custom_modules,
+        jobs=jobs,
     )
 
     if list_modules:
@@ -405,6 +421,9 @@ def check_androidqf(
     ),
 )
 @click.option("--verbose", "-v", is_flag=True, help=HELP_MSG_VERBOSE)
+@click.option(
+    "--jobs", type=click.IntRange(min=1), default=4, show_default=True, help=HELP_MSG_JOBS
+)
 @click.argument("LOGS_PATH", type=click.Path(exists=True))
 @click.pass_context
 def check_intrusion_logs(
@@ -416,6 +435,7 @@ def check_intrusion_logs(
     load_module,
     timezone,
     verbose,
+    jobs,
     logs_path,
 ):
     set_verbose_logging(verbose)
@@ -434,6 +454,7 @@ def check_intrusion_logs(
         disable_version_check=_get_disable_flags(ctx)[0],
         disable_indicator_check=_get_disable_flags(ctx)[1],
         custom_modules=custom_modules,
+        jobs=jobs,
     )
 
     if list_modules:

src/mvt/android/cmd_check_androidqf.py

@@ -52,6 +52,7 @@ def __init__(
         disable_version_check: bool = False,
         disable_indicator_check: bool = False,
         custom_modules: Optional[list[type[MVTModule]]] = None,
+        jobs: int = 4,
     ) -> None:
         super().__init__(
             target_path=target_path,
@@ -67,6 +68,7 @@ def __init__(
             disable_version_check=disable_version_check,
             disable_indicator_check=disable_indicator_check,
             custom_modules=custom_modules,
+            jobs=jobs,
         )
 
         self.platform = "android"
@@ -215,6 +217,7 @@ def run_bugreport_cmd(self) -> bool:
                 hashes=self.hashes,
                 sub_command=True,
                 custom_modules=self.custom_modules,
+                jobs=self.jobs,
             )
             cmd.from_zip(bugreport)
             cmd.run()
@@ -245,6 +248,7 @@ def run_backup_cmd(self) -> bool:
             hashes=self.hashes,
             sub_command=True,
             custom_modules=self.custom_modules,
+            jobs=self.jobs,
         )
         try:
             cmd.from_ab(backup)
@@ -325,6 +329,7 @@ def run_intrusion_logs_cmd(self) -> bool:
                 hashes=self.hashes,
                 sub_command=True,
                 custom_modules=self.custom_modules,
+                jobs=self.jobs,
             )
             cmd.run()
 

src/mvt/android/cmd_check_backup.py

@@ -47,6 +47,7 @@ def __init__(
         disable_version_check: bool = False,
         disable_indicator_check: bool = False,
         custom_modules: Optional[list[type[MVTModule]]] = None,
+        jobs: int = 4,
     ) -> None:
         super().__init__(
             target_path=target_path,
@@ -62,6 +63,7 @@ def __init__(
             disable_version_check=disable_version_check,
             disable_indicator_check=disable_indicator_check,
             custom_modules=custom_modules,
+            jobs=jobs,
         )
 
         self.platform = "android"

src/mvt/android/cmd_check_bugreport.py

@@ -34,6 +34,7 @@ def __init__(
         disable_version_check: bool = False,
         disable_indicator_check: bool = False,
         custom_modules: Optional[list[type[MVTModule]]] = None,
+        jobs: int = 4,
     ) -> None:
         super().__init__(
             target_path=target_path,
@@ -49,6 +50,7 @@ def __init__(
             disable_version_check=disable_version_check,
             disable_indicator_check=disable_indicator_check,
             custom_modules=custom_modules,
+            jobs=jobs,
         )
 
         self.platform = "android"

src/mvt/android/cmd_check_intrusion_logs.py

@@ -37,6 +37,7 @@ def __init__(
         disable_version_check: bool = False,
         disable_indicator_check: bool = False,
         custom_modules: Optional[list[type[MVTModule]]] = None,
+        jobs: int = 4,
     ) -> None:
         super().__init__(
             target_path=target_path,
@@ -52,6 +53,7 @@ def __init__(
             disable_version_check=disable_version_check,
             disable_indicator_check=disable_indicator_check,
             custom_modules=custom_modules,
+            jobs=jobs,
         )
 
         self.platform = "android"

src/mvt/android/modules/androidqf/aqf_packages.py

@@ -8,8 +8,6 @@
 import time
 from typing import Optional
 
-from rich.progress import track
-
 from mvt.android.utils import (
     BROWSER_INSTALLERS,
     PLAY_STORE_INSTALLERS,
@@ -153,10 +151,9 @@ def check_virustotal(self, delay: int = 0) -> None:
         if total_hashes == 0:
             return
 
-        progress_desc = f"Looking up {total_hashes} package files on VirusTotal..."
-        for index, file_hash in enumerate(
-            track(files_by_hash, description=progress_desc)
-        ):
+        self.log.info("Looking up %d package files on VirusTotal...", total_hashes)
+        for index, file_hash in enumerate(files_by_hash):
+            self.log.debug("VirusTotal lookup %d/%d", index + 1, total_hashes)
             try:
                 results = virustotal_lookup(file_hash)
             except VTNoKey as exc:

src/mvt/android/modules/androidqf/base.py

@@ -81,12 +81,13 @@ def _get_device_timezone(self):
         return None
 
     def _get_file_content(self, file_path):
-        if self.archive:
-            handle = self.archive.open(file_path)
-        else:
-            handle = open(os.path.join(self.parent_path, file_path), "rb")
+        with self.resource_lock:
+            if self.archive:
+                handle = self.archive.open(file_path)
+            else:
+                handle = open(os.path.join(self.parent_path, file_path), "rb")
 
-        data = handle.read()
-        handle.close()
+            data = handle.read()
+            handle.close()
 
-        return data
+            return data

src/mvt/android/modules/backup/base.py

@@ -55,21 +55,22 @@ def _get_files_by_pattern(self, pattern: str) -> list:
         return fnmatch.filter(self.files, pattern)
 
     def _get_file_content(self, file_path: str) -> bytes:
-        handle = None
-        if self.tar:
-            try:
-                member = self.tar.getmember(file_path)
-                handle = self.tar.extractfile(member)
-                if not handle:
-                    raise ValueError(f"Could not extract file: {file_path}")
-            except KeyError:
-                raise FileNotFoundError(f"File not found in tar: {file_path}")
-        elif self.backup_path:
-            handle = open(os.path.join(self.backup_path, file_path), "rb")
-        else:
-            raise ValueError("No backup path or tar file provided")
+        with self.resource_lock:
+            handle = None
+            if self.tar:
+                try:
+                    member = self.tar.getmember(file_path)
+                    handle = self.tar.extractfile(member)
+                    if not handle:
+                        raise ValueError(f"Could not extract file: {file_path}")
+                except KeyError:
+                    raise FileNotFoundError(f"File not found in tar: {file_path}")
+            elif self.backup_path:
+                handle = open(os.path.join(self.backup_path, file_path), "rb")
+            else:
+                raise ValueError("No backup path or tar file provided")
 
-        data = handle.read()
-        handle.close()
+            data = handle.read()
+            handle.close()
 
-        return data
+            return data