From 72c3a6712cd08abc37ef05e706470ca02a2aba2e Mon Sep 17 00:00:00 2001 From: Raffaello Date: Fri, 26 Jun 2026 17:59:55 +0800 Subject: [PATCH 01/17] feat: add multi-instance scheduler --- README.md | 15 + api/main.py | 24 +- api/routers/__init__.py | 3 +- api/routers/scheduler.py | 131 ++++++ api/scheduler/__init__.py | 5 + api/scheduler/manager.py | 405 ++++++++++++++++++ api/scheduler/schemas.py | 146 +++++++ api/scheduler/store.py | 373 ++++++++++++++++ api/schemas/crawler.py | 1 + cmd_arg/arg.py | 45 +- config/base_config.py | 8 +- docs/.vitepress/config.mjs | 1 + docs/index.md | 11 + ...77\347\224\250\346\214\207\345\215\227.md" | 179 ++++++++ ...43\347\240\201\347\273\223\346\236\204.md" | 9 +- ...66\346\236\204\346\226\207\346\241\243.md" | 47 ++ media_platform/bilibili/core.py | 6 +- media_platform/douyin/core.py | 6 +- media_platform/kuaishou/core.py | 8 +- media_platform/tieba/core.py | 8 +- media_platform/weibo/core.py | 6 +- media_platform/xhs/core.py | 6 +- media_platform/zhihu/core.py | 8 +- scheduler_webui/app.js | 227 ++++++++++ scheduler_webui/index.html | 164 +++++++ scheduler_webui/styles.css | 281 ++++++++++++ tests/test_cmd_arg_scheduler.py | 44 ++ tests/test_profile.py | 20 + tests/test_scheduler_api.py | 49 +++ tests/test_scheduler_manager.py | 52 +++ tests/test_scheduler_store.py | 51 +++ tools/cdp_browser.py | 7 +- tools/profile.py | 19 + 33 files changed, 2335 insertions(+), 30 deletions(-) create mode 100644 api/routers/scheduler.py create mode 100644 api/scheduler/__init__.py create mode 100644 api/scheduler/manager.py create mode 100644 api/scheduler/schemas.py create mode 100644 api/scheduler/store.py create mode 100644 "docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" create mode 100644 scheduler_webui/app.js create mode 100644 scheduler_webui/index.html create mode 100644 scheduler_webui/styles.css create mode 100644 tests/test_cmd_arg_scheduler.py create mode 100644 tests/test_profile.py create mode 100644 tests/test_scheduler_api.py create mode 100644 tests/test_scheduler_manager.py create mode 100644 tests/test_scheduler_store.py create mode 100644 tools/profile.py diff --git a/README.md b/README.md index 0c11c4553..ac1ee4dd0 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,9 @@ uv run main.py --platform xhs --lt qrcode --type search # 从配置文件中读取指定的帖子ID列表获取指定帖子的信息与评论信息 uv run main.py --platform xhs --lt qrcode --type detail +# 只维护登录态，不进入抓取流程 +uv run main.py --platform xhs --lt qrcode --type login + # 打开对应APP扫二维码登录 # 其他平台爬虫使用示例，执行下面的命令查看 @@ -211,6 +214,18 @@ uv run uvicorn api.main:app --port 8080 --reload 然后访问 `http://localhost:8080` 即可。 +#### 多实例调度器 + +启动同一个 API 服务后，访问 `http://localhost:8080/scheduler` 可打开多实例调度器。调度器支持创建多个独立账号实例，每个实例拥有独立浏览器 Profile、CDP 端口、登录态、默认爬取参数和任务队列。 + +调度器运行数据默认保存在 `data/scheduler/`： + +- `scheduler.db`：实例、任务、日志和产物索引 +- `profiles/{instance_id}/`：实例独立浏览器 Profile +- `artifacts/{instance_id}/{task_id}/`：任务抓取产物 + +详细说明请查看：[多实例调度器使用指南](docs/多实例调度器使用指南.md) + #### WebUI 功能特性 - 可视化配置爬虫参数（平台、登录方式、爬取类型等） diff --git a/api/main.py b/api/main.py index af49b152f..3bba80cbc 100644 --- a/api/main.py +++ b/api/main.py @@ -32,7 +32,7 @@ from fastapi.staticfiles import StaticFiles from fastapi.responses import FileResponse -from .routers import crawler_router, data_router, websocket_router +from .routers import crawler_router, data_router, scheduler_router, websocket_router # Project root directory (used for running subprocesses like uv run main.py) PROJECT_ROOT = Path(__file__).parent.parent @@ -45,6 +45,7 @@ # Get webui static files directory WEBUI_DIR = os.path.join(os.path.dirname(__file__), "webui") +SCHEDULER_WEBUI_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "scheduler_webui") # CORS configuration - allow frontend dev server access app.add_middleware( @@ -63,6 +64,7 @@ # Register routers app.include_router(crawler_router, prefix="/api") app.include_router(data_router, prefix="/api") +app.include_router(scheduler_router, prefix="/api") app.include_router(websocket_router, prefix="/api") @@ -85,6 +87,18 @@ async def health_check(): return {"status": "ok"} +@app.get("/scheduler") +async def serve_scheduler_frontend(): + """Return scheduler WebUI page.""" + index_path = os.path.join(SCHEDULER_WEBUI_DIR, "index.html") + if os.path.exists(index_path): + return FileResponse(index_path) + return { + "message": "MediaCrawler Scheduler WebUI", + "note": "Scheduler WebUI source was not found", + } + + @app.get("/api/env/check") async def check_environment(): """Check if MediaCrawler environment is configured correctly""" @@ -175,6 +189,7 @@ async def get_config_options(): {"value": "search", "label": "Search Mode"}, {"value": "detail", "label": "Detail Mode"}, {"value": "creator", "label": "Creator Mode"}, + {"value": "login", "label": "Login Only"}, ], "save_options": [ {"value": "jsonl", "label": "JSONL File"}, @@ -200,6 +215,13 @@ async def get_config_options(): # Mount other static files (e.g., vite.svg) app.mount("/static", StaticFiles(directory=WEBUI_DIR), name="webui-static") +if os.path.exists(SCHEDULER_WEBUI_DIR): + app.mount( + "/scheduler/static", + StaticFiles(directory=SCHEDULER_WEBUI_DIR), + name="scheduler-webui-static", + ) + if __name__ == "__main__": uvicorn.run(app, host="0.0.0.0", port=8080) diff --git a/api/routers/__init__.py b/api/routers/__init__.py index 123cbc066..fbd763550 100644 --- a/api/routers/__init__.py +++ b/api/routers/__init__.py @@ -18,6 +18,7 @@ from .crawler import router as crawler_router from .data import router as data_router +from .scheduler import router as scheduler_router from .websocket import router as websocket_router -__all__ = ["crawler_router", "data_router", "websocket_router"] +__all__ = ["crawler_router", "data_router", "scheduler_router", "websocket_router"] diff --git a/api/routers/scheduler.py b/api/routers/scheduler.py new file mode 100644 index 000000000..84e496777 --- /dev/null +++ b/api/routers/scheduler.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- + +from fastapi import APIRouter, HTTPException, Query + +from api.scheduler.manager import scheduler_manager +from api.scheduler.schemas import ( + ArtifactResponse, + InstanceCreateRequest, + InstanceResponse, + InstanceUpdateRequest, + SchedulerStatusResponse, + TaskCreateRequest, + TaskLogResponse, + TaskResponse, +) + +router = APIRouter(prefix="/scheduler", tags=["scheduler"]) + + +@router.get("/status", response_model=SchedulerStatusResponse) +async def scheduler_status(): + return scheduler_manager.status() + + +@router.get("/instances", response_model=list[InstanceResponse]) +async def list_instances(): + return scheduler_manager.list_instances() + + +@router.post("/instances", response_model=InstanceResponse) +async def create_instance(request: InstanceCreateRequest): + return scheduler_manager.create_instance(request) + + +@router.get("/instances/{instance_id}", response_model=InstanceResponse) +async def get_instance(instance_id: str): + instance = scheduler_manager.get_instance(instance_id) + if not instance: + raise HTTPException(status_code=404, detail="Instance not found") + return instance + + +@router.patch("/instances/{instance_id}", response_model=InstanceResponse) +async def update_instance(instance_id: str, request: InstanceUpdateRequest): + try: + instance = await scheduler_manager.update_instance(instance_id, request) + except RuntimeError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + if not instance: + raise HTTPException(status_code=404, detail="Instance not found") + return instance + + +@router.delete("/instances/{instance_id}") +async def delete_instance(instance_id: str): + try: + deleted = await scheduler_manager.delete_instance(instance_id) + except RuntimeError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + if not deleted: + raise HTTPException(status_code=404, detail="Instance not found") + return {"status": "ok", "message": "Instance deleted"} + + +@router.post("/instances/{instance_id}/login", response_model=TaskResponse) +async def login_instance(instance_id: str): + try: + return await scheduler_manager.create_login_task(instance_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail="Instance not found") from exc + except RuntimeError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.get("/tasks", response_model=list[TaskResponse]) +async def list_tasks( + instance_id: str | None = None, + limit: int = Query(default=100, ge=1, le=500), +): + return scheduler_manager.list_tasks(instance_id=instance_id, limit=limit) + + +@router.post("/tasks", response_model=TaskResponse) +async def create_task(request: TaskCreateRequest): + try: + return await scheduler_manager.create_task(request) + except KeyError as exc: + raise HTTPException(status_code=404, detail="Instance not found") from exc + except RuntimeError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.get("/tasks/{task_id}", response_model=TaskResponse) +async def get_task(task_id: str): + task = scheduler_manager.get_task(task_id) + if not task: + raise HTTPException(status_code=404, detail="Task not found") + return task + + +@router.post("/tasks/{task_id}/start", response_model=TaskResponse) +async def start_task(task_id: str): + try: + task = await scheduler_manager.start_task(task_id) + except RuntimeError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + if not task: + raise HTTPException(status_code=404, detail="Task not found") + return task + + +@router.post("/tasks/{task_id}/cancel", response_model=TaskResponse) +async def cancel_task(task_id: str): + task = await scheduler_manager.cancel_task(task_id) + if not task: + raise HTTPException(status_code=404, detail="Task not found") + return task + + +@router.get("/tasks/{task_id}/logs", response_model=list[TaskLogResponse]) +async def list_task_logs(task_id: str, limit: int = Query(default=300, ge=1, le=1000)): + if not scheduler_manager.get_task(task_id): + raise HTTPException(status_code=404, detail="Task not found") + return scheduler_manager.list_logs(task_id, limit=limit) + + +@router.get("/tasks/{task_id}/artifacts", response_model=list[ArtifactResponse]) +async def list_task_artifacts(task_id: str): + if not scheduler_manager.get_task(task_id): + raise HTTPException(status_code=404, detail="Task not found") + return scheduler_manager.list_artifacts(task_id) diff --git a/api/scheduler/__init__.py b/api/scheduler/__init__.py new file mode 100644 index 000000000..cffa114ed --- /dev/null +++ b/api/scheduler/__init__.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- + +from .manager import SchedulerManager, scheduler_manager + +__all__ = ["SchedulerManager", "scheduler_manager"] diff --git a/api/scheduler/manager.py b/api/scheduler/manager.py new file mode 100644 index 000000000..c2a61c864 --- /dev/null +++ b/api/scheduler/manager.py @@ -0,0 +1,405 @@ +# -*- coding: utf-8 -*- + +from __future__ import annotations + +import asyncio +import json +import os +import subprocess +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional + +from api.schemas.crawler import CrawlerTypeEnum +from .schemas import InstanceCreateRequest, InstanceUpdateRequest, TaskCreateRequest +from .store import PROJECT_ROOT, SchedulerStore, _json_dumps, utc_now + + +@dataclass +class InstanceRuntime: + instance_id: str + task_id: str + process: subprocess.Popen + read_task: Optional[asyncio.Task] = None + wait_task: Optional[asyncio.Task] = None + canceled: bool = False + + +class SchedulerManager: + """Manage multiple crawler subprocess instances.""" + + def __init__( + self, + store: SchedulerStore | None = None, + project_root: Path | None = None, + ) -> None: + self.store = store or SchedulerStore() + self.project_root = project_root or PROJECT_ROOT + self.base_dir = self.project_root / "data" / "scheduler" + self.profile_dir = self.base_dir / "profiles" + self.artifact_dir = self.base_dir / "artifacts" + self._lock = asyncio.Lock() + self._runtimes: dict[str, InstanceRuntime] = {} + + def list_instances(self) -> list[dict[str, Any]]: + return self.store.list_instances() + + def get_instance(self, instance_id: str) -> Optional[dict[str, Any]]: + return self.store.get_instance(instance_id) + + def create_instance(self, request: InstanceCreateRequest) -> dict[str, Any]: + payload = request.model_dump(mode="json") + instance_id = uuid.uuid4().hex + cdp_debug_port = request.cdp_debug_port or self._allocate_cdp_port() + profile_dir = request.browser_profile_dir.strip() + if not profile_dir: + profile_dir = str(self.profile_dir / instance_id) + else: + profile_path = Path(profile_dir) + if not profile_path.is_absolute(): + profile_dir = str(self.project_root / profile_path) + return self.store.create_instance(payload, profile_dir, cdp_debug_port, instance_id=instance_id) + + async def update_instance(self, instance_id: str, request: InstanceUpdateRequest) -> Optional[dict[str, Any]]: + async with self._lock: + instance = self.store.get_instance(instance_id) + if not instance: + return None + if instance["status"] in {"running", "stopping"}: + mutable_running_fields = {"status"} + changed_fields = { + key + for key, value in request.model_dump(exclude_unset=True).items() + if value is not None + } + if changed_fields - mutable_running_fields: + raise RuntimeError("running instance cannot be edited") + fields = request.model_dump(mode="json", exclude_unset=True) + if "default_params" in fields: + fields["default_params_json"] = _json_dumps(fields.pop("default_params")) + return self.store.update_instance(instance_id, **fields) + + async def delete_instance(self, instance_id: str) -> bool: + async with self._lock: + instance = self.store.get_instance(instance_id) + if not instance: + return False + if instance["status"] in {"running", "stopping"}: + raise RuntimeError("running instance cannot be deleted") + return self.store.delete_instance(instance_id) + + def list_tasks(self, instance_id: str | None = None, limit: int = 100) -> list[dict[str, Any]]: + return self.store.list_tasks(instance_id=instance_id, limit=limit) + + def get_task(self, task_id: str) -> Optional[dict[str, Any]]: + return self.store.get_task(task_id) + + async def create_task(self, request: TaskCreateRequest) -> dict[str, Any]: + async with self._lock: + instance = self.store.get_instance(request.instance_id) + if not instance: + raise KeyError("instance not found") + if instance["status"] == "disabled": + raise RuntimeError("instance is disabled") + artifact_dir = self.artifact_dir / request.instance_id + payload = request.model_dump(mode="json") + task = self.store.create_task(payload, str(artifact_dir / "pending")) + real_artifact_dir = artifact_dir / task["id"] + task = self.store.update_task(task["id"], artifact_dir=str(real_artifact_dir)) + if instance["status"] in {"idle", "error"} and request.instance_id not in self._runtimes: + await self._start_task_locked(task) + task = self.store.get_task(task["id"]) + return task + + async def create_login_task(self, instance_id: str) -> dict[str, Any]: + request = TaskCreateRequest(instance_id=instance_id, crawler_type=CrawlerTypeEnum.LOGIN, target_text="") + return await self.create_task(request) + + async def start_task(self, task_id: str) -> Optional[dict[str, Any]]: + async with self._lock: + task = self.store.get_task(task_id) + if not task: + return None + if task["status"] != "queued": + raise RuntimeError("only queued tasks can be started") + await self._start_task_locked(task) + return self.store.get_task(task_id) + + async def cancel_task(self, task_id: str) -> Optional[dict[str, Any]]: + async with self._lock: + task = self.store.get_task(task_id) + if not task: + return None + if task["status"] == "queued": + self.store.append_log(task_id, "Task canceled before start", "warning") + return self.store.update_task(task_id, status="canceled", finished_at=utc_now()) + if task["status"] != "running": + return task + runtime = self._runtimes.get(task["instance_id"]) + if runtime and runtime.task_id == task_id: + runtime.canceled = True + self.store.append_log(task_id, "Stopping crawler subprocess ...", "warning") + self.store.update_instance(task["instance_id"], status="stopping") + try: + runtime.process.terminate() + except ProcessLookupError: + pass + return self.store.update_task(task_id, status="canceled", finished_at=utc_now()) + + def list_logs(self, task_id: str, limit: int = 300) -> list[dict[str, Any]]: + return self.store.list_logs(task_id, limit) + + def list_artifacts(self, task_id: str) -> list[dict[str, Any]]: + return self.store.list_artifacts(task_id) + + def status(self) -> dict[str, int]: + return self.store.scheduler_counts() + + async def _start_task_locked(self, task: dict[str, Any]) -> None: + instance = self.store.get_instance(task["instance_id"]) + if not instance: + raise KeyError("instance not found") + if instance["id"] in self._runtimes: + return + + artifact_dir = Path(task["artifact_dir"]) + artifact_dir.mkdir(parents=True, exist_ok=True) + Path(instance["browser_profile_dir"]).mkdir(parents=True, exist_ok=True) + + cmd = self._build_command(instance, task) + self.store.append_log(task["id"], f"Starting crawler: {' '.join(cmd)}", "info") + try: + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + bufsize=1, + cwd=str(self.project_root), + env={**os.environ, "PYTHONUNBUFFERED": "1"}, + ) + except Exception as exc: + message = f"Failed to start crawler: {type(exc).__name__}: {exc}" + self.store.append_log(task["id"], message, "error") + self.store.update_task(task["id"], status="failed", error_message=message, finished_at=utc_now()) + self.store.update_instance(instance["id"], status="error", current_task_id=None, pid=None, last_error=message) + return + + runtime = InstanceRuntime(instance_id=instance["id"], task_id=task["id"], process=process) + runtime.read_task = asyncio.create_task(self._read_output(runtime)) + runtime.wait_task = asyncio.create_task(self._watch_process(runtime)) + self._runtimes[instance["id"]] = runtime + now = utc_now() + self.store.update_task(task["id"], status="running", pid=process.pid, started_at=now) + self.store.update_instance( + instance["id"], + status="running", + current_task_id=task["id"], + pid=process.pid, + last_error="", + ) + self.store.append_log(task["id"], f"Crawler subprocess started, pid={process.pid}", "success") + + def _build_command(self, instance: dict[str, Any], task: dict[str, Any]) -> list[str]: + params = {**instance.get("default_params", {}), **task.get("params", {})} + crawler_type = task["crawler_type"] + save_option = str(params.get("save_option", instance["save_option"])) + headless = self._as_bool(params.get("headless", instance["headless"])) + target_text = task.get("target_text", "") + + cmd = ["uv", "run", "python", "main.py"] + cmd.extend(["--platform", instance["platform"]]) + cmd.extend(["--lt", str(params.get("login_type", instance["login_type"]))]) + cmd.extend(["--type", crawler_type]) + cmd.extend(["--save_data_option", save_option]) + cmd.extend(["--save_data_path", task["artifact_dir"]]) + cmd.extend(["--instance_id", instance["id"]]) + cmd.extend(["--browser_profile_dir", instance["browser_profile_dir"]]) + cmd.extend(["--cdp_debug_port", str(instance["cdp_debug_port"])]) + cmd.extend(["--cdp_connect_existing", self._bool_arg(params.get("cdp_connect_existing", False))]) + cmd.extend(["--headless", self._bool_arg(headless)]) + + if crawler_type == "search": + keywords = target_text or str(params.get("keywords", "")) + if keywords: + cmd.extend(["--keywords", keywords]) + elif crawler_type == "detail": + specified_id = target_text or str(params.get("specified_id", "")) + if specified_id: + cmd.extend(["--specified_id", specified_id]) + elif crawler_type == "creator": + creator_id = target_text or str(params.get("creator_id", "")) + if creator_id: + cmd.extend(["--creator_id", creator_id]) + + option_map = { + "start": "--start", + "start_page": "--start", + "enable_comments": "--get_comment", + "enable_sub_comments": "--get_sub_comment", + "max_notes_count": "--crawler_max_notes_count", + "max_comments_count": "--max_comments_count_singlenotes", + "max_concurrency_num": "--max_concurrency_num", + "cookies": "--cookies", + "enable_ip_proxy": "--enable_ip_proxy", + "ip_proxy_pool_count": "--ip_proxy_pool_count", + "ip_proxy_provider_name": "--ip_proxy_provider_name", + "static_proxy_url": "--static_proxy_url", + } + for key, flag in option_map.items(): + if key not in params or params[key] in (None, ""): + continue + value = params[key] + if isinstance(value, bool): + value = self._bool_arg(value) + cmd.extend([flag, str(value)]) + return cmd + + async def _read_output(self, runtime: InstanceRuntime) -> None: + loop = asyncio.get_running_loop() + process = runtime.process + try: + while process.poll() is None and process.stdout: + line = await loop.run_in_executor(None, process.stdout.readline) + if not line: + break + self._append_process_log(runtime.task_id, line) + if process.stdout: + remaining = await loop.run_in_executor(None, process.stdout.read) + for line in remaining.splitlines(): + self._append_process_log(runtime.task_id, line) + except asyncio.CancelledError: + raise + except Exception as exc: + self.store.append_log(runtime.task_id, f"Log reader error: {type(exc).__name__}: {exc}", "error") + + async def _watch_process(self, runtime: InstanceRuntime) -> None: + loop = asyncio.get_running_loop() + return_code = await loop.run_in_executor(None, runtime.process.wait) + if runtime.read_task: + try: + await asyncio.wait_for(runtime.read_task, timeout=3) + except asyncio.TimeoutError: + runtime.read_task.cancel() + + async with self._lock: + task = self.store.get_task(runtime.task_id) + if not task: + self._runtimes.pop(runtime.instance_id, None) + return + if runtime.canceled or task["status"] == "canceled": + task_status = "canceled" + error_message = "" + instance_status = "idle" + level = "warning" + message = f"Crawler subprocess canceled, exit_code={return_code}" + elif return_code == 0: + task_status = "succeeded" + error_message = "" + instance_status = "idle" + level = "success" + message = "Crawler subprocess finished successfully" + else: + task_status = "failed" + error_message = f"Crawler subprocess exited with code {return_code}" + instance_status = "error" + level = "error" + message = error_message + + artifacts = self._scan_artifacts(Path(task["artifact_dir"])) + self.store.replace_artifacts(task["id"], artifacts) + self.store.update_task( + task["id"], + status=task_status, + exit_code=return_code, + error_message=error_message, + finished_at=utc_now(), + ) + self.store.update_instance( + runtime.instance_id, + status=instance_status, + current_task_id=None, + pid=None, + last_error=error_message, + ) + self.store.append_log(task["id"], message, level) + self._runtimes.pop(runtime.instance_id, None) + + if task_status in {"succeeded", "canceled"}: + next_task = self.store.get_next_queued_task(runtime.instance_id) + if next_task: + await self._start_task_locked(next_task) + + def _scan_artifacts(self, root: Path) -> list[dict[str, Any]]: + if not root.exists(): + return [] + artifacts: list[dict[str, Any]] = [] + for path in root.rglob("*"): + if not path.is_file(): + continue + stat = path.stat() + artifacts.append( + { + "path": str(path), + "type": path.suffix.lstrip(".") or "file", + "size": stat.st_size, + "modified_at": stat.st_mtime, + "record_count": self._count_records(path), + } + ) + return artifacts + + def _count_records(self, path: Path) -> Optional[int]: + try: + if path.suffix == ".jsonl": + with path.open("r", encoding="utf-8") as f: + return sum(1 for line in f if line.strip()) + if path.suffix == ".json": + with path.open("r", encoding="utf-8") as f: + data = json.load(f) + return len(data) if isinstance(data, list) else None + if path.suffix == ".csv": + with path.open("r", encoding="utf-8") as f: + lines = sum(1 for line in f if line.strip()) + return max(lines - 1, 0) + except Exception: + return None + return None + + def _append_process_log(self, task_id: str, line: str) -> None: + line = line.strip() + if not line: + return + self.store.append_log(task_id, line, self._parse_log_level(line)) + + def _parse_log_level(self, line: str) -> str: + line_upper = line.upper() + if "ERROR" in line_upper or "FAILED" in line_upper or "异常" in line: + return "error" + if "WARNING" in line_upper or "WARN" in line_upper: + return "warning" + if "SUCCESS" in line_upper or "完成" in line or "成功" in line: + return "success" + if "DEBUG" in line_upper: + return "debug" + return "info" + + def _allocate_cdp_port(self) -> int: + used_ports = set(self.store.list_cdp_ports()) + port = 9222 + while port in used_ports: + port += 1 + return port + + def _bool_arg(self, value: Any) -> str: + return "true" if self._as_bool(value) else "false" + + def _as_bool(self, value: Any) -> bool: + if isinstance(value, str): + return value.strip().lower() in {"1", "true", "yes", "y", "t", "on"} + return bool(value) + + +scheduler_manager = SchedulerManager() diff --git a/api/scheduler/schemas.py b/api/scheduler/schemas.py new file mode 100644 index 000000000..1ba2cd728 --- /dev/null +++ b/api/scheduler/schemas.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- + +from enum import Enum +from typing import Any, Dict, Optional + +from pydantic import BaseModel, Field, field_validator + +from api.schemas.crawler import ( + CrawlerTypeEnum, + LoginTypeEnum, + PlatformEnum, + SaveDataOptionEnum, +) + + +class InstanceStatusEnum(str, Enum): + """Scheduler instance status.""" + + IDLE = "idle" + RUNNING = "running" + STOPPING = "stopping" + ERROR = "error" + DISABLED = "disabled" + + +class TaskStatusEnum(str, Enum): + """Scheduler task status.""" + + QUEUED = "queued" + RUNNING = "running" + SUCCEEDED = "succeeded" + FAILED = "failed" + CANCELED = "canceled" + + +class InstanceCreateRequest(BaseModel): + """Create a crawler instance.""" + + name: str = Field(..., min_length=1, max_length=80) + platform: PlatformEnum + login_type: LoginTypeEnum = LoginTypeEnum.QRCODE + headless: bool = False + save_option: SaveDataOptionEnum = SaveDataOptionEnum.JSONL + browser_profile_dir: str = "" + cdp_debug_port: Optional[int] = Field(default=None, ge=1000, le=65535) + default_params: Dict[str, Any] = Field(default_factory=dict) + + @field_validator("name") + @classmethod + def strip_name(cls, value: str) -> str: + value = value.strip() + if not value: + raise ValueError("instance name cannot be empty") + return value + + +class InstanceUpdateRequest(BaseModel): + """Update mutable crawler instance settings.""" + + name: Optional[str] = Field(default=None, min_length=1, max_length=80) + login_type: Optional[LoginTypeEnum] = None + headless: Optional[bool] = None + save_option: Optional[SaveDataOptionEnum] = None + browser_profile_dir: Optional[str] = None + cdp_debug_port: Optional[int] = Field(default=None, ge=1000, le=65535) + default_params: Optional[Dict[str, Any]] = None + status: Optional[InstanceStatusEnum] = None + + @field_validator("name") + @classmethod + def strip_name(cls, value: Optional[str]) -> Optional[str]: + if value is None: + return value + value = value.strip() + if not value: + raise ValueError("instance name cannot be empty") + return value + + +class InstanceResponse(BaseModel): + id: str + name: str + platform: str + login_type: str + headless: bool + save_option: str + browser_profile_dir: str + cdp_debug_port: int + default_params: Dict[str, Any] + status: str + current_task_id: Optional[str] = None + pid: Optional[int] = None + last_error: str = "" + created_at: str + updated_at: str + + +class TaskCreateRequest(BaseModel): + """Create a crawler task bound to one instance.""" + + instance_id: str + crawler_type: CrawlerTypeEnum = CrawlerTypeEnum.SEARCH + target_text: str = "" + params: Dict[str, Any] = Field(default_factory=dict) + + +class TaskResponse(BaseModel): + id: str + instance_id: str + crawler_type: str + target_text: str + params: Dict[str, Any] + status: str + pid: Optional[int] = None + exit_code: Optional[int] = None + artifact_dir: str = "" + error_message: str = "" + created_at: str + updated_at: str + started_at: Optional[str] = None + finished_at: Optional[str] = None + + +class TaskLogResponse(BaseModel): + id: int + task_id: str + timestamp: str + level: str + message: str + + +class ArtifactResponse(BaseModel): + id: str + task_id: str + path: str + type: str + size: int + modified_at: float + record_count: Optional[int] = None + + +class SchedulerStatusResponse(BaseModel): + instances_total: int + running_instances: int + queued_tasks: int + running_tasks: int diff --git a/api/scheduler/store.py b/api/scheduler/store.py new file mode 100644 index 000000000..1aa97c667 --- /dev/null +++ b/api/scheduler/store.py @@ -0,0 +1,373 @@ +# -*- coding: utf-8 -*- + +from __future__ import annotations + +import json +import sqlite3 +import threading +import uuid +from datetime import datetime +from pathlib import Path +from typing import Any, Iterable, Optional + +PROJECT_ROOT = Path(__file__).resolve().parents[2] + + +def utc_now() -> str: + return datetime.now().isoformat(timespec="seconds") + + +def _json_dumps(value: Any) -> str: + return json.dumps(value or {}, ensure_ascii=False) + + +def _json_loads(value: str | None) -> dict[str, Any]: + if not value: + return {} + try: + loaded = json.loads(value) + except json.JSONDecodeError: + return {} + return loaded if isinstance(loaded, dict) else {} + + +class SchedulerStore: + """Small SQLite store for scheduler state.""" + + def __init__(self, db_path: str | Path | None = None) -> None: + self.db_path = Path(db_path) if db_path else PROJECT_ROOT / "data" / "scheduler" / "scheduler.db" + self.db_path.parent.mkdir(parents=True, exist_ok=True) + self._lock = threading.RLock() + self._init_db() + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + return conn + + def _init_db(self) -> None: + with self._lock, self._connect() as conn: + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS instances ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + platform TEXT NOT NULL, + login_type TEXT NOT NULL, + headless INTEGER NOT NULL DEFAULT 0, + save_option TEXT NOT NULL, + browser_profile_dir TEXT NOT NULL, + cdp_debug_port INTEGER NOT NULL, + default_params_json TEXT NOT NULL, + status TEXT NOT NULL, + current_task_id TEXT, + pid INTEGER, + last_error TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS tasks ( + id TEXT PRIMARY KEY, + instance_id TEXT NOT NULL, + crawler_type TEXT NOT NULL, + target_text TEXT NOT NULL DEFAULT '', + params_json TEXT NOT NULL, + status TEXT NOT NULL, + pid INTEGER, + exit_code INTEGER, + artifact_dir TEXT NOT NULL DEFAULT '', + error_message TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + started_at TEXT, + finished_at TEXT, + FOREIGN KEY(instance_id) REFERENCES instances(id) + ); + + CREATE TABLE IF NOT EXISTS task_logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + task_id TEXT NOT NULL, + timestamp TEXT NOT NULL, + level TEXT NOT NULL, + message TEXT NOT NULL, + FOREIGN KEY(task_id) REFERENCES tasks(id) + ); + + CREATE TABLE IF NOT EXISTS artifacts ( + id TEXT PRIMARY KEY, + task_id TEXT NOT NULL, + path TEXT NOT NULL, + type TEXT NOT NULL, + size INTEGER NOT NULL, + modified_at REAL NOT NULL, + record_count INTEGER, + FOREIGN KEY(task_id) REFERENCES tasks(id) + ); + """ + ) + conn.execute("CREATE INDEX IF NOT EXISTS idx_tasks_instance_status ON tasks(instance_id, status, created_at)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_task_logs_task ON task_logs(task_id, id)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_artifacts_task ON artifacts(task_id)") + + def create_instance( + self, + payload: dict[str, Any], + profile_dir: str, + cdp_debug_port: int, + instance_id: str | None = None, + ) -> dict[str, Any]: + now = utc_now() + instance_id = instance_id or uuid.uuid4().hex + with self._lock, self._connect() as conn: + conn.execute( + """ + INSERT INTO instances ( + id, name, platform, login_type, headless, save_option, + browser_profile_dir, cdp_debug_port, default_params_json, + status, current_task_id, pid, last_error, created_at, updated_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'idle', NULL, NULL, '', ?, ?) + """, + ( + instance_id, + payload["name"], + payload["platform"], + payload["login_type"], + 1 if payload.get("headless") else 0, + payload["save_option"], + profile_dir, + cdp_debug_port, + _json_dumps(payload.get("default_params")), + now, + now, + ), + ) + return self.get_instance(instance_id) + + def list_instances(self) -> list[dict[str, Any]]: + with self._lock, self._connect() as conn: + rows = conn.execute("SELECT * FROM instances ORDER BY created_at DESC").fetchall() + return [self._instance_from_row(row) for row in rows] + + def get_instance(self, instance_id: str) -> Optional[dict[str, Any]]: + with self._lock, self._connect() as conn: + row = conn.execute("SELECT * FROM instances WHERE id = ?", (instance_id,)).fetchone() + return self._instance_from_row(row) if row else None + + def update_instance(self, instance_id: str, **fields: Any) -> Optional[dict[str, Any]]: + if not fields: + return self.get_instance(instance_id) + allowed = { + "name", + "login_type", + "headless", + "save_option", + "browser_profile_dir", + "cdp_debug_port", + "default_params_json", + "status", + "current_task_id", + "pid", + "last_error", + } + assignments: list[str] = [] + values: list[Any] = [] + for key, value in fields.items(): + if key not in allowed: + continue + if key == "headless" and value is not None: + value = 1 if value else 0 + assignments.append(f"{key} = ?") + values.append(value) + if not assignments: + return self.get_instance(instance_id) + assignments.append("updated_at = ?") + values.append(utc_now()) + values.append(instance_id) + with self._lock, self._connect() as conn: + conn.execute(f"UPDATE instances SET {', '.join(assignments)} WHERE id = ?", values) + return self.get_instance(instance_id) + + def delete_instance(self, instance_id: str) -> bool: + with self._lock, self._connect() as conn: + row = conn.execute("SELECT status FROM instances WHERE id = ?", (instance_id,)).fetchone() + if not row: + return False + conn.execute("DELETE FROM instances WHERE id = ?", (instance_id,)) + return True + + def list_cdp_ports(self) -> list[int]: + with self._lock, self._connect() as conn: + rows = conn.execute("SELECT cdp_debug_port FROM instances").fetchall() + return [int(row["cdp_debug_port"]) for row in rows if row["cdp_debug_port"]] + + def create_task(self, payload: dict[str, Any], artifact_dir: str) -> dict[str, Any]: + now = utc_now() + task_id = uuid.uuid4().hex + with self._lock, self._connect() as conn: + conn.execute( + """ + INSERT INTO tasks ( + id, instance_id, crawler_type, target_text, params_json, + status, pid, exit_code, artifact_dir, error_message, + created_at, updated_at, started_at, finished_at + ) + VALUES (?, ?, ?, ?, ?, 'queued', NULL, NULL, ?, '', ?, ?, NULL, NULL) + """, + ( + task_id, + payload["instance_id"], + payload["crawler_type"], + payload.get("target_text", ""), + _json_dumps(payload.get("params")), + artifact_dir, + now, + now, + ), + ) + return self.get_task(task_id) + + def list_tasks(self, instance_id: str | None = None, limit: int = 100) -> list[dict[str, Any]]: + limit = max(1, min(limit, 500)) + with self._lock, self._connect() as conn: + if instance_id: + rows = conn.execute( + "SELECT * FROM tasks WHERE instance_id = ? ORDER BY created_at DESC LIMIT ?", + (instance_id, limit), + ).fetchall() + else: + rows = conn.execute("SELECT * FROM tasks ORDER BY created_at DESC LIMIT ?", (limit,)).fetchall() + return [self._task_from_row(row) for row in rows] + + def get_task(self, task_id: str) -> Optional[dict[str, Any]]: + with self._lock, self._connect() as conn: + row = conn.execute("SELECT * FROM tasks WHERE id = ?", (task_id,)).fetchone() + return self._task_from_row(row) if row else None + + def get_next_queued_task(self, instance_id: str) -> Optional[dict[str, Any]]: + with self._lock, self._connect() as conn: + row = conn.execute( + """ + SELECT * FROM tasks + WHERE instance_id = ? AND status = 'queued' + ORDER BY created_at ASC + LIMIT 1 + """, + (instance_id,), + ).fetchone() + return self._task_from_row(row) if row else None + + def update_task(self, task_id: str, **fields: Any) -> Optional[dict[str, Any]]: + if not fields: + return self.get_task(task_id) + allowed = { + "status", + "pid", + "exit_code", + "artifact_dir", + "error_message", + "started_at", + "finished_at", + } + assignments: list[str] = [] + values: list[Any] = [] + for key, value in fields.items(): + if key not in allowed: + continue + assignments.append(f"{key} = ?") + values.append(value) + if not assignments: + return self.get_task(task_id) + assignments.append("updated_at = ?") + values.append(utc_now()) + values.append(task_id) + with self._lock, self._connect() as conn: + conn.execute(f"UPDATE tasks SET {', '.join(assignments)} WHERE id = ?", values) + return self.get_task(task_id) + + def append_log(self, task_id: str, message: str, level: str = "info") -> dict[str, Any]: + timestamp = utc_now() + with self._lock, self._connect() as conn: + cursor = conn.execute( + "INSERT INTO task_logs (task_id, timestamp, level, message) VALUES (?, ?, ?, ?)", + (task_id, timestamp, level, message), + ) + log_id = int(cursor.lastrowid) + return { + "id": log_id, + "task_id": task_id, + "timestamp": timestamp, + "level": level, + "message": message, + } + + def list_logs(self, task_id: str, limit: int = 300) -> list[dict[str, Any]]: + limit = max(1, min(limit, 1000)) + with self._lock, self._connect() as conn: + rows = conn.execute( + """ + SELECT * FROM ( + SELECT * FROM task_logs + WHERE task_id = ? + ORDER BY id DESC + LIMIT ? + ) ORDER BY id ASC + """, + (task_id, limit), + ).fetchall() + return [dict(row) for row in rows] + + def replace_artifacts(self, task_id: str, artifacts: Iterable[dict[str, Any]]) -> None: + with self._lock, self._connect() as conn: + conn.execute("DELETE FROM artifacts WHERE task_id = ?", (task_id,)) + conn.executemany( + """ + INSERT INTO artifacts (id, task_id, path, type, size, modified_at, record_count) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + [ + ( + artifact.get("id") or uuid.uuid4().hex, + task_id, + artifact["path"], + artifact["type"], + artifact["size"], + artifact["modified_at"], + artifact.get("record_count"), + ) + for artifact in artifacts + ], + ) + + def list_artifacts(self, task_id: str) -> list[dict[str, Any]]: + with self._lock, self._connect() as conn: + rows = conn.execute( + "SELECT * FROM artifacts WHERE task_id = ? ORDER BY modified_at DESC", + (task_id,), + ).fetchall() + return [dict(row) for row in rows] + + def scheduler_counts(self) -> dict[str, int]: + with self._lock, self._connect() as conn: + instances_total = conn.execute("SELECT COUNT(*) FROM instances").fetchone()[0] + running_instances = conn.execute("SELECT COUNT(*) FROM instances WHERE status = 'running'").fetchone()[0] + queued_tasks = conn.execute("SELECT COUNT(*) FROM tasks WHERE status = 'queued'").fetchone()[0] + running_tasks = conn.execute("SELECT COUNT(*) FROM tasks WHERE status = 'running'").fetchone()[0] + return { + "instances_total": int(instances_total), + "running_instances": int(running_instances), + "queued_tasks": int(queued_tasks), + "running_tasks": int(running_tasks), + } + + def _instance_from_row(self, row: sqlite3.Row) -> dict[str, Any]: + data = dict(row) + data["headless"] = bool(data["headless"]) + data["default_params"] = _json_loads(data.pop("default_params_json", None)) + return data + + def _task_from_row(self, row: sqlite3.Row) -> dict[str, Any]: + data = dict(row) + data["params"] = _json_loads(data.pop("params_json", None)) + return data diff --git a/api/schemas/crawler.py b/api/schemas/crawler.py index 6eb3b1b7e..cc3ccc6d3 100644 --- a/api/schemas/crawler.py +++ b/api/schemas/crawler.py @@ -47,6 +47,7 @@ class CrawlerTypeEnum(str, Enum): SEARCH = "search" DETAIL = "detail" CREATOR = "creator" + LOGIN = "login" class SaveDataOptionEnum(str, Enum): diff --git a/cmd_arg/arg.py b/cmd_arg/arg.py index 20d1d3976..8bfc63578 100644 --- a/cmd_arg/arg.py +++ b/cmd_arg/arg.py @@ -63,6 +63,7 @@ class CrawlerTypeEnum(str, Enum): SEARCH = "search" DETAIL = "detail" CREATOR = "creator" + LOGIN = "login" class SaveDataOptionEnum(str, Enum): @@ -178,7 +179,7 @@ def main( CrawlerTypeEnum, typer.Option( "--type", - help="Crawler type (search=Search | detail=Detail | creator=Creator)", + help="Crawler type (search=Search | detail=Detail | creator=Creator | login=Login only)", rich_help_panel="Basic Configuration", ), ] = _coerce_enum(CrawlerTypeEnum, config.CRAWLER_TYPE, CrawlerTypeEnum.SEARCH), @@ -332,6 +333,39 @@ def main( rich_help_panel="Proxy Configuration", ), ] = config.STATIC_PROXY_URL, + instance_id: Annotated[ + str, + typer.Option( + "--instance_id", + help="Scheduler instance ID for multi-instance runs", + rich_help_panel="Scheduler Configuration", + ), + ] = config.INSTANCE_ID, + browser_profile_dir: Annotated[ + str, + typer.Option( + "--browser_profile_dir", + help="Browser profile directory for this crawler instance", + rich_help_panel="Scheduler Configuration", + ), + ] = config.BROWSER_PROFILE_DIR, + cdp_debug_port: Annotated[ + int, + typer.Option( + "--cdp_debug_port", + help="CDP debug port for this crawler instance", + rich_help_panel="Scheduler Configuration", + ), + ] = config.CDP_DEBUG_PORT, + cdp_connect_existing: Annotated[ + str, + typer.Option( + "--cdp_connect_existing", + help="Whether CDP mode connects to an existing browser", + rich_help_panel="Scheduler Configuration", + show_default=True, + ), + ] = str(config.CDP_CONNECT_EXISTING), ) -> SimpleNamespace: """MediaCrawler 命令行入口""" @@ -339,6 +373,7 @@ def main( enable_sub_comment = _to_bool(get_sub_comment) enable_headless = _to_bool(headless) enable_ip_proxy_value = _to_bool(enable_ip_proxy) + cdp_connect_existing_value = _to_bool(cdp_connect_existing) init_db_value = init_db.value if init_db else None # Parse specified_id and creator_id into lists @@ -365,6 +400,10 @@ def main( config.IP_PROXY_POOL_COUNT = ip_proxy_pool_count config.IP_PROXY_PROVIDER_NAME = ip_proxy_provider_name config.STATIC_PROXY_URL = static_proxy_url + config.INSTANCE_ID = instance_id + config.BROWSER_PROFILE_DIR = browser_profile_dir + config.CDP_DEBUG_PORT = cdp_debug_port + config.CDP_CONNECT_EXISTING = cdp_connect_existing_value # Set platform-specific ID lists for detail/creator mode if specified_id_list: @@ -415,6 +454,10 @@ def main( cookies=config.COOKIES, specified_id=specified_id, creator_id=creator_id, + instance_id=config.INSTANCE_ID, + browser_profile_dir=config.BROWSER_PROFILE_DIR, + cdp_debug_port=config.CDP_DEBUG_PORT, + cdp_connect_existing=config.CDP_CONNECT_EXISTING, ) command = typer.main.get_command(app) diff --git a/config/base_config.py b/config/base_config.py index 28a852e08..2d851b07c 100644 --- a/config/base_config.py +++ b/config/base_config.py @@ -28,7 +28,7 @@ LOGIN_TYPE = "qrcode" # qrcode or phone or cookie COOKIES = "" CRAWLER_TYPE = ( - "search" # Crawling type, search (keyword search) | detail (post details) | creator (creator homepage data) + "search" # Crawling type, search (keyword search) | detail (post details) | creator (creator homepage data) | login (login only) ) # Whether to enable IP proxy ENABLE_IP_PROXY = False @@ -62,6 +62,9 @@ # 如果端口被占用，系统会自动尝试下一个可用端口 CDP_DEBUG_PORT = 9222 +# 调度器多实例运行时使用的实例标识，仅用于日志、状态和产物隔离。 +INSTANCE_ID = "" + # 自定义浏览器路径（可选） # 如果为空，系统会自动检测 Chrome/Edge 的安装路径 # Windows 示例: "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe" @@ -95,6 +98,9 @@ # Browser file configuration cached by the user's browser USER_DATA_DIR = "%s_user_data_dir" # %s will be replaced by platform name +# 多实例运行时可显式指定浏览器用户数据目录；为空时沿用 USER_DATA_DIR。 +BROWSER_PROFILE_DIR = "" + # The number of pages to start crawling starts from the first page by default START_PAGE = 1 diff --git a/docs/.vitepress/config.mjs b/docs/.vitepress/config.mjs index 6863a419c..8af6d83ab 100644 --- a/docs/.vitepress/config.mjs +++ b/docs/.vitepress/config.mjs @@ -45,6 +45,7 @@ export default withMermaid(defineConfig({ items: [ {text: '基本使用', link: '/'}, {text: '项目架构文档', link: '/项目架构文档'}, + {text: '多实例调度器', link: '/多实例调度器使用指南'}, {text: '常见问题汇总', link: '/常见问题'}, {text: 'IP代理使用', link: '/代理使用'}, {text: '词云图使用', link: '/词云图使用配置'}, diff --git a/docs/index.md b/docs/index.md index 332d0c54b..50f2751bf 100644 --- a/docs/index.md +++ b/docs/index.md @@ -3,6 +3,7 @@ ## 项目文档 - [项目架构文档](项目架构文档.md) - 系统架构、模块设计、数据流向（含 Mermaid 图表） +- [多实例调度器使用指南](多实例调度器使用指南.md) - 多账号实例、任务队列、调度器 WebUI 和 API ## 推荐：使用 uv 管理依赖 @@ -47,6 +48,16 @@ uv run main.py --platform xhs --lt qrcode --type search --save_data_option db uv run main.py --help ``` +## 多实例调度器 + +如需管理多个独立账号实例，可启动 API 服务后访问调度器页面： + +```shell +uv run uvicorn api.main:app --port 8080 --reload +``` + +访问 `http://localhost:8080/scheduler` 创建实例、维护登录态、投递任务并查看日志和产物。详细说明见 [多实例调度器使用指南](多实例调度器使用指南.md)。 + ## 备选：Python 原生 venv（不推荐） > 如果爬取抖音或知乎，需要提前安装 Node.js，版本 `>= 16`。 ```shell diff --git "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" new file mode 100644 index 000000000..d0b83758e --- /dev/null +++ "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" @@ -0,0 +1,179 @@ +# 多实例调度器使用指南 + +本文档说明 MediaCrawler 多实例调度器的设计目标、运行方式、数据目录和 API。调度器用于在现有单实例爬虫基础上管理多个独立账号实例，每个实例拥有独立的登录态、浏览器 Profile、CDP 端口、默认爬取参数和任务队列。 + +## 1. 功能边界 + +多实例调度器解决以下问题： + +- 多个账号实例独立保存登录凭证和浏览器上下文，避免账号状态互相污染。 +- 每个实例维护自己的平台、登录方式、保存格式、CDP 端口和默认参数。 +- 调度器统一创建任务、启动爬虫子进程、停止任务、收集日志、扫描产物。 +- WebUI 提供实例创建、登录、任务投递、日志查看和产物查看。 + +调度器不改变平台爬虫的核心抓取逻辑。底层仍然通过 `main.py`、`cmd_arg/arg.py` 和各平台 `core.py` 执行搜索、详情、创作者和登录流程。 + +## 2. 架构图 + +```mermaid +flowchart TB + user["用户"] --> webui["/scheduler WebUI"] + webui --> api["/api/scheduler/*"] + + subgraph scheduler["调度器控制面"] + router["api/routers/scheduler.py"] + manager["api/scheduler/manager.py"] + store["api/scheduler/store.py
SQLite 状态库"] + end + + api --> router + router --> manager + manager --> store + + subgraph runtime["爬虫运行面"] + p1["实例 A 子进程
uv run python main.py"] + p2["实例 B 子进程
uv run python main.py"] + profile1["Profile A"] + profile2["Profile B"] + artifact1["产物 A"] + artifact2["产物 B"] + end + + manager --> p1 + manager --> p2 + p1 --> profile1 + p2 --> profile2 + p1 --> artifact1 + p2 --> artifact2 +``` + +## 3. 数据目录 + +调度器运行数据默认位于 `data/scheduler/`，该目录已被项目 `.gitignore` 忽略。 + +| 目录或文件 | 用途 | +| --- | --- | +| `data/scheduler/scheduler.db` | 实例、任务、日志和产物索引 | +| `data/scheduler/profiles/{instance_id}/` | 实例独立浏览器 Profile | +| `data/scheduler/artifacts/{instance_id}/{task_id}/` | 单个任务的抓取产物 | + +如果创建实例时手动填写浏览器 Profile 目录，调度器会使用该目录；相对路径会基于项目根目录解析。 + +## 4. 启动方式 + +启动现有 FastAPI 服务： + +```shell +uv run uvicorn api.main:app --port 8080 --reload +``` + +访问： + +- 单实例 WebUI：`http://localhost:8080` +- 多实例调度器：`http://localhost:8080/scheduler` +- API 文档：`http://localhost:8080/docs` + +## 5. 使用流程 + +1. 打开 `/scheduler`。 +2. 创建实例，选择平台、登录方式、保存格式，必要时指定 CDP 端口和默认参数。 +3. 点击实例行的“登录”，调度器会创建 `login` 任务，子进程只完成登录态维护，不进入抓取流程。 +4. 创建搜索、详情或创作者任务，调度器会把任务加入该实例队列。 +5. 查看任务状态、日志和产物列表。 + +实例处于 `running` 或 `stopping` 时不能修改核心配置，也不能删除实例。任务执行成功或被取消后，同实例队列中的下一个任务会自动启动；任务失败时实例进入 `error`，需要人工查看日志后再投递或启动后续任务。 + +## 6. 任务参数 + +实例的“默认参数 JSON”和任务的“任务参数 JSON”会合并，任务参数优先级更高。常用字段如下： + +| 字段 | 对应 CLI 参数 | 示例 | +| --- | --- | --- | +| `enable_comments` | `--get_comment` | `true` | +| `enable_sub_comments` | `--get_sub_comment` | `false` | +| `max_notes_count` | `--crawler_max_notes_count` | `20` | +| `max_comments_count` | `--max_comments_count_singlenotes` | `10` | +| `max_concurrency_num` | `--max_concurrency_num` | `1` | +| `cookies` | `--cookies` | `"a=b; c=d"` | +| `cdp_connect_existing` | `--cdp_connect_existing` | `false` | +| `enable_ip_proxy` | `--enable_ip_proxy` | `false` | +| `ip_proxy_pool_count` | `--ip_proxy_pool_count` | `2` | +| `ip_proxy_provider_name` | `--ip_proxy_provider_name` | `"static"` | +| `static_proxy_url` | `--static_proxy_url` | `"http://host:port"` | + +搜索任务的“目标”会映射为 `--keywords`，详情任务映射为 `--specified_id`，创作者任务映射为 `--creator_id`。 + +## 7. API 参考 + +### 7.1 实例 API + +| 方法 | 路径 | 说明 | +| --- | --- | --- | +| `GET` | `/api/scheduler/status` | 获取调度器统计 | +| `GET` | `/api/scheduler/instances` | 获取实例列表 | +| `POST` | `/api/scheduler/instances` | 创建实例 | +| `GET` | `/api/scheduler/instances/{instance_id}` | 获取实例详情 | +| `PATCH` | `/api/scheduler/instances/{instance_id}` | 修改实例配置 | +| `DELETE` | `/api/scheduler/instances/{instance_id}` | 删除空闲实例 | +| `POST` | `/api/scheduler/instances/{instance_id}/login` | 为实例创建登录任务 | + +创建实例示例： + +```json +{ + "name": "小红书账号 A", + "platform": "xhs", + "login_type": "qrcode", + "save_option": "jsonl", + "headless": false, + "default_params": { + "enable_comments": true, + "max_notes_count": 20 + } +} +``` + +### 7.2 任务 API + +| 方法 | 路径 | 说明 | +| --- | --- | --- | +| `GET` | `/api/scheduler/tasks` | 获取最近任务 | +| `POST` | `/api/scheduler/tasks` | 创建任务并在实例空闲时自动启动 | +| `GET` | `/api/scheduler/tasks/{task_id}` | 获取任务详情 | +| `POST` | `/api/scheduler/tasks/{task_id}/start` | 手动启动排队任务 | +| `POST` | `/api/scheduler/tasks/{task_id}/cancel` | 取消排队任务或停止运行任务 | +| `GET` | `/api/scheduler/tasks/{task_id}/logs` | 获取任务日志 | +| `GET` | `/api/scheduler/tasks/{task_id}/artifacts` | 获取任务产物索引 | + +创建任务示例： + +```json +{ + "instance_id": "实例 ID", + "crawler_type": "search", + "target_text": "编程副业,AI 工具", + "params": { + "enable_comments": true, + "max_comments_count": 10 + } +} +``` + +## 8. 与原单实例模式的关系 + +原单实例 WebUI 和 `/api/crawler/*` 接口仍然保留，适合临时手动运行一个爬虫任务。多实例调度器使用 `/api/scheduler/*`，每个任务都会启动独立子进程，并显式传入： + +- `--instance_id` +- `--browser_profile_dir` +- `--cdp_debug_port` +- `--cdp_connect_existing false` +- `--save_data_path data/scheduler/artifacts/{instance_id}/{task_id}` + +因此，多实例模式下账号登录态、浏览器状态和抓取产物都按实例或任务隔离。 + +## 9. 注意事项 + +- 本功能仍然应遵守项目非商业学习用途声明，不应扩大抓取规模或增加目标平台压力。 +- 多实例同时运行会同时启动多个浏览器上下文，需要预留足够 CPU、内存和端口。 +- 如果使用 Cookie 登录，可以把 `cookies` 放在实例默认参数中；如果使用二维码登录，建议先执行登录任务再投递抓取任务。 +- 如果任务失败，优先查看任务日志和对应实例的 `last_error` 字段。 diff --git "a/docs/\351\241\271\347\233\256\344\273\243\347\240\201\347\273\223\346\236\204.md" "b/docs/\351\241\271\347\233\256\344\273\243\347\240\201\347\273\223\346\236\204.md" index 6a5e2ed80..349e9fff1 100644 --- "a/docs/\351\241\271\347\233\256\344\273\243\347\240\201\347\273\223\346\236\204.md" +++ "b/docs/\351\241\271\347\233\256\344\273\243\347\240\201\347\273\223\346\236\204.md" @@ -11,6 +11,11 @@ MediaCrawler │ └── redis_cache.py # Redis缓存实现 ├── cmd_arg │ └── arg.py # 命令行参数定义 +├── api +│ ├── main.py # FastAPI 服务入口 +│ ├── routers # WebUI 与调度器 API 路由 +│ ├── scheduler # 多实例调度器状态库与进程管理 +│ └── webui # 单实例 WebUI 构建产物 ├── config │ ├── base_config.py # 基础配置 │ ├── db_config.py # 数据库配置 @@ -63,9 +68,11 @@ MediaCrawler │ ├── browser_launcher.py # 浏览器启动器 │ ├── cdp_browser.py # CDP浏览器控制 │ ├── crawler_util.py # 爬虫工具函数 +│ ├── profile.py # 浏览器 Profile 目录解析 │ ├── utils.py # 通用工具函数 │ └── ... +├── scheduler_webui # 多实例调度器静态管理页面源文件 ├── main.py # 程序入口, 支持 --init_db 参数来初始化数据库 ├── recv_sms.py # 短信转发HTTP SERVER接口 └── var.py # 全局上下文变量定义 -``` \ No newline at end of file +``` diff --git "a/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" "b/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" index 4a1c048c6..7592f2d86 100644 --- "a/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" +++ "b/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" @@ -142,6 +142,46 @@ flowchart LR --- +### 2.3 多实例调度器架构 + +在原有单实例爬虫链路之外，项目新增了一个独立调度器控制面。调度器不重写各平台抓取逻辑，而是通过 `uv run python main.py` 拉起多个子进程，并为每个实例注入独立的登录态目录、CDP 端口、默认参数和产物目录。 + +```mermaid +flowchart TB + scheduler_ui["/scheduler 调度器 WebUI"] --> scheduler_api["/api/scheduler/*"] + scheduler_api --> manager["SchedulerManager"] + manager --> sqlite["SchedulerStore
data/scheduler/scheduler.db"] + + manager --> task_a["实例 A 任务子进程"] + manager --> task_b["实例 B 任务子进程"] + + task_a --> profile_a["data/scheduler/profiles/A"] + task_b --> profile_b["data/scheduler/profiles/B"] + + task_a --> artifact_a["data/scheduler/artifacts/A/task"] + task_b --> artifact_b["data/scheduler/artifacts/B/task"] +``` + +调度器核心职责： + +- 维护实例配置：平台、登录方式、保存格式、CDP 端口、浏览器 Profile、默认参数。 +- 维护任务队列：搜索、详情、创作者和登录任务都绑定到具体实例。 +- 管理进程生命周期：实例空闲时启动队列任务，取消任务时终止对应子进程。 +- 采集运行状态：读取子进程 stdout，落库任务日志、退出码和错误信息。 +- 收集抓取产物：任务结束后扫描任务产物目录，记录文件类型、大小和记录数。 + +调度器相关代码位于： + +| 模块 | 说明 | +|------|------| +| `api/routers/scheduler.py` | 调度器 HTTP API | +| `api/scheduler/manager.py` | 任务队列、子进程和产物扫描 | +| `api/scheduler/store.py` | SQLite 状态库 | +| `scheduler_webui/` | 调度器管理页面 | +| `tools/profile.py` | 实例浏览器 Profile 路径解析 | + +--- + ## 3. 目录结构 ``` @@ -190,9 +230,16 @@ MediaCrawler/ │ ├── app_runner.py # 应用运行管理 │ ├── browser_launcher.py # 浏览器启动 │ ├── cdp_browser.py # CDP浏览器管理 +│ ├── profile.py # 多实例浏览器 Profile 路径解析 │ ├── crawler_util.py # 爬虫工具 │ └── async_file_writer.py # 异步文件写入 │ +├── api/ # WebUI API 与调度器 API +│ ├── routers/ # 路由层 +│ └── scheduler/ # 多实例调度器 +│ +├── scheduler_webui/ # 多实例调度器管理页面 +│ ├── model/ # 数据模型 │ └── m_{platform}.py # Pydantic模型 │ diff --git a/media_platform/bilibili/core.py b/media_platform/bilibili/core.py index 14b509977..bb3adbc59 100644 --- a/media_platform/bilibili/core.py +++ b/media_platform/bilibili/core.py @@ -23,7 +23,6 @@ # @Desc : Bilibili Crawler import asyncio -import os # import random # Removed as we now use fixed config.CRAWLER_MAX_SLEEP_SEC intervals from asyncio import Task from typing import Dict, List, Optional, Tuple, Union @@ -45,6 +44,7 @@ from store import bilibili as bilibili_store from tools import utils from tools.cdp_browser import CDPBrowserManager +from tools.profile import get_browser_profile_dir from var import crawler_type_var, source_keyword_var from .client import BilibiliClient @@ -129,6 +129,8 @@ async def start(self): continue else: await self.get_all_creator_details(config.BILI_CREATOR_ID_LIST) + elif config.CRAWLER_TYPE == "login": + utils.logger.info("[BilibiliCrawler.start] Login state is ready, skip crawling ...") else: pass utils.logger.info("[BilibiliCrawler.start] Bilibili Crawler finished ...") @@ -504,7 +506,7 @@ async def launch_browser( if config.SAVE_LOGIN_STATE: # feat issue #14 # we will save login state to avoid login every time - user_data_dir = os.path.join(os.getcwd(), "browser_data", config.USER_DATA_DIR % config.PLATFORM) # type: ignore + user_data_dir = get_browser_profile_dir() browser_context = await chromium.launch_persistent_context( user_data_dir=user_data_dir, accept_downloads=True, diff --git a/media_platform/douyin/core.py b/media_platform/douyin/core.py index c0e9372aa..30634acde 100644 --- a/media_platform/douyin/core.py +++ b/media_platform/douyin/core.py @@ -18,7 +18,6 @@ # 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 import asyncio -import os import random from asyncio import Task from typing import Any, Dict, List, Optional, Tuple @@ -37,6 +36,7 @@ from store import douyin as douyin_store from tools import utils from tools.cdp_browser import CDPBrowserManager +from tools.profile import get_browser_profile_dir from var import crawler_type_var, source_keyword_var from .client import DouYinClient @@ -121,6 +121,8 @@ async def start(self) -> None: elif config.CRAWLER_TYPE == "creator": # Get the information and comments of the specified creator await self.get_creators_and_videos() + elif config.CRAWLER_TYPE == "login": + utils.logger.info("[DouYinCrawler.start] Login state is ready, skip crawling ...") utils.logger.info("[DouYinCrawler.start] Douyin Crawler finished ...") @@ -337,7 +339,7 @@ async def launch_browser( ) -> BrowserContext: """Launch browser and create browser context""" if config.SAVE_LOGIN_STATE: - user_data_dir = os.path.join(os.getcwd(), "browser_data", config.USER_DATA_DIR % config.PLATFORM) # type: ignore + user_data_dir = get_browser_profile_dir() browser_context = await chromium.launch_persistent_context( user_data_dir=user_data_dir, accept_downloads=True, diff --git a/media_platform/kuaishou/core.py b/media_platform/kuaishou/core.py index 79773cb60..bbf36af32 100644 --- a/media_platform/kuaishou/core.py +++ b/media_platform/kuaishou/core.py @@ -19,7 +19,6 @@ import asyncio -import os # import random # Removed as we now use fixed config.CRAWLER_MAX_SLEEP_SEC intervals import time from asyncio import Task @@ -40,6 +39,7 @@ from store import kuaishou as kuaishou_store from tools import utils from tools.cdp_browser import CDPBrowserManager +from tools.profile import get_browser_profile_dir from var import comment_tasks_var, crawler_type_var, source_keyword_var from .client import KuaiShouClient @@ -122,6 +122,8 @@ async def start(self): elif config.CRAWLER_TYPE == "creator": # Get creator's information and their videos and comments await self.get_creators_and_videos() + elif config.CRAWLER_TYPE == "login": + utils.logger.info("[KuaishouCrawler.start] Login state is ready, skip crawling ...") else: pass @@ -338,9 +340,7 @@ async def launch_browser( "[KuaishouCrawler.launch_browser] Begin create browser context ..." ) if config.SAVE_LOGIN_STATE: - user_data_dir = os.path.join( - os.getcwd(), "browser_data", config.USER_DATA_DIR % config.PLATFORM - ) # type: ignore + user_data_dir = get_browser_profile_dir() browser_context = await chromium.launch_persistent_context( user_data_dir=user_data_dir, accept_downloads=True, diff --git a/media_platform/tieba/core.py b/media_platform/tieba/core.py index 940ab9f29..f9c94f838 100644 --- a/media_platform/tieba/core.py +++ b/media_platform/tieba/core.py @@ -19,7 +19,6 @@ import asyncio -import os from asyncio import Task from typing import Dict, List, Optional, Tuple @@ -38,6 +37,7 @@ from store import tieba as tieba_store from tools import utils from tools.cdp_browser import CDPBrowserManager +from tools.profile import get_browser_profile_dir from var import crawler_type_var, source_keyword_var from .client import BaiduTieBaClient @@ -140,6 +140,8 @@ async def start(self) -> None: elif config.CRAWLER_TYPE == "creator": # Get creator's information and their notes and comments await self.get_creators_and_notes() + elif config.CRAWLER_TYPE == "login": + utils.logger.info("[BaiduTieBaCrawler.start] Login state is ready, skip crawling ...") else: pass @@ -618,9 +620,7 @@ async def launch_browser( if config.SAVE_LOGIN_STATE: # feat issue #14 # we will save login state to avoid login every time - user_data_dir = os.path.join( - os.getcwd(), "browser_data", config.USER_DATA_DIR % config.PLATFORM - ) # type: ignore + user_data_dir = get_browser_profile_dir() browser_context = await chromium.launch_persistent_context( user_data_dir=user_data_dir, accept_downloads=True, diff --git a/media_platform/weibo/core.py b/media_platform/weibo/core.py index 4f34da510..8b5125621 100644 --- a/media_platform/weibo/core.py +++ b/media_platform/weibo/core.py @@ -23,7 +23,6 @@ # @Desc : Weibo crawler main workflow code import asyncio -import os # import random # Removed as we now use fixed config.CRAWLER_MAX_SLEEP_SEC intervals from asyncio import Task from typing import Dict, List, Optional, Tuple @@ -42,6 +41,7 @@ from store import weibo as weibo_store from tools import utils from tools.cdp_browser import CDPBrowserManager +from tools.profile import get_browser_profile_dir from var import crawler_type_var, source_keyword_var from .client import WeiboClient @@ -130,6 +130,8 @@ async def start(self): elif config.CRAWLER_TYPE == "creator": # Get creator's information and their notes and comments await self.get_creators_and_notes() + elif config.CRAWLER_TYPE == "login": + utils.logger.info("[WeiboCrawler.start] Login state is ready, skip crawling ...") else: pass utils.logger.info("[WeiboCrawler.start] Weibo Crawler finished ...") @@ -368,7 +370,7 @@ async def launch_browser( """Launch browser and create browser context""" utils.logger.info("[WeiboCrawler.launch_browser] Begin create browser context ...") if config.SAVE_LOGIN_STATE: - user_data_dir = os.path.join(os.getcwd(), "browser_data", config.USER_DATA_DIR % config.PLATFORM) # type: ignore + user_data_dir = get_browser_profile_dir() browser_context = await chromium.launch_persistent_context( user_data_dir=user_data_dir, accept_downloads=True, diff --git a/media_platform/xhs/core.py b/media_platform/xhs/core.py index 334fef395..c5447c28f 100644 --- a/media_platform/xhs/core.py +++ b/media_platform/xhs/core.py @@ -18,7 +18,6 @@ # 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 import asyncio -import os import random from asyncio import Task from typing import Dict, List, Optional @@ -39,6 +38,7 @@ from store import xhs as xhs_store from tools import utils from tools.cdp_browser import CDPBrowserManager +from tools.profile import get_browser_profile_dir from var import crawler_type_var, source_keyword_var from .client import XiaoHongShuClient @@ -121,6 +121,8 @@ async def start(self) -> None: elif config.CRAWLER_TYPE == "creator": # Get creator's information and their notes and comments await self.get_creators_and_notes() + elif config.CRAWLER_TYPE == "login": + utils.logger.info("[XiaoHongShuCrawler.start] Login state is ready, skip crawling ...") else: pass @@ -403,7 +405,7 @@ async def launch_browser( if config.SAVE_LOGIN_STATE: # feat issue #14 # we will save login state to avoid login every time - user_data_dir = os.path.join(os.getcwd(), "browser_data", config.USER_DATA_DIR % config.PLATFORM) # type: ignore + user_data_dir = get_browser_profile_dir() browser_context = await chromium.launch_persistent_context( user_data_dir=user_data_dir, accept_downloads=True, diff --git a/media_platform/zhihu/core.py b/media_platform/zhihu/core.py index 7b21e3b75..a55da4d00 100644 --- a/media_platform/zhihu/core.py +++ b/media_platform/zhihu/core.py @@ -20,7 +20,6 @@ # -*- coding: utf-8 -*- import asyncio -import os # import random # Removed as we now use fixed config.CRAWLER_MAX_SLEEP_SEC intervals from asyncio import Task from typing import Dict, List, Optional, Tuple, cast @@ -41,6 +40,7 @@ from store import zhihu as zhihu_store from tools import utils from tools.cdp_browser import CDPBrowserManager +from tools.profile import get_browser_profile_dir from var import crawler_type_var, source_keyword_var from .client import ZhiHuClient @@ -142,6 +142,8 @@ async def start(self) -> None: elif config.CRAWLER_TYPE == "creator": # Get creator's information and their notes and comments await self.get_creators_and_notes() + elif config.CRAWLER_TYPE == "login": + utils.logger.info("[ZhihuCrawler.start] Login state is ready, skip crawling ...") else: pass @@ -435,9 +437,7 @@ async def launch_browser( if config.SAVE_LOGIN_STATE: # feat issue #14 # we will save login state to avoid login every time - user_data_dir = os.path.join( - os.getcwd(), "browser_data", config.USER_DATA_DIR % config.PLATFORM - ) # type: ignore + user_data_dir = get_browser_profile_dir() browser_context = await chromium.launch_persistent_context( user_data_dir=user_data_dir, accept_downloads=True, diff --git a/scheduler_webui/app.js b/scheduler_webui/app.js new file mode 100644 index 000000000..7ef2baf0a --- /dev/null +++ b/scheduler_webui/app.js @@ -0,0 +1,227 @@ +const api = { + async get(path) { + const res = await fetch(path); + if (!res.ok) throw new Error(await errorText(res)); + return res.json(); + }, + async post(path, body) { + const res = await fetch(path, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + if (!res.ok) throw new Error(await errorText(res)); + return res.json(); + }, +}; + +let state = { + instances: [], + tasks: [], + selectedTaskId: "", +}; + +const els = { + summary: document.querySelector("#summary"), + refreshBtn: document.querySelector("#refreshBtn"), + instanceForm: document.querySelector("#instanceForm"), + taskForm: document.querySelector("#taskForm"), + instancesBody: document.querySelector("#instancesBody"), + tasksBody: document.querySelector("#tasksBody"), + taskInstanceSelect: document.querySelector("#taskInstanceSelect"), + instanceHint: document.querySelector("#instanceHint"), + taskHint: document.querySelector("#taskHint"), + logsBox: document.querySelector("#logsBox"), + artifactsList: document.querySelector("#artifactsList"), +}; + +function statusPill(value) { + return `${escapeHtml(value)}`; +} + +function escapeHtml(value) { + return String(value ?? "") + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} + +function parseJson(value, fallback = {}) { + const text = value.trim(); + if (!text) return fallback; + return JSON.parse(text); +} + +async function errorText(res) { + try { + const data = await res.json(); + return data.detail || res.statusText; + } catch { + return res.statusText; + } +} + +async function refreshAll() { + const [status, instances, tasks] = await Promise.all([ + api.get("/api/scheduler/status"), + api.get("/api/scheduler/instances"), + api.get("/api/scheduler/tasks?limit=100"), + ]); + state.instances = instances; + state.tasks = tasks; + renderStatus(status); + renderInstances(); + renderTasks(); + renderTaskSelect(); + if (state.selectedTaskId) await loadTaskDetail(state.selectedTaskId); +} + +function renderStatus(status) { + els.summary.textContent = + `实例 ${status.instances_total} 个，运行中 ${status.running_instances} 个，` + + `排队任务 ${status.queued_tasks} 个，运行任务 ${status.running_tasks} 个`; +} + +function renderInstances() { + els.instanceHint.textContent = `${state.instances.length} 个实例`; + els.instancesBody.innerHTML = state.instances + .map((item) => { + const canRun = !["running", "stopping"].includes(item.status); + return ` + + ${escapeHtml(item.name)}
${escapeHtml(item.id)} + ${escapeHtml(item.platform)} + ${statusPill(item.status)} + ${escapeHtml(item.cdp_debug_port)} + ${escapeHtml(item.current_task_id || "-")} + +
+ +
+ + + `; + }) + .join(""); +} + +function renderTaskSelect() { + els.taskInstanceSelect.innerHTML = state.instances + .map((item) => ``) + .join(""); +} + +function renderTasks() { + els.taskHint.textContent = `${state.tasks.length} 个最近任务`; + els.tasksBody.innerHTML = state.tasks + .map((item) => { + const instance = state.instances.find((it) => it.id === item.instance_id); + const canCancel = ["queued", "running"].includes(item.status); + return ` + + + ${escapeHtml(instance ? instance.name : item.instance_id)} + ${escapeHtml(item.crawler_type)} + ${statusPill(item.status)} + ${escapeHtml(item.target_text || "-")} + +
+ + +
+ + + `; + }) + .join(""); +} + +async function loadTaskDetail(taskId) { + const [logs, artifacts] = await Promise.all([ + api.get(`/api/scheduler/tasks/${taskId}/logs?limit=300`), + api.get(`/api/scheduler/tasks/${taskId}/artifacts`), + ]); + els.logsBox.textContent = logs.length + ? logs.map((log) => `[${log.timestamp}] [${log.level}] ${log.message}`).join("\n") + : "暂无日志。"; + els.artifactsList.innerHTML = artifacts.length + ? artifacts + .map((item) => { + const sizeKb = (item.size / 1024).toFixed(1); + const count = item.record_count == null ? "" : `，${item.record_count} 条`; + return `
  • ${escapeHtml(item.type)} ${escapeHtml(sizeKb)} KB${count}
    ${escapeHtml(item.path)}
    • `; + }) + .join("") + : "
  • 暂无产物。
    • "; +} + +els.refreshBtn.addEventListener("click", () => { + refreshAll().catch((err) => alert(err.message)); +}); + +els.instanceForm.addEventListener("submit", async (event) => { + event.preventDefault(); + const data = new FormData(event.currentTarget); + const body = { + name: data.get("name"), + platform: data.get("platform"), + login_type: data.get("login_type"), + save_option: data.get("save_option"), + headless: data.get("headless") === "on", + browser_profile_dir: data.get("browser_profile_dir") || "", + default_params: parseJson(String(data.get("default_params") || ""), {}), + }; + const port = String(data.get("cdp_debug_port") || "").trim(); + if (port) body.cdp_debug_port = Number(port); + await api.post("/api/scheduler/instances", body); + event.currentTarget.reset(); + await refreshAll(); +}); + +els.taskForm.addEventListener("submit", async (event) => { + event.preventDefault(); + const data = new FormData(event.currentTarget); + await api.post("/api/scheduler/tasks", { + instance_id: data.get("instance_id"), + crawler_type: data.get("crawler_type"), + target_text: data.get("target_text") || "", + params: parseJson(String(data.get("params") || ""), {}), + }); + event.currentTarget.reset(); + await refreshAll(); +}); + +document.addEventListener("click", async (event) => { + const target = event.target.closest("button[data-action]"); + if (!target) return; + const action = target.dataset.action; + const id = target.dataset.id; + try { + if (action === "login") { + const task = await api.post(`/api/scheduler/instances/${id}/login`, {}); + state.selectedTaskId = task.id; + await refreshAll(); + } + if (action === "select-task") { + state.selectedTaskId = id; + await loadTaskDetail(id); + } + if (action === "cancel-task") { + await api.post(`/api/scheduler/tasks/${id}/cancel`, {}); + state.selectedTaskId = id; + await refreshAll(); + } + } catch (err) { + alert(err.message); + } +}); + +refreshAll().catch((err) => { + els.summary.textContent = `读取失败：${err.message}`; +}); + +setInterval(() => { + refreshAll().catch(() => {}); +}, 5000); diff --git a/scheduler_webui/index.html b/scheduler_webui/index.html new file mode 100644 index 000000000..8d174e3f4 --- /dev/null +++ b/scheduler_webui/index.html @@ -0,0 +1,164 @@ + + + + + + MediaCrawler 多实例调度器 + + + +
    +
    +
    +

    MediaCrawler 多实例调度器

    +

    正在读取调度状态...

    +
    + +
    + +
    + + +
    +
    +
    +

    实例

    + +
    +
    + + + + + + + + + + + + +
    名称平台状态端口当前任务操作
    +
    +
    + +
    +

    新建任务

    +
    + + + + + +
    +
    + +
    +
    +

    任务

    + +
    +
    + + + + + + + + + + + + +
    任务实例类型状态目标操作
    +
    +
    + +
    +
    +

    日志

    + 
    选择一个任务查看日志。
    +
    +
    +

    产物

    +
      +
      +
      +
      +
      +
      + + + diff --git a/scheduler_webui/styles.css b/scheduler_webui/styles.css new file mode 100644 index 000000000..fac30cf57 --- /dev/null +++ b/scheduler_webui/styles.css @@ -0,0 +1,281 @@ +:root { + color-scheme: light; + --bg: #f6f7f4; + --panel: #ffffff; + --text: #202421; + --muted: #68736d; + --line: #d9ded6; + --teal: #0f766e; + --teal-strong: #0b5f59; + --amber: #b45309; + --red: #b42318; + --green: #15803d; + --blue: #1d4ed8; +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + background: var(--bg); + color: var(--text); + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; +} + +button, +input, +select, +textarea { + font: inherit; +} + +button { + min-height: 36px; + border: 1px solid var(--teal); + border-radius: 6px; + background: var(--teal); + color: #fff; + padding: 0 14px; + cursor: pointer; +} + +button:hover { + background: var(--teal-strong); +} + +button.secondary { + background: #fff; + color: var(--teal); +} + +button.danger { + border-color: var(--red); + background: var(--red); +} + +button:disabled { + cursor: not-allowed; + opacity: 0.55; +} + +input, +select, +textarea { + width: 100%; + border: 1px solid var(--line); + border-radius: 6px; + background: #fff; + color: var(--text); + padding: 9px 10px; +} + +textarea { + resize: vertical; +} + +label { + display: grid; + gap: 6px; + color: var(--muted); + font-size: 13px; +} + +table { + width: 100%; + border-collapse: collapse; + font-size: 14px; +} + +th, +td { + border-bottom: 1px solid var(--line); + padding: 10px; + text-align: left; + vertical-align: middle; +} + +th { + color: var(--muted); + font-size: 12px; + font-weight: 700; +} + +.app-shell { + width: min(1440px, 100%); + margin: 0 auto; + padding: 22px; +} + +.topbar { + display: flex; + align-items: center; + justify-content: space-between; + gap: 16px; + margin-bottom: 18px; +} + +.topbar h1 { + margin: 0; + font-size: 24px; + line-height: 1.2; +} + +.topbar p { + margin: 6px 0 0; + color: var(--muted); +} + +.layout { + display: grid; + grid-template-columns: 340px minmax(0, 1fr); + gap: 16px; + align-items: start; +} + +.panel { + border: 1px solid var(--line); + border-radius: 8px; + background: var(--panel); + padding: 16px; +} + +.panel h2 { + margin: 0 0 12px; + font-size: 16px; +} + +.stack { + display: grid; + gap: 14px; +} + +.sidebar { + position: sticky; + top: 16px; +} + +.section-title { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; +} + +.section-title span { + color: var(--muted); + font-size: 13px; +} + +.table-wrap { + overflow-x: auto; +} + +.checkline { + display: flex; + align-items: center; + gap: 8px; +} + +.checkline input { + width: auto; +} + +.task-grid { + display: grid; + grid-template-columns: repeat(2, minmax(180px, 1fr)); + gap: 12px; +} + +.task-grid .wide { + grid-column: 1 / -1; +} + +.detail-grid { + display: grid; + grid-template-columns: minmax(0, 1.4fr) minmax(260px, 0.6fr); + gap: 16px; +} + +.log-panel pre { + min-height: 260px; + max-height: 420px; + overflow: auto; + margin: 0; + border: 1px solid var(--line); + border-radius: 6px; + background: #171a18; + color: #e9f0ec; + padding: 12px; + white-space: pre-wrap; + word-break: break-word; +} + +.artifact-panel ul { + display: grid; + gap: 8px; + margin: 0; + padding: 0; + list-style: none; +} + +.artifact-panel li { + border: 1px solid var(--line); + border-radius: 6px; + padding: 10px; + word-break: break-all; +} + +.pill { + display: inline-flex; + align-items: center; + min-height: 24px; + border-radius: 999px; + padding: 0 9px; + background: #eef2f0; + color: var(--muted); + font-size: 12px; + font-weight: 700; +} + +.pill.running { + background: #dbeafe; + color: var(--blue); +} + +.pill.succeeded, +.pill.idle { + background: #dcfce7; + color: var(--green); +} + +.pill.failed, +.pill.error { + background: #fee2e2; + color: var(--red); +} + +.pill.queued, +.pill.stopping { + background: #fef3c7; + color: var(--amber); +} + +.actions { + display: flex; + flex-wrap: wrap; + gap: 8px; +} + +@media (max-width: 980px) { + .layout, + .detail-grid, + .task-grid { + grid-template-columns: 1fr; + } + + .sidebar { + position: static; + } +} diff --git a/tests/test_cmd_arg_scheduler.py b/tests/test_cmd_arg_scheduler.py new file mode 100644 index 000000000..ca79e227b --- /dev/null +++ b/tests/test_cmd_arg_scheduler.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +import pytest + +import config +from cmd_arg import parse_cmd + + +@pytest.mark.asyncio +async def test_cmd_arg_sets_scheduler_runtime_fields(): + original = { + "CRAWLER_TYPE": config.CRAWLER_TYPE, + "INSTANCE_ID": config.INSTANCE_ID, + "BROWSER_PROFILE_DIR": config.BROWSER_PROFILE_DIR, + "CDP_DEBUG_PORT": config.CDP_DEBUG_PORT, + "CDP_CONNECT_EXISTING": config.CDP_CONNECT_EXISTING, + } + + try: + result = await parse_cmd( + [ + "--platform", + "xhs", + "--type", + "login", + "--instance_id", + "inst-a", + "--browser_profile_dir", + "data/scheduler/profiles/inst-a", + "--cdp_debug_port", + "9233", + "--cdp_connect_existing", + "false", + ] + ) + + assert result.type == "login" + assert config.INSTANCE_ID == "inst-a" + assert config.BROWSER_PROFILE_DIR == "data/scheduler/profiles/inst-a" + assert config.CDP_DEBUG_PORT == 9233 + assert config.CDP_CONNECT_EXISTING is False + finally: + for key, value in original.items(): + setattr(config, key, value) diff --git a/tests/test_profile.py b/tests/test_profile.py new file mode 100644 index 000000000..b79d7aa14 --- /dev/null +++ b/tests/test_profile.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +import config +from tools.profile import get_browser_profile_dir + + +def test_get_browser_profile_dir_uses_explicit_path(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(config, "BROWSER_PROFILE_DIR", "data/profiles/inst-a") + + assert get_browser_profile_dir() == str(tmp_path / "data" / "profiles" / "inst-a") + + +def test_get_browser_profile_dir_uses_platform_default(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(config, "BROWSER_PROFILE_DIR", "") + monkeypatch.setattr(config, "PLATFORM", "xhs") + + assert get_browser_profile_dir() == str(tmp_path / "browser_data" / "xhs_user_data_dir") + assert get_browser_profile_dir(cdp=True) == str(tmp_path / "browser_data" / "cdp_xhs_user_data_dir") diff --git a/tests/test_scheduler_api.py b/tests/test_scheduler_api.py new file mode 100644 index 000000000..2d7caeed7 --- /dev/null +++ b/tests/test_scheduler_api.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +from fastapi.testclient import TestClient + +from api.main import app +from api.routers import scheduler as scheduler_router +from api.scheduler.manager import SchedulerManager +from api.scheduler.store import SchedulerStore + + +def test_scheduler_api_creates_instance_and_task(monkeypatch, tmp_path): + manager = SchedulerManager(store=SchedulerStore(tmp_path / "scheduler.db"), project_root=tmp_path) + + async def noop_start(task): + return None + + monkeypatch.setattr(manager, "_start_task_locked", noop_start) + monkeypatch.setattr(scheduler_router, "scheduler_manager", manager) + + client = TestClient(app) + instance_response = client.post( + "/api/scheduler/instances", + json={ + "name": "小红书账号 A", + "platform": "xhs", + "login_type": "qrcode", + "save_option": "jsonl", + }, + ) + assert instance_response.status_code == 200 + instance = instance_response.json() + assert instance["status"] == "idle" + + task_response = client.post( + "/api/scheduler/tasks", + json={ + "instance_id": instance["id"], + "crawler_type": "search", + "target_text": "编程副业", + "params": {"max_notes_count": 5}, + }, + ) + assert task_response.status_code == 200 + task = task_response.json() + assert task["instance_id"] == instance["id"] + assert task["status"] == "queued" + + logs_response = client.get(f"/api/scheduler/tasks/{task['id']}/logs") + assert logs_response.status_code == 200 diff --git a/tests/test_scheduler_manager.py b/tests/test_scheduler_manager.py new file mode 100644 index 000000000..2fd5feccc --- /dev/null +++ b/tests/test_scheduler_manager.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +from api.scheduler.manager import SchedulerManager +from api.scheduler.schemas import InstanceCreateRequest +from api.scheduler.store import SchedulerStore + + +def test_scheduler_manager_builds_isolated_command(tmp_path): + store = SchedulerStore(tmp_path / "scheduler.db") + manager = SchedulerManager(store=store, project_root=tmp_path) + instance = manager.create_instance( + InstanceCreateRequest( + name="抖音账号 A", + platform="dy", + browser_profile_dir=str(tmp_path / "profiles" / "dy-a"), + default_params={"enable_comments": False, "max_notes_count": 8}, + ) + ) + task = store.create_task( + { + "instance_id": instance["id"], + "crawler_type": "search", + "target_text": "AI 工具", + "params": {"enable_sub_comments": True}, + }, + str(tmp_path / "artifacts" / "task-a"), + ) + + cmd = manager._build_command(instance, task) + + assert cmd[:4] == ["uv", "run", "python", "main.py"] + assert cmd[cmd.index("--platform") + 1] == "dy" + assert cmd[cmd.index("--type") + 1] == "search" + assert cmd[cmd.index("--keywords") + 1] == "AI 工具" + assert cmd[cmd.index("--browser_profile_dir") + 1] == str(tmp_path / "profiles" / "dy-a") + assert cmd[cmd.index("--cdp_connect_existing") + 1] == "false" + assert cmd[cmd.index("--get_comment") + 1] == "false" + assert cmd[cmd.index("--get_sub_comment") + 1] == "true" + assert cmd[cmd.index("--crawler_max_notes_count") + 1] == "8" + + +def test_scheduler_manager_scans_artifacts(tmp_path): + manager = SchedulerManager(store=SchedulerStore(tmp_path / "scheduler.db"), project_root=tmp_path) + artifact_dir = tmp_path / "artifacts" + artifact_dir.mkdir() + artifact_file = artifact_dir / "data.jsonl" + artifact_file.write_text('{"id": 1}\n{"id": 2}\n', encoding="utf-8") + + artifacts = manager._scan_artifacts(artifact_dir) + + assert artifacts[0]["type"] == "jsonl" + assert artifacts[0]["record_count"] == 2 diff --git a/tests/test_scheduler_store.py b/tests/test_scheduler_store.py new file mode 100644 index 000000000..897266613 --- /dev/null +++ b/tests/test_scheduler_store.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- + +from api.scheduler.store import SchedulerStore + + +def test_scheduler_store_crud(tmp_path): + store = SchedulerStore(tmp_path / "scheduler.db") + instance = store.create_instance( + { + "name": "小红书账号 A", + "platform": "xhs", + "login_type": "qrcode", + "headless": False, + "save_option": "jsonl", + "default_params": {"enable_comments": True}, + }, + str(tmp_path / "profile"), + 9222, + instance_id="inst-a", + ) + + assert instance["id"] == "inst-a" + assert instance["default_params"] == {"enable_comments": True} + + task = store.create_task( + { + "instance_id": "inst-a", + "crawler_type": "search", + "target_text": "编程副业", + "params": {"max_notes_count": 5}, + }, + str(tmp_path / "artifacts" / "task-a"), + ) + store.append_log(task["id"], "started", "info") + store.replace_artifacts( + task["id"], + [ + { + "path": str(tmp_path / "artifacts" / "task-a" / "xhs.jsonl"), + "type": "jsonl", + "size": 12, + "modified_at": 1.0, + "record_count": 2, + } + ], + ) + + assert store.get_next_queued_task("inst-a")["id"] == task["id"] + assert store.list_logs(task["id"])[0]["message"] == "started" + assert store.list_artifacts(task["id"])[0]["record_count"] == 2 + assert store.scheduler_counts()["queued_tasks"] == 1 diff --git a/tools/cdp_browser.py b/tools/cdp_browser.py index bc5a76c12..8be01c0ec 100644 --- a/tools/cdp_browser.py +++ b/tools/cdp_browser.py @@ -30,6 +30,7 @@ import config from tools.browser_launcher import BrowserLauncher from tools import utils +from tools.profile import get_browser_profile_dir class CDPBrowserManager: @@ -254,11 +255,7 @@ async def _launch_browser(self, browser_path: str, headless: bool): # Set user data directory (if save login state is enabled) user_data_dir = None if config.SAVE_LOGIN_STATE: - user_data_dir = os.path.join( - os.getcwd(), - "browser_data", - f"cdp_{config.USER_DATA_DIR % config.PLATFORM}", - ) + user_data_dir = get_browser_profile_dir(cdp=True) os.makedirs(user_data_dir, exist_ok=True) utils.logger.info(f"[CDPBrowserManager] User data directory: {user_data_dir}") diff --git a/tools/profile.py b/tools/profile.py new file mode 100644 index 000000000..ce7d19f29 --- /dev/null +++ b/tools/profile.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +import os +from pathlib import Path +from typing import Optional + +import config + + +def get_browser_profile_dir(platform: Optional[str] = None, *, cdp: bool = False) -> str: + """Return browser profile directory for the current crawler run.""" + if config.BROWSER_PROFILE_DIR: + path = Path(config.BROWSER_PROFILE_DIR) + return str(path if path.is_absolute() else Path(os.getcwd()) / path) + + platform_name = platform or config.PLATFORM + dirname = config.USER_DATA_DIR % platform_name + if cdp: + dirname = f"cdp_{dirname}" + return str(Path(os.getcwd()) / "browser_data" / dirname) From a842293925fa88909cc99dae40280e8c155f012b Mon Sep 17 00:00:00 2001 From: Raffaello Date: Fri, 26 Jun 2026 21:15:56 +0800 Subject: [PATCH 02/17] feat: add content metric filters --- README.md | 5 + api/scheduler/manager.py | 3 + api/schemas/crawler.py | 15 +- api/services/crawler_manager.py | 4 + cmd_arg/arg.py | 15 ++ config/base_config.py | 4 + docs/index.md | 4 + ...77\347\224\250\346\214\207\345\215\227.md" | 59 +++++ ...77\347\224\250\346\214\207\345\215\227.md" | 9 +- ...66\346\236\204\346\226\207\346\241\243.md" | 11 +- media_platform/bilibili/core.py | 70 +++--- media_platform/douyin/core.py | 56 +++-- media_platform/kuaishou/core.py | 53 +++-- media_platform/tieba/core.py | 32 ++- media_platform/weibo/core.py | 41 ++-- media_platform/xhs/core.py | 59 +++-- media_platform/zhihu/core.py | 26 ++- tests/test_api_limits.py | 28 ++- tests/test_cmd_arg_scheduler.py | 5 + tests/test_content_filter.py | 68 ++++++ tests/test_scheduler_manager.py | 3 +- tools/content_filter.py | 218 ++++++++++++++++++ 22 files changed, 667 insertions(+), 121 deletions(-) create mode 100644 "docs/\345\206\205\345\256\271\350\277\207\346\273\244\344\275\277\347\224\250\346\214\207\345\215\227.md" create mode 100644 tests/test_content_filter.py create mode 100644 tools/content_filter.py diff --git a/README.md b/README.md index ac1ee4dd0..8bb86a648 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,9 @@ uv run playwright install # 从配置文件中读取关键词搜索相关的帖子并爬取帖子信息与评论 uv run main.py --platform xhs --lt qrcode --type search +# 按互动指标过滤爬取目标，例如只保存点赞数不低于 1000 的内容 +uv run main.py --platform xhs --lt qrcode --type search --content_filters '{"liked_count":{"min":1000}}' + # 从配置文件中读取指定的帖子ID列表获取指定帖子的信息与评论信息 uv run main.py --platform xhs --lt qrcode --type detail @@ -173,6 +176,8 @@ uv run main.py --platform xhs --lt qrcode --type login uv run main.py --help ``` +内容过滤支持不同平台的点赞、收藏、转发、评论等字段，详见 [内容过滤使用指南](docs/内容过滤使用指南.md)。 +
      🖥️ WebUI 可视化操作界面 diff --git a/api/scheduler/manager.py b/api/scheduler/manager.py index c2a61c864..c3d7c932f 100644 --- a/api/scheduler/manager.py +++ b/api/scheduler/manager.py @@ -241,6 +241,7 @@ def _build_command(self, instance: dict[str, Any], task: dict[str, Any]) -> list "enable_sub_comments": "--get_sub_comment", "max_notes_count": "--crawler_max_notes_count", "max_comments_count": "--max_comments_count_singlenotes", + "content_filters": "--content_filters", "max_concurrency_num": "--max_concurrency_num", "cookies": "--cookies", "enable_ip_proxy": "--enable_ip_proxy", @@ -254,6 +255,8 @@ def _build_command(self, instance: dict[str, Any], task: dict[str, Any]) -> list value = params[key] if isinstance(value, bool): value = self._bool_arg(value) + elif isinstance(value, (dict, list)): + value = json.dumps(value, ensure_ascii=False) cmd.extend([flag, str(value)]) return cmd diff --git a/api/schemas/crawler.py b/api/schemas/crawler.py index cc3ccc6d3..41d98d913 100644 --- a/api/schemas/crawler.py +++ b/api/schemas/crawler.py @@ -17,8 +17,10 @@ # 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 from enum import Enum -from typing import Optional, Literal -from pydantic import BaseModel, Field +from typing import Any, Optional, Literal +from pydantic import BaseModel, Field, model_validator + +from tools.content_filter import ContentFilterError, normalize_content_filters MAX_API_LIMIT_COUNT = 10000 @@ -77,6 +79,15 @@ class CrawlerStartRequest(BaseModel): headless: bool = False max_notes_count: Optional[int] = Field(default=None, ge=1, le=MAX_API_LIMIT_COUNT) max_comments_count: Optional[int] = Field(default=None, ge=1, le=MAX_API_LIMIT_COUNT) + content_filters: dict[str, Any] = Field(default_factory=dict) + + @model_validator(mode="after") + def validate_content_filters(self): + try: + self.content_filters = normalize_content_filters(self.platform.value, self.content_filters) + except ContentFilterError as exc: + raise ValueError(str(exc)) from exc + return self class CrawlerStatusResponse(BaseModel): diff --git a/api/services/crawler_manager.py b/api/services/crawler_manager.py index 9af954b3d..fdc130e86 100644 --- a/api/services/crawler_manager.py +++ b/api/services/crawler_manager.py @@ -17,6 +17,7 @@ # 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 import asyncio +import json import subprocess import signal import os @@ -231,6 +232,9 @@ def _build_command(self, config: CrawlerStartRequest) -> list: if config.max_comments_count is not None: cmd.extend(["--max_comments_count_singlenotes", str(config.max_comments_count)]) + if config.content_filters: + cmd.extend(["--content_filters", json.dumps(config.content_filters, ensure_ascii=False)]) + if config.cookies: cmd.extend(["--cookies", config.cookies]) diff --git a/cmd_arg/arg.py b/cmd_arg/arg.py index 8bfc63578..5595b819b 100644 --- a/cmd_arg/arg.py +++ b/cmd_arg/arg.py @@ -31,6 +31,7 @@ from typing_extensions import Annotated import config +from tools.content_filter import ContentFilterError, normalize_content_filters from tools.utils import str2bool @@ -284,6 +285,14 @@ def main( rich_help_panel="Basic Configuration", ), ] = config.CRAWLER_MAX_NOTES_COUNT, + content_filters: Annotated[ + str, + typer.Option( + "--content_filters", + help='Content metric filters JSON, for example {"liked_count":{"min":1000}}', + rich_help_panel="Basic Configuration", + ), + ] = "", max_concurrency_num: Annotated[ int, typer.Option( @@ -375,6 +384,10 @@ def main( enable_ip_proxy_value = _to_bool(enable_ip_proxy) cdp_connect_existing_value = _to_bool(cdp_connect_existing) init_db_value = init_db.value if init_db else None + try: + content_filters_value = normalize_content_filters(platform.value, content_filters or config.CONTENT_FILTERS) + except ContentFilterError as exc: + raise typer.BadParameter(str(exc)) from exc # Parse specified_id and creator_id into lists specified_id_list = [id.strip() for id in specified_id.split(",") if id.strip()] if specified_id else [] @@ -394,6 +407,7 @@ def main( config.COOKIES = cookies config.CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES = max_comments_count_singlenotes config.CRAWLER_MAX_NOTES_COUNT = crawler_max_notes_count + config.CONTENT_FILTERS = content_filters_value config.MAX_CONCURRENCY_NUM = max_concurrency_num config.SAVE_DATA_PATH = save_data_path config.ENABLE_IP_PROXY = enable_ip_proxy_value @@ -454,6 +468,7 @@ def main( cookies=config.COOKIES, specified_id=specified_id, creator_id=creator_id, + content_filters=config.CONTENT_FILTERS, instance_id=config.INSTANCE_ID, browser_profile_dir=config.BROWSER_PROFILE_DIR, cdp_debug_port=config.CDP_DEBUG_PORT, diff --git a/config/base_config.py b/config/base_config.py index 2d851b07c..6fecb649a 100644 --- a/config/base_config.py +++ b/config/base_config.py @@ -107,6 +107,10 @@ # Control the number of crawled videos/posts CRAWLER_MAX_NOTES_COUNT = 15 +# Content metric filters applied before storing media/comments. +# Example: {"liked_count": {"min": 1000}, "comment_count": {"max": 500}} +CONTENT_FILTERS = {} + # Controlling the number of concurrent crawlers MAX_CONCURRENCY_NUM = 1 diff --git a/docs/index.md b/docs/index.md index 50f2751bf..556466890 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,6 +4,7 @@ - [项目架构文档](项目架构文档.md) - 系统架构、模块设计、数据流向（含 Mermaid 图表） - [多实例调度器使用指南](多实例调度器使用指南.md) - 多账号实例、任务队列、调度器 WebUI 和 API +- [内容过滤使用指南](内容过滤使用指南.md) - 按点赞、收藏、转发、评论等互动指标筛选爬取目标 ## 推荐：使用 uv 管理依赖 @@ -35,6 +36,9 @@ uv run playwright install # 从配置中读取关键词搜索并爬取帖子与评论 uv run main.py --platform xhs --lt qrcode --type search +# 只保存点赞数不低于 1000 的搜索结果 +uv run main.py --platform xhs --lt qrcode --type search --content_filters '{"liked_count":{"min":1000}}' + # 从配置中读取指定帖子ID列表并爬取帖子与评论 uv run main.py --platform xhs --lt qrcode --type detail diff --git "a/docs/\345\206\205\345\256\271\350\277\207\346\273\244\344\275\277\347\224\250\346\214\207\345\215\227.md" "b/docs/\345\206\205\345\256\271\350\277\207\346\273\244\344\275\277\347\224\250\346\214\207\345\215\227.md" new file mode 100644 index 000000000..4bc6550ef --- /dev/null +++ "b/docs/\345\206\205\345\256\271\350\277\207\346\273\244\344\275\277\347\224\250\346\214\207\345\215\227.md" @@ -0,0 +1,59 @@ +# 内容过滤使用指南 + +内容过滤用于在爬取候选内容拿到详情后、保存内容和抓取评论/媒体前，按互动指标筛掉不需要的目标。它适用于关键词搜索、指定内容和创作者主页三种爬取模式。 + +## 1. 配置格式 + +CLI 参数名为 `--content_filters`，值是 JSON 字符串： + +```shell +uv run python main.py --platform xhs --type search --keywords "AI 工具" --content_filters '{"liked_count":{"min":1000},"comment_count":{"max":500}}' +``` + +过滤条件支持 `min`、`max`，边界包含等于值。多个字段同时配置时，内容必须全部满足才会保存。数值可以写数字，也可以写带单位的字符串，例如 `"1万"`、`"2.5万"`、`"1亿"`。 + +也可以在 `config/base_config.py` 中设置默认值： + +```python +CONTENT_FILTERS = { + "liked_count": {"min": 1000}, + "comment_count": {"min": 20, "max": 500}, +} +``` + +空配置 `{}` 表示不过滤，行为与原来一致。 + +## 2. 平台字段 + +| 平台 | 可用字段 | +| --- | --- | +| 小红书 `xhs` | `liked_count`、`collected_count`、`comment_count`、`share_count` | +| 抖音 `dy` | `liked_count`、`collected_count`、`comment_count`、`share_count` | +| 快手 `ks` | `liked_count`、`view_count` | +| B 站 `bili` | `liked_count`、`disliked_count`、`play_count`、`favorite_count`、`share_count`、`coin_count`、`danmaku_count`、`comment_count` | +| 微博 `wb` | `liked_count`、`comment_count`、`share_count` | +| 贴吧 `tieba` | `reply_count`、`reply_page_count` | +| 知乎 `zhihu` | `voteup_count`、`comment_count` | + +字段不支持时会在启动阶段报错，不会静默忽略。部分历史字段名保留了兼容别名，例如快手 `viewd_count`、微博 `comments_count`、B 站 `video_favorite_count`。 + +## 3. 调度器任务参数 + +多实例调度器的“默认参数 JSON”和“任务参数 JSON”都支持 `content_filters`： + +```json +{ + "enable_comments": true, + "max_notes_count": 20, + "content_filters": { + "liked_count": {"min": 1000}, + "comment_count": {"min": 20} + } +} +``` + +任务参数会覆盖实例默认参数。过滤发生在子进程内部，因此产物、日志和评论抓取都会只针对过滤后的内容。 + +## 4. 计数语义 + +过滤不会改变翻页和扫描预算。`CRAWLER_MAX_NOTES_COUNT`、`max_notes_count` 仍表示最多扫描多少候选内容或页数；如果 20 条候选里只有 3 条命中过滤条件，本次只保存和抓取这 3 条，不会继续翻页补齐到 20 条。 diff --git "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" index d0b83758e..911c64967 100644 --- "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" +++ "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" @@ -93,6 +93,7 @@ uv run uvicorn api.main:app --port 8080 --reload | `enable_sub_comments` | `--get_sub_comment` | `false` | | `max_notes_count` | `--crawler_max_notes_count` | `20` | | `max_comments_count` | `--max_comments_count_singlenotes` | `10` | +| `content_filters` | `--content_filters` | `{"liked_count":{"min":1000}}` | | `max_concurrency_num` | `--max_concurrency_num` | `1` | | `cookies` | `--cookies` | `"a=b; c=d"` | | `cdp_connect_existing` | `--cdp_connect_existing` | `false` | @@ -154,11 +155,17 @@ uv run uvicorn api.main:app --port 8080 --reload "target_text": "编程副业,AI 工具", "params": { "enable_comments": true, - "max_comments_count": 10 + "max_comments_count": 10, + "content_filters": { + "liked_count": {"min": 1000}, + "comment_count": {"min": 20} + } } } ``` +`content_filters` 会在内容入库、媒体下载和评论抓取前生效。过滤条件不改变 `max_notes_count` 的扫描预算；命中多少就保存多少，不会为了补齐数量继续翻页。 + ## 8. 与原单实例模式的关系 原单实例 WebUI 和 `/api/crawler/*` 接口仍然保留，适合临时手动运行一个爬虫任务。多实例调度器使用 `/api/scheduler/*`，每个任务都会启动独立子进程，并显式传入： diff --git "a/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" "b/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" index 7592f2d86..26938bd22 100644 --- "a/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" +++ "b/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" @@ -110,6 +110,7 @@ flowchart LR login["登录认证"] search["搜索/爬取"] parse["数据解析"] + filter["内容过滤"] comment["获取评论"] end @@ -131,15 +132,18 @@ flowchart LR browser --> login login --> search search --> parse - parse --> comment - parse --> content + parse --> filter + filter --> comment + filter --> content comment --> comments parse --> creator - parse --> media + filter --> media content & comments & creator --> file & db & nosql media --> file ``` +内容过滤由 `tools/content_filter.py` 提供统一规则，平台 `core.py` 在详情解析完成后调用。过滤命中的内容才会进入内容存储、媒体下载和评论抓取；创作者信息本身不受内容过滤影响。 + --- ### 2.3 多实例调度器架构 @@ -230,6 +234,7 @@ MediaCrawler/ │ ├── app_runner.py # 应用运行管理 │ ├── browser_launcher.py # 浏览器启动 │ ├── cdp_browser.py # CDP浏览器管理 +│ ├── content_filter.py # 内容互动指标过滤 │ ├── profile.py # 多实例浏览器 Profile 路径解析 │ ├── crawler_util.py # 爬虫工具 │ └── async_file_writer.py # 异步文件写入 diff --git a/media_platform/bilibili/core.py b/media_platform/bilibili/core.py index bb3adbc59..6f25f540a 100644 --- a/media_platform/bilibili/core.py +++ b/media_platform/bilibili/core.py @@ -44,6 +44,7 @@ from store import bilibili as bilibili_store from tools import utils from tools.cdp_browser import CDPBrowserManager +from tools.content_filter import filter_content_items, log_filter_result from tools.profile import get_browser_profile_dir from var import crawler_type_var, source_keyword_var @@ -225,12 +226,14 @@ async def search_by_keywords(self): except Exception as e: utils.logger.warning(f"[BilibiliCrawler.search_by_keywords] error in the task list. The video for this page will not be included. {e}") video_items = await asyncio.gather(*task_list) - for video_item in video_items: - if video_item: - video_id_list.append(video_item.get("View").get("aid")) - await bilibili_store.update_bilibili_video(video_item) - await bilibili_store.update_up_info(video_item) - await self.get_bilibili_video(video_item, semaphore) + valid_video_items = [video_item for video_item in video_items if video_item] + kept_video_items = filter_content_items("bili", valid_video_items) + log_filter_result("bili", "search", len(valid_video_items), len(kept_video_items), utils.logger) + for video_item in kept_video_items: + video_id_list.append(video_item.get("View").get("aid")) + await bilibili_store.update_bilibili_video(video_item) + await bilibili_store.update_up_info(video_item) + await self.get_bilibili_video(video_item, semaphore) page += 1 # Sleep after page navigation @@ -296,21 +299,26 @@ async def search_by_keywords_in_time_range(self, daily_limit: bool): semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM) task_list = [self.get_video_info_task(aid=video_item.get("aid"), bvid="", semaphore=semaphore) for video_item in video_list] video_items = await asyncio.gather(*task_list) - - for video_item in video_items: - if video_item: - if (daily_limit and total_notes_crawled_for_keyword >= config.CRAWLER_MAX_NOTES_COUNT): - break - if (not daily_limit and total_notes_crawled_for_keyword >= config.CRAWLER_MAX_NOTES_COUNT): - break - if notes_count_this_day >= config.MAX_NOTES_PER_DAY: - break - notes_count_this_day += 1 - total_notes_crawled_for_keyword += 1 - video_id_list.append(video_item.get("View").get("aid")) - await bilibili_store.update_bilibili_video(video_item) - await bilibili_store.update_up_info(video_item) - await self.get_bilibili_video(video_item, semaphore) + valid_video_items = [video_item for video_item in video_items if video_item] + kept_video_items = filter_content_items("bili", valid_video_items) + kept_video_item_ids = {id(video_item) for video_item in kept_video_items} + log_filter_result("bili", "search_time_range", len(valid_video_items), len(kept_video_items), utils.logger) + + for video_item in valid_video_items: + if (daily_limit and total_notes_crawled_for_keyword >= config.CRAWLER_MAX_NOTES_COUNT): + break + if (not daily_limit and total_notes_crawled_for_keyword >= config.CRAWLER_MAX_NOTES_COUNT): + break + if notes_count_this_day >= config.MAX_NOTES_PER_DAY: + break + notes_count_this_day += 1 + total_notes_crawled_for_keyword += 1 + if id(video_item) not in kept_video_item_ids: + continue + video_id_list.append(video_item.get("View").get("aid")) + await bilibili_store.update_bilibili_video(video_item) + await bilibili_store.update_up_info(video_item) + await self.get_bilibili_video(video_item, semaphore) page += 1 @@ -406,16 +414,18 @@ async def get_specified_videos(self, video_url_list: List[str]): semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM) task_list = [self.get_video_info_task(aid=0, bvid=video_id, semaphore=semaphore) for video_id in bvids_list] video_details = await asyncio.gather(*task_list) + valid_video_details = [video_detail for video_detail in video_details if video_detail is not None] + kept_video_details = filter_content_items("bili", valid_video_details) + log_filter_result("bili", "detail", len(valid_video_details), len(kept_video_details), utils.logger) video_aids_list = [] - for video_detail in video_details: - if video_detail is not None: - video_item_view: Dict = video_detail.get("View") - video_aid: str = video_item_view.get("aid") - if video_aid: - video_aids_list.append(video_aid) - await bilibili_store.update_bilibili_video(video_detail) - await bilibili_store.update_up_info(video_detail) - await self.get_bilibili_video(video_detail, semaphore) + for video_detail in kept_video_details: + video_item_view: Dict = video_detail.get("View") + video_aid: str = video_item_view.get("aid") + if video_aid: + video_aids_list.append(video_aid) + await bilibili_store.update_bilibili_video(video_detail) + await bilibili_store.update_up_info(video_detail) + await self.get_bilibili_video(video_detail, semaphore) await self.batch_get_video_comments(video_aids_list) async def get_video_info_task(self, aid: int, bvid: str, semaphore: asyncio.Semaphore) -> Optional[Dict]: diff --git a/media_platform/douyin/core.py b/media_platform/douyin/core.py index 30634acde..1a9c41466 100644 --- a/media_platform/douyin/core.py +++ b/media_platform/douyin/core.py @@ -36,6 +36,7 @@ from store import douyin as douyin_store from tools import utils from tools.cdp_browser import CDPBrowserManager +from tools.content_filter import filter_content_items, log_filter_result from tools.profile import get_browser_profile_dir from var import crawler_type_var, source_keyword_var @@ -164,13 +165,21 @@ async def search(self) -> None: break dy_search_id = posts_res.get("extra", {}).get("logid", "") page_aweme_list = [] + page_aweme_items = [] for post_item in posts_res.get("data"): try: aweme_info: Dict = (post_item.get("aweme_info") or post_item.get("aweme_mix_info", {}).get("mix_items")[0]) except TypeError: continue - aweme_list.append(aweme_info.get("aweme_id", "")) - page_aweme_list.append(aweme_info.get("aweme_id", "")) + page_aweme_items.append(aweme_info) + + kept_aweme_items = filter_content_items("dy", page_aweme_items) + log_filter_result("dy", "search", len(page_aweme_items), len(kept_aweme_items), utils.logger) + for aweme_info in kept_aweme_items: + aweme_id = aweme_info.get("aweme_id", "") + if aweme_id: + aweme_list.append(aweme_id) + page_aweme_list.append(aweme_id) await douyin_store.update_douyin_aweme(aweme_item=aweme_info) await self.get_aweme_media(aweme_item=aweme_info) @@ -211,11 +220,17 @@ async def get_specified_awemes(self): semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM) task_list = [self.get_aweme_detail(aweme_id=aweme_id, semaphore=semaphore) for aweme_id in aweme_id_list] aweme_details = await asyncio.gather(*task_list) - for aweme_detail in aweme_details: - if aweme_detail is not None: - await douyin_store.update_douyin_aweme(aweme_item=aweme_detail) - await self.get_aweme_media(aweme_item=aweme_detail) - await self.batch_get_note_comments(aweme_id_list) + valid_aweme_details = [aweme_detail for aweme_detail in aweme_details if aweme_detail is not None] + kept_aweme_details = filter_content_items("dy", valid_aweme_details) + log_filter_result("dy", "detail", len(valid_aweme_details), len(kept_aweme_details), utils.logger) + kept_aweme_ids = [] + for aweme_detail in kept_aweme_details: + aweme_id = aweme_detail.get("aweme_id") + if aweme_id: + kept_aweme_ids.append(aweme_id) + await douyin_store.update_douyin_aweme(aweme_item=aweme_detail) + await self.get_aweme_media(aweme_item=aweme_detail) + await self.batch_get_note_comments(kept_aweme_ids) async def get_aweme_detail(self, aweme_id: str, semaphore: asyncio.Semaphore) -> Any: """Get note detail""" @@ -290,12 +305,16 @@ async def get_creators_and_videos(self) -> None: await douyin_store.save_creator(user_id, creator=creator_info) # Get all video information of the creator - all_video_list = await self.dy_client.get_all_user_aweme_posts(sec_user_id=user_id, callback=self.fetch_creator_video_detail) + accepted_aweme_ids: List[str] = [] - video_ids = [video_item.get("aweme_id") for video_item in all_video_list] - await self.batch_get_note_comments(video_ids) + async def fetch_and_collect(video_list: List[Dict]): + accepted_aweme_ids.extend(await self.fetch_creator_video_detail(video_list)) - async def fetch_creator_video_detail(self, video_list: List[Dict]): + await self.dy_client.get_all_user_aweme_posts(sec_user_id=user_id, callback=fetch_and_collect) + + await self.batch_get_note_comments(accepted_aweme_ids) + + async def fetch_creator_video_detail(self, video_list: List[Dict]) -> List[str]: """ Concurrently obtain the specified post list and save the data """ @@ -303,10 +322,17 @@ async def fetch_creator_video_detail(self, video_list: List[Dict]): task_list = [self.get_aweme_detail(post_item.get("aweme_id"), semaphore) for post_item in video_list] note_details = await asyncio.gather(*task_list) - for aweme_item in note_details: - if aweme_item is not None: - await douyin_store.update_douyin_aweme(aweme_item=aweme_item) - await self.get_aweme_media(aweme_item=aweme_item) + valid_note_details = [aweme_item for aweme_item in note_details if aweme_item is not None] + kept_note_details = filter_content_items("dy", valid_note_details) + log_filter_result("dy", "creator", len(valid_note_details), len(kept_note_details), utils.logger) + kept_aweme_ids = [] + for aweme_item in kept_note_details: + aweme_id = aweme_item.get("aweme_id") + if aweme_id: + kept_aweme_ids.append(aweme_id) + await douyin_store.update_douyin_aweme(aweme_item=aweme_item) + await self.get_aweme_media(aweme_item=aweme_item) + return kept_aweme_ids async def create_douyin_client(self, httpx_proxy: Optional[str]) -> DouYinClient: """Create douyin client""" diff --git a/media_platform/kuaishou/core.py b/media_platform/kuaishou/core.py index bbf36af32..9ab420560 100644 --- a/media_platform/kuaishou/core.py +++ b/media_platform/kuaishou/core.py @@ -39,6 +39,7 @@ from store import kuaishou as kuaishou_store from tools import utils from tools.cdp_browser import CDPBrowserManager +from tools.content_filter import filter_content_items, log_filter_result from tools.profile import get_browser_profile_dir from var import comment_tasks_var, crawler_type_var, source_keyword_var @@ -171,8 +172,13 @@ async def search(self): ) break search_session_id = vision_search_photo.get("searchSessionId", "") - for video_detail in vision_search_photo.get("feeds"): - video_id_list.append(video_detail.get("photo", {}).get("id")) + feeds = vision_search_photo.get("feeds") or [] + kept_feeds = filter_content_items("ks", feeds) + log_filter_result("ks", "search", len(feeds), len(kept_feeds), utils.logger) + for video_detail in kept_feeds: + video_id = video_detail.get("photo", {}).get("id") + if video_id: + video_id_list.append(video_id) await kuaishou_store.update_kuaishou_video(video_item=video_detail) # batch fetch video comments @@ -203,10 +209,16 @@ async def get_specified_videos(self): for video_id in video_ids ] video_details = await asyncio.gather(*task_list) - for video_detail in video_details: - if video_detail is not None: - await kuaishou_store.update_kuaishou_video(video_detail) - await self.batch_get_video_comments(video_ids) + valid_video_details = [video_detail for video_detail in video_details if video_detail is not None] + kept_video_details = filter_content_items("ks", valid_video_details) + log_filter_result("ks", "detail", len(valid_video_details), len(kept_video_details), utils.logger) + kept_video_ids = [] + for video_detail in kept_video_details: + video_id = video_detail.get("photo", {}).get("id") + if video_id: + kept_video_ids.append(video_id) + await kuaishou_store.update_kuaishou_video(video_detail) + await self.batch_get_video_comments(kept_video_ids) async def get_video_info_task( self, video_id: str, semaphore: asyncio.Semaphore @@ -414,18 +426,20 @@ async def get_creators_and_videos(self) -> None: continue # Get all video information of the creator - all_video_list = await self.ks_client.get_all_videos_by_creator( + accepted_video_ids: List[str] = [] + + async def fetch_and_collect(video_list: List[Dict]): + accepted_video_ids.extend(await self.fetch_creator_video_detail(video_list)) + + await self.ks_client.get_all_videos_by_creator( user_id=user_id, crawl_interval=config.CRAWLER_MAX_SLEEP_SEC, - callback=self.fetch_creator_video_detail, + callback=fetch_and_collect, ) - video_ids = [ - video_item.get("photo", {}).get("id") for video_item in all_video_list - ] - await self.batch_get_video_comments(video_ids) + await self.batch_get_video_comments(accepted_video_ids) - async def fetch_creator_video_detail(self, video_list: List[Dict]): + async def fetch_creator_video_detail(self, video_list: List[Dict]) -> List[str]: """ Concurrently obtain the specified post list and save the data """ @@ -436,9 +450,16 @@ async def fetch_creator_video_detail(self, video_list: List[Dict]): ] video_details = await asyncio.gather(*task_list) - for video_detail in video_details: - if video_detail is not None: - await kuaishou_store.update_kuaishou_video(video_detail) + valid_video_details = [video_detail for video_detail in video_details if video_detail is not None] + kept_video_details = filter_content_items("ks", valid_video_details) + log_filter_result("ks", "creator", len(valid_video_details), len(kept_video_details), utils.logger) + kept_video_ids = [] + for video_detail in kept_video_details: + video_id = video_detail.get("photo", {}).get("id") + if video_id: + kept_video_ids.append(video_id) + await kuaishou_store.update_kuaishou_video(video_detail) + return kept_video_ids async def close(self): """Close browser context""" diff --git a/media_platform/tieba/core.py b/media_platform/tieba/core.py index f9c94f838..28bc527b9 100644 --- a/media_platform/tieba/core.py +++ b/media_platform/tieba/core.py @@ -37,6 +37,7 @@ from store import tieba as tieba_store from tools import utils from tools.cdp_browser import CDPBrowserManager +from tools.content_filter import filter_content_items, log_filter_result from tools.profile import get_browser_profile_dir from var import crawler_type_var, source_keyword_var @@ -266,10 +267,12 @@ async def get_specified_notes( ] note_details = await asyncio.gather(*task_list) note_details_model: List[TiebaNote] = [] - for note_detail in note_details: - if note_detail is not None: - note_details_model.append(note_detail) - await tieba_store.update_tieba_note(note_detail) + valid_note_details = [note_detail for note_detail in note_details if note_detail is not None] + kept_note_details = filter_content_items("tieba", valid_note_details) + log_filter_result("tieba", "detail", len(valid_note_details), len(kept_note_details), utils.logger) + for note_detail in kept_note_details: + note_details_model.append(note_detail) + await tieba_store.update_tieba_note(note_detail) await self.batch_get_note_comments(note_details_model) async def get_note_detail_async_task( @@ -383,18 +386,23 @@ async def get_creators_and_notes(self) -> None: raise Exception("Get creator info error") await tieba_store.save_creator(user_info=creator_info) + accepted_notes: List[TiebaNote] = [] + + async def save_filtered_notes(note_list: List[TiebaNote]): + kept_note_list = filter_content_items("tieba", note_list) + log_filter_result("tieba", "creator", len(note_list), len(kept_note_list), utils.logger) + accepted_notes.extend(kept_note_list) + await tieba_store.batch_update_tieba_notes(kept_note_list) # Get all note information of the creator - all_notes_list = ( - await self.tieba_client.get_all_notes_by_creator_url( - creator_url=creator_url, - crawl_interval=0, - callback=tieba_store.batch_update_tieba_notes, - max_note_count=config.CRAWLER_MAX_NOTES_COUNT, - ) + await self.tieba_client.get_all_notes_by_creator_url( + creator_url=creator_url, + crawl_interval=0, + callback=save_filtered_notes, + max_note_count=config.CRAWLER_MAX_NOTES_COUNT, ) - await self.batch_get_note_comments(all_notes_list) + await self.batch_get_note_comments(accepted_notes) else: utils.logger.error( diff --git a/media_platform/weibo/core.py b/media_platform/weibo/core.py index 8b5125621..fa784323b 100644 --- a/media_platform/weibo/core.py +++ b/media_platform/weibo/core.py @@ -41,6 +41,7 @@ from store import weibo as weibo_store from tools import utils from tools.cdp_browser import CDPBrowserManager +from tools.content_filter import filter_content_items, log_filter_result from tools.profile import get_browser_profile_dir from var import crawler_type_var, source_keyword_var @@ -175,13 +176,15 @@ async def search(self): note_list = filter_search_result_card(search_res.get("cards")) # If full text fetching is enabled, batch get full text of posts note_list = await self.batch_get_notes_full_text(note_list) - for note_item in note_list: - if note_item: - mblog: Dict = note_item.get("mblog") - if mblog: - note_id_list.append(mblog.get("id")) - await weibo_store.update_weibo_note(note_item) - await self.get_note_images(mblog) + valid_note_list = [note_item for note_item in note_list if note_item] + kept_note_list = filter_content_items("wb", valid_note_list) + log_filter_result("wb", "search", len(valid_note_list), len(kept_note_list), utils.logger) + for note_item in kept_note_list: + mblog: Dict = note_item.get("mblog") + if mblog: + note_id_list.append(mblog.get("id")) + await weibo_store.update_weibo_note(note_item) + await self.get_note_images(mblog) page += 1 @@ -199,10 +202,16 @@ async def get_specified_notes(self): semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM) task_list = [self.get_note_info_task(note_id=note_id, semaphore=semaphore) for note_id in config.WEIBO_SPECIFIED_ID_LIST] video_details = await asyncio.gather(*task_list) - for note_item in video_details: - if note_item: - await weibo_store.update_weibo_note(note_item) - await self.batch_get_notes_comments(config.WEIBO_SPECIFIED_ID_LIST) + valid_note_details = [note_item for note_item in video_details if note_item] + kept_note_details = filter_content_items("wb", valid_note_details) + log_filter_result("wb", "detail", len(valid_note_details), len(kept_note_details), utils.logger) + note_ids = [] + for note_item in kept_note_details: + mblog = note_item.get("mblog", {}) + if mblog.get("id"): + note_ids.append(mblog.get("id")) + await weibo_store.update_weibo_note(note_item) + await self.batch_get_notes_comments(note_ids) async def get_note_info_task(self, note_id: str, semaphore: asyncio.Semaphore) -> Optional[Dict]: """ @@ -317,22 +326,26 @@ async def get_creators_and_notes(self) -> None: if not createor_info: raise DataFetchError("Get creator info error") await weibo_store.save_creator(user_id, user_info=createor_info) + accepted_notes: List[Dict] = [] # Create a wrapper callback to get full text before saving data async def save_notes_with_full_text(note_list: List[Dict]): # If full text fetching is enabled, batch get full text first updated_note_list = await self.batch_get_notes_full_text(note_list) - await weibo_store.batch_update_weibo_notes(updated_note_list) + kept_note_list = filter_content_items("wb", updated_note_list) + log_filter_result("wb", "creator", len(updated_note_list), len(kept_note_list), utils.logger) + accepted_notes.extend(kept_note_list) + await weibo_store.batch_update_weibo_notes(kept_note_list) # Get all note information of the creator - all_notes_list = await self.wb_client.get_all_notes_by_creator_id( + await self.wb_client.get_all_notes_by_creator_id( creator_id=user_id, container_id=f"107603{user_id}", crawl_interval=0, callback=save_notes_with_full_text, ) - note_ids = [note_item.get("mblog", {}).get("id") for note_item in all_notes_list if note_item.get("mblog", {}).get("id")] + note_ids = [note_item.get("mblog", {}).get("id") for note_item in accepted_notes if note_item.get("mblog", {}).get("id")] await self.batch_get_notes_comments(note_ids) else: diff --git a/media_platform/xhs/core.py b/media_platform/xhs/core.py index c5447c28f..ff6ad32a4 100644 --- a/media_platform/xhs/core.py +++ b/media_platform/xhs/core.py @@ -38,6 +38,7 @@ from store import xhs as xhs_store from tools import utils from tools.cdp_browser import CDPBrowserManager +from tools.content_filter import filter_content_items, log_filter_result from tools.profile import get_browser_profile_dir from var import crawler_type_var, source_keyword_var @@ -170,11 +171,15 @@ async def search(self) -> None: ) for post_item in notes_res.get("items", {}) if post_item.get("model_type") not in ("rec_query", "hot_query") ] note_details = await asyncio.gather(*task_list) - for note_detail in note_details: - if note_detail: - await xhs_store.update_xhs_note(note_detail) - await self.get_notice_media(note_detail) - note_ids.append(note_detail.get("note_id")) + valid_note_details = [note_detail for note_detail in note_details if note_detail] + kept_note_details = filter_content_items("xhs", valid_note_details) + log_filter_result("xhs", "search", len(valid_note_details), len(kept_note_details), utils.logger) + for note_detail in kept_note_details: + await xhs_store.update_xhs_note(note_detail) + await self.get_notice_media(note_detail) + note_id = note_detail.get("note_id") + if note_id: + note_ids.append(note_id) xsec_tokens.append(note_detail.get("xsec_token")) page += 1 utils.logger.info(f"[XiaoHongShuCrawler.search] Note details: {note_details}") @@ -212,22 +217,29 @@ async def get_creators_and_notes(self) -> None: # Use fixed crawling interval crawl_interval = config.CRAWLER_MAX_SLEEP_SEC # Get all note information of the creator - all_notes_list = await self.xhs_client.get_all_notes_by_creator( + accepted_notes: List[Dict] = [] + + async def fetch_and_collect(note_list: List[Dict]): + accepted_notes.extend(await self.fetch_creator_notes_detail(note_list)) + + await self.xhs_client.get_all_notes_by_creator( user_id=user_id, crawl_interval=crawl_interval, - callback=self.fetch_creator_notes_detail, + callback=fetch_and_collect, xsec_token=creator_info.xsec_token, xsec_source=creator_info.xsec_source, ) note_ids = [] xsec_tokens = [] - for note_item in all_notes_list: - note_ids.append(note_item.get("note_id")) - xsec_tokens.append(note_item.get("xsec_token")) + for note_item in accepted_notes: + note_id = note_item.get("note_id") + if note_id: + note_ids.append(note_id) + xsec_tokens.append(note_item.get("xsec_token")) await self.batch_get_note_comments(note_ids, xsec_tokens) - async def fetch_creator_notes_detail(self, note_list: List[Dict]): + async def fetch_creator_notes_detail(self, note_list: List[Dict]) -> List[Dict]: """Concurrently obtain the specified post list and save the data""" semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM) task_list = [ @@ -240,10 +252,13 @@ async def fetch_creator_notes_detail(self, note_list: List[Dict]): ] note_details = await asyncio.gather(*task_list) - for note_detail in note_details: - if note_detail: - await xhs_store.update_xhs_note(note_detail) - await self.get_notice_media(note_detail) + valid_note_details = [note_detail for note_detail in note_details if note_detail] + kept_note_details = filter_content_items("xhs", valid_note_details) + log_filter_result("xhs", "creator", len(valid_note_details), len(kept_note_details), utils.logger) + for note_detail in kept_note_details: + await xhs_store.update_xhs_note(note_detail) + await self.get_notice_media(note_detail) + return kept_note_details async def get_specified_notes(self): """Get the information and comments of the specified post @@ -265,12 +280,16 @@ async def get_specified_notes(self): need_get_comment_note_ids = [] xsec_tokens = [] note_details = await asyncio.gather(*get_note_detail_task_list) - for note_detail in note_details: - if note_detail: - need_get_comment_note_ids.append(note_detail.get("note_id", "")) + valid_note_details = [note_detail for note_detail in note_details if note_detail] + kept_note_details = filter_content_items("xhs", valid_note_details) + log_filter_result("xhs", "detail", len(valid_note_details), len(kept_note_details), utils.logger) + for note_detail in kept_note_details: + note_id = note_detail.get("note_id", "") + if note_id: + need_get_comment_note_ids.append(note_id) xsec_tokens.append(note_detail.get("xsec_token", "")) - await xhs_store.update_xhs_note(note_detail) - await self.get_notice_media(note_detail) + await xhs_store.update_xhs_note(note_detail) + await self.get_notice_media(note_detail) await self.batch_get_note_comments(need_get_comment_note_ids, xsec_tokens) async def get_note_detail_async_task( diff --git a/media_platform/zhihu/core.py b/media_platform/zhihu/core.py index a55da4d00..a7d472d28 100644 --- a/media_platform/zhihu/core.py +++ b/media_platform/zhihu/core.py @@ -40,6 +40,7 @@ from store import zhihu as zhihu_store from tools import utils from tools.cdp_browser import CDPBrowserManager +from tools.content_filter import filter_content_items, log_filter_result from tools.profile import get_browser_profile_dir from var import crawler_type_var, source_keyword_var @@ -192,10 +193,12 @@ async def search(self) -> None: utils.logger.info(f"[ZhihuCrawler.search] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after page {page-1}") page += 1 - for content in content_list: + kept_content_list = filter_content_items("zhihu", content_list) + log_filter_result("zhihu", "search", len(content_list), len(kept_content_list), utils.logger) + for content in kept_content_list: await zhihu_store.update_zhihu_content(content) - await self.batch_get_content_comments(content_list) + await self.batch_get_content_comments(kept_content_list) except DataFetchError: utils.logger.error("[ZhihuCrawler.search] Search content error") return @@ -278,14 +281,21 @@ async def get_creators_and_notes(self) -> None: utils.logger.info( f"[ZhihuCrawler.get_creators_and_notes] Creator info: {createor_info}" ) + accepted_contents: List[ZhihuContent] = [] + + async def save_filtered_contents(contents: List[ZhihuContent]): + kept_contents = filter_content_items("zhihu", contents) + log_filter_result("zhihu", "creator", len(contents), len(kept_contents), utils.logger) + accepted_contents.extend(kept_contents) + await zhihu_store.batch_update_zhihu_contents(kept_contents) # By default, only answer information is extracted, uncomment below if articles and videos are needed # Get all anwser information of the creator - all_content_list = await self.zhihu_client.get_all_anwser_by_creator( + await self.zhihu_client.get_all_anwser_by_creator( url_token=user_url_token, crawl_interval=config.CRAWLER_MAX_SLEEP_SEC, - callback=zhihu_store.batch_update_zhihu_contents, + callback=save_filtered_contents, ) # Get all articles of the creator's contents @@ -303,7 +313,7 @@ async def get_creators_and_notes(self) -> None: # ) # Get all comments of the creator's contents - await self.batch_get_content_comments(all_content_list) + await self.batch_get_content_comments(accepted_contents) async def get_note_detail( self, full_note_url: str, semaphore: asyncio.Semaphore @@ -381,6 +391,7 @@ async def get_specified_notes(self): need_get_comment_notes: List[ZhihuContent] = [] note_details = await asyncio.gather(*get_note_detail_task_list) + valid_note_details: List[ZhihuContent] = [] for index, note_detail in enumerate(note_details): if not note_detail: utils.logger.info( @@ -389,6 +400,11 @@ async def get_specified_notes(self): continue note_detail = cast(ZhihuContent, note_detail) # only for type check + valid_note_details.append(note_detail) + + kept_note_details = filter_content_items("zhihu", valid_note_details) + log_filter_result("zhihu", "detail", len(valid_note_details), len(kept_note_details), utils.logger) + for note_detail in kept_note_details: need_get_comment_notes.append(note_detail) await zhihu_store.update_zhihu_content(note_detail) diff --git a/tests/test_api_limits.py b/tests/test_api_limits.py index 0cb65bc8d..2e6cab776 100644 --- a/tests/test_api_limits.py +++ b/tests/test_api_limits.py @@ -50,7 +50,8 @@ def test_crawler_manager_build_command(): crawler_type=CrawlerTypeEnum.SEARCH, keywords="test", max_notes_count=50, - max_comments_count=5 + max_comments_count=5, + content_filters={"liked_count": {"min": 1000}}, ) cmd2 = cm._build_command(req2) # Check that they are correctly added @@ -62,6 +63,10 @@ def test_crawler_manager_build_command(): idx_comments = cmd2.index("--max_comments_count_singlenotes") assert cmd2[idx_comments + 1] == "5" + assert "--content_filters" in cmd2 + idx_filters = cmd2.index("--content_filters") + assert cmd2[idx_filters + 1] == '{"liked_count": {"min": 1000.0}}' + def test_api_start_crawler_with_limits(): client = TestClient(app) @@ -75,7 +80,8 @@ def test_api_start_crawler_with_limits(): "crawler_type": "search", "keywords": "test", "max_notes_count": 50, - "max_comments_count": 5 + "max_comments_count": 5, + "content_filters": {"liked_count": {"min": 1000}}, }) assert response.status_code == 200 @@ -86,6 +92,7 @@ def test_api_start_crawler_with_limits(): assert called_request.platform == PlatformEnum.XHS assert called_request.max_notes_count == 50 assert called_request.max_comments_count == 5 + assert called_request.content_filters == {"liked_count": {"min": 1000.0}} def test_api_start_crawler_without_limits(): client = TestClient(app) @@ -135,3 +142,20 @@ def test_api_rejects_invalid_limits(field_name, value): assert response.status_code == 422 mock_start.assert_not_called() + + +def test_api_rejects_invalid_content_filter_field(): + client = TestClient(app) + payload = { + "platform": "dy", + "login_type": "qrcode", + "crawler_type": "search", + "keywords": "test", + "content_filters": {"view_count": {"min": 1}}, + } + + with patch("api.routers.crawler.crawler_manager.start", new_callable=AsyncMock) as mock_start: + response = client.post("/api/crawler/start", json=payload) + + assert response.status_code == 422 + mock_start.assert_not_called() diff --git a/tests/test_cmd_arg_scheduler.py b/tests/test_cmd_arg_scheduler.py index ca79e227b..7ee579beb 100644 --- a/tests/test_cmd_arg_scheduler.py +++ b/tests/test_cmd_arg_scheduler.py @@ -14,6 +14,7 @@ async def test_cmd_arg_sets_scheduler_runtime_fields(): "BROWSER_PROFILE_DIR": config.BROWSER_PROFILE_DIR, "CDP_DEBUG_PORT": config.CDP_DEBUG_PORT, "CDP_CONNECT_EXISTING": config.CDP_CONNECT_EXISTING, + "CONTENT_FILTERS": config.CONTENT_FILTERS, } try: @@ -31,14 +32,18 @@ async def test_cmd_arg_sets_scheduler_runtime_fields(): "9233", "--cdp_connect_existing", "false", + "--content_filters", + '{"liked_count":{"min":1000}}', ] ) assert result.type == "login" + assert result.content_filters == {"liked_count": {"min": 1000.0}} assert config.INSTANCE_ID == "inst-a" assert config.BROWSER_PROFILE_DIR == "data/scheduler/profiles/inst-a" assert config.CDP_DEBUG_PORT == 9233 assert config.CDP_CONNECT_EXISTING is False + assert config.CONTENT_FILTERS == {"liked_count": {"min": 1000.0}} finally: for key, value in original.items(): setattr(config, key, value) diff --git a/tests/test_content_filter.py b/tests/test_content_filter.py new file mode 100644 index 000000000..d5862c064 --- /dev/null +++ b/tests/test_content_filter.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +import pytest + +from model.m_baidu_tieba import TiebaNote +from model.m_zhihu import ZhihuContent +from tools.content_filter import ( + ContentFilterError, + filter_content_items, + normalize_content_filters, +) + + +def test_content_filters_parse_units_and_aliases(): + filters = normalize_content_filters( + "xhs", + {"liked_count": {"min": "1万"}, "collect_count": {"max": "2万"}}, + ) + + assert filters == { + "liked_count": {"min": 10000.0}, + "collected_count": {"max": 20000.0}, + } + + +def test_content_filters_filter_dict_platform_items(): + items = [ + {"interact_info": {"liked_count": "9999", "collected_count": 10}}, + {"interact_info": {"liked_count": "1.2万", "collected_count": 5}}, + ] + + kept = filter_content_items("xhs", items, {"liked_count": {"min": 10000}}) + + assert kept == [items[1]] + + +def test_content_filters_filter_bilibili_nested_metrics(): + items = [ + {"View": {"stat": {"like": 100, "favorite": 20}}}, + {"View": {"stat": {"like": 500, "favorite": 100}}}, + ] + + kept = filter_content_items( + "bili", + items, + {"liked_count": {"min": 300}, "favorite_count": {"min": 50}}, + ) + + assert kept == [items[1]] + + +def test_content_filters_filter_model_platform_items(): + tieba_items = [ + TiebaNote(note_id="1", title="a", note_url="u", tieba_name="t", tieba_link="l", total_replay_num=9), + TiebaNote(note_id="2", title="b", note_url="u", tieba_name="t", tieba_link="l", total_replay_num=10), + ] + zhihu_items = [ + ZhihuContent(content_id="a", voteup_count=99, comment_count=10), + ZhihuContent(content_id="b", voteup_count=100, comment_count=20), + ] + + assert filter_content_items("tieba", tieba_items, {"reply_count": {"min": 10}}) == [tieba_items[1]] + assert filter_content_items("zhihu", zhihu_items, {"voteup_count": {"min": 100}}) == [zhihu_items[1]] + + +def test_content_filters_reject_unsupported_field(): + with pytest.raises(ContentFilterError): + normalize_content_filters("dy", {"view_count": {"min": 1}}) diff --git a/tests/test_scheduler_manager.py b/tests/test_scheduler_manager.py index 2fd5feccc..4f53e713e 100644 --- a/tests/test_scheduler_manager.py +++ b/tests/test_scheduler_manager.py @@ -21,7 +21,7 @@ def test_scheduler_manager_builds_isolated_command(tmp_path): "instance_id": instance["id"], "crawler_type": "search", "target_text": "AI 工具", - "params": {"enable_sub_comments": True}, + "params": {"enable_sub_comments": True, "content_filters": {"liked_count": {"min": 1000}}}, }, str(tmp_path / "artifacts" / "task-a"), ) @@ -37,6 +37,7 @@ def test_scheduler_manager_builds_isolated_command(tmp_path): assert cmd[cmd.index("--get_comment") + 1] == "false" assert cmd[cmd.index("--get_sub_comment") + 1] == "true" assert cmd[cmd.index("--crawler_max_notes_count") + 1] == "8" + assert cmd[cmd.index("--content_filters") + 1] == '{"liked_count": {"min": 1000}}' def test_scheduler_manager_scans_artifacts(tmp_path): diff --git a/tools/content_filter.py b/tools/content_filter.py new file mode 100644 index 000000000..3e2a6fb0d --- /dev/null +++ b/tools/content_filter.py @@ -0,0 +1,218 @@ +# -*- coding: utf-8 -*- +import json +import re +from collections.abc import Callable +from typing import Any, Iterable, TypeVar + + +T = TypeVar("T") + + +class ContentFilterError(ValueError): + pass + + +MetricGetter = Callable[[Any], Any] + + +def _dig(item: Any, *path: str) -> Any: + value = item + for key in path: + if isinstance(value, dict): + value = value.get(key) + else: + value = getattr(value, key, None) + if value is None: + return None + return value + + +def _first_value(*getters: MetricGetter) -> MetricGetter: + def getter(item: Any) -> Any: + for get_value in getters: + value = get_value(item) + if value not in (None, ""): + return value + return None + + return getter + + +METRIC_GETTERS: dict[str, dict[str, MetricGetter]] = { + "xhs": { + "liked_count": _first_value(lambda item: _dig(item, "interact_info", "liked_count"), lambda item: _dig(item, "liked_count")), + "collected_count": _first_value(lambda item: _dig(item, "interact_info", "collected_count"), lambda item: _dig(item, "collected_count")), + "comment_count": _first_value(lambda item: _dig(item, "interact_info", "comment_count"), lambda item: _dig(item, "comment_count")), + "share_count": _first_value(lambda item: _dig(item, "interact_info", "share_count"), lambda item: _dig(item, "share_count")), + }, + "dy": { + "liked_count": _first_value(lambda item: _dig(item, "statistics", "digg_count"), lambda item: _dig(item, "liked_count")), + "collected_count": _first_value(lambda item: _dig(item, "statistics", "collect_count"), lambda item: _dig(item, "collected_count")), + "comment_count": _first_value(lambda item: _dig(item, "statistics", "comment_count"), lambda item: _dig(item, "comment_count")), + "share_count": _first_value(lambda item: _dig(item, "statistics", "share_count"), lambda item: _dig(item, "share_count")), + }, + "ks": { + "liked_count": _first_value(lambda item: _dig(item, "photo", "realLikeCount"), lambda item: _dig(item, "liked_count")), + "view_count": _first_value(lambda item: _dig(item, "photo", "viewCount"), lambda item: _dig(item, "view_count"), lambda item: _dig(item, "viewd_count")), + }, + "bili": { + "liked_count": _first_value(lambda item: _dig(item, "View", "stat", "like"), lambda item: _dig(item, "liked_count")), + "disliked_count": _first_value(lambda item: _dig(item, "View", "stat", "dislike"), lambda item: _dig(item, "disliked_count")), + "play_count": _first_value(lambda item: _dig(item, "View", "stat", "view"), lambda item: _dig(item, "video_play_count"), lambda item: _dig(item, "play_count")), + "favorite_count": _first_value(lambda item: _dig(item, "View", "stat", "favorite"), lambda item: _dig(item, "video_favorite_count"), lambda item: _dig(item, "favorite_count")), + "share_count": _first_value(lambda item: _dig(item, "View", "stat", "share"), lambda item: _dig(item, "video_share_count"), lambda item: _dig(item, "share_count")), + "coin_count": _first_value(lambda item: _dig(item, "View", "stat", "coin"), lambda item: _dig(item, "video_coin_count"), lambda item: _dig(item, "coin_count")), + "danmaku_count": _first_value(lambda item: _dig(item, "View", "stat", "danmaku"), lambda item: _dig(item, "video_danmaku"), lambda item: _dig(item, "danmaku_count")), + "comment_count": _first_value(lambda item: _dig(item, "View", "stat", "reply"), lambda item: _dig(item, "video_comment"), lambda item: _dig(item, "comment_count")), + }, + "wb": { + "liked_count": _first_value(lambda item: _dig(item, "mblog", "attitudes_count"), lambda item: _dig(item, "liked_count")), + "comment_count": _first_value(lambda item: _dig(item, "mblog", "comments_count"), lambda item: _dig(item, "comments_count"), lambda item: _dig(item, "comment_count")), + "share_count": _first_value(lambda item: _dig(item, "mblog", "reposts_count"), lambda item: _dig(item, "shared_count"), lambda item: _dig(item, "share_count")), + }, + "tieba": { + "reply_count": _first_value(lambda item: _dig(item, "total_replay_num"), lambda item: _dig(item, "total_reply_num"), lambda item: _dig(item, "reply_count")), + "reply_page_count": _first_value(lambda item: _dig(item, "total_replay_page"), lambda item: _dig(item, "total_reply_page"), lambda item: _dig(item, "reply_page_count")), + }, + "zhihu": { + "voteup_count": _first_value(lambda item: _dig(item, "voteup_count")), + "comment_count": _first_value(lambda item: _dig(item, "comment_count")), + }, +} + + +ALIASES: dict[str, dict[str, str]] = { + "xhs": {"like_count": "liked_count", "collect_count": "collected_count"}, + "dy": {"like_count": "liked_count", "collect_count": "collected_count"}, + "ks": {"viewd_count": "view_count", "like_count": "liked_count"}, + "bili": { + "like_count": "liked_count", + "view_count": "play_count", + "video_play_count": "play_count", + "video_favorite_count": "favorite_count", + "video_share_count": "share_count", + "video_coin_count": "coin_count", + "video_danmaku": "danmaku_count", + "video_comment": "comment_count", + "reply_count": "comment_count", + }, + "wb": {"like_count": "liked_count", "comments_count": "comment_count", "shared_count": "share_count", "reposts_count": "share_count"}, + "tieba": {"total_replay_num": "reply_count", "total_reply_num": "reply_count", "total_replay_page": "reply_page_count", "total_reply_page": "reply_page_count"}, + "zhihu": {"like_count": "voteup_count"}, +} + + +def parse_content_filters(value: Any) -> dict[str, Any]: + if value in (None, "", {}): + return {} + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError as exc: + raise ContentFilterError(f"content_filters must be valid JSON: {exc}") from exc + if not isinstance(value, dict): + raise ContentFilterError("content_filters must be a JSON object") + return value + + +def normalize_content_filters(platform: str, filters: Any) -> dict[str, dict[str, float]]: + raw_filters = parse_content_filters(filters) + if not raw_filters: + return {} + + platform_getters = METRIC_GETTERS.get(platform) + if not platform_getters: + raise ContentFilterError(f"unsupported platform for content filters: {platform}") + + aliases = ALIASES.get(platform, {}) + normalized: dict[str, dict[str, float]] = {} + for raw_name, raw_rule in raw_filters.items(): + metric = aliases.get(raw_name, raw_name) + if metric not in platform_getters: + supported = ", ".join(sorted(platform_getters)) + raise ContentFilterError(f"unsupported content filter field for {platform}: {raw_name}. Supported: {supported}") + normalized[metric] = _normalize_rule(raw_rule, raw_name) + return normalized + + +def filter_content_items(platform: str, items: Iterable[T], filters: Any = None) -> list[T]: + rules = normalize_content_filters(platform, _current_filters() if filters is None else filters) + if not rules: + return list(items) + return [item for item in items if match_content_filter(platform, item, rules)] + + +def match_content_filter(platform: str, item: Any, filters: Any = None) -> bool: + rules = normalize_content_filters(platform, _current_filters() if filters is None else filters) + if not rules: + return True + + getters = METRIC_GETTERS[platform] + for metric, rule in rules.items(): + value = _parse_number(getters[metric](item)) + if value is None: + return False + if "min" in rule and value < rule["min"]: + return False + if "max" in rule and value > rule["max"]: + return False + return True + + +def log_filter_result(platform: str, source: str, total: int, kept: int, logger: Any) -> None: + if total == kept: + return + logger.info(f"[content_filter] platform={platform}, source={source}, kept={kept}, skipped={total - kept}, total={total}") + + +def supported_filter_fields(platform: str) -> list[str]: + return sorted(METRIC_GETTERS.get(platform, {})) + + +def _current_filters() -> Any: + import config + + return getattr(config, "CONTENT_FILTERS", {}) + + +def _normalize_rule(rule: Any, field_name: str) -> dict[str, float]: + if isinstance(rule, (int, float, str)) and not isinstance(rule, bool): + value = _parse_number(rule) + if value is None: + raise ContentFilterError(f"content filter field {field_name} has invalid value: {rule}") + return {"min": value} + if not isinstance(rule, dict): + raise ContentFilterError(f"content filter field {field_name} must use min/max object") + + normalized: dict[str, float] = {} + for bound in ("min", "max"): + if bound not in rule or rule[bound] in (None, ""): + continue + value = _parse_number(rule[bound]) + if value is None: + raise ContentFilterError(f"content filter field {field_name}.{bound} has invalid value: {rule[bound]}") + normalized[bound] = value + if not normalized: + raise ContentFilterError(f"content filter field {field_name} must include min or max") + if "min" in normalized and "max" in normalized and normalized["min"] > normalized["max"]: + raise ContentFilterError(f"content filter field {field_name} min cannot exceed max") + return normalized + + +def _parse_number(value: Any) -> float | None: + if value is None or isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return float(value) + + text = str(value).strip().replace(",", "") + if not text: + return None + match = re.search(r"(-?\d+(?:\.\d+)?)\s*([万亿wWkK]?)", text) + if not match: + return None + + number = float(match.group(1)) + unit = match.group(2).lower() + multiplier = {"万": 10_000, "亿": 100_000_000, "w": 10_000, "k": 1_000}.get(unit, 1) + return number * multiplier From 6f9aedb001a27c6c69986a7a6cacd624e4b691dc Mon Sep 17 00:00:00 2001 From: Raffaello Date: Sun, 28 Jun 2026 17:01:37 +0800 Subject: [PATCH 03/17] feat: improve scheduler parameter forms --- README.md | 2 +- ...77\347\224\250\346\214\207\345\215\227.md" | 6 +- ...77\347\224\250\346\214\207\345\215\227.md" | 11 +- scheduler_webui/app.js | 222 +++++++++++++++++- scheduler_webui/index.html | 178 +++++++++++++- scheduler_webui/styles.css | 74 +++++- 6 files changed, 470 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 8bb86a648..5efd4ced1 100644 --- a/README.md +++ b/README.md @@ -221,7 +221,7 @@ uv run uvicorn api.main:app --port 8080 --reload #### 多实例调度器 -启动同一个 API 服务后，访问 `http://localhost:8080/scheduler` 可打开多实例调度器。调度器支持创建多个独立账号实例，每个实例拥有独立浏览器 Profile、CDP 端口、登录态、默认爬取参数和任务队列。 +启动同一个 API 服务后，访问 `http://localhost:8080/scheduler` 可打开多实例调度器。调度器支持创建多个独立账号实例，每个实例拥有独立浏览器 Profile、CDP 端口、登录态、默认爬取参数和任务队列。实例默认参数、任务覆盖参数、内容过滤、Cookie 和代理都可以在页面中通过输入框、下拉菜单配置，无需手写 JSON。 调度器运行数据默认保存在 `data/scheduler/`： diff --git "a/docs/\345\206\205\345\256\271\350\277\207\346\273\244\344\275\277\347\224\250\346\214\207\345\215\227.md" "b/docs/\345\206\205\345\256\271\350\277\207\346\273\244\344\275\277\347\224\250\346\214\207\345\215\227.md" index 4bc6550ef..c28ab0381 100644 --- "a/docs/\345\206\205\345\256\271\350\277\207\346\273\244\344\275\277\347\224\250\346\214\207\345\215\227.md" +++ "b/docs/\345\206\205\345\256\271\350\277\207\346\273\244\344\275\277\347\224\250\346\214\207\345\215\227.md" @@ -39,7 +39,9 @@ CONTENT_FILTERS = { ## 3. 调度器任务参数 -多实例调度器的“默认参数 JSON”和“任务参数 JSON”都支持 `content_filters`： +多实例调度器 WebUI 中，实例表单和任务表单都有“内容过滤”区域。点击“添加过滤条件”，选择指标并填写最小值或最大值即可，不需要手写 JSON。任务过滤条件会覆盖实例默认过滤条件；留空则沿用实例默认参数。 + +如果直接调用调度器 API，实例的 `default_params` 和任务的 `params` 都支持 `content_filters`： ```json { @@ -52,7 +54,7 @@ CONTENT_FILTERS = { } ``` -任务参数会覆盖实例默认参数。过滤发生在子进程内部，因此产物、日志和评论抓取都会只针对过滤后的内容。 +过滤发生在子进程内部，因此产物、日志和评论抓取都会只针对过滤后的内容。 ## 4. 计数语义 diff --git "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" index 911c64967..563b8b65f 100644 --- "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" +++ "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" @@ -76,7 +76,7 @@ uv run uvicorn api.main:app --port 8080 --reload ## 5. 使用流程 1. 打开 `/scheduler`。 -2. 创建实例，选择平台、登录方式、保存格式，必要时指定 CDP 端口和默认参数。 +2. 创建实例，选择平台、登录方式、保存格式，必要时指定 CDP 端口，并通过输入框、下拉菜单配置默认抓取参数、内容过滤、Cookie 和代理。 3. 点击实例行的“登录”，调度器会创建 `login` 任务，子进程只完成登录态维护，不进入抓取流程。 4. 创建搜索、详情或创作者任务，调度器会把任务加入该实例队列。 5. 查看任务状态、日志和产物列表。 @@ -85,7 +85,14 @@ uv run uvicorn api.main:app --port 8080 --reload ## 6. 任务参数 -实例的“默认参数 JSON”和任务的“任务参数 JSON”会合并，任务参数优先级更高。常用字段如下： +调度器 WebUI 已把实例默认参数和任务覆盖参数拆成表单控件，不需要手写 JSON： + +- 实例侧的“默认抓取参数”用于配置起始页、最大内容数、单条评论数、并发数、评论和二级评论。布尔项支持“项目默认 / 开启 / 关闭”，选择“项目默认”时不会写入覆盖值。 +- 任务侧的“任务覆盖参数”用于临时覆盖实例默认值。下拉项选择“沿用实例默认”时不会覆盖实例配置。 +- “内容过滤”按当前平台展示可选指标，点击“添加过滤条件”后填写最小值或最大值即可。数值支持 `1000`、`1万`、`2.5万` 这类写法。 +- “登录与代理”用于配置 Cookie、是否连接已有浏览器、是否启用代理、代理池数量、代理服务和固定代理地址。 + +WebUI 提交后会在内部生成 `default_params` 或 `params`。实例默认参数和任务参数仍按同一规则合并，任务参数优先级更高。API 调用仍使用 JSON，常用字段如下： | 字段 | 对应 CLI 参数 | 示例 | | --- | --- | --- | diff --git a/scheduler_webui/app.js b/scheduler_webui/app.js index 7ef2baf0a..98769fdbf 100644 --- a/scheduler_webui/app.js +++ b/scheduler_webui/app.js @@ -19,6 +19,52 @@ let state = { instances: [], tasks: [], selectedTaskId: "", + filterRows: { + instance: [], + task: [], + }, +}; + +const filterFields = { + xhs: [ + ["liked_count", "点赞数"], + ["collected_count", "收藏数"], + ["comment_count", "评论数"], + ["share_count", "转发数"], + ], + dy: [ + ["liked_count", "点赞数"], + ["collected_count", "收藏数"], + ["comment_count", "评论数"], + ["share_count", "转发数"], + ], + ks: [ + ["liked_count", "点赞数"], + ["view_count", "播放数"], + ], + bili: [ + ["liked_count", "点赞数"], + ["disliked_count", "点踩数"], + ["play_count", "播放数"], + ["favorite_count", "收藏数"], + ["share_count", "分享数"], + ["coin_count", "投币数"], + ["danmaku_count", "弹幕数"], + ["comment_count", "评论数"], + ], + wb: [ + ["liked_count", "点赞数"], + ["comment_count", "评论数"], + ["share_count", "转发数"], + ], + tieba: [ + ["reply_count", "回复数"], + ["reply_page_count", "回复页数"], + ], + zhihu: [ + ["voteup_count", "赞同数"], + ["comment_count", "评论数"], + ], }; const els = { @@ -33,6 +79,8 @@ const els = { taskHint: document.querySelector("#taskHint"), logsBox: document.querySelector("#logsBox"), artifactsList: document.querySelector("#artifactsList"), + instanceFilterRows: document.querySelector("#instanceFilterRows"), + taskFilterRows: document.querySelector("#taskFilterRows"), }; function statusPill(value) { @@ -48,12 +96,6 @@ function escapeHtml(value) { .replaceAll("'", "'"); } -function parseJson(value, fallback = {}) { - const text = value.trim(); - if (!text) return fallback; - return JSON.parse(text); -} - async function errorText(res) { try { const data = await res.json(); @@ -108,9 +150,14 @@ function renderInstances() { } function renderTaskSelect() { + const selectedId = els.taskInstanceSelect.value; els.taskInstanceSelect.innerHTML = state.instances .map((item) => ``) .join(""); + if (selectedId && state.instances.some((item) => item.id === selectedId)) { + els.taskInstanceSelect.value = selectedId; + } + syncFilterRowsWithPlatform("task"); } function renderTasks() { @@ -138,6 +185,145 @@ function renderTasks() { .join(""); } +function renderFilterRows(scope) { + const platform = currentPlatform(scope); + const target = scope === "instance" ? els.instanceFilterRows : els.taskFilterRows; + const rows = state.filterRows[scope]; + + target.innerHTML = rows.length + ? rows + .map((row, index) => ` +
      + + + + +
      + `) + .join("") + : `

      未设置过滤条件

      `; +} + +function metricOptions(platform, selectedMetric) { + return (filterFields[platform] || filterFields.xhs) + .map(([value, label]) => ``) + .join(""); +} + +function currentPlatform(scope) { + if (scope === "instance") { + return els.instanceForm.elements.platform.value || "xhs"; + } + const instanceId = els.taskInstanceSelect.value; + return state.instances.find((item) => item.id === instanceId)?.platform || "xhs"; +} + +function addFilterRow(scope) { + const [metric] = (filterFields[currentPlatform(scope)] || filterFields.xhs)[0]; + state.filterRows[scope].push({ metric, min: "", max: "" }); + renderFilterRows(scope); +} + +function removeFilterRow(scope, index) { + state.filterRows[scope].splice(index, 1); + renderFilterRows(scope); +} + +function syncFilterRowsWithPlatform(scope) { + const fields = filterFields[currentPlatform(scope)] || filterFields.xhs; + const allowed = new Set(fields.map(([value]) => value)); + const fallback = fields[0][0]; + state.filterRows[scope] = state.filterRows[scope].map((row) => ({ + ...row, + metric: allowed.has(row.metric) ? row.metric : fallback, + })); + renderFilterRows(scope); +} + +function updateFilterValue(target) { + const rowEl = target.closest(".filter-row"); + if (!rowEl) return; + const row = state.filterRows[rowEl.dataset.scope]?.[Number(rowEl.dataset.index)]; + if (!row) return; + row[target.dataset.field] = target.value; +} + +function collectFilters(scope) { + return state.filterRows[scope].reduce((filters, row) => { + const min = String(row.min || "").trim(); + const max = String(row.max || "").trim(); + if (!row.metric || (!min && !max)) return filters; + filters[row.metric] = {}; + if (min) filters[row.metric].min = min; + if (max) filters[row.metric].max = max; + return filters; + }, {}); +} + +function setNumberParam(params, key, value) { + const text = String(value || "").trim(); + if (text) params[key] = Number(text); +} + +function setStringParam(params, key, value) { + const text = String(value || "").trim(); + if (text) params[key] = text; +} + +function setBoolSelectParam(params, key, value) { + if (value === "true") params[key] = true; + if (value === "false") params[key] = false; +} + +function collectDefaultParams(data) { + const params = {}; + setBoolSelectParam(params, "enable_comments", data.get("default_enable_comments")); + setBoolSelectParam(params, "enable_sub_comments", data.get("default_enable_sub_comments")); + setBoolSelectParam(params, "cdp_connect_existing", data.get("default_cdp_connect_existing")); + setBoolSelectParam(params, "enable_ip_proxy", data.get("default_enable_ip_proxy")); + setNumberParam(params, "start_page", data.get("default_start_page")); + setNumberParam(params, "max_notes_count", data.get("default_max_notes_count")); + setNumberParam(params, "max_comments_count", data.get("default_max_comments_count")); + setNumberParam(params, "max_concurrency_num", data.get("default_max_concurrency_num")); + setNumberParam(params, "ip_proxy_pool_count", data.get("default_ip_proxy_pool_count")); + setStringParam(params, "cookies", data.get("default_cookies")); + setStringParam(params, "ip_proxy_provider_name", data.get("default_ip_proxy_provider_name")); + setStringParam(params, "static_proxy_url", data.get("default_static_proxy_url")); + const filters = collectFilters("instance"); + if (Object.keys(filters).length) params.content_filters = filters; + return params; +} + +function collectTaskParams(data) { + const params = {}; + setBoolSelectParam(params, "enable_comments", data.get("task_enable_comments")); + setBoolSelectParam(params, "enable_sub_comments", data.get("task_enable_sub_comments")); + setBoolSelectParam(params, "cdp_connect_existing", data.get("task_cdp_connect_existing")); + setBoolSelectParam(params, "enable_ip_proxy", data.get("task_enable_ip_proxy")); + setNumberParam(params, "start_page", data.get("task_start_page")); + setNumberParam(params, "max_notes_count", data.get("task_max_notes_count")); + setNumberParam(params, "max_comments_count", data.get("task_max_comments_count")); + setNumberParam(params, "max_concurrency_num", data.get("task_max_concurrency_num")); + setNumberParam(params, "ip_proxy_pool_count", data.get("task_ip_proxy_pool_count")); + setStringParam(params, "cookies", data.get("task_cookies")); + setStringParam(params, "ip_proxy_provider_name", data.get("task_ip_proxy_provider_name")); + setStringParam(params, "static_proxy_url", data.get("task_static_proxy_url")); + const filters = collectFilters("task"); + if (Object.keys(filters).length) params.content_filters = filters; + return params; +} + async function loadTaskDetail(taskId) { const [logs, artifacts] = await Promise.all([ api.get(`/api/scheduler/tasks/${taskId}/logs?limit=300`), @@ -171,12 +357,14 @@ els.instanceForm.addEventListener("submit", async (event) => { save_option: data.get("save_option"), headless: data.get("headless") === "on", browser_profile_dir: data.get("browser_profile_dir") || "", - default_params: parseJson(String(data.get("default_params") || ""), {}), + default_params: collectDefaultParams(data), }; const port = String(data.get("cdp_debug_port") || "").trim(); if (port) body.cdp_debug_port = Number(port); await api.post("/api/scheduler/instances", body); event.currentTarget.reset(); + state.filterRows.instance = []; + renderFilterRows("instance"); await refreshAll(); }); @@ -187,17 +375,33 @@ els.taskForm.addEventListener("submit", async (event) => { instance_id: data.get("instance_id"), crawler_type: data.get("crawler_type"), target_text: data.get("target_text") || "", - params: parseJson(String(data.get("params") || ""), {}), + params: collectTaskParams(data), }); event.currentTarget.reset(); + state.filterRows.task = []; + renderFilterRows("task"); await refreshAll(); }); +els.instanceForm.elements.platform.addEventListener("change", () => syncFilterRowsWithPlatform("instance")); +els.taskInstanceSelect.addEventListener("change", () => syncFilterRowsWithPlatform("task")); + +document.addEventListener("input", (event) => updateFilterValue(event.target)); +document.addEventListener("change", (event) => updateFilterValue(event.target)); + document.addEventListener("click", async (event) => { const target = event.target.closest("button[data-action]"); if (!target) return; const action = target.dataset.action; const id = target.dataset.id; + if (action === "add-filter") { + addFilterRow(target.dataset.scope); + return; + } + if (action === "remove-filter") { + removeFilterRow(target.dataset.scope, Number(target.dataset.index)); + return; + } try { if (action === "login") { const task = await api.post(`/api/scheduler/instances/${id}/login`, {}); @@ -218,6 +422,8 @@ document.addEventListener("click", async (event) => { } }); +renderFilterRows("instance"); +renderFilterRows("task"); refreshAll().catch((err) => { els.summary.textContent = `读取失败：${err.message}`; }); diff --git a/scheduler_webui/index.html b/scheduler_webui/index.html index 8d174e3f4..97ee26abd 100644 --- a/scheduler_webui/index.html +++ b/scheduler_webui/index.html @@ -62,10 +62,94 @@

      新建实例

      浏览器 Profile 目录 - + +
      +

      默认抓取参数

      +
      + + + + + + +
      +
      + +
      +

      内容过滤

      +
      + +
      + +
      +

      登录与代理

      + +
      + + + + +
      + +
      + - +
      +

      任务覆盖参数

      +
      + + + + + + +
      +

      内容过滤

      +
      + +
      +
      +

      任务登录与代理

      +
      + + + + +
      + + +
      diff --git a/scheduler_webui/styles.css b/scheduler_webui/styles.css index fac30cf57..6a01b7633 100644 --- a/scheduler_webui/styles.css +++ b/scheduler_webui/styles.css @@ -55,6 +55,14 @@ button.danger { background: var(--red); } +button.icon-button { + width: 36px; + min-width: 36px; + padding: 0; + font-size: 20px; + line-height: 1; +} + button:disabled { cursor: not-allowed; opacity: 0.55; @@ -129,7 +137,7 @@ th { .layout { display: grid; - grid-template-columns: 340px minmax(0, 1fr); + grid-template-columns: 400px minmax(0, 1fr); gap: 16px; align-items: start; } @@ -151,6 +159,27 @@ th { gap: 14px; } +.form-section { + display: grid; + gap: 12px; + padding-top: 14px; + border-top: 1px solid var(--line); +} + +.form-section h3 { + margin: 0; + color: var(--text); + font-size: 14px; + line-height: 1.3; +} + +.form-grid, +.task-params-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 10px; +} + .sidebar { position: sticky; top: 16px; @@ -198,6 +227,35 @@ th { gap: 16px; } +.filter-rows { + display: grid; + gap: 10px; +} + +.filter-row { + display: grid; + grid-template-columns: minmax(150px, 1.2fr) minmax(92px, 0.8fr) minmax(92px, 0.8fr) 36px; + gap: 8px; + align-items: end; +} + +.sidebar .filter-row { + grid-template-columns: repeat(2, minmax(0, 1fr)) 36px; +} + +.sidebar .filter-row label:first-child { + grid-column: 1 / -1; +} + +.empty-note { + margin: 0; + border: 1px dashed var(--line); + border-radius: 6px; + color: var(--muted); + padding: 10px; + font-size: 13px; +} + .log-panel pre { min-height: 260px; max-height: 420px; @@ -271,7 +329,8 @@ th { @media (max-width: 980px) { .layout, .detail-grid, - .task-grid { + .task-grid, + .task-params-grid { grid-template-columns: 1fr; } @@ -279,3 +338,14 @@ th { position: static; } } + +@media (max-width: 560px) { + .form-grid, + .filter-row { + grid-template-columns: 1fr; + } + + button.icon-button { + width: 100%; + } +} From b7953e3dcd76d88350d88d834887db42c3a4328c Mon Sep 17 00:00:00 2001 From: Raffaello Date: Sun, 28 Jun 2026 17:13:40 +0800 Subject: [PATCH 04/17] feat: reorder scheduler workflow layout --- ...77\347\224\250\346\214\207\345\215\227.md" | 8 +- scheduler_webui/index.html | 308 +++++++++--------- scheduler_webui/styles.css | 34 +- 3 files changed, 175 insertions(+), 175 deletions(-) diff --git "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" index 563b8b65f..84bd1552f 100644 --- "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" +++ "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" @@ -75,11 +75,13 @@ uv run uvicorn api.main:app --port 8080 --reload ## 5. 使用流程 +调度器页面自上而下排列为：实例列表、新建实例、任务列表、新建任务，最后是任务日志和产物详情。 + 1. 打开 `/scheduler`。 -2. 创建实例，选择平台、登录方式、保存格式，必要时指定 CDP 端口，并通过输入框、下拉菜单配置默认抓取参数、内容过滤、Cookie 和代理。 +2. 先查看实例列表；如果没有目标实例，再在下方创建实例，选择平台、登录方式、保存格式，必要时指定 CDP 端口，并通过输入框、下拉菜单配置默认抓取参数、内容过滤、Cookie 和代理。 3. 点击实例行的“登录”，调度器会创建 `login` 任务，子进程只完成登录态维护，不进入抓取流程。 -4. 创建搜索、详情或创作者任务，调度器会把任务加入该实例队列。 -5. 查看任务状态、日志和产物列表。 +4. 查看任务列表；需要新任务时，在任务列表下方创建搜索、详情或创作者任务，调度器会把任务加入该实例队列。 +5. 选择任务后查看日志和产物列表。 实例处于 `running` 或 `stopping` 时不能修改核心配置，也不能删除实例。任务执行成功或被取消后，同实例队列中的下一个任务会自动启动；任务失败时实例进入 `error`，需要人工查看日志后再投递或启动后续任务。 diff --git a/scheduler_webui/index.html b/scheduler_webui/index.html index 97ee26abd..652a0ee39 100644 --- a/scheduler_webui/index.html +++ b/scheduler_webui/index.html @@ -17,9 +17,31 @@

      MediaCrawler 多实例调度器

      - +
      -
      -
      -
      -

      实例

      - -
      -
      - - - - - - - - - - - - -
      名称平台状态端口当前任务操作
      -
      -
      +
      +
      +

      任务

      + +
      +
      + + + + + + + + + + + + +
      任务实例类型状态目标操作
      +
      +
      -
      -

      新建任务

      -
      - - - -
      -

      任务覆盖参数

      -
      - - - - - - -
      -

      内容过滤

      -
      - -
      -
      -

      任务登录与代理

      -
      - - - - -
      +
      +

      新建任务

      + + + + +
      +

      任务覆盖参数

      +
      + -
      - - -
      - -
      -
      -

      任务

      - -
      -
      - - - - - - - - - - - - -
      任务实例类型状态目标操作
      -
      -
      - -
      -
      -

      日志

      - 
      选择一个任务查看日志。
      + + + + +

      内容过滤

      +
      +
      -
      -

      产物

      -
        +
        +

        任务登录与代理

        +
        + + + + +
        + +
        + + +
        + +
        +
        +

        日志

        + 
        选择一个任务查看日志。
        +
        +
        +

        产物

        +
          diff --git a/scheduler_webui/styles.css b/scheduler_webui/styles.css index 6a01b7633..0d38eb84c 100644 --- a/scheduler_webui/styles.css +++ b/scheduler_webui/styles.css @@ -39,6 +39,7 @@ button { color: #fff; padding: 0 14px; cursor: pointer; + white-space: nowrap; } button:hover { @@ -137,9 +138,9 @@ th { .layout { display: grid; - grid-template-columns: 400px minmax(0, 1fr); - gap: 16px; - align-items: start; + grid-template-columns: 1fr; + gap: 14px; + align-items: stretch; } .panel { @@ -154,11 +155,6 @@ th { font-size: 16px; } -.stack { - display: grid; - gap: 14px; -} - .form-section { display: grid; gap: 12px; @@ -180,9 +176,16 @@ th { gap: 10px; } -.sidebar { - position: sticky; - top: 16px; +.instance-grid { + display: grid; + grid-template-columns: repeat(2, minmax(220px, 1fr)); + gap: 12px; +} + +.instance-grid .form-section, +.instance-grid .checkline, +.instance-grid button[type="submit"] { + grid-column: 1 / -1; } .section-title { @@ -239,11 +242,11 @@ th { align-items: end; } -.sidebar .filter-row { +.new-instance-panel .filter-row { grid-template-columns: repeat(2, minmax(0, 1fr)) 36px; } -.sidebar .filter-row label:first-child { +.new-instance-panel .filter-row label:first-child { grid-column: 1 / -1; } @@ -330,13 +333,10 @@ th { .layout, .detail-grid, .task-grid, + .instance-grid, .task-params-grid { grid-template-columns: 1fr; } - - .sidebar { - position: static; - } } @media (max-width: 560px) { From 415a9f5a99dc2b85591ffa6169bb2b3ee27c171b Mon Sep 17 00:00:00 2001 From: Raffaello Date: Sun, 28 Jun 2026 17:31:45 +0800 Subject: [PATCH 05/17] fix: make scheduler forms single column --- ...77\347\224\250\346\214\207\345\215\227.md" | 2 +- scheduler_webui/styles.css | 26 +++++-------------- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" index 84bd1552f..f7df75118 100644 --- "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" +++ "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" @@ -75,7 +75,7 @@ uv run uvicorn api.main:app --port 8080 --reload ## 5. 使用流程 -调度器页面自上而下排列为：实例列表、新建实例、任务列表、新建任务，最后是任务日志和产物详情。 +调度器页面自上而下排列为：实例列表、新建实例、任务列表、新建任务，最后是任务日志和产物详情。新建实例、新建任务和详情区采用纵向单列表单，避免在宽屏下左右跳读。 1. 打开 `/scheduler`。 2. 先查看实例列表；如果没有目标实例，再在下方创建实例，选择平台、登录方式、保存格式，必要时指定 CDP 端口，并通过输入框、下拉菜单配置默认抓取参数、内容过滤、Cookie 和代理。 diff --git a/scheduler_webui/styles.css b/scheduler_webui/styles.css index 0d38eb84c..c1cdf24c7 100644 --- a/scheduler_webui/styles.css +++ b/scheduler_webui/styles.css @@ -112,7 +112,7 @@ th { } .app-shell { - width: min(1440px, 100%); + width: min(1040px, 100%); margin: 0 auto; padding: 22px; } @@ -172,13 +172,13 @@ th { .form-grid, .task-params-grid { display: grid; - grid-template-columns: repeat(2, minmax(0, 1fr)); + grid-template-columns: 1fr; gap: 10px; } .instance-grid { display: grid; - grid-template-columns: repeat(2, minmax(220px, 1fr)); + grid-template-columns: 1fr; gap: 12px; } @@ -216,7 +216,7 @@ th { .task-grid { display: grid; - grid-template-columns: repeat(2, minmax(180px, 1fr)); + grid-template-columns: 1fr; gap: 12px; } @@ -226,7 +226,7 @@ th { .detail-grid { display: grid; - grid-template-columns: minmax(0, 1.4fr) minmax(260px, 0.6fr); + grid-template-columns: 1fr; gap: 16px; } @@ -237,19 +237,11 @@ th { .filter-row { display: grid; - grid-template-columns: minmax(150px, 1.2fr) minmax(92px, 0.8fr) minmax(92px, 0.8fr) 36px; + grid-template-columns: 1fr; gap: 8px; align-items: end; } -.new-instance-panel .filter-row { - grid-template-columns: repeat(2, minmax(0, 1fr)) 36px; -} - -.new-instance-panel .filter-row label:first-child { - grid-column: 1 / -1; -} - .empty-note { margin: 0; border: 1px dashed var(--line); @@ -330,11 +322,7 @@ th { } @media (max-width: 980px) { - .layout, - .detail-grid, - .task-grid, - .instance-grid, - .task-params-grid { + .layout { grid-template-columns: 1fr; } } From 416aa0d8993656e62c117c2817eb19f49bb9a612 Mon Sep 17 00:00:00 2001 From: Raffaello Date: Sun, 28 Jun 2026 17:59:17 +0800 Subject: [PATCH 06/17] feat: unify scheduler jobs and runs --- README.md | 8 +- api/routers/scheduler.py | 87 +++++ api/scheduler/manager.py | 145 +++++++- api/scheduler/schemas.py | 23 ++ api/scheduler/store.py | 44 ++- docs/index.md | 4 +- ...77\347\224\250\346\214\207\345\215\227.md" | 6 +- ...77\347\224\250\346\214\207\345\215\227.md" | 142 +++---- ...66\346\236\204\346\226\207\346\241\243.md" | 14 +- scheduler_webui/app.js | 352 +++++++++--------- scheduler_webui/index.html | 203 +++------- scheduler_webui/styles.css | 23 +- tests/test_scheduler_api.py | 40 ++ tests/test_scheduler_manager.py | 41 +- tests/test_scheduler_store.py | 7 + 15 files changed, 682 insertions(+), 457 deletions(-) diff --git a/README.md b/README.md index 5efd4ced1..fe678d769 100644 --- a/README.md +++ b/README.md @@ -221,13 +221,13 @@ uv run uvicorn api.main:app --port 8080 --reload #### 多实例调度器 -启动同一个 API 服务后，访问 `http://localhost:8080/scheduler` 可打开多实例调度器。调度器支持创建多个独立账号实例，每个实例拥有独立浏览器 Profile、CDP 端口、登录态、默认爬取参数和任务队列。实例默认参数、任务覆盖参数、内容过滤、Cookie 和代理都可以在页面中通过输入框、下拉菜单配置，无需手写 JSON。 +启动同一个 API 服务后，访问 `http://localhost:8080/scheduler` 可打开多实例调度器。调度器支持创建多个独立作业，每个作业拥有独立浏览器 Profile、CDP 端口、登录态、爬取目标和爬取参数。作业参数、内容过滤、Cookie 和代理都可以在页面中通过输入框、下拉菜单配置，无需手写 JSON。 调度器运行数据默认保存在 `data/scheduler/`： -- `scheduler.db`：实例、任务、日志和产物索引 -- `profiles/{instance_id}/`：实例独立浏览器 Profile -- `artifacts/{instance_id}/{task_id}/`：任务抓取产物 +- `scheduler.db`：作业配置、运行记录、日志和产物索引 +- `profiles/{job_id}/`：作业独立浏览器 Profile +- `artifacts/{job_id}/{task_id}/`：单次运行抓取产物 详细说明请查看：[多实例调度器使用指南](docs/多实例调度器使用指南.md) diff --git a/api/routers/scheduler.py b/api/routers/scheduler.py index 84e496777..63368a9b0 100644 --- a/api/routers/scheduler.py +++ b/api/routers/scheduler.py @@ -8,6 +8,9 @@ InstanceCreateRequest, InstanceResponse, InstanceUpdateRequest, + JobCreateRequest, + JobResponse, + JobUpdateRequest, SchedulerStatusResponse, TaskCreateRequest, TaskLogResponse, @@ -22,6 +25,90 @@ async def scheduler_status(): return scheduler_manager.status() +@router.get("/jobs", response_model=list[JobResponse]) +async def list_jobs(): + return scheduler_manager.list_jobs() + + +@router.post("/jobs", response_model=JobResponse) +async def create_job(request: JobCreateRequest): + return scheduler_manager.create_job(request) + + +@router.get("/jobs/{job_id}", response_model=JobResponse) +async def get_job(job_id: str): + job = scheduler_manager.get_job(job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + return job + + +@router.patch("/jobs/{job_id}", response_model=JobResponse) +async def update_job(job_id: str, request: JobUpdateRequest): + try: + job = await scheduler_manager.update_job(job_id, request) + except RuntimeError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + if not job: + raise HTTPException(status_code=404, detail="Job not found") + return job + + +@router.delete("/jobs/{job_id}") +async def delete_job(job_id: str): + try: + deleted = await scheduler_manager.delete_job(job_id) + except RuntimeError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + if not deleted: + raise HTTPException(status_code=404, detail="Job not found") + return {"status": "ok", "message": "Job deleted"} + + +@router.post("/jobs/{job_id}/login", response_model=TaskResponse) +async def login_job(job_id: str): + try: + return await scheduler_manager.login_job(job_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail="Job not found") from exc + except RuntimeError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.post("/jobs/{job_id}/run", response_model=TaskResponse) +async def run_job(job_id: str): + try: + return await scheduler_manager.run_job(job_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail="Job not found") from exc + except RuntimeError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.post("/jobs/{job_id}/stop", response_model=JobResponse) +async def stop_job(job_id: str): + job = await scheduler_manager.stop_job(job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + return job + + +@router.get("/jobs/{job_id}/logs", response_model=list[TaskLogResponse]) +async def list_job_logs(job_id: str, limit: int = Query(default=300, ge=1, le=1000)): + try: + return scheduler_manager.list_job_logs(job_id, limit=limit) + except KeyError as exc: + raise HTTPException(status_code=404, detail="Job not found") from exc + + +@router.get("/jobs/{job_id}/artifacts", response_model=list[ArtifactResponse]) +async def list_job_artifacts(job_id: str): + try: + return scheduler_manager.list_job_artifacts(job_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail="Job not found") from exc + + @router.get("/instances", response_model=list[InstanceResponse]) async def list_instances(): return scheduler_manager.list_instances() diff --git a/api/scheduler/manager.py b/api/scheduler/manager.py index c3d7c932f..1698d9d2a 100644 --- a/api/scheduler/manager.py +++ b/api/scheduler/manager.py @@ -12,7 +12,7 @@ from typing import Any, Optional from api.schemas.crawler import CrawlerTypeEnum -from .schemas import InstanceCreateRequest, InstanceUpdateRequest, TaskCreateRequest +from .schemas import JobCreateRequest, JobUpdateRequest, InstanceCreateRequest, InstanceUpdateRequest, TaskCreateRequest from .store import PROJECT_ROOT, SchedulerStore, _json_dumps, utc_now @@ -45,9 +45,15 @@ def __init__( def list_instances(self) -> list[dict[str, Any]]: return self.store.list_instances() + def list_jobs(self) -> list[dict[str, Any]]: + return self.store.list_instances() + def get_instance(self, instance_id: str) -> Optional[dict[str, Any]]: return self.store.get_instance(instance_id) + def get_job(self, job_id: str) -> Optional[dict[str, Any]]: + return self.store.get_instance(job_id) + def create_instance(self, request: InstanceCreateRequest) -> dict[str, Any]: payload = request.model_dump(mode="json") instance_id = uuid.uuid4().hex @@ -61,6 +67,9 @@ def create_instance(self, request: InstanceCreateRequest) -> dict[str, Any]: profile_dir = str(self.project_root / profile_path) return self.store.create_instance(payload, profile_dir, cdp_debug_port, instance_id=instance_id) + def create_job(self, request: JobCreateRequest) -> dict[str, Any]: + return self.create_instance(request) + async def update_instance(self, instance_id: str, request: InstanceUpdateRequest) -> Optional[dict[str, Any]]: async with self._lock: instance = self.store.get_instance(instance_id) @@ -78,8 +87,13 @@ async def update_instance(self, instance_id: str, request: InstanceUpdateRequest fields = request.model_dump(mode="json", exclude_unset=True) if "default_params" in fields: fields["default_params_json"] = _json_dumps(fields.pop("default_params")) + if "params" in fields: + fields["params_json"] = _json_dumps(fields.pop("params")) return self.store.update_instance(instance_id, **fields) + async def update_job(self, job_id: str, request: JobUpdateRequest) -> Optional[dict[str, Any]]: + return await self.update_instance(job_id, request) + async def delete_instance(self, instance_id: str) -> bool: async with self._lock: instance = self.store.get_instance(instance_id) @@ -89,6 +103,9 @@ async def delete_instance(self, instance_id: str) -> bool: raise RuntimeError("running instance cannot be deleted") return self.store.delete_instance(instance_id) + async def delete_job(self, job_id: str) -> bool: + return await self.delete_instance(job_id) + def list_tasks(self, instance_id: str | None = None, limit: int = 100) -> list[dict[str, Any]]: return self.store.list_tasks(instance_id=instance_id, limit=limit) @@ -100,22 +117,38 @@ async def create_task(self, request: TaskCreateRequest) -> dict[str, Any]: instance = self.store.get_instance(request.instance_id) if not instance: raise KeyError("instance not found") - if instance["status"] == "disabled": - raise RuntimeError("instance is disabled") - artifact_dir = self.artifact_dir / request.instance_id - payload = request.model_dump(mode="json") - task = self.store.create_task(payload, str(artifact_dir / "pending")) - real_artifact_dir = artifact_dir / task["id"] - task = self.store.update_task(task["id"], artifact_dir=str(real_artifact_dir)) - if instance["status"] in {"idle", "error"} and request.instance_id not in self._runtimes: - await self._start_task_locked(task) - task = self.store.get_task(task["id"]) + self._ensure_can_run(instance) + task = self._create_task_record( + request.instance_id, + request.crawler_type, + request.target_text, + request.params, + ) + self.store.update_instance(instance["id"], last_task_id=task["id"]) + await self._start_task_locked(task) + task = self.store.get_task(task["id"]) return task async def create_login_task(self, instance_id: str) -> dict[str, Any]: request = TaskCreateRequest(instance_id=instance_id, crawler_type=CrawlerTypeEnum.LOGIN, target_text="") return await self.create_task(request) + async def run_job(self, job_id: str, crawler_type: CrawlerTypeEnum | None = None) -> dict[str, Any]: + async with self._lock: + job = self.store.get_instance(job_id) + if not job: + raise KeyError("job not found") + self._ensure_can_run(job) + run_type = crawler_type or CrawlerTypeEnum(job["crawler_type"]) + target_text = "" if run_type == CrawlerTypeEnum.LOGIN else job.get("target_text", "") + task = self._create_task_record(job_id, run_type, target_text, job.get("params", {})) + self.store.update_instance(job_id, last_task_id=task["id"]) + await self._start_task_locked(task) + return self.store.get_task(task["id"]) + + async def login_job(self, job_id: str) -> dict[str, Any]: + return await self.run_job(job_id, CrawlerTypeEnum.LOGIN) + async def start_task(self, task_id: str) -> Optional[dict[str, Any]]: async with self._lock: task = self.store.get_task(task_id) @@ -123,6 +156,10 @@ async def start_task(self, task_id: str) -> Optional[dict[str, Any]]: return None if task["status"] != "queued": raise RuntimeError("only queued tasks can be started") + instance = self.store.get_instance(task["instance_id"]) + if not instance: + raise KeyError("instance not found") + self._ensure_can_run(instance) await self._start_task_locked(task) return self.store.get_task(task_id) @@ -147,15 +184,83 @@ async def cancel_task(self, task_id: str) -> Optional[dict[str, Any]]: pass return self.store.update_task(task_id, status="canceled", finished_at=utc_now()) + async def stop_job(self, job_id: str) -> Optional[dict[str, Any]]: + async with self._lock: + job = self.store.get_instance(job_id) + if not job: + return None + task_id = job.get("current_task_id") + if not task_id: + return job + task = self.store.get_task(task_id) + if not task or task["status"] != "running": + return job + runtime = self._runtimes.get(job_id) + if runtime and runtime.task_id == task_id: + runtime.canceled = True + self.store.append_log(task_id, "Stopping crawler subprocess ...", "warning") + self.store.update_instance(job_id, status="stopping") + try: + runtime.process.terminate() + except ProcessLookupError: + pass + self.store.update_task(task_id, status="canceled", finished_at=utc_now()) + return self.store.get_instance(job_id) + def list_logs(self, task_id: str, limit: int = 300) -> list[dict[str, Any]]: return self.store.list_logs(task_id, limit) + def list_job_logs(self, job_id: str, limit: int = 300) -> list[dict[str, Any]]: + job = self.store.get_instance(job_id) + if not job: + raise KeyError("job not found") + task_id = self._job_task_id(job) + return self.store.list_logs(task_id, limit) if task_id else [] + def list_artifacts(self, task_id: str) -> list[dict[str, Any]]: return self.store.list_artifacts(task_id) + def list_job_artifacts(self, job_id: str) -> list[dict[str, Any]]: + job = self.store.get_instance(job_id) + if not job: + raise KeyError("job not found") + task_id = self._job_task_id(job) + return self.store.list_artifacts(task_id) if task_id else [] + def status(self) -> dict[str, int]: return self.store.scheduler_counts() + def _create_task_record( + self, + instance_id: str, + crawler_type: CrawlerTypeEnum | str, + target_text: str, + params: dict[str, Any] | None, + ) -> dict[str, Any]: + artifact_dir = self.artifact_dir / instance_id + payload = { + "instance_id": instance_id, + "crawler_type": str(crawler_type.value if isinstance(crawler_type, CrawlerTypeEnum) else crawler_type), + "target_text": target_text or "", + "params": params or {}, + } + task = self.store.create_task(payload, str(artifact_dir / "pending")) + real_artifact_dir = artifact_dir / task["id"] + return self.store.update_task(task["id"], artifact_dir=str(real_artifact_dir)) + + def _ensure_can_run(self, instance: dict[str, Any]) -> None: + if instance["status"] == "disabled": + raise RuntimeError("job is disabled") + if instance["status"] in {"running", "stopping"} or instance["id"] in self._runtimes: + raise RuntimeError("job is already running") + + def _job_task_id(self, job: dict[str, Any]) -> str: + task_id = job.get("current_task_id") or job.get("last_task_id") + if task_id: + return task_id + latest_task = self.store.get_latest_task(job["id"]) + return latest_task["id"] if latest_task else "" + async def _start_task_locked(self, task: dict[str, Any]) -> None: instance = self.store.get_instance(task["instance_id"]) if not instance: @@ -184,7 +289,14 @@ async def _start_task_locked(self, task: dict[str, Any]) -> None: message = f"Failed to start crawler: {type(exc).__name__}: {exc}" self.store.append_log(task["id"], message, "error") self.store.update_task(task["id"], status="failed", error_message=message, finished_at=utc_now()) - self.store.update_instance(instance["id"], status="error", current_task_id=None, pid=None, last_error=message) + self.store.update_instance( + instance["id"], + status="error", + current_task_id=None, + last_task_id=task["id"], + pid=None, + last_error=message, + ) return runtime = InstanceRuntime(instance_id=instance["id"], task_id=task["id"], process=process) @@ -197,13 +309,14 @@ async def _start_task_locked(self, task: dict[str, Any]) -> None: instance["id"], status="running", current_task_id=task["id"], + last_task_id=task["id"], pid=process.pid, last_error="", ) self.store.append_log(task["id"], f"Crawler subprocess started, pid={process.pid}", "success") def _build_command(self, instance: dict[str, Any], task: dict[str, Any]) -> list[str]: - params = {**instance.get("default_params", {}), **task.get("params", {})} + params = {**instance.get("default_params", {}), **instance.get("params", {}), **task.get("params", {})} crawler_type = task["crawler_type"] save_option = str(params.get("save_option", instance["save_option"])) headless = self._as_bool(params.get("headless", instance["headless"])) @@ -324,17 +437,13 @@ async def _watch_process(self, runtime: InstanceRuntime) -> None: runtime.instance_id, status=instance_status, current_task_id=None, + last_task_id=task["id"], pid=None, last_error=error_message, ) self.store.append_log(task["id"], message, level) self._runtimes.pop(runtime.instance_id, None) - if task_status in {"succeeded", "canceled"}: - next_task = self.store.get_next_queued_task(runtime.instance_id) - if next_task: - await self._start_task_locked(next_task) - def _scan_artifacts(self, root: Path) -> list[dict[str, Any]]: if not root.exists(): return [] diff --git a/api/scheduler/schemas.py b/api/scheduler/schemas.py index 1ba2cd728..f14434142 100644 --- a/api/scheduler/schemas.py +++ b/api/scheduler/schemas.py @@ -44,6 +44,9 @@ class InstanceCreateRequest(BaseModel): browser_profile_dir: str = "" cdp_debug_port: Optional[int] = Field(default=None, ge=1000, le=65535) default_params: Dict[str, Any] = Field(default_factory=dict) + crawler_type: CrawlerTypeEnum = CrawlerTypeEnum.SEARCH + target_text: str = "" + params: Dict[str, Any] = Field(default_factory=dict) @field_validator("name") @classmethod @@ -58,12 +61,16 @@ class InstanceUpdateRequest(BaseModel): """Update mutable crawler instance settings.""" name: Optional[str] = Field(default=None, min_length=1, max_length=80) + platform: Optional[PlatformEnum] = None login_type: Optional[LoginTypeEnum] = None headless: Optional[bool] = None save_option: Optional[SaveDataOptionEnum] = None browser_profile_dir: Optional[str] = None cdp_debug_port: Optional[int] = Field(default=None, ge=1000, le=65535) default_params: Optional[Dict[str, Any]] = None + crawler_type: Optional[CrawlerTypeEnum] = None + target_text: Optional[str] = None + params: Optional[Dict[str, Any]] = None status: Optional[InstanceStatusEnum] = None @field_validator("name") @@ -87,14 +94,30 @@ class InstanceResponse(BaseModel): browser_profile_dir: str cdp_debug_port: int default_params: Dict[str, Any] + crawler_type: str + target_text: str + params: Dict[str, Any] status: str current_task_id: Optional[str] = None + last_task_id: Optional[str] = None pid: Optional[int] = None last_error: str = "" created_at: str updated_at: str +class JobCreateRequest(InstanceCreateRequest): + """Create a scheduler job.""" + + +class JobUpdateRequest(InstanceUpdateRequest): + """Update a scheduler job.""" + + +class JobResponse(InstanceResponse): + """Scheduler job configuration with current run status.""" + + class TaskCreateRequest(BaseModel): """Create a crawler task bound to one instance.""" diff --git a/api/scheduler/store.py b/api/scheduler/store.py index 1aa97c667..434a7bc75 100644 --- a/api/scheduler/store.py +++ b/api/scheduler/store.py @@ -59,8 +59,12 @@ def _init_db(self) -> None: browser_profile_dir TEXT NOT NULL, cdp_debug_port INTEGER NOT NULL, default_params_json TEXT NOT NULL, + crawler_type TEXT NOT NULL DEFAULT 'search', + target_text TEXT NOT NULL DEFAULT '', + params_json TEXT NOT NULL DEFAULT '{}', status TEXT NOT NULL, current_task_id TEXT, + last_task_id TEXT, pid INTEGER, last_error TEXT NOT NULL DEFAULT '', created_at TEXT NOT NULL, @@ -106,10 +110,23 @@ def _init_db(self) -> None: ); """ ) + self._ensure_instance_columns(conn) conn.execute("CREATE INDEX IF NOT EXISTS idx_tasks_instance_status ON tasks(instance_id, status, created_at)") conn.execute("CREATE INDEX IF NOT EXISTS idx_task_logs_task ON task_logs(task_id, id)") conn.execute("CREATE INDEX IF NOT EXISTS idx_artifacts_task ON artifacts(task_id)") + def _ensure_instance_columns(self, conn: sqlite3.Connection) -> None: + columns = {row["name"] for row in conn.execute("PRAGMA table_info(instances)").fetchall()} + additions = { + "crawler_type": "ALTER TABLE instances ADD COLUMN crawler_type TEXT NOT NULL DEFAULT 'search'", + "target_text": "ALTER TABLE instances ADD COLUMN target_text TEXT NOT NULL DEFAULT ''", + "params_json": "ALTER TABLE instances ADD COLUMN params_json TEXT NOT NULL DEFAULT '{}'", + "last_task_id": "ALTER TABLE instances ADD COLUMN last_task_id TEXT", + } + for column, statement in additions.items(): + if column not in columns: + conn.execute(statement) + def create_instance( self, payload: dict[str, Any], @@ -125,9 +142,10 @@ def create_instance( INSERT INTO instances ( id, name, platform, login_type, headless, save_option, browser_profile_dir, cdp_debug_port, default_params_json, - status, current_task_id, pid, last_error, created_at, updated_at + crawler_type, target_text, params_json, + status, current_task_id, last_task_id, pid, last_error, created_at, updated_at ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'idle', NULL, NULL, '', ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'idle', NULL, NULL, NULL, '', ?, ?) """, ( instance_id, @@ -139,6 +157,9 @@ def create_instance( profile_dir, cdp_debug_port, _json_dumps(payload.get("default_params")), + payload.get("crawler_type", "search"), + payload.get("target_text", ""), + _json_dumps(payload.get("params")), now, now, ), @@ -160,14 +181,19 @@ def update_instance(self, instance_id: str, **fields: Any) -> Optional[dict[str, return self.get_instance(instance_id) allowed = { "name", + "platform", "login_type", "headless", "save_option", "browser_profile_dir", "cdp_debug_port", "default_params_json", + "crawler_type", + "target_text", + "params_json", "status", "current_task_id", + "last_task_id", "pid", "last_error", } @@ -258,6 +284,19 @@ def get_next_queued_task(self, instance_id: str) -> Optional[dict[str, Any]]: ).fetchone() return self._task_from_row(row) if row else None + def get_latest_task(self, instance_id: str) -> Optional[dict[str, Any]]: + with self._lock, self._connect() as conn: + row = conn.execute( + """ + SELECT * FROM tasks + WHERE instance_id = ? + ORDER BY created_at DESC + LIMIT 1 + """, + (instance_id,), + ).fetchone() + return self._task_from_row(row) if row else None + def update_task(self, task_id: str, **fields: Any) -> Optional[dict[str, Any]]: if not fields: return self.get_task(task_id) @@ -365,6 +404,7 @@ def _instance_from_row(self, row: sqlite3.Row) -> dict[str, Any]: data = dict(row) data["headless"] = bool(data["headless"]) data["default_params"] = _json_loads(data.pop("default_params_json", None)) + data["params"] = _json_loads(data.pop("params_json", None)) return data def _task_from_row(self, row: sqlite3.Row) -> dict[str, Any]: diff --git a/docs/index.md b/docs/index.md index 556466890..6ab8c9b8d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -3,7 +3,7 @@ ## 项目文档 - [项目架构文档](项目架构文档.md) - 系统架构、模块设计、数据流向（含 Mermaid 图表） -- [多实例调度器使用指南](多实例调度器使用指南.md) - 多账号实例、任务队列、调度器 WebUI 和 API +- [多实例调度器使用指南](多实例调度器使用指南.md) - 多账号作业、运行记录、调度器 WebUI 和 API - [内容过滤使用指南](内容过滤使用指南.md) - 按点赞、收藏、转发、评论等互动指标筛选爬取目标 ## 推荐：使用 uv 管理依赖 @@ -60,7 +60,7 @@ uv run main.py --help uv run uvicorn api.main:app --port 8080 --reload ``` -访问 `http://localhost:8080/scheduler` 创建实例、维护登录态、投递任务并查看日志和产物。详细说明见 [多实例调度器使用指南](多实例调度器使用指南.md)。 +访问 `http://localhost:8080/scheduler` 创建作业、维护登录态、启动运行并查看日志和产物。详细说明见 [多实例调度器使用指南](多实例调度器使用指南.md)。 ## 备选：Python 原生 venv（不推荐） > 如果爬取抖音或知乎，需要提前安装 Node.js，版本 `>= 16`。 diff --git "a/docs/\345\206\205\345\256\271\350\277\207\346\273\244\344\275\277\347\224\250\346\214\207\345\215\227.md" "b/docs/\345\206\205\345\256\271\350\277\207\346\273\244\344\275\277\347\224\250\346\214\207\345\215\227.md" index c28ab0381..3d7863bae 100644 --- "a/docs/\345\206\205\345\256\271\350\277\207\346\273\244\344\275\277\347\224\250\346\214\207\345\215\227.md" +++ "b/docs/\345\206\205\345\256\271\350\277\207\346\273\244\344\275\277\347\224\250\346\214\207\345\215\227.md" @@ -37,11 +37,11 @@ CONTENT_FILTERS = { 字段不支持时会在启动阶段报错，不会静默忽略。部分历史字段名保留了兼容别名，例如快手 `viewd_count`、微博 `comments_count`、B 站 `video_favorite_count`。 -## 3. 调度器任务参数 +## 3. 调度器作业参数 -多实例调度器 WebUI 中，实例表单和任务表单都有“内容过滤”区域。点击“添加过滤条件”，选择指标并填写最小值或最大值即可，不需要手写 JSON。任务过滤条件会覆盖实例默认过滤条件；留空则沿用实例默认参数。 +多实例调度器 WebUI 中，作业表单有“内容过滤”区域。点击“添加过滤条件”，选择指标并填写最小值或最大值即可，不需要手写 JSON。 -如果直接调用调度器 API，实例的 `default_params` 和任务的 `params` 都支持 `content_filters`： +如果直接调用调度器 API，作业的 `params` 支持 `content_filters`： ```json { diff --git "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" index f7df75118..9565a1bfb 100644 --- "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" +++ "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" @@ -1,15 +1,13 @@ # 多实例调度器使用指南 -本文档说明 MediaCrawler 多实例调度器的设计目标、运行方式、数据目录和 API。调度器用于在现有单实例爬虫基础上管理多个独立账号实例，每个实例拥有独立的登录态、浏览器 Profile、CDP 端口、默认爬取参数和任务队列。 +本文档说明 MediaCrawler 多实例调度器的运行方式、数据目录和 API。调度器面向用户只暴露“作业”：一个作业同时包含账号实例、登录态、浏览器 Profile、爬取目标和爬取参数。每个作业同一时间只能运行一次；内部仍保留运行记录，用于日志、产物和退出状态追踪。 ## 1. 功能边界 -多实例调度器解决以下问题： - -- 多个账号实例独立保存登录凭证和浏览器上下文，避免账号状态互相污染。 -- 每个实例维护自己的平台、登录方式、保存格式、CDP 端口和默认参数。 -- 调度器统一创建任务、启动爬虫子进程、停止任务、收集日志、扫描产物。 -- WebUI 提供实例创建、登录、任务投递、日志查看和产物查看。 +- 每个作业维护自己的平台、登录方式、保存格式、CDP 端口、浏览器 Profile、爬取类型、目标和参数。 +- WebUI 提供作业创建、编辑、登录、运行、停止、日志查看和产物查看。 +- 调度器统一拉起 `uv run python main.py` 子进程，并按作业隔离登录态和产物目录。 +- 不再提供同一实例的任务队列；如果作业正在运行，再次运行会返回“作业正在运行”的错误。 调度器不改变平台爬虫的核心抓取逻辑。底层仍然通过 `main.py`、`cmd_arg/arg.py` 和各平台 `core.py` 执行搜索、详情、创作者和登录流程。 @@ -18,7 +16,7 @@ ```mermaid flowchart TB user["用户"] --> webui["/scheduler WebUI"] - webui --> api["/api/scheduler/*"] + webui --> api["/api/scheduler/jobs"] subgraph scheduler["调度器控制面"] router["api/routers/scheduler.py"] @@ -31,20 +29,20 @@ flowchart TB manager --> store subgraph runtime["爬虫运行面"] - p1["实例 A 子进程
          uv run python main.py"] - p2["实例 B 子进程
          uv run python main.py"] - profile1["Profile A"] - profile2["Profile B"] - artifact1["产物 A"] - artifact2["产物 B"] + job_a["作业 A 子进程
          uv run python main.py"] + job_b["作业 B 子进程
          uv run python main.py"] + profile_a["Profile A"] + profile_b["Profile B"] + artifact_a["产物 A"] + artifact_b["产物 B"] end - manager --> p1 - manager --> p2 - p1 --> profile1 - p2 --> profile2 - p1 --> artifact1 - p2 --> artifact2 + manager --> job_a + manager --> job_b + job_a --> profile_a + job_b --> profile_b + job_a --> artifact_a + job_b --> artifact_b ``` ## 3. 数据目录 @@ -53,11 +51,11 @@ flowchart TB | 目录或文件 | 用途 | | --- | --- | -| `data/scheduler/scheduler.db` | 实例、任务、日志和产物索引 | -| `data/scheduler/profiles/{instance_id}/` | 实例独立浏览器 Profile | -| `data/scheduler/artifacts/{instance_id}/{task_id}/` | 单个任务的抓取产物 | +| `data/scheduler/scheduler.db` | 作业配置、内部运行记录、日志和产物索引 | +| `data/scheduler/profiles/{job_id}/` | 作业独立浏览器 Profile | +| `data/scheduler/artifacts/{job_id}/{task_id}/` | 单次运行的抓取产物 | -如果创建实例时手动填写浏览器 Profile 目录，调度器会使用该目录；相对路径会基于项目根目录解析。 +如果创建作业时手动填写浏览器 Profile 目录，调度器会使用该目录；相对路径会基于项目根目录解析。 ## 4. 启动方式 @@ -75,26 +73,26 @@ uv run uvicorn api.main:app --port 8080 --reload ## 5. 使用流程 -调度器页面自上而下排列为：实例列表、新建实例、任务列表、新建任务，最后是任务日志和产物详情。新建实例、新建任务和详情区采用纵向单列表单，避免在宽屏下左右跳读。 +调度器页面自上而下排列为：作业列表、新建/编辑作业、日志、产物。 1. 打开 `/scheduler`。 -2. 先查看实例列表；如果没有目标实例，再在下方创建实例，选择平台、登录方式、保存格式，必要时指定 CDP 端口，并通过输入框、下拉菜单配置默认抓取参数、内容过滤、Cookie 和代理。 -3. 点击实例行的“登录”，调度器会创建 `login` 任务，子进程只完成登录态维护，不进入抓取流程。 -4. 查看任务列表；需要新任务时，在任务列表下方创建搜索、详情或创作者任务，调度器会把任务加入该实例队列。 -5. 选择任务后查看日志和产物列表。 +2. 创建作业，填写平台、登录方式、保存格式、爬取类型、目标、抓取参数、内容过滤、Cookie 和代理。 +3. 点击作业行的“登录”，调度器会创建一次内部 `login` 运行记录，只维护登录态，不进入抓取流程。 +4. 点击“运行”，调度器按作业配置创建一次内部运行记录并启动子进程。 +5. 选择作业后查看最近一次运行的日志和产物；作业正在运行时日志和产物指向当前运行。 -实例处于 `running` 或 `stopping` 时不能修改核心配置，也不能删除实例。任务执行成功或被取消后，同实例队列中的下一个任务会自动启动；任务失败时实例进入 `error`，需要人工查看日志后再投递或启动后续任务。 +作业处于 `running` 或 `stopping` 时不能编辑或删除，只能停止。作业失败后状态显示 `error`，可以修改配置后重新运行。 -## 6. 任务参数 +## 6. 作业参数 -调度器 WebUI 已把实例默认参数和任务覆盖参数拆成表单控件，不需要手写 JSON： +调度器 WebUI 已把参数拆成表单控件，不需要手写 JSON： -- 实例侧的“默认抓取参数”用于配置起始页、最大内容数、单条评论数、并发数、评论和二级评论。布尔项支持“项目默认 / 开启 / 关闭”，选择“项目默认”时不会写入覆盖值。 -- 任务侧的“任务覆盖参数”用于临时覆盖实例默认值。下拉项选择“沿用实例默认”时不会覆盖实例配置。 -- “内容过滤”按当前平台展示可选指标，点击“添加过滤条件”后填写最小值或最大值即可。数值支持 `1000`、`1万`、`2.5万` 这类写法。 -- “登录与代理”用于配置 Cookie、是否连接已有浏览器、是否启用代理、代理池数量、代理服务和固定代理地址。 +- 作业基础信息：平台、登录方式、保存格式、爬取类型、目标、CDP 端口和 Profile 目录。 +- 抓取参数：起始页、最大内容数、单条评论数、并发数、一级评论和二级评论。 +- 内容过滤：按当前平台展示可选指标，点击“添加过滤条件”后填写最小值或最大值。数值支持 `1000`、`1万`、`2.5万` 这类写法。 +- 登录与代理：Cookie、是否连接已有浏览器、是否启用代理、代理池数量、代理服务和固定代理地址。 -WebUI 提交后会在内部生成 `default_params` 或 `params`。实例默认参数和任务参数仍按同一规则合并，任务参数优先级更高。API 调用仍使用 JSON，常用字段如下： +WebUI 提交后会在内部生成作业 `params`。API 调用仍使用 JSON，常用字段如下： | 字段 | 对应 CLI 参数 | 示例 | | --- | --- | --- | @@ -111,60 +109,40 @@ WebUI 提交后会在内部生成 `default_params` 或 `params`。实例默认 | `ip_proxy_provider_name` | `--ip_proxy_provider_name` | `"static"` | | `static_proxy_url` | `--static_proxy_url` | `"http://host:port"` | -搜索任务的“目标”会映射为 `--keywords`，详情任务映射为 `--specified_id`，创作者任务映射为 `--creator_id`。 +作业目标会按爬取类型映射：搜索映射为 `--keywords`，详情映射为 `--specified_id`，创作者映射为 `--creator_id`。 ## 7. API 参考 -### 7.1 实例 API +### 7.1 作业 API | 方法 | 路径 | 说明 | | --- | --- | --- | | `GET` | `/api/scheduler/status` | 获取调度器统计 | -| `GET` | `/api/scheduler/instances` | 获取实例列表 | -| `POST` | `/api/scheduler/instances` | 创建实例 | -| `GET` | `/api/scheduler/instances/{instance_id}` | 获取实例详情 | -| `PATCH` | `/api/scheduler/instances/{instance_id}` | 修改实例配置 | -| `DELETE` | `/api/scheduler/instances/{instance_id}` | 删除空闲实例 | -| `POST` | `/api/scheduler/instances/{instance_id}/login` | 为实例创建登录任务 | - -创建实例示例： +| `GET` | `/api/scheduler/jobs` | 获取作业列表 | +| `POST` | `/api/scheduler/jobs` | 创建作业 | +| `GET` | `/api/scheduler/jobs/{job_id}` | 获取作业详情 | +| `PATCH` | `/api/scheduler/jobs/{job_id}` | 修改空闲作业配置 | +| `DELETE` | `/api/scheduler/jobs/{job_id}` | 删除空闲作业 | +| `POST` | `/api/scheduler/jobs/{job_id}/login` | 为作业创建登录运行 | +| `POST` | `/api/scheduler/jobs/{job_id}/run` | 按作业配置启动一次运行 | +| `POST` | `/api/scheduler/jobs/{job_id}/stop` | 停止当前运行 | +| `GET` | `/api/scheduler/jobs/{job_id}/logs` | 获取当前或最近一次运行日志 | +| `GET` | `/api/scheduler/jobs/{job_id}/artifacts` | 获取当前或最近一次运行产物 | + +创建作业示例： ```json { - "name": "小红书账号 A", + "name": "小红书作业 A", "platform": "xhs", "login_type": "qrcode", "save_option": "jsonl", - "headless": false, - "default_params": { - "enable_comments": true, - "max_notes_count": 20 - } -} -``` - -### 7.2 任务 API - -| 方法 | 路径 | 说明 | -| --- | --- | --- | -| `GET` | `/api/scheduler/tasks` | 获取最近任务 | -| `POST` | `/api/scheduler/tasks` | 创建任务并在实例空闲时自动启动 | -| `GET` | `/api/scheduler/tasks/{task_id}` | 获取任务详情 | -| `POST` | `/api/scheduler/tasks/{task_id}/start` | 手动启动排队任务 | -| `POST` | `/api/scheduler/tasks/{task_id}/cancel` | 取消排队任务或停止运行任务 | -| `GET` | `/api/scheduler/tasks/{task_id}/logs` | 获取任务日志 | -| `GET` | `/api/scheduler/tasks/{task_id}/artifacts` | 获取任务产物索引 | - -创建任务示例： - -```json -{ - "instance_id": "实例 ID", "crawler_type": "search", "target_text": "编程副业,AI 工具", + "headless": false, "params": { "enable_comments": true, - "max_comments_count": 10, + "max_notes_count": 20, "content_filters": { "liked_count": {"min": 1000}, "comment_count": {"min": 20} @@ -173,23 +151,25 @@ WebUI 提交后会在内部生成 `default_params` 或 `params`。实例默认 } ``` -`content_filters` 会在内容入库、媒体下载和评论抓取前生效。过滤条件不改变 `max_notes_count` 的扫描预算；命中多少就保存多少，不会为了补齐数量继续翻页。 +### 7.2 兼容 API + +旧 `/api/scheduler/instances` 和 `/api/scheduler/tasks` 接口暂时保留，用于兼容已有调用和测试。新版 WebUI 不再直接暴露“实例”和“任务”两个概念；内部 `tasks` 表只作为运行记录使用。 ## 8. 与原单实例模式的关系 -原单实例 WebUI 和 `/api/crawler/*` 接口仍然保留，适合临时手动运行一个爬虫任务。多实例调度器使用 `/api/scheduler/*`，每个任务都会启动独立子进程，并显式传入： +原单实例 WebUI 和 `/api/crawler/*` 接口仍然保留，适合临时手动运行一个爬虫任务。多实例调度器使用 `/api/scheduler/jobs/*`，每次运行都会启动独立子进程，并显式传入： - `--instance_id` - `--browser_profile_dir` - `--cdp_debug_port` - `--cdp_connect_existing false` -- `--save_data_path data/scheduler/artifacts/{instance_id}/{task_id}` +- `--save_data_path data/scheduler/artifacts/{job_id}/{task_id}` -因此，多实例模式下账号登录态、浏览器状态和抓取产物都按实例或任务隔离。 +因此，多实例模式下账号登录态、浏览器状态和抓取产物都按作业或单次运行隔离。 ## 9. 注意事项 - 本功能仍然应遵守项目非商业学习用途声明，不应扩大抓取规模或增加目标平台压力。 -- 多实例同时运行会同时启动多个浏览器上下文，需要预留足够 CPU、内存和端口。 -- 如果使用 Cookie 登录，可以把 `cookies` 放在实例默认参数中；如果使用二维码登录，建议先执行登录任务再投递抓取任务。 -- 如果任务失败，优先查看任务日志和对应实例的 `last_error` 字段。 +- 多个作业同时运行会同时启动多个浏览器上下文，需要预留足够 CPU、内存和端口。 +- 如果使用 Cookie 登录，可以把 `cookies` 放在作业参数中；如果使用二维码登录，建议先执行登录再运行抓取。 +- 如果作业失败，优先查看作业日志和 `last_error` 字段。 diff --git "a/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" "b/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" index 26938bd22..f7821f317 100644 --- "a/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" +++ "b/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" @@ -148,7 +148,7 @@ flowchart LR ### 2.3 多实例调度器架构 -在原有单实例爬虫链路之外，项目新增了一个独立调度器控制面。调度器不重写各平台抓取逻辑，而是通过 `uv run python main.py` 拉起多个子进程，并为每个实例注入独立的登录态目录、CDP 端口、默认参数和产物目录。 +在原有单实例爬虫链路之外，项目新增了一个独立调度器控制面。调度器不重写各平台抓取逻辑，而是通过 `uv run python main.py` 拉起多个子进程，并为每个作业注入独立的登录态目录、CDP 端口、爬取参数和产物目录。 ```mermaid flowchart TB @@ -156,8 +156,8 @@ flowchart TB scheduler_api --> manager["SchedulerManager"] manager --> sqlite["SchedulerStore
          data/scheduler/scheduler.db"] - manager --> task_a["实例 A 任务子进程"] - manager --> task_b["实例 B 任务子进程"] + manager --> task_a["作业 A 子进程"] + manager --> task_b["作业 B 子进程"] task_a --> profile_a["data/scheduler/profiles/A"] task_b --> profile_b["data/scheduler/profiles/B"] @@ -168,9 +168,9 @@ flowchart TB 调度器核心职责： -- 维护实例配置：平台、登录方式、保存格式、CDP 端口、浏览器 Profile、默认参数。 -- 维护任务队列：搜索、详情、创作者和登录任务都绑定到具体实例。 -- 管理进程生命周期：实例空闲时启动队列任务，取消任务时终止对应子进程。 +- 维护作业配置：平台、登录方式、保存格式、CDP 端口、浏览器 Profile、爬取类型、目标和参数。 +- 维护内部运行记录：登录、搜索、详情和创作者运行都绑定到具体作业。 +- 管理进程生命周期：作业空闲时才能启动运行，停止运行时终止对应子进程。 - 采集运行状态：读取子进程 stdout，落库任务日志、退出码和错误信息。 - 收集抓取产物：任务结束后扫描任务产物目录，记录文件类型、大小和记录数。 @@ -179,7 +179,7 @@ flowchart TB | 模块 | 说明 | |------|------| | `api/routers/scheduler.py` | 调度器 HTTP API | -| `api/scheduler/manager.py` | 任务队列、子进程和产物扫描 | +| `api/scheduler/manager.py` | 作业运行、子进程和产物扫描 | | `api/scheduler/store.py` | SQLite 状态库 | | `scheduler_webui/` | 调度器管理页面 | | `tools/profile.py` | 实例浏览器 Profile 路径解析 | diff --git a/scheduler_webui/app.js b/scheduler_webui/app.js index 98769fdbf..9e4327526 100644 --- a/scheduler_webui/app.js +++ b/scheduler_webui/app.js @@ -4,7 +4,7 @@ const api = { if (!res.ok) throw new Error(await errorText(res)); return res.json(); }, - async post(path, body) { + async post(path, body = {}) { const res = await fetch(path, { method: "POST", headers: { "Content-Type": "application/json" }, @@ -13,15 +13,28 @@ const api = { if (!res.ok) throw new Error(await errorText(res)); return res.json(); }, + async patch(path, body) { + const res = await fetch(path, { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + if (!res.ok) throw new Error(await errorText(res)); + return res.json(); + }, + async delete(path) { + const res = await fetch(path, { method: "DELETE" }); + if (!res.ok) throw new Error(await errorText(res)); + return res.json(); + }, }; let state = { - instances: [], - tasks: [], - selectedTaskId: "", + jobs: [], + selectedJobId: "", + editingJobId: "", filterRows: { - instance: [], - task: [], + job: [], }, }; @@ -70,17 +83,15 @@ const filterFields = { const els = { summary: document.querySelector("#summary"), refreshBtn: document.querySelector("#refreshBtn"), - instanceForm: document.querySelector("#instanceForm"), - taskForm: document.querySelector("#taskForm"), - instancesBody: document.querySelector("#instancesBody"), - tasksBody: document.querySelector("#tasksBody"), - taskInstanceSelect: document.querySelector("#taskInstanceSelect"), - instanceHint: document.querySelector("#instanceHint"), - taskHint: document.querySelector("#taskHint"), + jobForm: document.querySelector("#jobForm"), + jobsBody: document.querySelector("#jobsBody"), + jobHint: document.querySelector("#jobHint"), + formTitle: document.querySelector("#formTitle"), + jobSubmitBtn: document.querySelector("#jobSubmitBtn"), + cancelEditBtn: document.querySelector("#cancelEditBtn"), logsBox: document.querySelector("#logsBox"), artifactsList: document.querySelector("#artifactsList"), - instanceFilterRows: document.querySelector("#instanceFilterRows"), - taskFilterRows: document.querySelector("#taskFilterRows"), + jobFilterRows: document.querySelector("#jobFilterRows"), }; function statusPill(value) { @@ -106,77 +117,43 @@ async function errorText(res) { } async function refreshAll() { - const [status, instances, tasks] = await Promise.all([ + const [status, jobs] = await Promise.all([ api.get("/api/scheduler/status"), - api.get("/api/scheduler/instances"), - api.get("/api/scheduler/tasks?limit=100"), + api.get("/api/scheduler/jobs"), ]); - state.instances = instances; - state.tasks = tasks; + state.jobs = jobs; renderStatus(status); - renderInstances(); - renderTasks(); - renderTaskSelect(); - if (state.selectedTaskId) await loadTaskDetail(state.selectedTaskId); + renderJobs(); + if (state.selectedJobId) await loadJobDetail(state.selectedJobId); } function renderStatus(status) { - els.summary.textContent = - `实例 ${status.instances_total} 个，运行中 ${status.running_instances} 个，` + - `排队任务 ${status.queued_tasks} 个，运行任务 ${status.running_tasks} 个`; -} - -function renderInstances() { - els.instanceHint.textContent = `${state.instances.length} 个实例`; - els.instancesBody.innerHTML = state.instances - .map((item) => { - const canRun = !["running", "stopping"].includes(item.status); - return ` - - ${escapeHtml(item.name)}
          ${escapeHtml(item.id)} - ${escapeHtml(item.platform)} - ${statusPill(item.status)} - ${escapeHtml(item.cdp_debug_port)} - ${escapeHtml(item.current_task_id || "-")} - -
          - -
          - - - `; - }) - .join(""); -} - -function renderTaskSelect() { - const selectedId = els.taskInstanceSelect.value; - els.taskInstanceSelect.innerHTML = state.instances - .map((item) => ``) - .join(""); - if (selectedId && state.instances.some((item) => item.id === selectedId)) { - els.taskInstanceSelect.value = selectedId; - } - syncFilterRowsWithPlatform("task"); + els.summary.textContent = `作业 ${status.instances_total} 个，运行中 ${status.running_instances} 个`; } -function renderTasks() { - els.taskHint.textContent = `${state.tasks.length} 个最近任务`; - els.tasksBody.innerHTML = state.tasks - .map((item) => { - const instance = state.instances.find((it) => it.id === item.instance_id); - const canCancel = ["queued", "running"].includes(item.status); +function renderJobs() { + els.jobHint.textContent = `${state.jobs.length} 个作业`; + els.jobsBody.innerHTML = state.jobs + .map((job) => { + const canRun = !["running", "stopping"].includes(job.status); + const target = job.target_text || "-"; + const taskId = job.current_task_id || job.last_task_id || "-"; return ` - - ${escapeHtml(instance ? instance.name : item.instance_id)} - ${escapeHtml(item.crawler_type)} - ${statusPill(item.status)} - ${escapeHtml(item.target_text || "-")} + ${escapeHtml(job.name)}
          ${escapeHtml(job.id)} + ${escapeHtml(job.platform)} + ${escapeHtml(job.crawler_type)} + ${statusPill(job.status)} + ${escapeHtml(target)} + ${escapeHtml(taskId === "-" ? "-" : taskId.slice(0, 8))}
          - - + + + + + +
          @@ -186,11 +163,9 @@ function renderTasks() { } function renderFilterRows(scope) { - const platform = currentPlatform(scope); - const target = scope === "instance" ? els.instanceFilterRows : els.taskFilterRows; + const platform = currentPlatform(); const rows = state.filterRows[scope]; - - target.innerHTML = rows.length + els.jobFilterRows.innerHTML = rows.length ? rows .map((row, index) => `
          @@ -221,16 +196,12 @@ function metricOptions(platform, selectedMetric) { .join(""); } -function currentPlatform(scope) { - if (scope === "instance") { - return els.instanceForm.elements.platform.value || "xhs"; - } - const instanceId = els.taskInstanceSelect.value; - return state.instances.find((item) => item.id === instanceId)?.platform || "xhs"; +function currentPlatform() { + return els.jobForm.elements.platform.value || "xhs"; } function addFilterRow(scope) { - const [metric] = (filterFields[currentPlatform(scope)] || filterFields.xhs)[0]; + const [metric] = (filterFields[currentPlatform()] || filterFields.xhs)[0]; state.filterRows[scope].push({ metric, min: "", max: "" }); renderFilterRows(scope); } @@ -240,15 +211,15 @@ function removeFilterRow(scope, index) { renderFilterRows(scope); } -function syncFilterRowsWithPlatform(scope) { - const fields = filterFields[currentPlatform(scope)] || filterFields.xhs; +function syncFilterRowsWithPlatform() { + const fields = filterFields[currentPlatform()] || filterFields.xhs; const allowed = new Set(fields.map(([value]) => value)); const fallback = fields[0][0]; - state.filterRows[scope] = state.filterRows[scope].map((row) => ({ + state.filterRows.job = state.filterRows.job.map((row) => ({ ...row, metric: allowed.has(row.metric) ? row.metric : fallback, })); - renderFilterRows(scope); + renderFilterRows("job"); } function updateFilterValue(target) { @@ -259,8 +230,8 @@ function updateFilterValue(target) { row[target.dataset.field] = target.value; } -function collectFilters(scope) { - return state.filterRows[scope].reduce((filters, row) => { +function collectFilters() { + return state.filterRows.job.reduce((filters, row) => { const min = String(row.min || "").trim(); const max = String(row.max || "").trim(); if (!row.metric || (!min && !max)) return filters; @@ -286,48 +257,101 @@ function setBoolSelectParam(params, key, value) { if (value === "false") params[key] = false; } -function collectDefaultParams(data) { +function collectJobParams(data) { const params = {}; - setBoolSelectParam(params, "enable_comments", data.get("default_enable_comments")); - setBoolSelectParam(params, "enable_sub_comments", data.get("default_enable_sub_comments")); - setBoolSelectParam(params, "cdp_connect_existing", data.get("default_cdp_connect_existing")); - setBoolSelectParam(params, "enable_ip_proxy", data.get("default_enable_ip_proxy")); - setNumberParam(params, "start_page", data.get("default_start_page")); - setNumberParam(params, "max_notes_count", data.get("default_max_notes_count")); - setNumberParam(params, "max_comments_count", data.get("default_max_comments_count")); - setNumberParam(params, "max_concurrency_num", data.get("default_max_concurrency_num")); - setNumberParam(params, "ip_proxy_pool_count", data.get("default_ip_proxy_pool_count")); - setStringParam(params, "cookies", data.get("default_cookies")); - setStringParam(params, "ip_proxy_provider_name", data.get("default_ip_proxy_provider_name")); - setStringParam(params, "static_proxy_url", data.get("default_static_proxy_url")); - const filters = collectFilters("instance"); + setBoolSelectParam(params, "enable_comments", data.get("enable_comments")); + setBoolSelectParam(params, "enable_sub_comments", data.get("enable_sub_comments")); + setBoolSelectParam(params, "cdp_connect_existing", data.get("cdp_connect_existing")); + setBoolSelectParam(params, "enable_ip_proxy", data.get("enable_ip_proxy")); + setNumberParam(params, "start_page", data.get("start_page")); + setNumberParam(params, "max_notes_count", data.get("max_notes_count")); + setNumberParam(params, "max_comments_count", data.get("max_comments_count")); + setNumberParam(params, "max_concurrency_num", data.get("max_concurrency_num")); + setNumberParam(params, "ip_proxy_pool_count", data.get("ip_proxy_pool_count")); + setStringParam(params, "cookies", data.get("cookies")); + setStringParam(params, "ip_proxy_provider_name", data.get("ip_proxy_provider_name")); + setStringParam(params, "static_proxy_url", data.get("static_proxy_url")); + const filters = collectFilters(); if (Object.keys(filters).length) params.content_filters = filters; return params; } -function collectTaskParams(data) { - const params = {}; - setBoolSelectParam(params, "enable_comments", data.get("task_enable_comments")); - setBoolSelectParam(params, "enable_sub_comments", data.get("task_enable_sub_comments")); - setBoolSelectParam(params, "cdp_connect_existing", data.get("task_cdp_connect_existing")); - setBoolSelectParam(params, "enable_ip_proxy", data.get("task_enable_ip_proxy")); - setNumberParam(params, "start_page", data.get("task_start_page")); - setNumberParam(params, "max_notes_count", data.get("task_max_notes_count")); - setNumberParam(params, "max_comments_count", data.get("task_max_comments_count")); - setNumberParam(params, "max_concurrency_num", data.get("task_max_concurrency_num")); - setNumberParam(params, "ip_proxy_pool_count", data.get("task_ip_proxy_pool_count")); - setStringParam(params, "cookies", data.get("task_cookies")); - setStringParam(params, "ip_proxy_provider_name", data.get("task_ip_proxy_provider_name")); - setStringParam(params, "static_proxy_url", data.get("task_static_proxy_url")); - const filters = collectFilters("task"); - if (Object.keys(filters).length) params.content_filters = filters; - return params; +function collectJobBody(data) { + const body = { + name: data.get("name"), + platform: data.get("platform"), + login_type: data.get("login_type"), + save_option: data.get("save_option"), + crawler_type: data.get("crawler_type"), + target_text: data.get("target_text") || "", + headless: data.get("headless") === "on", + browser_profile_dir: data.get("browser_profile_dir") || "", + params: collectJobParams(data), + }; + const port = String(data.get("cdp_debug_port") || "").trim(); + if (port) body.cdp_debug_port = Number(port); + return body; +} + +function setSelect(name, value) { + els.jobForm.elements[name].value = value == null ? "" : String(value); } -async function loadTaskDetail(taskId) { +function setParamFields(params) { + const form = els.jobForm.elements; + form.start_page.value = params.start_page || ""; + form.max_notes_count.value = params.max_notes_count || ""; + form.max_comments_count.value = params.max_comments_count || ""; + form.max_concurrency_num.value = params.max_concurrency_num || ""; + form.ip_proxy_pool_count.value = params.ip_proxy_pool_count || ""; + form.cookies.value = params.cookies || ""; + form.static_proxy_url.value = params.static_proxy_url || ""; + setSelect("enable_comments", params.enable_comments === true ? "true" : params.enable_comments === false ? "false" : ""); + setSelect("enable_sub_comments", params.enable_sub_comments === true ? "true" : params.enable_sub_comments === false ? "false" : ""); + setSelect("cdp_connect_existing", params.cdp_connect_existing === true ? "true" : params.cdp_connect_existing === false ? "false" : ""); + setSelect("enable_ip_proxy", params.enable_ip_proxy === true ? "true" : params.enable_ip_proxy === false ? "false" : ""); + setSelect("ip_proxy_provider_name", params.ip_proxy_provider_name || ""); + state.filterRows.job = Object.entries(params.content_filters || {}).map(([metric, range]) => ({ + metric, + min: range?.min ?? "", + max: range?.max ?? "", + })); + syncFilterRowsWithPlatform(); +} + +function editJob(job) { + const form = els.jobForm.elements; + state.editingJobId = job.id; + els.formTitle.textContent = "编辑作业"; + els.jobSubmitBtn.textContent = "保存作业"; + els.cancelEditBtn.hidden = false; + form.name.value = job.name; + form.platform.value = job.platform; + form.login_type.value = job.login_type; + form.save_option.value = job.save_option; + form.crawler_type.value = job.crawler_type; + form.target_text.value = job.target_text || ""; + form.cdp_debug_port.value = job.cdp_debug_port || ""; + form.browser_profile_dir.value = job.browser_profile_dir || ""; + form.headless.checked = Boolean(job.headless); + setParamFields(job.params || {}); + els.jobForm.scrollIntoView({ behavior: "smooth", block: "start" }); +} + +function resetForm() { + state.editingJobId = ""; + els.formTitle.textContent = "新建作业"; + els.jobSubmitBtn.textContent = "创建作业"; + els.cancelEditBtn.hidden = true; + els.jobForm.reset(); + state.filterRows.job = []; + renderFilterRows("job"); +} + +async function loadJobDetail(jobId) { const [logs, artifacts] = await Promise.all([ - api.get(`/api/scheduler/tasks/${taskId}/logs?limit=300`), - api.get(`/api/scheduler/tasks/${taskId}/artifacts`), + api.get(`/api/scheduler/jobs/${jobId}/logs?limit=300`), + api.get(`/api/scheduler/jobs/${jobId}/artifacts`), ]); els.logsBox.textContent = logs.length ? logs.map((log) => `[${log.timestamp}] [${log.level}] ${log.message}`).join("\n") @@ -347,44 +371,22 @@ els.refreshBtn.addEventListener("click", () => { refreshAll().catch((err) => alert(err.message)); }); -els.instanceForm.addEventListener("submit", async (event) => { - event.preventDefault(); - const data = new FormData(event.currentTarget); - const body = { - name: data.get("name"), - platform: data.get("platform"), - login_type: data.get("login_type"), - save_option: data.get("save_option"), - headless: data.get("headless") === "on", - browser_profile_dir: data.get("browser_profile_dir") || "", - default_params: collectDefaultParams(data), - }; - const port = String(data.get("cdp_debug_port") || "").trim(); - if (port) body.cdp_debug_port = Number(port); - await api.post("/api/scheduler/instances", body); - event.currentTarget.reset(); - state.filterRows.instance = []; - renderFilterRows("instance"); - await refreshAll(); -}); +els.cancelEditBtn.addEventListener("click", resetForm); -els.taskForm.addEventListener("submit", async (event) => { +els.jobForm.addEventListener("submit", async (event) => { event.preventDefault(); const data = new FormData(event.currentTarget); - await api.post("/api/scheduler/tasks", { - instance_id: data.get("instance_id"), - crawler_type: data.get("crawler_type"), - target_text: data.get("target_text") || "", - params: collectTaskParams(data), - }); - event.currentTarget.reset(); - state.filterRows.task = []; - renderFilterRows("task"); + const body = collectJobBody(data); + if (state.editingJobId) { + await api.patch(`/api/scheduler/jobs/${state.editingJobId}`, body); + } else { + await api.post("/api/scheduler/jobs", body); + } + resetForm(); await refreshAll(); }); -els.instanceForm.elements.platform.addEventListener("change", () => syncFilterRowsWithPlatform("instance")); -els.taskInstanceSelect.addEventListener("change", () => syncFilterRowsWithPlatform("task")); +els.jobForm.elements.platform.addEventListener("change", syncFilterRowsWithPlatform); document.addEventListener("input", (event) => updateFilterValue(event.target)); document.addEventListener("change", (event) => updateFilterValue(event.target)); @@ -403,18 +405,33 @@ document.addEventListener("click", async (event) => { return; } try { - if (action === "login") { - const task = await api.post(`/api/scheduler/instances/${id}/login`, {}); - state.selectedTaskId = task.id; + if (action === "view-job") { + state.selectedJobId = id; + await loadJobDetail(id); + } + if (action === "edit-job") { + const job = state.jobs.find((item) => item.id === id) || await api.get(`/api/scheduler/jobs/${id}`); + editJob(job); + } + if (action === "login-job") { + await api.post(`/api/scheduler/jobs/${id}/login`); + state.selectedJobId = id; + await refreshAll(); + } + if (action === "run-job") { + await api.post(`/api/scheduler/jobs/${id}/run`); + state.selectedJobId = id; await refreshAll(); } - if (action === "select-task") { - state.selectedTaskId = id; - await loadTaskDetail(id); + if (action === "stop-job") { + await api.post(`/api/scheduler/jobs/${id}/stop`); + state.selectedJobId = id; + await refreshAll(); } - if (action === "cancel-task") { - await api.post(`/api/scheduler/tasks/${id}/cancel`, {}); - state.selectedTaskId = id; + if (action === "delete-job") { + await api.delete(`/api/scheduler/jobs/${id}`); + if (state.selectedJobId === id) state.selectedJobId = ""; + if (state.editingJobId === id) resetForm(); await refreshAll(); } } catch (err) { @@ -422,8 +439,7 @@ document.addEventListener("click", async (event) => { } }); -renderFilterRows("instance"); -renderFilterRows("task"); +renderFilterRows("job"); refreshAll().catch((err) => { els.summary.textContent = `读取失败：${err.message}`; }); diff --git a/scheduler_webui/index.html b/scheduler_webui/index.html index 652a0ee39..3e15aa4f7 100644 --- a/scheduler_webui/index.html +++ b/scheduler_webui/index.html @@ -3,24 +3,24 @@ - MediaCrawler 多实例调度器 + MediaCrawler 调度器
          -

          MediaCrawler 多实例调度器

          +

          MediaCrawler 调度器

          正在读取调度状态...

          -
          +
          -

          实例

          - +

          作业

          +
          @@ -28,22 +28,23 @@

          实例

          + - - + + - +
          名称 平台类型 状态端口当前任务目标最近运行 操作
          -
          -

          新建实例

          -
          +
          +

          新建作业

          + + +
          -

          默认抓取参数

          +

          抓取参数

          - - -
          - -
          -
          -

          任务

          - -
          -
          - - - - - - - - - - - - -
          任务实例类型状态目标操作
          -
          -
          - -
          -

          新建任务

          -
          - - - -
          -

          任务覆盖参数

          -
          - - - - - - -
          -

          内容过滤

          -
          - -
          -
          -

          任务登录与代理

          -
          - - - - -
          - - -
          - +
          + + +
          -
          +

          日志

          - 
          选择一个任务查看日志。
          + 
          选择一个作业查看日志。

          产物

          diff --git a/scheduler_webui/styles.css b/scheduler_webui/styles.css index c1cdf24c7..0d1c800ce 100644 --- a/scheduler_webui/styles.css +++ b/scheduler_webui/styles.css @@ -169,22 +169,21 @@ th { line-height: 1.3; } -.form-grid, -.task-params-grid { +.form-grid { display: grid; grid-template-columns: 1fr; gap: 10px; } -.instance-grid { +.job-grid { display: grid; grid-template-columns: 1fr; gap: 12px; } -.instance-grid .form-section, -.instance-grid .checkline, -.instance-grid button[type="submit"] { +.job-grid .form-section, +.job-grid .checkline, +.job-grid .form-actions { grid-column: 1 / -1; } @@ -214,14 +213,10 @@ th { width: auto; } -.task-grid { - display: grid; - grid-template-columns: 1fr; - gap: 12px; -} - -.task-grid .wide { - grid-column: 1 / -1; +.form-actions { + display: flex; + flex-wrap: wrap; + gap: 10px; } .detail-grid { diff --git a/tests/test_scheduler_api.py b/tests/test_scheduler_api.py index 2d7caeed7..49f3014d7 100644 --- a/tests/test_scheduler_api.py +++ b/tests/test_scheduler_api.py @@ -47,3 +47,43 @@ async def noop_start(task): logs_response = client.get(f"/api/scheduler/tasks/{task['id']}/logs") assert logs_response.status_code == 200 + + +def test_scheduler_api_creates_and_runs_job(monkeypatch, tmp_path): + manager = SchedulerManager(store=SchedulerStore(tmp_path / "scheduler.db"), project_root=tmp_path) + + async def noop_start(task): + return None + + monkeypatch.setattr(manager, "_start_task_locked", noop_start) + monkeypatch.setattr(scheduler_router, "scheduler_manager", manager) + + client = TestClient(app) + job_response = client.post( + "/api/scheduler/jobs", + json={ + "name": "小红书作业 A", + "platform": "xhs", + "login_type": "qrcode", + "save_option": "jsonl", + "crawler_type": "search", + "target_text": "AI 工具", + "params": {"max_notes_count": 5}, + }, + ) + assert job_response.status_code == 200 + job = job_response.json() + assert job["crawler_type"] == "search" + assert job["target_text"] == "AI 工具" + assert job["params"] == {"max_notes_count": 5} + + run_response = client.post(f"/api/scheduler/jobs/{job['id']}/run") + assert run_response.status_code == 200 + task = run_response.json() + assert task["instance_id"] == job["id"] + assert task["target_text"] == "AI 工具" + + refreshed_job = client.get(f"/api/scheduler/jobs/{job['id']}").json() + assert refreshed_job["last_task_id"] == task["id"] + assert client.get(f"/api/scheduler/jobs/{job['id']}/logs").status_code == 200 + assert client.get(f"/api/scheduler/jobs/{job['id']}/artifacts").status_code == 200 diff --git a/tests/test_scheduler_manager.py b/tests/test_scheduler_manager.py index 4f53e713e..935c18c1f 100644 --- a/tests/test_scheduler_manager.py +++ b/tests/test_scheduler_manager.py @@ -1,7 +1,11 @@ # -*- coding: utf-8 -*- +import asyncio + +import pytest + from api.scheduler.manager import SchedulerManager -from api.scheduler.schemas import InstanceCreateRequest +from api.scheduler.schemas import InstanceCreateRequest, JobCreateRequest from api.scheduler.store import SchedulerStore @@ -51,3 +55,38 @@ def test_scheduler_manager_scans_artifacts(tmp_path): assert artifacts[0]["type"] == "jsonl" assert artifacts[0]["record_count"] == 2 + + +def test_scheduler_manager_runs_job_without_queue(monkeypatch, tmp_path): + store = SchedulerStore(tmp_path / "scheduler.db") + manager = SchedulerManager(store=store, project_root=tmp_path) + job = manager.create_job( + JobCreateRequest( + name="小红书作业", + platform="xhs", + crawler_type="search", + target_text="AI 工具", + params={"max_notes_count": 3}, + ) + ) + + async def fake_start(task): + store.update_task(task["id"], status="running", pid=123) + store.update_instance( + task["instance_id"], + status="running", + current_task_id=task["id"], + last_task_id=task["id"], + pid=123, + ) + + monkeypatch.setattr(manager, "_start_task_locked", fake_start) + + task = asyncio.run(manager.run_job(job["id"])) + + assert task["crawler_type"] == "search" + assert task["target_text"] == "AI 工具" + assert task["params"] == {"max_notes_count": 3} + assert store.get_instance(job["id"])["last_task_id"] == task["id"] + with pytest.raises(RuntimeError, match="already running"): + asyncio.run(manager.run_job(job["id"])) diff --git a/tests/test_scheduler_store.py b/tests/test_scheduler_store.py index 897266613..2973e5bb0 100644 --- a/tests/test_scheduler_store.py +++ b/tests/test_scheduler_store.py @@ -13,6 +13,9 @@ def test_scheduler_store_crud(tmp_path): "headless": False, "save_option": "jsonl", "default_params": {"enable_comments": True}, + "crawler_type": "search", + "target_text": "编程副业", + "params": {"max_notes_count": 5}, }, str(tmp_path / "profile"), 9222, @@ -21,6 +24,9 @@ def test_scheduler_store_crud(tmp_path): assert instance["id"] == "inst-a" assert instance["default_params"] == {"enable_comments": True} + assert instance["crawler_type"] == "search" + assert instance["target_text"] == "编程副业" + assert instance["params"] == {"max_notes_count": 5} task = store.create_task( { @@ -46,6 +52,7 @@ def test_scheduler_store_crud(tmp_path): ) assert store.get_next_queued_task("inst-a")["id"] == task["id"] + assert store.get_latest_task("inst-a")["id"] == task["id"] assert store.list_logs(task["id"])[0]["message"] == "started" assert store.list_artifacts(task["id"])[0]["record_count"] == 2 assert store.scheduler_counts()["queued_tasks"] == 1 From 9d9af9354d19fae2c4b4a83b0233ecb3a705772f Mon Sep 17 00:00:00 2001 From: Raffaello Date: Tue, 30 Jun 2026 08:53:53 +0800 Subject: [PATCH 07/17] feat: adjust scheduler job workspace layout --- ...77\347\224\250\346\214\207\345\215\227.md" | 12 +- scheduler_webui/app.js | 144 ++++++++++++------ scheduler_webui/index.html | 58 +++---- scheduler_webui/styles.css | 85 +++++++++-- 4 files changed, 208 insertions(+), 91 deletions(-) diff --git "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" index 9565a1bfb..1a157d575 100644 --- "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" +++ "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" @@ -73,13 +73,17 @@ uv run uvicorn api.main:app --port 8080 --reload ## 5. 使用流程 -调度器页面自上而下排列为：作业列表、新建/编辑作业、日志、产物。 +调度器页面采用两列工作区： + +- 左列上方是作业列表，左列下方是当前选中作业的实时日志。 +- 右列上方是新建/编辑作业表单，右列下方是当前选中作业的产物列表。 1. 打开 `/scheduler`。 2. 创建作业，填写平台、登录方式、保存格式、爬取类型、目标、抓取参数、内容过滤、Cookie 和代理。 -3. 点击作业行的“登录”，调度器会创建一次内部 `login` 运行记录，只维护登录态，不进入抓取流程。 -4. 点击“运行”，调度器按作业配置创建一次内部运行记录并启动子进程。 -5. 选择作业后查看最近一次运行的日志和产物；作业正在运行时日志和产物指向当前运行。 +3. 点击作业列表中的条目即可选中作业，右侧表单会同步显示该作业当前配置，底部按钮会变为“保存更改”。 +4. 点击作业条目里的“登录”，调度器会创建一次内部 `login` 运行记录，只维护登录态，不进入抓取流程。 +5. 点击“运行”，调度器按作业配置创建一次内部运行记录并启动子进程。 +6. 选中作业后，左下实时日志和右下产物列表都会自动切换到该作业；作业正在运行时日志和产物指向当前运行。 作业处于 `running` 或 `stopping` 时不能编辑或删除，只能停止。作业失败后状态显示 `error`，可以修改配置后重新运行。 diff --git a/scheduler_webui/app.js b/scheduler_webui/app.js index 9e4327526..7545807e2 100644 --- a/scheduler_webui/app.js +++ b/scheduler_webui/app.js @@ -33,6 +33,7 @@ let state = { jobs: [], selectedJobId: "", editingJobId: "", + formDirty: false, filterRows: { job: [], }, @@ -124,7 +125,15 @@ async function refreshAll() { state.jobs = jobs; renderStatus(status); renderJobs(); - if (state.selectedJobId) await loadJobDetail(state.selectedJobId); + if (state.selectedJobId && !state.jobs.some((job) => job.id === state.selectedJobId)) { + resetForm(); + } + syncSelectedJobForm(); + if (state.selectedJobId) { + await loadJobDetail(state.selectedJobId); + } else { + clearJobDetail(); + } } function renderStatus(status) { @@ -133,33 +142,39 @@ function renderStatus(status) { function renderJobs() { els.jobHint.textContent = `${state.jobs.length} 个作业`; - els.jobsBody.innerHTML = state.jobs + els.jobsBody.innerHTML = state.jobs.length + ? state.jobs .map((job) => { const canRun = !["running", "stopping"].includes(job.status); const target = job.target_text || "-"; const taskId = job.current_task_id || job.last_task_id || "-"; + const selected = job.id === state.selectedJobId ? " selected" : ""; return ` - - ${escapeHtml(job.name)}
          ${escapeHtml(job.id)} - ${escapeHtml(job.platform)} - ${escapeHtml(job.crawler_type)} - ${statusPill(job.status)} - ${escapeHtml(target)} - ${escapeHtml(taskId === "-" ? "-" : taskId.slice(0, 8))} - -
          - - - - - - +
          +
          +
          + ${escapeHtml(job.name)} + ${escapeHtml(job.id)}
          - - + ${statusPill(job.status)} +
          +
          + ${escapeHtml(job.platform)} + ${escapeHtml(job.crawler_type)} + 运行 ${escapeHtml(taskId === "-" ? "-" : taskId.slice(0, 8))} +
          +
          ${escapeHtml(target)}
          +
          + + + + +
          +
          `; }) - .join(""); + .join("") + : `

          暂无作业。

          `; } function renderFilterRows(scope) { @@ -319,11 +334,11 @@ function setParamFields(params) { syncFilterRowsWithPlatform(); } -function editJob(job) { +function fillJobForm(job) { const form = els.jobForm.elements; state.editingJobId = job.id; - els.formTitle.textContent = "编辑作业"; - els.jobSubmitBtn.textContent = "保存作业"; + els.formTitle.textContent = "作业配置"; + els.jobSubmitBtn.textContent = "保存更改"; els.cancelEditBtn.hidden = false; form.name.value = job.name; form.platform.value = job.platform; @@ -335,17 +350,40 @@ function editJob(job) { form.browser_profile_dir.value = job.browser_profile_dir || ""; form.headless.checked = Boolean(job.headless); setParamFields(job.params || {}); - els.jobForm.scrollIntoView({ behavior: "smooth", block: "start" }); + state.formDirty = false; } function resetForm() { + state.selectedJobId = ""; state.editingJobId = ""; + state.formDirty = false; els.formTitle.textContent = "新建作业"; els.jobSubmitBtn.textContent = "创建作业"; els.cancelEditBtn.hidden = true; els.jobForm.reset(); state.filterRows.job = []; renderFilterRows("job"); + clearJobDetail(); + renderJobs(); +} + +function syncSelectedJobForm() { + if (!state.selectedJobId || state.formDirty) return; + const job = state.jobs.find((item) => item.id === state.selectedJobId); + if (job) fillJobForm(job); +} + +async function selectJob(jobId) { + const job = state.jobs.find((item) => item.id === jobId) || await api.get(`/api/scheduler/jobs/${jobId}`); + state.selectedJobId = job.id; + fillJobForm(job); + renderJobs(); + await loadJobDetail(job.id); +} + +function clearJobDetail() { + els.logsBox.textContent = "选择一个作业查看日志。"; + els.artifactsList.innerHTML = "
        • 选择一个作业查看产物。
          • "; } async function loadJobDetail(jobId) { @@ -377,50 +415,59 @@ els.jobForm.addEventListener("submit", async (event) => { event.preventDefault(); const data = new FormData(event.currentTarget); const body = collectJobBody(data); + let job; if (state.editingJobId) { - await api.patch(`/api/scheduler/jobs/${state.editingJobId}`, body); + job = await api.patch(`/api/scheduler/jobs/${state.editingJobId}`, body); } else { - await api.post("/api/scheduler/jobs", body); + job = await api.post("/api/scheduler/jobs", body); } - resetForm(); + state.selectedJobId = job.id; + state.editingJobId = job.id; + state.formDirty = false; await refreshAll(); }); els.jobForm.elements.platform.addEventListener("change", syncFilterRowsWithPlatform); -document.addEventListener("input", (event) => updateFilterValue(event.target)); -document.addEventListener("change", (event) => updateFilterValue(event.target)); +document.addEventListener("input", (event) => { + updateFilterValue(event.target); + if (event.target.closest("#jobForm")) state.formDirty = true; +}); +document.addEventListener("change", (event) => { + updateFilterValue(event.target); + if (event.target.closest("#jobForm")) state.formDirty = true; +}); document.addEventListener("click", async (event) => { const target = event.target.closest("button[data-action]"); - if (!target) return; - const action = target.dataset.action; - const id = target.dataset.id; - if (action === "add-filter") { - addFilterRow(target.dataset.scope); - return; - } - if (action === "remove-filter") { - removeFilterRow(target.dataset.scope, Number(target.dataset.index)); + if (!target) { + const item = event.target.closest("[data-job-id]"); + if (item) await selectJob(item.dataset.jobId).catch((err) => alert(err.message)); return; } + const action = target.dataset.action; + const id = target.dataset.id; try { - if (action === "view-job") { - state.selectedJobId = id; - await loadJobDetail(id); + if (action === "add-filter") { + addFilterRow(target.dataset.scope); + state.formDirty = true; + return; } - if (action === "edit-job") { - const job = state.jobs.find((item) => item.id === id) || await api.get(`/api/scheduler/jobs/${id}`); - editJob(job); + if (action === "remove-filter") { + removeFilterRow(target.dataset.scope, Number(target.dataset.index)); + state.formDirty = true; + return; } if (action === "login-job") { await api.post(`/api/scheduler/jobs/${id}/login`); state.selectedJobId = id; + state.formDirty = false; await refreshAll(); } if (action === "run-job") { await api.post(`/api/scheduler/jobs/${id}/run`); state.selectedJobId = id; + state.formDirty = false; await refreshAll(); } if (action === "stop-job") { @@ -439,6 +486,15 @@ document.addEventListener("click", async (event) => { } }); +document.addEventListener("keydown", async (event) => { + if (!["Enter", " "].includes(event.key)) return; + if (event.target.closest("button")) return; + const item = event.target.closest("[data-job-id]"); + if (!item) return; + event.preventDefault(); + await selectJob(item.dataset.jobId).catch((err) => alert(err.message)); +}); + renderFilterRows("job"); refreshAll().catch((err) => { els.summary.textContent = `读取失败：${err.message}`; diff --git a/scheduler_webui/index.html b/scheduler_webui/index.html index 3e15aa4f7..976a5f743 100644 --- a/scheduler_webui/index.html +++ b/scheduler_webui/index.html @@ -16,33 +16,29 @@

          MediaCrawler 调度器

          -
          -
          -
          -

          作业

          - -
          -
          - - - - - - - - - - - - - -
          名称平台类型状态目标最近运行操作
          -
          +
          +
          +
          +
          +

          作业列表

          + +
          +
          +
          + +
          +

          实时日志

          + 
          选择一个作业查看日志。
          +
          -
          -

          新建作业

          -
          +
          +
          +
          +

          新建作业

          + +
          +
          -
          - -
          - -
          -
          -

          日志

          - 
          选择一个作业查看日志。
          +
          +
          -

          产物

          +

          产物列表

            diff --git a/scheduler_webui/styles.css b/scheduler_webui/styles.css index 0d1c800ce..ff551db39 100644 --- a/scheduler_webui/styles.css +++ b/scheduler_webui/styles.css @@ -112,7 +112,7 @@ th { } .app-shell { - width: min(1040px, 100%); + width: min(1440px, 100%); margin: 0 auto; padding: 22px; } @@ -143,6 +143,17 @@ th { align-items: stretch; } +.scheduler-grid { + grid-template-columns: minmax(420px, 0.95fr) minmax(520px, 1.05fr); + align-items: start; +} + +.left-column, +.right-column { + display: grid; + gap: 14px; +} + .panel { border: 1px solid var(--line); border-radius: 8px; @@ -177,7 +188,7 @@ th { .job-grid { display: grid; - grid-template-columns: 1fr; + grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 12px; } @@ -203,6 +214,67 @@ th { overflow-x: auto; } +.jobs-list { + display: grid; + gap: 10px; +} + +.job-item { + display: grid; + gap: 10px; + width: 100%; + border: 1px solid var(--line); + border-radius: 8px; + background: #fff; + padding: 12px; + text-align: left; + cursor: pointer; +} + +.job-item:hover, +.job-item:focus-visible { + border-color: var(--teal); + outline: none; +} + +.job-item.selected { + border-color: var(--teal); + box-shadow: inset 3px 0 0 var(--teal); +} + +.job-main { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 12px; +} + +.job-name { + display: grid; + gap: 4px; + min-width: 0; +} + +.job-name strong, +.job-target { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.job-name small, +.job-meta, +.job-target { + color: var(--muted); + font-size: 12px; +} + +.job-meta { + display: flex; + flex-wrap: wrap; + gap: 8px; +} + .checkline { display: flex; align-items: center; @@ -219,12 +291,6 @@ th { gap: 10px; } -.detail-grid { - display: grid; - grid-template-columns: 1fr; - gap: 16px; -} - .filter-rows { display: grid; gap: 10px; @@ -317,12 +383,13 @@ th { } @media (max-width: 980px) { - .layout { + .scheduler-grid { grid-template-columns: 1fr; } } @media (max-width: 560px) { + .job-grid, .form-grid, .filter-row { grid-template-columns: 1fr; From 92d0cfa91c06a927e97c724f2873f7c544bb1615 Mon Sep 17 00:00:00 2001 From: Raffaello Date: Tue, 30 Jun 2026 09:03:43 +0800 Subject: [PATCH 08/17] feat: manage scheduler artifacts from job view --- api/routers/scheduler.py | 20 ++++++++ api/scheduler/manager.py | 47 +++++++++++++++++++ api/scheduler/store.py | 4 ++ ...77\347\224\250\346\214\207\345\215\227.md" | 8 +++- scheduler_webui/app.js | 22 ++++++++- scheduler_webui/index.html | 9 ++-- scheduler_webui/styles.css | 12 +++++ tests/test_scheduler_api.py | 28 +++++++++++ tests/test_scheduler_manager.py | 23 +++++++++ 9 files changed, 166 insertions(+), 7 deletions(-) diff --git a/api/routers/scheduler.py b/api/routers/scheduler.py index 63368a9b0..025a41511 100644 --- a/api/routers/scheduler.py +++ b/api/routers/scheduler.py @@ -109,6 +109,26 @@ async def list_job_artifacts(job_id: str): raise HTTPException(status_code=404, detail="Job not found") from exc +@router.post("/jobs/{job_id}/artifacts/{artifact_id}/open") +async def open_job_artifact(job_id: str, artifact_id: str): + try: + return scheduler_manager.open_job_artifact(job_id, artifact_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail="Artifact not found") from exc + except (RuntimeError, OSError) as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.delete("/jobs/{job_id}/artifacts/{artifact_id}") +async def delete_job_artifact(job_id: str, artifact_id: str): + try: + return scheduler_manager.delete_job_artifact(job_id, artifact_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail="Artifact not found") from exc + except (RuntimeError, OSError) as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + @router.get("/instances", response_model=list[InstanceResponse]) async def list_instances(): return scheduler_manager.list_instances() diff --git a/api/scheduler/manager.py b/api/scheduler/manager.py index 1698d9d2a..2cabe1464 100644 --- a/api/scheduler/manager.py +++ b/api/scheduler/manager.py @@ -6,6 +6,7 @@ import json import os import subprocess +import sys import uuid from dataclasses import dataclass from pathlib import Path @@ -227,6 +228,27 @@ def list_job_artifacts(self, job_id: str) -> list[dict[str, Any]]: task_id = self._job_task_id(job) return self.store.list_artifacts(task_id) if task_id else [] + def open_job_artifact(self, job_id: str, artifact_id: str) -> dict[str, str]: + _, artifact, path = self._job_artifact(job_id, artifact_id) + if not path.is_file(): + raise RuntimeError("artifact file not found") + if sys.platform == "darwin": + subprocess.Popen(["open", str(path)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + elif sys.platform.startswith("win"): + os.startfile(str(path)) # type: ignore[attr-defined] + else: + subprocess.Popen(["xdg-open", str(path)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + return {"status": "ok", "message": "Artifact opened"} + + def delete_job_artifact(self, job_id: str, artifact_id: str) -> dict[str, str]: + _, _, path = self._job_artifact(job_id, artifact_id) + if path.exists(): + if not path.is_file(): + raise RuntimeError("artifact path is not a file") + path.unlink() + self.store.delete_artifact(artifact_id) + return {"status": "ok", "message": "Artifact deleted"} + def status(self) -> dict[str, int]: return self.store.scheduler_counts() @@ -261,6 +283,31 @@ def _job_task_id(self, job: dict[str, Any]) -> str: latest_task = self.store.get_latest_task(job["id"]) return latest_task["id"] if latest_task else "" + def _job_artifact(self, job_id: str, artifact_id: str) -> tuple[str, dict[str, Any], Path]: + job = self.store.get_instance(job_id) + if not job: + raise KeyError("job not found") + task_id = self._job_task_id(job) + if not task_id: + raise KeyError("artifact not found") + artifact = next((item for item in self.store.list_artifacts(task_id) if item["id"] == artifact_id), None) + if not artifact: + raise KeyError("artifact not found") + path = self._safe_artifact_path(task_id, artifact["path"]) + return task_id, artifact, path + + def _safe_artifact_path(self, task_id: str, artifact_path: str) -> Path: + task = self.store.get_task(task_id) + if not task: + raise KeyError("task not found") + root = Path(task["artifact_dir"]).resolve() + path = Path(artifact_path).resolve() + try: + path.relative_to(root) + except ValueError as exc: + raise RuntimeError("artifact path is outside task directory") from exc + return path + async def _start_task_locked(self, task: dict[str, Any]) -> None: instance = self.store.get_instance(task["instance_id"]) if not instance: diff --git a/api/scheduler/store.py b/api/scheduler/store.py index 434a7bc75..7f065ded5 100644 --- a/api/scheduler/store.py +++ b/api/scheduler/store.py @@ -387,6 +387,10 @@ def list_artifacts(self, task_id: str) -> list[dict[str, Any]]: ).fetchall() return [dict(row) for row in rows] + def delete_artifact(self, artifact_id: str) -> None: + with self._lock, self._connect() as conn: + conn.execute("DELETE FROM artifacts WHERE id = ?", (artifact_id,)) + def scheduler_counts(self) -> dict[str, int]: with self._lock, self._connect() as conn: instances_total = conn.execute("SELECT COUNT(*) FROM instances").fetchone()[0] diff --git "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" index 1a157d575..e7199ea88 100644 --- "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" +++ "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" @@ -76,14 +76,16 @@ uv run uvicorn api.main:app --port 8080 --reload 调度器页面采用两列工作区： - 左列上方是作业列表，左列下方是当前选中作业的实时日志。 -- 右列上方是新建/编辑作业表单，右列下方是当前选中作业的产物列表。 +- 左列底部是当前选中作业的产物列表，支持打开文件和删除文件。 +- 右列是新建/编辑作业表单。 1. 打开 `/scheduler`。 2. 创建作业，填写平台、登录方式、保存格式、爬取类型、目标、抓取参数、内容过滤、Cookie 和代理。 3. 点击作业列表中的条目即可选中作业，右侧表单会同步显示该作业当前配置，底部按钮会变为“保存更改”。 4. 点击作业条目里的“登录”，调度器会创建一次内部 `login` 运行记录，只维护登录态，不进入抓取流程。 5. 点击“运行”，调度器按作业配置创建一次内部运行记录并启动子进程。 -6. 选中作业后，左下实时日志和右下产物列表都会自动切换到该作业；作业正在运行时日志和产物指向当前运行。 +6. 选中作业后，左侧实时日志和产物列表都会自动切换到该作业；作业正在运行时日志和产物指向当前运行。 +7. 产物列表中的“打开文件”会调用本机系统默认程序打开该文件；“删除文件”会二次确认，并且只允许删除当前作业已登记的产物文件。 作业处于 `running` 或 `stopping` 时不能编辑或删除，只能停止。作业失败后状态显示 `error`，可以修改配置后重新运行。 @@ -132,6 +134,8 @@ WebUI 提交后会在内部生成作业 `params`。API 调用仍使用 JSON， | `POST` | `/api/scheduler/jobs/{job_id}/stop` | 停止当前运行 | | `GET` | `/api/scheduler/jobs/{job_id}/logs` | 获取当前或最近一次运行日志 | | `GET` | `/api/scheduler/jobs/{job_id}/artifacts` | 获取当前或最近一次运行产物 | +| `POST` | `/api/scheduler/jobs/{job_id}/artifacts/{artifact_id}/open` | 打开已登记的产物文件 | +| `DELETE` | `/api/scheduler/jobs/{job_id}/artifacts/{artifact_id}` | 删除已登记的产物文件 | 创建作业示例： diff --git a/scheduler_webui/app.js b/scheduler_webui/app.js index 7545807e2..20552b197 100644 --- a/scheduler_webui/app.js +++ b/scheduler_webui/app.js @@ -399,7 +399,18 @@ async function loadJobDetail(jobId) { .map((item) => { const sizeKb = (item.size / 1024).toFixed(1); const count = item.record_count == null ? "" : `，${item.record_count} 条`; - return `
          • ${escapeHtml(item.type)} ${escapeHtml(sizeKb)} KB${count}
            ${escapeHtml(item.path)}
            • `; + return ` +
          • +
            + ${escapeHtml(item.type)} ${escapeHtml(sizeKb)} KB${count} +
            ${escapeHtml(item.path)} +
            +
            + + +
            +
            • + `; }) .join("") : "
          • 暂无产物。
            • "; @@ -481,6 +492,15 @@ document.addEventListener("click", async (event) => { if (state.editingJobId === id) resetForm(); await refreshAll(); } + if (action === "open-artifact") { + if (!state.selectedJobId) return; + await api.post(`/api/scheduler/jobs/${state.selectedJobId}/artifacts/${id}/open`); + } + if (action === "delete-artifact") { + if (!state.selectedJobId || !confirm("确定删除这个产物文件？")) return; + await api.delete(`/api/scheduler/jobs/${state.selectedJobId}/artifacts/${id}`); + await loadJobDetail(state.selectedJobId); + } } catch (err) { alert(err.message); } diff --git a/scheduler_webui/index.html b/scheduler_webui/index.html index 976a5f743..73736c0a7 100644 --- a/scheduler_webui/index.html +++ b/scheduler_webui/index.html @@ -30,6 +30,11 @@

            作业列表

            实时日志

            选择一个作业查看日志。
            + +
            +

            产物列表

            +
              +
              @@ -191,10 +196,6 @@

              登录与代理

              -
              -

              产物列表

              -
                -
                diff --git a/scheduler_webui/styles.css b/scheduler_webui/styles.css index ff551db39..218ea794f 100644 --- a/scheduler_webui/styles.css +++ b/scheduler_webui/styles.css @@ -341,6 +341,18 @@ th { word-break: break-all; } +.artifact-item { + display: grid; + gap: 10px; +} + +.artifact-info { + min-width: 0; + color: var(--text); + font-size: 13px; + line-height: 1.45; +} + .pill { display: inline-flex; align-items: center; diff --git a/tests/test_scheduler_api.py b/tests/test_scheduler_api.py index 49f3014d7..8a1abf39d 100644 --- a/tests/test_scheduler_api.py +++ b/tests/test_scheduler_api.py @@ -87,3 +87,31 @@ async def noop_start(task): assert refreshed_job["last_task_id"] == task["id"] assert client.get(f"/api/scheduler/jobs/{job['id']}/logs").status_code == 200 assert client.get(f"/api/scheduler/jobs/{job['id']}/artifacts").status_code == 200 + + +def test_scheduler_api_deletes_job_artifact(monkeypatch, tmp_path): + manager = SchedulerManager(store=SchedulerStore(tmp_path / "scheduler.db"), project_root=tmp_path) + monkeypatch.setattr(scheduler_router, "scheduler_manager", manager) + + client = TestClient(app) + job = client.post( + "/api/scheduler/jobs", + json={"name": "小红书作业 A", "platform": "xhs", "save_option": "jsonl"}, + ).json() + artifact_dir = tmp_path / "artifacts" / "task-a" + artifact_dir.mkdir(parents=True) + artifact_file = artifact_dir / "data.jsonl" + artifact_file.write_text('{"id": 1}\n', encoding="utf-8") + task = manager.store.create_task( + {"instance_id": job["id"], "crawler_type": "search", "target_text": "", "params": {}}, + str(artifact_dir), + ) + manager.store.update_instance(job["id"], last_task_id=task["id"]) + manager.store.replace_artifacts(task["id"], manager._scan_artifacts(artifact_dir)) + artifact = manager.store.list_artifacts(task["id"])[0] + + response = client.delete(f"/api/scheduler/jobs/{job['id']}/artifacts/{artifact['id']}") + + assert response.status_code == 200 + assert response.json()["status"] == "ok" + assert not artifact_file.exists() diff --git a/tests/test_scheduler_manager.py b/tests/test_scheduler_manager.py index 935c18c1f..fd5e1b77e 100644 --- a/tests/test_scheduler_manager.py +++ b/tests/test_scheduler_manager.py @@ -57,6 +57,29 @@ def test_scheduler_manager_scans_artifacts(tmp_path): assert artifacts[0]["record_count"] == 2 +def test_scheduler_manager_deletes_job_artifact(tmp_path): + store = SchedulerStore(tmp_path / "scheduler.db") + manager = SchedulerManager(store=store, project_root=tmp_path) + job = manager.create_job(JobCreateRequest(name="小红书作业", platform="xhs")) + artifact_dir = tmp_path / "artifacts" / "task-a" + artifact_dir.mkdir(parents=True) + artifact_file = artifact_dir / "data.jsonl" + artifact_file.write_text('{"id": 1}\n', encoding="utf-8") + task = store.create_task( + {"instance_id": job["id"], "crawler_type": "search", "target_text": "", "params": {}}, + str(artifact_dir), + ) + store.update_instance(job["id"], last_task_id=task["id"]) + store.replace_artifacts(task["id"], manager._scan_artifacts(artifact_dir)) + artifact = store.list_artifacts(task["id"])[0] + + result = manager.delete_job_artifact(job["id"], artifact["id"]) + + assert result["status"] == "ok" + assert not artifact_file.exists() + assert store.list_artifacts(task["id"]) == [] + + def test_scheduler_manager_runs_job_without_queue(monkeypatch, tmp_path): store = SchedulerStore(tmp_path / "scheduler.db") manager = SchedulerManager(store=store, project_root=tmp_path) From 48f58badd9398e00e7b2d479e146f86aa3faa30c Mon Sep 17 00:00:00 2001 From: Raffaello Date: Tue, 30 Jun 2026 09:21:18 +0800 Subject: [PATCH 09/17] fix: show scheduler artifact open feedback --- api/scheduler/manager.py | 22 ++++++++----- ...77\347\224\250\346\214\207\345\215\227.md" | 2 +- scheduler_webui/app.js | 22 ++++++++++++- tests/test_scheduler_manager.py | 31 +++++++++++++++++++ 4 files changed, 68 insertions(+), 9 deletions(-) diff --git a/api/scheduler/manager.py b/api/scheduler/manager.py index 2cabe1464..d22735788 100644 --- a/api/scheduler/manager.py +++ b/api/scheduler/manager.py @@ -232,13 +232,21 @@ def open_job_artifact(self, job_id: str, artifact_id: str) -> dict[str, str]: _, artifact, path = self._job_artifact(job_id, artifact_id) if not path.is_file(): raise RuntimeError("artifact file not found") - if sys.platform == "darwin": - subprocess.Popen(["open", str(path)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - elif sys.platform.startswith("win"): - os.startfile(str(path)) # type: ignore[attr-defined] - else: - subprocess.Popen(["xdg-open", str(path)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - return {"status": "ok", "message": "Artifact opened"} + try: + if sys.platform == "darwin": + cmd = ( + ["open", "-t", str(path)] + if path.suffix.lower() in {".jsonl", ".json", ".csv", ".txt", ".log"} + else ["open", str(path)] + ) + subprocess.run(cmd, check=True, capture_output=True, text=True) + elif sys.platform.startswith("win"): + os.startfile(str(path)) # type: ignore[attr-defined] + else: + subprocess.run(["xdg-open", str(path)], check=True, capture_output=True, text=True) + except subprocess.CalledProcessError as exc: + raise RuntimeError(exc.stderr.strip() or exc.stdout.strip() or "failed to open artifact") from exc + return {"status": "ok", "message": "Artifact opened", "path": artifact["path"]} def delete_job_artifact(self, job_id: str, artifact_id: str) -> dict[str, str]: _, _, path = self._job_artifact(job_id, artifact_id) diff --git "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" index e7199ea88..f756a525e 100644 --- "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" +++ "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" @@ -85,7 +85,7 @@ uv run uvicorn api.main:app --port 8080 --reload 4. 点击作业条目里的“登录”，调度器会创建一次内部 `login` 运行记录，只维护登录态，不进入抓取流程。 5. 点击“运行”，调度器按作业配置创建一次内部运行记录并启动子进程。 6. 选中作业后，左侧实时日志和产物列表都会自动切换到该作业；作业正在运行时日志和产物指向当前运行。 -7. 产物列表中的“打开文件”会调用本机系统默认程序打开该文件；“删除文件”会二次确认，并且只允许删除当前作业已登记的产物文件。 +7. 产物列表中的“打开文件”会调用本机系统打开该文件；`jsonl`、`json`、`csv`、`txt`、`log` 这类文本产物会使用系统文本编辑器打开。按钮会显示“打开中...”和“已打开”。“删除文件”会二次确认，并且只允许删除当前作业已登记的产物文件。 作业处于 `running` 或 `stopping` 时不能编辑或删除，只能停止。作业失败后状态显示 `error`，可以修改配置后重新运行。 diff --git a/scheduler_webui/app.js b/scheduler_webui/app.js index 20552b197..2a46eb88c 100644 --- a/scheduler_webui/app.js +++ b/scheduler_webui/app.js @@ -386,6 +386,24 @@ function clearJobDetail() { els.artifactsList.innerHTML = "
              • 选择一个作业查看产物。
                • "; } +async function withButtonFeedback(button, pendingText, doneText, action) { + const originalText = button.textContent; + button.disabled = true; + button.textContent = pendingText; + try { + await action(); + button.textContent = doneText; + setTimeout(() => { + button.textContent = originalText; + button.disabled = false; + }, 1500); + } catch (err) { + button.textContent = originalText; + button.disabled = false; + throw err; + } +} + async function loadJobDetail(jobId) { const [logs, artifacts] = await Promise.all([ api.get(`/api/scheduler/jobs/${jobId}/logs?limit=300`), @@ -494,7 +512,9 @@ document.addEventListener("click", async (event) => { } if (action === "open-artifact") { if (!state.selectedJobId) return; - await api.post(`/api/scheduler/jobs/${state.selectedJobId}/artifacts/${id}/open`); + await withButtonFeedback(target, "打开中...", "已打开", () => + api.post(`/api/scheduler/jobs/${state.selectedJobId}/artifacts/${id}/open`) + ); } if (action === "delete-artifact") { if (!state.selectedJobId || !confirm("确定删除这个产物文件？")) return; diff --git a/tests/test_scheduler_manager.py b/tests/test_scheduler_manager.py index fd5e1b77e..c4fccaad5 100644 --- a/tests/test_scheduler_manager.py +++ b/tests/test_scheduler_manager.py @@ -4,6 +4,7 @@ import pytest +from api.scheduler import manager as scheduler_manager_module from api.scheduler.manager import SchedulerManager from api.scheduler.schemas import InstanceCreateRequest, JobCreateRequest from api.scheduler.store import SchedulerStore @@ -80,6 +81,36 @@ def test_scheduler_manager_deletes_job_artifact(tmp_path): assert store.list_artifacts(task["id"]) == [] +def test_scheduler_manager_opens_job_artifact(monkeypatch, tmp_path): + store = SchedulerStore(tmp_path / "scheduler.db") + manager = SchedulerManager(store=store, project_root=tmp_path) + job = manager.create_job(JobCreateRequest(name="小红书作业", platform="xhs")) + artifact_dir = tmp_path / "artifacts" / "task-a" + artifact_dir.mkdir(parents=True) + artifact_file = artifact_dir / "data.jsonl" + artifact_file.write_text('{"id": 1}\n', encoding="utf-8") + task = store.create_task( + {"instance_id": job["id"], "crawler_type": "search", "target_text": "", "params": {}}, + str(artifact_dir), + ) + store.update_instance(job["id"], last_task_id=task["id"]) + store.replace_artifacts(task["id"], manager._scan_artifacts(artifact_dir)) + artifact = store.list_artifacts(task["id"])[0] + calls = [] + + def fake_run(cmd, check, capture_output, text): + calls.append((cmd, check, capture_output, text)) + + monkeypatch.setattr(scheduler_manager_module.sys, "platform", "darwin") + monkeypatch.setattr(scheduler_manager_module.subprocess, "run", fake_run) + + result = manager.open_job_artifact(job["id"], artifact["id"]) + + assert result["status"] == "ok" + assert result["path"] == str(artifact_file) + assert calls == [(["open", "-t", str(artifact_file)], True, True, True)] + + def test_scheduler_manager_runs_job_without_queue(monkeypatch, tmp_path): store = SchedulerStore(tmp_path / "scheduler.db") manager = SchedulerManager(store=store, project_root=tmp_path) From 123bc105dba670ec99a03b4960a6c1d177945465 Mon Sep 17 00:00:00 2001 From: Raffaello Date: Tue, 30 Jun 2026 09:46:17 +0800 Subject: [PATCH 10/17] feat: add publish time content filter --- README.md | 7 +- docs/index.md | 6 +- ...77\347\224\250\346\214\207\345\215\227.md" | 28 ++-- ...77\347\224\250\346\214\207\345\215\227.md" | 7 +- ...66\346\236\204\346\226\207\346\241\243.md" | 4 +- scheduler_webui/app.js | 142 ++++++++++++------ scheduler_webui/index.html | 2 +- scheduler_webui/styles.css | 11 +- tests/test_content_filter.py | 26 +++- tools/content_filter.py | 74 ++++++++- 10 files changed, 233 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index fe678d769..c60bd4b8e 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,10 @@ uv run playwright install # 从配置文件中读取关键词搜索相关的帖子并爬取帖子信息与评论 uv run main.py --platform xhs --lt qrcode --type search -# 按互动指标过滤爬取目标，例如只保存点赞数不低于 1000 的内容 +# 按发布时间和互动指标过滤爬取目标，例如只保存 2024-07-01 之后且点赞数不低于 1000 的内容 +uv run main.py --platform xhs --lt qrcode --type search --content_filters '{"publish_time":{"min":"2024-07-01"},"liked_count":{"min":1000}}' + +# 也可以只按互动指标过滤 uv run main.py --platform xhs --lt qrcode --type search --content_filters '{"liked_count":{"min":1000}}' # 从配置文件中读取指定的帖子ID列表获取指定帖子的信息与评论信息 @@ -176,7 +179,7 @@ uv run main.py --platform xhs --lt qrcode --type login uv run main.py --help ``` -内容过滤支持不同平台的点赞、收藏、转发、评论等字段，详见 [内容过滤使用指南](docs/内容过滤使用指南.md)。 +内容过滤支持不同平台的发布时间、点赞、收藏、转发、评论等字段，详见 [内容过滤使用指南](docs/内容过滤使用指南.md)。
                🖥️ WebUI 可视化操作界面 diff --git a/docs/index.md b/docs/index.md index 6ab8c9b8d..4b8e40cee 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,7 +4,7 @@ - [项目架构文档](项目架构文档.md) - 系统架构、模块设计、数据流向（含 Mermaid 图表） - [多实例调度器使用指南](多实例调度器使用指南.md) - 多账号作业、运行记录、调度器 WebUI 和 API -- [内容过滤使用指南](内容过滤使用指南.md) - 按点赞、收藏、转发、评论等互动指标筛选爬取目标 +- [内容过滤使用指南](内容过滤使用指南.md) - 按发布时间、点赞、收藏、转发、评论等条件筛选爬取目标 ## 推荐：使用 uv 管理依赖 @@ -36,8 +36,8 @@ uv run playwright install # 从配置中读取关键词搜索并爬取帖子与评论 uv run main.py --platform xhs --lt qrcode --type search -# 只保存点赞数不低于 1000 的搜索结果 -uv run main.py --platform xhs --lt qrcode --type search --content_filters '{"liked_count":{"min":1000}}' +# 只保存 2024-07-01 之后且点赞数不低于 1000 的搜索结果 +uv run main.py --platform xhs --lt qrcode --type search --content_filters '{"publish_time":{"min":"2024-07-01"},"liked_count":{"min":1000}}' # 从配置中读取指定帖子ID列表并爬取帖子与评论 uv run main.py --platform xhs --lt qrcode --type detail diff --git "a/docs/\345\206\205\345\256\271\350\277\207\346\273\244\344\275\277\347\224\250\346\214\207\345\215\227.md" "b/docs/\345\206\205\345\256\271\350\277\207\346\273\244\344\275\277\347\224\250\346\214\207\345\215\227.md" index 3d7863bae..1cce253d8 100644 --- "a/docs/\345\206\205\345\256\271\350\277\207\346\273\244\344\275\277\347\224\250\346\214\207\345\215\227.md" +++ "b/docs/\345\206\205\345\256\271\350\277\207\346\273\244\344\275\277\347\224\250\346\214\207\345\215\227.md" @@ -1,6 +1,6 @@ # 内容过滤使用指南 -内容过滤用于在爬取候选内容拿到详情后、保存内容和抓取评论/媒体前，按互动指标筛掉不需要的目标。它适用于关键词搜索、指定内容和创作者主页三种爬取模式。 +内容过滤用于在爬取候选内容拿到详情后、保存内容和抓取评论/媒体前，按发布时间或互动指标筛掉不需要的目标。它适用于关键词搜索、指定内容和创作者主页三种爬取模式。 ## 1. 配置格式 @@ -10,12 +10,19 @@ CLI 参数名为 `--content_filters`，值是 JSON 字符串： uv run python main.py --platform xhs --type search --keywords "AI 工具" --content_filters '{"liked_count":{"min":1000},"comment_count":{"max":500}}' ``` -过滤条件支持 `min`、`max`，边界包含等于值。多个字段同时配置时，内容必须全部满足才会保存。数值可以写数字，也可以写带单位的字符串，例如 `"1万"`、`"2.5万"`、`"1亿"`。 +过滤条件支持 `min`、`max`，边界包含等于值。多个字段同时配置时，内容必须全部满足才会保存。数值可以写数字，也可以写带单位的字符串，例如 `"1万"`、`"2.5万"`、`"1亿"`。发布时间使用 `publish_time` 字段，通常只配置 `min`，表示过滤掉早于设定时间的作品： + +```shell +uv run python main.py --platform xhs --type search --keywords "AI 工具" --content_filters '{"publish_time":{"min":"2024-07-01"},"liked_count":{"min":1000}}' +``` + +发布时间支持 `YYYY-MM-DD`、`YYYY-MM-DD HH:MM:SS`、秒级时间戳和毫秒级时间戳。 也可以在 `config/base_config.py` 中设置默认值： ```python CONTENT_FILTERS = { + "publish_time": {"min": "2024-07-01"}, "liked_count": {"min": 1000}, "comment_count": {"min": 20, "max": 500}, } @@ -27,19 +34,19 @@ CONTENT_FILTERS = { | 平台 | 可用字段 | | --- | --- | -| 小红书 `xhs` | `liked_count`、`collected_count`、`comment_count`、`share_count` | -| 抖音 `dy` | `liked_count`、`collected_count`、`comment_count`、`share_count` | -| 快手 `ks` | `liked_count`、`view_count` | -| B 站 `bili` | `liked_count`、`disliked_count`、`play_count`、`favorite_count`、`share_count`、`coin_count`、`danmaku_count`、`comment_count` | -| 微博 `wb` | `liked_count`、`comment_count`、`share_count` | -| 贴吧 `tieba` | `reply_count`、`reply_page_count` | -| 知乎 `zhihu` | `voteup_count`、`comment_count` | +| 小红书 `xhs` | `publish_time`、`liked_count`、`collected_count`、`comment_count`、`share_count` | +| 抖音 `dy` | `publish_time`、`liked_count`、`collected_count`、`comment_count`、`share_count` | +| 快手 `ks` | `publish_time`、`liked_count`、`view_count` | +| B 站 `bili` | `publish_time`、`liked_count`、`disliked_count`、`play_count`、`favorite_count`、`share_count`、`coin_count`、`danmaku_count`、`comment_count` | +| 微博 `wb` | `publish_time`、`liked_count`、`comment_count`、`share_count` | +| 贴吧 `tieba` | `publish_time`、`reply_count`、`reply_page_count` | +| 知乎 `zhihu` | `publish_time`、`voteup_count`、`comment_count` | 字段不支持时会在启动阶段报错，不会静默忽略。部分历史字段名保留了兼容别名，例如快手 `viewd_count`、微博 `comments_count`、B 站 `video_favorite_count`。 ## 3. 调度器作业参数 -多实例调度器 WebUI 中，作业表单有“内容过滤”区域。点击“添加过滤条件”，选择指标并填写最小值或最大值即可，不需要手写 JSON。 +多实例调度器 WebUI 中，作业表单有“内容过滤”区域。点击“添加过滤条件”，选择指标并填写最小值或最大值即可，不需要手写 JSON。选择“发布时间”时，表单会显示日期选择器，含义是“只保留不早于该日期发布的作品”。 如果直接调用调度器 API，作业的 `params` 支持 `content_filters`： @@ -48,6 +55,7 @@ CONTENT_FILTERS = { "enable_comments": true, "max_notes_count": 20, "content_filters": { + "publish_time": {"min": "2024-07-01"}, "liked_count": {"min": 1000}, "comment_count": {"min": 20} } diff --git "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" index f756a525e..3554bbc24 100644 --- "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" +++ "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" @@ -94,8 +94,8 @@ uv run uvicorn api.main:app --port 8080 --reload 调度器 WebUI 已把参数拆成表单控件，不需要手写 JSON： - 作业基础信息：平台、登录方式、保存格式、爬取类型、目标、CDP 端口和 Profile 目录。 -- 抓取参数：起始页、最大内容数、单条评论数、并发数、一级评论和二级评论。 -- 内容过滤：按当前平台展示可选指标，点击“添加过滤条件”后填写最小值或最大值。数值支持 `1000`、`1万`、`2.5万` 这类写法。 +- 抓取参数：起始页、最大内容数、单条评论数、并发数、一级评论和二级评论，页面中按两列排列，便于快速浏览。 +- 内容过滤：按当前平台展示可选指标，点击“添加过滤条件”后填写最小值或最大值。数值支持 `1000`、`1万`、`2.5万` 这类写法；选择“发布时间”时填写“不早于”日期，用于过滤掉早于设定时间的作品。 - 登录与代理：Cookie、是否连接已有浏览器、是否启用代理、代理池数量、代理服务和固定代理地址。 WebUI 提交后会在内部生成作业 `params`。API 调用仍使用 JSON，常用字段如下： @@ -106,7 +106,7 @@ WebUI 提交后会在内部生成作业 `params`。API 调用仍使用 JSON， | `enable_sub_comments` | `--get_sub_comment` | `false` | | `max_notes_count` | `--crawler_max_notes_count` | `20` | | `max_comments_count` | `--max_comments_count_singlenotes` | `10` | -| `content_filters` | `--content_filters` | `{"liked_count":{"min":1000}}` | +| `content_filters` | `--content_filters` | `{"publish_time":{"min":"2024-07-01"},"liked_count":{"min":1000}}` | | `max_concurrency_num` | `--max_concurrency_num` | `1` | | `cookies` | `--cookies` | `"a=b; c=d"` | | `cdp_connect_existing` | `--cdp_connect_existing` | `false` | @@ -152,6 +152,7 @@ WebUI 提交后会在内部生成作业 `params`。API 调用仍使用 JSON， "enable_comments": true, "max_notes_count": 20, "content_filters": { + "publish_time": {"min": "2024-07-01"}, "liked_count": {"min": 1000}, "comment_count": {"min": 20} } diff --git "a/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" "b/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" index f7821f317..4909caf4f 100644 --- "a/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" +++ "b/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" @@ -142,7 +142,7 @@ flowchart LR media --> file ``` -内容过滤由 `tools/content_filter.py` 提供统一规则，平台 `core.py` 在详情解析完成后调用。过滤命中的内容才会进入内容存储、媒体下载和评论抓取；创作者信息本身不受内容过滤影响。 +内容过滤由 `tools/content_filter.py` 提供统一规则，支持按发布时间、点赞、收藏、转发、评论等条件筛选。平台 `core.py` 在详情解析完成后调用，过滤命中的内容才会进入内容存储、媒体下载和评论抓取；创作者信息本身不受内容过滤影响。 --- @@ -234,7 +234,7 @@ MediaCrawler/ │ ├── app_runner.py # 应用运行管理 │ ├── browser_launcher.py # 浏览器启动 │ ├── cdp_browser.py # CDP浏览器管理 -│ ├── content_filter.py # 内容互动指标过滤 +│ ├── content_filter.py # 内容发布时间与互动指标过滤 │ ├── profile.py # 多实例浏览器 Profile 路径解析 │ ├── crawler_util.py # 爬虫工具 │ └── async_file_writer.py # 异步文件写入 diff --git a/scheduler_webui/app.js b/scheduler_webui/app.js index 2a46eb88c..37d159133 100644 --- a/scheduler_webui/app.js +++ b/scheduler_webui/app.js @@ -41,43 +41,50 @@ let state = { const filterFields = { xhs: [ - ["liked_count", "点赞数"], - ["collected_count", "收藏数"], - ["comment_count", "评论数"], - ["share_count", "转发数"], + ["publish_time", "发布时间", "date"], + ["liked_count", "点赞数", "number"], + ["collected_count", "收藏数", "number"], + ["comment_count", "评论数", "number"], + ["share_count", "转发数", "number"], ], dy: [ - ["liked_count", "点赞数"], - ["collected_count", "收藏数"], - ["comment_count", "评论数"], - ["share_count", "转发数"], + ["publish_time", "发布时间", "date"], + ["liked_count", "点赞数", "number"], + ["collected_count", "收藏数", "number"], + ["comment_count", "评论数", "number"], + ["share_count", "转发数", "number"], ], ks: [ - ["liked_count", "点赞数"], - ["view_count", "播放数"], + ["publish_time", "发布时间", "date"], + ["liked_count", "点赞数", "number"], + ["view_count", "播放数", "number"], ], bili: [ - ["liked_count", "点赞数"], - ["disliked_count", "点踩数"], - ["play_count", "播放数"], - ["favorite_count", "收藏数"], - ["share_count", "分享数"], - ["coin_count", "投币数"], - ["danmaku_count", "弹幕数"], - ["comment_count", "评论数"], + ["publish_time", "发布时间", "date"], + ["liked_count", "点赞数", "number"], + ["disliked_count", "点踩数", "number"], + ["play_count", "播放数", "number"], + ["favorite_count", "收藏数", "number"], + ["share_count", "分享数", "number"], + ["coin_count", "投币数", "number"], + ["danmaku_count", "弹幕数", "number"], + ["comment_count", "评论数", "number"], ], wb: [ - ["liked_count", "点赞数"], - ["comment_count", "评论数"], - ["share_count", "转发数"], + ["publish_time", "发布时间", "date"], + ["liked_count", "点赞数", "number"], + ["comment_count", "评论数", "number"], + ["share_count", "转发数", "number"], ], tieba: [ - ["reply_count", "回复数"], - ["reply_page_count", "回复页数"], + ["publish_time", "发布时间", "date"], + ["reply_count", "回复数", "number"], + ["reply_page_count", "回复页数", "number"], ], zhihu: [ - ["voteup_count", "赞同数"], - ["comment_count", "评论数"], + ["publish_time", "发布时间", "date"], + ["voteup_count", "赞同数", "number"], + ["comment_count", "评论数", "number"], ], }; @@ -182,35 +189,57 @@ function renderFilterRows(scope) { const rows = state.filterRows[scope]; els.jobFilterRows.innerHTML = rows.length ? rows - .map((row, index) => ` -
                - - - - -
                - `) + .map((row, index) => renderFilterRow(scope, platform, row, index)) .join("") : `

                未设置过滤条件

                `; } +function renderFilterRow(scope, platform, row, index) { + const type = filterFieldType(platform, row.metric); + const isDate = type === "date"; + const minControl = isDate + ? ` + + ` + : ` + + + `; + return ` +
                + + ${minControl} + +
                + `; +} + function metricOptions(platform, selectedMetric) { return (filterFields[platform] || filterFields.xhs) .map(([value, label]) => ``) .join(""); } +function filterFieldType(platform, metric) { + const fields = filterFields[platform] || filterFields.xhs; + const field = fields.find(([value]) => value === metric) || fields[0]; + return field[2] || "number"; +} + function currentPlatform() { return els.jobForm.elements.platform.value || "xhs"; } @@ -242,13 +271,21 @@ function updateFilterValue(target) { if (!rowEl) return; const row = state.filterRows[rowEl.dataset.scope]?.[Number(rowEl.dataset.index)]; if (!row) return; + if (target.dataset.field === "metric" && row.metric !== target.value) { + row.metric = target.value; + row.min = ""; + row.max = ""; + renderFilterRows(rowEl.dataset.scope); + return; + } row[target.dataset.field] = target.value; } function collectFilters() { return state.filterRows.job.reduce((filters, row) => { + const type = filterFieldType(currentPlatform(), row.metric); const min = String(row.min || "").trim(); - const max = String(row.max || "").trim(); + const max = type === "date" ? "" : String(row.max || "").trim(); if (!row.metric || (!min && !max)) return filters; filters[row.metric] = {}; if (min) filters[row.metric].min = min; @@ -312,6 +349,21 @@ function setSelect(name, value) { els.jobForm.elements[name].value = value == null ? "" : String(value); } +function dateInputValue(value) { + const text = String(value ?? "").trim(); + if (!text) return ""; + if (/^\d{4}-\d{2}-\d{2}/.test(text)) return text.slice(0, 10); + + const timestamp = Number(text); + if (!Number.isFinite(timestamp)) return ""; + const date = new Date((timestamp > 10_000_000_000 ? timestamp : timestamp * 1000)); + if (Number.isNaN(date.getTime())) return ""; + const year = String(date.getFullYear()); + const month = String(date.getMonth() + 1).padStart(2, "0"); + const day = String(date.getDate()).padStart(2, "0"); + return `${year}-${month}-${day}`; +} + function setParamFields(params) { const form = els.jobForm.elements; form.start_page.value = params.start_page || ""; diff --git a/scheduler_webui/index.html b/scheduler_webui/index.html index 73736c0a7..dc11886b5 100644 --- a/scheduler_webui/index.html +++ b/scheduler_webui/index.html @@ -101,7 +101,7 @@

                新建作业

                抓取参数

                -
                +
                - + diff --git a/scheduler_webui/styles.css b/scheduler_webui/styles.css index 7f2cdf77a..de5d5e61f 100644 --- a/scheduler_webui/styles.css +++ b/scheduler_webui/styles.css @@ -144,7 +144,7 @@ th { } .scheduler-grid { - grid-template-columns: minmax(420px, 0.95fr) minmax(520px, 1.05fr); + grid-template-columns: minmax(300px, 0.9fr) minmax(360px, 1.1fr); align-items: start; } @@ -243,7 +243,8 @@ th { .job-item.selected { border-color: var(--teal); - box-shadow: inset 3px 0 0 var(--teal); + background: #eefbf7; + box-shadow: inset 4px 0 0 var(--teal), 0 0 0 1px rgba(15, 118, 110, 0.16); } .job-main { @@ -402,7 +403,7 @@ th { gap: 8px; } -@media (max-width: 980px) { +@media (max-width: 760px) { .scheduler-grid { grid-template-columns: 1fr; } From 8c7545c17121a488831a3076d868a0d464194274 Mon Sep 17 00:00:00 2001 From: Raffaello Date: Tue, 30 Jun 2026 10:29:06 +0800 Subject: [PATCH 12/17] feat: show scheduler works and word cloud --- api/routers/scheduler.py | 13 ++ api/scheduler/manager.py | 182 ++++++++++++++++++ api/scheduler/schemas.py | 20 ++ ...77\347\224\250\346\214\207\345\215\227.md" | 4 +- ...66\346\236\204\346\226\207\346\241\243.md" | 1 + scheduler_webui/app.js | 62 +++++- scheduler_webui/index.html | 13 +- scheduler_webui/styles.css | 66 ++++++- tests/test_scheduler_api.py | 31 +++ tests/test_scheduler_manager.py | 48 +++++ 10 files changed, 435 insertions(+), 5 deletions(-) diff --git a/api/routers/scheduler.py b/api/routers/scheduler.py index 025a41511..825222929 100644 --- a/api/routers/scheduler.py +++ b/api/routers/scheduler.py @@ -4,6 +4,7 @@ from api.scheduler.manager import scheduler_manager from api.scheduler.schemas import ( + ArtifactSummaryResponse, ArtifactResponse, InstanceCreateRequest, InstanceResponse, @@ -109,6 +110,18 @@ async def list_job_artifacts(job_id: str): raise HTTPException(status_code=404, detail="Job not found") from exc +@router.get("/jobs/{job_id}/artifact-summary", response_model=ArtifactSummaryResponse) +async def list_job_artifact_summary( + job_id: str, + work_limit: int = Query(default=200, ge=1, le=500), + word_limit: int = Query(default=80, ge=1, le=200), +): + try: + return scheduler_manager.list_job_artifact_summary(job_id, work_limit=work_limit, word_limit=word_limit) + except KeyError as exc: + raise HTTPException(status_code=404, detail="Job not found") from exc + + @router.post("/jobs/{job_id}/artifacts/{artifact_id}/open") async def open_job_artifact(job_id: str, artifact_id: str): try: diff --git a/api/scheduler/manager.py b/api/scheduler/manager.py index d22735788..a43da22d9 100644 --- a/api/scheduler/manager.py +++ b/api/scheduler/manager.py @@ -3,12 +3,16 @@ from __future__ import annotations import asyncio +import csv import json import os +import re import subprocess import sys import uuid +from collections import Counter from dataclasses import dataclass +from datetime import datetime from pathlib import Path from typing import Any, Optional @@ -16,6 +20,32 @@ from .schemas import JobCreateRequest, JobUpdateRequest, InstanceCreateRequest, InstanceUpdateRequest, TaskCreateRequest from .store import PROJECT_ROOT, SchedulerStore, _json_dumps, utc_now +try: + import jieba +except ImportError: # pragma: no cover - dependency exists in normal project env + jieba = None + + +COMMENT_STOP_WORDS = { + "一个", + "不是", + "什么", + "这个", + "就是", + "还是", + "可以", + "没有", + "真的", + "怎么", + "哈哈", + "哈哈哈", + "感觉", + "我们", + "你们", + "他们", + "自己", +} + @dataclass class InstanceRuntime: @@ -228,6 +258,41 @@ def list_job_artifacts(self, job_id: str) -> list[dict[str, Any]]: task_id = self._job_task_id(job) return self.store.list_artifacts(task_id) if task_id else [] + def list_job_artifact_summary(self, job_id: str, work_limit: int = 200, word_limit: int = 80) -> dict[str, Any]: + job = self.store.get_instance(job_id) + if not job: + raise KeyError("job not found") + task_id = self._job_task_id(job) + if not task_id: + return {"works": [], "word_cloud": []} + + works: list[dict[str, Any]] = [] + seen_works: set[str] = set() + words: Counter[str] = Counter() + for artifact in self.store.list_artifacts(task_id): + path = self._safe_artifact_path(task_id, artifact["path"]) + if not path.is_file(): + continue + name = path.name.lower() + if "content" in name: + if len(works) >= work_limit: + continue + for record in self._iter_artifact_records(path): + item = self._work_item(job["platform"], record) + key = item.get("url") or item.get("id") + if not key or key in seen_works: + continue + seen_works.add(key) + works.append(item) + if len(works) >= work_limit: + break + elif "comment" in name: + for record in self._iter_artifact_records(path): + words.update(self._comment_words(record)) + + word_cloud = [{"text": word, "weight": count} for word, count in words.most_common(word_limit)] + return {"works": works, "word_cloud": word_cloud} + def open_job_artifact(self, job_id: str, artifact_id: str) -> dict[str, str]: _, artifact, path = self._job_artifact(job_id, artifact_id) if not path.is_file(): @@ -535,6 +600,123 @@ def _count_records(self, path: Path) -> Optional[int]: return None return None + def _iter_artifact_records(self, path: Path, limit: int = 5000): + try: + if path.suffix == ".jsonl": + with path.open("r", encoding="utf-8") as f: + for index, line in enumerate(f): + if index >= limit: + break + if line.strip(): + yield json.loads(line) + elif path.suffix == ".json": + with path.open("r", encoding="utf-8") as f: + data = json.load(f) + rows = data if isinstance(data, list) else data.get("data", []) if isinstance(data, dict) else [] + for record in rows[:limit]: + if isinstance(record, dict): + yield record + elif path.suffix == ".csv": + with path.open("r", encoding="utf-8") as f: + for index, record in enumerate(csv.DictReader(f)): + if index >= limit: + break + yield record + except Exception: + return + + def _work_item(self, platform: str, record: dict[str, Any]) -> dict[str, Any]: + work_id = self._first_record_value(record, "aweme_id", "note_id", "video_id", "bvid", "content_id", "id") + url = self._first_record_value( + record, + "aweme_url", + "note_url", + "video_url", + "content_url", + "source_url", + "url", + "web_url", + "link", + ) + if not url: + url = self._default_work_url(platform, work_id, record) + return { + "id": str(work_id or ""), + "title": self._first_record_value(record, "title", "desc", "note_title", "content", "content_text", "text") or "未命名作品", + "url": url, + "author": self._first_record_value(record, "nickname", "user_nickname", "author_name", "author", "screen_name") or "", + "publish_time": self._format_record_time(self._first_record_value(record, "create_time", "publish_time", "time", "created_time")), + "source_keyword": self._first_record_value(record, "source_keyword", "keyword") or "", + "metrics": { + "点赞": self._first_record_value(record, "liked_count", "like_count", "voteup_count"), + "收藏": self._first_record_value(record, "collected_count", "favorite_count", "video_favorite_count"), + "评论": self._first_record_value(record, "comment_count", "comments_count", "video_comment"), + "转发": self._first_record_value(record, "share_count", "shared_count", "video_share_count"), + }, + } + + def _comment_words(self, record: dict[str, Any]) -> list[str]: + text = self._first_record_value(record, "content", "comment_content", "text", "desc") + if not text: + return [] + tokens = jieba.lcut(str(text)) if jieba else re.findall(r"[\u4e00-\u9fff]{2,}|[A-Za-z0-9]{2,}", str(text)) + words: list[str] = [] + for token in tokens: + word = token.strip().lower() + if len(word) < 2 or word in COMMENT_STOP_WORDS or re.fullmatch(r"\d+", word): + continue + if not re.search(r"[\u4e00-\u9fffA-Za-z]", word): + continue + words.append(word) + return words + + def _first_record_value(self, record: dict[str, Any], *keys: str) -> Any: + for key in keys: + value = record.get(key) + if value not in (None, ""): + return value + return "" + + def _default_work_url(self, platform: str, work_id: Any, record: dict[str, Any] | None = None) -> str: + if not work_id: + return "" + record = record or {} + work_id = str(work_id) + if platform == "xhs": + token = self._first_record_value(record, "xsec_token") + suffix = f"?xsec_token={token}&xsec_source=pc_search" if token else "" + return f"https://www.xiaohongshu.com/explore/{work_id}{suffix}" + if platform == "dy": + return f"https://www.douyin.com/video/{work_id}" + if platform == "bili": + return f"https://www.bilibili.com/video/{work_id if work_id.upper().startswith('BV') else f'av{work_id}'}" + if platform == "ks": + return f"https://www.kuaishou.com/short-video/{work_id}" + if platform == "wb": + return f"https://m.weibo.cn/detail/{work_id}" + if platform == "tieba": + return f"https://tieba.baidu.com/p/{work_id}" + if platform == "zhihu": + question_id = self._first_record_value(record, "question_id") + content_type = str(self._first_record_value(record, "content_type")).lower() + if question_id: + return f"https://www.zhihu.com/question/{question_id}/answer/{work_id}" + if content_type == "zvideo": + return f"https://www.zhihu.com/zvideo/{work_id}" + return f"https://zhuanlan.zhihu.com/p/{work_id}" + return "" + + def _format_record_time(self, value: Any) -> str: + if value in (None, ""): + return "" + try: + timestamp = float(value) + except (TypeError, ValueError): + return str(value) + if timestamp > 10_000_000_000: + timestamp /= 1000 + return datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d %H:%M") + def _append_process_log(self, task_id: str, line: str) -> None: line = line.strip() if not line: diff --git a/api/scheduler/schemas.py b/api/scheduler/schemas.py index f14434142..bf6e29fc5 100644 --- a/api/scheduler/schemas.py +++ b/api/scheduler/schemas.py @@ -162,6 +162,26 @@ class ArtifactResponse(BaseModel): record_count: Optional[int] = None +class WorkItemResponse(BaseModel): + id: str = "" + title: str + url: str = "" + author: str = "" + publish_time: str = "" + source_keyword: str = "" + metrics: Dict[str, Any] = Field(default_factory=dict) + + +class WordCloudTermResponse(BaseModel): + text: str + weight: int + + +class ArtifactSummaryResponse(BaseModel): + works: list[WorkItemResponse] = Field(default_factory=list) + word_cloud: list[WordCloudTermResponse] = Field(default_factory=list) + + class SchedulerStatusResponse(BaseModel): instances_total: int running_instances: int diff --git "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" index 958a0ffb4..ca8064427 100644 --- "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" +++ "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" @@ -76,7 +76,7 @@ uv run uvicorn api.main:app --port 8080 --reload 调度器页面采用两列工作区： - 左列上方是作业列表，左列下方是当前选中作业的实时日志。 -- 左列底部是当前选中作业的产物列表，支持打开文件和删除文件。 +- 左列底部是当前选中作业的产物列表、作品列表和评论词云。产物列表支持打开文件和删除文件；作品列表中的“打开作品”会在浏览器新窗口打开作品网页，例如抖音作品会跳转到 `https://www.douyin.com/video/{aweme_id}`。 - 右列是新建/编辑作业表单。页面在常见桌面和 in-app browser 宽度下保持左右两列，只有窄屏手机宽度才折叠为单列。 1. 打开 `/scheduler`。 @@ -86,6 +86,7 @@ uv run uvicorn api.main:app --port 8080 --reload 5. 点击“运行”，调度器按作业配置创建一次内部运行记录并启动子进程。 6. 选中作业后，左侧实时日志和产物列表都会自动切换到该作业；作业正在运行时日志和产物指向当前运行。 7. 产物列表中的“打开文件”会调用本机系统打开该文件；`jsonl`、`json`、`csv`、`txt`、`log` 这类文本产物会使用系统文本编辑器打开。按钮会显示“打开中...”和“已打开”。“删除文件”会二次确认，并且只允许删除当前作业已登记的产物文件。 +8. 产物列表下方会从当前或最近一次运行的内容产物中生成作品列表，并从评论产物中生成评论词云。作品列表的“打开作品”只打开作品网页，不打开本地产物文件。当前支持小红书、抖音、快手、B 站、微博、贴吧、知乎的作品网页链接；如果产物里没有保存网页 URL，会按平台和作品 ID 生成官方网页地址。 作业处于 `running` 或 `stopping` 时不能编辑或删除，只能停止。作业失败后状态显示 `error`，可以修改配置后重新运行。 @@ -134,6 +135,7 @@ WebUI 提交后会在内部生成作业 `params`。API 调用仍使用 JSON， | `POST` | `/api/scheduler/jobs/{job_id}/stop` | 停止当前运行 | | `GET` | `/api/scheduler/jobs/{job_id}/logs` | 获取当前或最近一次运行日志 | | `GET` | `/api/scheduler/jobs/{job_id}/artifacts` | 获取当前或最近一次运行产物 | +| `GET` | `/api/scheduler/jobs/{job_id}/artifact-summary` | 从产物中读取作品列表和评论词云 | | `POST` | `/api/scheduler/jobs/{job_id}/artifacts/{artifact_id}/open` | 打开已登记的产物文件 | | `DELETE` | `/api/scheduler/jobs/{job_id}/artifacts/{artifact_id}` | 删除已登记的产物文件 | diff --git "a/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" "b/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" index 4909caf4f..aeeb02021 100644 --- "a/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" +++ "b/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" @@ -173,6 +173,7 @@ flowchart TB - 管理进程生命周期：作业空闲时才能启动运行，停止运行时终止对应子进程。 - 采集运行状态：读取子进程 stdout，落库任务日志、退出码和错误信息。 - 收集抓取产物：任务结束后扫描任务产物目录，记录文件类型、大小和记录数。 +- 派生产物视图：从已登记内容产物生成作品列表，从评论产物生成评论词云，不额外增加持久化表。 调度器相关代码位于： diff --git a/scheduler_webui/app.js b/scheduler_webui/app.js index 9f53774cb..d6bf35510 100644 --- a/scheduler_webui/app.js +++ b/scheduler_webui/app.js @@ -99,6 +99,9 @@ const els = { cancelEditBtn: document.querySelector("#cancelEditBtn"), logsBox: document.querySelector("#logsBox"), artifactsList: document.querySelector("#artifactsList"), + worksList: document.querySelector("#worksList"), + worksHint: document.querySelector("#worksHint"), + wordCloud: document.querySelector("#wordCloud"), jobFilterRows: document.querySelector("#jobFilterRows"), }; @@ -115,6 +118,15 @@ function escapeHtml(value) { .replaceAll("'", "'"); } +function safeHttpUrl(value) { + try { + const url = new URL(String(value || "")); + return ["http:", "https:"].includes(url.protocol) ? url.href : ""; + } catch { + return ""; + } +} + async function errorText(res) { try { const data = await res.json(); @@ -436,6 +448,9 @@ async function selectJob(jobId) { function clearJobDetail() { els.logsBox.textContent = "选择一个作业查看日志。"; els.artifactsList.innerHTML = "
              • 选择一个作业查看产物。
                • "; + els.worksHint.textContent = ""; + els.worksList.innerHTML = "
              • 选择一个作业查看作品。
                • "; + els.wordCloud.innerHTML = `

                选择一个作业查看评论词云。

                `; } async function withButtonFeedback(button, pendingText, doneText, action) { @@ -457,9 +472,10 @@ async function withButtonFeedback(button, pendingText, doneText, action) { } async function loadJobDetail(jobId) { - const [logs, artifacts] = await Promise.all([ + const [logs, artifacts, summary] = await Promise.all([ api.get(`/api/scheduler/jobs/${jobId}/logs?limit=300`), api.get(`/api/scheduler/jobs/${jobId}/artifacts`), + api.get(`/api/scheduler/jobs/${jobId}/artifact-summary?work_limit=100&word_limit=60`), ]); els.logsBox.textContent = logs.length ? logs.map((log) => `[${log.timestamp}] [${log.level}] ${log.message}`).join("\n") @@ -484,6 +500,50 @@ async function loadJobDetail(jobId) { }) .join("") : "
              • 暂无产物。
                • "; + renderWorks(summary.works || []); + renderWordCloud(summary.word_cloud || []); +} + +function renderWorks(works) { + els.worksHint.textContent = works.length ? `${works.length} 个作品` : ""; + els.worksList.innerHTML = works.length + ? works.map((item) => { + const url = safeHttpUrl(item.url); + const title = escapeHtml(item.title || item.id || "未命名作品"); + const metrics = Object.entries(item.metrics || {}) + .filter(([, value]) => value !== "" && value != null) + .map(([key, value]) => `${escapeHtml(key)} ${escapeHtml(value)}`) + .join(" · "); + const openLink = url + ? `打开作品` + : ""; + return ` +
              • + ${title} +
                + ${escapeHtml(item.author || "未知作者")} + ${item.publish_time ? ` · ${escapeHtml(item.publish_time)}` : ""} + ${item.source_keyword ? ` · ${escapeHtml(item.source_keyword)}` : ""} +
                + ${metrics ? `
                ${metrics}
                ` : ""} + ${openLink ? `
                ${openLink}
                ` : ""} +
                • + `; + }).join("") + : "
              • 暂无符合条件的作品。
                • "; +} + +function renderWordCloud(words) { + if (!words.length) { + els.wordCloud.innerHTML = `

                暂无评论词云。

                `; + return; + } + const maxWeight = Math.max(...words.map((item) => item.weight || 1)); + els.wordCloud.innerHTML = words.map((item) => { + const ratio = Math.max(0.7, (item.weight || 1) / maxWeight); + const size = Math.round(13 + ratio * 20); + return `${escapeHtml(item.text)}`; + }).join(""); } els.refreshBtn.addEventListener("click", () => { diff --git a/scheduler_webui/index.html b/scheduler_webui/index.html index df7580f29..279d9aa34 100644 --- a/scheduler_webui/index.html +++ b/scheduler_webui/index.html @@ -4,7 +4,7 @@ MediaCrawler 调度器 - +
                @@ -34,6 +34,15 @@

                实时日志

                产物列表

                  +
                  +

                  作品列表

                  + +
                  +
                    +
                    +

                    评论词云

                    +
                    +
                    @@ -199,6 +208,6 @@

                    登录与代理

                    - + diff --git a/scheduler_webui/styles.css b/scheduler_webui/styles.css index de5d5e61f..bd5fbe03c 100644 --- a/scheduler_webui/styles.css +++ b/scheduler_webui/styles.css @@ -42,15 +42,31 @@ button { white-space: nowrap; } +button, +.link-button { + text-decoration: none; +} + button:hover { background: var(--teal-strong); } -button.secondary { +button.secondary, +.link-button.secondary { background: #fff; color: var(--teal); } +.link-button { + display: inline-flex; + align-items: center; + min-height: 36px; + border: 1px solid var(--teal); + border-radius: 6px; + padding: 0 14px; + white-space: nowrap; +} + button.danger { border-color: var(--red); background: var(--red); @@ -362,6 +378,54 @@ th { line-height: 1.45; } +.subsection-title { + display: flex; + align-items: center; + justify-content: space-between; + gap: 10px; + margin-top: 14px; + padding-top: 14px; + border-top: 1px solid var(--line); +} + +.subsection-title h3 { + margin: 0; + font-size: 14px; +} + +.subsection-title span, +.work-meta { + color: var(--muted); + font-size: 12px; +} + +.work-item { + display: grid; + gap: 6px; +} + +.work-title { + color: var(--text); + font-size: 13px; + font-weight: 700; + line-height: 1.45; + text-decoration: none; +} + +.word-cloud { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 8px 10px; + min-height: 72px; +} + +.word-token { + color: var(--teal); + font-weight: 700; + line-height: 1.1; +} + .pill { display: inline-flex; align-items: center; diff --git a/tests/test_scheduler_api.py b/tests/test_scheduler_api.py index 8a1abf39d..19976d4da 100644 --- a/tests/test_scheduler_api.py +++ b/tests/test_scheduler_api.py @@ -115,3 +115,34 @@ def test_scheduler_api_deletes_job_artifact(monkeypatch, tmp_path): assert response.status_code == 200 assert response.json()["status"] == "ok" assert not artifact_file.exists() + + +def test_scheduler_api_returns_artifact_summary(monkeypatch, tmp_path): + manager = SchedulerManager(store=SchedulerStore(tmp_path / "scheduler.db"), project_root=tmp_path) + monkeypatch.setattr(scheduler_router, "scheduler_manager", manager) + + client = TestClient(app) + job = client.post( + "/api/scheduler/jobs", + json={"name": "抖音作业 A", "platform": "dy", "save_option": "jsonl"}, + ).json() + artifact_dir = tmp_path / "artifacts" / "task-a" + artifact_dir.mkdir(parents=True) + (artifact_dir / "search_contents.jsonl").write_text( + '{"aweme_id":"1","title":"作品 A","aweme_url":"https://www.douyin.com/video/1"}\n', + encoding="utf-8", + ) + (artifact_dir / "search_comments.jsonl").write_text('{"content":"郑州 郑州"}\n', encoding="utf-8") + task = manager.store.create_task( + {"instance_id": job["id"], "crawler_type": "search", "target_text": "", "params": {}}, + str(artifact_dir), + ) + manager.store.update_instance(job["id"], last_task_id=task["id"]) + manager.store.replace_artifacts(task["id"], manager._scan_artifacts(artifact_dir)) + + response = client.get(f"/api/scheduler/jobs/{job['id']}/artifact-summary") + + assert response.status_code == 200 + data = response.json() + assert data["works"][0]["title"] == "作品 A" + assert data["word_cloud"][0] == {"text": "郑州", "weight": 2} diff --git a/tests/test_scheduler_manager.py b/tests/test_scheduler_manager.py index c4fccaad5..7bb1e2339 100644 --- a/tests/test_scheduler_manager.py +++ b/tests/test_scheduler_manager.py @@ -58,6 +58,54 @@ def test_scheduler_manager_scans_artifacts(tmp_path): assert artifacts[0]["record_count"] == 2 +def test_scheduler_manager_builds_artifact_summary(tmp_path): + store = SchedulerStore(tmp_path / "scheduler.db") + manager = SchedulerManager(store=store, project_root=tmp_path) + job = manager.create_job(JobCreateRequest(name="抖音作业", platform="dy")) + artifact_dir = tmp_path / "artifacts" / "task-a" + artifact_dir.mkdir(parents=True) + (artifact_dir / "search_contents.jsonl").write_text( + '{"aweme_id":"1","title":"郑州作品","nickname":"作者","liked_count":"10","aweme_url":"https://www.douyin.com/video/1"}\n', + encoding="utf-8", + ) + (artifact_dir / "search_comments.jsonl").write_text( + '{"content":"郑州 城市 郑州"}\n{"content":"城市 交通"}\n', + encoding="utf-8", + ) + task = store.create_task( + {"instance_id": job["id"], "crawler_type": "search", "target_text": "", "params": {}}, + str(artifact_dir), + ) + store.update_instance(job["id"], last_task_id=task["id"]) + store.replace_artifacts(task["id"], manager._scan_artifacts(artifact_dir)) + + summary = manager.list_job_artifact_summary(job["id"]) + + assert summary["works"][0]["title"] == "郑州作品" + assert summary["works"][0]["url"] == "https://www.douyin.com/video/1" + assert {"text": "郑州", "weight": 2} in summary["word_cloud"] + + +def test_scheduler_manager_builds_platform_work_urls(tmp_path): + manager = SchedulerManager(store=SchedulerStore(tmp_path / "scheduler.db"), project_root=tmp_path) + + cases = [ + ("xhs", "abc", {"xsec_token": "token"}, "https://www.xiaohongshu.com/explore/abc?xsec_token=token&xsec_source=pc_search"), + ("dy", "123", {}, "https://www.douyin.com/video/123"), + ("ks", "123", {}, "https://www.kuaishou.com/short-video/123"), + ("bili", "456", {}, "https://www.bilibili.com/video/av456"), + ("bili", "BV1xx", {}, "https://www.bilibili.com/video/BV1xx"), + ("wb", "123", {}, "https://m.weibo.cn/detail/123"), + ("tieba", "123", {}, "https://tieba.baidu.com/p/123"), + ("zhihu", "789", {"question_id": "456"}, "https://www.zhihu.com/question/456/answer/789"), + ("zhihu", "789", {"content_type": "zvideo"}, "https://www.zhihu.com/zvideo/789"), + ("zhihu", "789", {"content_type": "article"}, "https://zhuanlan.zhihu.com/p/789"), + ] + + for platform, work_id, record, expected in cases: + assert manager._default_work_url(platform, work_id, record) == expected + + def test_scheduler_manager_deletes_job_artifact(tmp_path): store = SchedulerStore(tmp_path / "scheduler.db") manager = SchedulerManager(store=store, project_root=tmp_path) From a012bf37ecbd438a23b01c5b42b5beb7743c476e Mon Sep 17 00:00:00 2001 From: Raffaello Date: Tue, 30 Jun 2026 10:37:00 +0800 Subject: [PATCH 13/17] fix: move scheduler artifacts to right column --- ...77\347\224\250\346\214\207\345\215\227.md" | 10 ++++---- scheduler_webui/index.html | 25 +++++++++++-------- scheduler_webui/styles.css | 6 +++-- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" index ca8064427..c21e1d9a7 100644 --- "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" +++ "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" @@ -75,18 +75,18 @@ uv run uvicorn api.main:app --port 8080 --reload 调度器页面采用两列工作区： -- 左列上方是作业列表，左列下方是当前选中作业的实时日志。 -- 左列底部是当前选中作业的产物列表、作品列表和评论词云。产物列表支持打开文件和删除文件；作品列表中的“打开作品”会在浏览器新窗口打开作品网页，例如抖音作品会跳转到 `https://www.douyin.com/video/{aweme_id}`。 -- 右列是新建/编辑作业表单。页面在常见桌面和 in-app browser 宽度下保持左右两列，只有窄屏手机宽度才折叠为单列。 +- 左列上方是作业列表，中部是当前选中作业的实时日志，底部是作品列表。作品列表中的“打开作品”会在浏览器新窗口打开作品网页，例如抖音作品会跳转到 `https://www.douyin.com/video/{aweme_id}`。 +- 右列上方是新建/编辑作业表单，下方依次是评论词云和产物列表；产物列表位于右列最底部，支持打开文件和删除文件。 +- 页面在常见桌面和 in-app browser 宽度下保持左右两列，只有窄屏手机宽度才折叠为单列。 1. 打开 `/scheduler`。 2. 创建作业，填写平台、登录方式、保存格式、爬取类型、目标、抓取参数、内容过滤、Cookie 和代理。 3. 点击作业列表中的条目即可选中作业，右侧表单会同步显示该作业当前配置，底部按钮会变为“保存更改”。 4. 点击作业条目里的“登录”，调度器会创建一次内部 `login` 运行记录，只维护登录态，不进入抓取流程。 5. 点击“运行”，调度器按作业配置创建一次内部运行记录并启动子进程。 -6. 选中作业后，左侧实时日志和产物列表都会自动切换到该作业；作业正在运行时日志和产物指向当前运行。 +6. 选中作业后，实时日志、作品列表、评论词云和产物列表都会自动切换到该作业；作业正在运行时日志和产物指向当前运行。 7. 产物列表中的“打开文件”会调用本机系统打开该文件；`jsonl`、`json`、`csv`、`txt`、`log` 这类文本产物会使用系统文本编辑器打开。按钮会显示“打开中...”和“已打开”。“删除文件”会二次确认，并且只允许删除当前作业已登记的产物文件。 -8. 产物列表下方会从当前或最近一次运行的内容产物中生成作品列表，并从评论产物中生成评论词云。作品列表的“打开作品”只打开作品网页，不打开本地产物文件。当前支持小红书、抖音、快手、B 站、微博、贴吧、知乎的作品网页链接；如果产物里没有保存网页 URL，会按平台和作品 ID 生成官方网页地址。 +8. 作品列表会从当前或最近一次运行的内容产物中生成，评论词云会从评论产物中生成。作品列表的“打开作品”只打开作品网页，不打开本地产物文件。当前支持小红书、抖音、快手、B 站、微博、贴吧、知乎的作品网页链接；如果产物里没有保存网页 URL，会按平台和作品 ID 生成官方网页地址。 作业处于 `running` 或 `stopping` 时不能编辑或删除，只能停止。作业失败后状态显示 `error`，可以修改配置后重新运行。 diff --git a/scheduler_webui/index.html b/scheduler_webui/index.html index 279d9aa34..40b028bf6 100644 --- a/scheduler_webui/index.html +++ b/scheduler_webui/index.html @@ -4,7 +4,7 @@ MediaCrawler 调度器 - +
                    @@ -31,18 +31,12 @@

                    实时日志

                    选择一个作业查看日志。
                    -
                    -

                    产物列表

                    -
                      -
                      -

                      作品列表

                      +
                      +
                      +

                      作品列表

                        -
                        -

                        评论词云

                        -
                        -
                        @@ -205,9 +199,18 @@

                        登录与代理

                        +
                        +

                        评论词云

                        +
                        +
                        + +
                        +

                        产物列表

                        +
                          +
                          - + diff --git a/scheduler_webui/styles.css b/scheduler_webui/styles.css index bd5fbe03c..05675bfe6 100644 --- a/scheduler_webui/styles.css +++ b/scheduler_webui/styles.css @@ -351,7 +351,8 @@ th { word-break: break-word; } -.artifact-panel ul { +.artifact-panel ul, +.works-panel ul { display: grid; gap: 8px; margin: 0; @@ -359,7 +360,8 @@ th { list-style: none; } -.artifact-panel li { +.artifact-panel li, +.works-panel li { border: 1px solid var(--line); border-radius: 6px; padding: 10px; From 0857848f88cb3502072c14a674922eb656bfc35b Mon Sep 17 00:00:00 2001 From: Raffaello Date: Tue, 30 Jun 2026 10:46:54 +0800 Subject: [PATCH 14/17] fix: paginate scheduler works list --- ...77\347\224\250\346\214\207\345\215\227.md" | 2 +- scheduler_webui/app.js | 43 ++++++++++++++++--- scheduler_webui/index.html | 3 +- scheduler_webui/styles.css | 14 ++++++ 4 files changed, 55 insertions(+), 7 deletions(-) diff --git "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" index c21e1d9a7..855005541 100644 --- "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" +++ "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" @@ -86,7 +86,7 @@ uv run uvicorn api.main:app --port 8080 --reload 5. 点击“运行”，调度器按作业配置创建一次内部运行记录并启动子进程。 6. 选中作业后，实时日志、作品列表、评论词云和产物列表都会自动切换到该作业；作业正在运行时日志和产物指向当前运行。 7. 产物列表中的“打开文件”会调用本机系统打开该文件；`jsonl`、`json`、`csv`、`txt`、`log` 这类文本产物会使用系统文本编辑器打开。按钮会显示“打开中...”和“已打开”。“删除文件”会二次确认，并且只允许删除当前作业已登记的产物文件。 -8. 作品列表会从当前或最近一次运行的内容产物中生成，评论词云会从评论产物中生成。作品列表的“打开作品”只打开作品网页，不打开本地产物文件。当前支持小红书、抖音、快手、B 站、微博、贴吧、知乎的作品网页链接；如果产物里没有保存网页 URL，会按平台和作品 ID 生成官方网页地址。 +8. 作品列表会从当前或最近一次运行的内容产物中生成，并按 5 条一页分页展示；评论词云会从评论产物中生成。作品列表的“打开作品”只打开作品网页，不打开本地产物文件。当前支持小红书、抖音、快手、B 站、微博、贴吧、知乎的作品网页链接；如果产物里没有保存网页 URL，会按平台和作品 ID 生成官方网页地址。 作业处于 `running` 或 `stopping` 时不能编辑或删除，只能停止。作业失败后状态显示 `error`，可以修改配置后重新运行。 diff --git a/scheduler_webui/app.js b/scheduler_webui/app.js index d6bf35510..80ca53739 100644 --- a/scheduler_webui/app.js +++ b/scheduler_webui/app.js @@ -1,3 +1,5 @@ +const WORKS_PAGE_SIZE = 5; + const api = { async get(path) { const res = await fetch(path); @@ -34,6 +36,8 @@ let state = { selectedJobId: "", editingJobId: "", formDirty: false, + works: [], + worksPage: 1, filterRows: { job: [], }, @@ -101,6 +105,7 @@ const els = { artifactsList: document.querySelector("#artifactsList"), worksList: document.querySelector("#worksList"), worksHint: document.querySelector("#worksHint"), + worksPager: document.querySelector("#worksPager"), wordCloud: document.querySelector("#wordCloud"), jobFilterRows: document.querySelector("#jobFilterRows"), }; @@ -440,16 +445,20 @@ function syncSelectedJobForm() { async function selectJob(jobId) { const job = state.jobs.find((item) => item.id === jobId) || await api.get(`/api/scheduler/jobs/${jobId}`); state.selectedJobId = job.id; + state.worksPage = 1; fillJobForm(job); renderJobs(); await loadJobDetail(job.id); } function clearJobDetail() { + state.works = []; + state.worksPage = 1; els.logsBox.textContent = "选择一个作业查看日志。"; els.artifactsList.innerHTML = "
                        • 选择一个作业查看产物。
                          • "; els.worksHint.textContent = ""; els.worksList.innerHTML = "
                        • 选择一个作业查看作品。
                          • "; + els.worksPager.innerHTML = ""; els.wordCloud.innerHTML = `

                          选择一个作业查看评论词云。

                          `; } @@ -500,14 +509,21 @@ async function loadJobDetail(jobId) { }) .join("") : "
                        • 暂无产物。
                          • "; - renderWorks(summary.works || []); + state.works = summary.works || []; + renderWorks(); renderWordCloud(summary.word_cloud || []); } -function renderWorks(works) { - els.worksHint.textContent = works.length ? `${works.length} 个作品` : ""; - els.worksList.innerHTML = works.length - ? works.map((item) => { +function renderWorks() { + const works = state.works || []; + const totalPages = Math.max(1, Math.ceil(works.length / WORKS_PAGE_SIZE)); + state.worksPage = Math.min(Math.max(state.worksPage, 1), totalPages); + const start = (state.worksPage - 1) * WORKS_PAGE_SIZE; + const currentWorks = works.slice(start, start + WORKS_PAGE_SIZE); + + els.worksHint.textContent = works.length ? `${works.length} 个作品，第 ${state.worksPage}/${totalPages} 页` : ""; + els.worksList.innerHTML = currentWorks.length + ? currentWorks.map((item) => { const url = safeHttpUrl(item.url); const title = escapeHtml(item.title || item.id || "未命名作品"); const metrics = Object.entries(item.metrics || {}) @@ -531,6 +547,13 @@ function renderWorks(works) { `; }).join("") : "
                        • 暂无符合条件的作品。
                          • "; + els.worksPager.innerHTML = works.length + ? ` + + 第 ${state.worksPage} / ${totalPages} 页 + + ` + : ""; } function renderWordCloud(words) { @@ -591,6 +614,16 @@ document.addEventListener("click", async (event) => { const action = target.dataset.action; const id = target.dataset.id; try { + if (action === "works-prev") { + state.worksPage -= 1; + renderWorks(); + return; + } + if (action === "works-next") { + state.worksPage += 1; + renderWorks(); + return; + } if (action === "add-filter") { addFilterRow(target.dataset.scope); state.formDirty = true; diff --git a/scheduler_webui/index.html b/scheduler_webui/index.html index 40b028bf6..fe04c73a6 100644 --- a/scheduler_webui/index.html +++ b/scheduler_webui/index.html @@ -37,6 +37,7 @@

                          作品列表

                            +
                            @@ -211,6 +212,6 @@

                            产物列表

                            - + diff --git a/scheduler_webui/styles.css b/scheduler_webui/styles.css index 05675bfe6..453b2a137 100644 --- a/scheduler_webui/styles.css +++ b/scheduler_webui/styles.css @@ -368,6 +368,20 @@ th { word-break: break-all; } +.pager { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; + margin-top: 10px; +} + +.pager span { + color: var(--muted); + font-size: 12px; + white-space: nowrap; +} + .artifact-item { display: grid; gap: 10px; From 27f24f5cbeae21fb17a8a37140e73e21ff247892 Mon Sep 17 00:00:00 2001 From: Raffaello Date: Tue, 30 Jun 2026 10:58:00 +0800 Subject: [PATCH 15/17] fix: show eight works per scheduler page --- ...\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" | 2 +- scheduler_webui/app.js | 2 +- scheduler_webui/index.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" index 855005541..9281b1243 100644 --- "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" +++ "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" @@ -86,7 +86,7 @@ uv run uvicorn api.main:app --port 8080 --reload 5. 点击“运行”，调度器按作业配置创建一次内部运行记录并启动子进程。 6. 选中作业后，实时日志、作品列表、评论词云和产物列表都会自动切换到该作业；作业正在运行时日志和产物指向当前运行。 7. 产物列表中的“打开文件”会调用本机系统打开该文件；`jsonl`、`json`、`csv`、`txt`、`log` 这类文本产物会使用系统文本编辑器打开。按钮会显示“打开中...”和“已打开”。“删除文件”会二次确认，并且只允许删除当前作业已登记的产物文件。 -8. 作品列表会从当前或最近一次运行的内容产物中生成，并按 5 条一页分页展示；评论词云会从评论产物中生成。作品列表的“打开作品”只打开作品网页，不打开本地产物文件。当前支持小红书、抖音、快手、B 站、微博、贴吧、知乎的作品网页链接；如果产物里没有保存网页 URL，会按平台和作品 ID 生成官方网页地址。 +8. 作品列表会从当前或最近一次运行的内容产物中生成，并按 8 条一页分页展示；评论词云会从评论产物中生成。作品列表的“打开作品”只打开作品网页，不打开本地产物文件。当前支持小红书、抖音、快手、B 站、微博、贴吧、知乎的作品网页链接；如果产物里没有保存网页 URL，会按平台和作品 ID 生成官方网页地址。 作业处于 `running` 或 `stopping` 时不能编辑或删除，只能停止。作业失败后状态显示 `error`，可以修改配置后重新运行。 diff --git a/scheduler_webui/app.js b/scheduler_webui/app.js index 80ca53739..f544c3fb9 100644 --- a/scheduler_webui/app.js +++ b/scheduler_webui/app.js @@ -1,4 +1,4 @@ -const WORKS_PAGE_SIZE = 5; +const WORKS_PAGE_SIZE = 8; const api = { async get(path) { diff --git a/scheduler_webui/index.html b/scheduler_webui/index.html index fe04c73a6..2e63d85c4 100644 --- a/scheduler_webui/index.html +++ b/scheduler_webui/index.html @@ -212,6 +212,6 @@

                            产物列表

                            - + From 5f586ac51a1f0f595c11c6381c76b4e4d4b7afae Mon Sep 17 00:00:00 2001 From: Raffaello Date: Tue, 30 Jun 2026 14:03:52 +0800 Subject: [PATCH 16/17] fix: sort scheduler works by likes --- api/scheduler/manager.py | 21 ++++++++++++++----- ...77\347\224\250\346\214\207\345\215\227.md" | 2 +- ...66\346\236\204\346\226\207\346\241\243.md" | 2 +- tests/test_scheduler_manager.py | 10 +++++---- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/api/scheduler/manager.py b/api/scheduler/manager.py index a43da22d9..07c6089e5 100644 --- a/api/scheduler/manager.py +++ b/api/scheduler/manager.py @@ -275,8 +275,6 @@ def list_job_artifact_summary(self, job_id: str, work_limit: int = 200, word_lim continue name = path.name.lower() if "content" in name: - if len(works) >= work_limit: - continue for record in self._iter_artifact_records(path): item = self._work_item(job["platform"], record) key = item.get("url") or item.get("id") @@ -284,14 +282,13 @@ def list_job_artifact_summary(self, job_id: str, work_limit: int = 200, word_lim continue seen_works.add(key) works.append(item) - if len(works) >= work_limit: - break elif "comment" in name: for record in self._iter_artifact_records(path): words.update(self._comment_words(record)) + works.sort(key=lambda item: self._metric_number(item.get("metrics", {}).get("点赞")), reverse=True) word_cloud = [{"text": word, "weight": count} for word, count in words.most_common(word_limit)] - return {"works": works, "word_cloud": word_cloud} + return {"works": works[:work_limit], "word_cloud": word_cloud} def open_job_artifact(self, job_id: str, artifact_id: str) -> dict[str, str]: _, artifact, path = self._job_artifact(job_id, artifact_id) @@ -677,6 +674,20 @@ def _first_record_value(self, record: dict[str, Any], *keys: str) -> Any: return value return "" + def _metric_number(self, value: Any) -> float: + if isinstance(value, (int, float)): + return float(value) + text = str(value or "").replace(",", "").strip() + match = re.search(r"-?\d+(?:\.\d+)?", text) + if not match: + return 0.0 + number = float(match.group()) + if "亿" in text: + return number * 100000000 + if "万" in text or "w" in text.lower(): + return number * 10000 + return number + def _default_work_url(self, platform: str, work_id: Any, record: dict[str, Any] | None = None) -> str: if not work_id: return "" diff --git "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" index 9281b1243..65c5849a0 100644 --- "a/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" +++ "b/docs/\345\244\232\345\256\236\344\276\213\350\260\203\345\272\246\345\231\250\344\275\277\347\224\250\346\214\207\345\215\227.md" @@ -86,7 +86,7 @@ uv run uvicorn api.main:app --port 8080 --reload 5. 点击“运行”，调度器按作业配置创建一次内部运行记录并启动子进程。 6. 选中作业后，实时日志、作品列表、评论词云和产物列表都会自动切换到该作业；作业正在运行时日志和产物指向当前运行。 7. 产物列表中的“打开文件”会调用本机系统打开该文件；`jsonl`、`json`、`csv`、`txt`、`log` 这类文本产物会使用系统文本编辑器打开。按钮会显示“打开中...”和“已打开”。“删除文件”会二次确认，并且只允许删除当前作业已登记的产物文件。 -8. 作品列表会从当前或最近一次运行的内容产物中生成，并按 8 条一页分页展示；评论词云会从评论产物中生成。作品列表的“打开作品”只打开作品网页，不打开本地产物文件。当前支持小红书、抖音、快手、B 站、微博、贴吧、知乎的作品网页链接；如果产物里没有保存网页 URL，会按平台和作品 ID 生成官方网页地址。 +8. 作品列表会从当前或最近一次运行的内容产物中生成，按点赞数从高到低排序，并按 8 条一页分页展示；评论词云会从评论产物中生成。作品列表的“打开作品”只打开作品网页，不打开本地产物文件。当前支持小红书、抖音、快手、B 站、微博、贴吧、知乎的作品网页链接；如果产物里没有保存网页 URL，会按平台和作品 ID 生成官方网页地址。 作业处于 `running` 或 `stopping` 时不能编辑或删除，只能停止。作业失败后状态显示 `error`，可以修改配置后重新运行。 diff --git "a/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" "b/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" index aeeb02021..de9a5c8a7 100644 --- "a/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" +++ "b/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" @@ -173,7 +173,7 @@ flowchart TB - 管理进程生命周期：作业空闲时才能启动运行，停止运行时终止对应子进程。 - 采集运行状态：读取子进程 stdout，落库任务日志、退出码和错误信息。 - 收集抓取产物：任务结束后扫描任务产物目录，记录文件类型、大小和记录数。 -- 派生产物视图：从已登记内容产物生成作品列表，从评论产物生成评论词云，不额外增加持久化表。 +- 派生产物视图：从已登记内容产物生成按点赞数倒序的作品列表，从评论产物生成评论词云，不额外增加持久化表。 调度器相关代码位于： diff --git a/tests/test_scheduler_manager.py b/tests/test_scheduler_manager.py index 7bb1e2339..49b479061 100644 --- a/tests/test_scheduler_manager.py +++ b/tests/test_scheduler_manager.py @@ -65,7 +65,9 @@ def test_scheduler_manager_builds_artifact_summary(tmp_path): artifact_dir = tmp_path / "artifacts" / "task-a" artifact_dir.mkdir(parents=True) (artifact_dir / "search_contents.jsonl").write_text( - '{"aweme_id":"1","title":"郑州作品","nickname":"作者","liked_count":"10","aweme_url":"https://www.douyin.com/video/1"}\n', + '{"aweme_id":"1","title":"郑州作品","nickname":"作者","liked_count":"10","aweme_url":"https://www.douyin.com/video/1"}\n' + '{"aweme_id":"2","title":"高赞作品","nickname":"作者","liked_count":"1.2万","aweme_url":"https://www.douyin.com/video/2"}\n' + '{"aweme_id":"3","title":"中赞作品","nickname":"作者","liked_count":"200","aweme_url":"https://www.douyin.com/video/3"}\n', encoding="utf-8", ) (artifact_dir / "search_comments.jsonl").write_text( @@ -79,10 +81,10 @@ def test_scheduler_manager_builds_artifact_summary(tmp_path): store.update_instance(job["id"], last_task_id=task["id"]) store.replace_artifacts(task["id"], manager._scan_artifacts(artifact_dir)) - summary = manager.list_job_artifact_summary(job["id"]) + summary = manager.list_job_artifact_summary(job["id"], work_limit=2) - assert summary["works"][0]["title"] == "郑州作品" - assert summary["works"][0]["url"] == "https://www.douyin.com/video/1" + assert [work["title"] for work in summary["works"]] == ["高赞作品", "中赞作品"] + assert summary["works"][0]["url"] == "https://www.douyin.com/video/2" assert {"text": "郑州", "weight": 2} in summary["word_cloud"] From 6e4ba22adc3dc88f61c6f357da1df79e4368252a Mon Sep 17 00:00:00 2001 From: Raffaello Date: Thu, 2 Jul 2026 08:19:55 +0800 Subject: [PATCH 17/17] docs: trim scheduler contribution docs --- docs/.vitepress/config.mjs | 1 + ...43\347\240\201\347\273\223\346\236\204.md" | 9 +-- ...66\346\236\204\346\226\207\346\241\243.md" | 59 +------------------ 3 files changed, 5 insertions(+), 64 deletions(-) diff --git a/docs/.vitepress/config.mjs b/docs/.vitepress/config.mjs index 8af6d83ab..14bb03295 100644 --- a/docs/.vitepress/config.mjs +++ b/docs/.vitepress/config.mjs @@ -46,6 +46,7 @@ export default withMermaid(defineConfig({ {text: '基本使用', link: '/'}, {text: '项目架构文档', link: '/项目架构文档'}, {text: '多实例调度器', link: '/多实例调度器使用指南'}, + {text: '内容过滤', link: '/内容过滤使用指南'}, {text: '常见问题汇总', link: '/常见问题'}, {text: 'IP代理使用', link: '/代理使用'}, {text: '词云图使用', link: '/词云图使用配置'}, diff --git "a/docs/\351\241\271\347\233\256\344\273\243\347\240\201\347\273\223\346\236\204.md" "b/docs/\351\241\271\347\233\256\344\273\243\347\240\201\347\273\223\346\236\204.md" index 349e9fff1..6a5e2ed80 100644 --- "a/docs/\351\241\271\347\233\256\344\273\243\347\240\201\347\273\223\346\236\204.md" +++ "b/docs/\351\241\271\347\233\256\344\273\243\347\240\201\347\273\223\346\236\204.md" @@ -11,11 +11,6 @@ MediaCrawler │ └── redis_cache.py # Redis缓存实现 ├── cmd_arg │ └── arg.py # 命令行参数定义 -├── api -│ ├── main.py # FastAPI 服务入口 -│ ├── routers # WebUI 与调度器 API 路由 -│ ├── scheduler # 多实例调度器状态库与进程管理 -│ └── webui # 单实例 WebUI 构建产物 ├── config │ ├── base_config.py # 基础配置 │ ├── db_config.py # 数据库配置 @@ -68,11 +63,9 @@ MediaCrawler │ ├── browser_launcher.py # 浏览器启动器 │ ├── cdp_browser.py # CDP浏览器控制 │ ├── crawler_util.py # 爬虫工具函数 -│ ├── profile.py # 浏览器 Profile 目录解析 │ ├── utils.py # 通用工具函数 │ └── ... -├── scheduler_webui # 多实例调度器静态管理页面源文件 ├── main.py # 程序入口, 支持 --init_db 参数来初始化数据库 ├── recv_sms.py # 短信转发HTTP SERVER接口 └── var.py # 全局上下文变量定义 -``` +``` \ No newline at end of file diff --git "a/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" "b/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" index de9a5c8a7..4a1c048c6 100644 --- "a/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" +++ "b/docs/\351\241\271\347\233\256\346\236\266\346\236\204\346\226\207\346\241\243.md" @@ -110,7 +110,6 @@ flowchart LR login["登录认证"] search["搜索/爬取"] parse["数据解析"] - filter["内容过滤"] comment["获取评论"] end @@ -132,59 +131,15 @@ flowchart LR browser --> login login --> search search --> parse - parse --> filter - filter --> comment - filter --> content + parse --> comment + parse --> content comment --> comments parse --> creator - filter --> media + parse --> media content & comments & creator --> file & db & nosql media --> file ``` -内容过滤由 `tools/content_filter.py` 提供统一规则，支持按发布时间、点赞、收藏、转发、评论等条件筛选。平台 `core.py` 在详情解析完成后调用，过滤命中的内容才会进入内容存储、媒体下载和评论抓取；创作者信息本身不受内容过滤影响。 - ---- - -### 2.3 多实例调度器架构 - -在原有单实例爬虫链路之外，项目新增了一个独立调度器控制面。调度器不重写各平台抓取逻辑，而是通过 `uv run python main.py` 拉起多个子进程，并为每个作业注入独立的登录态目录、CDP 端口、爬取参数和产物目录。 - -```mermaid -flowchart TB - scheduler_ui["/scheduler 调度器 WebUI"] --> scheduler_api["/api/scheduler/*"] - scheduler_api --> manager["SchedulerManager"] - manager --> sqlite["SchedulerStore
                            data/scheduler/scheduler.db"] - - manager --> task_a["作业 A 子进程"] - manager --> task_b["作业 B 子进程"] - - task_a --> profile_a["data/scheduler/profiles/A"] - task_b --> profile_b["data/scheduler/profiles/B"] - - task_a --> artifact_a["data/scheduler/artifacts/A/task"] - task_b --> artifact_b["data/scheduler/artifacts/B/task"] -``` - -调度器核心职责： - -- 维护作业配置：平台、登录方式、保存格式、CDP 端口、浏览器 Profile、爬取类型、目标和参数。 -- 维护内部运行记录：登录、搜索、详情和创作者运行都绑定到具体作业。 -- 管理进程生命周期：作业空闲时才能启动运行，停止运行时终止对应子进程。 -- 采集运行状态：读取子进程 stdout，落库任务日志、退出码和错误信息。 -- 收集抓取产物：任务结束后扫描任务产物目录，记录文件类型、大小和记录数。 -- 派生产物视图：从已登记内容产物生成按点赞数倒序的作品列表，从评论产物生成评论词云，不额外增加持久化表。 - -调度器相关代码位于： - -| 模块 | 说明 | -|------|------| -| `api/routers/scheduler.py` | 调度器 HTTP API | -| `api/scheduler/manager.py` | 作业运行、子进程和产物扫描 | -| `api/scheduler/store.py` | SQLite 状态库 | -| `scheduler_webui/` | 调度器管理页面 | -| `tools/profile.py` | 实例浏览器 Profile 路径解析 | - --- ## 3. 目录结构 @@ -235,17 +190,9 @@ MediaCrawler/ │ ├── app_runner.py # 应用运行管理 │ ├── browser_launcher.py # 浏览器启动 │ ├── cdp_browser.py # CDP浏览器管理 -│ ├── content_filter.py # 内容发布时间与互动指标过滤 -│ ├── profile.py # 多实例浏览器 Profile 路径解析 │ ├── crawler_util.py # 爬虫工具 │ └── async_file_writer.py # 异步文件写入 │ -├── api/ # WebUI API 与调度器 API -│ ├── routers/ # 路由层 -│ └── scheduler/ # 多实例调度器 -│ -├── scheduler_webui/ # 多实例调度器管理页面 -│ ├── model/ # 数据模型 │ └── m_{platform}.py # Pydantic模型 │