diff --git a/.github/workflows/sync-catalog.yml b/.github/workflows/sync-catalog.yml new file mode 100644 index 0000000000..8887010a16 --- /dev/null +++ b/.github/workflows/sync-catalog.yml @@ -0,0 +1,65 @@ +name: Sync CLI skills to catalog + +# Mirrors skills/ into firecrawl/skills under skills/cli/. +# The catalog copy is a read-only mirror; PRs against it are redirected here. + +on: + push: + branches: [main] + paths: + - "skills/**" + - ".github/workflows/sync-catalog.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: catalog-sync + cancel-in-progress: false + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - name: Checkout CLI repo + uses: actions/checkout@v6 + with: + path: cli + + - name: Checkout catalog + uses: actions/checkout@v6 + with: + repository: firecrawl/skills + ssh-key: ${{ secrets.CATALOG_DEPLOY_KEY }} + path: catalog + + - name: Sync skills/cli + run: | + cd catalog + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + rm -rf skills/cli + mkdir -p skills/cli + cp -R ../cli/skills/. skills/cli/ + + cat > skills/cli/README.md <<'EOF' + # CLI skills (mirror) + + Mirror of [`firecrawl/cli`](https://github.com/firecrawl/cli/tree/main/skills) — do not edit here. + PR changes against `firecrawl/cli`; CI overwrites this directory on every sync. + EOF + + git add -A + if git diff --cached --quiet; then + echo "No changes to sync" + exit 0 + fi + + SHORT_SHA="${GITHUB_SHA:0:7}" + git commit -m "Sync from firecrawl/cli@${SHORT_SHA}" \ + -m "Source: https://github.com/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}" + + # Retry once if another sync landed first; never force-push. + git push origin main || { git pull --rebase origin main && git push origin main; } diff --git a/README.md b/README.md index 382ca36789..02dc6b2634 100644 --- a/README.md +++ b/README.md @@ -62,11 +62,18 @@ detected harnesses (all selected by default) so you can pick a subset. ### Agent skills -The init command installs all Firecrawl agent skill segments into AI coding agents (Cursor, Claude Code, Windsurf, etc.): +The init command installs the **CLI skills** by default and offers the **workflow skills** as optional extras, into AI coding agents (Cursor, Claude Code, Windsurf, etc.): -- **CLI skills** — teach agents how to use the Firecrawl CLI for live web work (search, scrape, interact, map, crawl, agent) -- **Build skills** — teach agents how to integrate Firecrawl into application code (choose endpoints, wire SDKs, set up API keys) -- **Workflow skills** — teach agents how to produce Firecrawl-powered deliverables such as research briefs, SEO audits, QA reports, lead lists, knowledge bases, and design-system extraction +- **CLI skills** — teach agents how to use the Firecrawl CLI for live web work (search, scrape, interact, map, crawl, agent). Installed by default. +- **Workflow skills** — teach agents how to produce Firecrawl-powered deliverables such as research briefs, SEO audits, QA reports, lead lists, knowledge bases, and design-system extraction. Interactive multi-select during init. + +All skill families live in the [`firecrawl/skills`](https://github.com/firecrawl/skills) catalog — including the **build skills** for integrating Firecrawl into application code: + +```bash +npx skills add firecrawl/skills +``` + +> Contributing skills? CLI skills (including the research/developer index skills) → PR this repo (`skills/`). Build/SDK skills → PR the [`firecrawl`](https://github.com/firecrawl/firecrawl) monorepo (`skills/`). Workflow skills → PR [`firecrawl/firecrawl-workflows`](https://github.com/firecrawl/firecrawl-workflows). The catalog ([`firecrawl/skills`](https://github.com/firecrawl/skills)) is read-only — never PR it directly. To reinstall skills manually: @@ -961,9 +968,9 @@ firecrawl x download https://docs.firecrawl.dev --include-paths "/features,/sdks ### Workflow Skills -The old experimental AI workflow commands have moved to the NPX-installable -[`firecrawl/firecrawl-workflows`](https://github.com/firecrawl/firecrawl-workflows) -skills package. Workflow skills infer from the user's request first and only ask +The old experimental AI workflow commands have moved to the +[`firecrawl/skills`](https://github.com/firecrawl/skills) catalog +(`skills/workflows/`). Workflow skills infer from the user's request first and only ask short clarifying questions when required inputs are missing. Install them with: ```bash diff --git a/skills/firecrawl-developer-index/SKILL.md b/skills/firecrawl-developer-index/SKILL.md index f804e24dfa..30ea7eaaa5 100644 --- a/skills/firecrawl-developer-index/SKILL.md +++ b/skills/firecrawl-developer-index/SKILL.md @@ -1,41 +1,59 @@ --- name: firecrawl-developer-index -description: | - Search issues, merged pull requests, READMEs, and documentation. Use when the question is how a library or API behaves, what an error means, or whether a bug was fixed; prefer this over a general web page. -allowed-tools: - - Bash(firecrawl *) - - Bash(npx firecrawl-cli *) +description: Search issues, merged pull requests, READMEs, and documentation. Use when the question is how a library or API behaves, what an error means, or whether a bug was fixed; prefer this over a general web page. --- -# firecrawl developer +# Firecrawl Developer Index -Answer a developer question from the primary source: the issue, the merged pull request that fixed it, or the README/docs passage that states the contract. +Answer a developer question from the primary source: the issue where the bug was reported, the merged pull request that fixed it, the README or documentation page that states the contract. A blog post that describes a behaviour is a weaker answer than the passage that defines it, so reach for the index first and the open web second. -## Quick start +There is **no fixed recipe**. Read the question, decide what kind it is, and choose the approach below. A literal error string wants a different move than "how do I do X". Don't run machinery a question doesn't call for. -```bash -mkdir -p .firecrawl -firecrawl developer "how do I configure retries" --limit 10 -o .firecrawl/developer.json --json -jq -r '.results[] | .id, .url, .passages[].text' .firecrawl/developer.json -``` +## The tools, and what each is uniquely good at -Run `firecrawl developer --help` for the full option list. +- HTTP: **`GET|POST https://api.firecrawl.dev/v2/search/developer`** + MCP: **`firecrawl_developer_search(query, k?, skills?)`** + CLI: **`firecrawl developer [--limit ]`** + Ranked results over the whole index. Each carries `id` (`issue:owner/repo#123`), `url`, and the **matched passages in markdown**, so tables and code blocks survive. The artifact kind is the `id` prefix: `doc:`, `issue:`, `pull_request:`, or `readme:`. + The default first move for a developer question. It is the only surface that returns the passages, which is what lets you answer instead of pointing at a page. + `k` / `--limit` is 1–100 and defaults to 10. `skills="only"` (HTTP/MCP only) restricts the search to agent-skill files. + Keyless; send `Authorization: Bearer $FIRECRAWL_API_KEY` for higher rate limits. -HTTP: `GET|POST https://api.firecrawl.dev/v2/search/developer`. MCP: `firecrawl_developer_search`. Each hit carries `id`, `url`, `passages`. Kind is the `id` prefix (`doc:`, `issue:`, `pull_request:`, `readme:`). Hits do not carry a `type` field. +- MCP: **`firecrawl_search(query, categories: ["developer"])`** + CLI: **`firecrawl search --categories developer`** + Developer hits in a `developer` group beside `web`, each with `url`, `title`, `description` (the matched passage), `position`, and `category: "developer"` — web results carry no `category`, so that is the field to key on when merging. + Use this when you are **already** running a web search and want developer sources weighed in the same call. It exposes none of the filters and no passage control. -**Done when:** the answer quotes a matched passage and cites its `url` (fall back to `url` when `title` is absent), or you have moved to the open web because the index had nothing to say. +- MCP: **`firecrawl_scrape(url)` / `firecrawl_search(query)`** + CLI: **`firecrawl scrape ` / `firecrawl search `** + General web fetch and search, for what no primary source states: a comparison between two libraries, an outage, a migration write-up, a project with no public repository or indexed docs. + Also the follow-through when a hit is the right page but you need all of it — `scrape` the result's `url`. -## Tips +## Filters, and what each one costs you -- Default first move is `firecrawl developer`. Use `search --categories developer` only when you are already running a web search and want developer hits in the same call (no passage control, no index filters). -- Literal error or stack trace: search the string plus the library name. On HTTP, `types=["issue","pull_request"]`. Strip paths, line numbers, and ids, then retry. -- API contract: `readme` and `doc` are authoritative. A merged PR supersedes an issue report. Never answer from an opening report alone. -- Scope last: search the whole index, then narrow with HTTP `types`, `repos`, or `sources`. If a scoped search is empty, read the echoed `indexed` flag before concluding the repo is missing. -- Repository filters (`language`, `topic`, `license`, `min_stars`, …) drop `doc` results unless you also pass `sources`. `types`, `repos`, `sources`, `passages`, and those repository filters are HTTP-only. -- Comparison, opinion, news, or an unindexed project: `firecrawl search`, then `firecrawl scrape`. +Only the HTTP surface takes these. On `GET`, pass `types=issue,pull_request` or repeat the parameter; on `POST`, pass arrays. All are optional. -## See also +- `types` — which of `doc`, `issue`, `pull_request`, `readme` to search. Defaults to all four. Narrowing here is the cheapest way to sharpen a query. +- `repos` (`owner/name`) scopes the repository half, meaning `issue`, `pull_request`, and `readme`; `sources` (documentation source ids, at most 20) scopes the documentation half, meaning `doc`. Passing both **unions** the halves rather than intersecting them. Both echo back in the response with `indexed: true|false` — that is how you tell "not in the index" from "found nothing". +- A filter that cannot match any requested `type` is a `400`, not an empty list: `repos` with no repository type in `types`, or `sources` without `doc`. +- `passages` (1–5, default 1) is the _maximum_ passages per result, not a guarantee. Raise it when one page is clearly the right page but the first passage is the wrong part of it. +- `language`, `topic`, `license`, `min_stars`, `max_stars`, `archived`, `fork` describe a **repository**. Most documentation pages in the index have no repository behind them, so no repository fact can admit or exclude one. Send any of these without a `sources` scope and the response holds repository evidence only — `issue`, `pull_request`, `readme`. That is the design, not an index fault: do not retry it and do not report the index broken. To keep documentation, drop the repository filters, or scope the documentation half with `sources` and read the `sources` echo to confirm the id is indexed. -- [firecrawl-search](../firecrawl-search/SKILL.md) — open web, or `search --categories developer` in the same call -- [firecrawl-scrape](../firecrawl-scrape/SKILL.md) — full page when a hit is right but you need all of it -- [firecrawl-research-index](../firecrawl-research-index/SKILL.md) — papers, not this index +## Match the approach to the question + +- **Literal error message or stack-trace string** → search the string itself plus the library name, with `types=["issue","pull_request"]`. Whoever hit it filed it. If nothing matches, strip the volatile parts (paths, line numbers, ids, addresses) and retry — the invariant middle of the message is what is indexed. +- **Conceptual "how do I do X"** → the full question in natural language, all four types. The answer is usually a `doc` or a `readme`; raise `passages` before raising `k`. +- **Known bug** → the issue reports it, the merged pull request _fixes_ it, and the fix is what you want. Search `types=["issue","pull_request"]`, then re-query the issue's own terms scoped to its repo with `types=["pull_request"]`. A merged PR's passages tell you what changed and in which direction. +- **API contract** ("what does X return", "is Y required", "what is the default") → `readme` and `doc` are authoritative and a blog post is not. Use `types=["readme","doc"]`. If the contract looks like it moved, follow up with `pull_request` for the change that moved it. +- **Version-specific behaviour** → an issue's opening report describes the broken version; its resolution supersedes it. Raise `passages` to see further into the thread, and read the resolution and the linked pull request before answering. Never answer from an opening report alone. +- **Scoped to one library** → `repos=["owner/name"]` when you know the slug, plus `sources` if you want its docs in the same call. If a scoped search comes back empty, read the echoed `indexed` flag first: `false` means nothing from that repo or source can ever match and no rephrasing will help — drop the scope and search the whole index, or go to the web. +- **Ecosystem-wide** ("which libraries do X", "who else hit this") → no scope. Use `language` / `topic` / `min_stars` to keep to maintained repositories, accepting that this gives up all `doc` results. +- **Agent skills and tooling conventions** → `skills="only"` (HTTP/MCP only). +- **Comparison, opinion, news, or an unindexed project** → the open web. `firecrawl_search`, then `firecrawl_scrape` whatever deserves a full read. Combining is often right: take the contract from the index and the trade-off from the web. + +## Principles + +- **Quote the passage, cite the `url`.** The passages are the evidence; hand them over rather than paraphrasing them into a claim the reader can't check. `title` is frequently absent on `doc` results — fall back to `url`. +- **A merge supersedes a report.** When an issue and a pull request disagree, the merged pull request is the current behaviour. Say which one you read. +- **Scope last, not first.** Search the whole index, then narrow with `types`, `repos`, or `sources` once you know what the hits look like. Scoping first hides the result that would have told you where to look. +- **Go to the web when the index has nothing to say.** Trade-offs, ecosystem opinion, and anything about an unindexed project are web questions. Don't force them through the index, and don't dress a general web page up as a primary source. diff --git a/skills/firecrawl-research-index/SKILL.md b/skills/firecrawl-research-index/SKILL.md index 77dc4aa881..eede538ec6 100644 --- a/skills/firecrawl-research-index/SKILL.md +++ b/skills/firecrawl-research-index/SKILL.md @@ -1,43 +1,69 @@ --- name: firecrawl-research-index -description: | - Find papers in Firecrawl's research paper index (PubMed, bioRxiv, medRxiv, arXiv). Use for literature-finding of any kind, including clinical and biomedical questions; `search --categories research` is a website filter, not this index. -allowed-tools: - - Bash(firecrawl *) - - Bash(npx firecrawl-cli *) +description: Find the papers that answer a research query in Firecrawl's research paper index — a corpus of paper abstracts whose largest share is biomedical and life-science literature (PubMed, bioRxiv, medRxiv), alongside arXiv preprints in CS, physics, and math — using semantic search, semantic and structural expansion, and in-body verification. Use this skill for literature-finding and paper-retrieval tasks of any kind, including clinical, biomedical, drug, gene, disease, and other life-science questions, whether the answer is a single paper or a full multi-paper set. The index is reached only through the `firecrawl_research_*` MCP tools or the `firecrawl research` CLI subcommands. Calling `firecrawl_search` with its `categories` option set to `["research"]` is a different feature — it filters ordinary web search to research-affiliated websites (the list includes PubMed, bioRxiv, medRxiv, arXiv, and publisher sites) and returns page results from them, without querying the paper records in this index. --- -# firecrawl research +# Firecrawl Research Index -Find the papers that answer a research query. When in doubt, return the relevant set (most relevant first) rather than one hit. +Find the research papers that answer a research query. Some questions have a single answer; many have several — and when in doubt, lean toward returning the fuller relevant set (most relevant first) rather than narrowing to one. A reader is better served seeing the neighboring methods and papers than having them silently dropped. -## Quick start +## What is in the index -```bash -mkdir -p .firecrawl -firecrawl research search-papers "CRISPR base editing off-target effects" \ - --limit 20 -o .firecrawl/papers.json --json -jq -r '.results[] | .primaryId, .title' .firecrawl/papers.json -``` +Paper abstracts, with full text reachable per paper. The largest share of the corpus is **biomedical and life-science** literature — **PubMed** journal articles plus **bioRxiv** and **medRxiv** preprints — so clinical, drug, gene, disease, epidemiology, and public-health questions are in scope. **arXiv** preprints cover computer science, physics, and mathematics. Coverage outside those sources is thinner: a paper that exists only behind a publisher paywall or in a niche venue may not be indexed, and the general web tools below are the fallback when it isn't. -Run `firecrawl research --help` for flags. MCP arguments use `paperId`, not `id`. +There is **no fixed recipe**. Read the query, decide what kind it is, and choose the approach below. Some queries need a single search; others need heavy structural/semantic expansion. Don't run machinery a query doesn't call for. -A successful `search-papers` response is `{success, results}`. Each hit carries `paperId`, `primaryId` (`pmid:`, `pmcid:`, `doi:`, or `arxiv:`), `ids`, `title`, `abstract`, and `score`. +## The tools, and what each is uniquely good at -**Done when:** the answer is a cited paper set (or the one named paper), each kept or dropped against a verified constraint, with `search-papers` as the first move unless the query already named an id. +- MCP: **`firecrawl_research_search_papers(query, k?)`** + CLI: **`firecrawl research search-papers [--k ]`** + Semantic (HyDE) search over **abstracts**. The natural first move for almost any query. + If results look thin or all-alike, re-run with a different framing (sibling domain, rival method, dataset/benchmark name) rather than giving up. -## Tips +- MCP: **`firecrawl_research_related_papers(seed_ids, intent, mode?, k?)`** + CLI: **`firecrawl research related-papers --intent [--mode ] [--k ]`** + Semantic and structural expansion, ranked to your `intent`. + This reaches papers semantic search _cannot_, and it's how you turn one good hit into the rest of a set. + `mode=similar` → niche siblings; `citers` → who uses/builds on the seeds; `references` → what they build on / compare against. -- `search-papers` is the first move. If results look thin or all-alike, re-run with a different framing (sibling domain, rival method, dataset/benchmark name). -- `related-papers` needs `--intent`. `mode=similar` for siblings, `citers` for who builds on the seeds, `references` for what they build on. -- `inspect-paper` is metadata for one id. `read-paper` is in-body passages for one constraint (sample size, method, affiliation). Use it to rule a paper out, not to gatekeep. -- `search --categories research` is a website filter. It returns pages from academic domains, not paper records in this index. -- Named paper ("the Qwen3 report") → one `search-papers`. Method / family / "papers that do X" → expand with `related-papers` and keep neighbors. -- Superlative / leaderboard questions live on the web: `firecrawl search` / `firecrawl scrape`, then `search-papers` each top entry. -- PubMed, bioRxiv, and medRxiv are the largest part of the corpus. Do not send a biomedical query to the open web on the assumption the index is arXiv-only. +- MCP: **`firecrawl_research_inspect_paper(id)`** + CLI: **`firecrawl research inspect-paper `** + Canonical metadata for **one** paper: title, abstract, authors, categories, source ids, and dates. + Use it after `search_papers` or `related_papers` when you need the complete citation/metadata for a candidate, or when you have an id from elsewhere and need to confirm what paper it resolves to. + This does **not** read the paper body; use `read_paper` for specific full-text questions. -## See also +- MCP: **`firecrawl_research_read_paper(id, question)`** + CLI: **`firecrawl research read-paper --question `** + In-body passages of **one** paper, to verify a load-bearing constraint (a method actually used, a score actually reported, an affiliation, what a paper compares to). + Use it to settle a specific doubt, not on everything. -- [firecrawl-search](../firecrawl-search/SKILL.md) — web pages, including `search --categories research` -- [firecrawl-scrape](../firecrawl-scrape/SKILL.md) — leaderboards and other non-paper pages -- [firecrawl-developer-index](../firecrawl-developer-index/SKILL.md) — issues, PRs, READMEs, and docs +- MCP: **`firecrawl_search(query, categories: ["research"])`** + CLI: **`firecrawl search --categories research`** + **Not this index.** This is a _website_ filter: it restricts a normal web search to a short list of research-affiliated domains — the list does include `pubmed.ncbi.nlm.nih.gov`, `biorxiv.org`, `medrxiv.org`, and `arxiv.org` alongside publisher sites — and returns page results in a `research` group beside `web`, each with `url`, `title`, `description` (the matched passage), `position`, and `category: "research"` — web results carry no `category`, so that is the field to key on when merging. + So it reaches those sites' **web pages**; what it does not do is query their **paper records** in this index — no semantic search over abstracts, no citation-graph or related-paper expansion, no canonical paper metadata, and no in-body passages. The results are ordinary web results. + Use it when you are **already** running a web search and want those sites weighed in the same call. For anything that is actually a paper-finding task, use `firecrawl_research_search_papers` and its siblings above. + +- MCP: **`firecrawl_search(query)` / `firecrawl_scrape(url)`** + CLI: **`firecrawl search ` / `firecrawl scrape `** + General **web** search and page fetch, for facts that don't live in paper abstracts: benchmark **leaderboards**, rankings, "who scores best / is largest / is most used." + Find the ranking on the web, then map the top entries back to papers with `search_papers`. + Reach for these only when the corpus can't answer the question on its own. + +## Match the approach to the query + +- **Single _named_ paper** ("the Qwen3 report") → one `search_papers`, done. This is the only case that truly wants exactly one paper. +- **Paper by description / by method or technique** ("the paper that introduced X", "training-free N-gram detection of AI text") → find the best match, then assume there's a _family_: expand with `related_papers` and **include the closely-related methods/papers too**. Even when one paper is the exact literal match, surface and keep its neighbors — don't narrow to the single best hit and reason the rest out. Only treat it as one-answer if the query names a specific paper. +- **Enumeration / method-family** ("papers that do X", "alternatives to Adam", "benchmarks for Y") → the answer is a _set_, and this is where `related_papers` earns its keep: expand several strong anchors with `mode=similar`, re-seed from new strong hits. One search is never enough here. +- **Exhibiting** ("papers that _use_ / exhibit property P") → the relevant papers apply P but their abstracts may not describe it. Go from P's defining paper outward via `citers`/`references`, and use `read_paper` to confirm a candidate actually uses P. +- **Superlative / leaderboard** ("best on benchmark X", "largest", "most popular") → the ranking lives on **leaderboards / the web**, not in any single abstract. Use `firecrawl_search` / `firecrawl_scrape` to find the benchmark's leaderboard or rankings, read off the top models/papers, then `search_papers` each to get its paper. As a fallback, search the benchmark and `read_paper` candidates for reported numbers. The hardest kind — cast wide. +- **Org / author filtered** ("from \", "by \") → topical match isn't enough; verify the affiliation/authorship (metadata or `read_paper`) before keeping a paper. +- **Compare-against** ("what does paper X benchmark against / build on") → the answer is _inside_ paper X: `read_paper(X, ...)` or `related_papers([X], ..., mode="references")`. + +## Principles + +- **Two different features share the word "research."** The paper index is `firecrawl_research_*` / `firecrawl research`. The `categories: ["research"]` option on `firecrawl_search` is a website filter — it does point web search at PubMed, bioRxiv, medRxiv, arXiv, and publisher sites, but what comes back is their web pages, not paper records. If a task is about finding papers, the tools in this skill are the ones that read the corpus; reaching for `categories: ["research"]` will quietly answer a different question. +- **Query shape and subject field are separate.** A clinical-trial question and a machine-learning question take the same shapes above; what differs is only which source the hits come from. Don't send a biomedical or life-science query to the open web on the assumption the corpus is arXiv-only — PubMed, bioRxiv, and medRxiv are the largest part of what `search_papers` reads. +- **When in doubt, include.** For any topic / method / comparison question, return the relevant _family_, not just the single best match — err toward keeping a plausibly-relevant paper rather than dropping it. The neighboring methods are part of a good answer; don't reason close work out just because one paper is the most exact match. +- **Follow the literature, and keep what you find.** The seminal source, the competing methods, the close neighbors are usually a hop away — use `related_papers`, and _include_ them, not just the first hit. Stopping at one good result is the most common way to leave the reader with half an answer. +- **Verify to exclude, not to gatekeep.** Use `read_paper` to rule a paper _out_ when a hard constraint clearly fails (wrong org/author, doesn't actually report the score). When a paper is plausibly relevant, lean toward keeping it rather than demanding proof. +- **Only drop the clearly off-topic.** Don't pad with papers you're confident are unrelated — but that's a high bar; most plausibly-relevant work should make the cut. diff --git a/src/__tests__/commands/init.test.ts b/src/__tests__/commands/init.test.ts index a777d17395..11e3b6249a 100644 --- a/src/__tests__/commands/init.test.ts +++ b/src/__tests__/commands/init.test.ts @@ -1,6 +1,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { execSync } from 'child_process'; import { handleInitCommand } from '../../commands/init'; +import { CLI_SKILLS, WORKFLOW_SKILLS } from '../../commands/skills-install'; + +const cliSkillFlags = `--skill ${CLI_SKILLS.join(' ')}`; +const workflowSkillFlags = `--skill ${WORKFLOW_SKILLS.join(' ')}`; const { installMcpMock, getApiKeyMock, confirmMock, checkboxMock } = vi.hoisted( () => ({ @@ -42,7 +46,7 @@ describe('handleInitCommand', () => { vi.restoreAllMocks(); }); - it('installs skills from all repos globally across all detected agents in non-interactive mode', async () => { + it('installs CLI and workflow skills from the catalog globally across all detected agents in non-interactive mode', async () => { await handleInitCommand({ yes: true, skipInstall: true, @@ -50,20 +54,21 @@ describe('handleInitCommand', () => { }); expect(execSync).toHaveBeenCalledWith( - 'npx -y skills add firecrawl/cli --full-depth --global --all --yes', + `npx -y skills add firecrawl/skills --full-depth --global --all --yes ${cliSkillFlags}`, expect.objectContaining({ stdio: ['ignore', 'pipe', 'pipe'] }) ); expect(execSync).toHaveBeenCalledWith( - 'npx -y skills add firecrawl/skills --full-depth --global --all --yes', + `npx -y skills add firecrawl/skills --full-depth --global --all --yes ${workflowSkillFlags}`, expect.objectContaining({ stdio: ['ignore', 'pipe', 'pipe'] }) ); - expect(execSync).toHaveBeenCalledWith( - 'npx -y skills add firecrawl/firecrawl-workflows --full-depth --global --all --yes', - expect.objectContaining({ stdio: ['ignore', 'pipe', 'pipe'] }) + // Build skills are intentionally no longer installed by init. + expect(execSync).not.toHaveBeenCalledWith( + expect.stringContaining('firecrawl-build'), + expect.anything() ); }); - it('scopes non-interactive skills install to one agent across all repos when provided', async () => { + it('scopes non-interactive skills install to one agent when provided', async () => { await handleInitCommand({ yes: true, skipInstall: true, @@ -72,15 +77,11 @@ describe('handleInitCommand', () => { }); expect(execSync).toHaveBeenCalledWith( - 'npx -y skills add firecrawl/cli --full-depth --global --yes --agent cursor', - expect.objectContaining({ stdio: ['ignore', 'pipe', 'pipe'] }) - ); - expect(execSync).toHaveBeenCalledWith( - 'npx -y skills add firecrawl/skills --full-depth --global --yes --agent cursor', + `npx -y skills add firecrawl/skills --full-depth --global --yes --agent cursor ${cliSkillFlags}`, expect.objectContaining({ stdio: ['ignore', 'pipe', 'pipe'] }) ); expect(execSync).toHaveBeenCalledWith( - 'npx -y skills add firecrawl/firecrawl-workflows --full-depth --global --yes --agent cursor', + `npx -y skills add firecrawl/skills --full-depth --global --yes --agent cursor ${workflowSkillFlags}`, expect.objectContaining({ stdio: ['ignore', 'pipe', 'pipe'] }) ); }); diff --git a/src/__tests__/commands/setup.test.ts b/src/__tests__/commands/setup.test.ts index 5dad063bc3..feb79073c8 100644 --- a/src/__tests__/commands/setup.test.ts +++ b/src/__tests__/commands/setup.test.ts @@ -19,7 +19,14 @@ import { installOpenClawMcp, installSkillsForAgent, } from '../../commands/setup'; -import { ALL_SKILL_REPOS } from '../../commands/skills-install'; +import { + ALL_SKILL_REPOS, + CLI_SKILLS, + WORKFLOW_SKILLS, +} from '../../commands/skills-install'; + +const cliSkillFlags = `--skill ${CLI_SKILLS.join(' ')}`; +const workflowSkillFlags = `--skill ${WORKFLOW_SKILLS.join(' ')}`; import { configureWebDefaults } from '../../utils/web-defaults'; import { getApiKey } from '../../utils/config'; @@ -56,37 +63,29 @@ describe('handleSetupCommand', () => { vi.restoreAllMocks(); }); - it('installs core and build skills globally across all detected agents by default', async () => { + it('installs the CLI skills from the catalog globally across all detected agents by default', async () => { await handleSetupCommand('skills', {}); expect(execSync).toHaveBeenCalledWith( - 'npx -y skills add firecrawl/cli --full-depth --global --all', - expect.objectContaining({ stdio: 'inherit' }) - ); - expect(execSync).toHaveBeenCalledWith( - 'npx -y skills add firecrawl/skills --full-depth --global --all', + `npx -y skills add firecrawl/skills --full-depth --global --all ${cliSkillFlags}`, expect.objectContaining({ stdio: 'inherit' }) ); }); - it('installs core and build skills globally for a specific agent without using --all', async () => { + it('installs the CLI skills globally for a specific agent without using --all', async () => { await handleSetupCommand('skills', { agent: 'cursor' }); expect(execSync).toHaveBeenCalledWith( - 'npx -y skills add firecrawl/cli --full-depth --global --agent cursor', - expect.objectContaining({ stdio: 'inherit' }) - ); - expect(execSync).toHaveBeenCalledWith( - 'npx -y skills add firecrawl/skills --full-depth --global --agent cursor', + `npx -y skills add firecrawl/skills --full-depth --global --agent cursor ${cliSkillFlags}`, expect.objectContaining({ stdio: 'inherit' }) ); }); - it('installs workflow skills as a separate setup option', async () => { + it('installs workflow skills from the catalog as a separate setup option', async () => { await handleSetupCommand('workflows', {}); expect(execSync).toHaveBeenCalledWith( - 'npx -y skills add firecrawl/firecrawl-workflows --full-depth --global --all', + `npx -y skills add firecrawl/skills --full-depth --global --all ${workflowSkillFlags}`, expect.objectContaining({ stdio: 'inherit' }) ); }); @@ -127,11 +126,7 @@ describe('handleSetupCommand', () => { await handleSetupCommand(undefined, { yes: true }); expect(execSync).toHaveBeenCalledWith( - 'npx -y skills add firecrawl/cli --full-depth --global --all --yes', - expect.objectContaining({ stdio: 'inherit' }) - ); - expect(execSync).toHaveBeenCalledWith( - 'npx -y skills add firecrawl/skills --full-depth --global --all --yes', + `npx -y skills add firecrawl/skills --full-depth --global --all --yes ${cliSkillFlags}`, expect.objectContaining({ stdio: 'inherit' }) ); expect(execFileSync).toHaveBeenCalledWith( @@ -866,7 +861,7 @@ describe('handleSetupCommand', () => { const installCalls = allCalls.filter(([cmd]) => cmd.includes('skills add') ); - expect(installCalls.length).toBe(2); + expect(installCalls.length).toBe(1); for (const [, opts] of installCalls) { expect(opts.env).toBeDefined(); expect(opts.env!.npm_command).toBeUndefined(); diff --git a/src/commands/init.ts b/src/commands/init.ts index 144bcfd848..4cf219df2a 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -9,11 +9,12 @@ import { isAuthenticated, browserLogin, interactiveLogin } from '../utils/auth'; import { saveCredentials } from '../utils/credentials'; import { updateConfig, getApiKey } from '../utils/config'; import { - ALL_SKILL_REPOS, buildSkillsInstallArgs, cleanNpmEnv, - SKILL_REPOS, - WORKFLOW_SKILL_REPOS, + CLI_SKILL_SELECTION, + WORKFLOW_SKILL_SELECTION, + WORKFLOW_SKILLS, + type SkillSelection, } from './skills-install'; import { hasNpx, @@ -110,16 +111,24 @@ export const TEMPLATES: TemplateEntry[] = [ }, ]; -/** Human-friendly labels for skill repos, shown during install. */ -const SKILL_REPO_LABELS: Record = { - 'firecrawl/cli': 'core firecrawl skills', - 'firecrawl/skills': 'skills to build with firecrawl', - 'firecrawl/firecrawl-workflows': 'firecrawl workflow skills', +/** + * Init selections extend the shared catalog selections with a retry hint. + * `setup skills`/`setup workflows` install the same selections, so the hint + * reinstalls exactly what init attempted. + */ +interface InitSkillSelection extends SkillSelection { + retryCommand: string; +} + +const INIT_CLI_SELECTION: InitSkillSelection = { + ...CLI_SKILL_SELECTION, + retryCommand: 'firecrawl setup skills', }; -function skillRepoLabel(repo: string): string { - return SKILL_REPO_LABELS[repo] ?? repo; -} +const INIT_WORKFLOW_SELECTION: InitSkillSelection = { + ...WORKFLOW_SKILL_SELECTION, + retryCommand: 'firecrawl setup workflows', +}; /** * Install one skill repo quietly. Captures `npx skills add` output instead of @@ -131,10 +140,10 @@ function skillRepoLabel(repo: string): string { * the final line. */ async function installSkillRepoQuiet( - repo: string, + selection: SkillSelection, options: InitOptions ): Promise { - const label = skillRepoLabel(repo); + const { repo, label } = selection; // Prefer the native installer when we can detect installed harnesses. // It creates symlinks to ~/.agents/skills/ (single source of truth) and @@ -154,6 +163,7 @@ async function installSkillRepoQuiet( const result = await installSkillsNative(repo, { agent: options.agent, quiet: true, + skills: selection.skills, }); const suffix = ` ${dim}(${result.skillCount})${reset}`; const linked = @@ -181,6 +191,7 @@ async function installSkillRepoQuiet( yes: options.yes || options.all || true, global: true, includeNpxYes: true, + skills: selection.skills, }); const isTty = process.stdout.isTTY; @@ -259,17 +270,17 @@ async function pickHarnesses(): Promise { * agent), so a multi-agent install does not inflate the total. */ async function installRepoAcrossAgents( - repo: string, + selection: SkillSelection, options: InitOptions, agents: string[] | null ): Promise { if (!agents) { - return installSkillRepoQuiet(repo, options); + return installSkillRepoQuiet(selection, options); } let count: number | null = null; for (const agent of agents) { - const c = await installSkillRepoQuiet(repo, { ...options, agent }); + const c = await installSkillRepoQuiet(selection, { ...options, agent }); if (count == null) count = c; } return count; @@ -522,12 +533,12 @@ async function stepIntegrations(options: InitOptions): Promise { message: 'Which integrations?', choices: [ { - name: 'Skills — install core/build Firecrawl skills for AI coding agents', + name: 'Skills — install core Firecrawl CLI skills for AI coding agents', value: 'skills', checked: true, }, { - name: 'Workflows — install Firecrawl workflow skills', + name: 'Workflows — pick Firecrawl workflow skills to install', value: 'workflows', checked: true, }, @@ -564,37 +575,46 @@ async function stepIntegrations(options: InitOptions): Promise { switch (integration) { case 'skills': { console.log(`\n Installing skills...`); - for (const repo of SKILL_REPOS) { - try { - const count = await installRepoAcrossAgents( - repo, - options, - targetAgents - ); - if (count != null) totalSkills = (totalSkills ?? 0) + count; - } catch { - console.error( - ` ${dim}Run "firecrawl setup skills" later to retry.${reset}` - ); - } + try { + const count = await installRepoAcrossAgents( + CLI_SKILL_SELECTION, + options, + targetAgents + ); + if (count != null) totalSkills = (totalSkills ?? 0) + count; + } catch { + console.error( + ` ${dim}Run "firecrawl setup skills" later to retry.${reset}` + ); } break; } case 'workflows': { + const { checkbox: pickWorkflows } = await import('@inquirer/prompts'); + const chosen = await pickWorkflows({ + message: 'Which workflow skills?', + choices: WORKFLOW_SKILLS.map((name) => ({ + name, + value: name, + checked: true, + })), + }); + if (chosen.length === 0) { + console.log(` ${dim}No workflow skills selected.${reset}`); + break; + } console.log(`\n Installing workflow skills...`); - for (const repo of WORKFLOW_SKILL_REPOS) { - try { - const count = await installRepoAcrossAgents( - repo, - options, - targetAgents - ); - if (count != null) totalSkills = (totalSkills ?? 0) + count; - } catch { - console.error( - ` ${dim}Run "firecrawl setup workflows" later to retry.${reset}` - ); - } + try { + const count = await installRepoAcrossAgents( + { ...WORKFLOW_SKILL_SELECTION, skills: chosen }, + options, + targetAgents + ); + if (count != null) totalSkills = (totalSkills ?? 0) + count; + } catch { + console.error( + ` ${dim}Run "firecrawl setup workflows" later to retry (installs all workflow skills).${reset}` + ); } break; } @@ -1031,17 +1051,13 @@ async function runNonInteractive(options: InitOptions): Promise { console.log( `${stepLabel()} Installing firecrawl skills for AI coding agents...` ); - for (const repo of ALL_SKILL_REPOS) { + for (const selection of [INIT_CLI_SELECTION, INIT_WORKFLOW_SELECTION]) { try { - const count = await installSkillRepoQuiet(repo, options); + const count = await installSkillRepoQuiet(selection, options); if (count != null) skillCount = (skillCount ?? 0) + count; } catch { - const retryCommand = - repo === 'firecrawl/firecrawl-workflows' - ? 'firecrawl setup workflows' - : 'firecrawl setup skills'; console.error( - `\n${dim}Failed to install skills from ${repo}. Retry with: ${retryCommand}${reset}` + `\n${dim}Failed to install ${selection.label}. Retry with: ${selection.retryCommand}${reset}` ); process.exit(1); } diff --git a/src/commands/setup.ts b/src/commands/setup.ts index 1ed3a4a0fd..727ea139cf 100644 --- a/src/commands/setup.ts +++ b/src/commands/setup.ts @@ -19,8 +19,10 @@ import { getApiKey } from '../utils/config'; import { buildSkillsInstallArgs, cleanNpmEnv, + CLI_SKILL_SELECTION, SKILL_REPOS, - WORKFLOW_SKILL_REPOS, + WORKFLOW_SKILL_SELECTION, + type SkillSelection, } from './skills-install'; import { hasNpx, installSkillsNative } from './skills-native'; import { @@ -284,10 +286,10 @@ export async function handleSetupCommand( switch (subcommand) { case 'skills': - await installSkills(options, SKILL_REPOS); + await installSkills(options, [CLI_SKILL_SELECTION]); break; case 'workflows': - await installSkills(options, WORKFLOW_SKILL_REPOS); + await installSkills(options, [WORKFLOW_SKILL_SELECTION]); break; case 'mcp': await installMcp(options); @@ -463,18 +465,20 @@ export async function handleMakeDefaultCommand( async function installSkills( options: SetupOptions, - repos: readonly string[] + selections: readonly SkillSelection[] ): Promise { - for (const repo of repos) { + for (const selection of selections) { + const { repo } = selection; if (options.nativeSkills) { try { const result = await installSkillsNative(repo, { agent: options.agent, quiet: options.quiet, + skills: selection.skills, }); if (options.quiet) { console.log( - ` ${green}✓${reset} ${skillRepoLabel(repo)} ${dim}(${result.skillCount})${reset}` + ` ${green}✓${reset} ${selection.label} ${dim}(${result.skillCount})${reset}` ); } } catch (error) { @@ -494,6 +498,7 @@ async function installSkills( global: true, yes: options.yes, includeNpxYes: true, + skills: selection.skills, }); const cmd = args.join(' '); @@ -509,7 +514,7 @@ async function installSkills( // Fallback: native install (no npx/Node required) try { - await installSkillsNative(repo); + await installSkillsNative(repo, { skills: selection.skills }); } catch (error) { console.error( `Failed to install skills from ${repo}:`, @@ -525,9 +530,11 @@ export async function installSkillsForAgent( options: SetupOptions = {}, repos: readonly string[] = SKILL_REPOS ): Promise { + // Legacy whole-repo path (used by `firecrawl launch`): install each repo + // in full, labeled by repo name. await installSkills( { ...options, agent, global: options.global ?? true }, - repos + repos.map((repo) => ({ repo, label: skillRepoLabel(repo) })) ); } diff --git a/src/commands/skills-install.ts b/src/commands/skills-install.ts index 609fd58807..5f7fc12f19 100644 --- a/src/commands/skills-install.ts +++ b/src/commands/skills-install.ts @@ -21,6 +21,79 @@ export const ALL_SKILL_REPOS = [ ...WORKFLOW_SKILL_REPOS, ] as const; +/** + * The skills catalog — the one promoted install source for every skill + * family. `firecrawl init` installs from here so install volume accrues to + * catalog counters on skills.sh. + */ +export const CATALOG_REPO = 'firecrawl/skills'; + +/** + * CLI skills, authored in firecrawl/cli and mirrored into the catalog under + * skills/cli/. Selection is name-based (`--skill`), so the catalog's + * directory layout doesn't matter. + */ +export const CLI_SKILLS = [ + 'firecrawl', + 'firecrawl-scrape', + 'firecrawl-search', + 'firecrawl-crawl', + 'firecrawl-map', + 'firecrawl-interact', + 'firecrawl-agent', + 'firecrawl-monitor', + 'firecrawl-parse', + 'firecrawl-download', + // Index skills: teach the `firecrawl research` / `firecrawl developer` + // CLI commands, so they ship with the CLI set. + 'firecrawl-research-index', + 'firecrawl-developer-index', +] as const; + +/** Workflow skills, authored in the catalog under skills/workflows/. */ +export const WORKFLOW_SKILLS = [ + 'firecrawl-workflows', + 'firecrawl-company-directories', + 'firecrawl-competitive-intel', + 'firecrawl-dashboard-reporting', + 'firecrawl-deep-research', + 'firecrawl-demo-walkthrough', + 'firecrawl-knowledge-base', + 'firecrawl-knowledge-ingest', + 'firecrawl-lead-gen', + 'firecrawl-lead-research', + 'firecrawl-market-research', + 'firecrawl-qa', + 'firecrawl-research-papers', + 'firecrawl-seo-audit', + 'firecrawl-shop', + 'firecrawl-website-design-clone', +] as const; + +/** + * A named subset of catalog skills to install. Shared by `init` and + * `setup skills`/`setup workflows` so a retry hint always reinstalls exactly + * the same set. Selection is name-based and independent of catalog layout. + */ +export interface SkillSelection { + repo: string; + /** Install only these skills (by name); omit for the whole repo. */ + skills?: readonly string[]; + label: string; +} + +export const CLI_SKILL_SELECTION: SkillSelection = { + repo: CATALOG_REPO, + skills: CLI_SKILLS, + label: 'core firecrawl skills', +}; + +export const WORKFLOW_SKILL_SELECTION: SkillSelection = { + repo: CATALOG_REPO, + skills: WORKFLOW_SKILLS, + label: 'firecrawl workflow skills', +}; + export interface SkillsInstallCommandOptions { agent?: string; all?: boolean; @@ -29,6 +102,8 @@ export interface SkillsInstallCommandOptions { includeNpxYes?: boolean; /** Repo to install from (defaults to firecrawl/cli) */ repo?: string; + /** Install only these skills (by name) instead of the whole repo. */ + skills?: readonly string[]; } export function buildSkillsInstallArgs( @@ -59,6 +134,11 @@ export function buildSkillsInstallArgs( args.push('--agent', options.agent); } + // `skills add` collects space-separated names after a single --skill flag. + if (options.skills) { + args.push('--skill', ...options.skills); + } + return args; } diff --git a/src/commands/skills-native.ts b/src/commands/skills-native.ts index c899d17e02..a728090f91 100644 --- a/src/commands/skills-native.ts +++ b/src/commands/skills-native.ts @@ -29,6 +29,8 @@ export interface NativeSkillsInstallOptions { agent?: string; /** Suppress per-repo status lines; caller will render its own summary. */ quiet?: boolean; + /** Install only these skills (by name) instead of the whole repo. */ + skills?: readonly string[]; } export interface NativeSkillsInstallResult { @@ -411,11 +413,22 @@ export async function installSkillsNative( throw new Error(`No ${SKILLS_SUBDIR}/ directory found in repository`); } - const skills = discoverSkills(skillsDir); + let skills = discoverSkills(skillsDir); if (skills.length === 0) { throw new Error('No skills found in repository'); } + if (options.skills) { + const wanted = new Set(options.skills); + skills = skills.filter((skill) => wanted.has(skill.name)); + const missing = options.skills.filter( + (name) => !skills.some((skill) => skill.name === name) + ); + if (missing.length > 0) { + throw new Error(`Skills not found in ${repo}: ${missing.join(', ')}`); + } + } + if (!options.quiet) { console.log(` ${dim}Found ${skills.length} skills${reset}`); }