diff --git a/README.en.md b/README.en.md index 7b1fa5d..8040931 100644 --- a/README.en.md +++ b/README.en.md @@ -148,6 +148,15 @@ ContextWeaver uses Tree-sitter to provide native AST parsing support for the fol | C++ | Yes | Yes | `.cpp`, `.hpp`, `.cc`, `.cxx` | | C# | Yes | Yes | `.cs` | +For text-based files that do not have Tree-sitter grammars wired in yet, ContextWeaver falls back to line-based plain-text chunks so important configs, UI descriptions, and scripts remain retrievable: + +| Category | File Extensions | +| ----------------- | ----------------------------------------------- | +| Android | `.kt`, `.kts`, `.xml`, `.gradle`, `.properties` | +| Config and data | `.yaml`, `.yml`, `.toml`, `.jsonc` | +| Web and documents | `.html`, `.htm`, `.css`, `.scss`, `.sass`, `.less`, `.mdx` | +| Scripts and specs | `.sh`, `.bash`, `.zsh`, `.sql`, `.proto`, `.graphql`, `.gql` | + ## Acknowledgements - [Linux DO](https://linux.do/) - An amazing technical community inspired this project diff --git a/README.md b/README.md index 1ce21fc..35a9893 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,15 @@ ContextWeaver 通过 Tree-sitter 原生支持以下编程语言的 AST 解析： | C++ | ✅ | ✅ | `.cpp`, `.hpp`, `.cc`, `.cxx` | | C# | ✅ | ✅ | `.cs` | +对于暂未接入 Tree-sitter 语法的文本型文件，ContextWeaver 会使用纯文本行分片兜底索引，保证代码库中的关键配置、UI 描述与脚本仍可被检索： + +| 类型 | 文件扩展名 | +| ----------- | ----------------------------------------------- | +| Android | `.kt`, `.kts`, `.xml`, `.gradle`, `.properties` | +| 配置与数据 | `.yaml`, `.yml`, `.toml`, `.jsonc` | +| Web 与文档 | `.html`, `.htm`, `.css`, `.scss`, `.sass`, `.less`, `.mdx` | +| 脚本与协议 | `.sh`, `.bash`, `.zsh`, `.sql`, `.proto`, `.graphql`, `.gql` | + ## 致谢 - [Linux DO](https://linux.do/) - 本项目的大量灵感来自这个非常哇塞的技术社区~ diff --git a/src/scanner/language.ts b/src/scanner/language.ts index 370f7aa..80adfab 100644 --- a/src/scanner/language.ts +++ b/src/scanner/language.ts @@ -1,7 +1,7 @@ /** - * 文件扩展名到语言标识的映射 + * 支持 AST 解析的文件扩展名到语言标识的映射 */ -const LANGUAGE_MAP: Record = { +const AST_LANGUAGE_MAP: Record = { '.ts': 'typescript', '.tsx': 'typescript', '.js': 'javascript', @@ -18,8 +18,48 @@ const LANGUAGE_MAP: Record = { '.cc': 'cpp', '.cxx': 'cpp', '.hpp': 'cpp', + '.cs': 'c_sharp', +}; + +/** + * 支持纯文本兜底索引的文件扩展名到语言标识的映射 + */ +const PLAIN_TEXT_LANGUAGE_MAP: Record = { '.md': 'markdown', + '.markdown': 'markdown', + '.mdx': 'markdown', '.json': 'json', + '.jsonc': 'json', + '.yaml': 'yaml', + '.yml': 'yaml', + '.toml': 'toml', + '.xml': 'xml', + '.svg': 'xml', + '.html': 'html', + '.htm': 'html', + '.css': 'css', + '.scss': 'scss', + '.sass': 'sass', + '.less': 'less', + '.kt': 'kotlin', + '.kts': 'kotlin', + '.gradle': 'gradle', + '.properties': 'properties', + '.proto': 'protobuf', + '.sql': 'sql', + '.graphql': 'graphql', + '.gql': 'graphql', + '.sh': 'shell', + '.bash': 'shell', + '.zsh': 'shell', +}; + +/** + * 文件扩展名到语言标识的映射 + */ +const LANGUAGE_MAP: Record = { + ...AST_LANGUAGE_MAP, + ...PLAIN_TEXT_LANGUAGE_MAP, }; /** @@ -27,6 +67,14 @@ const LANGUAGE_MAP: Record = { */ const ALLOWED_EXTENSIONS = new Set(Object.keys(LANGUAGE_MAP)); +/** + * 支持纯文本兜底索引的语言集合 + */ +const PLAIN_TEXT_FALLBACK_LANGUAGES = new Set([ + ...Object.values(AST_LANGUAGE_MAP), + ...Object.values(PLAIN_TEXT_LANGUAGE_MAP), +]); + /** * 根据文件路径获取语言标识 * @param filePath 文件路径 @@ -47,6 +95,15 @@ export function isAllowedExtension(filePath: string): boolean { return ALLOWED_EXTENSIONS.has(ext); } +/** + * 判断语言是否可以使用纯文本兜底分片 + * @param language 语言标识 + * @returns 是否支持纯文本兜底分片 + */ +export function canUsePlainTextFallback(language: string): boolean { + return PLAIN_TEXT_FALLBACK_LANGUAGES.has(language); +} + /** * 获取文件扩展名（包含点） * @param filePath 文件路径 diff --git a/src/scanner/processor.ts b/src/scanner/processor.ts index 1431049..08de285 100644 --- a/src/scanner/processor.ts +++ b/src/scanner/processor.ts @@ -10,19 +10,13 @@ import { } from '../chunking/index.js'; import { readFileWithEncoding } from '../utils/encoding.js'; import { sha256 } from './hash.js'; -import { getLanguage } from './language.js'; +import { canUsePlainTextFallback, getLanguage } from './language.js'; /** * 大文件阈值（字节） */ const MAX_FILE_SIZE = 100 * 1024; // 500KB -/** - * 需要兜底分片支持的目标语言集合 - * 这些语言的文件即使 AST 解析失败也会使用行分片保证可检索 - */ -const FALLBACK_LANGS = new Set(['python', 'go', 'rust', 'java', 'markdown', 'json']); - /** * 检查 JSON 文件是否应该跳过索引 * @@ -255,8 +249,8 @@ async function processFile( } } - // 兜底分片：对 FALLBACK_LANGS 语言，如果 AST 分片失败或返回空，使用行分片 - if (chunks.length === 0 && FALLBACK_LANGS.has(language)) { + // 兜底分片：允许文本型语言在没有 AST 分片结果时仍可检索 + if (chunks.length === 0 && canUsePlainTextFallback(language)) { chunks = splitter.splitPlainText(content, relPath, language); } diff --git a/tests/scanner/filter.test.ts b/tests/scanner/filter.test.ts index f07375d..ca31d9c 100644 --- a/tests/scanner/filter.test.ts +++ b/tests/scanner/filter.test.ts @@ -123,8 +123,13 @@ describe('scanner filter', () => { it('rejects file types that are not in the retrievable chunk allowlist', async () => { expect(isAllowedFile('src/app.ts')).toBe(true); - expect(isAllowedFile('scripts/dev.sh')).toBe(false); - expect(isAllowedFile('config/site.yaml')).toBe(false); + expect(isAllowedFile('src/App.cs')).toBe(true); + expect(isAllowedFile('app/src/main/java/MainActivity.kt')).toBe(true); + expect(isAllowedFile('app/src/main/res/layout/activity_main.xml')).toBe(true); + expect(isAllowedFile('app/build.gradle')).toBe(true); + expect(isAllowedFile('config/site.yaml')).toBe(true); + expect(isAllowedFile('scripts/dev.sh')).toBe(true); + expect(isAllowedFile('notes/plain.txt')).toBe(false); }); it('does not allow includePatterns to re-include gitignored files', async () => { diff --git a/tests/scanner/processor.test.ts b/tests/scanner/processor.test.ts new file mode 100644 index 0000000..ceac8c8 --- /dev/null +++ b/tests/scanner/processor.test.ts @@ -0,0 +1,64 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { processFiles } from '../../src/scanner/processor.js'; + +const tempDirs: string[] = []; + +async function createTempDir(prefix: string): Promise { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); + tempDirs.push(dir); + return dir; +} + +async function writeRepoFiles(repoRoot: string, files: Record): Promise { + await Promise.all( + Object.entries(files).map(async ([relativePath, content]) => { + const fullPath = path.join(repoRoot, relativePath); + await fs.mkdir(path.dirname(fullPath), { recursive: true }); + await fs.writeFile(fullPath, content, 'utf-8'); + }), + ); +} + +afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true }))); +}); + +describe('scanner processor', () => { + it('uses plain-text chunks for indexable languages without AST parsers', async () => { + const repoRoot = await createTempDir('cw-processor-'); + const files = { + 'app/src/main/java/com/example/MainActivity.kt': + 'class MainActivity {\n fun render() = Unit\n}\n', + 'app/src/main/res/layout/activity_main.xml': + '\n \n\n', + 'app/build.gradle': 'plugins { id "com.android.application" }\n', + 'config/site.yaml': 'site:\n title: ContextWeaver\n', + }; + + await writeRepoFiles(repoRoot, files); + + const results = await processFiles( + repoRoot, + Object.keys(files).map((relativePath) => path.join(repoRoot, relativePath)), + new Map(), + ); + + expect(results).toHaveLength(4); + for (const result of results) { + expect(result.status).toBe('added'); + expect(result.skipReason).toBeUndefined(); + expect(result.chunks.length).toBeGreaterThan(0); + expect(result.chunks[0]?.metadata.contextPath).toEqual([result.relPath]); + } + + expect(Object.fromEntries(results.map((result) => [result.relPath, result.language]))).toEqual({ + 'app/build.gradle': 'gradle', + 'app/src/main/java/com/example/MainActivity.kt': 'kotlin', + 'app/src/main/res/layout/activity_main.xml': 'xml', + 'config/site.yaml': 'yaml', + }); + }); +});