feat: 扩展文本型文件索引兜底支持

README.en.md

@@ -148,6 +148,15 @@ ContextWeaver uses Tree-sitter to provide native AST parsing support for the fol
 | C++        | Yes         | Yes               | `.cpp`, `.hpp`, `.cc`, `.cxx` |
 | C#         | Yes         | Yes               | `.cs`                         |
 
+For text-based files that do not have Tree-sitter grammars wired in yet, ContextWeaver falls back to line-based plain-text chunks so important configs, UI descriptions, and scripts remain retrievable:
+
+| Category          | File Extensions                                 |
+| ----------------- | ----------------------------------------------- |
+| Android           | `.kt`, `.kts`, `.xml`, `.gradle`, `.properties` |
+| Config and data   | `.yaml`, `.yml`, `.toml`, `.jsonc`              |
+| Web and documents | `.html`, `.htm`, `.css`, `.scss`, `.sass`, `.less`, `.mdx` |
+| Scripts and specs | `.sh`, `.bash`, `.zsh`, `.sql`, `.proto`, `.graphql`, `.gql` |
+
 ## Acknowledgements
 
 - [Linux DO](https://linux.do/) - An amazing technical community inspired this project

README.md

@@ -158,6 +158,15 @@ ContextWeaver 通过 Tree-sitter 原生支持以下编程语言的 AST 解析：
 | C++        | ✅       | ✅          | `.cpp`, `.hpp`, `.cc`, `.cxx` |
 | C#         | ✅       | ✅          | `.cs`                         |
 
+对于暂未接入 Tree-sitter 语法的文本型文件，ContextWeaver 会使用纯文本行分片兜底索引，保证代码库中的关键配置、UI 描述与脚本仍可被检索：
+
+| 类型        | 文件扩展名                                      |
+| ----------- | ----------------------------------------------- |
+| Android     | `.kt`, `.kts`, `.xml`, `.gradle`, `.properties` |
+| 配置与数据  | `.yaml`, `.yml`, `.toml`, `.jsonc`              |
+| Web 与文档  | `.html`, `.htm`, `.css`, `.scss`, `.sass`, `.less`, `.mdx` |
+| 脚本与协议  | `.sh`, `.bash`, `.zsh`, `.sql`, `.proto`, `.graphql`, `.gql` |
+
 ## 致谢
 
 - [Linux DO](https://linux.do/) - 本项目的大量灵感来自这个非常哇塞的技术社区~

src/scanner/language.ts

@@ -1,7 +1,7 @@
 /**
- * 文件扩展名到语言标识的映射
+ * 支持 AST 解析的文件扩展名到语言标识的映射
  */
-const LANGUAGE_MAP: Record<string, string> = {
+const AST_LANGUAGE_MAP: Record<string, string> = {
   '.ts': 'typescript',
   '.tsx': 'typescript',
   '.js': 'javascript',
@@ -18,15 +18,63 @@ const LANGUAGE_MAP: Record<string, string> = {
   '.cc': 'cpp',
   '.cxx': 'cpp',
   '.hpp': 'cpp',
+  '.cs': 'c_sharp',
+};
+
+/**
+ * 支持纯文本兜底索引的文件扩展名到语言标识的映射
+ */
+const PLAIN_TEXT_LANGUAGE_MAP: Record<string, string> = {
   '.md': 'markdown',
+  '.markdown': 'markdown',
+  '.mdx': 'markdown',
   '.json': 'json',
+  '.jsonc': 'json',
+  '.yaml': 'yaml',
+  '.yml': 'yaml',
+  '.toml': 'toml',
+  '.xml': 'xml',
+  '.svg': 'xml',
+  '.html': 'html',
+  '.htm': 'html',
+  '.css': 'css',
+  '.scss': 'scss',
+  '.sass': 'sass',
+  '.less': 'less',
+  '.kt': 'kotlin',
+  '.kts': 'kotlin',
+  '.gradle': 'gradle',
+  '.properties': 'properties',
+  '.proto': 'protobuf',
+  '.sql': 'sql',
+  '.graphql': 'graphql',
+  '.gql': 'graphql',
+  '.sh': 'shell',
+  '.bash': 'shell',
+  '.zsh': 'shell',
+};
+
+/**
+ * 文件扩展名到语言标识的映射
+ */
+const LANGUAGE_MAP: Record<string, string> = {
+  ...AST_LANGUAGE_MAP,
+  ...PLAIN_TEXT_LANGUAGE_MAP,
 };
 
 /**
  * 允许的文件扩展名白名单
  */
 const ALLOWED_EXTENSIONS = new Set(Object.keys(LANGUAGE_MAP));
 
+/**
+ * 支持纯文本兜底索引的语言集合
+ */
+const PLAIN_TEXT_FALLBACK_LANGUAGES = new Set([
+  ...Object.values(AST_LANGUAGE_MAP),
+  ...Object.values(PLAIN_TEXT_LANGUAGE_MAP),
+]);
+
 /**
  * 根据文件路径获取语言标识
  * @param filePath 文件路径
@@ -47,6 +95,15 @@ export function isAllowedExtension(filePath: string): boolean {
   return ALLOWED_EXTENSIONS.has(ext);
 }
 
+/**
+ * 判断语言是否可以使用纯文本兜底分片
+ * @param language 语言标识
+ * @returns 是否支持纯文本兜底分片
+ */
+export function canUsePlainTextFallback(language: string): boolean {
+  return PLAIN_TEXT_FALLBACK_LANGUAGES.has(language);
+}
+
 /**
  * 获取文件扩展名（包含点）
  * @param filePath 文件路径

src/scanner/processor.ts

@@ -10,19 +10,13 @@ import {
 } from '../chunking/index.js';
 import { readFileWithEncoding } from '../utils/encoding.js';
 import { sha256 } from './hash.js';
-import { getLanguage } from './language.js';
+import { canUsePlainTextFallback, getLanguage } from './language.js';
 
 /**
  * 大文件阈值（字节）
  */
 const MAX_FILE_SIZE = 100 * 1024; // 500KB
 
-/**
- * 需要兜底分片支持的目标语言集合
- * 这些语言的文件即使 AST 解析失败也会使用行分片保证可检索
- */
-const FALLBACK_LANGS = new Set(['python', 'go', 'rust', 'java', 'markdown', 'json']);
-
 /**
  * 检查 JSON 文件是否应该跳过索引
  *
@@ -255,8 +249,8 @@ async function processFile(
       }
     }
 
-    // 兜底分片：对 FALLBACK_LANGS 语言，如果 AST 分片失败或返回空，使用行分片
-    if (chunks.length === 0 && FALLBACK_LANGS.has(language)) {
+    // 兜底分片：允许文本型语言在没有 AST 分片结果时仍可检索
+    if (chunks.length === 0 && canUsePlainTextFallback(language)) {
       chunks = splitter.splitPlainText(content, relPath, language);
     }
 

tests/scanner/filter.test.ts

@@ -123,8 +123,13 @@ describe('scanner filter', () => {
 
   it('rejects file types that are not in the retrievable chunk allowlist', async () => {
     expect(isAllowedFile('src/app.ts')).toBe(true);
-    expect(isAllowedFile('scripts/dev.sh')).toBe(false);
-    expect(isAllowedFile('config/site.yaml')).toBe(false);
+    expect(isAllowedFile('src/App.cs')).toBe(true);
+    expect(isAllowedFile('app/src/main/java/MainActivity.kt')).toBe(true);
+    expect(isAllowedFile('app/src/main/res/layout/activity_main.xml')).toBe(true);
+    expect(isAllowedFile('app/build.gradle')).toBe(true);
+    expect(isAllowedFile('config/site.yaml')).toBe(true);
+    expect(isAllowedFile('scripts/dev.sh')).toBe(true);
+    expect(isAllowedFile('notes/plain.txt')).toBe(false);
   });
 
   it('does not allow includePatterns to re-include gitignored files', async () => {

tests/scanner/processor.test.ts

@@ -0,0 +1,64 @@
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { afterEach, describe, expect, it } from 'vitest';
+import { processFiles } from '../../src/scanner/processor.js';
+
+const tempDirs: string[] = [];
+
+async function createTempDir(prefix: string): Promise<string> {
+  const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix));
+  tempDirs.push(dir);
+  return dir;
+}
+
+async function writeRepoFiles(repoRoot: string, files: Record<string, string>): Promise<void> {
+  await Promise.all(
+    Object.entries(files).map(async ([relativePath, content]) => {
+      const fullPath = path.join(repoRoot, relativePath);
+      await fs.mkdir(path.dirname(fullPath), { recursive: true });
+      await fs.writeFile(fullPath, content, 'utf-8');
+    }),
+  );
+}
+
+afterEach(async () => {
+  await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })));
+});
+
+describe('scanner processor', () => {
+  it('uses plain-text chunks for indexable languages without AST parsers', async () => {
+    const repoRoot = await createTempDir('cw-processor-');
+    const files = {
+      'app/src/main/java/com/example/MainActivity.kt':
+        'class MainActivity {\n  fun render() = Unit\n}\n',
+      'app/src/main/res/layout/activity_main.xml':
+        '<LinearLayout>\n  <TextView android:id="@+id/title" />\n</LinearLayout>\n',
+      'app/build.gradle': 'plugins { id "com.android.application" }\n',
+      'config/site.yaml': 'site:\n  title: ContextWeaver\n',
+    };
+
+    await writeRepoFiles(repoRoot, files);
+
+    const results = await processFiles(
+      repoRoot,
+      Object.keys(files).map((relativePath) => path.join(repoRoot, relativePath)),
+      new Map(),
+    );
+
+    expect(results).toHaveLength(4);
+    for (const result of results) {
+      expect(result.status).toBe('added');
+      expect(result.skipReason).toBeUndefined();
+      expect(result.chunks.length).toBeGreaterThan(0);
+      expect(result.chunks[0]?.metadata.contextPath).toEqual([result.relPath]);
+    }
+
+    expect(Object.fromEntries(results.map((result) => [result.relPath, result.language]))).toEqual({
+      'app/build.gradle': 'gradle',
+      'app/src/main/java/com/example/MainActivity.kt': 'kotlin',
+      'app/src/main/res/layout/activity_main.xml': 'xml',
+      'config/site.yaml': 'yaml',
+    });
+  });
+});