--- -title: Fumadocs 文档站接入 -description: 使用 Next.js App Router 和 MDX 维护现代文档站。 ---- - -# Fumadocs 文档站接入 - -## 适用场景 - -Fumadocs 适合需要 MDX、组件化内容和现代前端体验的文档站。 - -## 接入 Modex - -在 `docs.yaml` 中声明 `type: fumadocs`，由 docsctl 执行构建命令并复制静态输出目录。 -
Authorize %s
This app wants to access Modex with these scopes:
%s`) +} + +func oauthRedirectError(w http.ResponseWriter, r *http.Request, redirectURI, state, code string) { + if redirectURI == "" { + writeError(w, http.StatusBadRequest, code, code) + return + } + u, err := url.Parse(redirectURI) + if err != nil { + writeError(w, http.StatusBadRequest, code, code) + return + } + q := u.Query() + q.Set("error", code) + if state != "" { + q.Set("state", state) + } + u.RawQuery = q.Encode() + http.Redirect(w, r, u.String(), http.StatusFound) +} + +func writeOAuthTokenResponse(w http.ResponseWriter, access, refresh string, scopes []string) { + writeJSON(w, http.StatusOK, map[string]any{ + "access_token": access, + "token_type": "Bearer", + "expires_in": int(oauthAccessTTL.Seconds()), + "refresh_token": refresh, + "scope": strings.Join(scopes, " "), + }) +} + +func writeOAuthTokenError(w http.ResponseWriter, status int, code, desc string) { + writeJSON(w, status, map[string]any{"error": code, "error_description": desc}) +} diff --git a/backend/internal/api/oauth_test.go b/backend/internal/api/oauth_test.go new file mode 100644 index 0000000..b655feb --- /dev/null +++ b/backend/internal/api/oauth_test.go @@ -0,0 +1,187 @@ +package api + +import ( + "encoding/base64" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "modex/backend/internal/store" +) + +func TestConnectedAppOAuthAuthorizationCodeFlow(t *testing.T) { + t.Setenv("SUPER_ADMIN_USERS", "dev") + t.Setenv("APP_BASE_URL", "http://modex.test") + + srv := New(store.NewSeededTestStore()) + handler := srv.Handler() + + // OIDC is the only login path, so mint a session cookie directly instead of + // driving a full provider round-trip. SUPER_ADMIN_USERS promotes "dev". + user := srv.app.Store().UpsertUser(store.User{ + ID: "u-dev", Username: "dev", DisplayName: "Dev User", + Email: "dev@example.com", Status: "active", + }) + login := httptest.NewRecorder() + if err := srv.app.Auth().CreateSession(login, user); err != nil { + t.Fatalf("create session: %v", err) + } + cookies := login.Result().Cookies() + + createBody := `{"name":"External MCP Client","redirect_uris":["https://client.example.com/oauth/modex/callback"],"scopes":["modex:mcp:read","modex:docs:read"],"trusted":true}` + create := httptest.NewRecorder() + createReq := httptest.NewRequest(http.MethodPost, "/api/admin/connected-apps", strings.NewReader(createBody)) + createReq.Header.Set("Content-Type", "application/json") + for _, c := range cookies { + createReq.AddCookie(c) + } + handler.ServeHTTP(create, createReq) + if create.Code != http.StatusCreated { + t.Fatalf("create app status = %d, body=%s", create.Code, create.Body.String()) + } + var app connectedAppResponse + if err := json.Unmarshal(create.Body.Bytes(), &app); err != nil { + t.Fatal(err) + } + if app.ClientID == "" || app.ClientSecret == "" { + t.Fatalf("missing client credentials: %+v", app) + } + + authURL := "/oauth/authorize?response_type=code&client_id=" + url.QueryEscape(app.ClientID) + + "&redirect_uri=" + url.QueryEscape("https://client.example.com/oauth/modex/callback") + + "&scope=" + url.QueryEscape("modex:mcp:read modex:docs:read") + + "&state=s1" + auth := httptest.NewRecorder() + authReq := httptest.NewRequest(http.MethodGet, authURL, nil) + for _, c := range cookies { + authReq.AddCookie(c) + } + handler.ServeHTTP(auth, authReq) + if auth.Code != http.StatusFound { + t.Fatalf("authorize status = %d, body=%s", auth.Code, auth.Body.String()) + } + loc, err := url.Parse(auth.Header().Get("Location")) + if err != nil { + t.Fatal(err) + } + code := loc.Query().Get("code") + if code == "" || loc.Query().Get("state") != "s1" { + t.Fatalf("bad authorize redirect: %s", loc.String()) + } + + form := url.Values{} + form.Set("grant_type", "authorization_code") + form.Set("code", code) + form.Set("redirect_uri", "https://client.example.com/oauth/modex/callback") + token := httptest.NewRecorder() + tokenReq := httptest.NewRequest(http.MethodPost, "/oauth/token", strings.NewReader(form.Encode())) + tokenReq.Header.Set("Content-Type", "application/x-www-form-urlencoded") + tokenReq.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(url.QueryEscape(app.ClientID)+":"+url.QueryEscape(app.ClientSecret)))) + handler.ServeHTTP(token, tokenReq) + if token.Code != http.StatusOK { + t.Fatalf("token status = %d, body=%s", token.Code, token.Body.String()) + } + var tok struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + Scope string `json:"scope"` + } + if err := json.Unmarshal(token.Body.Bytes(), &tok); err != nil { + t.Fatal(err) + } + if tok.AccessToken == "" || tok.RefreshToken == "" || !strings.Contains(tok.Scope, "modex:mcp:read") { + t.Fatalf("bad token response: %+v", tok) + } + + me := httptest.NewRecorder() + meReq := httptest.NewRequest(http.MethodGet, "/api/auth/me", nil) + meReq.Header.Set("Authorization", "Bearer "+tok.AccessToken) + handler.ServeHTTP(me, meReq) + if me.Code != http.StatusOK || !strings.Contains(me.Body.String(), `"username":"dev"`) { + t.Fatalf("bearer me status = %d, body=%s", me.Code, me.Body.String()) + } + + info := httptest.NewRecorder() + infoReq := httptest.NewRequest(http.MethodGet, "/api/mcp/token-info", nil) + infoReq.Header.Set("Authorization", "Bearer "+tok.AccessToken) + handler.ServeHTTP(info, infoReq) + if info.Code != http.StatusOK || !strings.Contains(info.Body.String(), `"modex:mcp:read"`) { + t.Fatalf("token info status = %d, body=%s", info.Code, info.Body.String()) + } + + admin := httptest.NewRecorder() + adminReq := httptest.NewRequest(http.MethodGet, "/api/admin/users", nil) + adminReq.Header.Set("Authorization", "Bearer "+tok.AccessToken) + handler.ServeHTTP(admin, adminReq) + if admin.Code != http.StatusUnauthorized { + t.Fatalf("oauth bearer should not enter admin APIs, status = %d, body=%s", admin.Code, admin.Body.String()) + } +} + +func TestCodexOAuthPublicClientAuthorizationCodeFlow(t *testing.T) { + t.Setenv("SUPER_ADMIN_USERS", "dev") + t.Setenv("APP_BASE_URL", "http://modex.test") + + srv := New(store.NewSeededTestStore()) + handler := srv.Handler() + + user := srv.app.Store().UpsertUser(store.User{ + ID: "u-dev", Username: "dev", DisplayName: "Dev User", + Email: "dev@example.com", Status: "active", + }) + login := httptest.NewRecorder() + if err := srv.app.Auth().CreateSession(login, user); err != nil { + t.Fatalf("create session: %v", err) + } + cookies := login.Result().Cookies() + + redirectURI := "http://127.0.0.1:49152/callback/modex" + authURL := "/oauth/authorize?response_type=code&client_id=" + url.QueryEscape(store.CodexOAuthClientID) + + "&redirect_uri=" + url.QueryEscape(redirectURI) + + "&scope=" + url.QueryEscape("modex:mcp:read modex:docs:read") + + "&state=codex-state" + auth := httptest.NewRecorder() + authReq := httptest.NewRequest(http.MethodGet, authURL, nil) + for _, c := range cookies { + authReq.AddCookie(c) + } + handler.ServeHTTP(auth, authReq) + if auth.Code != http.StatusFound { + t.Fatalf("authorize status = %d, body=%s", auth.Code, auth.Body.String()) + } + loc, err := url.Parse(auth.Header().Get("Location")) + if err != nil { + t.Fatal(err) + } + code := loc.Query().Get("code") + if code == "" || loc.Query().Get("state") != "codex-state" { + t.Fatalf("bad authorize redirect: %s", loc.String()) + } + + form := url.Values{} + form.Set("grant_type", "authorization_code") + form.Set("client_id", store.CodexOAuthClientID) + form.Set("code", code) + form.Set("redirect_uri", redirectURI) + token := httptest.NewRecorder() + tokenReq := httptest.NewRequest(http.MethodPost, "/oauth/token", strings.NewReader(form.Encode())) + tokenReq.Header.Set("Content-Type", "application/x-www-form-urlencoded") + handler.ServeHTTP(token, tokenReq) + if token.Code != http.StatusOK { + t.Fatalf("token status = %d, body=%s", token.Code, token.Body.String()) + } + var tok struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + Scope string `json:"scope"` + } + if err := json.Unmarshal(token.Body.Bytes(), &tok); err != nil { + t.Fatal(err) + } + if tok.AccessToken == "" || tok.RefreshToken == "" || !strings.Contains(tok.Scope, "modex:mcp:read") { + t.Fatalf("bad token response: %+v", tok) + } +} diff --git a/backend/internal/api/pagination.go b/backend/internal/api/pagination.go new file mode 100644 index 0000000..0107761 --- /dev/null +++ b/backend/internal/api/pagination.go @@ -0,0 +1,73 @@ +package api + +import ( + "net/http" + "strconv" + "strings" +) + +// pageResult is the envelope returned by admin list endpoints when the client +// opts into server-side pagination via ?page= / ?limit=. Endpoints stay +// backward-compatible: without those params they return the plain array so +// existing callers (dropdowns, homepage, identity derivation) keep working. +type pageResult[T any] struct { + Items []T `json:"items"` + Total int `json:"total"` + Page int `json:"page"` + Limit int `json:"limit"` +} + +// wantsPage reports whether the request opted into pagination. +func wantsPage(r *http.Request) bool { + q := r.URL.Query() + return q.Has("page") || q.Has("limit") +} + +func pageParams(r *http.Request) (page, limit int) { + page = atoiOr(r.URL.Query().Get("page"), 1) + limit = atoiOr(r.URL.Query().Get("limit"), 20) + if page < 1 { + page = 1 + } + if limit < 1 { + limit = 20 + } + if limit > 200 { + limit = 200 + } + return page, limit +} + +func atoiOr(s string, def int) int { + if n, err := strconv.Atoi(strings.TrimSpace(s)); err == nil { + return n + } + return def +} + +// paginate slices items for the given 1-based page and returns the envelope. +func paginate[T any](items []T, page, limit int) pageResult[T] { + total := len(items) + start := (page - 1) * limit + if start > total { + start = total + } + end := start + limit + if end > total { + end = total + } + out := append([]T{}, items[start:end]...) + return pageResult[T]{Items: out, Total: total, Page: page, Limit: limit} +} + +// keywordOf returns the normalized (trimmed, lowercased) keyword query param. +func keywordOf(r *http.Request) string { + return strings.ToLower(strings.TrimSpace(r.URL.Query().Get("keyword"))) +} + +func containsFold(haystack, needleLower string) bool { + if needleLower == "" { + return true + } + return strings.Contains(strings.ToLower(haystack), needleLower) +} diff --git a/backend/internal/api/posthog.go b/backend/internal/api/posthog.go new file mode 100644 index 0000000..54fc882 --- /dev/null +++ b/backend/internal/api/posthog.go @@ -0,0 +1,175 @@ +package api + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "strings" + "time" + + "modex/backend/internal/store" +) + +// errPosthogNotConfigured signals that PostHog query credentials are absent. +// The API falls back to the built-in analytics store in this case. +var errPosthogNotConfigured = errors.New("posthog not configured") + +// posthogHost returns the configured PostHog API host or the default. +func posthogHost() string { + host := strings.TrimRight(os.Getenv("POSTHOG_HOST"), "/") + if host == "" { + return "https://app.posthog.com" + } + return host +} + +// posthogConfigured reports whether the server-side PostHog query credentials +// are present. This is distinct from the frontend NEXT_PUBLIC_POSTHOG_KEY +// (capture key); querying read stats needs a personal/project API key. +func posthogConfigured() bool { + return os.Getenv("POSTHOG_PERSONAL_API_KEY") != "" && os.Getenv("POSTHOG_PROJECT_ID") != "" +} + +// PosthogConfigured and PosthogHost are exported wrappers used by package main +// (startup logging). +func PosthogConfigured() bool { return posthogConfigured() } +func PosthogHost() string { return posthogHost() } + +// posthogDocStats queries PostHog (HogQL) for the daily read trend and per-user +// reading totals/duration of one document. It returns an error when configured but +// the query fails, so callers can surface the problem instead of silently +// falling back. The event/property names match what the frontend captures: +// a "docs_page_view" event carrying a "doc_id" property. +func posthogDocStats(docID string, days int) (store.PageReadStats, error) { + if !posthogConfigured() { + return store.PageReadStats{}, errPosthogNotConfigured + } + host := posthogHost() + projectID := os.Getenv("POSTHOG_PROJECT_ID") + apiKey := os.Getenv("POSTHOG_PERSONAL_API_KEY") + + query := func(hogql string) ([][]any, error) { + body, _ := json.Marshal(map[string]any{ + "query": map[string]any{"kind": "HogQLQuery", "query": hogql}, + }) + url := fmt.Sprintf("%s/api/projects/%s/query/", host, projectID) + req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+apiKey) + req.Header.Set("Content-Type", "application/json") + client := &http.Client{Timeout: 8 * time.Second} + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("posthog returned %d", resp.StatusCode) + } + var out struct { + Results [][]any `json:"results"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return nil, err + } + return out.Results, nil + } + + esc := strings.ReplaceAll(docID, "'", "\\'") + dailyHogQL := fmt.Sprintf( + "SELECT toDate(timestamp) AS d, count() AS c FROM events "+ + "WHERE event = 'docs_page_view' AND properties.doc_id = '%s' "+ + "AND timestamp >= now() - INTERVAL %d DAY GROUP BY d ORDER BY d", + esc, days) + readersHogQL := fmt.Sprintf( + "SELECT reader, user_id, count() AS c, avg(duration) AS avg_duration, max(last) AS last_read "+ + "FROM (SELECT coalesce(person.properties.name, person.properties.email, distinct_id) AS reader, "+ + "distinct_id AS user_id, properties.read_id AS read_id, "+ + "max(toFloat64OrZero(toString(properties.duration_seconds))) AS duration, max(timestamp) AS last "+ + "FROM events WHERE event = 'docs_page_read' AND properties.doc_id = '%s' "+ + "AND timestamp >= now() - INTERVAL %d DAY GROUP BY reader, user_id, read_id) "+ + "GROUP BY reader, user_id ORDER BY c DESC LIMIT 200", + esc, days) + + dailyRows, err1 := query(dailyHogQL) + readerRows, err2 := query(readersHogQL) + if err1 != nil { + return store.PageReadStats{}, fmt.Errorf("posthog daily query failed: %w", err1) + } + if err2 != nil { + return store.PageReadStats{}, fmt.Errorf("posthog readers query failed: %w", err2) + } + + // Build a zero-filled day window so the chart has no gaps. + today := time.Now().UTC().Truncate(24 * time.Hour) + idx := map[string]int{} + daily := make([]store.DailyReadPoint, days) + for i := 0; i < days; i++ { + key := today.AddDate(0, 0, -(days - 1 - i)).Format("2006-01-02") + daily[i] = store.DailyReadPoint{Date: key, Count: 0} + idx[key] = i + } + total := 0 + for _, row := range dailyRows { + if len(row) < 2 { + continue + } + date := fmt.Sprintf("%v", row[0]) + if len(date) > 10 { + date = date[:10] + } + c := toInt(row[1]) + if i, ok := idx[date]; ok { + daily[i].Count = c + } + total += c + } + + readers := make([]store.ReaderStat, 0, len(readerRows)) + totalDuration := 0 + totalTimedReads := 0 + for _, row := range readerRows { + if len(row) < 5 { + continue + } + name := fmt.Sprintf("%v", row[0]) + if name == "" || name == "
`)
+
+ out := string(rewriteServedSiteRootRefs(input, "internal-wiki", "latest", "guide"))
+
+ base := "/api/docs/internal-wiki/latest/guide/site/"
+ for _, want := range []string{
+ `href="` + base + `assets/style.css"`,
+ `src="` + base + `assets/app.js"`,
+ base + `assets/a.png 1x`,
+ base + `assets/b.png 2x`,
+ `url('` + base + `assets/bg.png')`,
+ `href="` + base + `posts/cbb-shelf/faq.html"`,
+ `"` + base + `posts/cbb-shelf/faq.html"`,
+ `'` + base + `posts/cbb-shelf/system.html'`,
+ } {
+ if !strings.Contains(out, want) {
+ t.Fatalf("rewritten output missing %q: %s", want, out)
+ }
+ }
+ if strings.Contains(out, "/internal-tools/") {
+ t.Fatalf("legacy base still present: %s", out)
+ }
+ if !strings.Contains(out, `"/standards/sidebar-state"`) {
+ t.Fatalf("unrelated JS string path was rewritten: %s", out)
+ }
+}
+
+func TestSiteFileCandidatesSupportVitePressRoutes(t *testing.T) {
+ cases := map[string][]string{
+ "": {"index.html"},
+ "posts/cbb-shelf/system-overview": {"posts/cbb-shelf/system-overview", "posts/cbb-shelf/system-overview.html", "posts/cbb-shelf/system-overview/index.html", "index.html"},
+ "posts/cbb-shelf/system-overview.md": {"posts/cbb-shelf/system-overview.md", "posts/cbb-shelf/system-overview.html"},
+ "assets/style.css": {"assets/style.css"},
+ "/posts/process-tools/itr/user-guide/": {"posts/process-tools/itr/user-guide", "posts/process-tools/itr/user-guide.html", "posts/process-tools/itr/user-guide/index.html", "index.html"},
+ }
+ for input, want := range cases {
+ got := siteFileCandidates(input)
+ if strings.Join(got, "|") != strings.Join(want, "|") {
+ t.Fatalf("siteFileCandidates(%q) = %#v, want %#v", input, got, want)
+ }
+ }
+}
+
+func TestDeployErrorIncludesStageReport(t *testing.T) {
+ st := store.NewSeededTestStore()
+ if _, err := st.UpdateModule("DemoModule", store.Module{DeployToken: "secret"}); err != nil {
+ t.Fatalf("UpdateModule: %v", err)
+ }
+ srv := New(st)
+ req := httptest.NewRequest(http.MethodPost, "/api/deploy", bytes.NewReader(testDeployZip(t)))
+ req.Header.Set("Content-Type", "application/zip")
+ req.Header.Set("X-Modex-Deploy-Token", "wrong")
+ rr := httptest.NewRecorder()
+
+ srv.Handler().ServeHTTP(rr, req)
+
+ if rr.Code != http.StatusForbidden {
+ t.Fatalf("status = %d, want 403: %s", rr.Code, rr.Body.String())
+ }
+ var payload struct {
+ Error struct {
+ Code string `json:"code"`
+ } `json:"error"`
+ Deploy struct {
+ Stages []struct {
+ Name string `json:"name"`
+ Status string `json:"status"`
+ } `json:"stages"`
+ } `json:"deploy"`
+ }
+ if err := json.Unmarshal(rr.Body.Bytes(), &payload); err != nil {
+ t.Fatalf("decode response: %v", err)
+ }
+ if payload.Error.Code != "invalid_deploy_token" {
+ t.Fatalf("error code = %q", payload.Error.Code)
+ }
+ if len(payload.Deploy.Stages) != 2 || payload.Deploy.Stages[0].Name != "parse_artifact" || payload.Deploy.Stages[0].Status != "ok" || payload.Deploy.Stages[1].Name != "authenticate" || payload.Deploy.Stages[1].Status != "failed" {
+ t.Fatalf("unexpected deploy stages: %+v", payload.Deploy.Stages)
+ }
+}
+
+func TestDeployTokenSelectsDocumentSource(t *testing.T) {
+ st := store.NewSeededTestStore()
+ if _, err := st.UpdateModule("DemoModule", store.Module{DeployToken: "secret"}); err != nil {
+ t.Fatalf("UpdateModule: %v", err)
+ }
+ srv := New(st)
+ req := httptest.NewRequest(http.MethodPost, "/api/deploy", bytes.NewReader(testDeployZipForModule(t, "WrongModule")))
+ req.Header.Set("Content-Type", "application/zip")
+ req.Header.Set("X-Modex-Deploy-Token", "secret")
+ rr := httptest.NewRecorder()
+
+ srv.Handler().ServeHTTP(rr, req)
+
+ if rr.Code != http.StatusAccepted {
+ t.Fatalf("status = %d, want 202: %s", rr.Code, rr.Body.String())
+ }
+ if _, err := st.Page("DemoModule:latest:guide"); err != nil {
+ t.Fatalf("expected page to be indexed under token-owned module: %v", err)
+ }
+ if _, err := st.Page("WrongModule:latest:guide"); err == nil {
+ t.Fatal("page was indexed under artifact module instead of token-owned module")
+ }
+}
+
+func TestCanonicalizeDeployArtifactAppliesSplitMount(t *testing.T) {
+ artifact := deployArtifactForSplitTest("WrongModule")
+
+ got := canonicalizeDeployArtifact(artifact, store.Module{
+ ModuleKey: "DemoModule",
+ Name: "Demo Module",
+ DocType: "markdown",
+ Mount: "split",
+ })
+
+ if len(got.Manifest.Entries) != 2 {
+ t.Fatalf("entries = %#v, want two top-level groups", got.Manifest.Entries)
+ }
+ if got.Manifest.Entries[0].Key != "standard" || got.Manifest.Entries[1].Key != "tools" {
+ t.Fatalf("entry keys = %#v, want standard/tools", got.Manifest.Entries)
+ }
+ if len(got.Documents) != 2 {
+ t.Fatalf("documents = %#v, want two grouped documents", got.Documents)
+ }
+ if got.Documents[0].DocID != "DemoModule:latest:standard" || got.Documents[1].DocID != "DemoModule:latest:tools" {
+ t.Fatalf("doc ids = %#v", got.Documents)
+ }
+ if !strings.Contains(got.Documents[0].ContentMD, "Standard A") || !strings.Contains(got.Documents[1].ContentMD, "Tools B") {
+ t.Fatalf("grouped markdown content missing source text: %#v", got.Documents)
+ }
+}
+
+func TestCanonicalizeDeployArtifactRewritesContentResourceBase(t *testing.T) {
+ artifact := deploy.Artifact{
+ Metadata: deploy.Metadata{ModuleKey: "UnknownModule", ModuleName: "UnknownModule", DocsVersion: "latest", PackageVersion: "1.0.0"},
+ Manifest: deploy.Manifest{Entries: []deploy.Entry{{Key: "guide", Title: "Guide", Type: "markdown", Source: "docs"}}},
+ Documents: []deploy.DocumentRecord{{
+ DocID: "UnknownModule:latest:guide",
+ ModuleKey: "UnknownModule",
+ DocsVersion: "latest",
+ EntryKey: "guide",
+ EntryType: "markdown",
+ Title: "Guide",
+ Content: "see /api/docs/UnknownModule/latest/guide/site/images/shot.png",
+ ContentMD: "",
+ }},
+ }
+
+ got := canonicalizeDeployArtifact(artifact, store.Module{ModuleKey: "standards", Name: "Standards"})
+
+ if strings.Contains(got.Documents[0].ContentMD, "UnknownModule") || strings.Contains(got.Documents[0].Content, "UnknownModule") {
+ t.Fatalf("placeholder module not rewritten: %#v", got.Documents[0])
+ }
+ if !strings.Contains(got.Documents[0].ContentMD, "/api/docs/standards/latest/guide/site/images/shot.png") {
+ t.Fatalf("content not rewritten to resolved module: %q", got.Documents[0].ContentMD)
+ }
+}
+
+func TestCanonicalizeDeployArtifactPreservesSitePageDocIDs(t *testing.T) {
+ artifact := deploy.Artifact{
+ Metadata: deploy.Metadata{ModuleKey: "UnknownModule", ModuleName: "UnknownModule", DocsVersion: "latest"},
+ Manifest: deploy.Manifest{Entries: []deploy.Entry{{Key: "guide", Title: "Guide", Type: "vitepress", Source: "."}}},
+ Documents: []deploy.DocumentRecord{
+ {DocID: "UnknownModule:latest:guide/", ModuleKey: "UnknownModule", DocsVersion: "latest", EntryKey: "guide", EntryType: "vitepress", Title: "Home", Content: "home"},
+ {DocID: "UnknownModule:latest:guide/standards/coding/memory-safety", ModuleKey: "UnknownModule", DocsVersion: "latest", EntryKey: "guide", EntryType: "vitepress", Title: "Memory Safety", Content: "memory safety"},
+ },
+ }
+
+ got := canonicalizeDeployArtifact(artifact, store.Module{ModuleKey: "standards", Name: "Standards"})
+
+ if len(got.Documents) != 2 {
+ t.Fatalf("documents = %#v, want two pages", got.Documents)
+ }
+ if got.Documents[0].DocID != "standards:latest:guide/" {
+ t.Fatalf("first doc id = %q", got.Documents[0].DocID)
+ }
+ if got.Documents[1].DocID != "standards:latest:guide/standards/coding/memory-safety" {
+ t.Fatalf("second doc id = %q", got.Documents[1].DocID)
+ }
+}
+
+func TestModelEndpointAppendsExpectedSuffixOnce(t *testing.T) {
+ tests := []struct {
+ name string
+ base string
+ suffix string
+ want string
+ }{
+ {name: "embedding base", base: "https://api.example.com/v1", suffix: "/embeddings", want: "https://api.example.com/v1/embeddings"},
+ {name: "embedding endpoint", base: "https://api.example.com/v1/embeddings", suffix: "/embeddings", want: "https://api.example.com/v1/embeddings"},
+ {name: "rerank base", base: "https://api.example.com/v1/", suffix: "/rerank", want: "https://api.example.com/v1/rerank"},
+ {name: "rerank endpoint", base: "https://api.example.com/v1/rerank", suffix: "/rerank", want: "https://api.example.com/v1/rerank"},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := modelEndpoint(tt.base, tt.suffix); got != tt.want {
+ t.Fatalf("modelEndpoint() = %q, want %q", got, tt.want)
+ }
+ })
+ }
+}
+
+func TestChatEndpointUsesSelectedProtocol(t *testing.T) {
+ tests := []struct {
+ name string
+ protocol string
+ base string
+ model string
+ want string
+ }{
+ {name: "openai chat", protocol: "openai-chat", base: "https://api.example.com/v1", model: "gpt", want: "https://api.example.com/v1/chat/completions"},
+ {name: "responses", protocol: "openai-responses", base: "https://api.example.com/v1/", model: "gpt", want: "https://api.example.com/v1/responses"},
+ {name: "anthropic", protocol: "anthropic", base: "https://api.anthropic.com", model: "claude", want: "https://api.anthropic.com/v1/messages"},
+ {name: "gemini", protocol: "gemini", base: "https://generativelanguage.googleapis.com", model: "gemini pro", want: "https://generativelanguage.googleapis.com/v1beta/models/gemini%20pro:generateContent"},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := chatEndpoint(tt.base, tt.protocol, tt.model); got != tt.want {
+ t.Fatalf("chatEndpoint() = %q, want %q", got, tt.want)
+ }
+ })
+ }
+}
+
+func testDeployZip(t *testing.T) []byte {
+ return testDeployZipForModule(t, "DemoModule")
+}
+
+func deployArtifactForSplitTest(moduleKey string) deploy.Artifact {
+ return deploy.Artifact{
+ Metadata: deploy.Metadata{ModuleKey: moduleKey, ModuleName: moduleKey, DocsVersion: "latest", PackageVersion: "1.0.0"},
+ Manifest: deploy.Manifest{Entries: []deploy.Entry{
+ {Key: "guide-standard-a", Title: "Standard A", Type: "markdown", Source: "docs/standard/a.md"},
+ {Key: "guide-tools-b", Title: "Tools B", Type: "markdown", Source: "docs/tools/b.md"},
+ }},
+ Documents: []deploy.DocumentRecord{
+ {DocID: moduleKey + ":latest:guide-standard-a", ModuleKey: moduleKey, ModuleName: moduleKey, DocsVersion: "latest", EntryKey: "guide-standard-a", EntryType: "markdown", Title: "Standard A", SourceFile: "docs/standard/a.md", Content: "Standard text", ContentMD: "# Standard A"},
+ {DocID: moduleKey + ":latest:guide-tools-b", ModuleKey: moduleKey, ModuleName: moduleKey, DocsVersion: "latest", EntryKey: "guide-tools-b", EntryType: "markdown", Title: "Tools B", SourceFile: "docs/tools/b.md", Content: "Tools text", ContentMD: "# Tools B"},
+ },
+ }
+}
+
+func testDeployZipForModule(t *testing.T, moduleKey string) []byte {
+ t.Helper()
+ var buf bytes.Buffer
+ zw := zip.NewWriter(&buf)
+ files := map[string]string{
+ "metadata.json": `{"module_key":"` + moduleKey + `","module_name":"` + moduleKey + `","docs_version":"latest","package_version":"1.2.3"}`,
+ "manifest.json": `{"schema_version":"modex.docs/v1","generated_by":"test","entries":[{"key":"guide","title":"Guide","type":"markdown","source":"README.md"}]}`,
+ "nav.json": `[{"title":"Guide","path":"/guide"}]`,
+ "documents.jsonl": `{"doc_id":"` + moduleKey + `:latest:guide","module_key":"` + moduleKey + `","module_name":"` + moduleKey + `","docs_version":"latest","entry_key":"guide","entry_type":"markdown","title":"Guide","content":"Hello","path":"/docs/` + moduleKey + `/latest/guide"}` + "\n",
+ "llms.txt": "Guide\n",
+ }
+ for name, content := range files {
+ w, err := zw.Create(name)
+ if err != nil {
+ t.Fatalf("create zip entry: %v", err)
+ }
+ if _, err := w.Write([]byte(content)); err != nil {
+ t.Fatalf("write zip entry: %v", err)
+ }
+ }
+ if err := zw.Close(); err != nil {
+ t.Fatalf("close zip: %v", err)
+ }
+ return buf.Bytes()
+}
diff --git a/backend/internal/api/server_security_test.go b/backend/internal/api/server_security_test.go
new file mode 100644
index 0000000..7632aaf
--- /dev/null
+++ b/backend/internal/api/server_security_test.go
@@ -0,0 +1,168 @@
+package api
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ "modex/backend/internal/store"
+)
+
+func TestAdminReleaseRollbackRequiresLogin(t *testing.T) {
+ srv := New(store.NewSeededTestStore())
+ req := httptest.NewRequest(http.MethodPost, "/api/admin/releases/rel-demo-latest-001/rollback", nil)
+ rr := httptest.NewRecorder()
+
+ srv.Handler().ServeHTTP(rr, req)
+
+ if rr.Code != http.StatusUnauthorized {
+ t.Fatalf("status = %d, want %d", rr.Code, http.StatusUnauthorized)
+ }
+}
+
+func TestOptionalCurrentUserProbeIsAnonymousWithout401(t *testing.T) {
+ srv := New(store.NewTestStore())
+ req := httptest.NewRequest(http.MethodGet, "/api/auth/me?optional=1", nil)
+ rr := httptest.NewRecorder()
+
+ srv.Handler().ServeHTTP(rr, req)
+
+ if rr.Code != http.StatusOK {
+ t.Fatalf("status = %d, want %d: %s", rr.Code, http.StatusOK, rr.Body.String())
+ }
+ if strings.TrimSpace(rr.Body.String()) != "null" {
+ t.Fatalf("body = %q, want null", rr.Body.String())
+ }
+}
+
+func TestEmptyPublicModulesUsesJSONArray(t *testing.T) {
+ srv := New(store.NewTestStore())
+ req := httptest.NewRequest(http.MethodGet, "/api/modules", nil)
+ rr := httptest.NewRecorder()
+
+ srv.Handler().ServeHTTP(rr, req)
+
+ if rr.Code != http.StatusOK {
+ t.Fatalf("status = %d, want %d: %s", rr.Code, http.StatusOK, rr.Body.String())
+ }
+ if strings.TrimSpace(rr.Body.String()) != "[]" {
+ t.Fatalf("body = %q, want []", rr.Body.String())
+ }
+}
+
+func TestEmbedTextRequiresAdminSession(t *testing.T) {
+ srv := New(store.NewSeededTestStore())
+ req := httptest.NewRequest(http.MethodPost, "/api/embeddings/embed-text", strings.NewReader(`{"text":"hello"}`))
+ req.Header.Set("Content-Type", "application/json")
+ rr := httptest.NewRecorder()
+
+ srv.Handler().ServeHTTP(rr, req)
+
+ if rr.Code != http.StatusUnauthorized {
+ t.Fatalf("status = %d, want %d", rr.Code, http.StatusUnauthorized)
+ }
+}
+
+func TestAdminPluginsRequiresLogin(t *testing.T) {
+ srv := New(store.NewSeededTestStore())
+ req := httptest.NewRequest(http.MethodGet, "/api/admin/plugins", nil)
+ rr := httptest.NewRecorder()
+
+ srv.Handler().ServeHTTP(rr, req)
+
+ if rr.Code != http.StatusUnauthorized {
+ t.Fatalf("status = %d, want %d", rr.Code, http.StatusUnauthorized)
+ }
+}
+
+func TestMCPLogRequiresAuthenticatedCaller(t *testing.T) {
+ srv := New(store.NewSeededTestStore())
+ req := httptest.NewRequest(http.MethodPost, "/api/mcp/log", strings.NewReader(`{"tool_name":"search_docs"}`))
+ req.Header.Set("Content-Type", "application/json")
+ rr := httptest.NewRecorder()
+
+ srv.Handler().ServeHTTP(rr, req)
+
+ if rr.Code != http.StatusUnauthorized {
+ t.Fatalf("status = %d, want %d", rr.Code, http.StatusUnauthorized)
+ }
+}
+
+func TestMCPLogAcceptsPersonalMCPToken(t *testing.T) {
+ st := store.NewSeededTestStore()
+ current := st.CurrentUser()
+ if _, err := st.SetUserMCPToken(current.ID, "mcp-test-token"); err != nil {
+ t.Fatalf("SetUserMCPToken: %v", err)
+ }
+ srv := New(st)
+ req := httptest.NewRequest(http.MethodPost, "/api/mcp/log", strings.NewReader(`{"tool_name":"search_docs","query":"guide","result_count":1}`))
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", "Bearer mcp-test-token")
+ rr := httptest.NewRecorder()
+
+ srv.Handler().ServeHTTP(rr, req)
+
+ if rr.Code != http.StatusAccepted {
+ t.Fatalf("status = %d, want %d: %s", rr.Code, http.StatusAccepted, rr.Body.String())
+ }
+ logs := st.MCPLogs()
+ if len(logs) != 1 || logs[0].UserID != current.ID {
+ t.Fatalf("logs = %+v, want one log attributed to %s", logs, current.ID)
+ }
+}
+
+func TestMCPTokenInfoAcceptsPersonalMCPToken(t *testing.T) {
+ st := store.NewSeededTestStore()
+ current := st.CurrentUser()
+ if _, err := st.SetUserMCPToken(current.ID, "mcp-test-token"); err != nil {
+ t.Fatalf("SetUserMCPToken: %v", err)
+ }
+ req := httptest.NewRequest(http.MethodGet, "/api/mcp/token-info", nil)
+ req.Header.Set("Authorization", "Bearer mcp-test-token")
+ rr := httptest.NewRecorder()
+
+ New(st).Handler().ServeHTTP(rr, req)
+
+ if rr.Code != http.StatusOK {
+ t.Fatalf("status = %d: %s", rr.Code, rr.Body.String())
+ }
+ var info struct {
+ UserID string `json:"user_id"`
+ Scopes []string `json:"scopes"`
+ }
+ if err := json.Unmarshal(rr.Body.Bytes(), &info); err != nil {
+ t.Fatal(err)
+ }
+ if info.UserID != current.ID || len(info.Scopes) != 2 {
+ t.Fatalf("token info = %+v", info)
+ }
+}
+
+func TestUnknownAdminRouteReturnsNotFound(t *testing.T) {
+ srv := New(store.NewSeededTestStore())
+ req := httptest.NewRequest(http.MethodPost, "/api/admin/unknown-placeholder", nil)
+ rr := httptest.NewRecorder()
+
+ srv.Handler().ServeHTTP(rr, req)
+
+ if rr.Code != http.StatusNotFound {
+ t.Fatalf("status = %d, want %d", rr.Code, http.StatusNotFound)
+ }
+}
+
+func TestConfigExposesPluginDefaults(t *testing.T) {
+ srv := New(store.NewSeededTestStore())
+ req := httptest.NewRequest(http.MethodGet, "/api/config", nil)
+ rr := httptest.NewRecorder()
+
+ srv.Handler().ServeHTTP(rr, req)
+
+ if rr.Code != http.StatusOK {
+ t.Fatalf("status = %d, want 200", rr.Code)
+ }
+ if !strings.Contains(rr.Body.String(), `"kroki"`) || !strings.Contains(rr.Body.String(), `"plugins"`) {
+ t.Fatalf("config missing plugin defaults: %s", rr.Body.String())
+ }
+}
diff --git a/backend/internal/api/server_util.go b/backend/internal/api/server_util.go
new file mode 100644
index 0000000..a417017
--- /dev/null
+++ b/backend/internal/api/server_util.go
@@ -0,0 +1,353 @@
+package api
+
+import (
+ "bytes"
+ "context"
+ "crypto/rand"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "log"
+ "mime"
+ "net/http"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "time"
+
+ "modex/backend/internal/deploy"
+ "modex/backend/internal/store"
+
+ "github.com/minio/minio-go/v7"
+)
+
+func (s *Server) handleAdminAccepted(w http.ResponseWriter, r *http.Request) {
+ writeError(w, http.StatusNotFound, "not_found", "admin route not found")
+}
+
+func decodeBody(r *http.Request, v any) error {
+ return json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(v)
+}
+
+func (s *Server) writeMutation(w http.ResponseWriter, v any, successStatus int, err error) {
+ if err != nil {
+ writeResult(w, nil, err)
+ return
+ }
+ writeJSON(w, successStatus, v)
+}
+
+func writeResult(w http.ResponseWriter, v any, err error) {
+ if err == nil {
+ writeJSON(w, http.StatusOK, v)
+ return
+ }
+ if errors.Is(err, store.ErrNotFound) {
+ writeError(w, http.StatusNotFound, "not_found", "resource not found")
+ return
+ }
+ if errors.Is(err, store.ErrInvalid) {
+ writeError(w, http.StatusBadRequest, "invalid_input", err.Error())
+ return
+ }
+ if errors.Is(err, store.ErrConflict) {
+ writeError(w, http.StatusConflict, "conflict", err.Error())
+ return
+ }
+ writeError(w, http.StatusInternalServerError, "internal_error", err.Error())
+}
+
+func writeJSON(w http.ResponseWriter, status int, v any) {
+ w.Header().Set("Content-Type", "application/json; charset=utf-8")
+ w.WriteHeader(status)
+ _ = json.NewEncoder(w).Encode(v)
+}
+
+func writeError(w http.ResponseWriter, status int, code, message string) {
+ writeJSON(w, status, map[string]any{"error": map[string]string{"code": code, "message": message}})
+}
+
+func splitPath(path string) []string {
+ var out []string
+ for _, p := range strings.Split(path, "/") {
+ if p != "" {
+ out = append(out, p)
+ }
+ }
+ return out
+}
+
+func (s *Server) cors(next http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ origin := r.Header.Get("Origin")
+ if s.originAllowed(origin) {
+ w.Header().Set("Access-Control-Allow-Origin", origin)
+ w.Header().Set("Vary", "Origin")
+ w.Header().Set("Access-Control-Allow-Credentials", "true")
+ } else if origin == "" {
+ w.Header().Set("Access-Control-Allow-Origin", "*")
+ }
+ w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization")
+ w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
+ if r.Method == http.MethodOptions {
+ w.WriteHeader(http.StatusNoContent)
+ return
+ }
+ next.ServeHTTP(w, r)
+ })
+}
+
+func (s *Server) handleGitLabWebhook(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodPost {
+ writeError(w, http.StatusMethodNotAllowed, "method_not_allowed", "use POST")
+ return
+ }
+ secret := r.Header.Get("X-Gitlab-Token")
+ globalSecret := os.Getenv("GITLAB_WEBHOOK_SECRET")
+ if globalSecret != "" && secret != globalSecret {
+ writeError(w, http.StatusForbidden, "forbidden", "invalid webhook secret")
+ return
+ }
+ var payload struct {
+ ObjectKind string `json:"object_kind"`
+ Project struct {
+ PathWithNamespace string `json:"path_with_namespace"`
+ } `json:"project"`
+ Commits []struct {
+ ID string `json:"id"`
+ Message string `json:"message"`
+ Timestamp string `json:"timestamp"`
+ } `json:"commits"`
+ }
+ if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
+ writeError(w, http.StatusBadRequest, "bad_request", err.Error())
+ return
+ }
+ if payload.ObjectKind == "push" && len(payload.Commits) > 0 {
+ latest := payload.Commits[0]
+ log.Printf("GitLab webhook push for %s, latest commit %s: %s", payload.Project.PathWithNamespace, latest.ID, latest.Message)
+ // optionally update module last push info here if repo matches a module
+ }
+ writeJSON(w, http.StatusOK, map[string]any{"status": "received", "kind": payload.ObjectKind})
+}
+
+func (s *Server) originAllowed(origin string) bool {
+ if origin == "" {
+ return false
+ }
+ for _, allowed := range s.app.Auth().Config().CORSAllowOrigins {
+ if allowed == "*" || allowed == origin {
+ return true
+ }
+ }
+ return false
+}
+
+func recoverer(next http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ defer func() {
+ if rec := recover(); rec != nil {
+ writeError(w, http.StatusInternalServerError, "panic", fmt.Sprint(rec))
+ }
+ }()
+ next.ServeHTTP(w, r)
+ })
+}
+
+type statusRecorder struct {
+ http.ResponseWriter
+ status int
+}
+
+func (r *statusRecorder) Flush() {
+ if flusher, ok := r.ResponseWriter.(http.Flusher); ok {
+ flusher.Flush()
+ }
+}
+
+func (r *statusRecorder) WriteHeader(status int) {
+ r.status = status
+ r.ResponseWriter.WriteHeader(status)
+}
+
+func accessLogger(next http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ start := time.Now()
+ rr := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
+ next.ServeHTTP(rr, r)
+ if r.URL.Path == "/healthz" {
+ return
+ }
+ log.Printf("http method=%s path=%s status=%d duration_ms=%d remote=%s", r.Method, r.URL.Path, rr.status, time.Since(start).Milliseconds(), r.RemoteAddr)
+ })
+}
+
+type deployStep struct {
+ Name string `json:"name"`
+ Status string `json:"status"`
+ Error string `json:"error,omitempty"`
+ Note string `json:"note,omitempty"`
+}
+
+type deployReport struct {
+ Stages []deployStep `json:"stages"`
+}
+
+func newDeployReport() *deployReport {
+ return &deployReport{Stages: []deployStep{}}
+}
+
+func (r *deployReport) ok(name string) {
+ r.Stages = append(r.Stages, deployStep{Name: name, Status: "ok"})
+}
+
+func (r *deployReport) skip(name, note string) {
+ r.Stages = append(r.Stages, deployStep{Name: name, Status: "skipped", Note: note})
+}
+
+func (r *deployReport) fail(name string, err error) {
+ r.Stages = append(r.Stages, deployStep{Name: name, Status: "failed", Error: errorString(err)})
+ log.Printf("deploy stage failed stage=%s error=%v", name, err)
+}
+
+func writeDeployError(w http.ResponseWriter, status int, code, message string, report *deployReport) {
+ writeJSON(w, status, map[string]any{
+ "error": map[string]string{"code": code, "message": message},
+ "deploy": report,
+ })
+}
+
+func toStoreArtifact(a deploy.Artifact) store.DeployArtifact {
+ out := store.DeployArtifact{
+ ModuleKey: a.Metadata.ModuleKey,
+ ModuleName: a.Metadata.ModuleName,
+ DocsVersion: a.Metadata.DocsVersion,
+ PackageVersion: a.Metadata.PackageVersion,
+ Description: a.Metadata.Description,
+ Authors: append([]string(nil), a.Metadata.Authors...),
+ Edition: a.Metadata.Edition,
+ Keywords: append([]string(nil), a.Metadata.Keywords...),
+ RepoURL: a.Metadata.RepoURL,
+ RepoType: a.Metadata.RepoType,
+ Branch: a.Metadata.Branch,
+ CommitSHA: a.Metadata.CommitSHA,
+ Bytes: a.Bytes,
+ SiteHTML: map[string]string{},
+ SiteFiles: map[string][]byte{},
+ }
+ for _, e := range a.Manifest.Entries {
+ out.Entries = append(out.Entries, store.DeployEntry{Key: e.Key, Title: e.Title, Type: e.Type, Source: e.Source, Output: e.Output})
+ }
+ for _, d := range a.Documents {
+ out.Documents = append(out.Documents, store.DeployDocument{
+ DocID: d.DocID, ModuleKey: d.ModuleKey, ModuleName: d.ModuleName, DocsVersion: d.DocsVersion,
+ PackageVersion: d.PackageVersion, EntryKey: d.EntryKey, EntryType: d.EntryType, Title: d.Title,
+ Description: d.Description, Content: d.Content, ContentMD: d.ContentMD, Path: d.Path, SourceFile: d.SourceFile,
+ Keywords: append([]string(nil), d.Keywords...), Status: d.Status,
+ })
+ }
+ for _, n := range a.Nav {
+ out.Nav = append(out.Nav, toStoreNav(n))
+ }
+ for name, html := range a.SiteHTML {
+ out.SiteHTML[name] = html
+ }
+ for name, content := range a.SiteFiles {
+ // Hand over the bytes directly instead of cloning: the source artifact is
+ // discarded right after ingestion, and cloning a large (image/GIF-heavy)
+ // site here doubles peak memory and can OOM the backend on big deploys.
+ out.SiteFiles[name] = content
+ }
+ return out
+}
+
+func toStoreNav(n deploy.NavItem) store.NavItem {
+ out := store.NavItem{Title: n.Title, Path: n.Path}
+ for _, child := range n.Children {
+ out.Children = append(out.Children, toStoreNav(child))
+ }
+ return out
+}
+
+func (s *Server) uploadSiteFilesToMinIO(ctx context.Context, artifact deploy.Artifact, moduleKey, docsVersion string) error {
+ if s.minioClient == nil {
+ return nil
+ }
+ for name, content := range artifact.SiteFiles {
+ key := fmt.Sprintf("modules/%s/%s/%s", moduleKey, docsVersion, name)
+ ct := contentTypeForName(name, content)
+ _, err := s.minioClient.PutObject(ctx, s.minioBucket, key, bytes.NewReader(content), int64(len(content)), minio.PutObjectOptions{
+ ContentType: ct,
+ })
+ if err != nil {
+ return fmt.Errorf("upload %s: %w", key, err)
+ }
+ }
+ return nil
+}
+
+func (s *Server) cleanupUploadedSiteFiles(moduleKey, docsVersion string, files map[string][]byte) {
+ if s.minioClient == nil || len(files) == 0 {
+ return
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ for name := range files {
+ key := fmt.Sprintf("modules/%s/%s/%s", moduleKey, docsVersion, name)
+ if err := s.minioClient.RemoveObject(ctx, s.minioBucket, key, minio.RemoveObjectOptions{}); err != nil {
+ log.Printf("cleanup uploaded site file failed key=%s error=%v", key, err)
+ }
+ }
+}
+
+func contentTypeForName(name string, content []byte) string {
+ if ct := mime.TypeByExtension(filepath.Ext(name)); ct != "" {
+ return ct
+ }
+ if len(content) > 0 {
+ return http.DetectContentType(content)
+ }
+ return "application/octet-stream"
+}
+
+func errorString(err error) string {
+ if err == nil {
+ return ""
+ }
+ return err.Error()
+}
+
+func ternary[T any](cond bool, yes, no T) T {
+ if cond {
+ return yes
+ }
+ return no
+}
+
+func envInt64(key string, fallback int64) int64 {
+ v, err := strconv.ParseInt(os.Getenv(key), 10, 64)
+ if err != nil || v <= 0 {
+ return fallback
+ }
+ return v
+}
+
+func bearerToken(r *http.Request) string {
+ auth := r.Header.Get("Authorization")
+ const prefix = "Bearer "
+ if strings.HasPrefix(auth, prefix) {
+ return strings.TrimSpace(auth[len(prefix):])
+ }
+ return ""
+}
+
+func randomToken(n int) (string, error) {
+ b := make([]byte, n)
+ if _, err := rand.Read(b); err != nil {
+ return "", err
+ }
+ return base64.RawURLEncoding.EncodeToString(b), nil
+}
diff --git a/backend/internal/application/service.go b/backend/internal/application/service.go
new file mode 100644
index 0000000..792c7e4
--- /dev/null
+++ b/backend/internal/application/service.go
@@ -0,0 +1,88 @@
+package application
+
+import (
+ "modex/backend/internal/auth"
+ "modex/backend/internal/config"
+ "modex/backend/internal/embedding"
+ "modex/backend/internal/search"
+ "modex/backend/internal/store"
+)
+
+// Repository is the persistence boundary used by the application layer.
+// Implementations can be backed by pgx, an ORM, or a test double without
+// leaking database details into HTTP controllers.
+type Repository interface {
+ Close()
+}
+
+// Service owns the business-facing dependencies used by controllers.
+type Service struct {
+ store store.DataStore
+ auth *auth.Service
+ search search.Service
+ repository Repository
+}
+
+func New(st store.DataStore, vectors search.VectorStore, repository Repository) *Service {
+ return newService(st, vectors, repository, auth.NewService(auth.FromEnv()))
+}
+
+func NewConfigured(st store.DataStore, vectors search.VectorStore, repository Repository) (*Service, error) {
+ authService, err := auth.NewConfiguredService(auth.FromEnv())
+ if err != nil {
+ return nil, err
+ }
+ return newService(st, vectors, repository, authService), nil
+}
+
+func newService(st store.DataStore, vectors search.VectorStore, repository Repository, authService *auth.Service) *Service {
+ contentStore := search.ContentStore(st)
+ scoring := config.LoadSearchScoring()
+ provider := embedding.SettingsProvider{Load: func() embedding.Settings {
+ ai := contentStore.Settings().AI
+ return embedding.Settings{
+ BaseURL: ai.EmbeddingBaseURL,
+ Model: ai.EmbeddingModel,
+ APIKey: ai.EmbeddingAPIKey,
+ Dim: ai.EmbeddingDim,
+ }
+ }}
+ return &Service{
+ store: st,
+ auth: authService,
+ repository: repository,
+ search: search.Service{
+ Store: contentStore,
+ Embedder: provider,
+ Vectors: vectors,
+ KeywordWeight: positiveFloat(scoring.KeywordWeight, 0.6),
+ SemanticWeight: positiveFloat(scoring.SemanticWeight, 0.4),
+ },
+ }
+}
+
+func (s *Service) Store() store.DataStore {
+ return s.store
+}
+
+func (s *Service) Auth() *auth.Service {
+ return s.auth
+}
+
+func (s *Service) Search() *search.Service {
+ return &s.search
+}
+
+func (s *Service) Close() {
+ _ = s.auth.Close()
+ if s.repository != nil {
+ s.repository.Close()
+ }
+}
+
+func positiveFloat(v, fallback float64) float64 {
+ if v <= 0 {
+ return fallback
+ }
+ return v
+}
diff --git a/backend/internal/auth/config.go b/backend/internal/auth/config.go
index ece7888..d5eeef5 100644
--- a/backend/internal/auth/config.go
+++ b/backend/internal/auth/config.go
@@ -4,12 +4,14 @@ import (
"net/url"
"os"
"strings"
+ "time"
"modex/backend/internal/config"
+ "modex/backend/internal/dburl"
+ "modex/backend/internal/redisurl"
)
type Config struct {
- Mode string
AppBaseURL string
FrontendBaseURL string
IssuerURL string
@@ -26,8 +28,12 @@ type Config struct {
CookieDomain string
CookieSameSite string
CookieSecure bool
+ AutoLogin bool
CORSAllowOrigins []string
SuperAdmins []string
+ RedisURL string
+ DatabaseURL string
+ SessionTTL time.Duration
// UserMapping controls which OIDC claims are used for core user identity fields.
// Values are resolved from an optional config file + environment variable overrides.
@@ -42,14 +48,9 @@ func FromEnv() Config {
issuer = keycloakBase + "/realms/" + realm
}
appBase := strings.TrimRight(env("APP_BASE_URL", "http://localhost:8671"), "/")
- mode := env("AUTH_MODE", "mock")
- if mode == "keycloak" {
- mode = "oidc"
- }
cfg := Config{
- Mode: mode,
AppBaseURL: appBase,
- FrontendBaseURL: strings.TrimRight(env("FRONTEND_BASE_URL", "http://localhost:3000"), "/"),
+ FrontendBaseURL: strings.TrimRight(env("FRONTEND_BASE_URL", "http://localhost:3456"), "/"),
IssuerURL: issuer,
AuthURL: os.Getenv("OIDC_AUTH_URL"),
TokenURL: os.Getenv("OIDC_TOKEN_URL"),
@@ -64,12 +65,15 @@ func FromEnv() Config {
CookieDomain: os.Getenv("COOKIE_DOMAIN"),
CookieSameSite: env("COOKIE_SAME_SITE", "lax"),
CookieSecure: env("COOKIE_SECURE", "false") == "true",
- CORSAllowOrigins: splitList(env("CORS_ALLOW_ORIGINS", "http://localhost:3000")),
- SuperAdmins: splitList(os.Getenv("SUPER_ADMIN_USERS")),
+ AutoLogin: env("AUTO_LOGIN", "false") == "true",
+ CORSAllowOrigins: splitList(env("CORS_ALLOW_ORIGINS", "http://localhost:3456")),
+ SuperAdmins: splitList(os.Getenv("SUPER_ADMIN_USERS")),
+ RedisURL: redisurl.FromEnv(),
+ DatabaseURL: dburl.FromEnv(),
+ SessionTTL: envDuration("SESSION_TTL", 8*time.Hour),
- // UserMapping comes from a combination of (optional) config file + env overrides.
- // See internal/config for precedence rules and why some settings live in files
- // while connection secrets stay in the environment.
+ // UserMapping comes from the optional application config file. Connection
+ // secrets stay in environment variables; identity semantics stay in config.yaml.
UserMapping: resolveUserMapping(),
}
if cfg.AuthURL == "" && issuer != "" {
@@ -91,10 +95,10 @@ func FromEnv() Config {
}
func (c Config) LoginReady() bool {
- return c.Mode == "oidc" && c.AuthURL != "" && c.TokenURL != "" && c.UserInfoURL != "" && c.ClientID != ""
+ return c.IssuerURL != "" && c.AuthURL != "" && c.TokenURL != "" && c.ClientID != ""
}
-func (c Config) LoginURL(state string) string {
+func (c Config) LoginURL(state string, options ...string) string {
u, _ := url.Parse(c.AuthURL)
q := u.Query()
q.Set("client_id", c.ClientID)
@@ -102,10 +106,22 @@ func (c Config) LoginURL(state string) string {
q.Set("response_type", "code")
q.Set("scope", strings.Join(c.Scopes, " "))
q.Set("state", state)
+ for i := 0; i+1 < len(options); i += 2 {
+ q.Set(options[i], options[i+1])
+ }
u.RawQuery = q.Encode()
return u.String()
}
+func envDuration(key string, fallback time.Duration) time.Duration {
+ if value := os.Getenv(key); value != "" {
+ if parsed, err := time.ParseDuration(value); err == nil && parsed > 0 {
+ return parsed
+ }
+ }
+ return fallback
+}
+
func env(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
@@ -123,10 +139,8 @@ func splitList(v string) []string {
return out
}
-// resolveUserMapping loads the user attribute mapping with the following precedence:
-// 1. Values from config file (if CONFIG_FILE or conventional locations exist)
-// 2. OIDC_CLAIM_* environment variables (explicit overrides)
-// 3. Reasonable defaults (company prefers email as unique id)
+// resolveUserMapping loads user attribute mapping from config.yaml, then fills
+// defaults for fields omitted by the file.
func resolveUserMapping() config.UserMapping {
m := config.LoadUserMapping()
diff --git a/backend/internal/auth/service.go b/backend/internal/auth/service.go
index 8b2a7a1..b6527af 100644
--- a/backend/internal/auth/service.go
+++ b/backend/internal/auth/service.go
@@ -3,6 +3,8 @@ package auth
import (
"context"
"crypto/rand"
+ "crypto/sha256"
+ "crypto/subtle"
"encoding/base64"
"encoding/json"
"errors"
@@ -14,21 +16,45 @@ import (
"time"
"modex/backend/internal/store"
+
+ "github.com/coreos/go-oidc/v3/oidc"
)
type Service struct {
- cfg Config
- client *http.Client
- mu sync.RWMutex
- sessions map[string]store.User
+ cfg Config
+ client *http.Client
+ sessions sessionStore
+ providerMu sync.Mutex
+ verifier *oidc.IDTokenVerifier
}
func NewService(cfg Config) *Service {
- return &Service{
- cfg: cfg,
- client: &http.Client{Timeout: 20 * time.Second},
- sessions: map[string]store.User{},
+ return newService(cfg, newMemorySessionStore())
+}
+
+func NewConfiguredService(cfg Config) (*Service, error) {
+ if cfg.RedisURL != "" {
+ sessions, err := newRedisSessionStore(cfg.RedisURL)
+ if err != nil {
+ return nil, fmt.Errorf("connect Redis session store: %w", err)
+ }
+ return newService(cfg, sessions), nil
}
+ if cfg.DatabaseURL == "" {
+ return nil, errors.New("PostgreSQL connection is required when Redis is not configured")
+ }
+ sessions, err := newPostgresSessionStore(cfg.DatabaseURL)
+ if err != nil {
+ return nil, fmt.Errorf("connect PostgreSQL session store: %w", err)
+ }
+ return newService(cfg, sessions), nil
+}
+
+func newService(cfg Config, sessions sessionStore) *Service {
+ if cfg.SessionTTL <= 0 {
+ cfg.SessionTTL = 8 * time.Hour
+ }
+ return &Service{cfg: cfg, client: &http.Client{Timeout: 20 * time.Second}, sessions: sessions}
}
func (s *Service) Config() Config {
@@ -78,6 +104,18 @@ func (s *Service) BeginLogin(w http.ResponseWriter) (string, error) {
if err != nil {
return "", err
}
+ nonce, err := randomToken(32)
+ if err != nil {
+ return "", err
+ }
+ verifier, err := randomToken(48)
+ if err != nil {
+ return "", err
+ }
+ transaction := loginTransaction{Nonce: nonce, CodeVerifier: verifier}
+ if err := s.sessions.Set(context.Background(), oauthStateKey(state), transaction, 5*time.Minute); err != nil {
+ return "", fmt.Errorf("store OAuth2 transaction: %w", err)
+ }
http.SetCookie(w, &http.Cookie{
Name: s.cfg.StateCookie,
Value: state,
@@ -88,7 +126,13 @@ func (s *Service) BeginLogin(w http.ResponseWriter) (string, error) {
SameSite: sameSiteMode(s.cfg.CookieSameSite),
Secure: s.cfg.CookieSecure,
})
- return s.cfg.LoginURL(state), nil
+ challenge := sha256.Sum256([]byte(verifier))
+ return s.cfg.LoginURL(
+ state,
+ "nonce", nonce,
+ "code_challenge", base64.RawURLEncoding.EncodeToString(challenge[:]),
+ "code_challenge_method", "S256",
+ ), nil
}
func (s *Service) CompleteLogin(ctx context.Context, r *http.Request, w http.ResponseWriter) (store.User, error) {
@@ -105,14 +149,23 @@ func (s *Service) CompleteLogin(ctx context.Context, r *http.Request, w http.Res
return store.User{}, errors.New("missing OAuth2 code or state")
}
stateCookie, err := r.Cookie(s.cfg.StateCookie)
- if err != nil || stateCookie.Value != state {
+ if err != nil || subtle.ConstantTimeCompare([]byte(stateCookie.Value), []byte(state)) != 1 {
return store.User{}, errors.New("invalid OAuth2 state")
}
- token, err := s.exchangeCode(ctx, code)
+ var transaction loginTransaction
+ if err := s.sessions.Get(ctx, oauthStateKey(state), &transaction); err != nil {
+ return store.User{}, errors.New("expired or invalid OAuth2 transaction")
+ }
+ _ = s.sessions.Delete(ctx, oauthStateKey(state))
+ token, err := s.exchangeCode(ctx, code, transaction.CodeVerifier)
+ if err != nil {
+ return store.User{}, err
+ }
+ claims, err := s.verifyIDToken(ctx, token.IDToken, token.AccessToken, transaction.Nonce)
if err != nil {
return store.User{}, err
}
- user, err := s.fetchUserInfo(ctx, token.AccessToken, token.IDToken)
+ user, err := s.fetchUserInfo(ctx, token.AccessToken, claims)
if err != nil {
return store.User{}, err
}
@@ -123,24 +176,23 @@ func (s *Service) CompleteLogin(ctx context.Context, r *http.Request, w http.Res
return user, nil
}
-// CreateSession issues a session cookie for the given user. It is shared by the
-// OIDC callback and the local mock-login endpoint so both paths produce a real,
-// cookie-backed session.
+// CreateSession issues a session cookie for the given user after the OIDC
+// callback establishes a real, cookie-backed session.
func (s *Service) CreateSession(w http.ResponseWriter, user store.User) error {
user = s.applySuperAdmin(user)
sessionID, err := randomToken(32)
if err != nil {
return err
}
- s.mu.Lock()
- s.sessions[sessionID] = user
- s.mu.Unlock()
+ if err := s.sessions.Set(context.Background(), userSessionKey(sessionID), user, s.cfg.SessionTTL); err != nil {
+ return fmt.Errorf("store user session: %w", err)
+ }
http.SetCookie(w, &http.Cookie{
Name: s.cfg.SessionCookie,
Value: sessionID,
Domain: s.cfg.CookieDomain,
Path: "/",
- MaxAge: int((8 * time.Hour).Seconds()),
+ MaxAge: int(s.cfg.SessionTTL.Seconds()),
HttpOnly: true,
SameSite: sameSiteMode(s.cfg.CookieSameSite),
Secure: s.cfg.CookieSecure,
@@ -153,27 +205,27 @@ func (s *Service) CurrentUser(r *http.Request) (store.User, bool) {
if err != nil || cookie.Value == "" {
return store.User{}, false
}
- s.mu.RLock()
- defer s.mu.RUnlock()
- user, ok := s.sessions[cookie.Value]
- return user, ok
+ var user store.User
+ if err := s.sessions.Get(r.Context(), userSessionKey(cookie.Value), &user); err != nil {
+ return store.User{}, false
+ }
+ return user, true
}
func (s *Service) Logout(w http.ResponseWriter, r *http.Request) {
if cookie, err := r.Cookie(s.cfg.SessionCookie); err == nil {
- s.mu.Lock()
- delete(s.sessions, cookie.Value)
- s.mu.Unlock()
+ _ = s.sessions.Delete(r.Context(), userSessionKey(cookie.Value))
}
http.SetCookie(w, &http.Cookie{Name: s.cfg.SessionCookie, Value: "", Domain: s.cfg.CookieDomain, Path: "/", MaxAge: -1, HttpOnly: true, SameSite: sameSiteMode(s.cfg.CookieSameSite), Secure: s.cfg.CookieSecure})
}
-func (s *Service) exchangeCode(ctx context.Context, code string) (tokenResponse, error) {
+func (s *Service) exchangeCode(ctx context.Context, code, codeVerifier string) (tokenResponse, error) {
form := url.Values{}
form.Set("grant_type", "authorization_code")
form.Set("client_id", s.cfg.ClientID)
form.Set("code", code)
form.Set("redirect_uri", s.cfg.RedirectURL)
+ form.Set("code_verifier", codeVerifier)
if s.cfg.ClientSecret != "" {
form.Set("client_secret", s.cfg.ClientSecret)
}
@@ -200,16 +252,7 @@ func (s *Service) exchangeCode(ctx context.Context, code string) (tokenResponse,
return token, nil
}
-func (s *Service) fetchUserInfo(ctx context.Context, accessToken, idToken string) (store.User, error) {
- // Collect claims from both ID token (often has custom mappers like wxPhotoURL) and userinfo endpoint.
- claims := map[string]any{}
-
- if idToken != "" {
- if idClaims, err := parseJWTClaims(idToken); err == nil {
- mergeInto(claims, idClaims)
- }
- }
-
+func (s *Service) fetchUserInfo(ctx context.Context, accessToken string, claims map[string]any) (store.User, error) {
if s.cfg.UserInfoURL != "" && accessToken != "" {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.cfg.UserInfoURL, nil)
if err != nil {
@@ -239,7 +282,7 @@ func (s *Service) fetchUserInfo(ctx context.Context, accessToken, idToken string
// Stable internal ID: prefer the configured unique claim value; fall back to sub.
userID := uniqueID
if userID == "" {
- userID = getString(claims, "sub")
+ return store.User{}, errors.New("OIDC claims do not contain a stable user identifier")
}
// Username for login/display fallback.
@@ -261,7 +304,6 @@ func (s *Service) fetchUserInfo(ctx context.Context, accessToken, idToken string
avatar := pickClaim(claims, m.AvatarClaim, "picture", "avatar", "photo", "wxPhotoURL")
- groups := extractStringSlice(claims, "groups")
roles := extractStringSlice(claims, "roles")
return store.User{
@@ -271,11 +313,62 @@ func (s *Service) fetchUserInfo(ctx context.Context, accessToken, idToken string
Email: email,
Department: secondary,
Avatar: avatar,
- Groups: groups,
Roles: roles,
}, nil
}
+type loginTransaction struct {
+ Nonce string `json:"nonce"`
+ CodeVerifier string `json:"code_verifier"`
+}
+
+func userSessionKey(id string) string { return "modex:session:" + id }
+func oauthStateKey(state string) string { return "modex:oauth-state:" + state }
+
+func (s *Service) verifyIDToken(ctx context.Context, rawIDToken, accessToken, nonce string) (map[string]any, error) {
+ if rawIDToken == "" {
+ return nil, errors.New("OIDC token endpoint returned empty id_token")
+ }
+ verifier, err := s.idTokenVerifier(ctx)
+ if err != nil {
+ return nil, err
+ }
+ idToken, err := verifier.Verify(ctx, rawIDToken)
+ if err != nil {
+ return nil, fmt.Errorf("verify OIDC ID token: %w", err)
+ }
+ if nonce == "" || subtle.ConstantTimeCompare([]byte(idToken.Nonce), []byte(nonce)) != 1 {
+ return nil, errors.New("invalid OIDC nonce")
+ }
+ if idToken.AccessTokenHash != "" {
+ if err := idToken.VerifyAccessToken(accessToken); err != nil {
+ return nil, fmt.Errorf("verify OIDC access token hash: %w", err)
+ }
+ }
+ claims := map[string]any{}
+ if err := idToken.Claims(&claims); err != nil {
+ return nil, fmt.Errorf("decode verified OIDC claims: %w", err)
+ }
+ return claims, nil
+}
+
+func (s *Service) idTokenVerifier(ctx context.Context) (*oidc.IDTokenVerifier, error) {
+ s.providerMu.Lock()
+ defer s.providerMu.Unlock()
+ if s.verifier != nil {
+ return s.verifier, nil
+ }
+ provider, err := oidc.NewProvider(ctx, s.cfg.IssuerURL)
+ if err != nil {
+ return nil, fmt.Errorf("discover OIDC provider: %w", err)
+ }
+ s.verifier = provider.Verifier(&oidc.Config{ClientID: s.cfg.ClientID})
+ return s.verifier, nil
+}
+
+func (s *Service) Healthy(ctx context.Context) error { return s.sessions.Ping(ctx) }
+func (s *Service) Close() error { return s.sessions.Close() }
+
type tokenResponse struct {
AccessToken string `json:"access_token"`
IDToken string `json:"id_token"`
@@ -292,46 +385,6 @@ func randomToken(n int) (string, error) {
return base64.RawURLEncoding.EncodeToString(b), nil
}
-func first(values ...string) string {
- for _, v := range values {
- if v != "" {
- return v
- }
- }
- return ""
-}
-
-// parseJWTClaims extracts the payload claims from a JWT without signature validation.
-// This is safe here because the token was obtained directly from the token endpoint
-// after a successful authorization code exchange.
-func parseJWTClaims(token string) (map[string]any, error) {
- if token == "" {
- return nil, nil
- }
- parts := strings.Split(token, ".")
- if len(parts) != 3 {
- return nil, errors.New("invalid jwt format")
- }
- payload := parts[1]
- // Pad base64 if needed
- if pad := len(payload) % 4; pad != 0 {
- payload += strings.Repeat("=", 4-pad)
- }
- b, err := base64.URLEncoding.DecodeString(payload)
- if err != nil {
- // try raw encoding (no padding)
- b, err = base64.RawURLEncoding.DecodeString(parts[1])
- if err != nil {
- return nil, err
- }
- }
- var m map[string]any
- if err := json.Unmarshal(b, &m); err != nil {
- return nil, err
- }
- return m, nil
-}
-
// getString safely extracts a string value from claims (handles string, number, single-element slice).
func getString(m map[string]any, key string) string {
if m == nil || key == "" {
@@ -381,7 +434,7 @@ func mergeInto(dst, src map[string]any) {
}
}
-// extractStringSlice returns a []string for common group/role claims that may be string or []string.
+// extractStringSlice returns a []string for claims (e.g. roles) that may be string or []string.
func extractStringSlice(m map[string]any, key string) []string {
if m == nil {
return nil
diff --git a/backend/internal/auth/service_test.go b/backend/internal/auth/service_test.go
new file mode 100644
index 0000000..7d33929
--- /dev/null
+++ b/backend/internal/auth/service_test.go
@@ -0,0 +1,83 @@
+package auth
+
+import (
+ "context"
+ "net/http/httptest"
+ "net/url"
+ "strings"
+ "testing"
+ "time"
+
+ "modex/backend/internal/store"
+)
+
+func TestBeginLoginUsesNonceAndPKCE(t *testing.T) {
+ service := NewService(Config{
+ IssuerURL: "https://issuer.example.com",
+ AuthURL: "https://issuer.example.com/authorize",
+ TokenURL: "https://issuer.example.com/token",
+ ClientID: "modex",
+ RedirectURL: "https://modex.example.com/callback",
+ Scopes: []string{"openid"},
+ StateCookie: "oauth_state",
+ SessionCookie: "session",
+ SessionTTL: time.Hour,
+ })
+ recorder := httptest.NewRecorder()
+ loginURL, err := service.BeginLogin(recorder)
+ if err != nil {
+ t.Fatal(err)
+ }
+ parsed, err := url.Parse(loginURL)
+ if err != nil {
+ t.Fatal(err)
+ }
+ query := parsed.Query()
+ for _, key := range []string{"state", "nonce", "code_challenge"} {
+ if query.Get(key) == "" {
+ t.Fatalf("missing %s in login URL", key)
+ }
+ }
+ if query.Get("code_challenge_method") != "S256" {
+ t.Fatalf("code challenge method = %q", query.Get("code_challenge_method"))
+ }
+ if !strings.Contains(recorder.Header().Get("Set-Cookie"), "HttpOnly") {
+ t.Fatal("state cookie must be HttpOnly")
+ }
+}
+
+func TestMemoryBackedSessionLifecycle(t *testing.T) {
+ service := NewService(Config{SessionCookie: "session", SessionTTL: time.Hour})
+ recorder := httptest.NewRecorder()
+ user := store.User{ID: "user-1", Username: "alice"}
+ if err := service.CreateSession(recorder, user); err != nil {
+ t.Fatal(err)
+ }
+ response := recorder.Result()
+ cookies := response.Cookies()
+ if len(cookies) != 1 {
+ t.Fatalf("cookies = %d", len(cookies))
+ }
+ request := httptest.NewRequest("GET", "/", nil)
+ request.AddCookie(cookies[0])
+ got, ok := service.CurrentUser(request)
+ if !ok || got.ID != user.ID {
+ t.Fatalf("CurrentUser = %#v, %v", got, ok)
+ }
+ logout := httptest.NewRecorder()
+ service.Logout(logout, request)
+ if _, ok := service.CurrentUser(request); ok {
+ t.Fatal("session should be deleted on logout")
+ }
+}
+
+func TestMemorySessionExpires(t *testing.T) {
+ sessions := newMemorySessionStore()
+ if err := sessions.Set(context.Background(), "key", map[string]string{"id": "1"}, -time.Second); err != nil {
+ t.Fatal(err)
+ }
+ var value map[string]string
+ if err := sessions.Get(context.Background(), "key", &value); err != errSessionNotFound {
+ t.Fatalf("Get error = %v", err)
+ }
+}
diff --git a/backend/internal/auth/session_store.go b/backend/internal/auth/session_store.go
new file mode 100644
index 0000000..b09ec6f
--- /dev/null
+++ b/backend/internal/auth/session_store.go
@@ -0,0 +1,177 @@
+package auth
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "sync"
+ "time"
+
+ "github.com/jackc/pgx/v5"
+ "github.com/jackc/pgx/v5/pgxpool"
+ "github.com/redis/go-redis/v9"
+)
+
+var errSessionNotFound = errors.New("session not found")
+
+type sessionStore interface {
+ Set(ctx context.Context, key string, value any, ttl time.Duration) error
+ Get(ctx context.Context, key string, value any) error
+ Delete(ctx context.Context, key string) error
+ Ping(ctx context.Context) error
+ Close() error
+}
+
+type memorySession struct {
+ value []byte
+ expiresAt time.Time
+}
+
+type memorySessionStore struct {
+ mu sync.RWMutex
+ items map[string]memorySession
+}
+
+func newMemorySessionStore() *memorySessionStore {
+ return &memorySessionStore{items: map[string]memorySession{}}
+}
+
+func (s *memorySessionStore) Set(_ context.Context, key string, value any, ttl time.Duration) error {
+ raw, err := json.Marshal(value)
+ if err != nil {
+ return err
+ }
+ s.mu.Lock()
+ s.items[key] = memorySession{value: raw, expiresAt: time.Now().Add(ttl)}
+ s.mu.Unlock()
+ return nil
+}
+
+func (s *memorySessionStore) Get(_ context.Context, key string, value any) error {
+ s.mu.RLock()
+ item, ok := s.items[key]
+ s.mu.RUnlock()
+ if !ok || time.Now().After(item.expiresAt) {
+ if ok {
+ s.mu.Lock()
+ delete(s.items, key)
+ s.mu.Unlock()
+ }
+ return errSessionNotFound
+ }
+ return json.Unmarshal(item.value, value)
+}
+
+func (s *memorySessionStore) Delete(_ context.Context, key string) error {
+ s.mu.Lock()
+ delete(s.items, key)
+ s.mu.Unlock()
+ return nil
+}
+
+func (s *memorySessionStore) Ping(context.Context) error { return nil }
+func (s *memorySessionStore) Close() error { return nil }
+
+type postgresSessionStore struct {
+ pool *pgxpool.Pool
+}
+
+func newPostgresSessionStore(databaseURL string) (sessionStore, error) {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ pool, err := pgxpool.New(ctx, databaseURL)
+ if err != nil {
+ return nil, err
+ }
+ if err = pool.Ping(ctx); err != nil {
+ pool.Close()
+ return nil, err
+ }
+ if _, err = pool.Exec(ctx, `CREATE TABLE IF NOT EXISTS auth_session (
+ key TEXT PRIMARY KEY,
+ value_json JSONB NOT NULL,
+ expires_at TIMESTAMPTZ NOT NULL,
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+ )`); err != nil {
+ pool.Close()
+ return nil, err
+ }
+ return &postgresSessionStore{pool: pool}, nil
+}
+
+func (s *postgresSessionStore) Set(ctx context.Context, key string, value any, ttl time.Duration) error {
+ raw, err := json.Marshal(value)
+ if err != nil {
+ return err
+ }
+ _, err = s.pool.Exec(ctx, `INSERT INTO auth_session(key,value_json,expires_at,updated_at) VALUES($1,$2::jsonb,$3,now()) ON CONFLICT(key) DO UPDATE SET value_json=EXCLUDED.value_json,expires_at=EXCLUDED.expires_at,updated_at=now()`, key, string(raw), time.Now().UTC().Add(ttl))
+ return err
+}
+
+func (s *postgresSessionStore) Get(ctx context.Context, key string, value any) error {
+ var raw []byte
+ if _, err := s.pool.Exec(ctx, `DELETE FROM auth_session WHERE key=$1 AND expires_at<=now()`, key); err != nil {
+ return err
+ }
+ err := s.pool.QueryRow(ctx, `SELECT value_json FROM auth_session WHERE key=$1 AND expires_at>now()`, key).Scan(&raw)
+ if errors.Is(err, pgx.ErrNoRows) {
+ return errSessionNotFound
+ }
+ if err != nil {
+ return err
+ }
+ return json.Unmarshal(raw, value)
+}
+
+func (s *postgresSessionStore) Delete(ctx context.Context, key string) error {
+ _, err := s.pool.Exec(ctx, `DELETE FROM auth_session WHERE key=$1`, key)
+ return err
+}
+
+func (s *postgresSessionStore) Ping(ctx context.Context) error { return s.pool.Ping(ctx) }
+func (s *postgresSessionStore) Close() error { s.pool.Close(); return nil }
+
+type redisSessionStore struct {
+ client *redis.Client
+}
+
+func newRedisSessionStore(rawURL string) (sessionStore, error) {
+ options, err := redis.ParseURL(rawURL)
+ if err != nil {
+ return nil, err
+ }
+ client := redis.NewClient(options)
+ ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+ defer cancel()
+ if err := client.Ping(ctx).Err(); err != nil {
+ _ = client.Close()
+ return nil, err
+ }
+ return &redisSessionStore{client: client}, nil
+}
+
+func (s *redisSessionStore) Set(ctx context.Context, key string, value any, ttl time.Duration) error {
+ raw, err := json.Marshal(value)
+ if err != nil {
+ return err
+ }
+ return s.client.Set(ctx, key, raw, ttl).Err()
+}
+
+func (s *redisSessionStore) Get(ctx context.Context, key string, value any) error {
+ raw, err := s.client.Get(ctx, key).Bytes()
+ if errors.Is(err, redis.Nil) {
+ return errSessionNotFound
+ }
+ if err != nil {
+ return err
+ }
+ return json.Unmarshal(raw, value)
+}
+
+func (s *redisSessionStore) Delete(ctx context.Context, key string) error {
+ return s.client.Del(ctx, key).Err()
+}
+
+func (s *redisSessionStore) Ping(ctx context.Context) error { return s.client.Ping(ctx).Err() }
+func (s *redisSessionStore) Close() error { return s.client.Close() }
diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go
index e2ad3b6..850aa1c 100644
--- a/backend/internal/config/config.go
+++ b/backend/internal/config/config.go
@@ -2,6 +2,7 @@ package config
import (
"errors"
+ "fmt"
"log"
"os"
"path/filepath"
@@ -13,7 +14,8 @@ import (
// Use this for settings that describe application behavior/semantics rather than
// per-deployment infrastructure wiring (the latter belongs in environment variables).
type FileConfig struct {
- Auth AuthSection `yaml:"auth"`
+ Auth AuthSection `yaml:"auth"`
+ Search SearchSection `yaml:"search"`
}
type AuthSection struct {
@@ -23,6 +25,15 @@ type AuthSection struct {
UserMapping UserMapping `yaml:"user_mapping"`
}
+type SearchSection struct {
+ Scoring SearchScoring `yaml:"scoring"`
+}
+
+type SearchScoring struct {
+ KeywordWeight float64 `yaml:"keyword_weight"`
+ SemanticWeight float64 `yaml:"semantic_weight"`
+}
+
// UserMapping defines claim names coming from the identity provider (Keycloak etc.).
// These are used during OIDC login to populate the local User record.
type UserMapping struct {
@@ -43,18 +54,17 @@ type UserMapping struct {
SecondaryInfoClaim string `yaml:"secondary_info_claim"`
}
-// Load reads the application config.
-// Precedence (lowest to highest):
-// 1. Sensible defaults inside the code that calls this.
-// 2. Values from the YAML config file (if found).
-// 3. Explicit environment variable overrides (highest — allows per-deployment tweaks
-// without modifying the committed config file).
+// Load reads the application config. Sensible defaults live in the code that
+// calls this package; application-level behavior overrides live in YAML.
//
// The config file location is determined by:
// - CONFIG_FILE environment variable (highest priority for location)
// - Then a short list of conventional locations (./config.yaml, ./configs/config.yaml, /etc/modex/config.yaml)
func Load() (FileConfig, error) {
- path := findConfigPath()
+ path, err := findConfigPath()
+ if err != nil {
+ return FileConfig{}, err
+ }
if path == "" {
return FileConfig{}, nil // no config file configured or present — this is fine
}
@@ -73,40 +83,37 @@ func Load() (FileConfig, error) {
return fc, nil
}
-// LoadUserMapping returns the effective UserMapping after applying file + env precedence.
+// LoadUserMapping returns the UserMapping from the application config file.
func LoadUserMapping() UserMapping {
fc, err := Load()
if err != nil {
- log.Printf("warning: failed to load config file for user mapping, falling back to env only: %v", err)
+ log.Printf("warning: failed to load config file for user mapping: %v", err)
}
+ return fc.Auth.UserMapping
+}
- m := fc.Auth.UserMapping
-
- // Environment variables take highest priority (explicit overrides).
- // This preserves the previous pure-env behavior and allows emergency / per-env changes.
- if v := os.Getenv("OIDC_CLAIM_UNIQUE_ID"); v != "" {
- m.UniqueIDClaim = v
- }
- if v := os.Getenv("OIDC_CLAIM_AVATAR"); v != "" {
- m.AvatarClaim = v
- }
- if v := os.Getenv("OIDC_CLAIM_DISPLAY_NAME"); v != "" {
- m.DisplayNameClaim = v
- }
- if v := os.Getenv("OIDC_CLAIM_SECONDARY_INFO"); v != "" {
- m.SecondaryInfoClaim = v
+// LoadSearchScoring returns search ranking weights from the application config file.
+func LoadSearchScoring() SearchScoring {
+ fc, err := Load()
+ if err != nil {
+ log.Printf("warning: failed to load config file for search scoring: %v", err)
}
-
- return m
+ return fc.Search.Scoring
}
// findConfigPath returns the first existing config file path according to the lookup rules.
// It does not return an error if nothing is found (the caller decides what to do).
-func findConfigPath() string {
+func findConfigPath() (string, error) {
candidates := []string{}
if explicit := os.Getenv("CONFIG_FILE"); explicit != "" {
- candidates = append(candidates, explicit)
+ if _, err := os.Stat(explicit); err != nil {
+ if errors.Is(err, os.ErrNotExist) {
+ return "", fmt.Errorf("%w: %s", ErrConfigNotFound, explicit)
+ }
+ return "", err
+ }
+ return explicit, nil
}
// Conventional locations.
@@ -134,10 +141,10 @@ func findConfigPath() string {
continue
}
if _, err := os.Stat(p); err == nil {
- return p
+ return p, nil
}
}
- return ""
+ return "", nil
}
// MustLoad is like Load but logs and returns an empty config on error (never panics the server).
diff --git a/backend/internal/config/config_test.go b/backend/internal/config/config_test.go
new file mode 100644
index 0000000..fb129e1
--- /dev/null
+++ b/backend/internal/config/config_test.go
@@ -0,0 +1,58 @@
+package config
+
+import (
+ "errors"
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+func TestLoadRejectsMissingExplicitConfig(t *testing.T) {
+ t.Setenv("CONFIG_FILE", filepath.Join(t.TempDir(), "missing.yaml"))
+ _, err := Load()
+ if !errors.Is(err, ErrConfigNotFound) {
+ t.Fatalf("Load error = %v, want ErrConfigNotFound", err)
+ }
+}
+
+func TestLoadExplicitConfig(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "config.yaml")
+ if err := os.WriteFile(path, []byte("auth:\n user_mapping:\n unique_id_claim: sub\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ t.Setenv("CONFIG_FILE", path)
+ cfg, err := Load()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if cfg.Auth.UserMapping.UniqueIDClaim != "sub" {
+ t.Fatalf("unique id claim = %q", cfg.Auth.UserMapping.UniqueIDClaim)
+ }
+}
+
+func TestLoadUserMappingIgnoresOIDCClaimEnv(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "config.yaml")
+ if err := os.WriteFile(path, []byte("auth:\n user_mapping:\n unique_id_claim: sub\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ t.Setenv("CONFIG_FILE", path)
+ t.Setenv("OIDC_CLAIM_UNIQUE_ID", "email")
+
+ mapping := LoadUserMapping()
+ if mapping.UniqueIDClaim != "sub" {
+ t.Fatalf("unique id claim = %q, want config file value", mapping.UniqueIDClaim)
+ }
+}
+
+func TestLoadSearchScoring(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "config.yaml")
+ if err := os.WriteFile(path, []byte("search:\n scoring:\n keyword_weight: 0.7\n semantic_weight: 0.3\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ t.Setenv("CONFIG_FILE", path)
+
+ scoring := LoadSearchScoring()
+ if scoring.KeywordWeight != 0.7 || scoring.SemanticWeight != 0.3 {
+ t.Fatalf("search scoring = %+v", scoring)
+ }
+}
diff --git a/backend/internal/dburl/dburl.go b/backend/internal/dburl/dburl.go
new file mode 100644
index 0000000..e39500b
--- /dev/null
+++ b/backend/internal/dburl/dburl.go
@@ -0,0 +1,40 @@
+package dburl
+
+import (
+ "net"
+ "net/url"
+ "os"
+ "strings"
+)
+
+// FromEnv returns DATABASE_URL when explicitly set; otherwise it builds a
+// PostgreSQL URL from POSTGRES_* connection settings.
+func FromEnv() string {
+ if databaseURL := strings.TrimSpace(os.Getenv("DATABASE_URL")); databaseURL != "" {
+ return databaseURL
+ }
+ host := env("POSTGRES_HOST", "postgres")
+ port := env("POSTGRES_PORT", "5432")
+ database := env("POSTGRES_DB", "modex")
+ user := env("POSTGRES_USER", "modex")
+ password := env("POSTGRES_PASSWORD", "modex")
+ sslMode := env("POSTGRES_SSLMODE", "disable")
+
+ u := url.URL{
+ Scheme: "postgres",
+ User: url.UserPassword(user, password),
+ Host: net.JoinHostPort(host, port),
+ Path: "/" + database,
+ }
+ q := u.Query()
+ q.Set("sslmode", sslMode)
+ u.RawQuery = q.Encode()
+ return u.String()
+}
+
+func env(key, fallback string) string {
+ if value := strings.TrimSpace(os.Getenv(key)); value != "" {
+ return value
+ }
+ return fallback
+}
diff --git a/backend/internal/dburl/dburl_test.go b/backend/internal/dburl/dburl_test.go
new file mode 100644
index 0000000..594aa58
--- /dev/null
+++ b/backend/internal/dburl/dburl_test.go
@@ -0,0 +1,43 @@
+package dburl
+
+import (
+ "strings"
+ "testing"
+)
+
+func TestFromEnvPrefersDatabaseURL(t *testing.T) {
+ t.Setenv("DATABASE_URL", "postgres://custom:secret@db.example.com:6543/custom?sslmode=require")
+ t.Setenv("POSTGRES_HOST", "ignored")
+
+ if got := FromEnv(); got != "postgres://custom:secret@db.example.com:6543/custom?sslmode=require" {
+ t.Fatalf("FromEnv() = %q", got)
+ }
+}
+
+func TestFromEnvBuildsFromPostgresParts(t *testing.T) {
+ t.Setenv("POSTGRES_HOST", "10.0.0.5")
+ t.Setenv("POSTGRES_PORT", "15432")
+ t.Setenv("POSTGRES_DB", "modex_prod")
+ t.Setenv("POSTGRES_USER", "modex_user")
+ t.Setenv("POSTGRES_PASSWORD", "p@ss/word")
+ t.Setenv("POSTGRES_SSLMODE", "require")
+
+ got := FromEnv()
+ want := "postgres://modex_user:p%40ss%2Fword@10.0.0.5:15432/modex_prod?sslmode=require"
+ if got != want {
+ t.Fatalf("FromEnv() = %q, want %q", got, want)
+ }
+}
+
+func TestFromEnvDefaultsToComposePostgres(t *testing.T) {
+ got := FromEnv()
+ if !strings.HasPrefix(got, "postgres://modex:") {
+ t.Fatalf("FromEnv() = %q, want default modex credentials", got)
+ }
+ if !strings.Contains(got, "@postgres:5432/modex?") {
+ t.Fatalf("FromEnv() = %q, want default compose host and database", got)
+ }
+ if !strings.Contains(got, "sslmode=disable") {
+ t.Fatalf("FromEnv() = %q, want sslmode=disable", got)
+ }
+}
diff --git a/backend/internal/deploy/artifact.go b/backend/internal/deploy/artifact.go
index a30006a..6f84aec 100644
--- a/backend/internal/deploy/artifact.go
+++ b/backend/internal/deploy/artifact.go
@@ -61,6 +61,7 @@ type DocumentRecord struct {
Title string `json:"title"`
Description string `json:"description"`
Content string `json:"content"`
+ ContentMD string `json:"content_md,omitempty"`
Path string `json:"path"`
SourceFile string `json:"source_file"`
Keywords []string `json:"keywords"`
@@ -135,9 +136,6 @@ func ParseZip(r io.Reader, maxBytes int64) (Artifact, error) {
}
a.Documents = docs
}
- if strings.TrimSpace(a.Metadata.ModuleKey) == "" {
- return Artifact{}, fmt.Errorf("metadata.module_key is required")
- }
if strings.TrimSpace(a.Metadata.DocsVersion) == "" {
a.Metadata.DocsVersion = "latest"
}
diff --git a/backend/internal/embedding/provider.go b/backend/internal/embedding/provider.go
index 428a3ca..750057a 100644
--- a/backend/internal/embedding/provider.go
+++ b/backend/internal/embedding/provider.go
@@ -8,27 +8,39 @@ import (
"encoding/json"
"errors"
"fmt"
+ "io"
"net/http"
- "os"
+ "strings"
"time"
)
+// Dim is the system-wide embedding vector dimension. It must match the
+// docs_embedding.embedding column (vector(1024)) in schema.sql and the
+// admin settings UI. Changing it requires a column migration and a full
+// reindex, since pgvector columns are fixed-width and vectors of different
+// dimensions cannot coexist or be compared.
+const Dim = 1024
+
type Provider interface {
Name() string
EmbedText(ctx context.Context, text string) ([]float32, error)
EmbedBatch(ctx context.Context, texts []string) ([][]float32, error)
}
-type MockProvider struct {
+// FallbackProvider produces deterministic hash-based vectors when no real
+// embedding API is configured. The vectors carry no semantic meaning, so search
+// treats this provider as keyword-only; it exists so the system still runs
+// (ingest, indexing, keyword search) before an embedding provider is set up.
+type FallbackProvider struct {
Dim int
}
-func (p MockProvider) Name() string { return "mock" }
+func (p FallbackProvider) Name() string { return "fallback" }
-func (p MockProvider) EmbedText(ctx context.Context, text string) ([]float32, error) {
+func (p FallbackProvider) EmbedText(ctx context.Context, text string) ([]float32, error) {
dim := p.Dim
if dim <= 0 {
- dim = 384
+ dim = Dim
}
vec := make([]float32, dim)
seed := sha256.Sum256([]byte(text))
@@ -40,7 +52,7 @@ func (p MockProvider) EmbedText(ctx context.Context, text string) ([]float32, er
return vec, nil
}
-func (p MockProvider) EmbedBatch(ctx context.Context, texts []string) ([][]float32, error) {
+func (p FallbackProvider) EmbedBatch(ctx context.Context, texts []string) ([][]float32, error) {
out := make([][]float32, 0, len(texts))
for _, text := range texts {
vec, err := p.EmbedText(ctx, text)
@@ -52,37 +64,56 @@ func (p MockProvider) EmbedBatch(ctx context.Context, texts []string) ([][]float
return out, nil
}
-type HTTPProvider struct {
- URL string
- APIKey string
+// Settings is the admin-managed embedding configuration needed by Provider.
+type Settings struct {
+ BaseURL string
+ Model string
+ APIKey string
+ Dim int
+}
+
+// SettingsProvider resolves configuration for every request, so model changes
+// made in the admin page take effect without restarting the backend.
+type SettingsProvider struct {
+ Load func() Settings
Client *http.Client
}
-func (p HTTPProvider) Name() string { return "http" }
+func (p SettingsProvider) Name() string {
+ if cfg := p.settings(); cfg.BaseURL != "" && cfg.Model != "" {
+ return "admin"
+ }
+ return "fallback"
+}
-func (p HTTPProvider) EmbedText(ctx context.Context, text string) ([]float32, error) {
- batch, err := p.EmbedBatch(ctx, []string{text})
+func (p SettingsProvider) EmbedText(ctx context.Context, text string) ([]float32, error) {
+ vectors, err := p.EmbedBatch(ctx, []string{text})
if err != nil {
return nil, err
}
- if len(batch) == 0 {
+ if len(vectors) == 0 {
return nil, errors.New("embedding provider returned no vectors")
}
- return batch[0], nil
+ return vectors[0], nil
}
-func (p HTTPProvider) EmbedBatch(ctx context.Context, texts []string) ([][]float32, error) {
- if p.URL == "" {
- return nil, errors.New("EMBEDDING_HTTP_URL is empty")
+func (p SettingsProvider) EmbedBatch(ctx context.Context, texts []string) ([][]float32, error) {
+ cfg := p.settings()
+ if cfg.BaseURL == "" || cfg.Model == "" {
+ return FallbackProvider{Dim: cfg.Dim}.EmbedBatch(ctx, texts)
+ }
+ endpoint := strings.TrimRight(cfg.BaseURL, "/")
+ if !strings.HasSuffix(endpoint, "/embeddings") {
+ endpoint += "/embeddings"
}
- body, _ := json.Marshal(map[string]any{"texts": texts})
- req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.URL, bytes.NewReader(body))
+ body, _ := json.Marshal(map[string]any{"model": cfg.Model, "input": texts})
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
- if p.APIKey != "" {
- req.Header.Set("Authorization", "Bearer "+p.APIKey)
+ if cfg.APIKey != "" {
+ req.Header.Set("Authorization", "Bearer "+cfg.APIKey)
}
client := p.Client
if client == nil {
@@ -94,29 +125,40 @@ func (p HTTPProvider) EmbedBatch(ctx context.Context, texts []string) ([][]float
}
defer resp.Body.Close()
if resp.StatusCode >= 300 {
- return nil, fmt.Errorf("embedding http status %d", resp.StatusCode)
+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+ msg := strings.TrimSpace(string(body))
+ if msg == "" {
+ return nil, fmt.Errorf("embedding http status %d", resp.StatusCode)
+ }
+ return nil, fmt.Errorf("embedding http status %d: %s", resp.StatusCode, msg)
}
var decoded struct {
- Embeddings [][]float32 `json:"embeddings"`
+ Data []struct {
+ Embedding []float32 `json:"embedding"`
+ Index int `json:"index"`
+ } `json:"data"`
}
if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil {
return nil, err
}
- return decoded.Embeddings, nil
-}
-
-func FromEnv() Provider {
- dim := envInt("EMBEDDING_DIM", 384)
- if os.Getenv("EMBEDDING_PROVIDER") == "http" {
- return HTTPProvider{URL: os.Getenv("EMBEDDING_HTTP_URL"), APIKey: os.Getenv("EMBEDDING_HTTP_API_KEY")}
+ out := make([][]float32, len(decoded.Data))
+ for i, item := range decoded.Data {
+ idx := item.Index
+ if idx < 0 || idx >= len(out) {
+ idx = i
+ }
+ out[idx] = item.Embedding
}
- return MockProvider{Dim: dim}
+ return out, nil
}
-func envInt(key string, fallback int) int {
- var v int
- if _, err := fmt.Sscanf(os.Getenv(key), "%d", &v); err == nil && v > 0 {
- return v
+func (p SettingsProvider) settings() Settings {
+ if p.Load == nil {
+ return Settings{Dim: Dim}
+ }
+ cfg := p.Load()
+ if cfg.Dim <= 0 {
+ cfg.Dim = Dim
}
- return fallback
+ return cfg
}
diff --git a/backend/internal/embedding/provider_test.go b/backend/internal/embedding/provider_test.go
new file mode 100644
index 0000000..4a5a73e
--- /dev/null
+++ b/backend/internal/embedding/provider_test.go
@@ -0,0 +1,49 @@
+package embedding
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+)
+
+func TestSettingsProviderUsesCurrentAdminSettings(t *testing.T) {
+ var requestedModel string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/v1/embeddings" {
+ t.Fatalf("path = %q, want /v1/embeddings", r.URL.Path)
+ }
+ var body struct {
+ Model string `json:"model"`
+ Input []string `json:"input"`
+ }
+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+ t.Fatal(err)
+ }
+ requestedModel = body.Model
+ _ = json.NewEncoder(w).Encode(map[string]any{
+ "data": []any{map[string]any{"index": 0, "embedding": []float32{0.1, 0.2}}},
+ })
+ }))
+ defer server.Close()
+
+ cfg := Settings{Dim: 3}
+ provider := SettingsProvider{Load: func() Settings { return cfg }}
+ if provider.Name() != "fallback" {
+ t.Fatalf("provider name = %q, want fallback", provider.Name())
+ }
+ fallbackVector, err := provider.EmbedText(context.Background(), "hello")
+ if err != nil || len(fallbackVector) != 3 {
+ t.Fatalf("fallback vector len = %d, err = %v", len(fallbackVector), err)
+ }
+
+ cfg = Settings{BaseURL: server.URL + "/v1", Model: "embed-v2", APIKey: "secret"}
+ vector, err := provider.EmbedText(context.Background(), "hello")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if provider.Name() != "admin" || requestedModel != "embed-v2" || len(vector) != 2 {
+ t.Fatalf("name=%q model=%q vector=%v", provider.Name(), requestedModel, vector)
+ }
+}
diff --git a/backend/internal/redisurl/redisurl.go b/backend/internal/redisurl/redisurl.go
new file mode 100644
index 0000000..1174c4d
--- /dev/null
+++ b/backend/internal/redisurl/redisurl.go
@@ -0,0 +1,47 @@
+package redisurl
+
+import (
+ "net"
+ "net/url"
+ "os"
+ "strings"
+)
+
+// FromEnv returns REDIS_URL when explicitly set. When REDIS_URL is empty it
+// builds a Redis URL from REDIS_HOST/REDIS_PORT/REDIS_DB/REDIS_USER/REDIS_PASSWORD.
+// If REDIS_HOST is not set, Redis remains disabled and callers can fall back.
+func FromEnv() string {
+ if redisURL := strings.TrimSpace(os.Getenv("REDIS_URL")); redisURL != "" {
+ return redisURL
+ }
+ host := strings.TrimSpace(os.Getenv("REDIS_HOST"))
+ if host == "" {
+ return ""
+ }
+ port := env("REDIS_PORT", "6379")
+ db := strings.Trim(strings.TrimSpace(env("REDIS_DB", "0")), "/")
+ user := strings.TrimSpace(os.Getenv("REDIS_USER"))
+ password := os.Getenv("REDIS_PASSWORD")
+
+ u := url.URL{
+ Scheme: "redis",
+ Host: net.JoinHostPort(host, port),
+ Path: "/" + db,
+ }
+ switch {
+ case user != "" && password != "":
+ u.User = url.UserPassword(user, password)
+ case user != "":
+ u.User = url.User(user)
+ case password != "":
+ u.User = url.UserPassword("", password)
+ }
+ return u.String()
+}
+
+func env(key, fallback string) string {
+ if value := strings.TrimSpace(os.Getenv(key)); value != "" {
+ return value
+ }
+ return fallback
+}
diff --git a/backend/internal/redisurl/redisurl_test.go b/backend/internal/redisurl/redisurl_test.go
new file mode 100644
index 0000000..1bdb712
--- /dev/null
+++ b/backend/internal/redisurl/redisurl_test.go
@@ -0,0 +1,43 @@
+package redisurl
+
+import "testing"
+
+func TestFromEnvPrefersRedisURL(t *testing.T) {
+ t.Setenv("REDIS_URL", "redis://custom:6380/2")
+ t.Setenv("REDIS_HOST", "ignored")
+
+ if got := FromEnv(); got != "redis://custom:6380/2" {
+ t.Fatalf("FromEnv() = %q", got)
+ }
+}
+
+func TestFromEnvBuildsFromRedisParts(t *testing.T) {
+ t.Setenv("REDIS_HOST", "redis.internal")
+ t.Setenv("REDIS_PORT", "6380")
+ t.Setenv("REDIS_DB", "3")
+ t.Setenv("REDIS_USER", "modex")
+ t.Setenv("REDIS_PASSWORD", "p@ss/word")
+
+ got := FromEnv()
+ want := "redis://modex:p%40ss%2Fword@redis.internal:6380/3"
+ if got != want {
+ t.Fatalf("FromEnv() = %q, want %q", got, want)
+ }
+}
+
+func TestFromEnvBuildsPasswordOnlyURL(t *testing.T) {
+ t.Setenv("REDIS_HOST", "redis")
+ t.Setenv("REDIS_PASSWORD", "secret")
+
+ got := FromEnv()
+ want := "redis://:secret@redis:6379/0"
+ if got != want {
+ t.Fatalf("FromEnv() = %q, want %q", got, want)
+ }
+}
+
+func TestFromEnvDisabledWithoutHost(t *testing.T) {
+ if got := FromEnv(); got != "" {
+ t.Fatalf("FromEnv() = %q, want empty", got)
+ }
+}
diff --git a/backend/internal/repository/postgres.go b/backend/internal/repository/postgres.go
new file mode 100644
index 0000000..0b84e0c
--- /dev/null
+++ b/backend/internal/repository/postgres.go
@@ -0,0 +1,16 @@
+package repository
+
+import (
+ "context"
+
+ "modex/backend/internal/store"
+)
+
+// PostgresRepository is the formal-table business repository used in
+// production. It is exposed from this package so application assembly does not
+// depend on store implementation details directly.
+type PostgresRepository = store.PostgresRepository
+
+func OpenPostgres(ctx context.Context, databaseURL string) (*PostgresRepository, error) {
+ return store.OpenPostgresRepository(ctx, databaseURL)
+}
diff --git a/backend/internal/search/search.go b/backend/internal/search/search.go
index a2877a9..fe6567f 100644
--- a/backend/internal/search/search.go
+++ b/backend/internal/search/search.go
@@ -1,15 +1,25 @@
package search
import (
+ "bytes"
"context"
+ "encoding/json"
+ "fmt"
+ "log"
"math"
+ "net/http"
+ "regexp"
"sort"
"strings"
+ "time"
"modex/backend/internal/embedding"
"modex/backend/internal/store"
)
+const defaultEmbeddingInputRunes = 800
+const maxEmbeddingChunksPerDoc = 512
+
type Mode string
const (
@@ -34,6 +44,14 @@ type Request struct {
Filters Filters `json:"filters"`
Page int `json:"page"`
PageSize int `json:"page_size"`
+ // DefaultVersionsOnly limits results to each module's configured default
+ // docs version when the caller has not explicitly requested versions.
+ DefaultVersionsOnly bool `json:"default_versions_only"`
+ // Log marks an explicit, user-committed search (Enter / search button /
+ // result click) that should be persisted to the search log. Live
+ // as-you-type queries leave this false so the log isn't flooded.
+ Log bool `json:"log"`
+ ClickedDocID string `json:"clicked_doc_id"`
}
type Result struct {
@@ -67,9 +85,22 @@ type Response struct {
Facets map[string]map[string]int `json:"facets"`
}
+// ContentStore is the search-facing data boundary. Production uses the
+// PostgreSQL repository; tests may use MemoryStore as an explicit fake.
+type ContentStore interface {
+ Pages() []store.Page
+ Modules(categoryID, keyword string) []store.Module
+ Settings() store.Settings
+ Embedding(docID string) ([]float32, bool)
+ SetEmbedding(docID string, vector []float32)
+ ClearEmbeddings()
+ EmbeddingCount() int
+}
+
type Service struct {
- Store *store.Store
+ Store ContentStore
Embedder embedding.Provider
+ Vectors VectorStore
KeywordWeight float64
SemanticWeight float64
}
@@ -95,25 +126,71 @@ func (s Service) Search(ctx context.Context, req Request) (Response, error) {
pages := s.Store.Pages()
// Resolve module -> category breadcrumb once so each result carries a path.
breadcrumbs := map[string]string{}
+ defaultVersions := map[string]string{}
for _, m := range s.Store.Modules("", "") {
breadcrumbs[m.ModuleKey] = m.CategoryPath
+ defaultVersions[m.ModuleKey] = defaultVersionForModule(m)
+ }
+ candidates := make([]store.Page, 0, len(pages))
+ for _, p := range pages {
+ if !matchFilters(p, req.Filters) {
+ continue
+ }
+ if req.DefaultVersionsOnly && len(req.Filters.DocsVersions) == 0 {
+ if def := defaultVersions[p.ModuleKey]; def != "" && p.DocsVersion != def {
+ continue
+ }
+ }
+ candidates = append(candidates, p)
}
terms := matchTerms(req.Query)
// Semantic and hybrid modes need a query embedding; keyword mode skips it to
// avoid an unnecessary embedding-provider call.
var queryVec []float32
if req.Mode != ModeKeyword && strings.TrimSpace(req.Query) != "" {
- queryVec, _ = s.Embedder.EmbedText(ctx, req.Query)
+ var err error
+ queryVec, err = s.Embedder.EmbedText(ctx, s.truncateEmbeddingInput(req.Query))
+ if err != nil {
+ return Response{}, fmt.Errorf("embed query: %w", err)
+ }
}
- var scored []Result
- for _, p := range pages {
- if !matchFilters(p, req.Filters) {
- continue
+ // The fallback provider returns hash-based vectors (no real semantics). In
+ // hybrid mode that noise floats weak docs up and buries strong keyword hits
+ // (e.g. an "EventBus" page for the query "EventBus怎么用"), so when embeddings
+ // come from the fallback we score hybrid by keyword only. Explicit semantic
+ // mode still uses the vectors — the caller asked for vector ranking.
+ fallbackEmbed := s.Embedder.Name() == "fallback"
+ vectors := map[string][]float32{}
+ semanticScores := map[string]float64{}
+ if len(queryVec) > 0 {
+ docIDs := make([]string, 0, len(candidates))
+ for _, p := range candidates {
+ docIDs = append(docIDs, p.DocID)
+ }
+ if s.Vectors != nil {
+ var err error
+ semanticScores, err = s.Vectors.Similarities(ctx, queryVec, docIDs, len(docIDs))
+ if err != nil {
+ return Response{}, err
+ }
+ } else {
+ vectors, _ = s.embeddingBatch(docIDs)
}
+ }
+ var scored []Result
+ for _, p := range candidates {
kScore := keywordScore(req.Query, p)
var sScore float64
if len(queryVec) > 0 {
- sScore = cosine(queryVec, s.pageVector(ctx, p))
+ if s.Vectors != nil {
+ sScore = semanticScores[p.DocID]
+ } else {
+ var err error
+ sScore, err = s.pageSemanticScore(ctx, queryVec, p, vectors)
+ if err != nil {
+ return Response{}, err
+ }
+ }
}
var final float64
switch req.Mode {
@@ -122,7 +199,11 @@ func (s Service) Search(ctx context.Context, req Request) (Response, error) {
case ModeSemantic:
final = sScore
default:
- final = kScore*kw + sScore*sw
+ if fallbackEmbed {
+ final = kScore
+ } else {
+ final = kScore*kw + sScore*sw
+ }
req.Mode = ModeHybrid
}
if strings.TrimSpace(req.Query) != "" && final <= 0 {
@@ -135,7 +216,7 @@ func (s Service) Search(ctx context.Context, req Request) (Response, error) {
breadcrumb = p.ModuleName
}
scored = append(scored, Result{
- DocID: p.DocID, Title: p.Title, Snippet: snippet(req.Query, p.ContentText), Path: p.Path,
+ DocID: p.DocID, Title: plainText(p.Title), Snippet: snippet(req.Query, plainText(p.ContentText)), Path: p.Path,
Score: final, SearchMode: req.Mode, ModuleKey: p.ModuleKey, ModuleName: p.ModuleName,
DocsVersion: p.DocsVersion, PackageVersion: p.PackageVersion, EntryType: p.EntryType,
OwnerGroup: p.OwnerGroup, Status: p.Status, UpdatedAt: p.UpdatedAt.Format("2006-01-02T15:04:05Z07:00"),
@@ -143,6 +224,11 @@ func (s Service) Search(ctx context.Context, req Request) (Response, error) {
})
}
sort.Slice(scored, func(i, j int) bool { return scored[i].Score > scored[j].Score })
+ if reranked, err := s.rerank(ctx, req.Query, scored); err == nil {
+ scored = reranked
+ } else {
+ log.Printf("search rerank skipped: %v", err)
+ }
total := len(scored)
start := (req.Page - 1) * req.PageSize
if start > len(scored) {
@@ -155,6 +241,90 @@ func (s Service) Search(ctx context.Context, req Request) (Response, error) {
return Response{Query: req.Query, Mode: req.Mode, Page: req.Page, PageSize: req.PageSize, Total: total, Results: scored[start:end], Facets: facets(pages)}, nil
}
+func (s Service) rerank(ctx context.Context, query string, results []Result) ([]Result, error) {
+ ai := s.Store.Settings().AI
+ if strings.TrimSpace(query) == "" || ai.RerankBaseURL == "" || ai.RerankModel == "" || len(results) < 2 {
+ return results, nil
+ }
+ topK := ai.RerankTopK
+ if topK <= 0 || topK > len(results) {
+ topK = len(results)
+ }
+ documents := make([]string, topK)
+ for i := 0; i < topK; i++ {
+ documents[i] = results[i].Title + "\n" + results[i].Snippet
+ }
+ payload, _ := json.Marshal(map[string]any{
+ "model": ai.RerankModel, "query": query, "documents": documents, "top_n": topK,
+ })
+ endpoint := strings.TrimRight(ai.RerankBaseURL, "/")
+ if !strings.HasSuffix(endpoint, "/rerank") {
+ endpoint += "/rerank"
+ }
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(payload))
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("Content-Type", "application/json")
+ if ai.RerankAPIKey != "" {
+ req.Header.Set("Authorization", "Bearer "+ai.RerankAPIKey)
+ }
+ resp, err := (&http.Client{Timeout: 20 * time.Second}).Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode >= 300 {
+ return nil, fmt.Errorf("rerank http status %d", resp.StatusCode)
+ }
+ var decoded struct {
+ Results []struct {
+ Index int `json:"index"`
+ RelevanceScore float64 `json:"relevance_score"`
+ } `json:"results"`
+ }
+ if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil {
+ return nil, err
+ }
+ if len(decoded.Results) == 0 {
+ return nil, fmt.Errorf("rerank provider returned no results")
+ }
+ reranked := make([]Result, 0, len(results))
+ seen := make(map[int]bool, topK)
+ for _, item := range decoded.Results {
+ if item.Index < 0 || item.Index >= topK || seen[item.Index] {
+ continue
+ }
+ result := results[item.Index]
+ result.Score = item.RelevanceScore
+ reranked = append(reranked, result)
+ seen[item.Index] = true
+ }
+ for i := 0; i < topK; i++ {
+ if !seen[i] {
+ reranked = append(reranked, results[i])
+ }
+ }
+ return append(reranked, results[topK:]...), nil
+}
+
+func defaultVersionForModule(m store.Module) string {
+ for _, v := range m.AvailableVers {
+ if v.IsDefault && v.Status != "archived" {
+ return v.DocsVersion
+ }
+ }
+ if strings.TrimSpace(m.DefaultVersion) != "" {
+ return m.DefaultVersion
+ }
+ for _, v := range m.AvailableVers {
+ if v.Status != "archived" {
+ return v.DocsVersion
+ }
+ }
+ return ""
+}
+
func matchFilters(p store.Page, f Filters) bool {
return inAny(p.CategoryIDs, f.CategoryIDs) &&
inValue(p.ModuleKey, f.Modules) &&
@@ -165,8 +335,33 @@ func matchFilters(p store.Page, f Filters) bool {
inValue(p.Status, f.Status)
}
+// queryTokenRe splits a query into searchable tokens: runs of ASCII
+// letters/digits and runs of CJK (Han) characters. Plain whitespace splitting
+// fails on mixed input like "EventBus怎么用" (no spaces), collapsing it to one
+// token that matches nothing; this separates it into "eventbus" + "怎么用".
+var queryTokenRe = regexp.MustCompile(`[a-z0-9]+|\p{Han}+`)
+
+func queryTokens(query string) []string {
+ var out []string
+ for _, tok := range queryTokenRe.FindAllString(strings.ToLower(query), -1) {
+ r := []rune(tok)
+ // ASCII alnum run: keep whole (e.g. "eventbus").
+ if r[0] < 128 || len(r) == 1 {
+ out = append(out, tok)
+ continue
+ }
+ // CJK run: emit 2-char shingles so a query like "如何下载插件" matches docs
+ // containing "下载"/"插件" without a full word segmenter. Single chars would
+ // over-match; whole runs (the previous behaviour) under-match.
+ for i := 0; i+1 < len(r); i++ {
+ out = append(out, string(r[i:i+2]))
+ }
+ }
+ return out
+}
+
func keywordScore(query string, p store.Page) float64 {
- q := strings.Fields(strings.ToLower(query))
+ q := queryTokens(query)
if len(q) == 0 {
return 1
}
@@ -189,48 +384,194 @@ func keywordScore(query string, p store.Page) float64 {
return score / float64(len(q)*4)
}
-// Reindex (re)computes and caches an embedding for every page using the
+// Reindex (re)computes and caches embeddings for every page chunk using the
// configured provider. It powers POST /api/embeddings/reindex and pre-warms the
// cache so semantic search does not pay an embedding call on the hot path.
func (s Service) Reindex(ctx context.Context) (int, error) {
- s.Store.ClearEmbeddings()
- pages := s.Store.Pages()
+ if err := s.clearEmbeddings(ctx); err != nil {
+ return 0, err
+ }
+ return s.reindexPages(ctx, s.Store.Pages())
+}
+
+func (s Service) ReindexModuleVersion(ctx context.Context, moduleKey, docsVersion string) (int, error) {
+ prefix := moduleKey + ":" + docsVersion + ":"
+ if err := s.DeleteModuleVersionEmbeddings(ctx, moduleKey, docsVersion); err != nil {
+ return 0, err
+ }
+ pages := make([]store.Page, 0)
+ for _, p := range s.Store.Pages() {
+ if strings.HasPrefix(p.DocID, prefix) {
+ pages = append(pages, p)
+ }
+ }
+ return s.reindexPages(ctx, pages)
+}
+
+func (s Service) reindexPages(ctx context.Context, pages []store.Page) (int, error) {
count := 0
for _, p := range pages {
- text := embedText(p)
- if text == "" {
+ for _, chunk := range s.embeddingChunks(p) {
+ vec, err := s.Embedder.EmbedText(ctx, chunk.Content)
+ if err != nil {
+ return count, fmt.Errorf("embed page %s chunk %s: %w", p.DocID, chunk.ID, err)
+ }
+ if err := s.upsertEmbedding(ctx, p.DocID, chunk.ID, chunk.Content, vec); err != nil {
+ return count, err
+ }
+ count++
+ }
+ }
+ return count, nil
+}
+
+// pageSemanticScore returns the best precomputed chunk score for the page.
+// Missing embeddings are skipped; indexing is done by Reindex/ReindexModuleVersion
+// during document publish or explicit admin maintenance, not on the search path.
+func (s Service) pageSemanticScore(ctx context.Context, queryVec []float32, p store.Page, vectors map[string][]float32) (float64, error) {
+ best := 0.0
+ for _, chunk := range s.embeddingChunks(p) {
+ vec, ok := vectors[chunk.ID]
+ if !ok {
continue
}
- vec, err := s.Embedder.EmbedText(ctx, text)
- if err != nil {
- return count, err
+ if score := cosine(queryVec, vec); score > best {
+ best = score
}
- s.Store.SetEmbedding(p.DocID, vec)
- count++
}
- return count, nil
+ return best, nil
+}
+
+func (s Service) embeddingBatch(docIDs []string) (map[string][]float32, error) {
+ out := make(map[string][]float32, len(docIDs))
+ for _, p := range s.Store.Pages() {
+ for _, chunk := range s.embeddingChunks(p) {
+ if vector, ok := s.Store.Embedding(chunk.ID); ok {
+ out[chunk.ID] = vector
+ }
+ }
+ }
+ return out, nil
+}
+
+func (s Service) upsertEmbedding(ctx context.Context, docID, chunkID, content string, vector []float32) error {
+ if s.Vectors != nil {
+ return s.Vectors.UpsertChunk(ctx, docID, chunkID, content, vector)
+ }
+ s.Store.SetEmbedding(chunkID, vector)
+ return nil
}
-// pageVector returns the page's embedding from cache, computing and caching it
-// on demand the first time it is needed.
-func (s Service) pageVector(ctx context.Context, p store.Page) []float32 {
- if vec, ok := s.Store.Embedding(p.DocID); ok {
- return vec
+func (s Service) clearEmbeddings(ctx context.Context) error {
+ if s.Vectors != nil {
+ return s.Vectors.Clear(ctx)
}
- text := embedText(p)
+ s.Store.ClearEmbeddings()
+ return nil
+}
+
+func (s Service) EmbeddingCount(ctx context.Context) (int, error) {
+ if s.Vectors != nil {
+ return s.Vectors.Count(ctx)
+ }
+ return s.Store.EmbeddingCount(), nil
+}
+
+func (s Service) DeleteModuleVersionEmbeddings(ctx context.Context, moduleKey, docsVersion string) error {
+ prefix := moduleKey + ":" + docsVersion + ":"
+ if s.Vectors != nil {
+ return s.Vectors.DeletePrefix(ctx, prefix)
+ }
+ return nil
+}
+
+type embeddingChunk struct {
+ ID string
+ Content string
+}
+
+func (s Service) embeddingChunks(p store.Page) []embeddingChunk {
+ text := plainText(strings.TrimSpace(p.ContentText))
+ prefix := strings.TrimSpace(p.Title + "\n" + p.Description)
if text == "" {
- return nil
+ text = prefix
}
- vec, err := s.Embedder.EmbedText(ctx, text)
- if err != nil {
+ if strings.TrimSpace(text) == "" {
return nil
}
- s.Store.SetEmbedding(p.DocID, vec)
- return vec
+ limit := s.embeddingInputLimit()
+ overlap := s.Store.Settings().AI.ChunkOverlap
+ if overlap < 0 {
+ overlap = 0
+ }
+ if overlap >= limit {
+ overlap = limit / 5
+ }
+ prefixRunes := []rune(prefix)
+ if len(prefixRunes) > limit/4 {
+ prefix = string(prefixRunes[:limit/4])
+ prefixRunes = []rune(prefix)
+ }
+ bodyLimit := limit
+ if prefix != "" {
+ bodyLimit = limit - len(prefixRunes) - 1
+ if bodyLimit <= 0 {
+ bodyLimit = limit
+ prefix = ""
+ }
+ }
+ bodyRunes := []rune(text)
+ step := bodyLimit - overlap
+ if step <= 0 {
+ step = bodyLimit
+ }
+ chunks := make([]embeddingChunk, 0, (len(bodyRunes)/step)+1)
+ for start, index := 0, 0; start < len(bodyRunes) && index < maxEmbeddingChunksPerDoc; index++ {
+ end := start + bodyLimit
+ if end > len(bodyRunes) {
+ end = len(bodyRunes)
+ }
+ content := strings.TrimSpace(string(bodyRunes[start:end]))
+ if prefix != "" && content != prefix {
+ content = strings.TrimSpace(prefix + "\n" + content)
+ }
+ content = s.truncateEmbeddingInput(content)
+ if content != "" {
+ chunks = append(chunks, embeddingChunk{
+ ID: fmt.Sprintf("%s#chunk-%04d", p.DocID, index),
+ Content: content,
+ })
+ }
+ if end == len(bodyRunes) {
+ break
+ }
+ start += step
+ }
+ return chunks
}
-func embedText(p store.Page) string {
- return strings.TrimSpace(p.Title + "\n" + p.Description + "\n" + p.ContentText)
+func (s Service) embeddingInputLimit() int {
+ limit := s.Store.Settings().AI.ChunkSize
+ if limit <= 0 {
+ return defaultEmbeddingInputRunes
+ }
+ return limit
+}
+
+func (s Service) truncateEmbeddingInput(text string) string {
+ text = strings.TrimSpace(text)
+ if text == "" {
+ return ""
+ }
+ limit := s.embeddingInputLimit()
+ if limit <= 0 {
+ limit = defaultEmbeddingInputRunes
+ }
+ runes := []rune(text)
+ if len(runes) <= limit {
+ return text
+ }
+ return string(runes[:limit])
}
// cosine returns a 0..1 similarity (cosine distance shifted into [0,1]).
@@ -250,6 +591,44 @@ func cosine(a, b []float32) float64 {
return (dot/math.Sqrt(an*bn) + 1) / 2
}
+// Markdown-stripping patterns for plainText. Applied in order so links/images
+// resolve to their text before stray emphasis markers are removed.
+var (
+ mdCodeFence = regexp.MustCompile("(?s)```.*?```")
+ mdImage = regexp.MustCompile(`!\[([^\]]*)\]\([^)]*\)`)
+ mdLink = regexp.MustCompile(`\[([^\]]*)\]\([^)]*\)`)
+ mdInlineCode = regexp.MustCompile("`([^`]*)`")
+ mdTableSep = regexp.MustCompile(`(?m)^\s*\|?[\s:|-]{3,}\|?\s*$`)
+ mdHeadingHash = regexp.MustCompile(`(?m)(^|\s)#{1,6}\s+`)
+ mdBlockquote = regexp.MustCompile(`(?m)^\s{0,3}>\s?`)
+ mdListMarker = regexp.MustCompile(`(?m)^\s{0,3}([-*+]|\d+\.)\s+`)
+ mdHTMLTag = regexp.MustCompile(`<[^>]+>`)
+ mdEmphasis = regexp.MustCompile(`[*_~]{1,3}`)
+ mdWhitespace = regexp.MustCompile(`\s+`)
+)
+
+// plainText strips common Markdown syntax so search titles and snippets read as
+// clean prose instead of raw markup (** , #, [text](url), ``` ... ```, tables).
+// It is display-only; keyword scoring and embeddings still use the raw content.
+func plainText(s string) string {
+ if s == "" {
+ return ""
+ }
+ s = mdCodeFence.ReplaceAllString(s, " ")
+ s = mdImage.ReplaceAllString(s, "$1")
+ s = mdLink.ReplaceAllString(s, "$1")
+ s = mdInlineCode.ReplaceAllString(s, "$1")
+ s = mdTableSep.ReplaceAllString(s, " ")
+ s = mdHeadingHash.ReplaceAllString(s, "$1")
+ s = mdBlockquote.ReplaceAllString(s, "")
+ s = mdListMarker.ReplaceAllString(s, "")
+ s = mdHTMLTag.ReplaceAllString(s, "")
+ s = mdEmphasis.ReplaceAllString(s, "")
+ s = strings.ReplaceAll(s, "|", " ")
+ s = mdWhitespace.ReplaceAllString(s, " ")
+ return strings.TrimSpace(s)
+}
+
// snippet returns a rune-safe excerpt centered on the first matched term, with
// surrounding context before/after and ellipses, so callers can highlight the
// matched keywords. It never splits a multibyte (e.g. CJK) character.
@@ -298,8 +677,7 @@ func snippet(query, content string) string {
func matchTerms(query string) []string {
seen := map[string]bool{}
var out []string
- for _, f := range strings.Fields(strings.ToLower(query)) {
- f = strings.Trim(f, ",.;:!?，。、；：！？\"'")
+ for _, f := range queryTokens(query) {
if f == "" || seen[f] {
continue
}
diff --git a/backend/internal/search/search_test.go b/backend/internal/search/search_test.go
index 2073129..0077799 100644
--- a/backend/internal/search/search_test.go
+++ b/backend/internal/search/search_test.go
@@ -2,16 +2,234 @@ package search
import (
"context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "strings"
"testing"
"modex/backend/internal/embedding"
"modex/backend/internal/store"
)
+type tokenEmbedder struct {
+ token string
+}
+
+func (e tokenEmbedder) Name() string { return "token-test" }
+
+func (e tokenEmbedder) EmbedText(_ context.Context, text string) ([]float32, error) {
+ if strings.Contains(text, e.token) {
+ return []float32{1, 0}, nil
+ }
+ return []float32{0, 1}, nil
+}
+
+func (e tokenEmbedder) EmbedBatch(ctx context.Context, texts []string) ([][]float32, error) {
+ out := make([][]float32, len(texts))
+ for i, text := range texts {
+ vec, err := e.EmbedText(ctx, text)
+ if err != nil {
+ return nil, err
+ }
+ out[i] = vec
+ }
+ return out, nil
+}
+
+func TestRerankUsesAdminSettings(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/v1/rerank" {
+ t.Fatalf("path = %q, want /v1/rerank", r.URL.Path)
+ }
+ if got := r.Header.Get("Authorization"); got != "Bearer secret" {
+ t.Fatalf("authorization = %q", got)
+ }
+ _ = json.NewEncoder(w).Encode(map[string]any{"results": []any{
+ map[string]any{"index": 1, "relevance_score": 0.9},
+ map[string]any{"index": 0, "relevance_score": 0.2},
+ }})
+ }))
+ defer server.Close()
+
+ st := store.NewTestStore()
+ st.SaveAISettings(store.AISettings{
+ RerankBaseURL: server.URL + "/v1",
+ RerankModel: "rerank-v1",
+ RerankAPIKey: "secret",
+ RerankTopK: 2,
+ })
+ results, err := (Service{Store: st}).rerank(context.Background(), "query", []Result{
+ {DocID: "first", Title: "First"},
+ {DocID: "second", Title: "Second"},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if results[0].DocID != "second" || results[0].Score != 0.9 {
+ t.Fatalf("results = %#v", results)
+ }
+}
+
+func TestSearchFallsBackWhenRerankProviderFails(t *testing.T) {
+ server := httptest.NewServer(http.NotFoundHandler())
+ defer server.Close()
+
+ st := store.NewSeededTestStore()
+ st.SaveAISettings(store.AISettings{
+ RerankBaseURL: server.URL + "/v1",
+ RerankModel: "rerank-v1",
+ RerankTopK: 2,
+ })
+ s := Service{Store: st, Embedder: embedding.FallbackProvider{Dim: 256}, KeywordWeight: 0.6, SemanticWeight: 0.4}
+
+ resp, err := s.Search(context.Background(), Request{Query: "构建缓存", Mode: ModeKeyword, PageSize: 5})
+ if err != nil {
+ t.Fatalf("Search should fall back when rerank fails: %v", err)
+ }
+ if len(resp.Results) == 0 {
+ t.Fatal("expected search results without rerank")
+ }
+}
+
+func TestEmbeddingInputUsesChunkSizeBudget(t *testing.T) {
+ long := strings.Repeat("测", 1000)
+ defaultSvc := Service{Store: store.NewTestStore()}
+ if got := []rune(defaultSvc.truncateEmbeddingInput(long)); len(got) != 800 {
+ t.Fatalf("default truncated length = %d, want 800", len(got))
+ }
+
+ st := store.NewTestStore()
+ st.SaveAISettings(store.AISettings{ChunkSize: 320})
+ customSvc := Service{Store: st}
+ chunks := customSvc.embeddingChunks(store.Page{DocID: "doc", Title: "标题", ContentText: long})
+ if len(chunks) < 2 {
+ t.Fatalf("chunk count = %d, want multiple chunks", len(chunks))
+ }
+ if got := []rune(chunks[0].Content); len(got) != 320 {
+ t.Fatalf("custom truncated length = %d, want 320", len(got))
+ }
+}
+
+func TestEmbeddingChunksCoverWholeDocumentWithOverlap(t *testing.T) {
+ st := store.NewTestStore()
+ st.SaveAISettings(store.AISettings{ChunkSize: 10, ChunkOverlap: 2})
+ svc := Service{Store: st}
+ chunks := svc.embeddingChunks(store.Page{DocID: "doc", ContentText: "0123456789abcdefghij"})
+ if len(chunks) != 3 {
+ t.Fatalf("chunk count = %d, want 3", len(chunks))
+ }
+ if chunks[0].Content != "0123456789" || chunks[1].Content != "89abcdefgh" || chunks[2].Content != "ghij" {
+ t.Fatalf("chunks = %#v", chunks)
+ }
+}
+
+func TestSemanticSearchUsesPrebuiltLaterChunks(t *testing.T) {
+ st := store.NewTestStore()
+ st.SaveAISettings(store.AISettings{ChunkSize: 20})
+ _, err := st.IngestArtifact(store.DeployArtifact{
+ ModuleKey: "LongDocs",
+ ModuleName: "LongDocs",
+ DocsVersion: "latest",
+ Entries: []store.DeployEntry{{Key: "guide", Title: "Guide", Type: "markdown"}},
+ Documents: []store.DeployDocument{{
+ DocID: "LongDocs:latest:guide",
+ EntryKey: "guide",
+ EntryType: "markdown",
+ Title: "Long Guide",
+ Content: strings.Repeat("a", 60) + " needle",
+ Status: "active",
+ }},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ s := Service{Store: st, Embedder: tokenEmbedder{token: "needle"}}
+ if count, err := s.ReindexModuleVersion(context.Background(), "LongDocs", "latest"); err != nil {
+ t.Fatalf("ReindexModuleVersion: %v", err)
+ } else if count < 2 {
+ t.Fatalf("reindexed chunks = %d, want multiple chunks", count)
+ }
+ before := st.EmbeddingCount()
+ resp, err := s.Search(context.Background(), Request{Query: "needle", Mode: ModeSemantic, PageSize: 5})
+ if err != nil {
+ t.Fatalf("Search: %v", err)
+ }
+ if len(resp.Results) != 1 || resp.Results[0].DocID != "LongDocs:latest:guide" {
+ t.Fatalf("results = %#v", resp.Results)
+ }
+ if st.EmbeddingCount() != before {
+ t.Fatalf("search changed embedding count from %d to %d", before, st.EmbeddingCount())
+ }
+}
+
+type fakeVectorStore struct {
+ chunks map[string]fakeChunkVector
+ simCalls int
+}
+
+type fakeChunkVector struct {
+ docID string
+ vector []float32
+}
+
+func (f *fakeVectorStore) Existing(_ context.Context, docIDs []string) (map[string]bool, error) {
+ out := map[string]bool{}
+ wanted := map[string]bool{}
+ for _, id := range docIDs {
+ wanted[id] = true
+ }
+ for _, chunk := range f.chunks {
+ if wanted[chunk.docID] {
+ out[chunk.docID] = true
+ }
+ }
+ return out, nil
+}
+
+func (f *fakeVectorStore) Similarities(_ context.Context, query []float32, docIDs []string, _ int) (map[string]float64, error) {
+ f.simCalls++
+ out := map[string]float64{}
+ wanted := map[string]bool{}
+ for _, id := range docIDs {
+ wanted[id] = true
+ }
+ for _, chunk := range f.chunks {
+ if !wanted[chunk.docID] {
+ continue
+ }
+ if score := cosine(query, chunk.vector); score > out[chunk.docID] {
+ out[chunk.docID] = score
+ }
+ }
+ return out, nil
+}
+
+func (f *fakeVectorStore) UpsertChunk(_ context.Context, docID, chunkID, _ string, vector []float32) error {
+ f.chunks[chunkID] = fakeChunkVector{docID: docID, vector: append([]float32(nil), vector...)}
+ return nil
+}
+
+func (f *fakeVectorStore) Clear(context.Context) error {
+ f.chunks = map[string]fakeChunkVector{}
+ return nil
+}
+
+func (f *fakeVectorStore) DeletePrefix(_ context.Context, prefix string) error {
+ for id, chunk := range f.chunks {
+ if strings.HasPrefix(chunk.docID, prefix) {
+ delete(f.chunks, id)
+ }
+ }
+ return nil
+}
+
+func (f *fakeVectorStore) Count(context.Context) (int, error) { return len(f.chunks), nil }
+
func newService() Service {
return Service{
- Store: store.NewSeeded(),
- Embedder: embedding.MockProvider{Dim: 256},
+ Store: store.NewSeededTestStore(),
+ Embedder: embedding.FallbackProvider{Dim: 256},
KeywordWeight: 0.6,
SemanticWeight: 0.4,
}
@@ -54,6 +272,24 @@ func TestSemanticSearchReturnsRankedResults(t *testing.T) {
}
}
+func TestSemanticSearchUsesExternalVectorStoreWithoutMemoryCache(t *testing.T) {
+ s := newService()
+ vectors := &fakeVectorStore{chunks: map[string]fakeChunkVector{}}
+ s.Vectors = vectors
+ if _, err := s.Reindex(context.Background()); err != nil {
+ t.Fatalf("Reindex: %v", err)
+ }
+ if s.Store.EmbeddingCount() != 0 {
+ t.Fatalf("in-memory embeddings = %d, want 0", s.Store.EmbeddingCount())
+ }
+ if _, err := s.Search(context.Background(), Request{Query: "构建缓存", Mode: ModeSemantic}); err != nil {
+ t.Fatalf("Search: %v", err)
+ }
+ if vectors.simCalls != 1 {
+ t.Fatalf("similarity queries = %d, want one pgvector query", vectors.simCalls)
+ }
+}
+
func TestKeywordModeSkipsEmbeddingButStillMatches(t *testing.T) {
s := newService()
resp, err := s.Search(context.Background(), Request{Query: "构建缓存", Mode: ModeKeyword, PageSize: 5})
@@ -68,3 +304,61 @@ func TestKeywordModeSkipsEmbeddingButStillMatches(t *testing.T) {
t.Fatalf("keyword search should not populate embedding cache, got %d", s.Store.EmbeddingCount())
}
}
+
+func TestDefaultVersionsOnlyFiltersDuplicateOldVersions(t *testing.T) {
+ st := store.NewTestStore()
+ ingest := func(version, body string) {
+ t.Helper()
+ _, err := st.IngestArtifact(store.DeployArtifact{
+ ModuleKey: "Threadpool",
+ ModuleName: "Threadpool",
+ DocsVersion: version,
+ Entries: []store.DeployEntry{{Key: "guide", Title: "Guide", Type: "markdown"}},
+ Documents: []store.DeployDocument{{
+ DocID: "Threadpool:" + version + ":guide",
+ EntryKey: "guide",
+ EntryType: "markdown",
+ Title: "线程池配置",
+ Description: "线程池配置",
+ Content: body,
+ Status: "active",
+ }},
+ })
+ if err != nil {
+ t.Fatalf("IngestArtifact(%s): %v", version, err)
+ }
+ }
+ ingest("v1.0.0", "线程池配置 max_workers legacy")
+ ingest("v2.0.0", "线程池配置 max_workers current")
+
+ s := Service{Store: st, Embedder: embedding.FallbackProvider{Dim: 256}}
+ resp, err := s.Search(context.Background(), Request{Query: "max_workers", Mode: ModeKeyword, PageSize: 10, DefaultVersionsOnly: true})
+ if err != nil {
+ t.Fatalf("Search default versions: %v", err)
+ }
+ if len(resp.Results) != 1 || resp.Results[0].DocsVersion != "v2.0.0" {
+ t.Fatalf("default-version results = %#v, want only v2.0.0", resp.Results)
+ }
+
+ resp, err = s.Search(context.Background(), Request{Query: "max_workers", Mode: ModeKeyword, PageSize: 10, Filters: Filters{DocsVersions: []string{"v1.0.0"}}, DefaultVersionsOnly: true})
+ if err != nil {
+ t.Fatalf("Search explicit version: %v", err)
+ }
+ if len(resp.Results) != 1 || resp.Results[0].DocsVersion != "v1.0.0" {
+ t.Fatalf("explicit-version results = %#v, want v1.0.0", resp.Results)
+ }
+}
+
+func TestPlainText(t *testing.T) {
+ cases := []struct{ in, want string }{
+ {"**二、使用指南**", "二、使用指南"},
+ {"# **系统概述**\n## **产品定位**\n**[CBB系统](https://cbb.fsdev.cn/#/x)** 是一个集质量管控", "系统概述 产品定位 CBB系统 是一个集质量管控"},
+ {"```html\nCBB V25\n```\n- [0.概述](https://x)", "0.概述"},
+ {"use C# here", "use C# here"}, // hash not a heading marker
+ }
+ for _, c := range cases {
+ if got := plainText(c.in); got != c.want {
+ t.Errorf("plainText(%q) = %q, want %q", c.in, got, c.want)
+ }
+ }
+}
diff --git a/backend/internal/search/vector_store.go b/backend/internal/search/vector_store.go
new file mode 100644
index 0000000..760dee1
--- /dev/null
+++ b/backend/internal/search/vector_store.go
@@ -0,0 +1,14 @@
+package search
+
+import "context"
+
+// VectorStore persists document embeddings outside the process. Implementations
+// should return independent slices that callers may safely discard or modify.
+type VectorStore interface {
+ Existing(ctx context.Context, docIDs []string) (map[string]bool, error)
+ Similarities(ctx context.Context, query []float32, docIDs []string, limit int) (map[string]float64, error)
+ UpsertChunk(ctx context.Context, docID, chunkID, content string, vector []float32) error
+ Clear(ctx context.Context) error
+ DeletePrefix(ctx context.Context, docIDPrefix string) error
+ Count(ctx context.Context) (int, error)
+}
diff --git a/backend/internal/store/analytics.go b/backend/internal/store/analytics.go
new file mode 100644
index 0000000..2b45e1f
--- /dev/null
+++ b/backend/internal/store/analytics.go
@@ -0,0 +1,412 @@
+package store
+
+import (
+ "sort"
+ "strings"
+ "time"
+)
+
+func (s *MemoryStore) AddSearchLog(log SearchLog) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.searchLogs = append(s.searchLogs, log)
+}
+
+func (s *MemoryStore) SearchLogs() []SearchLog {
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ return append([]SearchLog(nil), s.searchLogs...)
+}
+
+func (s *MemoryStore) AddMCPLog(log MCPLog) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.mcpLogs = append(s.mcpLogs, log)
+}
+
+func (s *MemoryStore) MCPLogs() []MCPLog {
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ return append([]MCPLog(nil), s.mcpLogs...)
+}
+
+func (s *MemoryStore) AddDocFeedback(f DocFeedback) DocFeedback {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ if f.CreatedAt.IsZero() {
+ f.CreatedAt = time.Now().UTC()
+ }
+ if f.ID == "" {
+ f.ID = s.nextIDLocked("df")
+ }
+ for _, p := range s.pages {
+ if p.DocID == f.DocID {
+ f.PageID = p.ID
+ f.ModuleKey = p.ModuleKey
+ f.Title = p.Title
+ break
+ }
+ }
+ s.feedbacks = append(s.feedbacks, f)
+ return f
+}
+
+func (s *MemoryStore) DocFeedbacks() []DocFeedback {
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ return append([]DocFeedback(nil), s.feedbacks...)
+}
+
+// RecordPageView appends a page view and returns the stored record.
+func (s *MemoryStore) RecordPageView(pv PageView) PageView {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ if pv.ViewedAt.IsZero() {
+ pv.ViewedAt = time.Now().UTC()
+ }
+ if pv.ID == "" {
+ pv.ID = s.nextIDLocked("pv")
+ }
+ for _, p := range s.pages {
+ if p.DocID == pv.DocID {
+ pv.PageID = p.ID
+ pv.ModuleKey = p.ModuleKey
+ pv.ModuleName = p.ModuleName
+ pv.DocsVersion = p.DocsVersion
+ pv.EntryKey = p.EntryKey
+ pv.Title = p.Title
+ pv.Path = p.Path
+ break
+ }
+ }
+ s.pageViews = append(s.pageViews, pv)
+ return pv
+}
+
+// RecordReadProgress updates the latest matching page view with duration and
+// scroll depth for the given session and doc, or records a new view if none.
+func (s *MemoryStore) RecordReadProgress(docID, sessionID, readID string, durationSeconds int, scrollDepth float64) PageView {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ for i := len(s.pageViews) - 1; i >= 0; i-- {
+ pv := &s.pageViews[i]
+ if pv.DocID == docID && ((readID != "" && pv.ReadID == readID) || (readID == "" && pv.SessionID == sessionID)) {
+ if durationSeconds > pv.DurationSeconds {
+ pv.DurationSeconds = durationSeconds
+ }
+ if scrollDepth > pv.ScrollDepth {
+ pv.ScrollDepth = scrollDepth
+ }
+ return *pv
+ }
+ }
+ pv := PageView{
+ ID: s.nextIDLocked("pv"), DocID: docID, SessionID: sessionID, ReadID: readID,
+ DurationSeconds: durationSeconds, ScrollDepth: scrollDepth, ViewedAt: time.Now().UTC(),
+ }
+ for _, p := range s.pages {
+ if p.DocID == pv.DocID {
+ pv.PageID = p.ID
+ pv.ModuleKey = p.ModuleKey
+ pv.ModuleName = p.ModuleName
+ pv.DocsVersion = p.DocsVersion
+ pv.EntryKey = p.EntryKey
+ pv.Title = p.Title
+ pv.Path = p.Path
+ break
+ }
+ }
+ s.pageViews = append(s.pageViews, pv)
+ return pv
+}
+
+// PageAnalytics aggregates recorded views into per-page reading statistics.
+// Pages with no recorded views fall back to seeded read counts so the admin
+// dashboard is populated on a fresh start.
+func (s *MemoryStore) PageAnalytics() []PageStat {
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ now := time.Now().UTC()
+ week := now.AddDate(0, 0, -7)
+ month := now.AddDate(0, 0, -30)
+ type agg struct {
+ pv, reads7, reads30, durSum, durCount int
+ users map[string]struct{}
+ last time.Time
+ }
+ byDoc := map[string]*agg{}
+ for _, pv := range s.pageViews {
+ a := byDoc[pv.DocID]
+ if a == nil {
+ a = &agg{users: map[string]struct{}{}}
+ byDoc[pv.DocID] = a
+ }
+ a.pv++
+ uid := pv.UserID
+ if uid == "" {
+ uid = pv.SessionID
+ }
+ if uid != "" {
+ a.users[uid] = struct{}{}
+ }
+ if pv.ViewedAt.After(week) {
+ a.reads7++
+ }
+ if pv.ViewedAt.After(month) {
+ a.reads30++
+ }
+ if pv.DurationSeconds > 0 {
+ a.durSum += pv.DurationSeconds
+ a.durCount++
+ }
+ if pv.ViewedAt.After(a.last) {
+ a.last = pv.ViewedAt
+ }
+ }
+ var out []PageStat
+ for _, p := range s.pages {
+ stat := PageStat{DocID: p.DocID, Title: p.Title, ModuleKey: p.ModuleKey, ModuleName: p.ModuleName, DocsVersion: p.DocsVersion, Path: p.Path, LastViewedAt: p.UpdatedAt}
+ if a := byDoc[p.DocID]; a != nil {
+ stat.PV = a.pv
+ stat.UV = len(a.users)
+ stat.Reads7d = a.reads7
+ stat.Reads30d = a.reads30
+ stat.LastViewedAt = a.last
+ if a.durCount > 0 {
+ stat.AvgDurationSec = a.durSum / a.durCount
+ }
+ } else {
+ stat.Reads7d = s.seedReadsLocked(p.ModuleKey, true)
+ stat.Reads30d = s.seedReadsLocked(p.ModuleKey, false)
+ }
+ out = append(out, stat)
+ }
+ sort.Slice(out, func(i, j int) bool {
+ if out[i].PV != out[j].PV {
+ return out[i].PV > out[j].PV
+ }
+ return out[i].Reads30d > out[j].Reads30d
+ })
+ return out
+}
+
+func (s *MemoryStore) seedReadsLocked(moduleKey string, week bool) int {
+ for _, m := range s.modules {
+ if strings.EqualFold(m.ModuleKey, moduleKey) {
+ if week {
+ return m.Reads7d
+ }
+ return m.Reads30d
+ }
+ }
+ return 0
+}
+
+// PageReadStats aggregates recorded views for one document into a daily read
+// trend (last `days` days, inclusive of today) plus a per-reader breakdown.
+// Readers are keyed by user id, falling back to session id for anonymous views.
+func (s *MemoryStore) PageReadStats(docID string, days int) PageReadStats {
+ if days <= 0 {
+ days = 30
+ }
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+
+ now := time.Now().UTC()
+ today := now.Truncate(24 * time.Hour)
+ // Pre-seed every day in the window so the line chart has no gaps.
+ idxByDate := map[string]int{}
+ daily := make([]DailyReadPoint, days)
+ for i := 0; i < days; i++ {
+ d := today.AddDate(0, 0, -(days - 1 - i))
+ key := d.Format("2006-01-02")
+ daily[i] = DailyReadPoint{Date: key, Count: 0}
+ idxByDate[key] = i
+ }
+ windowStart := today.AddDate(0, 0, -(days - 1))
+
+ type ragg struct {
+ userID string
+ count int
+ duration int
+ last time.Time
+ }
+ readers := map[string]*ragg{}
+ total := 0
+ totalDuration := 0
+ timedReads := 0
+ for _, pv := range s.pageViews {
+ if pv.DocID != docID {
+ continue
+ }
+ total++
+ if pv.DurationSeconds > 0 {
+ totalDuration += pv.DurationSeconds
+ timedReads++
+ }
+ if !pv.ViewedAt.Before(windowStart) {
+ if i, ok := idxByDate[pv.ViewedAt.UTC().Format("2006-01-02")]; ok {
+ daily[i].Count++
+ }
+ }
+ key := pv.UserID
+ if key == "" {
+ key = "session:" + pv.SessionID
+ }
+ r := readers[key]
+ if r == nil {
+ r = &ragg{userID: pv.UserID}
+ readers[key] = r
+ }
+ r.count++
+ r.duration += pv.DurationSeconds
+ if pv.ViewedAt.After(r.last) {
+ r.last = pv.ViewedAt
+ }
+ }
+
+ out := PageReadStats{DocID: docID, Total: total, Daily: daily, Readers: []ReaderStat{}}
+ if timedReads > 0 {
+ out.AvgDurationSec = totalDuration / timedReads
+ }
+ for _, r := range readers {
+ name := "匿名"
+ if r.userID != "" {
+ if u, err := s.userByIDLocked(r.userID); err == nil {
+ if u.DisplayName != "" {
+ name = u.DisplayName
+ } else if u.Username != "" {
+ name = u.Username
+ }
+ } else {
+ name = r.userID
+ }
+ }
+ avgDuration := 0
+ if r.count > 0 {
+ avgDuration = r.duration / r.count
+ }
+ out.Readers = append(out.Readers, ReaderStat{Reader: name, UserID: r.userID, Count: r.count, AvgDurationSec: avgDuration, LastReadAt: r.last})
+ }
+ sort.Slice(out.Readers, func(i, j int) bool {
+ if out.Readers[i].Count != out.Readers[j].Count {
+ return out.Readers[i].Count > out.Readers[j].Count
+ }
+ return out.Readers[i].LastReadAt.After(out.Readers[j].LastReadAt)
+ })
+ return out
+}
+
+func (s *MemoryStore) UserFavorites(userID string) []UserFavorite {
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ out := []UserFavorite{}
+ for _, f := range s.favorites {
+ if f.UserID == userID {
+ out = append(out, f)
+ }
+ }
+ sort.Slice(out, func(i, j int) bool { return out[i].CreatedAt.After(out[j].CreatedAt) })
+ return out
+}
+
+func (s *MemoryStore) SetUserFavorite(userID, moduleKey string, favorite bool) ([]UserFavorite, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ userID = strings.TrimSpace(userID)
+ moduleKey = strings.TrimSpace(moduleKey)
+ if userID == "" || moduleKey == "" {
+ return nil, ErrInvalid
+ }
+ exists := false
+ for _, m := range s.modules {
+ if m.ModuleKey == moduleKey {
+ exists = true
+ break
+ }
+ }
+ if !exists {
+ return nil, ErrNotFound
+ }
+ for i := range s.favorites {
+ if s.favorites[i].UserID == userID && s.favorites[i].ModuleKey == moduleKey {
+ if favorite {
+ return s.userFavoritesLocked(userID), nil
+ }
+ s.favorites = append(s.favorites[:i], s.favorites[i+1:]...)
+ return s.userFavoritesLocked(userID), nil
+ }
+ }
+ if favorite {
+ s.favorites = append(s.favorites, UserFavorite{ID: s.nextIDLocked("fav"), UserID: userID, ModuleKey: moduleKey, CreatedAt: time.Now().UTC()})
+ }
+ return s.userFavoritesLocked(userID), nil
+}
+
+func (s *MemoryStore) userFavoritesLocked(userID string) []UserFavorite {
+ out := []UserFavorite{}
+ for _, f := range s.favorites {
+ if f.UserID == userID {
+ out = append(out, f)
+ }
+ }
+ sort.Slice(out, func(i, j int) bool { return out[i].CreatedAt.After(out[j].CreatedAt) })
+ return out
+}
+
+func (s *MemoryStore) UserRecentDocs(userID string, limit int) []UserRecentDoc {
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ if limit <= 0 {
+ limit = 30
+ }
+ out := []UserRecentDoc{}
+ for _, r := range s.recentDocs {
+ if r.UserID == userID {
+ out = append(out, r)
+ }
+ }
+ sort.Slice(out, func(i, j int) bool { return out[i].ViewedAt.After(out[j].ViewedAt) })
+ if len(out) > limit {
+ out = out[:limit]
+ }
+ return out
+}
+
+func (s *MemoryStore) RecordUserRecentDoc(userID string, recent UserRecentDoc) (UserRecentDoc, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ userID = strings.TrimSpace(userID)
+ recent.DocID = strings.TrimSpace(recent.DocID)
+ if userID == "" || recent.DocID == "" {
+ return UserRecentDoc{}, ErrInvalid
+ }
+ for _, p := range s.pages {
+ if p.DocID == recent.DocID {
+ recent.Title = firstNonEmpty(recent.Title, p.Title)
+ recent.ModuleKey = firstNonEmpty(recent.ModuleKey, p.ModuleKey)
+ recent.ModuleName = firstNonEmpty(recent.ModuleName, p.ModuleName)
+ recent.DocsVersion = firstNonEmpty(recent.DocsVersion, p.DocsVersion)
+ recent.EntryKey = firstNonEmpty(recent.EntryKey, p.EntryKey)
+ recent.Href = firstNonEmpty(recent.Href, "/docs/"+p.ModuleKey+"/"+p.DocsVersion+"/"+p.EntryKey)
+ break
+ }
+ }
+ if recent.ID == "" {
+ recent.ID = s.nextIDLocked("recent")
+ }
+ recent.UserID = userID
+ if recent.ViewedAt.IsZero() {
+ recent.ViewedAt = time.Now().UTC()
+ }
+ filtered := s.recentDocs[:0]
+ for _, item := range s.recentDocs {
+ if !(item.UserID == userID && item.DocID == recent.DocID) {
+ filtered = append(filtered, item)
+ }
+ }
+ s.recentDocs = append(filtered, recent)
+ return recent, nil
+}
+
+// CreateCategory adds a new category. Key is required and must be unique.
diff --git a/backend/internal/store/catalog_mutations.go b/backend/internal/store/catalog_mutations.go
new file mode 100644
index 0000000..b91ef39
--- /dev/null
+++ b/backend/internal/store/catalog_mutations.go
@@ -0,0 +1,776 @@
+package store
+
+import (
+ "mime"
+ "net/http"
+ "net/url"
+ "path"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+)
+
+func (s *MemoryStore) CreateCategory(c Category) (Category, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ // Key is system-generated from the name (dotted under the parent's key) so
+ // users never have to invent one. A user-supplied key is still honored.
+ if strings.TrimSpace(c.Key) == "" {
+ if strings.TrimSpace(c.Name) == "" {
+ return Category{}, ErrInvalid
+ }
+ c.Key = s.generateCategoryKeyLocked(c.Name, c.ParentID)
+ }
+ if c.ID == "" {
+ c.ID = c.Key
+ }
+ for _, existing := range s.categories {
+ if existing.ID == c.ID {
+ return Category{}, ErrConflict
+ }
+ }
+ if c.Status == "" {
+ c.Status = "active"
+ }
+ c.Children = nil
+ s.categories = append(s.categories, c)
+ return c, nil
+}
+
+// slugifyKey lowercases and keeps [a-z0-9-]; non-ASCII (e.g. Chinese) collapses
+// to empty, in which case callers fall back to a short unique token.
+func slugifyKey(name string) string {
+ var b strings.Builder
+ prevDash := false
+ for _, r := range strings.ToLower(strings.TrimSpace(name)) {
+ switch {
+ case (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9'):
+ b.WriteRune(r)
+ prevDash = false
+ case r == ' ' || r == '-' || r == '_' || r == '.' || r == '/':
+ if !prevDash && b.Len() > 0 {
+ b.WriteByte('-')
+ prevDash = true
+ }
+ }
+ }
+ return strings.Trim(b.String(), "-")
+}
+
+func (s *MemoryStore) generateCategoryKeyLocked(name, parentID string) string {
+ base := slugifyKey(name)
+ if base == "" {
+ base = "d" + strconv.FormatInt(time.Now().UnixNano()%1_000_000, 36)
+ }
+ prefix := ""
+ for _, c := range s.categories {
+ if c.ID == parentID {
+ prefix = c.Key + "."
+ break
+ }
+ }
+ taken := func(k string) bool {
+ for _, c := range s.categories {
+ if c.Key == k || c.ID == k {
+ return true
+ }
+ }
+ return false
+ }
+ key := prefix + base
+ for i := 2; taken(key); i++ {
+ key = prefix + base + "-" + strconv.Itoa(i)
+ }
+ return key
+}
+
+func (s *MemoryStore) generateTeamKeyLocked(name string) string {
+ base := slugifyKey(name)
+ if base == "" {
+ base = "team-" + strconv.FormatInt(time.Now().UnixNano()%1_000_000, 36)
+ }
+ taken := func(k string) bool {
+ for _, t := range s.teams {
+ if strings.EqualFold(t.Key, k) {
+ return true
+ }
+ }
+ return false
+ }
+ key := base
+ for i := 2; taken(key); i++ {
+ key = base + "-" + strconv.Itoa(i)
+ }
+ return key
+}
+
+// MoveCategory reparents a category and positions it at `index` among its new
+// siblings, renumbering sibling SortOrder so the tree order is stable. Rejects
+// moves that would create a cycle (into the node's own subtree).
+func (s *MemoryStore) MoveCategory(id, parentID string, index int) (Category, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ idx := -1
+ for i := range s.categories {
+ if s.categories[i].ID == id {
+ idx = i
+ break
+ }
+ }
+ if idx == -1 {
+ return Category{}, ErrNotFound
+ }
+ if parentID == id {
+ return Category{}, ErrInvalid
+ }
+ // Walk parent chain to reject cycles.
+ for p := parentID; p != ""; {
+ if p == id {
+ return Category{}, ErrInvalid
+ }
+ next := ""
+ for i := range s.categories {
+ if s.categories[i].ID == p {
+ next = s.categories[i].ParentID
+ break
+ }
+ }
+ p = next
+ }
+ if parentID != "" {
+ found := false
+ for i := range s.categories {
+ if s.categories[i].ID == parentID {
+ found = true
+ break
+ }
+ }
+ if !found {
+ return Category{}, ErrInvalid
+ }
+ }
+
+ s.categories[idx].ParentID = parentID
+
+ // Collect new siblings (same parent) in current order, excluding the moved
+ // node, then insert it at the requested index and renumber.
+ var sibs []int
+ for i := range s.categories {
+ if s.categories[i].ParentID == parentID && s.categories[i].ID != id {
+ sibs = append(sibs, i)
+ }
+ }
+ sort.SliceStable(sibs, func(a, b int) bool { return s.categories[sibs[a]].SortOrder < s.categories[sibs[b]].SortOrder })
+ order := make([]int, 0, len(sibs)+1)
+ if index < 0 {
+ index = 0
+ }
+ if index > len(sibs) {
+ index = len(sibs)
+ }
+ order = append(order, sibs[:index]...)
+ order = append(order, idx)
+ order = append(order, sibs[index:]...)
+ for pos, ci := range order {
+ s.categories[ci].SortOrder = (pos + 1) * 10
+ }
+ out := s.categories[idx]
+ out.Children = nil
+ return out, nil
+}
+
+func (s *MemoryStore) UpdateCategory(id string, c Category) (Category, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ for i := range s.categories {
+ if s.categories[i].ID == id {
+ if c.Name != "" {
+ s.categories[i].Name = c.Name
+ }
+ if c.Description != "" {
+ s.categories[i].Description = c.Description
+ }
+ if c.Icon != "" {
+ s.categories[i].Icon = c.Icon
+ }
+ if c.SortOrder != 0 {
+ s.categories[i].SortOrder = c.SortOrder
+ }
+ if c.Status != "" {
+ s.categories[i].Status = c.Status
+ }
+ if c.ParentID != "" {
+ s.categories[i].ParentID = c.ParentID
+ }
+ // Always accept ResponsibleTeam from patch (send "" explicitly to clear assignment to a team).
+ s.categories[i].ResponsibleTeam = c.ResponsibleTeam
+ out := s.categories[i]
+ out.Children = nil
+ return out, nil
+ }
+ }
+ return Category{}, ErrNotFound
+}
+
+func (s *MemoryStore) DeleteCategory(id string) error {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ for _, c := range s.categories {
+ if c.ParentID == id {
+ return ErrConflict
+ }
+ }
+ for i := range s.categories {
+ if s.categories[i].ID == id {
+ s.categories = append(s.categories[:i], s.categories[i+1:]...)
+ return nil
+ }
+ }
+ return ErrNotFound
+}
+
+func (s *MemoryStore) moduleKeyTakenLocked(key string) bool {
+ for _, m := range s.modules {
+ if strings.EqualFold(m.ModuleKey, key) {
+ return true
+ }
+ }
+ return false
+}
+
+func (s *MemoryStore) CreateModule(m Module) (Module, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ // Auto-generate module_key from the name (slug, unique) so admins never type one.
+ if strings.TrimSpace(m.ModuleKey) == "" {
+ if strings.TrimSpace(m.Name) == "" {
+ return Module{}, ErrInvalid
+ }
+ base := slugifyKey(m.Name)
+ if base == "" {
+ base = "doc-" + strconv.FormatInt(time.Now().UnixNano()%1_000_000, 36)
+ }
+ key := base
+ for i := 2; s.moduleKeyTakenLocked(key); i++ {
+ key = base + "-" + strconv.Itoa(i)
+ }
+ m.ModuleKey = key
+ }
+ for _, existing := range s.modules {
+ if strings.EqualFold(existing.ModuleKey, m.ModuleKey) {
+ return Module{}, ErrConflict
+ }
+ }
+ if m.ID == "" {
+ m.ID = s.nextIDLocked("m")
+ }
+ // Every doc source gets a deploy token for CI push auth.
+ if strings.TrimSpace(m.DeployToken) == "" {
+ m.DeployToken = "mdx_" + strconv.FormatInt(time.Now().UnixNano(), 36) + strconv.FormatInt(int64(len(s.modules)+1), 36)
+ }
+ if m.Name == "" {
+ m.Name = m.ModuleKey
+ }
+ if m.Status == "" {
+ m.Status = "active"
+ }
+ if m.DefaultVersion == "" {
+ m.DefaultVersion = "latest"
+ }
+ if m.CategoryPath == "" {
+ m.CategoryPath = s.categoryPathLocked(m.CategoryIDs)
+ }
+ m.UpdatedAt = time.Now().UTC()
+ m.AvailableVers = nil
+ m.DeployTokenSet = m.DeployToken != ""
+ s.modules = append(s.modules, m)
+ return m, nil
+}
+
+func (s *MemoryStore) UpdateModule(moduleKey string, patch Module) (Module, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ for i := range s.modules {
+ if strings.EqualFold(s.modules[i].ModuleKey, moduleKey) {
+ m := &s.modules[i]
+ if patch.Name != "" {
+ m.Name = patch.Name
+ }
+ if patch.Description != "" {
+ m.Description = patch.Description
+ }
+ if patch.OwnerGroup != "" {
+ m.OwnerGroup = patch.OwnerGroup
+ }
+ if patch.RepoType != "" {
+ m.RepoType = patch.RepoType
+ }
+ if patch.RepoURL != "" {
+ m.RepoURL = patch.RepoURL
+ }
+ if patch.DefaultVersion != "" {
+ m.DefaultVersion = patch.DefaultVersion
+ }
+ if patch.Visibility != "" {
+ m.Visibility = patch.Visibility
+ }
+ if patch.Status != "" {
+ m.Status = patch.Status
+ }
+ if patch.PackageVersion != "" {
+ m.PackageVersion = patch.PackageVersion
+ }
+ if patch.Channel != "" {
+ m.Channel = patch.Channel
+ }
+ if patch.Edition != "" {
+ m.Edition = patch.Edition
+ }
+ if patch.Keywords != nil {
+ m.Keywords = patch.Keywords
+ }
+ if patch.Maintainers != nil {
+ m.Maintainers = patch.Maintainers
+ }
+ if patch.CategoryIDs != nil {
+ m.CategoryIDs = patch.CategoryIDs
+ if patch.CategoryPath == "" {
+ m.CategoryPath = s.categoryPathLocked(m.CategoryIDs)
+ }
+ s.syncPageCategoriesForModuleLocked(m.ModuleKey, m.CategoryIDs)
+ }
+ if patch.CategoryPath != "" {
+ m.CategoryPath = patch.CategoryPath
+ }
+ if patch.SourceType != "" {
+ m.SourceType = patch.SourceType
+ }
+ if patch.DocType != "" {
+ m.DocType = patch.DocType
+ }
+ if patch.Mount != "" {
+ m.Mount = patch.Mount
+ }
+ if patch.GitLabBranch != "" {
+ m.GitLabBranch = patch.GitLabBranch
+ }
+ if patch.GitLabPath != "" {
+ m.GitLabPath = patch.GitLabPath
+ }
+ if patch.DeployToken != "" {
+ m.DeployToken = patch.DeployToken
+ }
+ m.UpdatedAt = time.Now().UTC()
+ out := *m
+ out.AvailableVers = s.versionsForLocked(m.ModuleKey)
+ out.DeployTokenSet = out.DeployToken != ""
+ return out, nil
+ }
+ }
+ return Module{}, ErrNotFound
+}
+
+func (s *MemoryStore) syncPageCategoriesForModuleLocked(moduleKey string, categoryIDs []string) {
+ for i := range s.pages {
+ if strings.EqualFold(s.pages[i].ModuleKey, moduleKey) {
+ s.pages[i].CategoryIDs = cloneStrings(categoryIDs)
+ }
+ }
+}
+
+func (s *MemoryStore) CreateVersion(moduleKey string, v Version) (Version, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ if _, err := s.moduleIndexLocked(moduleKey); err != nil {
+ return Version{}, err
+ }
+ if strings.TrimSpace(v.DocsVersion) == "" {
+ return Version{}, ErrInvalid
+ }
+ for _, existing := range s.versions {
+ if strings.EqualFold(existing.ModuleKey, moduleKey) && existing.DocsVersion == v.DocsVersion {
+ return Version{}, ErrConflict
+ }
+ }
+ v.ModuleKey = moduleKey
+ if v.ID == "" {
+ v.ID = s.nextIDLocked("v")
+ }
+ if v.DisplayName == "" {
+ v.DisplayName = v.DocsVersion
+ }
+ if v.Status == "" {
+ v.Status = "active"
+ }
+ v.CreatedAt = time.Now().UTC()
+ if v.IsDefault {
+ for i := range s.versions {
+ if strings.EqualFold(s.versions[i].ModuleKey, moduleKey) {
+ s.versions[i].IsDefault = false
+ }
+ }
+ if idx, err := s.moduleIndexLocked(moduleKey); err == nil {
+ s.modules[idx].DefaultVersion = v.DocsVersion
+ }
+ }
+ s.versions = append(s.versions, v)
+ return v, nil
+}
+
+func (s *MemoryStore) UpdateVersion(moduleKey, docsVersion string, patch Version) (Version, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ for i := range s.versions {
+ if strings.EqualFold(s.versions[i].ModuleKey, moduleKey) && s.versions[i].DocsVersion == docsVersion {
+ v := &s.versions[i]
+ if patch.DisplayName != "" {
+ v.DisplayName = patch.DisplayName
+ }
+ if patch.VersionType != "" {
+ v.VersionType = patch.VersionType
+ }
+ if patch.Status != "" {
+ v.Status = patch.Status
+ }
+ if patch.SourceBranch != "" {
+ v.SourceBranch = patch.SourceBranch
+ }
+ if patch.PackageVersion != "" {
+ v.PackageVersion = patch.PackageVersion
+ }
+ if patch.Channel != "" {
+ v.Channel = patch.Channel
+ }
+ if patch.Edition != "" {
+ v.Edition = patch.Edition
+ }
+ if patch.SupportStatus != "" {
+ v.SupportStatus = patch.SupportStatus
+ }
+ if patch.IsDefault {
+ for j := range s.versions {
+ if strings.EqualFold(s.versions[j].ModuleKey, moduleKey) {
+ s.versions[j].IsDefault = false
+ }
+ }
+ v.IsDefault = true
+ if idx, err := s.moduleIndexLocked(moduleKey); err == nil {
+ s.modules[idx].DefaultVersion = v.DocsVersion
+ }
+ }
+ return *v, nil
+ }
+ }
+ return Version{}, ErrNotFound
+}
+
+func (s *MemoryStore) CreateEntry(moduleKey, docsVersion string, e Entry) (Entry, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ if _, err := s.moduleIndexLocked(moduleKey); err != nil {
+ return Entry{}, err
+ }
+ if strings.TrimSpace(e.EntryKey) == "" {
+ return Entry{}, ErrInvalid
+ }
+ for _, existing := range s.entries {
+ if strings.EqualFold(existing.ModuleKey, moduleKey) && existing.DocsVersion == docsVersion && existing.EntryKey == e.EntryKey {
+ return Entry{}, ErrConflict
+ }
+ }
+ e.ModuleKey = moduleKey
+ e.DocsVersion = docsVersion
+ if e.ID == "" {
+ e.ID = s.nextIDLocked("e")
+ }
+ if e.EntryType == "" {
+ e.EntryType = "markdown"
+ }
+ if e.Builder == "" {
+ e.Builder = e.EntryType
+ }
+ if e.IndexStatus == "" {
+ e.IndexStatus = "pending"
+ }
+ if e.Status == "" {
+ e.Status = "active"
+ }
+ e.CreatedAt = time.Now().UTC()
+ s.entries = append(s.entries, e)
+ return e, nil
+}
+
+func (s *MemoryStore) UpdateEntry(entryID string, patch Entry) (Entry, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ for i := range s.entries {
+ if s.entries[i].ID == entryID {
+ e := &s.entries[i]
+ if patch.Title != "" {
+ e.Title = patch.Title
+ }
+ if patch.EntryType != "" {
+ e.EntryType = patch.EntryType
+ }
+ if patch.Builder != "" {
+ e.Builder = patch.Builder
+ }
+ if patch.Source != "" {
+ e.Source = patch.Source
+ }
+ if patch.StorageURI != "" {
+ e.StorageURI = patch.StorageURI
+ }
+ if patch.NavURI != "" {
+ e.NavURI = patch.NavURI
+ }
+ if patch.IndexStatus != "" {
+ e.IndexStatus = patch.IndexStatus
+ }
+ if patch.SortOrder != 0 {
+ e.SortOrder = patch.SortOrder
+ }
+ if patch.Status != "" {
+ e.Status = patch.Status
+ }
+ e.IsPrimary = patch.IsPrimary
+ return *e, nil
+ }
+ }
+ return Entry{}, ErrNotFound
+}
+
+func (s *MemoryStore) DeleteEntry(entryID string) error {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ for i := range s.entries {
+ if s.entries[i].ID == entryID {
+ s.entries = append(s.entries[:i], s.entries[i+1:]...)
+ return nil
+ }
+ }
+ return ErrNotFound
+}
+
+func (s *MemoryStore) Release(releaseID string) (Release, error) {
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ for _, r := range s.releases {
+ if r.ReleaseID == releaseID || r.ID == releaseID {
+ return r, nil
+ }
+ }
+ return Release{}, ErrNotFound
+}
+
+// RollbackRelease marks the target release as rolled back. A real
+// implementation would also re-point storage and search to the prior artifact.
+func (s *MemoryStore) RollbackRelease(releaseID string) (Release, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ for i := range s.releases {
+ if s.releases[i].ReleaseID == releaseID || s.releases[i].ID == releaseID {
+ s.releases[i].Status = "rolled_back"
+ return s.releases[i], nil
+ }
+ }
+ return Release{}, ErrNotFound
+}
+
+func (s *MemoryStore) moduleIndexLocked(moduleKey string) (int, error) {
+ for i := range s.modules {
+ if strings.EqualFold(s.modules[i].ModuleKey, moduleKey) {
+ return i, nil
+ }
+ }
+ return -1, ErrNotFound
+}
+
+func (s *MemoryStore) nextIDLocked(prefix string) string {
+ s.seq++
+ return prefix + "-" + strconv.FormatInt(s.seq, 10) + "-" + strconv.FormatInt(time.Now().UnixNano(), 36)
+}
+
+func (s *MemoryStore) versionsForLocked(moduleKey string) []Version {
+ var out []Version
+ for _, v := range s.versions {
+ if strings.EqualFold(v.ModuleKey, moduleKey) {
+ out = append(out, v)
+ }
+ }
+ return out
+}
+
+func contains(xs []string, target string) bool {
+ for _, x := range xs {
+ if x == target {
+ return true
+ }
+ }
+ return false
+}
+
+func routeKey(moduleKey, docsVersion, entryKey string) string {
+ if entryKey == "" {
+ return strings.ToLower(moduleKey) + ":" + docsVersion
+ }
+ return strings.ToLower(moduleKey) + ":" + docsVersion + ":" + entryKey
+}
+
+func siteFileKey(moduleKey, docsVersion, entryKey, name string) string {
+ return routeKey(moduleKey, docsVersion, entryKey) + ":" + path.Clean(strings.TrimPrefix(name, "/"))
+}
+
+// isSiteBuilderType reports whether an entry ships a pre-built static site
+// (VitePress/VuePress/Fumadocs) rather than Markdown rendered by Modex.
+func isSiteBuilderType(entryType string) bool {
+ switch strings.ToLower(entryType) {
+ case "vitepress", "vuepress", "fumadocs":
+ return true
+ default:
+ return false
+ }
+}
+
+// docPagePath builds the in-app URL for an indexed page. For site-builder
+// entries it preserves the per-page route as a ?p= deep link so a search hit
+// opens the matched page inside the embedded site, instead of collapsing every
+// page to the entry root. Markdown/static entries keep the plain entry URL.
+func docPagePath(moduleKey, docsVersion, entryKey, entryType, route string) string {
+ base := "/docs/" + moduleKey + "/" + docsVersion + "/" + entryKey
+ route = strings.TrimSpace(route)
+ if isSiteBuilderType(entryType) && route != "" && route != "/" && !strings.HasPrefix(route, "/docs/") {
+ return base + "?p=" + url.QueryEscape(route)
+ }
+ return base
+}
+
+func cloneStrings(xs []string) []string {
+ if xs == nil {
+ return nil
+ }
+ return append([]string(nil), xs...)
+}
+
+func cloneNav(xs []NavItem) []NavItem {
+ if xs == nil {
+ return nil
+ }
+ out := make([]NavItem, len(xs))
+ for i, x := range xs {
+ out[i] = x
+ out[i].Children = cloneNav(x.Children)
+ }
+ return out
+}
+
+func firstNonEmpty(values ...string) string {
+ for _, v := range values {
+ if strings.TrimSpace(v) != "" {
+ return v
+ }
+ }
+ return ""
+}
+
+func firstString(xs []string) string {
+ if len(xs) == 0 {
+ return ""
+ }
+ return xs[0]
+}
+
+func coalesceStrings(primary, fallback []string) []string {
+ if len(primary) > 0 {
+ return primary
+ }
+ return fallback
+}
+
+func removeEntries(entries []Entry, moduleKey, docsVersion string) []Entry {
+ out := entries[:0]
+ for _, e := range entries {
+ if strings.EqualFold(e.ModuleKey, moduleKey) && e.DocsVersion == docsVersion {
+ continue
+ }
+ out = append(out, e)
+ }
+ return out
+}
+
+func removePages(pages []Page, moduleKey, docsVersion string) []Page {
+ out := pages[:0]
+ for _, p := range pages {
+ if strings.EqualFold(p.ModuleKey, moduleKey) && p.DocsVersion == docsVersion {
+ continue
+ }
+ out = append(out, p)
+ }
+ return out
+}
+
+func entryKeyFromDocID(docID string) string {
+ parts := strings.Split(docID, ":")
+ if len(parts) == 0 {
+ return ""
+ }
+ return parts[len(parts)-1]
+}
+
+func entryTypeForEntry(entries []DeployEntry, entryKey string) string {
+ for _, e := range entries {
+ if e.Key == entryKey {
+ return e.Type
+ }
+ }
+ return "markdown"
+}
+
+func titleForEntry(entries []DeployEntry, entryKey string) string {
+ for _, e := range entries {
+ if e.Key == entryKey {
+ return e.Title
+ }
+ }
+ return entryKey
+}
+
+func htmlForEntry(files map[string]string, entryKey string) string {
+ for _, name := range []string{"site/" + entryKey + "/index.html", "site/" + entryKey + ".html"} {
+ if html := files[name]; html != "" {
+ return html
+ }
+ }
+ prefix := "site/" + entryKey + "/"
+ for name, html := range files {
+ if strings.HasPrefix(name, prefix) && strings.HasSuffix(name, ".html") {
+ return html
+ }
+ }
+ return ""
+}
+
+func splitSiteFile(name string) (string, string, bool) {
+ name = path.Clean(strings.TrimPrefix(name, "/"))
+ parts := strings.Split(name, "/")
+ if len(parts) < 3 || parts[0] != "site" || parts[1] == "" {
+ return "", "", false
+ }
+ rel := path.Join(parts[2:]...)
+ if rel == "." || rel == "" {
+ rel = "index.html"
+ }
+ return parts[1], rel, true
+}
+
+func contentTypeForName(name string, content []byte) string {
+ if ct := mime.TypeByExtension(path.Ext(name)); ct != "" {
+ return ct
+ }
+ if len(content) > 0 {
+ return http.DetectContentType(content)
+ }
+ return "application/octet-stream"
+}
diff --git a/backend/internal/store/datastore.go b/backend/internal/store/datastore.go
new file mode 100644
index 0000000..69d5e55
--- /dev/null
+++ b/backend/internal/store/datastore.go
@@ -0,0 +1,104 @@
+package store
+
+import "time"
+
+// DataStore is the business data boundary used by the application. Production
+// injects PostgresRepository; MemoryStore exists only as a deterministic test
+// fake. Methods intentionally preserve the existing service API while storage
+// is migrated away from process-local state.
+type DataStore interface {
+ Settings() Settings
+ SaveAISettings(AISettings) Settings
+ CurrentUser() User
+ Users(string) []User
+ UserByID(string) (User, error)
+ UserByMCPToken(string) (User, error)
+ SetUserMCPToken(string, string) (User, error)
+ CreateUser(User) (User, error)
+ UpdateUser(string, User) (User, error)
+ DeleteUser(string) error
+ UpsertUser(User) User
+ Teams() []Team
+ Team(string) (Team, error)
+ CreateTeam(Team) (Team, error)
+ UpdateTeam(string, Team) (Team, error)
+ DeleteTeam(string) error
+ AddTeamMember(string, string) (Team, error)
+ RemoveTeamMember(string, string) (Team, error)
+ SetTeamLeader(string, string) (Team, error)
+ TeamMembers(string) []string
+ TeamKeysForUser(User) []string
+ AllCategories() []Category
+ CategoryName(string) string
+ CategoryTree() []Category
+ CreateCategory(Category) (Category, error)
+ MoveCategory(string, string, int) (Category, error)
+ UpdateCategory(string, Category) (Category, error)
+ DeleteCategory(string) error
+ Modules(string, string) []Module
+ Module(string) (Module, error)
+ ModuleByDeployToken(string) (Module, error)
+ CreateModule(Module) (Module, error)
+ UpdateModule(string, Module) (Module, error)
+ Versions(string) []Version
+ CreateVersion(string, Version) (Version, error)
+ UpdateVersion(string, string, Version) (Version, error)
+ Entries(string, string) []Entry
+ EntryModuleKey(string) (string, bool)
+ CreateEntry(string, string, Entry) (Entry, error)
+ UpdateEntry(string, Entry) (Entry, error)
+ DeleteEntry(string) error
+ Releases() []Release
+ Release(string) (Release, error)
+ RollbackRelease(string) (Release, error)
+ Page(string) (Page, error)
+ PageByRoute(string, string, string) (Page, error)
+ Pages() []Page
+ Nav(string, string) []NavItem
+ PageHTML(string, string, string) string
+ SiteFile(string, string, string, string) (SiteFile, error)
+ SiteObjects() map[string]SiteFile
+ ClearSiteAssets()
+ Embedding(string) ([]float32, bool)
+ SetEmbedding(string, []float32)
+ EmbeddingCount() int
+ ClearEmbeddings()
+ IngestArtifact(DeployArtifact) (DeployResult, error)
+ AddSearchLog(SearchLog)
+ SearchLogs() []SearchLog
+ AddMCPLog(MCPLog)
+ MCPLogs() []MCPLog
+ AddDocFeedback(DocFeedback) DocFeedback
+ DocFeedbacks() []DocFeedback
+ RecordPageView(PageView) PageView
+ RecordReadProgress(string, string, string, int, float64) PageView
+ PageAnalytics() []PageStat
+ PageReadStats(string, int) PageReadStats
+ UserFavorites(string) []UserFavorite
+ SetUserFavorite(string, string, bool) ([]UserFavorite, error)
+ UserRecentDocs(string, int) []UserRecentDoc
+ RecordUserRecentDoc(string, UserRecentDoc) (UserRecentDoc, error)
+ ConnectedApps() []ConnectedApp
+ ConnectedAppByClientID(string) (ConnectedApp, error)
+ CreateConnectedApp(ConnectedApp, string) (ConnectedApp, error)
+ UpdateConnectedApp(string, ConnectedApp) (ConnectedApp, error)
+ DeleteConnectedApp(string) error
+ VerifyConnectedAppSecret(string, string) (ConnectedApp, error)
+ CreateOAuthCode(string, string, string, []string, string, time.Duration) (OAuthGrant, error)
+ RedeemOAuthCode(string, string, string, string, string, time.Duration, time.Duration) (OAuthGrant, ConnectedApp, User, error)
+ RefreshOAuthToken(string, string, string, string, time.Duration, time.Duration) (OAuthGrant, ConnectedApp, User, error)
+ UserByOAuthAccessToken(string) (User, ConnectedApp, OAuthGrant, error)
+ RevokeOAuthToken(string, string) bool
+ PluginStates() []PluginState
+ SavePluginSettings(map[string]PluginSetting) []PluginState
+ PluginEffective() map[string]PluginSetting
+ UploadedPlugins() []UploadedPlugin
+ EnabledUploadedPlugins() []UploadedPlugin
+ SaveUploadedPlugin(UploadedPlugin) (UploadedPlugin, error)
+ DeleteUploadedPlugin(string) bool
+ SnippetData() ([]Snippet, map[string]string)
+ SaveSnippetData([]Snippet, map[string]string) ([]Snippet, map[string]string)
+}
+
+var _ DataStore = (*MemoryStore)(nil)
+var _ DataStore = (*PostgresRepository)(nil)
diff --git a/backend/internal/store/ingest.go b/backend/internal/store/ingest.go
new file mode 100644
index 0000000..3b833ac
--- /dev/null
+++ b/backend/internal/store/ingest.go
@@ -0,0 +1,253 @@
+package store
+
+import (
+ "strconv"
+ "strings"
+ "time"
+)
+
+func (s *MemoryStore) IngestArtifact(a DeployArtifact) (DeployResult, error) {
+ if strings.TrimSpace(a.ModuleKey) == "" || strings.TrimSpace(a.DocsVersion) == "" || len(a.Entries) == 0 || len(a.Documents) == 0 {
+ return DeployResult{}, ErrInvalid
+ }
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ now := time.Now().UTC()
+ moduleName := firstNonEmpty(a.ModuleName, a.ModuleKey)
+ moduleIdx, err := s.moduleIndexLocked(a.ModuleKey)
+ if err != nil {
+ s.modules = append(s.modules, Module{
+ ID: s.nextIDLocked("m"),
+ ModuleKey: a.ModuleKey,
+ Name: moduleName,
+ Description: a.Description,
+ OwnerGroup: firstNonEmpty(firstString(a.Authors), "docs"),
+ RepoType: firstNonEmpty(a.RepoType, "git"),
+ RepoURL: a.RepoURL,
+ SourceType: "gitlab",
+ GitLabBranch: a.Branch,
+ DefaultVersion: a.DocsVersion,
+ Visibility: "internal",
+ Status: "active",
+ PackageName: a.ModuleKey,
+ PackageVersion: a.PackageVersion,
+ Channel: "docs",
+ Edition: a.Edition,
+ Keywords: cloneStrings(a.Keywords),
+ Maintainers: cloneStrings(a.Authors),
+ LastSyncedCommit: a.CommitSHA,
+ LastSyncedAt: now,
+ UpdatedAt: now,
+ })
+ moduleIdx = len(s.modules) - 1
+ } else {
+ m := &s.modules[moduleIdx]
+ m.Name = moduleName
+ if a.Description != "" {
+ m.Description = a.Description
+ }
+ m.DefaultVersion = a.DocsVersion
+ if a.PackageVersion != "" {
+ m.PackageVersion = a.PackageVersion
+ }
+ if a.Edition != "" {
+ m.Edition = a.Edition
+ }
+ if len(a.Keywords) > 0 {
+ m.Keywords = cloneStrings(a.Keywords)
+ }
+ if len(a.Authors) > 0 {
+ m.Maintainers = cloneStrings(a.Authors)
+ if m.OwnerGroup == "" {
+ m.OwnerGroup = a.Authors[0]
+ }
+ }
+ if m.Status == "" {
+ m.Status = "active"
+ }
+ if m.Visibility == "" {
+ m.Visibility = "internal"
+ }
+ // Refresh source metadata from each CI push (repo/branch/commit).
+ if a.RepoURL != "" {
+ m.RepoURL = a.RepoURL
+ }
+ if a.RepoType != "" {
+ m.RepoType = a.RepoType
+ }
+ if m.SourceType == "" || a.TriggerType == "pipeline" || a.RepoType == "gitlab" {
+ m.SourceType = deploySourceType(a.RepoType, a.RepoURL)
+ if m.SourceType == "manual" && a.TriggerType == "pipeline" {
+ m.SourceType = "gitlab"
+ }
+ }
+ if a.Branch != "" {
+ m.GitLabBranch = a.Branch
+ }
+ if a.CommitSHA != "" {
+ m.LastSyncedCommit = a.CommitSHA
+ }
+ m.LastSyncedAt = now
+ m.UpdatedAt = now
+ }
+ // Preserve existing category assignment and rebuild the display path from
+ // current categories so the admin UI and module cards show the right labels.
+ module := &s.modules[moduleIdx]
+ module.CategoryPath = s.categoryPathLocked(module.CategoryIDs)
+ versionFound := false
+ for i := range s.versions {
+ if strings.EqualFold(s.versions[i].ModuleKey, a.ModuleKey) && s.versions[i].DocsVersion == a.DocsVersion {
+ v := &s.versions[i]
+ v.DisplayName = firstNonEmpty(v.DisplayName, a.DocsVersion)
+ v.IsDefault = true
+ v.Status = "active"
+ v.PackageVersion = a.PackageVersion
+ v.Edition = a.Edition
+ if v.VersionType == "" {
+ v.VersionType = "release"
+ }
+ if v.SupportStatus == "" {
+ v.SupportStatus = "supported"
+ }
+ versionFound = true
+ } else if strings.EqualFold(s.versions[i].ModuleKey, a.ModuleKey) {
+ s.versions[i].IsDefault = false
+ }
+ }
+ if !versionFound {
+ s.versions = append(s.versions, Version{
+ ID: s.nextIDLocked("v"),
+ ModuleKey: a.ModuleKey,
+ DocsVersion: a.DocsVersion,
+ DisplayName: a.DocsVersion,
+ VersionType: "release",
+ IsDefault: true,
+ Status: "active",
+ PackageVersion: a.PackageVersion,
+ Channel: firstNonEmpty(module.Channel, "docs"),
+ Edition: a.Edition,
+ SupportStatus: "supported",
+ CreatedAt: now,
+ })
+ }
+ s.entries = removeEntries(s.entries, a.ModuleKey, a.DocsVersion)
+ for i, e := range a.Entries {
+ s.entries = append(s.entries, Entry{
+ ID: s.nextIDLocked("e"),
+ ModuleKey: a.ModuleKey,
+ DocsVersion: a.DocsVersion,
+ EntryKey: e.Key,
+ Title: e.Title,
+ EntryType: firstNonEmpty(e.Type, "markdown"),
+ Builder: firstNonEmpty(e.Type, "markdown"),
+ Source: e.Source,
+ StorageURI: "memory://" + routeKey(a.ModuleKey, a.DocsVersion, e.Key),
+ NavURI: "memory://" + routeKey(a.ModuleKey, a.DocsVersion, ""),
+ IndexStatus: "indexed",
+ IsPrimary: i == 0,
+ SortOrder: i + 1,
+ Status: "active",
+ CreatedAt: now,
+ })
+ }
+ s.pages = removePages(s.pages, a.ModuleKey, a.DocsVersion)
+ // Drop cached embeddings for this module/version so re-published content is
+ // re-embedded on the next reindex (or lazily during search).
+ if s.embeddings != nil {
+ embPrefix := a.ModuleKey + ":" + a.DocsVersion + ":"
+ for docID := range s.embeddings {
+ if strings.HasPrefix(docID, embPrefix) {
+ delete(s.embeddings, docID)
+ }
+ }
+ }
+ for _, d := range a.Documents {
+ entryKey := firstNonEmpty(d.EntryKey, entryKeyFromDocID(d.DocID))
+ docID := firstNonEmpty(d.DocID, a.ModuleKey+":"+a.DocsVersion+":"+entryKey)
+ s.pages = append(s.pages, Page{
+ ID: s.nextIDLocked("p"),
+ DocID: docID,
+ ModuleKey: a.ModuleKey,
+ ModuleName: moduleName,
+ DocsVersion: a.DocsVersion,
+ PackageVersion: firstNonEmpty(d.PackageVersion, a.PackageVersion),
+ EntryKey: entryKey,
+ EntryType: firstNonEmpty(d.EntryType, entryTypeForEntry(a.Entries, entryKey)),
+ Title: firstNonEmpty(d.Title, titleForEntry(a.Entries, entryKey)),
+ Description: d.Description,
+ Path: docPagePath(a.ModuleKey, a.DocsVersion, entryKey, firstNonEmpty(d.EntryType, entryTypeForEntry(a.Entries, entryKey)), d.Path),
+ SourceFile: d.SourceFile,
+ DocType: firstNonEmpty(d.EntryType, entryTypeForEntry(a.Entries, entryKey)),
+ Status: firstNonEmpty(d.Status, "active"),
+ OwnerGroup: module.OwnerGroup,
+ CategoryIDs: cloneStrings(module.CategoryIDs),
+ Tags: cloneStrings(coalesceStrings(d.Keywords, a.Keywords)),
+ ContentText: d.Content,
+ ContentMD: d.ContentMD,
+ UpdatedAt: now,
+ })
+ }
+ if s.navs == nil {
+ s.navs = map[string][]NavItem{}
+ }
+ s.navs[routeKey(a.ModuleKey, a.DocsVersion, "")] = cloneNav(a.Nav)
+ if s.html == nil {
+ s.html = map[string]string{}
+ }
+ if s.siteFiles == nil {
+ s.siteFiles = map[string]SiteFile{}
+ }
+ for k := range s.html {
+ prefix := routeKey(a.ModuleKey, a.DocsVersion, "") + ":"
+ if strings.HasPrefix(k, prefix) {
+ delete(s.html, k)
+ }
+ }
+ for k := range s.siteFiles {
+ prefix := routeKey(a.ModuleKey, a.DocsVersion, "") + ":"
+ if strings.HasPrefix(k, prefix) {
+ delete(s.siteFiles, k)
+ }
+ }
+ for _, e := range a.Entries {
+ html := htmlForEntry(a.SiteHTML, e.Key)
+ if html != "" {
+ s.html[routeKey(a.ModuleKey, a.DocsVersion, e.Key)] = html
+ }
+ }
+ for name, content := range a.SiteFiles {
+ entryKey, relName, ok := splitSiteFile(name)
+ if !ok {
+ continue
+ }
+ s.siteFiles[siteFileKey(a.ModuleKey, a.DocsVersion, entryKey, relName)] = SiteFile{
+ Name: relName, Content: append([]byte(nil), content...), ContentType: contentTypeForName(relName, content),
+ }
+ }
+ rel := Release{
+ ID: s.nextIDLocked("r"),
+ ReleaseID: "rel-" + strings.ToLower(a.ModuleKey) + "-" + strings.ToLower(a.DocsVersion) + "-" + strconv.FormatInt(now.UnixNano(), 36),
+ ModuleKey: a.ModuleKey,
+ DocsVersion: a.DocsVersion,
+ Publisher: firstNonEmpty(firstString(a.Authors), "docsctl"),
+ BuildSystem: "docsctl",
+ TriggerType: firstNonEmpty(a.TriggerType, "manual"),
+ SourceIP: a.SourceIP,
+ ArtifactVersion: now.Format("20060102.150405"),
+ PackageVersion: a.PackageVersion,
+ StorageURI: "memory://" + a.ModuleKey + "/" + a.DocsVersion + "/docs-artifact.zip",
+ Status: "published",
+ PublishedAt: now,
+ CreatedAt: now,
+ }
+ s.releases = append(s.releases, rel)
+ return DeployResult{Release: rel, PagesIndexed: len(a.Documents), EntriesIndexed: len(a.Entries), HTMLFiles: len(a.SiteHTML), SiteFiles: len(a.SiteFiles), BytesReceived: a.Bytes}, nil
+}
+
+func deploySourceType(repoType, repoURL string) string {
+ repoType = strings.ToLower(strings.TrimSpace(repoType))
+ if repoType == "gitlab" || strings.Contains(strings.ToLower(repoURL), "gitlab") {
+ return "gitlab"
+ }
+ return firstNonEmpty(repoType, "manual")
+}
diff --git a/backend/internal/store/memory.go b/backend/internal/store/memory.go
index e7f3f8e..6cd9f5a 100644
--- a/backend/internal/store/memory.go
+++ b/backend/internal/store/memory.go
@@ -2,11 +2,8 @@ package store
import (
"errors"
- "mime"
- "net/http"
"path"
"sort"
- "strconv"
"strings"
"sync"
"time"
@@ -18,11 +15,13 @@ var (
ErrConflict = errors.New("resource already exists")
)
-type Store struct {
+// MemoryStore is a test fake. Production assembly injects PostgresRepository.
+type MemoryStore struct {
mu sync.RWMutex
user User
users []User
- groups []Group
+ apps []ConnectedApp
+ grants []OAuthGrant
teams []Team
categories []Category
modules []Module
@@ -32,7 +31,10 @@ type Store struct {
pages []Page
searchLogs []SearchLog
mcpLogs []MCPLog
+ feedbacks []DocFeedback
pageViews []PageView
+ favorites []UserFavorite
+ recentDocs []UserRecentDoc
navs map[string][]NavItem
html map[string]string
siteFiles map[string]SiteFile
@@ -42,7 +44,7 @@ type Store struct {
}
// Settings returns a copy of the persisted platform settings.
-func (s *Store) Settings() Settings {
+func (s *MemoryStore) Settings() Settings {
s.mu.RLock()
defer s.mu.RUnlock()
return s.settings
@@ -50,24 +52,29 @@ func (s *Store) Settings() Settings {
// SaveAISettings updates the AI connection. An empty AskAPIKey keeps the
// previously stored key (so a masked round-trip from the UI does not wipe it).
-func (s *Store) SaveAISettings(ai AISettings) Settings {
+func (s *MemoryStore) SaveAISettings(ai AISettings) Settings {
s.mu.Lock()
defer s.mu.Unlock()
if strings.TrimSpace(ai.AskAPIKey) == "" {
ai.AskAPIKey = s.settings.AI.AskAPIKey
}
+ if strings.TrimSpace(ai.EmbeddingAPIKey) == "" {
+ ai.EmbeddingAPIKey = s.settings.AI.EmbeddingAPIKey
+ }
+ if strings.TrimSpace(ai.RerankAPIKey) == "" {
+ ai.RerankAPIKey = s.settings.AI.RerankAPIKey
+ }
ai.UpdatedAt = time.Now().UTC()
s.settings.AI = ai
return s.settings
}
-// New returns a completely empty store. This is the default for a fresh start
-// ("从0开始"). No demo users, categories, modules or documents are created.
-// Use NewSeeded() only for tests or when you explicitly want demo data.
-func New() *Store {
- return &Store{
+// NewTestStore returns an empty deterministic fake for unit tests.
+func NewTestStore() *MemoryStore {
+ return &MemoryStore{
users: []User{},
- groups: []Group{},
+ apps: []ConnectedApp{builtinCodexOAuthApp(time.Now().UTC())},
+ grants: []OAuthGrant{},
teams: []Team{},
categories: []Category{},
modules: []Module{},
@@ -77,7 +84,10 @@ func New() *Store {
pages: []Page{},
searchLogs: []SearchLog{},
mcpLogs: []MCPLog{},
+ feedbacks: []DocFeedback{},
pageViews: []PageView{},
+ favorites: []UserFavorite{},
+ recentDocs: []UserRecentDoc{},
navs: map[string][]NavItem{},
html: map[string]string{},
siteFiles: map[string]SiteFile{},
@@ -85,32 +95,29 @@ func New() *Store {
}
}
-func NewSeeded() *Store {
+func NewSeededTestStore() *MemoryStore {
now := time.Now().UTC()
- s := &Store{
- user: User{ID: "u-dev", Username: "dev", DisplayName: "研发用户", Email: "dev@example.com", Department: "工程化", Groups: []string{"cad-team", "engineering"}, Roles: []string{"admin"}},
+ s := &MemoryStore{
+ user: User{ID: "u-dev", Username: "dev", DisplayName: "研发用户", Email: "dev@example.com", Department: "工程化", Roles: []string{"admin"}},
users: []User{
- {ID: "u-dev", Username: "dev", DisplayName: "研发用户", Email: "dev@example.com", Department: "工程化", Groups: []string{"cad-team", "engineering"}, Roles: []string{"admin"}, Source: "seed", Status: "active", CreatedAt: now, UpdatedAt: now},
- {ID: "u-alice", Username: "alice", DisplayName: "Alice", Email: "alice@example.com", Department: "CAD", Groups: []string{"cad-team"}, Roles: []string{"maintainer"}, Source: "seed", Status: "active", CreatedAt: now, UpdatedAt: now},
- {ID: "u-bob", Username: "bob", DisplayName: "Bob", Email: "bob@example.com", Department: "前端", Groups: []string{"frontend-platform"}, Roles: []string{"viewer"}, Source: "seed", Status: "active", CreatedAt: now, UpdatedAt: now},
- },
- groups: []Group{
- {ID: "g-admin", GroupKey: "admin", Name: "平台管理员", Source: "seed", CreatedAt: now, UpdatedAt: now},
- {ID: "g-engineering", GroupKey: "engineering", Name: "工程化", Source: "seed", CreatedAt: now, UpdatedAt: now},
- {ID: "g-cad", GroupKey: "cad-team", Name: "CAD 团队", Source: "seed", CreatedAt: now, UpdatedAt: now},
- {ID: "g-frontend", GroupKey: "frontend-platform", Name: "前端平台", Source: "seed", CreatedAt: now, UpdatedAt: now},
- {ID: "g-standards", GroupKey: "standards", Name: "研发规范", Source: "seed", CreatedAt: now, UpdatedAt: now},
+ {ID: "u-dev", Username: "dev", DisplayName: "研发用户", Email: "dev@example.com", Department: "工程化", Roles: []string{"admin"}, Source: "seed", Status: "active", CreatedAt: now, UpdatedAt: now},
+ {ID: "u-alice", Username: "alice", DisplayName: "Alice", Email: "alice@example.com", Department: "CAD", Roles: []string{"maintainer"}, Source: "seed", Status: "active", CreatedAt: now, UpdatedAt: now},
+ {ID: "u-bob", Username: "bob", DisplayName: "Bob", Email: "bob@example.com", Department: "前端", Roles: []string{"viewer"}, Source: "seed", Status: "active", CreatedAt: now, UpdatedAt: now},
},
+ apps: []ConnectedApp{builtinCodexOAuthApp(now)},
+ grants: []OAuthGrant{},
teams: []Team{
- {ID: "t-cad", Key: "cad-team", Name: "CAD 团队", Description: "CAD 内核与插件文档维护团队", Leader: "alice", Members: []string{"alice", "bob"}, CreatedAt: now, UpdatedAt: now},
- {ID: "t-eng", Key: "engineering", Name: "工程化团队", Description: "工程化平台与 CBB 规范维护", Leader: "dev", Members: []string{"dev", "alice"}, CreatedAt: now, UpdatedAt: now},
- {ID: "t-fe", Key: "frontend-platform", Name: "前端平台团队", Description: "前端文档框架与组件规范", Leader: "bob", Members: []string{"bob", "dev"}, CreatedAt: now, UpdatedAt: now},
- {ID: "t-std", Key: "standards", Name: "研发规范团队", Description: "通用研发规范、流程与工具规范维护团队 (参考 GitBook/Mintlify 层级领域)", Leader: "alice", Members: []string{"alice", "dev"}, CreatedAt: now, UpdatedAt: now},
+ {ID: "t-cad", Key: "cad-team", Name: "CAD 团队", Description: "CAD 内核与插件文档维护团队", Leaders: []string{"alice"}, Members: []string{"alice", "bob"}, CreatedAt: now, UpdatedAt: now},
+ {ID: "t-eng", Key: "engineering", Name: "工程化团队", Description: "工程化平台与 CBB 规范维护", Leaders: []string{"dev"}, Members: []string{"dev", "alice"}, CreatedAt: now, UpdatedAt: now},
+ {ID: "t-fe", Key: "frontend-platform", Name: "前端平台团队", Description: "前端文档框架与组件规范", Leaders: []string{"bob"}, Members: []string{"bob", "dev"}, CreatedAt: now, UpdatedAt: now},
+ {ID: "t-std", Key: "standards", Name: "研发规范团队", Description: "通用研发规范、流程与工具规范维护团队 (参考 GitBook/Mintlify 层级领域)", Leaders: []string{"alice"}, Members: []string{"alice", "dev"}, CreatedAt: now, UpdatedAt: now},
},
navs: map[string][]NavItem{},
html: map[string]string{},
siteFiles: map[string]SiteFile{},
embeddings: map[string][]float32{},
+ favorites: []UserFavorite{},
+ recentDocs: []UserRecentDoc{},
categories: []Category{
{ID: "engineering", Key: "engineering", Name: "工程化", Description: "研发效能、构建、CI/CD 与质量平台", Icon: "wrench", SortOrder: 10, Status: "active", ResponsibleTeam: "engineering"},
{ID: "engineering.cbb", ParentID: "engineering", Key: "engineering.cbb", Name: "CBB", Description: "CBB 构建与模块治理", Icon: "package", SortOrder: 11, Status: "active", ResponsibleTeam: "engineering"},
@@ -146,17 +153,17 @@ func NewSeeded() *Store {
{ID: "e-fumadocs-guide", ModuleKey: "FumadocsKit", DocsVersion: "latest", EntryKey: "guide", Title: "Fumadocs 文档站接入", EntryType: "fumadocs", Builder: "fumadocs", Source: "content/docs", StorageURI: "minio://modex/FumadocsKit/latest/site/guide/index.html", NavURI: "minio://modex/FumadocsKit/latest/nav.json", IndexStatus: "indexed", IsPrimary: true, SortOrder: 1, Status: "active", CreatedAt: now},
}
s.pages = []Page{
- {ID: "p-demo-guide", DocID: "DemoModule:latest:guide", ModuleKey: "DemoModule", ModuleName: "DemoModule", DocsVersion: "latest", PackageVersion: "1.2.3", EntryKey: "guide", EntryType: "markdown", Title: "模块落地指导", Description: "面向业务开发人员的模块接入、部署、接口和异常处理说明。", Path: "/docs/DemoModule/latest/guide", SourceFile: "docs/integration-guide.md", DocType: "markdown", Status: "active", OwnerGroup: "cad-team", CategoryIDs: []string{"cad", "cad.demo"}, Tags: []string{"demo", "cad"}, ContentText: "模块落地指导说明如何接入 DemoModule，包括接口设计、部署运行、异常处理、风险影响面和发布检查。", UpdatedAt: now},
- {ID: "p-demo-maintenance", DocID: "DemoModule:latest:maintenance", ModuleKey: "DemoModule", ModuleName: "DemoModule", DocsVersion: "latest", PackageVersion: "1.2.3", EntryKey: "maintenance", EntryType: "markdown", Title: "模块维护说明", Description: "面向维护开发人员的架构、设计、流程和维护说明。", Path: "/docs/DemoModule/latest/maintenance", SourceFile: "docs/maintenance-guide.md", DocType: "markdown", Status: "active", OwnerGroup: "cad-team", CategoryIDs: []string{"cad", "cad.demo"}, Tags: []string{"demo", "cad", "architecture"}, ContentText: "模块维护说明包含总体架构、设计原则、模块结构、核心流程、时序逻辑、前后端设计和质量可维护性要求。", UpdatedAt: now},
- {ID: "p-cbb-build", DocID: "CBB:latest:build-cache", ModuleKey: "CBB", ModuleName: "CBB 文档", DocsVersion: "latest", PackageVersion: "2.8.0", EntryKey: "build-cache", EntryType: "markdown", Title: "构建缓存清理", Description: "CBB 构建缓存清理和常见构建问题排查。", Path: "/docs/CBB/latest/build-cache", SourceFile: "docs/build-cache.md", DocType: "markdown", Status: "active", OwnerGroup: "engineering", CategoryIDs: []string{"engineering", "engineering.cbb"}, Tags: []string{"cbb", "ci", "build"}, ContentText: "构建缓存清理用于解决依赖缓存、编译缓存和 CI 工作区残留导致的构建异常。可以重新拉取依赖并清理本地缓存。", UpdatedAt: now},
+ {ID: "p-demo-guide", DocID: "DemoModule:latest:guide", ModuleKey: "DemoModule", ModuleName: "DemoModule", DocsVersion: "latest", PackageVersion: "1.2.3", EntryKey: "guide", EntryType: "markdown", Title: "模块落地指导", Description: "面向业务开发人员的模块接入、部署、接口和异常处理说明。", Path: "/docs/DemoModule/latest/guide", SourceFile: "docs/integration-guide.md", DocType: "markdown", Status: "active", OwnerGroup: "cad-team", CategoryIDs: []string{"cad", "cad.demo"}, Tags: []string{"demo", "cad"}, ContentText: "模块落地指导说明如何接入 DemoModule，包括接口设计、部署运行、异常处理、风险影响面和发布检查。", ContentMD: seedDemoGuideMD, UpdatedAt: now},
+ {ID: "p-demo-maintenance", DocID: "DemoModule:latest:maintenance", ModuleKey: "DemoModule", ModuleName: "DemoModule", DocsVersion: "latest", PackageVersion: "1.2.3", EntryKey: "maintenance", EntryType: "markdown", Title: "模块维护说明", Description: "面向维护开发人员的架构、设计、流程和维护说明。", Path: "/docs/DemoModule/latest/maintenance", SourceFile: "docs/maintenance-guide.md", DocType: "markdown", Status: "active", OwnerGroup: "cad-team", CategoryIDs: []string{"cad", "cad.demo"}, Tags: []string{"demo", "cad", "architecture"}, ContentText: "模块维护说明包含总体架构、设计原则、模块结构、核心流程、时序逻辑、前后端设计和质量可维护性要求。", ContentMD: seedDemoMaintenanceMD, UpdatedAt: now},
+ {ID: "p-cbb-build", DocID: "CBB:latest:build-cache", ModuleKey: "CBB", ModuleName: "CBB 文档", DocsVersion: "latest", PackageVersion: "2.8.0", EntryKey: "build-cache", EntryType: "markdown", Title: "构建缓存清理", Description: "CBB 构建缓存清理和常见构建问题排查。", Path: "/docs/CBB/latest/build-cache", SourceFile: "docs/build-cache.md", DocType: "markdown", Status: "active", OwnerGroup: "engineering", CategoryIDs: []string{"engineering", "engineering.cbb"}, Tags: []string{"cbb", "ci", "build"}, ContentText: "构建缓存清理用于解决依赖缓存、编译缓存和 CI 工作区残留导致的构建异常。可以重新拉取依赖并清理本地缓存。", ContentMD: seedCBBBuildCacheMD, UpdatedAt: now},
{ID: "p-vuepress-guide", DocID: "VuePressGuide:latest:guide", ModuleKey: "VuePressGuide", ModuleName: "VuePressGuide", DocsVersion: "latest", PackageVersion: "0.4.0", EntryKey: "guide", EntryType: "vuepress", Title: "VuePress 文档站接入", Description: "VuePress 文档通过 docsctl 执行构建命令并复制 dist 输出目录。", Path: "/docs/VuePressGuide/latest/guide", SourceFile: "docs/README.md", DocType: "vuepress", Status: "active", OwnerGroup: "frontend-platform", CategoryIDs: []string{"frontend", "frontend.docs"}, Tags: []string{"vuepress", "frontend", "markdown"}, ContentText: "VuePress 文档站接入说明如何声明 docs.yaml、执行 npm run docs:build、复制 docs/.vuepress/dist 并生成标准文档包。", UpdatedAt: now.Add(-2 * time.Hour)},
{ID: "p-fumadocs-guide", DocID: "FumadocsKit:latest:guide", ModuleKey: "FumadocsKit", ModuleName: "FumadocsKit", DocsVersion: "latest", PackageVersion: "0.2.0", EntryKey: "guide", EntryType: "fumadocs", Title: "Fumadocs 文档站接入", Description: "Fumadocs 文档通过 Next.js 与 MDX 构建，适合现代前端文档站。", Path: "/docs/FumadocsKit/latest/guide", SourceFile: "content/docs/index.mdx", DocType: "fumadocs", Status: "active", OwnerGroup: "frontend-platform", CategoryIDs: []string{"frontend", "frontend.docs"}, Tags: []string{"fumadocs", "nextjs", "mdx", "frontend"}, ContentText: "Fumadocs 文档站接入说明如何维护 MDX 内容、运行 Next.js 构建、输出静态站点并交给 Modex 进行搜索和 MCP 读取。", UpdatedAt: now.Add(-90 * time.Minute)},
}
- s.releases = []Release{{ID: "r-demo-1", ReleaseID: "rel-demo-latest-001", ModuleKey: "DemoModule", DocsVersion: "latest", CommitSHA: "d34db33f", Branch: "main", Publisher: "alice", PipelineURL: "https://gitlab.example.com/cad/demo-module/-/pipelines/1", BuildSystem: "gitlab", BuildID: "1", ArtifactVersion: "20260609.1", PackageVersion: "1.2.3", StorageURI: "minio://modex/DemoModule/latest/docs-artifact.zip", Status: "published", PublishedAt: now, CreatedAt: now}}
+ s.releases = []Release{{ID: "r-demo-1", ReleaseID: "rel-demo-latest-001", ModuleKey: "DemoModule", DocsVersion: "latest", CommitSHA: "d34db33f", Branch: "main", Publisher: "alice", PipelineURL: "https://gitlab.example.com/cad/demo-module/-/pipelines/1", BuildSystem: "gitlab", BuildID: "1", TriggerType: "pipeline", SourceIP: "192.0.2.10", ArtifactVersion: "20260609.1", PackageVersion: "1.2.3", StorageURI: "minio://modex/DemoModule/latest/docs-artifact.zip", Status: "published", PublishedAt: now, CreatedAt: now}}
return s
}
-func (s *Store) CurrentUser() User {
+func (s *MemoryStore) CurrentUser() User {
s.mu.RLock()
defer s.mu.RUnlock()
return s.user
@@ -164,7 +171,7 @@ func (s *Store) CurrentUser() User {
// Users returns all users, optionally filtered by a case-insensitive keyword
// matching username, display name, email, or department.
-func (s *Store) Users(keyword string) []User {
+func (s *MemoryStore) Users(keyword string) []User {
s.mu.RLock()
defer s.mu.RUnlock()
q := strings.ToLower(strings.TrimSpace(keyword))
@@ -179,9 +186,15 @@ func (s *Store) Users(keyword string) []User {
return out
}
-func (s *Store) UserByID(id string) (User, error) {
+func (s *MemoryStore) UserByID(id string) (User, error) {
s.mu.RLock()
defer s.mu.RUnlock()
+ return s.userByIDLocked(id)
+}
+
+// userByIDLocked looks up a user without acquiring the lock; callers must hold
+// s.mu (read or write).
+func (s *MemoryStore) userByIDLocked(id string) (User, error) {
for _, u := range s.users {
if u.ID == id {
return u, nil
@@ -190,7 +203,31 @@ func (s *Store) UserByID(id string) (User, error) {
return User{}, ErrNotFound
}
-func (s *Store) CreateUser(u User) (User, error) {
+func (s *MemoryStore) UserByMCPToken(token string) (User, error) {
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ for _, u := range s.users {
+ if u.MCPToken == token {
+ return u, nil
+ }
+ }
+ return User{}, ErrNotFound
+}
+
+func (s *MemoryStore) SetUserMCPToken(id string, token string) (User, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ for i := range s.users {
+ if s.users[i].ID == id {
+ s.users[i].MCPToken = token
+ s.users[i].UpdatedAt = time.Now().UTC()
+ return s.users[i], nil
+ }
+ }
+ return User{}, ErrNotFound
+}
+
+func (s *MemoryStore) CreateUser(u User) (User, error) {
s.mu.Lock()
defer s.mu.Unlock()
if strings.TrimSpace(u.Username) == "" {
@@ -218,11 +255,10 @@ func (s *Store) CreateUser(u User) (User, error) {
u.UpdatedAt = now
// SuperAdmin can be set at creation time (usually only by another super admin via the admin UI).
s.users = append(s.users, u)
- s.ensureGroupsLocked(u.Groups)
return u, nil
}
-func (s *Store) UpdateUser(id string, patch User) (User, error) {
+func (s *MemoryStore) UpdateUser(id string, patch User) (User, error) {
s.mu.Lock()
defer s.mu.Unlock()
for i := range s.users {
@@ -239,10 +275,6 @@ func (s *Store) UpdateUser(id string, patch User) (User, error) {
if patch.Department != "" {
u.Department = patch.Department
}
- if patch.Groups != nil {
- u.Groups = patch.Groups
- s.ensureGroupsLocked(patch.Groups)
- }
if patch.Roles != nil {
u.Roles = patch.Roles
}
@@ -260,7 +292,7 @@ func (s *Store) UpdateUser(id string, patch User) (User, error) {
return User{}, ErrNotFound
}
-func (s *Store) DeleteUser(id string) error {
+func (s *MemoryStore) DeleteUser(id string) error {
s.mu.Lock()
defer s.mu.Unlock()
for i := range s.users {
@@ -273,13 +305,12 @@ func (s *Store) DeleteUser(id string) error {
}
// UpsertUser syncs an identity from the SSO provider into the user directory on
-// login, recording groups and refreshing the last-login timestamp. Manual role
-// assignments are preserved unless the provider supplies roles.
-func (s *Store) UpsertUser(u User) User {
+// login, refreshing the last-login timestamp. Manual role assignments are
+// preserved unless the provider supplies roles.
+func (s *MemoryStore) UpsertUser(u User) User {
s.mu.Lock()
defer s.mu.Unlock()
now := time.Now().UTC()
- s.ensureGroupsLocked(u.Groups)
for i := range s.users {
existing := &s.users[i]
if existing.ID == u.ID || (u.Username != "" && strings.EqualFold(existing.Username, u.Username)) {
@@ -295,9 +326,6 @@ func (s *Store) UpsertUser(u User) User {
if u.Avatar != "" {
existing.Avatar = u.Avatar
}
- if len(u.Groups) > 0 {
- existing.Groups = u.Groups
- }
if len(u.Roles) > 0 {
existing.Roles = u.Roles
}
@@ -322,79 +350,8 @@ func (s *Store) UpsertUser(u User) User {
return u
}
-func (s *Store) Groups() []Group {
- s.mu.RLock()
- defer s.mu.RUnlock()
- out := append([]Group{}, s.groups...) // non-nil
- sort.Slice(out, func(i, j int) bool { return out[i].GroupKey < out[j].GroupKey })
- return out
-}
-
-func (s *Store) CreateGroup(g Group) (Group, error) {
- s.mu.Lock()
- defer s.mu.Unlock()
- if strings.TrimSpace(g.GroupKey) == "" {
- return Group{}, ErrInvalid
- }
- for _, existing := range s.groups {
- if strings.EqualFold(existing.GroupKey, g.GroupKey) {
- return Group{}, ErrConflict
- }
- }
- if g.ID == "" {
- g.ID = s.nextIDLocked("g")
- }
- if g.Name == "" {
- g.Name = g.GroupKey
- }
- if g.Source == "" {
- g.Source = "manual"
- }
- now := time.Now().UTC()
- g.CreatedAt = now
- g.UpdatedAt = now
- s.groups = append(s.groups, g)
- return g, nil
-}
-
-// ensureGroupsLocked auto-registers any group keys referenced by a user so the
-// group directory stays consistent. Caller must hold the write lock.
-func (s *Store) ensureGroupsLocked(keys []string) {
- for _, key := range keys {
- key = strings.TrimSpace(key)
- if key == "" {
- continue
- }
- found := false
- for _, g := range s.groups {
- if strings.EqualFold(g.GroupKey, key) {
- found = true
- break
- }
- }
- if !found {
- now := time.Now().UTC()
- s.groups = append(s.groups, Group{ID: s.nextIDLocked("g"), GroupKey: key, Name: key, Source: "auto", CreatedAt: now, UpdatedAt: now})
- }
- }
-}
-
-// ensureTeamGroupLocked ensures a team's key exists as a group (for owner_group compat).
-func (s *Store) ensureTeamGroupLocked(key string) {
- if strings.TrimSpace(key) == "" {
- return
- }
- for _, g := range s.groups {
- if strings.EqualFold(g.GroupKey, key) {
- return
- }
- }
- now := time.Now().UTC()
- s.groups = append(s.groups, Group{ID: s.nextIDLocked("g"), GroupKey: key, Name: key, Source: "team", CreatedAt: now, UpdatedAt: now})
-}
-
// Teams returns all teams sorted by key.
-func (s *Store) Teams() []Team {
+func (s *MemoryStore) Teams() []Team {
s.mu.RLock()
defer s.mu.RUnlock()
out := append([]Team{}, s.teams...) // always non-nil, even if s.teams was nil
@@ -403,7 +360,7 @@ func (s *Store) Teams() []Team {
}
// Team returns a team by key (preferred) or ID.
-func (s *Store) Team(key string) (Team, error) {
+func (s *MemoryStore) Team(key string) (Team, error) {
s.mu.RLock()
defer s.mu.RUnlock()
k := strings.ToLower(strings.TrimSpace(key))
@@ -417,7 +374,7 @@ func (s *Store) Team(key string) (Team, error) {
// CreateTeam creates a maintenance team. Key is required and unique. Leader is
// auto-added to members if provided and not present.
-func (s *Store) CreateTeam(t Team) (Team, error) {
+func (s *MemoryStore) CreateTeam(t Team) (Team, error) {
s.mu.Lock()
defer s.mu.Unlock()
if strings.TrimSpace(t.Key) == "" {
@@ -437,19 +394,20 @@ func (s *Store) CreateTeam(t Team) (Team, error) {
if t.Name == "" {
t.Name = t.Key
}
- if t.Leader != "" && !contains(t.Members, t.Leader) {
- t.Members = append([]string{t.Leader}, t.Members...)
+ for _, l := range t.Leaders {
+ if l != "" && !contains(t.Members, l) {
+ t.Members = append(t.Members, l)
+ }
}
now := time.Now().UTC()
t.CreatedAt = now
t.UpdatedAt = now
s.teams = append(s.teams, t)
- s.ensureTeamGroupLocked(t.Key)
return t, nil
}
// UpdateTeam patches name/desc/leader/members.
-func (s *Store) UpdateTeam(key string, patch Team) (Team, error) {
+func (s *MemoryStore) UpdateTeam(key string, patch Team) (Team, error) {
s.mu.Lock()
defer s.mu.Unlock()
for i := range s.teams {
@@ -463,27 +421,26 @@ func (s *Store) UpdateTeam(key string, patch Team) (Team, error) {
if patch.Description != "" {
tm.Description = patch.Description
}
- if patch.Leader != "" {
- tm.Leader = patch.Leader
- if !contains(tm.Members, patch.Leader) {
- tm.Members = append(tm.Members, patch.Leader)
- }
+ if patch.Leaders != nil {
+ tm.Leaders = cloneStrings(patch.Leaders)
}
if patch.Members != nil {
tm.Members = cloneStrings(patch.Members)
- if tm.Leader != "" && !contains(tm.Members, tm.Leader) {
- tm.Members = append([]string{tm.Leader}, tm.Members...)
+ }
+ // Every leader must also be a member.
+ for _, l := range tm.Leaders {
+ if l != "" && !contains(tm.Members, l) {
+ tm.Members = append(tm.Members, l)
}
}
tm.UpdatedAt = time.Now().UTC()
- s.ensureTeamGroupLocked(tm.Key)
return *tm, nil
}
return Team{}, ErrNotFound
}
// DeleteTeam removes a team (no cascade; modules/categories keep the key as string ref).
-func (s *Store) DeleteTeam(key string) error {
+func (s *MemoryStore) DeleteTeam(key string) error {
s.mu.Lock()
defer s.mu.Unlock()
for i := range s.teams {
@@ -496,7 +453,7 @@ func (s *Store) DeleteTeam(key string) error {
}
// AddTeamMember adds a username (or id) to the team's members. Idempotent.
-func (s *Store) AddTeamMember(key, member string) (Team, error) {
+func (s *MemoryStore) AddTeamMember(key, member string) (Team, error) {
s.mu.Lock()
defer s.mu.Unlock()
member = strings.TrimSpace(member)
@@ -518,7 +475,7 @@ func (s *Store) AddTeamMember(key, member string) (Team, error) {
}
// RemoveTeamMember removes a member. Leader is not auto-removed (call SetTeamLeader first if needed).
-func (s *Store) RemoveTeamMember(key, member string) (Team, error) {
+func (s *MemoryStore) RemoveTeamMember(key, member string) (Team, error) {
s.mu.Lock()
defer s.mu.Unlock()
member = strings.TrimSpace(member)
@@ -541,7 +498,7 @@ func (s *Store) RemoveTeamMember(key, member string) (Team, error) {
}
// SetTeamLeader sets leader and ensures they are in members.
-func (s *Store) SetTeamLeader(key, leader string) (Team, error) {
+func (s *MemoryStore) SetTeamLeader(key, leader string) (Team, error) {
s.mu.Lock()
defer s.mu.Unlock()
leader = strings.TrimSpace(leader)
@@ -553,7 +510,9 @@ func (s *Store) SetTeamLeader(key, leader string) (Team, error) {
continue
}
tm := &s.teams[i]
- tm.Leader = leader
+ if !contains(tm.Leaders, leader) {
+ tm.Leaders = append(tm.Leaders, leader)
+ }
if !contains(tm.Members, leader) {
tm.Members = append(tm.Members, leader)
}
@@ -564,7 +523,7 @@ func (s *Store) SetTeamLeader(key, leader string) (Team, error) {
}
// TeamMembers returns the member list for a team (for permission/display).
-func (s *Store) TeamMembers(key string) []string {
+func (s *MemoryStore) TeamMembers(key string) []string {
s.mu.RLock()
defer s.mu.RUnlock()
for _, t := range s.teams {
@@ -575,8 +534,36 @@ func (s *Store) TeamMembers(key string) []string {
return nil
}
+// TeamKeysForUser returns the keys of every team the user belongs to, counting
+// both the designated leader and listed members.
+func (s *MemoryStore) TeamKeysForUser(u User) []string {
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ var keys []string
+ for _, t := range s.teams {
+ // Leaders are always also members, so checking Members covers both.
+ for _, m := range t.Members {
+ if strings.EqualFold(m, u.Username) || strings.EqualFold(m, u.ID) {
+ keys = append(keys, t.Key)
+ break
+ }
+ }
+ }
+ return keys
+}
+
+// AllCategories returns a flat copy of every category (with ParentID links).
+func (s *MemoryStore) AllCategories() []Category {
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ out := make([]Category, len(s.categories))
+ copy(out, s.categories)
+ s.attachResponsibleTeamInfoLocked(out)
+ return out
+}
+
// CategoryName returns the display name for a category id (or the id if unknown).
-func (s *Store) CategoryName(id string) string {
+func (s *MemoryStore) CategoryName(id string) string {
s.mu.RLock()
defer s.mu.RUnlock()
for _, c := range s.categories {
@@ -587,8 +574,26 @@ func (s *Store) CategoryName(id string) string {
return id
}
+func (s *MemoryStore) categoryPathLocked(ids []string) string {
+ if len(ids) == 0 {
+ return ""
+ }
+ var parts []string
+ for _, id := range ids {
+ name := id
+ for _, c := range s.categories {
+ if c.ID == id {
+ name = c.Name
+ break
+ }
+ }
+ parts = append(parts, name)
+ }
+ return strings.Join(parts, " / ")
+}
+
// EntryModuleKey returns the module key that owns an entry, for permission checks.
-func (s *Store) EntryModuleKey(entryID string) (string, bool) {
+func (s *MemoryStore) EntryModuleKey(entryID string) (string, bool) {
s.mu.RLock()
defer s.mu.RUnlock()
for _, e := range s.entries {
@@ -599,13 +604,14 @@ func (s *Store) EntryModuleKey(entryID string) (string, bool) {
return "", false
}
-func (s *Store) CategoryTree() []Category {
+func (s *MemoryStore) CategoryTree() []Category {
s.mu.RLock()
defer s.mu.RUnlock()
byParent := map[string][]Category{}
for _, c := range s.categories {
cp := c
cp.Children = nil
+ cp.ResponsibleTeamInfo = s.responsibleTeamInfoLocked(cp.ResponsibleTeam)
byParent[c.ParentID] = append(byParent[c.ParentID], cp)
}
var attach func(parent string) []Category
@@ -624,10 +630,34 @@ func (s *Store) CategoryTree() []Category {
return res
}
-func (s *Store) Modules(categoryID, keyword string) []Module {
+func (s *MemoryStore) attachResponsibleTeamInfoLocked(categories []Category) {
+ for i := range categories {
+ categories[i].ResponsibleTeamInfo = s.responsibleTeamInfoLocked(categories[i].ResponsibleTeam)
+ }
+}
+
+func (s *MemoryStore) responsibleTeamInfoLocked(teamKey string) *TeamSummary {
+ if strings.TrimSpace(teamKey) == "" {
+ return nil
+ }
+ for _, t := range s.teams {
+ if strings.EqualFold(t.Key, teamKey) {
+ return &TeamSummary{
+ Key: t.Key,
+ Name: t.Name,
+ Description: t.Description,
+ Leaders: cloneStrings(t.Leaders),
+ Members: cloneStrings(t.Members),
+ }
+ }
+ }
+ return nil
+}
+
+func (s *MemoryStore) Modules(categoryID, keyword string) []Module {
s.mu.RLock()
defer s.mu.RUnlock()
- var out []Module
+ out := make([]Module, 0)
q := strings.ToLower(keyword)
for _, m := range s.modules {
if categoryID != "" && !contains(m.CategoryIDs, categoryID) {
@@ -638,33 +668,53 @@ func (s *Store) Modules(categoryID, keyword string) []Module {
}
cp := m
cp.AvailableVers = s.versionsForLocked(m.ModuleKey)
+ cp.DeployTokenSet = cp.DeployToken != ""
out = append(out, cp)
}
return out
}
-func (s *Store) Module(moduleKey string) (Module, error) {
+func (s *MemoryStore) Module(moduleKey string) (Module, error) {
s.mu.RLock()
defer s.mu.RUnlock()
for _, m := range s.modules {
if strings.EqualFold(m.ModuleKey, moduleKey) {
m.AvailableVers = s.versionsForLocked(m.ModuleKey)
+ m.DeployTokenSet = m.DeployToken != ""
return m, nil
}
}
return Module{}, ErrNotFound
}
-func (s *Store) Versions(moduleKey string) []Version {
+func (s *MemoryStore) ModuleByDeployToken(token string) (Module, error) {
+ token = strings.TrimSpace(token)
+ if token == "" {
+ return Module{}, ErrNotFound
+ }
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ for _, m := range s.modules {
+ if m.DeployToken == token {
+ cp := m
+ cp.DeployTokenSet = cp.DeployToken != ""
+ cp.AvailableVers = s.versionsForLocked(m.ModuleKey)
+ return cp, nil
+ }
+ }
+ return Module{}, ErrNotFound
+}
+
+func (s *MemoryStore) Versions(moduleKey string) []Version {
s.mu.RLock()
defer s.mu.RUnlock()
return s.versionsForLocked(moduleKey)
}
-func (s *Store) Entries(moduleKey, docsVersion string) []Entry {
+func (s *MemoryStore) Entries(moduleKey, docsVersion string) []Entry {
s.mu.RLock()
defer s.mu.RUnlock()
- var out []Entry
+ out := make([]Entry, 0)
for _, e := range s.entries {
if strings.EqualFold(e.ModuleKey, moduleKey) && e.DocsVersion == docsVersion {
out = append(out, e)
@@ -674,7 +724,7 @@ func (s *Store) Entries(moduleKey, docsVersion string) []Entry {
return out
}
-func (s *Store) Releases() []Release {
+func (s *MemoryStore) Releases() []Release {
s.mu.RLock()
defer s.mu.RUnlock()
out := append([]Release(nil), s.releases...)
@@ -682,7 +732,7 @@ func (s *Store) Releases() []Release {
return out
}
-func (s *Store) Page(docID string) (Page, error) {
+func (s *MemoryStore) Page(docID string) (Page, error) {
s.mu.RLock()
defer s.mu.RUnlock()
for _, p := range s.pages {
@@ -693,7 +743,7 @@ func (s *Store) Page(docID string) (Page, error) {
return Page{}, ErrNotFound
}
-func (s *Store) PageByRoute(moduleKey, docsVersion, entryKey string) (Page, error) {
+func (s *MemoryStore) PageByRoute(moduleKey, docsVersion, entryKey string) (Page, error) {
s.mu.RLock()
defer s.mu.RUnlock()
for _, p := range s.pages {
@@ -704,19 +754,19 @@ func (s *Store) PageByRoute(moduleKey, docsVersion, entryKey string) (Page, erro
return Page{}, ErrNotFound
}
-func (s *Store) Nav(moduleKey, docsVersion string) []NavItem {
+func (s *MemoryStore) Nav(moduleKey, docsVersion string) []NavItem {
s.mu.RLock()
defer s.mu.RUnlock()
return cloneNav(s.navs[routeKey(moduleKey, docsVersion, "")])
}
-func (s *Store) PageHTML(moduleKey, docsVersion, entryKey string) string {
+func (s *MemoryStore) PageHTML(moduleKey, docsVersion, entryKey string) string {
s.mu.RLock()
defer s.mu.RUnlock()
return s.html[routeKey(moduleKey, docsVersion, entryKey)]
}
-func (s *Store) SiteFile(moduleKey, docsVersion, entryKey, name string) (SiteFile, error) {
+func (s *MemoryStore) SiteFile(moduleKey, docsVersion, entryKey, name string) (SiteFile, error) {
s.mu.RLock()
defer s.mu.RUnlock()
if name == "" {
@@ -730,14 +780,14 @@ func (s *Store) SiteFile(moduleKey, docsVersion, entryKey, name string) (SiteFil
return f, nil
}
-func (s *Store) Pages() []Page {
+func (s *MemoryStore) Pages() []Page {
s.mu.RLock()
defer s.mu.RUnlock()
return append([]Page(nil), s.pages...)
}
// Embedding returns the cached embedding vector for a document, if present.
-func (s *Store) Embedding(docID string) ([]float32, bool) {
+func (s *MemoryStore) Embedding(docID string) ([]float32, bool) {
s.mu.RLock()
defer s.mu.RUnlock()
v, ok := s.embeddings[docID]
@@ -748,7 +798,7 @@ func (s *Store) Embedding(docID string) ([]float32, bool) {
}
// SetEmbedding stores (or replaces) the embedding vector for a document.
-func (s *Store) SetEmbedding(docID string, vec []float32) {
+func (s *MemoryStore) SetEmbedding(docID string, vec []float32) {
s.mu.Lock()
defer s.mu.Unlock()
if s.embeddings == nil {
@@ -758,1116 +808,57 @@ func (s *Store) SetEmbedding(docID string, vec []float32) {
}
// EmbeddingCount reports how many documents currently have cached embeddings.
-func (s *Store) EmbeddingCount() int {
+func (s *MemoryStore) EmbeddingCount() int {
s.mu.RLock()
defer s.mu.RUnlock()
return len(s.embeddings)
}
// ClearEmbeddings drops the entire embedding cache (used before a full reindex).
-func (s *Store) ClearEmbeddings() {
+func (s *MemoryStore) ClearEmbeddings() {
s.mu.Lock()
defer s.mu.Unlock()
s.embeddings = map[string][]float32{}
}
-func (s *Store) IngestArtifact(a DeployArtifact) (DeployResult, error) {
- if strings.TrimSpace(a.ModuleKey) == "" || strings.TrimSpace(a.DocsVersion) == "" || len(a.Entries) == 0 || len(a.Documents) == 0 {
- return DeployResult{}, ErrInvalid
- }
- s.mu.Lock()
- defer s.mu.Unlock()
- now := time.Now().UTC()
- moduleName := firstNonEmpty(a.ModuleName, a.ModuleKey)
- moduleIdx, err := s.moduleIndexLocked(a.ModuleKey)
- if err != nil {
- s.modules = append(s.modules, Module{
- ID: s.nextIDLocked("m"),
- ModuleKey: a.ModuleKey,
- Name: moduleName,
- Description: a.Description,
- OwnerGroup: firstNonEmpty(firstString(a.Authors), "docs"),
- RepoType: firstNonEmpty(a.RepoType, "git"),
- RepoURL: a.RepoURL,
- SourceType: "gitlab",
- GitLabBranch: a.Branch,
- DefaultVersion: a.DocsVersion,
- Visibility: "internal",
- Status: "active",
- PackageName: a.ModuleKey,
- PackageVersion: a.PackageVersion,
- Channel: "docs",
- Edition: a.Edition,
- Keywords: cloneStrings(a.Keywords),
- Maintainers: cloneStrings(a.Authors),
- LastSyncedCommit: a.CommitSHA,
- LastSyncedAt: now,
- UpdatedAt: now,
- })
- moduleIdx = len(s.modules) - 1
- } else {
- m := &s.modules[moduleIdx]
- m.Name = moduleName
- if a.Description != "" {
- m.Description = a.Description
- }
- m.DefaultVersion = a.DocsVersion
- if a.PackageVersion != "" {
- m.PackageVersion = a.PackageVersion
- }
- if a.Edition != "" {
- m.Edition = a.Edition
- }
- if len(a.Keywords) > 0 {
- m.Keywords = cloneStrings(a.Keywords)
- }
- if len(a.Authors) > 0 {
- m.Maintainers = cloneStrings(a.Authors)
- if m.OwnerGroup == "" {
- m.OwnerGroup = a.Authors[0]
- }
- }
- if m.Status == "" {
- m.Status = "active"
- }
- if m.Visibility == "" {
- m.Visibility = "internal"
- }
- // Refresh source metadata from each CI push (repo/branch/commit).
- if a.RepoURL != "" {
- m.RepoURL = a.RepoURL
- }
- if a.RepoType != "" {
- m.RepoType = a.RepoType
- }
- if a.Branch != "" {
- m.GitLabBranch = a.Branch
- }
- if a.CommitSHA != "" {
- m.LastSyncedCommit = a.CommitSHA
- }
- m.LastSyncedAt = now
- m.UpdatedAt = now
- }
- module := s.modules[moduleIdx]
- versionFound := false
- for i := range s.versions {
- if strings.EqualFold(s.versions[i].ModuleKey, a.ModuleKey) && s.versions[i].DocsVersion == a.DocsVersion {
- v := &s.versions[i]
- v.DisplayName = firstNonEmpty(v.DisplayName, a.DocsVersion)
- v.IsDefault = true
- v.Status = "active"
- v.PackageVersion = a.PackageVersion
- v.Edition = a.Edition
- if v.VersionType == "" {
- v.VersionType = "release"
- }
- if v.SupportStatus == "" {
- v.SupportStatus = "supported"
- }
- versionFound = true
- } else if strings.EqualFold(s.versions[i].ModuleKey, a.ModuleKey) {
- s.versions[i].IsDefault = false
- }
- }
- if !versionFound {
- s.versions = append(s.versions, Version{
- ID: s.nextIDLocked("v"),
- ModuleKey: a.ModuleKey,
- DocsVersion: a.DocsVersion,
- DisplayName: a.DocsVersion,
- VersionType: "release",
- IsDefault: true,
- Status: "active",
- PackageVersion: a.PackageVersion,
- Channel: firstNonEmpty(module.Channel, "docs"),
- Edition: a.Edition,
- SupportStatus: "supported",
- CreatedAt: now,
- })
- }
- s.entries = removeEntries(s.entries, a.ModuleKey, a.DocsVersion)
- for i, e := range a.Entries {
- s.entries = append(s.entries, Entry{
- ID: s.nextIDLocked("e"),
- ModuleKey: a.ModuleKey,
- DocsVersion: a.DocsVersion,
- EntryKey: e.Key,
- Title: e.Title,
- EntryType: firstNonEmpty(e.Type, "markdown"),
- Builder: firstNonEmpty(e.Type, "markdown"),
- Source: e.Source,
- StorageURI: "memory://" + routeKey(a.ModuleKey, a.DocsVersion, e.Key),
- NavURI: "memory://" + routeKey(a.ModuleKey, a.DocsVersion, ""),
- IndexStatus: "indexed",
- IsPrimary: i == 0,
- SortOrder: i + 1,
- Status: "active",
- CreatedAt: now,
- })
- }
- s.pages = removePages(s.pages, a.ModuleKey, a.DocsVersion)
- // Drop cached embeddings for this module/version so re-published content is
- // re-embedded on the next reindex (or lazily during search).
- if s.embeddings != nil {
- embPrefix := a.ModuleKey + ":" + a.DocsVersion + ":"
- for docID := range s.embeddings {
- if strings.HasPrefix(docID, embPrefix) {
- delete(s.embeddings, docID)
- }
- }
- }
- for _, d := range a.Documents {
- entryKey := firstNonEmpty(d.EntryKey, entryKeyFromDocID(d.DocID))
- docID := firstNonEmpty(d.DocID, a.ModuleKey+":"+a.DocsVersion+":"+entryKey)
- s.pages = append(s.pages, Page{
- ID: s.nextIDLocked("p"),
- DocID: docID,
- ModuleKey: a.ModuleKey,
- ModuleName: moduleName,
- DocsVersion: a.DocsVersion,
- PackageVersion: firstNonEmpty(d.PackageVersion, a.PackageVersion),
- EntryKey: entryKey,
- EntryType: firstNonEmpty(d.EntryType, entryTypeForEntry(a.Entries, entryKey)),
- Title: firstNonEmpty(d.Title, titleForEntry(a.Entries, entryKey)),
- Description: d.Description,
- Path: "/docs/" + a.ModuleKey + "/" + a.DocsVersion + "/" + entryKey,
- SourceFile: d.SourceFile,
- DocType: firstNonEmpty(d.EntryType, entryTypeForEntry(a.Entries, entryKey)),
- Status: firstNonEmpty(d.Status, "active"),
- OwnerGroup: module.OwnerGroup,
- CategoryIDs: cloneStrings(module.CategoryIDs),
- Tags: cloneStrings(coalesceStrings(d.Keywords, a.Keywords)),
- ContentText: d.Content,
- UpdatedAt: now,
- })
- }
- if s.navs == nil {
- s.navs = map[string][]NavItem{}
- }
- s.navs[routeKey(a.ModuleKey, a.DocsVersion, "")] = cloneNav(a.Nav)
- if s.html == nil {
- s.html = map[string]string{}
- }
- if s.siteFiles == nil {
- s.siteFiles = map[string]SiteFile{}
- }
- for k := range s.html {
- prefix := routeKey(a.ModuleKey, a.DocsVersion, "") + ":"
- if strings.HasPrefix(k, prefix) {
- delete(s.html, k)
- }
- }
- for k := range s.siteFiles {
- prefix := routeKey(a.ModuleKey, a.DocsVersion, "") + ":"
- if strings.HasPrefix(k, prefix) {
- delete(s.siteFiles, k)
- }
- }
- for _, e := range a.Entries {
- html := htmlForEntry(a.SiteHTML, e.Key)
- if html != "" {
- s.html[routeKey(a.ModuleKey, a.DocsVersion, e.Key)] = html
- }
- }
- for name, content := range a.SiteFiles {
- entryKey, relName, ok := splitSiteFile(name)
- if !ok {
- continue
- }
- s.siteFiles[siteFileKey(a.ModuleKey, a.DocsVersion, entryKey, relName)] = SiteFile{
- Name: relName, Content: append([]byte(nil), content...), ContentType: contentTypeForName(relName, content),
- }
- }
- rel := Release{
- ID: s.nextIDLocked("r"),
- ReleaseID: "rel-" + strings.ToLower(a.ModuleKey) + "-" + strings.ToLower(a.DocsVersion) + "-" + strconv.FormatInt(now.UnixNano(), 36),
- ModuleKey: a.ModuleKey,
- DocsVersion: a.DocsVersion,
- Publisher: firstNonEmpty(firstString(a.Authors), "docsctl"),
- BuildSystem: "docsctl",
- ArtifactVersion: now.Format("20060102.150405"),
- PackageVersion: a.PackageVersion,
- StorageURI: "memory://" + a.ModuleKey + "/" + a.DocsVersion + "/docs-artifact.zip",
- Status: "published",
- PublishedAt: now,
- CreatedAt: now,
- }
- s.releases = append(s.releases, rel)
- return DeployResult{Release: rel, PagesIndexed: len(a.Documents), EntriesIndexed: len(a.Entries), HTMLFiles: len(a.SiteHTML), SiteFiles: len(a.SiteFiles), BytesReceived: a.Bytes}, nil
-}
-
-func (s *Store) AddSearchLog(log SearchLog) {
- s.mu.Lock()
- defer s.mu.Unlock()
- s.searchLogs = append(s.searchLogs, log)
-}
-
-func (s *Store) SearchLogs() []SearchLog {
- s.mu.RLock()
- defer s.mu.RUnlock()
- return append([]SearchLog(nil), s.searchLogs...)
-}
-
-func (s *Store) AddMCPLog(log MCPLog) {
- s.mu.Lock()
- defer s.mu.Unlock()
- s.mcpLogs = append(s.mcpLogs, log)
-}
-
-func (s *Store) MCPLogs() []MCPLog {
- s.mu.RLock()
- defer s.mu.RUnlock()
- return append([]MCPLog(nil), s.mcpLogs...)
-}
-
-// RecordPageView appends a page view and returns the stored record.
-func (s *Store) RecordPageView(pv PageView) PageView {
- s.mu.Lock()
- defer s.mu.Unlock()
- if pv.ViewedAt.IsZero() {
- pv.ViewedAt = time.Now().UTC()
- }
- if pv.ID == "" {
- pv.ID = s.nextIDLocked("pv")
- }
- for _, p := range s.pages {
- if p.DocID == pv.DocID {
- pv.PageID = p.ID
- pv.ModuleKey = p.ModuleKey
- pv.DocsVersion = p.DocsVersion
- break
- }
- }
- s.pageViews = append(s.pageViews, pv)
- return pv
-}
-
-// RecordReadProgress updates the latest matching page view with duration and
-// scroll depth for the given session and doc, or records a new view if none.
-func (s *Store) RecordReadProgress(docID, sessionID string, durationSeconds int, scrollDepth float64) {
- s.mu.Lock()
- defer s.mu.Unlock()
- for i := len(s.pageViews) - 1; i >= 0; i-- {
- pv := &s.pageViews[i]
- if pv.DocID == docID && pv.SessionID == sessionID {
- if durationSeconds > pv.DurationSeconds {
- pv.DurationSeconds = durationSeconds
- }
- if scrollDepth > pv.ScrollDepth {
- pv.ScrollDepth = scrollDepth
- }
- return
- }
- }
- s.pageViews = append(s.pageViews, PageView{
- ID: s.nextIDLocked("pv"), DocID: docID, SessionID: sessionID,
- DurationSeconds: durationSeconds, ScrollDepth: scrollDepth, ViewedAt: time.Now().UTC(),
- })
-}
-
-// PageAnalytics aggregates recorded views into per-page reading statistics.
-// Pages with no recorded views fall back to seeded read counts so the admin
-// dashboard is populated on a fresh start.
-func (s *Store) PageAnalytics() []PageStat {
- s.mu.RLock()
- defer s.mu.RUnlock()
- now := time.Now().UTC()
- week := now.AddDate(0, 0, -7)
- month := now.AddDate(0, 0, -30)
- type agg struct {
- pv, reads7, reads30, durSum, durCount int
- users map[string]struct{}
- last time.Time
- }
- byDoc := map[string]*agg{}
- for _, pv := range s.pageViews {
- a := byDoc[pv.DocID]
- if a == nil {
- a = &agg{users: map[string]struct{}{}}
- byDoc[pv.DocID] = a
- }
- a.pv++
- uid := pv.UserID
- if uid == "" {
- uid = pv.SessionID
- }
- if uid != "" {
- a.users[uid] = struct{}{}
- }
- if pv.ViewedAt.After(week) {
- a.reads7++
- }
- if pv.ViewedAt.After(month) {
- a.reads30++
- }
- if pv.DurationSeconds > 0 {
- a.durSum += pv.DurationSeconds
- a.durCount++
- }
- if pv.ViewedAt.After(a.last) {
- a.last = pv.ViewedAt
- }
- }
- var out []PageStat
- for _, p := range s.pages {
- stat := PageStat{DocID: p.DocID, Title: p.Title, ModuleKey: p.ModuleKey, ModuleName: p.ModuleName, DocsVersion: p.DocsVersion, Path: p.Path, LastViewedAt: p.UpdatedAt}
- if a := byDoc[p.DocID]; a != nil {
- stat.PV = a.pv
- stat.UV = len(a.users)
- stat.Reads7d = a.reads7
- stat.Reads30d = a.reads30
- stat.LastViewedAt = a.last
- if a.durCount > 0 {
- stat.AvgDurationSec = a.durSum / a.durCount
- }
- } else {
- stat.Reads7d = s.seedReadsLocked(p.ModuleKey, true)
- stat.Reads30d = s.seedReadsLocked(p.ModuleKey, false)
- }
- out = append(out, stat)
- }
- sort.Slice(out, func(i, j int) bool {
- if out[i].PV != out[j].PV {
- return out[i].PV > out[j].PV
- }
- return out[i].Reads30d > out[j].Reads30d
- })
- return out
-}
-
-func (s *Store) seedReadsLocked(moduleKey string, week bool) int {
- for _, m := range s.modules {
- if strings.EqualFold(m.ModuleKey, moduleKey) {
- if week {
- return m.Reads7d
- }
- return m.Reads30d
- }
- }
- return 0
-}
-
-// CreateCategory adds a new category. Key is required and must be unique.
-func (s *Store) CreateCategory(c Category) (Category, error) {
- s.mu.Lock()
- defer s.mu.Unlock()
- // Key is system-generated from the name (dotted under the parent's key) so
- // users never have to invent one. A user-supplied key is still honored.
- if strings.TrimSpace(c.Key) == "" {
- if strings.TrimSpace(c.Name) == "" {
- return Category{}, ErrInvalid
- }
- c.Key = s.generateCategoryKeyLocked(c.Name, c.ParentID)
- }
- if c.ID == "" {
- c.ID = c.Key
- }
- for _, existing := range s.categories {
- if existing.ID == c.ID {
- return Category{}, ErrConflict
- }
- }
- if c.Status == "" {
- c.Status = "active"
- }
- c.Children = nil
- s.categories = append(s.categories, c)
- return c, nil
-}
-
-// slugifyKey lowercases and keeps [a-z0-9-]; non-ASCII (e.g. Chinese) collapses
-// to empty, in which case callers fall back to a short unique token.
-func slugifyKey(name string) string {
- var b strings.Builder
- prevDash := false
- for _, r := range strings.ToLower(strings.TrimSpace(name)) {
- switch {
- case (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9'):
- b.WriteRune(r)
- prevDash = false
- case r == ' ' || r == '-' || r == '_' || r == '.' || r == '/':
- if !prevDash && b.Len() > 0 {
- b.WriteByte('-')
- prevDash = true
- }
- }
- }
- return strings.Trim(b.String(), "-")
-}
-
-func (s *Store) generateCategoryKeyLocked(name, parentID string) string {
- base := slugifyKey(name)
- if base == "" {
- base = "d" + strconv.FormatInt(time.Now().UnixNano()%1_000_000, 36)
- }
- prefix := ""
- for _, c := range s.categories {
- if c.ID == parentID {
- prefix = c.Key + "."
- break
- }
- }
- taken := func(k string) bool {
- for _, c := range s.categories {
- if c.Key == k || c.ID == k {
- return true
- }
- }
- return false
- }
- key := prefix + base
- for i := 2; taken(key); i++ {
- key = prefix + base + "-" + strconv.Itoa(i)
- }
- return key
-}
-
-func (s *Store) generateTeamKeyLocked(name string) string {
- base := slugifyKey(name)
- if base == "" {
- base = "team-" + strconv.FormatInt(time.Now().UnixNano()%1_000_000, 36)
- }
- taken := func(k string) bool {
- for _, t := range s.teams {
- if strings.EqualFold(t.Key, k) {
- return true
- }
- }
- return false
- }
- key := base
- for i := 2; taken(key); i++ {
- key = base + "-" + strconv.Itoa(i)
- }
- return key
-}
-
-// MoveCategory reparents a category and positions it at `index` among its new
-// siblings, renumbering sibling SortOrder so the tree order is stable. Rejects
-// moves that would create a cycle (into the node's own subtree).
-func (s *Store) MoveCategory(id, parentID string, index int) (Category, error) {
+// ClearSiteAssets drops cached rendered HTML and static site files from memory.
+// Deployed environments can serve these assets from MinIO instead, which keeps
+// the backend RSS from growing with image-heavy or large documentation sites.
+func (s *MemoryStore) ClearSiteAssets() {
s.mu.Lock()
defer s.mu.Unlock()
-
- idx := -1
- for i := range s.categories {
- if s.categories[i].ID == id {
- idx = i
- break
- }
- }
- if idx == -1 {
- return Category{}, ErrNotFound
- }
- if parentID == id {
- return Category{}, ErrInvalid
- }
- // Walk parent chain to reject cycles.
- for p := parentID; p != ""; {
- if p == id {
- return Category{}, ErrInvalid
- }
- next := ""
- for i := range s.categories {
- if s.categories[i].ID == p {
- next = s.categories[i].ParentID
- break
- }
- }
- p = next
- }
- if parentID != "" {
- found := false
- for i := range s.categories {
- if s.categories[i].ID == parentID {
- found = true
- break
- }
- }
- if !found {
- return Category{}, ErrInvalid
- }
- }
-
- s.categories[idx].ParentID = parentID
-
- // Collect new siblings (same parent) in current order, excluding the moved
- // node, then insert it at the requested index and renumber.
- var sibs []int
- for i := range s.categories {
- if s.categories[i].ParentID == parentID && s.categories[i].ID != id {
- sibs = append(sibs, i)
- }
- }
- sort.SliceStable(sibs, func(a, b int) bool { return s.categories[sibs[a]].SortOrder < s.categories[sibs[b]].SortOrder })
- order := make([]int, 0, len(sibs)+1)
- if index < 0 {
- index = 0
- }
- if index > len(sibs) {
- index = len(sibs)
- }
- order = append(order, sibs[:index]...)
- order = append(order, idx)
- order = append(order, sibs[index:]...)
- for pos, ci := range order {
- s.categories[ci].SortOrder = (pos + 1) * 10
- }
- out := s.categories[idx]
- out.Children = nil
- return out, nil
-}
-
-func (s *Store) UpdateCategory(id string, c Category) (Category, error) {
- s.mu.Lock()
- defer s.mu.Unlock()
- for i := range s.categories {
- if s.categories[i].ID == id {
- if c.Name != "" {
- s.categories[i].Name = c.Name
- }
- if c.Description != "" {
- s.categories[i].Description = c.Description
- }
- if c.Icon != "" {
- s.categories[i].Icon = c.Icon
- }
- if c.SortOrder != 0 {
- s.categories[i].SortOrder = c.SortOrder
- }
- if c.Status != "" {
- s.categories[i].Status = c.Status
- }
- if c.ParentID != "" {
- s.categories[i].ParentID = c.ParentID
- }
- // Always accept ResponsibleTeam from patch (send "" explicitly to clear assignment to a team).
- s.categories[i].ResponsibleTeam = c.ResponsibleTeam
- out := s.categories[i]
- out.Children = nil
- return out, nil
- }
- }
- return Category{}, ErrNotFound
-}
-
-func (s *Store) DeleteCategory(id string) error {
- s.mu.Lock()
- defer s.mu.Unlock()
- for _, c := range s.categories {
- if c.ParentID == id {
- return ErrConflict
- }
- }
- for i := range s.categories {
- if s.categories[i].ID == id {
- s.categories = append(s.categories[:i], s.categories[i+1:]...)
- return nil
- }
- }
- return ErrNotFound
-}
-
-func (s *Store) moduleKeyTakenLocked(key string) bool {
- for _, m := range s.modules {
- if strings.EqualFold(m.ModuleKey, key) {
- return true
- }
- }
- return false
-}
-
-func (s *Store) CreateModule(m Module) (Module, error) {
- s.mu.Lock()
- defer s.mu.Unlock()
- // Auto-generate module_key from the name (slug, unique) so admins never type one.
- if strings.TrimSpace(m.ModuleKey) == "" {
- if strings.TrimSpace(m.Name) == "" {
- return Module{}, ErrInvalid
- }
- base := slugifyKey(m.Name)
- if base == "" {
- base = "doc-" + strconv.FormatInt(time.Now().UnixNano()%1_000_000, 36)
- }
- key := base
- for i := 2; s.moduleKeyTakenLocked(key); i++ {
- key = base + "-" + strconv.Itoa(i)
- }
- m.ModuleKey = key
- }
- for _, existing := range s.modules {
- if strings.EqualFold(existing.ModuleKey, m.ModuleKey) {
- return Module{}, ErrConflict
- }
- }
- if m.ID == "" {
- m.ID = s.nextIDLocked("m")
- }
- // Every doc source gets a deploy token for CI push auth.
- if strings.TrimSpace(m.DeployToken) == "" {
- m.DeployToken = "mdx_" + strconv.FormatInt(time.Now().UnixNano(), 36) + strconv.FormatInt(int64(len(s.modules)+1), 36)
- }
- if m.Name == "" {
- m.Name = m.ModuleKey
- }
- if m.Status == "" {
- m.Status = "active"
- }
- if m.DefaultVersion == "" {
- m.DefaultVersion = "latest"
- }
- m.UpdatedAt = time.Now().UTC()
- m.AvailableVers = nil
- s.modules = append(s.modules, m)
- return m, nil
-}
-
-func (s *Store) UpdateModule(moduleKey string, patch Module) (Module, error) {
- s.mu.Lock()
- defer s.mu.Unlock()
- for i := range s.modules {
- if strings.EqualFold(s.modules[i].ModuleKey, moduleKey) {
- m := &s.modules[i]
- if patch.Name != "" {
- m.Name = patch.Name
- }
- if patch.Description != "" {
- m.Description = patch.Description
- }
- if patch.OwnerGroup != "" {
- m.OwnerGroup = patch.OwnerGroup
- }
- if patch.RepoType != "" {
- m.RepoType = patch.RepoType
- }
- if patch.RepoURL != "" {
- m.RepoURL = patch.RepoURL
- }
- if patch.DefaultVersion != "" {
- m.DefaultVersion = patch.DefaultVersion
- }
- if patch.Visibility != "" {
- m.Visibility = patch.Visibility
- }
- if patch.Status != "" {
- m.Status = patch.Status
- }
- if patch.PackageVersion != "" {
- m.PackageVersion = patch.PackageVersion
- }
- if patch.Channel != "" {
- m.Channel = patch.Channel
- }
- if patch.Edition != "" {
- m.Edition = patch.Edition
- }
- if patch.Keywords != nil {
- m.Keywords = patch.Keywords
- }
- if patch.Maintainers != nil {
- m.Maintainers = patch.Maintainers
- }
- if patch.CategoryIDs != nil {
- m.CategoryIDs = patch.CategoryIDs
- }
- if patch.CategoryPath != "" {
- m.CategoryPath = patch.CategoryPath
- }
- if patch.SourceType != "" {
- m.SourceType = patch.SourceType
- }
- if patch.DocType != "" {
- m.DocType = patch.DocType
- }
- if patch.Mount != "" {
- m.Mount = patch.Mount
- }
- if patch.GitLabBranch != "" {
- m.GitLabBranch = patch.GitLabBranch
- }
- if patch.GitLabPath != "" {
- m.GitLabPath = patch.GitLabPath
- }
- if patch.DeployToken != "" {
- m.DeployToken = patch.DeployToken
- }
- m.UpdatedAt = time.Now().UTC()
- out := *m
- out.AvailableVers = s.versionsForLocked(m.ModuleKey)
- return out, nil
- }
- }
- return Module{}, ErrNotFound
-}
-
-func (s *Store) CreateVersion(moduleKey string, v Version) (Version, error) {
- s.mu.Lock()
- defer s.mu.Unlock()
- if _, err := s.moduleIndexLocked(moduleKey); err != nil {
- return Version{}, err
- }
- if strings.TrimSpace(v.DocsVersion) == "" {
- return Version{}, ErrInvalid
- }
- for _, existing := range s.versions {
- if strings.EqualFold(existing.ModuleKey, moduleKey) && existing.DocsVersion == v.DocsVersion {
- return Version{}, ErrConflict
- }
- }
- v.ModuleKey = moduleKey
- if v.ID == "" {
- v.ID = s.nextIDLocked("v")
- }
- if v.DisplayName == "" {
- v.DisplayName = v.DocsVersion
- }
- if v.Status == "" {
- v.Status = "active"
- }
- v.CreatedAt = time.Now().UTC()
- if v.IsDefault {
- for i := range s.versions {
- if strings.EqualFold(s.versions[i].ModuleKey, moduleKey) {
- s.versions[i].IsDefault = false
- }
- }
- if idx, err := s.moduleIndexLocked(moduleKey); err == nil {
- s.modules[idx].DefaultVersion = v.DocsVersion
- }
- }
- s.versions = append(s.versions, v)
- return v, nil
-}
-
-func (s *Store) UpdateVersion(moduleKey, docsVersion string, patch Version) (Version, error) {
- s.mu.Lock()
- defer s.mu.Unlock()
- for i := range s.versions {
- if strings.EqualFold(s.versions[i].ModuleKey, moduleKey) && s.versions[i].DocsVersion == docsVersion {
- v := &s.versions[i]
- if patch.DisplayName != "" {
- v.DisplayName = patch.DisplayName
- }
- if patch.VersionType != "" {
- v.VersionType = patch.VersionType
- }
- if patch.Status != "" {
- v.Status = patch.Status
- }
- if patch.SourceBranch != "" {
- v.SourceBranch = patch.SourceBranch
- }
- if patch.PackageVersion != "" {
- v.PackageVersion = patch.PackageVersion
- }
- if patch.Channel != "" {
- v.Channel = patch.Channel
- }
- if patch.Edition != "" {
- v.Edition = patch.Edition
- }
- if patch.SupportStatus != "" {
- v.SupportStatus = patch.SupportStatus
- }
- if patch.IsDefault {
- for j := range s.versions {
- if strings.EqualFold(s.versions[j].ModuleKey, moduleKey) {
- s.versions[j].IsDefault = false
- }
- }
- v.IsDefault = true
- if idx, err := s.moduleIndexLocked(moduleKey); err == nil {
- s.modules[idx].DefaultVersion = v.DocsVersion
- }
- }
- return *v, nil
- }
- }
- return Version{}, ErrNotFound
-}
-
-func (s *Store) CreateEntry(moduleKey, docsVersion string, e Entry) (Entry, error) {
- s.mu.Lock()
- defer s.mu.Unlock()
- if _, err := s.moduleIndexLocked(moduleKey); err != nil {
- return Entry{}, err
- }
- if strings.TrimSpace(e.EntryKey) == "" {
- return Entry{}, ErrInvalid
- }
- for _, existing := range s.entries {
- if strings.EqualFold(existing.ModuleKey, moduleKey) && existing.DocsVersion == docsVersion && existing.EntryKey == e.EntryKey {
- return Entry{}, ErrConflict
- }
- }
- e.ModuleKey = moduleKey
- e.DocsVersion = docsVersion
- if e.ID == "" {
- e.ID = s.nextIDLocked("e")
- }
- if e.EntryType == "" {
- e.EntryType = "markdown"
- }
- if e.Builder == "" {
- e.Builder = e.EntryType
- }
- if e.IndexStatus == "" {
- e.IndexStatus = "pending"
- }
- if e.Status == "" {
- e.Status = "active"
- }
- e.CreatedAt = time.Now().UTC()
- s.entries = append(s.entries, e)
- return e, nil
-}
-
-func (s *Store) UpdateEntry(entryID string, patch Entry) (Entry, error) {
- s.mu.Lock()
- defer s.mu.Unlock()
- for i := range s.entries {
- if s.entries[i].ID == entryID {
- e := &s.entries[i]
- if patch.Title != "" {
- e.Title = patch.Title
- }
- if patch.EntryType != "" {
- e.EntryType = patch.EntryType
- }
- if patch.Builder != "" {
- e.Builder = patch.Builder
- }
- if patch.Source != "" {
- e.Source = patch.Source
- }
- if patch.StorageURI != "" {
- e.StorageURI = patch.StorageURI
- }
- if patch.NavURI != "" {
- e.NavURI = patch.NavURI
- }
- if patch.IndexStatus != "" {
- e.IndexStatus = patch.IndexStatus
- }
- if patch.SortOrder != 0 {
- e.SortOrder = patch.SortOrder
- }
- if patch.Status != "" {
- e.Status = patch.Status
- }
- e.IsPrimary = patch.IsPrimary
- return *e, nil
- }
- }
- return Entry{}, ErrNotFound
+ s.html = map[string]string{}
+ s.siteFiles = map[string]SiteFile{}
}
-func (s *Store) DeleteEntry(entryID string) error {
- s.mu.Lock()
- defer s.mu.Unlock()
- for i := range s.entries {
- if s.entries[i].ID == entryID {
- s.entries = append(s.entries[:i], s.entries[i+1:]...)
- return nil
- }
- }
- return ErrNotFound
-}
-
-func (s *Store) Release(releaseID string) (Release, error) {
+// SiteObjects returns the currently cached site assets keyed by their MinIO
+// object path. It is used once at startup to migrate legacy snapshot data.
+func (s *MemoryStore) SiteObjects() map[string]SiteFile {
s.mu.RLock()
defer s.mu.RUnlock()
- for _, r := range s.releases {
- if r.ReleaseID == releaseID || r.ID == releaseID {
- return r, nil
- }
- }
- return Release{}, ErrNotFound
-}
-
-// RollbackRelease marks the target release as rolled back. A real
-// implementation would also re-point storage and search to the prior artifact.
-func (s *Store) RollbackRelease(releaseID string) (Release, error) {
- s.mu.Lock()
- defer s.mu.Unlock()
- for i := range s.releases {
- if s.releases[i].ReleaseID == releaseID || s.releases[i].ID == releaseID {
- s.releases[i].Status = "rolled_back"
- return s.releases[i], nil
- }
- }
- return Release{}, ErrNotFound
-}
-
-func (s *Store) moduleIndexLocked(moduleKey string) (int, error) {
- for i := range s.modules {
- if strings.EqualFold(s.modules[i].ModuleKey, moduleKey) {
- return i, nil
- }
- }
- return -1, ErrNotFound
-}
-
-func (s *Store) nextIDLocked(prefix string) string {
- s.seq++
- return prefix + "-" + strconv.FormatInt(s.seq, 10) + "-" + strconv.FormatInt(time.Now().UnixNano(), 36)
-}
-
-func (s *Store) versionsForLocked(moduleKey string) []Version {
- var out []Version
- for _, v := range s.versions {
- if strings.EqualFold(v.ModuleKey, moduleKey) {
- out = append(out, v)
- }
- }
- return out
-}
-
-func contains(xs []string, target string) bool {
- for _, x := range xs {
- if x == target {
- return true
- }
- }
- return false
-}
-
-func routeKey(moduleKey, docsVersion, entryKey string) string {
- if entryKey == "" {
- return strings.ToLower(moduleKey) + ":" + docsVersion
- }
- return strings.ToLower(moduleKey) + ":" + docsVersion + ":" + entryKey
-}
-
-func siteFileKey(moduleKey, docsVersion, entryKey, name string) string {
- return routeKey(moduleKey, docsVersion, entryKey) + ":" + path.Clean(strings.TrimPrefix(name, "/"))
-}
-
-func cloneStrings(xs []string) []string {
- if xs == nil {
- return nil
- }
- return append([]string(nil), xs...)
-}
-
-func cloneNav(xs []NavItem) []NavItem {
- if xs == nil {
- return nil
- }
- out := make([]NavItem, len(xs))
- for i, x := range xs {
- out[i] = x
- out[i].Children = cloneNav(x.Children)
- }
- return out
-}
-
-func firstNonEmpty(values ...string) string {
- for _, v := range values {
- if strings.TrimSpace(v) != "" {
- return v
- }
- }
- return ""
-}
-
-func firstString(xs []string) string {
- if len(xs) == 0 {
- return ""
- }
- return xs[0]
-}
-
-func coalesceStrings(primary, fallback []string) []string {
- if len(primary) > 0 {
- return primary
- }
- return fallback
-}
-
-func removeEntries(entries []Entry, moduleKey, docsVersion string) []Entry {
- out := entries[:0]
- for _, e := range entries {
- if strings.EqualFold(e.ModuleKey, moduleKey) && e.DocsVersion == docsVersion {
+ out := make(map[string]SiteFile, len(s.html)+len(s.siteFiles))
+ moduleNames := make(map[string]string, len(s.modules))
+ for _, module := range s.modules {
+ moduleNames[strings.ToLower(module.ModuleKey)] = module.ModuleKey
+ }
+ for key, html := range s.html {
+ parts := strings.SplitN(key, ":", 3)
+ if len(parts) != 3 {
continue
}
- out = append(out, e)
+ moduleKey := firstNonEmpty(moduleNames[parts[0]], parts[0])
+ name := path.Join("modules", moduleKey, parts[1], "site", parts[2], "index.html")
+ out[name] = SiteFile{Name: name, Content: []byte(html), ContentType: "text/html; charset=utf-8"}
}
- return out
-}
-
-func removePages(pages []Page, moduleKey, docsVersion string) []Page {
- out := pages[:0]
- for _, p := range pages {
- if strings.EqualFold(p.ModuleKey, moduleKey) && p.DocsVersion == docsVersion {
+ for key, file := range s.siteFiles {
+ parts := strings.SplitN(key, ":", 4)
+ if len(parts) != 4 {
continue
}
- out = append(out, p)
+ moduleKey := firstNonEmpty(moduleNames[parts[0]], parts[0])
+ name := path.Join("modules", moduleKey, parts[1], "site", parts[2], parts[3])
+ file.Name = name
+ out[name] = file
}
return out
}
-
-func entryKeyFromDocID(docID string) string {
- parts := strings.Split(docID, ":")
- if len(parts) == 0 {
- return ""
- }
- return parts[len(parts)-1]
-}
-
-func entryTypeForEntry(entries []DeployEntry, entryKey string) string {
- for _, e := range entries {
- if e.Key == entryKey {
- return e.Type
- }
- }
- return "markdown"
-}
-
-func titleForEntry(entries []DeployEntry, entryKey string) string {
- for _, e := range entries {
- if e.Key == entryKey {
- return e.Title
- }
- }
- return entryKey
-}
-
-func htmlForEntry(files map[string]string, entryKey string) string {
- for _, name := range []string{"site/" + entryKey + "/index.html", "site/" + entryKey + ".html"} {
- if html := files[name]; html != "" {
- return html
- }
- }
- prefix := "site/" + entryKey + "/"
- for name, html := range files {
- if strings.HasPrefix(name, prefix) && strings.HasSuffix(name, ".html") {
- return html
- }
- }
- return ""
-}
-
-func splitSiteFile(name string) (string, string, bool) {
- name = path.Clean(strings.TrimPrefix(name, "/"))
- parts := strings.Split(name, "/")
- if len(parts) < 3 || parts[0] != "site" || parts[1] == "" {
- return "", "", false
- }
- rel := path.Join(parts[2:]...)
- if rel == "." || rel == "" {
- rel = "index.html"
- }
- return parts[1], rel, true
-}
-
-func contentTypeForName(name string, content []byte) string {
- if ct := mime.TypeByExtension(path.Ext(name)); ct != "" {
- return ct
- }
- if len(content) > 0 {
- return http.DetectContentType(content)
- }
- return "application/octet-stream"
-}
diff --git a/backend/internal/store/memory_test.go b/backend/internal/store/memory_test.go
index 3ec3231..87b7469 100644
--- a/backend/internal/store/memory_test.go
+++ b/backend/internal/store/memory_test.go
@@ -1,66 +1,37 @@
package store
import (
- "path/filepath"
"testing"
)
-func TestSnapshotRoundTrip(t *testing.T) {
- s := NewSeeded()
- if _, err := s.CreateUser(User{Username: "carol", Roles: []string{"viewer"}}); err != nil {
- t.Fatalf("CreateUser: %v", err)
- }
- s.RecordPageView(PageView{DocID: "DemoModule:latest:guide", SessionID: "x"})
- s.SetEmbedding("DemoModule:latest:guide", []float32{0.1, 0.2, 0.3})
-
- path := filepath.Join(t.TempDir(), "snap.json")
- if err := s.Save(path); err != nil {
- t.Fatalf("Save: %v", err)
- }
-
- loaded, err := Load(path)
+func TestSiteObjectsUseCanonicalModuleKey(t *testing.T) {
+ s := NewTestStore()
+ _, err := s.IngestArtifact(DeployArtifact{
+ ModuleKey: "RuntimeDocs",
+ DocsVersion: "latest",
+ Entries: []DeployEntry{{Key: "guide", Title: "Guide"}},
+ Documents: []DeployDocument{{DocID: "RuntimeDocs:latest:guide", EntryKey: "guide", Title: "Guide", Content: "body"}},
+ SiteHTML: map[string]string{"site/guide/index.html": "Guide
"}, + SiteFiles: map[string][]byte{"site/guide/assets/app.css": []byte("body{}")}, + }) if err != nil { - t.Fatalf("Load: %v", err) - } - if got, want := len(loaded.Users("")), len(s.Users("")); got != want { - t.Fatalf("users after reload = %d, want %d", got, want) - } - if _, err := loaded.UserByID(""); err == nil { - t.Fatal("expected lookup of empty id to fail") - } - if loaded.EmbeddingCount() != 1 { - t.Fatalf("embeddings after reload = %d, want 1", loaded.EmbeddingCount()) + t.Fatalf("IngestArtifact: %v", err) } - stats := loaded.PageAnalytics() - var pv int - for _, st := range stats { - if st.DocID == "DemoModule:latest:guide" { - pv = st.PV + objects := s.SiteObjects() + for _, key := range []string{ + "modules/RuntimeDocs/latest/site/guide/index.html", + "modules/RuntimeDocs/latest/site/guide/assets/app.css", + } { + if _, ok := objects[key]; !ok { + t.Fatalf("missing MinIO object %q in %#v", key, objects) } } - if pv != 1 { - t.Fatalf("page view after reload = %d, want 1", pv) - } - // Verify new IDs continue past the persisted sequence (no collisions). - created, err := loaded.CreateModule(Module{ModuleKey: "PersistedModule"}) - if err != nil { - t.Fatalf("CreateModule after reload: %v", err) - } - if created.ID == "" { - t.Fatal("expected generated module ID after reload") - } -} - -func TestLoadMissingReturnsNotFound(t *testing.T) { - if _, err := Load(filepath.Join(t.TempDir(), "nope.json")); err != ErrNotFound { - t.Fatalf("err = %v, want ErrNotFound", err) - } } -func TestUserCRUDAndGroupAutoRegister(t *testing.T) { - s := NewSeeded() +func TestUserCRUD(t *testing.T) { + s := NewSeededTestStore() - created, err := s.CreateUser(User{Username: "carol", Department: "测试", Groups: []string{"qa-team"}, Roles: []string{"viewer"}}) + created, err := s.CreateUser(User{Username: "carol", Department: "测试", Roles: []string{"viewer"}}) if err != nil { t.Fatalf("CreateUser: %v", err) } @@ -70,17 +41,6 @@ func TestUserCRUDAndGroupAutoRegister(t *testing.T) { if _, err := s.CreateUser(User{Username: "CAROL"}); err != ErrConflict { t.Fatalf("duplicate username err = %v, want ErrConflict", err) } - // qa-team should have been auto-registered as a group. - var hasQA bool - for _, g := range s.Groups() { - if g.GroupKey == "qa-team" { - hasQA = true - } - } - if !hasQA { - t.Fatal("expected qa-team group to be auto-registered") - } - updated, err := s.UpdateUser(created.ID, User{Roles: []string{"maintainer"}, Status: "disabled"}) if err != nil { t.Fatalf("UpdateUser: %v", err) @@ -98,11 +58,11 @@ func TestUserCRUDAndGroupAutoRegister(t *testing.T) { } func TestUpsertUserSyncsOnLogin(t *testing.T) { - s := NewSeeded() + s := NewSeededTestStore() before := len(s.Users("")) // Existing seeded user alice: upsert should update, not duplicate. - u := s.UpsertUser(User{ID: "u-alice", Username: "alice", Department: "新部门", Groups: []string{"cad-team", "release"}}) + u := s.UpsertUser(User{ID: "u-alice", Username: "alice", Department: "新部门"}) if u.Source != "oidc" || u.Department != "新部门" { t.Fatalf("upsert existing wrong: %+v", u) } @@ -114,20 +74,20 @@ func TestUpsertUserSyncsOnLogin(t *testing.T) { } // New identity from provider: should be created. - s.UpsertUser(User{Username: "dave", Email: "dave@example.com", Groups: []string{"ops"}}) + s.UpsertUser(User{Username: "dave", Email: "dave@example.com"}) if got := len(s.Users("")); got != before+1 { t.Fatalf("expected new user added, count %d -> %d", before, got) } } func TestPageAnalyticsAggregatesViews(t *testing.T) { - s := NewSeeded() + s := NewSeededTestStore() doc := "DemoModule:latest:guide" s.RecordPageView(PageView{DocID: doc, SessionID: "a", DurationSeconds: 10}) s.RecordPageView(PageView{DocID: doc, SessionID: "a"}) s.RecordPageView(PageView{DocID: doc, SessionID: "b"}) - s.RecordReadProgress(doc, "b", 30, 0.9) + s.RecordReadProgress(doc, "b", "", 30, 0.9) var found bool for _, st := range s.PageAnalytics() { @@ -154,7 +114,7 @@ func TestPageAnalyticsAggregatesViews(t *testing.T) { } func TestPageAnalyticsFallsBackToSeedReads(t *testing.T) { - s := NewSeeded() + s := NewSeededTestStore() for _, st := range s.PageAnalytics() { if st.DocID == "CBB:latest:build-cache" { if st.Reads30d == 0 { @@ -167,7 +127,7 @@ func TestPageAnalyticsFallsBackToSeedReads(t *testing.T) { } func TestModuleVersionEntryCRUD(t *testing.T) { - s := NewSeeded() + s := NewSeededTestStore() if _, err := s.CreateModule(Module{ModuleKey: "NCKernel", CategoryIDs: []string{"nc"}}); err != nil { t.Fatalf("CreateModule: %v", err) @@ -207,14 +167,14 @@ func TestModuleVersionEntryCRUD(t *testing.T) { } func TestVersionRequiresExistingModule(t *testing.T) { - s := NewSeeded() + s := NewSeededTestStore() if _, err := s.CreateVersion("Ghost", Version{DocsVersion: "latest"}); err != ErrNotFound { t.Fatalf("err = %v, want ErrNotFound", err) } } func TestRollbackRelease(t *testing.T) { - s := NewSeeded() + s := NewSeededTestStore() rel, err := s.RollbackRelease("rel-demo-latest-001") if err != nil { t.Fatalf("RollbackRelease: %v", err) @@ -228,7 +188,7 @@ func TestRollbackRelease(t *testing.T) { } func TestIngestArtifactPublishesPagesNavAndHTML(t *testing.T) { - s := NewSeeded() + s := NewSeededTestStore() result, err := s.IngestArtifact(DeployArtifact{ ModuleKey: "RuntimeDocs", ModuleName: "Runtime Docs", @@ -279,3 +239,36 @@ func TestIngestArtifactPublishesPagesNavAndHTML(t *testing.T) { t.Fatalf("content type = %q", file.ContentType) } } + +func TestUpdateModulePropagatesCategoriesToIndexedPages(t *testing.T) { + s := NewSeededTestStore() + if _, err := s.CreateModule(Module{ModuleKey: "RuntimeDocs", Name: "Runtime Docs", CategoryIDs: []string{"engineering"}}); err != nil { + t.Fatalf("CreateModule: %v", err) + } + if _, err := s.IngestArtifact(DeployArtifact{ + ModuleKey: "RuntimeDocs", + ModuleName: "Runtime Docs", + DocsVersion: "latest", + Entries: []DeployEntry{{Key: "guide", Title: "Guide", Type: "markdown"}}, + Documents: []DeployDocument{{DocID: "RuntimeDocs:latest:guide", EntryKey: "guide", Title: "Guide", Content: "runtime documentation content"}}, + }); err != nil { + t.Fatalf("IngestArtifact: %v", err) + } + page, err := s.Page("RuntimeDocs:latest:guide") + if err != nil { + t.Fatalf("Page before update: %v", err) + } + if len(page.CategoryIDs) != 1 || page.CategoryIDs[0] != "engineering" { + t.Fatalf("initial page categories = %#v", page.CategoryIDs) + } + if _, err := s.UpdateModule("RuntimeDocs", Module{CategoryIDs: []string{"frontend", "frontend.docs"}}); err != nil { + t.Fatalf("UpdateModule: %v", err) + } + page, err = s.Page("RuntimeDocs:latest:guide") + if err != nil { + t.Fatalf("Page after update: %v", err) + } + if len(page.CategoryIDs) != 2 || page.CategoryIDs[0] != "frontend" || page.CategoryIDs[1] != "frontend.docs" { + t.Fatalf("updated page categories = %#v", page.CategoryIDs) + } +} diff --git a/backend/internal/store/models.go b/backend/internal/store/models.go index 65bfa5f..1dc3b1d 100644 --- a/backend/internal/store/models.go +++ b/backend/internal/store/models.go @@ -9,7 +9,6 @@ type User struct { Email string `json:"email"` Department string `json:"department"` Avatar string `json:"avatar,omitempty"` - Groups []string `json:"groups"` Roles []string `json:"roles"` // ManagedCategories lists the platform/category IDs this user may manage. // Super admins manage everything regardless of this list. @@ -20,48 +19,86 @@ type User struct { // It is combined with SUPER_ADMIN_USERS env (see auth service) so that // super admin status can be granted either statically (env, for bootstrap) // or dynamically via the admin UI. - SuperAdmin bool `json:"is_super_admin,omitempty"` + SuperAdmin bool `json:"is_super_admin,omitempty"` + // MCPToken is the user's personal bearer token for the MCP server, so MCP + // calls can be attributed to them. Never serialized (revealed via /api/me/mcp-token). + MCPToken string `json:"-"` LastLoginAt time.Time `json:"last_login_at,omitempty"` CreatedAt time.Time `json:"created_at,omitempty"` UpdatedAt time.Time `json:"updated_at,omitempty"` } -type Group struct { - ID string `json:"id"` - GroupKey string `json:"group_key"` - Name string `json:"name"` - Source string `json:"source"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` +type ConnectedApp struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + ClientID string `json:"client_id"` + ClientSecretHash string `json:"client_secret_hash,omitempty"` + RedirectURIs []string `json:"redirect_uris"` + Scopes []string `json:"scopes"` + Trusted bool `json:"trusted"` + Enabled bool `json:"enabled"` + CreatedBy string `json:"created_by,omitempty"` + LastUsedAt time.Time `json:"last_used_at,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type OAuthGrant struct { + ID string `json:"id"` + AppID string `json:"app_id"` + UserID string `json:"user_id"` + CodeHash string `json:"code_hash,omitempty"` + AccessTokenHash string `json:"access_token_hash,omitempty"` + RefreshTokenHash string `json:"refresh_token_hash,omitempty"` + RedirectURI string `json:"redirect_uri,omitempty"` + Scopes []string `json:"scopes,omitempty"` + CodeExpiresAt time.Time `json:"code_expires_at,omitempty"` + AccessExpiresAt time.Time `json:"access_expires_at,omitempty"` + RefreshExpiresAt time.Time `json:"refresh_expires_at,omitempty"` + RevokedAt time.Time `json:"revoked_at,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` } // Team represents a document maintenance team (文档维护团队). // A team has a leader (负责人) who can add/remove members. Teams can be // assigned as responsible_party for one or more Categories (领域/分类), // owning the doc structure and maintenance under those domains. -// Team.Key can be used as owner_group / group reference for compatibility. +// Team.Key is also used as the module/page owner_group string. type Team struct { - ID string `json:"id"` - Key string `json:"key"` - Name string `json:"name"` - Description string `json:"description"` - Leader string `json:"leader"` - Members []string `json:"members"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` + ID string `json:"id"` + Key string `json:"key"` + Name string `json:"name"` + Description string `json:"description"` + // Leaders are the team's responsible people (at least one). Every leader is + // also kept in Members. + Leaders []string `json:"leaders"` + Members []string `json:"members"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type TeamSummary struct { + Key string `json:"key"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Leaders []string `json:"leaders"` + Members []string `json:"members,omitempty"` } type Category struct { - ID string `json:"id"` - ParentID string `json:"parent_id,omitempty"` - Key string `json:"key"` - Name string `json:"name"` - Description string `json:"description"` - Icon string `json:"icon"` - SortOrder int `json:"sort_order"` - Status string `json:"status"` - ResponsibleTeam string `json:"responsible_team,omitempty"` - Children []Category `json:"children,omitempty"` + ID string `json:"id"` + ParentID string `json:"parent_id,omitempty"` + Key string `json:"key"` + Name string `json:"name"` + Description string `json:"description"` + Icon string `json:"icon"` + SortOrder int `json:"sort_order"` + Status string `json:"status"` + ResponsibleTeam string `json:"responsible_team,omitempty"` + ResponsibleTeamInfo *TeamSummary `json:"responsible_team_info,omitempty"` + Children []Category `json:"children,omitempty"` } type Module struct { @@ -83,6 +120,7 @@ type Module struct { Maintainers []string `json:"maintainers"` CategoryIDs []string `json:"category_ids"` CategoryPath string `json:"category_path"` + CreatedBy string `json:"created_by,omitempty"` // GitLab / source integration (similar to Mintlify deploy integration) SourceType string `json:"source_type,omitempty"` // "gitlab", "manual" @@ -94,6 +132,9 @@ type Module struct { LastSyncedCommit string `json:"last_synced_commit,omitempty"` LastSyncedAt time.Time `json:"last_synced_at,omitempty"` + // Read-only flag for admins (the actual token is secret and exposed only via /deploy-token). + DeployTokenSet bool `json:"deploy_token_set,omitempty"` + AvailableVers []Version `json:"available_versions,omitempty"` UpdatedAt time.Time `json:"updated_at"` Reads7d int `json:"reads_7d"` @@ -145,6 +186,8 @@ type Release struct { PipelineURL string `json:"pipeline_url"` BuildSystem string `json:"build_system"` BuildID string `json:"build_id"` + TriggerType string `json:"trigger_type"` + SourceIP string `json:"source_ip"` ArtifactVersion string `json:"artifact_version"` PackageVersion string `json:"package_version"` StorageURI string `json:"storage_uri"` @@ -173,23 +216,65 @@ type Page struct { Tags []string `json:"tags"` ContentText string `json:"content_text"` ContentHTML string `json:"content_html,omitempty"` + ContentMD string `json:"content_md,omitempty"` UpdatedAt time.Time `json:"updated_at"` } -// AISettings holds the admin-configured large-model connection used to power -// AI answers (RAG). It targets any OpenAI-compatible /chat/completions endpoint -// (OpenAI, DeepSeek, Qwen/DashScope-compat, local vLLM/Ollama, …). +// DefaultAskSystemPrompt is the built-in RAG system prompt used when an admin +// has not configured a custom one. It is exposed via the settings API so the +// admin UI can pre-fill the editor and offer a "reset to default" action. +const DefaultAskSystemPrompt = "你是企业研发文档助手。只依据提供的【文档片段】回答用户问题，使用简洁中文；若片段中没有答案，明确说明未在文档中找到，不要编造。回答末尾不要重复罗列来源。" + +// AISettings holds the admin-configured model and retrieval settings used to +// power AI answers (RAG), semantic search, reranking, and recall evaluation. type AISettings struct { - AskBaseURL string `json:"ask_base_url"` // e.g. https://api.openai.com/v1 - AskModel string `json:"ask_model"` // e.g. gpt-4o-mini, deepseek-chat - AskAPIKey string `json:"ask_api_key"` // secret; masked when read back - AskSystemPrompt string `json:"ask_system_prompt"` // optional override - UpdatedAt time.Time `json:"updated_at"` + // AskProtocol selects the API format of the chat endpoint: + // "openai-chat" (default), "openai-responses", "anthropic", or "gemini". + AskProtocol string `json:"ask_protocol"` + AskBaseURL string `json:"ask_base_url"` // e.g. https://api.openai.com/v1 + AskModel string `json:"ask_model"` // fetched from the endpoint + AskAPIKey string `json:"ask_api_key"` // secret; masked when read back + AskSystemPrompt string `json:"ask_system_prompt"` // optional override + // AskMaxTokens caps the answer length. 0 means "use the engine default". + // (Required by Anthropic; optional for the others.) + AskMaxTokens int `json:"ask_max_tokens,omitempty"` + // AskTemperature controls sampling. nil means "use the engine default" + // (so an explicit 0 for deterministic output is still distinguishable). + AskTemperature *float64 `json:"ask_temperature,omitempty"` + + EmbeddingBaseURL string `json:"embedding_base_url,omitempty"` + EmbeddingModel string `json:"embedding_model,omitempty"` + EmbeddingAPIKey string `json:"embedding_api_key,omitempty"` // secret; masked when read back + EmbeddingDim int `json:"embedding_dim,omitempty"` // fixed at embedding.Dim (1024); the configured model must output this many dims + + RerankBaseURL string `json:"rerank_base_url,omitempty"` + RerankModel string `json:"rerank_model,omitempty"` + RerankAPIKey string `json:"rerank_api_key,omitempty"` // secret; masked when read back + RerankTopK int `json:"rerank_top_k,omitempty"` + + ChunkStrategy string `json:"chunk_strategy,omitempty"` // fixed, heading, markdown, semantic + ChunkSize int `json:"chunk_size,omitempty"` + ChunkOverlap int `json:"chunk_overlap,omitempty"` + + RecallTestQuery string `json:"recall_test_query,omitempty"` + RecallTestTopK int `json:"recall_test_top_k,omitempty"` + RecallTestDocIDs string `json:"recall_test_doc_ids,omitempty"` // newline/comma separated expected doc ids + + UpdatedAt time.Time `json:"updated_at"` } // Settings is the persisted, admin-editable platform configuration. type Settings struct { AI AISettings `json:"ai"` + // Plugins holds per-plugin enable/config overrides keyed by plugin key. + // Absent keys fall back to the built-in catalog defaults (see plugins.go). + Plugins map[string]PluginSetting `json:"plugins,omitempty"` + // Snippets and Variables power reusable doc content (see snippets.go). + Snippets []Snippet `json:"snippets,omitempty"` + Variables map[string]string `json:"variables,omitempty"` + // UploadedPlugins are admin-imported, sandbox-rendered JSX plugins + // (see uploaded_plugins.go). Enable/disable reuses Plugins overrides. + UploadedPlugins []UploadedPlugin `json:"uploaded_plugins,omitempty"` } type NavItem struct { @@ -217,6 +302,7 @@ type DeployDocument struct { Title string `json:"title"` Description string `json:"description"` Content string `json:"content"` + ContentMD string `json:"content_md,omitempty"` Path string `json:"path"` SourceFile string `json:"source_file"` Keywords []string `json:"keywords"` @@ -236,6 +322,8 @@ type DeployArtifact struct { RepoType string Branch string CommitSHA string + TriggerType string + SourceIP string Entries []DeployEntry Documents []DeployDocument Nav []NavItem @@ -262,6 +350,7 @@ type SiteFile struct { type SearchLog struct { ID string `json:"id"` UserID string `json:"user_id"` + IPAddress string `json:"ip_address,omitempty"` // recorded for anonymous searches Query string `json:"query"` Mode string `json:"mode"` FiltersJSON string `json:"filters_json"` @@ -280,19 +369,57 @@ type MCPLog struct { CreatedAt time.Time `json:"created_at"` } +type DocFeedback struct { + ID string `json:"id"` + DocID string `json:"doc_id"` + PageID string `json:"page_id"` + ModuleKey string `json:"module_key"` + Title string `json:"title"` + Rating string `json:"rating"` + Comment string `json:"comment"` + UserID string `json:"user_id"` + SessionID string `json:"session_id"` + CreatedAt time.Time `json:"created_at"` +} + type PageView struct { ID string `json:"id"` PageID string `json:"page_id"` DocID string `json:"doc_id"` ModuleKey string `json:"module_key"` + ModuleName string `json:"module_name,omitempty"` DocsVersion string `json:"docs_version"` + EntryKey string `json:"entry_key,omitempty"` + Title string `json:"title,omitempty"` + Path string `json:"path,omitempty"` UserID string `json:"user_id"` SessionID string `json:"session_id"` + ReadID string `json:"read_id,omitempty"` DurationSeconds int `json:"duration_seconds"` ScrollDepth float64 `json:"scroll_depth"` ViewedAt time.Time `json:"viewed_at"` } +type UserFavorite struct { + ID string `json:"id"` + UserID string `json:"user_id"` + ModuleKey string `json:"module_key"` + CreatedAt time.Time `json:"created_at"` +} + +type UserRecentDoc struct { + ID string `json:"id"` + UserID string `json:"user_id"` + DocID string `json:"doc_id"` + Title string `json:"title"` + ModuleKey string `json:"module_key"` + ModuleName string `json:"module_name"` + DocsVersion string `json:"docs_version"` + EntryKey string `json:"entry_key"` + Href string `json:"href"` + ViewedAt time.Time `json:"viewed_at"` +} + // PageStat is an aggregated reading-statistics row for one document page. type PageStat struct { DocID string `json:"doc_id"` @@ -308,3 +435,28 @@ type PageStat struct { AvgDurationSec int `json:"avg_duration_seconds"` LastViewedAt time.Time `json:"last_viewed_at"` } + +// DailyReadPoint is one day's read count for a single page (line-chart point). +type DailyReadPoint struct { + Date string `json:"date"` // YYYY-MM-DD (UTC) + Count int `json:"count"` +} + +// ReaderStat is one reader's aggregated read activity for a single page. +type ReaderStat struct { + Reader string `json:"reader"` // display name, username, or "匿名" + UserID string `json:"user_id"` + Count int `json:"count"` + AvgDurationSec int `json:"avg_duration_seconds"` + LastReadAt time.Time `json:"last_read_at"` +} + +// PageReadStats is the per-page reading detail surfaced behind the doc-page +// "eye" popover: a daily read trend plus a per-reader breakdown. +type PageReadStats struct { + DocID string `json:"doc_id"` + Total int `json:"total"` + AvgDurationSec int `json:"avg_duration_seconds"` + Daily []DailyReadPoint `json:"daily"` + Readers []ReaderStat `json:"readers"` +} diff --git a/backend/internal/store/oauth.go b/backend/internal/store/oauth.go new file mode 100644 index 0000000..ec708a3 --- /dev/null +++ b/backend/internal/store/oauth.go @@ -0,0 +1,315 @@ +package store + +import ( + "crypto/sha256" + "encoding/hex" + "strings" + "time" +) + +const CodexOAuthClientID = "codex-cli" + +func builtinCodexOAuthApp(now time.Time) ConnectedApp { + return ConnectedApp{ + ID: "app-codex-cli", + Name: "Codex CLI MCP OAuth", + Description: "Built-in public OAuth client for Codex MCP login.", + ClientID: CodexOAuthClientID, + ClientSecretHash: "", + RedirectURIs: []string{"http://localhost", "http://127.0.0.1", "http://[::1]"}, + Scopes: []string{"modex:mcp:read", "modex:docs:read"}, + Trusted: true, + Enabled: true, + CreatedAt: now, + UpdatedAt: now, + } +} + +func (s *MemoryStore) ConnectedApps() []ConnectedApp { + s.mu.RLock() + defer s.mu.RUnlock() + out := append([]ConnectedApp{}, s.apps...) + return out +} + +func (s *MemoryStore) ConnectedAppByClientID(clientID string) (ConnectedApp, error) { + s.mu.RLock() + defer s.mu.RUnlock() + for _, app := range s.apps { + if app.ClientID == clientID { + return app, nil + } + } + return ConnectedApp{}, ErrNotFound +} + +func (s *MemoryStore) CreateConnectedApp(app ConnectedApp, clientSecret string) (ConnectedApp, error) { + s.mu.Lock() + defer s.mu.Unlock() + if strings.TrimSpace(app.Name) == "" || strings.TrimSpace(app.ClientID) == "" || len(app.RedirectURIs) == 0 { + return ConnectedApp{}, ErrInvalid + } + for _, existing := range s.apps { + if strings.EqualFold(existing.ClientID, app.ClientID) { + return ConnectedApp{}, ErrConflict + } + } + if app.ID == "" { + app.ID = s.nextIDLocked("app") + } + app.Name = strings.TrimSpace(app.Name) + app.ClientID = strings.TrimSpace(app.ClientID) + app.RedirectURIs = cleanNonEmpty(app.RedirectURIs) + app.Scopes = normalizeScopes(app.Scopes) + if len(app.Scopes) == 0 { + app.Scopes = []string{"modex:mcp:read"} + } + app.ClientSecretHash = hashToken(clientSecret) + now := time.Now().UTC() + app.CreatedAt = now + app.UpdatedAt = now + s.apps = append(s.apps, app) + return app, nil +} + +func (s *MemoryStore) UpdateConnectedApp(id string, patch ConnectedApp) (ConnectedApp, error) { + s.mu.Lock() + defer s.mu.Unlock() + for i := range s.apps { + if s.apps[i].ID != id { + continue + } + app := &s.apps[i] + if strings.TrimSpace(patch.Name) != "" { + app.Name = strings.TrimSpace(patch.Name) + } + app.Description = patch.Description + if patch.RedirectURIs != nil { + uris := cleanNonEmpty(patch.RedirectURIs) + if len(uris) == 0 { + return ConnectedApp{}, ErrInvalid + } + app.RedirectURIs = uris + } + if patch.Scopes != nil { + app.Scopes = normalizeScopes(patch.Scopes) + } + app.Trusted = patch.Trusted + app.Enabled = patch.Enabled + app.UpdatedAt = time.Now().UTC() + return *app, nil + } + return ConnectedApp{}, ErrNotFound +} + +func (s *MemoryStore) DeleteConnectedApp(id string) error { + s.mu.Lock() + defer s.mu.Unlock() + for i := range s.apps { + if s.apps[i].ID == id { + s.apps = append(s.apps[:i], s.apps[i+1:]...) + now := time.Now().UTC() + for j := range s.grants { + if s.grants[j].AppID == id && s.grants[j].RevokedAt.IsZero() { + s.grants[j].RevokedAt = now + s.grants[j].UpdatedAt = now + } + } + return nil + } + } + return ErrNotFound +} + +func (s *MemoryStore) VerifyConnectedAppSecret(clientID, clientSecret string) (ConnectedApp, error) { + s.mu.RLock() + defer s.mu.RUnlock() + h := hashToken(clientSecret) + for _, app := range s.apps { + if app.ClientID == clientID && app.ClientSecretHash == h && app.Enabled { + return app, nil + } + } + return ConnectedApp{}, ErrNotFound +} + +func (s *MemoryStore) CreateOAuthCode(appID, userID, redirectURI string, scopes []string, code string, ttl time.Duration) (OAuthGrant, error) { + s.mu.Lock() + defer s.mu.Unlock() + if appID == "" || userID == "" || redirectURI == "" || code == "" { + return OAuthGrant{}, ErrInvalid + } + now := time.Now().UTC() + g := OAuthGrant{ + ID: s.nextIDLocked("grant"), + AppID: appID, + UserID: userID, + CodeHash: hashToken(code), + RedirectURI: redirectURI, + Scopes: normalizeScopes(scopes), + CodeExpiresAt: now.Add(ttl), + CreatedAt: now, + UpdatedAt: now, + } + s.grants = append(s.grants, g) + return g, nil +} + +func (s *MemoryStore) RedeemOAuthCode(clientID, code, redirectURI, accessToken, refreshToken string, accessTTL, refreshTTL time.Duration) (OAuthGrant, ConnectedApp, User, error) { + s.mu.Lock() + defer s.mu.Unlock() + now := time.Now().UTC() + for i := range s.grants { + g := &s.grants[i] + if g.CodeHash != hashToken(code) || g.RedirectURI != redirectURI || !g.RevokedAt.IsZero() || now.After(g.CodeExpiresAt) { + continue + } + app, ok := s.appByIDLocked(g.AppID) + if !ok || app.ClientID != clientID || !app.Enabled { + return OAuthGrant{}, ConnectedApp{}, User{}, ErrNotFound + } + user, err := s.userByIDLocked(g.UserID) + if err != nil { + return OAuthGrant{}, ConnectedApp{}, User{}, err + } + g.CodeHash = "" + g.AccessTokenHash = hashToken(accessToken) + g.RefreshTokenHash = hashToken(refreshToken) + g.AccessExpiresAt = now.Add(accessTTL) + g.RefreshExpiresAt = now.Add(refreshTTL) + g.UpdatedAt = now + s.touchAppLocked(g.AppID, now) + return *g, app, user, nil + } + return OAuthGrant{}, ConnectedApp{}, User{}, ErrNotFound +} + +func (s *MemoryStore) RefreshOAuthToken(clientID, refreshToken, accessToken, nextRefreshToken string, accessTTL, refreshTTL time.Duration) (OAuthGrant, ConnectedApp, User, error) { + s.mu.Lock() + defer s.mu.Unlock() + now := time.Now().UTC() + for i := range s.grants { + g := &s.grants[i] + if g.RefreshTokenHash != hashToken(refreshToken) || !g.RevokedAt.IsZero() || now.After(g.RefreshExpiresAt) { + continue + } + app, ok := s.appByIDLocked(g.AppID) + if !ok || app.ClientID != clientID || !app.Enabled { + return OAuthGrant{}, ConnectedApp{}, User{}, ErrNotFound + } + user, err := s.userByIDLocked(g.UserID) + if err != nil { + return OAuthGrant{}, ConnectedApp{}, User{}, err + } + g.AccessTokenHash = hashToken(accessToken) + g.RefreshTokenHash = hashToken(nextRefreshToken) + g.AccessExpiresAt = now.Add(accessTTL) + g.RefreshExpiresAt = now.Add(refreshTTL) + g.UpdatedAt = now + s.touchAppLocked(g.AppID, now) + return *g, app, user, nil + } + return OAuthGrant{}, ConnectedApp{}, User{}, ErrNotFound +} + +func (s *MemoryStore) UserByOAuthAccessToken(token string) (User, ConnectedApp, OAuthGrant, error) { + s.mu.Lock() + defer s.mu.Unlock() + now := time.Now().UTC() + h := hashToken(token) + for i := range s.grants { + g := &s.grants[i] + if g.AccessTokenHash != h || !g.RevokedAt.IsZero() || now.After(g.AccessExpiresAt) { + continue + } + app, ok := s.appByIDLocked(g.AppID) + if !ok || !app.Enabled { + return User{}, ConnectedApp{}, OAuthGrant{}, ErrNotFound + } + user, err := s.userByIDLocked(g.UserID) + if err != nil { + return User{}, ConnectedApp{}, OAuthGrant{}, err + } + g.UpdatedAt = now + s.touchAppLocked(g.AppID, now) + return user, app, *g, nil + } + return User{}, ConnectedApp{}, OAuthGrant{}, ErrNotFound +} + +func (s *MemoryStore) RevokeOAuthToken(clientID, token string) bool { + s.mu.Lock() + defer s.mu.Unlock() + now := time.Now().UTC() + h := hashToken(token) + revoked := false + for i := range s.grants { + g := &s.grants[i] + app, ok := s.appByIDLocked(g.AppID) + if !ok || app.ClientID != clientID { + continue + } + if g.AccessTokenHash == h || g.RefreshTokenHash == h || g.CodeHash == h { + g.RevokedAt = now + g.UpdatedAt = now + revoked = true + } + } + return revoked +} + +func (s *MemoryStore) appByIDLocked(id string) (ConnectedApp, bool) { + for _, app := range s.apps { + if app.ID == id { + return app, true + } + } + return ConnectedApp{}, false +} + +func (s *MemoryStore) touchAppLocked(id string, at time.Time) { + for i := range s.apps { + if s.apps[i].ID == id { + s.apps[i].LastUsedAt = at + s.apps[i].UpdatedAt = at + return + } + } +} + +func hashToken(token string) string { + if token == "" { + return "" + } + sum := sha256.Sum256([]byte(token)) + return hex.EncodeToString(sum[:]) +} + +func cleanNonEmpty(items []string) []string { + var out []string + for _, item := range items { + if item = strings.TrimSpace(item); item != "" { + out = append(out, item) + } + } + return out +} + +func normalizeScopes(scopes []string) []string { + seen := map[string]struct{}{} + var out []string + for _, scope := range scopes { + for _, part := range strings.FieldsFunc(scope, func(r rune) bool { return r == ' ' || r == ',' }) { + part = strings.TrimSpace(part) + if part == "" { + continue + } + if _, ok := seen[part]; ok { + continue + } + seen[part] = struct{}{} + out = append(out, part) + } + } + return out +} diff --git a/backend/internal/store/persist.go b/backend/internal/store/persist.go deleted file mode 100644 index bd07974..0000000 --- a/backend/internal/store/persist.go +++ /dev/null @@ -1,175 +0,0 @@ -package store - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" -) - -// snapshot is the serializable projection of the full in-memory store. It is -// used to persist state to disk so the registry survives process restarts until -// the relational (PostgreSQL) backend lands. -type snapshot struct { - Version int `json:"version"` - User User `json:"user"` - Users []User `json:"users"` - Groups []Group `json:"groups"` - Teams []Team `json:"teams"` - Categories []Category `json:"categories"` - Modules []Module `json:"modules"` - Versions []Version `json:"versions"` - Entries []Entry `json:"entries"` - Releases []Release `json:"releases"` - Pages []Page `json:"pages"` - SearchLogs []SearchLog `json:"search_logs"` - MCPLogs []MCPLog `json:"mcp_logs"` - PageViews []PageView `json:"page_views"` - Navs map[string][]NavItem `json:"navs"` - HTML map[string]string `json:"html"` - SiteFiles map[string]SiteFile `json:"site_files"` - Embeddings map[string][]float32 `json:"embeddings"` - Settings Settings `json:"settings"` - Seq int64 `json:"seq"` -} - -func (s *Store) toSnapshot() snapshot { - s.mu.RLock() - defer s.mu.RUnlock() - return snapshot{ - Version: 1, - User: s.user, - Users: s.users, - Groups: s.groups, - Teams: s.teams, - Categories: s.categories, - Modules: s.modules, - Versions: s.versions, - Entries: s.entries, - Releases: s.releases, - Pages: s.pages, - SearchLogs: s.searchLogs, - MCPLogs: s.mcpLogs, - PageViews: s.pageViews, - Navs: s.navs, - HTML: s.html, - SiteFiles: s.siteFiles, - Embeddings: s.embeddings, - Settings: s.settings, - Seq: s.seq, - } -} - -func storeFromSnapshot(snap snapshot) *Store { - s := &Store{ - user: snap.User, - users: snap.Users, - groups: snap.Groups, - teams: snap.Teams, - categories: snap.Categories, - modules: snap.Modules, - versions: snap.Versions, - entries: snap.Entries, - releases: snap.Releases, - pages: snap.Pages, - searchLogs: snap.SearchLogs, - mcpLogs: snap.MCPLogs, - pageViews: snap.PageViews, - navs: snap.Navs, - html: snap.HTML, - siteFiles: snap.SiteFiles, - embeddings: snap.Embeddings, - settings: snap.Settings, - seq: snap.Seq, - } - // Ensure slices are never nil (important for empty start and JSON nulls from old snapshots) - if s.users == nil { - s.users = []User{} - } - if s.groups == nil { - s.groups = []Group{} - } - if s.teams == nil { - s.teams = []Team{} - } - if s.categories == nil { - s.categories = []Category{} - } - if s.modules == nil { - s.modules = []Module{} - } - if s.versions == nil { - s.versions = []Version{} - } - if s.entries == nil { - s.entries = []Entry{} - } - if s.releases == nil { - s.releases = []Release{} - } - if s.pages == nil { - s.pages = []Page{} - } - if s.searchLogs == nil { - s.searchLogs = []SearchLog{} - } - if s.mcpLogs == nil { - s.mcpLogs = []MCPLog{} - } - if s.pageViews == nil { - s.pageViews = []PageView{} - } - if s.navs == nil { - s.navs = map[string][]NavItem{} - } - if s.html == nil { - s.html = map[string]string{} - } - if s.siteFiles == nil { - s.siteFiles = map[string]SiteFile{} - } - if s.embeddings == nil { - s.embeddings = map[string][]float32{} - } - if s.teams == nil { - s.teams = []Team{} - } - return s -} - -// Save atomically writes the store snapshot to path (writing to a temp file and -// renaming to avoid a torn file on crash). -func (s *Store) Save(path string) error { - if path == "" { - return fmt.Errorf("empty snapshot path") - } - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - return err - } - data, err := json.Marshal(s.toSnapshot()) - if err != nil { - return err - } - tmp := path + ".tmp" - if err := os.WriteFile(tmp, data, 0o644); err != nil { - return err - } - return os.Rename(tmp, path) -} - -// Load reads a store snapshot from path. It returns ErrNotFound when the file -// does not exist so callers can fall back to a freshly seeded store. -func Load(path string) (*Store, error) { - data, err := os.ReadFile(path) - if err != nil { - if os.IsNotExist(err) { - return nil, ErrNotFound - } - return nil, err - } - var snap snapshot - if err := json.Unmarshal(data, &snap); err != nil { - return nil, fmt.Errorf("snapshot %s: %w", path, err) - } - return storeFromSnapshot(snap), nil -} diff --git a/backend/internal/store/plugins.go b/backend/internal/store/plugins.go new file mode 100644 index 0000000..e3d3b5e --- /dev/null +++ b/backend/internal/store/plugins.go @@ -0,0 +1,170 @@ +package store + +import "strings" + +// PluginSetting is the persisted per-plugin override (enable flag + config +// values). It lives inside Settings, so it is snapshotted with everything else. +type PluginSetting struct { + Enabled bool `json:"enabled"` + Config map[string]string `json:"config,omitempty"` +} + +// PluginField describes one configurable value of a plugin for the admin UI. +type PluginField struct { + Key string `json:"key"` + Label string `json:"label"` + Placeholder string `json:"placeholder,omitempty"` + Default string `json:"default,omitempty"` +} + +// PluginDef is a static catalog entry for a built-in plugin. The catalog is the +// single source of truth; admins only toggle/configure these, they cannot add +// arbitrary third-party code. +type PluginDef struct { + Key string `json:"key"` + Name string `json:"name"` + Description string `json:"description"` + Category string `json:"category"` + DefaultEnabled bool `json:"default_enabled"` + Fields []PluginField `json:"fields,omitempty"` +} + +// PluginState is a catalog entry merged with its saved override (admin view). +// Uploaded (imported) plugins carry extra fields so the admin UI can show their +// provenance, kind and source. +type PluginState struct { + PluginDef + Enabled bool `json:"enabled"` + Config map[string]string `json:"config"` + Uploaded bool `json:"uploaded,omitempty"` + Kind string `json:"kind,omitempty"` + Tag string `json:"tag,omitempty"` + Lang string `json:"lang,omitempty"` + Code string `json:"code,omitempty"` +} + +// pluginCatalog is the curated set of built-in doc-engine plugins. +var pluginCatalog = []PluginDef{ + {Key: "kroki", Name: "Kroki 图表", Category: "diagram", DefaultEnabled: true, + Description: "用 Kroki 渲染 PlantUML、Graphviz、C4、DITAA、D2 等图表代码块。", + Fields: []PluginField{{Key: "base_url", Label: "Kroki 服务地址", Placeholder: "https://kroki.io"}}}, + {Key: "mermaid", Name: "Mermaid 图表", Category: "diagram", DefaultEnabled: true, + Description: "渲染 Mermaid 流程图、时序图、甘特图等图表。"}, + {Key: "math", Name: "数学公式", Category: "math", DefaultEnabled: true, + Description: "用 KaTeX 渲染行内公式和块级 LaTeX 公式。"}, + {Key: "github_alerts", Name: "GitHub 提示块", Category: "content", DefaultEnabled: true, + Description: "把 GitHub 风格的 NOTE、WARNING 等提示块转为提示卡片。"}, + {Key: "toc", Name: "自动目录", Category: "content", DefaultEnabled: true, + Description: "把目录占位标记替换为页面内目录导航。"}, + {Key: "footnotes", Name: "脚注", Category: "content", DefaultEnabled: true, + Description: "渲染 Markdown 脚注。"}, + {Key: "snippets", Name: "可复用片段", Category: "content", DefaultEnabled: true, + Description: "支持变量插值与可复用内容片段。"}, + {Key: "openapi", Name: "OpenAPI / API 调试台", Category: "api", DefaultEnabled: true, + Description: "渲染交互式 API 调试台与从 OpenAPI 规范生成的接口参考。", + Fields: []PluginField{{Key: "default_spec_url", Label: "默认 OpenAPI 规范地址", Placeholder: "https://example.com/openapi.json"}}}, +} + +// PluginCatalog returns the static built-in plugin catalog. +func PluginCatalog() []PluginDef { return pluginCatalog } + +// mergePlugins overlays saved overrides onto the catalog defaults, then appends +// uploaded plugins (which default to disabled). +func mergePlugins(overrides map[string]PluginSetting, uploaded []UploadedPlugin) []PluginState { + out := make([]PluginState, 0, len(pluginCatalog)+len(uploaded)) + for _, def := range pluginCatalog { + st := PluginState{PluginDef: def, Enabled: def.DefaultEnabled, Config: map[string]string{}} + for _, f := range def.Fields { + if f.Default != "" { + st.Config[f.Key] = f.Default + } + } + if ov, ok := overrides[def.Key]; ok { + st.Enabled = ov.Enabled + for k, v := range ov.Config { + st.Config[k] = v + } + } + out = append(out, st) + } + for _, up := range uploaded { + st := PluginState{ + PluginDef: PluginDef{Key: up.Key, Name: up.Name, Description: up.Description, Category: up.Category, DefaultEnabled: false}, + Enabled: false, + Config: map[string]string{}, + Uploaded: true, + Kind: up.Kind, + Tag: up.Tag, + Lang: up.Lang, + Code: up.Code, + } + if ov, ok := overrides[up.Key]; ok { + st.Enabled = ov.Enabled + } + out = append(out, st) + } + return out +} + +// PluginStates returns the catalog merged with saved overrides (admin view). +func (s *MemoryStore) PluginStates() []PluginState { + s.mu.RLock() + overrides := s.settings.Plugins + uploaded := s.settings.UploadedPlugins + s.mu.RUnlock() + return mergePlugins(overrides, uploaded) +} + +// SavePluginSettings persists enable/config overrides, ignoring unknown plugin +// keys and config fields not declared in the catalog. +func (s *MemoryStore) SavePluginSettings(overrides map[string]PluginSetting) []PluginState { + allowed := map[string]map[string]bool{} + for _, def := range pluginCatalog { + fs := map[string]bool{} + for _, f := range def.Fields { + fs[f.Key] = true + } + allowed[def.Key] = fs + } + s.mu.RLock() + uploaded := s.settings.UploadedPlugins + for _, up := range uploaded { + if _, ok := allowed[up.Key]; !ok { + allowed[up.Key] = map[string]bool{} // uploaded plugins: enable flag only + } + } + s.mu.RUnlock() + + clean := map[string]PluginSetting{} + for key, ov := range overrides { + fields, known := allowed[key] + if !known { + continue + } + cfg := map[string]string{} + for k, v := range ov.Config { + if fields[k] { + if tv := strings.TrimSpace(v); tv != "" { + cfg[k] = tv + } + } + } + clean[key] = PluginSetting{Enabled: ov.Enabled, Config: cfg} + } + s.mu.Lock() + s.settings.Plugins = clean + uploaded = s.settings.UploadedPlugins + s.mu.Unlock() + return mergePlugins(clean, uploaded) +} + +// PluginEffective returns a slim enabled+config map for the public config API, +// consumed by the doc renderer to drive conditional plugins. +func (s *MemoryStore) PluginEffective() map[string]PluginSetting { + states := s.PluginStates() + out := make(map[string]PluginSetting, len(states)) + for _, st := range states { + out[st.Key] = PluginSetting{Enabled: st.Enabled, Config: st.Config} + } + return out +} diff --git a/backend/internal/store/plugins_test.go b/backend/internal/store/plugins_test.go new file mode 100644 index 0000000..0915fa5 --- /dev/null +++ b/backend/internal/store/plugins_test.go @@ -0,0 +1,68 @@ +package store + +import "testing" + +func findState(states []PluginState, key string) (PluginState, bool) { + for _, s := range states { + if s.Key == key { + return s, true + } + } + return PluginState{}, false +} + +func TestPluginStatesDefaults(t *testing.T) { + states := NewTestStore().PluginStates() + if len(states) != len(pluginCatalog) { + t.Fatalf("states = %d, want %d", len(states), len(pluginCatalog)) + } + kroki, ok := findState(states, "kroki") + if !ok { + t.Fatal("kroki plugin missing from catalog") + } + if !kroki.Enabled { + t.Error("kroki should be enabled by default") + } + if kroki.Category != "diagram" || len(kroki.Fields) == 0 || kroki.Fields[0].Key != "base_url" { + t.Errorf("kroki def unexpected: %+v", kroki.PluginDef) + } +} + +func TestSavePluginSettingsOverrideAndFilter(t *testing.T) { + st := NewTestStore() + st.SavePluginSettings(map[string]PluginSetting{ + "kroki": {Enabled: false, Config: map[string]string{"base_url": " http://kroki.internal:8000 ", "bogus": "x"}}, + "unknown": {Enabled: true}, + }) + states := st.PluginStates() + + kroki, _ := findState(states, "kroki") + if kroki.Enabled { + t.Error("kroki should be disabled after override") + } + if got := kroki.Config["base_url"]; got != "http://kroki.internal:8000" { + t.Errorf("base_url = %q, want trimmed value", got) + } + if _, ok := kroki.Config["bogus"]; ok { + t.Error("unknown config field should be dropped") + } + if _, ok := findState(states, "unknown"); ok { + t.Error("unknown plugin key should not appear") + } + // A plugin we did not touch keeps its default-enabled state. + if math, _ := findState(states, "math"); !math.Enabled { + t.Error("untouched math plugin should stay enabled") + } +} + +func TestPluginEffective(t *testing.T) { + st := NewTestStore() + st.SavePluginSettings(map[string]PluginSetting{"math": {Enabled: false}}) + eff := st.PluginEffective() + if eff["math"].Enabled { + t.Error("math should be disabled in effective config") + } + if !eff["kroki"].Enabled { + t.Error("kroki should remain enabled in effective config") + } +} diff --git a/backend/internal/store/postgres_analytics.go b/backend/internal/store/postgres_analytics.go new file mode 100644 index 0000000..d6c6bdb --- /dev/null +++ b/backend/internal/store/postgres_analytics.go @@ -0,0 +1,282 @@ +package store + +import ( + "context" + "sort" + "strings" + "time" +) + +func (p *PostgresRepository) AddSearchLog(log SearchLog) { + if log.ID == "" { + log.ID = databaseID("sl") + } + if log.SearchedAt.IsZero() { + log.SearchedAt = time.Now().UTC() + } + _, _ = p.pool.Exec(context.Background(), `INSERT INTO docs_search_log(id,user_id,ip_address,query,mode,filters_json,result_count,clicked_doc_id,searched_at) VALUES($1,$2,$3,$4,$5,NULLIF($6,'')::jsonb,$7,$8,$9)`, log.ID, log.UserID, log.IPAddress, log.Query, log.Mode, log.FiltersJSON, log.ResultCount, log.ClickedDocID, log.SearchedAt) +} +func (p *PostgresRepository) SearchLogs() []SearchLog { + var logs []SearchLog + if err := loadQuery(context.Background(), p.pool, `SELECT (to_jsonb(t)-'filters_json')||jsonb_build_object('filters_json',COALESCE(filters_json::text,'')) FROM docs_search_log t ORDER BY searched_at DESC,id`, &logs); err != nil { + return []SearchLog{} + } + return logs +} +func (p *PostgresRepository) AddMCPLog(log MCPLog) { + if log.ID == "" { + log.ID = databaseID("ml") + } + if log.CreatedAt.IsZero() { + log.CreatedAt = time.Now().UTC() + } + _, _ = p.pool.Exec(context.Background(), `INSERT INTO docs_mcp_log(id,tool_name,user_id,query,input_json,result_count,created_at) VALUES($1,$2,$3,$4,NULLIF($5,'')::jsonb,$6,$7)`, log.ID, log.ToolName, log.UserID, log.Query, log.InputJSON, log.ResultCount, log.CreatedAt) +} +func (p *PostgresRepository) MCPLogs() []MCPLog { + var logs []MCPLog + if err := loadQuery(context.Background(), p.pool, `SELECT (to_jsonb(t)-'input_json')||jsonb_build_object('input_json',COALESCE(input_json::text,'')) FROM docs_mcp_log t ORDER BY created_at DESC,id`, &logs); err != nil { + return []MCPLog{} + } + return logs +} + +func (p *PostgresRepository) AddDocFeedback(feedback DocFeedback) DocFeedback { + if feedback.ID == "" { + feedback.ID = databaseID("df") + } + if feedback.CreatedAt.IsZero() { + feedback.CreatedAt = time.Now().UTC() + } + if page, err := p.Page(feedback.DocID); err == nil { + feedback.PageID = page.ID + feedback.ModuleKey = page.ModuleKey + feedback.Title = page.Title + } + saved, err := queryJSONOne[DocFeedback](context.Background(), p.pool, `INSERT INTO docs_feedback(id,doc_id,page_id,module_key,title,rating,comment,user_id,session_id,created_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) RETURNING to_jsonb(docs_feedback)`, feedback.ID, feedback.DocID, feedback.PageID, feedback.ModuleKey, feedback.Title, feedback.Rating, feedback.Comment, feedback.UserID, feedback.SessionID, feedback.CreatedAt) + if err != nil { + return feedback + } + return saved +} +func (p *PostgresRepository) DocFeedbacks() []DocFeedback { + var values []DocFeedback + if err := loadQuery(context.Background(), p.pool, `SELECT to_jsonb(t) FROM docs_feedback t ORDER BY created_at DESC,id`, &values); err != nil { + return []DocFeedback{} + } + return values +} + +func (p *PostgresRepository) enrichPageView(view *PageView) { + if page, err := p.Page(view.DocID); err == nil { + view.PageID = page.ID + view.ModuleKey = page.ModuleKey + view.ModuleName = page.ModuleName + view.DocsVersion = page.DocsVersion + view.EntryKey = page.EntryKey + view.Title = page.Title + view.Path = page.Path + } +} +func (p *PostgresRepository) RecordPageView(view PageView) PageView { + if view.ID == "" { + view.ID = databaseID("pv") + } + if view.ViewedAt.IsZero() { + view.ViewedAt = time.Now().UTC() + } + p.enrichPageView(&view) + moduleID, versionID := "", "" + if view.ModuleKey != "" { + if module, err := p.Module(view.ModuleKey); err == nil { + moduleID = module.ID + for _, version := range module.AvailableVers { + if version.DocsVersion == view.DocsVersion { + versionID = version.ID + break + } + } + } + } + saved, err := queryJSONOne[PageView](context.Background(), p.pool, `INSERT INTO docs_page_view(id,page_id,module_id,version_id,doc_id,module_key,module_name,docs_version,entry_key,title,path,user_id,session_id,read_id,duration_seconds,scroll_depth,viewed_at) VALUES($1,NULLIF($2,''),NULLIF($3,''),NULLIF($4,''),$5,$6,$7,$8,$9,$10,$11,NULLIF($12,''),$13,$14,$15,$16,$17) RETURNING (to_jsonb(docs_page_view)-'module_id'-'version_id')`, view.ID, view.PageID, moduleID, versionID, view.DocID, view.ModuleKey, view.ModuleName, view.DocsVersion, view.EntryKey, view.Title, view.Path, view.UserID, view.SessionID, view.ReadID, view.DurationSeconds, view.ScrollDepth, view.ViewedAt) + if err != nil { + return view + } + return saved +} +func (p *PostgresRepository) RecordReadProgress(docID, sessionID, readID string, durationSeconds int, scrollDepth float64) PageView { + ctx := context.Background() + condition := `doc_id=$1 AND (($2<>'' AND read_id=$2) OR ($2='' AND session_id=$3))` + view, err := queryJSONOne[PageView](ctx, p.pool, `UPDATE docs_page_view SET duration_seconds=GREATEST(COALESCE(duration_seconds,0),$4),scroll_depth=GREATEST(COALESCE(scroll_depth,0),$5) WHERE id=(SELECT id FROM docs_page_view WHERE `+condition+` ORDER BY viewed_at DESC LIMIT 1) RETURNING (to_jsonb(docs_page_view)-'module_id'-'version_id')`, docID, readID, sessionID, durationSeconds, scrollDepth) + if err == nil { + return view + } + return p.RecordPageView(PageView{DocID: docID, SessionID: sessionID, ReadID: readID, DurationSeconds: durationSeconds, ScrollDepth: scrollDepth}) +} + +func (p *PostgresRepository) pageViews() []PageView { + var values []PageView + if err := loadQuery(context.Background(), p.pool, `SELECT (to_jsonb(t)-'module_id'-'version_id') FROM docs_page_view t ORDER BY viewed_at,id`, &values); err != nil { + return []PageView{} + } + return values +} +func (p *PostgresRepository) PageAnalytics() []PageStat { + var values []PageStat + query := `SELECT jsonb_build_object( + 'doc_id',p.doc_id,'title',p.title,'module_key',m.module_key,'module_name',m.name, + 'docs_version',v.docs_version,'path',p.path,'pv',count(pv.id), + 'uv',count(DISTINCT COALESCE(NULLIF(pv.user_id,''),NULLIF(pv.session_id,''))), + 'reads_7d',CASE WHEN count(pv.id)=0 THEN m.reads_7d ELSE count(pv.id) FILTER(WHERE pv.viewed_at>now()-interval '7 days') END, + 'reads_30d',CASE WHEN count(pv.id)=0 THEN m.reads_30d ELSE count(pv.id) FILTER(WHERE pv.viewed_at>now()-interval '30 days') END, + 'avg_duration_seconds',COALESCE(avg(NULLIF(pv.duration_seconds,0))::int,0), + 'last_viewed_at',COALESCE(max(pv.viewed_at),p.updated_at) + ) FROM docs_page p JOIN docs_module m ON m.id=p.module_id JOIN docs_version v ON v.id=p.version_id LEFT JOIN docs_page_view pv ON pv.doc_id=p.doc_id GROUP BY p.id,m.id,v.id ORDER BY count(pv.id) DESC,m.reads_30d DESC` + if err := loadQuery(context.Background(), p.pool, query, &values); err != nil { + return []PageStat{} + } + return values +} +func (p *PostgresRepository) PageReadStats(docID string, days int) PageReadStats { + if days <= 0 { + days = 30 + } + now := time.Now().UTC() + today := now.Truncate(24 * time.Hour) + daily := make([]DailyReadPoint, days) + index := map[string]int{} + for i := 0; i < days; i++ { + date := today.AddDate(0, 0, -(days - 1 - i)).Format("2006-01-02") + daily[i] = DailyReadPoint{Date: date} + index[date] = i + } + type readerAggregate struct { + userID string + count, duration int + last time.Time + } + readers := map[string]*readerAggregate{} + total, totalDuration, timed := 0, 0, 0 + window := today.AddDate(0, 0, -(days - 1)) + for _, view := range p.pageViews() { + if view.DocID != docID { + continue + } + total++ + if view.DurationSeconds > 0 { + totalDuration += view.DurationSeconds + timed++ + } + if !view.ViewedAt.Before(window) { + if i, ok := index[view.ViewedAt.UTC().Format("2006-01-02")]; ok { + daily[i].Count++ + } + } + key := view.UserID + if key == "" { + key = "session:" + view.SessionID + } + aggregate := readers[key] + if aggregate == nil { + aggregate = &readerAggregate{userID: view.UserID} + readers[key] = aggregate + } + aggregate.count++ + aggregate.duration += view.DurationSeconds + if view.ViewedAt.After(aggregate.last) { + aggregate.last = view.ViewedAt + } + } + result := PageReadStats{DocID: docID, Total: total, Daily: daily, Readers: []ReaderStat{}} + if timed > 0 { + result.AvgDurationSec = totalDuration / timed + } + users := map[string]User{} + for _, user := range p.Users("") { + users[user.ID] = user + } + for _, aggregate := range readers { + name := "匿名" + if aggregate.userID != "" { + if user, ok := users[aggregate.userID]; ok { + name = firstNonEmpty(user.DisplayName, user.Username) + } else { + name = aggregate.userID + } + } + average := 0 + if aggregate.count > 0 { + average = aggregate.duration / aggregate.count + } + result.Readers = append(result.Readers, ReaderStat{Reader: name, UserID: aggregate.userID, Count: aggregate.count, AvgDurationSec: average, LastReadAt: aggregate.last}) + } + sort.Slice(result.Readers, func(i, j int) bool { + if result.Readers[i].Count != result.Readers[j].Count { + return result.Readers[i].Count > result.Readers[j].Count + } + return result.Readers[i].LastReadAt.After(result.Readers[j].LastReadAt) + }) + return result +} + +func (p *PostgresRepository) UserFavorites(userID string) []UserFavorite { + var values []UserFavorite + if err := loadQuery(context.Background(), p.pool, `SELECT to_jsonb(t) FROM user_favorite t WHERE user_id=$1 ORDER BY created_at DESC`, &values, userID); err != nil { + return []UserFavorite{} + } + return values +} +func (p *PostgresRepository) SetUserFavorite(userID, moduleKey string, favorite bool) ([]UserFavorite, error) { + userID = strings.TrimSpace(userID) + moduleKey = strings.TrimSpace(moduleKey) + if userID == "" || moduleKey == "" { + return nil, ErrInvalid + } + if _, err := p.Module(moduleKey); err != nil { + return nil, err + } + ctx := context.Background() + if favorite { + _, err := p.pool.Exec(ctx, `INSERT INTO user_favorite(id,user_id,module_key,created_at) VALUES($1,$2,$3,now()) ON CONFLICT(user_id,module_key) DO NOTHING`, databaseID("fav"), userID, moduleKey) + if err != nil { + return nil, err + } + } else { + if _, err := p.pool.Exec(ctx, `DELETE FROM user_favorite WHERE user_id=$1 AND module_key=$2`, userID, moduleKey); err != nil { + return nil, err + } + } + return p.UserFavorites(userID), nil +} +func (p *PostgresRepository) UserRecentDocs(userID string, limit int) []UserRecentDoc { + if limit <= 0 { + limit = 30 + } + var values []UserRecentDoc + if err := loadQuery(context.Background(), p.pool, `SELECT to_jsonb(t) FROM user_recent_doc t WHERE user_id=$1 ORDER BY viewed_at DESC LIMIT $2`, &values, userID, limit); err != nil { + return []UserRecentDoc{} + } + return values +} +func (p *PostgresRepository) RecordUserRecentDoc(userID string, recent UserRecentDoc) (UserRecentDoc, error) { + userID = strings.TrimSpace(userID) + recent.DocID = strings.TrimSpace(recent.DocID) + if userID == "" || recent.DocID == "" { + return UserRecentDoc{}, ErrInvalid + } + if page, err := p.Page(recent.DocID); err == nil { + recent.Title = firstNonEmpty(recent.Title, page.Title) + recent.ModuleKey = firstNonEmpty(recent.ModuleKey, page.ModuleKey) + recent.ModuleName = firstNonEmpty(recent.ModuleName, page.ModuleName) + recent.DocsVersion = firstNonEmpty(recent.DocsVersion, page.DocsVersion) + recent.EntryKey = firstNonEmpty(recent.EntryKey, page.EntryKey) + recent.Href = firstNonEmpty(recent.Href, "/docs/"+page.ModuleKey+"/"+page.DocsVersion+"/"+page.EntryKey) + } + if recent.ID == "" { + recent.ID = databaseID("recent") + } + recent.UserID = userID + if recent.ViewedAt.IsZero() { + recent.ViewedAt = time.Now().UTC() + } + return queryJSONOne[UserRecentDoc](context.Background(), p.pool, `INSERT INTO user_recent_doc(id,user_id,doc_id,title,module_key,module_name,docs_version,entry_key,href,viewed_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) ON CONFLICT(user_id,doc_id) DO UPDATE SET title=EXCLUDED.title,module_key=EXCLUDED.module_key,module_name=EXCLUDED.module_name,docs_version=EXCLUDED.docs_version,entry_key=EXCLUDED.entry_key,href=EXCLUDED.href,viewed_at=EXCLUDED.viewed_at RETURNING to_jsonb(user_recent_doc)`, recent.ID, recent.UserID, recent.DocID, recent.Title, recent.ModuleKey, recent.ModuleName, recent.DocsVersion, recent.EntryKey, recent.Href, recent.ViewedAt) +} diff --git a/backend/internal/store/postgres_catalog.go b/backend/internal/store/postgres_catalog.go new file mode 100644 index 0000000..be718df --- /dev/null +++ b/backend/internal/store/postgres_catalog.go @@ -0,0 +1,690 @@ +package store + +import ( + "context" + "encoding/json" + "errors" + "path" + "sort" + "strconv" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +func (p *PostgresRepository) AllCategories() []Category { + var categories []Category + if err := loadQuery(context.Background(), p.pool, `SELECT (to_jsonb(c)-'created_at'-'updated_at') || jsonb_build_object( + 'responsible_team_info', + CASE WHEN t.id IS NULL THEN NULL ELSE jsonb_build_object( + 'key', t.key, + 'name', t.name, + 'description', t.description, + 'leaders', t.leaders, + 'members', t.members + ) END + ) + FROM docs_category c + LEFT JOIN teams t ON lower(t.key)=lower(c.responsible_team) + ORDER BY c.sort_order,c.id`, &categories); err != nil { + return []Category{} + } + return categories +} + +func (p *PostgresRepository) CategoryName(id string) string { + var name string + if err := p.pool.QueryRow(context.Background(), `SELECT name FROM docs_category WHERE id=$1`, id).Scan(&name); err != nil { + return id + } + return name +} + +func (p *PostgresRepository) categoryPath(ids []string) string { + if len(ids) == 0 { + return "" + } + parts := make([]string, 0, len(ids)) + for _, id := range ids { + parts = append(parts, p.CategoryName(id)) + } + return strings.Join(parts, " / ") +} + +func (p *PostgresRepository) CategoryTree() []Category { + byParent := map[string][]Category{} + for _, category := range p.AllCategories() { + category.Children = nil + byParent[category.ParentID] = append(byParent[category.ParentID], category) + } + var attach func(string) []Category + attach = func(parent string) []Category { + nodes := byParent[parent] + sort.SliceStable(nodes, func(i, j int) bool { return nodes[i].SortOrder < nodes[j].SortOrder }) + for i := range nodes { + nodes[i].Children = attach(nodes[i].ID) + } + return nodes + } + result := attach("") + if result == nil { + return []Category{} + } + return result +} + +func (p *PostgresRepository) createCategoryKey(ctx context.Context, name, parentID string) string { + base := slugifyKey(name) + if base == "" { + base = "d" + strconv.FormatInt(time.Now().UnixNano()%1_000_000, 36) + } + prefix := "" + _ = p.pool.QueryRow(ctx, `SELECT key||'.' FROM docs_category WHERE id=$1`, parentID).Scan(&prefix) + for index := 1; ; index++ { + key := prefix + base + if index > 1 { + key += "-" + strconv.Itoa(index) + } + var exists bool + _ = p.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM docs_category WHERE key=$1 OR id=$1)`, key).Scan(&exists) + if !exists { + return key + } + } +} + +func (p *PostgresRepository) CreateCategory(category Category) (Category, error) { + ctx := context.Background() + if strings.TrimSpace(category.Key) == "" { + if strings.TrimSpace(category.Name) == "" { + return Category{}, ErrInvalid + } + category.Key = p.createCategoryKey(ctx, category.Name, category.ParentID) + } + if category.ID == "" { + category.ID = category.Key + } + if category.Status == "" { + category.Status = "active" + } + created, err := queryJSONOne[Category](ctx, p.pool, `INSERT INTO docs_category(id,parent_id,key,name,description,icon,sort_order,status,responsible_team) VALUES($1,NULLIF($2,''),$3,$4,$5,$6,$7,$8,$9) RETURNING to_jsonb(docs_category)-'created_at'-'updated_at'`, category.ID, category.ParentID, category.Key, category.Name, category.Description, category.Icon, category.SortOrder, category.Status, category.ResponsibleTeam) + return created, postgresError(err) +} + +func (p *PostgresRepository) MoveCategory(id, parentID string, index int) (Category, error) { + ctx := context.Background() + tx, err := p.pool.Begin(ctx) + if err != nil { + return Category{}, err + } + defer tx.Rollback(ctx) + category, err := queryJSONOne[Category](ctx, tx, `SELECT to_jsonb(c)-'created_at'-'updated_at' FROM docs_category c WHERE id=$1 FOR UPDATE`, id) + if err != nil { + return Category{}, err + } + if parentID == id { + return Category{}, ErrInvalid + } + if parentID != "" { + var valid bool + if err = tx.QueryRow(ctx, `WITH RECURSIVE parents AS (SELECT id,parent_id FROM docs_category WHERE id=$1 UNION ALL SELECT c.id,c.parent_id FROM docs_category c JOIN parents p ON c.id=p.parent_id) SELECT EXISTS(SELECT 1 FROM docs_category WHERE id=$1) AND NOT EXISTS(SELECT 1 FROM parents WHERE id=$2)`, parentID, id).Scan(&valid); err != nil || !valid { + return Category{}, ErrInvalid + } + } + rows, err := tx.Query(ctx, `SELECT id FROM docs_category WHERE parent_id IS NOT DISTINCT FROM NULLIF($1,'') AND id<>$2 ORDER BY sort_order,id FOR UPDATE`, parentID, id) + if err != nil { + return Category{}, err + } + var siblings []string + for rows.Next() { + var sibling string + if err = rows.Scan(&sibling); err != nil { + rows.Close() + return Category{}, err + } + siblings = append(siblings, sibling) + } + rows.Close() + if index < 0 { + index = 0 + } + if index > len(siblings) { + index = len(siblings) + } + siblings = append(siblings, "") + copy(siblings[index+1:], siblings[index:]) + siblings[index] = id + if _, err = tx.Exec(ctx, `UPDATE docs_category SET parent_id=NULLIF($2,'') WHERE id=$1`, id, parentID); err != nil { + return Category{}, err + } + for position, sibling := range siblings { + if _, err = tx.Exec(ctx, `UPDATE docs_category SET sort_order=$2 WHERE id=$1`, sibling, (position+1)*10); err != nil { + return Category{}, err + } + } + category.ParentID, category.SortOrder = parentID, (index+1)*10 + return category, tx.Commit(ctx) +} + +func (p *PostgresRepository) UpdateCategory(id string, patch Category) (Category, error) { + current, err := queryJSONOne[Category](context.Background(), p.pool, `SELECT to_jsonb(c)-'created_at'-'updated_at' FROM docs_category c WHERE id=$1`, id) + if err != nil { + return Category{}, err + } + if patch.Name != "" { + current.Name = patch.Name + } + if patch.Description != "" { + current.Description = patch.Description + } + if patch.Icon != "" { + current.Icon = patch.Icon + } + if patch.SortOrder != 0 { + current.SortOrder = patch.SortOrder + } + if patch.Status != "" { + current.Status = patch.Status + } + if patch.ParentID != "" { + current.ParentID = patch.ParentID + } + current.ResponsibleTeam = patch.ResponsibleTeam + return queryJSONOne[Category](context.Background(), p.pool, `UPDATE docs_category SET parent_id=NULLIF($2,''),name=$3,description=$4,icon=$5,sort_order=$6,status=$7,responsible_team=$8 WHERE id=$1 RETURNING to_jsonb(docs_category)-'created_at'-'updated_at'`, id, current.ParentID, current.Name, current.Description, current.Icon, current.SortOrder, current.Status, current.ResponsibleTeam) +} + +func (p *PostgresRepository) DeleteCategory(id string) error { + command, err := p.pool.Exec(context.Background(), `DELETE FROM docs_category WHERE id=$1 AND NOT EXISTS(SELECT 1 FROM docs_category WHERE parent_id=$1)`, id) + if err != nil { + return postgresError(err) + } + if command.RowsAffected() == 0 { + var exists bool + _ = p.pool.QueryRow(context.Background(), `SELECT EXISTS(SELECT 1 FROM docs_category WHERE id=$1)`, id).Scan(&exists) + if exists { + return ErrConflict + } + return ErrNotFound + } + return nil +} + +func (p *PostgresRepository) Module(moduleKey string) (Module, error) { + var raw []byte + var deployToken string + err := p.pool.QueryRow(context.Background(), `SELECT (to_jsonb(m)-'default_version_id'-'created_at'-'deploy_token') || jsonb_build_object('category_ids',COALESCE((SELECT jsonb_agg(mc.category_id ORDER BY mc.is_primary DESC,mc.category_id) FROM docs_module_category mc WHERE mc.module_id=m.id),'[]'::jsonb),'category_path',COALESCE(NULLIF(m.category_path,''),(SELECT string_agg(c.name,' / ' ORDER BY mc.is_primary DESC,mc.category_id) FROM docs_module_category mc JOIN docs_category c ON c.id=mc.category_id WHERE mc.module_id=m.id),''),'deploy_token_set',COALESCE(m.deploy_token,'')<>''),COALESCE(m.deploy_token,'') FROM docs_module m WHERE lower(module_key)=lower($1)`, moduleKey).Scan(&raw, &deployToken) + if errors.Is(err, pgx.ErrNoRows) { + return Module{}, ErrNotFound + } + if err != nil { + return Module{}, err + } + var module Module + if err = json.Unmarshal(raw, &module); err != nil { + return Module{}, err + } + module.DeployToken = deployToken + module.AvailableVers = p.Versions(module.ModuleKey) + return module, nil +} + +func (p *PostgresRepository) ModuleByDeployToken(token string) (Module, error) { + token = strings.TrimSpace(token) + if token == "" { + return Module{}, ErrNotFound + } + var raw []byte + var deployToken string + err := p.pool.QueryRow(context.Background(), `SELECT (to_jsonb(m)-'default_version_id'-'created_at'-'deploy_token') || jsonb_build_object('category_ids',COALESCE((SELECT jsonb_agg(mc.category_id ORDER BY mc.is_primary DESC,mc.category_id) FROM docs_module_category mc WHERE mc.module_id=m.id),'[]'::jsonb),'category_path',COALESCE(NULLIF(m.category_path,''),(SELECT string_agg(c.name,' / ' ORDER BY mc.is_primary DESC,mc.category_id) FROM docs_module_category mc JOIN docs_category c ON c.id=mc.category_id WHERE mc.module_id=m.id),''),'deploy_token_set',COALESCE(m.deploy_token,'')<>''),COALESCE(m.deploy_token,'') FROM docs_module m WHERE deploy_token=$1`, token).Scan(&raw, &deployToken) + if errors.Is(err, pgx.ErrNoRows) { + return Module{}, ErrNotFound + } + if err != nil { + return Module{}, err + } + var module Module + if err = json.Unmarshal(raw, &module); err != nil { + return Module{}, err + } + module.DeployToken = deployToken + module.AvailableVers = p.Versions(module.ModuleKey) + return module, nil +} + +func (p *PostgresRepository) uniqueModuleKey(name string) string { + base := slugifyKey(name) + if base == "" { + base = "doc-" + strconv.FormatInt(time.Now().UnixNano()%1_000_000, 36) + } + for index := 1; ; index++ { + key := base + if index > 1 { + key += "-" + strconv.Itoa(index) + } + var exists bool + _ = p.pool.QueryRow(context.Background(), `SELECT EXISTS(SELECT 1 FROM docs_module WHERE lower(module_key)=lower($1))`, key).Scan(&exists) + if !exists { + return key + } + } +} + +func (p *PostgresRepository) CreateModule(module Module) (Module, error) { + if strings.TrimSpace(module.ModuleKey) == "" { + if strings.TrimSpace(module.Name) == "" { + return Module{}, ErrInvalid + } + module.ModuleKey = p.uniqueModuleKey(module.Name) + } + if module.ID == "" { + module.ID = databaseID("m") + } + if module.DeployToken == "" { + module.DeployToken = "mdx_" + strconv.FormatInt(time.Now().UnixNano(), 36) + } + if module.Name == "" { + module.Name = module.ModuleKey + } + if module.Status == "" { + module.Status = "active" + } + if module.DefaultVersion == "" { + module.DefaultVersion = "latest" + } + if module.CategoryPath == "" { + module.CategoryPath = p.categoryPath(module.CategoryIDs) + } + ctx := context.Background() + tx, err := p.pool.Begin(ctx) + if err != nil { + return Module{}, err + } + defer tx.Rollback(ctx) + created, err := queryJSONOne[Module](ctx, tx, `INSERT INTO docs_module(id,module_key,name,description,owner_group,repo_type,repo_url,default_version,visibility,status,package_name,package_version,channel,edition,keywords,maintainers,category_path,source_type,doc_type,mount,gitlab_branch,gitlab_path,deploy_token,last_synced_commit,last_synced_at,created_by,reads_7d,reads_30d,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15::jsonb,$16::jsonb,$17,$18,$19,$20,$21,$22,$23,$24,NULLIF($25,'')::timestamptz,NULLIF($26,''),$27,$28,now()) RETURNING to_jsonb(docs_module)-'default_version_id'-'created_at'`, module.ID, module.ModuleKey, module.Name, module.Description, module.OwnerGroup, module.RepoType, module.RepoURL, module.DefaultVersion, module.Visibility, module.Status, module.PackageName, module.PackageVersion, module.Channel, module.Edition, mustJSON(module.Keywords), mustJSON(module.Maintainers), module.CategoryPath, module.SourceType, module.DocType, module.Mount, module.GitLabBranch, module.GitLabPath, module.DeployToken, module.LastSyncedCommit, timeText(module.LastSyncedAt), module.CreatedBy, module.Reads7d, module.Reads30d) + if err != nil { + return Module{}, postgresError(err) + } + if err = p.syncModuleCategories(ctx, tx, created.ID, module.CategoryIDs); err != nil { + return Module{}, err + } + created.CategoryIDs = cloneStrings(module.CategoryIDs) + created.DeployToken = module.DeployToken + created.DeployTokenSet = created.DeployToken != "" + return created, tx.Commit(ctx) +} + +func (p *PostgresRepository) syncModuleCategories(ctx context.Context, tx interface { + Exec(context.Context, string, ...any) (pgconn.CommandTag, error) +}, moduleID string, ids []string) error { + if _, err := tx.Exec(ctx, `DELETE FROM docs_module_category WHERE module_id=$1`, moduleID); err != nil { + return err + } + for index, id := range ids { + if _, err := tx.Exec(ctx, `INSERT INTO docs_module_category(module_id,category_id,is_primary) VALUES($1,$2,$3)`, moduleID, id, index == 0); err != nil { + return err + } + } + return nil +} + +func (p *PostgresRepository) UpdateModule(moduleKey string, patch Module) (Module, error) { + current, err := p.Module(moduleKey) + if err != nil { + return Module{}, err + } + if patch.Name != "" { + current.Name = patch.Name + } + if patch.Description != "" { + current.Description = patch.Description + } + if patch.OwnerGroup != "" { + current.OwnerGroup = patch.OwnerGroup + } + if patch.RepoType != "" { + current.RepoType = patch.RepoType + } + if patch.RepoURL != "" { + current.RepoURL = patch.RepoURL + } + if patch.DefaultVersion != "" { + current.DefaultVersion = patch.DefaultVersion + } + if patch.Visibility != "" { + current.Visibility = patch.Visibility + } + if patch.Status != "" { + current.Status = patch.Status + } + if patch.PackageVersion != "" { + current.PackageVersion = patch.PackageVersion + } + if patch.Channel != "" { + current.Channel = patch.Channel + } + if patch.Edition != "" { + current.Edition = patch.Edition + } + if patch.Keywords != nil { + current.Keywords = patch.Keywords + } + if patch.Maintainers != nil { + current.Maintainers = patch.Maintainers + } + if patch.CategoryIDs != nil { + current.CategoryIDs = patch.CategoryIDs + if patch.CategoryPath == "" { + current.CategoryPath = p.categoryPath(current.CategoryIDs) + } + } + if patch.CategoryPath != "" { + current.CategoryPath = patch.CategoryPath + } + if patch.SourceType != "" { + current.SourceType = patch.SourceType + } + if patch.DocType != "" { + current.DocType = patch.DocType + } + if patch.Mount != "" { + current.Mount = patch.Mount + } + if patch.GitLabBranch != "" { + current.GitLabBranch = patch.GitLabBranch + } + if patch.GitLabPath != "" { + current.GitLabPath = patch.GitLabPath + } + if patch.DeployToken != "" { + current.DeployToken = patch.DeployToken + } + if patch.CreatedBy != "" { + current.CreatedBy = patch.CreatedBy + } + ctx := context.Background() + tx, err := p.pool.Begin(ctx) + if err != nil { + return Module{}, err + } + defer tx.Rollback(ctx) + updated, err := queryJSONOne[Module](ctx, tx, `UPDATE docs_module SET name=$2,description=$3,owner_group=$4,repo_type=$5,repo_url=$6,default_version=$7,visibility=$8,status=$9,package_version=$10,channel=$11,edition=$12,keywords=$13::jsonb,maintainers=$14::jsonb,category_path=$15,source_type=$16,doc_type=$17,mount=$18,gitlab_branch=$19,gitlab_path=$20,deploy_token=$21,created_by=NULLIF($22,''),updated_at=now() WHERE id=$1 RETURNING to_jsonb(docs_module)-'default_version_id'-'created_at'`, current.ID, current.Name, current.Description, current.OwnerGroup, current.RepoType, current.RepoURL, current.DefaultVersion, current.Visibility, current.Status, current.PackageVersion, current.Channel, current.Edition, mustJSON(current.Keywords), mustJSON(current.Maintainers), current.CategoryPath, current.SourceType, current.DocType, current.Mount, current.GitLabBranch, current.GitLabPath, current.DeployToken, current.CreatedBy) + if err != nil { + return Module{}, err + } + if patch.CategoryIDs != nil { + if err = p.syncModuleCategories(ctx, tx, current.ID, current.CategoryIDs); err != nil { + return Module{}, err + } + if _, err = tx.Exec(ctx, `UPDATE docs_page SET category_ids=$2::jsonb WHERE module_id=$1`, current.ID, mustJSON(current.CategoryIDs)); err != nil { + return Module{}, err + } + } + updated.CategoryIDs = current.CategoryIDs + updated.AvailableVers = current.AvailableVers + updated.DeployToken = current.DeployToken + updated.DeployTokenSet = updated.DeployToken != "" + return updated, tx.Commit(ctx) +} + +func (p *PostgresRepository) Versions(moduleKey string) []Version { + var values []Version + if err := loadQuery(context.Background(), p.pool, `SELECT (to_jsonb(v)-'module_id'-'updated_at')||jsonb_build_object('module_key',m.module_key) FROM docs_version v JOIN docs_module m ON m.id=v.module_id WHERE lower(m.module_key)=lower($1) ORDER BY v.created_at`, &values, moduleKey); err != nil { + return []Version{} + } + return values +} +func (p *PostgresRepository) Entries(moduleKey, docsVersion string) []Entry { + var values []Entry + if err := loadQuery(context.Background(), p.pool, `SELECT (to_jsonb(e)-'module_id'-'version_id'-'updated_at')||jsonb_build_object('module_key',m.module_key,'docs_version',v.docs_version) FROM docs_entry e JOIN docs_module m ON m.id=e.module_id JOIN docs_version v ON v.id=e.version_id WHERE lower(m.module_key)=lower($1) AND v.docs_version=$2 ORDER BY e.sort_order,e.id`, &values, moduleKey, docsVersion); err != nil { + return []Entry{} + } + return values +} +func (p *PostgresRepository) EntryModuleKey(entryID string) (string, bool) { + var key string + err := p.pool.QueryRow(context.Background(), `SELECT m.module_key FROM docs_entry e JOIN docs_module m ON m.id=e.module_id WHERE e.id=$1`, entryID).Scan(&key) + return key, err == nil +} + +func (p *PostgresRepository) CreateVersion(moduleKey string, v Version) (Version, error) { + if strings.TrimSpace(v.DocsVersion) == "" { + return Version{}, ErrInvalid + } + m, err := p.Module(moduleKey) + if err != nil { + return Version{}, err + } + if v.ID == "" { + v.ID = databaseID("v") + } + if v.DisplayName == "" { + v.DisplayName = v.DocsVersion + } + if v.Status == "" { + v.Status = "active" + } + ctx := context.Background() + tx, err := p.pool.Begin(ctx) + if err != nil { + return Version{}, err + } + defer tx.Rollback(ctx) + if v.IsDefault { + _, err = tx.Exec(ctx, `UPDATE docs_version SET is_default=false WHERE module_id=$1`, m.ID) + if err == nil { + _, err = tx.Exec(ctx, `UPDATE docs_module SET default_version=$2 WHERE id=$1`, m.ID, v.DocsVersion) + } + if err != nil { + return Version{}, err + } + } + created, err := queryJSONOne[Version](ctx, tx, `WITH changed AS (INSERT INTO docs_version(id,module_id,docs_version,display_name,version_type,is_default,status,source_branch,package_version,channel,edition,support_status,created_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,now()) RETURNING *) SELECT (to_jsonb(changed)-'module_id'-'updated_at')||jsonb_build_object('module_key',$13::text) FROM changed`, v.ID, m.ID, v.DocsVersion, v.DisplayName, v.VersionType, v.IsDefault, v.Status, v.SourceBranch, v.PackageVersion, v.Channel, v.Edition, v.SupportStatus, m.ModuleKey) + if err != nil { + return Version{}, postgresError(err) + } + return created, tx.Commit(ctx) +} + +func (p *PostgresRepository) UpdateVersion(moduleKey, docsVersion string, patch Version) (Version, error) { + m, err := p.Module(moduleKey) + if err != nil { + return Version{}, err + } + var current Version + for _, v := range m.AvailableVers { + if v.DocsVersion == docsVersion { + current = v + break + } + } + if current.ID == "" { + return Version{}, ErrNotFound + } + if patch.DisplayName != "" { + current.DisplayName = patch.DisplayName + } + if patch.VersionType != "" { + current.VersionType = patch.VersionType + } + if patch.Status != "" { + current.Status = patch.Status + } + if patch.SourceBranch != "" { + current.SourceBranch = patch.SourceBranch + } + if patch.PackageVersion != "" { + current.PackageVersion = patch.PackageVersion + } + if patch.Channel != "" { + current.Channel = patch.Channel + } + if patch.Edition != "" { + current.Edition = patch.Edition + } + if patch.SupportStatus != "" { + current.SupportStatus = patch.SupportStatus + } + ctx := context.Background() + tx, err := p.pool.Begin(ctx) + if err != nil { + return Version{}, err + } + defer tx.Rollback(ctx) + if patch.IsDefault { + if _, err = tx.Exec(ctx, `UPDATE docs_version SET is_default=false WHERE module_id=$1`, m.ID); err != nil { + return Version{}, err + } + if _, err = tx.Exec(ctx, `UPDATE docs_module SET default_version=$2 WHERE id=$1`, m.ID, docsVersion); err != nil { + return Version{}, err + } + current.IsDefault = true + } + updated, err := queryJSONOne[Version](ctx, tx, `WITH changed AS (UPDATE docs_version SET display_name=$2,version_type=$3,is_default=$4,status=$5,source_branch=$6,package_version=$7,channel=$8,edition=$9,support_status=$10 WHERE id=$1 RETURNING *) SELECT (to_jsonb(changed)-'module_id'-'updated_at')||jsonb_build_object('module_key',$11::text) FROM changed`, current.ID, current.DisplayName, current.VersionType, current.IsDefault, current.Status, current.SourceBranch, current.PackageVersion, current.Channel, current.Edition, current.SupportStatus, m.ModuleKey) + if err != nil { + return Version{}, err + } + return updated, tx.Commit(ctx) +} + +func (p *PostgresRepository) CreateEntry(moduleKey, docsVersion string, e Entry) (Entry, error) { + m, err := p.Module(moduleKey) + if err != nil { + return Entry{}, err + } + var versionID string + if err = p.pool.QueryRow(context.Background(), `SELECT id FROM docs_version WHERE module_id=$1 AND docs_version=$2`, m.ID, docsVersion).Scan(&versionID); err != nil { + return Entry{}, ErrNotFound + } + if strings.TrimSpace(e.EntryKey) == "" { + return Entry{}, ErrInvalid + } + if e.ID == "" { + e.ID = databaseID("e") + } + if e.EntryType == "" { + e.EntryType = "markdown" + } + if e.Builder == "" { + e.Builder = e.EntryType + } + if e.IndexStatus == "" { + e.IndexStatus = "pending" + } + if e.Status == "" { + e.Status = "active" + } + created, err := queryJSONOne[Entry](context.Background(), p.pool, `INSERT INTO docs_entry(id,module_id,version_id,entry_key,title,entry_type,builder,source,storage_uri,nav_uri,index_status,is_primary,sort_order,status,created_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,now()) RETURNING (to_jsonb(docs_entry)-'module_id'-'version_id'-'updated_at')||jsonb_build_object('module_key',$15::text,'docs_version',$16::text)`, e.ID, m.ID, versionID, e.EntryKey, e.Title, e.EntryType, e.Builder, e.Source, e.StorageURI, e.NavURI, e.IndexStatus, e.IsPrimary, e.SortOrder, e.Status, m.ModuleKey, docsVersion) + return created, postgresError(err) +} + +func (p *PostgresRepository) UpdateEntry(entryID string, patch Entry) (Entry, error) { + key, ok := p.EntryModuleKey(entryID) + if !ok { + return Entry{}, ErrNotFound + } + var current Entry + for _, version := range p.Versions(key) { + for _, entry := range p.Entries(key, version.DocsVersion) { + if entry.ID == entryID { + current = entry + break + } + } + } + if current.ID == "" { + return Entry{}, ErrNotFound + } + if patch.Title != "" { + current.Title = patch.Title + } + if patch.EntryType != "" { + current.EntryType = patch.EntryType + } + if patch.Builder != "" { + current.Builder = patch.Builder + } + if patch.Source != "" { + current.Source = patch.Source + } + if patch.StorageURI != "" { + current.StorageURI = patch.StorageURI + } + if patch.NavURI != "" { + current.NavURI = patch.NavURI + } + if patch.IndexStatus != "" { + current.IndexStatus = patch.IndexStatus + } + if patch.SortOrder != 0 { + current.SortOrder = patch.SortOrder + } + if patch.Status != "" { + current.Status = patch.Status + } + current.IsPrimary = patch.IsPrimary + return queryJSONOne[Entry](context.Background(), p.pool, `UPDATE docs_entry SET title=$2,entry_type=$3,builder=$4,source=$5,storage_uri=$6,nav_uri=$7,index_status=$8,is_primary=$9,sort_order=$10,status=$11 WHERE id=$1 RETURNING (to_jsonb(docs_entry)-'module_id'-'version_id'-'updated_at')||jsonb_build_object('module_key',$12::text,'docs_version',$13::text)`, entryID, current.Title, current.EntryType, current.Builder, current.Source, current.StorageURI, current.NavURI, current.IndexStatus, current.IsPrimary, current.SortOrder, current.Status, current.ModuleKey, current.DocsVersion) +} +func (p *PostgresRepository) DeleteEntry(entryID string) error { + command, err := p.pool.Exec(context.Background(), `DELETE FROM docs_entry WHERE id=$1`, entryID) + if err != nil { + return err + } + if command.RowsAffected() == 0 { + return ErrNotFound + } + return nil +} + +func (p *PostgresRepository) Releases() []Release { + var values []Release + if err := loadQuery(context.Background(), p.pool, `SELECT (to_jsonb(r)-'module_id'-'version_id')||jsonb_build_object('module_key',m.module_key,'docs_version',v.docs_version) FROM docs_release r JOIN docs_module m ON m.id=r.module_id JOIN docs_version v ON v.id=r.version_id ORDER BY r.published_at DESC`, &values); err != nil { + return []Release{} + } + return values +} +func (p *PostgresRepository) Release(releaseID string) (Release, error) { + return queryJSONOne[Release](context.Background(), p.pool, `SELECT (to_jsonb(r)-'module_id'-'version_id')||jsonb_build_object('module_key',m.module_key,'docs_version',v.docs_version) FROM docs_release r JOIN docs_module m ON m.id=r.module_id JOIN docs_version v ON v.id=r.version_id WHERE r.release_id=$1 OR r.id=$1`, releaseID) +} +func (p *PostgresRepository) RollbackRelease(releaseID string) (Release, error) { + return queryJSONOne[Release](context.Background(), p.pool, `WITH updated AS (UPDATE docs_release SET status='rolled_back' WHERE release_id=$1 OR id=$1 RETURNING *) SELECT (to_jsonb(r)-'module_id'-'version_id')||jsonb_build_object('module_key',m.module_key,'docs_version',v.docs_version) FROM updated r JOIN docs_module m ON m.id=r.module_id JOIN docs_version v ON v.id=r.version_id`, releaseID) +} + +func (p *PostgresRepository) Page(docID string) (Page, error) { + return queryJSONOne[Page](context.Background(), p.pool, `SELECT (to_jsonb(p)-'module_id'-'version_id'-'entry_id'-'release_id'-'last_verified_at'-'created_at')||jsonb_build_object('module_key',m.module_key,'module_name',m.name,'docs_version',v.docs_version,'package_version',v.package_version,'entry_key',COALESCE(e.entry_key,''),'entry_type',COALESCE(e.entry_type,'')) FROM docs_page p JOIN docs_module m ON m.id=p.module_id JOIN docs_version v ON v.id=p.version_id LEFT JOIN docs_entry e ON e.id=p.entry_id WHERE p.doc_id=$1`, docID) +} +func (p *PostgresRepository) PageByRoute(moduleKey, docsVersion, entryKey string) (Page, error) { + return queryJSONOne[Page](context.Background(), p.pool, `SELECT (to_jsonb(p)-'module_id'-'version_id'-'entry_id'-'release_id'-'last_verified_at'-'created_at')||jsonb_build_object('module_key',m.module_key,'module_name',m.name,'docs_version',v.docs_version,'package_version',v.package_version,'entry_key',COALESCE(e.entry_key,''),'entry_type',COALESCE(e.entry_type,'')) FROM docs_page p JOIN docs_module m ON m.id=p.module_id JOIN docs_version v ON v.id=p.version_id LEFT JOIN docs_entry e ON e.id=p.entry_id WHERE lower(m.module_key)=lower($1) AND v.docs_version=$2 AND e.entry_key=$3 LIMIT 1`, moduleKey, docsVersion, entryKey) +} +func (p *PostgresRepository) Nav(moduleKey, docsVersion string) []NavItem { + var raw []byte + if err := p.pool.QueryRow(context.Background(), `SELECT items_json FROM docs_nav WHERE lower(module_key)=lower($1) AND docs_version=$2`, moduleKey, docsVersion).Scan(&raw); err != nil { + return []NavItem{} + } + var nav []NavItem + if json.Unmarshal(raw, &nav) != nil { + return []NavItem{} + } + return nav +} +func (p *PostgresRepository) PageHTML(moduleKey, docsVersion, entryKey string) string { + page, err := p.PageByRoute(moduleKey, docsVersion, entryKey) + if err != nil { + return "" + } + return page.ContentHTML +} +func (p *PostgresRepository) SiteFile(moduleKey, docsVersion, entryKey, name string) (SiteFile, error) { + if name == "" { + name = "index.html" + } + var file SiteFile + err := p.pool.QueryRow(context.Background(), `SELECT name,content,content_type FROM docs_site_file WHERE lower(module_key)=lower($1) AND docs_version=$2 AND entry_key=$3 AND name=$4`, moduleKey, docsVersion, entryKey, path.Clean(strings.TrimPrefix(name, "/"))).Scan(&file.Name, &file.Content, &file.ContentType) + if errors.Is(err, pgx.ErrNoRows) { + return SiteFile{}, ErrNotFound + } + return file, err +} +func (p *PostgresRepository) ClearSiteAssets() {} +func (p *PostgresRepository) SiteObjects() map[string]SiteFile { return map[string]SiteFile{} } diff --git a/backend/internal/store/postgres_identity.go b/backend/internal/store/postgres_identity.go new file mode 100644 index 0000000..ba1cfc3 --- /dev/null +++ b/backend/internal/store/postgres_identity.go @@ -0,0 +1,379 @@ +package store + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +type rowQueryer interface { + QueryRow(context.Context, string, ...any) pgx.Row +} + +func queryJSONOne[T any](ctx context.Context, q rowQueryer, query string, args ...any) (T, error) { + var zero T + var raw []byte + if err := q.QueryRow(ctx, query, args...).Scan(&raw); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return zero, ErrNotFound + } + return zero, err + } + var value T + if err := json.Unmarshal(raw, &value); err != nil { + return zero, err + } + return value, nil +} + +func postgresError(err error) error { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == "23505" { + return ErrConflict + } + return err +} + +func databaseID(prefix string) string { + return fmt.Sprintf("%s-%d", prefix, time.Now().UTC().UnixNano()) +} + +func (p *PostgresRepository) CurrentUser() User { + user, err := queryJSONOne[User](context.Background(), p.pool, `SELECT to_jsonb(u) FROM users u ORDER BY username LIMIT 1`) + if err != nil { + return User{} + } + return user +} + +func (p *PostgresRepository) Users(keyword string) []User { + ctx := context.Background() + q := "%" + strings.TrimSpace(keyword) + "%" + var users []User + if err := loadQuery(ctx, p.pool, `SELECT to_jsonb(u) FROM users u + WHERE $1='' OR username ILIKE $1 OR display_name ILIKE $1 OR email ILIKE $1 OR department ILIKE $1 + ORDER BY username`, &users, q); err != nil { + return []User{} + } + return users +} + +func (p *PostgresRepository) UserByID(id string) (User, error) { + var raw []byte + var mcpToken string + err := p.pool.QueryRow(context.Background(), `SELECT to_jsonb(u),COALESCE(mcp_token,'') FROM users u WHERE id=$1`, id).Scan(&raw, &mcpToken) + if errors.Is(err, pgx.ErrNoRows) { + return User{}, ErrNotFound + } + if err != nil { + return User{}, err + } + var user User + if err = json.Unmarshal(raw, &user); err != nil { + return User{}, err + } + user.MCPToken = mcpToken + return user, nil +} + +func (p *PostgresRepository) UserByMCPToken(token string) (User, error) { + user, err := queryJSONOne[User](context.Background(), p.pool, `SELECT to_jsonb(u) FROM users u WHERE mcp_token=$1`, token) + if err == nil { + user.MCPToken = token + } + return user, err +} + +func (p *PostgresRepository) SetUserMCPToken(id, token string) (User, error) { + user, err := queryJSONOne[User](context.Background(), p.pool, `UPDATE users SET mcp_token=$2,updated_at=now() WHERE id=$1 RETURNING to_jsonb(users)`, id, token) + if err == nil { + user.MCPToken = token + } + return user, err +} + +func (p *PostgresRepository) CreateUser(u User) (User, error) { + u.Username = strings.TrimSpace(u.Username) + if u.Username == "" { + return User{}, ErrInvalid + } + if u.ID == "" { + u.ID = databaseID("u") + } + if u.DisplayName == "" { + u.DisplayName = u.Username + } + if u.Source == "" { + u.Source = "manual" + } + if u.Status == "" { + u.Status = "active" + } + ctx := context.Background() + tx, err := p.pool.Begin(ctx) + if err != nil { + return User{}, err + } + defer tx.Rollback(ctx) + created, err := queryJSONOne[User](ctx, tx, `INSERT INTO users + (id,username,display_name,email,department,avatar,roles_json,managed_categories_json,source,status,is_super_admin,mcp_token,created_at,updated_at) + VALUES($1,$2,$3,$4,$5,$6,$7::jsonb,$8::jsonb,$9,$10,$11,$12,now(),now()) RETURNING to_jsonb(users)`, + u.ID, u.Username, u.DisplayName, u.Email, u.Department, u.Avatar, mustJSONArray(u.Roles), mustJSONArray(u.ManagedCategories), u.Source, u.Status, u.SuperAdmin, u.MCPToken) + if err != nil { + return User{}, postgresError(err) + } + return created, tx.Commit(ctx) +} + +func (p *PostgresRepository) UpdateUser(id string, patch User) (User, error) { + ctx := context.Background() + tx, err := p.pool.Begin(ctx) + if err != nil { + return User{}, err + } + defer tx.Rollback(ctx) + current, err := queryJSONOne[User](ctx, tx, `SELECT to_jsonb(u) FROM users u WHERE id=$1 FOR UPDATE`, id) + if err != nil { + return User{}, err + } + if patch.DisplayName != "" { + current.DisplayName = patch.DisplayName + } + if patch.Email != "" { + current.Email = patch.Email + } + if patch.Department != "" { + current.Department = patch.Department + } + if patch.Roles != nil { + current.Roles = patch.Roles + } + if patch.ManagedCategories != nil { + current.ManagedCategories = patch.ManagedCategories + } + if patch.Status != "" { + current.Status = patch.Status + } + current.SuperAdmin = patch.SuperAdmin + updated, err := queryJSONOne[User](ctx, tx, `UPDATE users SET display_name=$2,email=$3,department=$4,roles_json=$5::jsonb,managed_categories_json=$6::jsonb,status=$7,is_super_admin=$8,updated_at=now() WHERE id=$1 RETURNING to_jsonb(users)`, id, current.DisplayName, current.Email, current.Department, mustJSONArray(current.Roles), mustJSONArray(current.ManagedCategories), current.Status, current.SuperAdmin) + if err != nil { + return User{}, err + } + return updated, tx.Commit(ctx) +} + +func (p *PostgresRepository) DeleteUser(id string) error { + command, err := p.pool.Exec(context.Background(), `DELETE FROM users WHERE id=$1`, id) + if err != nil { + return err + } + if command.RowsAffected() == 0 { + return ErrNotFound + } + return nil +} + +func (p *PostgresRepository) UpsertUser(u User) User { + u.Username = strings.TrimSpace(u.Username) + if u.ID == "" { + u.ID = databaseID("u") + } + if u.Status == "" { + u.Status = "active" + } + ctx := context.Background() + tx, err := p.pool.Begin(ctx) + if err != nil { + return u + } + defer tx.Rollback(ctx) + result, err := queryJSONOne[User](ctx, tx, `INSERT INTO users + (id,username,display_name,email,department,avatar,roles_json,managed_categories_json,source,status,is_super_admin,last_login_at,created_at,updated_at) + VALUES($1,$2,$3,$4,$5,$6,$7::jsonb,$8::jsonb,'oidc','active',$9,now(),now(),now()) + ON CONFLICT(username) DO UPDATE SET display_name=COALESCE(NULLIF(EXCLUDED.display_name,''),users.display_name),email=COALESCE(NULLIF(EXCLUDED.email,''),users.email),department=COALESCE(NULLIF(EXCLUDED.department,''),users.department),avatar=COALESCE(NULLIF(EXCLUDED.avatar,''),users.avatar),roles_json=CASE WHEN jsonb_typeof(EXCLUDED.roles_json)='array' AND jsonb_array_length(EXCLUDED.roles_json)>0 THEN EXCLUDED.roles_json WHEN jsonb_typeof(users.roles_json)='array' THEN users.roles_json ELSE '[]'::jsonb END,managed_categories_json=CASE WHEN jsonb_typeof(users.managed_categories_json)='array' THEN users.managed_categories_json ELSE '[]'::jsonb END,source='oidc',status='active',last_login_at=now(),updated_at=now() + RETURNING to_jsonb(users)`, u.ID, u.Username, u.DisplayName, u.Email, u.Department, u.Avatar, mustJSONArray(u.Roles), mustJSONArray(u.ManagedCategories), u.SuperAdmin) + if err != nil { + return u + } + if tx.Commit(ctx) != nil { + return u + } + return result +} + +func (p *PostgresRepository) Teams() []Team { + var teams []Team + if err := loadQuery(context.Background(), p.pool, `SELECT to_jsonb(t) FROM teams t ORDER BY key`, &teams); err != nil { + return []Team{} + } + return teams +} + +func (p *PostgresRepository) Team(key string) (Team, error) { + return queryJSONOne[Team](context.Background(), p.pool, `SELECT to_jsonb(t) FROM teams t WHERE lower(key)=lower($1) OR id=$1`, key) +} + +func normalizeTeam(t Team) Team { + if t.Name == "" { + t.Name = t.Key + } + for _, leader := range t.Leaders { + if leader != "" && !contains(t.Members, leader) { + t.Members = append(t.Members, leader) + } + } + return t +} + +func (p *PostgresRepository) uniqueTeamKey(ctx context.Context, name string) string { + base := slugifyKey(name) + if base == "" { + base = "team-" + strconv.FormatInt(time.Now().UnixNano()%1_000_000, 36) + } + for index := 1; ; index++ { + key := base + if index > 1 { + key += "-" + strconv.Itoa(index) + } + var exists bool + _ = p.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM teams WHERE lower(key)=lower($1))`, key).Scan(&exists) + if !exists { + return key + } + } +} + +func (p *PostgresRepository) CreateTeam(t Team) (Team, error) { + ctx := context.Background() + t.Key = strings.TrimSpace(t.Key) + if t.Key == "" { + if strings.TrimSpace(t.Name) == "" { + return Team{}, ErrInvalid + } + t.Key = p.uniqueTeamKey(ctx, t.Name) + } + if t.ID == "" { + t.ID = databaseID("t") + } + t = normalizeTeam(t) + tx, err := p.pool.Begin(ctx) + if err != nil { + return Team{}, err + } + defer tx.Rollback(ctx) + created, err := queryJSONOne[Team](ctx, tx, `INSERT INTO teams(id,key,name,description,leaders,members,created_at,updated_at) VALUES($1,$2,$3,$4,$5::jsonb,$6::jsonb,now(),now()) RETURNING to_jsonb(teams)`, t.ID, t.Key, t.Name, t.Description, mustJSON(t.Leaders), mustJSON(t.Members)) + if err != nil { + return Team{}, postgresError(err) + } + return created, tx.Commit(ctx) +} + +func (p *PostgresRepository) UpdateTeam(key string, patch Team) (Team, error) { + current, err := p.Team(key) + if err != nil { + return Team{}, err + } + if patch.Name != "" { + current.Name = patch.Name + } + if patch.Description != "" { + current.Description = patch.Description + } + if patch.Leaders != nil { + current.Leaders = cloneStrings(patch.Leaders) + } + if patch.Members != nil { + current.Members = cloneStrings(patch.Members) + } + current = normalizeTeam(current) + return queryJSONOne[Team](context.Background(), p.pool, `UPDATE teams SET name=$2,description=$3,leaders=$4::jsonb,members=$5::jsonb,updated_at=now() WHERE id=$1 RETURNING to_jsonb(teams)`, current.ID, current.Name, current.Description, mustJSON(current.Leaders), mustJSON(current.Members)) +} + +func (p *PostgresRepository) DeleteTeam(key string) error { + command, err := p.pool.Exec(context.Background(), `DELETE FROM teams WHERE lower(key)=lower($1) OR id=$1`, key) + if err != nil { + return err + } + if command.RowsAffected() == 0 { + return ErrNotFound + } + return nil +} + +func (p *PostgresRepository) AddTeamMember(key, member string) (Team, error) { + member = strings.TrimSpace(member) + if member == "" { + return Team{}, ErrInvalid + } + t, err := p.Team(key) + if err != nil { + return Team{}, err + } + if !contains(t.Members, member) { + t.Members = append(t.Members, member) + } + return queryJSONOne[Team](context.Background(), p.pool, `UPDATE teams SET members=$2::jsonb,updated_at=now() WHERE id=$1 RETURNING to_jsonb(teams)`, t.ID, mustJSON(t.Members)) +} + +func (p *PostgresRepository) RemoveTeamMember(key, member string) (Team, error) { + t, err := p.Team(key) + if err != nil { + return Team{}, err + } + members := make([]string, 0, len(t.Members)) + for _, value := range t.Members { + if !strings.EqualFold(value, strings.TrimSpace(member)) { + members = append(members, value) + } + } + return queryJSONOne[Team](context.Background(), p.pool, `UPDATE teams SET members=$2::jsonb,updated_at=now() WHERE id=$1 RETURNING to_jsonb(teams)`, t.ID, mustJSON(members)) +} + +func (p *PostgresRepository) SetTeamLeader(key, leader string) (Team, error) { + leader = strings.TrimSpace(leader) + if leader == "" { + return Team{}, ErrInvalid + } + t, err := p.Team(key) + if err != nil { + return Team{}, err + } + if !contains(t.Leaders, leader) { + t.Leaders = append(t.Leaders, leader) + } + if !contains(t.Members, leader) { + t.Members = append(t.Members, leader) + } + return queryJSONOne[Team](context.Background(), p.pool, `UPDATE teams SET leaders=$2::jsonb,members=$3::jsonb,updated_at=now() WHERE id=$1 RETURNING to_jsonb(teams)`, t.ID, mustJSON(t.Leaders), mustJSON(t.Members)) +} + +func (p *PostgresRepository) TeamMembers(key string) []string { + t, err := p.Team(key) + if err != nil { + return nil + } + return cloneStrings(t.Members) +} + +func (p *PostgresRepository) TeamKeysForUser(u User) []string { + var keys []string + rows, err := p.pool.Query(context.Background(), `SELECT key FROM teams WHERE members ? $1 OR members ? $2 ORDER BY key`, u.Username, u.ID) + if err != nil { + return keys + } + defer rows.Close() + for rows.Next() { + var key string + if rows.Scan(&key) == nil { + keys = append(keys, key) + } + } + return keys +} diff --git a/backend/internal/store/postgres_ingest.go b/backend/internal/store/postgres_ingest.go new file mode 100644 index 0000000..39d31ab --- /dev/null +++ b/backend/internal/store/postgres_ingest.go @@ -0,0 +1,180 @@ +package store + +import ( + "context" + "strconv" + "strings" + "time" +) + +func (p *PostgresRepository) IngestArtifact(artifact DeployArtifact) (DeployResult, error) { + if strings.TrimSpace(artifact.ModuleKey) == "" || strings.TrimSpace(artifact.DocsVersion) == "" || len(artifact.Entries) == 0 || len(artifact.Documents) == 0 { + return DeployResult{}, ErrInvalid + } + ctx := context.Background() + tx, err := p.pool.Begin(ctx) + if err != nil { + return DeployResult{}, err + } + defer tx.Rollback(ctx) + now := time.Now().UTC() + moduleName := firstNonEmpty(artifact.ModuleName, artifact.ModuleKey) + module, err := queryJSONOne[Module](ctx, tx, `SELECT (to_jsonb(m)-'default_version_id'-'created_at')||jsonb_build_object('category_ids',COALESCE((SELECT jsonb_agg(mc.category_id ORDER BY mc.is_primary DESC,mc.category_id) FROM docs_module_category mc WHERE mc.module_id=m.id),'[]'::jsonb)) FROM docs_module m WHERE lower(module_key)=lower($1) FOR UPDATE`, artifact.ModuleKey) + if err == ErrNotFound { + module = Module{ID: databaseID("m"), ModuleKey: artifact.ModuleKey, Name: moduleName, Description: artifact.Description, OwnerGroup: firstNonEmpty(firstString(artifact.Authors), "docs"), RepoType: firstNonEmpty(artifact.RepoType, "git"), RepoURL: artifact.RepoURL, SourceType: "gitlab", GitLabBranch: artifact.Branch, DefaultVersion: artifact.DocsVersion, Visibility: "internal", Status: "active", PackageName: artifact.ModuleKey, PackageVersion: artifact.PackageVersion, Channel: "docs", Edition: artifact.Edition, Keywords: cloneStrings(artifact.Keywords), Maintainers: cloneStrings(artifact.Authors), LastSyncedCommit: artifact.CommitSHA, LastSyncedAt: now} + module, err = queryJSONOne[Module](ctx, tx, `INSERT INTO docs_module(id,module_key,name,description,owner_group,repo_type,repo_url,default_version,visibility,status,package_name,package_version,channel,edition,keywords,maintainers,source_type,gitlab_branch,last_synced_commit,last_synced_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15::jsonb,$16::jsonb,$17,$18,$19,$20,$20) RETURNING to_jsonb(docs_module)-'default_version_id'-'created_at'`, module.ID, module.ModuleKey, module.Name, module.Description, module.OwnerGroup, module.RepoType, module.RepoURL, module.DefaultVersion, module.Visibility, module.Status, module.PackageName, module.PackageVersion, module.Channel, module.Edition, mustJSON(module.Keywords), mustJSON(module.Maintainers), module.SourceType, module.GitLabBranch, module.LastSyncedCommit, now) + } else if err == nil { + module.Name = moduleName + if artifact.Description != "" { + module.Description = artifact.Description + } + module.DefaultVersion = artifact.DocsVersion + if artifact.PackageVersion != "" { + module.PackageVersion = artifact.PackageVersion + } + if artifact.Edition != "" { + module.Edition = artifact.Edition + } + if len(artifact.Keywords) > 0 { + module.Keywords = cloneStrings(artifact.Keywords) + } + if len(artifact.Authors) > 0 { + module.Maintainers = cloneStrings(artifact.Authors) + if module.OwnerGroup == "" { + module.OwnerGroup = artifact.Authors[0] + } + } + module.Status = firstNonEmpty(module.Status, "active") + module.Visibility = firstNonEmpty(module.Visibility, "internal") + if artifact.RepoURL != "" { + module.RepoURL = artifact.RepoURL + } + if artifact.RepoType != "" { + module.RepoType = artifact.RepoType + } + if module.SourceType == "" || artifact.TriggerType == "pipeline" || artifact.RepoType == "gitlab" { + module.SourceType = deploySourceType(artifact.RepoType, artifact.RepoURL) + if module.SourceType == "manual" && artifact.TriggerType == "pipeline" { + module.SourceType = "gitlab" + } + } + if artifact.Branch != "" { + module.GitLabBranch = artifact.Branch + } + if artifact.CommitSHA != "" { + module.LastSyncedCommit = artifact.CommitSHA + } + module.LastSyncedAt = now + module, err = queryJSONOne[Module](ctx, tx, `UPDATE docs_module SET name=$2,description=$3,owner_group=$4,repo_type=$5,repo_url=$6,default_version=$7,visibility=$8,status=$9,package_version=$10,edition=$11,keywords=$12::jsonb,maintainers=$13::jsonb,gitlab_branch=$14,last_synced_commit=$15,last_synced_at=$16,source_type=$17,updated_at=$16 WHERE id=$1 RETURNING to_jsonb(docs_module)-'default_version_id'-'created_at'`, module.ID, module.Name, module.Description, module.OwnerGroup, module.RepoType, module.RepoURL, module.DefaultVersion, module.Visibility, module.Status, module.PackageVersion, module.Edition, mustJSON(module.Keywords), mustJSON(module.Maintainers), module.GitLabBranch, module.LastSyncedCommit, now, module.SourceType) + } + if err != nil { + return DeployResult{}, err + } + var categoryIDs []string + rows, err := tx.Query(ctx, `SELECT category_id FROM docs_module_category WHERE module_id=$1 ORDER BY is_primary DESC,category_id`, module.ID) + if err != nil { + return DeployResult{}, err + } + for rows.Next() { + var id string + if err = rows.Scan(&id); err != nil { + rows.Close() + return DeployResult{}, err + } + categoryIDs = append(categoryIDs, id) + } + rows.Close() + module.CategoryIDs = categoryIDs + var categoryPath string + _ = tx.QueryRow(ctx, `SELECT COALESCE(string_agg(c.name,' / ' ORDER BY mc.is_primary DESC,c.sort_order,c.id),'') FROM docs_module_category mc JOIN docs_category c ON c.id=mc.category_id WHERE mc.module_id=$1`, module.ID).Scan(&categoryPath) + module.CategoryPath = categoryPath + if _, err = tx.Exec(ctx, `UPDATE docs_module SET category_path=$2 WHERE id=$1`, module.ID, categoryPath); err != nil { + return DeployResult{}, err + } + if _, err = tx.Exec(ctx, `UPDATE docs_version SET is_default=false WHERE module_id=$1`, module.ID); err != nil { + return DeployResult{}, err + } + version, err := queryJSONOne[Version](ctx, tx, `WITH changed AS (INSERT INTO docs_version(id,module_id,docs_version,display_name,version_type,is_default,status,package_version,channel,edition,support_status,created_at,updated_at) VALUES($1,$2,$3,$3,'release',true,'active',$4,$5,$6,'supported',$7,$7) ON CONFLICT(module_id,docs_version) DO UPDATE SET display_name=COALESCE(NULLIF(docs_version.display_name,''),EXCLUDED.display_name),version_type=COALESCE(NULLIF(docs_version.version_type,''),'release'),is_default=true,status='active',package_version=EXCLUDED.package_version,edition=EXCLUDED.edition,support_status=COALESCE(NULLIF(docs_version.support_status,''),'supported'),updated_at=EXCLUDED.updated_at RETURNING *) SELECT (to_jsonb(changed)-'module_id'-'updated_at')||jsonb_build_object('module_key',$8::text) FROM changed`, databaseID("v"), module.ID, artifact.DocsVersion, artifact.PackageVersion, firstNonEmpty(module.Channel, "docs"), artifact.Edition, now, module.ModuleKey) + if err != nil { + return DeployResult{}, err + } + if _, err = tx.Exec(ctx, `UPDATE docs_page_view SET page_id=NULL WHERE page_id IN(SELECT id FROM docs_page WHERE module_id=$1 AND version_id=$2)`, module.ID, version.ID); err != nil { + return DeployResult{}, err + } + if _, err = tx.Exec(ctx, `UPDATE docs_embedding SET page_id=NULL,entry_id=NULL WHERE module_id=$1 AND version_id=$2`, module.ID, version.ID); err != nil { + return DeployResult{}, err + } + if _, err = tx.Exec(ctx, `DELETE FROM docs_page WHERE module_id=$1 AND version_id=$2`, module.ID, version.ID); err != nil { + return DeployResult{}, err + } + if _, err = tx.Exec(ctx, `DELETE FROM docs_entry WHERE module_id=$1 AND version_id=$2`, module.ID, version.ID); err != nil { + return DeployResult{}, err + } + entryIDs := map[string]string{} + for index, entry := range artifact.Entries { + id := databaseID("e") + entryIDs[entry.Key] = id + if _, err = tx.Exec(ctx, `INSERT INTO docs_entry(id,module_id,version_id,entry_key,title,entry_type,builder,source,storage_uri,nav_uri,index_status,is_primary,sort_order,status,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$6,$7,$8,$9,'indexed',$10,$11,'active',$12,$12)`, id, module.ID, version.ID, entry.Key, entry.Title, firstNonEmpty(entry.Type, "markdown"), entry.Source, "minio://modex/modules/"+module.ModuleKey+"/"+artifact.DocsVersion+"/site/"+entry.Key, "minio://modex/modules/"+module.ModuleKey+"/"+artifact.DocsVersion+"/nav.json", index == 0, index+1, now); err != nil { + return DeployResult{}, err + } + } + seenDocIDs := make(map[string]bool, len(artifact.Documents)) + for _, document := range artifact.Documents { + entryKey := firstNonEmpty(document.EntryKey, entryKeyFromDocID(document.DocID)) + docID := firstNonEmpty(document.DocID, artifact.ModuleKey+":"+artifact.DocsVersion+":"+entryKey) + // docs_page.doc_id is globally UNIQUE; a duplicate within one artifact + // would abort the whole deploy. Skip repeats (keeping the first) so a + // malformed artifact degrades to a partial index instead of a 400. + if seenDocIDs[docID] { + continue + } + seenDocIDs[docID] = true + entryType := firstNonEmpty(document.EntryType, entryTypeForEntry(artifact.Entries, entryKey)) + contentHTML := htmlForEntry(artifact.SiteHTML, entryKey) + if _, err = tx.Exec(ctx, ` + INSERT INTO docs_page(id,module_id,version_id,entry_id,doc_id,title,description,path,source_file,doc_type,status,owner_group,tags,category_ids,content_text,content_html,content_md,updated_at,created_at) + VALUES($1,$2,$3,NULLIF($4,''),$5,$6,$7,$8,$9,$10,$11,$12,$13::jsonb,$14::jsonb,$15,$16,$17,$18,$18) + ON CONFLICT(doc_id) DO UPDATE SET + module_id=EXCLUDED.module_id, + version_id=EXCLUDED.version_id, + entry_id=EXCLUDED.entry_id, + title=EXCLUDED.title, + description=EXCLUDED.description, + path=EXCLUDED.path, + source_file=EXCLUDED.source_file, + doc_type=EXCLUDED.doc_type, + status=EXCLUDED.status, + owner_group=EXCLUDED.owner_group, + tags=EXCLUDED.tags, + category_ids=EXCLUDED.category_ids, + content_text=EXCLUDED.content_text, + content_html=EXCLUDED.content_html, + content_md=EXCLUDED.content_md, + updated_at=EXCLUDED.updated_at`, databaseID("p"), module.ID, version.ID, entryIDs[entryKey], docID, firstNonEmpty(document.Title, titleForEntry(artifact.Entries, entryKey)), document.Description, docPagePath(artifact.ModuleKey, artifact.DocsVersion, entryKey, entryType, document.Path), document.SourceFile, entryType, firstNonEmpty(document.Status, "active"), module.OwnerGroup, mustJSON(coalesceStrings(document.Keywords, artifact.Keywords)), mustJSON(categoryIDs), document.Content, contentHTML, document.ContentMD, now); err != nil { + return DeployResult{}, err + } + } + if _, err = tx.Exec(ctx, `INSERT INTO docs_nav(module_key,docs_version,items_json,updated_at) VALUES($1,$2,$3::jsonb,$4) ON CONFLICT(module_key,docs_version) DO UPDATE SET items_json=EXCLUDED.items_json,updated_at=EXCLUDED.updated_at`, module.ModuleKey, artifact.DocsVersion, mustJSON(artifact.Nav), now); err != nil { + return DeployResult{}, err + } + if _, err = tx.Exec(ctx, `DELETE FROM docs_site_file WHERE lower(module_key)=lower($1) AND docs_version=$2`, module.ModuleKey, artifact.DocsVersion); err != nil { + return DeployResult{}, err + } + for name, content := range artifact.SiteFiles { + entryKey, relativeName, ok := splitSiteFile(name) + if !ok { + continue + } + if _, err = tx.Exec(ctx, `INSERT INTO docs_site_file(module_key,docs_version,entry_key,name,content,content_type,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7)`, module.ModuleKey, artifact.DocsVersion, entryKey, relativeName, content, contentTypeForName(relativeName, content), now); err != nil { + return DeployResult{}, err + } + } + release := Release{ID: databaseID("r"), ReleaseID: "rel-" + strings.ToLower(artifact.ModuleKey) + "-" + strings.ToLower(artifact.DocsVersion) + "-" + strconv.FormatInt(now.UnixNano(), 36), ModuleKey: module.ModuleKey, DocsVersion: artifact.DocsVersion, CommitSHA: artifact.CommitSHA, Branch: artifact.Branch, Publisher: firstNonEmpty(firstString(artifact.Authors), "docsctl"), BuildSystem: "docsctl", TriggerType: firstNonEmpty(artifact.TriggerType, "manual"), SourceIP: artifact.SourceIP, ArtifactVersion: now.Format("20060102.150405"), PackageVersion: artifact.PackageVersion, StorageURI: "minio://modex/modules/" + module.ModuleKey + "/" + artifact.DocsVersion + "/docs-artifact.zip", Status: "published", PublishedAt: now, CreatedAt: now} + if _, err = tx.Exec(ctx, `INSERT INTO docs_release(id,module_id,version_id,release_id,commit_sha,branch,publisher,build_system,trigger_type,source_ip,artifact_version,package_version,storage_uri,status,published_at,created_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$15)`, release.ID, module.ID, version.ID, release.ReleaseID, release.CommitSHA, release.Branch, release.Publisher, release.BuildSystem, release.TriggerType, release.SourceIP, release.ArtifactVersion, release.PackageVersion, release.StorageURI, release.Status, now); err != nil { + return DeployResult{}, err + } + if err = tx.Commit(ctx); err != nil { + return DeployResult{}, err + } + return DeployResult{Release: release, PagesIndexed: len(artifact.Documents), EntriesIndexed: len(artifact.Entries), HTMLFiles: len(artifact.SiteHTML), SiteFiles: len(artifact.SiteFiles), BytesReceived: artifact.Bytes}, nil +} diff --git a/backend/internal/store/postgres_oauth.go b/backend/internal/store/postgres_oauth.go new file mode 100644 index 0000000..ecc45a7 --- /dev/null +++ b/backend/internal/store/postgres_oauth.go @@ -0,0 +1,180 @@ +package store + +import ( + "context" + "strings" + "time" +) + +func (p *PostgresRepository) ConnectedApps() []ConnectedApp { + var apps []ConnectedApp + if err := loadQuery(context.Background(), p.pool, `SELECT to_jsonb(a) FROM connected_app a ORDER BY created_at DESC,id`, &apps); err != nil { + return []ConnectedApp{} + } + return apps +} +func (p *PostgresRepository) ConnectedAppByClientID(clientID string) (ConnectedApp, error) { + return queryJSONOne[ConnectedApp](context.Background(), p.pool, `SELECT to_jsonb(a) FROM connected_app a WHERE client_id=$1`, clientID) +} +func (p *PostgresRepository) CreateConnectedApp(app ConnectedApp, clientSecret string) (ConnectedApp, error) { + app.Name = strings.TrimSpace(app.Name) + app.ClientID = strings.TrimSpace(app.ClientID) + app.RedirectURIs = cleanNonEmpty(app.RedirectURIs) + app.Scopes = normalizeScopes(app.Scopes) + if app.Name == "" || app.ClientID == "" || len(app.RedirectURIs) == 0 { + return ConnectedApp{}, ErrInvalid + } + if len(app.Scopes) == 0 { + app.Scopes = []string{"modex:mcp:read"} + } + if app.ID == "" { + app.ID = databaseID("app") + } + app.ClientSecretHash = hashToken(clientSecret) + created, err := queryJSONOne[ConnectedApp](context.Background(), p.pool, `INSERT INTO connected_app(id,name,description,client_id,client_secret_hash,redirect_uris,scopes,trusted,enabled,created_by,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6::jsonb,$7::jsonb,$8,$9,NULLIF($10,''),now(),now()) RETURNING to_jsonb(connected_app)`, app.ID, app.Name, app.Description, app.ClientID, app.ClientSecretHash, mustJSON(app.RedirectURIs), mustJSON(app.Scopes), app.Trusted, app.Enabled, app.CreatedBy) + return created, postgresError(err) +} +func (p *PostgresRepository) UpdateConnectedApp(id string, patch ConnectedApp) (ConnectedApp, error) { + app, err := queryJSONOne[ConnectedApp](context.Background(), p.pool, `SELECT to_jsonb(a) FROM connected_app a WHERE id=$1`, id) + if err != nil { + return ConnectedApp{}, err + } + if strings.TrimSpace(patch.Name) != "" { + app.Name = strings.TrimSpace(patch.Name) + } + app.Description = patch.Description + if patch.RedirectURIs != nil { + app.RedirectURIs = cleanNonEmpty(patch.RedirectURIs) + if len(app.RedirectURIs) == 0 { + return ConnectedApp{}, ErrInvalid + } + } + if patch.Scopes != nil { + app.Scopes = normalizeScopes(patch.Scopes) + } + app.Trusted = patch.Trusted + app.Enabled = patch.Enabled + return queryJSONOne[ConnectedApp](context.Background(), p.pool, `UPDATE connected_app SET name=$2,description=$3,redirect_uris=$4::jsonb,scopes=$5::jsonb,trusted=$6,enabled=$7,updated_at=now() WHERE id=$1 RETURNING to_jsonb(connected_app)`, id, app.Name, app.Description, mustJSON(app.RedirectURIs), mustJSON(app.Scopes), app.Trusted, app.Enabled) +} +func (p *PostgresRepository) DeleteConnectedApp(id string) error { + ctx := context.Background() + tx, err := p.pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + if _, err = tx.Exec(ctx, `DELETE FROM oauth_grant WHERE app_id=$1`, id); err != nil { + return err + } + command, err := tx.Exec(ctx, `DELETE FROM connected_app WHERE id=$1`, id) + if err != nil { + return err + } + if command.RowsAffected() == 0 { + return ErrNotFound + } + return tx.Commit(ctx) +} +func (p *PostgresRepository) VerifyConnectedAppSecret(clientID, clientSecret string) (ConnectedApp, error) { + return queryJSONOne[ConnectedApp](context.Background(), p.pool, `SELECT to_jsonb(a) FROM connected_app a WHERE client_id=$1 AND client_secret_hash=$2 AND enabled=true`, clientID, hashToken(clientSecret)) +} + +func (p *PostgresRepository) CreateOAuthCode(appID, userID, redirectURI string, scopes []string, code string, ttl time.Duration) (OAuthGrant, error) { + if appID == "" || userID == "" || redirectURI == "" || code == "" { + return OAuthGrant{}, ErrInvalid + } + return queryJSONOne[OAuthGrant](context.Background(), p.pool, `INSERT INTO oauth_grant(id,app_id,user_id,code_hash,redirect_uri,scopes,code_expires_at,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6::jsonb,$7,now(),now()) RETURNING to_jsonb(oauth_grant)`, databaseID("grant"), appID, userID, hashToken(code), redirectURI, mustJSON(normalizeScopes(scopes)), time.Now().UTC().Add(ttl)) +} + +func (p *PostgresRepository) RedeemOAuthCode(clientID, code, redirectURI, accessToken, refreshToken string, accessTTL, refreshTTL time.Duration) (OAuthGrant, ConnectedApp, User, error) { + ctx := context.Background() + tx, err := p.pool.Begin(ctx) + if err != nil { + return OAuthGrant{}, ConnectedApp{}, User{}, err + } + defer tx.Rollback(ctx) + grant, err := queryJSONOne[OAuthGrant](ctx, tx, `SELECT to_jsonb(g) FROM oauth_grant g JOIN connected_app a ON a.id=g.app_id WHERE g.code_hash=$1 AND g.redirect_uri=$2 AND g.revoked_at IS NULL AND g.code_expires_at>now() AND a.client_id=$3 AND a.enabled=true FOR UPDATE OF g`, hashToken(code), redirectURI, clientID) + if err != nil { + return OAuthGrant{}, ConnectedApp{}, User{}, err + } + app, err := queryJSONOne[ConnectedApp](ctx, tx, `SELECT to_jsonb(a) FROM connected_app a WHERE id=$1`, grant.AppID) + if err != nil { + return OAuthGrant{}, ConnectedApp{}, User{}, err + } + user, err := queryJSONOne[User](ctx, tx, `SELECT to_jsonb(u) FROM users u WHERE id=$1`, grant.UserID) + if err != nil { + return OAuthGrant{}, ConnectedApp{}, User{}, err + } + now := time.Now().UTC() + grant, err = queryJSONOne[OAuthGrant](ctx, tx, `UPDATE oauth_grant SET code_hash='',access_token_hash=$2,refresh_token_hash=$3,access_expires_at=$4,refresh_expires_at=$5,updated_at=$6 WHERE id=$1 RETURNING to_jsonb(oauth_grant)`, grant.ID, hashToken(accessToken), hashToken(refreshToken), now.Add(accessTTL), now.Add(refreshTTL), now) + if err != nil { + return OAuthGrant{}, ConnectedApp{}, User{}, err + } + if _, err = tx.Exec(ctx, `UPDATE connected_app SET last_used_at=$2,updated_at=$2 WHERE id=$1`, app.ID, now); err != nil { + return OAuthGrant{}, ConnectedApp{}, User{}, err + } + return grant, app, user, tx.Commit(ctx) +} + +func (p *PostgresRepository) RefreshOAuthToken(clientID, refreshToken, accessToken, nextRefreshToken string, accessTTL, refreshTTL time.Duration) (OAuthGrant, ConnectedApp, User, error) { + ctx := context.Background() + tx, err := p.pool.Begin(ctx) + if err != nil { + return OAuthGrant{}, ConnectedApp{}, User{}, err + } + defer tx.Rollback(ctx) + grant, err := queryJSONOne[OAuthGrant](ctx, tx, `SELECT to_jsonb(g) FROM oauth_grant g JOIN connected_app a ON a.id=g.app_id WHERE g.refresh_token_hash=$1 AND g.revoked_at IS NULL AND g.refresh_expires_at>now() AND a.client_id=$2 AND a.enabled=true FOR UPDATE OF g`, hashToken(refreshToken), clientID) + if err != nil { + return OAuthGrant{}, ConnectedApp{}, User{}, err + } + app, err := queryJSONOne[ConnectedApp](ctx, tx, `SELECT to_jsonb(a) FROM connected_app a WHERE id=$1`, grant.AppID) + if err != nil { + return OAuthGrant{}, ConnectedApp{}, User{}, err + } + user, err := queryJSONOne[User](ctx, tx, `SELECT to_jsonb(u) FROM users u WHERE id=$1`, grant.UserID) + if err != nil { + return OAuthGrant{}, ConnectedApp{}, User{}, err + } + now := time.Now().UTC() + grant, err = queryJSONOne[OAuthGrant](ctx, tx, `UPDATE oauth_grant SET access_token_hash=$2,refresh_token_hash=$3,access_expires_at=$4,refresh_expires_at=$5,updated_at=$6 WHERE id=$1 RETURNING to_jsonb(oauth_grant)`, grant.ID, hashToken(accessToken), hashToken(nextRefreshToken), now.Add(accessTTL), now.Add(refreshTTL), now) + if err != nil { + return OAuthGrant{}, ConnectedApp{}, User{}, err + } + if _, err = tx.Exec(ctx, `UPDATE connected_app SET last_used_at=$2,updated_at=$2 WHERE id=$1`, app.ID, now); err != nil { + return OAuthGrant{}, ConnectedApp{}, User{}, err + } + return grant, app, user, tx.Commit(ctx) +} + +func (p *PostgresRepository) UserByOAuthAccessToken(token string) (User, ConnectedApp, OAuthGrant, error) { + ctx := context.Background() + tx, err := p.pool.Begin(ctx) + if err != nil { + return User{}, ConnectedApp{}, OAuthGrant{}, err + } + defer tx.Rollback(ctx) + grant, err := queryJSONOne[OAuthGrant](ctx, tx, `SELECT to_jsonb(g) FROM oauth_grant g JOIN connected_app a ON a.id=g.app_id WHERE g.access_token_hash=$1 AND g.revoked_at IS NULL AND g.access_expires_at>now() AND a.enabled=true FOR UPDATE OF g`, hashToken(token)) + if err != nil { + return User{}, ConnectedApp{}, OAuthGrant{}, err + } + app, err := queryJSONOne[ConnectedApp](ctx, tx, `SELECT to_jsonb(a) FROM connected_app a WHERE id=$1`, grant.AppID) + if err != nil { + return User{}, ConnectedApp{}, OAuthGrant{}, err + } + user, err := queryJSONOne[User](ctx, tx, `SELECT to_jsonb(u) FROM users u WHERE id=$1`, grant.UserID) + if err != nil { + return User{}, ConnectedApp{}, OAuthGrant{}, err + } + now := time.Now().UTC() + if _, err = tx.Exec(ctx, `UPDATE oauth_grant SET updated_at=$2 WHERE id=$1`, grant.ID, now); err != nil { + return User{}, ConnectedApp{}, OAuthGrant{}, err + } + if _, err = tx.Exec(ctx, `UPDATE connected_app SET last_used_at=$2,updated_at=$2 WHERE id=$1`, app.ID, now); err != nil { + return User{}, ConnectedApp{}, OAuthGrant{}, err + } + return user, app, grant, tx.Commit(ctx) +} +func (p *PostgresRepository) RevokeOAuthToken(clientID, token string) bool { + command, err := p.pool.Exec(context.Background(), `UPDATE oauth_grant g SET revoked_at=now(),updated_at=now() FROM connected_app a WHERE a.id=g.app_id AND a.client_id=$1 AND (g.access_token_hash=$2 OR g.refresh_token_hash=$2 OR g.code_hash=$2)`, clientID, hashToken(token)) + return err == nil && command.RowsAffected() > 0 +} diff --git a/backend/internal/store/postgres_repository.go b/backend/internal/store/postgres_repository.go new file mode 100644 index 0000000..0be5707 --- /dev/null +++ b/backend/internal/store/postgres_repository.go @@ -0,0 +1,95 @@ +package store + +import ( + "context" + _ "embed" + "encoding/json" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +//go:embed schema.sql +var relationalSchema string + +// PostgresRepository is the production business store. Every method executes +// its read or write against PostgreSQL; it does not cache a process-local copy +// of application state. +type PostgresRepository struct { + pool *pgxpool.Pool +} + +func OpenPostgresRepository(ctx context.Context, databaseURL string) (*PostgresRepository, error) { + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + return nil, err + } + for { + if err := pool.Ping(ctx); err == nil { + break + } else { + select { + case <-ctx.Done(): + pool.Close() + return nil, fmt.Errorf("connect PostgreSQL: %w", err) + case <-time.After(500 * time.Millisecond): + } + } + } + if _, err := pool.Exec(ctx, relationalSchema); err != nil { + pool.Close() + return nil, fmt.Errorf("apply relational schema: %w", err) + } + return &PostgresRepository{pool: pool}, nil +} + +func (p *PostgresRepository) Close() { + if p != nil && p.pool != nil { + p.pool.Close() + } +} + +type queryer interface { + Query(context.Context, string, ...any) (pgx.Rows, error) +} + +func loadQuery[T any](ctx context.Context, q queryer, query string, out *[]T, args ...any) error { + rows, err := q.Query(ctx, query, args...) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var raw []byte + if err := rows.Scan(&raw); err != nil { + return err + } + var value T + if err := json.Unmarshal(raw, &value); err != nil { + return fmt.Errorf("decode relational record: %w", err) + } + *out = append(*out, value) + } + return rows.Err() +} + +func mustJSON(value any) string { + encoded, _ := json.Marshal(value) + return string(encoded) +} + +func mustJSONArray[T any](value []T) string { + if value == nil { + value = []T{} + } + return mustJSON(value) +} + +func timeText(value time.Time) string { + if value.IsZero() { + return "" + } + return value.UTC().Format(time.RFC3339Nano) +} diff --git a/backend/internal/store/postgres_repository_test.go b/backend/internal/store/postgres_repository_test.go new file mode 100644 index 0000000..be9ec38 --- /dev/null +++ b/backend/internal/store/postgres_repository_test.go @@ -0,0 +1,164 @@ +package store + +import ( + "context" + "fmt" + "os" + "testing" + "time" +) + +func TestPostgresRepositoryRequestLevelCRUD(t *testing.T) { + databaseURL := os.Getenv("TEST_DATABASE_URL") + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + repository, err := OpenPostgresRepository(ctx, databaseURL) + if err != nil { + t.Fatalf("OpenPostgresRepository: %v", err) + } + defer repository.Close() + + suffix := fmt.Sprint(time.Now().UnixNano()) + user, err := repository.CreateUser(User{ID: "u-db-" + suffix, Username: "db-" + suffix, DisplayName: "DB User"}) + if err != nil { + t.Fatalf("CreateUser: %v", err) + } + team, err := repository.CreateTeam(Team{Name: "DB Team " + suffix, Leaders: []string{user.Username}}) + if err != nil { + t.Fatalf("CreateTeam without key: %v", err) + } + if team.Key == "" || team.Name == "" || len(team.Leaders) != 1 || len(team.Members) != 1 || team.Members[0] != user.Username { + t.Fatalf("CreateTeam generated unexpected team: %+v", team) + } + category, err := repository.CreateCategory(Category{ID: "cat-" + suffix, Key: "cat-" + suffix, Name: "DB Category"}) + if err != nil { + t.Fatalf("CreateCategory: %v", err) + } + module, err := repository.CreateModule(Module{ID: "m-db-" + suffix, ModuleKey: "module-" + suffix, Name: "DB Module", CategoryIDs: []string{category.ID}}) + if err != nil { + t.Fatalf("CreateModule: %v", err) + } + version, err := repository.CreateVersion(module.ModuleKey, Version{ID: "v-db-" + suffix, DocsVersion: "latest", IsDefault: true}) + if err != nil { + t.Fatalf("CreateVersion: %v", err) + } + _, err = repository.CreateEntry(module.ModuleKey, version.DocsVersion, Entry{ID: "e-db-" + suffix, EntryKey: "guide", Title: "Guide"}) + if err != nil { + t.Fatalf("CreateEntry: %v", err) + } + secondRepository, err := OpenPostgresRepository(ctx, databaseURL) + if err != nil { + t.Fatalf("open second repository: %v", err) + } + defer secondRepository.Close() + if _, err = secondRepository.Module(module.ModuleKey); err != nil { + t.Fatalf("second repository did not observe committed module: %v", err) + } + app, err := repository.CreateConnectedApp(ConnectedApp{ID: "app-db-" + suffix, Name: "Repository Test", ClientID: "repository-" + suffix, RedirectURIs: []string{"http://localhost/callback"}, CreatedBy: user.ID, Enabled: true}, "secret") + if err != nil { + t.Fatalf("CreateConnectedApp: %v", err) + } + t.Cleanup(func() { + cleanup, openErr := OpenPostgresRepository(context.Background(), databaseURL) + if openErr != nil { + return + } + defer cleanup.Close() + _ = cleanup.DeleteConnectedApp(app.ID) + _, _ = cleanup.pool.Exec(context.Background(), `DELETE FROM docs_page_view WHERE module_id=$1`, module.ID) + _, _ = cleanup.pool.Exec(context.Background(), `DELETE FROM user_favorite WHERE module_key=$1`, module.ModuleKey) + _, _ = cleanup.pool.Exec(context.Background(), `DELETE FROM user_recent_doc WHERE module_key=$1`, module.ModuleKey) + _, _ = cleanup.pool.Exec(context.Background(), `DELETE FROM docs_feedback WHERE module_key=$1`, module.ModuleKey) + _, _ = cleanup.pool.Exec(context.Background(), `DELETE FROM docs_embedding WHERE module_id=$1`, module.ID) + _, _ = cleanup.pool.Exec(context.Background(), `DELETE FROM docs_page WHERE module_id=$1`, module.ID) + _, _ = cleanup.pool.Exec(context.Background(), `DELETE FROM docs_release WHERE module_id=$1`, module.ID) + _, _ = cleanup.pool.Exec(context.Background(), `DELETE FROM docs_entry WHERE module_id=$1`, module.ID) + _, _ = cleanup.pool.Exec(context.Background(), `DELETE FROM docs_site_file WHERE module_key=$1`, module.ModuleKey) + _, _ = cleanup.pool.Exec(context.Background(), `DELETE FROM docs_nav WHERE module_key=$1`, module.ModuleKey) + _, _ = cleanup.pool.Exec(context.Background(), `DELETE FROM docs_version WHERE id=$1`, version.ID) + _, _ = cleanup.pool.Exec(context.Background(), `DELETE FROM docs_module_category WHERE module_id=$1`, module.ID) + _, _ = cleanup.pool.Exec(context.Background(), `DELETE FROM docs_module WHERE id=$1`, module.ID) + _ = cleanup.DeleteCategory(category.ID) + _ = cleanup.DeleteTeam(team.Key) + _ = cleanup.DeleteUser(user.ID) + }) + + updatedUser, err := repository.SetUserMCPToken(user.ID, "mcp-direct") + if err != nil || updatedUser.MCPToken != "mcp-direct" { + t.Fatalf("SetUserMCPToken = %#v, %v", updatedUser, err) + } + loadedModule, err := repository.Module(module.ModuleKey) + if err != nil { + t.Fatalf("Module: %v", err) + } + if len(loadedModule.CategoryIDs) != 1 || loadedModule.CategoryIDs[0] != category.ID { + t.Fatalf("module categories = %#v", loadedModule.CategoryIDs) + } + if loadedModule.DeployToken == "" || !loadedModule.DeployTokenSet { + t.Fatal("module deploy token was not loaded for internal authentication") + } + if len(repository.Entries(module.ModuleKey, version.DocsVersion)) != 1 { + t.Fatal("entry was not read directly after insert") + } + if _, err := repository.VerifyConnectedAppSecret(app.ClientID, "secret"); err != nil { + t.Fatalf("VerifyConnectedAppSecret: %v", err) + } + + result, err := repository.IngestArtifact(DeployArtifact{ + ModuleKey: module.ModuleKey, ModuleName: module.Name, DocsVersion: "latest", + Authors: []string{user.Username}, CommitSHA: "abc123", + Entries: []DeployEntry{{Key: "guide", Title: "Guide", Type: "markdown"}}, + Documents: []DeployDocument{{DocID: module.ModuleKey + ":latest:guide", EntryKey: "guide", Title: "Guide", Content: "database-backed content"}}, + Nav: []NavItem{{Title: "Guide", Path: "/guide"}}, + SiteFiles: map[string][]byte{"site/guide/assets/app.css": []byte("body{}")}, + }) + if err != nil { + t.Fatalf("IngestArtifact: %v", err) + } + if result.PagesIndexed != 1 || len(repository.Releases()) == 0 { + t.Fatalf("ingest result = %#v", result) + } + docID := module.ModuleKey + ":latest:guide" + page, err := repository.Page(docID) + if err != nil || page.ContentText != "database-backed content" { + t.Fatalf("Page = %#v, %v", page, err) + } + if _, err := repository.SiteFile(module.ModuleKey, "latest", "guide", "assets/app.css"); err != nil { + t.Fatalf("SiteFile: %v", err) + } + view := repository.RecordPageView(PageView{DocID: docID, UserID: user.ID, SessionID: "session-" + suffix, ReadID: "read-" + suffix}) + view = repository.RecordReadProgress(docID, view.SessionID, view.ReadID, 42, 0.8) + if view.DurationSeconds != 42 || view.ScrollDepth != 0.8 { + t.Fatalf("read progress = %#v", view) + } + if _, err := repository.SetUserFavorite(user.ID, module.ModuleKey, true); err != nil { + t.Fatalf("SetUserFavorite: %v", err) + } + if _, err := repository.RecordUserRecentDoc(user.ID, UserRecentDoc{DocID: docID}); err != nil { + t.Fatalf("RecordUserRecentDoc: %v", err) + } + if len(repository.UserFavorites(user.ID)) != 1 || len(repository.UserRecentDocs(user.ID, 10)) != 1 { + t.Fatal("personal activity was not immediately readable") + } + + grant, err := repository.CreateOAuthCode(app.ID, user.ID, app.RedirectURIs[0], app.Scopes, "code-"+suffix, time.Minute) + if err != nil { + t.Fatalf("CreateOAuthCode: %v", err) + } + grant, _, _, err = repository.RedeemOAuthCode(app.ClientID, "code-"+suffix, app.RedirectURIs[0], "access-"+suffix, "refresh-"+suffix, time.Minute, time.Hour) + if err != nil || grant.AccessTokenHash == "" { + t.Fatalf("RedeemOAuthCode = %#v, %v", grant, err) + } + if _, _, _, err = repository.UserByOAuthAccessToken("access-" + suffix); err != nil { + t.Fatalf("UserByOAuthAccessToken: %v", err) + } + if _, _, _, err = repository.RefreshOAuthToken(app.ClientID, "refresh-"+suffix, "access-next-"+suffix, "refresh-next-"+suffix, time.Minute, time.Hour); err != nil { + t.Fatalf("RefreshOAuthToken: %v", err) + } + if !repository.RevokeOAuthToken(app.ClientID, "refresh-next-"+suffix) { + t.Fatal("RevokeOAuthToken did not update a row") + } +} diff --git a/backend/internal/store/postgres_search.go b/backend/internal/store/postgres_search.go new file mode 100644 index 0000000..5bf0876 --- /dev/null +++ b/backend/internal/store/postgres_search.go @@ -0,0 +1,103 @@ +package store + +import ( + "context" + "encoding/json" + "strings" +) + +// Pages reads the searchable document projection directly from PostgreSQL. +// It intentionally does not use PostgresRepository.load: request paths must +// not materialize an in-memory copy of the complete business store. +func (p *PostgresRepository) Pages() []Page { + var pages []Page + err := loadQuery(context.Background(), p.pool, ` + SELECT (to_jsonb(p)-'module_id'-'version_id'-'entry_id'-'release_id'-'last_verified_at'-'created_at') || + jsonb_build_object( + 'module_key',m.module_key, + 'module_name',m.name, + 'docs_version',v.docs_version, + 'package_version',v.package_version, + 'entry_key',COALESCE(e.entry_key,''), + 'entry_type',COALESCE(e.entry_type,'') + ) + FROM docs_page p + JOIN docs_module m ON m.id=p.module_id + JOIN docs_version v ON v.id=p.version_id + LEFT JOIN docs_entry e ON e.id=p.entry_id + ORDER BY p.id`, &pages) + if err != nil { + return []Page{} + } + return pages +} + +// Modules queries the catalog directly. Category membership is assembled by a +// correlated relational query rather than from a process-local snapshot. +func (p *PostgresRepository) Modules(categoryID, keyword string) []Module { + var modules []Module + args := []any{} + where := []string{"1=1"} + if categoryID != "" { + args = append(args, categoryID) + where = append(where, `EXISTS (SELECT 1 FROM docs_module_category f WHERE f.module_id=m.id AND f.category_id=$1)`) + } + if strings.TrimSpace(keyword) != "" { + args = append(args, "%"+strings.TrimSpace(keyword)+"%") + where = append(where, `(m.name ILIKE $`+itoa(len(args))+` OR m.module_key ILIKE $`+itoa(len(args))+` OR m.repo_url ILIKE $`+itoa(len(args))+`)`) + } + query := `SELECT (to_jsonb(m)-'default_version_id'-'created_at'-'deploy_token') || + jsonb_build_object( + 'category_ids',COALESCE((SELECT jsonb_agg(mc.category_id ORDER BY mc.is_primary DESC,mc.category_id) FROM docs_module_category mc WHERE mc.module_id=m.id),'[]'::jsonb), + 'category_path',COALESCE(NULLIF(m.category_path,''),(SELECT string_agg(c.name,' / ' ORDER BY mc.is_primary DESC,mc.category_id) FROM docs_module_category mc JOIN docs_category c ON c.id=mc.category_id WHERE mc.module_id=m.id),''), + 'deploy_token_set',COALESCE(m.deploy_token,'')<>'' + ) + FROM docs_module m WHERE ` + strings.Join(where, " AND ") + ` ORDER BY m.updated_at DESC,m.id` + if err := loadQuery(context.Background(), p.pool, query, &modules, args...); err != nil { + return []Module{} + } + for i := range modules { + modules[i].AvailableVers = p.Versions(modules[i].ModuleKey) + } + return modules +} + +func (p *PostgresRepository) Settings() Settings { + var raw []byte + if err := p.pool.QueryRow(context.Background(), `SELECT value_json FROM platform_settings WHERE key='main'`).Scan(&raw); err != nil { + return Settings{} + } + var settings Settings + if err := json.Unmarshal(raw, &settings); err != nil { + return Settings{} + } + return settings +} + +// Embeddings are owned by vectorstore.Postgres in production. These methods +// satisfy the fallback contract used by unit tests and non-vector deployments; +// a PostgreSQL application always configures the external vector store. +func (p *PostgresRepository) Embedding(string) ([]float32, bool) { return nil, false } +func (p *PostgresRepository) SetEmbedding(string, []float32) {} +func (p *PostgresRepository) ClearEmbeddings() { + _, _ = p.pool.Exec(context.Background(), `TRUNCATE docs_embedding`) +} +func (p *PostgresRepository) EmbeddingCount() int { + var count int + _ = p.pool.QueryRow(context.Background(), `SELECT count(*) FROM docs_embedding`).Scan(&count) + return count +} + +func itoa(value int) string { + if value == 0 { + return "0" + } + var digits [20]byte + i := len(digits) + for value > 0 { + i-- + digits[i] = byte('0' + value%10) + value /= 10 + } + return string(digits[i:]) +} diff --git a/backend/internal/store/postgres_settings.go b/backend/internal/store/postgres_settings.go new file mode 100644 index 0000000..9ebf9fc --- /dev/null +++ b/backend/internal/store/postgres_settings.go @@ -0,0 +1,246 @@ +package store + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" +) + +func (p *PostgresRepository) updateSettings(mutator func(*Settings) error) (Settings, error) { + ctx := context.Background() + tx, err := p.pool.Begin(ctx) + if err != nil { + return Settings{}, err + } + defer tx.Rollback(ctx) + var raw []byte + err = tx.QueryRow(ctx, `SELECT value_json FROM platform_settings WHERE key='main' FOR UPDATE`).Scan(&raw) + settings := Settings{} + if err == nil { + if err = json.Unmarshal(raw, &settings); err != nil { + return Settings{}, err + } + } + if err != nil { + _, err = tx.Exec(ctx, `INSERT INTO platform_settings(key,value_json) VALUES('main','{}'::jsonb) ON CONFLICT(key) DO NOTHING`) + if err != nil { + return Settings{}, err + } + } + if err = mutator(&settings); err != nil { + return Settings{}, err + } + if _, err = tx.Exec(ctx, `INSERT INTO platform_settings(key,value_json) VALUES('main',$1::jsonb) ON CONFLICT(key) DO UPDATE SET value_json=EXCLUDED.value_json`, mustJSON(settings)); err != nil { + return Settings{}, err + } + return settings, tx.Commit(ctx) +} + +func (p *PostgresRepository) SaveAISettings(ai AISettings) Settings { + settings, _ := p.updateSettings(func(current *Settings) error { + if strings.TrimSpace(ai.AskAPIKey) == "" { + ai.AskAPIKey = current.AI.AskAPIKey + } + if strings.TrimSpace(ai.EmbeddingAPIKey) == "" { + ai.EmbeddingAPIKey = current.AI.EmbeddingAPIKey + } + if strings.TrimSpace(ai.RerankAPIKey) == "" { + ai.RerankAPIKey = current.AI.RerankAPIKey + } + ai.UpdatedAt = time.Now().UTC() + current.AI = ai + return nil + }) + return settings +} + +func (p *PostgresRepository) PluginStates() []PluginState { + settings := p.Settings() + return mergePlugins(settings.Plugins, settings.UploadedPlugins) +} +func (p *PostgresRepository) PluginEffective() map[string]PluginSetting { + states := p.PluginStates() + result := make(map[string]PluginSetting, len(states)) + for _, state := range states { + result[state.Key] = PluginSetting{Enabled: state.Enabled, Config: state.Config} + } + return result +} + +func cleanPluginSettings(overrides map[string]PluginSetting, uploaded []UploadedPlugin) map[string]PluginSetting { + allowed := map[string]map[string]bool{} + for _, def := range pluginCatalog { + fields := map[string]bool{} + for _, field := range def.Fields { + fields[field.Key] = true + } + allowed[def.Key] = fields + } + for _, plugin := range uploaded { + if _, ok := allowed[plugin.Key]; !ok { + allowed[plugin.Key] = map[string]bool{} + } + } + clean := map[string]PluginSetting{} + for key, override := range overrides { + fields, known := allowed[key] + if !known { + continue + } + config := map[string]string{} + for name, value := range override.Config { + if fields[name] && strings.TrimSpace(value) != "" { + config[name] = strings.TrimSpace(value) + } + } + clean[key] = PluginSetting{Enabled: override.Enabled, Config: config} + } + return clean +} + +func (p *PostgresRepository) SavePluginSettings(overrides map[string]PluginSetting) []PluginState { + settings, _ := p.updateSettings(func(current *Settings) error { + current.Plugins = cleanPluginSettings(overrides, current.UploadedPlugins) + return nil + }) + return mergePlugins(settings.Plugins, settings.UploadedPlugins) +} +func (p *PostgresRepository) UploadedPlugins() []UploadedPlugin { + return append([]UploadedPlugin(nil), p.Settings().UploadedPlugins...) +} +func (p *PostgresRepository) EnabledUploadedPlugins() []UploadedPlugin { + settings := p.Settings() + result := []UploadedPlugin{} + for _, plugin := range settings.UploadedPlugins { + if override, ok := settings.Plugins[plugin.Key]; ok && override.Enabled { + result = append(result, plugin) + } + } + return result +} + +func validateUploadedPlugin(plugin UploadedPlugin) (UploadedPlugin, error) { + plugin.Key = strings.TrimSpace(plugin.Key) + plugin.Name = strings.TrimSpace(plugin.Name) + plugin.Tag = strings.TrimSpace(plugin.Tag) + plugin.Lang = strings.TrimSpace(plugin.Lang) + plugin.Category = strings.TrimSpace(plugin.Category) + if plugin.Category == "" { + plugin.Category = "custom" + } + plugin.Format = "jsx" + if !keyRe.MatchString(plugin.Key) { + return UploadedPlugin{}, fmt.Errorf("key 需为小写字母/数字/连字符（如 my-plugin）") + } + for _, definition := range pluginCatalog { + if definition.Key == plugin.Key { + return UploadedPlugin{}, fmt.Errorf("key 与内置插件冲突：%s", plugin.Key) + } + } + if plugin.Name == "" { + return UploadedPlugin{}, fmt.Errorf("name 不能为空") + } + if strings.TrimSpace(plugin.Code) == "" { + return UploadedPlugin{}, fmt.Errorf("code 不能为空") + } + switch plugin.Kind { + case "component": + if !tagRe.MatchString(plugin.Tag) { + return UploadedPlugin{}, fmt.Errorf("component 插件需要大写开头的 tag（如 Figma）") + } + plugin.Lang = "" + case "fence": + if !lngRe.MatchString(plugin.Lang) { + return UploadedPlugin{}, fmt.Errorf("fence 插件需要小写的 lang（如 figma）") + } + plugin.Tag = "" + default: + return UploadedPlugin{}, fmt.Errorf("kind 必须是 component 或 fence") + } + return plugin, nil +} +func (p *PostgresRepository) SaveUploadedPlugin(plugin UploadedPlugin) (UploadedPlugin, error) { + validated, err := validateUploadedPlugin(plugin) + if err != nil { + return UploadedPlugin{}, err + } + settings, err := p.updateSettings(func(current *Settings) error { + validated.UpdatedAt = time.Now().UTC() + for index, existing := range current.UploadedPlugins { + if existing.Key == validated.Key { + validated.Version = existing.Version + 1 + current.UploadedPlugins[index] = validated + return nil + } + } + validated.Version = 1 + current.UploadedPlugins = append(current.UploadedPlugins, validated) + return nil + }) + if err != nil { + return UploadedPlugin{}, err + } + for _, saved := range settings.UploadedPlugins { + if saved.Key == validated.Key { + return saved, nil + } + } + return UploadedPlugin{}, ErrNotFound +} +func (p *PostgresRepository) DeleteUploadedPlugin(key string) bool { + deleted := false + _, err := p.updateSettings(func(current *Settings) error { + for index, plugin := range current.UploadedPlugins { + if plugin.Key == key { + current.UploadedPlugins = append(current.UploadedPlugins[:index], current.UploadedPlugins[index+1:]...) + delete(current.Plugins, key) + deleted = true + break + } + } + return nil + }) + return err == nil && deleted +} + +func cleanSnippetData(snippets []Snippet, variables map[string]string) ([]Snippet, map[string]string) { + cleanSnippets := make([]Snippet, 0, len(snippets)) + seen := map[string]int{} + for _, snippet := range snippets { + key := strings.TrimSpace(snippet.Key) + if key == "" { + continue + } + entry := Snippet{Key: key, Name: strings.TrimSpace(snippet.Name), Content: snippet.Content} + if index, ok := seen[key]; ok { + cleanSnippets[index] = entry + } else { + seen[key] = len(cleanSnippets) + cleanSnippets = append(cleanSnippets, entry) + } + } + cleanVariables := map[string]string{} + for key, value := range variables { + key = strings.TrimSpace(key) + if key != "" { + cleanVariables[key] = strings.TrimSpace(value) + } + } + return cleanSnippets, cleanVariables +} +func (p *PostgresRepository) SnippetData() ([]Snippet, map[string]string) { + settings := p.Settings() + snippets := append([]Snippet(nil), settings.Snippets...) + variables := map[string]string{} + for key, value := range settings.Variables { + variables[key] = value + } + return snippets, variables +} +func (p *PostgresRepository) SaveSnippetData(snippets []Snippet, variables map[string]string) ([]Snippet, map[string]string) { + snippets, variables = cleanSnippetData(snippets, variables) + _, _ = p.updateSettings(func(current *Settings) error { current.Snippets = snippets; current.Variables = variables; return nil }) + return snippets, variables +} diff --git a/backend/internal/store/schema.sql b/backend/internal/store/schema.sql new file mode 100644 index 0000000..cbd485f --- /dev/null +++ b/backend/internal/store/schema.sql @@ -0,0 +1,443 @@ +CREATE EXTENSION IF NOT EXISTS vector; + +CREATE TABLE IF NOT EXISTS auth_session ( + key TEXT PRIMARY KEY, + value_json JSONB NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS users ( + id TEXT PRIMARY KEY, + username TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, + email TEXT, + department TEXT, + avatar TEXT, + roles_json JSONB NOT NULL DEFAULT '[]'::jsonb, + managed_categories_json JSONB NOT NULL DEFAULT '[]'::jsonb, + source TEXT, + status TEXT, + is_super_admin BOOLEAN NOT NULL DEFAULT false, + mcp_token TEXT, + last_login_at TIMESTAMPTZ, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +-- Teams: document maintenance teams with leader + members. +-- A team can be assigned as responsible_team on docs_category (领域 owner). +-- Members/leaders get implicit management rights on assigned domains (see canManageViaResponsibleTeam). +CREATE TABLE IF NOT EXISTS teams ( + id TEXT PRIMARY KEY, + key TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + description TEXT, + leaders JSONB NOT NULL DEFAULT '[]'::jsonb, + members JSONB NOT NULL DEFAULT '[]'::jsonb, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS connected_app ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + description TEXT, + client_id TEXT NOT NULL UNIQUE, + client_secret_hash TEXT, + redirect_uris JSONB DEFAULT '[]'::jsonb, + scopes JSONB DEFAULT '[]'::jsonb, + trusted BOOLEAN DEFAULT false, + enabled BOOLEAN DEFAULT true, + created_by TEXT REFERENCES users(id), + last_used_at TIMESTAMPTZ, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +-- Built-in public OAuth client for Codex MCP OAuth login. Codex CLI supports +-- `--oauth-client-id` but does not take a client secret for MCP login, so this +-- client intentionally has an empty secret and only allows local loopback +-- callbacks. +INSERT INTO connected_app(id,name,description,client_id,client_secret_hash,redirect_uris,scopes,trusted,enabled,created_at,updated_at) +VALUES( + 'app-codex-cli', + 'Codex CLI MCP OAuth', + 'Built-in public OAuth client for Codex MCP login.', + 'codex-cli', + '', + '["http://localhost","http://127.0.0.1","http://[::1]"]'::jsonb, + '["modex:mcp:read","modex:docs:read"]'::jsonb, + true, + true, + now(), + now() +) +ON CONFLICT(client_id) DO NOTHING; + +CREATE TABLE IF NOT EXISTS oauth_grant ( + id TEXT PRIMARY KEY, + app_id TEXT REFERENCES connected_app(id), + user_id TEXT REFERENCES users(id), + code_hash TEXT, + access_token_hash TEXT, + refresh_token_hash TEXT, + redirect_uri TEXT, + scopes JSONB DEFAULT '[]'::jsonb, + code_expires_at TIMESTAMPTZ, + access_expires_at TIMESTAMPTZ, + refresh_expires_at TIMESTAMPTZ, + revoked_at TIMESTAMPTZ, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_oauth_grant_code_hash ON oauth_grant(code_hash); +CREATE INDEX IF NOT EXISTS idx_oauth_grant_access_token_hash ON oauth_grant(access_token_hash); +CREATE INDEX IF NOT EXISTS idx_oauth_grant_refresh_token_hash ON oauth_grant(refresh_token_hash); + +CREATE TABLE IF NOT EXISTS docs_category ( + id TEXT PRIMARY KEY, + parent_id TEXT REFERENCES docs_category(id), + key TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + description TEXT, + icon TEXT, + sort_order INT DEFAULT 0, + status TEXT DEFAULT 'active', + responsible_team TEXT, -- team key owning this 领域 (domain); members get mgmt rights + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS docs_module ( + id TEXT PRIMARY KEY, + module_key TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + description TEXT, + owner_group TEXT, + repo_type TEXT, + repo_url TEXT, + default_version TEXT, + visibility TEXT DEFAULT 'internal', + status TEXT DEFAULT 'active', + package_name TEXT, + package_version TEXT, + channel TEXT, + edition TEXT, + keywords JSONB DEFAULT '[]'::jsonb, + maintainers JSONB DEFAULT '[]'::jsonb, + category_path TEXT, + source_type TEXT, + doc_type TEXT, + mount TEXT, + gitlab_branch TEXT, + gitlab_path TEXT, + deploy_token TEXT, + last_synced_commit TEXT, + last_synced_at TIMESTAMPTZ, + created_by TEXT REFERENCES users(id), + reads_7d INT NOT NULL DEFAULT 0, + reads_30d INT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +CREATE UNIQUE INDEX IF NOT EXISTS docs_module_deploy_token_uidx + ON docs_module(deploy_token) + WHERE deploy_token IS NOT NULL AND deploy_token <> ''; + +CREATE TABLE IF NOT EXISTS docs_module_category ( + module_id TEXT REFERENCES docs_module(id), + category_id TEXT REFERENCES docs_category(id), + is_primary BOOLEAN DEFAULT false, + PRIMARY KEY (module_id, category_id) +); + +CREATE TABLE IF NOT EXISTS docs_version ( + id TEXT PRIMARY KEY, + module_id TEXT REFERENCES docs_module(id), + docs_version TEXT NOT NULL, + display_name TEXT, + version_type TEXT, + is_default BOOLEAN DEFAULT false, + status TEXT DEFAULT 'active', + source_branch TEXT, + package_version TEXT, + channel TEXT, + edition TEXT, + support_status TEXT, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + UNIQUE(module_id, docs_version) +); + +CREATE TABLE IF NOT EXISTS docs_entry ( + id TEXT PRIMARY KEY, + module_id TEXT REFERENCES docs_module(id), + version_id TEXT REFERENCES docs_version(id), + entry_key TEXT NOT NULL, + title TEXT NOT NULL, + entry_type TEXT NOT NULL, + builder TEXT, + source TEXT, + storage_uri TEXT, + nav_uri TEXT, + index_status TEXT, + is_primary BOOLEAN DEFAULT false, + sort_order INT DEFAULT 0, + status TEXT DEFAULT 'active', + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS docs_release ( + id TEXT PRIMARY KEY, + module_id TEXT REFERENCES docs_module(id), + version_id TEXT REFERENCES docs_version(id), + release_id TEXT NOT NULL UNIQUE, + commit_sha TEXT, + branch TEXT, + publisher TEXT, + pipeline_url TEXT, + build_system TEXT, + build_id TEXT, + trigger_type TEXT, + source_ip TEXT, + artifact_version TEXT, + package_version TEXT, + storage_uri TEXT, + status TEXT, + published_at TIMESTAMPTZ, + created_at TIMESTAMPTZ DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS docs_page ( + id TEXT PRIMARY KEY, + module_id TEXT REFERENCES docs_module(id), + version_id TEXT REFERENCES docs_version(id), + entry_id TEXT REFERENCES docs_entry(id), + release_id TEXT REFERENCES docs_release(id), + doc_id TEXT NOT NULL UNIQUE, + title TEXT NOT NULL, + description TEXT, + path TEXT, + source_file TEXT, + doc_type TEXT, + status TEXT DEFAULT 'active', + owner_group TEXT, + tags JSONB DEFAULT '[]'::jsonb, + category_ids JSONB DEFAULT '[]'::jsonb, + content_text TEXT, + content_html TEXT, + content_md TEXT, + updated_at TIMESTAMPTZ, + last_verified_at TIMESTAMPTZ, + created_at TIMESTAMPTZ DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS docs_page_view ( + id TEXT PRIMARY KEY, + page_id TEXT REFERENCES docs_page(id), + module_id TEXT REFERENCES docs_module(id), + version_id TEXT REFERENCES docs_version(id), + doc_id TEXT, + module_key TEXT, + module_name TEXT, + docs_version TEXT, + entry_key TEXT, + title TEXT, + path TEXT, + user_id TEXT REFERENCES users(id), + session_id TEXT, + read_id TEXT, + duration_seconds INT, + scroll_depth NUMERIC, + viewed_at TIMESTAMPTZ DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS user_favorite ( + id TEXT PRIMARY KEY, + user_id TEXT REFERENCES users(id), + module_key TEXT NOT NULL, + created_at TIMESTAMPTZ DEFAULT now(), + UNIQUE(user_id, module_key) +); + +CREATE TABLE IF NOT EXISTS user_recent_doc ( + id TEXT PRIMARY KEY, + user_id TEXT REFERENCES users(id), + doc_id TEXT NOT NULL, + title TEXT, + module_key TEXT, + module_name TEXT, + docs_version TEXT, + entry_key TEXT, + href TEXT, + viewed_at TIMESTAMPTZ DEFAULT now(), + UNIQUE(user_id, doc_id) +); + +CREATE TABLE IF NOT EXISTS docs_search_log ( + id TEXT PRIMARY KEY, + user_id TEXT, + ip_address TEXT, + query TEXT, + mode TEXT, + filters_json JSONB, + result_count INT, + clicked_doc_id TEXT, + searched_at TIMESTAMPTZ DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS docs_embedding ( + id BIGSERIAL PRIMARY KEY, + page_id TEXT REFERENCES docs_page(id), + doc_id TEXT, + chunk_id TEXT, + module_id TEXT REFERENCES docs_module(id), + version_id TEXT REFERENCES docs_version(id), + entry_id TEXT REFERENCES docs_entry(id), + content TEXT, + embedding vector(1024), + embedding_json JSONB, + metadata_json JSONB, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +DROP INDEX IF EXISTS idx_docs_embedding_doc_id; +CREATE UNIQUE INDEX IF NOT EXISTS idx_docs_embedding_chunk_id ON docs_embedding(chunk_id); + +CREATE TABLE IF NOT EXISTS docs_mcp_log ( + id TEXT PRIMARY KEY, + tool_name TEXT, + user_id TEXT, + query TEXT, + input_json JSONB, + result_count INT, + created_at TIMESTAMPTZ DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS docs_feedback ( + id TEXT PRIMARY KEY, + doc_id TEXT NOT NULL, + page_id TEXT, + module_key TEXT, + title TEXT, + rating TEXT, + comment TEXT, + user_id TEXT, + session_id TEXT, + created_at TIMESTAMPTZ DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS docs_nav ( + module_key TEXT NOT NULL, + docs_version TEXT NOT NULL, + items_json JSONB NOT NULL DEFAULT '[]'::jsonb, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (module_key, docs_version) +); + +CREATE TABLE IF NOT EXISTS docs_site_file ( + module_key TEXT NOT NULL, + docs_version TEXT NOT NULL, + entry_key TEXT NOT NULL, + name TEXT NOT NULL, + content BYTEA NOT NULL, + content_type TEXT, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (module_key, docs_version, entry_key, name) +); + +CREATE TABLE IF NOT EXISTS platform_settings ( + key TEXT PRIMARY KEY, + value_json JSONB NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +DROP TABLE IF EXISTS store_metadata; +DROP TABLE IF EXISTS modex_store_snapshot; +-- Upgrade databases created by earlier releases. CREATE TABLE IF NOT EXISTS +-- does not add or change columns on an existing installation. +ALTER TABLE users ADD COLUMN IF NOT EXISTS avatar TEXT; +ALTER TABLE users ADD COLUMN IF NOT EXISTS roles_json JSONB NOT NULL DEFAULT '[]'::jsonb; +ALTER TABLE users ADD COLUMN IF NOT EXISTS managed_categories_json JSONB NOT NULL DEFAULT '[]'::jsonb; +ALTER TABLE users ADD COLUMN IF NOT EXISTS source TEXT; +ALTER TABLE users ADD COLUMN IF NOT EXISTS status TEXT; +ALTER TABLE users ADD COLUMN IF NOT EXISTS is_super_admin BOOLEAN NOT NULL DEFAULT false; +ALTER TABLE users ADD COLUMN IF NOT EXISTS mcp_token TEXT; +ALTER TABLE users ADD COLUMN IF NOT EXISTS last_login_at TIMESTAMPTZ; + +-- Older deployments briefly allowed non-array JSON values in these fields. +-- Keep them normalized so login upserts and user serialization can safely treat +-- them as string arrays. +UPDATE users SET roles_json='[]'::jsonb WHERE jsonb_typeof(roles_json) <> 'array'; +UPDATE users SET managed_categories_json='[]'::jsonb WHERE jsonb_typeof(managed_categories_json) <> 'array'; + +-- Drop the legacy Keycloak group mirror. Team membership (teams + responsible_team) +-- is the only authorization model; SSO groups were never consulted for access. +DROP TABLE IF EXISTS user_groups; +DROP TABLE IF EXISTS groups; +ALTER TABLE users DROP COLUMN IF EXISTS groups_json; + +ALTER TABLE teams ADD COLUMN IF NOT EXISTS leaders JSONB NOT NULL DEFAULT '[]'::jsonb; +ALTER TABLE teams ADD COLUMN IF NOT EXISTS members JSONB NOT NULL DEFAULT '[]'::jsonb; + +ALTER TABLE docs_module ADD COLUMN IF NOT EXISTS default_version TEXT; +ALTER TABLE docs_module ADD COLUMN IF NOT EXISTS category_path TEXT; +ALTER TABLE docs_module ADD COLUMN IF NOT EXISTS source_type TEXT; +ALTER TABLE docs_module ADD COLUMN IF NOT EXISTS doc_type TEXT; +ALTER TABLE docs_module ADD COLUMN IF NOT EXISTS mount TEXT; +ALTER TABLE docs_module ADD COLUMN IF NOT EXISTS gitlab_branch TEXT; +ALTER TABLE docs_module ADD COLUMN IF NOT EXISTS gitlab_path TEXT; +ALTER TABLE docs_module ADD COLUMN IF NOT EXISTS deploy_token TEXT; +ALTER TABLE docs_module ADD COLUMN IF NOT EXISTS last_synced_commit TEXT; +ALTER TABLE docs_module ADD COLUMN IF NOT EXISTS last_synced_at TIMESTAMPTZ; +ALTER TABLE docs_module ADD COLUMN IF NOT EXISTS created_by TEXT REFERENCES users(id); +ALTER TABLE docs_module ADD COLUMN IF NOT EXISTS reads_7d INT NOT NULL DEFAULT 0; +ALTER TABLE docs_module ADD COLUMN IF NOT EXISTS reads_30d INT NOT NULL DEFAULT 0; + +CREATE UNIQUE INDEX IF NOT EXISTS docs_module_deploy_token_uidx + ON docs_module(deploy_token) + WHERE deploy_token IS NOT NULL AND deploy_token <> ''; + +ALTER TABLE docs_page ADD COLUMN IF NOT EXISTS category_ids JSONB NOT NULL DEFAULT '[]'::jsonb; +ALTER TABLE docs_page ADD COLUMN IF NOT EXISTS content_html TEXT; +ALTER TABLE docs_page ADD COLUMN IF NOT EXISTS content_md TEXT; + +ALTER TABLE docs_page_view ALTER COLUMN id DROP DEFAULT; +ALTER TABLE docs_page_view ALTER COLUMN id TYPE TEXT USING id::text; +ALTER TABLE docs_page_view ADD COLUMN IF NOT EXISTS doc_id TEXT; +ALTER TABLE docs_page_view ADD COLUMN IF NOT EXISTS module_key TEXT; +ALTER TABLE docs_page_view ADD COLUMN IF NOT EXISTS module_name TEXT; +ALTER TABLE docs_page_view ADD COLUMN IF NOT EXISTS docs_version TEXT; +ALTER TABLE docs_page_view ADD COLUMN IF NOT EXISTS entry_key TEXT; +ALTER TABLE docs_page_view ADD COLUMN IF NOT EXISTS title TEXT; +ALTER TABLE docs_page_view ADD COLUMN IF NOT EXISTS path TEXT; +ALTER TABLE docs_page_view ADD COLUMN IF NOT EXISTS read_id TEXT; +ALTER TABLE docs_search_log ALTER COLUMN id DROP DEFAULT; +ALTER TABLE docs_search_log ALTER COLUMN id TYPE TEXT USING id::text; +ALTER TABLE docs_search_log ADD COLUMN IF NOT EXISTS ip_address TEXT; +ALTER TABLE docs_mcp_log ALTER COLUMN id DROP DEFAULT; +ALTER TABLE docs_mcp_log ALTER COLUMN id TYPE TEXT USING id::text; + +-- Remove JSON mirrors from development builds. Rows are reconstructed only +-- from typed columns and foreign-key relationships. +ALTER TABLE users DROP COLUMN IF EXISTS record_json; +ALTER TABLE teams DROP COLUMN IF EXISTS record_json; +ALTER TABLE connected_app DROP COLUMN IF EXISTS record_json; +ALTER TABLE oauth_grant DROP COLUMN IF EXISTS record_json; +ALTER TABLE docs_category DROP COLUMN IF EXISTS record_json; +ALTER TABLE docs_module DROP COLUMN IF EXISTS record_json; +ALTER TABLE docs_version DROP COLUMN IF EXISTS record_json; +ALTER TABLE docs_entry DROP COLUMN IF EXISTS record_json; +ALTER TABLE docs_release DROP COLUMN IF EXISTS record_json; +ALTER TABLE docs_release ADD COLUMN IF NOT EXISTS trigger_type TEXT; +ALTER TABLE docs_release ADD COLUMN IF NOT EXISTS source_ip TEXT; +ALTER TABLE docs_page DROP COLUMN IF EXISTS record_json; +ALTER TABLE docs_page_view DROP COLUMN IF EXISTS record_json; +ALTER TABLE docs_search_log DROP COLUMN IF EXISTS record_json; +ALTER TABLE docs_mcp_log DROP COLUMN IF EXISTS record_json; +ALTER TABLE docs_feedback DROP COLUMN IF EXISTS record_json; diff --git a/backend/internal/store/seed.go b/backend/internal/store/seed.go new file mode 100644 index 0000000..947af3f --- /dev/null +++ b/backend/internal/store/seed.go @@ -0,0 +1,13 @@ +package store + +import _ "embed" + +// Seed markdown keeps the optional local development dataset self-contained. +var ( + //go:embed seeddata/demo-guide.md + seedDemoGuideMD string + //go:embed seeddata/demo-maintenance.md + seedDemoMaintenanceMD string + //go:embed seeddata/cbb-build-cache.md + seedCBBBuildCacheMD string +) diff --git a/backend/internal/store/seeddata/cbb-build-cache.md b/backend/internal/store/seeddata/cbb-build-cache.md new file mode 100644 index 0000000..4ace66f --- /dev/null +++ b/backend/internal/store/seeddata/cbb-build-cache.md @@ -0,0 +1,37 @@ +--- +title: 构建缓存清理 +description: CBB 构建缓存清理和常见构建问题排查。 +--- + +当依赖缓存、编译缓存或 CI 工作区残留导致构建异常时，可按本页步骤清理。 + +${mdx.replace(/[<&>]/g, (c) => ({ "<": "<", ">": ">", "&": "&" })[c])}# VuePress 文档站接入 - -## 概览 - -这个示例模拟一个 VuePress 文档站。真实项目可以把 `build` 改成 `pnpm docs:build`。 - -## 发布流程 - -1. 在仓库维护 `docs.yaml`。 -2. 执行 VuePress 构建。 -3. docsctl 复制 `docs/.vuepress/dist` 到标准文档包。 -
${markdown.replace(/[<&>]/g, (c) => ({ "<": "<", ">": ">", "&": "&" })[c])}
-
-
-
-
- 总 PV
- {data.total_pv}
-
-
- 近 7 天阅读
- {data.reads_7d}
-
-
- 页面数
- {data.popular_pages.length}
-
-
- 事件类型
- {data.events.length}
-
-
-
-
-
| 文档 | -模块 | -版本 | -PV | -UV | -近7天 | -近30天 | -平均时长(s) | -
|---|---|---|---|---|---|---|---|
|
- {p.title}
- {p.doc_id}
- |
- {p.module_name} | -{p.docs_version} | -{p.pv} | -{p.uv} | -{p.reads_7d} | -{p.reads_30d} | -{p.avg_duration_seconds} | -
埋点事件：{data.events.join(" / ")}
-
-
-
-
+
+
+
{hasChildren && open ? (
@@ -98,6 +98,7 @@ function TreeNode({
}
export default function AdminCategoriesPage() {
+ const { t } = useI18n();
const [categories, setCategories] = useState{error}
: null}
-
{flatten(categories).length} 个领域节点 · 拖动卡片排序或改层级
+ {flatten(categories).length} {t("admin.categories.category_nodes_drag_cards_to_reorder_or_change")}
-
+ {isSuper ? : null}
-
+
-
- 暂无领域
- 点击「新增顶级领域」开始创建层级结构。支持任意嵌套，可绑定负责团队。
-
{categories.map((cat) => (
@@ -266,35 +273,35 @@ export default function AdminCategoriesPage() {
setModalOpen(false)}
- title={data.id ? "编辑领域" : data.parent_id ? "新增子领域" : "新增顶级领域"}
- subtitle="顶层领域需超管权限，子领域可由父领域管理员或负责团队创建"
+ title={data.id ? t("admin.categories.edit_category") : data.parent_id ? t("admin.categories.add_subcategory") : t("admin.categories.add_top_level_category")}
+ subtitle={t("admin.categories.top_level_categories_require_admin_privileges_subcategories_can")}
footer={
<>
-
-
+
+
}
>
diff --git a/frontend/app/admin/connected-apps/page.tsx b/frontend/app/admin/connected-apps/page.tsx
new file mode 100644
index 0000000..9ee7272
--- /dev/null
+++ b/frontend/app/admin/connected-apps/page.tsx
@@ -0,0 +1,331 @@
+"use client";
+
+import { useEffect, useState } from "react";
+import { Check, Copy, KeyRound, Link2, Loader2, Pencil, Plus, Trash2 } from "lucide-react";
+import { AdminShell } from "@/components/admin-shell";
+import { Modal } from "@/components/ui/modal";
+import { Switch } from "@/components/ui/switch";
+import { EmptyState } from "@/components/ui/empty-state";
+import { Combobox, type ComboOption } from "@/components/ui/combobox";
+import {
+ createConnectedApp,
+ deleteConnectedApp,
+ getConnectedApps,
+ updateConnectedApp,
+ type ConnectedApp,
+ type ConnectedAppDraft,
+} from "@/lib/api";
+import { useI18n } from "@/lib/i18n";
+
+const DEFAULT_SCOPES = ["modex:mcp:read", "modex:docs:read"];
+
+type Draft = ConnectedAppDraft & {
+ id?: string;
+ redirect_text: string;
+};
+
+const emptyDraft: Draft = {
+ name: "",
+ description: "",
+ client_id: "",
+ redirect_uris: [],
+ scopes: [...DEFAULT_SCOPES],
+ trusted: false,
+ enabled: true,
+ redirect_text: "",
+};
+
+function lines(value: string) {
+ return value
+ .split(/\r?\n|,/)
+ .map((v) => v.trim())
+ .filter(Boolean);
+}
+
+function toDraft(app?: ConnectedApp): Draft {
+ if (!app) return emptyDraft;
+ return {
+ id: app.id,
+ name: app.name,
+ description: app.description || "",
+ client_id: app.client_id,
+ redirect_uris: app.redirect_uris || [],
+ scopes: app.scopes || [],
+ trusted: !!app.trusted,
+ enabled: !!app.enabled,
+ redirect_text: (app.redirect_uris || []).join("\n"),
+ };
+}
+
+function payload(draft: Draft): ConnectedAppDraft {
+ return {
+ name: draft.name.trim(),
+ description: draft.description?.trim(),
+ redirect_uris: lines(draft.redirect_text),
+ scopes: draft.scopes,
+ trusted: draft.trusted,
+ enabled: draft.enabled,
+ };
+}
+
+async function copy(text: string, done: (v: boolean) => void) {
+ try {
+ await navigator.clipboard.writeText(text);
+ done(true);
+ setTimeout(() => done(false), 1500);
+ } catch {
+ done(false);
+ }
+}
+
+export default function AdminConnectedAppsPage() {
+ const { t } = useI18n();
+ const SCOPE_OPTIONS: ComboOption[] = [
+ { value: "modex:mcp:read", label: "modex:mcp:read", hint: t("admin.connectedApps.allow_mcp_tools_to_read_documents") },
+ { value: "modex:docs:read", label: "modex:docs:read", hint: t("admin.connectedApps.allow_document_reading_api") },
+ ];
+ const [apps, setApps] = useState([]);
+ const [loading, setLoading] = useState(true);
+ const [saving, setSaving] = useState(false);
+ const [error, setError] = useState("");
+ const [modalOpen, setModalOpen] = useState(false);
+ const [draft, setDraft] = useState(emptyDraft);
+ const [createdSecret, setCreatedSecret] = useState("");
+ const [copied, setCopied] = useState("");
+
+ async function load() {
+ setLoading(true);
+ setError("");
+ try {
+ const res = await getConnectedApps();
+ setApps(res.apps || []);
+ } catch (e) {
+ setError(String(e));
+ } finally {
+ setLoading(false);
+ }
+ }
+
+ useEffect(() => {
+ load();
+ }, []);
+
+ function openCreate() {
+ setDraft(emptyDraft);
+ setCreatedSecret("");
+ setError("");
+ setModalOpen(true);
+ }
+
+ function openEdit(app: ConnectedApp) {
+ setDraft(toDraft(app));
+ setCreatedSecret("");
+ setError("");
+ setModalOpen(true);
+ }
+
+ async function submit() {
+ const body = payload(draft);
+ if (!body.name || !body.redirect_uris.length) {
+ setError(t("admin.connectedApps.name_and_redirect_uri_are_required"));
+ return;
+ }
+ setSaving(true);
+ setError("");
+ try {
+ if (draft.id) {
+ await updateConnectedApp(draft.id, body);
+ setModalOpen(false);
+ } else {
+ const created = await createConnectedApp(body);
+ setCreatedSecret(created.client_secret || "");
+ setDraft(toDraft(created));
+ }
+ await load();
+ } catch (e) {
+ setError(String(e));
+ } finally {
+ setSaving(false);
+ }
+ }
+
+ async function remove(app: ConnectedApp) {
+ if (!confirm(t("admin.connectedApps.delete_app_value1_existing_authorizations_will_be_revoked", { value1: app.name }))) return;
+ setError("");
+ try {
+ await deleteConnectedApp(app.id);
+ await load();
+ } catch (e) {
+ setError(String(e));
+ }
+ }
+
+ return (
+
+ {error ? setModalOpen(false)}
+ title={draft.id ? t("admin.connectedApps.edit_app_value1", { value1: draft.name }) : t("admin.connectedApps.create_application_link")}
+ subtitle={t("admin.connectedApps.the_client_secret_is_displayed_only_once_upon")}
+ width={720}
+ footer={
+ <>
+
+
+
+ }
+ >
+ {createdSecret ? (
+
-
- setData({ ...data, name: e.target.value })} />
- {data.id ? 标识 {data.key}（系统生成，不可修改） : 标识由系统自动生成。}
+
+ setData({ ...data, name: e.target.value })} />
+ {data.id ? {t("admin.categories.id")} {data.key}{t("admin.categories.system_generated_immutable")} : {t("admin.categories.id_is_auto_generated_by_the_system")}}
-
- setData({ ...data, description: e.target.value })} />
+
+ setData({ ...data, description: e.target.value })} />
-
- setData({ ...data, parent_id: v[0] || "" })} multiple={false} placeholder="选择父领域…" />
+
+ setData({ ...data, parent_id: v[0] || "" })} multiple={false} placeholder={t("admin.categories.select_parent_category")} />
-
+
setData({ ...data, icon })} />
-
- setData({ ...data, responsible_team: v[0] || "" })} multiple={false} placeholder="选择负责团队…" />
+
+ setData({ ...data, responsible_team: v[0] || "" })} multiple={false} placeholder={t("admin.categories.select_owning_team")} />
{error}
: null}
+
+
+
+
+
+
+ OAuth Applications
+
+ {t("admin.connectedApps.recommended_scope")}modex:mcp:read、modex:docs:read
+
+
+
+
+
+
+
+
+
+
+
+ | {t("admin.connectedApps.apply")} | +Client ID | +Scopes | +{t("admin.releases.status")} | +{t("admin.connectedApps.recently_used")} | +{t("admin.modules.actions")} | +
|---|---|---|---|---|---|
|
+ {app.name}
+ {app.description ? {app.description} : null}
+ |
+ {app.client_id} |
+
+
+ {(app.scopes || []).map((scope) => {scope})}
+
+ |
+ + + {app.enabled ? t("admin.connectedApps.enable") : t("admin.connectedApps.deactivate")} + + {app.trusted ? Trusted : null} + | ++ {app.last_used_at ? new Date(app.last_used_at).toLocaleString() : "—"} + | +
+
+
+
+
+ |
+
|
+ |
+ |||||
+
+
+ ) : null}
+
+
+
+
+
+ {t("admin.connectedApps.save_this_securely_in_your_external_application_immediately")}
+
+
+ setDraft({ ...draft, name: e.target.value })} />
+ {t("admin.connectedApps.client_id_is_auto_generated_and_displayed_in")}
+
+
+ {draft.id ? (
+
+
+
+ {t("admin.connectedApps.client_id_cannot_be_modified_after_app_creation")}
+
+ ) : null}
+
+
+
+ setDraft({ ...draft, description: e.target.value })} />
+
+
+
+
+