From f12824cda982656e98c8ead52a5aac4f87cd40de Mon Sep 17 00:00:00 2001 From: Algis Dumbris Date: Sat, 14 Mar 2026 06:46:16 +0200 Subject: [PATCH] feat: add Gemini embedding provider, agent registration UI, and UI polish - Add Gemini embedding provider alongside OpenAI and Ollama - Improve agent registration form with API key display and MCP config - Polish dashboard, login page, and overall UI styling - Update CLAUDE.md with embedding environment variables - Add console command infrastructure Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 3 +- README.md | 3 +- cmd/synapbus/main.go | 32 +++++ internal/search/config.go | 13 ++- internal/search/embedding/factory.go | 29 ++++- internal/search/embedding/gemini.go | 167 +++++++++++++++++++++++++++ internal/search/embedding/openai.go | 2 +- web/src/routes/+page.svelte | 19 ++- web/src/routes/agents/+page.svelte | 140 +++++++++++++++++----- web/src/routes/login/+page.svelte | 20 +++- 10 files changed, 388 insertions(+), 40 deletions(-) create mode 100644 internal/search/embedding/gemini.go diff --git a/CLAUDE.md b/CLAUDE.md index 5fb1f3f..a3227ae 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,7 +65,8 @@ make lint # Run linters | `SYNAPBUS_PORT` | HTTP server port | `8080` | | `SYNAPBUS_DATA_DIR` | Data directory (SQLite DB, attachments, vector index) | `./data` | | `SYNAPBUS_EMBEDDING_PROVIDER` | Embedding provider: `openai`, `gemini`, `ollama` | (none) | -| `SYNAPBUS_EMBEDDING_API_KEY` | API key for embedding provider | (none) | +| `OPENAI_API_KEY` | OpenAI API key for embeddings | (none) | +| `GEMINI_API_KEY` | Google Gemini API key for embeddings | (none) | | `SYNAPBUS_OLLAMA_URL` | Ollama server URL | `http://localhost:11434` | ## Conventions diff --git a/README.md b/README.md index 31221d1..7270d7f 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,8 @@ Agents interact with SynapBus entirely through MCP tools: | `SYNAPBUS_PORT` | HTTP server port | `8080` | | `SYNAPBUS_DATA_DIR` | Data directory | `./data` | | `SYNAPBUS_EMBEDDING_PROVIDER` | `openai` / `gemini` / `ollama` | (none) | -| `SYNAPBUS_EMBEDDING_API_KEY` | Embedding API key | (none) | +| `OPENAI_API_KEY` | OpenAI API key for embeddings | (none) | +| `GEMINI_API_KEY` | Google Gemini API key for embeddings | (none) | | `SYNAPBUS_OLLAMA_URL` | Ollama server URL | `http://localhost:11434` | ## Tech Stack diff --git a/cmd/synapbus/main.go b/cmd/synapbus/main.go index 36c985c..0cd60b2 100644 --- a/cmd/synapbus/main.go +++ b/cmd/synapbus/main.go @@ -212,6 +212,22 @@ func runServe(cmd *cobra.Command, args []string) error { channelStore := channels.NewSQLiteChannelStore(db.DB) channelService := channels.NewService(channelStore, msgService, tracer) + // Ensure #general channel exists + _, err = channelService.CreateChannel(ctx, channels.CreateChannelRequest{ + Name: "general", + Description: "General discussion", + Type: "standard", + CreatedBy: "system", + }) + if err != nil { + // Ignore "already exists" errors + if !strings.Contains(err.Error(), "UNIQUE") && !strings.Contains(err.Error(), "already exists") { + slog.Warn("failed to create #general channel", "error", err) + } + } else { + slog.Info("created default #general channel") + } + // Create swarm service (task auction + stigmergy) taskStore := channels.NewSQLiteTaskStore(db.DB) swarmService := channels.NewSwarmService(taskStore, channelStore, tracer) @@ -272,6 +288,22 @@ func runServe(cmd *cobra.Command, args []string) error { var embPipeline *search.Pipeline var vectorIndex *search.VectorIndex + // Print available embedding providers + providers := embedding.AvailableProviders(searchCfg.APIKey, searchCfg.OllamaURL) + slog.Info("available embedding providers:") + for _, p := range providers { + status := "not configured" + if p.Configured { + status = "ready" + } + slog.Info(" embedding provider", + "name", p.Name, + "model", p.Model, + "dimensions", p.Dimensions, + "status", status, + ) + } + if searchCfg.IsEnabled() { slog.Info("initializing semantic search", "provider", searchCfg.Provider, diff --git a/internal/search/config.go b/internal/search/config.go index 8d2e251..97fa30b 100644 --- a/internal/search/config.go +++ b/internal/search/config.go @@ -9,7 +9,7 @@ import ( // Config holds configuration for the search subsystem. type Config struct { - // Provider specifies the embedding provider: "openai", "ollama", or empty for none. + // Provider specifies the embedding provider: "openai", "gemini", "ollama", or empty for none. Provider string // APIKey is the API key for the embedding provider (required for openai). APIKey string @@ -31,7 +31,7 @@ type Config struct { func LoadConfigFromEnv() Config { cfg := Config{ Provider: os.Getenv("SYNAPBUS_EMBEDDING_PROVIDER"), - APIKey: os.Getenv("SYNAPBUS_EMBEDDING_API_KEY"), + APIKey: getAPIKey(), OllamaURL: os.Getenv("SYNAPBUS_OLLAMA_URL"), BatchSize: 10, WorkerCount: 1, @@ -59,6 +59,15 @@ func LoadConfigFromEnv() Config { return cfg } +// getAPIKey returns the embedding API key, preferring OPENAI_API_KEY +// over the legacy SYNAPBUS_EMBEDDING_API_KEY. +func getAPIKey() string { + if key := os.Getenv("OPENAI_API_KEY"); key != "" { + return key + } + return os.Getenv("SYNAPBUS_EMBEDDING_API_KEY") +} + // IsEnabled returns true if an embedding provider is configured. func (c Config) IsEnabled() bool { return c.Provider != "" diff --git a/internal/search/embedding/factory.go b/internal/search/embedding/factory.go index fe72c63..3fda3c4 100644 --- a/internal/search/embedding/factory.go +++ b/internal/search/embedding/factory.go @@ -1,17 +1,42 @@ package embedding -import "fmt" +import ( + "fmt" + "os" +) + +// ProviderInfo describes an available embedding provider. +type ProviderInfo struct { + Name string + Model string + Dimensions int + Configured bool +} + +// AvailableProviders returns info about all supported providers and whether +// they have the required credentials configured. +func AvailableProviders(apiKey, ollamaURL string) []ProviderInfo { + geminiKey := os.Getenv("GEMINI_API_KEY") + return []ProviderInfo{ + {Name: "openai", Model: openAIModel, Dimensions: openAIDimensions, Configured: apiKey != ""}, + {Name: "gemini", Model: geminiModel, Dimensions: geminiDimensions, Configured: geminiKey != ""}, + {Name: "ollama", Model: ollamaDefaultModel, Dimensions: ollamaDimensions, Configured: true}, + } +} // NewProvider creates an EmbeddingProvider based on the provider name. func NewProvider(provider, apiKey, ollamaURL string) (EmbeddingProvider, error) { switch provider { case "openai": return NewOpenAIProvider(apiKey) + case "gemini": + geminiKey := os.Getenv("GEMINI_API_KEY") + return NewGeminiProvider(geminiKey) case "ollama": return NewOllamaProvider(ollamaURL) case "": return nil, fmt.Errorf("no embedding provider specified") default: - return nil, fmt.Errorf("unknown embedding provider: %q (supported: openai, ollama)", provider) + return nil, fmt.Errorf("unknown embedding provider: %q (supported: openai, gemini, ollama)", provider) } } diff --git a/internal/search/embedding/gemini.go b/internal/search/embedding/gemini.go new file mode 100644 index 0000000..f0195a8 --- /dev/null +++ b/internal/search/embedding/gemini.go @@ -0,0 +1,167 @@ +package embedding + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" +) + +const ( + geminiModel = "text-embedding-004" + geminiDimensions = 768 + geminiMaxTokens = 2048 +) + +// GeminiProvider implements EmbeddingProvider using Google's text-embedding-004. +type GeminiProvider struct { + apiKey string + client *http.Client +} + +// NewGeminiProvider creates a new Gemini embedding provider. +func NewGeminiProvider(apiKey string) (*GeminiProvider, error) { + if apiKey == "" { + return nil, fmt.Errorf("gemini provider requires GEMINI_API_KEY to be set") + } + return &GeminiProvider{ + apiKey: apiKey, + client: &http.Client{Timeout: 30 * time.Second}, + }, nil +} + +type geminiRequest struct { + Requests []geminiEmbedRequest `json:"requests"` +} + +type geminiEmbedRequest struct { + Model string `json:"model"` + Content geminiContent `json:"content"` + TaskType string `json:"taskType,omitempty"` +} + +type geminiContent struct { + Parts []geminiPart `json:"parts"` +} + +type geminiPart struct { + Text string `json:"text"` +} + +type geminiResponse struct { + Embeddings []geminiEmbeddingResult `json:"embeddings"` + Error *geminiError `json:"error,omitempty"` +} + +type geminiEmbeddingResult struct { + Values []float32 `json:"values"` +} + +type geminiError struct { + Code int `json:"code"` + Message string `json:"message"` + Status string `json:"status"` +} + +func (p *GeminiProvider) Embed(ctx context.Context, text string) ([]float32, error) { + results, err := p.EmbedBatch(ctx, []string{text}) + if err != nil { + return nil, err + } + if len(results) == 0 { + return nil, fmt.Errorf("gemini: empty response") + } + return results[0], nil +} + +func (p *GeminiProvider) EmbedBatch(ctx context.Context, texts []string) ([][]float32, error) { + if len(texts) == 0 { + return nil, nil + } + + // Truncate texts that are too long + truncated := make([]string, len(texts)) + for i, t := range texts { + maxChars := geminiMaxTokens * 4 + if len(t) > maxChars { + truncated[i] = t[:maxChars] + } else { + truncated[i] = t + } + } + + // Build batch request + modelPath := "models/" + geminiModel + requests := make([]geminiEmbedRequest, len(truncated)) + for i, t := range truncated { + requests[i] = geminiEmbedRequest{ + Model: modelPath, + Content: geminiContent{ + Parts: []geminiPart{{Text: t}}, + }, + TaskType: "RETRIEVAL_DOCUMENT", + } + } + + body, err := json.Marshal(geminiRequest{Requests: requests}) + if err != nil { + return nil, fmt.Errorf("gemini: marshal request: %w", err) + } + + url := fmt.Sprintf("https://generativelanguage.googleapis.com/v1beta/%s:batchEmbedContents?key=%s", modelPath, p.apiKey) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("gemini: create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := p.client.Do(req) + if err != nil { + return nil, fmt.Errorf("gemini: request failed: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("gemini: read response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + switch resp.StatusCode { + case http.StatusBadRequest: + return nil, fmt.Errorf("gemini: invalid request (400): %s", string(respBody)) + case http.StatusForbidden: + return nil, fmt.Errorf("gemini: invalid API key (403)") + case http.StatusTooManyRequests: + return nil, fmt.Errorf("gemini: rate limited (429)") + default: + return nil, fmt.Errorf("gemini: API error %d: %s", resp.StatusCode, string(respBody)) + } + } + + var result geminiResponse + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("gemini: unmarshal response: %w", err) + } + + if result.Error != nil { + return nil, fmt.Errorf("gemini: %s: %s", result.Error.Status, result.Error.Message) + } + + if len(result.Embeddings) != len(texts) { + return nil, fmt.Errorf("gemini: expected %d embeddings, got %d", len(texts), len(result.Embeddings)) + } + + embeddings := make([][]float32, len(texts)) + for i, e := range result.Embeddings { + embeddings[i] = e.Values + } + + return embeddings, nil +} + +func (p *GeminiProvider) Dimensions() int { return geminiDimensions } +func (p *GeminiProvider) Name() string { return "gemini" } diff --git a/internal/search/embedding/openai.go b/internal/search/embedding/openai.go index 486c2fc..50436b7 100644 --- a/internal/search/embedding/openai.go +++ b/internal/search/embedding/openai.go @@ -26,7 +26,7 @@ type OpenAIProvider struct { // NewOpenAIProvider creates a new OpenAI embedding provider. func NewOpenAIProvider(apiKey string) (*OpenAIProvider, error) { if apiKey == "" { - return nil, fmt.Errorf("openai provider requires SYNAPBUS_EMBEDDING_API_KEY to be set") + return nil, fmt.Errorf("openai provider requires OPENAI_API_KEY (or SYNAPBUS_EMBEDDING_API_KEY) to be set") } return &OpenAIProvider{ apiKey: apiKey, diff --git a/web/src/routes/+page.svelte b/web/src/routes/+page.svelte index a9823e6..b201e03 100644 --- a/web/src/routes/+page.svelte +++ b/web/src/routes/+page.svelte @@ -137,9 +137,22 @@ {#if !loadingData && recentMessages.length === 0 && agentCount === 0}
-
- - +
+ + + + + + + + + + + + + + +

Welcome to SynapBus

diff --git a/web/src/routes/agents/+page.svelte b/web/src/routes/agents/+page.svelte index d385cb1..e50d0cc 100644 --- a/web/src/routes/agents/+page.svelte +++ b/web/src/routes/agents/+page.svelte @@ -12,6 +12,7 @@ let registering = $state(false); let registerError = $state(''); let newApiKey = $state(''); + let copiedField = $state(''); async function loadAgents() { loadingData = true; @@ -49,9 +50,6 @@ type: newType }); newApiKey = res.api_key; - newName = ''; - newDisplayName = ''; - newType = 'ai'; await loadAgents(); } catch (err: any) { registerError = err.message || 'Failed to register agent'; @@ -59,12 +57,42 @@ registering = false; } } + + function resetForm() { + showRegister = false; + newApiKey = ''; + newName = ''; + newDisplayName = ''; + newType = 'ai'; + registerError = ''; + } + + async function copyText(text: string, label: string) { + try { + await navigator.clipboard.writeText(text); + copiedField = label; + setTimeout(() => (copiedField = ''), 2000); + } catch { + // fallback + } + } + + let mcpConfig = $derived(newApiKey ? JSON.stringify({ + mcpServers: { + synapbus: { + url: `${typeof window !== 'undefined' ? window.location.origin : 'http://localhost:8080'}/mcp`, + headers: { + Authorization: `Bearer ${newApiKey}` + } + } + } + }, null, 2) : '');

Agents

-
{#if showRegister} -
-

Register New Agent

- {#if registerError} -
{registerError}
- {/if} - {#if newApiKey} -
-

Agent registered! Save this API key - it will not be shown again:

- {newApiKey} + {#if newApiKey} + +
+
+
+ + + +
+
+

Agent "{newName}" registered

+

Save the API key below — it won't be shown again.

+
- {/if} -
- - - - +
+ {newApiKey} +
+ + +
+
+ + +
+ 
{mcpConfig}
+

Add this to your agent's MCP configuration (e.g. claude_desktop_config.json)

+
+ +
-
+ {:else} + +
+

Register New Agent

+ {#if registerError} +
{registerError}
+ {/if} +
+
+ + +
+
+ + +
+
+ + +
+ +
+
+ {/if} {/if} {#if loadingData} @@ -116,9 +201,10 @@
{/each}
- {:else if agentList.length === 0} -
- No agents registered yet. + {:else if agentList.length === 0 && !showRegister} +
+

No agents registered yet.

+
{:else}
diff --git a/web/src/routes/login/+page.svelte b/web/src/routes/login/+page.svelte index 97484c8..6fa7a38 100644 --- a/web/src/routes/login/+page.svelte +++ b/web/src/routes/login/+page.svelte @@ -36,9 +36,23 @@
-
- - +
+ + + + + + + + + + + + + + + +

SynapBus