Algis DumbrisandClaude Opus 4.6
5dcb9e6149
Add HNSW-based vector search with configurable embedding providers (OpenAI, Ollama) and automatic FTS5 fallback when no provider is configured. Background pipeline embeds messages asynchronously on ingest, stores vectors in a pure-Go HNSW index, and retries on failure with exponential backoff. The search_messages MCP tool now supports search_mode (auto/semantic/fulltext) and returns ranked results with similarity scores. All existing tests continue to pass, CGO_ENABLED=0 cross-compilation verified. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
27 lines
1.1 KiB
SQL
27 lines
1.1 KiB
SQL
-- Semantic search: embeddings tracking and queue
|
|
-- Note: actual vectors are stored in the HNSW index file on disk.
|
|
-- This table tracks which messages have been embedded and by which provider.
|
|
|
|
CREATE TABLE IF NOT EXISTS embeddings (
|
|
message_id INTEGER PRIMARY KEY REFERENCES messages(id) ON DELETE CASCADE,
|
|
provider TEXT NOT NULL,
|
|
model TEXT NOT NULL,
|
|
dimensions INTEGER NOT NULL,
|
|
embedded_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS embedding_queue (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
message_id INTEGER NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
|
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'processing', 'completed', 'failed')),
|
|
attempts INTEGER NOT NULL DEFAULT 0,
|
|
last_error TEXT,
|
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
completed_at TIMESTAMP
|
|
);
|
|
|
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_embedding_queue_message ON embedding_queue(message_id);
|
|
CREATE INDEX IF NOT EXISTS idx_embedding_queue_status ON embedding_queue(status);
|
|
|
|
INSERT INTO schema_migrations (version) VALUES (5);
|