ваше сообщение коммита

vector-search/.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+*.pyc
+*.pkl
+*.faiss
+.env 

vector-search/Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3.10-slim
+WORKDIR /app
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+COPY . .
+EXPOSE 8001
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8001"] 

vector-search/README.md

@@ -0,0 +1,25 @@
+# Vector Search Service
+
+## Запуск локально
+```
+pip install -r requirements.txt
+uvicorn app:app --reload --host 0.0.0.0 --port 8001
+```
+
+## Запуск в Docker
+```
+docker build -t vector-search .
+docker run -p 8001:8001 vector-search
+```
+
+## Эндпоинты
+- POST /upsert — добавить/обновить строки
+- POST /search — поиск похожих
+- POST /delete — удалить строки
+- POST /rebuild — пересоздать индекс
+- GET /health — проверка статуса
+
+## Пример запроса /upsert
+```
+curl -X POST http://localhost:8001/upsert -H "Content-Type: application/json" -d '{"table_id": "t1", "rows": [{"row_id": "1", "text": "Пример", "metadata": {}}]}'
+``` 

vector-search/app.py

@@ -0,0 +1,128 @@
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from typing import List, Dict, Any
+import os
+import requests
+from vector_store import VectorStore
+
+OLLAMA_BASE_URL = os.getenv('OLLAMA_BASE_URL', 'http://ollama:11434')
+EMBED_MODEL = os.getenv('OLLAMA_EMBED_MODEL', 'mxbai-embed-large')
+
+app = FastAPI()
+store = VectorStore()
+
+class UpsertRow(BaseModel):
+    row_id: str
+    text: str
+    metadata: Dict[str, Any] = {}
+
+class UpsertRequest(BaseModel):
+    table_id: str
+    rows: List[UpsertRow]
+
+class SearchRequest(BaseModel):
+    table_id: str
+    query: str
+    top_k: int = 3
+
+class SearchResult(BaseModel):
+    row_id: str
+    score: float
+    metadata: Dict[str, Any] = {}
+
+class SearchResponse(BaseModel):
+    results: List[SearchResult]
+
+class DeleteRequest(BaseModel):
+    table_id: str
+    row_ids: List[str]
+
+class RebuildRequest(BaseModel):
+    table_id: str
+    rows: List[UpsertRow]
+
+# --- Ollama embedding ---
+def get_embedding(text: str) -> list:
+    print(f"[DEBUG] Getting embedding for text: '{text[:50]}...' (length: {len(text)})")
+    print(f"[DEBUG] Using OLLAMA_BASE_URL: {OLLAMA_BASE_URL}")
+    print(f"[DEBUG] Using EMBED_MODEL: {EMBED_MODEL}")
+    
+    try:
+        resp = requests.post(f"{OLLAMA_BASE_URL}/api/embeddings", json={
+            "model": EMBED_MODEL,
+            "prompt": text
+        }, timeout=30)
+        print(f"[DEBUG] Ollama response status: {resp.status_code}")
+        
+        if not resp.ok:
+            print(f"[ERROR] Ollama API error: {resp.status_code} - {resp.text}")
+            raise HTTPException(status_code=resp.status_code, detail=f"Ollama API error: {resp.text}")
+        
+        data = resp.json()
+        print(f"[DEBUG] Ollama response keys: {list(data.keys())}")
+        
+        if 'embedding' in data:
+            print(f"[DEBUG] Found embedding in data['embedding'], length: {len(data['embedding'])}")
+            return data['embedding']
+        if 'data' in data and isinstance(data['data'], list) and 'embedding' in data['data'][0]:
+            print(f"[DEBUG] Found embedding in data['data'][0]['embedding'], length: {len(data['data'][0]['embedding'])}")
+            return data['data'][0]['embedding']
+        
+        print(f"[ERROR] No embedding found in response: {data}")
+        raise ValueError('No embedding in Ollama response')
+    except requests.exceptions.RequestException as e:
+        print(f"[ERROR] Request exception: {e}")
+        raise HTTPException(status_code=422, detail=f"Failed to connect to Ollama: {e}")
+    except Exception as e:
+        print(f"[ERROR] Unexpected error: {e}")
+        raise HTTPException(status_code=422, detail=f"Failed to get embedding: {e}")
+
+@app.post("/upsert")
+def upsert_rows(req: UpsertRequest):
+    print(f"[DEBUG] Upsert request: table_id={req.table_id}, rows_count={len(req.rows)}")
+    rows_to_upsert = []
+    for i, row in enumerate(req.rows):
+        print(f"[DEBUG] Processing row {i}: row_id={row.row_id}, text_length={len(row.text)}")
+        try:
+            emb = get_embedding(row.text)
+            print(f"[DEBUG] Got embedding for row {i}: length={len(emb)}")
+            rows_to_upsert.append({
+                'row_id': row.row_id,
+                'embedding': emb,
+                'metadata': row.metadata
+            })
+        except Exception as e:
+            print(f"[ERROR] Failed to get embedding for row {i}: {e}")
+            raise HTTPException(status_code=422, detail=f"Failed to get embedding: {e}")
+    
+    print(f"[DEBUG] Upserting {len(rows_to_upsert)} rows to store")
+    store.upsert(req.table_id, rows_to_upsert)
+    return {"success": True}
+
+@app.post("/search", response_model=SearchResponse)
+def search(req: SearchRequest):
+    emb = get_embedding(req.query)
+    results = store.search(req.table_id, emb, req.top_k)
+    return {"results": results}
+
+@app.post("/delete")
+def delete(req: DeleteRequest):
+    store.delete(req.table_id, req.row_ids)
+    return {"success": True}
+
+@app.post("/rebuild")
+def rebuild(req: RebuildRequest):
+    rows_to_upsert = []
+    for row in req.rows:
+        emb = get_embedding(row.text)
+        rows_to_upsert.append({
+            'row_id': row.row_id,
+            'embedding': emb,
+            'metadata': row.metadata
+        })
+    store.rebuild(req.table_id, rows_to_upsert)
+    return {"success": True}
+
+@app.get("/health")
+def health():
+    return {"status": "ok"} 

vector-search/requirements.txt

@@ -0,0 +1,5 @@
+fastapi
+uvicorn
+faiss-cpu
+requests
+pydantic 

vector-search/schemas.py

@@ -0,0 +1 @@
+# Здесь можно разместить расширенные схемы Pydantic для API 

vector-search/vector_store.py

@@ -0,0 +1,102 @@
+# Заглушка для работы с FAISS
+
+import os
+import pickle
+import faiss
+import numpy as np
+from typing import List, Dict, Any
+
+INDEX_DIR = os.path.join(os.path.dirname(__file__), 'indexes')
+
+class VectorStore:
+    def __init__(self):
+        os.makedirs(INDEX_DIR, exist_ok=True)
+        self.index_cache = {}  # table_id: (faiss_index, meta)
+
+    def _index_path(self, table_id):
+        return os.path.join(INDEX_DIR, f'table_{table_id}.faiss')
+    def _meta_path(self, table_id):
+        return os.path.join(INDEX_DIR, f'table_{table_id}_meta.pkl')
+
+    def load(self, table_id):
+        idx_path = self._index_path(table_id)
+        meta_path = self._meta_path(table_id)
+        if os.path.exists(idx_path) and os.path.exists(meta_path):
+            index = faiss.read_index(idx_path)
+            with open(meta_path, 'rb') as f:
+                meta = pickle.load(f)
+            self.index_cache[table_id] = (index, meta)
+            return index, meta
+        return None, None
+
+    def save(self, table_id, index, meta):
+        faiss.write_index(index, self._index_path(table_id))
+        with open(self._meta_path(table_id), 'wb') as f:
+            pickle.dump(meta, f)
+        self.index_cache[table_id] = (index, meta)
+
+    def upsert(self, table_id, rows: List[Dict]):
+        # rows: [{row_id, embedding, metadata}]
+        index, meta = self.load(table_id)
+        if index is None:
+            dim = len(rows[0]['embedding'])
+            index = faiss.IndexFlatL2(dim)
+            meta = []
+        # Удаляем дубликаты row_id
+        existing_ids = {m['row_id'] for m in meta}
+        new_rows = [r for r in rows if r['row_id'] not in existing_ids]
+        if not new_rows:
+            return
+        vectors = np.array([r['embedding'] for r in new_rows]).astype('float32')
+        index.add(vectors)
+        meta.extend(new_rows)
+        self.save(table_id, index, meta)
+
+    def search(self, table_id, query_embedding, top_k=3):
+        index, meta = self.load(table_id)
+        if index is None or not meta:
+            return []
+        query = np.array([query_embedding]).astype('float32')
+        D, I = index.search(query, top_k)
+        results = []
+        for idx, dist in zip(I[0], D[0]):
+            if idx < 0 or idx >= len(meta):
+                continue
+            m = meta[idx]
+            results.append({
+                'row_id': m['row_id'],
+                'score': float(-dist),  # FAISS: чем меньше dist, тем ближе
+                'metadata': m['metadata']
+            })
+        return results
+
+    def delete(self, table_id, row_ids: List[str]):
+        index, meta = self.load(table_id)
+        if index is None or not meta:
+            return
+        # FAISS не поддерживает удаление, пересоздаём индекс
+        new_meta = [m for m in meta if m['row_id'] not in row_ids]
+        if not new_meta:
+            # Удаляем файлы
+            try:
+                os.remove(self._index_path(table_id))
+                os.remove(self._meta_path(table_id))
+            except Exception:
+                pass
+            self.index_cache.pop(table_id, None)
+            return
+        dim = len(new_meta[0]['embedding'])
+        new_index = faiss.IndexFlatL2(dim)
+        vectors = np.array([m['embedding'] for m in new_meta]).astype('float32')
+        new_index.add(vectors)
+        self.save(table_id, new_index, new_meta)
+
+    def rebuild(self, table_id, rows: List[Dict]):
+        # rows: [{row_id, embedding, metadata}]
+        if not rows:
+            return
+        dim = len(rows[0]['embedding'])
+        index = faiss.IndexFlatL2(dim)
+        vectors = np.array([r['embedding'] for r in rows]).astype('float32')
+        index.add(vectors)
+        self.save(table_id, index, rows)