ваше сообщение коммита

2025-06-26 20:34:58 +03:00
parent 25f1286c93
commit 1f4024d5be
36 changed files with 1709 additions and 967 deletions
--- a/backend/services/aiAssistantSettingsService.js
+++ b/backend/services/aiAssistantSettingsService.js
@@ -20,19 +20,21 @@ async function getSettings() {
  return {
    ...settings,
    telegramBot,
-    supportEmail
+    supportEmail,
+    embedding_model: settings.embedding_model
  };
 }

-async function upsertSettings({ system_prompt, selected_rag_tables, languages, model, rules, updated_by, telegram_settings_id, email_settings_id, system_message }) {
+async function upsertSettings({ system_prompt, selected_rag_tables, languages, model, embedding_model, rules, updated_by, telegram_settings_id, email_settings_id, system_message }) {
  const { rows } = await db.getQuery()(
-    `INSERT INTO ${TABLE} (id, system_prompt, selected_rag_tables, languages, model, rules, updated_at, updated_by, telegram_settings_id, email_settings_id, system_message)
-     VALUES (1, $1, $2, $3, $4, $5, NOW(), $6, $7, $8, $9)
+    `INSERT INTO ${TABLE} (id, system_prompt, selected_rag_tables, languages, model, embedding_model, rules, updated_at, updated_by, telegram_settings_id, email_settings_id, system_message)
+     VALUES (1, $1, $2, $3, $4, $5, $6, NOW(), $7, $8, $9, $10)
     ON CONFLICT (id) DO UPDATE SET
       system_prompt = EXCLUDED.system_prompt,
       selected_rag_tables = EXCLUDED.selected_rag_tables,
       languages = EXCLUDED.languages,
       model = EXCLUDED.model,
+       embedding_model = EXCLUDED.embedding_model,
       rules = EXCLUDED.rules,
       updated_at = NOW(),
       updated_by = EXCLUDED.updated_by,
@@ -40,7 +42,7 @@ async function upsertSettings({ system_prompt, selected_rag_tables, languages, m
       email_settings_id = EXCLUDED.email_settings_id,
       system_message = EXCLUDED.system_message
     RETURNING *`,
-    [system_prompt, selected_rag_tables, languages, model, rules, updated_by, telegram_settings_id, email_settings_id, system_message]
+    [system_prompt, selected_rag_tables, languages, model, embedding_model, rules, updated_by, telegram_settings_id, email_settings_id, system_message]
  );
  return rows[0];
 }
--- a/backend/services/aiProviderSettingsService.js
+++ b/backend/services/aiProviderSettingsService.js
@@ -12,17 +12,18 @@ async function getProviderSettings(provider) {
  return rows[0] || null;
 }

-async function upsertProviderSettings({ provider, api_key, base_url, selected_model }) {
+async function upsertProviderSettings({ provider, api_key, base_url, selected_model, embedding_model }) {
  const { rows } = await db.getQuery()(
-    `INSERT INTO ${TABLE} (provider, api_key, base_url, selected_model, updated_at)
-     VALUES ($1, $2, $3, $4, NOW())
+    `INSERT INTO ${TABLE} (provider, api_key, base_url, selected_model, embedding_model, updated_at)
+     VALUES ($1, $2, $3, $4, $5, NOW())
     ON CONFLICT (provider) DO UPDATE SET
       api_key = EXCLUDED.api_key,
       base_url = EXCLUDED.base_url,
       selected_model = EXCLUDED.selected_model,
+       embedding_model = EXCLUDED.embedding_model,
       updated_at = NOW()
     RETURNING *`,
-    [provider, api_key, base_url, selected_model]
+    [provider, api_key, base_url, selected_model, embedding_model]
  );
  return rows[0];
 }
@@ -97,10 +98,28 @@ async function verifyProviderKey(provider, { api_key, base_url } = {}) {
  }
 }

+async function getAllLLMModels() {
+  const { rows } = await db.getQuery()(
+    `SELECT provider, selected_model FROM ${TABLE} WHERE selected_model IS NOT NULL AND selected_model <> ''`
+  );
+  // Возвращаем массив объектов { id, provider }
+  return rows.map(r => ({ id: r.selected_model, provider: r.provider }));
+}
+
+async function getAllEmbeddingModels() {
+  const { rows } = await db.getQuery()(
+    `SELECT provider, embedding_model FROM ${TABLE} WHERE embedding_model IS NOT NULL AND embedding_model <> ''`
+  );
+  // Возвращаем массив объектов { id, provider }
+  return rows.map(r => ({ id: r.embedding_model, provider: r.provider }));
+}
+
 module.exports = {
  getProviderSettings,
  upsertProviderSettings,
  deleteProviderSettings,
  getProviderModels,
  verifyProviderKey,
+  getAllLLMModels,
+  getAllEmbeddingModels,
 }; 
--- a/backend/services/dbSettingsService.js
+++ b/backend/services/dbSettingsService.js
@@ -0,0 +1,27 @@
+const db = require('../db');
+
+class DbSettingsService {
+  async getSettings() {
+    const { rows } = await db.getQuery()('SELECT * FROM db_settings WHERE id = 1');
+    return rows[0];
+  }
+
+  async upsertSettings({ db_host, db_port, db_name, db_user, db_password }) {
+    const { rows } = await db.getQuery()(
+      `INSERT INTO db_settings (id, db_host, db_port, db_name, db_user, db_password, updated_at)
+       VALUES (1, $1, $2, $3, $4, $5, NOW())
+       ON CONFLICT (id) DO UPDATE SET
+         db_host = EXCLUDED.db_host,
+         db_port = EXCLUDED.db_port,
+         db_name = EXCLUDED.db_name,
+         db_user = EXCLUDED.db_user,
+         db_password = EXCLUDED.db_password,
+         updated_at = NOW()
+       RETURNING *`,
+      [db_host, db_port, db_name, db_user, db_password]
+    );
+    return rows[0];
+  }
+}
+
+module.exports = new DbSettingsService(); 
--- a/backend/services/emailBot.js
+++ b/backend/services/emailBot.js
@@ -309,6 +309,11 @@ class EmailBotService {
      throw err;
    }
  }
+
+  async getAllEmailSettings() {
+    const { rows } = await db.getQuery()('SELECT id, from_email FROM email_settings ORDER BY id');
+    return rows;
+  }
 }

 console.log('[EmailBot] module.exports = EmailBotService');
--- a/backend/services/ragService.js
+++ b/backend/services/ragService.js
@@ -1,10 +1,15 @@
-const { OpenAIEmbeddings } = require('@langchain/openai');
 const { HNSWLib } = require('@langchain/community/vectorstores/hnswlib');
 const db = require('../db');
 const { ChatOllama } = require('@langchain/ollama');
+const { OllamaEmbeddings } = require('@langchain/ollama');
+const { getProviderSettings } = require('./aiProviderSettingsService');
+const { OpenAIEmbeddings } = require('@langchain/openai');
+
+console.log('[RAG] ragService.js loaded');

 async function getTableData(tableId) {
  const columns = (await db.getQuery()('SELECT * FROM user_columns WHERE table_id = $1', [tableId])).rows;
+  console.log('RAG getTableData: columns:', columns);
  const rows = (await db.getQuery()('SELECT * FROM user_rows WHERE table_id = $1', [tableId])).rows;
  const cellValues = (await db.getQuery()('SELECT * FROM user_cell_values WHERE row_id IN (SELECT id FROM user_rows WHERE table_id = $1)', [tableId])).rows;

@@ -17,7 +22,7 @@ async function getTableData(tableId) {
  const priorityColId = getColId('priority');
  const dateColId = getColId('date');

-  return rows.map(row => {
+  const data = rows.map(row => {
    const cells = cellValues.filter(cell => cell.row_id === row.id);
    return {
      id: row.id,
@@ -30,35 +35,107 @@ async function getTableData(tableId) {
      date: cells.find(c => c.column_id === dateColId)?.value,
    };
  });
+  const questions = data.map(row => row.question);
+  console.log('RAG getTableData: questions:', questions);
+  if (!questions.length) {
+    console.warn('RAG getTableData: questions array is empty! Проверьте структуру колонок и наличие данных.');
+  }
+  return data;
 }

-async function ragAnswer({ tableId, userQuestion, userTags = [], product = null }) {
+async function getEmbeddingsProvider(providerName = 'ollama') {
+  const settings = await getProviderSettings(providerName);
+  if (!settings) throw new Error('Embeddings provider settings not found');
+  switch (providerName) {
+    case 'openai':
+      return new OpenAIEmbeddings({
+        apiKey: settings.api_key,
+        baseURL: settings.base_url,
+        model: settings.selected_model || undefined,
+      });
+    case 'ollama': {
+      // Fallback: если не задан base_url, пробуем env, host.docker.internal, localhost
+      let baseUrl = settings.base_url;
+      if (!baseUrl) {
+        baseUrl = process.env.OLLAMA_BASE_URL;
+      }
+      if (!baseUrl) {
+        // Если в Docker — используем host.docker.internal
+        baseUrl = 'http://host.docker.internal:11434';
+      }
+      // Если всё равно нет — последний fallback
+      if (!baseUrl) {
+        baseUrl = 'http://localhost:11434';
+      }
+      return new OllamaEmbeddings({
+        model: settings.embedding_model || process.env.OLLAMA_EMBED_MODEL || 'mxbai-embed-large',
+        baseUrl,
+      });
+    }
+    // case 'gemini':
+    //   return new GeminiEmbeddings({ apiKey: settings.api_key });
+    // Добавьте другие провайдеры по аналогии
+    default:
+      throw new Error('Unknown embeddings provider: ' + providerName);
+  }
+}
+
+async function ragAnswer({ tableId, userQuestion, userTags = [], product = null, embeddingsProvider = 'ollama', threshold = 0.3 }) {
+  console.log('[RAG] Используется провайдер эмбеддингов:', embeddingsProvider);
  const data = await getTableData(tableId);
-  const questions = data.map(row => row.question);
+  // Триммируем вопросы для чистоты сравнения
+  const questions = data.map(row => row.question && typeof row.question === 'string' ? row.question.trim() : row.question);
+
+  // Получаем embeddings-инстанс динамически
+  const embeddingsInstance = await getEmbeddingsProvider(embeddingsProvider);

  // Получаем embedding для всех вопросов
-  const embeddings = await new OpenAIEmbeddings().embedDocuments(questions);
+  const embeddings = await embeddingsInstance.embedDocuments(questions);
+  console.log('Questions embedding length:', embeddings[0]?.length, 'Total questions:', questions.length);
+
+  // Получаем embedding для вопроса пользователя (trim)
+  const userQuestionTrimmed = userQuestion && typeof userQuestion === 'string' ? userQuestion.trim() : userQuestion;
+  const [userEmbedding] = await embeddingsInstance.embedDocuments([userQuestionTrimmed]);
+  console.log('User embedding length:', userEmbedding?.length, 'User question:', userQuestionTrimmed);
+
+  // Явно сравниваем embeddings (отладка)
+  console.log('[RAG] Embedding сравнение:');
+  embeddings.forEach((emb, idx) => {
+    const dot = emb.reduce((sum, v, i) => sum + v * userEmbedding[i], 0);
+    console.log(`  [${idx}] dot-product: ${dot} | question: "${questions[idx]}"`);
+  });
+
+  // Создаём массив метаданных для каждого вопроса
+  const metadatas = data.map(row => ({
+    id: row.id,
+    answer: row.answer,
+    userTags: row.userTags,
+    context: row.context,
+    product: row.product,
+    priority: row.priority,
+    date: row.date,
+  }));

  // Создаём векторное хранилище
-  const vectorStore = await HNSWLib.fromTexts(questions, data, new OpenAIEmbeddings());
-
-  // Получаем embedding для вопроса пользователя
-  const [userEmbedding] = await new OpenAIEmbeddings().embedDocuments([userQuestion]);
+  const vectorStore = await HNSWLib.fromTexts(questions, metadatas, embeddingsInstance);

  // Ищем наиболее похожие вопросы (top-3)
  const results = await vectorStore.similaritySearchVectorWithScore(userEmbedding, 3);
+  console.log('[RAG] Результаты поиска по векторам (score):', results.map(([doc, score]) => ({ ...doc.metadata, score })));

  // Фильтруем по тегам/продукту, если нужно
-  let filtered = results.map(([row, score]) => ({ ...row, score }));
+  let filtered = results.map(([doc, score]) => ({ ...doc.metadata, score }));
  if (userTags.length) {
    filtered = filtered.filter(row => row.userTags && userTags.some(tag => row.userTags.includes(tag)));
  }
  if (product) {
    filtered = filtered.filter(row => row.product === product);
  }
+  console.log('[RAG] Отфильтрованные результаты:', filtered);

-  // Берём лучший результат
-  const best = filtered[0];
+  // Берём лучший результат с учётом порога
+  const best = filtered.find(row => row.score >= threshold);
+  console.log(`[RAG] Выбранный ответ (порог ${threshold}):`, best);

  // Формируем ответ
  return {
--- a/backend/services/telegramBot.js
+++ b/backend/services/telegramBot.js
@@ -424,9 +424,16 @@ function clearSettingsCache() {
  telegramSettingsCache = null;
 }

+async function getAllBots() {
+  const { rows } = await db.getQuery()('SELECT id, bot_username FROM telegram_settings ORDER BY id');
+  return rows;
+}
+
 module.exports = {
+  getTelegramSettings,
  getBot,
  stopBot,
  initTelegramAuth,
  clearSettingsCache,
+  getAllBots,
 };