diff --git a/apps/backend/hub_backend/settings.py b/apps/backend/hub_backend/settings.py index bb93ed1..6e2987a 100644 --- a/apps/backend/hub_backend/settings.py +++ b/apps/backend/hub_backend/settings.py @@ -143,6 +143,7 @@ HUB_AI_REQUEST_TIMEOUT = float(os.environ.get("HUB_AI_REQUEST_TIMEOUT", "30")) HUB_AI_MAX_RETRIES = int(os.environ.get("HUB_AI_MAX_RETRIES", "2")) HUB_AI_GLOBAL_DAILY_COST_LIMIT_MICROS = int(os.environ.get("HUB_AI_GLOBAL_DAILY_COST_LIMIT_MICROS", "0")) # 0 = без лимита HUB_AI_PRICING: dict = {} # переопределение цен micro-USD/токен по модели +HUB_AI_EMBEDDING_MODEL = os.environ.get("HUB_AI_EMBEDDING_MODEL", "openai/text-embedding-3-small") # Password reset link lifetime. UI обещает 30 минут (default_token_generator uses this setting). PASSWORD_RESET_TIMEOUT = int(os.environ.get("PASSWORD_RESET_TIMEOUT", str(30 * 60))) diff --git a/apps/backend/hub_platform/ai/chunking.py b/apps/backend/hub_platform/ai/chunking.py new file mode 100644 index 0000000..b47efa7 --- /dev/null +++ b/apps/backend/hub_platform/ai/chunking.py @@ -0,0 +1,14 @@ +def chunk_text(text: str, *, max_chars: int = 800) -> list[str]: + """Split markdown into chunks by paragraphs, packing up to max_chars.""" + paragraphs = [paragraph.strip() for paragraph in text.split("\n\n") if paragraph.strip()] + chunks: list[str] = [] + current = "" + for paragraph in paragraphs: + if current and len(current) + len(paragraph) + 2 > max_chars: + chunks.append(current) + current = paragraph + else: + current = f"{current}\n\n{paragraph}" if current else paragraph + if current: + chunks.append(current) + return chunks diff --git a/apps/backend/hub_platform/ai/doc_views.py b/apps/backend/hub_platform/ai/doc_views.py index 567312d..6a335f3 100644 --- a/apps/backend/hub_platform/ai/doc_views.py +++ b/apps/backend/hub_platform/ai/doc_views.py @@ -46,6 +46,10 @@ class _DocConfig(APIView): request=request, ) + def after_publish(self, version) -> None: + # Хук для типоспецифичного действия после публикации версии. + pass + class DocumentListCreateView(_DocConfig): def get(self, request: Request) -> Response: @@ -128,6 +132,7 @@ class DocumentPublishVersionView(_DocConfig): except self.version_model.DoesNotExist: return Response({"detail": "Version not found"}, status=404) doc_service.publish_version(version=target) + self.after_publish(target) document = self._document(request, document_id) self._audit(request, "version_published", document) return Response({"document": self.payload(document)}) @@ -184,6 +189,11 @@ class _KnowledgeConfig: supports_inclusion = True audit_prefix = "ai.knowledge" + def after_publish(self, version) -> None: + from hub_platform.ai.indexing import reindex_knowledge_version + + reindex_knowledge_version(version) + class _PromptConfig: document_model = PromptDocument diff --git a/apps/backend/hub_platform/ai/indexing.py b/apps/backend/hub_platform/ai/indexing.py new file mode 100644 index 0000000..b62edb9 --- /dev/null +++ b/apps/backend/hub_platform/ai/indexing.py @@ -0,0 +1,24 @@ +from django.conf import settings + +from hub_platform.ai.chunking import chunk_text +from hub_platform.ai.invocation import embed_texts +from hub_platform.ai.models import KnowledgeDocumentVersion, KnowledgeFragment + + +def reindex_knowledge_version(version: KnowledgeDocumentVersion) -> list[KnowledgeFragment]: + """Rebuild fragments + embeddings for a published knowledge version (ADR-HUB-0016).""" + KnowledgeFragment.objects.filter(version=version).delete() + chunks = chunk_text(version.content) + if not chunks: + return [] + embeddings = embed_texts( + product=version.document.product, + texts=chunks, + model=settings.HUB_AI_EMBEDDING_MODEL, + purpose="knowledge_index", + ) + fragments = [ + KnowledgeFragment(version=version, chunk_index=index, content=chunk, embedding=result.vector) + for index, (chunk, result) in enumerate(zip(chunks, embeddings)) + ] + return KnowledgeFragment.objects.bulk_create(fragments) diff --git a/apps/backend/hub_platform/ai/invocation.py b/apps/backend/hub_platform/ai/invocation.py index a065b83..8028314 100644 --- a/apps/backend/hub_platform/ai/invocation.py +++ b/apps/backend/hub_platform/ai/invocation.py @@ -12,7 +12,7 @@ from hub_platform.ai.provider.resilience import CircuitBreaker, call_with_resili _breaker = CircuitBreaker() -def invoke_chat(*, product, messages: list[ChatMessage], purpose: str, release=None, model: str | None = None, params: dict | None = None) -> ChatResult: +def invoke_chat(*, product, messages: list[ChatMessage], purpose: str, release=None, model: str | None = None, params: dict | None = None, used_fragment_ids: list | None = None) -> ChatResult: agent = product.ai_agent model = model or agent.model @@ -48,6 +48,7 @@ def invoke_chat(*, product, messages: list[ChatMessage], purpose: str, release=N prompt_tokens=result.prompt_tokens, completion_tokens=result.completion_tokens, total_tokens=result.total_tokens, cost_micros=pricing.cost_micros(result.model, result.prompt_tokens, result.completion_tokens), latency_ms=int((time.monotonic() - started) * 1000), status=LlmInvocationStatus.SUCCESS, + used_fragment_ids=used_fragment_ids or [], ) return result diff --git a/apps/backend/hub_platform/ai/migrations/0006_llminvocation_used_fragment_ids_knowledgefragment.py b/apps/backend/hub_platform/ai/migrations/0006_llminvocation_used_fragment_ids_knowledgefragment.py new file mode 100644 index 0000000..c2cc344 --- /dev/null +++ b/apps/backend/hub_platform/ai/migrations/0006_llminvocation_used_fragment_ids_knowledgefragment.py @@ -0,0 +1,37 @@ +# Generated by Django 5.2.15 on 2026-06-24 16:40 + +import django.db.models.deletion +import pgvector.django.vector +from django.db import migrations, models +from pgvector.django import VectorExtension + + +class Migration(migrations.Migration): + + dependencies = [ + ('ai', '0005_llminvocation'), + ] + + operations = [ + VectorExtension(), + migrations.AddField( + model_name='llminvocation', + name='used_fragment_ids', + field=models.JSONField(blank=True, default=list), + ), + migrations.CreateModel( + name='KnowledgeFragment', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('chunk_index', models.PositiveIntegerField()), + ('content', models.TextField()), + ('embedding', pgvector.django.vector.VectorField(blank=True, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('version', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='fragments', to='ai.knowledgedocumentversion')), + ], + options={ + 'ordering': ['version_id', 'chunk_index'], + 'constraints': [models.UniqueConstraint(fields=('version', 'chunk_index'), name='uniq_fragment_version_chunk')], + }, + ), + ] diff --git a/apps/backend/hub_platform/ai/models.py b/apps/backend/hub_platform/ai/models.py index 07ce791..7f3a938 100644 --- a/apps/backend/hub_platform/ai/models.py +++ b/apps/backend/hub_platform/ai/models.py @@ -1,6 +1,7 @@ from django.conf import settings from django.core.exceptions import ValidationError from django.db import models +from pgvector.django import VectorField # Один основной sales-агент на продукт (ADR-HUB-0007). DEFAULT_AI_MODEL = "openai/gpt-4o-mini" @@ -111,6 +112,23 @@ class KnowledgeDocumentVersion(_BaseDocumentVersion): constraints = [models.UniqueConstraint(fields=["document", "version"], name="uniq_knowledge_version")] +class KnowledgeFragment(models.Model): + # Чанк опубликованной версии знания + его эмбеддинг (pgvector). ADR-HUB-0016. + version = models.ForeignKey(KnowledgeDocumentVersion, on_delete=models.CASCADE, related_name="fragments") + chunk_index = models.PositiveIntegerField() + content = models.TextField() + # Размерность не фиксируется: совместимость локального и production embedding-провайдера. + embedding = VectorField(null=True, blank=True) + created_at = models.DateTimeField(auto_now_add=True) + + class Meta: + ordering = ["version_id", "chunk_index"] + constraints = [models.UniqueConstraint(fields=["version", "chunk_index"], name="uniq_fragment_version_chunk")] + + def __str__(self) -> str: + return f"fragment:{self.version_id}/{self.chunk_index}" + + class PromptDocumentVersion(_BaseDocumentVersion): document = models.ForeignKey(PromptDocument, on_delete=models.CASCADE, related_name="versions") @@ -205,6 +223,7 @@ class LlmInvocation(models.Model): latency_ms = models.PositiveIntegerField(default=0) status = models.CharField(max_length=16, choices=LlmInvocationStatus.choices, default=LlmInvocationStatus.SUCCESS) error = models.TextField(blank=True) + used_fragment_ids = models.JSONField(default=list, blank=True) created_at = models.DateTimeField(auto_now_add=True, db_index=True) class Meta: diff --git a/apps/backend/hub_platform/ai/retrieval.py b/apps/backend/hub_platform/ai/retrieval.py new file mode 100644 index 0000000..b8847c1 --- /dev/null +++ b/apps/backend/hub_platform/ai/retrieval.py @@ -0,0 +1,48 @@ +from django.conf import settings +from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVector +from pgvector.django import CosineDistance + +from hub_platform.ai.invocation import embed_texts +from hub_platform.ai.models import KnowledgeFragment, ProductAIRelease + + +def _release_fragments(release: ProductAIRelease): + version_ids = release.knowledge_versions.values_list("knowledge_version_id", flat=True) + return KnowledgeFragment.objects.filter(version_id__in=version_ids).select_related("version__document") + + +def lexical_search(release: ProductAIRelease, query: str, *, limit: int = 5) -> list[KnowledgeFragment]: + if not query.strip(): + return [] + search_query = SearchQuery(query, search_type="websearch") + return list( + _release_fragments(release) + .annotate(rank=SearchRank(SearchVector("content"), search_query)) + .filter(rank__gt=0) + .order_by("-rank")[:limit] + ) + + +def semantic_search(release: ProductAIRelease, query_vector: list[float], *, limit: int = 5) -> list[KnowledgeFragment]: + return list( + _release_fragments(release) + .filter(embedding__isnull=False) + .order_by(CosineDistance("embedding", query_vector))[:limit] + ) + + +class KnowledgeRetriever: + """Hybrid retriever: semantic (pgvector) primary, lexical (Postgres FTS) complementary.""" + + def retrieve(self, *, release: ProductAIRelease, query: str, limit: int = 5) -> list[KnowledgeFragment]: + query_vector = embed_texts( + product=release.product, + texts=[query], + model=settings.HUB_AI_EMBEDDING_MODEL, + purpose="retrieval_query", + )[0].vector + semantic = semantic_search(release, query_vector, limit=limit) + lexical = lexical_search(release, query, limit=limit) + seen = {fragment.id for fragment in semantic} + merged = semantic + [fragment for fragment in lexical if fragment.id not in seen] + return merged[:limit] diff --git a/apps/backend/hub_platform/ai/tests.py b/apps/backend/hub_platform/ai/tests.py index 5194182..9bafd4c 100644 --- a/apps/backend/hub_platform/ai/tests.py +++ b/apps/backend/hub_platform/ai/tests.py @@ -387,3 +387,74 @@ class ChatInvocationTests(TestCase): with self.assertRaises(ai_limits.LimitExceeded): invoke_chat(product=self.product, messages=[ChatMessage(role="user", content="hi")], purpose="test_chat") self.assertTrue(LlmInvocation.objects.filter(product=self.product, status=LlmInvocationStatus.BLOCKED).exists()) + + def test_invocation_records_used_fragment_ids(self) -> None: + from hub_platform.ai.invocation import invoke_chat + from hub_platform.ai.models import LlmInvocation + from hub_platform.ai.provider.base import ChatMessage + + invoke_chat( + product=self.product, + messages=[ChatMessage(role="user", content="hi")], + purpose="test_chat", + used_fragment_ids=[11, 22], + ) + invocation = LlmInvocation.objects.get(product=self.product, operation="chat") + self.assertEqual(invocation.used_fragment_ids, [11, 22]) + + +class ChunkingTests(TestCase): + def test_packs_paragraphs_into_chunks(self) -> None: + from hub_platform.ai.chunking import chunk_text + + text = "\n\n".join(["paragraph " + str(i) + " " + "x" * 200 for i in range(10)]) + chunks = chunk_text(text, max_chars=500) + self.assertGreater(len(chunks), 1) + self.assertTrue(all(len(chunk) <= 700 for chunk in chunks)) + + +class KnowledgeRetrievalTests(TestCase): + def setUp(self) -> None: + bootstrap_edevs_owner(email="owner@edevs.tech", password="temporary-password") + self.client = APIClient() + self.client.login(username="owner@edevs.tech", password="temporary-password") + document_id = self.client.post( + "/api/v1/ai/knowledge/", + data=json.dumps( + { + "product": "firepage", + "code": "faq", + "title": "FAQ", + "category": "FAQ", + "content": "Refund policy details here.\n\nDelivery and shipping information.", + } + ), + content_type="application/json", + ).json()["document"]["id"] + self.client.post(f"/api/v1/ai/knowledge/{document_id}/versions/1/publish/") + + from hub_platform.ai import releases + + self.product = Product.objects.get(code="firepage") + owner = HumanUser.objects.get(email="owner@edevs.tech") + self.release = releases.create_draft_release(product=self.product, author=owner) + + def test_publish_builds_fragments_with_embeddings(self) -> None: + from hub_platform.ai.models import KnowledgeFragment + + fragments = KnowledgeFragment.objects.filter(version__document__code="faq") + self.assertGreaterEqual(fragments.count(), 1) + self.assertTrue(all(fragment.embedding is not None for fragment in fragments)) + + def test_retriever_returns_release_scoped_fragments(self) -> None: + from hub_platform.ai.retrieval import KnowledgeRetriever + + results = KnowledgeRetriever().retrieve(release=self.release, query="refund", limit=5) + self.assertGreaterEqual(len(results), 1) + self.assertLessEqual(len(results), 5) + + def test_lexical_search_matches_content(self) -> None: + from hub_platform.ai.retrieval import lexical_search + + results = lexical_search(self.release, "Delivery", limit=5) + self.assertTrue(any("delivery" in fragment.content.lower() for fragment in results)) diff --git a/apps/backend/requirements.txt b/apps/backend/requirements.txt index 8e00eb4..732affb 100644 --- a/apps/backend/requirements.txt +++ b/apps/backend/requirements.txt @@ -1,6 +1,7 @@ Django>=5.2,<5.3 djangorestframework>=3.16,<3.17 cryptography>=43,<45 +pgvector>=0.3,<0.4 psycopg[binary]>=3.2,<3.3 redis>=5.2,<5.3 uvicorn[standard]>=0.34,<0.35 diff --git a/compose.yaml b/compose.yaml index 95f3f66..13962a1 100644 --- a/compose.yaml +++ b/compose.yaml @@ -5,7 +5,7 @@ x-hub-env: &hub-env services: postgres: - image: postgres:16-alpine + image: pgvector/pgvector:pg16 environment: POSTGRES_DB: ${POSTGRES_DB:-edevs_hub} POSTGRES_USER: ${POSTGRES_USER:-edevs_hub}