✨ feat(ai): knowledge retrieval with pgvector (E04 slice 3)

- ADR-HUB-0016: retrieval on PostgreSQL (FTS) + pgvector (semantic); db image
  switched to pgvector/pgvector:pg16, extension enabled via migration
- KnowledgeFragment stores chunks + embeddings (dimension-agnostic vector so
  local and OpenRouter embedding sizes coexist)
- publishing a knowledge version re-chunks and embeds it via the provider
- KnowledgeRetriever: semantic (cosine) primary + lexical (FTS) complementary,
  scoped to the release's pinned knowledge versions
- LlmInvocation records used_fragment_ids (ADR-HUB-0007)
- tests for chunking, indexing, retrieval scope and lexical match

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

apps/backend/hub_backend/settings.py

@@ -143,6 +143,7 @@ HUB_AI_REQUEST_TIMEOUT = float(os.environ.get("HUB_AI_REQUEST_TIMEOUT", "30"))
 HUB_AI_MAX_RETRIES = int(os.environ.get("HUB_AI_MAX_RETRIES", "2"))
 HUB_AI_GLOBAL_DAILY_COST_LIMIT_MICROS = int(os.environ.get("HUB_AI_GLOBAL_DAILY_COST_LIMIT_MICROS", "0"))  # 0 = без лимита
 HUB_AI_PRICING: dict = {}  # переопределение цен micro-USD/токен по модели
+HUB_AI_EMBEDDING_MODEL = os.environ.get("HUB_AI_EMBEDDING_MODEL", "openai/text-embedding-3-small")
 
 # Password reset link lifetime. UI обещает 30 минут (default_token_generator uses this setting).
 PASSWORD_RESET_TIMEOUT = int(os.environ.get("PASSWORD_RESET_TIMEOUT", str(30 * 60)))

apps/backend/hub_platform/ai/chunking.py

@@ -0,0 +1,14 @@
+def chunk_text(text: str, *, max_chars: int = 800) -> list[str]:
+    """Split markdown into chunks by paragraphs, packing up to max_chars."""
+    paragraphs = [paragraph.strip() for paragraph in text.split("\n\n") if paragraph.strip()]
+    chunks: list[str] = []
+    current = ""
+    for paragraph in paragraphs:
+        if current and len(current) + len(paragraph) + 2 > max_chars:
+            chunks.append(current)
+            current = paragraph
+        else:
+            current = f"{current}\n\n{paragraph}" if current else paragraph
+    if current:
+        chunks.append(current)
+    return chunks

apps/backend/hub_platform/ai/doc_views.py

@@ -46,6 +46,10 @@ class _DocConfig(APIView):
             request=request,
         )
 
+    def after_publish(self, version) -> None:
+        # Хук для типоспецифичного действия после публикации версии.
+        pass
+
 
 class DocumentListCreateView(_DocConfig):
     def get(self, request: Request) -> Response:
@@ -128,6 +132,7 @@ class DocumentPublishVersionView(_DocConfig):
         except self.version_model.DoesNotExist:
             return Response({"detail": "Version not found"}, status=404)
         doc_service.publish_version(version=target)
+        self.after_publish(target)
         document = self._document(request, document_id)
         self._audit(request, "version_published", document)
         return Response({"document": self.payload(document)})
@@ -184,6 +189,11 @@ class _KnowledgeConfig:
     supports_inclusion = True
     audit_prefix = "ai.knowledge"
 
+    def after_publish(self, version) -> None:
+        from hub_platform.ai.indexing import reindex_knowledge_version
+
+        reindex_knowledge_version(version)
+
 
 class _PromptConfig:
     document_model = PromptDocument

apps/backend/hub_platform/ai/indexing.py

@@ -0,0 +1,24 @@
+from django.conf import settings
+
+from hub_platform.ai.chunking import chunk_text
+from hub_platform.ai.invocation import embed_texts
+from hub_platform.ai.models import KnowledgeDocumentVersion, KnowledgeFragment
+
+
+def reindex_knowledge_version(version: KnowledgeDocumentVersion) -> list[KnowledgeFragment]:
+    """Rebuild fragments + embeddings for a published knowledge version (ADR-HUB-0016)."""
+    KnowledgeFragment.objects.filter(version=version).delete()
+    chunks = chunk_text(version.content)
+    if not chunks:
+        return []
+    embeddings = embed_texts(
+        product=version.document.product,
+        texts=chunks,
+        model=settings.HUB_AI_EMBEDDING_MODEL,
+        purpose="knowledge_index",
+    )
+    fragments = [
+        KnowledgeFragment(version=version, chunk_index=index, content=chunk, embedding=result.vector)
+        for index, (chunk, result) in enumerate(zip(chunks, embeddings))
+    ]
+    return KnowledgeFragment.objects.bulk_create(fragments)

apps/backend/hub_platform/ai/invocation.py

@@ -12,7 +12,7 @@ from hub_platform.ai.provider.resilience import CircuitBreaker, call_with_resili
 _breaker = CircuitBreaker()
 
 
-def invoke_chat(*, product, messages: list[ChatMessage], purpose: str, release=None, model: str | None = None, params: dict | None = None) -> ChatResult:
+def invoke_chat(*, product, messages: list[ChatMessage], purpose: str, release=None, model: str | None = None, params: dict | None = None, used_fragment_ids: list | None = None) -> ChatResult:
     agent = product.ai_agent
     model = model or agent.model
 
@@ -48,6 +48,7 @@ def invoke_chat(*, product, messages: list[ChatMessage], purpose: str, release=N
         prompt_tokens=result.prompt_tokens, completion_tokens=result.completion_tokens, total_tokens=result.total_tokens,
         cost_micros=pricing.cost_micros(result.model, result.prompt_tokens, result.completion_tokens),
         latency_ms=int((time.monotonic() - started) * 1000), status=LlmInvocationStatus.SUCCESS,
+        used_fragment_ids=used_fragment_ids or [],
     )
     return result
 

apps/backend/hub_platform/ai/migrations/0006_llminvocation_used_fragment_ids_knowledgefragment.py

@@ -0,0 +1,37 @@
+# Generated by Django 5.2.15 on 2026-06-24 16:40
+
+import django.db.models.deletion
+import pgvector.django.vector
+from django.db import migrations, models
+from pgvector.django import VectorExtension
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ai', '0005_llminvocation'),
+    ]
+
+    operations = [
+        VectorExtension(),
+        migrations.AddField(
+            model_name='llminvocation',
+            name='used_fragment_ids',
+            field=models.JSONField(blank=True, default=list),
+        ),
+        migrations.CreateModel(
+            name='KnowledgeFragment',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('chunk_index', models.PositiveIntegerField()),
+                ('content', models.TextField()),
+                ('embedding', pgvector.django.vector.VectorField(blank=True, null=True)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('version', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='fragments', to='ai.knowledgedocumentversion')),
+            ],
+            options={
+                'ordering': ['version_id', 'chunk_index'],
+                'constraints': [models.UniqueConstraint(fields=('version', 'chunk_index'), name='uniq_fragment_version_chunk')],
+            },
+        ),
+    ]

apps/backend/hub_platform/ai/models.py

@@ -1,6 +1,7 @@
 from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.db import models
+from pgvector.django import VectorField
 
 # Один основной sales-агент на продукт (ADR-HUB-0007).
 DEFAULT_AI_MODEL = "openai/gpt-4o-mini"
@@ -111,6 +112,23 @@ class KnowledgeDocumentVersion(_BaseDocumentVersion):
         constraints = [models.UniqueConstraint(fields=["document", "version"], name="uniq_knowledge_version")]
 
 
+class KnowledgeFragment(models.Model):
+    # Чанк опубликованной версии знания + его эмбеддинг (pgvector). ADR-HUB-0016.
+    version = models.ForeignKey(KnowledgeDocumentVersion, on_delete=models.CASCADE, related_name="fragments")
+    chunk_index = models.PositiveIntegerField()
+    content = models.TextField()
+    # Размерность не фиксируется: совместимость локального и production embedding-провайдера.
+    embedding = VectorField(null=True, blank=True)
+    created_at = models.DateTimeField(auto_now_add=True)
+
+    class Meta:
+        ordering = ["version_id", "chunk_index"]
+        constraints = [models.UniqueConstraint(fields=["version", "chunk_index"], name="uniq_fragment_version_chunk")]
+
+    def __str__(self) -> str:
+        return f"fragment:{self.version_id}/{self.chunk_index}"
+
+
 class PromptDocumentVersion(_BaseDocumentVersion):
     document = models.ForeignKey(PromptDocument, on_delete=models.CASCADE, related_name="versions")
 
@@ -205,6 +223,7 @@ class LlmInvocation(models.Model):
     latency_ms = models.PositiveIntegerField(default=0)
     status = models.CharField(max_length=16, choices=LlmInvocationStatus.choices, default=LlmInvocationStatus.SUCCESS)
     error = models.TextField(blank=True)
+    used_fragment_ids = models.JSONField(default=list, blank=True)
     created_at = models.DateTimeField(auto_now_add=True, db_index=True)
 
     class Meta:

apps/backend/hub_platform/ai/retrieval.py

@@ -0,0 +1,48 @@
+from django.conf import settings
+from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVector
+from pgvector.django import CosineDistance
+
+from hub_platform.ai.invocation import embed_texts
+from hub_platform.ai.models import KnowledgeFragment, ProductAIRelease
+
+
+def _release_fragments(release: ProductAIRelease):
+    version_ids = release.knowledge_versions.values_list("knowledge_version_id", flat=True)
+    return KnowledgeFragment.objects.filter(version_id__in=version_ids).select_related("version__document")
+
+
+def lexical_search(release: ProductAIRelease, query: str, *, limit: int = 5) -> list[KnowledgeFragment]:
+    if not query.strip():
+        return []
+    search_query = SearchQuery(query, search_type="websearch")
+    return list(
+        _release_fragments(release)
+        .annotate(rank=SearchRank(SearchVector("content"), search_query))
+        .filter(rank__gt=0)
+        .order_by("-rank")[:limit]
+    )
+
+
+def semantic_search(release: ProductAIRelease, query_vector: list[float], *, limit: int = 5) -> list[KnowledgeFragment]:
+    return list(
+        _release_fragments(release)
+        .filter(embedding__isnull=False)
+        .order_by(CosineDistance("embedding", query_vector))[:limit]
+    )
+
+
+class KnowledgeRetriever:
+    """Hybrid retriever: semantic (pgvector) primary, lexical (Postgres FTS) complementary."""
+
+    def retrieve(self, *, release: ProductAIRelease, query: str, limit: int = 5) -> list[KnowledgeFragment]:
+        query_vector = embed_texts(
+            product=release.product,
+            texts=[query],
+            model=settings.HUB_AI_EMBEDDING_MODEL,
+            purpose="retrieval_query",
+        )[0].vector
+        semantic = semantic_search(release, query_vector, limit=limit)
+        lexical = lexical_search(release, query, limit=limit)
+        seen = {fragment.id for fragment in semantic}
+        merged = semantic + [fragment for fragment in lexical if fragment.id not in seen]
+        return merged[:limit]

apps/backend/hub_platform/ai/tests.py

@@ -387,3 +387,74 @@ class ChatInvocationTests(TestCase):
         with self.assertRaises(ai_limits.LimitExceeded):
             invoke_chat(product=self.product, messages=[ChatMessage(role="user", content="hi")], purpose="test_chat")
         self.assertTrue(LlmInvocation.objects.filter(product=self.product, status=LlmInvocationStatus.BLOCKED).exists())
+
+    def test_invocation_records_used_fragment_ids(self) -> None:
+        from hub_platform.ai.invocation import invoke_chat
+        from hub_platform.ai.models import LlmInvocation
+        from hub_platform.ai.provider.base import ChatMessage
+
+        invoke_chat(
+            product=self.product,
+            messages=[ChatMessage(role="user", content="hi")],
+            purpose="test_chat",
+            used_fragment_ids=[11, 22],
+        )
+        invocation = LlmInvocation.objects.get(product=self.product, operation="chat")
+        self.assertEqual(invocation.used_fragment_ids, [11, 22])
+
+
+class ChunkingTests(TestCase):
+    def test_packs_paragraphs_into_chunks(self) -> None:
+        from hub_platform.ai.chunking import chunk_text
+
+        text = "\n\n".join(["paragraph " + str(i) + " " + "x" * 200 for i in range(10)])
+        chunks = chunk_text(text, max_chars=500)
+        self.assertGreater(len(chunks), 1)
+        self.assertTrue(all(len(chunk) <= 700 for chunk in chunks))
+
+
+class KnowledgeRetrievalTests(TestCase):
+    def setUp(self) -> None:
+        bootstrap_edevs_owner(email="owner@edevs.tech", password="temporary-password")
+        self.client = APIClient()
+        self.client.login(username="owner@edevs.tech", password="temporary-password")
+        document_id = self.client.post(
+            "/api/v1/ai/knowledge/",
+            data=json.dumps(
+                {
+                    "product": "firepage",
+                    "code": "faq",
+                    "title": "FAQ",
+                    "category": "FAQ",
+                    "content": "Refund policy details here.\n\nDelivery and shipping information.",
+                }
+            ),
+            content_type="application/json",
+        ).json()["document"]["id"]
+        self.client.post(f"/api/v1/ai/knowledge/{document_id}/versions/1/publish/")
+
+        from hub_platform.ai import releases
+
+        self.product = Product.objects.get(code="firepage")
+        owner = HumanUser.objects.get(email="owner@edevs.tech")
+        self.release = releases.create_draft_release(product=self.product, author=owner)
+
+    def test_publish_builds_fragments_with_embeddings(self) -> None:
+        from hub_platform.ai.models import KnowledgeFragment
+
+        fragments = KnowledgeFragment.objects.filter(version__document__code="faq")
+        self.assertGreaterEqual(fragments.count(), 1)
+        self.assertTrue(all(fragment.embedding is not None for fragment in fragments))
+
+    def test_retriever_returns_release_scoped_fragments(self) -> None:
+        from hub_platform.ai.retrieval import KnowledgeRetriever
+
+        results = KnowledgeRetriever().retrieve(release=self.release, query="refund", limit=5)
+        self.assertGreaterEqual(len(results), 1)
+        self.assertLessEqual(len(results), 5)
+
+    def test_lexical_search_matches_content(self) -> None:
+        from hub_platform.ai.retrieval import lexical_search
+
+        results = lexical_search(self.release, "Delivery", limit=5)
+        self.assertTrue(any("delivery" in fragment.content.lower() for fragment in results))

apps/backend/requirements.txt

@@ -1,6 +1,7 @@
 Django>=5.2,<5.3
 djangorestframework>=3.16,<3.17
 cryptography>=43,<45
+pgvector>=0.3,<0.4
 psycopg[binary]>=3.2,<3.3
 redis>=5.2,<5.3
 uvicorn[standard]>=0.34,<0.35

compose.yaml

@@ -5,7 +5,7 @@ x-hub-env: &hub-env
 
 services:
   postgres:
-    image: postgres:16-alpine
+    image: pgvector/pgvector:pg16
     environment:
       POSTGRES_DB: ${POSTGRES_DB:-edevs_hub}
       POSTGRES_USER: ${POSTGRES_USER:-edevs_hub}