diff --git a/.claude/launch.json b/.claude/launch.json deleted file mode 100644 index f42f358..0000000 --- a/.claude/launch.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "version": "0.0.1", - "configurations": [ - { - "name": "internal-ui", - "runtimeExecutable": "npm", - "runtimeArgs": ["run", "dev", "--workspace", "@chatballs/internal-ui", "--", "--host", "127.0.0.1"], - "cwd": ".", - "port": 5173 - } - ] -} diff --git a/.gitattributes b/.gitattributes index 1941046..b16c3c0 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5,6 +5,21 @@ *.sh text eol=lf chatballs text eol=lf +# Python читают и переписывают линуксовые инструменты (ruff в контейнере, CI). +# На CRLF-чекауте ruff принимал за перевод строки одиночный возврат каретки и +# писал его в вывод: файл оставался рабочим, но обрастал лишними CR. Двадцать +# один такой файл успел доехать до репозитория. +*.py text eol=lf + +# Dockerfile и манифесты стека читают Linux-инструменты: CR в них ломает +# RUN-строки и heredoc'и ровно так же, как шебанг. +Dockerfile text eol=lf +Dockerfile.* text eol=lf +*.Dockerfile text eol=lf +compose*.yaml text eol=lf +Caddyfile text eol=lf +*.sql text eol=lf + *.bat text eol=crlf *.cmd text eol=crlf *.ps1 text eol=crlf diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 0000000..15bbb6b --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,60 @@ +# Быстрые проверки на каждый push и pull request. +# +# До этого на GitHub был только релизный workflow по тегу, то есть до первого +# тега ничего не проверялось вовсе. Здесь — то, что стоит секунды и ловит +# ошибки раньше человека: линтер бэкенда и тесты deployment CLI (они держат +# свойство «установка — один compose.yaml без host-mount'ов»). +# +# Бэкенд-сьют и e2e сюда не вынесены намеренно: им нужны Postgres, Redis и +# полчаса, их место — отдельный пайплайн, а не проверка на каждый коммит. + +name: checks + +on: + push: + branches: ["**"] + pull_request: + +permissions: + contents: read + +jobs: + lint: + name: ruff + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Установить ruff той же версии, что и в образе + run: | + set -eu + # Единственный источник версии — requirements бэкенда, чтобы CI и + # контейнер не разъезжались. + constraint="$(grep -E '^ruff' apps/backend/requirements.txt)" + python -m pip install --disable-pip-version-check "$constraint" + ruff --version + + # Конфигурация линтера лежит в корневом pyproject.toml (src = apps/backend). + - name: ruff check + run: ruff check --output-format=github apps/backend + + cli: + name: deployment CLI + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - run: python -m pip install --disable-pip-version-check pytest pyyaml + + # Тесты гоняют настоящий bash-скрипт chatballs с замоканными docker и + # flock, плюс проверяют, что production-манифест остаётся + # самодостаточным (tests/cli/test_release_compose.py). + - run: python -m pytest -q tests/cli diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 751754d..69bcbae 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,9 +1,16 @@ -# Публикация образов Chatballs в GitHub Container Registry. +# Публикация Chatballs: образы в GitHub Container Registry + релизный compose.yaml. # -# Смысл: человек не должен собирать продукт, чтобы его запустить. По тегу vX.Y.Z -# workflow собирает backend и frontend, публикует их в ghcr.io и прикладывает к -# релизу release.env — файл со ссылками на образы по digest. С ним запуск -# сводится к «docker compose up» без единой сборки. +# Смысл: человек не должен ни собирать продукт, ни распаковывать бандл, чтобы +# его запустить. По тегу vX.Y.Z workflow собирает четыре образа (backend, +# frontend, gateway с Caddyfile внутри, postgres с init-скриптами внутри), +# публикует их в ghcr.io и прикладывает к релизу один файл — compose.yaml с +# ссылками по digest. Установка на чистый хост: +# +# curl -fsSL <ссылка на compose.yaml> -o compose.yaml +# docker compose up -d --wait +# +# release.env кладётся рядом для тех, кто разворачивает через `chatballs` +# (закрытый контур): там нужны те же digest'ы отдельным файлом. # # Ничего настраивать не нужно: путь образов выводится из github.repository, # публикация идёт встроенным GITHUB_TOKEN. @@ -28,9 +35,9 @@ env: APP_DIR: . REGISTRY: ghcr.io # Сторонние образы пинуются тем же способом, что и свои: по digest. - POSTGRES_IMAGE: pgvector/pgvector:pg16 + POSTGRES_BASE_IMAGE: pgvector/pgvector:pg16 + GATEWAY_BASE_IMAGE: caddy:2.8.4 REDIS_IMAGE: redis:7-alpine - GATEWAY_IMAGE: caddy:2.8.4 COTURN_IMAGE: coturn/coturn:4.6 jobs: @@ -58,6 +65,20 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + # Базовые образы закрепляем до сборки: тогда digest наших образов + # однозначно отвечает известному входу, а не плавающему тегу. + - name: Digest базовых образов + id: bases + run: | + set -eu + digest_of() { + docker buildx imagetools inspect "$1" --format '{{ "{{json .Manifest.Digest}}" }}' | tr -d '"' + } + echo "postgres=${POSTGRES_BASE_IMAGE}@$(digest_of "$POSTGRES_BASE_IMAGE")" >> "$GITHUB_OUTPUT" + echo "gateway=${GATEWAY_BASE_IMAGE}@$(digest_of "$GATEWAY_BASE_IMAGE")" >> "$GITHUB_OUTPUT" + echo "redis=${REDIS_IMAGE}@$(digest_of "$REDIS_IMAGE")" >> "$GITHUB_OUTPUT" + echo "coturn=${COTURN_IMAGE}@$(digest_of "$COTURN_IMAGE")" >> "$GITHUB_OUTPUT" + - name: Backend id: backend uses: docker/build-push-action@v6 @@ -84,40 +105,92 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max - - name: release.env со ссылками по digest + # Шлюз и база — свои образы: Caddyfile и init-скрипты живут внутри них, + # а не монтируются с хоста. Ради этого установка и стала одним файлом. + - name: Gateway + id: gateway + uses: docker/build-push-action@v6 + with: + context: ${{ env.APP_DIR }} + file: ${{ env.APP_DIR }}/deploy/docker/gateway.Dockerfile + build-args: | + CHATBALLS_GATEWAY_BASE_IMAGE=${{ steps.bases.outputs.gateway }} + push: true + tags: | + ${{ env.REGISTRY }}/${{ steps.version.outputs.repo }}/gateway:${{ steps.version.outputs.value }} + ${{ env.REGISTRY }}/${{ steps.version.outputs.repo }}/gateway:latest + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Postgres + id: postgres + uses: docker/build-push-action@v6 + with: + context: ${{ env.APP_DIR }} + file: ${{ env.APP_DIR }}/deploy/docker/postgres.Dockerfile + build-args: | + CHATBALLS_POSTGRES_BASE_IMAGE=${{ steps.bases.outputs.postgres }} + push: true + tags: | + ${{ env.REGISTRY }}/${{ steps.version.outputs.repo }}/postgres:${{ steps.version.outputs.value }} + ${{ env.REGISTRY }}/${{ steps.version.outputs.repo }}/postgres:latest + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Релизный compose.yaml и release.env + id: artifacts run: | set -eu version="${{ steps.version.outputs.value }}" - repo="${{ steps.version.outputs.repo }}" - prefix="${{ env.REGISTRY }}/$repo" + prefix="${{ env.REGISTRY }}/${{ steps.version.outputs.repo }}" - # Digest своих образов отдаёт сам build-push-action; сторонние - # спрашиваем у реестра — :latest в релизе недопустим. - third_party_digest() { - docker buildx imagetools inspect "$1" --format '{{ "{{json .Manifest.Digest}}" }}' | tr -d '"' - } + backend="$prefix/backend:$version@${{ steps.backend.outputs.digest }}" + frontend="$prefix/frontend:$version@${{ steps.frontend.outputs.digest }}" + gateway="$prefix/gateway:$version@${{ steps.gateway.outputs.digest }}" + postgres="$prefix/postgres:$version@${{ steps.postgres.outputs.digest }}" + redis="${{ steps.bases.outputs.redis }}" + coturn="${{ steps.bases.outputs.coturn }}" + + python3 scripts/pin-release-compose.py \ + --source compose.yaml \ + --output dist/compose.yaml \ + --version "$version" \ + --pin "CHATBALLS_BACKEND_IMAGE=$backend" \ + --pin "CHATBALLS_FRONTEND_IMAGE=$frontend" \ + --pin "CHATBALLS_GATEWAY_IMAGE=$gateway" \ + --pin "CHATBALLS_POSTGRES_IMAGE=$postgres" \ + --pin "CHATBALLS_REDIS_IMAGE=$redis" \ + --pin "CHATBALLS_COTURN_IMAGE=$coturn" { - echo "# release.env — сгенерирован ${{ github.workflow }} для $version." - echo "# Все образы закреплены по digest: :latest источником релиза не является." + echo "# release.env — digest-пины релиза $version для \`chatballs deploy\`." + echo "# Тем, кто ставит одной командой, он не нужен: всё уже внутри compose.yaml." echo "CHATBALLS_VERSION=$version" - echo "CHATBALLS_BACKEND_IMAGE=$prefix/backend:$version@${{ steps.backend.outputs.digest }}" - echo "CHATBALLS_FRONTEND_IMAGE=$prefix/frontend:$version@${{ steps.frontend.outputs.digest }}" - echo "CHATBALLS_POSTGRES_IMAGE=${POSTGRES_IMAGE}@$(third_party_digest "$POSTGRES_IMAGE")" - echo "CHATBALLS_REDIS_IMAGE=${REDIS_IMAGE}@$(third_party_digest "$REDIS_IMAGE")" - echo "CHATBALLS_GATEWAY_IMAGE=${GATEWAY_IMAGE}@$(third_party_digest "$GATEWAY_IMAGE")" - echo "CHATBALLS_COTURN_IMAGE=${COTURN_IMAGE}@$(third_party_digest "$COTURN_IMAGE")" - } > release.env + echo "CHATBALLS_BACKEND_IMAGE=$backend" + echo "CHATBALLS_FRONTEND_IMAGE=$frontend" + echo "CHATBALLS_GATEWAY_IMAGE=$gateway" + echo "CHATBALLS_POSTGRES_IMAGE=$postgres" + echo "CHATBALLS_REDIS_IMAGE=$redis" + echo "CHATBALLS_COTURN_IMAGE=$coturn" + } > dist/release.env - cat release.env + # Файл, который скачает человек, обязан быть валидным сам по себе — + # без переменных окружения и без чего-либо рядом. + ( cd dist && docker compose -f compose.yaml config -q ) + + cat dist/compose.yaml - uses: actions/upload-artifact@v4 with: - name: release.env - path: release.env + name: release-compose + path: | + dist/compose.yaml + dist/release.env - - name: Приложить release.env к релизу + - name: Приложить к релизу if: startsWith(github.ref, 'refs/tags/') uses: softprops/action-gh-release@v2 with: - files: release.env + files: | + dist/compose.yaml + dist/release.env diff --git a/.gitignore b/.gitignore index 9ca6802..6596a18 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,6 @@ data/ # Архив, создаваемый pack.bat Chatballs.zip + +# Служебный каталог агента: локальные настройки запуска, а не часть продукта. +.claude/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index a928ddf..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,29 +0,0 @@ -# GitLab CI/CD — canonical pipeline Chatballs (ADR-HUB-0028 / PLAN-CHATBALLS-0002). -# Application orchestration is owned only by the release-bundled `chatballs` CLI. - -stages: - - validate - - test - - build - - release - - deploy - -variables: - APP_DIR: "." - DOCKER_TLS_CERTDIR: "" - DOCKER_DRIVER: overlay2 - - BACKEND_IMAGE: "$CI_REGISTRY_IMAGE/backend:$CI_COMMIT_SHA" - FRONTEND_IMAGE: "$CI_REGISTRY_IMAGE/frontend:$CI_COMMIT_SHA" - BACKEND_LATEST_IMAGE: "$CI_REGISTRY_IMAGE/backend:latest" - FRONTEND_LATEST_IMAGE: "$CI_REGISTRY_IMAGE/frontend:latest" - - POSTGRES_IMAGE: "pgvector/pgvector:pg16" - REDIS_IMAGE: "redis:7-alpine" - GATEWAY_IMAGE: "caddy:2.8.4" - COTURN_IMAGE: "coturn/coturn:4.6" - -include: - - local: "/.gitlab/ci/validate-test.yml" - - local: "/.gitlab/ci/build-release.yml" - - local: "/.gitlab/ci/deploy.yml" diff --git a/.gitlab/ci/build-release.yml b/.gitlab/ci/build-release.yml deleted file mode 100644 index 8779c1d..0000000 --- a/.gitlab/ci/build-release.yml +++ /dev/null @@ -1,125 +0,0 @@ -images:build: - stage: build - image: docker:27 - services: - - name: docker:27-dind - alias: docker - needs: - - backend:test - - frontend:build - - gateway:validate - - cli:test - before_script: - - echo "$CI_REGISTRY_PASSWORD" | docker login "$CI_REGISTRY" -u "$CI_REGISTRY_USER" --password-stdin - - docker buildx version - script: - - | - set -eu - VERSION="${CI_COMMIT_TAG:-$CI_COMMIT_SHA}" - case "$VERSION" in - *[!A-Za-z0-9._-]*|'') - echo "Unsupported release version: $VERSION" >&2 - exit 1 - ;; - esac - - docker build \ - -f "$APP_DIR/apps/backend/Dockerfile.production" \ - -t "$BACKEND_IMAGE" \ - -t "$BACKEND_LATEST_IMAGE" \ - "$APP_DIR" - docker build \ - -f "$APP_DIR/deploy/docker/frontend.Dockerfile" \ - -t "$FRONTEND_IMAGE" \ - -t "$FRONTEND_LATEST_IMAGE" \ - "$APP_DIR" - - docker push "$BACKEND_IMAGE" - docker push "$FRONTEND_IMAGE" - - if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then - docker push "$BACKEND_LATEST_IMAGE" - docker push "$FRONTEND_LATEST_IMAGE" - fi - - resolve_digest_ref() { - image="$1" - manifest="$(mktemp)" - docker buildx imagetools inspect --raw "$image" > "$manifest" - test -s "$manifest" - digest="sha256:$(sha256sum "$manifest" | awk '{print $1}')" - rm -f "$manifest" - printf '%s@%s' "$image" "$digest" - } - - mkdir -p "$APP_DIR/.ci" - cat > "$APP_DIR/.ci/release.env" < checksums.txt - sha256sum -c checksums.txt - ) - - tar -czf "$DIST_DIR/chatballs-release.tar.gz" "$BUNDLE_DIR" - printf 'CHATBALLS_RELEASE_VERSION=%s\n' "$VERSION" > "$DIST_DIR/release-metadata.env" - printf 'CHATBALLS_RELEASE_ARCHIVE=%s/dist/chatballs-release.tar.gz\n' "$APP_DIR" >> "$DIST_DIR/release-metadata.env" - artifacts: - name: "chatballs-$CI_COMMIT_REF_SLUG" - paths: - - "$APP_DIR/dist/chatballs-release.tar.gz" - - "$APP_DIR/dist/release-metadata.env" - reports: - dotenv: "$APP_DIR/dist/release-metadata.env" - expire_in: 1 year - when: on_success - rules: - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' - - if: '$CI_COMMIT_TAG' diff --git a/.gitlab/ci/deploy.yml b/.gitlab/ci/deploy.yml deleted file mode 100644 index 869a668..0000000 --- a/.gitlab/ci/deploy.yml +++ /dev/null @@ -1,129 +0,0 @@ -deploy:production: - stage: deploy - image: alpine:3.20 - needs: - - job: release:bundle - artifacts: true - resource_group: production - interruptible: false - variables: - DEPLOY_ROOT: "/opt/chatballs" - before_script: - - apk add --no-cache bash openssh-client - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh - - printf '%s\n' "$DEPLOY_SSH_PRIVATE_KEY" > ~/.ssh/id_ed25519 - - chmod 600 ~/.ssh/id_ed25519 - - ssh-keyscan -p "${DEPLOY_PORT:-22}" "$DEPLOY_HOST" >> ~/.ssh/known_hosts - script: - - | - set -eu - REMOTE_ARCHIVE="/tmp/chatballs-${CI_PIPELINE_ID}-${CI_JOB_ID}.tar.gz" - scp -P "${DEPLOY_PORT:-22}" \ - "$CHATBALLS_RELEASE_ARCHIVE" \ - "$DEPLOY_USER@$DEPLOY_HOST:$REMOTE_ARCHIVE" - - ssh -p "${DEPLOY_PORT:-22}" "$DEPLOY_USER@$DEPLOY_HOST" \ - bash -s -- "$DEPLOY_ROOT" "$CHATBALLS_RELEASE_VERSION" "$REMOTE_ARCHIVE" <<'REMOTE' - set -Eeuo pipefail - - root="$1" - version="$2" - archive="$3" - release_name="chatballs-$version" - target="$root/releases/$release_name" - incoming="" - - cleanup() { - if [[ -n "$incoming" ]]; then - rm -rf "$incoming" - fi - rm -f "$archive" - } - trap cleanup EXIT - - sudo mkdir -p \ - "$root/releases" \ - "$root/instance/state" \ - "$root/instance/data" \ - "$root/instance/backups" \ - "$root/instance/logs" - sudo chown "$(id -u):$(id -g)" \ - "$root" \ - "$root/releases" \ - "$root/instance" \ - "$root/instance/state" \ - "$root/instance/data" \ - "$root/instance/backups" \ - "$root/instance/logs" - - incoming="$(mktemp -d "$root/releases/.incoming-${release_name}.XXXXXX")" - tar -xzf "$archive" -C "$incoming" - source_dir="$incoming/$release_name" - - [[ -d "$source_dir" ]] || { - echo "Release directory is missing in bundle: $release_name" >&2 - exit 1 - } - [[ -f "$source_dir/checksums.txt" ]] || { - echo "checksums.txt is missing in release bundle" >&2 - exit 1 - } - ( - cd "$source_dir" - sha256sum -c checksums.txt - ) - - if [[ -e "$target" ]]; then - [[ -f "$target/checksums.txt" ]] || { - echo "Existing release has no checksums: $target" >&2 - exit 1 - } - ( - cd "$target" - sha256sum -c checksums.txt - ) - incoming_manifest="$(sha256sum "$source_dir/checksums.txt" | awk '{print $1}')" - target_manifest="$(sha256sum "$target/checksums.txt" | awk '{print $1}')" - [[ "$incoming_manifest" = "$target_manifest" ]] || { - echo "Release $version already exists with different content" >&2 - exit 1 - } - else - mv "$source_dir" "$target" - fi - - chmod +x "$target/chatballs" - REMOTE - - - | - set -eu - printf '%s' "$CI_REGISTRY_PASSWORD" \ - | ssh -p "${DEPLOY_PORT:-22}" "$DEPLOY_USER@$DEPLOY_HOST" \ - "docker login '$CI_REGISTRY' -u '$CI_REGISTRY_USER' --password-stdin" - - - | - set -eu - ssh -p "${DEPLOY_PORT:-22}" "$DEPLOY_USER@$DEPLOY_HOST" \ - bash -s -- "$DEPLOY_ROOT" "$CHATBALLS_RELEASE_VERSION" <<'REMOTE' - set -Eeuo pipefail - - root="$1" - version="$2" - release_dir="$root/releases/chatballs-$version" - - CHATBALLS_INSTANCE_DIR="$root/instance" \ - "$release_dir/chatballs" deploy --non-interactive - - ln -sfnT "releases/chatballs-$version" "$root/current" - REMOTE - after_script: - - | - ssh -p "${DEPLOY_PORT:-22}" "$DEPLOY_USER@$DEPLOY_HOST" \ - "docker logout '$CI_REGISTRY' >/dev/null 2>&1 || true" || true - environment: - name: production - url: https://${CHATBALLS_APP_DOMAIN} - rules: - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' - when: manual diff --git a/.gitlab/ci/validate-test.yml b/.gitlab/ci/validate-test.yml deleted file mode 100644 index 415d0b0..0000000 --- a/.gitlab/ci/validate-test.yml +++ /dev/null @@ -1,86 +0,0 @@ -frontend:build: - stage: validate - image: node:22-alpine - before_script: - - cd "$APP_DIR" - - npm ci - script: - - npm run typecheck - - npm run build - rules: - - if: '$CI_COMMIT_BRANCH' - - if: '$CI_COMMIT_TAG' - -gateway:validate: - stage: validate - image: docker:27 - services: - - name: docker:27-dind - alias: docker - before_script: - - mkdir -p "$APP_DIR/.ci-instance/data/caddy" "$APP_DIR/.ci-instance/data/caddy-config" - script: - - | - set -eu - # Никакого .env: продукт поднимается со значениями по умолчанию, и - # проверять его надо ровно так же, как он ставится у человека. - export CHATBALLS_INSTANCE_DIR="$CI_PROJECT_DIR/$APP_DIR/.ci-instance" - export CHATBALLS_RELEASE_DIR="$CI_PROJECT_DIR/$APP_DIR" - - docker compose \ - --project-directory "$CHATBALLS_INSTANCE_DIR" \ - -f "$APP_DIR/compose.yaml" \ - config -q - - docker compose \ - --project-directory "$CHATBALLS_INSTANCE_DIR" \ - -f "$APP_DIR/compose.yaml" \ - run --rm --no-deps gateway \ - caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile - rules: - - if: '$CI_COMMIT_BRANCH' - - if: '$CI_COMMIT_TAG' - -backend:test: - stage: test - image: python:3.12-slim - services: - - name: pgvector/pgvector:pg16 - alias: postgres - - name: redis:7-alpine - alias: redis - variables: - CHATBALLS_DEBUG: "true" - CHATBALLS_SECRET_KEY: "ci-test-secret" - CHATBALLS_FIELD_ENCRYPTION_KEY: "" - POSTGRES_DB: "chatballs_test" - POSTGRES_USER: "chatballs" - POSTGRES_PASSWORD: "chatballs" - POSTGRES_HOST: "postgres" - POSTGRES_PORT: "5432" - REDIS_URL: "redis://redis:6379/0" - before_script: - - cd "$APP_DIR/apps/backend" - - python -m pip install --upgrade pip - - pip install -r requirements.txt - script: - - python manage.py check - - pytest - rules: - - if: '$CI_COMMIT_BRANCH' - - if: '$CI_COMMIT_TAG' - -cli:test: - stage: test - image: python:3.12-slim - before_script: - - apt-get update - - apt-get install -y --no-install-recommends bash coreutils util-linux - - rm -rf /var/lib/apt/lists/* - - python -m pip install --no-cache-dir pytest - script: - - cd "$APP_DIR" - - pytest -c tests/cli/pytest.ini tests/cli - rules: - - if: '$CI_COMMIT_BRANCH' - - if: '$CI_COMMIT_TAG' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 63cfddd..1345cb6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,12 @@ repos: + # Тот же линтер и та же конфигурация, что в CI (.github/workflows/checks.yml). + # ruff-format здесь намеренно нет: он переписал бы 374 файла разом, и это + # отдельное решение, а не побочный эффект установки хука. - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.12.0 hooks: - id: ruff args: ["--fix"] - - id: ruff-format - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: diff --git a/README.md b/README.md index 2f11e20..062f7c7 100644 --- a/README.md +++ b/README.md @@ -2,51 +2,71 @@ Canonical implementation workspace for Chatballs. -## Быстрый старт (одна минута) +## Установка (одна команда) -Нужен только Docker (Docker Desktop на Windows/macOS или Docker Engine с Compose -на Linux). Ни одной переменной задавать не нужно и негде: у продукта нет `.env`. -Организацию, владельца, домены, интеграции, почту и хранилище человек настраивает -в интерфейсе. +Нужен только Docker с плагином Compose. Ни одной переменной задавать не нужно и +негде: у продукта нет `.env`. Организацию, владельца, домены, интеграции, почту и +хранилище человек настраивает в интерфейсе. + +Весь дистрибутив — один файл `compose.yaml` со страницы релиза: ссылки на образы +в нём закреплены по digest, а Caddyfile, init-скрипты базы и генератор секретов +лежат внутри образов. Рядом с файлом ничего лежать не должно. + +Linux / macOS: + +```bash +curl -fsSL https://github.com/dartdavros/chatballs/releases/latest/download/compose.yaml -o compose.yaml +docker compose up -d --wait +``` Windows (PowerShell): +```powershell +curl.exe -fsSL https://github.com/dartdavros/chatballs/releases/latest/download/compose.yaml -o compose.yaml +docker compose up -d --wait +``` + +`--wait` держит команду до готовности стека: когда она вернула управление, +установка отвечает. Откройте **http://localhost** (или адрес сервера) — вместо +входа система покажет **мастер первого запуска**: название организации, ваше имя, +e-mail и пароль владельца, переключатель «Установить демо-данные». После кнопки +«Начать» вы сразу в приложении под владельцем. Мастер доступен только пока в +системе нет ни одной организации; после создания владельца он закрывается +навсегда. + +Секреты инстанса (ключ подписи, пароли ролей БД) генерирует сам первый старт и +держит в томе `chatballs-secrets`. Состояние установки живёт в именованных томах +`chatballs-*` — установка не зависит от того, из какого каталога её запустили. + +Обновление — тот же файл новой версии и та же команда: + +```bash +curl -fsSL https://github.com/dartdavros/chatballs/releases/latest/download/compose.yaml -o compose.yaml +docker compose up -d --wait +``` + +Миграции прогоняет one-shot сервис `init` на каждом старте. Откат — прежняя +копия `compose.yaml` и снова та же команда. + +### Запуск из исходников (разработка) + +Тем, кто правит код, релизный файл не нужен: стек собирается локально. + ```powershell git clone chatballs cd chatballs .\scripts\start.ps1 ``` -Linux / macOS: - ```bash git clone chatballs cd chatballs ./scripts/start.sh ``` -Настраивать нечего: файла `.env` у продукта нет, секреты инстанса (ключ -подписи, пароли ролей БД) генерирует сам первый старт и держит их в томе -`chatballs-secrets`. - -Если положить рядом со скриптом `release.env` со страницы релиза, образы -скачаются из реестра по digest и сборки не будет — это самый быстрый путь. -Без `release.env` стек собирается из исходников: так работают те, кто правит -код. Когда в логах появится готовность, откройте -**http://localhost** — вместо входа система покажет **мастер первого запуска**: -название организации, ваше имя, e-mail и пароль владельца, переключатель -«Установить демо-данные». После кнопки «Начать» вы сразу в приложении под -владельцем. Мастер доступен только пока в системе нет ни одной организации; -после создания владельца он закрывается навсегда. - -### Доступ по http и переход на TLS - -Свежая установка отвечает по обычному http — по адресу сервера, пока домена и -сертификата ещё нет. Продукт не уводит себя на https принудительно: этим -занимается шлюз, когда у него появляется настоящий домен и сертификат. -Жёсткость транспорта включается сама по факту TLS: запрос пришёл по https — -cookie получают префикс `__Host-`, флаг `Secure` и HSTS; по http — обычные -имена без `Secure`. Настраивать для этого нечего. +Первая сборка занимает минуты (`npm ci` + `pip install`). Dev-контур держит +состояние в `./data`, публикует порты Postgres/Redis и подменяет frontend на +Vite с HMR — см. `compose.dev.yaml`. ### Демо-данные @@ -79,7 +99,11 @@ docker compose run --rm backend-app python manage.py seed_demo --organization Response: try: diff --git a/apps/backend/chatballs/ai/indexing.py b/apps/backend/chatballs/ai/indexing.py index 0292e3d..eccd75c 100644 --- a/apps/backend/chatballs/ai/indexing.py +++ b/apps/backend/chatballs/ai/indexing.py @@ -30,7 +30,7 @@ def _store_fragments(*, organization, chunks: list[str], **source) -> list[Knowl embedding=vector, **source, ) - for index, (chunk, vector) in enumerate(zip(chunks, vectors)) + for index, (chunk, vector) in enumerate(zip(chunks, vectors, strict=False)) ] return KnowledgeFragment.objects.bulk_create(fragments) diff --git a/apps/backend/chatballs/ai/knowledge_policy.py b/apps/backend/chatballs/ai/knowledge_policy.py index 70b250d..4f77e9a 100644 --- a/apps/backend/chatballs/ai/knowledge_policy.py +++ b/apps/backend/chatballs/ai/knowledge_policy.py @@ -3,8 +3,8 @@ from __future__ import annotations from django.core.exceptions import PermissionDenied from django.db.models import QuerySet -from chatballs.identity.policy import has_capability_any_scope from chatballs.ai.models import AIAgent, Knowledge +from chatballs.identity.policy import has_capability_any_scope from chatballs.tenancy.context import TenantContext AI_VIEW = "ai.view" diff --git a/apps/backend/chatballs/ai/limits.py b/apps/backend/chatballs/ai/limits.py index 045fcd2..7853692 100644 --- a/apps/backend/chatballs/ai/limits.py +++ b/apps/backend/chatballs/ai/limits.py @@ -1,32 +1,32 @@ -from django.conf import settings -from django.db.models import Sum -from django.utils import timezone - -from chatballs.ai.models import LlmInvocation, LlmInvocationStatus - - -class LimitExceeded(Exception): - pass - - -def _day_start(): - now = timezone.localtime() - return now.replace(hour=0, minute=0, second=0, microsecond=0) - - -def daily_cost_micros(channel=None) -> int: - queryset = LlmInvocation.objects.filter(created_at__gte=_day_start(), status=LlmInvocationStatus.SUCCESS) - if channel is not None: - queryset = queryset.filter(channel=channel) - return queryset.aggregate(total=Sum("cost_micros"))["total"] or 0 - - -def assert_within_limits(channel, agent) -> None: - global_limit = settings.CHATBALLS_AI_GLOBAL_DAILY_COST_LIMIT_MICROS - if global_limit and daily_cost_micros() >= global_limit: - raise LimitExceeded("Global daily AI cost limit reached") - # Канальный лимит хранится в целых центах USD (dailyCostUsd); расход учитывается - # в micro-USD. 1 цент = 10 000 micro-USD. - channel_limit = (agent.limits or {}).get("dailyCostUsd") - if channel_limit and daily_cost_micros(channel) >= int(channel_limit) * 10_000: - raise LimitExceeded("Channel daily AI cost limit reached") +from django.conf import settings +from django.db.models import Sum +from django.utils import timezone + +from chatballs.ai.models import LlmInvocation, LlmInvocationStatus + + +class LimitExceeded(Exception): + pass + + +def _day_start(): + now = timezone.localtime() + return now.replace(hour=0, minute=0, second=0, microsecond=0) + + +def daily_cost_micros(channel=None) -> int: + queryset = LlmInvocation.objects.filter(created_at__gte=_day_start(), status=LlmInvocationStatus.SUCCESS) + if channel is not None: + queryset = queryset.filter(channel=channel) + return queryset.aggregate(total=Sum("cost_micros"))["total"] or 0 + + +def assert_within_limits(channel, agent) -> None: + global_limit = settings.CHATBALLS_AI_GLOBAL_DAILY_COST_LIMIT_MICROS + if global_limit and daily_cost_micros() >= global_limit: + raise LimitExceeded("Global daily AI cost limit reached") + # Канальный лимит хранится в целых центах USD (dailyCostUsd); расход учитывается + # в micro-USD. 1 цент = 10 000 micro-USD. + channel_limit = (agent.limits or {}).get("dailyCostUsd") + if channel_limit and daily_cost_micros(channel) >= int(channel_limit) * 10_000: + raise LimitExceeded("Channel daily AI cost limit reached") diff --git a/apps/backend/chatballs/ai/models.py b/apps/backend/chatballs/ai/models.py index 5c7fae9..7b0b2da 100644 --- a/apps/backend/chatballs/ai/models.py +++ b/apps/backend/chatballs/ai/models.py @@ -1,19 +1,11 @@ import uuid - - from django.core.exceptions import ValidationError - from django.db import models - from pgvector.django import VectorField - - from chatballs.tenancy.models import TenantRelationModel - - # Один основной агент на канал обработки (ADR-HUB-0019, ADR-CHATBALLS-0023). DEFAULT_AI_MODEL = "anthropic/claude-sonnet-4.6" @@ -88,13 +80,9 @@ class Knowledge(models.Model): - def clean(self) -> None: + def __str__(self) -> str: - super().clean() - - if self.category_id is not None and self.category.organization_id != self.organization_id: - - raise ValidationError({"category": "Category belongs to another organization"}) + return f"knowledge:{self.organization_id}/{self.title}" @@ -106,9 +94,13 @@ class Knowledge(models.Model): - def __str__(self) -> str: + def clean(self) -> None: - return f"knowledge:{self.organization_id}/{self.title}" + super().clean() + + if self.category_id is not None and self.category.organization_id != self.organization_id: + + raise ValidationError({"category": "Category belongs to another organization"}) @@ -119,15 +111,10 @@ class Knowledge(models.Model): # models after Knowledge exists so they are registered without growing this file. from chatballs.ai.knowledge_models import ( # noqa: E402, F401 - KnowledgeCategory, - ) - - - def attachment_upload_path(instance: "KnowledgeAttachment", filename: str) -> str: organization = instance.knowledge.organization @@ -198,8 +185,6 @@ class KnowledgeAttachment(TenantRelationModel): from django.urls import reverse - - from chatballs.identity.instance_settings import public_base_url diff --git a/apps/backend/chatballs/ai/pricing.py b/apps/backend/chatballs/ai/pricing.py index ac92a5a..e100291 100644 --- a/apps/backend/chatballs/ai/pricing.py +++ b/apps/backend/chatballs/ai/pricing.py @@ -1,17 +1,17 @@ -from django.conf import settings - -# micro-USD за токен (1 USD = 1_000_000 micro); значение = цена в USD за 1M токенов. -# Fallback на случай, если провайдер не вернул фактическую стоимость (usage.cost). -# Реальные/уточнённые цены задаются через CHATBALLS_AI_PRICING. -DEFAULT_PRICING = { - "openai/gpt-4o-mini": {"prompt": 0.15, "completion": 0.60}, - "anthropic/claude-sonnet-4.6": {"prompt": 3.0, "completion": 15.0}, -} - - -def cost_micros(model: str, prompt_tokens: int, completion_tokens: int) -> int: - table = {**DEFAULT_PRICING, **getattr(settings, "CHATBALLS_AI_PRICING", {})} - price = table.get(model) - if not price: - return 0 - return round(prompt_tokens * price["prompt"] + completion_tokens * price["completion"]) +from django.conf import settings + +# micro-USD за токен (1 USD = 1_000_000 micro); значение = цена в USD за 1M токенов. +# Fallback на случай, если провайдер не вернул фактическую стоимость (usage.cost). +# Реальные/уточнённые цены задаются через CHATBALLS_AI_PRICING. +DEFAULT_PRICING = { + "openai/gpt-4o-mini": {"prompt": 0.15, "completion": 0.60}, + "anthropic/claude-sonnet-4.6": {"prompt": 3.0, "completion": 15.0}, +} + + +def cost_micros(model: str, prompt_tokens: int, completion_tokens: int) -> int: + table = {**DEFAULT_PRICING, **getattr(settings, "CHATBALLS_AI_PRICING", {})} + price = table.get(model) + if not price: + return 0 + return round(prompt_tokens * price["prompt"] + completion_tokens * price["completion"]) diff --git a/apps/backend/chatballs/ai/provider/custom.py b/apps/backend/chatballs/ai/provider/custom.py index e737a8c..96dd90a 100644 --- a/apps/backend/chatballs/ai/provider/custom.py +++ b/apps/backend/chatballs/ai/provider/custom.py @@ -1,9 +1,6 @@ from chatballs.ai.provider.openrouter import OpenRouterProvider - - - class CustomProvider(OpenRouterProvider): """Generic OpenAI-compatible BYOK adapter (ADR-CHATBALLS-0034). diff --git a/apps/backend/chatballs/ai/provider/demo.py b/apps/backend/chatballs/ai/provider/demo.py index 4fb892e..648207c 100644 --- a/apps/backend/chatballs/ai/provider/demo.py +++ b/apps/backend/chatballs/ai/provider/demo.py @@ -15,7 +15,13 @@ import hashlib import math import re -from chatballs.ai.provider.base import ChatMessage, ChatResult, EmbeddingResult, LLMProvider, ProviderError +from chatballs.ai.provider.base import ( + ChatMessage, + ChatResult, + EmbeddingResult, + LLMProvider, + ProviderError, +) EMBEDDING_DIM = 16 KNOWLEDGE_MARKER = "Отвечай только на основе этих знаний:" diff --git a/apps/backend/chatballs/ai/provider/openai_http.py b/apps/backend/chatballs/ai/provider/openai_http.py index 4a0b2a1..358de71 100644 --- a/apps/backend/chatballs/ai/provider/openai_http.py +++ b/apps/backend/chatballs/ai/provider/openai_http.py @@ -32,26 +32,15 @@ do not duplicate it. Adapters stay responsible for their own product semantics from __future__ import annotations - - import http.client - import json - import urllib.error - import urllib.request - - from chatballs.ai.provider.base import ChatMessage, ChatResult, EmbeddingResult, ProviderError - from chatballs.integrations.proxy import build_opener - - - def post_json(*, base_url: str, path: str, api_key: str, payload: dict, timeout: float, proxy_url: str = "") -> dict: """POST a JSON body to {base_url}{path} with Bearer auth; return parsed JSON. diff --git a/apps/backend/chatballs/ai/provider/resilience.py b/apps/backend/chatballs/ai/provider/resilience.py index 7ae4404..704da81 100644 --- a/apps/backend/chatballs/ai/provider/resilience.py +++ b/apps/backend/chatballs/ai/provider/resilience.py @@ -1,5 +1,5 @@ import time -from typing import Callable +from collections.abc import Callable from chatballs.ai.provider.base import ProviderError diff --git a/apps/backend/chatballs/ai/test_agent_cards.py b/apps/backend/chatballs/ai/test_agent_cards.py index 58da8ff..3778eb2 100644 --- a/apps/backend/chatballs/ai/test_agent_cards.py +++ b/apps/backend/chatballs/ai/test_agent_cards.py @@ -1,7 +1,6 @@ import json from django.test import TestCase -from chatballs.testing import TenantAPIClient as APIClient from chatballs.ai.models import AIAgent, AIAgentStatus from chatballs.channels.models import Channel @@ -17,6 +16,7 @@ from chatballs.integrations.models import ( IntegrationKind, IntegrationProvider, ) +from chatballs.testing import TenantAPIClient as APIClient class AgentCardTestCase(TestCase): diff --git a/apps/backend/chatballs/ai/test_demo_provider.py b/apps/backend/chatballs/ai/test_demo_provider.py index 67b5291..5619bc6 100644 --- a/apps/backend/chatballs/ai/test_demo_provider.py +++ b/apps/backend/chatballs/ai/test_demo_provider.py @@ -8,10 +8,17 @@ from chatballs.ai.provider.routing import _provider_from_integration from chatballs.ai.runtime import HANDOFF_TOKEN from chatballs.identity.bootstrap import bootstrap_owner from chatballs.identity.models import Organization -from chatballs.integrations.models import Integration, IntegrationKind, IntegrationProvider, IntegrationStatus +from chatballs.integrations.models import ( + Integration, + IntegrationKind, + IntegrationProvider, + IntegrationStatus, +) from chatballs.integrations.services import ( IntegrationInput, create_integration, +) +from chatballs.integrations.services import ( # Алиас обязателен: имя test_* на уровне модуля pytest собирает как тест # и падает на ненайденных фикстурах (как в integrations/tests.py). test_integration as run_integration_test, diff --git a/apps/backend/chatballs/ai/tests.py b/apps/backend/chatballs/ai/tests.py index 7d9f759..fbf41e0 100644 --- a/apps/backend/chatballs/ai/tests.py +++ b/apps/backend/chatballs/ai/tests.py @@ -3,13 +3,14 @@ import tempfile from django.core.files.uploadedfile import SimpleUploadedFile from django.test import TestCase, override_settings -from chatballs.testing import TenantAPIClient as APIClient, system_tenant_context from chatballs.ai.knowledge_categories import ensure_uncategorized_category from chatballs.ai.models import AIAgent, AIAgentStatus, Knowledge, KnowledgeFragment from chatballs.channels.models import Channel from chatballs.identity.bootstrap import bootstrap_owner from chatballs.identity.models import EmployeeRole, HumanUser, Organization, OrganizationMembership +from chatballs.testing import TenantAPIClient as APIClient +from chatballs.testing import system_tenant_context _MEDIA_ROOT = tempfile.mkdtemp(prefix="hub-test-media-") @@ -284,7 +285,11 @@ class ResilienceTests(TestCase): def test_circuit_breaker_opens_after_threshold(self) -> None: from chatballs.ai.provider.base import ProviderError - from chatballs.ai.provider.resilience import CircuitBreaker, CircuitBreakerOpen, call_with_resilience + from chatballs.ai.provider.resilience import ( + CircuitBreaker, + CircuitBreakerOpen, + call_with_resilience, + ) breaker = CircuitBreaker(failure_threshold=2, reset_timeout=999) diff --git a/apps/backend/chatballs/ai/views.py b/apps/backend/chatballs/ai/views.py index 88f7f53..dcd1723 100644 --- a/apps/backend/chatballs/ai/views.py +++ b/apps/backend/chatballs/ai/views.py @@ -5,7 +5,7 @@ from rest_framework.response import Response from rest_framework.views import APIView from chatballs.ai.api_errors import validation_error_response -from chatballs.api.pagination import page_payload, paginate +from chatballs.ai.indexing import reindex_knowledge from chatballs.ai.knowledge_api_inputs import knowledge_filters, knowledge_input from chatballs.ai.knowledge_import import import_knowledge_documents from chatballs.ai.knowledge_policy import ( @@ -19,7 +19,6 @@ from chatballs.ai.knowledge_services import ( delete_knowledge, update_knowledge, ) -from chatballs.ai.indexing import reindex_knowledge from chatballs.ai.models import Knowledge from chatballs.ai.selectors import ( apply_knowledge_filters, @@ -29,6 +28,7 @@ from chatballs.ai.selectors import ( writable_knowledge_item_for_employee, ) from chatballs.ai.serializers import attachment_payload, knowledge_payload +from chatballs.api.pagination import page_payload, paginate from chatballs.api.permissions import HasCapability from chatballs.identity.audit import record_audit_event from chatballs.identity.models import AuditEvent diff --git a/apps/backend/chatballs/api/exceptions.py b/apps/backend/chatballs/api/exceptions.py index 92e6cfb..e61238d 100644 --- a/apps/backend/chatballs/api/exceptions.py +++ b/apps/backend/chatballs/api/exceptions.py @@ -1,17 +1,14 @@ from typing import Any -from rest_framework import status -from rest_framework.response import Response from rest_framework.views import exception_handler as drf_exception_handler - def _flatten(data: Any) -> str: if isinstance(data, str): return data if isinstance(data, dict): return "; ".join(_flatten(value) for value in data.values()) - if isinstance(data, (list, tuple)): + if isinstance(data, list | tuple): return "; ".join(_flatten(item) for item in data) return str(data) diff --git a/apps/backend/chatballs/api/permissions.py b/apps/backend/chatballs/api/permissions.py index 2a15275..75acab7 100644 --- a/apps/backend/chatballs/api/permissions.py +++ b/apps/backend/chatballs/api/permissions.py @@ -12,6 +12,11 @@ class HasCapability(BasePermission): Views declare ``required_capability`` or a method keyed ``required_capabilities`` mapping. Object/resource scope is still checked by the view after loading the canonical resource. + + Организационная область не объявляется вьюхой и не отключается: без + членства в организации проверка не проходит вообще. Раньше рядом стоял + атрибут ``require_organization_scope = True``, который никто не читал — он + выглядел как переключатель там, где переключателя нет. """ message = "Required capability is missing" diff --git a/apps/backend/chatballs/calls/consumers.py b/apps/backend/chatballs/calls/consumers.py index f1c0565..c64057e 100644 --- a/apps/backend/chatballs/calls/consumers.py +++ b/apps/backend/chatballs/calls/consumers.py @@ -17,10 +17,9 @@ from channels.generic.websocket import AsyncJsonWebsocketConsumer from chatballs.calls import signaling from chatballs.calls.errors import CallTokenError -from chatballs.calls.models import TERMINAL_CALL_STATUSES +from chatballs.calls.models import TERMINAL_CALL_STATUSES, ParticipantSide from chatballs.calls.permissions import staff_call_access_valid from chatballs.calls.services import authorize_call_access_context -from chatballs.calls.models import ParticipantSide from chatballs.tenancy.database import run_tenant_operation logger = logging.getLogger(__name__) diff --git a/apps/backend/chatballs/calls/lifecycle.py b/apps/backend/chatballs/calls/lifecycle.py index c7b0633..8c06170 100644 --- a/apps/backend/chatballs/calls/lifecycle.py +++ b/apps/backend/chatballs/calls/lifecycle.py @@ -12,7 +12,6 @@ from chatballs.calls.models import ( ParticipantSide, ) from chatballs.conversations.models import Message, MessageAuthor -from chatballs.tenancy.context import TenantContext ALLOWED_TRANSITIONS = { CallStatus.REQUESTED: { diff --git a/apps/backend/chatballs/calls/metrics.py b/apps/backend/chatballs/calls/metrics.py index 5dc5266..9895ff4 100644 --- a/apps/backend/chatballs/calls/metrics.py +++ b/apps/backend/chatballs/calls/metrics.py @@ -46,7 +46,7 @@ def record_call_metric( local = _sanitize_candidate_type(local_candidate_type) remote = _sanitize_candidate_type(remote_candidate_type) rtt: int | None = None - if isinstance(round_trip_ms, (int, float)) and not isinstance(round_trip_ms, bool): + if isinstance(round_trip_ms, int | float) and not isinstance(round_trip_ms, bool): rtt = max(0, min(_MAX_ROUND_TRIP_MS, int(round_trip_ms))) CallMetric.objects.update_or_create( call_session_id=call_session_id, diff --git a/apps/backend/chatballs/calls/models.py b/apps/backend/chatballs/calls/models.py index 3e7a24d..a2ca5e9 100644 --- a/apps/backend/chatballs/calls/models.py +++ b/apps/backend/chatballs/calls/models.py @@ -113,15 +113,15 @@ class CallSession(models.Model): ] indexes = [models.Index(fields=["organization", "status"])] + def __str__(self) -> str: + return f"call:{self.id}/{self.status}" + @property def duration_seconds(self) -> int | None: if self.connected_at is None or self.ended_at is None: return None return max(0, int((self.ended_at - self.connected_at).total_seconds())) - def __str__(self) -> str: - return f"call:{self.id}/{self.status}" - class CallInvite(TenantRelationModel): tenant_relation_fields = ("call_session", "connection_identity") diff --git a/apps/backend/chatballs/calls/routing.py b/apps/backend/chatballs/calls/routing.py index b560057..86432fd 100644 --- a/apps/backend/chatballs/calls/routing.py +++ b/apps/backend/chatballs/calls/routing.py @@ -1,7 +1,11 @@ from django.urls import path from chatballs.calls.consumers import CallSignalingConsumer +from chatballs.http.ws_middleware import SameOriginWebSocketMiddleware +# Сигналинг звонка аутентифицируется call access token, а не сессией, поэтому +# переименование cookie ему не нужно. Проверка Origin — нужна: страницу звонка +# открывает браузер, и чужой сайт не должен открывать сокет за него. websocket_urlpatterns = [ - path("ws/calls/", CallSignalingConsumer.as_asgi()), + path("ws/calls/", SameOriginWebSocketMiddleware(CallSignalingConsumer.as_asgi())), ] diff --git a/apps/backend/chatballs/calls/services.py b/apps/backend/chatballs/calls/services.py index 364725f..562fe86 100644 --- a/apps/backend/chatballs/calls/services.py +++ b/apps/backend/chatballs/calls/services.py @@ -14,7 +14,6 @@ from chatballs.calls.errors import ( CallTokenError, ) from chatballs.calls.lifecycle import transition_call -from chatballs.integrations.features import call_allowed from chatballs.calls.metrics import record_call_metric from chatballs.calls.models import ( TERMINAL_CALL_STATUSES, @@ -53,6 +52,7 @@ from chatballs.conversations.models import ( ) from chatballs.conversations.services import ClaimError, claim_locked_conversation from chatballs.events.services import DomainEvent, enqueue_event +from chatballs.integrations.features import call_allowed from chatballs.integrations.models import IntegrationProvider from chatballs.tenancy.context import TenantContext diff --git a/apps/backend/chatballs/calls/signaling.py b/apps/backend/chatballs/calls/signaling.py index 8cba96d..51cf286 100644 --- a/apps/backend/chatballs/calls/signaling.py +++ b/apps/backend/chatballs/calls/signaling.py @@ -10,16 +10,17 @@ from django.utils import timezone from chatballs.calls.errors import CallInvalidTransition from chatballs.calls.lifecycle import finish_call, transition_call from chatballs.calls.models import ( + TERMINAL_CALL_STATUSES, CallParticipant, CallSession, CallStatus, ParticipantConnectionState, - TERMINAL_CALL_STATUSES, ) from chatballs.calls.serializers import public_call_state_payload from chatballs.calls.services import record_call_metric from chatballs.tenancy.context import TenantContext + def _call(context: TenantContext, call_id) -> CallSession: return CallSession.objects.select_related("initiated_by").get( id=call_id, organization=context.organization diff --git a/apps/backend/chatballs/calls/tests/test_api.py b/apps/backend/chatballs/calls/tests/test_api.py index 5e081f9..a892451 100644 --- a/apps/backend/chatballs/calls/tests/test_api.py +++ b/apps/backend/chatballs/calls/tests/test_api.py @@ -1,11 +1,10 @@ import json -from chatballs.testing import TenantAPIClient as APIClient - from chatballs.calls.models import CallKind, CallSession, CallStatus, ParticipantSide from chatballs.calls.tests.helpers import CallTestCase, create_call_request from chatballs.calls.tokens import verify_call_access_token from chatballs.conversations.models import ControlMode +from chatballs.testing import TenantAPIClient as APIClient class InternalCallApiTests(CallTestCase): diff --git a/apps/backend/chatballs/calls/tests/test_flow.py b/apps/backend/chatballs/calls/tests/test_flow.py index 095ccca..74e410a 100644 --- a/apps/backend/chatballs/calls/tests/test_flow.py +++ b/apps/backend/chatballs/calls/tests/test_flow.py @@ -3,13 +3,12 @@ TG/MAX через outbox, истечение и системные событи import json from datetime import timedelta -from urllib.parse import parse_qs, urlparse from unittest import mock +from urllib.parse import parse_qs, urlparse from django.utils import timezone -from chatballs.testing import TenantAPIClient as APIClient, tenant_context_for -from chatballs.calls.event_handlers import handle_call_invite_send +from chatballs.calls.event_handlers import CallInviteDeliveryError, handle_call_invite_send from chatballs.calls.models import ( CallEndedBy, CallInvite, @@ -18,14 +17,16 @@ from chatballs.calls.models import ( InviteDeliveryStatus, ) from chatballs.calls.services import ( - open_call_for_identity, decline_call_for_identity, + open_call_for_identity, ) from chatballs.calls.tests.helpers import CallTestCase, create_call_request, expire_stale_calls from chatballs.calls.tokens import hash_invite_token from chatballs.conversations.models import Conversation, Message from chatballs.events.models import OutboxEvent from chatballs.integrations.models import Integration, IntegrationKind, IntegrationProvider +from chatballs.testing import TenantAPIClient as APIClient +from chatballs.testing import tenant_context_for class CancelCallApiTests(CallTestCase): @@ -279,7 +280,7 @@ class MessengerDeliveryTests(CallTestCase): with mock.patch( "chatballs.calls.event_handlers.transports.send_call_invite", return_value=False ): - with self.assertRaises(Exception): + with self.assertRaises(CallInviteDeliveryError): handle_call_invite_send( {"callSessionId": str(created.call_session.id)}, tenant_context_for(self.owner, self.organization), diff --git a/apps/backend/chatballs/calls/tests/test_signaling.py b/apps/backend/chatballs/calls/tests/test_signaling.py index ba772b8..7c27f0e 100644 --- a/apps/backend/chatballs/calls/tests/test_signaling.py +++ b/apps/backend/chatballs/calls/tests/test_signaling.py @@ -270,7 +270,7 @@ class AuthDeadlineTests(CallDomainMixin, TransactionTestCase): self.assertEqual(output["code"], consumers.AUTH_TIMEOUT_CLOSE) def test_authenticated_socket_survives_the_deadline(self) -> None: - created, customer_token = None, None + customer_token = None def prepare(): create_call_request(conversation_id=self.conversation.id, initiator=self.owner) diff --git a/apps/backend/chatballs/calls/tests/test_tokens.py b/apps/backend/chatballs/calls/tests/test_tokens.py index dd57626..5cadd30 100644 --- a/apps/backend/chatballs/calls/tests/test_tokens.py +++ b/apps/backend/chatballs/calls/tests/test_tokens.py @@ -1,5 +1,5 @@ -from datetime import timedelta import uuid +from datetime import timedelta from django.test import override_settings from django.utils import timezone diff --git a/apps/backend/chatballs/calls/tokens.py b/apps/backend/chatballs/calls/tokens.py index 7ff9451..0182e5d 100644 --- a/apps/backend/chatballs/calls/tokens.py +++ b/apps/backend/chatballs/calls/tokens.py @@ -32,7 +32,7 @@ class CallAccessClaims: def _derived_secret(purpose: str) -> bytes: return hmac.new( (settings.SECRET_KEY or "").encode("utf-8"), - f"hub:{purpose}:v1".encode("utf-8"), + f"hub:{purpose}:v1".encode(), hashlib.sha256, ).digest() diff --git a/apps/backend/chatballs/channels/apps.py b/apps/backend/chatballs/channels/apps.py index 1cec473..a478e30 100644 --- a/apps/backend/chatballs/channels/apps.py +++ b/apps/backend/chatballs/channels/apps.py @@ -1,8 +1,8 @@ -from django.apps import AppConfig - - -class ChannelsConfig(AppConfig): - default_auto_field = "django.db.models.BigAutoField" - label = "channels" - name = "chatballs.channels" - verbose_name = "Processing channels (AI context)" +from django.apps import AppConfig + + +class ChannelsConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + label = "channels" + name = "chatballs.channels" + verbose_name = "Processing channels (AI context)" diff --git a/apps/backend/chatballs/conversations/admin.py b/apps/backend/chatballs/conversations/admin.py index ed848a6..5eb0280 100644 --- a/apps/backend/chatballs/conversations/admin.py +++ b/apps/backend/chatballs/conversations/admin.py @@ -1,21 +1,21 @@ -from django.contrib import admin - -from chatballs.conversations.models import Contact, Conversation, Message - - -@admin.register(Conversation) -class ConversationAdmin(admin.ModelAdmin): - list_display = ("id", "channel", "contact", "lifecycle", "control_mode", "last_activity_at") - list_filter = ("lifecycle", "control_mode") - - -@admin.register(Contact) -class ContactAdmin(admin.ModelAdmin): - list_display = ("id", "name", "organization", "created_at") - search_fields = ("name",) - - -@admin.register(Message) -class MessageAdmin(admin.ModelAdmin): - list_display = ("id", "conversation", "author_type", "created_at") - list_filter = ("author_type",) +from django.contrib import admin + +from chatballs.conversations.models import Contact, Conversation, Message + + +@admin.register(Conversation) +class ConversationAdmin(admin.ModelAdmin): + list_display = ("id", "channel", "contact", "lifecycle", "control_mode", "last_activity_at") + list_filter = ("lifecycle", "control_mode") + + +@admin.register(Contact) +class ContactAdmin(admin.ModelAdmin): + list_display = ("id", "name", "organization", "created_at") + search_fields = ("name",) + + +@admin.register(Message) +class MessageAdmin(admin.ModelAdmin): + list_display = ("id", "conversation", "author_type", "created_at") + list_filter = ("author_type",) diff --git a/apps/backend/chatballs/conversations/chat_extras_views.py b/apps/backend/chatballs/conversations/chat_extras_views.py index 8d8ab42..6681410 100644 --- a/apps/backend/chatballs/conversations/chat_extras_views.py +++ b/apps/backend/chatballs/conversations/chat_extras_views.py @@ -15,10 +15,10 @@ from rest_framework.views import APIView from chatballs.api.permissions import HasCapability from chatballs.conversations.models import ( + ControlMode, Conversation, ConversationLabel, ConversationPriority, - ControlMode, LifecycleState, ReplyTemplate, ) diff --git a/apps/backend/chatballs/conversations/contacts_merge.py b/apps/backend/chatballs/conversations/contacts_merge.py index 1d8c478..aa2b57b 100644 --- a/apps/backend/chatballs/conversations/contacts_merge.py +++ b/apps/backend/chatballs/conversations/contacts_merge.py @@ -12,7 +12,7 @@ from django.core.exceptions import ValidationError from django.db import transaction from django.utils import timezone -from chatballs.conversations.models import ConnectionIdentity, ContactMerge, Contact, Conversation +from chatballs.conversations.models import ConnectionIdentity, Contact, ContactMerge, Conversation from chatballs.identity.audit import record_audit_event # Поля карточки, которые дозаполняются из исходного контакта, если у целевого diff --git a/apps/backend/chatballs/conversations/html_sanitizer.py b/apps/backend/chatballs/conversations/html_sanitizer.py index b00d6c2..7f99d6c 100644 --- a/apps/backend/chatballs/conversations/html_sanitizer.py +++ b/apps/backend/chatballs/conversations/html_sanitizer.py @@ -4,7 +4,6 @@ import html from html.parser import HTMLParser from urllib.parse import urlsplit - _ALLOWED_TAGS = { "a", "b", diff --git a/apps/backend/chatballs/conversations/ingest.py b/apps/backend/chatballs/conversations/ingest.py index 9d5d65b..d73e846 100644 --- a/apps/backend/chatballs/conversations/ingest.py +++ b/apps/backend/chatballs/conversations/ingest.py @@ -48,15 +48,24 @@ _ROLE = { def _already_processed(context: TenantContext, source: str, external_id: str, text: str) -> bool: + """Отметить сообщение обработанным; True — оно уже приходило. + + Вставка идёт своей точкой сохранения. Вызывают эту функцию изнутри + транзакции (воркер держит ``tenant_atomic`` на весь цикл поллинга), а + IntegrityError в Postgres обрывает транзакцию целиком: без savepoint + первый же повтор сообщения ронял не дедупликацию, а весь цикл — со всеми + остальными подключениями организации. + """ payload_hash = hashlib.sha256(text.encode("utf-8")).hexdigest()[:32] try: - InboxEvent.objects.create( - source=source, - external_event_id=external_id, - payload_hash=payload_hash, - ownership=EventOwnership.TENANT, - organization=context.organization, - ) + with transaction.atomic(): + InboxEvent.objects.create( + source=source, + external_event_id=external_id, + payload_hash=payload_hash, + ownership=EventOwnership.TENANT, + organization=context.organization, + ) return False except IntegrityError: return True @@ -89,7 +98,10 @@ class TranscriptionJob: def prepare_transcription(channel, message: Message) -> TranscriptionJob | None: """Шаг в транзакции: провайдер организации, модель и байты аудио.""" from chatballs.ai.provider.factory import get_provider - from chatballs.ai.provider.routing import DEFAULT_TRANSCRIPTION_MODEL, resolve_transcription_model + from chatballs.ai.provider.routing import ( + DEFAULT_TRANSCRIPTION_MODEL, + resolve_transcription_model, + ) if not message.audio: return None diff --git a/apps/backend/chatballs/conversations/maintenance.py b/apps/backend/chatballs/conversations/maintenance.py index 077f181..feb1990 100644 --- a/apps/backend/chatballs/conversations/maintenance.py +++ b/apps/backend/chatballs/conversations/maintenance.py @@ -1,17 +1,10 @@ import logging - from datetime import timedelta - - from django.utils import timezone - - from chatballs.conversations.models import Conversation, LifecycleState - - logger = logging.getLogger(__name__) diff --git a/apps/backend/chatballs/conversations/migrations/0012_message_audio_message_audio_content_type_and_more.py b/apps/backend/chatballs/conversations/migrations/0012_message_audio_message_audio_content_type_and_more.py index 252103b..3d82a8c 100644 --- a/apps/backend/chatballs/conversations/migrations/0012_message_audio_message_audio_content_type_and_more.py +++ b/apps/backend/chatballs/conversations/migrations/0012_message_audio_message_audio_content_type_and_more.py @@ -1,44 +1,44 @@ -# Generated by Django 5.2.15 on 2026-09-04 20:40 - -import chatballs.conversations.models -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('conversations', '0011_conversation_archived_at_conversation_note_and_more'), - ] - - operations = [ - migrations.AddField( - model_name='message', - name='audio', - field=models.FileField(blank=True, max_length=512, upload_to=chatballs.conversations.models.message_audio_upload_path), - ), - migrations.AddField( - model_name='message', - name='audio_content_type', - field=models.CharField(blank=True, max_length=64), - ), - migrations.AddField( - model_name='message', - name='duration_seconds', - field=models.PositiveIntegerField(default=0), - ), - migrations.AddField( - model_name='message', - name='transcript', - field=models.TextField(blank=True), - ), - migrations.AddField( - model_name='message', - name='transcript_status', - field=models.CharField(choices=[('NONE', 'Не расшифровано'), ('READY', 'Готова'), ('FAILED', 'Ошибка')], default='NONE', max_length=8), - ), - migrations.AlterField( - model_name='message', - name='kind', - field=models.CharField(blank=True, choices=[('', 'Текст'), ('contact_request', 'Запрос контакта'), ('contact', 'Контакт'), ('voice', 'Голосовое сообщение')], default='', max_length=32), - ), - ] +# Generated by Django 5.2.15 on 2026-09-04 20:40 + +import chatballs.conversations.models +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('conversations', '0011_conversation_archived_at_conversation_note_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='message', + name='audio', + field=models.FileField(blank=True, max_length=512, upload_to=chatballs.conversations.models.message_audio_upload_path), + ), + migrations.AddField( + model_name='message', + name='audio_content_type', + field=models.CharField(blank=True, max_length=64), + ), + migrations.AddField( + model_name='message', + name='duration_seconds', + field=models.PositiveIntegerField(default=0), + ), + migrations.AddField( + model_name='message', + name='transcript', + field=models.TextField(blank=True), + ), + migrations.AddField( + model_name='message', + name='transcript_status', + field=models.CharField(choices=[('NONE', 'Не расшифровано'), ('READY', 'Готова'), ('FAILED', 'Ошибка')], default='NONE', max_length=8), + ), + migrations.AlterField( + model_name='message', + name='kind', + field=models.CharField(blank=True, choices=[('', 'Текст'), ('contact_request', 'Запрос контакта'), ('contact', 'Контакт'), ('voice', 'Голосовое сообщение')], default='', max_length=32), + ), + ] diff --git a/apps/backend/chatballs/conversations/poller.py b/apps/backend/chatballs/conversations/poller.py index 23a6d72..8d58f54 100644 --- a/apps/backend/chatballs/conversations/poller.py +++ b/apps/backend/chatballs/conversations/poller.py @@ -1,38 +1,38 @@ -import logging - -from chatballs.conversations import transports -from chatballs.conversations.ingest import ingest_inbound -from chatballs.integrations.models import Integration - -logger = logging.getLogger(__name__) - - -def poll_all_messengers(context) -> int: - """Poll every messenger connection bound to a channel; ingest inbound. Returns count.""" - # Сервисные боты уведомлений поллятся отдельно (notifications.binding). - # Фильтр по config — в Python: JSON-lookup в .exclude() отбрасывает и строки - # без ключа purpose (NULL в SQL), т.е. все клиентские боты. - integrations = [ - integration - for integration in Integration.objects.filter( - organization=context.organization, - provider__in=transports.SUPPORTED_PROVIDERS, - is_active=True, - channel__isnull=False, - channel__is_active=True, - ).exclude(secret="") - if integration.config.get("purpose") != "notifications" - ] - total = 0 - for integration in integrations: - messages, new_marker = transports.poll(integration) - for inbound in messages: - try: - ingest_inbound(integration, inbound) - total += 1 - except Exception: # pragma: no cover - logger.exception("Ingest failed for integration %s", integration.id) - if new_marker and new_marker != integration.poll_marker: - integration.poll_marker = new_marker - integration.save(update_fields=["poll_marker", "updated_at"]) - return total +import logging + +from chatballs.conversations import transports +from chatballs.conversations.ingest import ingest_inbound +from chatballs.integrations.models import Integration + +logger = logging.getLogger(__name__) + + +def poll_all_messengers(context) -> int: + """Poll every messenger connection bound to a channel; ingest inbound. Returns count.""" + # Сервисные боты уведомлений поллятся отдельно (notifications.binding). + # Фильтр по config — в Python: JSON-lookup в .exclude() отбрасывает и строки + # без ключа purpose (NULL в SQL), т.е. все клиентские боты. + integrations = [ + integration + for integration in Integration.objects.filter( + organization=context.organization, + provider__in=transports.SUPPORTED_PROVIDERS, + is_active=True, + channel__isnull=False, + channel__is_active=True, + ).exclude(secret="") + if integration.config.get("purpose") != "notifications" + ] + total = 0 + for integration in integrations: + messages, new_marker = transports.poll(integration) + for inbound in messages: + try: + ingest_inbound(integration, inbound) + total += 1 + except Exception: # pragma: no cover + logger.exception("Ingest failed for integration %s", integration.id) + if new_marker and new_marker != integration.poll_marker: + integration.poll_marker = new_marker + integration.save(update_fields=["poll_marker", "updated_at"]) + return total diff --git a/apps/backend/chatballs/conversations/reporting_views.py b/apps/backend/chatballs/conversations/reporting_views.py index ed1c6b7..eb09ed4 100644 --- a/apps/backend/chatballs/conversations/reporting_views.py +++ b/apps/backend/chatballs/conversations/reporting_views.py @@ -1,16 +1,15 @@ +from django.core.exceptions import ValidationError from rest_framework.request import Request from rest_framework.response import Response -from django.core.exceptions import ValidationError - from chatballs.api.pagination import page_payload, paginate from chatballs.conversations.clients import client_detail, client_row, clients_queryset from chatballs.conversations.contacts_merge import merge_contacts, revert_merge from chatballs.conversations.models import Contact -from chatballs.identity.models import EmployeeRole from chatballs.conversations.stats import sales_overview_stats from chatballs.conversations.view_base import ConversationViewBase from chatballs.identity.audit import record_audit_event +from chatballs.identity.models import EmployeeRole class ConversationStatsView(ConversationViewBase): diff --git a/apps/backend/chatballs/conversations/routing.py b/apps/backend/chatballs/conversations/routing.py index 02364bf..ffa4842 100644 --- a/apps/backend/chatballs/conversations/routing.py +++ b/apps/backend/chatballs/conversations/routing.py @@ -2,13 +2,18 @@ from channels.auth import AuthMiddlewareStack from django.urls import path from chatballs.conversations.consumers import ConversationEventsConsumer +from chatballs.http.ws_middleware import websocket_boundary # Оповещения о диалогах аутентифицируются сессией того же SPA — отдельного # токена, как у сигналинга звонков, здесь не нужно: сокет открывает тот же # браузер. Организация стоит в адресе, как и во всём HTTP-слое. +# +# websocket_boundary снаружи AuthMiddlewareStack: он приводит имена cookie к +# тем, по которым Channels ищет сессию (по TLS браузер держит __Host-…), и +# отбивает хендшейк с чужим Origin. websocket_urlpatterns = [ path( "ws/organizations//conversations/", - AuthMiddlewareStack(ConversationEventsConsumer.as_asgi()), + websocket_boundary(AuthMiddlewareStack(ConversationEventsConsumer.as_asgi())), ), ] diff --git a/apps/backend/chatballs/conversations/serializers.py b/apps/backend/chatballs/conversations/serializers.py index 80d738e..733f7db 100644 --- a/apps/backend/chatballs/conversations/serializers.py +++ b/apps/backend/chatballs/conversations/serializers.py @@ -1,6 +1,5 @@ from django.db.models import Count, Max, Q -from chatballs.integrations.features import features_payload from chatballs.conversations.models import ( ConnectionIdentity, Conversation, @@ -9,6 +8,7 @@ from chatballs.conversations.models import ( MessageKind, ) from chatballs.identity.avatars import user_avatar_url_in +from chatballs.integrations.features import features_payload from chatballs.integrations.models import IntegrationProvider diff --git a/apps/backend/chatballs/conversations/services.py b/apps/backend/chatballs/conversations/services.py index 4aa051a..695f388 100644 --- a/apps/backend/chatballs/conversations/services.py +++ b/apps/backend/chatballs/conversations/services.py @@ -1,39 +1,21 @@ from django.db import transaction - from django.utils import timezone - - from chatballs.conversations import transports - from chatballs.conversations.models import ( - ConnectionIdentity, - - Conversation, - ControlMode, - + Conversation, ExpectedResponder, - LifecycleState, - Message, - MessageAuthor, - MessageKind, - ) - from chatballs.identity.models import EmployeeRole - from chatballs.integrations.models import IntegrationProvider - from chatballs.tenancy.context import TenantContext - - CONTACT_REQUEST_TEXT = "Поделитесь, пожалуйста, контактом — нажмите кнопку ниже." CONTACT_REQUEST_TEXT_WEB = "Поделитесь, пожалуйста, номером телефона." diff --git a/apps/backend/chatballs/conversations/stats.py b/apps/backend/chatballs/conversations/stats.py index 1554655..105a39f 100644 --- a/apps/backend/chatballs/conversations/stats.py +++ b/apps/backend/chatballs/conversations/stats.py @@ -15,8 +15,8 @@ from django.utils import timezone from chatballs.ai.models import LlmInvocation from chatballs.channels.selectors import channels_in_organization from chatballs.conversations.models import ( - Conversation, ControlMode, + Conversation, ExpectedResponder, LifecycleState, Message, diff --git a/apps/backend/chatballs/conversations/test_authorization.py b/apps/backend/chatballs/conversations/test_authorization.py index 65cb8df..608b18f 100644 --- a/apps/backend/chatballs/conversations/test_authorization.py +++ b/apps/backend/chatballs/conversations/test_authorization.py @@ -1,5 +1,4 @@ from django.test import TestCase -from chatballs.testing import TenantAPIClient as APIClient from chatballs.channels.models import Channel from chatballs.conversations.models import Contact, Conversation @@ -10,6 +9,7 @@ from chatballs.identity.models import ( Organization, OrganizationMembership, ) +from chatballs.testing import TenantAPIClient as APIClient class ConversationVisibilityTests(TestCase): diff --git a/apps/backend/chatballs/conversations/test_chat_extras.py b/apps/backend/chatballs/conversations/test_chat_extras.py index a5adeee..b4cc341 100644 --- a/apps/backend/chatballs/conversations/test_chat_extras.py +++ b/apps/backend/chatballs/conversations/test_chat_extras.py @@ -2,15 +2,14 @@ import json from django.test import TestCase from django.utils import timezone -from chatballs.testing import TenantAPIClient as APIClient from chatballs.channels.models import Channel from chatballs.conversations.models import ( Contact, + ControlMode, Conversation, ConversationLabel, ConversationPriority, - ControlMode, LifecycleState, ReplyTemplate, ) @@ -21,6 +20,7 @@ from chatballs.identity.models import ( Organization, OrganizationMembership, ) +from chatballs.testing import TenantAPIClient as APIClient class ChatExtrasTestCase(TestCase): diff --git a/apps/backend/chatballs/conversations/test_contacts_merge.py b/apps/backend/chatballs/conversations/test_contacts_merge.py index bd2f817..21d17ae 100644 --- a/apps/backend/chatballs/conversations/test_contacts_merge.py +++ b/apps/backend/chatballs/conversations/test_contacts_merge.py @@ -5,7 +5,7 @@ from django.test import TestCase from chatballs.channels.models import Channel from chatballs.conversations.contacts_merge import merge_contacts, revert_merge -from chatballs.conversations.models import ConnectionIdentity, Contact, ContactMerge, Conversation +from chatballs.conversations.models import ConnectionIdentity, Contact, Conversation from chatballs.identity.models import ( AuditEvent, EmployeeRole, diff --git a/apps/backend/chatballs/conversations/test_email_transport.py b/apps/backend/chatballs/conversations/test_email_transport.py index 6613e6e..7137c31 100644 --- a/apps/backend/chatballs/conversations/test_email_transport.py +++ b/apps/backend/chatballs/conversations/test_email_transport.py @@ -3,39 +3,22 @@ from email import message_from_bytes, policy - from email.message import EmailMessage as MimeMessage - from unittest import mock - - from django.http import QueryDict - from django.test import TestCase - - from chatballs.channels.models import Channel - -from chatballs.conversations.models import Contact, Conversation, MessageAuthor - from chatballs.conversations.clients import client_detail, client_row, clients_queryset - +from chatballs.conversations.models import Contact, Conversation, MessageAuthor from chatballs.conversations.selectors import conversation_messages - from chatballs.conversations.serializers import conversation_payload, message_payload - from chatballs.conversations.transports import email as email_transport - from chatballs.identity.bootstrap import bootstrap_owner - from chatballs.identity.models import Organization - from chatballs.integrations.models import Integration, IntegrationKind, IntegrationProvider - - EMAIL_CONFIG = { "email": "support@example.com", @@ -499,7 +482,6 @@ class EmailIngestThreadMetaTests(TestCase): def _ingest(self, *, external_id: str, subject: str, message_id: str) -> None: from chatballs.conversations.ingest import ingest_inbound - from chatballs.conversations.transports.base import InboundMessage diff --git a/apps/backend/chatballs/conversations/test_ingest_dedup.py b/apps/backend/chatballs/conversations/test_ingest_dedup.py new file mode 100644 index 0000000..1151f28 --- /dev/null +++ b/apps/backend/chatballs/conversations/test_ingest_dedup.py @@ -0,0 +1,53 @@ +"""Повторная доставка входящего не должна ломать транзакцию. + +Дедупликация ставит запись в inbox и ловит IntegrityError на повторе. Ловить +его без точки сохранения нельзя: Postgres обрывает транзакцию целиком, и +следующий же запрос падает с TransactionManagementError. А вызывают это +изнутри транзакции — воркер держит ``tenant_atomic`` на весь цикл поллинга, +так что первый повтор ронял не дедупликацию, а весь цикл организации. +""" + +from __future__ import annotations + +from django.db import transaction +from django.test import TestCase + +from chatballs.conversations.ingest import _already_processed +from chatballs.events.models import InboxEvent +from chatballs.identity.bootstrap import bootstrap_owner +from chatballs.tenancy.database import tenant_atomic +from chatballs.testing import system_tenant_context + + +class InboundDeduplicationTests(TestCase): + def setUp(self) -> None: + result = bootstrap_owner(email="ingest-owner@example.com", password="Owner-Password-2026!") + self.context = system_tenant_context(result.organization) + + def test_repeat_is_reported_without_breaking_the_transaction(self) -> None: + with tenant_atomic(self.context): + first = _already_processed(self.context, "telegram:1", "update-42", "привет") + second = _already_processed(self.context, "telegram:1", "update-42", "привет") + + self.assertFalse(first) + self.assertTrue(second) + + # Главное: транзакция жива и дальше в ней можно работать. Раньше + # именно здесь всё и разваливалось. + self.assertEqual( + InboxEvent.objects.filter(external_event_id="update-42").count(), 1 + ) + + def test_repeat_does_not_roll_back_work_done_earlier(self) -> None: + with transaction.atomic(): + with tenant_atomic(self.context): + _already_processed(self.context, "max:7", "update-1", "первое") + _already_processed(self.context, "max:7", "update-1", "первое") + _already_processed(self.context, "max:7", "update-2", "второе") + + self.assertEqual(InboxEvent.objects.filter(source="max:7").count(), 2) + + def test_different_sources_do_not_collide(self) -> None: + with tenant_atomic(self.context): + self.assertFalse(_already_processed(self.context, "telegram:1", "shared-id", "текст")) + self.assertFalse(_already_processed(self.context, "max:2", "shared-id", "текст")) diff --git a/apps/backend/chatballs/conversations/test_lifecycle.py b/apps/backend/chatballs/conversations/test_lifecycle.py index f3da8eb..c61121d 100644 --- a/apps/backend/chatballs/conversations/test_lifecycle.py +++ b/apps/backend/chatballs/conversations/test_lifecycle.py @@ -1,57 +1,30 @@ import json - from unittest import mock - - from django.test import TestCase - - from chatballs.channels.models import Channel - from chatballs.conversations.ingest import ingest_inbound - from chatballs.conversations.models import ( - ConnectionIdentity, - Contact, - ControlMode, - Conversation, - ExpectedResponder, - LifecycleState, - MessageAuthor, - ) - from chatballs.conversations.transports.base import InboundMessage - from chatballs.identity.bootstrap import bootstrap_owner - from chatballs.identity.models import HumanUser, Organization - from chatballs.integrations.models import ( - Integration, - IntegrationKind, - IntegrationProvider, - ) - from chatballs.testing import TenantAPIClient as APIClient - - - def _connection(channel: Channel) -> Integration: return Integration.objects.create( diff --git a/apps/backend/chatballs/conversations/test_voice_messages.py b/apps/backend/chatballs/conversations/test_voice_messages.py index 37173af..f749f93 100644 --- a/apps/backend/chatballs/conversations/test_voice_messages.py +++ b/apps/backend/chatballs/conversations/test_voice_messages.py @@ -1,10 +1,9 @@ from unittest import mock from django.core.files.base import ContentFile -from django.db import connections from django.core.files.uploadedfile import SimpleUploadedFile +from django.db import connections from django.test import TestCase, TransactionTestCase -from chatballs.testing import TenantAPIClient as APIClient from chatballs.ai.provider.local import LocalProvider from chatballs.channels.models import Channel @@ -20,7 +19,6 @@ from chatballs.conversations.models import ( ) from chatballs.conversations.transports.base import InboundMessage from chatballs.identity.bootstrap import bootstrap_owner -from chatballs.tenancy.database import current_tenant_id, tenant_atomic from chatballs.identity.models import ( EmployeeRole, HumanUser, @@ -32,6 +30,8 @@ from chatballs.integrations.models import ( IntegrationKind, IntegrationProvider, ) +from chatballs.tenancy.database import current_tenant_id, tenant_atomic +from chatballs.testing import TenantAPIClient as APIClient class VoiceFixtureMixin: diff --git a/apps/backend/chatballs/conversations/tests.py b/apps/backend/chatballs/conversations/tests.py index f92a18a..c751dc0 100644 --- a/apps/backend/chatballs/conversations/tests.py +++ b/apps/backend/chatballs/conversations/tests.py @@ -1,13 +1,11 @@ import json - from unittest import mock from django.test import TestCase, override_settings -from chatballs.testing import TenantAPIClient as APIClient from chatballs.ai.limits import LimitExceeded -from chatballs.ai.provider.base import ProviderError from chatballs.ai.models import AIAgent, AIAgentStatus +from chatballs.ai.provider.base import ProviderError from chatballs.channels.models import Channel from chatballs.conversations.models import ( ConnectionIdentity, @@ -19,9 +17,9 @@ from chatballs.conversations.models import ( MessageAuthor, MessageKind, ) -from chatballs.conversations.transports.base import InboundMessage from chatballs.conversations.transports import max as max_transport from chatballs.conversations.transports import telegram as telegram_transport +from chatballs.conversations.transports.base import InboundMessage from chatballs.identity.bootstrap import bootstrap_owner from chatballs.identity.models import ( EmployeeRole, @@ -31,6 +29,7 @@ from chatballs.identity.models import ( ) from chatballs.integrations.models import Integration, IntegrationKind, IntegrationProvider from chatballs.notifications.models import Notification, NotificationAudience, NotificationType +from chatballs.testing import TenantAPIClient as APIClient def _messenger_connection(channel): diff --git a/apps/backend/chatballs/conversations/transports/base.py b/apps/backend/chatballs/conversations/transports/base.py index 9f58a6a..5d3d692 100644 --- a/apps/backend/chatballs/conversations/transports/base.py +++ b/apps/backend/chatballs/conversations/transports/base.py @@ -13,7 +13,6 @@ from django.conf import settings from chatballs.integrations.outbound import ensure_downloadable from chatballs.integrations.proxy import build_opener - MAX_ATTACHMENT_BYTES = 20 * 1024 * 1024 @@ -94,9 +93,15 @@ def download_bytes( ``allowed_host`` — хост из ``base_url`` подключения, который владелец назвал сам. """ - ensure_downloadable(url, allowed_host=allowed_host, via_proxy=bool(proxy_url)) + def guard(candidate: str) -> None: + ensure_downloadable(candidate, allowed_host=allowed_host, via_proxy=bool(proxy_url)) + + guard(url) request = urllib.request.Request(url) - with build_opener(proxy_url).open(request, timeout=settings.CHATBALLS_AI_REQUEST_TIMEOUT) as response: + # Та же проверка на каждый редирект: провайдер отдаёт адрес данными, и + # 302 увёл бы скачивание туда, куда исходный адрес не пустили. + opener = build_opener(proxy_url, validate_redirect=guard) + with opener.open(request, timeout=settings.CHATBALLS_AI_REQUEST_TIMEOUT) as response: data = response.read(max_bytes + 1) if len(data) > max_bytes: raise ValueError("Файл больше допустимого размера") diff --git a/apps/backend/chatballs/conversations/transports/email.py b/apps/backend/chatballs/conversations/transports/email.py index 4e0a499..6f99b24 100644 --- a/apps/backend/chatballs/conversations/transports/email.py +++ b/apps/backend/chatballs/conversations/transports/email.py @@ -19,7 +19,13 @@ from email.utils import parseaddr from django.conf import settings from chatballs.conversations.html_sanitizer import sanitize_email_html -from chatballs.conversations.transports.base import MAX_ATTACHMENT_BYTES, InboundFile, InboundMessage, guess_content_type, safe_filename +from chatballs.conversations.transports.base import ( + MAX_ATTACHMENT_BYTES, + InboundFile, + InboundMessage, + guess_content_type, + safe_filename, +) logger = logging.getLogger(__name__) diff --git a/apps/backend/chatballs/conversations/urls.py b/apps/backend/chatballs/conversations/urls.py index bccc4ac..66fcffa 100644 --- a/apps/backend/chatballs/conversations/urls.py +++ b/apps/backend/chatballs/conversations/urls.py @@ -1,6 +1,12 @@ from django.urls import path -from chatballs.conversations import attachment_views, chat_extras_views, reporting_views, views, voice_views +from chatballs.conversations import ( + attachment_views, + chat_extras_views, + reporting_views, + views, + voice_views, +) urlpatterns = [ path("", views.ConversationListView.as_view(), name="conversation-list"), diff --git a/apps/backend/chatballs/conversations/view_base.py b/apps/backend/chatballs/conversations/view_base.py index ffa6346..ee63044 100644 --- a/apps/backend/chatballs/conversations/view_base.py +++ b/apps/backend/chatballs/conversations/view_base.py @@ -1,43 +1,43 @@ -from rest_framework.request import Request -from rest_framework.views import APIView - -from chatballs.api.permissions import HasCapability -from chatballs.conversations.models import Conversation -from chatballs.conversations.selectors import conversation_for_context -from chatballs.identity.audit import record_audit_event -from chatballs.identity.policy import require_capability - - -class ConversationViewBase(APIView): - permission_classes = [HasCapability] - required_capability = "conversations.view" - - def _org(self, request: Request): - return request.tenant_context.organization - - def _conversation( - self, - request: Request, - conversation_id: int, - capability: str = "conversations.view", - ) -> Conversation: - conversation = conversation_for_context( - context=request.tenant_context, conversation_id=conversation_id - ) - if not require_capability( - request.tenant_context.membership, capability, conversation - ): - raise Conversation.DoesNotExist - return conversation - - def _audit( - self, request: Request, action: str, conversation: Conversation - ) -> None: - record_audit_event( - action=f"conversations.{action}", - actor=request.user, - organization=self._org(request), - object_type="Conversation", - object_id=str(conversation.id), - request=request, - ) +from rest_framework.request import Request +from rest_framework.views import APIView + +from chatballs.api.permissions import HasCapability +from chatballs.conversations.models import Conversation +from chatballs.conversations.selectors import conversation_for_context +from chatballs.identity.audit import record_audit_event +from chatballs.identity.policy import require_capability + + +class ConversationViewBase(APIView): + permission_classes = [HasCapability] + required_capability = "conversations.view" + + def _org(self, request: Request): + return request.tenant_context.organization + + def _conversation( + self, + request: Request, + conversation_id: int, + capability: str = "conversations.view", + ) -> Conversation: + conversation = conversation_for_context( + context=request.tenant_context, conversation_id=conversation_id + ) + if not require_capability( + request.tenant_context.membership, capability, conversation + ): + raise Conversation.DoesNotExist + return conversation + + def _audit( + self, request: Request, action: str, conversation: Conversation + ) -> None: + record_audit_event( + action=f"conversations.{action}", + actor=request.user, + organization=self._org(request), + object_type="Conversation", + object_id=str(conversation.id), + request=request, + ) diff --git a/apps/backend/chatballs/conversations/views.py b/apps/backend/chatballs/conversations/views.py index 90fe8e7..522a602 100644 --- a/apps/backend/chatballs/conversations/views.py +++ b/apps/backend/chatballs/conversations/views.py @@ -2,6 +2,12 @@ from django.db import transaction from rest_framework.request import Request from rest_framework.response import Response +from chatballs.api.pagination import ( + cursor_id, + window, + window_payload, + window_size, +) from chatballs.conversations.models import ( ControlMode, Conversation, @@ -9,12 +15,6 @@ from chatballs.conversations.models import ( LifecycleState, Message, ) -from chatballs.api.pagination import ( - cursor_id, - window, - window_payload, - window_size, -) from chatballs.conversations.selectors import ( MESSAGES_NEWER_KEYS, MESSAGES_OLDER_KEYS, diff --git a/apps/backend/chatballs/events/handlers.py b/apps/backend/chatballs/events/handlers.py index c5780d5..0ff8c0b 100644 --- a/apps/backend/chatballs/events/handlers.py +++ b/apps/backend/chatballs/events/handlers.py @@ -1,5 +1,5 @@ import logging -from typing import Callable +from collections.abc import Callable from chatballs.events.models import OutboxEvent from chatballs.events.services import tenant_context_for_event diff --git a/apps/backend/chatballs/http/test_ws_middleware.py b/apps/backend/chatballs/http/test_ws_middleware.py new file mode 100644 index 0000000..2dc9c5e --- /dev/null +++ b/apps/backend/chatballs/http/test_ws_middleware.py @@ -0,0 +1,122 @@ +"""WebSocket-обвязка: имена cookie по TLS и проверка Origin. + +Установка с сертификатом отдаёт браузеру только ``__Host-``-имена, а Channels +ищет сессию строго по ``settings.SESSION_COOKIE_NAME``. Пока это не сходилось, +каждый сокет на https закрывался как неаутентифицированный — и живые +обновления диалогов молча не работали, притом что REST на той же странице +работал. +""" + +from __future__ import annotations + +import asyncio + +from django.test import SimpleTestCase, override_settings + +from chatballs.http.ws_middleware import ( + SameOriginWebSocketMiddleware, + TlsAwareCookieASGIMiddleware, +) + +PLAIN = "chatballs_app_session" +HARDENED = "__Host-chatballs-app-session" +COOKIE_NAMES = {PLAIN: HARDENED, "chatballs_app_csrftoken": "__Host-chatballs-app-csrf"} + + +class _Recorder: + """Внутреннее приложение: запоминает scope, до которого дошёл запрос.""" + + def __init__(self) -> None: + self.scope: dict | None = None + + async def __call__(self, scope, receive, send): + self.scope = scope + + +def _scope(*, scheme: str = "wss", cookie: str = "", origin: str = "", host: str = "app.example") -> dict: + headers = [(b"host", host.encode())] + if cookie: + headers.append((b"cookie", cookie.encode())) + if origin: + headers.append((b"origin", origin.encode())) + return {"type": "websocket", "scheme": scheme, "headers": headers} + + +def _cookies(scope: dict) -> dict[str, str]: + raw = next((value for key, value in scope["headers"] if key == b"cookie"), b"") + items = [part.strip() for part in raw.decode().split(";") if part.strip()] + return dict(item.split("=", 1) for item in items) + + +@override_settings(CHATBALLS_TLS_COOKIE_NAMES=COOKIE_NAMES) +class TlsAwareCookieASGIMiddlewareTests(SimpleTestCase): + def _run(self, scope: dict) -> dict: + inner = _Recorder() + asyncio.run(TlsAwareCookieASGIMiddleware(inner)(scope, None, None)) + assert inner.scope is not None + return inner.scope + + def test_hardened_cookie_is_delivered_under_the_plain_name(self) -> None: + scope = self._run(_scope(cookie=f"{HARDENED}=session-value")) + + self.assertEqual(_cookies(scope)[PLAIN], "session-value") + + def test_plain_cookie_alone_is_dropped_over_tls(self) -> None: + """По TLS обычное имя мы не выдаём — значит пришло оно не от нас.""" + scope = self._run(_scope(cookie=f"{PLAIN}=planted")) + + self.assertNotIn(PLAIN, _cookies(scope)) + + def test_hardened_cookie_wins_over_a_planted_plain_one(self) -> None: + scope = self._run(_scope(cookie=f"{PLAIN}=planted; {HARDENED}=real")) + + self.assertEqual(_cookies(scope)[PLAIN], "real") + + def test_plain_http_scope_is_untouched(self) -> None: + scope = self._run(_scope(scheme="ws", cookie=f"{PLAIN}=session-value")) + + self.assertEqual(_cookies(scope)[PLAIN], "session-value") + + def test_scope_without_cookies_passes_through(self) -> None: + scope = self._run(_scope()) + + self.assertEqual(_cookies(scope), {}) + + +class SameOriginWebSocketMiddlewareTests(SimpleTestCase): + def _run(self, scope: dict) -> tuple[dict | None, list[dict]]: + inner = _Recorder() + sent: list[dict] = [] + + async def send(message): + sent.append(message) + + asyncio.run(SameOriginWebSocketMiddleware(inner)(scope, None, send)) + return inner.scope, sent + + def test_same_origin_handshake_passes(self) -> None: + scope, sent = self._run(_scope(origin="https://app.example", host="app.example")) + + self.assertIsNotNone(scope) + self.assertEqual(sent, []) + + def test_foreign_origin_is_closed(self) -> None: + scope, sent = self._run(_scope(origin="https://evil.example", host="app.example")) + + self.assertIsNone(scope) + self.assertEqual(sent, [{"type": "websocket.close", "code": 4403}]) + + def test_origin_with_matching_port_passes(self) -> None: + scope, sent = self._run( + _scope(origin="http://localhost:5173", host="localhost:5173") + ) + + self.assertIsNotNone(scope) + self.assertEqual(sent, []) + + def test_client_without_origin_is_allowed(self) -> None: + """Origin шлёт браузер; клиенты без него — не то, от чего мы защищаемся.""" + scope, sent = self._run(_scope(host="app.example")) + + self.assertIsNotNone(scope) + self.assertEqual(sent, []) diff --git a/apps/backend/chatballs/http/ws_middleware.py b/apps/backend/chatballs/http/ws_middleware.py new file mode 100644 index 0000000..39281c6 --- /dev/null +++ b/apps/backend/chatballs/http/ws_middleware.py @@ -0,0 +1,117 @@ +"""ASGI-обвязка WebSocket: имена cookie по факту TLS и проверка Origin. + +HTTP-слой продукта переименовывает cookie по протоколу запроса +(``chatballs.http.middleware.TlsAwareCookieMiddleware``): по https браузер +держит ``__Host-…``, по http — обычное имя. Django-middleware на +WebSocket-хендшейк не выполняется, а Channels ищет cookie строго по +``settings.SESSION_COOKIE_NAME``. Из-за этого на установке с TLS сокет не +находил сессию вообще: браузер присылал только защищённое имя, и каждое +подключение закрывалось как неаутентифицированное — живые обновления диалогов +молча переставали работать. + +Здесь то же правило применяется к scope до ``AuthMiddlewareStack``. +""" + +from __future__ import annotations + +from collections.abc import Callable +from http.cookies import SimpleCookie +from urllib.parse import urlsplit + +from django.conf import settings + + +def _tls_cookie_pairs() -> tuple[tuple[str, str], ...]: + mapping = getattr(settings, "CHATBALLS_TLS_COOKIE_NAMES", {}) + return tuple((plain, hardened) for plain, hardened in mapping.items() if plain != hardened) + + +def _header(scope: dict, name: bytes) -> bytes: + for key, value in scope.get("headers") or (): + if key.lower() == name: + return value + return b"" + + +def _replace_header(scope: dict, name: bytes, value: bytes) -> None: + headers = [(key, item) for key, item in (scope.get("headers") or ()) if key.lower() != name] + if value: + headers.append((name, value)) + scope["headers"] = headers + + +class TlsAwareCookieASGIMiddleware: + """По wss отдаёт вглубь защищённые cookie под обычными именами.""" + + def __init__(self, inner: Callable) -> None: + self.inner = inner + + async def __call__(self, scope, receive, send): + if scope.get("type") == "websocket" and scope.get("scheme") in {"wss", "https"}: + self._rewrite(scope) + return await self.inner(scope, receive, send) + + @staticmethod + def _rewrite(scope: dict) -> None: + pairs = _tls_cookie_pairs() + if not pairs: + return + raw = _header(scope, b"cookie") + if not raw: + return + jar = SimpleCookie() + jar.load(raw.decode("latin-1")) + values = {key: morsel.value for key, morsel in jar.items()} + changed = False + for plain, hardened in pairs: + if hardened in values: + # Защищённое имя всегда сильнее обычного — то же правило, что и + # в HTTP-слое: cookie с префиксом __Host- браузер принимает + # только с самого хоста и только по TLS. + if values.get(plain) != values[hardened]: + values[plain] = values[hardened] + changed = True + elif plain in values: + # По TLS обычное имя мы не выдаём — значит пришло не от нас. + del values[plain] + changed = True + if not changed: + return + rebuilt = "; ".join(f"{key}={value}" for key, value in values.items()) + _replace_header(scope, b"cookie", rebuilt.encode("latin-1")) + + +class SameOriginWebSocketMiddleware: + """Отклоняет хендшейк, у которого Origin не совпадает с Host. + + Сокет открывает то же SPA, что и REST, поэтому Origin у него всегда наш. + Кросс-сайтовый запрос сейчас и так остаётся без cookie (``SameSite=Lax``), + но полагаться на один барьер не стоит: у cookie этот флаг настраиваемый. + Клиенты без браузера Origin не присылают — их не трогаем. + """ + + def __init__(self, inner: Callable) -> None: + self.inner = inner + + async def __call__(self, scope, receive, send): + if scope.get("type") == "websocket" and not self._allowed(scope): + await send({"type": "websocket.close", "code": 4403}) + return + return await self.inner(scope, receive, send) + + @staticmethod + def _allowed(scope: dict) -> bool: + origin = _header(scope, b"origin").decode("latin-1").strip() + if not origin: + return True + host = _header(scope, b"host").decode("latin-1").strip().lower() + if not host: + return False + origin_host = (urlsplit(origin).netloc or "").lower() + return origin_host == host + + +def websocket_boundary(inner: Callable) -> Callable: + """Обе проверки одним вызовом — порядок важен, Origin проверяется первым.""" + + return SameOriginWebSocketMiddleware(TlsAwareCookieASGIMiddleware(inner)) diff --git a/apps/backend/chatballs/identity/administration_payloads.py b/apps/backend/chatballs/identity/administration_payloads.py index a082920..54548e7 100644 --- a/apps/backend/chatballs/identity/administration_payloads.py +++ b/apps/backend/chatballs/identity/administration_payloads.py @@ -13,7 +13,6 @@ from chatballs.identity.audit_catalog import ( ) from chatballs.identity.models import AuditEvent, Organization - ORGANIZATION_CHANGE_ACTIONS = ( "administration.organization_updated", "administration.logo_updated", diff --git a/apps/backend/chatballs/identity/administration_services.py b/apps/backend/chatballs/identity/administration_services.py index a6c586f..b13e3c7 100644 --- a/apps/backend/chatballs/identity/administration_services.py +++ b/apps/backend/chatballs/identity/administration_services.py @@ -18,7 +18,6 @@ from chatballs.tenancy.storage_quota import ( reserve_storage, ) - MAX_LOGO_BYTES = 2 * 1024 * 1024 diff --git a/apps/backend/chatballs/identity/administration_views.py b/apps/backend/chatballs/identity/administration_views.py index 2524546..d9dec80 100644 --- a/apps/backend/chatballs/identity/administration_views.py +++ b/apps/backend/chatballs/identity/administration_views.py @@ -4,8 +4,8 @@ from datetime import datetime, time, timedelta from django.core.exceptions import ValidationError from django.db.models import Q -from django.utils import timezone as django_timezone from django.http import FileResponse +from django.utils import timezone as django_timezone from rest_framework.parsers import FormParser, MultiPartParser from rest_framework.permissions import IsAuthenticated from rest_framework.request import Request @@ -50,7 +50,6 @@ class OrganizationSettingsView(APIView): "GET": "settings.view", "PATCH": "settings.manage", } - require_organization_scope = True def get(self, request: Request) -> Response: return Response( @@ -93,7 +92,6 @@ class OrganizationLogoView(APIView): "POST": "settings.manage", "DELETE": "settings.manage", } - require_organization_scope = True def get_permissions(self): if self.request.method == "GET": @@ -173,7 +171,6 @@ class AuditListView(APIView): permission_classes = [HasCapability] required_capability = "audit.view" - require_organization_scope = True def get(self, request: Request) -> Response: organization_id = request.tenant_context.organization_id diff --git a/apps/backend/chatballs/identity/auth/__init__.py b/apps/backend/chatballs/identity/auth/__init__.py index 1b5d296..f8d344d 100644 --- a/apps/backend/chatballs/identity/auth/__init__.py +++ b/apps/backend/chatballs/identity/auth/__init__.py @@ -5,9 +5,9 @@ from chatballs.identity.auth.password_reset import ( PasswordResetValidateView, ) from chatballs.identity.auth.profile import ( + ChangeTemporaryPasswordView, ProfileAppearanceView, ProfileAvatarView, - ChangeTemporaryPasswordView, ProfilePasswordView, ProfileRevokeOtherSessionsView, ProfileSessionsView, @@ -26,6 +26,8 @@ __all__ = [ "PasswordResetValidateView", "PasswordResetConfirmView", "ProfileUpdateView", + "ProfileAppearanceView", + "ProfileAvatarView", "ProfilePasswordView", "ProfileTotpStartView", "ProfileTotpDisableView", diff --git a/apps/backend/chatballs/identity/auth/password_reset.py b/apps/backend/chatballs/identity/auth/password_reset.py index 6e9d631..460c533 100644 --- a/apps/backend/chatballs/identity/auth/password_reset.py +++ b/apps/backend/chatballs/identity/auth/password_reset.py @@ -16,6 +16,7 @@ from chatballs.events.services import DomainEvent, enqueue_event from chatballs.identity.audit import record_audit_event from chatballs.identity.event_handlers import PASSWORD_RESET_REQUESTED from chatballs.identity.models import AuditResult, HumanUser +from chatballs.identity.sessions import revoke_user_sessions def _user_from_reset_link(uid: str, token: str) -> HumanUser | None: @@ -98,9 +99,15 @@ class PasswordResetConfirmView(APIView): user.must_change_password = False user.password_changed_at = timezone.now() user.save(update_fields=["password", "must_change_password", "password_changed_at"]) + # Пароль сбрасывают именно тогда, когда доступ к учётной записи мог + # оказаться у чужого. Оставить его сессии живыми — значит не сделать + # ничего: смена пароля из профиля и админский сброс их завершают, + # этот путь обязан вести себя так же. + revoked = revoke_user_sessions(user.id) record_audit_event( action="identity.password_reset_completed", actor=user, + payload={"revoked": revoked}, request=request, ) - return Response({"ok": True}) + return Response({"ok": True, "revoked": revoked}) diff --git a/apps/backend/chatballs/identity/auth/profile.py b/apps/backend/chatballs/identity/auth/profile.py index f064486..e982963 100644 --- a/apps/backend/chatballs/identity/auth/profile.py +++ b/apps/backend/chatballs/identity/auth/profile.py @@ -11,10 +11,10 @@ from rest_framework.views import APIView from chatballs.identity.audit import record_audit_event from chatballs.identity.auth.common import _revoke_other_user_sessions, _user_payload -from chatballs.identity.sessions import list_user_sessions -from chatballs.tenancy.ingress import user_requires_totp from chatballs.identity.avatars import delete_user_avatar, replace_user_avatar from chatballs.identity.models import HumanUser +from chatballs.identity.sessions import list_user_sessions +from chatballs.tenancy.ingress import user_requires_totp class ProfileUpdateView(APIView): @@ -115,12 +115,24 @@ class ProfilePasswordView(APIView): class ProfileTotpStartView(APIView): + """Начало настройки 2FA: выдать пользователю новый секрет. + + Выключать этим уже включённую 2FA нельзя. Иначе достаточно было бы + угнанной сессии: отключение (``ProfileTotpDisableView``) спрашивает + текущий пароль и не даёт обойти требование организации, а этот эндпоинт + молча делал ровно то же самое без единой проверки. + """ + permission_classes = [IsAuthenticated] def post(self, request: Request) -> Response: - request.user.totp_enabled = False + if request.user.totp_enabled: + return Response( + {"detail": "TOTP уже включена: сначала отключите её текущим паролем"}, + status=409, + ) request.user.totp_secret = "" - request.user.save(update_fields=["totp_enabled", "totp_secret"]) + request.user.save(update_fields=["totp_secret"]) record_audit_event( action="identity.profile_totp_setup_started", actor=request.user, diff --git a/apps/backend/chatballs/identity/crypto.py b/apps/backend/chatballs/identity/crypto.py index 7dcf078..c064b88 100644 --- a/apps/backend/chatballs/identity/crypto.py +++ b/apps/backend/chatballs/identity/crypto.py @@ -59,8 +59,34 @@ def decrypt_secret(token: str) -> str: return "" +def ciphertext_length(plaintext_chars: int) -> int: + """Сколько символов занимает Fernet-токен для строки такой длины. + + В колонке лежит не значение, а шифротекст: Fernet добавляет версию, + метку времени, IV и подпись, дополняет до блока AES и кодирует всё в + base64. Строка в 512 символов уже не помещается в varchar(512) — запись + падала бы на длинном пароле SMTP или ключе S3. Считаем по худшему случаю: + 4 байта на символ (UTF-8). + """ + payload = plaintext_chars * 4 + blocks = payload // 16 + 1 # PKCS#7 всегда добавляет хотя бы один байт + raw = 57 + 16 * blocks # 57 = версия + timestamp + IV + HMAC + return (raw + 2) // 3 * 4 # base64 без переносов + + class EncryptedCharField(models.CharField): - """CharField прозрачно шифрующий значение в БД (Fernet).""" + """CharField, прозрачно шифрующий значение в БД (Fernet). + + ``max_length`` описывает открытое значение — то, что вводит человек. Под + колонку берётся длина шифротекста: иначе ограничение поля и ограничение + столбца означают разное, и запись падает уже в базе. + """ + + def __init__(self, *args, **kwargs) -> None: + self.plaintext_max_length = kwargs.get("max_length") + if self.plaintext_max_length: + kwargs["max_length"] = ciphertext_length(self.plaintext_max_length) + super().__init__(*args, **kwargs) def from_db_value(self, value, expression, connection): # noqa: ANN001 if not value: diff --git a/apps/backend/chatballs/identity/demo_seed/loaders/conversations.py b/apps/backend/chatballs/identity/demo_seed/loaders/conversations.py index a53662c..7f853a8 100644 --- a/apps/backend/chatballs/identity/demo_seed/loaders/conversations.py +++ b/apps/backend/chatballs/identity/demo_seed/loaders/conversations.py @@ -11,8 +11,8 @@ from django.core.files.base import ContentFile from chatballs.conversations.models import ( ConnectionIdentity, Contact, - ConversationLabel, Conversation, + ConversationLabel, ConversationRead, Message, MessageAuthor, @@ -147,10 +147,11 @@ def _ensure_conversation(context: TenantContext, refs: DemoRefs, item: dict, cur channel = refs.channels[item["agent"]] connection = refs.integrations[item["connection"]] if item.get("webGuest"): - contact, identity, external_chat_id = _web_guest(context, refs, item) + # Identity гостя заводит _web_guest; диалогу она не нужна — его + # единственный источник identity это контакт (Conversation.contact). + contact, _identity, external_chat_id = _web_guest(context, refs, item) else: contact = refs.contacts.get(item.get("contact")) - identity = refs.identities.get(item.get("identity")) external_chat_id = item["externalChatId"] existing = Conversation.objects.filter( diff --git a/apps/backend/chatballs/identity/demo_views.py b/apps/backend/chatballs/identity/demo_views.py index abf463b..bc913f9 100644 --- a/apps/backend/chatballs/identity/demo_views.py +++ b/apps/backend/chatballs/identity/demo_views.py @@ -30,7 +30,6 @@ class DemoDataView(APIView): "POST": "company.manage", "DELETE": "company.manage", } - require_organization_scope = True def get(self, request: Request) -> Response: return Response(service.demo_status(request.tenant_context.organization)) diff --git a/apps/backend/chatballs/identity/employee_support.py b/apps/backend/chatballs/identity/employee_support.py index 3c18f1a..a78c942 100644 --- a/apps/backend/chatballs/identity/employee_support.py +++ b/apps/backend/chatballs/identity/employee_support.py @@ -1,7 +1,7 @@ from rest_framework.request import Request -from chatballs.identity.governance import employee_management_flags from chatballs.identity.avatars import user_avatar_url +from chatballs.identity.governance import employee_management_flags from chatballs.identity.models import AuditEvent, OrganizationMembership diff --git a/apps/backend/chatballs/identity/employee_views.py b/apps/backend/chatballs/identity/employee_views.py index b1f9b65..cd7f40a 100644 --- a/apps/backend/chatballs/identity/employee_views.py +++ b/apps/backend/chatballs/identity/employee_views.py @@ -8,6 +8,7 @@ from rest_framework.views import APIView from chatballs.api.pagination import page_payload, paginate from chatballs.events.services import DomainEvent, enqueue_event from chatballs.identity.audit import record_audit_event +from chatballs.identity.employee_password import clean_password_mode, issue_initial_password from chatballs.identity.employee_selectors import employees_for from chatballs.identity.employee_support import employee_payload, get_owned_profile from chatballs.identity.employee_validation import ( @@ -16,7 +17,6 @@ from chatballs.identity.employee_validation import ( deny_employee_action, resolve_groups, ) -from chatballs.identity.employee_password import clean_password_mode, issue_initial_password from chatballs.identity.event_handlers import INITIAL_ACCESS_REQUESTED from chatballs.identity.governance import EmployeeAction, can_create_role, can_manage_employee from chatballs.identity.group_models import EmployeeGroupMember diff --git a/apps/backend/chatballs/identity/group_models.py b/apps/backend/chatballs/identity/group_models.py index 5e3913d..c04814d 100644 --- a/apps/backend/chatballs/identity/group_models.py +++ b/apps/backend/chatballs/identity/group_models.py @@ -29,17 +29,17 @@ class EmployeeGroup(models.Model): ) ] - def clean(self) -> None: - self.name = self.name.strip() - if not self.name: - raise ValidationError({"name": "Group name is required"}) + def __str__(self) -> str: + return f"{self.organization.slug}/{self.name}" def save(self, *args, **kwargs) -> None: self.full_clean() super().save(*args, **kwargs) - def __str__(self) -> str: - return f"{self.organization.slug}/{self.name}" + def clean(self) -> None: + self.name = self.name.strip() + if not self.name: + raise ValidationError({"name": "Group name is required"}) class EmployeeGroupMember(TenantRelationModel): diff --git a/apps/backend/chatballs/identity/instance_settings.py b/apps/backend/chatballs/identity/instance_settings.py index ecd43b4..67f74ea 100644 --- a/apps/backend/chatballs/identity/instance_settings.py +++ b/apps/backend/chatballs/identity/instance_settings.py @@ -24,6 +24,12 @@ class InstanceSettings(models.Model): # Хост без схемы и порта: «crm.example.com» или «203.0.113.10». public_host = models.CharField(max_length=253, blank=True, default="") + # Предыдущий адрес: остаётся принятым, чтобы смена адреса не выбрасывала + # того, кто её делает. Владелец меняет адрес заранее — до того, как домен + # начал резолвиться и получил сертификат, — и сидит при этом на старом. + # Без этого он получал «Invalid host» через десять секунд после + # сохранения, а мастер уже закрыт: вернуться было бы неоткуда. + previous_public_host = models.CharField(max_length=253, blank=True, default="") # Схема, по которой установку открывают снаружи. Меняется вместе с # адресом, когда перед установкой появляется домен и сертификат. public_scheme = models.CharField(max_length=5, blank=True, default="") @@ -62,7 +68,7 @@ class InstanceSettings(models.Model): _CACHE_TTL_SECONDS = 10.0 _lock = threading.Lock() -_cached: tuple[float, str] | None = None +_cached: tuple[float, tuple[str, str]] | None = None def invalidate_cache() -> None: @@ -71,8 +77,8 @@ def invalidate_cache() -> None: _cached = None -def public_host() -> str: - """Адрес установки, запомненный мастером, или пустая строка.""" +def _hosts() -> tuple[str, str]: + """Текущий и предыдущий адрес установки (оба могут быть пустыми).""" global _cached now = time.monotonic() @@ -81,14 +87,26 @@ def public_host() -> str: return _cached[1] try: row = InstanceSettings.objects.filter(pk=InstanceSettings.SINGLETON_PK).first() - value = row.public_host if row is not None else "" + value = (row.public_host, row.previous_public_host) if row is not None else ("", "") except Exception: # таблицы ещё нет (первые миграции) - return "" + return ("", "") with _lock: _cached = (now, value) return value +def public_host() -> str: + """Адрес установки, запомненный мастером, или пустая строка.""" + + return _hosts()[0] + + +def accepted_hosts() -> tuple[str, ...]: + """Адреса, которые установка признаёт своими: текущий и предыдущий.""" + + return tuple(host for host in _hosts() if host) + + def remember_public_host(raw_host: str, scheme: str = "http") -> None: """Запомнить адрес, на котором прошли мастер, если он ещё не задан.""" diff --git a/apps/backend/chatballs/identity/instance_views.py b/apps/backend/chatballs/identity/instance_views.py index 4b263c2..b52ddd6 100644 --- a/apps/backend/chatballs/identity/instance_views.py +++ b/apps/backend/chatballs/identity/instance_views.py @@ -88,7 +88,11 @@ class InstanceAddressView(APIView): {"detail": next(iter(errors.values())), "errors": errors}, status=400 ) - fields = ["public_host", "public_scheme", "updated_at"] + fields = ["public_host", "public_scheme", "previous_public_host", "updated_at"] + if host != row.public_host: + # Прежний адрес остаётся принятым: владелец меняет адрес заранее, + # сидя на старом, и не должен выпасть из установки в тот же миг. + row.previous_public_host = row.public_host row.public_host = host row.public_scheme = scheme diff --git a/apps/backend/chatballs/identity/migrations/0016_channel_capabilities.py b/apps/backend/chatballs/identity/migrations/0016_channel_capabilities.py index 4f7abe2..19cfbc0 100644 --- a/apps/backend/chatballs/identity/migrations/0016_channel_capabilities.py +++ b/apps/backend/chatballs/identity/migrations/0016_channel_capabilities.py @@ -1,82 +1,82 @@ -# SPEC-HUB-0027 §5.1/§5.3, ADR-HUB-0037 §9 — этап 1. -# -# Вводит `channels.view` / `channels.manage` и выдаёт их существующим профилям -# доступа по текущим `ai.view` / `ai.manage`, чтобы никто не потерял доступ в -# момент выката. -# -# Scope в этой модели живёт на `EmployeeAccessAssignment`, а capability — на -# `AccessProfile`. Поэтому «с тем же scope» достигается тем, что мы вообще не -# трогаем назначения: каждое действующее назначение профиля продолжает работать -# со своим scope. Обе новые capability допускают ORGANIZATION и DEPARTMENT, так -# что набор допустимых scope профиля (`allowed_profile_scopes`) не сужается. -from django.db import migrations, models - -AI_VIEW = "ai.view" -AI_MANAGE = "ai.manage" -CHANNELS_VIEW = "channels.view" -CHANNELS_MANAGE = "channels.manage" - - -def _grant(apps, *, source: str, target: str) -> None: - AccessProfile = apps.get_model("identity", "AccessProfile") - AccessProfileCapability = apps.get_model("identity", "AccessProfileCapability") - - already_granted = set( - AccessProfileCapability.objects.filter(capability_code=target).values_list( - "access_profile_id", flat=True - ) - ) - # organization_id берётся у профиля, а не выводится: триггер - # chatballs.enforce_tenant_fk требует совпадения с владельцем профиля. - profiles = ( - AccessProfile.objects.filter(capability_links__capability_code=source) - .values_list("id", "organization_id") - .distinct() - ) - AccessProfileCapability.objects.bulk_create( - [ - AccessProfileCapability( - access_profile_id=profile_id, - organization_id=organization_id, - capability_code=target, - ) - for profile_id, organization_id in profiles - if profile_id not in already_granted - ] - ) - - -def grant_channel_capabilities(apps, schema_editor): - _grant(apps, source=AI_VIEW, target=CHANNELS_VIEW) - _grant(apps, source=AI_MANAGE, target=CHANNELS_MANAGE) - - -def revoke_channel_capabilities(apps, schema_editor): - # В отличие от 0010 откат обязан удалить выданные строки: следом - # восстанавливается прежний check-constraint реестра, и строки с кодом вне - # списка сделали бы обратную миграцию невыполнимой. - AccessProfileCapability = apps.get_model("identity", "AccessProfileCapability") - AccessProfileCapability.objects.filter( - capability_code__in=(CHANNELS_VIEW, CHANNELS_MANAGE) - ).delete() - - -class Migration(migrations.Migration): - - dependencies = [ - ('identity', '0015_organization_status'), - ] - - operations = [ - migrations.RemoveConstraint( - model_name='accessprofilecapability', - name='access_profile_capability_registry', - ), - migrations.AddConstraint( - model_name='accessprofilecapability', - constraint=models.CheckConstraint(condition=models.Q(('capability_code__in', ['company.view', 'company.manage', 'departments.view', 'departments.manage', 'employees.view', 'employees.manage', 'products.view', 'products.manage', 'channels.view', 'channels.manage', 'ai.view', 'ai.manage', 'ai.publish', 'integrations.view', 'integrations.manage', 'secrets.manage', 'settings.view', 'settings.manage', 'audit.view', 'conversations.view', 'conversations.operate', 'conversations.call', 'customers.view', 'customers.manage', 'sales.view', 'sales.operate', 'sales.correct', 'sales_sources.manage', 'support.view', 'support.operate', 'notifications.manage'])), name='access_profile_capability_registry'), - ), - # Порядок важен: при откате Django исполняет операции в обратном порядке, - # поэтому строки удаляются до восстановления старого constraint. - migrations.RunPython(grant_channel_capabilities, revoke_channel_capabilities), - ] +# SPEC-HUB-0027 §5.1/§5.3, ADR-HUB-0037 §9 — этап 1. +# +# Вводит `channels.view` / `channels.manage` и выдаёт их существующим профилям +# доступа по текущим `ai.view` / `ai.manage`, чтобы никто не потерял доступ в +# момент выката. +# +# Scope в этой модели живёт на `EmployeeAccessAssignment`, а capability — на +# `AccessProfile`. Поэтому «с тем же scope» достигается тем, что мы вообще не +# трогаем назначения: каждое действующее назначение профиля продолжает работать +# со своим scope. Обе новые capability допускают ORGANIZATION и DEPARTMENT, так +# что набор допустимых scope профиля (`allowed_profile_scopes`) не сужается. +from django.db import migrations, models + +AI_VIEW = "ai.view" +AI_MANAGE = "ai.manage" +CHANNELS_VIEW = "channels.view" +CHANNELS_MANAGE = "channels.manage" + + +def _grant(apps, *, source: str, target: str) -> None: + AccessProfile = apps.get_model("identity", "AccessProfile") + AccessProfileCapability = apps.get_model("identity", "AccessProfileCapability") + + already_granted = set( + AccessProfileCapability.objects.filter(capability_code=target).values_list( + "access_profile_id", flat=True + ) + ) + # organization_id берётся у профиля, а не выводится: триггер + # chatballs.enforce_tenant_fk требует совпадения с владельцем профиля. + profiles = ( + AccessProfile.objects.filter(capability_links__capability_code=source) + .values_list("id", "organization_id") + .distinct() + ) + AccessProfileCapability.objects.bulk_create( + [ + AccessProfileCapability( + access_profile_id=profile_id, + organization_id=organization_id, + capability_code=target, + ) + for profile_id, organization_id in profiles + if profile_id not in already_granted + ] + ) + + +def grant_channel_capabilities(apps, schema_editor): + _grant(apps, source=AI_VIEW, target=CHANNELS_VIEW) + _grant(apps, source=AI_MANAGE, target=CHANNELS_MANAGE) + + +def revoke_channel_capabilities(apps, schema_editor): + # В отличие от 0010 откат обязан удалить выданные строки: следом + # восстанавливается прежний check-constraint реестра, и строки с кодом вне + # списка сделали бы обратную миграцию невыполнимой. + AccessProfileCapability = apps.get_model("identity", "AccessProfileCapability") + AccessProfileCapability.objects.filter( + capability_code__in=(CHANNELS_VIEW, CHANNELS_MANAGE) + ).delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ('identity', '0015_organization_status'), + ] + + operations = [ + migrations.RemoveConstraint( + model_name='accessprofilecapability', + name='access_profile_capability_registry', + ), + migrations.AddConstraint( + model_name='accessprofilecapability', + constraint=models.CheckConstraint(condition=models.Q(('capability_code__in', ['company.view', 'company.manage', 'departments.view', 'departments.manage', 'employees.view', 'employees.manage', 'products.view', 'products.manage', 'channels.view', 'channels.manage', 'ai.view', 'ai.manage', 'ai.publish', 'integrations.view', 'integrations.manage', 'secrets.manage', 'settings.view', 'settings.manage', 'audit.view', 'conversations.view', 'conversations.operate', 'conversations.call', 'customers.view', 'customers.manage', 'sales.view', 'sales.operate', 'sales.correct', 'sales_sources.manage', 'support.view', 'support.operate', 'notifications.manage'])), name='access_profile_capability_registry'), + ), + # Порядок важен: при откате Django исполняет операции в обратном порядке, + # поэтому строки удаляются до восстановления старого constraint. + migrations.RunPython(grant_channel_capabilities, revoke_channel_capabilities), + ] diff --git a/apps/backend/chatballs/identity/migrations/0020_employeegroup_employeegroupmember_and_more.py b/apps/backend/chatballs/identity/migrations/0020_employeegroup_employeegroupmember_and_more.py index 1deb3eb..cd522ca 100644 --- a/apps/backend/chatballs/identity/migrations/0020_employeegroup_employeegroupmember_and_more.py +++ b/apps/backend/chatballs/identity/migrations/0020_employeegroup_employeegroupmember_and_more.py @@ -1,173 +1,173 @@ -# Generated by Django 5.2.15 on 2026-09-03 22:23 - -import django.db.models.deletion -import django.db.models.functions.text -from django.db import migrations, models - -# При откате Django пересоздаёт снесённые таблицы «голыми» — без ownership и -# грантов, которые исходно раздавала tenancy/0003 (она при откате не -# переприменяется). SECURITY DEFINER-триггеры (enforce_tenant_fk) тогда не могут -# читать identity_department и migration-тесты падают на старых состояниях. -# Первый operation ниже — noop вперёд; его reverse выполняется ПОСЛЕДНИМ при -# откате (операции разворачиваются в обратном порядке), когда таблицы уже -# пересозданы, и возвращает им владельца и гранты. RLS на старых состояниях -# тестами не используется, поэтому политики не восстанавливаем. -_RESTORE_GRANTS_SQL = "\n".join( - f""" -ALTER TABLE {table} OWNER TO chatballs_schema; -GRANT ALL ON {table} TO chatballs_schema; -GRANT SELECT, INSERT, UPDATE, DELETE ON {table} TO chatballs_runtime_app; -""" - for table in ( - "identity_department", - "identity_accessprofile", - "identity_accessprofilecapability", - "identity_employeeaccessassignment", - ) -) - - -class Migration(migrations.Migration): - - dependencies = [ - ('ai', '0014_remove_knowledgedepartment_department_and_more'), - ('channels', '0006_remove_channel_department'), - ('identity', '0019_drop_sales_capabilities'), - ('notifications', '0008_remove_notification_department'), - ('support_portals', '0008_remove_supportportal_department'), - ] - - operations = [ - migrations.RunSQL(migrations.RunSQL.noop, _RESTORE_GRANTS_SQL), - migrations.CreateModel( - name='EmployeeGroup', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=120)), - ('created_at', models.DateTimeField(auto_now_add=True)), - ], - options={ - 'ordering': ['name'], - }, - ), - migrations.CreateModel( - name='EmployeeGroupMember', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('created_at', models.DateTimeField(auto_now_add=True)), - ], - ), - # Снимаем constraint'ы удаляемых моделей до удаления их полей: иначе - # state хранит constraint на несуществующее поле и обратная миграция - # (reverse DeleteModel в migration-тестах) падает при create_model. - migrations.RemoveConstraint( - model_name='accessprofile', - name='uniq_access_profile_org_name_ci', - ), - migrations.RemoveConstraint( - model_name='accessprofilecapability', - name='uniq_access_profile_capability', - ), - migrations.RemoveConstraint( - model_name='department', - name='uniq_department_org_code', - ), - migrations.RemoveConstraint( - model_name='employeeaccessassignment', - name='access_assignment_scope_department', - ), - migrations.RemoveConstraint( - model_name='employeeaccessassignment', - name='uniq_active_org_access_assignment', - ), - migrations.RemoveConstraint( - model_name='employeeaccessassignment', - name='uniq_active_dept_access_assignment', - ), - migrations.RemoveField( - model_name='accessprofile', - name='organization', - ), - migrations.RemoveField( - model_name='accessprofilecapability', - name='access_profile', - ), - migrations.RemoveField( - model_name='employeeaccessassignment', - name='access_profile', - ), - migrations.RemoveField( - model_name='accessprofilecapability', - name='organization', - ), - migrations.RemoveField( - model_name='department', - name='organization', - ), - # Сначала снимаем constraint, зависящий от primary_department: дроп - # колонки удалил бы его каскадно и RemoveConstraint ниже упал бы. - migrations.RemoveConstraint( - model_name='organizationmembership', - name='owner_is_company_level', - ), - migrations.RemoveField( - model_name='organizationmembership', - name='primary_department', - ), - migrations.RemoveField( - model_name='employeeaccessassignment', - name='department', - ), - migrations.RemoveField( - model_name='employeeaccessassignment', - name='assigned_by', - ), - migrations.RemoveField( - model_name='employeeaccessassignment', - name='employee', - ), - migrations.RemoveField( - model_name='employeeaccessassignment', - name='organization', - ), - migrations.AddField( - model_name='employeegroup', - name='organization', - field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='employee_groups', to='identity.organization'), - ), - migrations.AddField( - model_name='employeegroupmember', - name='employee', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='group_links', to='identity.organizationmembership'), - ), - migrations.AddField( - model_name='employeegroupmember', - name='group', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='member_links', to='identity.employeegroup'), - ), - migrations.AddField( - model_name='employeegroupmember', - name='organization', - field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='+', to='identity.organization'), - ), - migrations.DeleteModel( - name='AccessProfile', - ), - migrations.DeleteModel( - name='AccessProfileCapability', - ), - migrations.DeleteModel( - name='Department', - ), - migrations.DeleteModel( - name='EmployeeAccessAssignment', - ), - migrations.AddConstraint( - model_name='employeegroup', - constraint=models.UniqueConstraint(django.db.models.functions.text.Lower('name'), models.F('organization'), name='uniq_employee_group_org_name_ci'), - ), - migrations.AddConstraint( - model_name='employeegroupmember', - constraint=models.UniqueConstraint(fields=('group', 'employee'), name='uniq_employee_group_member'), - ), - ] +# Generated by Django 5.2.15 on 2026-09-03 22:23 + +import django.db.models.deletion +import django.db.models.functions.text +from django.db import migrations, models + +# При откате Django пересоздаёт снесённые таблицы «голыми» — без ownership и +# грантов, которые исходно раздавала tenancy/0003 (она при откате не +# переприменяется). SECURITY DEFINER-триггеры (enforce_tenant_fk) тогда не могут +# читать identity_department и migration-тесты падают на старых состояниях. +# Первый operation ниже — noop вперёд; его reverse выполняется ПОСЛЕДНИМ при +# откате (операции разворачиваются в обратном порядке), когда таблицы уже +# пересозданы, и возвращает им владельца и гранты. RLS на старых состояниях +# тестами не используется, поэтому политики не восстанавливаем. +_RESTORE_GRANTS_SQL = "\n".join( + f""" +ALTER TABLE {table} OWNER TO chatballs_schema; +GRANT ALL ON {table} TO chatballs_schema; +GRANT SELECT, INSERT, UPDATE, DELETE ON {table} TO chatballs_runtime_app; +""" + for table in ( + "identity_department", + "identity_accessprofile", + "identity_accessprofilecapability", + "identity_employeeaccessassignment", + ) +) + + +class Migration(migrations.Migration): + + dependencies = [ + ('ai', '0014_remove_knowledgedepartment_department_and_more'), + ('channels', '0006_remove_channel_department'), + ('identity', '0019_drop_sales_capabilities'), + ('notifications', '0008_remove_notification_department'), + ('support_portals', '0008_remove_supportportal_department'), + ] + + operations = [ + migrations.RunSQL(migrations.RunSQL.noop, _RESTORE_GRANTS_SQL), + migrations.CreateModel( + name='EmployeeGroup', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=120)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ], + options={ + 'ordering': ['name'], + }, + ), + migrations.CreateModel( + name='EmployeeGroupMember', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ], + ), + # Снимаем constraint'ы удаляемых моделей до удаления их полей: иначе + # state хранит constraint на несуществующее поле и обратная миграция + # (reverse DeleteModel в migration-тестах) падает при create_model. + migrations.RemoveConstraint( + model_name='accessprofile', + name='uniq_access_profile_org_name_ci', + ), + migrations.RemoveConstraint( + model_name='accessprofilecapability', + name='uniq_access_profile_capability', + ), + migrations.RemoveConstraint( + model_name='department', + name='uniq_department_org_code', + ), + migrations.RemoveConstraint( + model_name='employeeaccessassignment', + name='access_assignment_scope_department', + ), + migrations.RemoveConstraint( + model_name='employeeaccessassignment', + name='uniq_active_org_access_assignment', + ), + migrations.RemoveConstraint( + model_name='employeeaccessassignment', + name='uniq_active_dept_access_assignment', + ), + migrations.RemoveField( + model_name='accessprofile', + name='organization', + ), + migrations.RemoveField( + model_name='accessprofilecapability', + name='access_profile', + ), + migrations.RemoveField( + model_name='employeeaccessassignment', + name='access_profile', + ), + migrations.RemoveField( + model_name='accessprofilecapability', + name='organization', + ), + migrations.RemoveField( + model_name='department', + name='organization', + ), + # Сначала снимаем constraint, зависящий от primary_department: дроп + # колонки удалил бы его каскадно и RemoveConstraint ниже упал бы. + migrations.RemoveConstraint( + model_name='organizationmembership', + name='owner_is_company_level', + ), + migrations.RemoveField( + model_name='organizationmembership', + name='primary_department', + ), + migrations.RemoveField( + model_name='employeeaccessassignment', + name='department', + ), + migrations.RemoveField( + model_name='employeeaccessassignment', + name='assigned_by', + ), + migrations.RemoveField( + model_name='employeeaccessassignment', + name='employee', + ), + migrations.RemoveField( + model_name='employeeaccessassignment', + name='organization', + ), + migrations.AddField( + model_name='employeegroup', + name='organization', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='employee_groups', to='identity.organization'), + ), + migrations.AddField( + model_name='employeegroupmember', + name='employee', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='group_links', to='identity.organizationmembership'), + ), + migrations.AddField( + model_name='employeegroupmember', + name='group', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='member_links', to='identity.employeegroup'), + ), + migrations.AddField( + model_name='employeegroupmember', + name='organization', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='+', to='identity.organization'), + ), + migrations.DeleteModel( + name='AccessProfile', + ), + migrations.DeleteModel( + name='AccessProfileCapability', + ), + migrations.DeleteModel( + name='Department', + ), + migrations.DeleteModel( + name='EmployeeAccessAssignment', + ), + migrations.AddConstraint( + model_name='employeegroup', + constraint=models.UniqueConstraint(django.db.models.functions.text.Lower('name'), models.F('organization'), name='uniq_employee_group_org_name_ci'), + ), + migrations.AddConstraint( + model_name='employeegroupmember', + constraint=models.UniqueConstraint(fields=('group', 'employee'), name='uniq_employee_group_member'), + ), + ] diff --git a/apps/backend/chatballs/identity/migrations/0033_instance_previous_public_host.py b/apps/backend/chatballs/identity/migrations/0033_instance_previous_public_host.py new file mode 100644 index 0000000..7fc44cb --- /dev/null +++ b/apps/backend/chatballs/identity/migrations/0033_instance_previous_public_host.py @@ -0,0 +1,23 @@ +"""Прежний адрес установки остаётся принятым после смены адреса. + +Владелец меняет адрес в «Настройках» заранее — до того, как новый домен начал +резолвиться и получил сертификат, — и сидит при этом на старом. Пока принятым +был только новый адрес, сохранение выбрасывало его из установки через десять +секунд (TTL кэша), а мастер первого запуска уже закрыт: вернуться было неоткуда. +""" + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("identity", "0032_remove_organization_tax_regime_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="instancesettings", + name="previous_public_host", + field=models.CharField(blank=True, default="", max_length=253), + ), + ] diff --git a/apps/backend/chatballs/identity/migrations/0034_encrypted_column_widths.py b/apps/backend/chatballs/identity/migrations/0034_encrypted_column_widths.py new file mode 100644 index 0000000..e364bf5 --- /dev/null +++ b/apps/backend/chatballs/identity/migrations/0034_encrypted_column_widths.py @@ -0,0 +1,34 @@ +"""Колонка вмещает шифротекст, а не открытое значение. + +``EncryptedCharField`` хранит Fernet-токен: версия, метка времени, IV, подпись, +дополнение до блока AES и base64 поверх всего. Значение в 512 символов +занимает почти 2.9 КБ, и в ``varchar(512)`` не помещалось — длинный пароль +SMTP или ключ S3 ронял запись уже в базе. ``max_length`` поля по-прежнему +описывает открытое значение; ширину столбца считает +``chatballs.identity.crypto.ciphertext_length``. +""" + +from django.db import migrations + +import chatballs.identity.crypto + + +class Migration(migrations.Migration): + dependencies = [ + ("identity", "0033_instance_previous_public_host"), + ] + + operations = [ + migrations.AlterField( + model_name="humanuser", + name="totp_secret", + field=chatballs.identity.crypto.EncryptedCharField(blank=True, max_length=1444), + ), + migrations.AlterField( + model_name="instancesettings", + name="email_password", + field=chatballs.identity.crypto.EncryptedCharField( + blank=True, default="", max_length=2828 + ), + ), + ] diff --git a/apps/backend/chatballs/identity/models.py b/apps/backend/chatballs/identity/models.py index e82cd00..9fc9e0a 100644 --- a/apps/backend/chatballs/identity/models.py +++ b/apps/backend/chatballs/identity/models.py @@ -16,7 +16,7 @@ from chatballs.identity.crypto import EncryptedCharField class HumanUserManager(UserManager): use_in_migrations = True - def _create_user(self, email: str, password: str | None, **extra_fields: object) -> "HumanUser": + def _create_user(self, email: str, password: str | None, **extra_fields: object) -> HumanUser: if not email: raise ValueError("The email must be set") email = self.normalize_email(email) @@ -25,12 +25,12 @@ class HumanUserManager(UserManager): user.save(using=self._db) return user - def create_user(self, email: str, password: str | None = None, **extra_fields: object) -> "HumanUser": + def create_user(self, email: str, password: str | None = None, **extra_fields: object) -> HumanUser: extra_fields.setdefault("is_staff", False) extra_fields.setdefault("is_superuser", False) return self._create_user(email, password, **extra_fields) - def create_superuser(self, email: str, password: str | None = None, **extra_fields: object) -> "HumanUser": + def create_superuser(self, email: str, password: str | None = None, **extra_fields: object) -> HumanUser: extra_fields.setdefault("is_staff", True) extra_fields.setdefault("is_superuser", True) if extra_fields.get("is_staff") is not True: diff --git a/apps/backend/chatballs/identity/test_auth_hardening.py b/apps/backend/chatballs/identity/test_auth_hardening.py new file mode 100644 index 0000000..c0b2a86 --- /dev/null +++ b/apps/backend/chatballs/identity/test_auth_hardening.py @@ -0,0 +1,133 @@ +"""Границы, которые обходились через соседний эндпоинт. + +Два места вели себя не так, как обещает соседняя ручка того же экрана: + +- начало настройки 2FA молча выключало уже включённую, минуя и пароль, и + требование организации, — то есть было бесплатным способом снять 2FA для + того, у кого уже есть чужая сессия; +- сброс пароля по письму оставлял чужие сессии живыми, хотя пароль сбрасывают + как раз тогда, когда доступ мог оказаться у чужого. +""" + +from __future__ import annotations + +from django.contrib.auth.tokens import default_token_generator +from django.contrib.sessions.models import Session +from django.test import TestCase +from django.utils.encoding import force_bytes +from django.utils.http import urlsafe_base64_encode + +from chatballs.identity.auth.totp_utils import _generate_totp_secret +from chatballs.identity.bootstrap import bootstrap_owner +from chatballs.testing import TenantAPIClient + +PASSWORD = "Owner-Password-2026!" +NEW_PASSWORD = "Owner-Password-2027!" + + +class TotpSetupStartTests(TestCase): + def setUp(self) -> None: + self.result = bootstrap_owner(email="totp-owner@example.com", password=PASSWORD) + self.owner = self.result.owner + self.client = TenantAPIClient() + self.client.force_authenticate(self.owner) + + def test_start_issues_a_secret_while_totp_is_off(self) -> None: + self.owner.totp_secret = "OLDSECRET" + self.owner.save(update_fields=["totp_secret"]) + + response = self.client.post("/api/v1/auth/profile/totp/start/") + + self.assertEqual(response.status_code, 200, response.content) + self.owner.refresh_from_db() + self.assertEqual(self.owner.totp_secret, "") + self.assertFalse(self.owner.totp_enabled) + + def test_start_cannot_disable_enabled_totp(self) -> None: + secret = _generate_totp_secret() + self.owner.totp_secret = secret + self.owner.totp_enabled = True + self.owner.save(update_fields=["totp_secret", "totp_enabled"]) + + response = self.client.post("/api/v1/auth/profile/totp/start/") + + self.assertEqual(response.status_code, 409, response.content) + self.owner.refresh_from_db() + self.assertTrue(self.owner.totp_enabled) + self.assertEqual(self.owner.totp_secret, secret) + + def test_disable_still_requires_the_current_password(self) -> None: + self.owner.totp_secret = _generate_totp_secret() + self.owner.totp_enabled = True + self.owner.save(update_fields=["totp_secret", "totp_enabled"]) + + rejected = self.client.post( + "/api/v1/auth/profile/totp/disable/", + {"currentPassword": "wrong-password"}, + format="json", + ) + self.assertEqual(rejected.status_code, 400, rejected.content) + + accepted = self.client.post( + "/api/v1/auth/profile/totp/disable/", + {"currentPassword": PASSWORD}, + format="json", + ) + self.assertEqual(accepted.status_code, 200, accepted.content) + self.owner.refresh_from_db() + self.assertFalse(self.owner.totp_enabled) + + +class PasswordResetSessionTests(TestCase): + def setUp(self) -> None: + self.result = bootstrap_owner(email="reset-owner@example.com", password=PASSWORD) + self.owner = self.result.owner + + def _reset_link(self) -> tuple[str, str]: + # Токен считается от текущего состояния пользователя (в том числе + # last_login), поэтому берём его после всех входов. + self.owner.refresh_from_db() + return ( + urlsafe_base64_encode(force_bytes(self.owner.pk)), + default_token_generator.make_token(self.owner), + ) + + def _session_keys(self) -> set[str]: + keys = set() + for session in Session.objects.all(): + if str(session.get_decoded().get("_auth_user_id", "")) == str(self.owner.pk): + keys.add(session.session_key) + return keys + + def test_reset_revokes_existing_sessions(self) -> None: + stolen = TenantAPIClient() + self.assertTrue(stolen.login(email=self.owner.email, password=PASSWORD)) + self.assertTrue(self._session_keys()) + + uid, token = self._reset_link() + anonymous = TenantAPIClient() + response = anonymous.post( + "/api/v1/auth/password-reset/confirm/", + {"uid": uid, "token": token, "newPassword": NEW_PASSWORD}, + format="json", + ) + + self.assertEqual(response.status_code, 200, response.content) + self.assertGreaterEqual(response.json()["revoked"], 1) + self.assertEqual(self._session_keys(), set()) + + # Прежняя сессия больше не открывает приложение. + session = stolen.get("/api/v1/auth/session/") + self.assertFalse(session.json()["authenticated"]) + + def test_reset_still_sets_the_new_password(self) -> None: + uid, token = self._reset_link() + TenantAPIClient().post( + "/api/v1/auth/password-reset/confirm/", + {"uid": uid, "token": token, "newPassword": NEW_PASSWORD}, + format="json", + ) + + self.owner.refresh_from_db() + self.assertTrue(self.owner.check_password(NEW_PASSWORD)) + self.assertFalse(self.owner.must_change_password) diff --git a/apps/backend/chatballs/identity/test_crypto_columns.py b/apps/backend/chatballs/identity/test_crypto_columns.py new file mode 100644 index 0000000..87d332e --- /dev/null +++ b/apps/backend/chatballs/identity/test_crypto_columns.py @@ -0,0 +1,52 @@ +"""Шифротекст должен помещаться в колонку. + +``EncryptedCharField`` кладёт в базу Fernet-токен: версия, метка времени, IV, +подпись, дополнение до блока AES и base64 поверх всего. Значение в 512 +символов занимает почти 2.9 КБ — в varchar(512) оно не помещалось, и длинный +пароль SMTP или ключ S3 ронял запись уже в базе. +""" + +from __future__ import annotations + +from django.test import TestCase + +from chatballs.identity.crypto import ciphertext_length, decrypt_secret, encrypt_secret +from chatballs.identity.instance_settings import InstanceSettings +from chatballs.tenancy.storage_settings import StorageSettings + + +class CiphertextLengthTests(TestCase): + def test_estimate_covers_the_real_token(self) -> None: + for plaintext_chars in (1, 16, 255, 512, 1024): + value = "щ" * plaintext_chars # 2 байта на символ в UTF-8 + self.assertLessEqual( + len(encrypt_secret(value)), + ciphertext_length(plaintext_chars), + f"оценка мала для {plaintext_chars} символов", + ) + + def test_round_trip_survives(self) -> None: + value = "п" * 400 + self.assertEqual(decrypt_secret(encrypt_secret(value)), value) + + +class EncryptedColumnWidthTests(TestCase): + def test_long_smtp_password_is_stored(self) -> None: + password = "Пароль-" + "x" * 500 + row = InstanceSettings.load() + row.email_host = "smtp.example.test" + row.email_password = password + row.save(update_fields=["email_host", "email_password", "updated_at"]) + + row.refresh_from_db() + self.assertEqual(row.email_password, password) + + def test_long_s3_keys_are_stored(self) -> None: + secret = "S" * 500 + row = StorageSettings.load() + row.s3_access_key = "A" * 500 + row.s3_secret_key = secret + row.save(update_fields=["s3_access_key", "s3_secret_key"]) + + row.refresh_from_db() + self.assertEqual(row.s3_secret_key, secret) diff --git a/apps/backend/chatballs/identity/test_instance_address.py b/apps/backend/chatballs/identity/test_instance_address.py new file mode 100644 index 0000000..a02ca7c --- /dev/null +++ b/apps/backend/chatballs/identity/test_instance_address.py @@ -0,0 +1,121 @@ +"""Адрес установки: смена не должна выбрасывать того, кто её делает. + +Владелец меняет адрес заранее — до того, как новый домен начал резолвиться и +получил сертификат, — и сидит при этом на старом. Пока принятым был только +новый адрес, сохранение отвечало «Invalid host» через десять секунд (TTL +кэша), а мастер первого запуска уже закрыт: вернуться было неоткуда. + +Здесь же — запрет вешать портал помощи на адрес самой установки: SPA решает, +что рисовать, по ответу ``/api/v1/help/``, и такой портал подменял бы +сотрудникам приложение своим Help Center. +""" + +from __future__ import annotations + +from django.core.exceptions import ValidationError +from django.test import TestCase + +from chatballs.identity.bootstrap import bootstrap_owner +from chatballs.identity.instance_settings import ( + InstanceSettings, + accepted_hosts, + invalidate_cache, +) +from chatballs.support_portals.models import SupportPortal +from chatballs.testing import TenantAPIClient + +PASSWORD = "Owner-Password-2026!" + + +class InstanceAddressChangeTests(TestCase): + def setUp(self) -> None: + self.result = bootstrap_owner(email="address-owner@example.com", password=PASSWORD) + self.client = TenantAPIClient() + self.client.force_authenticate(self.result.owner) + row = InstanceSettings.load() + row.public_host = "203.0.113.10" + row.public_scheme = "http" + row.previous_public_host = "" + row.save(update_fields=["public_host", "public_scheme", "previous_public_host", "updated_at"]) + invalidate_cache() + self.addCleanup(invalidate_cache) + + def _patch(self, host: str, scheme: str = "https"): + return self.client.patch( + "/api/v1/company/administration/instance/", + {"publicHost": host, "publicScheme": scheme}, + format="json", + ) + + def test_previous_address_stays_accepted(self) -> None: + response = self._patch("crm.example.test") + + self.assertEqual(response.status_code, 200, response.content) + invalidate_cache() + self.assertEqual(set(accepted_hosts()), {"crm.example.test", "203.0.113.10"}) + + def test_old_address_still_answers_after_the_change(self) -> None: + self._patch("crm.example.test") + invalidate_cache() + + response = self.client.get("/api/v1/auth/session/", HTTP_HOST="203.0.113.10") + + self.assertEqual(response.status_code, 200, response.content) + + def test_stranger_host_is_still_rejected(self) -> None: + self._patch("crm.example.test") + invalidate_cache() + + response = self.client.get("/api/v1/auth/session/", HTTP_HOST="evil.example") + + self.assertEqual(response.status_code, 400) + + def test_only_one_previous_address_is_kept(self) -> None: + self._patch("first.example.test") + self._patch("second.example.test") + invalidate_cache() + + self.assertEqual(set(accepted_hosts()), {"second.example.test", "first.example.test"}) + + def test_saving_the_same_address_does_not_shift_history(self) -> None: + self._patch("crm.example.test") + self._patch("crm.example.test") + invalidate_cache() + + self.assertEqual(set(accepted_hosts()), {"crm.example.test", "203.0.113.10"}) + + +class PortalDomainCollisionTests(TestCase): + def setUp(self) -> None: + self.result = bootstrap_owner(email="portal-owner@example.com", password=PASSWORD) + row = InstanceSettings.load() + row.public_host = "crm.example.test" + row.previous_public_host = "203.0.113.10" + row.save(update_fields=["public_host", "previous_public_host", "updated_at"]) + invalidate_cache() + self.addCleanup(invalidate_cache) + + def _portal(self, custom_domain: str) -> SupportPortal: + return SupportPortal( + organization=self.result.organization, + slug="help", + name="Help", + custom_domain=custom_domain, + ) + + def test_installation_address_cannot_become_a_portal_domain(self) -> None: + with self.assertRaises(ValidationError) as error: + self._portal("crm.example.test").clean() + + self.assertIn("custom_domain", error.exception.message_dict) + + def test_previous_installation_address_is_also_refused(self) -> None: + with self.assertRaises(ValidationError): + self._portal("203.0.113.10").clean() + + def test_unrelated_domain_is_allowed(self) -> None: + portal = self._portal("help.example.test") + + portal.clean() # не должно бросать + + self.assertEqual(portal.custom_domain, "help.example.test") diff --git a/apps/backend/chatballs/identity/test_invitation_accept.py b/apps/backend/chatballs/identity/test_invitation_accept.py index 4b09371..a442a73 100644 --- a/apps/backend/chatballs/identity/test_invitation_accept.py +++ b/apps/backend/chatballs/identity/test_invitation_accept.py @@ -17,6 +17,8 @@ from chatballs.identity.models import ( OrganizationMembership, OrganizationStatus, ) + + def _pending_org(slug: str = "pending-org") -> Organization: return Organization.objects.create( name="Pending Org", diff --git a/apps/backend/chatballs/identity/test_seed_demo.py b/apps/backend/chatballs/identity/test_seed_demo.py index 3c62423..963c331 100644 --- a/apps/backend/chatballs/identity/test_seed_demo.py +++ b/apps/backend/chatballs/identity/test_seed_demo.py @@ -8,7 +8,6 @@ import tempfile from django.apps import apps from django.db import models from django.test import TestCase, override_settings -from chatballs.testing import TenantAPIClient as APIClient from chatballs.ai.models import AIAgent, AIAgentStatus, Knowledge, KnowledgeAttachment from chatballs.conversations.models import ( @@ -27,6 +26,7 @@ from chatballs.identity.models import HumanUser, Organization, OrganizationMembe from chatballs.identity.setup import SetupInput, complete_setup from chatballs.tenancy.context import TenantActorKind, TenantContext from chatballs.tenancy.database import tenant_atomic +from chatballs.testing import TenantAPIClient as APIClient _MEDIA_ROOT = tempfile.mkdtemp(prefix="hub-demo-media-") diff --git a/apps/backend/chatballs/identity/tests.py b/apps/backend/chatballs/identity/tests.py index cbabfe5..1aa9b24 100644 --- a/apps/backend/chatballs/identity/tests.py +++ b/apps/backend/chatballs/identity/tests.py @@ -9,15 +9,14 @@ from django.core import mail from django.test import Client, TestCase, override_settings from django.utils.encoding import force_bytes from django.utils.http import urlsafe_base64_encode -from chatballs.testing import TenantAPIClient as APIClient from rest_framework.throttling import ScopedRateThrottle -from chatballs.identity.bootstrap import bootstrap_owner from chatballs.identity.auth.totp_utils import ( TOTP_CHALLENGE_TTL_SECONDS, TOTP_STARTED_KEY, _totp_code, ) +from chatballs.identity.bootstrap import bootstrap_owner from chatballs.identity.models import ( AuditEvent, EmployeeGroup, @@ -27,6 +26,7 @@ from chatballs.identity.models import ( OrganizationMembership, ) from chatballs.identity.policy import ResourceScope, authorize +from chatballs.testing import TenantAPIClient as APIClient _LOCMEM_CACHE = {"default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}} diff --git a/apps/backend/chatballs/integrations/admin.py b/apps/backend/chatballs/integrations/admin.py index f00b223..7af71b3 100644 --- a/apps/backend/chatballs/integrations/admin.py +++ b/apps/backend/chatballs/integrations/admin.py @@ -1,11 +1,11 @@ -from django.contrib import admin - -from chatballs.integrations.models import Integration - - -@admin.register(Integration) -class IntegrationAdmin(admin.ModelAdmin): - list_display = ("name", "provider", "kind", "status", "last_checked_at") - list_filter = ("provider", "kind", "status") - search_fields = ("name",) - readonly_fields = ("last_checked_at", "last_error", "created_at", "updated_at") +from django.contrib import admin + +from chatballs.integrations.models import Integration + + +@admin.register(Integration) +class IntegrationAdmin(admin.ModelAdmin): + list_display = ("name", "provider", "kind", "status", "last_checked_at") + list_filter = ("provider", "kind", "status") + search_fields = ("name",) + readonly_fields = ("last_checked_at", "last_error", "created_at", "updated_at") diff --git a/apps/backend/chatballs/integrations/apps.py b/apps/backend/chatballs/integrations/apps.py index b19ef09..e8e5a56 100644 --- a/apps/backend/chatballs/integrations/apps.py +++ b/apps/backend/chatballs/integrations/apps.py @@ -1,8 +1,8 @@ -from django.apps import AppConfig - - -class IntegrationsConfig(AppConfig): - default_auto_field = "django.db.models.BigAutoField" - label = "integrations" - name = "chatballs.integrations" - verbose_name = "Integrations: providers and connections" +from django.apps import AppConfig + + +class IntegrationsConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + label = "integrations" + name = "chatballs.integrations" + verbose_name = "Integrations: providers and connections" diff --git a/apps/backend/chatballs/integrations/migrations/0001_initial.py b/apps/backend/chatballs/integrations/migrations/0001_initial.py index 9237772..c2113d5 100644 --- a/apps/backend/chatballs/integrations/migrations/0001_initial.py +++ b/apps/backend/chatballs/integrations/migrations/0001_initial.py @@ -1,38 +1,38 @@ -# Generated by Django 5.2.15 on 2026-06-27 21:12 - -import django.db.models.deletion -import chatballs.identity.crypto -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ('identity', '0006_alter_employeeprofile_totp_secret'), - ] - - operations = [ - migrations.CreateModel( - name='Integration', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('kind', models.CharField(choices=[('LLM_PROVIDER', 'LLM-провайдер'), ('MESSENGER', 'Подключение-мессенджер')], max_length=16)), - ('provider', models.CharField(choices=[('OPENROUTER', 'OpenRouter'), ('MAX', 'MAX'), ('TELEGRAM', 'Telegram'), ('WEB', 'Web-виджет')], max_length=16)), - ('name', models.CharField(max_length=255)), - ('secret', chatballs.identity.crypto.EncryptedCharField(blank=True, max_length=1024)), - ('config', models.JSONField(blank=True, default=dict)), - ('status', models.CharField(choices=[('UNCHECKED', 'Не проверено'), ('OK', 'Подключено'), ('ERROR', 'Ошибка')], default='UNCHECKED', max_length=16)), - ('last_checked_at', models.DateTimeField(blank=True, null=True)), - ('last_error', models.TextField(blank=True)), - ('created_at', models.DateTimeField(auto_now_add=True)), - ('updated_at', models.DateTimeField(auto_now=True)), - ('organization', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='integrations', to='identity.organization')), - ], - options={ - 'ordering': ['provider', 'name'], - 'constraints': [models.UniqueConstraint(fields=('organization', 'provider', 'name'), name='uniq_integration_org_provider_name')], - }, - ), - ] +# Generated by Django 5.2.15 on 2026-06-27 21:12 + +import django.db.models.deletion +import chatballs.identity.crypto +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('identity', '0006_alter_employeeprofile_totp_secret'), + ] + + operations = [ + migrations.CreateModel( + name='Integration', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('kind', models.CharField(choices=[('LLM_PROVIDER', 'LLM-провайдер'), ('MESSENGER', 'Подключение-мессенджер')], max_length=16)), + ('provider', models.CharField(choices=[('OPENROUTER', 'OpenRouter'), ('MAX', 'MAX'), ('TELEGRAM', 'Telegram'), ('WEB', 'Web-виджет')], max_length=16)), + ('name', models.CharField(max_length=255)), + ('secret', chatballs.identity.crypto.EncryptedCharField(blank=True, max_length=1024)), + ('config', models.JSONField(blank=True, default=dict)), + ('status', models.CharField(choices=[('UNCHECKED', 'Не проверено'), ('OK', 'Подключено'), ('ERROR', 'Ошибка')], default='UNCHECKED', max_length=16)), + ('last_checked_at', models.DateTimeField(blank=True, null=True)), + ('last_error', models.TextField(blank=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('organization', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='integrations', to='identity.organization')), + ], + options={ + 'ordering': ['provider', 'name'], + 'constraints': [models.UniqueConstraint(fields=('organization', 'provider', 'name'), name='uniq_integration_org_provider_name')], + }, + ), + ] diff --git a/apps/backend/chatballs/integrations/migrations/0008_encrypted_column_width.py b/apps/backend/chatballs/integrations/migrations/0008_encrypted_column_width.py new file mode 100644 index 0000000..6d63900 --- /dev/null +++ b/apps/backend/chatballs/integrations/migrations/0008_encrypted_column_width.py @@ -0,0 +1,19 @@ +"""Секрет интеграции: колонка под шифротекст (см. identity.0034).""" + +from django.db import migrations + +import chatballs.identity.crypto + + +class Migration(migrations.Migration): + dependencies = [ + ("integrations", "0007_integration_feature_flags"), + ] + + operations = [ + migrations.AlterField( + model_name="integration", + name="secret", + field=chatballs.identity.crypto.EncryptedCharField(blank=True, max_length=5560), + ), + ] diff --git a/apps/backend/chatballs/integrations/proxy.py b/apps/backend/chatballs/integrations/proxy.py index 2fa8e95..214dc27 100644 --- a/apps/backend/chatballs/integrations/proxy.py +++ b/apps/backend/chatballs/integrations/proxy.py @@ -38,6 +38,40 @@ class _RefusedDataHandler(urllib.request.DataHandler): raise OutboundUrlRejected("Схема data: в исходящих запросах запрещена") +class _GuardedRedirectHandler(urllib.request.HTTPRedirectHandler): + """Проверяет каждый Location, а не только исходный адрес. + + Политика исходящих (``integrations.outbound``) проверяет адрес до запроса. + Но urllib сам ходит по редиректам, и ответ подставного провайдера мог + вернуть 302 на ``http://169.254.169.254/…`` — проверку прошёл один адрес, + а сходили по другому. + + Проверка стоит в ``http_error_302``, а не только в ``redirect_request``: + свою проверку схемы urllib делает раньше и отвечает на неё ``HTTPError``, + из-за чего запрет выглядел бы сетевой ошибкой, а не отказом политики. + """ + + def __init__(self, validate) -> None: + self._validate = validate + + def http_error_302(self, req, fp, code, msg, headers): # noqa: ANN001 + location = headers.get("location") or headers.get("uri") or "" + if location: + self._validate(urllib.parse.urljoin(req.full_url, location)) + return super().http_error_302(req, fp, code, msg, headers) + + # urllib связывает остальные коды с базовым методом на этапе создания + # класса, поэтому переопределения одного http_error_302 мало. + http_error_301 = http_error_302 + http_error_303 = http_error_302 + http_error_307 = http_error_302 + http_error_308 = http_error_302 + + def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: ANN001 + self._validate(newurl) + return super().redirect_request(req, fp, code, msg, headers, newurl) + + def _blocked_scheme_handlers() -> list[urllib.request.BaseHandler]: """Заглушки вместо file/ftp/data. @@ -50,9 +84,16 @@ def _blocked_scheme_handlers() -> list[urllib.request.BaseHandler]: return [_RefusedFileHandler(), _RefusedFTPHandler(), _RefusedDataHandler()] -def build_opener(proxy_url: str): - """urllib opener, проксирующий http/https/socks5 запросы. Пустой proxy_url → без прокси.""" +def build_opener(proxy_url: str, *, validate_redirect=None): + """urllib opener, проксирующий http/https/socks5 запросы. + + Пустой ``proxy_url`` → без прокси. ``validate_redirect`` — проверка адреса, + на который ответ просит перейти: её передают там, где сам адрес пришёл + данными от провайдера, а не из настроек подключения. + """ blocked = _blocked_scheme_handlers() + if validate_redirect is not None: + blocked.append(_GuardedRedirectHandler(validate_redirect)) if not proxy_url: return urllib.request.build_opener(*blocked) scheme = urllib.parse.urlparse(proxy_url).scheme.lower() diff --git a/apps/backend/chatballs/integrations/serializers.py b/apps/backend/chatballs/integrations/serializers.py index ef1cbd9..b5cb8ec 100644 --- a/apps/backend/chatballs/integrations/serializers.py +++ b/apps/backend/chatballs/integrations/serializers.py @@ -1,5 +1,50 @@ +from urllib.parse import urlsplit, urlunsplit + from chatballs.integrations.models import Integration +# Пароль прокси наружу не отдаётся: в списке подключений его видел бы каждый, +# у кого есть право смотреть интеграции, а сам адрес попадал бы в логи и +# историю браузера вместе с ним. Пустое поле при сохранении означает +# «оставить прежний» — ровно как у секрета интеграции. +PROXY_PASSWORD_MASK = "••••••••" + + +def mask_proxy_url(url: str) -> str: + """Адрес прокси без пароля: «socks5://user:••••••••@host:1080».""" + if not url: + return "" + parsed = urlsplit(url) + if not parsed.password: + return url + host = parsed.hostname or "" + if parsed.port: + host = f"{host}:{parsed.port}" + userinfo = f"{parsed.username or ''}:{PROXY_PASSWORD_MASK}" + return urlunsplit( + (parsed.scheme, f"{userinfo}@{host}", parsed.path, parsed.query, parsed.fragment) + ) + + +def restore_proxy_password(submitted: str, stored: str) -> str: + """Вернуть сохранённый пароль, если пришла маска того же прокси. + + Форма отправляет конфигурацию целиком, поэтому без этого замаскированное + значение сохранилось бы вместо настоящего пароля и прокси перестал бы + работать при первом же редактировании соседнего поля. + """ + if not submitted or PROXY_PASSWORD_MASK not in submitted or not stored: + return submitted + new, old = urlsplit(submitted), urlsplit(stored) + same_target = ( + new.scheme == old.scheme + and (new.hostname or "") == (old.hostname or "") + and new.port == old.port + and (new.username or "") == (old.username or "") + ) + if not same_target or not old.password: + return submitted + return stored + def secret_mask(secret: str) -> str: """Маска секрета для колонки «Секрет» (кадр N3): только публичный префикс @@ -26,7 +71,7 @@ def integration_payload(integration: Integration) -> dict[str, object]: "baseUrl": integration.config.get("base_url", ""), "defaultModel": integration.config.get("default_model", ""), "transcriptionModel": integration.config.get("transcription_model", ""), - "proxyUrl": integration.config.get("proxy_url", ""), + "proxyUrl": mask_proxy_url(str(integration.config.get("proxy_url", ""))), "botId": integration.config.get("bot_id", ""), "botUsername": integration.config.get("bot_username", ""), "botName": integration.config.get("bot_name", ""), diff --git a/apps/backend/chatballs/integrations/test_email.py b/apps/backend/chatballs/integrations/test_email.py index 9e460e1..4a8cf12 100644 --- a/apps/backend/chatballs/integrations/test_email.py +++ b/apps/backend/chatballs/integrations/test_email.py @@ -4,30 +4,17 @@ from unittest import mock - - from django.core.exceptions import ValidationError - from django.test import TestCase - - from chatballs.identity.bootstrap import bootstrap_owner - from chatballs.identity.models import Organization - from chatballs.integrations import checks - from chatballs.integrations.models import IntegrationKind, IntegrationProvider, IntegrationStatus - from chatballs.integrations.serializers import integration_payload - from chatballs.integrations.services import IntegrationInput, create_integration - from chatballs.testing import system_tenant_context - - EMAIL_INPUT = { "email": "Support@example.com", diff --git a/apps/backend/chatballs/integrations/test_outbound_redirects.py b/apps/backend/chatballs/integrations/test_outbound_redirects.py new file mode 100644 index 0000000..39111f7 --- /dev/null +++ b/apps/backend/chatballs/integrations/test_outbound_redirects.py @@ -0,0 +1,75 @@ +"""Редирект не должен уводить скачивание туда, куда исходный адрес не пустили. + +Политика исходящих проверяет адрес до запроса, но urllib сам ходит по +редиректам: ответ подставного провайдера возвращал 302 на внутренний адрес, и +хаб шёл туда своими руками. Проверка теперь висит на каждом Location. +""" + +from __future__ import annotations + +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer + +from django.test import SimpleTestCase + +from chatballs.conversations.transports.base import download_bytes +from chatballs.integrations.outbound import OutboundUrlRejected + +PAYLOAD = b"file-content" + + +class _Handler(BaseHTTPRequestHandler): + redirect_to = "" + + def do_GET(self): # noqa: N802 + if self.path == "/file": + self.send_response(200) + self.send_header("Content-Length", str(len(PAYLOAD))) + self.end_headers() + self.wfile.write(PAYLOAD) + return + self.send_response(302) + self.send_header("Location", type(self).redirect_to) + self.end_headers() + + def log_message(self, *args): # тишина в выводе тестов + return + + +class OutboundRedirectTests(SimpleTestCase): + def setUp(self) -> None: + self.server = HTTPServer(("127.0.0.1", 0), _Handler) + self.host, self.port = self.server.server_address + thread = threading.Thread(target=self.server.serve_forever, daemon=True) + thread.start() + self.addCleanup(self.server.shutdown) + self.addCleanup(self.server.server_close) + # Хост из base_url подключения владелец назвал сам — он разрешён даже + # будучи внутренним. Именно так провайдер и живёт у self-hosted. + self.allowed_host = "127.0.0.1" + self.base = f"http://127.0.0.1:{self.port}" + + def test_direct_download_from_the_configured_host_works(self) -> None: + data = download_bytes(f"{self.base}/file", allowed_host=self.allowed_host) + + self.assertEqual(data, PAYLOAD) + + def test_redirect_inside_the_configured_host_is_followed(self) -> None: + _Handler.redirect_to = f"{self.base}/file" + + data = download_bytes(f"{self.base}/start", allowed_host=self.allowed_host) + + self.assertEqual(data, PAYLOAD) + + def test_redirect_to_a_foreign_internal_address_is_rejected(self) -> None: + # Классическая цель SSRF: метаданные облачной машины. + _Handler.redirect_to = "http://169.254.169.254/latest/meta-data/" + + with self.assertRaises(OutboundUrlRejected): + download_bytes(f"{self.base}/start", allowed_host=self.allowed_host) + + def test_redirect_to_a_forbidden_scheme_is_rejected(self) -> None: + _Handler.redirect_to = "file:///run/chatballs/secrets/secret_key" + + with self.assertRaises(OutboundUrlRejected): + download_bytes(f"{self.base}/start", allowed_host=self.allowed_host) diff --git a/apps/backend/chatballs/integrations/test_proxy_masking.py b/apps/backend/chatballs/integrations/test_proxy_masking.py new file mode 100644 index 0000000..0c4ee1f --- /dev/null +++ b/apps/backend/chatballs/integrations/test_proxy_masking.py @@ -0,0 +1,108 @@ +"""Пароль прокси не уходит наружу и не теряется при редактировании. + +Секрет интеграции в ответе маскируется, а адрес прокси отдавался целиком — +вместе с ``user:pass@``. Его видит каждый, у кого есть право смотреть +интеграции, и он же оседает в логах и истории браузера. + +Маскировать мало: форма отправляет конфигурацию целиком, поэтому замаскированное +значение сохранилось бы вместо настоящего пароля при правке соседнего поля. +""" + +from __future__ import annotations + +from django.test import TestCase + +from chatballs.identity.bootstrap import bootstrap_owner +from chatballs.integrations.models import IntegrationKind, IntegrationProvider +from chatballs.integrations.serializers import ( + PROXY_PASSWORD_MASK, + integration_payload, + mask_proxy_url, + restore_proxy_password, +) +from chatballs.integrations.services import IntegrationInput, create_integration, update_integration +from chatballs.testing import system_tenant_context + +PROXY = "socks5://proxy-user:s3cr3t-pass@proxy.example:1080" + + +class ProxyMaskTests(TestCase): + def test_password_is_replaced_and_the_rest_is_kept(self) -> None: + masked = mask_proxy_url(PROXY) + + self.assertNotIn("s3cr3t-pass", masked) + self.assertIn("proxy-user", masked) + self.assertIn("proxy.example:1080", masked) + self.assertIn(PROXY_PASSWORD_MASK, masked) + + def test_proxy_without_password_is_untouched(self) -> None: + self.assertEqual( + mask_proxy_url("http://proxy.example:3128"), "http://proxy.example:3128" + ) + + def test_empty_stays_empty(self) -> None: + self.assertEqual(mask_proxy_url(""), "") + + def test_mask_of_the_same_proxy_restores_the_stored_password(self) -> None: + self.assertEqual(restore_proxy_password(mask_proxy_url(PROXY), PROXY), PROXY) + + def test_mask_of_another_host_is_not_restored(self) -> None: + submitted = mask_proxy_url("socks5://proxy-user:other@elsewhere.example:1080") + + self.assertEqual(restore_proxy_password(submitted, PROXY), submitted) + + def test_new_password_replaces_the_stored_one(self) -> None: + submitted = "socks5://proxy-user:brand-new@proxy.example:1080" + + self.assertEqual(restore_proxy_password(submitted, PROXY), submitted) + + +class ProxyPayloadTests(TestCase): + def setUp(self) -> None: + result = bootstrap_owner(email="proxy-owner@example.com", password="Owner-Password-2026!") + self.context = system_tenant_context(result.organization) + self.integration = create_integration( + context=self.context, + data=IntegrationInput( + provider=IntegrationProvider.OPENROUTER, + name="LLM", + secret="sk-or-v1-secret-value", + config={"baseUrl": "https://openrouter.ai/api/v1", "proxyUrl": PROXY}, + ), + ) + + def test_payload_never_carries_the_proxy_password(self) -> None: + payload = integration_payload(self.integration) + + self.assertNotIn("s3cr3t-pass", str(payload)) + self.assertIn(PROXY_PASSWORD_MASK, payload["config"]["proxyUrl"]) + + def test_stored_value_keeps_the_real_password(self) -> None: + self.integration.refresh_from_db() + + self.assertEqual(self.integration.config["proxy_url"], PROXY) + + def test_editing_another_field_keeps_the_proxy_password(self) -> None: + """Форма возвращает то, что ей отдали, — включая маску.""" + from chatballs.integrations.views import _keep_proxy_password + + submitted = { + "baseUrl": "https://openrouter.ai/api/v1", + "proxyUrl": integration_payload(self.integration)["config"]["proxyUrl"], + "defaultModel": "openai/gpt-4o-mini", + } + restored = _keep_proxy_password(submitted, self.integration) + + updated = update_integration( + context=self.context, + integration=self.integration, + data=IntegrationInput( + provider=IntegrationProvider.OPENROUTER, + name="LLM", + secret=None, + config=restored, + ), + ) + + self.assertEqual(updated.config["proxy_url"], PROXY) + self.assertEqual(updated.kind, IntegrationKind.LLM_PROVIDER) diff --git a/apps/backend/chatballs/integrations/tests.py b/apps/backend/chatballs/integrations/tests.py index 3481aa4..6f494c3 100644 --- a/apps/backend/chatballs/integrations/tests.py +++ b/apps/backend/chatballs/integrations/tests.py @@ -1,45 +1,31 @@ import json - import urllib.request - from unittest import mock - - from django.core.exceptions import ValidationError - from django.test import TestCase - - from chatballs.identity.bootstrap import bootstrap_owner - from chatballs.identity.models import Organization - from chatballs.integrations import checks - -from chatballs.integrations.models import Integration, IntegrationKind, IntegrationProvider, IntegrationStatus - +from chatballs.integrations.models import ( + Integration, + IntegrationKind, + IntegrationProvider, + IntegrationStatus, +) from chatballs.integrations.serializers import integration_payload - from chatballs.integrations.services import ( - IntegrationInput, - create_integration, - + update_integration, +) +from chatballs.integrations.services import ( test_integration as run_integration_test, - - update_integration, - ) - from chatballs.testing import system_tenant_context - - - def _fake_response(status: int, body: dict): response = mock.MagicMock() @@ -497,7 +483,6 @@ class BuildOpenerSocksTests(TestCase): def test_socks_without_pysocks_raises_value_error(self) -> None: from chatballs.integrations import proxy - from chatballs.integrations.proxy import build_opener diff --git a/apps/backend/chatballs/integrations/urls.py b/apps/backend/chatballs/integrations/urls.py index 17cb7ac..e88653e 100644 --- a/apps/backend/chatballs/integrations/urls.py +++ b/apps/backend/chatballs/integrations/urls.py @@ -1,9 +1,9 @@ -from django.urls import path - -from chatballs.integrations import views - -urlpatterns = [ - path("", views.IntegrationListView.as_view(), name="integration-list"), - path("/", views.IntegrationDetailView.as_view(), name="integration-detail"), - path("/test/", views.IntegrationTestView.as_view(), name="integration-test"), -] +from django.urls import path + +from chatballs.integrations import views + +urlpatterns = [ + path("", views.IntegrationListView.as_view(), name="integration-list"), + path("/", views.IntegrationDetailView.as_view(), name="integration-detail"), + path("/test/", views.IntegrationTestView.as_view(), name="integration-test"), +] diff --git a/apps/backend/chatballs/integrations/views.py b/apps/backend/chatballs/integrations/views.py index ff8992d..84bc8de 100644 --- a/apps/backend/chatballs/integrations/views.py +++ b/apps/backend/chatballs/integrations/views.py @@ -11,7 +11,7 @@ from chatballs.integrations.selectors import ( integration_for_context, integrations_for_context, ) -from chatballs.integrations.serializers import integration_payload +from chatballs.integrations.serializers import integration_payload, restore_proxy_password from chatballs.integrations.services import ( IntegrationInput, create_integration, @@ -23,6 +23,7 @@ from chatballs.integrations.services import ( def _input(body: dict[str, object], *, current: Integration | None = None) -> IntegrationInput: config = body.get("config", current.config if current else {}) + config = _keep_proxy_password(config, current) raw_channel = body.get("channelId", current.channel_id if current else None) channel_id = int(raw_channel) if isinstance(raw_channel, int) or (isinstance(raw_channel, str) and raw_channel.isdigit()) else None return IntegrationInput( @@ -39,6 +40,22 @@ def _input(body: dict[str, object], *, current: Integration | None = None) -> In ) +def _keep_proxy_password(config: object, current: Integration | None) -> object: + """Маска пароля прокси из ответа не должна затирать настоящий пароль.""" + if not isinstance(config, dict) or current is None: + return config + submitted = str(config.get("proxyUrl", config.get("proxy_url", "")) or "") + if not submitted: + return config + restored = restore_proxy_password(submitted, str(current.config.get("proxy_url", ""))) + if restored == submitted: + return config + config = dict(config) + config.pop("proxy_url", None) + config["proxyUrl"] = restored + return config + + def _validation_error(error: Exception) -> Response: if isinstance(error, ValidationError): if hasattr(error, "message_dict"): @@ -64,7 +81,6 @@ def _audit(request: Request, action: str, integration: Integration) -> None: class IntegrationListView(APIView): permission_classes = [HasCapability] required_capabilities = {"GET": "integrations.view", "POST": "integrations.manage"} - require_organization_scope = True def get(self, request: Request) -> Response: items = integrations_for_context(request.tenant_context) @@ -84,7 +100,6 @@ class IntegrationListView(APIView): class IntegrationDetailView(APIView): permission_classes = [HasCapability] required_capability = "integrations.manage" - require_organization_scope = True def _get(self, request: Request, integration_id: int) -> Integration: return integration_for_context( @@ -119,7 +134,6 @@ class IntegrationDetailView(APIView): class IntegrationTestView(APIView): permission_classes = [HasCapability] required_capability = "integrations.manage" - require_organization_scope = True def post(self, request: Request, integration_id: int) -> Response: try: diff --git a/apps/backend/chatballs/notifications/admin.py b/apps/backend/chatballs/notifications/admin.py index c2428ba..2fae084 100644 --- a/apps/backend/chatballs/notifications/admin.py +++ b/apps/backend/chatballs/notifications/admin.py @@ -1,15 +1,15 @@ -from django.contrib import admin - -from chatballs.notifications.models import Notification, NotificationRead - - -@admin.register(Notification) -class NotificationAdmin(admin.ModelAdmin): - list_display = ("type", "audience", "level", "title", "organization", "created_at") - list_filter = ("type", "audience", "level") - search_fields = ("title", "body", "dedup_key") - - -@admin.register(NotificationRead) -class NotificationReadAdmin(admin.ModelAdmin): - list_display = ("notification", "user", "read_at") +from django.contrib import admin + +from chatballs.notifications.models import Notification, NotificationRead + + +@admin.register(Notification) +class NotificationAdmin(admin.ModelAdmin): + list_display = ("type", "audience", "level", "title", "organization", "created_at") + list_filter = ("type", "audience", "level") + search_fields = ("title", "body", "dedup_key") + + +@admin.register(NotificationRead) +class NotificationReadAdmin(admin.ModelAdmin): + list_display = ("notification", "user", "read_at") diff --git a/apps/backend/chatballs/notifications/apps.py b/apps/backend/chatballs/notifications/apps.py index 8b1cc0f..4f25ddd 100644 --- a/apps/backend/chatballs/notifications/apps.py +++ b/apps/backend/chatballs/notifications/apps.py @@ -1,11 +1,11 @@ -from django.apps import AppConfig - - -class NotificationsConfig(AppConfig): - default_auto_field = "django.db.models.BigAutoField" - label = "notifications" - name = "chatballs.notifications" - verbose_name = "Notifications" - - def ready(self) -> None: - from chatballs.notifications import event_handlers # noqa: F401 (register outbox handlers) +from django.apps import AppConfig + + +class NotificationsConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + label = "notifications" + name = "chatballs.notifications" + verbose_name = "Notifications" + + def ready(self) -> None: + from chatballs.notifications import event_handlers # noqa: F401 (register outbox handlers) diff --git a/apps/backend/chatballs/notifications/delivery.py b/apps/backend/chatballs/notifications/delivery.py index 4da29f3..8600511 100644 --- a/apps/backend/chatballs/notifications/delivery.py +++ b/apps/backend/chatballs/notifications/delivery.py @@ -10,10 +10,8 @@ from __future__ import annotations import logging -from django.conf import settings - -from chatballs.identity.instance_settings import public_base_url from chatballs.conversations import transports +from chatballs.identity.instance_settings import public_base_url from chatballs.identity.models import OrganizationMembership from chatballs.notifications.models import ( MessengerBinding, diff --git a/apps/backend/chatballs/notifications/serializers.py b/apps/backend/chatballs/notifications/serializers.py index 07b9b90..59f11e2 100644 --- a/apps/backend/chatballs/notifications/serializers.py +++ b/apps/backend/chatballs/notifications/serializers.py @@ -1,15 +1,15 @@ -from chatballs.notifications.models import Notification - - -def notification_payload(notification: Notification, *, unread: bool) -> dict[str, object]: - return { - "id": notification.id, - "type": notification.type, - "level": notification.level, - "title": notification.title, - "body": notification.body, - "targetRoute": notification.target_route, - "targetId": notification.target_id, - "createdAt": notification.created_at.isoformat(), - "unread": unread, - } +from chatballs.notifications.models import Notification + + +def notification_payload(notification: Notification, *, unread: bool) -> dict[str, object]: + return { + "id": notification.id, + "type": notification.type, + "level": notification.level, + "title": notification.title, + "body": notification.body, + "targetRoute": notification.target_route, + "targetId": notification.target_id, + "createdAt": notification.created_at.isoformat(), + "unread": unread, + } diff --git a/apps/backend/chatballs/notifications/tests.py b/apps/backend/chatballs/notifications/tests.py index 74ca332..df65f6d 100644 --- a/apps/backend/chatballs/notifications/tests.py +++ b/apps/backend/chatballs/notifications/tests.py @@ -1,7 +1,6 @@ from unittest import mock from django.test import TestCase -from chatballs.testing import TenantAPIClient as APIClient, tenant_context_for from chatballs.conversations.transports.base import InboundMessage from chatballs.events.handlers import dispatch @@ -11,9 +10,16 @@ from chatballs.identity.models import HumanUser, Organization from chatballs.integrations.models import Integration, IntegrationKind, IntegrationProvider from chatballs.notifications.binding import deep_link, handle_notifier_inbound, issue_binding_code from chatballs.notifications.delivery import NOTIFICATION_CREATED -from chatballs.notifications.models import MessengerBinding, MessengerBindingCode, NotificationAudience, NotificationType -from chatballs.notifications.services import notify +from chatballs.notifications.models import ( + MessengerBinding, + MessengerBindingCode, + NotificationAudience, + NotificationType, +) from chatballs.notifications.selectors import visible_for +from chatballs.notifications.services import notify +from chatballs.testing import TenantAPIClient as APIClient +from chatballs.testing import tenant_context_for def _notifier(organization, provider=IntegrationProvider.TELEGRAM, username="chatballs_notify_bot"): diff --git a/apps/backend/chatballs/notifications/urls.py b/apps/backend/chatballs/notifications/urls.py index 2110bb7..d9d6bce 100644 --- a/apps/backend/chatballs/notifications/urls.py +++ b/apps/backend/chatballs/notifications/urls.py @@ -1,10 +1,10 @@ -from django.urls import path - -from chatballs.notifications import views - -urlpatterns = [ - path("", views.NotificationListView.as_view(), name="notification-list"), - path("read/", views.NotificationReadView.as_view(), name="notification-read"), - path("messenger-bindings/", views.MessengerBindingListView.as_view(), name="messenger-binding-list"), - path("messenger-bindings//", views.MessengerBindingDetailView.as_view(), name="messenger-binding-detail"), -] +from django.urls import path + +from chatballs.notifications import views + +urlpatterns = [ + path("", views.NotificationListView.as_view(), name="notification-list"), + path("read/", views.NotificationReadView.as_view(), name="notification-read"), + path("messenger-bindings/", views.MessengerBindingListView.as_view(), name="messenger-binding-list"), + path("messenger-bindings//", views.MessengerBindingDetailView.as_view(), name="messenger-binding-detail"), +] diff --git a/apps/backend/chatballs/support_portals/addressing.py b/apps/backend/chatballs/support_portals/addressing.py index 1383e32..7bc9e74 100644 --- a/apps/backend/chatballs/support_portals/addressing.py +++ b/apps/backend/chatballs/support_portals/addressing.py @@ -6,7 +6,6 @@ from django.conf import settings from django.core.exceptions import ValidationError from django.db import models - HOST_LABEL_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$") @@ -24,6 +23,16 @@ def validate_domain(value: str) -> str: return domain +def _installation_hosts() -> tuple[str, ...]: + """Адреса установки; пусто, если таблицы настроек ещё нет (ранние миграции).""" + from chatballs.identity.instance_settings import accepted_hosts + + try: + return accepted_hosts() + except Exception: + return () + + def hosted_domain(portal_key: str) -> str: base_domain = normalize_domain(settings.CHATBALLS_HELP_BASE_DOMAIN) return validate_domain(f"{portal_key}.{base_domain}") @@ -40,6 +49,13 @@ def clean_portal_domains(portal) -> None: for value in getattr(settings, "CHATBALLS_APP_PRIMARY_HOSTS", []) if value and not value.startswith(".") } + # Адрес самой установки — тоже адрес приложения, хотя в статическом + # списке его нет: он живёт в настройках. Портал, повешенный на него, + # подменял бы сотрудникам приложение своим Help Center: SPA пробует + # /api/v1/help/ и по ответу решает, что рисовать. + application_hosts |= { + normalize_domain(host) for host in _installation_hosts() if host + } if portal.custom_domain in application_hosts: raise ValidationError( {"custom_domain": "Домен внутреннего приложения использовать нельзя"} diff --git a/apps/backend/chatballs/support_portals/admin.py b/apps/backend/chatballs/support_portals/admin.py index 26730d0..4423bd7 100644 --- a/apps/backend/chatballs/support_portals/admin.py +++ b/apps/backend/chatballs/support_portals/admin.py @@ -8,7 +8,6 @@ from chatballs.support_portals.models import ( SupportPortal, ) - admin.site.register( ( SupportPortal, diff --git a/apps/backend/chatballs/support_portals/content_views.py b/apps/backend/chatballs/support_portals/content_views.py index 1a1f4fe..324f163 100644 --- a/apps/backend/chatballs/support_portals/content_views.py +++ b/apps/backend/chatballs/support_portals/content_views.py @@ -19,9 +19,6 @@ from chatballs.support_portals.content_services import ( ) from chatballs.support_portals.models import PortalArticle from chatballs.support_portals.portal_views import PortalBaseView - -# Библиотека статей плотнее списка порталов — своя страница (кадр PT3). -ARTICLES_PAGE_SIZE = 25 from chatballs.support_portals.selectors import ( category_article_counts, portal_articles_queryset, @@ -33,6 +30,9 @@ from chatballs.support_portals.serializers import ( revision_payload, ) +# Библиотека статей плотнее списка порталов — своя страница (кадр PT3). +ARTICLES_PAGE_SIZE = 25 + def _article(portal, article_id: int) -> PortalArticle | None: return ( diff --git a/apps/backend/chatballs/support_portals/domain_services.py b/apps/backend/chatballs/support_portals/domain_services.py index ee9a246..c71c562 100644 --- a/apps/backend/chatballs/support_portals/domain_services.py +++ b/apps/backend/chatballs/support_portals/domain_services.py @@ -2,12 +2,12 @@ from __future__ import annotations import dns.exception import dns.resolver -from django.conf import settings from django.core.exceptions import ValidationError from django.utils import timezone from chatballs.support_portals.addressing import normalize_domain, validate_domain from chatballs.support_portals.models import SupportPortal +from chatballs.support_portals.public_address import help_public_ipv4 from chatballs.support_portals.statuses import PortalStatus @@ -37,7 +37,8 @@ def verify_custom_domain(portal: SupportPortal) -> SupportPortal: # и установка принадлежат одному владельцу (README дизайн-базлайна, решение # 6). Остаётся техническая проверка «ведёт ли домен на этот сервер» — она # нужна, чтобы выписать сертификат. - if settings.CHATBALLS_HELP_PUBLIC_IPV4: + server_ipv4 = help_public_ipv4() + if server_ipv4: try: address_answers = dns.resolver.resolve(portal.custom_domain, "A") addresses = { @@ -53,7 +54,7 @@ def verify_custom_domain(portal: SupportPortal) -> SupportPortal: raise ValidationError( {"customDomain": "A-запись домена пока не найдена"} ) from error - if settings.CHATBALLS_HELP_PUBLIC_IPV4 not in addresses: + if server_ipv4 not in addresses: raise ValidationError( {"customDomain": "A-запись домена указывает не на сервер Chatballs"} ) diff --git a/apps/backend/chatballs/support_portals/gateway_views.py b/apps/backend/chatballs/support_portals/gateway_views.py index b9cc657..f262fb3 100644 --- a/apps/backend/chatballs/support_portals/gateway_views.py +++ b/apps/backend/chatballs/support_portals/gateway_views.py @@ -3,18 +3,35 @@ from rest_framework.request import Request from rest_framework.response import Response from rest_framework.views import APIView +from chatballs.identity.instance_settings import accepted_hosts from chatballs.support_portals.addressing import normalize_domain from chatballs.tenancy.ingress import support_portal_route class HelpDomainAuthorizationView(APIView): - """Caddy on-demand TLS authorization for published portal hosts.""" + """Caddy on-demand TLS authorization. + + Шлюз спрашивает разрешение перед выпуском сертификата на каждый новый хост. + Разрешены два вида адресов, и оба человек задал сам в интерфейсе: + + - адрес самой установки (мастер первого запуска запомнил его, владелец + меняет в «Настройках») — без этого коробка навсегда оставалась бы на + http: свой домен ей выписать было нечем; + - домены опубликованных порталов помощи. + + Всё остальное — 404, иначе любой указавший на нас домен заставлял бы + установку ходить в ACME. + """ authentication_classes: list = [] permission_classes = [AllowAny] def get(self, request: Request) -> Response: domain = normalize_domain(str(request.query_params.get("domain", ""))) - if not domain or support_portal_route(domain) is None: + if not domain: + return Response(status=404) + if domain in {normalize_domain(item) for item in accepted_hosts()}: + return Response(status=204) + if support_portal_route(domain) is None: return Response(status=404) return Response(status=204) diff --git a/apps/backend/chatballs/support_portals/host_boundary.py b/apps/backend/chatballs/support_portals/host_boundary.py index ab10c56..9d688f8 100644 --- a/apps/backend/chatballs/support_portals/host_boundary.py +++ b/apps/backend/chatballs/support_portals/host_boundary.py @@ -39,12 +39,13 @@ class SupportPortalHostBoundaryMiddleware: or str(connections["default"].settings_dict["NAME"]).startswith("test_") ): return True - # Адрес, который человек ввёл в браузере на первом запуске: продукт - # запомнил его в настройках установки и признаёт своим. - from chatballs.identity.instance_settings import public_host + # Адреса, которые человек задал сам: тот, на котором прошли мастер, и + # предыдущий — чтобы смена адреса в «Настройках» не выбрасывала того, + # кто её делает, до того как новый домен вообще заработал. + from chatballs.identity.instance_settings import accepted_hosts try: - if host and host == public_host(): + if host and host in {normalize_domain(item) for item in accepted_hosts()}: return True except Exception: pass @@ -76,6 +77,10 @@ def _instance_is_unconfigured() -> bool: except Exception: # База ещё не поднялась: на этом этапе хост тем более не проверить. return True - if not needs_setup: + if not needs_setup and not settings.TESTING: + # Кэш живёт на весь процесс — в проде это ровно то, что нужно + # (после установки признак больше не меняется). В тестах каждый тест + # начинается с пустой базы, и первый же из них, создавший + # организацию, закрывал бы мастер всем остальным в том же процессе. _configured = True return needs_setup diff --git a/apps/backend/chatballs/support_portals/public_address.py b/apps/backend/chatballs/support_portals/public_address.py new file mode 100644 index 0000000..336afec --- /dev/null +++ b/apps/backend/chatballs/support_portals/public_address.py @@ -0,0 +1,84 @@ +"""Адрес, на который владелец направляет A-запись домена портала. + +Раньше это была переменная окружения с обязательной проверкой при старте, а у +коробки её задавать негде — поэтому значение по умолчанию схлопывалось в +``127.0.0.1``, и продукт печатал владельцу инструкцию «направьте домен на +loopback». Теперь источник тот же, что и у всего остального адресного: адрес +самой установки, который человек ввёл в мастере первого запуска и меняет в +«Настройках». + +Порядок: явная переменная окружения (контуры, которые ведут конфигурацию +сами) → адрес установки, если это IPv4 → его A-запись, если это домен. Пусто +— значит показывать нечего: до мастера адреса ещё нет. +""" + +from __future__ import annotations + +import threading +import time +from ipaddress import IPv4Address + +from django.conf import settings + +_CACHE_TTL_SECONDS = 60.0 +_lock = threading.Lock() +_cached: tuple[float, str] | None = None + + +def invalidate_cache() -> None: + global _cached + with _lock: + _cached = None + + +def _as_ipv4(value: str) -> str: + try: + return str(IPv4Address(value.strip())) + except ValueError: + return "" + + +def _resolve_a_record(hostname: str) -> str: + import dns.exception + import dns.resolver + + try: + answers = dns.resolver.resolve(hostname, "A") + except ( + dns.resolver.NoAnswer, + dns.resolver.NXDOMAIN, + dns.resolver.NoNameservers, + dns.exception.Timeout, + ): + return "" + for answer in answers: + address = _as_ipv4(getattr(answer, "address", str(answer).rstrip("."))) + if address: + return address + return "" + + +def help_public_ipv4() -> str: + """IPv4 установки для инструкции «направьте A-запись сюда»; пусто — нечего показать.""" + + configured = str(getattr(settings, "CHATBALLS_HELP_PUBLIC_IPV4", "") or "") + if configured: + return configured + + global _cached + now = time.monotonic() + with _lock: + if _cached is not None and now - _cached[0] < _CACHE_TTL_SECONDS: + return _cached[1] + + from chatballs.identity.instance_settings import public_host + + try: + host = public_host() + except Exception: # таблицы ещё нет (ранние миграции) + return "" + address = _as_ipv4(host) or (_resolve_a_record(host) if host else "") + + with _lock: + _cached = (now, address) + return address diff --git a/apps/backend/chatballs/support_portals/public_views.py b/apps/backend/chatballs/support_portals/public_views.py index 551be0a..0af2129 100644 --- a/apps/backend/chatballs/support_portals/public_views.py +++ b/apps/backend/chatballs/support_portals/public_views.py @@ -1,8 +1,8 @@ from rest_framework.permissions import AllowAny from rest_framework.request import Request from rest_framework.response import Response -from rest_framework.views import APIView from rest_framework.throttling import ScopedRateThrottle +from rest_framework.views import APIView from chatballs.identity.models import Organization from chatballs.support_portals.content_services import record_feedback diff --git a/apps/backend/chatballs/support_portals/serializers.py b/apps/backend/chatballs/support_portals/serializers.py index b5e2e12..2ca631d 100644 --- a/apps/backend/chatballs/support_portals/serializers.py +++ b/apps/backend/chatballs/support_portals/serializers.py @@ -1,4 +1,3 @@ -from django.conf import settings from chatballs.integrations.models import IntegrationProvider, IntegrationStatus from chatballs.support_portals.addressing import portal_public_url @@ -10,9 +9,11 @@ from chatballs.support_portals.models import ( PortalCategory, SupportPortal, ) +from chatballs.support_portals.public_address import help_public_ipv4 def portal_payload(portal: SupportPortal, *, counts: dict | None = None) -> dict: + server_ipv4 = help_public_ipv4() if portal.custom_domain else "" public_url = portal_public_url( hosted=portal.hosted_domain, custom=portal.custom_domain, @@ -28,9 +29,9 @@ def portal_payload(portal: SupportPortal, *, counts: dict | None = None) -> dict { "name": portal.custom_domain, "type": "A", - "value": settings.CHATBALLS_HELP_PUBLIC_IPV4, + "value": server_ipv4, } - if portal.custom_domain and settings.CHATBALLS_HELP_PUBLIC_IPV4 + if portal.custom_domain and server_ipv4 else None ), "customDomainVerifiedAt": portal.custom_domain_verified_at, diff --git a/apps/backend/chatballs/support_portals/tests/test_public_address.py b/apps/backend/chatballs/support_portals/tests/test_public_address.py new file mode 100644 index 0000000..bb83277 --- /dev/null +++ b/apps/backend/chatballs/support_portals/tests/test_public_address.py @@ -0,0 +1,55 @@ +"""IPv4 для A-записи домена портала берётся из адреса установки. + +Раньше это была переменная окружения с обязательной проверкой при старте, а у +коробки её задавать негде — значение по умолчанию схлопывалось в 127.0.0.1, и +продукт печатал владельцу инструкцию «направьте домен на loopback». +""" + +from __future__ import annotations + +from django.test import TestCase, override_settings + +from chatballs.identity.instance_settings import InstanceSettings, invalidate_cache +from chatballs.support_portals import public_address + + +class HelpPublicIpv4Tests(TestCase): + def setUp(self) -> None: + public_address.invalidate_cache() + invalidate_cache() + self.addCleanup(public_address.invalidate_cache) + self.addCleanup(invalidate_cache) + + def _set_host(self, host: str) -> None: + row = InstanceSettings.load() + row.public_host = host + row.save(update_fields=["public_host", "updated_at"]) + invalidate_cache() + public_address.invalidate_cache() + + @override_settings(CHATBALLS_HELP_PUBLIC_IPV4="") + def test_ip_installation_address_is_used_as_is(self) -> None: + self._set_host("203.0.113.10") + + self.assertEqual(public_address.help_public_ipv4(), "203.0.113.10") + + @override_settings(CHATBALLS_HELP_PUBLIC_IPV4="") + def test_no_address_yet_means_nothing_to_show(self) -> None: + self._set_host("") + + self.assertEqual(public_address.help_public_ipv4(), "") + + @override_settings(CHATBALLS_HELP_PUBLIC_IPV4="") + def test_domain_installation_address_is_resolved(self) -> None: + self._set_host("crm.example.test") + original = public_address._resolve_a_record + public_address._resolve_a_record = lambda host: "198.51.100.7" + self.addCleanup(setattr, public_address, "_resolve_a_record", original) + + self.assertEqual(public_address.help_public_ipv4(), "198.51.100.7") + + @override_settings(CHATBALLS_HELP_PUBLIC_IPV4="192.0.2.5") + def test_explicit_setting_wins(self) -> None: + self._set_host("203.0.113.10") + + self.assertEqual(public_address.help_public_ipv4(), "192.0.2.5") diff --git a/apps/backend/chatballs/support_portals/tests/test_public_api.py b/apps/backend/chatballs/support_portals/tests/test_public_api.py index 92e4834..2eb9144 100644 --- a/apps/backend/chatballs/support_portals/tests/test_public_api.py +++ b/apps/backend/chatballs/support_portals/tests/test_public_api.py @@ -2,8 +2,9 @@ from django.core.files.uploadedfile import SimpleUploadedFile from django.test import TestCase, override_settings from django.utils import timezone -from chatballs.identity.bootstrap import bootstrap_owner from chatballs.channels.models import Channel +from chatballs.identity.bootstrap import bootstrap_owner +from chatballs.identity.instance_settings import InstanceSettings, invalidate_cache from chatballs.support_portals.models import PortalArticleFeedback, SupportPortal from chatballs.testing import TenantAPIClient from chatballs.webchat.testing import create_web_widget @@ -205,6 +206,39 @@ class PublicSupportPortalTests(TestCase): sorted([attached_doc["name"], attached_image["name"]]), ) + @override_settings(ROOT_URLCONF="chatballs_backend.urls_platform") + def test_gateway_authorizes_the_installation_address(self) -> None: + """Свой домен установки шлюз обязан уметь закрыть сертификатом. + + Адрес коробка знает только от человека: мастер первого запуска + запомнил, на чём его открыли, владелец меняет это в «Настройках». + Пока ask-эндпоинт отвечал 404 на всё, кроме порталов, установка + оставалась на http навсегда — выписать сертификат было нечем. + """ + invalidate_cache() + unknown = self.client.get( + "/api/v1/gateway/help-domain/", {"domain": "crm.example.test"} + ) + self.assertEqual(unknown.status_code, 404, unknown.content) + + row = InstanceSettings.load() + row.public_host = "crm.example.test" + row.public_scheme = "https" + row.save(update_fields=["public_host", "public_scheme", "updated_at"]) + invalidate_cache() + + allowed = self.client.get( + "/api/v1/gateway/help-domain/", {"domain": "CRM.example.test." } + ) + stranger = self.client.get( + "/api/v1/gateway/help-domain/", {"domain": "someone-else.example"} + ) + empty = self.client.get("/api/v1/gateway/help-domain/") + + self.assertEqual(allowed.status_code, 204, allowed.content) + self.assertEqual(stranger.status_code, 404, stranger.content) + self.assertEqual(empty.status_code, 404, empty.content) + @override_settings(ROOT_URLCONF="chatballs_backend.urls_platform") def test_gateway_authorizes_only_published_portal_domains(self) -> None: custom_domain = "help.app.example" diff --git a/apps/backend/chatballs/tenancy/database.py b/apps/backend/chatballs/tenancy/database.py index e326a4a..a42a537 100644 --- a/apps/backend/chatballs/tenancy/database.py +++ b/apps/backend/chatballs/tenancy/database.py @@ -1,8 +1,8 @@ from __future__ import annotations -from collections.abc import Iterator +from collections.abc import Callable, Iterator from contextlib import contextmanager -from typing import Any, Callable +from typing import Any from django.db import DEFAULT_DB_ALIAS, connections, transaction diff --git a/apps/backend/chatballs/tenancy/ingress.py b/apps/backend/chatballs/tenancy/ingress.py index 21563b0..88aa52c 100644 --- a/apps/backend/chatballs/tenancy/ingress.py +++ b/apps/backend/chatballs/tenancy/ingress.py @@ -7,7 +7,6 @@ from django.conf import settings from django.db import connections - @dataclass(frozen=True, slots=True) class IngressRoute: organization_id: int diff --git a/apps/backend/chatballs/tenancy/middleware.py b/apps/backend/chatballs/tenancy/middleware.py index 7b04248..535714d 100644 --- a/apps/backend/chatballs/tenancy/middleware.py +++ b/apps/backend/chatballs/tenancy/middleware.py @@ -1,8 +1,8 @@ from __future__ import annotations -from collections.abc import Callable import re import uuid +from collections.abc import Callable from django.http import Http404, HttpRequest, HttpResponse from django.urls import Resolver404, resolve diff --git a/apps/backend/chatballs/tenancy/migrations/0007_platform_row_select_policies.py b/apps/backend/chatballs/tenancy/migrations/0007_platform_row_select_policies.py index 4b47347..9ed42ae 100644 --- a/apps/backend/chatballs/tenancy/migrations/0007_platform_row_select_policies.py +++ b/apps/backend/chatballs/tenancy/migrations/0007_platform_row_select_policies.py @@ -1,27 +1,27 @@ -from django.db import migrations - -# Django ORM inserts use INSERT ... RETURNING id; PostgreSQL applies SELECT -# policies to rows returned by RETURNING, so the app role needs SELECT -# visibility of the platform-scope rows it is allowed to insert -# (chatballs_app_platform_audit_insert / chatballs_app_platform_outbox_insert). -# Without these policies a login-failed audit write fails with -# "new row violates row-level security policy". -CREATE_POLICIES = """ -DROP POLICY IF EXISTS chatballs_app_platform_audit_select ON identity_auditevent; -CREATE POLICY chatballs_app_platform_audit_select ON identity_auditevent - FOR SELECT TO chatballs_runtime_app USING (organization_id IS NULL); -DROP POLICY IF EXISTS chatballs_app_platform_outbox_select ON events_outboxevent; -CREATE POLICY chatballs_app_platform_outbox_select ON events_outboxevent - FOR SELECT TO chatballs_runtime_app - USING (ownership = 'PLATFORM' AND organization_id IS NULL); -""" - -DROP_POLICIES = """ -DROP POLICY IF EXISTS chatballs_app_platform_audit_select ON identity_auditevent; -DROP POLICY IF EXISTS chatballs_app_platform_outbox_select ON events_outboxevent; -""" - - -class Migration(migrations.Migration): - dependencies = [("tenancy", "0006_storage_reserved_bytes")] - operations = [migrations.RunSQL(CREATE_POLICIES, DROP_POLICIES)] +from django.db import migrations + +# Django ORM inserts use INSERT ... RETURNING id; PostgreSQL applies SELECT +# policies to rows returned by RETURNING, so the app role needs SELECT +# visibility of the platform-scope rows it is allowed to insert +# (chatballs_app_platform_audit_insert / chatballs_app_platform_outbox_insert). +# Without these policies a login-failed audit write fails with +# "new row violates row-level security policy". +CREATE_POLICIES = """ +DROP POLICY IF EXISTS chatballs_app_platform_audit_select ON identity_auditevent; +CREATE POLICY chatballs_app_platform_audit_select ON identity_auditevent + FOR SELECT TO chatballs_runtime_app USING (organization_id IS NULL); +DROP POLICY IF EXISTS chatballs_app_platform_outbox_select ON events_outboxevent; +CREATE POLICY chatballs_app_platform_outbox_select ON events_outboxevent + FOR SELECT TO chatballs_runtime_app + USING (ownership = 'PLATFORM' AND organization_id IS NULL); +""" + +DROP_POLICIES = """ +DROP POLICY IF EXISTS chatballs_app_platform_audit_select ON identity_auditevent; +DROP POLICY IF EXISTS chatballs_app_platform_outbox_select ON events_outboxevent; +""" + + +class Migration(migrations.Migration): + dependencies = [("tenancy", "0006_storage_reserved_bytes")] + operations = [migrations.RunSQL(CREATE_POLICIES, DROP_POLICIES)] diff --git a/apps/backend/chatballs/tenancy/migrations/0009_tenant_guards_security_definer.py b/apps/backend/chatballs/tenancy/migrations/0009_tenant_guards_security_definer.py index 4010764..20851e0 100644 --- a/apps/backend/chatballs/tenancy/migrations/0009_tenant_guards_security_definer.py +++ b/apps/backend/chatballs/tenancy/migrations/0009_tenant_guards_security_definer.py @@ -1,43 +1,43 @@ -"""Guard-функции C04 должны видеть строки поверх RLS (SECURITY DEFINER). - -Функции проверяют целостность связей между тенантами и обязаны читать -родительскую строку независимо от политик вызывающей роли. Как SECURITY -INVOKER их внутренний SELECT подчинялся RLS: из кросс-тенантного пути без -выставленного `chatballs.organization_id` (outbox-воркер claim'ит событие -любой организации) родитель не виден, parent_org = NULL, и проверка -`IS DISTINCT FROM` ложно срабатывала — легитимный UPDATE падал с -`cross-tenant relation`, что уводило воркер в краш-петлю. - -SECURITY DEFINER исполняет их от chatballs_schema (policy USING (true)), -поэтому сравнение идёт по фактическим данным. Проверка не ослабляется: -настоящее нарушение по-прежнему приводит к RAISE. search_path закреплён, -как того требует безопасность SECURITY DEFINER-функций. -""" - -from django.db import migrations - - -GUARD_FUNCTIONS = ( - "chatballs.enforce_tenant_fk()", - "chatballs.enforce_tenant_user()", - "chatballs.enforce_tenant_pair()", -) - - -def set_security_definer(apps, schema_editor): - for function in GUARD_FUNCTIONS: - schema_editor.execute( - f"ALTER FUNCTION {function} SECURITY DEFINER SET search_path = public, pg_temp" - ) - - -def set_security_invoker(apps, schema_editor): - for function in GUARD_FUNCTIONS: - schema_editor.execute( - f"ALTER FUNCTION {function} SECURITY INVOKER RESET search_path" - ) - - -class Migration(migrations.Migration): - dependencies = [("tenancy", "0008_ai_knowledge_scope_guards")] - operations = [migrations.RunPython(set_security_definer, set_security_invoker)] +"""Guard-функции C04 должны видеть строки поверх RLS (SECURITY DEFINER). + +Функции проверяют целостность связей между тенантами и обязаны читать +родительскую строку независимо от политик вызывающей роли. Как SECURITY +INVOKER их внутренний SELECT подчинялся RLS: из кросс-тенантного пути без +выставленного `chatballs.organization_id` (outbox-воркер claim'ит событие +любой организации) родитель не виден, parent_org = NULL, и проверка +`IS DISTINCT FROM` ложно срабатывала — легитимный UPDATE падал с +`cross-tenant relation`, что уводило воркер в краш-петлю. + +SECURITY DEFINER исполняет их от chatballs_schema (policy USING (true)), +поэтому сравнение идёт по фактическим данным. Проверка не ослабляется: +настоящее нарушение по-прежнему приводит к RAISE. search_path закреплён, +как того требует безопасность SECURITY DEFINER-функций. +""" + +from django.db import migrations + + +GUARD_FUNCTIONS = ( + "chatballs.enforce_tenant_fk()", + "chatballs.enforce_tenant_user()", + "chatballs.enforce_tenant_pair()", +) + + +def set_security_definer(apps, schema_editor): + for function in GUARD_FUNCTIONS: + schema_editor.execute( + f"ALTER FUNCTION {function} SECURITY DEFINER SET search_path = public, pg_temp" + ) + + +def set_security_invoker(apps, schema_editor): + for function in GUARD_FUNCTIONS: + schema_editor.execute( + f"ALTER FUNCTION {function} SECURITY INVOKER RESET search_path" + ) + + +class Migration(migrations.Migration): + dependencies = [("tenancy", "0008_ai_knowledge_scope_guards")] + operations = [migrations.RunPython(set_security_definer, set_security_invoker)] diff --git a/apps/backend/chatballs/tenancy/migrations/0030_encrypted_column_widths.py b/apps/backend/chatballs/tenancy/migrations/0030_encrypted_column_widths.py new file mode 100644 index 0000000..ea2f986 --- /dev/null +++ b/apps/backend/chatballs/tenancy/migrations/0030_encrypted_column_widths.py @@ -0,0 +1,28 @@ +"""Ключи S3: колонки под шифротекст (см. identity.0034).""" + +from django.db import migrations + +import chatballs.identity.crypto + + +class Migration(migrations.Migration): + dependencies = [ + ("tenancy", "0029_drop_product_support"), + ] + + operations = [ + migrations.AlterField( + model_name="storagesettings", + name="s3_access_key", + field=chatballs.identity.crypto.EncryptedCharField( + blank=True, default="", max_length=2828 + ), + ), + migrations.AlterField( + model_name="storagesettings", + name="s3_secret_key", + field=chatballs.identity.crypto.EncryptedCharField( + blank=True, default="", max_length=2828 + ), + ), + ] diff --git a/apps/backend/chatballs/tenancy/models.py b/apps/backend/chatballs/tenancy/models.py index 6e0d67e..058bb61 100644 --- a/apps/backend/chatballs/tenancy/models.py +++ b/apps/backend/chatballs/tenancy/models.py @@ -19,6 +19,10 @@ class TenantRelationModel(models.Model): class Meta: abstract = True + def save(self, *args: object, **kwargs: object) -> None: + self.validate_tenant_relations() + super().save(*args, **kwargs) + def _related_organization_ids(self) -> set[int]: organization_ids: set[int] = set() for field_name in self.tenant_relation_fields: @@ -48,10 +52,6 @@ class TenantRelationModel(models.Model): super().clean() self.validate_tenant_relations() - def save(self, *args: object, **kwargs: object) -> None: - self.validate_tenant_relations() - super().save(*args, **kwargs) - class OrganizationStorageUsage(models.Model): """Authoritative storage_bytes usage counter for one organization. @@ -82,6 +82,9 @@ class OrganizationStorageUsage(models.Model): ), ] + def __str__(self) -> str: + return f"storage:{self.organization_id}/{self.bytes_used + self.reserved_bytes}" + class StorageReservation(models.Model): """An in-flight storage_bytes reservation keyed by an idempotency token, so a diff --git a/apps/backend/chatballs/tenancy/storage_settings.py b/apps/backend/chatballs/tenancy/storage_settings.py index 1434bc1..3ce4826 100644 --- a/apps/backend/chatballs/tenancy/storage_settings.py +++ b/apps/backend/chatballs/tenancy/storage_settings.py @@ -62,13 +62,16 @@ class StorageSettings(models.Model): class Meta: verbose_name = "Настройки хранилища" + def __str__(self) -> str: + return f"storage-settings:{self.backend.lower()}" + def save(self, *args, **kwargs): self.pk = self.SINGLETON_PK super().save(*args, **kwargs) invalidate_cache() @classmethod - def load(cls) -> "StorageSettings": + def load(cls) -> StorageSettings: obj, _ = cls.objects.get_or_create(pk=cls.SINGLETON_PK) return obj diff --git a/apps/backend/chatballs/tenancy/storage_views.py b/apps/backend/chatballs/tenancy/storage_views.py index 3b002ce..5e3fa0f 100644 --- a/apps/backend/chatballs/tenancy/storage_views.py +++ b/apps/backend/chatballs/tenancy/storage_views.py @@ -12,7 +12,6 @@ POST company/administration/storage/migrate/ — перенести локал from __future__ import annotations from django.core.exceptions import ValidationError -from rest_framework.permissions import IsAuthenticated from rest_framework.request import Request from rest_framework.response import Response from rest_framework.views import APIView diff --git a/apps/backend/chatballs/tenancy/test_rls.py b/apps/backend/chatballs/tenancy/test_rls.py index ad3f446..bd74134 100644 --- a/apps/backend/chatballs/tenancy/test_rls.py +++ b/apps/backend/chatballs/tenancy/test_rls.py @@ -1,43 +1,23 @@ from django.db import DatabaseError, connection, transaction - from django.test import TransactionTestCase - - from chatballs.ai.knowledge_categories import ensure_uncategorized_category - from chatballs.ai.models import AIAgent, Knowledge - from chatballs.channels.models import Channel - from chatballs.identity.group_models import EmployeeGroup - from chatballs.identity.models import ( - AuditEvent, - AuditResult, - EmployeeRole, - HumanUser, - Organization, - OrganizationMembership, - ) - from chatballs.tenancy.database import current_tenant_id, set_local_tenant - from chatballs.tenancy.models import StorageReservation - from chatballs.testing import TenantAPIClient - - - class RowLevelSecurityTests(TransactionTestCase): reset_sequences = True diff --git a/apps/backend/chatballs/webchat/apps.py b/apps/backend/chatballs/webchat/apps.py index 7de75ba..f0a65a6 100644 --- a/apps/backend/chatballs/webchat/apps.py +++ b/apps/backend/chatballs/webchat/apps.py @@ -1,8 +1,8 @@ -from django.apps import AppConfig - - -class WebchatConfig(AppConfig): - default_auto_field = "django.db.models.BigAutoField" - label = "webchat" - name = "chatballs.webchat" - verbose_name = "Web chat widget" +from django.apps import AppConfig + + +class WebchatConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + label = "webchat" + name = "chatballs.webchat" + verbose_name = "Web chat widget" diff --git a/apps/backend/chatballs/webchat/services.py b/apps/backend/chatballs/webchat/services.py index da15ecf..3f4b0fa 100644 --- a/apps/backend/chatballs/webchat/services.py +++ b/apps/backend/chatballs/webchat/services.py @@ -12,8 +12,8 @@ from chatballs.conversations.ingest import ingest_inbound from chatballs.conversations.models import ( ConnectionIdentity, Contact, - Conversation, ControlMode, + Conversation, LifecycleState, MessageKind, ) @@ -218,7 +218,11 @@ def post_voice(session: WebSession, *, content: bytes, content_type: str, durati def post_file(session: WebSession, *, content: bytes, filename: str, content_type: str, caption: str = "") -> None: """Файл из виджета: байты приходят телом запроса, подпись — текстом.""" - from chatballs.conversations.transports.base import InboundFile, guess_content_type, safe_filename + from chatballs.conversations.transports.base import ( + InboundFile, + guess_content_type, + safe_filename, + ) name = safe_filename(filename) mime = content_type or guess_content_type(name) diff --git a/apps/backend/chatballs/webchat/views.py b/apps/backend/chatballs/webchat/views.py index c8de47c..1bc3329 100644 --- a/apps/backend/chatballs/webchat/views.py +++ b/apps/backend/chatballs/webchat/views.py @@ -8,8 +8,11 @@ from rest_framework.request import Request from rest_framework.response import Response from rest_framework.views import APIView +from chatballs.conversations.attachment_views import ( + attachment_response, + validate_upload, +) from chatballs.conversations.models import Message, MessageKind -from chatballs.conversations.attachment_views import MAX_FILE_BYTES, attachment_response, validate_upload from chatballs.conversations.voice_views import ALLOWED_AUDIO_TYPES, MAX_VOICE_BYTES from chatballs.identity.models import Organization from chatballs.integrations.features import voice_messages_allowed diff --git a/apps/backend/chatballs_backend/settings_base.py b/apps/backend/chatballs_backend/settings_base.py index 88e21f8..8e3629e 100644 --- a/apps/backend/chatballs_backend/settings_base.py +++ b/apps/backend/chatballs_backend/settings_base.py @@ -251,20 +251,18 @@ CHATBALLS_HELP_BASE_DOMAIN = os.environ.get( ).strip().lower().rstrip(".") CHATBALLS_HELP_PUBLIC_SCHEME = os.environ.get("CHATBALLS_HELP_PUBLIC_SCHEME", "https").strip().lower() CHATBALLS_HELP_PUBLIC_PORT = os.environ.get("CHATBALLS_HELP_PUBLIC_PORT", "").strip() -_default_help_public_ipv4 = os.environ.get( - "CHATBALLS_WEB_LISTENING_IP", - "", -).strip() -if _default_help_public_ipv4 in {"", "0.0.0.0", "::"}: - _default_help_public_ipv4 = "127.0.0.1" if CHATBALLS_HELP_BASE_DOMAIN == "localhost" else "" -CHATBALLS_HELP_PUBLIC_IPV4 = os.environ.get( - "CHATBALLS_HELP_PUBLIC_IPV4", - _default_help_public_ipv4, -).strip() -if not DEBUG and not TESTING and not CHATBALLS_HELP_PUBLIC_IPV4: - raise ImproperlyConfigured( - "CHATBALLS_HELP_PUBLIC_IPV4 or a non-wildcard CHATBALLS_WEB_LISTENING_IP is required" - ) +# Адрес, на который владелец направляет A-запись домена портала. Штатный +# источник — сам адрес установки (его знает только она сама, см. +# chatballs.support_portals.public_address); переменные ниже остаются +# переопределением для контуров, которые ведут конфигурацию сами. Пустое +# значение — не ошибка установки: до мастера первого запуска адреса просто +# ещё нет, а порталов с доменами тем более. +_configured_help_ipv4 = os.environ.get("CHATBALLS_HELP_PUBLIC_IPV4", "").strip() +if not _configured_help_ipv4: + _listening_ip = os.environ.get("CHATBALLS_WEB_LISTENING_IP", "").strip() + if _listening_ip not in {"", "0.0.0.0", "::"}: + _configured_help_ipv4 = _listening_ip +CHATBALLS_HELP_PUBLIC_IPV4 = _configured_help_ipv4 if CHATBALLS_HELP_PUBLIC_IPV4: try: IPv4Address(CHATBALLS_HELP_PUBLIC_IPV4) diff --git a/apps/backend/chatballs_backend/settings_database.py b/apps/backend/chatballs_backend/settings_database.py index c902b96..1e251a9 100644 --- a/apps/backend/chatballs_backend/settings_database.py +++ b/apps/backend/chatballs_backend/settings_database.py @@ -1,93 +1,93 @@ -import os - -from django.core.exceptions import ImproperlyConfigured - -from chatballs_backend.settings_env import env_secret - - -def _credentials() -> tuple[dict[str, str], dict[str, str]]: - # Имена ролей — константы продукта, а не настройка установки: их создаёт - # deploy/postgres/init-runtime-roles.sh при первом старте. Переменные - # окружения остаются переопределением для нестандартных установок. - users = { - "app": os.environ.get("POSTGRES_APP_USER", "chatballs_app"), - "platform": os.environ.get("POSTGRES_PLATFORM_USER", "chatballs_platform"), - "migration": os.environ.get( - "POSTGRES_MIGRATION_USER", "chatballs_migration" - ), - } - # Пароли ролей генерирует первый старт стека в том с секретами; человек их - # не вводит и не хранит. Переменные окружения остаются переопределением. - fallback = env_secret("POSTGRES_PASSWORD", "postgres_password", "chatballs") - passwords = { - "app": env_secret("POSTGRES_APP_PASSWORD", "postgres_app_password", fallback), - "platform": env_secret( - "POSTGRES_PLATFORM_PASSWORD", "postgres_platform_password", fallback - ), - "migration": env_secret( - "POSTGRES_MIGRATION_PASSWORD", "postgres_migration_password", fallback - ), - } - return users, passwords - - -def _pool_options(*, testing: bool) -> dict | None: - # ASGI-серверы исполняют ORM в короткоживущих потоках sync_to_async; - # persistent-соединения (CONN_MAX_AGE > 0) в таких потоках осиротевают и - # исчерпывают max_connections Postgres. Вместо них — psycopg pool на процесс: - # соединения возвращаются в пул независимо от потока и ограничены сверху. - # CHATBALLS_DB_POOL_MAX=0 отключает пул (короткоживущие соединения на запрос). - if testing: - return None - max_size = int(os.environ.get("CHATBALLS_DB_POOL_MAX", "4")) - if max_size <= 0: - return None - return { - "min_size": int(os.environ.get("CHATBALLS_DB_POOL_MIN", "1")), - "max_size": max_size, - "timeout": float(os.environ.get("CHATBALLS_DB_POOL_TIMEOUT", "10")), - } - - -def build_databases(*, debug: bool, testing: bool) -> dict[str, dict]: - role = os.environ.get("CHATBALLS_DB_ROLE", "app").lower() - if role not in {"app", "platform", "migration"}: - raise ImproperlyConfigured("CHATBALLS_DB_ROLE must be app, platform or migration") - users, passwords = _credentials() - if not debug and not testing and len(set(users.values())) != 3: - raise ImproperlyConfigured( - "App, platform and migration database users must be distinct" - ) - pool = _pool_options(testing=testing) - - def config(selected_role: str) -> dict: - return { - "ENGINE": "django.db.backends.postgresql", - "NAME": os.environ.get("POSTGRES_DB", "chatballs"), - "USER": users[selected_role], - "PASSWORD": passwords[selected_role], - "HOST": os.environ.get("POSTGRES_HOST", "postgres"), - "PORT": os.environ.get("POSTGRES_PORT", "5432"), - # Пул несовместим с persistent-соединениями: с ним CONN_MAX_AGE - # обязан быть 0, а без пула persistent-режим возвращать нельзя - # (см. _pool_options). - "CONN_MAX_AGE": 0, - "OPTIONS": {"pool": dict(pool)} if pool else {}, - } - - databases = { - "default": config("migration" if testing else role), - "platform": config("platform"), - } - if testing: - # Тесты создают свою БД и подключаются владельцем кластера. Его пароль - # приходит оттуда же, откуда у остальных ролей: файл секрета инстанса, - # переменная окружения — переопределение. - databases["default"]["USER"] = os.environ.get( - "POSTGRES_USER", "chatballs_bootstrap" - ) - databases["default"]["PASSWORD"] = env_secret( - "POSTGRES_PASSWORD", "postgres_password", "chatballs" - ) - databases["platform"]["TEST"] = {"MIRROR": "default"} - return databases +import os + +from django.core.exceptions import ImproperlyConfigured + +from chatballs_backend.settings_env import env_secret + + +def _credentials() -> tuple[dict[str, str], dict[str, str]]: + # Имена ролей — константы продукта, а не настройка установки: их создаёт + # deploy/postgres/init-runtime-roles.sh при первом старте. Переменные + # окружения остаются переопределением для нестандартных установок. + users = { + "app": os.environ.get("POSTGRES_APP_USER", "chatballs_app"), + "platform": os.environ.get("POSTGRES_PLATFORM_USER", "chatballs_platform"), + "migration": os.environ.get( + "POSTGRES_MIGRATION_USER", "chatballs_migration" + ), + } + # Пароли ролей генерирует первый старт стека в том с секретами; человек их + # не вводит и не хранит. Переменные окружения остаются переопределением. + fallback = env_secret("POSTGRES_PASSWORD", "postgres_password", "chatballs") + passwords = { + "app": env_secret("POSTGRES_APP_PASSWORD", "postgres_app_password", fallback), + "platform": env_secret( + "POSTGRES_PLATFORM_PASSWORD", "postgres_platform_password", fallback + ), + "migration": env_secret( + "POSTGRES_MIGRATION_PASSWORD", "postgres_migration_password", fallback + ), + } + return users, passwords + + +def _pool_options(*, testing: bool) -> dict | None: + # ASGI-серверы исполняют ORM в короткоживущих потоках sync_to_async; + # persistent-соединения (CONN_MAX_AGE > 0) в таких потоках осиротевают и + # исчерпывают max_connections Postgres. Вместо них — psycopg pool на процесс: + # соединения возвращаются в пул независимо от потока и ограничены сверху. + # CHATBALLS_DB_POOL_MAX=0 отключает пул (короткоживущие соединения на запрос). + if testing: + return None + max_size = int(os.environ.get("CHATBALLS_DB_POOL_MAX", "4")) + if max_size <= 0: + return None + return { + "min_size": int(os.environ.get("CHATBALLS_DB_POOL_MIN", "1")), + "max_size": max_size, + "timeout": float(os.environ.get("CHATBALLS_DB_POOL_TIMEOUT", "10")), + } + + +def build_databases(*, debug: bool, testing: bool) -> dict[str, dict]: + role = os.environ.get("CHATBALLS_DB_ROLE", "app").lower() + if role not in {"app", "platform", "migration"}: + raise ImproperlyConfigured("CHATBALLS_DB_ROLE must be app, platform or migration") + users, passwords = _credentials() + if not debug and not testing and len(set(users.values())) != 3: + raise ImproperlyConfigured( + "App, platform and migration database users must be distinct" + ) + pool = _pool_options(testing=testing) + + def config(selected_role: str) -> dict: + return { + "ENGINE": "django.db.backends.postgresql", + "NAME": os.environ.get("POSTGRES_DB", "chatballs"), + "USER": users[selected_role], + "PASSWORD": passwords[selected_role], + "HOST": os.environ.get("POSTGRES_HOST", "postgres"), + "PORT": os.environ.get("POSTGRES_PORT", "5432"), + # Пул несовместим с persistent-соединениями: с ним CONN_MAX_AGE + # обязан быть 0, а без пула persistent-режим возвращать нельзя + # (см. _pool_options). + "CONN_MAX_AGE": 0, + "OPTIONS": {"pool": dict(pool)} if pool else {}, + } + + databases = { + "default": config("migration" if testing else role), + "platform": config("platform"), + } + if testing: + # Тесты создают свою БД и подключаются владельцем кластера. Его пароль + # приходит оттуда же, откуда у остальных ролей: файл секрета инстанса, + # переменная окружения — переопределение. + databases["default"]["USER"] = os.environ.get( + "POSTGRES_USER", "chatballs_bootstrap" + ) + databases["default"]["PASSWORD"] = env_secret( + "POSTGRES_PASSWORD", "postgres_password", "chatballs" + ) + databases["platform"]["TEST"] = {"MIRROR": "default"} + return databases diff --git a/apps/backend/chatballs_backend/urls_platform.py b/apps/backend/chatballs_backend/urls_platform.py index e2e481a..f841152 100644 --- a/apps/backend/chatballs_backend/urls_platform.py +++ b/apps/backend/chatballs_backend/urls_platform.py @@ -1,4 +1,5 @@ from django.urls import include, path + from chatballs.support_portals.gateway_views import HelpDomainAuthorizationView urlpatterns = [ diff --git a/apps/internal-ui/src/features/integrations/IntegrationForm.tsx b/apps/internal-ui/src/features/integrations/IntegrationForm.tsx index 208183d..f1cb72d 100644 --- a/apps/internal-ui/src/features/integrations/IntegrationForm.tsx +++ b/apps/internal-ui/src/features/integrations/IntegrationForm.tsx @@ -152,7 +152,10 @@ export function IntegrationForm({ initial, kind, onClose, onSaved }: { initial: )} {!isWeb && !isEmail && !isDemo && ( - + <> + +
Пароль прокси наружу не отдаётся: вместо него точки. Оставьте их как есть — прежний пароль сохранится; чтобы сменить, впишите новый целиком
+ )} {meta.hasModel && ( diff --git a/compose.dev.yaml b/compose.dev.yaml index 2ea2c55..723ae53 100644 --- a/compose.dev.yaml +++ b/compose.dev.yaml @@ -5,13 +5,28 @@ # Не определяет отдельную production-топологию. services: + # Секреты в dev генерирует тот же сервис на локально собранном backend-образе. + secrets: + build: + context: . + dockerfile: apps/backend/Dockerfile + postgres: + build: + context: . + dockerfile: deploy/docker/postgres.Dockerfile ports: - "${POSTGRES_HOST_PORT:-5432}:5432" + # Dev держит состояние в рабочем каталоге: базу видно, её легко снести + # и легко подсмотреть. В коробке это именованные тома (compose.yaml). + volumes: + - ./data/postgres:/var/lib/postgresql/data redis: ports: - "${REDIS_HOST_PORT:-6379}:6379" + volumes: + - ./data/redis:/data # Dev: migrate через init, без collectstatic (испечён только в prod-образе). init: @@ -47,6 +62,7 @@ services: CHATBALLS_HELP_PUBLIC_PORT: "" volumes: - ./apps/backend:/app/apps/backend + - ./data/media:/app/apps/backend/media depends_on: init: condition: service_completed_successfully @@ -98,6 +114,7 @@ services: CHATBALLS_DELIVERY_MODE: ${CHATBALLS_DELIVERY_MODE:-CLOUD} volumes: - ./apps/backend:/app/apps/backend + - ./data/media:/app/apps/backend/media # Dev: frontend-сервис превращается в Vite dev-сервер internal-ui (HMR). frontend: @@ -105,6 +122,10 @@ services: context: . dockerfile: apps/internal-ui/Dockerfile command: npm run dev -- --host 0.0.0.0 --port 5173 + # В коробке frontend — nginx на 80, и его healthcheck нужен `up --wait`. + # Здесь это Vite на 5173: тот же тест был бы всегда красным. + healthcheck: + disable: true environment: # Direct Vite remains same-origin for cookies/CSRF and proxies only to the # real local Django service. This variable exists only in dev compose. diff --git a/compose.yaml b/compose.yaml index 275e594..58d1bde 100644 --- a/compose.yaml +++ b/compose.yaml @@ -3,32 +3,41 @@ # Локальная разработка — через override compose.dev.yaml; этот файл не содержит # dev source mounts, dev-команд и host-портов (кроме public 80/443 у gateway). # -# Image references берутся из release.env (immutable digest refs, см. ADR §release). -# В dev они не задаются — срабатывают безопасные defaults, а override подменяет -# image на build. +# Этот файл — весь дистрибутив. Рядом с ним не должно лежать ничего: ни +# Caddyfile, ни init-скриптов базы, ни генератора секретов — всё это внутри +# образов (deploy/docker/*.Dockerfile). Поэтому установка на чистый хост +# выглядит так: +# +# curl -fsSL <ссылка на compose.yaml со страницы релиза> -o compose.yaml +# docker compose up -d --wait +# +# В релизной копии ссылки на образы уже закреплены по digest (их подставляет +# CI). Значения по умолчанию ниже — для сборки из исходников: с +# compose.dev.yaml они собираются локально. # # У продукта нет .env: секреты инстанса генерирует сервис secrets при первом # старте, всё остальное человек настраивает в UI. -# Запуск production/box: -# docker compose --env-file release.env up -d -# Запуск разработки: -# docker compose -f compose.yaml -f compose.dev.yaml up services: # Секреты инстанса (ключ подписи, пароли ролей БД) генерирует первый старт; # они живут в томе chatballs-secrets. Человек их не вводит и не хранит: # установка — одна команда, всё остальное настраивается в UI. Скрипт # идемпотентен — перезапуск не меняет пароли работающей базы. + # + # Крутится на backend-образе: он всё равно нужен стеку, а скрипт запечён в + # него (apps/backend/Dockerfile.production). secrets: - image: ${CHATBALLS_POSTGRES_IMAGE:-pgvector/pgvector:pg16} - entrypoint: ["/bin/sh", "/chatballs-generate-secrets.sh"] + image: ${CHATBALLS_BACKEND_IMAGE:-chatballs-backend:dev} + entrypoint: ["/bin/sh", "/usr/local/bin/chatballs-generate-secrets.sh"] + # Скрипт создаёт каталог и выставляет права — это делает root, а + # production-образ работает под hub. + user: root restart: "no" volumes: - - ${CHATBALLS_RELEASE_DIR:-.}/deploy/secrets/generate-instance-secrets.sh:/chatballs-generate-secrets.sh:ro - chatballs-secrets:/run/chatballs/secrets postgres: - image: ${CHATBALLS_POSTGRES_IMAGE:-pgvector/pgvector:pg16} + image: ${CHATBALLS_POSTGRES_IMAGE:-chatballs-postgres:dev} restart: unless-stopped environment: POSTGRES_DB: ${POSTGRES_DB:-chatballs} @@ -45,9 +54,7 @@ services: condition: service_completed_successfully volumes: - chatballs-secrets:/run/chatballs/secrets:ro - - ${CHATBALLS_INSTANCE_DIR:-.}/data/postgres:/var/lib/postgresql/data - - ${CHATBALLS_RELEASE_DIR:-.}/deploy/postgres/init-runtime-roles.sh:/docker-entrypoint-initdb.d/20-chatballs-runtime-roles.sh:ro - - ${CHATBALLS_RELEASE_DIR:-.}/deploy/postgres/reassign-schema-ownership.sql:/chatballs-reassign-ownership.sql:ro + - chatballs-postgres:/var/lib/postgresql/data healthcheck: test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] interval: 10s @@ -59,7 +66,7 @@ services: restart: unless-stopped command: ["redis-server", "--appendonly", "yes"] volumes: - - ${CHATBALLS_INSTANCE_DIR:-.}/data/redis:/data + - chatballs-redis:/data healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 10s @@ -97,9 +104,10 @@ services: CHATBALLS_DB_ROLE: app volumes: - chatballs-secrets:/run/chatballs/secrets:ro - # Legacy source media retained for the separately approved copy/hash - # migration. Production writes use the required S3 backend in C04. - - ${CHATBALLS_INSTANCE_DIR:-.}/data/media:/app/apps/backend/media + # Локальные файлы — именованный том, а не каталог с хоста. Том наследует + # владельца из образа (hub), поэтому загрузки работают и на Linux, где + # bind-mount достался бы контейнеру как root:root и падал с EACCES. + - chatballs-media:/app/apps/backend/media depends_on: init: condition: service_completed_successfully @@ -188,7 +196,7 @@ services: CHATBALLS_DB_ROLE: app volumes: - chatballs-secrets:/run/chatballs/secrets:ro - - ${CHATBALLS_INSTANCE_DIR:-.}/data/media:/app/apps/backend/media + - chatballs-media:/app/apps/backend/media depends_on: backend-app: condition: service_healthy @@ -203,25 +211,32 @@ services: depends_on: backend-app: condition: service_healthy + healthcheck: + # Без него `up --wait` считает контейнер готовым сразу после старта, и + # человек открывает адрес раньше, чем nginx поднял конфигурацию. + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1/"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 5s # Gateway: единственный HTTP/HTTPS public boundary (ADR-CHATBALLS-0028 §gateway). # Caddy: TLS termination, ACME, HTTP->HTTPS redirect, WebSocket upgrade (native). - # Маршрутизация публичных путей делегируется frontend (internal). Coturn не - # проксируется через Caddy — отдельная network boundary (profile calls). + # Caddyfile запечён в образ (deploy/docker/gateway.Dockerfile) — с хоста + # ничего не монтируется. Маршрутизация публичных путей делегируется frontend + # (internal). Coturn не проксируется через Caddy — отдельная network boundary + # (profile calls). gateway: - image: ${CHATBALLS_GATEWAY_IMAGE:-caddy:2.8.4} + image: ${CHATBALLS_GATEWAY_IMAGE:-chatballs-gateway:dev} restart: unless-stopped environment: - CHATBALLS_APP_DOMAIN: ${CHATBALLS_APP_DOMAIN:-localhost} CHATBALLS_PLATFORM_DOMAIN: ${CHATBALLS_PLATFORM_DOMAIN:-platform.localhost} - CHATBALLS_ACME_EMAIL: ${CHATBALLS_ACME_EMAIL:-} ports: - "${CHATBALLS_WEB_LISTENING_IP:-0.0.0.0}:80:80" - "${CHATBALLS_WEB_LISTENING_IP:-0.0.0.0}:443:443" volumes: - - ${CHATBALLS_RELEASE_DIR:-.}/Caddyfile:/etc/caddy/Caddyfile:ro - - ${CHATBALLS_INSTANCE_DIR:-.}/data/caddy:/data - - ${CHATBALLS_INSTANCE_DIR:-.}/data/caddy-config:/config + - chatballs-caddy-data:/data + - chatballs-caddy-config:/config depends_on: frontend: condition: service_started @@ -276,10 +291,11 @@ services: - --no-tlsv1 - --no-tlsv1_1 volumes: - # LE-сертификат TURN-хоста (turns:), скопированный под uid coturn (nobody) - # renewal deploy-hook'ом. См. docs по развёртыванию TURN-over-TLS. + # LE-сертификат TURN-хоста (turns:) кладёт на хост renewal deploy-hook — + # это единственное место, где стек смотрит наружу файлом, и только при + # включённом профиле calls. Каталог задаётся явно. - chatballs-secrets:/run/chatballs/secrets:ro - - ${CHATBALLS_INSTANCE_DIR:-.}/data/coturn-certs:/etc/coturn/certs:ro + - ${CHATBALLS_TURN_CERTS_DIR:-./data/coturn-certs}:/etc/coturn/certs:ro healthcheck: # Allocation smoke: STUN binding к собственному listener на выделенном IP. test: ["CMD", "turnutils_stunclient", "-p", "${CHATBALLS_TURN_LISTENING_PORT:-3478}", "${CHATBALLS_TURN_LISTENING_IP:-0.0.0.0}"] @@ -291,3 +307,11 @@ services: volumes: # Секреты инстанса: генерируются при первом старте, живут только здесь. chatballs-secrets: + # Состояние установки. Именованные тома вместо каталогов с хоста: установка + # не зависит от того, из какого каталога её запустили, и не упирается в + # владельца каталога на хосте. + chatballs-postgres: + chatballs-redis: + chatballs-media: + chatballs-caddy-data: + chatballs-caddy-config: diff --git a/deploy/docker/gateway.Dockerfile b/deploy/docker/gateway.Dockerfile new file mode 100644 index 0000000..bbed80d --- /dev/null +++ b/deploy/docker/gateway.Dockerfile @@ -0,0 +1,9 @@ +# Шлюз Chatballs: Caddy со своим Caddyfile внутри образа. +# +# Конфигурация шлюза — часть релиза, а не файл, который человек кладёт рядом: +# установка сводится к одному compose.yaml и не требует ничего распаковывать. +ARG CHATBALLS_GATEWAY_BASE_IMAGE=caddy:2.8.4 +FROM ${CHATBALLS_GATEWAY_BASE_IMAGE} + +COPY Caddyfile /etc/caddy/Caddyfile +RUN caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile diff --git a/deploy/docker/postgres.Dockerfile b/deploy/docker/postgres.Dockerfile new file mode 100644 index 0000000..35a7179 --- /dev/null +++ b/deploy/docker/postgres.Dockerfile @@ -0,0 +1,18 @@ +# PostgreSQL Chatballs: тот же pgvector, но со своими init-скриптами внутри. +# +# Раньше compose монтировал эти файлы с хоста. Из-за этого установка требовала +# рядом распакованный репозиторий, а забытый в релизе файл превращался в +# молча созданный Docker'ом пустой каталог — и стек падал на первом старте. +# Теперь всё, что нужно базе, лежит в образе. +ARG CHATBALLS_POSTGRES_BASE_IMAGE=pgvector/pgvector:pg16 +FROM ${CHATBALLS_POSTGRES_BASE_IMAGE} + +# Роли app/platform/migration и расширение vector — при инициализации кластера. +COPY deploy/postgres/init-runtime-roles.sh /docker-entrypoint-initdb.d/20-chatballs-runtime-roles.sh +# Нормализация владельца public-схемы: вызывается вручную из `chatballs deploy`. +COPY deploy/postgres/reassign-schema-ownership.sql /chatballs-reassign-ownership.sql + +# Бит исполнения не переживает checkout на Windows, а без него entrypoint +# источает скрипт вместо запуска — и `exit 1` внутри убивает инициализацию. +RUN chmod 0755 /docker-entrypoint-initdb.d/20-chatballs-runtime-roles.sh \ + && chmod 0644 /chatballs-reassign-ownership.sql diff --git a/deploy/nginx/frontend.production.conf b/deploy/nginx/frontend.production.conf index 6dbc5a7..5a37758 100644 --- a/deploy/nginx/frontend.production.conf +++ b/deploy/nginx/frontend.production.conf @@ -23,6 +23,14 @@ server { proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; } + # Готовность стека — внутренний сигнал: её спрашивают healthcheck'и и + # smoke изнутри сети. Снаружи она сообщала бы состояние базы и Redis + # любому желающему. Liveness (/health/live/) остаётся открытым: по нему + # балансировщик отличает живой контейнер от мёртвого. + location = /api/v1/health/ready/ { + return 404; + } + location /api/ { proxy_pass http://backend-app:8000/api/; proxy_http_version 1.1; diff --git a/pyproject.toml b/pyproject.toml index 57a4579..0144311 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,10 +2,19 @@ target-version = "py312" line-length = 100 src = ["apps/backend"] +# Миграции пишет Django, а не человек: переносить в них строки и переставлять +# импорты бессмысленно — файл всё равно перегенерируется как был. +extend-exclude = ["**/migrations/*"] [tool.ruff.lint] select = ["E", "F", "I", "UP", "B", "DJ"] -ignore = ["DJ001"] +# DJ001 — null=True на текстовых полях: в схеме это осознанно. +# E501 — длину строки правит форматтер, а не линтер. `ruff format` в этом +# репозитории не принят (он переписал бы 374 файла), и пока это отдельное +# решение, гейтить длину строки в CI нечестно: правило было бы красным всегда +# и его просто перестали бы читать. Всё остальное из E, F, I, UP, B и DJ +# проверяется и обязано быть зелёным. +ignore = ["DJ001", "E501"] # pytest config lives in apps/backend/pytest.ini so it is also picked up inside # the backend container (workdir /app/apps/backend, which does not mount this file). diff --git a/scripts/check.ps1 b/scripts/check.ps1 index 22ac329..04a46d1 100644 --- a/scripts/check.ps1 +++ b/scripts/check.ps1 @@ -33,6 +33,10 @@ function Invoke-Step { } # Сначала быстрые проверки, затем длинные: backend-сьют идёт около 20 минут. +# Линтер запускается из корня репозитория: конфигурация лежит в pyproject.toml, +# а в контейнер смонтирован только apps/backend — без корня ruff взял бы +# правила по умолчанию вместо проектных и молча пропускал бы половину. +Invoke-Step "backend · ruff" { docker @compose run --rm --no-deps -v "${PWD}:/repo" -w /repo backend-app ruff check apps/backend } Invoke-Step "internal-ui · typecheck" { docker @compose run --rm --no-deps frontend npm run typecheck } Invoke-Step "web-chat · typecheck" { docker @compose run --rm --no-deps web-chat npm run typecheck } Invoke-Step "internal-ui · vitest" { docker @compose run --rm --no-deps frontend npm run test } diff --git a/scripts/pin-release-compose.py b/scripts/pin-release-compose.py new file mode 100644 index 0000000..fd6a305 --- /dev/null +++ b/scripts/pin-release-compose.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Готовит релизную копию compose.yaml: ссылки на образы закреплены по digest. + +Установка на чистый хост — это один файл: человек скачивает compose.yaml со +страницы релиза и делает `docker compose up -d --wait`. Значит в этом файле не +должно остаться ни одной подстановки, которая молча возьмёт `:dev` или плавающий +тег, если переменной в окружении нет. + +Скрипт переписывает ровно значения по умолчанию внутри ``${VAR:-...}`` для +известных ключей образов и падает, если хоть один ключ не найден или пришёл без +digest. Переопределение переменной окружения остаётся возможным — это нужно +staging и облаку, которые ведут конфигурацию сами. + +Использование: + pin-release-compose.py --source compose.yaml --output dist/compose.yaml \\ + --version 1.0.0 --pin CHATBALLS_BACKEND_IMAGE=ghcr.io/...@sha256:... ... +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +IMAGE_KEYS = ( + "CHATBALLS_BACKEND_IMAGE", + "CHATBALLS_FRONTEND_IMAGE", + "CHATBALLS_POSTGRES_IMAGE", + "CHATBALLS_REDIS_IMAGE", + "CHATBALLS_GATEWAY_IMAGE", + "CHATBALLS_COTURN_IMAGE", +) + +DIGEST_RE = re.compile(r"@sha256:[0-9a-fA-F]{64}$") + +HEADER = """# Chatballs {version} — релизная копия compose.yaml. +# +# Установка на чистый хост с одним докером: +# +# docker compose up -d --wait +# +# Больше рядом ничего не нужно: Caddyfile, init-скрипты базы и генератор +# секретов лежат внутри образов. Файл сгенерирован автоматически из +# compose.yaml релиза {version}; править его руками не нужно — обновление +# сводится к тому, чтобы скачать этот файл новой версии и повторить команду. +# +""" + + +def pin(source: str, pins: dict[str, str], version: str) -> str: + text = source + for key, ref in pins.items(): + pattern = re.compile(r"\$\{" + re.escape(key) + r":-[^}]*\}") + replaced, count = pattern.subn(ref, text) + if count == 0: + raise SystemExit(f"{key}: подстановка ${{{key}:-...}} не найдена в compose.yaml") + text = replaced + leftover = [key for key in IMAGE_KEYS if f"${{{key}" in text] + if leftover: + raise SystemExit("не закреплены ссылки на образы: " + ", ".join(leftover)) + return HEADER.format(version=version) + text + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--source", default="compose.yaml") + parser.add_argument("--output", required=True) + parser.add_argument("--version", required=True) + parser.add_argument( + "--pin", + action="append", + default=[], + metavar="KEY=REF", + help="ссылка на образ по digest, по одной на каждый ключ", + ) + args = parser.parse_args(argv) + + pins: dict[str, str] = {} + for item in args.pin: + key, _, ref = item.partition("=") + if key not in IMAGE_KEYS: + raise SystemExit(f"неизвестный ключ образа: {key}") + if not DIGEST_RE.search(ref): + raise SystemExit(f"{key}: ссылка обязана быть закреплена по @sha256") + pins[key] = ref + + missing = [key for key in IMAGE_KEYS if key not in pins] + if missing: + raise SystemExit("не переданы ссылки на образы: " + ", ".join(missing)) + + source = Path(args.source).read_text(encoding="utf-8") + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(pin(source, pins, args.version), encoding="utf-8", newline="\n") + print(f"{output}: закреплено ссылок — {len(pins)}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/start.ps1 b/scripts/start.ps1 index af0c35f..5992881 100644 --- a/scripts/start.ps1 +++ b/scripts/start.ps1 @@ -1,21 +1,23 @@ -# Локальный запуск Chatballs (Windows). +# Локальный запуск Chatballs из исходников (Windows). +# +# Это путь разработчика: стек собирается из репозитория. Установка продукта +# выглядит иначе и этого скрипта не требует — там один compose.yaml со +# страницы релиза и `docker compose up -d --wait` (см. README). # # Ни одной переменной задавать не нужно и негде: .env у продукта нет. Секреты -# инстанса генерирует первый старт (сервис secrets), всё остальное — организацию, -# владельца, домены, почту, интеграции — человек настраивает в UI. -# -# Если рядом лежит release.env (скачан со страницы релиза), образы берутся из -# реестра по digest — запуск занимает минуты вместо сборки. Без него стек -# собирается из исходников: так работают те, кто правит код. +# инстанса генерирует первый старт (сервис secrets), всё остальное — +# организацию, владельца, домены, почту, интеграции — человек настраивает в UI. +param( + [ValidateSet("Cloud", "SelfHosted")] + [string] $Mode = "Cloud" +) + $ErrorActionPreference = "Stop" Set-Location (Join-Path $PSScriptRoot "..") -if (Test-Path "release.env") { - Write-Output "release.env найден: образы берутся из реестра, сборки не будет." - docker compose --env-file release.env -f compose.yaml up -} -else { - Write-Output "release.env нет: собираем из исходников (для готовых образов скачайте release.env со страницы релиза)." - docker compose -f compose.yaml -f compose.dev.yaml up --build -} +$delivery = if ($Mode -eq "SelfHosted") { "SELF_HOSTED" } else { "CLOUD" } +Write-Output "Сборка из исходников, режим поставки: $delivery." + +$env:CHATBALLS_DELIVERY_MODE = $delivery +docker compose -f compose.yaml -f compose.dev.yaml up --build diff --git a/scripts/start.sh b/scripts/start.sh index 9408e00..f75613a 100644 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -1,21 +1,39 @@ #!/usr/bin/env sh -# Локальный запуск Chatballs (Linux/macOS). +# Локальный запуск Chatballs из исходников (Linux/macOS). +# +# Это путь разработчика: стек собирается из репозитория. Установка продукта +# выглядит иначе и этого скрипта не требует — там один compose.yaml со +# страницы релиза и `docker compose up -d --wait` (см. README). # # Ни одной переменной задавать не нужно и негде: .env у продукта нет. Секреты -# инстанса генерирует первый старт (сервис secrets), всё остальное — организацию, -# владельца, домены, почту, интеграции — человек настраивает в UI. +# инстанса генерирует первый старт (сервис secrets), всё остальное — +# организацию, владельца, домены, почту, интеграции — человек настраивает в UI. # -# Если рядом лежит release.env (скачан со страницы релиза), образы берутся из -# реестра по digest — запуск занимает минуты вместо сборки. Без него стек -# собирается из исходников: так работают те, кто правит код. +# Режим поставки: --mode cloud (по умолчанию) или --mode self-hosted. set -eu cd "$(dirname "$0")/.." -if [ -f release.env ]; then - echo "release.env найден: образы берутся из реестра, сборки не будет." - exec docker compose --env-file release.env -f compose.yaml up -fi +mode="CLOUD" +while [ $# -gt 0 ]; do + case "$1" in + --mode) + shift + case "${1:-}" in + cloud|CLOUD) mode="CLOUD" ;; + self-hosted|SELF_HOSTED|selfhosted) mode="SELF_HOSTED" ;; + *) echo "Неизвестный режим: ${1:-}. Допустимо: cloud, self-hosted" >&2; exit 2 ;; + esac + shift + ;; + -h|--help) + echo "Использование: $0 [--mode cloud|self-hosted]" >&2 + exit 0 + ;; + *) echo "Неизвестный аргумент: $1" >&2; exit 2 ;; + esac +done -echo "release.env нет: собираем из исходников (для готовых образов скачайте release.env со страницы релиза)." -exec docker compose -f compose.yaml -f compose.dev.yaml up --build +echo "Сборка из исходников, режим поставки: $mode." +CHATBALLS_DELIVERY_MODE="$mode" \ + exec docker compose -f compose.yaml -f compose.dev.yaml up --build diff --git a/tests/cli/conftest.py b/tests/cli/conftest.py index 602b6ff..1319075 100644 --- a/tests/cli/conftest.py +++ b/tests/cli/conftest.py @@ -21,7 +21,9 @@ import pytest REPO_RELEASE_ROOT = Path(__file__).resolve().parents[2] # code/chatballs -RELEASE_FILES = ["compose.yaml", "Caddyfile"] +# Релиз — это один compose.yaml. Caddyfile, init-скрипты базы и генератор +# секретов лежат внутри образов, поэтому в бандле их нет. +RELEASE_FILES = ["compose.yaml"] LIB_GLOB_DIR = "deploy/cli/lib" diff --git a/tests/cli/test_chatballs_cli.py b/tests/cli/test_chatballs_cli.py index fb6b2d9..7f63934 100644 --- a/tests/cli/test_chatballs_cli.py +++ b/tests/cli/test_chatballs_cli.py @@ -232,7 +232,7 @@ def test_deploy_fails_when_release_checksum_is_invalid(fake_env): fake_env.install_flock(held=False) - (fake_env.release / "Caddyfile").write_text("tampered\n", encoding="utf-8") + (fake_env.release / "compose.yaml").write_text("tampered\n", encoding="utf-8") r = _run(fake_env, "deploy", "--non-interactive") diff --git a/tests/cli/test_release_compose.py b/tests/cli/test_release_compose.py new file mode 100644 index 0000000..d4d5ad8 --- /dev/null +++ b/tests/cli/test_release_compose.py @@ -0,0 +1,121 @@ +"""compose.yaml обязан быть самодостаточным: установка — один файл. + +Раньше стек монтировал с хоста Caddyfile, init-скрипты базы и генератор +секретов. Из-за этого установка требовала рядом распакованный репозиторий, а +файл, забытый при сборке релиза, Docker молча подменял пустым каталогом — и +стек падал на первом старте, уже у человека. + +Эти тесты держат свойство, а не текущий текст файла: в production-манифесте нет +ни одного bind-mount (кроме сертификатов TURN у опционального профиля calls), а +все ссылки на образы поддаются закреплению по digest. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +COMPOSE = REPO_ROOT / "compose.yaml" +PIN_SCRIPT = REPO_ROOT / "scripts" / "pin-release-compose.py" + +# Единственное исключение: сертификат TURN-хоста кладёт на хост renewal-хук, +# и только при включённом профиле calls. +BIND_MOUNT_EXCEPTIONS = {"coturn"} + +DIGEST = "sha256:" + "a" * 64 +IMAGE_KEYS = ( + "CHATBALLS_BACKEND_IMAGE", + "CHATBALLS_FRONTEND_IMAGE", + "CHATBALLS_POSTGRES_IMAGE", + "CHATBALLS_REDIS_IMAGE", + "CHATBALLS_GATEWAY_IMAGE", + "CHATBALLS_COTURN_IMAGE", +) + + +def _compose() -> dict: + return yaml.safe_load(COMPOSE.read_text(encoding="utf-8")) + + +def _volume_source(entry) -> str: + if isinstance(entry, str): + return entry.split(":", 1)[0] + return str(entry.get("source", "")) + + +def _is_bind(source: str) -> bool: + """Bind-mount — всё, что указывает на путь, а не на именованный том.""" + return source.startswith((".", "/", "~")) or "${" in source + + +@pytest.mark.parametrize("service", sorted(_compose()["services"])) +def test_service_has_no_host_bind_mounts(service: str) -> None: + definition = _compose()["services"][service] + binds = [ + source + for entry in definition.get("volumes", []) + if _is_bind(source := _volume_source(entry)) + ] + if service in BIND_MOUNT_EXCEPTIONS: + pytest.skip(f"{service}: bind-mount разрешён явно") + assert binds == [], ( + f"{service}: манифест монтирует с хоста {binds}. " + "Установка — один compose.yaml: всё, что нужно сервису, кладётся в образ." + ) + + +def test_named_volumes_are_declared() -> None: + compose = _compose() + declared = set(compose.get("volumes") or {}) + used = { + source + for definition in compose["services"].values() + for entry in definition.get("volumes", []) + if not _is_bind(source := _volume_source(entry)) + } + assert used <= declared, f"не объявлены тома: {sorted(used - declared)}" + + +def test_every_image_reference_can_be_pinned(tmp_path: Path) -> None: + """Все шесть ключей образов присутствуют и закрепляются по digest. + + Если из манифеста уйдёт (или переименуется) хоть один ключ, релизный + compose.yaml уедет с плавающим тегом — а найдётся это уже у человека. + """ + output = tmp_path / "compose.yaml" + pins: list[str] = [] + for key in IMAGE_KEYS: + pins += ["--pin", f"{key}=registry.test/{key.lower()}:1.0.0@{DIGEST}"] + + result = subprocess.run( + [sys.executable, str(PIN_SCRIPT), "--source", str(COMPOSE), + "--output", str(output), "--version", "1.0.0-test", *pins], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + + pinned = output.read_text(encoding="utf-8") + for key in IMAGE_KEYS: + assert f"${{{key}" not in pinned, f"{key} остался подстановкой в релизном файле" + + +def test_pinning_rejects_floating_tag(tmp_path: Path) -> None: + pins: list[str] = [] + for key in IMAGE_KEYS: + ref = "registry.test/x:1.0.0" if key == "CHATBALLS_BACKEND_IMAGE" else f"registry.test/x:1.0.0@{DIGEST}" + pins += ["--pin", f"{key}={ref}"] + + result = subprocess.run( + [sys.executable, str(PIN_SCRIPT), "--source", str(COMPOSE), + "--output", str(tmp_path / "compose.yaml"), "--version", "1.0.0-test", *pins], + capture_output=True, + text=True, + ) + assert result.returncode != 0 + assert "@sha256" in result.stdout + result.stderr