feat: self-hosted log drains (#28297)

* feat: initial log drain creation, sans rules creation. * feat: add rules posting * add project settings to self hosted and adapt log drains * feat: log drains crud implementation, env var update * feat: local log drains is working! rules provisioning refined * fix: add filtering * feat: finish implementing CRUD of local log drains. * chore: formatting * only allow navigation to log drains * rm unnecessary checks * rm log * rm logs * rm log * fix type err * turbofix for turboissue --------- Co-authored-by: Jordi Enric <jordi.err@gmail.com>
2026-10-05 09:25:06 +03:00 · 2025-10-30 10:37:32 +00:00
1 parent 02783c3179
commit 5cfd10aae6
@@ -34,16 +34,7 @@ parameters:
    default: ''
    type: 'string'
    description: |
-      Allows you to pass in an API key that will used for authentication for ingestion only. This is intended for programmatic usage where an external program sets the API key. A default ingestion API key will be automatically generated.
-  - id: 'LOGFLARE_PRIVATE_ACCESS_TOKEN' # {string} A unique identifier for this param.
-    title: 'LOGFLARE_PRIVATE_ACCESS_TOKEN' # {string} Any name.
-    tags: ['general'] # {string[]} These tags are useful for grouping parameters
-    links: [] # {string[]} These tags are useful for grouping parameters
-    required: true
-    default: ''
-    type: 'string'
-    description: |
-      Allows you to pass in an Management API key that will used for authentication. This is intended for programmatic usage where an external program sets the API key. This key is considered secret.
+      Allows you to pass in an API key that will used for authentication. This is intended for programmatic usage where an external program sets the API key.  If this value is not provided, the default API key will be automatically generated.
  - id: 'LOGFLARE_SUPABASE_MODE' # {string} A unique identifier for this param.
    title: 'LOGFLARE_SUPABASE_MODE' # {string} Any name.
    tags: ['general'] # {string[]} These tags are useful for grouping parameters
@@ -14,3 +14,24 @@ info:
    - id: general
      title: General Settings
      description: General server settings.
+
+# This section is an array of public functions which a user might need to execute.
+parameters:
+  - id: 'LOGFLARE_SINGLE_TENANT' # {string} A unique identifier for this param.
+    title: 'LOGFLARE_SINGLE_TENANT' # {string} Any name.
+    tags: ['general'] # {string[]} These tags are useful for grouping parameters
+    links: [] # {string[]} These tags are useful for grouping parameters
+    required: true
+    default: 'true'
+    type: 'boolean'
+    description: |
+      This is will seed a singular user into the database, and will disable browser authentication. All browser usage will default to this user. Inviting team users and other team-related functionality is currently not supported for self-hosted. Logflare self-hosted is currently intended for single-user experience only.
+  - id: 'LOGFLARE_PUBLIC_ACCESS_TOKEN' # {string} A unique identifier for this param.
+    title: 'LOGFLARE_PUBLIC_ACCESS_TOKEN' # {string} Any name.
+    tags: ['general'] # {string[]} These tags are useful for grouping parameters
+    links: [] # {string[]} These tags are useful for grouping parameters
+    required: true
+    default: ''
+    type: 'string'
+    description: |
+      Allows you to pass in an API key that will used for authentication. This is intended for programmatic usage where an external program sets the API key. It is advised to use the UI to configure the access tokens instead. If this value is not provided, the default API key will be automatically generated.
@@ -5,7 +5,7 @@ import { useForm } from 'react-hook-form'
 import { toast } from 'sonner'
 import { z } from 'zod'

-import { useFlag, useParams } from 'common'
+import { IS_PLATFORM, useFlag, useParams } from 'common'
 import { DocsButton } from 'components/ui/DocsButton'
 import { LogDrainData, useLogDrainsQuery } from 'data/log-drains/log-drains-query'
 import { DOCS_URL } from 'lib/constants'
@@ -265,7 +265,8 @@ export function LogDrainDestinationSheetForm({

                // Temp check to make sure the name is unique
                const logDrainName = form.getValues('name')
-                const logDrainExists = logDrains?.find((drain) => drain.name === logDrainName)
+                const logDrainExists =
+                  !!logDrains?.length && logDrains?.find((drain) => drain.name === logDrainName)
                if (logDrainExists && mode === 'create') {
                  toast.error('Log drain name already exists')
                  return
@@ -569,7 +570,9 @@ export function LogDrainDestinationSheetForm({
        </SheetSection>

        <div className="mt-auto">
-          <SheetSection className="border-t bg-background-alternative-200 mt-auto">
+          <SheetSection
+            className={`border-t bg-background-alternative-200 mt-auto ${!IS_PLATFORM ? 'hidden' : ''}`}
+          >
            <FormItemLayout
              isReactForm={false}
              layout="horizontal"
@@ -56,6 +56,7 @@ export function LogDrains({
    }
  )
  const sentryEnabled = useFlag('SentryLogDrain')
+  const hasLogDrains = !!logDrains?.length

  const { mutate: deleteLogDrain } = useDeleteLogDrainMutation({
    onSuccess: () => {
@@ -90,7 +91,7 @@ export function LogDrains({
    )
  }

-  if (!isLoading && logDrains?.length === 0) {
+  if (!isLoading && !hasLogDrains) {
    return (
      <div className="grid lg:grid-cols-2 gap-3">
        {LOG_DRAIN_TYPES.filter((t) => t.value !== 'sentry' || sentryEnabled).map((src) => (
@@ -195,16 +195,12 @@ export const generateOtherRoutes = (
 export const generateSettingsRoutes = (ref?: string, project?: Project): Route[] => {
  const settingsMenu = generateSettingsMenu(ref as string)
  return [
-    ...(IS_PLATFORM
-      ? [
-          {
-            key: 'settings',
-            label: 'Project Settings',
-            icon: <Settings size={ICON_SIZE} strokeWidth={ICON_STROKE_WIDTH} />,
-            link: ref && `/project/${ref}/settings/general`,
-            items: settingsMenu,
-          },
-        ]
-      : []),
+    {
+      key: 'settings',
+      label: 'Project Settings',
+      icon: <Settings size={ICON_SIZE} strokeWidth={ICON_STROKE_WIDTH} />,
+      link: ref && `/project/${ref}/settings/general`,
+      items: settingsMenu,
+    },
  ]
 }
@@ -21,12 +21,6 @@ const SettingsLayout = ({ title, children }: PropsWithChildren<SettingsLayoutPro
  const { data: project } = useSelectedProjectQuery()
  const { data: organization } = useSelectedOrganizationQuery()

-  useEffect(() => {
-    if (!IS_PLATFORM) {
-      router.push('/project/default')
-    }
-  }, [router])
-
  // billing pages live under /billing/invoices and /billing/subscription, etc
  // so we need to pass the [5]th part of the url to the menu
  const page = router.pathname.includes('billing')
@@ -20,6 +20,21 @@ export const generateSettingsMenu = (
    billing?: boolean
  }
 ): ProductMenuGroup[] => {
+  if (!IS_PLATFORM) {
+    return [
+      {
+        title: 'Project Settings',
+        items: [
+          {
+            name: `Log Drains`,
+            key: `log-drains`,
+            url: `/project/${ref}/settings/log-drains`,
+            items: [],
+          },
+        ],
+      },
+    ]
+  }
  const isProjectBuilding = project?.status === PROJECT_STATUS.COMING_UP
  const buildingUrl = `/project/${ref}`

@@ -28,7 +43,6 @@ export const generateSettingsMenu = (
  const edgeFunctionsEnabled = features?.edgeFunctions ?? true
  const storageEnabled = features?.storage ?? true
  const legacyJwtKeysEnabled = features?.legacyJwtKeys ?? true
-  const logDrainsEnabled = features?.logDrains ?? true
  const billingEnabled = features?.billing ?? true

  return [
@@ -41,64 +55,55 @@ export const generateSettingsMenu = (
          url: `/project/${ref}/settings/general`,
          items: [],
        },
-        ...(IS_PLATFORM
-          ? [
-              {
-                name: 'Compute and Disk',
-                key: 'compute-and-disk',
-                url: `/project/${ref}/settings/compute-and-disk`,
-                items: [],
-              },
-            ]
-          : []),
+        {
+          name: 'Compute and Disk',
+          key: 'compute-and-disk',
+          url: `/project/${ref}/settings/compute-and-disk`,
+          items: [],
+        },
        {
          name: 'Infrastructure',
          key: 'infrastructure',
          url: isProjectBuilding ? buildingUrl : `/project/${ref}/settings/infrastructure`,
          items: [],
        },
-        ...(IS_PLATFORM
-          ? [
-              {
-                name: 'Integrations',
-                key: 'integrations',
-                url: `/project/${ref}/settings/integrations`,
-                items: [],
-              },
-              ...(logDrainsEnabled
-                ? [
-                    {
-                      name: `Log Drains`,
-                      key: `log-drains`,
-                      url: `/project/${ref}/settings/log-drains`,
-                      items: [],
-                    },
-                  ]
-                : []),
-              {
-                name: 'Data API',
-                key: 'api',
-                url: isProjectBuilding ? buildingUrl : `/project/${ref}/settings/api`,
-                items: [],
-              },
-              {
-                name: 'API Keys',
-                key: 'api-keys',
-                url: `/project/${ref}/settings/api-keys`,
-                items: [],
-                label: 'NEW',
-              },
-              {
-                name: 'JWT Keys',
-                key: 'jwt',
-                url: legacyJwtKeysEnabled
-                  ? `/project/${ref}/settings/jwt`
-                  : `/project/${ref}/settings/jwt/signing-keys`,
-                items: [],
-                label: 'NEW',
-              },
-            ]
-          : []),
+
+        {
+          name: 'Integrations',
+          key: 'integrations',
+          url: `/project/${ref}/settings/integrations`,
+          items: [],
+        },
+
+        {
+          name: 'Data API',
+          key: 'api',
+          url: isProjectBuilding ? buildingUrl : `/project/${ref}/settings/api`,
+          items: [],
+        },
+        {
+          name: 'API Keys',
+          key: 'api-keys',
+          url: `/project/${ref}/settings/api-keys`,
+          items: [],
+          label: 'NEW',
+        },
+        {
+          name: 'JWT Keys',
+          key: 'jwt',
+          url: legacyJwtKeysEnabled
+            ? `/project/${ref}/settings/jwt`
+            : `/project/${ref}/settings/jwt/signing-keys`,
+          items: [],
+          label: 'NEW',
+        },
+
+        {
+          name: `Log Drains`,
+          key: `log-drains`,
+          url: `/project/${ref}/settings/log-drains`,
+          items: [],
+        },
        {
          name: 'Add Ons',
          key: 'addons',
@@ -125,7 +130,7 @@ export const generateSettingsMenu = (
          items: [],
          rightIcon: <ArrowUpRight strokeWidth={1} className="h-4 w-4" />,
        },
-        ...(IS_PLATFORM && authEnabled
+        ...(authEnabled
          ? [
              {
                name: 'Authentication',
@@ -138,7 +143,7 @@ export const generateSettingsMenu = (
              },
            ]
          : []),
-        ...(IS_PLATFORM && storageEnabled
+        ...(storageEnabled
          ? [
              {
                name: 'Storage',
@@ -149,7 +154,7 @@ export const generateSettingsMenu = (
              },
            ]
          : []),
-        ...(IS_PLATFORM && edgeFunctionsEnabled
+        ...(edgeFunctionsEnabled
          ? [
              {
                name: 'Edge Functions',
@@ -0,0 +1,125 @@
+import { NextApiRequest, NextApiResponse } from 'next'
+import apiWrapper from 'lib/api/apiWrapper'
+import { PROJECT_ANALYTICS_URL } from 'lib/constants/api'
+
+export default (req: NextApiRequest, res: NextApiResponse) => apiWrapper(req, res, handler)
+
+async function handler(req: NextApiRequest, res: NextApiResponse) {
+  const { method } = req
+  const { ref } = req.query
+
+  const missingEnvVars = envVarsSet()
+
+  if (missingEnvVars !== true) {
+    return res
+      .status(500)
+      .json({ error: { message: `${missingEnvVars.join(', ')} env variables are not set` } })
+  }
+
+  const baseUrl = PROJECT_ANALYTICS_URL
+  if (!baseUrl) {
+    return res.status(500).json({ error: { message: `LOGFLARE_URL env variable is not set` } })
+  }
+
+  switch (method) {
+    case 'GET':
+      // list log drains
+      const url = new URL(baseUrl)
+      url.pathname = '/api/backends'
+      url.search = new URLSearchParams({
+        'metadata[type]': 'log-drain',
+      }).toString()
+      const resp = await fetch(url, {
+        method: 'GET',
+        headers: {
+          Authorization: `Bearer ${process.env.LOGFLARE_PRIVATE_ACCESS_TOKEN}`,
+          'Content-Type': 'application/json',
+          Accept: 'application/json',
+        },
+      }).then((res) => {
+        return res.json()
+      })
+
+      return res.status(200).json(resp)
+    case 'POST':
+      // create the log drain
+      const postUrl = new URL(baseUrl)
+      postUrl.pathname = '/api/backends'
+      const postResult = await fetch(postUrl, {
+        body: JSON.stringify({
+          ...req.body,
+          config: req.body.config,
+          metadata: {
+            type: 'log-drain',
+          },
+        }),
+        method: 'POST',
+        headers: {
+          Authorization: `Bearer ${process.env.LOGFLARE_PRIVATE_ACCESS_TOKEN}`,
+          'Content-Type': 'application/json',
+          Accept: 'application/json',
+        },
+      }).then(async (r) => await r.json())
+
+      const sourcesGetUrl = new URL(baseUrl)
+      sourcesGetUrl.pathname = '/api/sources'
+      const sources = await fetch(sourcesGetUrl, {
+        method: 'GET',
+        headers: {
+          Authorization: `Bearer ${process.env.LOGFLARE_PRIVATE_ACCESS_TOKEN}`,
+          'Content-Type': 'application/json',
+          Accept: 'application/json',
+        },
+      }).then((r) => r.json())
+
+      const params = sources
+        .filter((source: { name: string; metadata: { type: string } }) =>
+          [
+            'cloudflare.logs.prod',
+            'deno-relay-logs',
+            'deno-subhosting-events',
+            'gotrue.logs.prod',
+            'pgbouncer.logs.prod',
+            'postgrest.logs.prod',
+            'postgres.logs',
+            'realtime.logs.prod',
+            'storage.logs.prod.2',
+          ].includes(source.name.toLocaleLowerCase())
+        )
+        .map((source: { name: string; id: number }) => {
+          return { backend_id: postResult.id, lql_string: `~".*?"`, source_id: source.id }
+        })
+      const rulesPostUrl = new URL(baseUrl)
+      rulesPostUrl.pathname = '/api/rules'
+      await Promise.all(
+        params.map((param: any) =>
+          fetch(rulesPostUrl, {
+            method: 'POST',
+            body: JSON.stringify(param),
+            headers: {
+              Authorization: `Bearer ${process.env.LOGFLARE_PRIVATE_ACCESS_TOKEN}`,
+              'Content-Type': 'application/json',
+              Accept: 'application/json',
+            },
+          })
+        )
+      )
+      return res.status(201).json(postResult)
+
+    default:
+      res.setHeader('Allow', ['GET', 'POST', 'PUT', 'DELETE'])
+      res.status(405).json({ data: null, error: { message: `Method ${method} Not Allowed` } })
+  }
+}
+
+const envVarsSet = () => {
+  const missingEnvVars = [
+    process.env.LOGFLARE_PRIVATE_ACCESS_TOKEN ? null : 'LOGFLARE_PRIVATE_ACCESS_TOKEN',
+    process.env.LOGFLARE_URL ? null : 'LOGFLARE_URL',
+  ].filter((v) => v)
+  if (missingEnvVars.length == 0) {
+    return true
+  } else {
+    return missingEnvVars
+  }
+}
@@ -0,0 +1,93 @@
+import { NextApiRequest, NextApiResponse } from 'next'
+import apiWrapper from 'lib/api/apiWrapper'
+import { PROJECT_ANALYTICS_URL } from 'lib/constants/api'
+
+export default (req: NextApiRequest, res: NextApiResponse) => apiWrapper(req, res, handler)
+
+async function handler(req: NextApiRequest, res: NextApiResponse) {
+  const { method } = req
+  const { uuid } = req.query
+
+  const missingEnvVars = envVarsSet()
+
+  if (missingEnvVars !== true) {
+    return res
+      .status(500)
+      .json({ error: { message: `${missingEnvVars.join(', ')} env variables are not set` } })
+  }
+
+  const baseUrl = PROJECT_ANALYTICS_URL
+  if (!baseUrl) {
+    return res.status(500).json({ error: { message: `LOGFLARE_URL env variable is not set` } })
+  }
+
+  switch (method) {
+    case 'GET':
+      // get log drain
+      const url = new URL(baseUrl)
+      url.pathname = `/api/backends/${uuid}`
+      const result = await fetch(url, {
+        method: 'GET',
+        headers: {
+          Authorization: `Bearer ${process.env.LOGFLARE_PRIVATE_ACCESS_TOKEN}`,
+          'Content-Type': 'application/json',
+          Accept: 'application/json',
+        },
+      }).then((r) => r.json())
+
+      return res.status(200).json(result)
+    case 'PUT':
+      // create the log drain
+      const putUrl = new URL(baseUrl)
+      putUrl.pathname = `/api/backends/${uuid}`
+      delete req.body['metadata']
+      const putResult = await fetch(putUrl, {
+        body: JSON.stringify(req.body),
+        method: 'PUT',
+        headers: {
+          Authorization: `Bearer ${process.env.LOGFLARE_PRIVATE_ACCESS_TOKEN}`,
+          'Content-Type': 'application/json',
+          Accept: 'application/json',
+        },
+      })
+        .then(async (r) => await r.json())
+        .catch((err) => {
+          console.error('error updating log drain', err)
+          return res.status(500).json({ error: { message: 'Error updating log drain' } })
+        })
+      return res.status(200).json(putResult)
+
+    case 'DELETE':
+      // create the log drain
+      const deleteUrl = new URL(baseUrl)
+      deleteUrl.pathname = `/api/backends/${uuid}`
+
+      await fetch(deleteUrl, {
+        headers: {
+          Authorization: `Bearer ${process.env.LOGFLARE_PRIVATE_ACCESS_TOKEN}`,
+          'Content-Type': 'application/json',
+          Accept: 'application/json',
+        },
+        method: 'DELETE',
+      }).catch((err) => {
+        console.error('error deleting log drain', err)
+        return res.status(500).json({ error: { message: 'Error deleting log drain' } })
+      })
+      return res.status(204).json({ error: null })
+    default:
+      res.setHeader('Allow', ['GET', 'POST'])
+      res.status(405).json({ data: null, error: { message: `Method ${method} Not Allowed` } })
+  }
+}
+
+const envVarsSet = () => {
+  const missingEnvVars = [
+    process.env.LOGFLARE_PRIVATE_ACCESS_TOKEN ? null : 'LOGFLARE_PRIVATE_ACCESS_TOKEN',
+    process.env.LOGFLARE_URL ? null : 'LOGFLARE_URL',
+  ].filter((v) => v)
+  if (missingEnvVars.length == 0) {
+    return true
+  } else {
+    return missingEnvVars
+  }
+}
@@ -12,6 +12,9 @@ import { useIsFeatureEnabled } from 'hooks/misc/useIsFeatureEnabled'
 import { useSelectedOrganizationQuery } from 'hooks/misc/useSelectedOrganization'
 import { useSelectedProjectQuery } from 'hooks/misc/useSelectedProject'
 import type { NextPageWithLayout } from 'types'
+import { useRouter } from 'next/router'
+import { useEffect } from 'react'
+import { IS_PLATFORM } from 'common'

 const ProjectSettings: NextPageWithLayout = () => {
  const { data: project } = useSelectedProjectQuery()
@@ -20,6 +23,13 @@ const ProjectSettings: NextPageWithLayout = () => {
  const isBranch = !!project?.parent_project_ref
  const { projectsTransfer: projectTransferEnabled, projectSettingsCustomDomains } =
    useIsFeatureEnabled(['projects:transfer', 'project_settings:custom_domains'])
+  const router = useRouter()
+
+  useEffect(() => {
+    if (!IS_PLATFORM) {
+      router.push(`/project/default/settings/log-drains`)
+    }
+  }, [router])

  const { data: subscription } = useOrgSubscriptionQuery({ orgSlug: selectedOrganization?.slug })
  const hasHipaaAddon = subscriptionHasHipaaAddon(subscription)
@@ -45,7 +45,11 @@ services:
      SUPABASE_SERVICE_KEY: ${SERVICE_ROLE_KEY}
      AUTH_JWT_SECRET: ${JWT_SECRET}

+      # LOGFLARE_API_KEY is deprecated
+      LOGFLARE_API_KEY: ${LOGFLARE_PUBLIC_ACCESS_TOKEN}
+      LOGFLARE_PUBLIC_ACCESS_TOKEN: ${LOGFLARE_PUBLIC_ACCESS_TOKEN}
      LOGFLARE_PRIVATE_ACCESS_TOKEN: ${LOGFLARE_PRIVATE_ACCESS_TOKEN}
+
      LOGFLARE_URL: http://analytics:4000
      NEXT_PUBLIC_ENABLE_LOGS: true
      # Comment to use Big Query backend for analytics
@@ -25,6 +25,7 @@ const defaultEnv = {
  SENTRY_IGNORE_API_RESOLUTION_ERROR: '1',
  LOGFLARE_URL: 'http://127.0.0.1:54327',
  LOGFLARE_PRIVATE_ACCESS_TOKEN: 'api-key',
+  LOGFLARE_API_KEY: 'api-key',
  NEXT_PUBLIC_SITE_URL: 'http://localhost:8082',
  NEXT_PUBLIC_GOTRUE_URL: '$SUPABASE_PUBLIC_URL/auth/v1',
  NEXT_PUBLIC_HCAPTCHA_SITE_KEY: '10000000-ffff-ffff-ffff-000000000001',
@@ -72,17 +72,17 @@ enable_confirmations = false

 # Uncomment the following to use gh oAuth app locally with your own test app
 # to use env vars locally, run > `source ./supabase/.env && supabase [command...]`
-[auth.external.github]
-enabled = true
-client_id = "env(GITHUB_CLIENT_ID)"
-secret = "env(GITHUB_SECRET)"
-# Overrides the default auth redirectUrl.
-redirect_uri = "http://localhost:54321/auth/v1/callback"
+# [auth.external.github]
+# enabled = true
+# client_id = "env(GITHUB_CLIENT_ID)"
+# secret = "env(GITHUB_SECRET)"
+# # Overrides the default auth redirectUrl.
+# redirect_uri = "http://localhost:54321/auth/v1/callback"

-[remotes.prod.auth.external.github]
-enabled = true
-client_id = "env(GITHUB_CLIENT_ID)"
-secret = "env(GITHUB_SECRET)"
+# [remotes.prod.auth.external.github]
+# enabled = true
+# client_id = "env(GITHUB_CLIENT_ID)"
+# secret = "env(GITHUB_SECRET)"

 # Use an external OAuth provider. The full list of providers are: `apple`, `azure`, `bitbucket`,
 # `discord`, `facebook`, `figma`, `github`, `gitlab`, `google`, `keycloak`, `linkedin`, `linkedin_oidc`, `notion`,
@@ -100,8 +100,10 @@
        "DEFAULT_ORGANIZATION_NAME",
        "OPENAI_API_KEY",
        "AUTH_JWT_SECRET",
-        "LOGFLARE_URL",
+        "LOGFLARE_API_KEY",
+        "LOGFLARE_PUBLIC_ACCESS_TOKEN",
        "LOGFLARE_PRIVATE_ACCESS_TOKEN",
+        "LOGFLARE_URL",
        "SENTRY_ORG",
        "SENTRY_PROJECT",
        "SENTRY_AUTH_TOKEN",