diff --git a/.github/workflows/docs-lint-v2.yml b/.github/workflows/docs-lint-v2.yml index f967f56e94366..3e3e27370a3ff 100644 --- a/.github/workflows/docs-lint-v2.yml +++ b/.github/workflows/docs-lint-v2.yml @@ -45,6 +45,6 @@ jobs: run: | set -o pipefail git diff --name-only origin/$BASE_REF HEAD \ - | grep -E "^apps/docs/content/guides/(getting-started|ai|api|auth|database|deployment|functions)/" \ + | { grep -E "^apps/docs/content/guides/(getting-started|ai|api|auth|database|deployment|functions)/" || test $? = 1; } \ | xargs -r supa-mdx-lint --format rdf \ | reviewdog -f=rdjsonl -reporter=github-pr-review diff --git a/apps/docs/app/page.tsx b/apps/docs/app/page.tsx index 3ad2489ead11a..66911206162da 100644 --- a/apps/docs/app/page.tsx +++ b/apps/docs/app/page.tsx @@ -79,12 +79,12 @@ const postgresIntegrations = [ href: '/guides/cron', description: 'Schedule and monitor recurring Jobs', }, - // { - // title: 'Queues', - // icon: 'queue', - // href: '/guides/queue', - // description: 'Postgres-native pull queues', - // }, + { + title: 'Queues', + icon: 'queues', + href: '/guides/queues', + description: 'Durable Message Queues with guaranteed delivery', + }, ] const selfHostingOptions = [ diff --git a/apps/docs/components/Navigation/NavigationMenu/MenuIconPicker.tsx b/apps/docs/components/Navigation/NavigationMenu/MenuIconPicker.tsx index 906e4a5b0feb1..24358664a09d9 100644 --- a/apps/docs/components/Navigation/NavigationMenu/MenuIconPicker.tsx +++ b/apps/docs/components/Navigation/NavigationMenu/MenuIconPicker.tsx @@ -95,7 +95,7 @@ function getMenuIcon(menuKey: string, width: number = 16, height: number = 16, c return case 'cron': return - case 'queue': + case 'queues': return default: return diff --git a/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts b/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts index 6c1479d9f0cb6..481c0c944a65e 100644 --- a/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts +++ b/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.constants.ts @@ -63,12 +63,12 @@ export const GLOBAL_MENU_ITEMS: GlobalMenuItems = [ href: '/guides/cron', level: 'cron', }, - // { - // label: 'Queues', - // icon: 'queue', - // href: '/guides/queues', - // level: 'queue', - // }, + { + label: 'Queues', + icon: 'queues', + href: '/guides/queues', + level: 'queues', + }, ], ], }, @@ -1146,6 +1146,25 @@ export const cron: NavMenuConstant = { ], } +export const queues: NavMenuConstant = { + icon: 'queues', + title: 'Queues', + url: '/guides/queues', + items: [ + { name: 'Overview', url: '/guides/queues' }, + { + name: 'Getting Started', + url: undefined, + items: [{ name: 'Quickstart', url: '/guides/queues/quickstart' }], + }, + { + name: 'References', + url: undefined, + items: [{ name: 'API', url: '/guides/queues/api' }], + }, + ], +} + export const api: NavMenuConstant = { icon: 'rest', title: 'REST API', diff --git a/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.tsx b/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.tsx index 634fc00ecf190..fa77229f9997f 100644 --- a/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.tsx +++ b/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.tsx @@ -16,6 +16,7 @@ enum MenuId { Storage = 'storage', Ai = 'ai', Cron = 'cron', + Queues = 'queues', Platform = 'platform', Deployment = 'deployment', MonitoringTroubleshooting = 'monitoring_troubleshooting', @@ -78,6 +79,10 @@ const menus: Menu[] = [ id: MenuId.Graphql, type: 'guide', }, + { + id: MenuId.Queues, + type: 'guide', + }, { id: MenuId.Auth, type: 'guide', diff --git a/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.utils.ts b/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.utils.ts index 5367a318ac831..94c31dd2b0df2 100644 --- a/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.utils.ts +++ b/apps/docs/components/Navigation/NavigationMenu/NavigationMenu.utils.ts @@ -129,6 +129,8 @@ export const getMenuId = (pathname: string | null) => { return MenuId.MonitoringTroubleshooting case pathname.startsWith('platform'): return MenuId.Platform + case pathname.startsWith('queues'): + return MenuId.Queues case pathname.startsWith('realtime'): return MenuId.Realtime case pathname.startsWith('resources'): diff --git a/apps/docs/content/guides/deployment/database-migrations.mdx b/apps/docs/content/guides/deployment/database-migrations.mdx index 0f9ba80f34d1e..3ad7b9f47841e 100644 --- a/apps/docs/content/guides/deployment/database-migrations.mdx +++ b/apps/docs/content/guides/deployment/database-migrations.mdx @@ -3,28 +3,28 @@ id: 'database-migrations' title: 'Database Migrations' description: 'How to manage schema migrations for your Supabase project.' subtitle: 'How to manage schema migrations for your Supabase project.' -video: 'https://www.youtube-nocookie.com/v/vyHyYpvjaks' -tocVideo: 'vyHyYpvjaks' +video: 'https://www.youtube-nocookie.com/v/Kx5nHBmIxyQ' +tocVideo: 'Kx5nHBmIxyQ' --- -Database schema changes are managed through "migrations". Database migrations are a common way of tracking changes to your database over time. +Database migrations are SQL statements that create, update, or delete your existing database schemas. They are a common way of tracking changes to your database over time. ## Schema migrations For this guide, we'll create a table called `employees` and see how we can make changes to it. +You will need to [install](/docs/guides/local-development#quickstart) the Supabase CLI and start the local development stack. + - To get started, generate a [new migration](/docs/reference/cli/supabase-migration-new) to store the SQL needed to create our `employees` table. - -```bash +```bash Terminal supabase migration new create_employees_table ``` @@ -37,18 +37,17 @@ supabase migration new create_employees_table - This creates a new migration: supabase/migrations/\ - _create_employees_table.sql. + This creates a new migration file in supabase/migrations directory. - To that file, add the SQL to create this `employees` table + To that file, add the SQL to create this `employees` table. -```sql -create table employees ( +```sql supabase/migrations/_create_employees_table.sql +create table if not exists employees ( id bigint primary key generated always as identity, - name text, + name text not null, email text, created_at timestamptz default now() ); @@ -62,16 +61,16 @@ create table employees ( - - Now that you have a migration file, you can run this migration and create the `employees` table. + + Run this migration to create the `employees` table. - Use the `reset` command here to reset the database to the current migrations + Now you can visit your new `employees` table in the local Dashboard. -```bash -supabase db reset +```bash Terminal +supabase migration up ``` @@ -83,15 +82,13 @@ supabase db reset - Now you can visit your new `employees` table in the Dashboard. - - Next, modify your `employees` table by adding a column for department. Create a new migration file for that. + Next, modify your `employees` table by adding a column for `department`. -```bash -supabase migration new add_department_to_employees_table +```bash Terminal +supabase migration new add_department_column ``` @@ -103,15 +100,12 @@ supabase migration new add_department_to_employees_table - This creates a new migration file: supabase/migrations/\ - _add_department_to_employees_table.sql. - - To that file, add the SQL to create a new department column + To that new migration file, add the SQL to create a new `department` column. -```sql +```sql supabase/migrations/_add_department_column.sql alter table if exists public.employees add department text default 'Hooli'; ``` @@ -121,22 +115,44 @@ add department text default 'Hooli'; -### Add sample data + -Now that you are managing your database with migrations scripts, it would be great have some seed data to use every time you reset the database. + + + Run this migration to update your existing `employees` table. + -For this, you can create a seed script in `supabase/seed.sql`. + + +```bash Terminal +supabase migration up +``` + + + + + + +Finally, you should see the `department` column added to your `employees` table in the local Dashboard. + +View the [complete code](https://github.com/supabase/supabase/tree/master/examples/database/employees) for this example. + +### Seeding data + +Now that you are managing your database with migrations scripts, it would be great have some seed data to use every time you reset the database. - Insert data into your `employees` table with your `supabase/seed.sql` file. + Create a seed script in supabase/seed.sql. + + To that file, add the SQL to insert data into your `employees` table. -```sql +```sql supabase/seed.sql insert into public.employees (name) values @@ -154,7 +170,7 @@ values - Reset your database (apply current migrations), and populate with seed data + Reset your database to reapply migrations and populate with seed data. @@ -198,60 +214,62 @@ The last step is deploying these changes to a live Supabase project. ## Deploy your project -You've been developing your project locally, making changes to your tables via migrations. It's time to deploy your project to the Supabase Platform and start scaling up to millions of users! Head over to [Supabase](https://supabase.com/dashboard) and create a new project to deploy to. +You've been developing your project locally, making changes to your tables via migrations. It's time to deploy your project to the Supabase Platform and start scaling up to millions of users! -### Log in to the Supabase CLI +Head over to [Supabase](https://supabase.com/dashboard) and create a new project to deploy to. - + + + + + [Login](/docs/reference/cli/usage#supabase-login) to the Supabase CLI using an auto-generated Personal Access Token. + + + ```bash Terminal supabase login ``` -```bash npx -npx supabase login -``` - - - -### Link your project - -Associate your project with your remote project using [`supabase link`](/docs/reference/cli/usage#supabase-link). - -```bash -supabase link --project-ref -# You can get from your project's dashboard URL: https://supabase.com/dashboard/project/ + -supabase db pull -# Capture any changes that you have made to your remote database before you went through the steps above -# If you have not made any changes to the remote database, skip this step -``` + + -`supabase/migrations` is now populated with a migration in `_remote_schema.sql`. -This migration captures any changes required for your local database to match the schema of your remote Supabase project. + -Review the generated migration file and once happy, apply the changes to your local instance: + + + [Link](/docs/reference/cli/usage#supabase-link) to your remote project by selecting from the on-screen prompt. + -```bash -# To apply the new migration to your local database: -supabase migration up + -# To reset your local database completely: -supabase db reset +```bash Terminal +supabase link ``` - + -There are a few commands required to link your project. We are in the process of consolidating these commands into a single command. Bear with us! + + - + -### Deploy database changes + + + [Push](/docs/reference/cli/usage#supabase-db-push) your migrations to the remote database. + -Deploy any local database migrations using [`db push`](/docs/reference/cli/usage#supabase-db-push): + -```sh +```bash Terminal supabase db push ``` -Visiting your live project on [Supabase](https://supabase.com/dashboard), you'll see a new `employees` table, complete with the `department` column you added in the second migration above. + + + + + +Visiting your live project on [Supabase](https://supabase.com/dashboard/project/_), you'll see a new `employees` table, complete with the `department` column you added in the second migration above. diff --git a/apps/docs/content/guides/queues.mdx b/apps/docs/content/guides/queues.mdx new file mode 100644 index 0000000000000..8130a357edcc4 --- /dev/null +++ b/apps/docs/content/guides/queues.mdx @@ -0,0 +1,34 @@ +--- +title: Supabase Queues +subtitle: 'Durable Message Queues with Guaranteed Delivery in Postgres' +--- + +Supabase Queues is a Postgres-native durable Message Queue system with guaranteed delivery built on the [pgmq database extension](https://github.com/tembo-io/pgmq). It offers developers a seamless way to persist and process Messages in the background while improving the resiliency and scalability of their applications and services. + +Queues couples the reliability of Postgres with the simplicity Supabase's platform and developer experience, enabling developers to manage Background Tasks with zero configuration. + +## Features + +- **Postgres Native** +
+ Built on top of the `pgmq` database extension, create and manage Queues with any Postgres tooling. +- **Guaranteed Message Delivery** +
+ Messages added to Queues are guaranteed to be delivered to your consumers. +- **Exactly Once Message Delivery** +
A Message is delivered exactly once to a consumer within a customizable visibility window. +- **Message Durability and Archival** +
+ Messages are stored in Postgres and you can choose to archive them for analytical or auditing purposes. +- **Granular Authorization** +
+ Control client-side consumer access to Queues with API permissions and Row Level Security (RLS) policies. +- **Queue Management and Monitoring** +
+ Create, manage, and monitor Queues and Messages in the Supabase Dashboard. + +## Resources + +- [Quickstart](/docs/guides/queues/quickstart) +- [API Reference](/docs/guides/queues/api) +- [`pgmq` GitHub Repository](https://github.com/tembo-io/pgmq) diff --git a/apps/docs/content/guides/queues/api.mdx b/apps/docs/content/guides/queues/api.mdx new file mode 100644 index 0000000000000..01b1aed2688e0 --- /dev/null +++ b/apps/docs/content/guides/queues/api.mdx @@ -0,0 +1,62 @@ +--- +title: API +--- + +{/* */} +When you create a Queue in Supabase, you can choose to create helper database functions in the `pgmq_public` schema. This schema exposes operations to manage Queue Messages to consumers client-side, but does not expose functions for creating or dropping Queues. + +Database functions in `pgmq_public` can be exposed via Supabase Data API so consumers client-side can call them. Visit the [Quickstart](/docs/guides/queues/quickstart) for an example. + +## `pgmq_public.pop(queue_name)` + +Retrieves the next available message and deletes it from the specified Queue. + +- `queue_name` (text): Queue name + +--- + +### `pgmq_public.send(queue_name, message, sleep_seconds)` + +Adds a Message to the specified Queue, optionally delaying its visibility to all consumers by a number of seconds. + +- `queue_name` (text): Queue name +- `message` (jsonb): Message payload to send +- `sleep_seconds` (integer, optional): Delay message visibility by specified seconds. Defaults to 0 + +--- + +### `pgmq_public.send_batch(queue_name, messages, sleep_seconds)` + +Adds a batch of Messages to the specified Queue, optionally delaying their availability to all consumers by a number of seconds. + +- `queue_name` (text): Queue name +- `messages` (jsonb[]): Array of message payloads to send +- `sleep_seconds` (integer, optional): Delay messages visibility by specified seconds. Defaults to 0 + +--- + +### `pgmq_public.archive(queue_name, message_id)` + +Archives a Message by moving it from the Queue table to the Queue's archive table. + +- `queue_name` (text): Queue name +- `message_id` (bigint): ID of the Message to archive + +--- + +### `pgmq_public.delete(queue_name, message_id)` + +Permanently deletes a Message from the specified Queue. + +- `queue_name` (text): Queue name +- `message_id` (bigint): ID of the Message to delete + +--- + +### `pgmq_public.read(queue_name, sleep_seconds, n)` + +Reads up to "n" Messages from the specified Queue with an optional "sleep_seconds" (visibility timeout). + +- `queue_name` (text): Queue name +- `sleep_seconds` (integer): Visibility timeout in seconds +- `n` (integer): Maximum number of Messages to read diff --git a/apps/docs/content/guides/queues/quickstart.mdx b/apps/docs/content/guides/queues/quickstart.mdx new file mode 100644 index 0000000000000..426f057ea6c93 --- /dev/null +++ b/apps/docs/content/guides/queues/quickstart.mdx @@ -0,0 +1,195 @@ +--- +title: Quickstart +subtitle: 'Learn how to use Supabase Queues to add and read messages' +--- + +{/* */} +This guide is an introduction to interacting with Supabase Queues via the Dashboard and official client library. Check out [Queues API Reference](/docs/guides/queues/api) for more details on our API. + +## Concepts + +Supabase Queues is a pull-based Message Queue consisting of three main components: Queues, Messages, and Queue Types. + +### Pull-Based Queue + +A pull-based Queue is a Message storage and delivery system where consumers actively fetch Messages when they're ready to process them - similar to constantly refreshing a webpage to display the latest updates. Our pull-based Queues process Messages in a First-In-First-Out (FIFO) manner without priority levels. + +### Message + +A Message in a Queue is a json object that is stored until a consumer explicitly processes and removes it, like a task waiting in a to-do list until someone checks and completes it. + +### Queue Types + +Supabase Queues offers three types of Queues: + +- **Basic Queue**: A durable Queue that stores Messages in a logged table. +- **Unlogged Queue**: A transient Queue that stores Messages in an unlogged table for better performance but may result in loss of Queue Messages. + +- **Partitioned Queue** (_Coming Soon_): A durable and scalable Queue that stores Messages in multiple table partitions for better performance. + +## Create Queues + +To get started, navigate to the [Supabase Queues](/dashboard/project/_/integrations/queues/overview) Postgres Module under Integrations in the Dashboard and enable the `pgmq` extension. + + + +`pgmq` extension is available in Postgres version 15.6.1.143 or later. + + + +Supabase Dashboard Integrations page, showing the Queues Postgres Module + +On the [Queues page](/dashboard/project/_/integrations/queues/queues): + +- Click **Add a new queue** button + + + +If you've already created a Queue click the **Create a queue** button instead. + + + +- Name your queue + + + +Queue names can only be lowercase and hyphens and underscores are permitted. + + + +- Select your [Queue Type](#queue-types) + +Create a Queue from the Supabase Dashboard + +### What happens when you create a queue? + +Every new Queue creates two tables in the `pgmq` schema. These tables are `pgmq.q_` to store and process active messages and `pgmq.a_` to store any archived messages. + +A "Basic Queue" will create `pgmq.q_` and `pgmq.a_` tables as logged tables. + +However, an "Unlogged Queue" will create `pgmq.q_` as an unlogged table for better performance while sacrificing durability. The `pgmq.a_` table will still be created as a logged table so your archived messages remain safe and secure. + +## Expose Queues to client-side consumers + +Queues, by default, are not exposed over Supabase Data API and are only accessible via Postgres clients. + +However, you may grant client-side consumers access to your Queues by enabling the Supabase Data API and granting permissions to the Queues API, which is a collection of database functions in the `pgmq_public` schema that wraps the database functions in the `pgmq` schema. + +This is to prevent direct access to the `pgmq` schema and its tables (RLS is not enabled by default on any tables) and database functions. + +To get started, navigate to the Queues [Settings page](/dashboard/project/_/integrations/queues/settings) and toggle on “Expose Queues via PostgREST”. Once enabled, Supabase creates and exposes a `pgmq_public` schema containing database function wrappers to a subset of `pgmq`'s database functions. + +Screenshot of Queues settings with toggle to expose to PostgREST + +### Enable RLS on your tables in `pgmq` schema + +For security purposes, you must enable Row Level Security (RLS) on all Queue tables (all tables in `pgmq` schema that begin with `q_`) if the Data API is enabled. + +You’ll want to create RLS policies for any Queues you want your client-side consumers to interact with. + +Screenshot of creating an RLS policy from the Queues settings + +### Grant permissions to `pgmq_public` database functions + +On top of enabling RLS and writing RLS policies on the underlying Queue tables, you must grant the correct permissions to the `pgmq_public` database functions for each Data API role. + +The permissions required for each Queue API database function: + +| **Operations** | **Permissions Required** | +| ------------------- | ------------------------ | +| `send` `send_batch` | `Select` `Insert` | +| `read` `pop` | `Select` `Update` | +| `archive` `delete` | `Select` `Delete` | + +Screenshot of configuring API access for roles from the Queues settings + + + +`postgres` and `service_role` roles should never be exposed client-side. + + + +### Enqueuing and Dequeuing Messages + +Once your Queue has been created, you can begin enqueuing and dequeuing Messages. + +Here's a TypeScript example using the official Supabase client library: + +```tsx +import { createClient } from '@supabase/supabase-js' + +const supabaseUrl = 'supabaseURL' +const supabaseKey = 'supabaseKey' + +const supabase = createClient(supabaseUrl, supabaseKey) + +const QueuesTest: React.FC = () => { + //Add a Message + const sendToQueue = async () => { + const result = await supabase.rpc('send', { + queue_name: 'foo', + message: { hello: 'world' }, + sleep_seconds: 30, + }) + console.log(result) + } + + //Dequeue Message + const popFromQueue = async () => { + const result = await supabase.rpc('pop', { queue_name: 'foo' }) + console.log(result) + } + + return ( +
+

Queue Test Component

+ + +
+ ) +} + +export default QueuesTest +``` diff --git a/apps/docs/features/docs/GuidesMdx.utils.tsx b/apps/docs/features/docs/GuidesMdx.utils.tsx index aaef7455e70e8..1c8b28a4282c2 100644 --- a/apps/docs/features/docs/GuidesMdx.utils.tsx +++ b/apps/docs/features/docs/GuidesMdx.utils.tsx @@ -35,6 +35,7 @@ const PUBLISHED_SECTIONS = [ 'local-development', 'monitoring-troubleshooting', 'platform', + 'queues', 'realtime', 'resources', 'self-hosting', diff --git a/apps/docs/layouts/MainSkeleton.tsx b/apps/docs/layouts/MainSkeleton.tsx index dbee0c2a53808..3eaf636e930e5 100644 --- a/apps/docs/layouts/MainSkeleton.tsx +++ b/apps/docs/layouts/MainSkeleton.tsx @@ -34,6 +34,10 @@ const levelsData = { icon: 'cron', name: 'Cron', }, + queues: { + icon: 'queues', + name: 'Queues', + }, api: { icon: 'rest', name: 'REST API', diff --git a/apps/docs/public/img/queues-quickstart-create.png b/apps/docs/public/img/queues-quickstart-create.png new file mode 100644 index 0000000000000..30ed87e885661 Binary files /dev/null and b/apps/docs/public/img/queues-quickstart-create.png differ diff --git a/apps/docs/public/img/queues-quickstart-install.png b/apps/docs/public/img/queues-quickstart-install.png new file mode 100644 index 0000000000000..6ca44f07e1cc9 Binary files /dev/null and b/apps/docs/public/img/queues-quickstart-install.png differ diff --git a/apps/docs/public/img/queues-quickstart-rls.png b/apps/docs/public/img/queues-quickstart-rls.png new file mode 100644 index 0000000000000..ed112b31b7f8d Binary files /dev/null and b/apps/docs/public/img/queues-quickstart-rls.png differ diff --git a/apps/docs/public/img/queues-quickstart-roles-light.png b/apps/docs/public/img/queues-quickstart-roles-light.png new file mode 100644 index 0000000000000..e43792164a97d Binary files /dev/null and b/apps/docs/public/img/queues-quickstart-roles-light.png differ diff --git a/apps/docs/public/img/queues-quickstart-roles.png b/apps/docs/public/img/queues-quickstart-roles.png new file mode 100644 index 0000000000000..ae72156c430c1 Binary files /dev/null and b/apps/docs/public/img/queues-quickstart-roles.png differ diff --git a/apps/docs/public/img/queues-quickstart-settings.png b/apps/docs/public/img/queues-quickstart-settings.png new file mode 100644 index 0000000000000..a8138e6c5f101 Binary files /dev/null and b/apps/docs/public/img/queues-quickstart-settings.png differ diff --git a/apps/studio/components/interfaces/Integrations/Integration/IntegrationOverviewTab.tsx b/apps/studio/components/interfaces/Integrations/Integration/IntegrationOverviewTab.tsx index d86dbac261bfb..d0fff0d5b9de9 100644 --- a/apps/studio/components/interfaces/Integrations/Integration/IntegrationOverviewTab.tsx +++ b/apps/studio/components/interfaces/Integrations/Integration/IntegrationOverviewTab.tsx @@ -6,7 +6,7 @@ import { Markdown } from 'components/interfaces/Markdown' import { useDatabaseExtensionsQuery } from 'data/database-extensions/database-extensions-query' import { useSelectedProject } from 'hooks/misc/useSelectedProject' import { Badge, Separator } from 'ui' -import { Admonition } from 'ui-patterns' +import { Admonition } from 'ui-patterns/admonition' import { INTEGRATIONS } from '../Landing/Integrations.constants' import { BuiltBySection } from './BuildBySection' import { MarkdownContent } from './MarkdownContent' @@ -26,23 +26,26 @@ export const IntegrationOverviewTab = ({ const integration = INTEGRATIONS.find((i) => i.id === id) - const dependsOnExtension = (integration?.requiredExtensions ?? []).length > 0 - const { data: extensions } = useDatabaseExtensionsQuery({ projectRef: project?.ref, connectionString: project?.connectionString, }) - const neededExtensions = (extensions ?? []).filter((ext) => - (integration?.requiredExtensions ?? []).includes(ext.name) - ) - - const hasMissingExtensions = neededExtensions.some((x) => !x.installed_version) - if (!integration) { return
Unsupported integration type
} + const dependsOnExtension = (integration.requiredExtensions ?? []).length > 0 + + const installableExtensions = (extensions ?? []).filter((ext) => + (integration.requiredExtensions ?? []).includes(ext.name) + ) + const hasToInstallExtensions = installableExtensions.some((x) => !x.installed_version) + + // The integration requires extensions that are not available to install on the current database image + const hasMissingExtensions = + installableExtensions.length !== integration.requiredExtensions.length + return (
@@ -65,19 +68,23 @@ export const IntegrationOverviewTab = ({ className="max-w-full" content={`This integration uses the ${integration.requiredExtensions.map((x) => `\`${x}\``).join(', ')} extension${integration.requiredExtensions.length > 1 ? 's' : ''} directly in your Postgres database. - ${hasMissingExtensions ? `Install ${integration.requiredExtensions.length > 1 ? 'these' : 'this'} database extension${integration.requiredExtensions.length > 1 ? 's' : ''} to use ${integration.name} in your project.` : ''} + ${hasToInstallExtensions && !hasMissingExtensions ? `Install ${integration.requiredExtensions.length > 1 ? 'these' : 'this'} database extension${integration.requiredExtensions.length > 1 ? 's' : ''} to use ${integration.name} in your project.` : ''} `} /> -
- {neededExtensions.map((extension) => ( - - ))} -
+ {hasMissingExtensions ? ( + integration.missingExtensionsAlert + ) : ( +
+ {installableExtensions.map((extension) => ( + + ))} +
+ )}
)} - {!!actions && !hasMissingExtensions &&
{actions}
} + {!!actions && !hasToInstallExtensions &&
{actions}
} {children} diff --git a/apps/studio/components/interfaces/Integrations/Landing/Integrations.constants.tsx b/apps/studio/components/interfaces/Integrations/Landing/Integrations.constants.tsx index 752ee4eea1a7c..b2babf911f9f7 100644 --- a/apps/studio/components/interfaces/Integrations/Landing/Integrations.constants.tsx +++ b/apps/studio/components/interfaces/Integrations/Landing/Integrations.constants.tsx @@ -6,6 +6,7 @@ import { ComponentType, ReactNode } from 'react' import { GenericSkeletonLoader } from 'components/ui/ShimmeringLoader' import { BASE_PATH } from 'lib/constants' import { cn } from 'ui' +import { UpgradeDatabaseAlert } from '../Queues/UpgradeDatabaseAlert' import { WRAPPERS } from '../Wrappers/Wrappers.constants' import { WrapperMeta } from '../Wrappers/Wrappers.types' @@ -35,6 +36,8 @@ export type IntegrationDefinition = { websiteUrl: string } requiredExtensions: string[] + /** Optional component to render if the integration requires extensions that are not available on the current database image */ + missingExtensionsAlert?: ReactNode navigation?: Navigation[] navigate: ( id: string, @@ -53,6 +56,7 @@ const supabaseIntegrations: IntegrationDefinition[] = [ id: 'queues', type: 'postgres_extension' as const, requiredExtensions: ['pgmq'], + missingExtensionsAlert: , name: `Queues`, icon: ({ className, ...props } = {}) => ( diff --git a/apps/studio/components/interfaces/Integrations/Landing/useInstalledIntegrations.tsx b/apps/studio/components/interfaces/Integrations/Landing/useInstalledIntegrations.tsx index 8d5696fab5d59..6ba86da73ac49 100644 --- a/apps/studio/components/interfaces/Integrations/Landing/useInstalledIntegrations.tsx +++ b/apps/studio/components/interfaces/Integrations/Landing/useInstalledIntegrations.tsx @@ -65,26 +65,12 @@ export const useInstalledIntegrations = () => { }).sort((a, b) => a.name.localeCompare(b.name)) }, [wrappers, extensions, isHooksEnabled]) - // available integrations are all integrations that can be installed. If an extension is on available to be installed - // on the project, that integration is hidden. - const availableIntegrations = useMemo(() => { - return INTEGRATIONS.filter((i) => { - // special handling for supabase webhooks - if (i.id === 'webhooks') { - return true - } - if (i.type === 'wrapper') { - return true - } - if (i.type === 'postgres_extension') { - return i.requiredExtensions.every((extName) => { - const foundExtension = (extensions ?? []).find((ext) => ext.name === extName) - return !!foundExtension - }) - } - return false - }).sort((a, b) => a.name.localeCompare(b.name)) - }, [extensions]) + // available integrations are all integrations that can be installed. If an integration can't be installed (needed + // extensions are not available on this DB image), the UI will provide a tooltip explaining why. + const availableIntegrations = useMemo( + () => INTEGRATIONS.sort((a, b) => a.name.localeCompare(b.name)), + [] + ) const error = fdwError || extensionsError || schemasError const isLoading = isSchemasLoading || isFDWLoading || isExtensionsLoading diff --git a/apps/studio/components/interfaces/Integrations/Queues/OverviewTab.tsx b/apps/studio/components/interfaces/Integrations/Queues/OverviewTab.tsx index bd7598b1a9ea1..dba1d5108468c 100644 --- a/apps/studio/components/interfaces/Integrations/Queues/OverviewTab.tsx +++ b/apps/studio/components/interfaces/Integrations/Queues/OverviewTab.tsx @@ -1,10 +1,10 @@ -import { useSelectedProject } from 'hooks/misc/useSelectedProject' -import { IntegrationOverviewTab } from '../Integration/IntegrationOverviewTab' +import { useParams } from 'common' import { useQueuesExposePostgrestStatusQuery } from 'data/database-queues/database-queues-expose-postgrest-status-query' -import { Admonition } from 'ui-patterns' -import { Button } from 'ui' +import { useSelectedProject } from 'hooks/misc/useSelectedProject' import Link from 'next/link' -import { useParams } from 'common' +import { Button } from 'ui' +import { Admonition } from 'ui-patterns' +import { IntegrationOverviewTab } from '../Integration/IntegrationOverviewTab' export const QueuesOverviewTab = () => { const { ref } = useParams() @@ -24,12 +24,12 @@ export const QueuesOverviewTab = () => { title="Queues can be managed via any Supabase client library or PostgREST endpoints" >

- You may choose to toggle the exposure of Queues through PostgREST via the queues + You may choose to toggle the exposure of Queues through Data APIs via the queues settings

diff --git a/apps/studio/components/interfaces/Integrations/Queues/QueuesSettings.tsx b/apps/studio/components/interfaces/Integrations/Queues/QueuesSettings.tsx index f0eaf0d2e76a0..67d28db12f409 100644 --- a/apps/studio/components/interfaces/Integrations/Queues/QueuesSettings.tsx +++ b/apps/studio/components/interfaces/Integrations/Queues/QueuesSettings.tsx @@ -12,11 +12,15 @@ import { FormPanelContent, FormPanelFooter, } from 'components/ui/Forms/FormPanel' +import { useProjectPostgrestConfigQuery } from 'data/config/project-postgrest-config-query' +import { useProjectPostgrestConfigUpdateMutation } from 'data/config/project-postgrest-config-update-mutation' import { useQueuesExposePostgrestStatusQuery } from 'data/database-queues/database-queues-expose-postgrest-status-query' import { QUEUES_SCHEMA, useDatabaseQueueToggleExposeMutation, } from 'data/database-queues/database-queues-toggle-postgrest-mutation' +import { useTableUpdateMutation } from 'data/tables/table-update-mutation' +import { useTablesQuery } from 'data/tables/tables-query' import { useCheckPermissions } from 'hooks/misc/useCheckPermissions' import { useSelectedProject } from 'hooks/misc/useSelectedProject' import { @@ -28,12 +32,8 @@ import { Switch, } from 'ui' import { Admonition } from 'ui-patterns' -import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout' -import { useProjectPostgrestConfigUpdateMutation } from 'data/config/project-postgrest-config-update-mutation' -import { useProjectPostgrestConfigQuery } from 'data/config/project-postgrest-config-query' -import { useTablesQuery } from 'data/tables/tables-query' import ConfirmationModal from 'ui-patterns/Dialogs/ConfirmationModal' -import { useTableUpdateMutation } from 'data/tables/table-update-mutation' +import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout' // [Joshen] Not convinced with the UI and layout but getting the functionality out first @@ -179,7 +179,7 @@ export const QueuesSettings = () => {
diff --git a/apps/studio/components/interfaces/Integrations/Queues/UpgradeDatabaseAlert.tsx b/apps/studio/components/interfaces/Integrations/Queues/UpgradeDatabaseAlert.tsx new file mode 100644 index 0000000000000..ccdf6feedaf68 --- /dev/null +++ b/apps/studio/components/interfaces/Integrations/Queues/UpgradeDatabaseAlert.tsx @@ -0,0 +1,28 @@ +import Link from 'next/link' + +import { useSelectedProject } from 'hooks/misc/useSelectedProject' +import { Button } from 'ui' +import { Admonition } from 'ui-patterns/admonition' + +export const UpgradeDatabaseAlert = () => { + const project = useSelectedProject() + + return ( + +
+

+ This integration requires the pgmq extension which is not available on this + version of Postgres. The extension is available on version 15.6.1.143 and higher. +

+
+ + + ) +} diff --git a/apps/studio/static-data/integrations/queues/overview.md b/apps/studio/static-data/integrations/queues/overview.md index 09bebecd8c32a..0ec0a1b590e47 100644 --- a/apps/studio/static-data/integrations/queues/overview.md +++ b/apps/studio/static-data/integrations/queues/overview.md @@ -1 +1 @@ -Queues uses the `pgmq` Postgres extension, which is a lightweight message queue like [AWS SQS](https://aws.amazon.com/sqs/) and [RSMQ](https://github.com/smrchy/rsmq) but on Postgres. +Queues uses the `pgmq` Postgres extension, which is a durable Message Queue, like [AWS SQS](https://aws.amazon.com/sqs/) and [RSMQ](https://github.com/smrchy/rsmq), with guaranteed delivery in Postgres. diff --git a/apps/www/_blog/2024-12-04-supabase-cron.mdx b/apps/www/_blog/2024-12-04-supabase-cron.mdx index 9566886fa9652..972e74129672b 100644 --- a/apps/www/_blog/2024-12-04-supabase-cron.mdx +++ b/apps/www/_blog/2024-12-04-supabase-cron.mdx @@ -35,7 +35,7 @@ Supabase Cron is built on the powerful [`pg_cron`](https://github.com/citusdata/ It's a Supabase policy to [support existing tools](https://supabase.com/docs/guides/getting-started/architecture#support-existing-tools) wherever possible, and the Citus Data team have generously licensed their extension with the OSI-compatible [PostgreSQL license](https://github.com/citusdata/pg_cron?tab=PostgreSQL-1-ov-file). -We're very thankful to all the contributors and we look forward to continue working with the community. +We're very thankful to all the contributors and we look forward to our continued work with the community. diff --git a/apps/www/_blog/2024-12-05-high-performance-disks.mdx b/apps/www/_blog/2024-12-05-high-performance-disks.mdx new file mode 100644 index 0000000000000..da19133548009 --- /dev/null +++ b/apps/www/_blog/2024-12-05-high-performance-disks.mdx @@ -0,0 +1,91 @@ +--- +title: 'Scaling Database Storage: Introducing High Performance Disks' +description: Store up to 60 TB of data with 100x improved durability and 5x more IOPS +author: pcnc,jonny,gregpapas +image: launch-week-13/high-performance-disks/og.jpg +thumb: launch-week-13/high-performance-disks/thumb.jpg +categories: + - product +tags: + - launch-week + - postgres +date: '2024-12-05T00:00:01' +toc_depth: 3 +launchweek: '13' +--- + +At Supabase, we're constantly working to provide developers with the tools they need to build powerful applications at scale. In our previous [blog post](https://supabase.com/blog/supabase-clickhouse-partnership#improved-disk-management) we mentioned a new type of high performance disk. Today, we’re here to tell you all about it. + +With advanced disks you can store up to 60 TB of data with 100x improved durability, and provision up to 5x more IOPS than the default disks we offer. + +## A Two-Pronged Approach to Disk Scalability + +We've been tackling disk scalability from two angles. On the software side, our implementation of [Oriole DB's index-organized tables](https://www.orioledb.com/blog/orioledb-beta7-benchmarks) significantly reduces disk I/O operations, improving performance without additional hardware resources. However, this focus for this post is on our infrastructure improvements: the introduction of new disk options that allow advanced scaling for your Postgres databases. + +## Expanded Capacity + +One of the most significant improvements is our increased storage capacity. We've moved beyond our previous 16 TB limit, now offering up to 60 TB of storage for your largest databases. But with greater capacity comes the need for enhanced performance - particularly in how quickly your database can read and write data. This makes IOPS (Input/Output Operations Per Second) especially important. + +To address these needs, we’re introducing high performance disks. These high performance disks can handle up to 80,000 IOPS - a 5x increase from the 16,000 IOPS limit of our general purpose disks. + +## Understanding Performance: IOPS and Throughput + +IOPS is a critical metric that measures how many read and write operations your database can perform each second. Think of it as the "speed limit" for your database's ability to access stored data. Higher IOPS means faster database operations, which translates to better application performance, especially for data-intensive workloads. + +Throughput, measured in MiB/s (Mebibytes per second), is equally important as it determines how much total data can flow through your disk at once. While IOPS tells you how many individual read/write operations can happen per second, throughput determines the total volume of data that can be moved. With our general purpose disks, you start with a baseline throughput of 125 MiB/s, which can be provisioned up to 1,000 MiB/s. Our new high performance disks automatically scale throughput with IOPS, providing better performance for data-intensive workloads. + +Effective throughput and IOPS also depends on your compute instance size. You can check out our [compute and disk documentation](https://supabase.com/docs/guides/platform/compute-and-disk#compute-size) for more detail. + +## Durability: Keeping Your Data Safe + +Another benefit of our High performance disks is increased durability. Our new disks offer 99.999% durability, a 100x increase over our standard disk. This means that if you’re using high performance disk, you will almost never need to worry about disk failure — say goodbye to recovery from backups. + +## Consolidated Disk and Compute User Interface + +With these advanced options comes complexity—both in the number of options available, and how they interplay with compute settings. To address this we've completely redesigned our disk management interface to coexist and interoperate with our compute upgrade UI. When designing the new UI, we adhered to the following principles: + +### Transparent Billing + +- Real-time cost updates as you adjust your disk configuration + +Real-time cost updates as you adjust your disk configuration + +- Clear breakdown of how each change affects your bill + +Clear breakdown of how each change affects your bill + +### Updated Disk Size Insights + +The Disk Size Usage graph breaks down the space used by the Database, Write-Ahead Log, and the System, rather than simply showing "used space." + +Updated disk size insights + +### Preventing Footguns + +The effective IOPS is limited by both your compute add-on and disk configuration and it is technically possible to over provision the disk throughput and IOPS with the instance not being able to make full use of it. For example, to achieve maximum IOPS (80,000), you'll need a 16XL or larger compute instance. The dashboard warns you when it detects scenarios like these. + +Preventing footguns + +## Pricing + +The pricing for high performance disks start at $0.195 per GB, and you can additionally provision IOPS at $0.119 per IOPS. The storage pricing for general purpose disks remains unchanged, and you can provision IOPS at $0.024 per IOPS and 0.095$ per Mbps throughput. + +For more details on pricing breakdown vs. general purpose disk, check out our [pricing page](https://supabase.com/pricing). + +## Getting Started + +Ready to try out our new disk options? Visit [Compute and Disk](https://supabase.com/dashboard/project/_/settings/compute-and-disk) in the Supabase Dashboard. + +We're excited to see what you'll build with these new capabilities. As always, we're committed to providing the tools you need to scale your applications effectively while maintaining the simplicity and developer experience you've come to expect from Supabase. diff --git a/apps/www/_blog/2024-12-05-supabase-queues.mdx b/apps/www/_blog/2024-12-05-supabase-queues.mdx new file mode 100644 index 0000000000000..83d4723c833f8 --- /dev/null +++ b/apps/www/_blog/2024-12-05-supabase-queues.mdx @@ -0,0 +1,221 @@ +--- +title: 'Supabase Queues' +description: Durable Message Queues with Guaranteed Delivery in Postgres +author: ivasilov,oli_rice +image: launch-week-13/day-4-supabase-queues/og.png +thumb: launch-week-13/day-4-supabase-queues/thumb.png +categories: + - developers + - postgres +tags: + - queues + - postgres +date: '2024-12-05T00:00:02' +toc_depth: 3 +launchweek: '13' +--- + +Today we're releasing [Supabase Queues](/modules/queues), for durable background task processing. + +Supabase Queues is a Postgres-native, durable Message Queue with guaranteed delivery, improving the scalability and resiliency of your applications. It's designed to work seamlessly with the entire Supabase platform. + +
+