diff --git a/docs-v2/pages/workflows/building-workflows/code/nodejs/using-data-stores.mdx b/docs-v2/pages/workflows/building-workflows/code/nodejs/using-data-stores.mdx index 92c298eae44c3..2a00ad0496578 100644 --- a/docs-v2/pages/workflows/building-workflows/code/nodejs/using-data-stores.mdx +++ b/docs-v2/pages/workflows/building-workflows/code/nodejs/using-data-stores.mdx @@ -6,23 +6,23 @@ In Node.js (Javascript) code steps, you can also store and retrieve data within Add data stores to steps as props. By adding the store as a prop, it's available under `this`. -For example, you can define a data store as a data prop, and reference it at `this.data`: +For example, you can define a data store as a dataStore prop, and reference it at `this.dataStore`: ```javascript export default defineComponent({ props: { - // Define that the "data" variable in our component is a data store - data: { type: "data_store" }, + // Define that the "dataStore" variable in our component is a data store + dataStore: { type: "data_store" }, }, async run({ steps, $ }) { - // Now we can access the data store at "this.data" - await this.data.get("email"); + // Now we can access the data store at "this.dataStore" + await this.dataStore.get("email"); }, }); ``` -**`props` injects variables into `this`**. See how we declared the `data` prop in the `props` object, and then accessed it at `this.data` in the `run` method. +**`props` injects variables into `this`**. See how we declared the `dataStore` prop in the `props` object, and then accessed it at `this.dataStore` in the `run` method. @@ -37,20 +37,62 @@ Once you've defined a data store prop for your component, then you'll be able to ## Saving data -Data Stores are key-value stores. Save data within a Data Store using the `this.data.set` method. The first argument is the _key_ where the data should be held, and the second argument is the _value_ assigned to that key. +Data Stores are key-value stores. Save data within a Data Store using the `this.dataStore.set` method. The first argument is the _key_ where the data should be held, and the second argument is the _value_ assigned to that key. ```javascript export default defineComponent({ props: { - data: { type: "data_store" }, + dataStore: { type: "data_store" }, }, async run({ steps, $ }) { // Store a timestamp each time this step is executed in the workflow - await this.data.set("lastRanAt", new Date()); + await this.dataStore.set("lastRanAt", new Date()); }, }); ``` +### Setting expiration (TTL) for records + +You can set an expiration time for a record by passing a TTL (Time-To-Live) option as the third argument to the `set` method. The TTL value is specified in seconds: + +```javascript +export default defineComponent({ + props: { + dataStore: { type: "data_store" }, + }, + async run({ steps, $ }) { + // Store a temporary value that will expire after 1 hour (3600 seconds) + await this.dataStore.set("temporaryToken", "abc123", { ttl: 3600 }); + + // Store a value that will expire after 1 day + await this.dataStore.set("dailyMetric", 42, { ttl: 86400 }); + }, +}); +``` + +When the TTL period elapses, the record will be automatically deleted from the data store. + +### Updating TTL for existing records + +You can update the TTL for an existing record using the `setTtl` method: + +```javascript +export default defineComponent({ + props: { + dataStore: { type: "data_store" }, + }, + async run({ steps, $ }) { + // Update an existing record to expire after 30 minutes + await this.dataStore.setTtl("temporaryToken", 1800); + + // Remove expiration from a record + await this.dataStore.setTtl("temporaryToken", null); + }, +}); +``` + +This is useful for extending the lifetime of temporary data or removing expiration from records that should now be permanent. + ## Retrieving keys Fetch all the keys in a given Data Store using the `keys` method: @@ -58,11 +100,11 @@ Fetch all the keys in a given Data Store using the `keys` method: ```javascript export default defineComponent({ props: { - data: { type: "data_store" }, + dataStore: { type: "data_store" }, }, async run({ steps, $ }) { // Return a list of all the keys in a given Data Store - return await this.data.keys(); + return await this.dataStore.keys(); }, }); ``` @@ -74,11 +116,11 @@ If you need to check whether a specific `key` exists in a Data Store, you can pa ```javascript export default defineComponent({ props: { - data: { type: "data_store" }, + dataStore: { type: "data_store" }, }, async run({ steps, $ }) { // Check if a specific key exists in your Data Store - return await this.data.has("lastRanAt"); + return await this.dataStore.has("lastRanAt"); }, }); ``` @@ -90,11 +132,11 @@ You can retrieve data with the Data Store using the `get` method. Pass the _key_ ```javascript export default defineComponent({ props: { - data: { type: "data_store" }, + dataStore: { type: "data_store" }, }, async run({ steps, $ }) { // Check if the lastRanAt key exists - const lastRanAt = await this.data.get("lastRanAt"); + const lastRanAt = await this.dataStore.get("lastRanAt"); }, }); ``` @@ -106,11 +148,11 @@ Use an async iterator to efficiently retrieve all records or keys in your data s ```javascript export default defineComponent({ props: { - data: { type: "data_store" }, + dataStore: { type: "data_store" }, }, async run({ steps, $ }) { const records = {}; - for await (const [k,v] of this.data) { + for await (const [k,v] of this.dataStore) { records[k] = v; } return records; @@ -125,14 +167,14 @@ To delete or update the _value_ of an individual record, use the `set` method fo ```javascript export default defineComponent({ props: { - data: { type: "data_store" }, + dataStore: { type: "data_store" }, }, async run({ steps, $ }) { // Update the value associated with the key, myKey - await this.data.set("myKey", "newValue"); + await this.dataStore.set("myKey", "newValue"); // Remove the value but retain the key - await this.data.set("myKey", ""); + await this.dataStore.set("myKey", ""); }, }); ``` @@ -144,11 +186,11 @@ To delete individual records in a Data Store, use the `delete` method for a spec ```javascript export default defineComponent({ props: { - data: { type: "data_store" }, + dataStore: { type: "data_store" }, }, async run({ steps, $ }) { // Delete the lastRanAt record - const lastRanAt = await this.data.delete("lastRanAt"); + const lastRanAt = await this.dataStore.delete("lastRanAt"); }, }); ``` @@ -160,11 +202,11 @@ If you need to delete all records in a given Data Store, you can use the `clear` ```javascript export default defineComponent({ props: { - data: { type: "data_store" }, + dataStore: { type: "data_store" }, }, async run({ steps, $ }) { // Delete all records from a specific Data Store - return await this.data.clear(); + return await this.dataStore.clear(); }, }); ``` @@ -201,16 +243,16 @@ You can use a data store as a counter. For example, this code counts the number ```javascript export default defineComponent({ props: { - data: { type: "data_store" }, + dataStore: { type: "data_store" }, }, async run({ steps, $ }) { // By default, all database entries are undefined. // It's wise to set a default value so our code as an initial value to work with - const counter = (await this.data.get("counter")) ?? 0; + const counter = (await this.dataStore.get("counter")) ?? 0; // On the first run "counter" will be 0 and we'll increment it to 1 // The next run will increment the counter to 2, and so forth - await this.data.set("counter", counter + 1); + await this.dataStore.set("counter", counter + 1); }, }); ``` @@ -224,12 +266,12 @@ For example, this workflow's trigger contains an email address from a potential ```javascript export default defineComponent({ props: { - data: { type: "data_store" }, + dataStore: { type: "data_store" }, }, async run({ steps, $ }) { const email = steps.trigger.event.body.new_customer_email; // Retrieve the past recorded emails from other runs - const emails = (await this.data.get("emails")) ?? []; + const emails = (await this.dataStore.get("emails")) ?? []; // If the current email being passed from our webhook is already in our list, exit early if (emails.includes(email)) { @@ -237,14 +279,50 @@ export default defineComponent({ } // Add the current email to the list of past emails so we can detect it in the future runs - await this.data.set("emails", [...emails, email]); + await this.dataStore.set("emails", [...emails, email]); }, }); ``` -## Data store limitations +## TTL use case: temporary caching and rate limiting + +TTL functionality is particularly useful for implementing temporary caching and rate limiting. Here's an example of a simple rate limiter that prevents a user from making more than 5 requests per hour: + +```javascript +export default defineComponent({ + props: { + dataStore: { type: "data_store" }, + }, + async run({ steps, $ }) { + const userId = steps.trigger.event.userId; + const rateKey = `ratelimit:${userId}`; + + // Try to get current rate limit counter + let requests = await this.dataStore.get(rateKey); + + if (requests === undefined) { + // First request from this user in the time window + await this.dataStore.set(rateKey, 1, { ttl: 3600 }); // Expire after 1 hour + return { allowed: true, remaining: 4 }; + } + + if (requests >= 5) { + // Rate limit exceeded + return { allowed: false, error: "Rate limit exceeded", retryAfter: "1 hour" }; + } + + // Increment the counter + await this.dataStore.set(rateKey, requests + 1); + return { allowed: true, remaining: 4 - requests }; + }, +}); +``` -Data Stores are only currently available in Node.js and Python steps. They are not yet available [Bash](/workflows/building-workflows/code/bash/) or [Go](/workflows/building-workflows/code/go/). +This pattern can be extended for various temporary caching scenarios like: +- Session tokens with automatic expiration +- Short-lived feature flags +- Temporary access grants +- Time-based promotional codes ### Supported data types diff --git a/docs-v2/pages/workflows/building-workflows/code/python/using-data-stores.mdx b/docs-v2/pages/workflows/building-workflows/code/python/using-data-stores.mdx index 240f78313d9fe..bab598ff78d24 100644 --- a/docs-v2/pages/workflows/building-workflows/code/python/using-data-stores.mdx +++ b/docs-v2/pages/workflows/building-workflows/code/python/using-data-stores.mdx @@ -2,7 +2,7 @@ import Callout from '@/components/Callout' # Using Data Stores -You can store and retrieve data from [Data stores](/workflows/data-management/data-stores/) in Python without connecting to a 3rd party database. +You can store and retrieve data from [Data Stores](/workflows/data-management/data-stores/) in Python without connecting to a 3rd party database. Add a data store as a input to a Python step, then access it in your Python `handler` with `pd.inputs["data_store"]`. @@ -42,6 +42,42 @@ def handler(pd: "pipedream"): data_store["last_ran_at"] = datetime.now().isoformat() ``` +### Setting expiration (TTL) for records + +You can set an expiration time for a record by passing a TTL (Time-To-Live) option as the third argument to the `set` method. The TTL value is specified in seconds: + +```python +def handler(pd: "pipedream"): + # Access the data store under the pd.inputs + data_store = pd.inputs["data_store"] + + # Store a temporary value that will expire after 1 hour (3600 seconds) + data_store.set("temporaryToken", "abc123", ttl=3600) + + # Store a value that will expire after 1 day + data_store.set("dailyMetric", 42, ttl=86400) +``` + +When the TTL period elapses, the record will be automatically deleted from the data store. + +### Updating TTL for existing records + +You can update the TTL for an existing record using the `set_ttl` method: + +```python +def handler(pd: "pipedream"): + # Access the data store under the pd.inputs + data_store = pd.inputs["data_store"] + + # Update an existing record to expire after 30 minutes + data_store.set_ttl("temporaryToken", ttl=1800) + + # Remove expiration from a record + data_store.set_ttl("temporaryToken", ttl=None) +``` + +This is useful for extending the lifetime of temporary data or removing expiration from records that should now be permanent. + ## Retrieving keys Fetch all the keys in a given data store using the `keys` method: @@ -285,6 +321,40 @@ def handler(pd: "pipedream"): return new_email ``` +## TTL use case: temporary caching and rate limiting + +TTL functionality is particularly useful for implementing temporary caching and rate limiting. Here's an example of a simple rate limiter that prevents a user from making more than 5 requests per hour: + +```python +def handler(pd: "pipedream"): + # Access the data store + data_store = pd.inputs["data_store"] + user_id = pd.steps["trigger"]["event"]["user_id"] + rate_key = f"ratelimit:{user_id}" + + # Try to get current rate limit counter + requests_num = data_store.get("rate_key") + + if not requests_num: + # First request from this user in the time window + data_store.set(rate_key, 1, ttl=3600) # Expire after 1 hour + return { "allowed": True, "remaining": 4 } + + if requests_num >= 5: + # Rate limit exceeded + return { "allowed": False, "error": "Rate limit exceeded", "retry_after": "1 hour" } + + # Increment the counter + data_store["rate_key"] = requests_num + 1 + return { "allowed": True, "remaining": 4 - requests_num } +``` + +This pattern can be extended for various temporary caching scenarios like: +- Session tokens with automatic expiration +- Short-lived feature flags +- Temporary access grants +- Time-based promotional codes + ### Supported data types Data stores can hold any JSON-serializable data within the storage limits. This includes data types including: diff --git a/docs-v2/pages/workflows/data-management/data-stores.mdx b/docs-v2/pages/workflows/data-management/data-stores.mdx index a6763e5710ae3..5152f2cdf7d4f 100644 --- a/docs-v2/pages/workflows/data-management/data-stores.mdx +++ b/docs-v2/pages/workflows/data-management/data-stores.mdx @@ -13,9 +13,10 @@ import VideoPlayer from "@/components/VideoPlayer"; Data stores are useful for: - Storing and retrieving data at a specific key +- Setting automatic expiration times for temporary data (TTL) - Counting or summing values over time - Retrieving JSON-serializable data across workflow executions -- Caching +- Caching and rate limiting - And any other case where you'd use a key-value store You can connect to the same data store across workflows, so they're also great for sharing state across different services. @@ -41,6 +42,7 @@ Configure the action: 1. **Select or create a Data Store** — create a new data store or choose an existing data store. 2. **Key** - the unique ID for this data that you'll use for lookup later 3. **Value** - The data to store at the specified `key` +4. **Time to Live (TTL)** - (Optional) The number of seconds until this record expires and is automatically deleted. Leave blank for records that should not expire. ![Configure the action](/images/data-stores/configuring-data-store-update-action.png) @@ -74,6 +76,23 @@ Configure the action: ![Get record action](/images/data-stores/configure-data-store-retrieve-record.png) +### Setting or updating record expiration (TTL) + +You can set automatic expiration times for records using the **Update record expiration** action: + +1. Add a new step to your workflow. +2. Search for the **Data Stores** app and select it. +3. Select the **Update record expiration** pre-built action. + +Configure the action: + +1. **Select a Data Store** - select the data store containing the record to modify +2. **Key** - the key for the record you want to update the expiration for +3. **Expiration Type** - choose from preset expiration times (1 hour, 1 day, 1 week, etc.) or select "Custom value" to enter a specific time in seconds +4. **Custom TTL (seconds)** - (only if "Custom value" is selected) enter the number of seconds until the record expires + +To remove expiration from a record, select "No expiration" as the expiration type. + ### Deleting Data To delete a single record from your data store, use the **Delete a single record** action in a step: