From 2cd57b6365d7a47eeab385cb72532f95e72531af Mon Sep 17 00:00:00 2001 From: Michael Lim Date: Wed, 23 Oct 2024 17:37:45 -0700 Subject: [PATCH 01/18] Adding evals for natural language workflow building. --- .../eval-test-suite-0-100-filtered.json | 877 ++++++++++++++++++ ...-test-suite-101-150-verified-filtered.json | 455 +++++++++ ...-test-suite-151-200-verified-filtered.json | 466 ++++++++++ ...l-test-suite-201-250-complex-filtered.json | 513 ++++++++++ .../eval-test-suite-251-300-new-filtered.json | 461 +++++++++ .../eval-test-suite-ai-focus-filtered.json | 214 +++++ ...st-suite-graduated-corrected-filtered.json | 187 ++++ ...l-test-suite-invalid-queries-filtered.json | 310 +++++++ ...val-test-suite-no-components-filtered.json | 276 ++++++ 9 files changed, 3759 insertions(+) create mode 100644 packages/evals/component_retrieval/eval-test-suite-0-100-filtered.json create mode 100644 packages/evals/component_retrieval/eval-test-suite-101-150-verified-filtered.json create mode 100644 packages/evals/component_retrieval/eval-test-suite-151-200-verified-filtered.json create mode 100644 packages/evals/component_retrieval/eval-test-suite-201-250-complex-filtered.json create mode 100644 packages/evals/component_retrieval/eval-test-suite-251-300-new-filtered.json create mode 100644 packages/evals/component_retrieval/eval-test-suite-ai-focus-filtered.json create mode 100644 packages/evals/component_retrieval/eval-test-suite-graduated-corrected-filtered.json create mode 100644 packages/evals/component_retrieval/eval-test-suite-invalid-queries-filtered.json create mode 100644 packages/evals/component_retrieval/eval-test-suite-no-components-filtered.json diff --git a/packages/evals/component_retrieval/eval-test-suite-0-100-filtered.json b/packages/evals/component_retrieval/eval-test-suite-0-100-filtered.json new file mode 100644 index 0000000000000..4e8b3f7f59ce3 --- /dev/null +++ b/packages/evals/component_retrieval/eval-test-suite-0-100-filtered.json @@ -0,0 +1,877 @@ +{ + "evaluationTests": [ + { + "query": "When a lead is created in Salesforce, create a task in Asana and send a message in Slack", + "triggers": [ + "salesforce_rest_api-new-record-instant" + ], + "actions": [ + "asana-create-task", + "slack-send-message" + ], + "persona": "complex-workflow" + }, + { + "query": "add new hubspot contacts to mailchimp", + "triggers": [ + "hubspot-new-contact" + ], + "actions": [ + "mailchimp-add-or-update-subscriber" + ], + "persona": "succinct" + }, + { + "query": "how do I get a phone number", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "when new emails arrive in Gmail, use OpenAI to summarize them and post to Slack", + "triggers": [ + "gmail-new-email" + ], + "actions": [ + "openai-chat", + "slack-send-message" + ], + "persona": "complex-workflow" + }, + { + "query": "watch airtable for new records", + "triggers": [ + "airtable_oauth-new-records" + ], + "actions": [], + "persona": "succinct" + }, + { + "query": "create jira ticket", + "triggers": [], + "actions": [ + "jira-create-issue" + ], + "persona": "succinct" + }, + { + "query": "When new Zendesk tickets arrive, analyze sentiment with OpenAI and prioritize in Linear", + "triggers": [ + "zendesk-new-ticket" + ], + "actions": [ + "openai-chat", + "linear-create-issue" + ], + "persona": "complex-workflow" + }, + { + "query": "how to make database connection", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "When someone mentions our company on Twitter, create a Zendesk ticket", + "triggers": [ + "twitter-new-mention-received-by-user" + ], + "actions": [ + "zendesk-create-ticket" + ], + "persona": "complex-workflow" + }, + { + "query": "use gpt to answer intercom messages", + "triggers": [ + "intercom-new-conversation" + ], + "actions": [ + "openai-chat" + ], + "persona": "task-oriented" + }, + { + "query": "I would like to automatically add my Calendly meetings to Google Calendar", + "triggers": [ + "calendly_v2-new-event-scheduled" + ], + "actions": [ + "google_calendar-create-event" + ], + "persona": "verbose" + }, + { + "query": "noshan page updated", + "triggers": [ + "notion-updated-page" + ], + "actions": [], + "persona": "sloppy" + }, + { + "query": "create new linear issue", + "triggers": [], + "actions": [ + "linear-create-issue" + ], + "persona": "succinct" + }, + { + "query": "When a customer submits a response in Typeform, send it through GPT for analysis and create a Notion page", + "triggers": [ + "typeform-new-submission" + ], + "actions": [ + "openai-chat", + "notion-create-page-from-database" + ], + "persona": "complex-workflow" + }, + { + "query": "new row in googel sheet", + "triggers": [ + "google_sheets-new-row-added" + ], + "actions": [], + "persona": "sloppy" + }, + { + "query": "create task in monday", + "triggers": [], + "actions": [ + "monday-create-item" + ], + "persona": "succinct" + }, + { + "query": "When a new deal is created in HubSpot, transcribe the discovery call with OpenAI and update the contact", + "triggers": [ + "hubspot-new-deal" + ], + "actions": [ + "openai-create-transcription", + "hubspot-create-or-update-contact" + ], + "persona": "complex-workflow" + }, + { + "query": "what is the weather today", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "send gong recordings to slack", + "triggers": [ + "gong-new-call" + ], + "actions": [ + "slack-send-message" + ], + "persona": "task-oriented" + }, + { + "query": "When a new customer is added in Stripe, create their account in Notion", + "triggers": [ + "stripe-new-customer" + ], + "actions": [ + "notion-create-page-from-database" + ], + "persona": "complex-workflow" + }, + { + "query": "update asana task status", + "triggers": [], + "actions": [ + "asana-update-task" + ], + "persona": "succinct" + }, + { + "query": "When Sentry reports an error, create an issue in Linear", + "triggers": [ + "sentry-issue-event" + ], + "actions": [ + "linear-create-issue" + ], + "persona": "verbose" + }, + { + "query": "convert file to pdf", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "When a form is submitted on Webflow, create contact in HubSpot", + "triggers": [ + "webflow-new-form-submission" + ], + "actions": [ + "hubspot-create-or-update-contact" + ], + "persona": "complex-workflow" + }, + { + "query": "watch 4 new stripe payments", + "triggers": [ + "stripe-new-payment" + ], + "actions": [], + "persona": "sloppy" + }, + { + "query": "When support tickets come in after hours, use GPT to draft a response and send it through Gmail", + "triggers": [ + "zendesk-new-ticket" + ], + "actions": [ + "openai-chat", + "gmail-send-email" + ], + "persona": "complex-workflow" + }, + { + "query": "When customer feedback arrives in Delighted, analyze sentiment with AI and update Salesforce", + "triggers": [], + "actions": [ + "openai-chat", + "salesforce_rest_api-update-contact" + ], + "persona": "complex-workflow" + }, + { + "query": "upload file to dropbox", + "triggers": [], + "actions": [ + "dropbox-upload-file" + ], + "persona": "succinct" + }, + { + "query": "I need new Linear issues to be automatically added to our Notion database", + "triggers": [ + "linear-issue-created-instant" + ], + "actions": [ + "notion-create-page-from-database" + ], + "persona": "verbose" + }, + { + "query": "why is my code not working", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "When an opportunity changes in Salesforce, update the record in HubSpot", + "triggers": [], + "actions": [ + "hubspot-create-or-update-contact" + ], + "persona": "complex-workflow" + }, + { + "query": "create new document in Google Docs", + "triggers": [], + "actions": [ + "google_docs-create-document" + ], + "persona": "succinct" + }, + { + "query": "update zendesk ticket", + "triggers": [], + "actions": [ + "zendesk-update-ticket" + ], + "persona": "succinct" + }, + { + "query": "analyze customer support tickets with GPT", + "triggers": [], + "actions": [ + "openai-chat" + ], + "persona": "task-oriented" + }, + { + "query": "When a deal closes in Pipedrive, generate an invoice in QuickBooks", + "triggers": [ + "pipedrive-updated-deal" + ], + "actions": [ + "quickbooks-create-invoice" + ], + "persona": "complex-workflow" + }, + { + "query": "add contact to salesforce", + "triggers": [], + "actions": [ + "salesforce_rest_api-create-contact" + ], + "persona": "succinct" + }, + { + "query": "When a customer replies to our survey, use AI to analyze it and update their profile in HubSpot", + "triggers": [ + "survey_monkey-new-survey-response" + ], + "actions": [ + "openai-chat", + "hubspot-create-or-update-contact" + ], + "persona": "complex-workflow" + }, + { + "query": "cant find my data", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "send Linear issues to Slack", + "triggers": [ + "linear-issue-created-instant" + ], + "actions": [ + "slack-send-message" + ], + "persona": "verbose" + }, + { + "query": "create new noshun page", + "triggers": [], + "actions": [ + "notion-create-page" + ], + "persona": "sloppy" + }, + { + "query": "When a customer signs up, add them to Mailchimp", + "triggers": [], + "actions": [ + "mailchimp-add-or-update-subscriber" + ], + "persona": "task-oriented" + }, + { + "query": "create basecamp project", + "triggers": [], + "actions": [], + "persona": "succinct" + }, + { + "query": "translate new Notion pages with GPT", + "triggers": [ + "notion-new-page" + ], + "actions": [ + "openai-chat" + ], + "persona": "task-oriented" + }, + { + "query": "When a critical error occurs in Sentry, create PagerDuty incident", + "triggers": [ + "sentry-issue-event" + ], + "actions": [ + "pagerduty-trigger-incident" + ], + "persona": "complex-workflow" + }, + { + "query": "add row 2 spreadsheet", + "triggers": [], + "actions": [ + "google_sheets-add-single-row" + ], + "persona": "sloppy" + }, + { + "query": "I would like to be notified in Slack when new issues are created in Linear", + "triggers": [ + "linear-issue-created-instant" + ], + "actions": [ + "slack-send-message" + ], + "persona": "verbose" + }, + { + "query": "create task in asna", + "triggers": [], + "actions": [ + "asana-create-task" + ], + "persona": "sloppy" + }, + { + "query": "When a large transaction occurs in Stripe, notify me in Slack", + "triggers": [ + "stripe-new-payment" + ], + "actions": [ + "slack-send-message" + ], + "persona": "task-oriented" + }, + { + "query": "post message in discord", + "triggers": [], + "actions": [ + "discord-send-message" + ], + "persona": "succinct" + }, + { + "query": "When someone fills out our Typeform, use GPT to qualify the lead and add to HubSpot", + "triggers": [ + "typeform-new-submission" + ], + "actions": [ + "openai-chat", + "hubspot-create-or-update-contact" + ], + "persona": "complex-workflow" + }, + { + "query": "how to center text", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "create new page in Confluence", + "triggers": [], + "actions": [ + "onenote-create-page" + ], + "persona": "succinct" + }, + { + "query": "When a meeting ends in Zoom, transcribe it with OpenAI and save to Notion", + "triggers": [ + "zoom-meeting-ended" + ], + "actions": [ + "openai-create-transcription", + "notion-create-page-from-database" + ], + "persona": "complex-workflow" + }, + { + "query": "update clickup task", + "triggers": [], + "actions": [ + "clickup-update-task" + ], + "persona": "succinct" + }, + { + "query": "I need all my Gmail attachments to be analyzed by GPT", + "triggers": [ + "gmail-new-email" + ], + "actions": [ + "openai-chat" + ], + "persona": "verbose" + }, + { + "query": "create quickboks invoice", + "triggers": [], + "actions": [ + "quickbooks-create-invoice" + ], + "persona": "sloppy" + }, + { + "query": "When a deal updates in Close.com, sync it to HubSpot", + "triggers": [], + "actions": [ + "hubspot-create-or-update-contact" + ], + "persona": "task-oriented" + }, + { + "query": "When a sales call is recorded in Gong, transcribe with OpenAI and update the deal in Salesforce", + "triggers": [ + "gong-new-call" + ], + "actions": [ + "openai-create-transcription", + "salesforce_rest_api-update-opportunity" + ], + "persona": "complex-workflow" + }, + { + "query": "generate email content with GPT", + "triggers": [], + "actions": [ + "openai-chat" + ], + "persona": "succinct" + }, + { + "query": "I would like all new Airtable records to be enriched using OpenAI", + "triggers": [ + "airtable_oauth-new-records" + ], + "actions": [ + "openai-chat" + ], + "persona": "verbose" + }, + { + "query": "make chatgpt write emails", + "triggers": [], + "actions": [ + "openai-chat" + ], + "persona": "sloppy" + }, + { + "query": "When customers respond in Help Scout, analyze sentiment with GPT and update their status in HubSpot", + "triggers": [], + "actions": [ + "openai-chat", + "hubspot-create-or-update-contact" + ], + "persona": "complex-workflow" + }, + { + "query": "sync my google drive", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "create new event in google calendar", + "triggers": [], + "actions": [ + "google_calendar-create-event" + ], + "persona": "succinct" + }, + { + "query": "When a customer's subscription changes in Chargebee, update their record in Salesforce", + "triggers": [], + "actions": [ + "salesforce_rest_api-update-contact" + ], + "persona": "complex-workflow" + }, + { + "query": "post 2 twiter", + "triggers": [], + "actions": [ + "twitter-create-tweet" + ], + "persona": "sloppy" + }, + { + "query": "I need all Salesforce opportunities to be automatically added to our Monday.com board", + "triggers": [ + "salesforce_rest_api-new-record-instant" + ], + "actions": [ + "monday-create-item" + ], + "persona": "verbose" + }, + { + "query": "When new issues are created in GitHub, use GPT to analyze severity and create PagerDuty incidents", + "triggers": [], + "actions": [ + "openai-chat", + "pagerduty-trigger-incident" + ], + "persona": "complex-workflow" + }, + { + "query": "make new asana project", + "triggers": [], + "actions": [ + "asana-create-project" + ], + "persona": "succinct" + }, + { + "query": "When customer reviews come in through Trustpilot, analyze them with AI and notify in Slack", + "triggers": [], + "actions": [ + "openai-chat", + "slack-send-message" + ], + "persona": "complex-workflow" + }, + { + "query": "upload files to dropbox", + "triggers": [], + "actions": [ + "dropbox-upload-file" + ], + "persona": "succinct" + }, + { + "query": "where is my data stored", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "send mailchmp campaign", + "triggers": [], + "actions": [ + "mailchimp-create-campaign" + ], + "persona": "sloppy" + }, + { + "query": "I want new Shopify orders to be automatically processed in NetSuite", + "triggers": [ + "shopify_developer_app-new-order-created" + ], + "actions": [], + "persona": "verbose" + }, + { + "query": "When a document is uploaded to Google Drive, use GPT to summarize it and post in Slack", + "triggers": [ + "google_drive-new-files-instant" + ], + "actions": [ + "openai-chat", + "slack-send-message" + ], + "persona": "complex-workflow" + }, + { + "query": "create linear ticket", + "triggers": [], + "actions": [ + "linear-create-issue" + ], + "persona": "succinct" + }, + { + "query": "When a user submits feedback in Canny, analyze it with AI and create tasks in ClickUp", + "triggers": [], + "actions": [ + "openai-chat", + "clickup-create-task" + ], + "persona": "complex-workflow" + }, + { + "query": "send invoice via xero", + "triggers": [], + "actions": [ + "xero_accounting_api-email-an-invoice" + ], + "persona": "task-oriented" + }, + { + "query": "hubspt deal updated", + "triggers": [ + "hubspot-deal-updated" + ], + "actions": [], + "persona": "sloppy" + }, + { + "query": "I would like to automatically generate reports when new data appears in Amplitude", + "triggers": [], + "actions": [ + "google_sheets-add-single-row" + ], + "persona": "verbose" + }, + { + "query": "When a new video is uploaded to Vimeo, transcribe it with OpenAI and create a Notion page", + "triggers": [ + "vimeo-new-video-of-mine" + ], + "actions": [ + "openai-create-transcription", + "notion-create-page-from-database" + ], + "persona": "complex-workflow" + }, + { + "query": "analyze text with gpt", + "triggers": [], + "actions": [ + "openai-chat" + ], + "persona": "succinct" + }, + { + "query": "When a trial user signs up in Stripe, add them to HubSpot and start onboarding sequence in Customer.io", + "triggers": [ + "stripe-new-customer" + ], + "actions": [ + "hubspot-create-or-update-contact" + ], + "persona": "complex-workflow" + }, + { + "query": "create new folder in gdrive", + "triggers": [], + "actions": [ + "google_drive-create-folder" + ], + "persona": "succinct" + }, + { + "query": "update record in airetable", + "triggers": [], + "actions": [ + "airtable_oauth-update-record" + ], + "persona": "sloppy" + }, + { + "query": "help me configure postgres", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "When an event starts in Google Calendar, send a message in Telegram", + "triggers": [ + "google_calendar-upcoming-event-alert" + ], + "actions": [ + "telegram_bot_api-send-text-message-or-reply" + ], + "persona": "complex-workflow" + }, + { + "query": "create task in todoist", + "triggers": [], + "actions": [ + "todoist-create-task" + ], + "persona": "succinct" + }, + { + "query": "I want to automatically catalog all my Loom videos in Notion", + "triggers": [], + "actions": [ + "notion-create-page-from-database" + ], + "persona": "verbose" + }, + { + "query": "When a form is submitted in Reform, analyze responses with GPT and update Airtable", + "triggers": [], + "actions": [ + "openai-chat", + "airtable_oauth-update-record" + ], + "persona": "complex-workflow" + }, + { + "query": "make new trello board", + "triggers": [], + "actions": [], + "persona": "succinct" + }, + { + "query": "sync contacts between platforms", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "When a product is updated in Productboard, notify stakeholders in Slack", + "triggers": [ + "productboard-new-feature" + ], + "actions": [ + "slack-send-message" + ], + "persona": "task-oriented" + }, + { + "query": "genrate presentation with ai", + "triggers": [], + "actions": [ + "openai-chat" + ], + "persona": "sloppy" + }, + { + "query": "I need all customer conversations from Front to be analyzed by GPT", + "triggers": [ + "frontapp-new-conversation-state-change" + ], + "actions": [ + "openai-chat" + ], + "persona": "verbose" + }, + { + "query": "When someone stars our GitHub repo, send a thank you email with GPT", + "triggers": [], + "actions": [ + "openai-chat", + "gmail-send-email" + ], + "persona": "complex-workflow" + }, + { + "query": "create new card trelo", + "triggers": [], + "actions": [ + "trello-create-card" + ], + "persona": "sloppy" + }, + { + "query": "track time in toggl", + "triggers": [], + "actions": [], + "persona": "succinct" + }, + { + "query": "I would like to be notified in Slack when new videos are posted to YouTube", + "triggers": [ + "youtube_data_api-new-videos-in-channel" + ], + "actions": [ + "slack-send-message" + ], + "persona": "verbose" + }, + { + "query": "When a customer reaches a usage threshold in Stripe, notify the success team in Slack", + "triggers": [ + "stripe-subscription-updated" + ], + "actions": [ + "slack-send-message" + ], + "persona": "task-oriented" + }, + { + "query": "forward emails to telegram", + "triggers": [ + "gmail-new-email" + ], + "actions": [ + "telegram_bot_api-send-text-message-or-reply" + ], + "persona": "task-oriented" + } + ] +} \ No newline at end of file diff --git a/packages/evals/component_retrieval/eval-test-suite-101-150-verified-filtered.json b/packages/evals/component_retrieval/eval-test-suite-101-150-verified-filtered.json new file mode 100644 index 0000000000000..3f55d7f14c602 --- /dev/null +++ b/packages/evals/component_retrieval/eval-test-suite-101-150-verified-filtered.json @@ -0,0 +1,455 @@ +{ + "evaluationTests": [ + { + "query": "When a deal closes in Salesforce, create invoice in QuickBooks", + "triggers": [], + "actions": [ + "quickbooks-create-invoice" + ], + "persona": "complex-workflow" + }, + { + "query": "add new subscriber to mailchimp", + "triggers": [], + "actions": [ + "mailchimp-add-subscriber-to-tag" + ], + "persona": "succinct" + }, + { + "query": "I would like GPT to analyze new Typeform responses", + "triggers": [ + "typeform-new-submission" + ], + "actions": [ + "openai-chat" + ], + "persona": "verbose" + }, + { + "query": "When a new row appears in Google Sheets, create a Notion page", + "triggers": [ + "google_sheets-new-row-added" + ], + "actions": [ + "notion-create-page-from-database" + ], + "persona": "complex-workflow" + }, + { + "query": "salesfrce contact update", + "triggers": [], + "actions": [ + "salesforce_rest_api-update-contact" + ], + "persona": "sloppy" + }, + { + "query": "create task in linear", + "triggers": [], + "actions": [ + "linear-create-issue" + ], + "persona": "succinct" + }, + { + "query": "When new Jira issues are created, post them to Slack", + "triggers": [ + "jira-issue-created" + ], + "actions": [ + "slack-send-message" + ], + "persona": "complex-workflow" + }, + { + "query": "create trello card", + "triggers": [], + "actions": [ + "trello-create-card" + ], + "persona": "succinct" + }, + { + "query": "I need all new Calendly bookings to be added to Google Calendar", + "triggers": [ + "calendly_v2-new-event-scheduled" + ], + "actions": [ + "google_calendar-create-event" + ], + "persona": "verbose" + }, + { + "query": "update deal in hubspt", + "triggers": [], + "actions": [ + "hubspot-create-or-update-contact" + ], + "persona": "sloppy" + }, + { + "query": "When a video is uploaded to YouTube, transcribe it with OpenAI", + "triggers": [ + "youtube_data_api-new-videos" + ], + "actions": [ + "openai-create-transcription" + ], + "persona": "complex-workflow" + }, + { + "query": "send gmail message", + "triggers": [], + "actions": [ + "gmail-send-email" + ], + "persona": "succinct" + }, + { + "query": "When files are added to Dropbox, create a Notion page", + "triggers": [ + "dropbox-new-file" + ], + "actions": [ + "notion-create-page-from-database" + ], + "persona": "complex-workflow" + }, + { + "query": "use gpt for email", + "triggers": [], + "actions": [ + "openai-chat" + ], + "persona": "succinct" + }, + { + "query": "I would like new Twitter mentions to create Zendesk tickets", + "triggers": [ + "twitter-new-mention-received-by-user" + ], + "actions": [ + "zendesk-create-ticket" + ], + "persona": "verbose" + }, + { + "query": "When a new form submission happens in Typeform, add to Airtable", + "triggers": [ + "typeform-new-submission" + ], + "actions": [ + "airtable_oauth-create-single-record" + ], + "persona": "complex-workflow" + }, + { + "query": "add row 2 sheets", + "triggers": [], + "actions": [ + "google_sheets-add-single-row" + ], + "persona": "sloppy" + }, + { + "query": "create discord message", + "triggers": [], + "actions": [ + "discord-send-message" + ], + "persona": "succinct" + }, + { + "query": "When new records appear in Airtable, send them to Slack", + "triggers": [ + "airtable_oauth-new-records" + ], + "actions": [ + "slack-send-message" + ], + "persona": "complex-workflow" + }, + { + "query": "update cell in googel sheets", + "triggers": [], + "actions": [ + "google_sheets-update-cell" + ], + "persona": "sloppy" + }, + { + "query": "When a payment happens in Stripe, update HubSpot contact", + "triggers": [ + "stripe-new-payment" + ], + "actions": [ + "hubspot-create-or-update-contact" + ], + "persona": "complex-workflow" + }, + { + "query": "make asana task", + "triggers": [], + "actions": [ + "asana-create-task" + ], + "persona": "sloppy" + }, + { + "query": "I want GPT to analyze my Gmail messages", + "triggers": [ + "gmail-new-email" + ], + "actions": [ + "openai-chat" + ], + "persona": "verbose" + }, + { + "query": "When new issues are created in Linear, post to Discord", + "triggers": [ + "linear-issue-created-instant" + ], + "actions": [ + "discord-send-message" + ], + "persona": "complex-workflow" + }, + { + "query": "create notion database", + "triggers": [], + "actions": [ + "notion-create-page" + ], + "persona": "succinct" + }, + { + "query": "When GitHub issues are created, create Linear tickets", + "triggers": [], + "actions": [ + "linear-create-issue" + ], + "persona": "complex-workflow" + }, + { + "query": "make fresh ticket", + "triggers": [], + "actions": [ + "freshdesk-create-ticket" + ], + "persona": "sloppy" + }, + { + "query": "I would like to send Telegram messages when Jira tickets update", + "triggers": [ + "jira-issue-updated" + ], + "actions": [ + "telegram_bot_api-send-text-message-or-reply" + ], + "persona": "verbose" + }, + { + "query": "When deals change stage in HubSpot, update records in Salesforce", + "triggers": [ + "hubspot-deal-updated" + ], + "actions": [ + "salesforce_rest_api-update-contact" + ], + "persona": "complex-workflow" + }, + { + "query": "create new doc", + "triggers": [], + "actions": [ + "google_docs-create-document" + ], + "persona": "succinct" + }, + { + "query": "send telgram msg", + "triggers": [], + "actions": [ + "telegram_bot_api-send-text-message-or-reply" + ], + "persona": "sloppy" + }, + { + "query": "When RSS feed updates, post to Discord", + "triggers": [ + "rss-new-item-in-feed" + ], + "actions": [ + "discord-send-message" + ], + "persona": "task-oriented" + }, + { + "query": "I need form submissions from JotForm in Google Sheets", + "triggers": [ + "jotform-new-submission" + ], + "actions": [ + "google_sheets-add-single-row" + ], + "persona": "verbose" + }, + { + "query": "When calendar events start in Google Calendar, send Slack message", + "triggers": [ + "google_calendar-upcoming-event-alert" + ], + "actions": [ + "slack-send-message" + ], + "persona": "complex-workflow" + }, + { + "query": "update zendesk ticket", + "triggers": [], + "actions": [ + "zendesk-update-ticket" + ], + "persona": "succinct" + }, + { + "query": "send campayn email", + "triggers": [], + "actions": [ + "gmail-send-email" + ], + "persona": "sloppy" + }, + { + "query": "When tasks are completed in Todoist, update Trello", + "triggers": [ + "todoist-completed-task" + ], + "actions": [], + "persona": "complex-workflow" + }, + { + "query": "add row to airtble", + "triggers": [], + "actions": [ + "airtable_oauth-create-single-record" + ], + "persona": "sloppy" + }, + { + "query": "I want new Slack messages to be analyzed by GPT", + "triggers": [ + "slack-new-message-in-channels" + ], + "actions": [ + "openai-chat" + ], + "persona": "verbose" + }, + { + "query": "When new files are added to Google Drive, create Notion pages", + "triggers": [ + "google_drive-new-files-instant" + ], + "actions": [ + "notion-create-page-from-database" + ], + "persona": "complex-workflow" + }, + { + "query": "create monday item", + "triggers": [], + "actions": [ + "monday-create-item" + ], + "persona": "succinct" + }, + { + "query": "When email attachments arrive in Gmail, use OpenAI to analyze them", + "triggers": [ + "gmail-new-email" + ], + "actions": [ + "openai-chat" + ], + "persona": "complex-workflow" + }, + { + "query": "add subscriber mailchmp", + "triggers": [], + "actions": [ + "mailchimp-add-subscriber-to-tag" + ], + "persona": "sloppy" + }, + { + "query": "I want to be notified in Telegram about new Twitter mentions", + "triggers": [ + "twitter-new-mention-received-by-user" + ], + "actions": [ + "telegram_bot_api-send-text-message-or-reply" + ], + "persona": "verbose" + }, + { + "query": "When Zoom meetings end, send recordings to Slack", + "triggers": [ + "zoom-recording-completed" + ], + "actions": [ + "slack-send-message" + ], + "persona": "complex-workflow" + }, + { + "query": "make noshan page", + "triggers": [], + "actions": [ + "notion-create-page" + ], + "persona": "sloppy" + }, + { + "query": "When new comments appear in Discord, use GPT to moderate them", + "triggers": [ + "discord-new-message" + ], + "actions": [ + "openai-chat" + ], + "persona": "complex-workflow" + }, + { + "query": "create gsheet row", + "triggers": [], + "actions": [ + "google_sheets-add-single-row" + ], + "persona": "succinct" + }, + { + "query": "I need Typeform responses to be analyzed by GPT and stored in Airtable", + "triggers": [ + "typeform-new-submission" + ], + "actions": [ + "openai-chat", + "airtable_oauth-create-single-record" + ], + "persona": "verbose" + }, + { + "query": "When new starred messages in Slack, create tasks in Linear", + "triggers": [ + "slack-new-reaction-added" + ], + "actions": [ + "linear-create-issue" + ], + "persona": "complex-workflow" + } + ] +} \ No newline at end of file diff --git a/packages/evals/component_retrieval/eval-test-suite-151-200-verified-filtered.json b/packages/evals/component_retrieval/eval-test-suite-151-200-verified-filtered.json new file mode 100644 index 0000000000000..9cae5a27b27c1 --- /dev/null +++ b/packages/evals/component_retrieval/eval-test-suite-151-200-verified-filtered.json @@ -0,0 +1,466 @@ +{ + "evaluationTests": [ + { + "query": "When a row is added to Google Sheets in a shared drive, create a Linear ticket", + "triggers": [ + "google_sheets-new-row-added" + ], + "actions": [ + "linear-create-issue" + ], + "persona": "complex-workflow" + }, + { + "query": "update airtable record", + "triggers": [], + "actions": [ + "airtable_oauth-update-record" + ], + "persona": "succinct" + }, + { + "query": "I would like to forward all my Gmail attachments to Telegram", + "triggers": [ + "gmail-new-email" + ], + "actions": [ + "telegram_bot_api-send-text-message-or-reply" + ], + "persona": "verbose" + }, + { + "query": "send msg in slck", + "triggers": [], + "actions": [ + "slack-send-message" + ], + "persona": "sloppy" + }, + { + "query": "When a message is posted to Discord, analyze with OpenAI", + "triggers": [ + "discord-new-message" + ], + "actions": [ + "openai-chat" + ], + "persona": "complex-workflow" + }, + { + "query": "add new google calendar event", + "triggers": [], + "actions": [ + "google_calendar-create-event" + ], + "persona": "succinct" + }, + { + "query": "When a new tweet mentions our company, send to Slack channel", + "triggers": [ + "twitter-new-mention-received-by-user" + ], + "actions": [ + "slack-send-message" + ], + "persona": "task-oriented" + }, + { + "query": "create nw spreadsheet", + "triggers": [], + "actions": [ + "google_sheets-create-spreadsheet" + ], + "persona": "sloppy" + }, + { + "query": "I want my Typeform submissions to be saved in Notion databases", + "triggers": [ + "typeform-new-submission" + ], + "actions": [ + "notion-create-page-from-database" + ], + "persona": "verbose" + }, + { + "query": "When a Notion page updates, send a message to Telegram", + "triggers": [ + "notion-updated-page" + ], + "actions": [ + "telegram_bot_api-send-text-message-or-reply" + ], + "persona": "complex-workflow" + }, + { + "query": "post 2 discord", + "triggers": [], + "actions": [ + "discord-send-message" + ], + "persona": "sloppy" + }, + { + "query": "When files are uploaded to Dropbox, copy them to Google Drive", + "triggers": [ + "dropbox-new-file" + ], + "actions": [ + "google_drive-upload-file" + ], + "persona": "complex-workflow" + }, + { + "query": "create trello board", + "triggers": [], + "actions": [], + "persona": "succinct" + }, + { + "query": "I need new RSS items to be posted to Discord", + "triggers": [ + "rss-new-item-in-feed" + ], + "actions": [ + "discord-send-message" + ], + "persona": "verbose" + }, + { + "query": "When a form is submitted in JotForm, use GPT to analyze it", + "triggers": [ + "jotform-new-submission" + ], + "actions": [ + "openai-chat" + ], + "persona": "complex-workflow" + }, + { + "query": "make linear issue", + "triggers": [], + "actions": [ + "linear-create-issue" + ], + "persona": "succinct" + }, + { + "query": "add row 2 googlesheets", + "triggers": [], + "actions": [ + "google_sheets-add-single-row" + ], + "persona": "sloppy" + }, + { + "query": "When a YouTube video is posted, share it on Twitter", + "triggers": [ + "youtube_data_api-new-videos" + ], + "actions": [ + "twitter-create-tweet" + ], + "persona": "complex-workflow" + }, + { + "query": "forward email to telegram", + "triggers": [ + "gmail-new-email" + ], + "actions": [ + "telegram_bot_api-send-text-message-or-reply" + ], + "persona": "task-oriented" + }, + { + "query": "When new Slack messages contain specific keywords, create Linear issues", + "triggers": [ + "slack-new-message-in-channels" + ], + "actions": [ + "linear-create-issue" + ], + "persona": "complex-workflow" + }, + { + "query": "updaet notion page", + "triggers": [], + "actions": [ + "notion-update-page" + ], + "persona": "sloppy" + }, + { + "query": "create google doc", + "triggers": [], + "actions": [ + "google_docs-create-document" + ], + "persona": "succinct" + }, + { + "query": "I would like my Gmail messages to be summarized by GPT", + "triggers": [ + "gmail-new-email" + ], + "actions": [ + "openai-chat" + ], + "persona": "verbose" + }, + { + "query": "When calendar events are updated, post to Slack", + "triggers": [ + "google_calendar-new-or-updated-event-instant" + ], + "actions": [ + "slack-send-message" + ], + "persona": "complex-workflow" + }, + { + "query": "download gdrive file", + "triggers": [], + "actions": [ + "google_drive-download-file" + ], + "persona": "succinct" + }, + { + "query": "When a new page is created in Notion, use GPT to summarize it", + "triggers": [ + "notion-new-page" + ], + "actions": [ + "openai-chat" + ], + "persona": "complex-workflow" + }, + { + "query": "send telagrem message", + "triggers": [], + "actions": [ + "telegram_bot_api-send-text-message-or-reply" + ], + "persona": "sloppy" + }, + { + "query": "I want to track my liked tweets in a spreadsheet", + "triggers": [ + "twitter-my-tweets" + ], + "actions": [ + "google_sheets-add-single-row" + ], + "persona": "verbose" + }, + { + "query": "When files are modified in Google Drive, update Notion", + "triggers": [ + "google_drive-new-or-modified-files" + ], + "actions": [ + "notion-update-page" + ], + "persona": "complex-workflow" + }, + { + "query": "analyze text gpt", + "triggers": [], + "actions": [ + "openai-chat" + ], + "persona": "succinct" + }, + { + "query": "update row in sheets", + "triggers": [], + "actions": [ + "google_sheets-update-row" + ], + "persona": "task-oriented" + }, + { + "query": "When Zoom recordings complete, transcribe with OpenAI", + "triggers": [ + "zoom-recording-completed" + ], + "actions": [ + "openai-create-transcription" + ], + "persona": "complex-workflow" + }, + { + "query": "make asana projct", + "triggers": [], + "actions": [ + "asana-create-project" + ], + "persona": "sloppy" + }, + { + "query": "I would like to receive Discord messages in Telegram", + "triggers": [ + "discord-new-message" + ], + "actions": [ + "telegram_bot_api-send-text-message-or-reply" + ], + "persona": "verbose" + }, + { + "query": "When Slack reactions are added, create Trello cards", + "triggers": [ + "slack-new-reaction-added" + ], + "actions": [ + "trello-create-card" + ], + "persona": "complex-workflow" + }, + { + "query": "download dropbox file", + "triggers": [], + "actions": [ + "dropbox-download-file-to-tmp" + ], + "persona": "succinct" + }, + { + "query": "get cell from sheet", + "triggers": [], + "actions": [ + "google_sheets-get-cell" + ], + "persona": "task-oriented" + }, + { + "query": "When new Airtable records are created, use GPT to classify them", + "triggers": [ + "airtable_oauth-new-records" + ], + "actions": [ + "openai-chat" + ], + "persona": "complex-workflow" + }, + { + "query": "reply 2 slack thread", + "triggers": [], + "actions": [ + "slack-reply-to-a-message" + ], + "persona": "sloppy" + }, + { + "query": "I need new Twitter mentions to create Linear issues", + "triggers": [ + "twitter-new-mention-received-by-user" + ], + "actions": [ + "linear-create-issue" + ], + "persona": "verbose" + }, + { + "query": "When files change in shared Google Drive, notify in Slack", + "triggers": [ + "google_drive-changes-to-specific-files-shared-drive" + ], + "actions": [ + "slack-send-message" + ], + "persona": "complex-workflow" + }, + { + "query": "create gmail draft", + "triggers": [], + "actions": [ + "gmail-create-draft" + ], + "persona": "succinct" + }, + { + "query": "move file in gdrive", + "triggers": [], + "actions": [ + "google_drive-move-file" + ], + "persona": "sloppy" + }, + { + "query": "When new Direct Messages arrive in Slack, forward to email", + "triggers": [ + "slack-new-direct-message" + ], + "actions": [ + "gmail-send-email" + ], + "persona": "complex-workflow" + }, + { + "query": "I want my Google Calendar events to show in Slack", + "triggers": [ + "google_calendar-new-event" + ], + "actions": [ + "slack-send-message" + ], + "persona": "verbose" + }, + { + "query": "When Discord reactions are added, update Notion", + "triggers": [ + "discord-reaction-added" + ], + "actions": [ + "notion-update-page" + ], + "persona": "complex-workflow" + }, + { + "query": "post direct msg slack", + "triggers": [], + "actions": [ + "slack-send-message" + ], + "persona": "sloppy" + }, + { + "query": "create new worksheet", + "triggers": [], + "actions": [ + "google_sheets-create-worksheet" + ], + "persona": "succinct" + }, + { + "query": "When RSS feed updates, let GPT analyze the content", + "triggers": [ + "rss-new-item-in-feed" + ], + "actions": [ + "openai-chat" + ], + "persona": "complex-workflow" + }, + { + "query": "send message to discord channel", + "triggers": [], + "actions": [ + "discord-send-message" + ], + "persona": "task-oriented" + }, + { + "query": "When worksheets are added to Google Sheets, create Notion pages", + "triggers": [ + "google_sheets-new-worksheet" + ], + "actions": [ + "notion-create-page-from-database" + ], + "persona": "complex-workflow" + } + ] +} \ No newline at end of file diff --git a/packages/evals/component_retrieval/eval-test-suite-201-250-complex-filtered.json b/packages/evals/component_retrieval/eval-test-suite-201-250-complex-filtered.json new file mode 100644 index 0000000000000..11ea6994d42d1 --- /dev/null +++ b/packages/evals/component_retrieval/eval-test-suite-201-250-complex-filtered.json @@ -0,0 +1,513 @@ +{ + "evaluationTests": [ + { + "query": "When new Gmail attachments arrive, use GPT to analyze them and create Linear issues for action items", + "triggers": [ + "gmail-new-email" + ], + "actions": [ + "openai-chat", + "linear-create-issue" + ], + "persona": "complex-workflow" + }, + { + "query": "translate notion pages with gpt", + "triggers": [ + "notion-new-page" + ], + "actions": [ + "openai-chat" + ], + "persona": "succinct" + }, + { + "query": "I need all Typeform submissions to be analyzed by GPT and then posted to both Slack and Notion", + "triggers": [ + "typeform-new-submission" + ], + "actions": [ + "openai-chat", + "slack-send-message", + "notion-create-page-from-database" + ], + "persona": "verbose" + }, + { + "query": "When Discord messages mention specific keywords, use GPT to analyze sentiment and post to Slack threads", + "triggers": [ + "discord-new-message" + ], + "actions": [ + "openai-chat", + "slack-reply-to-a-message" + ], + "persona": "complex-workflow" + }, + { + "query": "send msg 2 multiple slack channls", + "triggers": [], + "actions": [ + "slack-send-message" + ], + "persona": "sloppy" + }, + { + "query": "When RSS feeds have new items, use GPT to summarize and post summaries to Telegram", + "triggers": [ + "rss-new-item-in-feed" + ], + "actions": [ + "openai-chat", + "telegram_bot_api-send-text-message-or-reply" + ], + "persona": "complex-workflow" + }, + { + "query": "analyze zoom recordings with gpt", + "triggers": [ + "zoom-recording-completed" + ], + "actions": [ + "openai-chat" + ], + "persona": "task-oriented" + }, + { + "query": "I would like new Twitter mentions to be analyzed by GPT and important ones saved to Notion", + "triggers": [ + "twitter-new-mention-received-by-user" + ], + "actions": [ + "openai-chat", + "notion-create-page-from-database" + ], + "persona": "verbose" + }, + { + "query": "When Google Drive files are modified, update Sheets log and notify in Slack", + "triggers": [ + "google_drive-new-or-modified-files" + ], + "actions": [ + "google_sheets-add-single-row", + "slack-send-message" + ], + "persona": "complex-workflow" + }, + { + "query": "foward emails to slck", + "triggers": [ + "gmail-new-email" + ], + "actions": [ + "slack-send-message" + ], + "persona": "sloppy" + }, + { + "query": "When new Airtable records appear, let GPT analyze them and create Linear issues", + "triggers": [ + "airtable_oauth-new-records" + ], + "actions": [ + "openai-chat", + "linear-create-issue" + ], + "persona": "complex-workflow" + }, + { + "query": "move files between drives", + "triggers": [], + "actions": [ + "google_drive-move-file" + ], + "persona": "succinct" + }, + { + "query": "I need YouTube video transcriptions analyzed by GPT and saved to Notion", + "triggers": [ + "youtube_data_api-new-videos" + ], + "actions": [ + "openai-create-transcription", + "openai-chat", + "notion-create-page-from-database" + ], + "persona": "verbose" + }, + { + "query": "When calendar events are created, use GPT to generate agendas and send via Gmail", + "triggers": [ + "google_calendar-new-event" + ], + "actions": [ + "openai-chat", + "gmail-send-email" + ], + "persona": "complex-workflow" + }, + { + "query": "post message in discord thread", + "triggers": [], + "actions": [ + "discord-send-message" + ], + "persona": "task-oriented" + }, + { + "query": "When Slack messages are starred, transcribe any attachments and create Linear tickets", + "triggers": [ + "slack-new-reaction-added" + ], + "actions": [ + "openai-create-transcription", + "linear-create-issue" + ], + "persona": "complex-workflow" + }, + { + "query": "analyize email with gpt", + "triggers": [ + "gmail-new-email" + ], + "actions": [ + "openai-chat" + ], + "persona": "sloppy" + }, + { + "query": "When files are added to Dropbox, copy to Drive and update Notion log", + "triggers": [ + "dropbox-new-file" + ], + "actions": [ + "google_drive-upload-file", + "notion-create-page-from-database" + ], + "persona": "complex-workflow" + }, + { + "query": "send multiple discord msgs", + "triggers": [], + "actions": [ + "discord-send-message" + ], + "persona": "sloppy" + }, + { + "query": "When new form submissions arrive, analyze with GPT and update spreadsheet", + "triggers": [ + "typeform-new-submission" + ], + "actions": [ + "openai-chat", + "google_sheets-add-single-row" + ], + "persona": "complex-workflow" + }, + { + "query": "share files in telegram", + "triggers": [], + "actions": [ + "telegram_bot_api-send-text-message-or-reply" + ], + "persona": "succinct" + }, + { + "query": "I would like GPT to analyze my Slack messages and create tasks in Linear", + "triggers": [ + "slack-new-message-in-channels" + ], + "actions": [ + "openai-chat", + "linear-create-issue" + ], + "persona": "verbose" + }, + { + "query": "When Discord reactions are added, update Notion and notify in Telegram", + "triggers": [ + "discord-reaction-added" + ], + "actions": [ + "notion-update-page", + "telegram_bot_api-send-text-message-or-reply" + ], + "persona": "complex-workflow" + }, + { + "query": "create multiple sheets rows", + "triggers": [], + "actions": [ + "google_sheets-add-multiple-rows" + ], + "persona": "succinct" + }, + { + "query": "When RSS items mention specific terms, analyze with GPT and create Linear issues", + "triggers": [ + "rss-new-item-in-feed" + ], + "actions": [ + "openai-chat", + "linear-create-issue" + ], + "persona": "complex-workflow" + }, + { + "query": "reply 2 telegram msg", + "triggers": [], + "actions": [ + "telegram_bot_api-send-text-message-or-reply" + ], + "persona": "sloppy" + }, + { + "query": "I need new Google Calendar events to be summarized by GPT and posted to Slack", + "triggers": [ + "google_calendar-new-event" + ], + "actions": [ + "openai-chat", + "slack-send-message" + ], + "persona": "verbose" + }, + { + "query": "When Notion pages update, let GPT analyze changes and notify in Discord", + "triggers": [ + "notion-updated-page" + ], + "actions": [ + "openai-chat", + "discord-send-message" + ], + "persona": "complex-workflow" + }, + { + "query": "get data from sheets range", + "triggers": [], + "actions": [ + "google_sheets-get-values-in-range" + ], + "persona": "succinct" + }, + { + "query": "When new YouTube videos are posted, create transcripts and Notion pages", + "triggers": [ + "youtube_data_api-new-videos" + ], + "actions": [ + "openai-create-transcription", + "notion-create-page-from-database" + ], + "persona": "complex-workflow" + }, + { + "query": "updaet multiple sheet rows", + "triggers": [], + "actions": [ + "google_sheets-update-row" + ], + "persona": "sloppy" + }, + { + "query": "I want GPT to analyze Gmail attachments and create Notion documents", + "triggers": [ + "gmail-new-email" + ], + "actions": [ + "openai-chat", + "notion-create-page-from-database" + ], + "persona": "verbose" + }, + { + "query": "When Slack threads are updated, use GPT for sentiment and notify in Telegram", + "triggers": [ + "slack-new-message-in-channels" + ], + "actions": [ + "openai-chat", + "telegram_bot_api-send-text-message-or-reply" + ], + "persona": "complex-workflow" + }, + { + "query": "merge multiple rss feeds", + "triggers": [], + "actions": [ + "rss-merge-rss-feeds" + ], + "persona": "task-oriented" + }, + { + "query": "When files are modified in shared drive, update Sheets and Notion", + "triggers": [ + "google_drive-changes-to-specific-files-shared-drive" + ], + "actions": [ + "google_sheets-add-single-row", + "notion-create-page-from-database" + ], + "persona": "complex-workflow" + }, + { + "query": "transcribe audio to txt", + "triggers": [], + "actions": [ + "openai-create-transcription" + ], + "persona": "sloppy" + }, + { + "query": "I need Zoom recordings to be transcribed and summarized in Slack", + "triggers": [ + "zoom-recording-completed" + ], + "actions": [ + "openai-create-transcription", + "slack-send-message" + ], + "persona": "verbose" + }, + { + "query": "When new Airtable records appear, use GPT to analyze and update the same record", + "triggers": [ + "airtable_oauth-new-records" + ], + "actions": [ + "openai-chat", + "airtable_oauth-update-record" + ], + "persona": "complex-workflow" + }, + { + "query": "download multiple gdrive files", + "triggers": [], + "actions": [ + "google_drive-download-file" + ], + "persona": "succinct" + }, + { + "query": "When Discord messages arrive, translate with GPT and post to Telegram", + "triggers": [ + "discord-new-message" + ], + "actions": [ + "openai-chat", + "telegram_bot_api-send-text-message-or-reply" + ], + "persona": "complex-workflow" + }, + { + "query": "make google docs from txt", + "triggers": [], + "actions": [ + "google_docs-create-document-from-text" + ], + "persona": "sloppy" + }, + { + "query": "I would like GPT to analyze Telegram messages and create Linear issues", + "triggers": [ + "telegram_bot_api-message-updates" + ], + "actions": [ + "openai-chat", + "linear-create-issue" + ], + "persona": "verbose" + }, + { + "query": "When spreadsheet updates, let GPT analyze changes and send in Slack", + "triggers": [ + "google_sheets-new-updates" + ], + "actions": [ + "openai-chat", + "slack-send-message" + ], + "persona": "complex-workflow" + }, + { + "query": "add text to google doc", + "triggers": [], + "actions": [ + "google_docs-append-text" + ], + "persona": "succinct" + }, + { + "query": "When YouTube videos are liked, download and transcribe them", + "triggers": [ + "youtube_data_api-new-liked-videos" + ], + "actions": [ + "openai-create-transcription" + ], + "persona": "complex-workflow" + }, + { + "query": "forward emails to channels", + "triggers": [ + "gmail-new-email" + ], + "actions": [ + "slack-send-message" + ], + "persona": "task-oriented" + }, + { + "query": "When RSS feeds update, create docs and share in Telegram", + "triggers": [ + "rss-new-item-in-feed" + ], + "actions": [ + "google_docs-create-document", + "telegram_bot_api-send-text-message-or-reply" + ], + "persona": "complex-workflow" + }, + { + "query": "genrate email with AI", + "triggers": [], + "actions": [ + "openai-chat" + ], + "persona": "sloppy" + }, + { + "query": "When Slack messages are starred, create Notion pages and Linear tickets", + "triggers": [ + "slack-new-reaction-added" + ], + "actions": [ + "notion-create-page-from-database", + "linear-create-issue" + ], + "persona": "complex-workflow" + }, + { + "query": "create sheet from template", + "triggers": [], + "actions": [ + "google_sheets-create-spreadsheet" + ], + "persona": "succinct" + }, + { + "query": "When calendar events start, generate agenda with GPT and send via Telegram", + "triggers": [ + "google_calendar-upcoming-event-alert" + ], + "actions": [ + "openai-chat", + "telegram_bot_api-send-text-message-or-reply" + ], + "persona": "complex-workflow" + } + ] +} \ No newline at end of file diff --git a/packages/evals/component_retrieval/eval-test-suite-251-300-new-filtered.json b/packages/evals/component_retrieval/eval-test-suite-251-300-new-filtered.json new file mode 100644 index 0000000000000..f9390aa022f84 --- /dev/null +++ b/packages/evals/component_retrieval/eval-test-suite-251-300-new-filtered.json @@ -0,0 +1,461 @@ +{ + "evaluationTests": [ + { + "query": "When a Webflow form is submitted, create contact in Mailchimp and send PagerDuty alert", + "triggers": [ + "webflow-new-form-submission" + ], + "actions": [ + "mailchimp-add-subscriber-to-tag", + "pagerduty-trigger-incident" + ], + "persona": "complex-workflow" + }, + { + "query": "create medium blog post", + "triggers": [], + "actions": [ + "medium-create-post" + ], + "persona": "succinct" + }, + { + "query": "send push notification using pushover", + "triggers": [], + "actions": [ + "pushover-push-notification" + ], + "persona": "task-oriented" + }, + { + "query": "When there's a new hot post on Reddit subreddit, send to Telegram", + "triggers": [ + "reddit-new-hot-posts-on-a-subreddit" + ], + "actions": [ + "telegram_bot_api-send-text-message-or-reply" + ], + "persona": "complex-workflow" + }, + { + "query": "mkae pushbullet notifcation", + "triggers": [], + "actions": [ + "pushbullet-send-push" + ], + "persona": "sloppy" + }, + { + "query": "I would like to be notified when someone completes their Calendly booking", + "triggers": [ + "calendly_v2-invitee-created" + ], + "actions": [], + "persona": "verbose" + }, + { + "query": "When new Spotify playlist is created, post to Discord", + "triggers": [ + "spotify-new-playlist" + ], + "actions": [ + "discord-send-message" + ], + "persona": "complex-workflow" + }, + { + "query": "upload file 2 imgbb", + "triggers": [], + "actions": [ + "imgbb-upload-picture" + ], + "persona": "sloppy" + }, + { + "query": "create postmark email", + "triggers": [], + "actions": [ + "postmark-send-single-email" + ], + "persona": "succinct" + }, + { + "query": "When a new story mentions our keyword on Hacker News, create Linear ticket", + "triggers": [ + "hacker_news-new-stories-by-keyword" + ], + "actions": [ + "linear-create-issue" + ], + "persona": "complex-workflow" + }, + { + "query": "I need my WordPress posts to be automatically shared on LinkedIn", + "triggers": [], + "actions": [ + "linkedin-create-text-post-user" + ], + "persona": "verbose" + }, + { + "query": "make aws lambda functon", + "triggers": [], + "actions": [ + "aws-lambda-invoke-function" + ], + "persona": "sloppy" + }, + { + "query": "When a new video is added to my YouTube playlist, add to Spotify", + "triggers": [ + "youtube_data_api-new-videos-in-playlist" + ], + "actions": [ + "spotify-add-items-to-playlist" + ], + "persona": "complex-workflow" + }, + { + "query": "create coda row", + "triggers": [], + "actions": [ + "coda-create-rows" + ], + "persona": "succinct" + }, + { + "query": "When my Strava activity is created, post to Facebook Page", + "triggers": [ + "strava-activity-created" + ], + "actions": [ + "facebook_pages-create-post" + ], + "persona": "complex-workflow" + }, + { + "query": "send emergency pushover alert", + "triggers": [], + "actions": [ + "pushover-emergency-push-notification" + ], + "persona": "task-oriented" + }, + { + "query": "I would like to sync my Google Tasks with Microsoft To Do", + "triggers": [ + "google_tasks-new-task-added" + ], + "actions": [ + "microsofttodo-create-task" + ], + "persona": "verbose" + }, + { + "query": "When someone comments on Figma, create Motion task", + "triggers": [ + "figma-new-comment" + ], + "actions": [ + "motion-create-task" + ], + "persona": "complex-workflow" + }, + { + "query": "creat cloudinary upload", + "triggers": [], + "actions": [ + "cloudinary-upload-media-asset" + ], + "persona": "sloppy" + }, + { + "query": "When there's a new commit in Bitbucket, post to Microsoft Teams", + "triggers": [ + "bitbucket-new-commit" + ], + "actions": [ + "microsoft_teams-send-channel-message" + ], + "persona": "complex-workflow" + }, + { + "query": "send line notification", + "triggers": [], + "actions": [ + "line-send-notification-message" + ], + "persona": "succinct" + }, + { + "query": "I need all my Raindrop bookmarks to be saved in Notion", + "triggers": [ + "raindrop-new-bookmark" + ], + "actions": [ + "notion-create-page-from-database" + ], + "persona": "verbose" + }, + { + "query": "When a new highlight is made in Readwise, create Google Doc", + "triggers": [ + "readwise-new-highlights" + ], + "actions": [ + "google_docs-create-document" + ], + "persona": "complex-workflow" + }, + { + "query": "add contact 2 mautic", + "triggers": [], + "actions": [ + "mautic-create-contact" + ], + "persona": "sloppy" + }, + { + "query": "When a new form is answered in Google Forms, add to Airtable", + "triggers": [ + "google_forms-new-form-answer" + ], + "actions": [ + "airtable_oauth-create-single-record" + ], + "persona": "complex-workflow" + }, + { + "query": "create onenote page", + "triggers": [], + "actions": [ + "onenote-create-page" + ], + "persona": "succinct" + }, + { + "query": "I want my Mastodon posts to be automatically shared on Twitter", + "triggers": [ + "mastodon-new-status" + ], + "actions": [ + "twitter-create-tweet" + ], + "persona": "verbose" + }, + { + "query": "When a WhatsApp message is received, analyze with GPT and send to Telegram", + "triggers": [ + "whatsapp_business-new-message-sent" + ], + "actions": [ + "openai-chat", + "telegram_bot_api-send-text-message-or-reply" + ], + "persona": "complex-workflow" + }, + { + "query": "create algolia objct", + "triggers": [], + "actions": [ + "algolia-create-objects" + ], + "persona": "sloppy" + }, + { + "query": "When files are added to Box, backup to S3", + "triggers": [ + "box-new-file" + ], + "actions": [ + "aws-s3-upload-file" + ], + "persona": "task-oriented" + }, + { + "query": "make cisco webex msg", + "triggers": [], + "actions": [ + "cisco_webex-create-message" + ], + "persona": "sloppy" + }, + { + "query": "I need my Google Classroom assignments to be tracked in Todoist", + "triggers": [ + "google_classroom-new-assignment" + ], + "actions": [ + "todoist-create-task" + ], + "persona": "verbose" + }, + { + "query": "When new contacts are added to ActiveCampaign, sync to Klaviyo", + "triggers": [ + "activecampaign-new-or-updated-contact" + ], + "actions": [ + "klaviyo-add-member-to-list" + ], + "persona": "complex-workflow" + }, + { + "query": "send chat_api whatsapp", + "triggers": [], + "actions": [ + "chat_api_for_whatsapp-send-message" + ], + "persona": "sloppy" + }, + { + "query": "When someone reacts to a Discord message, create Trello card", + "triggers": [ + "discord-reaction-added" + ], + "actions": [ + "trello-create-card" + ], + "persona": "complex-workflow" + }, + { + "query": "post giphy to discord", + "triggers": [], + "actions": [ + "discord-send-message" + ], + "persona": "succinct" + }, + { + "query": "I would like new Pocket items to be saved to Instapaper", + "triggers": [ + "pocket-new-item" + ], + "actions": [ + "instapaper-add-url" + ], + "persona": "verbose" + }, + { + "query": "When Twitch stream starts, post to social media", + "triggers": [ + "twitch-streams-by-streamer" + ], + "actions": [ + "twitter-create-tweet", + "facebook_pages-create-post" + ], + "persona": "complex-workflow" + }, + { + "query": "make freshsales contact", + "triggers": [], + "actions": [], + "persona": "sloppy" + }, + { + "query": "When new article appears in Inoreader folder, save to Notion", + "triggers": [ + "inoreader-new-article-in-folder" + ], + "actions": [ + "notion-create-page-from-database" + ], + "persona": "complex-workflow" + }, + { + "query": "create servicenow incident", + "triggers": [], + "actions": [], + "persona": "succinct" + }, + { + "query": "I want to receive Bandwidth SMS messages in Telegram", + "triggers": [ + "bandwidth-new-incoming-sms" + ], + "actions": [ + "telegram_bot_api-send-text-message-or-reply" + ], + "persona": "verbose" + }, + { + "query": "When a new comment is posted on WordPress, analyze with GPT", + "triggers": [], + "actions": [ + "openai-chat" + ], + "persona": "complex-workflow" + }, + { + "query": "send mailgun email", + "triggers": [], + "actions": [ + "mailgun-send-email" + ], + "persona": "succinct" + }, + { + "query": "When RingCentral records a call, transcribe with OpenAI", + "triggers": [ + "ringcentral-new-call-recording" + ], + "actions": [ + "openai-create-transcription" + ], + "persona": "complex-workflow" + }, + { + "query": "create shipstation evnt", + "triggers": [], + "actions": [], + "persona": "sloppy" + }, + { + "query": "I need my Google Contacts to sync with Microsoft Outlook", + "triggers": [ + "google_contacts-contact-created" + ], + "actions": [ + "microsoft_outlook-create-contact" + ], + "persona": "verbose" + }, + { + "query": "When new Squarespace transaction occurs, update Quickbooks", + "triggers": [ + "squarespace-new-create-transaction" + ], + "actions": [ + "quickbooks-create-invoice" + ], + "persona": "complex-workflow" + }, + { + "query": "create zoho mail", + "triggers": [], + "actions": [ + "zoho_mail-send-email" + ], + "persona": "succinct" + }, + { + "query": "When Firebase document changes, send Line notification", + "triggers": [ + "firebase_admin_sdk-new-doc-in-firestore-collection" + ], + "actions": [ + "line-send-notification-message" + ], + "persona": "complex-workflow" + }, + { + "query": "add pinterest pin", + "triggers": [], + "actions": [ + "pinterest-create-pin" + ], + "persona": "succinct" + } + ] +} \ No newline at end of file diff --git a/packages/evals/component_retrieval/eval-test-suite-ai-focus-filtered.json b/packages/evals/component_retrieval/eval-test-suite-ai-focus-filtered.json new file mode 100644 index 0000000000000..afa6ed0658439 --- /dev/null +++ b/packages/evals/component_retrieval/eval-test-suite-ai-focus-filtered.json @@ -0,0 +1,214 @@ +{ + "evaluationTests": [ + { + "query": "When new support tickets come in through Zendesk, analyze sentiment with GPT and prioritize in Linear based on urgency", + "triggers": [ + "zendesk-new-ticket" + ], + "actions": [ + "openai-chat", + "linear-create-issue" + ], + "persona": "complex-workflow" + }, + { + "query": "I need sales calls recorded in Gong to be transcribed and summarized for the team in Slack", + "triggers": [ + "gong-new-call" + ], + "actions": [ + "openai-create-transcription", + "openai-chat", + "slack-send-message" + ], + "persona": "verbose" + }, + { + "query": "When leads submit Typeform responses, use GPT to qualify them and update their status in HubSpot", + "triggers": [ + "typeform-new-submission" + ], + "actions": [ + "openai-chat", + "hubspot-create-or-update-contact" + ], + "persona": "complex-workflow" + }, + { + "query": "Get help from AI to classify and organize our Notion knowledge base", + "triggers": [ + "notion-new-page" + ], + "actions": [ + "openai-chat", + "notion-create-page-from-database" + ], + "persona": "task-oriented" + }, + { + "query": "When customers message us on Intercom, analyze intent with GPT before creating tickets", + "triggers": [ + "intercom-new-conversation" + ], + "actions": [ + "openai-chat", + "linear-create-issue" + ], + "persona": "complex-workflow" + }, + { + "query": "Analyze customer feedback from Delighted with AI and update account health in Salesforce", + "triggers": [], + "actions": [ + "openai-chat", + "salesforce_rest_api-update-contact" + ], + "persona": "task-oriented" + }, + { + "query": "When new videos are uploaded to Zoom, I want them transcribed and summarized for the team", + "triggers": [ + "zoom-recording-completed" + ], + "actions": [ + "openai-create-transcription", + "openai-chat", + "slack-send-message" + ], + "persona": "verbose" + }, + { + "query": "Use AI to analyze Github issues and suggest priority levels in Linear", + "triggers": [], + "actions": [ + "openai-chat", + "linear-create-issue" + ], + "persona": "task-oriented" + }, + { + "query": "When documents are uploaded to Google Drive, use GPT to generate summaries in Notion", + "triggers": [ + "google_drive-new-files-instant" + ], + "actions": [ + "openai-chat", + "notion-create-page-from-database" + ], + "persona": "complex-workflow" + }, + { + "query": "Analyze customer churn risk based on Intercom conversations using GPT", + "triggers": [ + "intercom-new-conversation" + ], + "actions": [ + "openai-chat", + "hubspot-create-or-update-contact" + ], + "persona": "task-oriented" + }, + { + "query": "When new RSS articles mention our company, use AI to analyze sentiment and alert team", + "triggers": [ + "rss-new-item-in-feed" + ], + "actions": [ + "openai-chat", + "slack-send-message" + ], + "persona": "complex-workflow" + }, + { + "query": "I need GPT to help categorize incoming feature requests from Canny into our product roadmap", + "triggers": [], + "actions": [ + "openai-chat", + "notion-create-page-from-database" + ], + "persona": "verbose" + }, + { + "query": "When customers respond to our Typeform survey, analyze trends with AI and update dashboards", + "triggers": [ + "typeform-new-submission" + ], + "actions": [ + "openai-chat", + "google_sheets-add-single-row" + ], + "persona": "complex-workflow" + }, + { + "query": "Use GPT to analyze Calendly meeting notes and create action items in Asana", + "triggers": [ + "calendly_v2-new-event-scheduled" + ], + "actions": [ + "openai-chat", + "asana-create-task" + ], + "persona": "task-oriented" + }, + { + "query": "When support team sends emails in Gmail, let AI check tone and suggest improvements", + "triggers": [ + "gmail-new-email" + ], + "actions": [ + "openai-chat" + ], + "persona": "complex-workflow" + }, + { + "query": "I need customer conversations from Help Scout to be analyzed for product feedback", + "triggers": [], + "actions": [ + "openai-chat", + "notion-create-page-from-database" + ], + "persona": "verbose" + }, + { + "query": "When new comments appear on our YouTube videos, use AI to moderate and flag issues", + "triggers": [ + "youtube_data_api-new-comment-posted" + ], + "actions": [ + "openai-chat", + "slack-send-message" + ], + "persona": "complex-workflow" + }, + { + "query": "Use GPT to analyze Twitter mentions and create support tickets when needed", + "triggers": [ + "twitter-new-mention-received-by-user" + ], + "actions": [ + "openai-chat", + "linear-create-issue" + ], + "persona": "task-oriented" + }, + { + "query": "When deals close in Salesforce, use AI to generate personalized onboarding docs", + "triggers": [], + "actions": [ + "openai-chat", + "google_docs-create-document" + ], + "persona": "complex-workflow" + }, + { + "query": "I want GPT to help write better commit messages for our Github repos", + "triggers": [ + "github-new-commit" + ], + "actions": [ + "openai-chat" + ], + "persona": "verbose" + } + ] +} \ No newline at end of file diff --git a/packages/evals/component_retrieval/eval-test-suite-graduated-corrected-filtered.json b/packages/evals/component_retrieval/eval-test-suite-graduated-corrected-filtered.json new file mode 100644 index 0000000000000..d36db6d29071d --- /dev/null +++ b/packages/evals/component_retrieval/eval-test-suite-graduated-corrected-filtered.json @@ -0,0 +1,187 @@ +{ + "evaluationTests": [ + { + "query": "send slack message", + "triggers": [], + "actions": [ + "slack-send-message" + ], + "persona": "succinct" + }, + { + "query": "create issue in linear", + "triggers": [], + "actions": [ + "linear-create-issue" + ], + "persona": "succinct" + }, + { + "query": "sync my contacts", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "When new emails arrive in Gmail, save attachments to Google Drive", + "triggers": [ + "gmail-new-email" + ], + "actions": [ + "google_drive-upload-file" + ], + "persona": "verbose" + }, + { + "query": "add roblox purchases to quickbooks", + "triggers": [], + "actions": [ + "quickbooks-create-invoice" + ], + "persona": "no-component" + }, + { + "query": "When someone fills out my Typeform, notify me in Slack", + "triggers": [ + "typeform-new-submission" + ], + "actions": [ + "slack-send-message" + ], + "persona": "verbose" + }, + { + "query": "automate my business processes", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "When new videos are uploaded to YouTube, transcribe with OpenAI and post summary to Discord", + "triggers": [ + "youtube_data_api-new-videos" + ], + "actions": [ + "openai-create-transcription", + "openai-chat", + "discord-send-message" + ], + "persona": "complex-workflow" + }, + { + "query": "When customer submits form in AirKit, create contact in HubSpot", + "triggers": [], + "actions": [ + "hubspot-create-or-update-contact" + ], + "persona": "no-component" + }, + { + "query": "When a deal closes in Salesforce, generate invoice in QuickBooks and notify accounting in Slack", + "triggers": [], + "actions": [ + "quickbooks-create-invoice", + "slack-send-message" + ], + "persona": "complex-workflow" + }, + { + "query": "make my team more productive", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "When new support tickets come in through Zendesk, analyze sentiment with GPT and create tasks in Linear", + "triggers": [ + "zendesk-new-ticket" + ], + "actions": [ + "openai-chat", + "linear-create-issue" + ], + "persona": "complex-workflow" + }, + { + "query": "When deals update in Copper CRM, sync to Oracle NetSuite", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "When files are added to Dropbox, use GPT to analyze content, create summaries in Notion, and notify relevant teams in Slack", + "triggers": [ + "dropbox-new-file" + ], + "actions": [ + "openai-chat", + "notion-create-page-from-database", + "slack-send-message" + ], + "persona": "complex-workflow" + }, + { + "query": "help me with reporting", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "When a customer books through Calendly, add to HubSpot, create Asana task, and start onboarding sequence", + "triggers": [ + "calendly_v2-new-event-scheduled" + ], + "actions": [ + "hubspot-create-or-update-contact", + "asana-create-task" + ], + "persona": "complex-workflow" + }, + { + "query": "When feedback is submitted in UserVoice, analyze with GPT and create ticket in Jira", + "triggers": [], + "actions": [ + "openai-chat", + "jira-create-issue" + ], + "persona": "no-component" + }, + { + "query": "When new recordings appear in Gong, transcribe with OpenAI, analyze for key insights, update deal notes in Salesforce, and notify sales team in Slack", + "triggers": [ + "gong-new-call" + ], + "actions": [ + "openai-create-transcription", + "openai-chat", + "salesforce_rest_api-update-contact", + "slack-send-message" + ], + "persona": "complex-workflow" + }, + { + "query": "When customer reaches usage threshold in Stripe, check status in ChargeBee, update account in HubSpot, and trigger customer success workflow", + "triggers": [ + "stripe-subscription-updated" + ], + "actions": [ + "hubspot-create-or-update-contact" + ], + "persona": "complex-workflow" + }, + { + "query": "When new document is uploaded to Google Drive, use GPT for content analysis, create summaries in Notion, add action items in Linear, notify teams in Slack, and schedule follow-up in Google Calendar", + "triggers": [ + "google_drive-new-files-instant" + ], + "actions": [ + "openai-chat", + "notion-create-page-from-database", + "linear-create-issue", + "slack-send-message", + "google_calendar-create-event" + ], + "persona": "complex-workflow" + } + ] +} \ No newline at end of file diff --git a/packages/evals/component_retrieval/eval-test-suite-invalid-queries-filtered.json b/packages/evals/component_retrieval/eval-test-suite-invalid-queries-filtered.json new file mode 100644 index 0000000000000..7cb0a18d18af9 --- /dev/null +++ b/packages/evals/component_retrieval/eval-test-suite-invalid-queries-filtered.json @@ -0,0 +1,310 @@ +{ + "evaluationTests": [ + { + "query": "sync my data between systems", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "send notifications when something important happens", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "create a daily report", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "automate my sales process", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "connect my database", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "help me with customer support", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "track user activity", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "move files automatically", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "update my calendar", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "make my workflow more efficient", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "integrate with my existing tools", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "analyze my customer feedback", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "automate document creation", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "send reminders to my team", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "track project progress", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "monitor system performance", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "backup my important files", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "generate custom reports", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "create new users", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "manage my contacts", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "update status automatically", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "help me with social media", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "organize my documents", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "sync my calendars", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "manage my team better", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "automate invoice processing", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "track employee time", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "monitor website changes", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "schedule my posts", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "manage my inventory", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "sync my contact lists", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "automate email responses", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "create task assignments", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "track project deadlines", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "manage my subscriptions", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "update my database", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "automate customer onboarding", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "sync team communications", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "manage project resources", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "track sales performance", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "automate data entry", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "generate reports weekly", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "manage my workflows", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "integrate payment processing", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "automate file transfers", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "sync my documents", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "manage customer relationships", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "track employee performance", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "automate marketing campaigns", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "manage my leads", + "triggers": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "sync my apps", + "triggers": [], + "actions": [], + "persona": "invalid-query" + } + ] +} \ No newline at end of file diff --git a/packages/evals/component_retrieval/eval-test-suite-no-components-filtered.json b/packages/evals/component_retrieval/eval-test-suite-no-components-filtered.json new file mode 100644 index 0000000000000..ebee7aa2a18bd --- /dev/null +++ b/packages/evals/component_retrieval/eval-test-suite-no-components-filtered.json @@ -0,0 +1,276 @@ +{ + "evaluationTests": [ + { + "query": "When someone books a demo in Chili Piper, create a contact in Apollo", + "triggers": [], + "actions": [ + "apollo_io-create-contact" + ], + "persona": "no-component" + }, + { + "query": "When we get a new review on G2, post it to Slack", + "triggers": [], + "actions": [ + "slack-send-message" + ], + "persona": "no-component" + }, + { + "query": "Add new Gong calls to my Guru knowledge base", + "triggers": [ + "gong-new-call" + ], + "actions": [], + "persona": "no-component" + }, + { + "query": "When someone signs our DocuSign NDA, grant access in Okta", + "triggers": [ + "docusign-envelope-sent-or-complete" + ], + "actions": [], + "persona": "no-component" + }, + { + "query": "Create a new project in Wrike when Typeform is submitted", + "triggers": [ + "typeform-new-submission" + ], + "actions": [], + "persona": "no-component" + }, + { + "query": "When customer churns in Chargebee, remove access in Auth0 and revoke Workato license", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "Send new Snowflake query results to PowerBI", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "When new issue is created in Azure DevOps, create ticket in ServiceNow", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "Add rowing sessions from Hydrow to my Apple Health", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "When someone mentions us on Blind, send to Discord", + "triggers": [], + "actions": [ + "discord-send-message" + ], + "persona": "no-component" + }, + { + "query": "Create tasks in Monday.com from Shortcut stories", + "triggers": [], + "actions": [ + "monday-create-item" + ], + "persona": "no-component" + }, + { + "query": "When a new invoice is created in NetSuite, sync to Sage Intacct", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "Create Lattice goals from Asana projects", + "triggers": [ + "asana-new-project" + ], + "actions": [], + "persona": "no-component" + }, + { + "query": "When new candidate appears in Lever, create user in JumpCloud", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "Add new LeanIX components to our Architecture Handbook in Confluence", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "When a deal closes in Close.com, create project in TeamGantt", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "Create Coda doc from new Pitch presentation", + "triggers": [], + "actions": [ + "coda-create-rows" + ], + "persona": "no-component" + }, + { + "query": "When contract is approved in Ironclad, create workspace in Retool", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "Add new Gong snippets to Chorus.ai library", + "triggers": [ + "gong-new-call" + ], + "actions": [], + "persona": "no-component" + }, + { + "query": "When employee is terminated in Namely, revoke access in Rippling", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "Sync product specs from Productboard to Aha!", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "Create Metabase dashboard when new Amplitude cohort is created", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "When new report runs in Looker, send to Tableau", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "Add new Greenhouse candidates to iCIMS", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "When feature is deployed in LaunchDarkly, update status in StatusPage", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "Create Lucidchart diagram from Miro board", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "When new user signs up in Mixpanel, create profile in Pendo", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "Add new 6sense accounts to RollWorks", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "When new survey response in Qualtrics, add to Gainsight", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "Create tasks in Wrike from Clubhouse stories", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "When customer submits proof in Filestage, add to Bynder", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "Create Qatalog process from new Notion template", + "triggers": [ + "notion-new-page" + ], + "actions": [], + "persona": "no-component" + }, + { + "query": "When user requests access in Mesh Directory, create ticket in Jira Service Management", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "Add new candidates from SmartRecruiters to Workday", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "When new post is created in Discourse, share on Yammer", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "Create monday.com pulse from Range check-in", + "triggers": [], + "actions": [ + "monday-create-item" + ], + "persona": "no-component" + }, + { + "query": "When new task is created in Process Street, add to Teamwork", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "Add new Demodesk recordings to Gong", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "When new article is published on Dev.to, cross-post to Hashnode", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "Create UserPilot experience from new Pendo guide", + "triggers": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "When new document is uploaded to Box, scan with Nightfall", + "triggers": [ + "box-new-file" + ], + "actions": [], + "persona": "no-component" + } + ] +} \ No newline at end of file From a9b710ae9afb543f8f611a362970dfa8421d9ac2 Mon Sep 17 00:00:00 2001 From: "Dylan J. Sather" Date: Wed, 23 Oct 2024 20:50:16 -0700 Subject: [PATCH 02/18] Adding evaluator.mjs --- .../evals/component_retrieval/.env.example | 2 + packages/evals/component_retrieval/.gitignore | 1 + .../evals/component_retrieval/evaluator.mjs | 110 ++++++++++++++++++ .../component_retrieval/package-lock.json | 80 +++++++++++++ .../evals/component_retrieval/package.json | 10 ++ 5 files changed, 203 insertions(+) create mode 100644 packages/evals/component_retrieval/.env.example create mode 100644 packages/evals/component_retrieval/.gitignore create mode 100644 packages/evals/component_retrieval/evaluator.mjs create mode 100644 packages/evals/component_retrieval/package-lock.json create mode 100644 packages/evals/component_retrieval/package.json diff --git a/packages/evals/component_retrieval/.env.example b/packages/evals/component_retrieval/.env.example new file mode 100644 index 0000000000000..49fc5c72ee867 --- /dev/null +++ b/packages/evals/component_retrieval/.env.example @@ -0,0 +1,2 @@ +API_BASE_URL=https://api.dylburger.gkes.pipedream.net +PIPEDREAM_API_KEY= diff --git a/packages/evals/component_retrieval/.gitignore b/packages/evals/component_retrieval/.gitignore new file mode 100644 index 0000000000000..4c49bd78f1d08 --- /dev/null +++ b/packages/evals/component_retrieval/.gitignore @@ -0,0 +1 @@ +.env diff --git a/packages/evals/component_retrieval/evaluator.mjs b/packages/evals/component_retrieval/evaluator.mjs new file mode 100644 index 0000000000000..7dabb7ca181b0 --- /dev/null +++ b/packages/evals/component_retrieval/evaluator.mjs @@ -0,0 +1,110 @@ +#!/usr/bin/env node + +import "dotenv/config"; +import fs from "fs/promises"; +import path from "path"; +import { diff } from "json-diff"; + +const GREEN_CHECK = "\x1b[32m✔\x1b[0m"; +const RED_CROSS = "\x1b[31m✖\x1b[0m"; + +let totalEvals = 0; +let totalSuccesses = 0; + +const apiHost = process.env.API_BASE_URL || "https://api.pipedream.com"; + +// json-diff shows the diff as __old and __new keys, which isn't descriptive, +// so we replace them with custom labels +function customDiff(original, updated, oldLabel = "expected", newLabel = "actual") { + const result = diff(original, updated); + + function replaceLabels(obj) { + if (Array.isArray(obj)) { + return obj.map(replaceLabels); + } else if (typeof obj !== "object" || obj === null) { + return obj; + } + + const newObj = {}; + for (const key in obj) { + if (key === "__old") { + newObj[oldLabel] = replaceLabels(obj[key]); + } else if (key === "__new") { + newObj[newLabel] = replaceLabels(obj[key]); + } else { + newObj[key] = replaceLabels(obj[key]); + } + } + return newObj; + } + + return replaceLabels(result); +} + +async function processEvalFile(filePath) { + try { + const content = await fs.readFile(filePath, "utf-8"); + const evalData = JSON.parse(content); + + for (const evalTest of evalData.evaluationTests) { + totalEvals++; + const { + query, triggers, actions, + } = evalTest; + + const encodedQuery = encodeURIComponent(query); + const apiUrl = `${apiHost}/v1/components/search?query=${encodedQuery}`; + + const response = await fetch(apiUrl, { + headers: { + "Content-Type": "application/json", + "Authorization": `Bearer ${process.env.PIPEDREAM_API_KEY}`, + }, + }); + const apiData = await response.json(); + + // Compare actual and expected + const apiTriggers = apiData?.triggers ?? []; + const apiActions = apiData?.actions ?? []; + const triggersMatch = + JSON.stringify(apiTriggers.sort()) === JSON.stringify(triggers.sort()); + const actionsMatch = + JSON.stringify(apiActions.sort()) === JSON.stringify(actions.sort()); + + if (triggersMatch && actionsMatch) { + totalSuccesses++; + console.log(`${GREEN_CHECK} Success for query: "${query}"`); + } else { + console.log(`${RED_CROSS} Failure for query: "${query}"`); + console.log("Differences:"); + console.log(customDiff({ + triggers, + actions, + }, apiData)); + } + } + } catch (error) { + console.error(`Error processing file ${filePath}:`, error.message); + } +} + +async function main() { + const evalFiles = process.argv.slice(2); + + if (evalFiles.length === 0) { + console.error("Please provide at least one eval JSON file."); + process.exit(1); + } + + for (const file of evalFiles) { + const filePath = path.resolve(file); + await processEvalFile(filePath); + } + + const successRate = ((totalSuccesses / totalEvals) * 100).toFixed(2); + console.log(`\nTotal Evals: ${totalEvals}`); + console.log(`Total Successes: ${totalSuccesses}`); + console.log(`Success Rate: ${successRate}%`); +} + +main(); diff --git a/packages/evals/component_retrieval/package-lock.json b/packages/evals/component_retrieval/package-lock.json new file mode 100644 index 0000000000000..eeebfb5eb738d --- /dev/null +++ b/packages/evals/component_retrieval/package-lock.json @@ -0,0 +1,80 @@ +{ + "name": "@pipedream/evals-component_retrieval", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@pipedream/evals-component_retrieval", + "version": "0.0.1", + "dependencies": { + "dotenv": "^16.4.5", + "json-diff": "^1.0.6" + } + }, + "node_modules/@ewoudenberg/difflib": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@ewoudenberg/difflib/-/difflib-0.1.0.tgz", + "integrity": "sha512-OU5P5mJyD3OoWYMWY+yIgwvgNS9cFAU10f+DDuvtogcWQOoJIsQ4Hy2McSfUfhKjq8L0FuWVb4Rt7kgA+XK86A==", + "dependencies": { + "heap": ">= 0.2.0" + } + }, + "node_modules/colors": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", + "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==", + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/dotenv": { + "version": "16.4.5", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", + "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dreamopt": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/dreamopt/-/dreamopt-0.8.0.tgz", + "integrity": "sha512-vyJTp8+mC+G+5dfgsY+r3ckxlz+QMX40VjPQsZc5gxVAxLmi64TBoVkP54A/pRAXMXsbu2GMMBrZPxNv23waMg==", + "dependencies": { + "wordwrap": ">=0.0.2" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/heap": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/heap/-/heap-0.2.7.tgz", + "integrity": "sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==" + }, + "node_modules/json-diff": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/json-diff/-/json-diff-1.0.6.tgz", + "integrity": "sha512-tcFIPRdlc35YkYdGxcamJjllUhXWv4n2rK9oJ2RsAzV4FBkuV4ojKEDgcZ+kpKxDmJKv+PFK65+1tVVOnSeEqA==", + "dependencies": { + "@ewoudenberg/difflib": "0.1.0", + "colors": "^1.4.0", + "dreamopt": "~0.8.0" + }, + "bin": { + "json-diff": "bin/json-diff.js" + }, + "engines": { + "node": "*" + } + }, + "node_modules/wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==" + } + } +} diff --git a/packages/evals/component_retrieval/package.json b/packages/evals/component_retrieval/package.json new file mode 100644 index 0000000000000..5b5ca619bc3f6 --- /dev/null +++ b/packages/evals/component_retrieval/package.json @@ -0,0 +1,10 @@ +{ + "name": "@pipedream/evals-component_retrieval", + "version": "0.0.1", + "description": "Eval for Pipedream AI component retrieval API", + "main": "evaluator.mjs", + "dependencies": { + "dotenv": "^16.4.5", + "json-diff": "^1.0.6" + } +} From ab34992403484b3e6f7127050fd04e009290584f Mon Sep 17 00:00:00 2001 From: "Dylan J. Sather" Date: Wed, 23 Oct 2024 20:51:35 -0700 Subject: [PATCH 03/18] pnpm-lock --- pnpm-lock.yaml | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ac41840fa8951..ef82584bfc25b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11879,6 +11879,14 @@ importers: playwright-core: 1.41.2 puppeteer-core: 21.11.0 + packages/evals/component_retrieval: + specifiers: + dotenv: ^16.4.5 + json-diff: ^1.0.6 + dependencies: + dotenv: 16.4.5 + json-diff: 1.0.6 + packages/prompts: specifiers: typescript: ^5.0.4 @@ -16264,6 +16272,12 @@ packages: engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} dev: true + /@ewoudenberg/difflib/0.1.0: + resolution: {integrity: sha512-OU5P5mJyD3OoWYMWY+yIgwvgNS9cFAU10f+DDuvtogcWQOoJIsQ4Hy2McSfUfhKjq8L0FuWVb4Rt7kgA+XK86A==} + dependencies: + heap: 0.2.7 + dev: false + /@exodus/schemasafe/1.3.0: resolution: {integrity: sha512-5Aap/GaRupgNx/feGBwLLTVv8OQFfv3pq2lPRzPg9R+IOBnDgghTGW7l7EuVXOvg5cc/xSAlRW8rBrjIC3Nvqw==} dev: false @@ -23490,6 +23504,11 @@ packages: resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} dev: true + /colors/1.4.0: + resolution: {integrity: sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==} + engines: {node: '>=0.1.90'} + dev: false + /colorspace/1.1.4: resolution: {integrity: sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==} dependencies: @@ -24549,6 +24568,11 @@ packages: is-obj: 2.0.0 dev: false + /dotenv/16.4.5: + resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==} + engines: {node: '>=12'} + dev: false + /dotenv/8.6.0: resolution: {integrity: sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g==} engines: {node: '>=10'} @@ -24558,6 +24582,13 @@ packages: resolution: {integrity: sha512-t5ouWOpItmHrm0J0+bX/cFrIjBFWnJkk5LbIJq6bbU/M4aLX2c3LrM4QYsBptwvlPe3WzdpQefQ0v1pe/A5wjg==} dev: false + /dreamopt/0.8.0: + resolution: {integrity: sha512-vyJTp8+mC+G+5dfgsY+r3ckxlz+QMX40VjPQsZc5gxVAxLmi64TBoVkP54A/pRAXMXsbu2GMMBrZPxNv23waMg==} + engines: {node: '>=0.4.0'} + dependencies: + wordwrap: 1.0.0 + dev: false + /dropbox/10.34.0_@types+node-fetch@2.6.6: resolution: {integrity: sha512-5jb5/XzU0fSnq36/hEpwT5/QIep7MgqKuxghEG44xCu7HruOAjPdOb3x0geXv5O/hd0nHpQpWO+r5MjYTpMvJg==} engines: {node: '>=0.10.3'} @@ -26984,6 +27015,10 @@ packages: engines: {node: '>=10.0.0'} dev: false + /heap/0.2.7: + resolution: {integrity: sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==} + dev: false + /hexoid/1.0.0: resolution: {integrity: sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==} engines: {node: '>=8'} @@ -28626,6 +28661,15 @@ packages: /json-buffer/3.0.1: resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} + /json-diff/1.0.6: + resolution: {integrity: sha512-tcFIPRdlc35YkYdGxcamJjllUhXWv4n2rK9oJ2RsAzV4FBkuV4ojKEDgcZ+kpKxDmJKv+PFK65+1tVVOnSeEqA==} + hasBin: true + dependencies: + '@ewoudenberg/difflib': 0.1.0 + colors: 1.4.0 + dreamopt: 0.8.0 + dev: false + /json-parse-better-errors/1.0.2: resolution: {integrity: sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==} dev: true @@ -35729,7 +35773,7 @@ packages: dev: false /verror/1.10.0: - resolution: {integrity: sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==} + resolution: {integrity: sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=} engines: {'0': node >=0.6.0} dependencies: assert-plus: 1.0.0 From 74b4b5bcb9fd1c5917e3337b99c33a1b95071013 Mon Sep 17 00:00:00 2001 From: Michael Lim Date: Thu, 24 Oct 2024 11:43:04 -0700 Subject: [PATCH 04/18] Updating incorrect evals. --- .../eval-test-suite-0-100-filtered.json | 2 +- ...-test-suite-151-200-verified-filtered.json | 6 +- ...val-test-suite-no-components-filtered.json | 2 +- ...eval-test-suite-similar-apps-filtered.json | 707 ++++++++++++++++++ 4 files changed, 712 insertions(+), 5 deletions(-) create mode 100644 packages/evals/component_retrieval/eval-test-suite-similar-apps-filtered.json diff --git a/packages/evals/component_retrieval/eval-test-suite-0-100-filtered.json b/packages/evals/component_retrieval/eval-test-suite-0-100-filtered.json index 4e8b3f7f59ce3..46d06a08a78a3 100644 --- a/packages/evals/component_retrieval/eval-test-suite-0-100-filtered.json +++ b/packages/evals/component_retrieval/eval-test-suite-0-100-filtered.json @@ -30,7 +30,7 @@ { "query": "when new emails arrive in Gmail, use OpenAI to summarize them and post to Slack", "triggers": [ - "gmail-new-email" + "gmail-new-email-received" ], "actions": [ "openai-chat", diff --git a/packages/evals/component_retrieval/eval-test-suite-151-200-verified-filtered.json b/packages/evals/component_retrieval/eval-test-suite-151-200-verified-filtered.json index 9cae5a27b27c1..57a3a5577c88d 100644 --- a/packages/evals/component_retrieval/eval-test-suite-151-200-verified-filtered.json +++ b/packages/evals/component_retrieval/eval-test-suite-151-200-verified-filtered.json @@ -21,7 +21,7 @@ { "query": "I would like to forward all my Gmail attachments to Telegram", "triggers": [ - "gmail-new-email" + "gmail-new-email-received" ], "actions": [ "telegram_bot_api-send-text-message-or-reply" @@ -165,7 +165,7 @@ { "query": "forward email to telegram", "triggers": [ - "gmail-new-email" + "gmail-new-email-received" ], "actions": [ "telegram_bot_api-send-text-message-or-reply" @@ -201,7 +201,7 @@ { "query": "I would like my Gmail messages to be summarized by GPT", "triggers": [ - "gmail-new-email" + "gmail-new-email-received" ], "actions": [ "openai-chat" diff --git a/packages/evals/component_retrieval/eval-test-suite-no-components-filtered.json b/packages/evals/component_retrieval/eval-test-suite-no-components-filtered.json index ebee7aa2a18bd..1f065d8e5b7cd 100644 --- a/packages/evals/component_retrieval/eval-test-suite-no-components-filtered.json +++ b/packages/evals/component_retrieval/eval-test-suite-no-components-filtered.json @@ -48,7 +48,7 @@ }, { "query": "Send new Snowflake query results to PowerBI", - "triggers": [], + "triggers": ["snowflake-query-results"], "actions": [], "persona": "no-component" }, diff --git a/packages/evals/component_retrieval/eval-test-suite-similar-apps-filtered.json b/packages/evals/component_retrieval/eval-test-suite-similar-apps-filtered.json new file mode 100644 index 0000000000000..ce22a121776b0 --- /dev/null +++ b/packages/evals/component_retrieval/eval-test-suite-similar-apps-filtered.json @@ -0,0 +1,707 @@ +{ + "evaluationTests": [ + { + "query": "add new row in google spreadsheet", + "triggers": [], + "actions": [ + "google_sheets-add-single-row" + ], + "persona": "similar-app" + }, + { + "query": "When someone books through Calendly, add event to Google Calendar", + "triggers": [ + "calendly_v2-new-event-scheduled" + ], + "actions": [ + "google_calendar-create-event" + ], + "persona": "similar-app" + }, + { + "query": "create new task in microsoft todo", + "triggers": [], + "actions": [ + "microsofttodo-create-task" + ], + "persona": "similar-app" + }, + { + "query": "When files are uploaded to Box, create Google Doc", + "triggers": [ + "box-new-file" + ], + "actions": [ + "google_docs-create-document" + ], + "persona": "similar-app" + }, + { + "query": "send message using Discord bot", + "triggers": [], + "actions": [ + "discord_bot-send-message" + ], + "persona": "similar-app" + }, + { + "query": "When TidyCal appointment is booked, create Google Calendar event", + "triggers": [ + "tidycal-new-appointment-booked" + ], + "actions": [ + "google_calendar-create-event" + ], + "persona": "similar-app" + }, + { + "query": "analyze text with Azure OpenAI", + "triggers": [], + "actions": [ + "azure_openai_service-chat" + ], + "persona": "similar-app" + }, + { + "query": "When new email in Gmail, forward to Outlook", + "triggers": [ + "gmail-new-email-received" + ], + "actions": [ + "microsoft_outlook-send-email" + ], + "persona": "similar-app" + }, + { + "query": "When Box gets new file, copy to Dropbox", + "triggers": [ + "box-new-file" + ], + "actions": [ + "dropbox-upload-file" + ], + "persona": "similar-app" + }, + { + "query": "send message through Discord webhook", + "triggers": [], + "actions": [ + "discord-send-message" + ], + "persona": "similar-app" + }, + { + "query": "When files are added to OneDrive folder, save to Google Drive", + "triggers": [ + "microsoft_onedrive-new-file-in-folder" + ], + "actions": [ + "google_drive-upload-file" + ], + "persona": "similar-app" + }, + { + "query": "create new task in Google Tasks when Todoist task is created", + "triggers": [ + "todoist-new-task" + ], + "actions": [ + "google_tasks-create-task" + ], + "persona": "similar-app" + }, + { + "query": "When new Outlook calendar event created, add to Google Calendar", + "triggers": [ + "microsoft_outlook_calendar-new-calendar-event" + ], + "actions": [ + "google_calendar-create-event" + ], + "persona": "similar-app" + }, + { + "query": "use GPT-4 to analyze text with OpenAI", + "triggers": [], + "actions": [ + "openai-chat" + ], + "persona": "similar-app" + }, + { + "query": "When Dropbox file is modified, upload to OneDrive", + "triggers": [ + "dropbox-new-file" + ], + "actions": [ + "microsoft_onedrive-upload-file" + ], + "persona": "similar-app" + }, + { + "query": "send email through Gmail when Zoho Mail receives new message", + "triggers": [ + "zoho_mail-new-email" + ], + "actions": [ + "gmail-send-email" + ], + "persona": "similar-app" + }, + { + "query": "When Google Calendar event starts, create Zoom meeting", + "triggers": [ + "google_calendar-upcoming-event-alert" + ], + "actions": [ + "zoom-create-meeting" + ], + "persona": "similar-app" + }, + { + "query": "use Claude to analyze text", + "triggers": [], + "actions": [ + "anthropic-chat" + ], + "persona": "similar-app" + }, + { + "query": "When Microsoft Todo task is created, add to Todoist", + "triggers": [ + "microsofttodo-new-task-created" + ], + "actions": [ + "todoist-create-task" + ], + "persona": "similar-app" + }, + { + "query": "create new task in Asana when ClickUp task is created", + "triggers": [ + "clickup-new-task" + ], + "actions": [ + "asana-create-task" + ], + "persona": "similar-app" + }, + { + "query": "send transactional email through Mailgun", + "triggers": [], + "actions": [ + "mailgun-send-email" + ], + "persona": "similar-app" + }, + { + "query": "When new subscriber in Mailchimp, add contact to Mailerlite", + "triggers": [ + "mailchimp-new-subscriber" + ], + "actions": [ + "mailerlite-subscribe-to-group" + ], + "persona": "similar-app" + }, + { + "query": "create document in Microsoft OneDrive when Google Drive gets new file", + "triggers": [ + "google_drive-new-files-instant" + ], + "actions": [ + "microsoft_onedrive-upload-file" + ], + "persona": "similar-app" + }, + { + "query": "use Palm API to analyze text", + "triggers": [], + "actions": [ + "google_palm_api-chat" + ], + "persona": "similar-app" + }, + { + "query": "When Zoom recording completes, save to Google Drive and create Box folder", + "triggers": [ + "zoom-recording-completed" + ], + "actions": [ + "google_drive-upload-file", + "box-upload-file" + ], + "persona": "similar-app" + }, + { + "query": "add new row in google spreadsheet", + "triggers": [], + "actions": [ + "google_sheets-add-single-row" + ], + "persona": "similar-app" + }, + { + "query": "When someone books through Calendly, add event to Google Calendar", + "triggers": [ + "calendly_v2-new-event-scheduled" + ], + "actions": [ + "google_calendar-create-event" + ], + "persona": "similar-app" + }, + { + "query": "create new task in microsoft todo", + "triggers": [], + "actions": [ + "microsofttodo-create-task" + ], + "persona": "similar-app" + }, + { + "query": "When files are uploaded to Box, create Google Doc", + "triggers": [ + "box-new-file" + ], + "actions": [ + "google_docs-create-document" + ], + "persona": "similar-app" + }, + { + "query": "send message using Discord bot", + "triggers": [], + "actions": [ + "discord_bot-send-message" + ], + "persona": "similar-app" + }, + { + "query": "When TidyCal appointment is booked, create Google Calendar event", + "triggers": [ + "tidycal-new-appointment-booked" + ], + "actions": [ + "google_calendar-create-event" + ], + "persona": "similar-app" + }, + { + "query": "analyze text with Azure OpenAI", + "triggers": [], + "actions": [ + "azure_openai_service-chat" + ], + "persona": "similar-app" + }, + { + "query": "When new email in Gmail, forward to Outlook", + "triggers": [ + "gmail-new-email" + ], + "actions": [ + "microsoft_outlook-send-email" + ], + "persona": "similar-app" + }, + { + "query": "When Box gets new file, copy to Dropbox", + "triggers": [ + "box-new-file" + ], + "actions": [ + "dropbox-upload-file" + ], + "persona": "similar-app" + }, + { + "query": "send message through Discord webhook", + "triggers": [], + "actions": [ + "discord-send-message" + ], + "persona": "similar-app" + }, + { + "query": "When files are added to OneDrive folder, save to Google Drive", + "triggers": [ + "microsoft_onedrive-new-file-in-folder" + ], + "actions": [ + "google_drive-upload-file" + ], + "persona": "similar-app" + }, + { + "query": "create new task in Google Tasks when Todoist task is created", + "triggers": [ + "todoist-new-task" + ], + "actions": [ + "google_tasks-create-task" + ], + "persona": "similar-app" + }, + { + "query": "When new Outlook calendar event created, add to Google Calendar", + "triggers": [ + "microsoft_outlook_calendar-new-calendar-event" + ], + "actions": [ + "google_calendar-create-event" + ], + "persona": "similar-app" + }, + { + "query": "use GPT-4 to analyze text with OpenAI", + "triggers": [], + "actions": [ + "openai-chat" + ], + "persona": "similar-app" + }, + { + "query": "When Dropbox file is modified, upload to OneDrive", + "triggers": [ + "dropbox-new-file" + ], + "actions": [ + "microsoft_onedrive-upload-file" + ], + "persona": "similar-app" + }, + { + "query": "send email through Gmail when Zoho Mail receives new message", + "triggers": [ + "zoho_mail-new-email" + ], + "actions": [ + "gmail-send-email" + ], + "persona": "similar-app" + }, + { + "query": "When Google Calendar event starts, create Zoom meeting", + "triggers": [ + "google_calendar-upcoming-event-alert" + ], + "actions": [ + "zoom-create-meeting" + ], + "persona": "similar-app" + }, + { + "query": "use Claude to analyze text", + "triggers": [], + "actions": [ + "anthropic-chat" + ], + "persona": "similar-app" + }, + { + "query": "When Microsoft Todo task is created, add to Todoist", + "triggers": [ + "microsofttodo-new-task-created" + ], + "actions": [ + "todoist-create-task" + ], + "persona": "similar-app" + }, + { + "query": "create new task in Asana when ClickUp task is created", + "triggers": [ + "clickup-new-task" + ], + "actions": [ + "asana-create-task" + ], + "persona": "similar-app" + }, + { + "query": "send transactional email through Mailgun", + "triggers": [], + "actions": [ + "mailgun-send-email" + ], + "persona": "similar-app" + }, + { + "query": "When new subscriber in Mailchimp, add contact to Mailerlite", + "triggers": [ + "mailchimp-new-subscriber" + ], + "actions": [ + "mailerlite-subscribe-to-group" + ], + "persona": "similar-app" + }, + { + "query": "create document in Microsoft OneDrive when Google Drive gets new file", + "triggers": [ + "google_drive-new-files-instant" + ], + "actions": [ + "microsoft_onedrive-upload-file" + ], + "persona": "similar-app" + }, + { + "query": "use Palm API to analyze text", + "triggers": [], + "actions": [ + "google_palm_api-chat" + ], + "persona": "similar-app" + }, + { + "query": "When Zoom recording completes, save to Google Drive and create Box folder", + "triggers": [ + "zoom-recording-completed" + ], + "actions": [ + "google_drive-upload-file", + "box-upload-file" + ], + "persona": "similar-app" + }, + { + "query": "When Google Forms gets new response, add to Typeform analytics", + "triggers": [ + "google_forms-new-form-answer" + ], + "actions": [ + "typeform-list-responses" + ], + "persona": "similar-app" + }, + { + "query": "create new presentation in Google Slides when Dropbox gets new file", + "triggers": [ + "dropbox-new-file" + ], + "actions": [ + "google_slides-create-presentation" + ], + "persona": "similar-app" + }, + { + "query": "send message on Microsoft Teams channel", + "triggers": [], + "actions": [ + "microsoft_teams-send-channel-message" + ], + "persona": "similar-app" + }, + { + "query": "When new Gmail attachment arrives, save to Box", + "triggers": [ + "gmail-new-attachment-received" + ], + "actions": [ + "box-upload-file" + ], + "persona": "similar-app" + }, + { + "query": "use Azure OpenAI to translate text", + "triggers": [], + "actions": [ + "azure_openai_service-translate-text" + ], + "persona": "similar-app" + }, + { + "query": "When Microsoft Outlook gets email with label, create Google Calendar event", + "triggers": [ + "microsoft_outlook-new-email" + ], + "actions": [ + "google_calendar-create-event" + ], + "persona": "similar-app" + }, + { + "query": "send notifications through MailerLite", + "triggers": [], + "actions": [ + "mailerlite-subscribe-to-group" + ], + "persona": "similar-app" + }, + { + "query": "When task is completed in Todoist, update Microsoft Todo", + "triggers": [ + "todoist-completed-task" + ], + "actions": [ + "microsofttodo-update-task" + ], + "persona": "similar-app" + }, + { + "query": "use Cohere to analyze sentiment", + "triggers": [], + "actions": [ + "cohere_platform-chat" + ], + "persona": "similar-app" + }, + { + "query": "When someone books through TidyCal, add to Calendly availability", + "triggers": [ + "tidycal-new-appointment-booked" + ], + "actions": [], + "persona": "similar-app" + }, + { + "query": "send chat message through Discord bot when Slack gets message", + "triggers": [ + "slack-new-message-in-channels" + ], + "actions": [ + "discord_bot-send-message" + ], + "persona": "similar-app" + }, + { + "query": "When file changes in OneDrive, sync to Google Drive and Box", + "triggers": [ + "microsoft_onedrive-new-file-created" + ], + "actions": [ + "google_drive-upload-file", + "box-upload-file" + ], + "persona": "similar-app" + }, + { + "query": "create task in Google Tasks when Microsoft Todo list is created", + "triggers": [ + "microsofttodo-new-list-created" + ], + "actions": [ + "google_tasks-create-task" + ], + "persona": "similar-app" + }, + { + "query": "use Mailgun to send transactional emails when Mailchimp campaign is sent", + "triggers": [ + "mailchimp-new-campaign" + ], + "actions": [ + "mailgun-send-email" + ], + "persona": "similar-app" + }, + { + "query": "When Google Calendar event ends, update Outlook Calendar", + "triggers": [ + "google_calendar-event-cancelled" + ], + "actions": [ + "microsoft_outlook_calendar-update-calendar-event" + ], + "persona": "similar-app" + }, + { + "query": "summarize text with OpenAI GPT-4", + "triggers": [], + "actions": [ + "openai-summarize" + ], + "persona": "similar-app" + }, + { + "query": "When Zoho Mail gets new email, forward through Microsoft Outlook", + "triggers": [ + "zoho_mail-new-email" + ], + "actions": [ + "microsoft_outlook-send-email" + ], + "persona": "similar-app" + }, + { + "query": "create new worksheet in Google Sheets when Smartsheet updates row", + "triggers": [ + "smartsheet-new-row-updated" + ], + "actions": [ + "google_sheets-create-worksheet" + ], + "persona": "similar-app" + }, + { + "query": "use Anthropic Claude to generate email content", + "triggers": [], + "actions": [ + "anthropic-chat" + ], + "persona": "similar-app" + }, + { + "query": "When Box gets new file, upload to OneDrive and Google Drive", + "triggers": [ + "box-new-file" + ], + "actions": [ + "microsoft_onedrive-upload-file", + "google_drive-upload-file" + ], + "persona": "similar-app" + }, + { + "query": "analyze text with Google Palm API", + "triggers": [], + "actions": [ + "google_palm_api-generate-text" + ], + "persona": "similar-app" + }, + { + "query": "When Dropbox folder changes, sync to Box and OneDrive", + "triggers": [ + "dropbox-all-updates" + ], + "actions": [ + "box-upload-file", + "microsoft_onedrive-upload-file" + ], + "persona": "similar-app" + }, + { + "query": "create event in Zoho Calendar when Google Calendar event is updated", + "triggers": [ + "google_calendar-new-or-updated-event-instant" + ], + "actions": [ + "zoho_calendar-create-or-update-event" + ], + "persona": "similar-app" + }, + { + "query": "When Discord gets reaction, notify through Microsoft Teams", + "triggers": [ + "discord-reaction-added" + ], + "actions": [ + "microsoft_teams-send-channel-message" + ], + "persona": "similar-app" + }, + { + "query": "send Mailjet msg when MailerLite subscriber is added", + "triggers": [ + "mailerlite-subscriber-added-from-form" + ], + "actions": [ + "mailjet-send-message" + ], + "persona": "similar-app" + } + ] +} \ No newline at end of file From 4f35c8c5e5830ddd4c47da65387c8caab619b64e Mon Sep 17 00:00:00 2001 From: Biz Melesse Date: Thu, 24 Oct 2024 12:12:04 -0700 Subject: [PATCH 05/18] Output eval results as a csv --- .../evals/component_retrieval/evaluator.mjs | 35 ++++++++++++++++++- .../component_retrieval/package-lock.json | 29 +++++++++++++++ .../evals/component_retrieval/package.json | 1 + 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/packages/evals/component_retrieval/evaluator.mjs b/packages/evals/component_retrieval/evaluator.mjs index 7dabb7ca181b0..a0a9f2d9122a2 100644 --- a/packages/evals/component_retrieval/evaluator.mjs +++ b/packages/evals/component_retrieval/evaluator.mjs @@ -4,12 +4,14 @@ import "dotenv/config"; import fs from "fs/promises"; import path from "path"; import { diff } from "json-diff"; +import { json2csv } from "json-2-csv"; const GREEN_CHECK = "\x1b[32m✔\x1b[0m"; const RED_CROSS = "\x1b[31m✖\x1b[0m"; let totalEvals = 0; let totalSuccesses = 0; +let apiResults = [] const apiHost = process.env.API_BASE_URL || "https://api.pipedream.com"; @@ -41,7 +43,23 @@ function customDiff(original, updated, oldLabel = "expected", newLabel = "actual return replaceLabels(result); } +async function exportToCsv(filePath, limit, threshold) { + const csvData = json2csv(apiResults, { + fields: ["query", "evalTriggers", "apiTriggers", "evalActions", "apiActions", "success"] + }); + const parts = filePath.split("/") + const path = parts[parts.length -1].split(".json")[0] + await fs.writeFile(`./csv/${path}-${limit}-${threshold}.csv`, csvData); +} + +function arrayToString(items) { + if (items) return items.join(",") + return "" +} + async function processEvalFile(filePath) { + const limit = 3 + const threshold = 0.65 try { const content = await fs.readFile(filePath, "utf-8"); const evalData = JSON.parse(content); @@ -53,7 +71,7 @@ async function processEvalFile(filePath) { } = evalTest; const encodedQuery = encodeURIComponent(query); - const apiUrl = `${apiHost}/v1/components/search?query=${encodedQuery}`; + const apiUrl = `${apiHost}/v1/components/search?query=${encodedQuery}&similarity_threshold=${threshold}&limit=${limit}`; const response = await fetch(apiUrl, { headers: { @@ -63,16 +81,20 @@ async function processEvalFile(filePath) { }); const apiData = await response.json(); + // Compare actual and expected const apiTriggers = apiData?.triggers ?? []; const apiActions = apiData?.actions ?? []; + const triggersMatch = JSON.stringify(apiTriggers.sort()) === JSON.stringify(triggers.sort()); const actionsMatch = JSON.stringify(apiActions.sort()) === JSON.stringify(actions.sort()); + let success = false if (triggersMatch && actionsMatch) { totalSuccesses++; + success = true console.log(`${GREEN_CHECK} Success for query: "${query}"`); } else { console.log(`${RED_CROSS} Failure for query: "${query}"`); @@ -82,10 +104,21 @@ async function processEvalFile(filePath) { actions, }, apiData)); } + + const record = { + query: query.replace("\"", ""), + apiTriggers: arrayToString(apiTriggers), + apiActions: arrayToString(apiActions), + evalTriggers: arrayToString(triggers), + evalActions: arrayToString(actions), + success: success + }; + apiResults.push(record) } } catch (error) { console.error(`Error processing file ${filePath}:`, error.message); } + await exportToCsv(filePath, limit, threshold) } async function main() { diff --git a/packages/evals/component_retrieval/package-lock.json b/packages/evals/component_retrieval/package-lock.json index eeebfb5eb738d..0f1736b44be18 100644 --- a/packages/evals/component_retrieval/package-lock.json +++ b/packages/evals/component_retrieval/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.1", "dependencies": { "dotenv": "^16.4.5", + "json-2-csv": "^5.5.6", "json-diff": "^1.0.6" } }, @@ -28,6 +29,22 @@ "node": ">=0.1.90" } }, + "node_modules/deeks": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/deeks/-/deeks-3.1.0.tgz", + "integrity": "sha512-e7oWH1LzIdv/prMQ7pmlDlaVoL64glqzvNgkgQNgyec9ORPHrT2jaOqMtRyqJuwWjtfb6v+2rk9pmaHj+F137A==", + "engines": { + "node": ">= 16" + } + }, + "node_modules/doc-path": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/doc-path/-/doc-path-4.1.1.tgz", + "integrity": "sha512-h1ErTglQAVv2gCnOpD3sFS6uolDbOKHDU1BZq+Kl3npPqroU3dYL42lUgMfd5UimlwtRgp7C9dLGwqQ5D2HYgQ==", + "engines": { + "node": ">=16" + } + }, "node_modules/dotenv": { "version": "16.4.5", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", @@ -55,6 +72,18 @@ "resolved": "https://registry.npmjs.org/heap/-/heap-0.2.7.tgz", "integrity": "sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==" }, + "node_modules/json-2-csv": { + "version": "5.5.6", + "resolved": "https://registry.npmjs.org/json-2-csv/-/json-2-csv-5.5.6.tgz", + "integrity": "sha512-N673XbJgHwUq9JreKpk530jSywPF/rEAQ08dV99QQpkluP/4HTwshpoP9hmDz26iSFqu7eNAPgyJfu/77HvPGA==", + "dependencies": { + "deeks": "3.1.0", + "doc-path": "4.1.1" + }, + "engines": { + "node": ">= 16" + } + }, "node_modules/json-diff": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/json-diff/-/json-diff-1.0.6.tgz", diff --git a/packages/evals/component_retrieval/package.json b/packages/evals/component_retrieval/package.json index 5b5ca619bc3f6..6f16913e997f5 100644 --- a/packages/evals/component_retrieval/package.json +++ b/packages/evals/component_retrieval/package.json @@ -5,6 +5,7 @@ "main": "evaluator.mjs", "dependencies": { "dotenv": "^16.4.5", + "json-2-csv": "^5.5.6", "json-diff": "^1.0.6" } } From a95b5b6c458d859b61ab187a7b1020d03a8cbb08 Mon Sep 17 00:00:00 2001 From: "Dylan J. Sather" Date: Thu, 24 Oct 2024 18:09:40 -0700 Subject: [PATCH 06/18] triggers -> sources on eval JSON Adding precision, recall, F1 score to metrics Add --debug flag --- packages/evals/component_retrieval/.gitignore | 1 + .../eval-test-suite-0-100-filtered.json | 613 +++++------------- ...-test-suite-101-150-verified-filtered.json | 351 +++------- ...-test-suite-151-200-verified-filtered.json | 360 +++------- ...l-test-suite-201-250-complex-filtered.json | 372 +++-------- .../eval-test-suite-251-300-new-filtered.json | 344 +++------- .../eval-test-suite-ai-focus-filtered.json | 160 ++--- ...st-suite-graduated-corrected-filtered.json | 108 +-- ...l-test-suite-invalid-queries-filtered.json | 104 +-- ...val-test-suite-no-components-filtered.json | 122 ++-- ...eval-test-suite-similar-apps-filtered.json | 553 +++++----------- .../evals/component_retrieval/evaluator.mjs | 215 ++++-- .../component_retrieval/package-lock.json | 9 + .../evals/component_retrieval/package.json | 1 + pnpm-lock.yaml | 20 + 15 files changed, 1079 insertions(+), 2254 deletions(-) diff --git a/packages/evals/component_retrieval/.gitignore b/packages/evals/component_retrieval/.gitignore index 4c49bd78f1d08..e7bf0c84d3e5c 100644 --- a/packages/evals/component_retrieval/.gitignore +++ b/packages/evals/component_retrieval/.gitignore @@ -1 +1,2 @@ .env +csv diff --git a/packages/evals/component_retrieval/eval-test-suite-0-100-filtered.json b/packages/evals/component_retrieval/eval-test-suite-0-100-filtered.json index 46d06a08a78a3..e7d7e64d1af1c 100644 --- a/packages/evals/component_retrieval/eval-test-suite-0-100-filtered.json +++ b/packages/evals/component_retrieval/eval-test-suite-0-100-filtered.json @@ -2,153 +2,103 @@ "evaluationTests": [ { "query": "When a lead is created in Salesforce, create a task in Asana and send a message in Slack", - "triggers": [ - "salesforce_rest_api-new-record-instant" - ], - "actions": [ - "asana-create-task", - "slack-send-message" - ], + "sources": ["salesforce_rest_api-new-record-instant"], + "actions": ["asana-create-task", "slack-send-message"], "persona": "complex-workflow" }, { "query": "add new hubspot contacts to mailchimp", - "triggers": [ - "hubspot-new-contact" - ], - "actions": [ - "mailchimp-add-or-update-subscriber" - ], + "sources": ["hubspot-new-contact"], + "actions": ["mailchimp-add-or-update-subscriber"], "persona": "succinct" }, { "query": "how do I get a phone number", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "when new emails arrive in Gmail, use OpenAI to summarize them and post to Slack", - "triggers": [ - "gmail-new-email-received" - ], - "actions": [ - "openai-chat", - "slack-send-message" - ], + "sources": ["gmail-new-email-received"], + "actions": ["openai-chat", "slack-send-message"], "persona": "complex-workflow" }, { "query": "watch airtable for new records", - "triggers": [ - "airtable_oauth-new-records" - ], + "sources": ["airtable_oauth-new-records"], "actions": [], "persona": "succinct" }, { "query": "create jira ticket", - "triggers": [], - "actions": [ - "jira-create-issue" - ], + "sources": [], + "actions": ["jira-create-issue"], "persona": "succinct" }, { "query": "When new Zendesk tickets arrive, analyze sentiment with OpenAI and prioritize in Linear", - "triggers": [ - "zendesk-new-ticket" - ], - "actions": [ - "openai-chat", - "linear-create-issue" - ], + "sources": ["zendesk-new-ticket"], + "actions": ["openai-chat", "linear-create-issue"], "persona": "complex-workflow" }, { "query": "how to make database connection", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "When someone mentions our company on Twitter, create a Zendesk ticket", - "triggers": [ - "twitter-new-mention-received-by-user" - ], - "actions": [ - "zendesk-create-ticket" - ], + "sources": ["twitter-new-mention-received-by-user"], + "actions": ["zendesk-create-ticket"], "persona": "complex-workflow" }, { "query": "use gpt to answer intercom messages", - "triggers": [ - "intercom-new-conversation" - ], - "actions": [ - "openai-chat" - ], + "sources": ["intercom-new-conversation"], + "actions": ["openai-chat"], "persona": "task-oriented" }, { "query": "I would like to automatically add my Calendly meetings to Google Calendar", - "triggers": [ - "calendly_v2-new-event-scheduled" - ], - "actions": [ - "google_calendar-create-event" - ], + "sources": ["calendly_v2-new-event-scheduled"], + "actions": ["google_calendar-create-event"], "persona": "verbose" }, { "query": "noshan page updated", - "triggers": [ - "notion-updated-page" - ], + "sources": ["notion-updated-page"], "actions": [], "persona": "sloppy" }, { "query": "create new linear issue", - "triggers": [], - "actions": [ - "linear-create-issue" - ], + "sources": [], + "actions": ["linear-create-issue"], "persona": "succinct" }, { "query": "When a customer submits a response in Typeform, send it through GPT for analysis and create a Notion page", - "triggers": [ - "typeform-new-submission" - ], - "actions": [ - "openai-chat", - "notion-create-page-from-database" - ], + "sources": ["typeform-new-submission"], + "actions": ["openai-chat", "notion-create-page-from-database"], "persona": "complex-workflow" }, { "query": "new row in googel sheet", - "triggers": [ - "google_sheets-new-row-added" - ], + "sources": ["google_sheets-new-row-added"], "actions": [], "persona": "sloppy" }, { "query": "create task in monday", - "triggers": [], - "actions": [ - "monday-create-item" - ], + "sources": [], + "actions": ["monday-create-item"], "persona": "succinct" }, { "query": "When a new deal is created in HubSpot, transcribe the discovery call with OpenAI and update the contact", - "triggers": [ - "hubspot-new-deal" - ], + "sources": ["hubspot-new-deal"], "actions": [ "openai-create-transcription", "hubspot-create-or-update-contact" @@ -157,309 +107,217 @@ }, { "query": "what is the weather today", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "send gong recordings to slack", - "triggers": [ - "gong-new-call" - ], - "actions": [ - "slack-send-message" - ], + "sources": ["gong-new-call"], + "actions": ["slack-send-message"], "persona": "task-oriented" }, { "query": "When a new customer is added in Stripe, create their account in Notion", - "triggers": [ - "stripe-new-customer" - ], - "actions": [ - "notion-create-page-from-database" - ], + "sources": ["stripe-new-customer"], + "actions": ["notion-create-page-from-database"], "persona": "complex-workflow" }, { "query": "update asana task status", - "triggers": [], - "actions": [ - "asana-update-task" - ], + "sources": [], + "actions": ["asana-update-task"], "persona": "succinct" }, { "query": "When Sentry reports an error, create an issue in Linear", - "triggers": [ - "sentry-issue-event" - ], - "actions": [ - "linear-create-issue" - ], + "sources": ["sentry-issue-event"], + "actions": ["linear-create-issue"], "persona": "verbose" }, { "query": "convert file to pdf", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "When a form is submitted on Webflow, create contact in HubSpot", - "triggers": [ - "webflow-new-form-submission" - ], - "actions": [ - "hubspot-create-or-update-contact" - ], + "sources": ["webflow-new-form-submission"], + "actions": ["hubspot-create-or-update-contact"], "persona": "complex-workflow" }, { "query": "watch 4 new stripe payments", - "triggers": [ - "stripe-new-payment" - ], + "sources": ["stripe-new-payment"], "actions": [], "persona": "sloppy" }, { "query": "When support tickets come in after hours, use GPT to draft a response and send it through Gmail", - "triggers": [ - "zendesk-new-ticket" - ], - "actions": [ - "openai-chat", - "gmail-send-email" - ], + "sources": ["zendesk-new-ticket"], + "actions": ["openai-chat", "gmail-send-email"], "persona": "complex-workflow" }, { "query": "When customer feedback arrives in Delighted, analyze sentiment with AI and update Salesforce", - "triggers": [], - "actions": [ - "openai-chat", - "salesforce_rest_api-update-contact" - ], + "sources": [], + "actions": ["openai-chat", "salesforce_rest_api-update-contact"], "persona": "complex-workflow" }, { "query": "upload file to dropbox", - "triggers": [], - "actions": [ - "dropbox-upload-file" - ], + "sources": [], + "actions": ["dropbox-upload-file"], "persona": "succinct" }, { "query": "I need new Linear issues to be automatically added to our Notion database", - "triggers": [ - "linear-issue-created-instant" - ], - "actions": [ - "notion-create-page-from-database" - ], + "sources": ["linear-issue-created-instant"], + "actions": ["notion-create-page-from-database"], "persona": "verbose" }, { "query": "why is my code not working", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "When an opportunity changes in Salesforce, update the record in HubSpot", - "triggers": [], - "actions": [ - "hubspot-create-or-update-contact" - ], + "sources": [], + "actions": ["hubspot-create-or-update-contact"], "persona": "complex-workflow" }, { "query": "create new document in Google Docs", - "triggers": [], - "actions": [ - "google_docs-create-document" - ], + "sources": [], + "actions": ["google_docs-create-document"], "persona": "succinct" }, { "query": "update zendesk ticket", - "triggers": [], - "actions": [ - "zendesk-update-ticket" - ], + "sources": [], + "actions": ["zendesk-update-ticket"], "persona": "succinct" }, { "query": "analyze customer support tickets with GPT", - "triggers": [], - "actions": [ - "openai-chat" - ], + "sources": [], + "actions": ["openai-chat"], "persona": "task-oriented" }, { "query": "When a deal closes in Pipedrive, generate an invoice in QuickBooks", - "triggers": [ - "pipedrive-updated-deal" - ], - "actions": [ - "quickbooks-create-invoice" - ], + "sources": ["pipedrive-updated-deal"], + "actions": ["quickbooks-create-invoice"], "persona": "complex-workflow" }, { "query": "add contact to salesforce", - "triggers": [], - "actions": [ - "salesforce_rest_api-create-contact" - ], + "sources": [], + "actions": ["salesforce_rest_api-create-contact"], "persona": "succinct" }, { "query": "When a customer replies to our survey, use AI to analyze it and update their profile in HubSpot", - "triggers": [ - "survey_monkey-new-survey-response" - ], - "actions": [ - "openai-chat", - "hubspot-create-or-update-contact" - ], + "sources": ["survey_monkey-new-survey-response"], + "actions": ["openai-chat", "hubspot-create-or-update-contact"], "persona": "complex-workflow" }, { "query": "cant find my data", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "send Linear issues to Slack", - "triggers": [ - "linear-issue-created-instant" - ], - "actions": [ - "slack-send-message" - ], + "sources": ["linear-issue-created-instant"], + "actions": ["slack-send-message"], "persona": "verbose" }, { "query": "create new noshun page", - "triggers": [], - "actions": [ - "notion-create-page" - ], + "sources": [], + "actions": ["notion-create-page"], "persona": "sloppy" }, { "query": "When a customer signs up, add them to Mailchimp", - "triggers": [], - "actions": [ - "mailchimp-add-or-update-subscriber" - ], + "sources": [], + "actions": ["mailchimp-add-or-update-subscriber"], "persona": "task-oriented" }, { "query": "create basecamp project", - "triggers": [], + "sources": [], "actions": [], "persona": "succinct" }, { "query": "translate new Notion pages with GPT", - "triggers": [ - "notion-new-page" - ], - "actions": [ - "openai-chat" - ], + "sources": ["notion-new-page"], + "actions": ["openai-chat"], "persona": "task-oriented" }, { "query": "When a critical error occurs in Sentry, create PagerDuty incident", - "triggers": [ - "sentry-issue-event" - ], - "actions": [ - "pagerduty-trigger-incident" - ], + "sources": ["sentry-issue-event"], + "actions": ["pagerduty-trigger-incident"], "persona": "complex-workflow" }, { "query": "add row 2 spreadsheet", - "triggers": [], - "actions": [ - "google_sheets-add-single-row" - ], + "sources": [], + "actions": ["google_sheets-add-single-row"], "persona": "sloppy" }, { "query": "I would like to be notified in Slack when new issues are created in Linear", - "triggers": [ - "linear-issue-created-instant" - ], - "actions": [ - "slack-send-message" - ], + "sources": ["linear-issue-created-instant"], + "actions": ["slack-send-message"], "persona": "verbose" }, { "query": "create task in asna", - "triggers": [], - "actions": [ - "asana-create-task" - ], + "sources": [], + "actions": ["asana-create-task"], "persona": "sloppy" }, { "query": "When a large transaction occurs in Stripe, notify me in Slack", - "triggers": [ - "stripe-new-payment" - ], - "actions": [ - "slack-send-message" - ], + "sources": ["stripe-new-payment"], + "actions": ["slack-send-message"], "persona": "task-oriented" }, { "query": "post message in discord", - "triggers": [], - "actions": [ - "discord-send-message" - ], + "sources": [], + "actions": ["discord-send-message"], "persona": "succinct" }, { "query": "When someone fills out our Typeform, use GPT to qualify the lead and add to HubSpot", - "triggers": [ - "typeform-new-submission" - ], - "actions": [ - "openai-chat", - "hubspot-create-or-update-contact" - ], + "sources": ["typeform-new-submission"], + "actions": ["openai-chat", "hubspot-create-or-update-contact"], "persona": "complex-workflow" }, { "query": "how to center text", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "create new page in Confluence", - "triggers": [], - "actions": [ - "onenote-create-page" - ], + "sources": [], + "actions": ["onenote-create-page"], "persona": "succinct" }, { "query": "When a meeting ends in Zoom, transcribe it with OpenAI and save to Notion", - "triggers": [ - "zoom-meeting-ended" - ], + "sources": ["zoom-meeting-ended"], "actions": [ "openai-create-transcription", "notion-create-page-from-database" @@ -468,43 +326,31 @@ }, { "query": "update clickup task", - "triggers": [], - "actions": [ - "clickup-update-task" - ], + "sources": [], + "actions": ["clickup-update-task"], "persona": "succinct" }, { "query": "I need all my Gmail attachments to be analyzed by GPT", - "triggers": [ - "gmail-new-email" - ], - "actions": [ - "openai-chat" - ], + "sources": ["gmail-new-email"], + "actions": ["openai-chat"], "persona": "verbose" }, { "query": "create quickboks invoice", - "triggers": [], - "actions": [ - "quickbooks-create-invoice" - ], + "sources": [], + "actions": ["quickbooks-create-invoice"], "persona": "sloppy" }, { "query": "When a deal updates in Close.com, sync it to HubSpot", - "triggers": [], - "actions": [ - "hubspot-create-or-update-contact" - ], + "sources": [], + "actions": ["hubspot-create-or-update-contact"], "persona": "task-oriented" }, { "query": "When a sales call is recorded in Gong, transcribe with OpenAI and update the deal in Salesforce", - "triggers": [ - "gong-new-call" - ], + "sources": ["gong-new-call"], "actions": [ "openai-create-transcription", "salesforce_rest_api-update-opportunity" @@ -513,192 +359,139 @@ }, { "query": "generate email content with GPT", - "triggers": [], - "actions": [ - "openai-chat" - ], + "sources": [], + "actions": ["openai-chat"], "persona": "succinct" }, { "query": "I would like all new Airtable records to be enriched using OpenAI", - "triggers": [ - "airtable_oauth-new-records" - ], - "actions": [ - "openai-chat" - ], + "sources": ["airtable_oauth-new-records"], + "actions": ["openai-chat"], "persona": "verbose" }, { "query": "make chatgpt write emails", - "triggers": [], - "actions": [ - "openai-chat" - ], + "sources": [], + "actions": ["openai-chat"], "persona": "sloppy" }, { "query": "When customers respond in Help Scout, analyze sentiment with GPT and update their status in HubSpot", - "triggers": [], - "actions": [ - "openai-chat", - "hubspot-create-or-update-contact" - ], + "sources": [], + "actions": ["openai-chat", "hubspot-create-or-update-contact"], "persona": "complex-workflow" }, { "query": "sync my google drive", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "create new event in google calendar", - "triggers": [], - "actions": [ - "google_calendar-create-event" - ], + "sources": [], + "actions": ["google_calendar-create-event"], "persona": "succinct" }, { "query": "When a customer's subscription changes in Chargebee, update their record in Salesforce", - "triggers": [], - "actions": [ - "salesforce_rest_api-update-contact" - ], + "sources": [], + "actions": ["salesforce_rest_api-update-contact"], "persona": "complex-workflow" }, { "query": "post 2 twiter", - "triggers": [], - "actions": [ - "twitter-create-tweet" - ], + "sources": [], + "actions": ["twitter-create-tweet"], "persona": "sloppy" }, { "query": "I need all Salesforce opportunities to be automatically added to our Monday.com board", - "triggers": [ - "salesforce_rest_api-new-record-instant" - ], - "actions": [ - "monday-create-item" - ], + "sources": ["salesforce_rest_api-new-record-instant"], + "actions": ["monday-create-item"], "persona": "verbose" }, { "query": "When new issues are created in GitHub, use GPT to analyze severity and create PagerDuty incidents", - "triggers": [], - "actions": [ - "openai-chat", - "pagerduty-trigger-incident" - ], + "sources": [], + "actions": ["openai-chat", "pagerduty-trigger-incident"], "persona": "complex-workflow" }, { "query": "make new asana project", - "triggers": [], - "actions": [ - "asana-create-project" - ], + "sources": [], + "actions": ["asana-create-project"], "persona": "succinct" }, { "query": "When customer reviews come in through Trustpilot, analyze them with AI and notify in Slack", - "triggers": [], - "actions": [ - "openai-chat", - "slack-send-message" - ], + "sources": [], + "actions": ["openai-chat", "slack-send-message"], "persona": "complex-workflow" }, { "query": "upload files to dropbox", - "triggers": [], - "actions": [ - "dropbox-upload-file" - ], + "sources": [], + "actions": ["dropbox-upload-file"], "persona": "succinct" }, { "query": "where is my data stored", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "send mailchmp campaign", - "triggers": [], - "actions": [ - "mailchimp-create-campaign" - ], + "sources": [], + "actions": ["mailchimp-create-campaign"], "persona": "sloppy" }, { "query": "I want new Shopify orders to be automatically processed in NetSuite", - "triggers": [ - "shopify_developer_app-new-order-created" - ], + "sources": ["shopify_developer_app-new-order-created"], "actions": [], "persona": "verbose" }, { "query": "When a document is uploaded to Google Drive, use GPT to summarize it and post in Slack", - "triggers": [ - "google_drive-new-files-instant" - ], - "actions": [ - "openai-chat", - "slack-send-message" - ], + "sources": ["google_drive-new-files-instant"], + "actions": ["openai-chat", "slack-send-message"], "persona": "complex-workflow" }, { "query": "create linear ticket", - "triggers": [], - "actions": [ - "linear-create-issue" - ], + "sources": [], + "actions": ["linear-create-issue"], "persona": "succinct" }, { "query": "When a user submits feedback in Canny, analyze it with AI and create tasks in ClickUp", - "triggers": [], - "actions": [ - "openai-chat", - "clickup-create-task" - ], + "sources": [], + "actions": ["openai-chat", "clickup-create-task"], "persona": "complex-workflow" }, { "query": "send invoice via xero", - "triggers": [], - "actions": [ - "xero_accounting_api-email-an-invoice" - ], + "sources": [], + "actions": ["xero_accounting_api-email-an-invoice"], "persona": "task-oriented" }, { "query": "hubspt deal updated", - "triggers": [ - "hubspot-deal-updated" - ], + "sources": ["hubspot-deal-updated"], "actions": [], "persona": "sloppy" }, { "query": "I would like to automatically generate reports when new data appears in Amplitude", - "triggers": [], - "actions": [ - "google_sheets-add-single-row" - ], + "sources": [], + "actions": ["google_sheets-add-single-row"], "persona": "verbose" }, { "query": "When a new video is uploaded to Vimeo, transcribe it with OpenAI and create a Notion page", - "triggers": [ - "vimeo-new-video-of-mine" - ], + "sources": ["vimeo-new-video-of-mine"], "actions": [ "openai-create-transcription", "notion-create-page-from-database" @@ -707,171 +500,123 @@ }, { "query": "analyze text with gpt", - "triggers": [], - "actions": [ - "openai-chat" - ], + "sources": [], + "actions": ["openai-chat"], "persona": "succinct" }, { "query": "When a trial user signs up in Stripe, add them to HubSpot and start onboarding sequence in Customer.io", - "triggers": [ - "stripe-new-customer" - ], - "actions": [ - "hubspot-create-or-update-contact" - ], + "sources": ["stripe-new-customer"], + "actions": ["hubspot-create-or-update-contact"], "persona": "complex-workflow" }, { "query": "create new folder in gdrive", - "triggers": [], - "actions": [ - "google_drive-create-folder" - ], + "sources": [], + "actions": ["google_drive-create-folder"], "persona": "succinct" }, { "query": "update record in airetable", - "triggers": [], - "actions": [ - "airtable_oauth-update-record" - ], + "sources": [], + "actions": ["airtable_oauth-update-record"], "persona": "sloppy" }, { "query": "help me configure postgres", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "When an event starts in Google Calendar, send a message in Telegram", - "triggers": [ - "google_calendar-upcoming-event-alert" - ], - "actions": [ - "telegram_bot_api-send-text-message-or-reply" - ], + "sources": ["google_calendar-upcoming-event-alert"], + "actions": ["telegram_bot_api-send-text-message-or-reply"], "persona": "complex-workflow" }, { "query": "create task in todoist", - "triggers": [], - "actions": [ - "todoist-create-task" - ], + "sources": [], + "actions": ["todoist-create-task"], "persona": "succinct" }, { "query": "I want to automatically catalog all my Loom videos in Notion", - "triggers": [], - "actions": [ - "notion-create-page-from-database" - ], + "sources": [], + "actions": ["notion-create-page-from-database"], "persona": "verbose" }, { "query": "When a form is submitted in Reform, analyze responses with GPT and update Airtable", - "triggers": [], - "actions": [ - "openai-chat", - "airtable_oauth-update-record" - ], + "sources": [], + "actions": ["openai-chat", "airtable_oauth-update-record"], "persona": "complex-workflow" }, { "query": "make new trello board", - "triggers": [], + "sources": [], "actions": [], "persona": "succinct" }, { "query": "sync contacts between platforms", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "When a product is updated in Productboard, notify stakeholders in Slack", - "triggers": [ - "productboard-new-feature" - ], - "actions": [ - "slack-send-message" - ], + "sources": ["productboard-new-feature"], + "actions": ["slack-send-message"], "persona": "task-oriented" }, { "query": "genrate presentation with ai", - "triggers": [], - "actions": [ - "openai-chat" - ], + "sources": [], + "actions": ["openai-chat"], "persona": "sloppy" }, { "query": "I need all customer conversations from Front to be analyzed by GPT", - "triggers": [ - "frontapp-new-conversation-state-change" - ], - "actions": [ - "openai-chat" - ], + "sources": ["frontapp-new-conversation-state-change"], + "actions": ["openai-chat"], "persona": "verbose" }, { "query": "When someone stars our GitHub repo, send a thank you email with GPT", - "triggers": [], - "actions": [ - "openai-chat", - "gmail-send-email" - ], + "sources": [], + "actions": ["openai-chat", "gmail-send-email"], "persona": "complex-workflow" }, { "query": "create new card trelo", - "triggers": [], - "actions": [ - "trello-create-card" - ], + "sources": [], + "actions": ["trello-create-card"], "persona": "sloppy" }, { "query": "track time in toggl", - "triggers": [], + "sources": [], "actions": [], "persona": "succinct" }, { "query": "I would like to be notified in Slack when new videos are posted to YouTube", - "triggers": [ - "youtube_data_api-new-videos-in-channel" - ], - "actions": [ - "slack-send-message" - ], + "sources": ["youtube_data_api-new-videos-in-channel"], + "actions": ["slack-send-message"], "persona": "verbose" }, { "query": "When a customer reaches a usage threshold in Stripe, notify the success team in Slack", - "triggers": [ - "stripe-subscription-updated" - ], - "actions": [ - "slack-send-message" - ], + "sources": ["stripe-subscription-updated"], + "actions": ["slack-send-message"], "persona": "task-oriented" }, { "query": "forward emails to telegram", - "triggers": [ - "gmail-new-email" - ], - "actions": [ - "telegram_bot_api-send-text-message-or-reply" - ], + "sources": ["gmail-new-email"], + "actions": ["telegram_bot_api-send-text-message-or-reply"], "persona": "task-oriented" } ] -} \ No newline at end of file +} diff --git a/packages/evals/component_retrieval/eval-test-suite-101-150-verified-filtered.json b/packages/evals/component_retrieval/eval-test-suite-101-150-verified-filtered.json index 3f55d7f14c602..bbb90aa9f3375 100644 --- a/packages/evals/component_retrieval/eval-test-suite-101-150-verified-filtered.json +++ b/packages/evals/component_retrieval/eval-test-suite-101-150-verified-filtered.json @@ -2,454 +2,303 @@ "evaluationTests": [ { "query": "When a deal closes in Salesforce, create invoice in QuickBooks", - "triggers": [], - "actions": [ - "quickbooks-create-invoice" - ], + "sources": [], + "actions": ["quickbooks-create-invoice"], "persona": "complex-workflow" }, { "query": "add new subscriber to mailchimp", - "triggers": [], - "actions": [ - "mailchimp-add-subscriber-to-tag" - ], + "sources": [], + "actions": ["mailchimp-add-subscriber-to-tag"], "persona": "succinct" }, { "query": "I would like GPT to analyze new Typeform responses", - "triggers": [ - "typeform-new-submission" - ], - "actions": [ - "openai-chat" - ], + "sources": ["typeform-new-submission"], + "actions": ["openai-chat"], "persona": "verbose" }, { "query": "When a new row appears in Google Sheets, create a Notion page", - "triggers": [ - "google_sheets-new-row-added" - ], - "actions": [ - "notion-create-page-from-database" - ], + "sources": ["google_sheets-new-row-added"], + "actions": ["notion-create-page-from-database"], "persona": "complex-workflow" }, { "query": "salesfrce contact update", - "triggers": [], - "actions": [ - "salesforce_rest_api-update-contact" - ], + "sources": [], + "actions": ["salesforce_rest_api-update-contact"], "persona": "sloppy" }, { "query": "create task in linear", - "triggers": [], - "actions": [ - "linear-create-issue" - ], + "sources": [], + "actions": ["linear-create-issue"], "persona": "succinct" }, { "query": "When new Jira issues are created, post them to Slack", - "triggers": [ - "jira-issue-created" - ], - "actions": [ - "slack-send-message" - ], + "sources": ["jira-issue-created"], + "actions": ["slack-send-message"], "persona": "complex-workflow" }, { "query": "create trello card", - "triggers": [], - "actions": [ - "trello-create-card" - ], + "sources": [], + "actions": ["trello-create-card"], "persona": "succinct" }, { "query": "I need all new Calendly bookings to be added to Google Calendar", - "triggers": [ - "calendly_v2-new-event-scheduled" - ], - "actions": [ - "google_calendar-create-event" - ], + "sources": ["calendly_v2-new-event-scheduled"], + "actions": ["google_calendar-create-event"], "persona": "verbose" }, { "query": "update deal in hubspt", - "triggers": [], - "actions": [ - "hubspot-create-or-update-contact" - ], + "sources": [], + "actions": ["hubspot-create-or-update-contact"], "persona": "sloppy" }, { "query": "When a video is uploaded to YouTube, transcribe it with OpenAI", - "triggers": [ - "youtube_data_api-new-videos" - ], - "actions": [ - "openai-create-transcription" - ], + "sources": ["youtube_data_api-new-videos"], + "actions": ["openai-create-transcription"], "persona": "complex-workflow" }, { "query": "send gmail message", - "triggers": [], - "actions": [ - "gmail-send-email" - ], + "sources": [], + "actions": ["gmail-send-email"], "persona": "succinct" }, { "query": "When files are added to Dropbox, create a Notion page", - "triggers": [ - "dropbox-new-file" - ], - "actions": [ - "notion-create-page-from-database" - ], + "sources": ["dropbox-new-file"], + "actions": ["notion-create-page-from-database"], "persona": "complex-workflow" }, { "query": "use gpt for email", - "triggers": [], - "actions": [ - "openai-chat" - ], + "sources": [], + "actions": ["openai-chat"], "persona": "succinct" }, { "query": "I would like new Twitter mentions to create Zendesk tickets", - "triggers": [ - "twitter-new-mention-received-by-user" - ], - "actions": [ - "zendesk-create-ticket" - ], + "sources": ["twitter-new-mention-received-by-user"], + "actions": ["zendesk-create-ticket"], "persona": "verbose" }, { "query": "When a new form submission happens in Typeform, add to Airtable", - "triggers": [ - "typeform-new-submission" - ], - "actions": [ - "airtable_oauth-create-single-record" - ], + "sources": ["typeform-new-submission"], + "actions": ["airtable_oauth-create-single-record"], "persona": "complex-workflow" }, { "query": "add row 2 sheets", - "triggers": [], - "actions": [ - "google_sheets-add-single-row" - ], + "sources": [], + "actions": ["google_sheets-add-single-row"], "persona": "sloppy" }, { "query": "create discord message", - "triggers": [], - "actions": [ - "discord-send-message" - ], + "sources": [], + "actions": ["discord-send-message"], "persona": "succinct" }, { "query": "When new records appear in Airtable, send them to Slack", - "triggers": [ - "airtable_oauth-new-records" - ], - "actions": [ - "slack-send-message" - ], + "sources": ["airtable_oauth-new-records"], + "actions": ["slack-send-message"], "persona": "complex-workflow" }, { "query": "update cell in googel sheets", - "triggers": [], - "actions": [ - "google_sheets-update-cell" - ], + "sources": [], + "actions": ["google_sheets-update-cell"], "persona": "sloppy" }, { "query": "When a payment happens in Stripe, update HubSpot contact", - "triggers": [ - "stripe-new-payment" - ], - "actions": [ - "hubspot-create-or-update-contact" - ], + "sources": ["stripe-new-payment"], + "actions": ["hubspot-create-or-update-contact"], "persona": "complex-workflow" }, { "query": "make asana task", - "triggers": [], - "actions": [ - "asana-create-task" - ], + "sources": [], + "actions": ["asana-create-task"], "persona": "sloppy" }, { "query": "I want GPT to analyze my Gmail messages", - "triggers": [ - "gmail-new-email" - ], - "actions": [ - "openai-chat" - ], + "sources": ["gmail-new-email"], + "actions": ["openai-chat"], "persona": "verbose" }, { "query": "When new issues are created in Linear, post to Discord", - "triggers": [ - "linear-issue-created-instant" - ], - "actions": [ - "discord-send-message" - ], + "sources": ["linear-issue-created-instant"], + "actions": ["discord-send-message"], "persona": "complex-workflow" }, { "query": "create notion database", - "triggers": [], - "actions": [ - "notion-create-page" - ], + "sources": [], + "actions": ["notion-create-page"], "persona": "succinct" }, { "query": "When GitHub issues are created, create Linear tickets", - "triggers": [], - "actions": [ - "linear-create-issue" - ], + "sources": [], + "actions": ["linear-create-issue"], "persona": "complex-workflow" }, { "query": "make fresh ticket", - "triggers": [], - "actions": [ - "freshdesk-create-ticket" - ], + "sources": [], + "actions": ["freshdesk-create-ticket"], "persona": "sloppy" }, { "query": "I would like to send Telegram messages when Jira tickets update", - "triggers": [ - "jira-issue-updated" - ], - "actions": [ - "telegram_bot_api-send-text-message-or-reply" - ], + "sources": ["jira-issue-updated"], + "actions": ["telegram_bot_api-send-text-message-or-reply"], "persona": "verbose" }, { "query": "When deals change stage in HubSpot, update records in Salesforce", - "triggers": [ - "hubspot-deal-updated" - ], - "actions": [ - "salesforce_rest_api-update-contact" - ], + "sources": ["hubspot-deal-updated"], + "actions": ["salesforce_rest_api-update-contact"], "persona": "complex-workflow" }, { "query": "create new doc", - "triggers": [], - "actions": [ - "google_docs-create-document" - ], + "sources": [], + "actions": ["google_docs-create-document"], "persona": "succinct" }, { "query": "send telgram msg", - "triggers": [], - "actions": [ - "telegram_bot_api-send-text-message-or-reply" - ], + "sources": [], + "actions": ["telegram_bot_api-send-text-message-or-reply"], "persona": "sloppy" }, { "query": "When RSS feed updates, post to Discord", - "triggers": [ - "rss-new-item-in-feed" - ], - "actions": [ - "discord-send-message" - ], + "sources": ["rss-new-item-in-feed"], + "actions": ["discord-send-message"], "persona": "task-oriented" }, { "query": "I need form submissions from JotForm in Google Sheets", - "triggers": [ - "jotform-new-submission" - ], - "actions": [ - "google_sheets-add-single-row" - ], + "sources": ["jotform-new-submission"], + "actions": ["google_sheets-add-single-row"], "persona": "verbose" }, { "query": "When calendar events start in Google Calendar, send Slack message", - "triggers": [ - "google_calendar-upcoming-event-alert" - ], - "actions": [ - "slack-send-message" - ], + "sources": ["google_calendar-upcoming-event-alert"], + "actions": ["slack-send-message"], "persona": "complex-workflow" }, { "query": "update zendesk ticket", - "triggers": [], - "actions": [ - "zendesk-update-ticket" - ], + "sources": [], + "actions": ["zendesk-update-ticket"], "persona": "succinct" }, { "query": "send campayn email", - "triggers": [], - "actions": [ - "gmail-send-email" - ], + "sources": [], + "actions": ["gmail-send-email"], "persona": "sloppy" }, { "query": "When tasks are completed in Todoist, update Trello", - "triggers": [ - "todoist-completed-task" - ], + "sources": ["todoist-completed-task"], "actions": [], "persona": "complex-workflow" }, { "query": "add row to airtble", - "triggers": [], - "actions": [ - "airtable_oauth-create-single-record" - ], + "sources": [], + "actions": ["airtable_oauth-create-single-record"], "persona": "sloppy" }, { "query": "I want new Slack messages to be analyzed by GPT", - "triggers": [ - "slack-new-message-in-channels" - ], - "actions": [ - "openai-chat" - ], + "sources": ["slack-new-message-in-channels"], + "actions": ["openai-chat"], "persona": "verbose" }, { "query": "When new files are added to Google Drive, create Notion pages", - "triggers": [ - "google_drive-new-files-instant" - ], - "actions": [ - "notion-create-page-from-database" - ], + "sources": ["google_drive-new-files-instant"], + "actions": ["notion-create-page-from-database"], "persona": "complex-workflow" }, { "query": "create monday item", - "triggers": [], - "actions": [ - "monday-create-item" - ], + "sources": [], + "actions": ["monday-create-item"], "persona": "succinct" }, { "query": "When email attachments arrive in Gmail, use OpenAI to analyze them", - "triggers": [ - "gmail-new-email" - ], - "actions": [ - "openai-chat" - ], + "sources": ["gmail-new-email"], + "actions": ["openai-chat"], "persona": "complex-workflow" }, { "query": "add subscriber mailchmp", - "triggers": [], - "actions": [ - "mailchimp-add-subscriber-to-tag" - ], + "sources": [], + "actions": ["mailchimp-add-subscriber-to-tag"], "persona": "sloppy" }, { "query": "I want to be notified in Telegram about new Twitter mentions", - "triggers": [ - "twitter-new-mention-received-by-user" - ], - "actions": [ - "telegram_bot_api-send-text-message-or-reply" - ], + "sources": ["twitter-new-mention-received-by-user"], + "actions": ["telegram_bot_api-send-text-message-or-reply"], "persona": "verbose" }, { "query": "When Zoom meetings end, send recordings to Slack", - "triggers": [ - "zoom-recording-completed" - ], - "actions": [ - "slack-send-message" - ], + "sources": ["zoom-recording-completed"], + "actions": ["slack-send-message"], "persona": "complex-workflow" }, { "query": "make noshan page", - "triggers": [], - "actions": [ - "notion-create-page" - ], + "sources": [], + "actions": ["notion-create-page"], "persona": "sloppy" }, { "query": "When new comments appear in Discord, use GPT to moderate them", - "triggers": [ - "discord-new-message" - ], - "actions": [ - "openai-chat" - ], + "sources": ["discord-new-message"], + "actions": ["openai-chat"], "persona": "complex-workflow" }, { "query": "create gsheet row", - "triggers": [], - "actions": [ - "google_sheets-add-single-row" - ], + "sources": [], + "actions": ["google_sheets-add-single-row"], "persona": "succinct" }, { "query": "I need Typeform responses to be analyzed by GPT and stored in Airtable", - "triggers": [ - "typeform-new-submission" - ], - "actions": [ - "openai-chat", - "airtable_oauth-create-single-record" - ], + "sources": ["typeform-new-submission"], + "actions": ["openai-chat", "airtable_oauth-create-single-record"], "persona": "verbose" }, { "query": "When new starred messages in Slack, create tasks in Linear", - "triggers": [ - "slack-new-reaction-added" - ], - "actions": [ - "linear-create-issue" - ], + "sources": ["slack-new-reaction-added"], + "actions": ["linear-create-issue"], "persona": "complex-workflow" } ] -} \ No newline at end of file +} diff --git a/packages/evals/component_retrieval/eval-test-suite-151-200-verified-filtered.json b/packages/evals/component_retrieval/eval-test-suite-151-200-verified-filtered.json index 57a3a5577c88d..73b878b9c0ec4 100644 --- a/packages/evals/component_retrieval/eval-test-suite-151-200-verified-filtered.json +++ b/packages/evals/component_retrieval/eval-test-suite-151-200-verified-filtered.json @@ -2,465 +2,309 @@ "evaluationTests": [ { "query": "When a row is added to Google Sheets in a shared drive, create a Linear ticket", - "triggers": [ - "google_sheets-new-row-added" - ], - "actions": [ - "linear-create-issue" - ], + "sources": ["google_sheets-new-row-added"], + "actions": ["linear-create-issue"], "persona": "complex-workflow" }, { "query": "update airtable record", - "triggers": [], - "actions": [ - "airtable_oauth-update-record" - ], + "sources": [], + "actions": ["airtable_oauth-update-record"], "persona": "succinct" }, { "query": "I would like to forward all my Gmail attachments to Telegram", - "triggers": [ - "gmail-new-email-received" - ], - "actions": [ - "telegram_bot_api-send-text-message-or-reply" - ], + "sources": ["gmail-new-email-received"], + "actions": ["telegram_bot_api-send-text-message-or-reply"], "persona": "verbose" }, { "query": "send msg in slck", - "triggers": [], - "actions": [ - "slack-send-message" - ], + "sources": [], + "actions": ["slack-send-message"], "persona": "sloppy" }, { "query": "When a message is posted to Discord, analyze with OpenAI", - "triggers": [ - "discord-new-message" - ], - "actions": [ - "openai-chat" - ], + "sources": ["discord-new-message"], + "actions": ["openai-chat"], "persona": "complex-workflow" }, { "query": "add new google calendar event", - "triggers": [], - "actions": [ - "google_calendar-create-event" - ], + "sources": [], + "actions": ["google_calendar-create-event"], "persona": "succinct" }, { "query": "When a new tweet mentions our company, send to Slack channel", - "triggers": [ - "twitter-new-mention-received-by-user" - ], - "actions": [ - "slack-send-message" - ], + "sources": ["twitter-new-mention-received-by-user"], + "actions": ["slack-send-message"], "persona": "task-oriented" }, { "query": "create nw spreadsheet", - "triggers": [], - "actions": [ - "google_sheets-create-spreadsheet" - ], + "sources": [], + "actions": ["google_sheets-create-spreadsheet"], "persona": "sloppy" }, { "query": "I want my Typeform submissions to be saved in Notion databases", - "triggers": [ - "typeform-new-submission" - ], - "actions": [ - "notion-create-page-from-database" - ], + "sources": ["typeform-new-submission"], + "actions": ["notion-create-page-from-database"], "persona": "verbose" }, { "query": "When a Notion page updates, send a message to Telegram", - "triggers": [ - "notion-updated-page" - ], - "actions": [ - "telegram_bot_api-send-text-message-or-reply" - ], + "sources": ["notion-updated-page"], + "actions": ["telegram_bot_api-send-text-message-or-reply"], "persona": "complex-workflow" }, { "query": "post 2 discord", - "triggers": [], - "actions": [ - "discord-send-message" - ], + "sources": [], + "actions": ["discord-send-message"], "persona": "sloppy" }, { "query": "When files are uploaded to Dropbox, copy them to Google Drive", - "triggers": [ - "dropbox-new-file" - ], - "actions": [ - "google_drive-upload-file" - ], + "sources": ["dropbox-new-file"], + "actions": ["google_drive-upload-file"], "persona": "complex-workflow" }, { "query": "create trello board", - "triggers": [], + "sources": [], "actions": [], "persona": "succinct" }, { "query": "I need new RSS items to be posted to Discord", - "triggers": [ - "rss-new-item-in-feed" - ], - "actions": [ - "discord-send-message" - ], + "sources": ["rss-new-item-in-feed"], + "actions": ["discord-send-message"], "persona": "verbose" }, { "query": "When a form is submitted in JotForm, use GPT to analyze it", - "triggers": [ - "jotform-new-submission" - ], - "actions": [ - "openai-chat" - ], + "sources": ["jotform-new-submission"], + "actions": ["openai-chat"], "persona": "complex-workflow" }, { "query": "make linear issue", - "triggers": [], - "actions": [ - "linear-create-issue" - ], + "sources": [], + "actions": ["linear-create-issue"], "persona": "succinct" }, { "query": "add row 2 googlesheets", - "triggers": [], - "actions": [ - "google_sheets-add-single-row" - ], + "sources": [], + "actions": ["google_sheets-add-single-row"], "persona": "sloppy" }, { "query": "When a YouTube video is posted, share it on Twitter", - "triggers": [ - "youtube_data_api-new-videos" - ], - "actions": [ - "twitter-create-tweet" - ], + "sources": ["youtube_data_api-new-videos"], + "actions": ["twitter-create-tweet"], "persona": "complex-workflow" }, { "query": "forward email to telegram", - "triggers": [ - "gmail-new-email-received" - ], - "actions": [ - "telegram_bot_api-send-text-message-or-reply" - ], + "sources": ["gmail-new-email-received"], + "actions": ["telegram_bot_api-send-text-message-or-reply"], "persona": "task-oriented" }, { "query": "When new Slack messages contain specific keywords, create Linear issues", - "triggers": [ - "slack-new-message-in-channels" - ], - "actions": [ - "linear-create-issue" - ], + "sources": ["slack-new-message-in-channels"], + "actions": ["linear-create-issue"], "persona": "complex-workflow" }, { "query": "updaet notion page", - "triggers": [], - "actions": [ - "notion-update-page" - ], + "sources": [], + "actions": ["notion-update-page"], "persona": "sloppy" }, { "query": "create google doc", - "triggers": [], - "actions": [ - "google_docs-create-document" - ], + "sources": [], + "actions": ["google_docs-create-document"], "persona": "succinct" }, { "query": "I would like my Gmail messages to be summarized by GPT", - "triggers": [ - "gmail-new-email-received" - ], - "actions": [ - "openai-chat" - ], + "sources": ["gmail-new-email-received"], + "actions": ["openai-chat"], "persona": "verbose" }, { "query": "When calendar events are updated, post to Slack", - "triggers": [ - "google_calendar-new-or-updated-event-instant" - ], - "actions": [ - "slack-send-message" - ], + "sources": ["google_calendar-new-or-updated-event-instant"], + "actions": ["slack-send-message"], "persona": "complex-workflow" }, { "query": "download gdrive file", - "triggers": [], - "actions": [ - "google_drive-download-file" - ], + "sources": [], + "actions": ["google_drive-download-file"], "persona": "succinct" }, { "query": "When a new page is created in Notion, use GPT to summarize it", - "triggers": [ - "notion-new-page" - ], - "actions": [ - "openai-chat" - ], + "sources": ["notion-new-page"], + "actions": ["openai-chat"], "persona": "complex-workflow" }, { "query": "send telagrem message", - "triggers": [], - "actions": [ - "telegram_bot_api-send-text-message-or-reply" - ], + "sources": [], + "actions": ["telegram_bot_api-send-text-message-or-reply"], "persona": "sloppy" }, { "query": "I want to track my liked tweets in a spreadsheet", - "triggers": [ - "twitter-my-tweets" - ], - "actions": [ - "google_sheets-add-single-row" - ], + "sources": ["twitter-my-tweets"], + "actions": ["google_sheets-add-single-row"], "persona": "verbose" }, { "query": "When files are modified in Google Drive, update Notion", - "triggers": [ - "google_drive-new-or-modified-files" - ], - "actions": [ - "notion-update-page" - ], + "sources": ["google_drive-new-or-modified-files"], + "actions": ["notion-update-page"], "persona": "complex-workflow" }, { "query": "analyze text gpt", - "triggers": [], - "actions": [ - "openai-chat" - ], + "sources": [], + "actions": ["openai-chat"], "persona": "succinct" }, { "query": "update row in sheets", - "triggers": [], - "actions": [ - "google_sheets-update-row" - ], + "sources": [], + "actions": ["google_sheets-update-row"], "persona": "task-oriented" }, { "query": "When Zoom recordings complete, transcribe with OpenAI", - "triggers": [ - "zoom-recording-completed" - ], - "actions": [ - "openai-create-transcription" - ], + "sources": ["zoom-recording-completed"], + "actions": ["openai-create-transcription"], "persona": "complex-workflow" }, { "query": "make asana projct", - "triggers": [], - "actions": [ - "asana-create-project" - ], + "sources": [], + "actions": ["asana-create-project"], "persona": "sloppy" }, { "query": "I would like to receive Discord messages in Telegram", - "triggers": [ - "discord-new-message" - ], - "actions": [ - "telegram_bot_api-send-text-message-or-reply" - ], + "sources": ["discord-new-message"], + "actions": ["telegram_bot_api-send-text-message-or-reply"], "persona": "verbose" }, { "query": "When Slack reactions are added, create Trello cards", - "triggers": [ - "slack-new-reaction-added" - ], - "actions": [ - "trello-create-card" - ], + "sources": ["slack-new-reaction-added"], + "actions": ["trello-create-card"], "persona": "complex-workflow" }, { "query": "download dropbox file", - "triggers": [], - "actions": [ - "dropbox-download-file-to-tmp" - ], + "sources": [], + "actions": ["dropbox-download-file-to-tmp"], "persona": "succinct" }, { "query": "get cell from sheet", - "triggers": [], - "actions": [ - "google_sheets-get-cell" - ], + "sources": [], + "actions": ["google_sheets-get-cell"], "persona": "task-oriented" }, { "query": "When new Airtable records are created, use GPT to classify them", - "triggers": [ - "airtable_oauth-new-records" - ], - "actions": [ - "openai-chat" - ], + "sources": ["airtable_oauth-new-records"], + "actions": ["openai-chat"], "persona": "complex-workflow" }, { "query": "reply 2 slack thread", - "triggers": [], - "actions": [ - "slack-reply-to-a-message" - ], + "sources": [], + "actions": ["slack-reply-to-a-message"], "persona": "sloppy" }, { "query": "I need new Twitter mentions to create Linear issues", - "triggers": [ - "twitter-new-mention-received-by-user" - ], - "actions": [ - "linear-create-issue" - ], + "sources": ["twitter-new-mention-received-by-user"], + "actions": ["linear-create-issue"], "persona": "verbose" }, { "query": "When files change in shared Google Drive, notify in Slack", - "triggers": [ - "google_drive-changes-to-specific-files-shared-drive" - ], - "actions": [ - "slack-send-message" - ], + "sources": ["google_drive-changes-to-specific-files-shared-drive"], + "actions": ["slack-send-message"], "persona": "complex-workflow" }, { "query": "create gmail draft", - "triggers": [], - "actions": [ - "gmail-create-draft" - ], + "sources": [], + "actions": ["gmail-create-draft"], "persona": "succinct" }, { "query": "move file in gdrive", - "triggers": [], - "actions": [ - "google_drive-move-file" - ], + "sources": [], + "actions": ["google_drive-move-file"], "persona": "sloppy" }, { "query": "When new Direct Messages arrive in Slack, forward to email", - "triggers": [ - "slack-new-direct-message" - ], - "actions": [ - "gmail-send-email" - ], + "sources": ["slack-new-direct-message"], + "actions": ["gmail-send-email"], "persona": "complex-workflow" }, { "query": "I want my Google Calendar events to show in Slack", - "triggers": [ - "google_calendar-new-event" - ], - "actions": [ - "slack-send-message" - ], + "sources": ["google_calendar-new-event"], + "actions": ["slack-send-message"], "persona": "verbose" }, { "query": "When Discord reactions are added, update Notion", - "triggers": [ - "discord-reaction-added" - ], - "actions": [ - "notion-update-page" - ], + "sources": ["discord-reaction-added"], + "actions": ["notion-update-page"], "persona": "complex-workflow" }, { "query": "post direct msg slack", - "triggers": [], - "actions": [ - "slack-send-message" - ], + "sources": [], + "actions": ["slack-send-message"], "persona": "sloppy" }, { "query": "create new worksheet", - "triggers": [], - "actions": [ - "google_sheets-create-worksheet" - ], + "sources": [], + "actions": ["google_sheets-create-worksheet"], "persona": "succinct" }, { "query": "When RSS feed updates, let GPT analyze the content", - "triggers": [ - "rss-new-item-in-feed" - ], - "actions": [ - "openai-chat" - ], + "sources": ["rss-new-item-in-feed"], + "actions": ["openai-chat"], "persona": "complex-workflow" }, { "query": "send message to discord channel", - "triggers": [], - "actions": [ - "discord-send-message" - ], + "sources": [], + "actions": ["discord-send-message"], "persona": "task-oriented" }, { "query": "When worksheets are added to Google Sheets, create Notion pages", - "triggers": [ - "google_sheets-new-worksheet" - ], - "actions": [ - "notion-create-page-from-database" - ], + "sources": ["google_sheets-new-worksheet"], + "actions": ["notion-create-page-from-database"], "persona": "complex-workflow" } ] -} \ No newline at end of file +} diff --git a/packages/evals/component_retrieval/eval-test-suite-201-250-complex-filtered.json b/packages/evals/component_retrieval/eval-test-suite-201-250-complex-filtered.json index 11ea6994d42d1..c216b44f78fec 100644 --- a/packages/evals/component_retrieval/eval-test-suite-201-250-complex-filtered.json +++ b/packages/evals/component_retrieval/eval-test-suite-201-250-complex-filtered.json @@ -2,30 +2,19 @@ "evaluationTests": [ { "query": "When new Gmail attachments arrive, use GPT to analyze them and create Linear issues for action items", - "triggers": [ - "gmail-new-email" - ], - "actions": [ - "openai-chat", - "linear-create-issue" - ], + "sources": ["gmail-new-email"], + "actions": ["openai-chat", "linear-create-issue"], "persona": "complex-workflow" }, { "query": "translate notion pages with gpt", - "triggers": [ - "notion-new-page" - ], - "actions": [ - "openai-chat" - ], + "sources": ["notion-new-page"], + "actions": ["openai-chat"], "persona": "succinct" }, { "query": "I need all Typeform submissions to be analyzed by GPT and then posted to both Slack and Notion", - "triggers": [ - "typeform-new-submission" - ], + "sources": ["typeform-new-submission"], "actions": [ "openai-chat", "slack-send-message", @@ -35,100 +24,61 @@ }, { "query": "When Discord messages mention specific keywords, use GPT to analyze sentiment and post to Slack threads", - "triggers": [ - "discord-new-message" - ], - "actions": [ - "openai-chat", - "slack-reply-to-a-message" - ], + "sources": ["discord-new-message"], + "actions": ["openai-chat", "slack-reply-to-a-message"], "persona": "complex-workflow" }, { "query": "send msg 2 multiple slack channls", - "triggers": [], - "actions": [ - "slack-send-message" - ], + "sources": [], + "actions": ["slack-send-message"], "persona": "sloppy" }, { "query": "When RSS feeds have new items, use GPT to summarize and post summaries to Telegram", - "triggers": [ - "rss-new-item-in-feed" - ], - "actions": [ - "openai-chat", - "telegram_bot_api-send-text-message-or-reply" - ], + "sources": ["rss-new-item-in-feed"], + "actions": ["openai-chat", "telegram_bot_api-send-text-message-or-reply"], "persona": "complex-workflow" }, { "query": "analyze zoom recordings with gpt", - "triggers": [ - "zoom-recording-completed" - ], - "actions": [ - "openai-chat" - ], + "sources": ["zoom-recording-completed"], + "actions": ["openai-chat"], "persona": "task-oriented" }, { "query": "I would like new Twitter mentions to be analyzed by GPT and important ones saved to Notion", - "triggers": [ - "twitter-new-mention-received-by-user" - ], - "actions": [ - "openai-chat", - "notion-create-page-from-database" - ], + "sources": ["twitter-new-mention-received-by-user"], + "actions": ["openai-chat", "notion-create-page-from-database"], "persona": "verbose" }, { "query": "When Google Drive files are modified, update Sheets log and notify in Slack", - "triggers": [ - "google_drive-new-or-modified-files" - ], - "actions": [ - "google_sheets-add-single-row", - "slack-send-message" - ], + "sources": ["google_drive-new-or-modified-files"], + "actions": ["google_sheets-add-single-row", "slack-send-message"], "persona": "complex-workflow" }, { "query": "foward emails to slck", - "triggers": [ - "gmail-new-email" - ], - "actions": [ - "slack-send-message" - ], + "sources": ["gmail-new-email"], + "actions": ["slack-send-message"], "persona": "sloppy" }, { "query": "When new Airtable records appear, let GPT analyze them and create Linear issues", - "triggers": [ - "airtable_oauth-new-records" - ], - "actions": [ - "openai-chat", - "linear-create-issue" - ], + "sources": ["airtable_oauth-new-records"], + "actions": ["openai-chat", "linear-create-issue"], "persona": "complex-workflow" }, { "query": "move files between drives", - "triggers": [], - "actions": [ - "google_drive-move-file" - ], + "sources": [], + "actions": ["google_drive-move-file"], "persona": "succinct" }, { "query": "I need YouTube video transcriptions analyzed by GPT and saved to Notion", - "triggers": [ - "youtube_data_api-new-videos" - ], + "sources": ["youtube_data_api-new-videos"], "actions": [ "openai-create-transcription", "openai-chat", @@ -138,49 +88,31 @@ }, { "query": "When calendar events are created, use GPT to generate agendas and send via Gmail", - "triggers": [ - "google_calendar-new-event" - ], - "actions": [ - "openai-chat", - "gmail-send-email" - ], + "sources": ["google_calendar-new-event"], + "actions": ["openai-chat", "gmail-send-email"], "persona": "complex-workflow" }, { "query": "post message in discord thread", - "triggers": [], - "actions": [ - "discord-send-message" - ], + "sources": [], + "actions": ["discord-send-message"], "persona": "task-oriented" }, { "query": "When Slack messages are starred, transcribe any attachments and create Linear tickets", - "triggers": [ - "slack-new-reaction-added" - ], - "actions": [ - "openai-create-transcription", - "linear-create-issue" - ], + "sources": ["slack-new-reaction-added"], + "actions": ["openai-create-transcription", "linear-create-issue"], "persona": "complex-workflow" }, { "query": "analyize email with gpt", - "triggers": [ - "gmail-new-email" - ], - "actions": [ - "openai-chat" - ], + "sources": ["gmail-new-email"], + "actions": ["openai-chat"], "persona": "sloppy" }, { "query": "When files are added to Dropbox, copy to Drive and update Notion log", - "triggers": [ - "dropbox-new-file" - ], + "sources": ["dropbox-new-file"], "actions": [ "google_drive-upload-file", "notion-create-page-from-database" @@ -189,47 +121,31 @@ }, { "query": "send multiple discord msgs", - "triggers": [], - "actions": [ - "discord-send-message" - ], + "sources": [], + "actions": ["discord-send-message"], "persona": "sloppy" }, { "query": "When new form submissions arrive, analyze with GPT and update spreadsheet", - "triggers": [ - "typeform-new-submission" - ], - "actions": [ - "openai-chat", - "google_sheets-add-single-row" - ], + "sources": ["typeform-new-submission"], + "actions": ["openai-chat", "google_sheets-add-single-row"], "persona": "complex-workflow" }, { "query": "share files in telegram", - "triggers": [], - "actions": [ - "telegram_bot_api-send-text-message-or-reply" - ], + "sources": [], + "actions": ["telegram_bot_api-send-text-message-or-reply"], "persona": "succinct" }, { "query": "I would like GPT to analyze my Slack messages and create tasks in Linear", - "triggers": [ - "slack-new-message-in-channels" - ], - "actions": [ - "openai-chat", - "linear-create-issue" - ], + "sources": ["slack-new-message-in-channels"], + "actions": ["openai-chat", "linear-create-issue"], "persona": "verbose" }, { "query": "When Discord reactions are added, update Notion and notify in Telegram", - "triggers": [ - "discord-reaction-added" - ], + "sources": ["discord-reaction-added"], "actions": [ "notion-update-page", "telegram_bot_api-send-text-message-or-reply" @@ -238,66 +154,43 @@ }, { "query": "create multiple sheets rows", - "triggers": [], - "actions": [ - "google_sheets-add-multiple-rows" - ], + "sources": [], + "actions": ["google_sheets-add-multiple-rows"], "persona": "succinct" }, { "query": "When RSS items mention specific terms, analyze with GPT and create Linear issues", - "triggers": [ - "rss-new-item-in-feed" - ], - "actions": [ - "openai-chat", - "linear-create-issue" - ], + "sources": ["rss-new-item-in-feed"], + "actions": ["openai-chat", "linear-create-issue"], "persona": "complex-workflow" }, { "query": "reply 2 telegram msg", - "triggers": [], - "actions": [ - "telegram_bot_api-send-text-message-or-reply" - ], + "sources": [], + "actions": ["telegram_bot_api-send-text-message-or-reply"], "persona": "sloppy" }, { "query": "I need new Google Calendar events to be summarized by GPT and posted to Slack", - "triggers": [ - "google_calendar-new-event" - ], - "actions": [ - "openai-chat", - "slack-send-message" - ], + "sources": ["google_calendar-new-event"], + "actions": ["openai-chat", "slack-send-message"], "persona": "verbose" }, { "query": "When Notion pages update, let GPT analyze changes and notify in Discord", - "triggers": [ - "notion-updated-page" - ], - "actions": [ - "openai-chat", - "discord-send-message" - ], + "sources": ["notion-updated-page"], + "actions": ["openai-chat", "discord-send-message"], "persona": "complex-workflow" }, { "query": "get data from sheets range", - "triggers": [], - "actions": [ - "google_sheets-get-values-in-range" - ], + "sources": [], + "actions": ["google_sheets-get-values-in-range"], "persona": "succinct" }, { "query": "When new YouTube videos are posted, create transcripts and Notion pages", - "triggers": [ - "youtube_data_api-new-videos" - ], + "sources": ["youtube_data_api-new-videos"], "actions": [ "openai-create-transcription", "notion-create-page-from-database" @@ -306,47 +199,31 @@ }, { "query": "updaet multiple sheet rows", - "triggers": [], - "actions": [ - "google_sheets-update-row" - ], + "sources": [], + "actions": ["google_sheets-update-row"], "persona": "sloppy" }, { "query": "I want GPT to analyze Gmail attachments and create Notion documents", - "triggers": [ - "gmail-new-email" - ], - "actions": [ - "openai-chat", - "notion-create-page-from-database" - ], + "sources": ["gmail-new-email"], + "actions": ["openai-chat", "notion-create-page-from-database"], "persona": "verbose" }, { "query": "When Slack threads are updated, use GPT for sentiment and notify in Telegram", - "triggers": [ - "slack-new-message-in-channels" - ], - "actions": [ - "openai-chat", - "telegram_bot_api-send-text-message-or-reply" - ], + "sources": ["slack-new-message-in-channels"], + "actions": ["openai-chat", "telegram_bot_api-send-text-message-or-reply"], "persona": "complex-workflow" }, { "query": "merge multiple rss feeds", - "triggers": [], - "actions": [ - "rss-merge-rss-feeds" - ], + "sources": [], + "actions": ["rss-merge-rss-feeds"], "persona": "task-oriented" }, { "query": "When files are modified in shared drive, update Sheets and Notion", - "triggers": [ - "google_drive-changes-to-specific-files-shared-drive" - ], + "sources": ["google_drive-changes-to-specific-files-shared-drive"], "actions": [ "google_sheets-add-single-row", "notion-create-page-from-database" @@ -355,116 +232,73 @@ }, { "query": "transcribe audio to txt", - "triggers": [], - "actions": [ - "openai-create-transcription" - ], + "sources": [], + "actions": ["openai-create-transcription"], "persona": "sloppy" }, { "query": "I need Zoom recordings to be transcribed and summarized in Slack", - "triggers": [ - "zoom-recording-completed" - ], - "actions": [ - "openai-create-transcription", - "slack-send-message" - ], + "sources": ["zoom-recording-completed"], + "actions": ["openai-create-transcription", "slack-send-message"], "persona": "verbose" }, { "query": "When new Airtable records appear, use GPT to analyze and update the same record", - "triggers": [ - "airtable_oauth-new-records" - ], - "actions": [ - "openai-chat", - "airtable_oauth-update-record" - ], + "sources": ["airtable_oauth-new-records"], + "actions": ["openai-chat", "airtable_oauth-update-record"], "persona": "complex-workflow" }, { "query": "download multiple gdrive files", - "triggers": [], - "actions": [ - "google_drive-download-file" - ], + "sources": [], + "actions": ["google_drive-download-file"], "persona": "succinct" }, { "query": "When Discord messages arrive, translate with GPT and post to Telegram", - "triggers": [ - "discord-new-message" - ], - "actions": [ - "openai-chat", - "telegram_bot_api-send-text-message-or-reply" - ], + "sources": ["discord-new-message"], + "actions": ["openai-chat", "telegram_bot_api-send-text-message-or-reply"], "persona": "complex-workflow" }, { "query": "make google docs from txt", - "triggers": [], - "actions": [ - "google_docs-create-document-from-text" - ], + "sources": [], + "actions": ["google_docs-create-document-from-text"], "persona": "sloppy" }, { "query": "I would like GPT to analyze Telegram messages and create Linear issues", - "triggers": [ - "telegram_bot_api-message-updates" - ], - "actions": [ - "openai-chat", - "linear-create-issue" - ], + "sources": ["telegram_bot_api-message-updates"], + "actions": ["openai-chat", "linear-create-issue"], "persona": "verbose" }, { "query": "When spreadsheet updates, let GPT analyze changes and send in Slack", - "triggers": [ - "google_sheets-new-updates" - ], - "actions": [ - "openai-chat", - "slack-send-message" - ], + "sources": ["google_sheets-new-updates"], + "actions": ["openai-chat", "slack-send-message"], "persona": "complex-workflow" }, { "query": "add text to google doc", - "triggers": [], - "actions": [ - "google_docs-append-text" - ], + "sources": [], + "actions": ["google_docs-append-text"], "persona": "succinct" }, { "query": "When YouTube videos are liked, download and transcribe them", - "triggers": [ - "youtube_data_api-new-liked-videos" - ], - "actions": [ - "openai-create-transcription" - ], + "sources": ["youtube_data_api-new-liked-videos"], + "actions": ["openai-create-transcription"], "persona": "complex-workflow" }, { "query": "forward emails to channels", - "triggers": [ - "gmail-new-email" - ], - "actions": [ - "slack-send-message" - ], + "sources": ["gmail-new-email"], + "actions": ["slack-send-message"], "persona": "task-oriented" }, { "query": "When RSS feeds update, create docs and share in Telegram", - "triggers": [ - "rss-new-item-in-feed" - ], + "sources": ["rss-new-item-in-feed"], "actions": [ "google_docs-create-document", "telegram_bot_api-send-text-message-or-reply" @@ -473,41 +307,27 @@ }, { "query": "genrate email with AI", - "triggers": [], - "actions": [ - "openai-chat" - ], + "sources": [], + "actions": ["openai-chat"], "persona": "sloppy" }, { "query": "When Slack messages are starred, create Notion pages and Linear tickets", - "triggers": [ - "slack-new-reaction-added" - ], - "actions": [ - "notion-create-page-from-database", - "linear-create-issue" - ], + "sources": ["slack-new-reaction-added"], + "actions": ["notion-create-page-from-database", "linear-create-issue"], "persona": "complex-workflow" }, { "query": "create sheet from template", - "triggers": [], - "actions": [ - "google_sheets-create-spreadsheet" - ], + "sources": [], + "actions": ["google_sheets-create-spreadsheet"], "persona": "succinct" }, { "query": "When calendar events start, generate agenda with GPT and send via Telegram", - "triggers": [ - "google_calendar-upcoming-event-alert" - ], - "actions": [ - "openai-chat", - "telegram_bot_api-send-text-message-or-reply" - ], + "sources": ["google_calendar-upcoming-event-alert"], + "actions": ["openai-chat", "telegram_bot_api-send-text-message-or-reply"], "persona": "complex-workflow" } ] -} \ No newline at end of file +} diff --git a/packages/evals/component_retrieval/eval-test-suite-251-300-new-filtered.json b/packages/evals/component_retrieval/eval-test-suite-251-300-new-filtered.json index f9390aa022f84..4e0f3e6a199cf 100644 --- a/packages/evals/component_retrieval/eval-test-suite-251-300-new-filtered.json +++ b/packages/evals/component_retrieval/eval-test-suite-251-300-new-filtered.json @@ -2,9 +2,7 @@ "evaluationTests": [ { "query": "When a Webflow form is submitted, create contact in Mailchimp and send PagerDuty alert", - "triggers": [ - "webflow-new-form-submission" - ], + "sources": ["webflow-new-form-submission"], "actions": [ "mailchimp-add-subscriber-to-tag", "pagerduty-trigger-incident" @@ -13,449 +11,303 @@ }, { "query": "create medium blog post", - "triggers": [], - "actions": [ - "medium-create-post" - ], + "sources": [], + "actions": ["medium-create-post"], "persona": "succinct" }, { "query": "send push notification using pushover", - "triggers": [], - "actions": [ - "pushover-push-notification" - ], + "sources": [], + "actions": ["pushover-push-notification"], "persona": "task-oriented" }, { "query": "When there's a new hot post on Reddit subreddit, send to Telegram", - "triggers": [ - "reddit-new-hot-posts-on-a-subreddit" - ], - "actions": [ - "telegram_bot_api-send-text-message-or-reply" - ], + "sources": ["reddit-new-hot-posts-on-a-subreddit"], + "actions": ["telegram_bot_api-send-text-message-or-reply"], "persona": "complex-workflow" }, { "query": "mkae pushbullet notifcation", - "triggers": [], - "actions": [ - "pushbullet-send-push" - ], + "sources": [], + "actions": ["pushbullet-send-push"], "persona": "sloppy" }, { "query": "I would like to be notified when someone completes their Calendly booking", - "triggers": [ - "calendly_v2-invitee-created" - ], + "sources": ["calendly_v2-invitee-created"], "actions": [], "persona": "verbose" }, { "query": "When new Spotify playlist is created, post to Discord", - "triggers": [ - "spotify-new-playlist" - ], - "actions": [ - "discord-send-message" - ], + "sources": ["spotify-new-playlist"], + "actions": ["discord-send-message"], "persona": "complex-workflow" }, { "query": "upload file 2 imgbb", - "triggers": [], - "actions": [ - "imgbb-upload-picture" - ], + "sources": [], + "actions": ["imgbb-upload-picture"], "persona": "sloppy" }, { "query": "create postmark email", - "triggers": [], - "actions": [ - "postmark-send-single-email" - ], + "sources": [], + "actions": ["postmark-send-single-email"], "persona": "succinct" }, { "query": "When a new story mentions our keyword on Hacker News, create Linear ticket", - "triggers": [ - "hacker_news-new-stories-by-keyword" - ], - "actions": [ - "linear-create-issue" - ], + "sources": ["hacker_news-new-stories-by-keyword"], + "actions": ["linear-create-issue"], "persona": "complex-workflow" }, { "query": "I need my WordPress posts to be automatically shared on LinkedIn", - "triggers": [], - "actions": [ - "linkedin-create-text-post-user" - ], + "sources": [], + "actions": ["linkedin-create-text-post-user"], "persona": "verbose" }, { "query": "make aws lambda functon", - "triggers": [], - "actions": [ - "aws-lambda-invoke-function" - ], + "sources": [], + "actions": ["aws-lambda-invoke-function"], "persona": "sloppy" }, { "query": "When a new video is added to my YouTube playlist, add to Spotify", - "triggers": [ - "youtube_data_api-new-videos-in-playlist" - ], - "actions": [ - "spotify-add-items-to-playlist" - ], + "sources": ["youtube_data_api-new-videos-in-playlist"], + "actions": ["spotify-add-items-to-playlist"], "persona": "complex-workflow" }, { "query": "create coda row", - "triggers": [], - "actions": [ - "coda-create-rows" - ], + "sources": [], + "actions": ["coda-create-rows"], "persona": "succinct" }, { "query": "When my Strava activity is created, post to Facebook Page", - "triggers": [ - "strava-activity-created" - ], - "actions": [ - "facebook_pages-create-post" - ], + "sources": ["strava-activity-created"], + "actions": ["facebook_pages-create-post"], "persona": "complex-workflow" }, { "query": "send emergency pushover alert", - "triggers": [], - "actions": [ - "pushover-emergency-push-notification" - ], + "sources": [], + "actions": ["pushover-emergency-push-notification"], "persona": "task-oriented" }, { "query": "I would like to sync my Google Tasks with Microsoft To Do", - "triggers": [ - "google_tasks-new-task-added" - ], - "actions": [ - "microsofttodo-create-task" - ], + "sources": ["google_tasks-new-task-added"], + "actions": ["microsofttodo-create-task"], "persona": "verbose" }, { "query": "When someone comments on Figma, create Motion task", - "triggers": [ - "figma-new-comment" - ], - "actions": [ - "motion-create-task" - ], + "sources": ["figma-new-comment"], + "actions": ["motion-create-task"], "persona": "complex-workflow" }, { "query": "creat cloudinary upload", - "triggers": [], - "actions": [ - "cloudinary-upload-media-asset" - ], + "sources": [], + "actions": ["cloudinary-upload-media-asset"], "persona": "sloppy" }, { "query": "When there's a new commit in Bitbucket, post to Microsoft Teams", - "triggers": [ - "bitbucket-new-commit" - ], - "actions": [ - "microsoft_teams-send-channel-message" - ], + "sources": ["bitbucket-new-commit"], + "actions": ["microsoft_teams-send-channel-message"], "persona": "complex-workflow" }, { "query": "send line notification", - "triggers": [], - "actions": [ - "line-send-notification-message" - ], + "sources": [], + "actions": ["line-send-notification-message"], "persona": "succinct" }, { "query": "I need all my Raindrop bookmarks to be saved in Notion", - "triggers": [ - "raindrop-new-bookmark" - ], - "actions": [ - "notion-create-page-from-database" - ], + "sources": ["raindrop-new-bookmark"], + "actions": ["notion-create-page-from-database"], "persona": "verbose" }, { "query": "When a new highlight is made in Readwise, create Google Doc", - "triggers": [ - "readwise-new-highlights" - ], - "actions": [ - "google_docs-create-document" - ], + "sources": ["readwise-new-highlights"], + "actions": ["google_docs-create-document"], "persona": "complex-workflow" }, { "query": "add contact 2 mautic", - "triggers": [], - "actions": [ - "mautic-create-contact" - ], + "sources": [], + "actions": ["mautic-create-contact"], "persona": "sloppy" }, { "query": "When a new form is answered in Google Forms, add to Airtable", - "triggers": [ - "google_forms-new-form-answer" - ], - "actions": [ - "airtable_oauth-create-single-record" - ], + "sources": ["google_forms-new-form-answer"], + "actions": ["airtable_oauth-create-single-record"], "persona": "complex-workflow" }, { "query": "create onenote page", - "triggers": [], - "actions": [ - "onenote-create-page" - ], + "sources": [], + "actions": ["onenote-create-page"], "persona": "succinct" }, { "query": "I want my Mastodon posts to be automatically shared on Twitter", - "triggers": [ - "mastodon-new-status" - ], - "actions": [ - "twitter-create-tweet" - ], + "sources": ["mastodon-new-status"], + "actions": ["twitter-create-tweet"], "persona": "verbose" }, { "query": "When a WhatsApp message is received, analyze with GPT and send to Telegram", - "triggers": [ - "whatsapp_business-new-message-sent" - ], - "actions": [ - "openai-chat", - "telegram_bot_api-send-text-message-or-reply" - ], + "sources": ["whatsapp_business-new-message-sent"], + "actions": ["openai-chat", "telegram_bot_api-send-text-message-or-reply"], "persona": "complex-workflow" }, { "query": "create algolia objct", - "triggers": [], - "actions": [ - "algolia-create-objects" - ], + "sources": [], + "actions": ["algolia-create-objects"], "persona": "sloppy" }, { "query": "When files are added to Box, backup to S3", - "triggers": [ - "box-new-file" - ], - "actions": [ - "aws-s3-upload-file" - ], + "sources": ["box-new-file"], + "actions": ["aws-s3-upload-file"], "persona": "task-oriented" }, { "query": "make cisco webex msg", - "triggers": [], - "actions": [ - "cisco_webex-create-message" - ], + "sources": [], + "actions": ["cisco_webex-create-message"], "persona": "sloppy" }, { "query": "I need my Google Classroom assignments to be tracked in Todoist", - "triggers": [ - "google_classroom-new-assignment" - ], - "actions": [ - "todoist-create-task" - ], + "sources": ["google_classroom-new-assignment"], + "actions": ["todoist-create-task"], "persona": "verbose" }, { "query": "When new contacts are added to ActiveCampaign, sync to Klaviyo", - "triggers": [ - "activecampaign-new-or-updated-contact" - ], - "actions": [ - "klaviyo-add-member-to-list" - ], + "sources": ["activecampaign-new-or-updated-contact"], + "actions": ["klaviyo-add-member-to-list"], "persona": "complex-workflow" }, { "query": "send chat_api whatsapp", - "triggers": [], - "actions": [ - "chat_api_for_whatsapp-send-message" - ], + "sources": [], + "actions": ["chat_api_for_whatsapp-send-message"], "persona": "sloppy" }, { "query": "When someone reacts to a Discord message, create Trello card", - "triggers": [ - "discord-reaction-added" - ], - "actions": [ - "trello-create-card" - ], + "sources": ["discord-reaction-added"], + "actions": ["trello-create-card"], "persona": "complex-workflow" }, { "query": "post giphy to discord", - "triggers": [], - "actions": [ - "discord-send-message" - ], + "sources": [], + "actions": ["discord-send-message"], "persona": "succinct" }, { "query": "I would like new Pocket items to be saved to Instapaper", - "triggers": [ - "pocket-new-item" - ], - "actions": [ - "instapaper-add-url" - ], + "sources": ["pocket-new-item"], + "actions": ["instapaper-add-url"], "persona": "verbose" }, { "query": "When Twitch stream starts, post to social media", - "triggers": [ - "twitch-streams-by-streamer" - ], - "actions": [ - "twitter-create-tweet", - "facebook_pages-create-post" - ], + "sources": ["twitch-streams-by-streamer"], + "actions": ["twitter-create-tweet", "facebook_pages-create-post"], "persona": "complex-workflow" }, { "query": "make freshsales contact", - "triggers": [], + "sources": [], "actions": [], "persona": "sloppy" }, { "query": "When new article appears in Inoreader folder, save to Notion", - "triggers": [ - "inoreader-new-article-in-folder" - ], - "actions": [ - "notion-create-page-from-database" - ], + "sources": ["inoreader-new-article-in-folder"], + "actions": ["notion-create-page-from-database"], "persona": "complex-workflow" }, { "query": "create servicenow incident", - "triggers": [], + "sources": [], "actions": [], "persona": "succinct" }, { "query": "I want to receive Bandwidth SMS messages in Telegram", - "triggers": [ - "bandwidth-new-incoming-sms" - ], - "actions": [ - "telegram_bot_api-send-text-message-or-reply" - ], + "sources": ["bandwidth-new-incoming-sms"], + "actions": ["telegram_bot_api-send-text-message-or-reply"], "persona": "verbose" }, { "query": "When a new comment is posted on WordPress, analyze with GPT", - "triggers": [], - "actions": [ - "openai-chat" - ], + "sources": [], + "actions": ["openai-chat"], "persona": "complex-workflow" }, { "query": "send mailgun email", - "triggers": [], - "actions": [ - "mailgun-send-email" - ], + "sources": [], + "actions": ["mailgun-send-email"], "persona": "succinct" }, { "query": "When RingCentral records a call, transcribe with OpenAI", - "triggers": [ - "ringcentral-new-call-recording" - ], - "actions": [ - "openai-create-transcription" - ], + "sources": ["ringcentral-new-call-recording"], + "actions": ["openai-create-transcription"], "persona": "complex-workflow" }, { "query": "create shipstation evnt", - "triggers": [], + "sources": [], "actions": [], "persona": "sloppy" }, { "query": "I need my Google Contacts to sync with Microsoft Outlook", - "triggers": [ - "google_contacts-contact-created" - ], - "actions": [ - "microsoft_outlook-create-contact" - ], + "sources": ["google_contacts-contact-created"], + "actions": ["microsoft_outlook-create-contact"], "persona": "verbose" }, { "query": "When new Squarespace transaction occurs, update Quickbooks", - "triggers": [ - "squarespace-new-create-transaction" - ], - "actions": [ - "quickbooks-create-invoice" - ], + "sources": ["squarespace-new-create-transaction"], + "actions": ["quickbooks-create-invoice"], "persona": "complex-workflow" }, { "query": "create zoho mail", - "triggers": [], - "actions": [ - "zoho_mail-send-email" - ], + "sources": [], + "actions": ["zoho_mail-send-email"], "persona": "succinct" }, { "query": "When Firebase document changes, send Line notification", - "triggers": [ - "firebase_admin_sdk-new-doc-in-firestore-collection" - ], - "actions": [ - "line-send-notification-message" - ], + "sources": ["firebase_admin_sdk-new-doc-in-firestore-collection"], + "actions": ["line-send-notification-message"], "persona": "complex-workflow" }, { "query": "add pinterest pin", - "triggers": [], - "actions": [ - "pinterest-create-pin" - ], + "sources": [], + "actions": ["pinterest-create-pin"], "persona": "succinct" } ] -} \ No newline at end of file +} diff --git a/packages/evals/component_retrieval/eval-test-suite-ai-focus-filtered.json b/packages/evals/component_retrieval/eval-test-suite-ai-focus-filtered.json index afa6ed0658439..515bcc1a192cb 100644 --- a/packages/evals/component_retrieval/eval-test-suite-ai-focus-filtered.json +++ b/packages/evals/component_retrieval/eval-test-suite-ai-focus-filtered.json @@ -2,20 +2,13 @@ "evaluationTests": [ { "query": "When new support tickets come in through Zendesk, analyze sentiment with GPT and prioritize in Linear based on urgency", - "triggers": [ - "zendesk-new-ticket" - ], - "actions": [ - "openai-chat", - "linear-create-issue" - ], + "sources": ["zendesk-new-ticket"], + "actions": ["openai-chat", "linear-create-issue"], "persona": "complex-workflow" }, { "query": "I need sales calls recorded in Gong to be transcribed and summarized for the team in Slack", - "triggers": [ - "gong-new-call" - ], + "sources": ["gong-new-call"], "actions": [ "openai-create-transcription", "openai-chat", @@ -25,51 +18,31 @@ }, { "query": "When leads submit Typeform responses, use GPT to qualify them and update their status in HubSpot", - "triggers": [ - "typeform-new-submission" - ], - "actions": [ - "openai-chat", - "hubspot-create-or-update-contact" - ], + "sources": ["typeform-new-submission"], + "actions": ["openai-chat", "hubspot-create-or-update-contact"], "persona": "complex-workflow" }, { "query": "Get help from AI to classify and organize our Notion knowledge base", - "triggers": [ - "notion-new-page" - ], - "actions": [ - "openai-chat", - "notion-create-page-from-database" - ], + "sources": ["notion-new-page"], + "actions": ["openai-chat", "notion-create-page-from-database"], "persona": "task-oriented" }, { "query": "When customers message us on Intercom, analyze intent with GPT before creating tickets", - "triggers": [ - "intercom-new-conversation" - ], - "actions": [ - "openai-chat", - "linear-create-issue" - ], + "sources": ["intercom-new-conversation"], + "actions": ["openai-chat", "linear-create-issue"], "persona": "complex-workflow" }, { "query": "Analyze customer feedback from Delighted with AI and update account health in Salesforce", - "triggers": [], - "actions": [ - "openai-chat", - "salesforce_rest_api-update-contact" - ], + "sources": [], + "actions": ["openai-chat", "salesforce_rest_api-update-contact"], "persona": "task-oriented" }, { "query": "When new videos are uploaded to Zoom, I want them transcribed and summarized for the team", - "triggers": [ - "zoom-recording-completed" - ], + "sources": ["zoom-recording-completed"], "actions": [ "openai-create-transcription", "openai-chat", @@ -79,136 +52,81 @@ }, { "query": "Use AI to analyze Github issues and suggest priority levels in Linear", - "triggers": [], - "actions": [ - "openai-chat", - "linear-create-issue" - ], + "sources": [], + "actions": ["openai-chat", "linear-create-issue"], "persona": "task-oriented" }, { "query": "When documents are uploaded to Google Drive, use GPT to generate summaries in Notion", - "triggers": [ - "google_drive-new-files-instant" - ], - "actions": [ - "openai-chat", - "notion-create-page-from-database" - ], + "sources": ["google_drive-new-files-instant"], + "actions": ["openai-chat", "notion-create-page-from-database"], "persona": "complex-workflow" }, { "query": "Analyze customer churn risk based on Intercom conversations using GPT", - "triggers": [ - "intercom-new-conversation" - ], - "actions": [ - "openai-chat", - "hubspot-create-or-update-contact" - ], + "sources": ["intercom-new-conversation"], + "actions": ["openai-chat", "hubspot-create-or-update-contact"], "persona": "task-oriented" }, { "query": "When new RSS articles mention our company, use AI to analyze sentiment and alert team", - "triggers": [ - "rss-new-item-in-feed" - ], - "actions": [ - "openai-chat", - "slack-send-message" - ], + "sources": ["rss-new-item-in-feed"], + "actions": ["openai-chat", "slack-send-message"], "persona": "complex-workflow" }, { "query": "I need GPT to help categorize incoming feature requests from Canny into our product roadmap", - "triggers": [], - "actions": [ - "openai-chat", - "notion-create-page-from-database" - ], + "sources": [], + "actions": ["openai-chat", "notion-create-page-from-database"], "persona": "verbose" }, { "query": "When customers respond to our Typeform survey, analyze trends with AI and update dashboards", - "triggers": [ - "typeform-new-submission" - ], - "actions": [ - "openai-chat", - "google_sheets-add-single-row" - ], + "sources": ["typeform-new-submission"], + "actions": ["openai-chat", "google_sheets-add-single-row"], "persona": "complex-workflow" }, { "query": "Use GPT to analyze Calendly meeting notes and create action items in Asana", - "triggers": [ - "calendly_v2-new-event-scheduled" - ], - "actions": [ - "openai-chat", - "asana-create-task" - ], + "sources": ["calendly_v2-new-event-scheduled"], + "actions": ["openai-chat", "asana-create-task"], "persona": "task-oriented" }, { "query": "When support team sends emails in Gmail, let AI check tone and suggest improvements", - "triggers": [ - "gmail-new-email" - ], - "actions": [ - "openai-chat" - ], + "sources": ["gmail-new-email"], + "actions": ["openai-chat"], "persona": "complex-workflow" }, { "query": "I need customer conversations from Help Scout to be analyzed for product feedback", - "triggers": [], - "actions": [ - "openai-chat", - "notion-create-page-from-database" - ], + "sources": [], + "actions": ["openai-chat", "notion-create-page-from-database"], "persona": "verbose" }, { "query": "When new comments appear on our YouTube videos, use AI to moderate and flag issues", - "triggers": [ - "youtube_data_api-new-comment-posted" - ], - "actions": [ - "openai-chat", - "slack-send-message" - ], + "sources": ["youtube_data_api-new-comment-posted"], + "actions": ["openai-chat", "slack-send-message"], "persona": "complex-workflow" }, { "query": "Use GPT to analyze Twitter mentions and create support tickets when needed", - "triggers": [ - "twitter-new-mention-received-by-user" - ], - "actions": [ - "openai-chat", - "linear-create-issue" - ], + "sources": ["twitter-new-mention-received-by-user"], + "actions": ["openai-chat", "linear-create-issue"], "persona": "task-oriented" }, { "query": "When deals close in Salesforce, use AI to generate personalized onboarding docs", - "triggers": [], - "actions": [ - "openai-chat", - "google_docs-create-document" - ], + "sources": [], + "actions": ["openai-chat", "google_docs-create-document"], "persona": "complex-workflow" }, { "query": "I want GPT to help write better commit messages for our Github repos", - "triggers": [ - "github-new-commit" - ], - "actions": [ - "openai-chat" - ], + "sources": ["github-new-commit"], + "actions": ["openai-chat"], "persona": "verbose" } ] -} \ No newline at end of file +} diff --git a/packages/evals/component_retrieval/eval-test-suite-graduated-corrected-filtered.json b/packages/evals/component_retrieval/eval-test-suite-graduated-corrected-filtered.json index d36db6d29071d..6b651ffbdc09e 100644 --- a/packages/evals/component_retrieval/eval-test-suite-graduated-corrected-filtered.json +++ b/packages/evals/component_retrieval/eval-test-suite-graduated-corrected-filtered.json @@ -2,65 +2,49 @@ "evaluationTests": [ { "query": "send slack message", - "triggers": [], - "actions": [ - "slack-send-message" - ], + "sources": [], + "actions": ["slack-send-message"], "persona": "succinct" }, { "query": "create issue in linear", - "triggers": [], - "actions": [ - "linear-create-issue" - ], + "sources": [], + "actions": ["linear-create-issue"], "persona": "succinct" }, { "query": "sync my contacts", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "When new emails arrive in Gmail, save attachments to Google Drive", - "triggers": [ - "gmail-new-email" - ], - "actions": [ - "google_drive-upload-file" - ], + "sources": ["gmail-new-email"], + "actions": ["google_drive-upload-file"], "persona": "verbose" }, { "query": "add roblox purchases to quickbooks", - "triggers": [], - "actions": [ - "quickbooks-create-invoice" - ], + "sources": [], + "actions": ["quickbooks-create-invoice"], "persona": "no-component" }, { "query": "When someone fills out my Typeform, notify me in Slack", - "triggers": [ - "typeform-new-submission" - ], - "actions": [ - "slack-send-message" - ], + "sources": ["typeform-new-submission"], + "actions": ["slack-send-message"], "persona": "verbose" }, { "query": "automate my business processes", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "When new videos are uploaded to YouTube, transcribe with OpenAI and post summary to Discord", - "triggers": [ - "youtube_data_api-new-videos" - ], + "sources": ["youtube_data_api-new-videos"], "actions": [ "openai-create-transcription", "openai-chat", @@ -70,49 +54,37 @@ }, { "query": "When customer submits form in AirKit, create contact in HubSpot", - "triggers": [], - "actions": [ - "hubspot-create-or-update-contact" - ], + "sources": [], + "actions": ["hubspot-create-or-update-contact"], "persona": "no-component" }, { "query": "When a deal closes in Salesforce, generate invoice in QuickBooks and notify accounting in Slack", - "triggers": [], - "actions": [ - "quickbooks-create-invoice", - "slack-send-message" - ], + "sources": [], + "actions": ["quickbooks-create-invoice", "slack-send-message"], "persona": "complex-workflow" }, { "query": "make my team more productive", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "When new support tickets come in through Zendesk, analyze sentiment with GPT and create tasks in Linear", - "triggers": [ - "zendesk-new-ticket" - ], - "actions": [ - "openai-chat", - "linear-create-issue" - ], + "sources": ["zendesk-new-ticket"], + "actions": ["openai-chat", "linear-create-issue"], "persona": "complex-workflow" }, { "query": "When deals update in Copper CRM, sync to Oracle NetSuite", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "When files are added to Dropbox, use GPT to analyze content, create summaries in Notion, and notify relevant teams in Slack", - "triggers": [ - "dropbox-new-file" - ], + "sources": ["dropbox-new-file"], "actions": [ "openai-chat", "notion-create-page-from-database", @@ -122,35 +94,25 @@ }, { "query": "help me with reporting", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "When a customer books through Calendly, add to HubSpot, create Asana task, and start onboarding sequence", - "triggers": [ - "calendly_v2-new-event-scheduled" - ], - "actions": [ - "hubspot-create-or-update-contact", - "asana-create-task" - ], + "sources": ["calendly_v2-new-event-scheduled"], + "actions": ["hubspot-create-or-update-contact", "asana-create-task"], "persona": "complex-workflow" }, { "query": "When feedback is submitted in UserVoice, analyze with GPT and create ticket in Jira", - "triggers": [], - "actions": [ - "openai-chat", - "jira-create-issue" - ], + "sources": [], + "actions": ["openai-chat", "jira-create-issue"], "persona": "no-component" }, { "query": "When new recordings appear in Gong, transcribe with OpenAI, analyze for key insights, update deal notes in Salesforce, and notify sales team in Slack", - "triggers": [ - "gong-new-call" - ], + "sources": ["gong-new-call"], "actions": [ "openai-create-transcription", "openai-chat", @@ -161,19 +123,13 @@ }, { "query": "When customer reaches usage threshold in Stripe, check status in ChargeBee, update account in HubSpot, and trigger customer success workflow", - "triggers": [ - "stripe-subscription-updated" - ], - "actions": [ - "hubspot-create-or-update-contact" - ], + "sources": ["stripe-subscription-updated"], + "actions": ["hubspot-create-or-update-contact"], "persona": "complex-workflow" }, { "query": "When new document is uploaded to Google Drive, use GPT for content analysis, create summaries in Notion, add action items in Linear, notify teams in Slack, and schedule follow-up in Google Calendar", - "triggers": [ - "google_drive-new-files-instant" - ], + "sources": ["google_drive-new-files-instant"], "actions": [ "openai-chat", "notion-create-page-from-database", @@ -184,4 +140,4 @@ "persona": "complex-workflow" } ] -} \ No newline at end of file +} diff --git a/packages/evals/component_retrieval/eval-test-suite-invalid-queries-filtered.json b/packages/evals/component_retrieval/eval-test-suite-invalid-queries-filtered.json index 7cb0a18d18af9..144eb4de15116 100644 --- a/packages/evals/component_retrieval/eval-test-suite-invalid-queries-filtered.json +++ b/packages/evals/component_retrieval/eval-test-suite-invalid-queries-filtered.json @@ -2,309 +2,309 @@ "evaluationTests": [ { "query": "sync my data between systems", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "send notifications when something important happens", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "create a daily report", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "automate my sales process", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "connect my database", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "help me with customer support", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "track user activity", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "move files automatically", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "update my calendar", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "make my workflow more efficient", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "integrate with my existing tools", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "analyze my customer feedback", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "automate document creation", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "send reminders to my team", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "track project progress", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "monitor system performance", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "backup my important files", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "generate custom reports", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "create new users", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "manage my contacts", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "update status automatically", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "help me with social media", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "organize my documents", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "sync my calendars", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "manage my team better", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "automate invoice processing", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "track employee time", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "monitor website changes", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "schedule my posts", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "manage my inventory", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "sync my contact lists", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "automate email responses", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "create task assignments", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "track project deadlines", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "manage my subscriptions", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "update my database", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "automate customer onboarding", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "sync team communications", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "manage project resources", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "track sales performance", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "automate data entry", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "generate reports weekly", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "manage my workflows", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "integrate payment processing", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "automate file transfers", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "sync my documents", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "manage customer relationships", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "track employee performance", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "automate marketing campaigns", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "manage my leads", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" }, { "query": "sync my apps", - "triggers": [], + "sources": [], "actions": [], "persona": "invalid-query" } ] -} \ No newline at end of file +} diff --git a/packages/evals/component_retrieval/eval-test-suite-no-components-filtered.json b/packages/evals/component_retrieval/eval-test-suite-no-components-filtered.json index 1f065d8e5b7cd..029710c7c42ec 100644 --- a/packages/evals/component_retrieval/eval-test-suite-no-components-filtered.json +++ b/packages/evals/component_retrieval/eval-test-suite-no-components-filtered.json @@ -2,275 +2,249 @@ "evaluationTests": [ { "query": "When someone books a demo in Chili Piper, create a contact in Apollo", - "triggers": [], - "actions": [ - "apollo_io-create-contact" - ], + "sources": [], + "actions": ["apollo_io-create-contact"], "persona": "no-component" }, { "query": "When we get a new review on G2, post it to Slack", - "triggers": [], - "actions": [ - "slack-send-message" - ], + "sources": [], + "actions": ["slack-send-message"], "persona": "no-component" }, { "query": "Add new Gong calls to my Guru knowledge base", - "triggers": [ - "gong-new-call" - ], + "sources": ["gong-new-call"], "actions": [], "persona": "no-component" }, { "query": "When someone signs our DocuSign NDA, grant access in Okta", - "triggers": [ - "docusign-envelope-sent-or-complete" - ], + "sources": ["docusign-envelope-sent-or-complete"], "actions": [], "persona": "no-component" }, { "query": "Create a new project in Wrike when Typeform is submitted", - "triggers": [ - "typeform-new-submission" - ], + "sources": ["typeform-new-submission"], "actions": [], "persona": "no-component" }, { "query": "When customer churns in Chargebee, remove access in Auth0 and revoke Workato license", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "Send new Snowflake query results to PowerBI", - "triggers": ["snowflake-query-results"], + "sources": ["snowflake-query-results"], "actions": [], "persona": "no-component" }, { "query": "When new issue is created in Azure DevOps, create ticket in ServiceNow", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "Add rowing sessions from Hydrow to my Apple Health", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "When someone mentions us on Blind, send to Discord", - "triggers": [], - "actions": [ - "discord-send-message" - ], + "sources": [], + "actions": ["discord-send-message"], "persona": "no-component" }, { "query": "Create tasks in Monday.com from Shortcut stories", - "triggers": [], - "actions": [ - "monday-create-item" - ], + "sources": [], + "actions": ["monday-create-item"], "persona": "no-component" }, { "query": "When a new invoice is created in NetSuite, sync to Sage Intacct", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "Create Lattice goals from Asana projects", - "triggers": [ - "asana-new-project" - ], + "sources": ["asana-new-project"], "actions": [], "persona": "no-component" }, { "query": "When new candidate appears in Lever, create user in JumpCloud", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "Add new LeanIX components to our Architecture Handbook in Confluence", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "When a deal closes in Close.com, create project in TeamGantt", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "Create Coda doc from new Pitch presentation", - "triggers": [], - "actions": [ - "coda-create-rows" - ], + "sources": [], + "actions": ["coda-create-rows"], "persona": "no-component" }, { "query": "When contract is approved in Ironclad, create workspace in Retool", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "Add new Gong snippets to Chorus.ai library", - "triggers": [ - "gong-new-call" - ], + "sources": ["gong-new-call"], "actions": [], "persona": "no-component" }, { "query": "When employee is terminated in Namely, revoke access in Rippling", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "Sync product specs from Productboard to Aha!", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "Create Metabase dashboard when new Amplitude cohort is created", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "When new report runs in Looker, send to Tableau", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "Add new Greenhouse candidates to iCIMS", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "When feature is deployed in LaunchDarkly, update status in StatusPage", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "Create Lucidchart diagram from Miro board", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "When new user signs up in Mixpanel, create profile in Pendo", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "Add new 6sense accounts to RollWorks", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "When new survey response in Qualtrics, add to Gainsight", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "Create tasks in Wrike from Clubhouse stories", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "When customer submits proof in Filestage, add to Bynder", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "Create Qatalog process from new Notion template", - "triggers": [ - "notion-new-page" - ], + "sources": ["notion-new-page"], "actions": [], "persona": "no-component" }, { "query": "When user requests access in Mesh Directory, create ticket in Jira Service Management", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "Add new candidates from SmartRecruiters to Workday", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "When new post is created in Discourse, share on Yammer", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "Create monday.com pulse from Range check-in", - "triggers": [], - "actions": [ - "monday-create-item" - ], + "sources": [], + "actions": ["monday-create-item"], "persona": "no-component" }, { "query": "When new task is created in Process Street, add to Teamwork", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "Add new Demodesk recordings to Gong", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "When new article is published on Dev.to, cross-post to Hashnode", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "Create UserPilot experience from new Pendo guide", - "triggers": [], + "sources": [], "actions": [], "persona": "no-component" }, { "query": "When new document is uploaded to Box, scan with Nightfall", - "triggers": [ - "box-new-file" - ], + "sources": ["box-new-file"], "actions": [], "persona": "no-component" } ] -} \ No newline at end of file +} diff --git a/packages/evals/component_retrieval/eval-test-suite-similar-apps-filtered.json b/packages/evals/component_retrieval/eval-test-suite-similar-apps-filtered.json index ce22a121776b0..4d374c6989007 100644 --- a/packages/evals/component_retrieval/eval-test-suite-similar-apps-filtered.json +++ b/packages/evals/component_retrieval/eval-test-suite-similar-apps-filtered.json @@ -2,706 +2,453 @@ "evaluationTests": [ { "query": "add new row in google spreadsheet", - "triggers": [], - "actions": [ - "google_sheets-add-single-row" - ], + "sources": [], + "actions": ["google_sheets-add-single-row"], "persona": "similar-app" }, { "query": "When someone books through Calendly, add event to Google Calendar", - "triggers": [ - "calendly_v2-new-event-scheduled" - ], - "actions": [ - "google_calendar-create-event" - ], + "sources": ["calendly_v2-new-event-scheduled"], + "actions": ["google_calendar-create-event"], "persona": "similar-app" }, { "query": "create new task in microsoft todo", - "triggers": [], - "actions": [ - "microsofttodo-create-task" - ], + "sources": [], + "actions": ["microsofttodo-create-task"], "persona": "similar-app" }, { "query": "When files are uploaded to Box, create Google Doc", - "triggers": [ - "box-new-file" - ], - "actions": [ - "google_docs-create-document" - ], + "sources": ["box-new-file"], + "actions": ["google_docs-create-document"], "persona": "similar-app" }, { "query": "send message using Discord bot", - "triggers": [], - "actions": [ - "discord_bot-send-message" - ], + "sources": [], + "actions": ["discord_bot-send-message"], "persona": "similar-app" }, { "query": "When TidyCal appointment is booked, create Google Calendar event", - "triggers": [ - "tidycal-new-appointment-booked" - ], - "actions": [ - "google_calendar-create-event" - ], + "sources": ["tidycal-new-appointment-booked"], + "actions": ["google_calendar-create-event"], "persona": "similar-app" }, { "query": "analyze text with Azure OpenAI", - "triggers": [], - "actions": [ - "azure_openai_service-chat" - ], + "sources": [], + "actions": ["azure_openai_service-chat"], "persona": "similar-app" }, { "query": "When new email in Gmail, forward to Outlook", - "triggers": [ - "gmail-new-email-received" - ], - "actions": [ - "microsoft_outlook-send-email" - ], + "sources": ["gmail-new-email-received"], + "actions": ["microsoft_outlook-send-email"], "persona": "similar-app" }, { "query": "When Box gets new file, copy to Dropbox", - "triggers": [ - "box-new-file" - ], - "actions": [ - "dropbox-upload-file" - ], + "sources": ["box-new-file"], + "actions": ["dropbox-upload-file"], "persona": "similar-app" }, { "query": "send message through Discord webhook", - "triggers": [], - "actions": [ - "discord-send-message" - ], + "sources": [], + "actions": ["discord-send-message"], "persona": "similar-app" }, { "query": "When files are added to OneDrive folder, save to Google Drive", - "triggers": [ - "microsoft_onedrive-new-file-in-folder" - ], - "actions": [ - "google_drive-upload-file" - ], + "sources": ["microsoft_onedrive-new-file-in-folder"], + "actions": ["google_drive-upload-file"], "persona": "similar-app" }, { "query": "create new task in Google Tasks when Todoist task is created", - "triggers": [ - "todoist-new-task" - ], - "actions": [ - "google_tasks-create-task" - ], + "sources": ["todoist-new-task"], + "actions": ["google_tasks-create-task"], "persona": "similar-app" }, { "query": "When new Outlook calendar event created, add to Google Calendar", - "triggers": [ - "microsoft_outlook_calendar-new-calendar-event" - ], - "actions": [ - "google_calendar-create-event" - ], + "sources": ["microsoft_outlook_calendar-new-calendar-event"], + "actions": ["google_calendar-create-event"], "persona": "similar-app" }, { "query": "use GPT-4 to analyze text with OpenAI", - "triggers": [], - "actions": [ - "openai-chat" - ], + "sources": [], + "actions": ["openai-chat"], "persona": "similar-app" }, { "query": "When Dropbox file is modified, upload to OneDrive", - "triggers": [ - "dropbox-new-file" - ], - "actions": [ - "microsoft_onedrive-upload-file" - ], + "sources": ["dropbox-new-file"], + "actions": ["microsoft_onedrive-upload-file"], "persona": "similar-app" }, { "query": "send email through Gmail when Zoho Mail receives new message", - "triggers": [ - "zoho_mail-new-email" - ], - "actions": [ - "gmail-send-email" - ], + "sources": ["zoho_mail-new-email"], + "actions": ["gmail-send-email"], "persona": "similar-app" }, { "query": "When Google Calendar event starts, create Zoom meeting", - "triggers": [ - "google_calendar-upcoming-event-alert" - ], - "actions": [ - "zoom-create-meeting" - ], + "sources": ["google_calendar-upcoming-event-alert"], + "actions": ["zoom-create-meeting"], "persona": "similar-app" }, { "query": "use Claude to analyze text", - "triggers": [], - "actions": [ - "anthropic-chat" - ], + "sources": [], + "actions": ["anthropic-chat"], "persona": "similar-app" }, { "query": "When Microsoft Todo task is created, add to Todoist", - "triggers": [ - "microsofttodo-new-task-created" - ], - "actions": [ - "todoist-create-task" - ], + "sources": ["microsofttodo-new-task-created"], + "actions": ["todoist-create-task"], "persona": "similar-app" }, { "query": "create new task in Asana when ClickUp task is created", - "triggers": [ - "clickup-new-task" - ], - "actions": [ - "asana-create-task" - ], + "sources": ["clickup-new-task"], + "actions": ["asana-create-task"], "persona": "similar-app" }, { "query": "send transactional email through Mailgun", - "triggers": [], - "actions": [ - "mailgun-send-email" - ], + "sources": [], + "actions": ["mailgun-send-email"], "persona": "similar-app" }, { "query": "When new subscriber in Mailchimp, add contact to Mailerlite", - "triggers": [ - "mailchimp-new-subscriber" - ], - "actions": [ - "mailerlite-subscribe-to-group" - ], + "sources": ["mailchimp-new-subscriber"], + "actions": ["mailerlite-subscribe-to-group"], "persona": "similar-app" }, { "query": "create document in Microsoft OneDrive when Google Drive gets new file", - "triggers": [ - "google_drive-new-files-instant" - ], - "actions": [ - "microsoft_onedrive-upload-file" - ], + "sources": ["google_drive-new-files-instant"], + "actions": ["microsoft_onedrive-upload-file"], "persona": "similar-app" }, { "query": "use Palm API to analyze text", - "triggers": [], - "actions": [ - "google_palm_api-chat" - ], + "sources": [], + "actions": ["google_palm_api-chat"], "persona": "similar-app" }, { "query": "When Zoom recording completes, save to Google Drive and create Box folder", - "triggers": [ - "zoom-recording-completed" - ], - "actions": [ - "google_drive-upload-file", - "box-upload-file" - ], + "sources": ["zoom-recording-completed"], + "actions": ["google_drive-upload-file", "box-upload-file"], "persona": "similar-app" }, { "query": "add new row in google spreadsheet", - "triggers": [], - "actions": [ - "google_sheets-add-single-row" - ], + "sources": [], + "actions": ["google_sheets-add-single-row"], "persona": "similar-app" }, { "query": "When someone books through Calendly, add event to Google Calendar", - "triggers": [ - "calendly_v2-new-event-scheduled" - ], - "actions": [ - "google_calendar-create-event" - ], + "sources": ["calendly_v2-new-event-scheduled"], + "actions": ["google_calendar-create-event"], "persona": "similar-app" }, { "query": "create new task in microsoft todo", - "triggers": [], - "actions": [ - "microsofttodo-create-task" - ], + "sources": [], + "actions": ["microsofttodo-create-task"], "persona": "similar-app" }, { "query": "When files are uploaded to Box, create Google Doc", - "triggers": [ - "box-new-file" - ], - "actions": [ - "google_docs-create-document" - ], + "sources": ["box-new-file"], + "actions": ["google_docs-create-document"], "persona": "similar-app" }, { "query": "send message using Discord bot", - "triggers": [], - "actions": [ - "discord_bot-send-message" - ], + "sources": [], + "actions": ["discord_bot-send-message"], "persona": "similar-app" }, { "query": "When TidyCal appointment is booked, create Google Calendar event", - "triggers": [ - "tidycal-new-appointment-booked" - ], - "actions": [ - "google_calendar-create-event" - ], + "sources": ["tidycal-new-appointment-booked"], + "actions": ["google_calendar-create-event"], "persona": "similar-app" }, { "query": "analyze text with Azure OpenAI", - "triggers": [], - "actions": [ - "azure_openai_service-chat" - ], + "sources": [], + "actions": ["azure_openai_service-chat"], "persona": "similar-app" }, { "query": "When new email in Gmail, forward to Outlook", - "triggers": [ - "gmail-new-email" - ], - "actions": [ - "microsoft_outlook-send-email" - ], + "sources": ["gmail-new-email"], + "actions": ["microsoft_outlook-send-email"], "persona": "similar-app" }, { "query": "When Box gets new file, copy to Dropbox", - "triggers": [ - "box-new-file" - ], - "actions": [ - "dropbox-upload-file" - ], + "sources": ["box-new-file"], + "actions": ["dropbox-upload-file"], "persona": "similar-app" }, { "query": "send message through Discord webhook", - "triggers": [], - "actions": [ - "discord-send-message" - ], + "sources": [], + "actions": ["discord-send-message"], "persona": "similar-app" }, { "query": "When files are added to OneDrive folder, save to Google Drive", - "triggers": [ - "microsoft_onedrive-new-file-in-folder" - ], - "actions": [ - "google_drive-upload-file" - ], + "sources": ["microsoft_onedrive-new-file-in-folder"], + "actions": ["google_drive-upload-file"], "persona": "similar-app" }, { "query": "create new task in Google Tasks when Todoist task is created", - "triggers": [ - "todoist-new-task" - ], - "actions": [ - "google_tasks-create-task" - ], + "sources": ["todoist-new-task"], + "actions": ["google_tasks-create-task"], "persona": "similar-app" }, { "query": "When new Outlook calendar event created, add to Google Calendar", - "triggers": [ - "microsoft_outlook_calendar-new-calendar-event" - ], - "actions": [ - "google_calendar-create-event" - ], + "sources": ["microsoft_outlook_calendar-new-calendar-event"], + "actions": ["google_calendar-create-event"], "persona": "similar-app" }, { "query": "use GPT-4 to analyze text with OpenAI", - "triggers": [], - "actions": [ - "openai-chat" - ], + "sources": [], + "actions": ["openai-chat"], "persona": "similar-app" }, { "query": "When Dropbox file is modified, upload to OneDrive", - "triggers": [ - "dropbox-new-file" - ], - "actions": [ - "microsoft_onedrive-upload-file" - ], + "sources": ["dropbox-new-file"], + "actions": ["microsoft_onedrive-upload-file"], "persona": "similar-app" }, { "query": "send email through Gmail when Zoho Mail receives new message", - "triggers": [ - "zoho_mail-new-email" - ], - "actions": [ - "gmail-send-email" - ], + "sources": ["zoho_mail-new-email"], + "actions": ["gmail-send-email"], "persona": "similar-app" }, { "query": "When Google Calendar event starts, create Zoom meeting", - "triggers": [ - "google_calendar-upcoming-event-alert" - ], - "actions": [ - "zoom-create-meeting" - ], + "sources": ["google_calendar-upcoming-event-alert"], + "actions": ["zoom-create-meeting"], "persona": "similar-app" }, { "query": "use Claude to analyze text", - "triggers": [], - "actions": [ - "anthropic-chat" - ], + "sources": [], + "actions": ["anthropic-chat"], "persona": "similar-app" }, { "query": "When Microsoft Todo task is created, add to Todoist", - "triggers": [ - "microsofttodo-new-task-created" - ], - "actions": [ - "todoist-create-task" - ], + "sources": ["microsofttodo-new-task-created"], + "actions": ["todoist-create-task"], "persona": "similar-app" }, { "query": "create new task in Asana when ClickUp task is created", - "triggers": [ - "clickup-new-task" - ], - "actions": [ - "asana-create-task" - ], + "sources": ["clickup-new-task"], + "actions": ["asana-create-task"], "persona": "similar-app" }, { "query": "send transactional email through Mailgun", - "triggers": [], - "actions": [ - "mailgun-send-email" - ], + "sources": [], + "actions": ["mailgun-send-email"], "persona": "similar-app" }, { "query": "When new subscriber in Mailchimp, add contact to Mailerlite", - "triggers": [ - "mailchimp-new-subscriber" - ], - "actions": [ - "mailerlite-subscribe-to-group" - ], + "sources": ["mailchimp-new-subscriber"], + "actions": ["mailerlite-subscribe-to-group"], "persona": "similar-app" }, { "query": "create document in Microsoft OneDrive when Google Drive gets new file", - "triggers": [ - "google_drive-new-files-instant" - ], - "actions": [ - "microsoft_onedrive-upload-file" - ], + "sources": ["google_drive-new-files-instant"], + "actions": ["microsoft_onedrive-upload-file"], "persona": "similar-app" }, { "query": "use Palm API to analyze text", - "triggers": [], - "actions": [ - "google_palm_api-chat" - ], + "sources": [], + "actions": ["google_palm_api-chat"], "persona": "similar-app" }, { "query": "When Zoom recording completes, save to Google Drive and create Box folder", - "triggers": [ - "zoom-recording-completed" - ], - "actions": [ - "google_drive-upload-file", - "box-upload-file" - ], + "sources": ["zoom-recording-completed"], + "actions": ["google_drive-upload-file", "box-upload-file"], "persona": "similar-app" }, { "query": "When Google Forms gets new response, add to Typeform analytics", - "triggers": [ - "google_forms-new-form-answer" - ], - "actions": [ - "typeform-list-responses" - ], + "sources": ["google_forms-new-form-answer"], + "actions": ["typeform-list-responses"], "persona": "similar-app" }, { "query": "create new presentation in Google Slides when Dropbox gets new file", - "triggers": [ - "dropbox-new-file" - ], - "actions": [ - "google_slides-create-presentation" - ], + "sources": ["dropbox-new-file"], + "actions": ["google_slides-create-presentation"], "persona": "similar-app" }, { "query": "send message on Microsoft Teams channel", - "triggers": [], - "actions": [ - "microsoft_teams-send-channel-message" - ], + "sources": [], + "actions": ["microsoft_teams-send-channel-message"], "persona": "similar-app" }, { "query": "When new Gmail attachment arrives, save to Box", - "triggers": [ - "gmail-new-attachment-received" - ], - "actions": [ - "box-upload-file" - ], + "sources": ["gmail-new-attachment-received"], + "actions": ["box-upload-file"], "persona": "similar-app" }, { "query": "use Azure OpenAI to translate text", - "triggers": [], - "actions": [ - "azure_openai_service-translate-text" - ], + "sources": [], + "actions": ["azure_openai_service-translate-text"], "persona": "similar-app" }, { "query": "When Microsoft Outlook gets email with label, create Google Calendar event", - "triggers": [ - "microsoft_outlook-new-email" - ], - "actions": [ - "google_calendar-create-event" - ], + "sources": ["microsoft_outlook-new-email"], + "actions": ["google_calendar-create-event"], "persona": "similar-app" }, { "query": "send notifications through MailerLite", - "triggers": [], - "actions": [ - "mailerlite-subscribe-to-group" - ], + "sources": [], + "actions": ["mailerlite-subscribe-to-group"], "persona": "similar-app" }, { "query": "When task is completed in Todoist, update Microsoft Todo", - "triggers": [ - "todoist-completed-task" - ], - "actions": [ - "microsofttodo-update-task" - ], + "sources": ["todoist-completed-task"], + "actions": ["microsofttodo-update-task"], "persona": "similar-app" }, { "query": "use Cohere to analyze sentiment", - "triggers": [], - "actions": [ - "cohere_platform-chat" - ], + "sources": [], + "actions": ["cohere_platform-chat"], "persona": "similar-app" }, { "query": "When someone books through TidyCal, add to Calendly availability", - "triggers": [ - "tidycal-new-appointment-booked" - ], + "sources": ["tidycal-new-appointment-booked"], "actions": [], "persona": "similar-app" }, { "query": "send chat message through Discord bot when Slack gets message", - "triggers": [ - "slack-new-message-in-channels" - ], - "actions": [ - "discord_bot-send-message" - ], + "sources": ["slack-new-message-in-channels"], + "actions": ["discord_bot-send-message"], "persona": "similar-app" }, { "query": "When file changes in OneDrive, sync to Google Drive and Box", - "triggers": [ - "microsoft_onedrive-new-file-created" - ], - "actions": [ - "google_drive-upload-file", - "box-upload-file" - ], + "sources": ["microsoft_onedrive-new-file-created"], + "actions": ["google_drive-upload-file", "box-upload-file"], "persona": "similar-app" }, { "query": "create task in Google Tasks when Microsoft Todo list is created", - "triggers": [ - "microsofttodo-new-list-created" - ], - "actions": [ - "google_tasks-create-task" - ], + "sources": ["microsofttodo-new-list-created"], + "actions": ["google_tasks-create-task"], "persona": "similar-app" }, { "query": "use Mailgun to send transactional emails when Mailchimp campaign is sent", - "triggers": [ - "mailchimp-new-campaign" - ], - "actions": [ - "mailgun-send-email" - ], + "sources": ["mailchimp-new-campaign"], + "actions": ["mailgun-send-email"], "persona": "similar-app" }, { "query": "When Google Calendar event ends, update Outlook Calendar", - "triggers": [ - "google_calendar-event-cancelled" - ], - "actions": [ - "microsoft_outlook_calendar-update-calendar-event" - ], + "sources": ["google_calendar-event-cancelled"], + "actions": ["microsoft_outlook_calendar-update-calendar-event"], "persona": "similar-app" }, { "query": "summarize text with OpenAI GPT-4", - "triggers": [], - "actions": [ - "openai-summarize" - ], + "sources": [], + "actions": ["openai-summarize"], "persona": "similar-app" }, { "query": "When Zoho Mail gets new email, forward through Microsoft Outlook", - "triggers": [ - "zoho_mail-new-email" - ], - "actions": [ - "microsoft_outlook-send-email" - ], + "sources": ["zoho_mail-new-email"], + "actions": ["microsoft_outlook-send-email"], "persona": "similar-app" }, { "query": "create new worksheet in Google Sheets when Smartsheet updates row", - "triggers": [ - "smartsheet-new-row-updated" - ], - "actions": [ - "google_sheets-create-worksheet" - ], + "sources": ["smartsheet-new-row-updated"], + "actions": ["google_sheets-create-worksheet"], "persona": "similar-app" }, { "query": "use Anthropic Claude to generate email content", - "triggers": [], - "actions": [ - "anthropic-chat" - ], + "sources": [], + "actions": ["anthropic-chat"], "persona": "similar-app" }, { "query": "When Box gets new file, upload to OneDrive and Google Drive", - "triggers": [ - "box-new-file" - ], - "actions": [ - "microsoft_onedrive-upload-file", - "google_drive-upload-file" - ], + "sources": ["box-new-file"], + "actions": ["microsoft_onedrive-upload-file", "google_drive-upload-file"], "persona": "similar-app" }, { "query": "analyze text with Google Palm API", - "triggers": [], - "actions": [ - "google_palm_api-generate-text" - ], + "sources": [], + "actions": ["google_palm_api-generate-text"], "persona": "similar-app" }, { "query": "When Dropbox folder changes, sync to Box and OneDrive", - "triggers": [ - "dropbox-all-updates" - ], - "actions": [ - "box-upload-file", - "microsoft_onedrive-upload-file" - ], + "sources": ["dropbox-all-updates"], + "actions": ["box-upload-file", "microsoft_onedrive-upload-file"], "persona": "similar-app" }, { "query": "create event in Zoho Calendar when Google Calendar event is updated", - "triggers": [ - "google_calendar-new-or-updated-event-instant" - ], - "actions": [ - "zoho_calendar-create-or-update-event" - ], + "sources": ["google_calendar-new-or-updated-event-instant"], + "actions": ["zoho_calendar-create-or-update-event"], "persona": "similar-app" }, { "query": "When Discord gets reaction, notify through Microsoft Teams", - "triggers": [ - "discord-reaction-added" - ], - "actions": [ - "microsoft_teams-send-channel-message" - ], + "sources": ["discord-reaction-added"], + "actions": ["microsoft_teams-send-channel-message"], "persona": "similar-app" }, { "query": "send Mailjet msg when MailerLite subscriber is added", - "triggers": [ - "mailerlite-subscriber-added-from-form" - ], - "actions": [ - "mailjet-send-message" - ], + "sources": ["mailerlite-subscriber-added-from-form"], + "actions": ["mailjet-send-message"], "persona": "similar-app" } ] -} \ No newline at end of file +} diff --git a/packages/evals/component_retrieval/evaluator.mjs b/packages/evals/component_retrieval/evaluator.mjs index a0a9f2d9122a2..df78ab6319d20 100644 --- a/packages/evals/component_retrieval/evaluator.mjs +++ b/packages/evals/component_retrieval/evaluator.mjs @@ -5,13 +5,17 @@ import fs from "fs/promises"; import path from "path"; import { diff } from "json-diff"; import { json2csv } from "json-2-csv"; +import { program } from "commander"; const GREEN_CHECK = "\x1b[32m✔\x1b[0m"; const RED_CROSS = "\x1b[31m✖\x1b[0m"; +let totalPrecision = 0; +let totalRecall = 0; +let totalF1Score = 0; let totalEvals = 0; let totalSuccesses = 0; -let apiResults = [] +let apiResults = []; const apiHost = process.env.API_BASE_URL || "https://api.pipedream.com"; @@ -45,84 +49,157 @@ function customDiff(original, updated, oldLabel = "expected", newLabel = "actual async function exportToCsv(filePath, limit, threshold) { const csvData = json2csv(apiResults, { - fields: ["query", "evalTriggers", "apiTriggers", "evalActions", "apiActions", "success"] + fields: [ + "query", + "evalTriggers", + "apiTriggers", + "evalActions", + "apiActions", + "precision", + "recall", + "f1Score", + ], }); - const parts = filePath.split("/") - const path = parts[parts.length -1].split(".json")[0] + const parts = filePath.split("/"); + const path = parts[parts.length - 1].split(".json")[0]; + try { + await fs.access("./csv"); + } catch (error) { + await fs.mkdir("./csv"); + } await fs.writeFile(`./csv/${path}-${limit}-${threshold}.csv`, csvData); } function arrayToString(items) { - if (items) return items.join(",") - return "" + if (items) return items.join(","); + return ""; } -async function processEvalFile(filePath) { - const limit = 3 - const threshold = 0.65 - try { - const content = await fs.readFile(filePath, "utf-8"); - const evalData = JSON.parse(content); - - for (const evalTest of evalData.evaluationTests) { - totalEvals++; - const { - query, triggers, actions, - } = evalTest; - - const encodedQuery = encodeURIComponent(query); - const apiUrl = `${apiHost}/v1/components/search?query=${encodedQuery}&similarity_threshold=${threshold}&limit=${limit}`; - - const response = await fetch(apiUrl, { - headers: { - "Content-Type": "application/json", - "Authorization": `Bearer ${process.env.PIPEDREAM_API_KEY}`, - }, - }); - const apiData = await response.json(); - - - // Compare actual and expected - const apiTriggers = apiData?.triggers ?? []; - const apiActions = apiData?.actions ?? []; - - const triggersMatch = - JSON.stringify(apiTriggers.sort()) === JSON.stringify(triggers.sort()); - const actionsMatch = +async function processEvalFile(filePath, options) { + const limit = 2; + const threshold = 0.7; + const content = await fs.readFile(filePath, "utf-8"); + const evalData = JSON.parse(content); + + for (const evalTest of evalData.evaluationTests) { + totalEvals++; + const { + query, sources, actions, + } = evalTest; + + const encodedQuery = encodeURIComponent(query); + const apiUrl = `${apiHost}/v1/components/search?query=${encodedQuery}&similarity_threshold=${threshold}&limit=${limit}`; + + const response = await fetch(apiUrl, { + headers: { + "Content-Type": "application/json", + "Authorization": `Bearer ${process.env.PIPEDREAM_API_KEY}`, + }, + }); + const apiData = await response.json(); + if (options.debug) { + console.log(`\nQuery: "${query}"`); + console.log(JSON.stringify(apiData, null, 2)); + } + + // Calculate precision, recall, and F1 score + const expectedItems = [ + ...(sources || []), + ...(actions || []), + ]; + const retrievedTriggers = apiData?.sources ?? []; + const retrievedActions = apiData?.actions ?? []; + const retrievedItems = [ + ...retrievedTriggers, + ...retrievedActions, + ]; + + const correctlyRetrievedItems = retrievedItems.filter((item) => expectedItems.includes(item)); + const numCorrectlyRetrieved = correctlyRetrievedItems.length; + const numRetrieved = retrievedItems.length; + const numExpected = expectedItems.length; + + let precision, recall, f1Score; + + if (numExpected === 0 && numRetrieved === 0) { + // This case is expected — our test has no expected items, and the API returned no items + precision = 1; + recall = 1; + f1Score = 1; + } else { + // Standard calculations + precision = numRetrieved > 0 + ? numCorrectlyRetrieved / numRetrieved + : 0; + recall = numExpected > 0 + ? numCorrectlyRetrieved / numExpected + : 0; + f1Score = (precision + recall) > 0 + ? (2 * precision * recall) / (precision + recall) + : 0; + } + + // Accumulate totals for averaging later + totalPrecision += precision; + totalRecall += recall; + totalF1Score += f1Score; + + console.log(`Precision: ${precision.toFixed(2)}`); + console.log(`Recall: ${recall.toFixed(2)}`); + console.log(`F1 Score: ${f1Score.toFixed(2)}`); + + // Compare actual and expected data for debugging + const apiTriggers = apiData?.sources ?? []; + const apiActions = apiData?.actions ?? []; + + const sourcesMatch = + JSON.stringify(apiTriggers.sort()) === JSON.stringify(sources.sort()); + const actionsMatch = JSON.stringify(apiActions.sort()) === JSON.stringify(actions.sort()); - let success = false - if (triggersMatch && actionsMatch) { - totalSuccesses++; - success = true - console.log(`${GREEN_CHECK} Success for query: "${query}"`); - } else { - console.log(`${RED_CROSS} Failure for query: "${query}"`); + let success = false; + if (sourcesMatch && actionsMatch) { + totalSuccesses++; + success = true; + console.log(`${GREEN_CHECK} Success for query: "${query}"`); + } else { + console.log(`${RED_CROSS} Failure for query: "${query}"`); + if (options.debug) { console.log("Differences:"); console.log(customDiff({ - triggers, + sources, actions, }, apiData)); } - - const record = { - query: query.replace("\"", ""), - apiTriggers: arrayToString(apiTriggers), - apiActions: arrayToString(apiActions), - evalTriggers: arrayToString(triggers), - evalActions: arrayToString(actions), - success: success - }; - apiResults.push(record) } - } catch (error) { - console.error(`Error processing file ${filePath}:`, error.message); + + const record = { + query: query.replace("\"", ""), + apiTriggers: arrayToString(apiTriggers), + apiActions: arrayToString(apiActions), + evalTriggers: arrayToString(sources), + evalActions: arrayToString(actions), + success: success, + }; + apiResults.push(record); } - await exportToCsv(filePath, limit, threshold) + await exportToCsv(filePath, limit, threshold); } async function main() { - const evalFiles = process.argv.slice(2); + program + .version("1.0.0") + .usage("[options] ") + .option("-d, --debug", "output extra debugging") + .parse(process.argv); + + const options = program.opts(); + const evalFiles = program.args; + + if (evalFiles.length === 0) { + console.error("Please provide at least one eval JSON file."); + process.exit(1); + } if (evalFiles.length === 0) { console.error("Please provide at least one eval JSON file."); @@ -131,13 +208,25 @@ async function main() { for (const file of evalFiles) { const filePath = path.resolve(file); - await processEvalFile(filePath); + await processEvalFile(filePath, options); } const successRate = ((totalSuccesses / totalEvals) * 100).toFixed(2); console.log(`\nTotal Evals: ${totalEvals}`); - console.log(`Total Successes: ${totalSuccesses}`); - console.log(`Success Rate: ${successRate}%`); + console.log(`Perfect Matches: ${totalSuccesses}`); + console.log(`Perfect Match Rate: ${successRate}%`); + + const averagePrecision = totalPrecision / totalEvals; + const averageRecall = totalRecall / totalEvals; + const averageF1Score = totalF1Score / totalEvals; + + // Convert average F1 Score to success percentage + const successPercentage = (averageF1Score * 100).toFixed(2); + + console.log(`\nAverage Precision: ${averagePrecision.toFixed(2)}`); + console.log(`Average Recall: ${averageRecall.toFixed(2)}`); + console.log(`Average F1 Score: ${averageF1Score.toFixed(2)}`); + console.log(`\nOverall Success Percentage: ${successPercentage}%`); } main(); diff --git a/packages/evals/component_retrieval/package-lock.json b/packages/evals/component_retrieval/package-lock.json index 0f1736b44be18..cb696761e03e7 100644 --- a/packages/evals/component_retrieval/package-lock.json +++ b/packages/evals/component_retrieval/package-lock.json @@ -8,6 +8,7 @@ "name": "@pipedream/evals-component_retrieval", "version": "0.0.1", "dependencies": { + "commander": "^12.1.0", "dotenv": "^16.4.5", "json-2-csv": "^5.5.6", "json-diff": "^1.0.6" @@ -29,6 +30,14 @@ "node": ">=0.1.90" } }, + "node_modules/commander": { + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", + "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", + "engines": { + "node": ">=18" + } + }, "node_modules/deeks": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/deeks/-/deeks-3.1.0.tgz", diff --git a/packages/evals/component_retrieval/package.json b/packages/evals/component_retrieval/package.json index 6f16913e997f5..0833e990c43fb 100644 --- a/packages/evals/component_retrieval/package.json +++ b/packages/evals/component_retrieval/package.json @@ -4,6 +4,7 @@ "description": "Eval for Pipedream AI component retrieval API", "main": "evaluator.mjs", "dependencies": { + "commander": "^12.1.0", "dotenv": "^16.4.5", "json-2-csv": "^5.5.6", "json-diff": "^1.0.6" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6927c8e000fc6..75c8acf3d2ee3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11900,9 +11900,11 @@ importers: packages/evals/component_retrieval: specifiers: dotenv: ^16.4.5 + json-2-csv: ^5.5.6 json-diff: ^1.0.6 dependencies: dotenv: 16.4.5 + json-2-csv: 5.5.6 json-diff: 1.0.6 packages/prompts: @@ -24232,6 +24234,11 @@ packages: engines: {node: '>= 12'} dev: false + /deeks/3.1.0: + resolution: {integrity: sha512-e7oWH1LzIdv/prMQ7pmlDlaVoL64glqzvNgkgQNgyec9ORPHrT2jaOqMtRyqJuwWjtfb6v+2rk9pmaHj+F137A==} + engines: {node: '>= 16'} + dev: false + /deep-assign/3.0.0: resolution: {integrity: sha512-YX2i9XjJ7h5q/aQ/IM9PEwEnDqETAIYbggmdDB3HLTlSgo1CxPsj6pvhPG68rq6SVE0+p+6Ywsm5fTYNrYtBWw==} engines: {node: '>=0.10.0'} @@ -24514,6 +24521,11 @@ packages: engines: {node: '>=12'} dev: false + /doc-path/4.1.1: + resolution: {integrity: sha512-h1ErTglQAVv2gCnOpD3sFS6uolDbOKHDU1BZq+Kl3npPqroU3dYL42lUgMfd5UimlwtRgp7C9dLGwqQ5D2HYgQ==} + engines: {node: '>=16'} + dev: false + /doctrine/3.0.0: resolution: {integrity: sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==} engines: {node: '>=6.0.0'} @@ -28670,6 +28682,14 @@ packages: doc-path: 3.1.0 dev: false + /json-2-csv/5.5.6: + resolution: {integrity: sha512-N673XbJgHwUq9JreKpk530jSywPF/rEAQ08dV99QQpkluP/4HTwshpoP9hmDz26iSFqu7eNAPgyJfu/77HvPGA==} + engines: {node: '>= 16'} + dependencies: + deeks: 3.1.0 + doc-path: 4.1.1 + dev: false + /json-bigint/1.0.0: resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==} dependencies: From 80f2256c901f6bab9d44a18df6ca1f8f1938979c Mon Sep 17 00:00:00 2001 From: "Dylan J. Sather" Date: Thu, 24 Oct 2024 18:13:06 -0700 Subject: [PATCH 07/18] pnpm --- pnpm-lock.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 75c8acf3d2ee3..2ab4757d95bf4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11899,10 +11899,12 @@ importers: packages/evals/component_retrieval: specifiers: + commander: ^12.1.0 dotenv: ^16.4.5 json-2-csv: ^5.5.6 json-diff: ^1.0.6 dependencies: + commander: 12.1.0 dotenv: 16.4.5 json-2-csv: 5.5.6 json-diff: 1.0.6 @@ -23571,6 +23573,11 @@ packages: engines: {node: '>=16'} dev: false + /commander/12.1.0: + resolution: {integrity: sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==} + engines: {node: '>=18'} + dev: false + /commander/2.20.3: resolution: {integrity: sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==} From 58f7eb919c01da52541317966c5a0a8b7568a7d7 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Fri, 25 Oct 2024 10:23:08 -0400 Subject: [PATCH 08/18] New Components - pdf_app_net (#14406) * new components * pnpm-lock.yaml * fix summary --- .../actions/compress-pdf/compress-pdf.mjs | 35 ++++++ .../actions/image-to-pdf/image-to-pdf.mjs | 37 ++++++ .../actions/split-pdf/split-pdf.mjs | 41 +++++++ components/pdf_app_net/package.json | 7 +- components/pdf_app_net/pdf_app_net.app.mjs | 50 +++++++- pnpm-lock.yaml | 107 +++++++++--------- 6 files changed, 218 insertions(+), 59 deletions(-) create mode 100644 components/pdf_app_net/actions/compress-pdf/compress-pdf.mjs create mode 100644 components/pdf_app_net/actions/image-to-pdf/image-to-pdf.mjs create mode 100644 components/pdf_app_net/actions/split-pdf/split-pdf.mjs diff --git a/components/pdf_app_net/actions/compress-pdf/compress-pdf.mjs b/components/pdf_app_net/actions/compress-pdf/compress-pdf.mjs new file mode 100644 index 0000000000000..6b129f6511fba --- /dev/null +++ b/components/pdf_app_net/actions/compress-pdf/compress-pdf.mjs @@ -0,0 +1,35 @@ +import pdfApp from "../../pdf_app_net.app.mjs"; + +export default { + key: "pdf_app_net-compress-pdf", + name: "Compress PDF", + description: "Compress a PDF File with PDF-app.net. [See the documentation](https://pdf-app.net/apidocumentation)", + version: "0.0.1", + type: "action", + props: { + pdfApp, + fileUrl: { + type: "string", + label: "File URL", + description: "The URL of a .pdf file to compress", + }, + fileName: { + propDefinition: [ + pdfApp, + "fileName", + ], + }, + }, + async run({ $ }) { + const response = await this.pdfApp.compressPdf({ + $, + data: { + fileUrl: this.fileUrl, + async: false, + fileName: this.fileName, + }, + }); + $.export("$summary", "Successfully compressed PDF File"); + return response; + }, +}; diff --git a/components/pdf_app_net/actions/image-to-pdf/image-to-pdf.mjs b/components/pdf_app_net/actions/image-to-pdf/image-to-pdf.mjs new file mode 100644 index 0000000000000..6dc818dea2371 --- /dev/null +++ b/components/pdf_app_net/actions/image-to-pdf/image-to-pdf.mjs @@ -0,0 +1,37 @@ +import pdfApp from "../../pdf_app_net.app.mjs"; + +export default { + key: "pdf_app_net-image-to-pdf", + name: "Image to PDF", + description: "Convert an image from a URL to a PDF File with PDF-app.net. [See the documentation](https://pdf-app.net/apidocumentation)", + version: "0.0.1", + type: "action", + props: { + pdfApp, + imageUrl: { + type: "string", + label: "Image URL", + description: "The URL of an image file to convert", + }, + fileName: { + propDefinition: [ + pdfApp, + "fileName", + ], + }, + }, + async run({ $ }) { + const response = await this.pdfApp.imageToPdf({ + $, + data: { + imageUrls: [ + this.imageUrl, + ], + async: false, + fileName: this.fileName, + }, + }); + $.export("$summary", "Successfully converted image to PDF File"); + return response; + }, +}; diff --git a/components/pdf_app_net/actions/split-pdf/split-pdf.mjs b/components/pdf_app_net/actions/split-pdf/split-pdf.mjs new file mode 100644 index 0000000000000..765f73f8bb154 --- /dev/null +++ b/components/pdf_app_net/actions/split-pdf/split-pdf.mjs @@ -0,0 +1,41 @@ +import pdfApp from "../../pdf_app_net.app.mjs"; + +export default { + key: "pdf_app_net-split-pdf", + name: "Split PDF", + description: "Split a PDF into multiple PDFs containing the specified number of pages. [See the documentation](https://pdf-app.net/apidocumentation)", + version: "0.0.1", + type: "action", + props: { + pdfApp, + fileUrl: { + type: "string", + label: "File URL", + description: "The URL of a .pdf file to split", + }, + pagesPerSplit: { + type: "integer", + label: "Pages", + description: "Each PDF will contain the specified number of pages. For instance, if the original PDF contains 6 pages, and `Pages` is `2`, the result will be 3 files, each containing 2 pages.", + }, + fileName: { + propDefinition: [ + pdfApp, + "fileName", + ], + }, + }, + async run({ $ }) { + const response = await this.pdfApp.splitPdf({ + $, + data: { + fileUrl: this.fileUrl, + pagesPerSplit: this.pagesPerSplit, + async: false, + fileName: this.fileName, + }, + }); + $.export("$summary", "Successfully split PDF File"); + return response; + }, +}; diff --git a/components/pdf_app_net/package.json b/components/pdf_app_net/package.json index 0caed215a13ac..a36082cf96c5a 100644 --- a/components/pdf_app_net/package.json +++ b/components/pdf_app_net/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/pdf_app_net", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream PDF-app.net Components", "main": "pdf_app_net.app.mjs", "keywords": [ @@ -11,5 +11,8 @@ "author": "Pipedream (https://pipedream.com/)", "publishConfig": { "access": "public" + }, + "dependencies": { + "@pipedream/platform": "^3.0.3" } -} \ No newline at end of file +} diff --git a/components/pdf_app_net/pdf_app_net.app.mjs b/components/pdf_app_net/pdf_app_net.app.mjs index 05f72b9c077ed..0fd7c75e8ed56 100644 --- a/components/pdf_app_net/pdf_app_net.app.mjs +++ b/components/pdf_app_net/pdf_app_net.app.mjs @@ -1,11 +1,51 @@ +import { axios } from "@pipedream/platform"; + export default { type: "app", app: "pdf_app_net", - propDefinitions: {}, + propDefinitions: { + fileName: { + type: "string", + label: "File Name", + description: "The name of the file", + optional: true, + }, + }, methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); + _baseUrl() { + return "https://api.pdf-app.net"; + }, + _makeRequest({ + $ = this, + path, + ...opts + }) { + return axios($, { + method: "POST", + url: `${this._baseUrl()}${path}`, + headers: { + "Authorization": `${this.$auth.api_key}`, + }, + ...opts, + }); + }, + compressPdf(opts = {}) { + return this._makeRequest({ + path: "/compress_PDF", + ...opts, + }); + }, + imageToPdf(opts = {}) { + return this._makeRequest({ + path: "/image_to_pdf", + ...opts, + }); + }, + splitPdf(opts = {}) { + return this._makeRequest({ + path: "/splitt_PDF", + ...opts, + }); }, }, -}; \ No newline at end of file +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2ab4757d95bf4..35eb6e18d23ba 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7235,7 +7235,10 @@ importers: specifiers: {} components/pdf_app_net: - specifiers: {} + specifiers: + '@pipedream/platform': ^3.0.3 + dependencies: + '@pipedream/platform': 3.0.3 components/pdf_charts: specifiers: @@ -13175,6 +13178,55 @@ packages: - aws-crt dev: false + /@aws-sdk/client-sso-oidc/3.600.0_tdq3komn4zwyd65w7klbptsu34: + resolution: {integrity: sha512-7+I8RWURGfzvChyNQSyj5/tKrqRbzRl7H+BnTOf/4Vsw1nFOi5ROhlhD4X/Y0QCTacxnaoNcIrqnY7uGGvVRzw==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/client-sts': 3.600.0 + '@aws-sdk/core': 3.598.0 + '@aws-sdk/credential-provider-node': 3.600.0_f7n47caigsrjd2lr2szmwfuee4 + '@aws-sdk/middleware-host-header': 3.598.0 + '@aws-sdk/middleware-logger': 3.598.0 + '@aws-sdk/middleware-recursion-detection': 3.598.0 + '@aws-sdk/middleware-user-agent': 3.598.0 + '@aws-sdk/region-config-resolver': 3.598.0 + '@aws-sdk/types': 3.598.0 + '@aws-sdk/util-endpoints': 3.598.0 + '@aws-sdk/util-user-agent-browser': 3.598.0 + '@aws-sdk/util-user-agent-node': 3.598.0 + '@smithy/config-resolver': 3.0.3 + '@smithy/core': 2.2.3 + '@smithy/fetch-http-handler': 3.2.1 + '@smithy/hash-node': 3.0.2 + '@smithy/invalid-dependency': 3.0.2 + '@smithy/middleware-content-length': 3.0.2 + '@smithy/middleware-endpoint': 3.0.4 + '@smithy/middleware-retry': 3.0.6 + '@smithy/middleware-serde': 3.0.3 + '@smithy/middleware-stack': 3.0.3 + '@smithy/node-config-provider': 3.1.3 + '@smithy/node-http-handler': 3.1.2 + '@smithy/protocol-http': 4.0.3 + '@smithy/smithy-client': 3.1.6 + '@smithy/types': 3.3.0 + '@smithy/url-parser': 3.0.3 + '@smithy/util-base64': 3.0.0 + '@smithy/util-body-length-browser': 3.0.0 + '@smithy/util-body-length-node': 3.0.0 + '@smithy/util-defaults-mode-browser': 3.0.6 + '@smithy/util-defaults-mode-node': 3.0.6 + '@smithy/util-endpoints': 2.0.3 + '@smithy/util-middleware': 3.0.3 + '@smithy/util-retry': 3.0.2 + '@smithy/util-utf8': 3.0.0 + tslib: 2.6.3 + transitivePeerDependencies: + - '@aws-sdk/client-sts' + - aws-crt + dev: false + /@aws-sdk/client-sso/3.423.0: resolution: {integrity: sha512-znIufHkwhCIePgaYciIs3x/+BpzR57CZzbCKHR9+oOvGyufEPPpUT5bFLvbwTgfiVkTjuk6sG/ES3U5Bc+xtrA==} engines: {node: '>=14.0.0'} @@ -13410,7 +13462,7 @@ packages: dependencies: '@aws-crypto/sha256-browser': 5.2.0 '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/client-sso-oidc': 3.600.0 + '@aws-sdk/client-sso-oidc': 3.600.0_tdq3komn4zwyd65w7klbptsu34 '@aws-sdk/core': 3.598.0 '@aws-sdk/credential-provider-node': 3.600.0_f7n47caigsrjd2lr2szmwfuee4 '@aws-sdk/middleware-host-header': 3.598.0 @@ -13452,55 +13504,6 @@ packages: - aws-crt dev: false - /@aws-sdk/client-sts/3.600.0_dseaa2p5u2yk67qiepewcq3hkq: - resolution: {integrity: sha512-KQG97B7LvTtTiGmjlrG1LRAY8wUvCQzrmZVV5bjrJ/1oXAU7DITYwVbSJeX9NWg6hDuSk0VE3MFwIXS2SvfLIA==} - engines: {node: '>=16.0.0'} - dependencies: - '@aws-crypto/sha256-browser': 5.2.0 - '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/client-sso-oidc': 3.600.0 - '@aws-sdk/core': 3.598.0 - '@aws-sdk/credential-provider-node': 3.600.0_f7n47caigsrjd2lr2szmwfuee4 - '@aws-sdk/middleware-host-header': 3.598.0 - '@aws-sdk/middleware-logger': 3.598.0 - '@aws-sdk/middleware-recursion-detection': 3.598.0 - '@aws-sdk/middleware-user-agent': 3.598.0 - '@aws-sdk/region-config-resolver': 3.598.0 - '@aws-sdk/types': 3.598.0 - '@aws-sdk/util-endpoints': 3.598.0 - '@aws-sdk/util-user-agent-browser': 3.598.0 - '@aws-sdk/util-user-agent-node': 3.598.0 - '@smithy/config-resolver': 3.0.3 - '@smithy/core': 2.2.3 - '@smithy/fetch-http-handler': 3.2.1 - '@smithy/hash-node': 3.0.2 - '@smithy/invalid-dependency': 3.0.2 - '@smithy/middleware-content-length': 3.0.2 - '@smithy/middleware-endpoint': 3.0.4 - '@smithy/middleware-retry': 3.0.6 - '@smithy/middleware-serde': 3.0.3 - '@smithy/middleware-stack': 3.0.3 - '@smithy/node-config-provider': 3.1.3 - '@smithy/node-http-handler': 3.1.2 - '@smithy/protocol-http': 4.0.3 - '@smithy/smithy-client': 3.1.6 - '@smithy/types': 3.3.0 - '@smithy/url-parser': 3.0.3 - '@smithy/util-base64': 3.0.0 - '@smithy/util-body-length-browser': 3.0.0 - '@smithy/util-body-length-node': 3.0.0 - '@smithy/util-defaults-mode-browser': 3.0.6 - '@smithy/util-defaults-mode-node': 3.0.6 - '@smithy/util-endpoints': 2.0.3 - '@smithy/util-middleware': 3.0.3 - '@smithy/util-retry': 3.0.2 - '@smithy/util-utf8': 3.0.0 - tslib: 2.6.3 - transitivePeerDependencies: - - '@aws-sdk/client-sso-oidc' - - aws-crt - dev: false - /@aws-sdk/core/3.556.0: resolution: {integrity: sha512-vJaSaHw2kPQlo11j/Rzuz0gk1tEaKdz+2ser0f0qZ5vwFlANjt08m/frU17ctnVKC1s58bxpctO/1P894fHLrA==} engines: {node: '>=14.0.0'} @@ -17799,7 +17802,7 @@ packages: '@aws-sdk/client-sns': 3.423.0 '@aws-sdk/client-sqs': 3.423.0 '@aws-sdk/client-ssm': 3.423.0 - '@aws-sdk/client-sts': 3.600.0_dseaa2p5u2yk67qiepewcq3hkq + '@aws-sdk/client-sts': 3.600.0 '@aws-sdk/s3-request-presigner': 3.609.0 '@pipedream/helper_functions': 0.3.12 '@pipedream/platform': 1.6.6 From 3cc761b28a71da2df956258b1d1e19c5bf60120e Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Fri, 25 Oct 2024 10:30:36 -0400 Subject: [PATCH 09/18] renew webhooks (#14386) --- components/smartsuite/package.json | 2 +- components/smartsuite/sources/common/base.mjs | 8 ++++++++ .../new-record-created-instant.mjs | 2 +- .../record-updated-instant/record-updated-instant.mjs | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/components/smartsuite/package.json b/components/smartsuite/package.json index 97321cd418ca0..159f2af208cbf 100644 --- a/components/smartsuite/package.json +++ b/components/smartsuite/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/smartsuite", - "version": "0.1.0", + "version": "0.1.1", "description": "Pipedream SmartSuite Components", "main": "smartsuite.app.mjs", "keywords": [ diff --git a/components/smartsuite/sources/common/base.mjs b/components/smartsuite/sources/common/base.mjs index 0aa060f46bdae..0f64216840c67 100644 --- a/components/smartsuite/sources/common/base.mjs +++ b/components/smartsuite/sources/common/base.mjs @@ -5,6 +5,14 @@ export default { smartsuite, db: "$.service.db", http: "$.interface.http", + timer: { + label: "Webhook renewal schedule", + description: "The SmartSuite API requires occasional renewal of webhooks. **This runs in the background, so you should not need to modify this schedule**.", + type: "$.interface.timer", + static: { + intervalSeconds: 24 * 60 * 60, // once per day + }, + }, solutionId: { propDefinition: [ smartsuite, diff --git a/components/smartsuite/sources/new-record-created-instant/new-record-created-instant.mjs b/components/smartsuite/sources/new-record-created-instant/new-record-created-instant.mjs index 08e2ea1e249c8..41661125c0593 100644 --- a/components/smartsuite/sources/new-record-created-instant/new-record-created-instant.mjs +++ b/components/smartsuite/sources/new-record-created-instant/new-record-created-instant.mjs @@ -6,7 +6,7 @@ export default { key: "smartsuite-new-record-created-instant", name: "New Record Created (Instant)", description: "Emit new event when a new record is created", - version: "0.0.1", + version: "0.0.2", type: "source", dedupe: "unique", methods: { diff --git a/components/smartsuite/sources/record-updated-instant/record-updated-instant.mjs b/components/smartsuite/sources/record-updated-instant/record-updated-instant.mjs index 97dfe46577e61..332e121b9fbda 100644 --- a/components/smartsuite/sources/record-updated-instant/record-updated-instant.mjs +++ b/components/smartsuite/sources/record-updated-instant/record-updated-instant.mjs @@ -6,7 +6,7 @@ export default { key: "smartsuite-record-updated-instant", name: "Record Updated (Instant)", description: "Emit new event when an existing record is updated", - version: "0.0.1", + version: "0.0.2", type: "source", dedupe: "unique", methods: { From 019f1cb24f7a976c183d857bd7057d29578c3905 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Fri, 25 Oct 2024 11:38:43 -0400 Subject: [PATCH 10/18] OpenAI - Add audio functionality to Chat action (#14367) * accept audio input * versions * add configuration error --- components/openai/actions/chat/chat.mjs | 15 +++++++++++++-- .../classify-items-into-categories.mjs | 2 +- components/openai/actions/common/common.mjs | 15 +++++++++++++++ .../create-embeddings/create-embeddings.mjs | 2 +- .../create-transcription/create-transcription.mjs | 2 +- .../openai/actions/send-prompt/send-prompt.mjs | 2 +- components/openai/actions/summarize/summarize.mjs | 2 +- .../actions/translate-text/translate-text.mjs | 2 +- components/openai/package.json | 2 +- 9 files changed, 35 insertions(+), 9 deletions(-) diff --git a/components/openai/actions/chat/chat.mjs b/components/openai/actions/chat/chat.mjs index 8dbc88773f03d..8f741e50e9551 100644 --- a/components/openai/actions/chat/chat.mjs +++ b/components/openai/actions/chat/chat.mjs @@ -1,11 +1,12 @@ import openai from "../../openai.app.mjs"; import common from "../common/common.mjs"; import constants from "../../common/constants.mjs"; +import { ConfigurationError } from "@pipedream/platform"; export default { ...common, name: "Chat", - version: "0.2.0", + version: "0.2.1", key: "openai-chat", description: "The Chat API, using the `gpt-3.5-turbo` or `gpt-4` model. [See the documentation](https://platform.openai.com/docs/api-reference/chat)", type: "action", @@ -38,7 +39,13 @@ export default { images: { label: "Images", type: "string[]", - description: "Provide one or more images to [OpenAI's vision model](https://platform.openai.com/docs/guides/vision). Accepts URLs or base64 encoded strings. Compatible with the `gpt4-vision-preview model`", + description: "Provide one or more images to [OpenAI's vision model](https://platform.openai.com/docs/guides/vision). Accepts URLs or base64 encoded strings. Compatible with the `gpt4-vision-preview` model", + optional: true, + }, + audio: { + type: "string", + label: "Audio", + description: "Provide the file path to an audio file in the `/tmp` directory. For use with the `gpt-4o-audio-preview` model. Currently supports `wav` and `mp3` files.", optional: true, }, responseFormat: { @@ -65,6 +72,10 @@ export default { }; }, async run({ $ }) { + if (this.audio && !this.modelId.includes("gpt-4o-audio-preview")) { + throw new ConfigurationError("Use of audio files requires using the `gpt-4o-audio-preview` model."); + } + const args = this._getChatArgs(); const response = await this.openai.createChatCompletion({ diff --git a/components/openai/actions/classify-items-into-categories/classify-items-into-categories.mjs b/components/openai/actions/classify-items-into-categories/classify-items-into-categories.mjs index 99d154d97b24e..7e30c012cffa5 100644 --- a/components/openai/actions/classify-items-into-categories/classify-items-into-categories.mjs +++ b/components/openai/actions/classify-items-into-categories/classify-items-into-categories.mjs @@ -3,7 +3,7 @@ import common from "../common/common-helper.mjs"; export default { ...common, name: "Classify Items into Categories", - version: "0.1.0", + version: "0.1.1", key: "openai-classify-items-into-categories", description: "Classify items into specific categories using the Chat API. [See the documentation](https://platform.openai.com/docs/api-reference/chat)", type: "action", diff --git a/components/openai/actions/common/common.mjs b/components/openai/actions/common/common.mjs index 9642b8c71d175..41474b809d1d6 100644 --- a/components/openai/actions/common/common.mjs +++ b/components/openai/actions/common/common.mjs @@ -1,6 +1,7 @@ import { ConfigurationError } from "@pipedream/platform"; import constants from "../../common/constants.mjs"; import { parse } from "../../common/helpers.mjs"; +import fs from "fs"; const CHAT_DOCS_MESSAGE_FORMAT_URL = "https://platform.openai.com/docs/guides/chat/introduction"; @@ -92,6 +93,20 @@ export default { } } + if (this.audio) { + const fileContent = fs.readFileSync(this.audio.includes("tmp/") + ? this.audio + : `/tmp/${this.audio}`).toString("base64"); + const extension = this.audio.match(/\.(\w+)$/)?.[1]; + content.push({ + type: "input_audio", + input_audio: { + data: fileContent, + format: extension, + }, + }); + } + content.push({ "type": "text", "text": this.userMessage, diff --git a/components/openai/actions/create-embeddings/create-embeddings.mjs b/components/openai/actions/create-embeddings/create-embeddings.mjs index a353efa67c4c5..20d400b6deb7b 100644 --- a/components/openai/actions/create-embeddings/create-embeddings.mjs +++ b/components/openai/actions/create-embeddings/create-embeddings.mjs @@ -4,7 +4,7 @@ import common from "../common/common.mjs"; export default { name: "Create Embeddings", - version: "0.0.12", + version: "0.0.13", key: "openai-create-embeddings", description: "Get a vector representation of a given input that can be easily consumed by machine learning models and algorithms. [See the documentation](https://platform.openai.com/docs/api-reference/embeddings)", type: "action", diff --git a/components/openai/actions/create-transcription/create-transcription.mjs b/components/openai/actions/create-transcription/create-transcription.mjs index 41071a891d774..6fcec2e43d091 100644 --- a/components/openai/actions/create-transcription/create-transcription.mjs +++ b/components/openai/actions/create-transcription/create-transcription.mjs @@ -24,7 +24,7 @@ const pipelineAsync = promisify(stream.pipeline); export default { name: "Create Transcription (Whisper)", - version: "0.1.12", + version: "0.1.13", key: "openai-create-transcription", description: "Transcribes audio into the input language. [See the documentation](https://platform.openai.com/docs/api-reference/audio/create).", type: "action", diff --git a/components/openai/actions/send-prompt/send-prompt.mjs b/components/openai/actions/send-prompt/send-prompt.mjs index 1c31ff5a1e515..fa1bf3abae291 100644 --- a/components/openai/actions/send-prompt/send-prompt.mjs +++ b/components/openai/actions/send-prompt/send-prompt.mjs @@ -4,7 +4,7 @@ import common from "../common/common.mjs"; export default { ...common, name: "Create Completion (Send Prompt)", - version: "0.1.11", + version: "0.1.12", key: "openai-send-prompt", description: "OpenAI recommends using the **Chat** action for the latest `gpt-3.5-turbo` API, since it's faster and 10x cheaper. This action creates a completion for the provided prompt and parameters using the older `/completions` API. [See the documentation](https://beta.openai.com/docs/api-reference/completions/create)", type: "action", diff --git a/components/openai/actions/summarize/summarize.mjs b/components/openai/actions/summarize/summarize.mjs index a91046342ebfc..36e43c22ff4f9 100644 --- a/components/openai/actions/summarize/summarize.mjs +++ b/components/openai/actions/summarize/summarize.mjs @@ -4,7 +4,7 @@ import constants from "../../common/constants.mjs"; export default { ...common, name: "Summarize Text", - version: "0.1.0", + version: "0.1.1", key: "openai-summarize", description: "Summarizes text using the Chat API. [See the documentation](https://platform.openai.com/docs/api-reference/chat)", type: "action", diff --git a/components/openai/actions/translate-text/translate-text.mjs b/components/openai/actions/translate-text/translate-text.mjs index 0559dcb8a25b3..9e2758532b2ad 100644 --- a/components/openai/actions/translate-text/translate-text.mjs +++ b/components/openai/actions/translate-text/translate-text.mjs @@ -9,7 +9,7 @@ const langOptions = lang.LANGUAGES.map((l) => ({ export default { ...common, name: "Translate Text (Whisper)", - version: "0.1.0", + version: "0.1.1", key: "openai-translate-text", description: "Translate text from one language to another using the Chat API. [See the documentation](https://platform.openai.com/docs/api-reference/chat)", type: "action", diff --git a/components/openai/package.json b/components/openai/package.json index 6542529a89813..79c6ebea67a8f 100644 --- a/components/openai/package.json +++ b/components/openai/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/openai", - "version": "0.6.0", + "version": "0.6.1", "description": "Pipedream OpenAI Components", "main": "openai.app.mjs", "keywords": [ From 724d098fec4f5355e35972a7f414cc7c30387df7 Mon Sep 17 00:00:00 2001 From: danhsiung <35384182+danhsiung@users.noreply.github.com> Date: Fri, 25 Oct 2024 11:33:03 -0700 Subject: [PATCH 11/18] Adding app scaffolding for jina_reader --- components/jina_reader/jina_reader.app.mjs | 11 +++++++++++ components/jina_reader/package.json | 15 +++++++++++++++ pnpm-lock.yaml | 3 +++ 3 files changed, 29 insertions(+) create mode 100644 components/jina_reader/jina_reader.app.mjs create mode 100644 components/jina_reader/package.json diff --git a/components/jina_reader/jina_reader.app.mjs b/components/jina_reader/jina_reader.app.mjs new file mode 100644 index 0000000000000..69d3fc9ad6e7e --- /dev/null +++ b/components/jina_reader/jina_reader.app.mjs @@ -0,0 +1,11 @@ +export default { + type: "app", + app: "jina_reader", + propDefinitions: {}, + methods: { + // this.$auth contains connected account data + authKeys() { + console.log(Object.keys(this.$auth)); + }, + }, +}; \ No newline at end of file diff --git a/components/jina_reader/package.json b/components/jina_reader/package.json new file mode 100644 index 0000000000000..d663710124034 --- /dev/null +++ b/components/jina_reader/package.json @@ -0,0 +1,15 @@ +{ + "name": "@pipedream/jina_reader", + "version": "0.0.1", + "description": "Pipedream Jina Reader Components", + "main": "jina_reader.app.mjs", + "keywords": [ + "pipedream", + "jina_reader" + ], + "homepage": "https://pipedream.com/apps/jina_reader", + "author": "Pipedream (https://pipedream.com/)", + "publishConfig": { + "access": "public" + } +} \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 35eb6e18d23ba..87ed460af7085 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5099,6 +5099,9 @@ importers: form-data: 4.0.0 mime: 4.0.4 + components/jina_reader: + specifiers: {} + components/jira: specifiers: '@pipedream/platform': ^3.0.1 From 8f89ee2f868fb48e964008a8f7d71f3d0e7dba82 Mon Sep 17 00:00:00 2001 From: Jorge Cortes Date: Fri, 25 Oct 2024 14:51:52 -0500 Subject: [PATCH 12/18] Runware: new action component (#14380) --- .../actions/request-task/request-task.mjs | 176 ++++++++++++++++++ components/runware/common/constants.mjs | 35 ++++ components/runware/package.json | 8 +- components/runware/runware.app.mjs | 157 +++++++++++++++- pnpm-lock.yaml | 7 +- 5 files changed, 376 insertions(+), 7 deletions(-) create mode 100644 components/runware/actions/request-task/request-task.mjs create mode 100644 components/runware/common/constants.mjs diff --git a/components/runware/actions/request-task/request-task.mjs b/components/runware/actions/request-task/request-task.mjs new file mode 100644 index 0000000000000..509660bd5ad75 --- /dev/null +++ b/components/runware/actions/request-task/request-task.mjs @@ -0,0 +1,176 @@ +import { v4 as uuid } from "uuid"; +import app from "../../runware.app.mjs"; + +export default { + key: "runware-request-task", + name: "Request Task", + description: "Request one task to be processed by the Runware API. [See the documentation](https://docs.runware.ai/en/image-inference/api-reference).", + version: "0.0.1", + type: "action", + props: { + app, + taskType: { + propDefinition: [ + app, + "taskType", + ], + }, + outputType: { + propDefinition: [ + app, + "outputType", + ], + }, + outputFormat: { + propDefinition: [ + app, + "outputFormat", + ], + }, + uploadEndpoint: { + propDefinition: [ + app, + "uploadEndpoint", + ], + }, + checkNSFW: { + propDefinition: [ + app, + "checkNSFW", + ], + }, + includeCost: { + propDefinition: [ + app, + "includeCost", + ], + }, + positivePrompt: { + propDefinition: [ + app, + "positivePrompt", + ], + }, + negativePrompt: { + propDefinition: [ + app, + "negativePrompt", + ], + }, + seedImage: { + propDefinition: [ + app, + "seedImage", + ], + }, + maskImage: { + propDefinition: [ + app, + "maskImage", + ], + }, + strength: { + propDefinition: [ + app, + "strength", + ], + }, + height: { + propDefinition: [ + app, + "height", + ], + }, + width: { + propDefinition: [ + app, + "width", + ], + }, + model: { + propDefinition: [ + app, + "model", + ], + }, + steps: { + propDefinition: [ + app, + "steps", + ], + }, + scheduler: { + propDefinition: [ + app, + "scheduler", + ], + }, + seed: { + propDefinition: [ + app, + "seed", + ], + }, + numberResults: { + propDefinition: [ + app, + "numberResults", + ], + }, + }, + async run({ $ }) { + const { + app, + taskType, + outputType, + outputFormat, + uploadEndpoint, + checkNSFW, + includeCost, + positivePrompt, + negativePrompt, + seedImage, + maskImage, + strength, + height, + width, + model, + steps, + scheduler, + seed, + numberResults, + } = this; + + const response = await app.post({ + $, + data: [ + { + taskUUID: uuid(), + taskType, + outputType, + outputFormat, + uploadEndpoint, + checkNSFW, + includeCost, + positivePrompt, + negativePrompt, + seedImage, + maskImage, + strength, + height, + width, + model, + steps, + scheduler, + seed: seed + ? parseInt(seed) + : undefined, + numberResults, + }, + ], + }); + + $.export("$summary", `Successfully requested task with UUID \`${response.data[0].taskUUID}\`.`); + return response; + }, +}; diff --git a/components/runware/common/constants.mjs b/components/runware/common/constants.mjs new file mode 100644 index 0000000000000..01df0f6f95e74 --- /dev/null +++ b/components/runware/common/constants.mjs @@ -0,0 +1,35 @@ +const BASE_URL = "https://api.runware.ai"; +const VERSION_PATH = "/v1"; + +const TASK_TYPE = { + IMAGE_INFERENCE: { + value: "imageInference", + label: "Image Inference", + }, + IMAGE_CONTROL_NET_PREPROCESS: { + value: "imageControlNetPreProcess", + label: "Image Control Net Preprocess", + }, + IMAGE_UPSCALE: { + value: "imageUpscale", + label: "Image Upscale", + }, + IMAGE_BACKGROUND_REMOVAL: { + value: "imageBackgroundRemoval", + label: "Image Background Removal", + }, + IMAGE_CAPTION: { + value: "imageCaption", + label: "Image Caption", + }, + PROMPT_ENHANCE: { + value: "promptEnhance", + label: "Prompt Enhance", + }, +}; + +export default { + BASE_URL, + VERSION_PATH, + TASK_TYPE, +}; diff --git a/components/runware/package.json b/components/runware/package.json index e910a1ab6268e..02fb5aff6d31d 100644 --- a/components/runware/package.json +++ b/components/runware/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/runware", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream Runware Components", "main": "runware.app.mjs", "keywords": [ @@ -11,5 +11,9 @@ "author": "Pipedream (https://pipedream.com/)", "publishConfig": { "access": "public" + }, + "dependencies": { + "@pipedream/platform": "3.0.3", + "uuid": "^10.0.0" } -} \ No newline at end of file +} diff --git a/components/runware/runware.app.mjs b/components/runware/runware.app.mjs index c2dd1ce6d7cf9..eda79b83d54b0 100644 --- a/components/runware/runware.app.mjs +++ b/components/runware/runware.app.mjs @@ -1,11 +1,160 @@ +import { axios } from "@pipedream/platform"; +import constants from "./common/constants.mjs"; + export default { type: "app", app: "runware", - propDefinitions: {}, + propDefinitions: { + taskType: { + type: "string", + label: "Task Type", + description: "The type of task to be processed by the Runware API.", + options: Object.values(constants.TASK_TYPE), + }, + outputType: { + type: "string", + label: "Output Type", + description: "Specifies the output type in which the image is returned.", + optional: true, + options: [ + "base64Data", + "dataURI", + "URL", + ], + }, + outputFormat: { + type: "string", + label: "Output Format", + description: "Specifies the format of the output image.", + optional: true, + options: [ + "PNG", + "JPG", + "WEBP", + ], + }, + uploadEndpoint: { + type: "string", + label: "Upload Endpoint", + description: "This parameter allows you to specify a URL to which the generated image will be uploaded as binary image data using the HTTP PUT method. For example, an S3 bucket URL can be used as the upload endpoint. When the image is ready, it will be uploaded to the specified URL.", + optional: true, + }, + checkNSFW: { + type: "boolean", + label: "Check NSFW", + description: "This parameter is used to enable or disable the NSFW check. When enabled, the API will check if the image contains NSFW (not safe for work) content. This check is done using a pre-trained model that detects adult content in images. When the check is enabled, the API will return `NSFWContent: true` in the response object if the image is flagged as potentially sensitive content. If the image is not flagged, the API will return `NSFWContent: false`. If this parameter is not used, the parameter `NSFWContent` will not be included in the response object. Adds `0.1` seconds to image inference time and incurs additional costs. The NSFW filter occasionally returns false positives and very rarely false negatives.", + optional: true, + }, + includeCost: { + type: "boolean", + label: "Include Cost", + description: "If set to `true`, the cost to perform the task will be included in the response object. Defaults to `false`.", + optional: true, + }, + positivePrompt: { + type: "string", + label: "Positive Prompt", + description: "A positive prompt is a text instruction to guide the model on generating the image. It is usually a sentence or a paragraph that provides positive guidance for the task. This parameter is essential to shape the desired results. For example, if the positive prompt is `dragon drinking coffee`, the model will generate an image of a dragon drinking coffee. The more detailed the prompt, the more accurate the results. The length of the prompt must be between 4 and 2000 characters.", + }, + negativePrompt: { + type: "string", + label: "Negative Prompt", + description: "A negative prompt is a text instruction to guide the model on generating the image. It is usually a sentence or a paragraph that provides negative guidance for the task. This parameter helps to avoid certain undesired results. For example, if the negative prompt is `red dragon, cup`, the model will follow the positive prompt but will avoid generating an image of a red dragon or including a cup. The more detailed the prompt, the more accurate the results. The length of the prompt must be between 4 and 2000 characters.", + optional: true, + }, + seedImage: { + type: "string", + label: "Seed Image", + description: "When doing Image-to-Image, Inpainting or Outpainting, this parameter is **required**. Specifies the seed image to be used for the diffusion process. The image can be specified in one of the following formats:\n - An UUID v4 string of a [previously uploaded image](https://docs.runware.ai/en/getting-started/image-upload) or a [generated image](https://docs.runware.ai/en/image-inference/api-reference).\n - A data URI string representing the image. The data URI must be in the format `data:;base64,` followed by the base64-encoded image. For example: `data:image/png;base64,iVBORw0KGgo...`.\n - A base64 encoded image without the data URI prefix. For example: `iVBORw0KGgo...`.\n - A URL pointing to the image. The image must be accessible publicly. Supported formats are: PNG, JPG and WEBP.", + optional: true, + }, + maskImage: { + type: "string", + label: "Mask Image", + description: "When doing Inpainting or Outpainting, this parameter is **required**. Specifies the mask image to be used for the inpainting process. The image can be specified in one of the following formats:\n - An UUID v4 string of a [previously uploaded image](https://docs.runware.ai/en/getting-started/image-upload) or a [generated image](https://docs.runware.ai/en/image-inference/api-reference).\n - A data URI string representing the image. The data URI must be in the format `data:;base64,` followed by the base64-encoded image. For example: `data:image/png;base64,iVBORw0KGgo...`.\n - A base64 encoded image without the data URI prefix. For example: `iVBORw0KGgo...`.\n - A URL pointing to the image. The image must be accessible publicly. Supported formats are: PNG, JPG and WEBP.", + optional: true, + }, + strength: { + type: "string", + label: "Strength", + description: "When doing Image-to-Image, Inpainting or Outpainting, this parameter is used to determine the influence of the **Seed Image** image in the generated output. A higher value results in more influence from the original image, while a lower value allows more creative deviation. Min: `0` Max: `1` and Default: `0.8`.", + optional: true, + }, + height: { + type: "integer", + label: "Height", + description: "Used to define the height dimension of the generated image. Certain models perform better with specific dimensions. The value must be divisible by 64, eg: `512`, `576`, `640` ... `2048`.", + min: 512, + max: 2048, + }, + width: { + type: "integer", + label: "Width", + description: "Used to define the width dimension of the generated image. Certain models perform better with specific dimensions. The value must be divisible by 64, eg: `512`, `576`, `640` ... `2048`.", + min: 512, + max: 2048, + }, + model: { + type: "string", + label: "Model", + description: "This identifier is a unique string that represents a specific model. You can find the AIR identifier of the model you want to use in our [Model Explorer](https://docs.runware.ai/en/image-inference/models#model-explorer), which is a tool that allows you to search for models based on their characteristics. More information about the AIR system can be found in the [Models page](https://docs.runware.ai/en/image-inference/models). Eg. `civitai:78605@83390`.", + }, + steps: { + type: "integer", + label: "Steps", + description: "The number of steps is the number of iterations the model will perform to generate the image. The higher the number of steps, the more detailed the image will be. However, increasing the number of steps will also increase the time it takes to generate the image and may not always result in a better image (some [schedulers](https://docs.runware.ai/en/image-inference/api-reference#request-scheduler) work differently). When using your own models you can specify a new default value for the number of steps. Defaults to `20`.", + min: 1, + max: 100, + optional: true, + }, + scheduler: { + type: "string", + label: "Scheduler", + description: "An scheduler is a component that manages the inference process. Different schedulers can be used to achieve different results like more detailed images, faster inference, or more accurate results. The default scheduler is the one that the model was trained with, but you can choose a different one to get different results. Schedulers are explained in more detail in the [Schedulers page](https://docs.runware.ai/en/image-inference/schedulers).", + optional: true, + }, + seed: { + type: "string", + label: "Seed", + description: "A seed is a value used to randomize the image generation. If you want to make images reproducible (generate the same image multiple times), you can use the same seed value. When requesting multiple images with the same seed, the seed will be incremented by 1 (+1) for each image generated. Min: `0` Max: `9223372036854776000`. Defaults to `Random`.", + optional: true, + }, + numberResults: { + type: "integer", + label: "Number Of Results", + description: "The number of images to generate from the specified prompt. If **Seed** is set, it will be incremented by 1 (+1) for each image generated.", + optional: true, + }, + }, methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); + getUrl(path = "") { + return `${constants.BASE_URL}${constants.VERSION_PATH}${path}`; + }, + getHeaders(headers) { + return { + ...headers, + "Content-Type": "application/json", + "Authorization": `Bearer ${this.$auth.api_key}`, + }; + }, + async makeRequest({ + $ = this, path, headers, ...args + }) { + const response = await axios($, { + ...args, + url: this.getUrl(path), + headers: this.getHeaders(headers), + }); + if (response.errors) { + throw new Error(JSON.stringify(response.errors)); + } + return response; + }, + post(args = {}) { + return this.makeRequest({ + method: "POST", + ...args, + }); }, }, }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 87ed460af7085..119b497039009 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8524,7 +8524,12 @@ importers: specifiers: {} components/runware: - specifiers: {} + specifiers: + '@pipedream/platform': 3.0.3 + uuid: ^10.0.0 + dependencies: + '@pipedream/platform': 3.0.3 + uuid: 10.0.0 components/rytr: specifiers: {} From d514b8d7d7cf95351505924d1dfd230beda4d395 Mon Sep 17 00:00:00 2001 From: Luan Cazarine Date: Mon, 28 Oct 2024 10:43:45 -0300 Subject: [PATCH 13/18] New Components - smstools (#14378) * smstools init * [Components] smstools #14370 Sources - New Inbound Message Actions - Add Contact - Add Contact Opt Out - Send SMS * pnpm update * Update components/smstools/smstools.app.mjs --------- Co-authored-by: michelle0927 --- .../add-contact-opt-out.mjs | 34 ++++ .../actions/add-contact/add-contact.mjs | 114 ++++++++++++ .../smstools/actions/send-sms/send-sms.mjs | 72 ++++++++ components/smstools/package.json | 7 +- components/smstools/smstools.app.mjs | 164 +++++++++++++++++- .../new-inbound-message.mjs | 64 +++++++ .../new-inbound-message/test-event.mjs | 8 + pnpm-lock.yaml | 107 ++++++------ 8 files changed, 511 insertions(+), 59 deletions(-) create mode 100644 components/smstools/actions/add-contact-opt-out/add-contact-opt-out.mjs create mode 100644 components/smstools/actions/add-contact/add-contact.mjs create mode 100644 components/smstools/actions/send-sms/send-sms.mjs create mode 100644 components/smstools/sources/new-inbound-message/new-inbound-message.mjs create mode 100644 components/smstools/sources/new-inbound-message/test-event.mjs diff --git a/components/smstools/actions/add-contact-opt-out/add-contact-opt-out.mjs b/components/smstools/actions/add-contact-opt-out/add-contact-opt-out.mjs new file mode 100644 index 0000000000000..a696fba6d52cd --- /dev/null +++ b/components/smstools/actions/add-contact-opt-out/add-contact-opt-out.mjs @@ -0,0 +1,34 @@ +import { ConfigurationError } from "@pipedream/platform"; +import smstools from "../../smstools.app.mjs"; + +export default { + key: "smstools-add-contact-opt-out", + name: "Add Contact to Opt-Out List", + description: "Adds a selected contact to the opt-out list, stopping further communications. [See the documentation](https://www.smstools.com/en/sms-gateway-api/add_optout)", + version: "0.0.1", + type: "action", + props: { + smstools, + contactNumber: { + propDefinition: [ + smstools, + "contactNumber", + ], + }, + }, + async run({ $ }) { + try { + const response = await this.smstools.addOptOut({ + $, + data: { + number: this.contactNumber, + }, + }); + + $.export("$summary", `Successfully added contact number ${this.contactNumber} to the opt-out list.`); + return response; + } catch (e) { + throw new ConfigurationError("The number is already opted-out or does not exist in the database."); + } + }, +}; diff --git a/components/smstools/actions/add-contact/add-contact.mjs b/components/smstools/actions/add-contact/add-contact.mjs new file mode 100644 index 0000000000000..082c219cf366a --- /dev/null +++ b/components/smstools/actions/add-contact/add-contact.mjs @@ -0,0 +1,114 @@ +import { ConfigurationError } from "@pipedream/platform"; +import smstools from "../../smstools.app.mjs"; + +export default { + key: "smstools-add-contact", + name: "Add Contact to Group", + description: "Adds a new contact to an existing contact list. [See the documentation](https://www.smstools.com/en/sms-gateway-api/add_contact)", + version: "0.0.1", + type: "action", + props: { + smstools, + phone: { + type: "string", + label: "Phone Number", + description: "The phone number of the contact.", + }, + groupid: { + propDefinition: [ + smstools, + "groupId", + ], + }, + firstName: { + type: "string", + label: "First Name", + description: "First name of the contact.", + optional: true, + }, + lastName: { + type: "string", + label: "Last Name", + description: "Last name of the contact.", + optional: true, + }, + birthday: { + type: "string", + label: "Birthday", + description: "Birthday of the contact. **Format: YYYY-MM-DD**.", + optional: true, + }, + extra1: { + type: "string", + label: "Extra 1", + description: "Extra field 1 for the contact.", + optional: true, + }, + extra2: { + type: "string", + label: "Extra 2", + description: "Extra field 2 for the contact.", + optional: true, + }, + extra3: { + type: "string", + label: "Extra 3", + description: "Extra field 3 for the contact.", + optional: true, + }, + extra4: { + type: "string", + label: "Extra 4", + description: "Extra field 4 for the contact.", + optional: true, + }, + extra5: { + type: "string", + label: "Extra 5", + description: "Extra field 5 for the contact.", + optional: true, + }, + extra6: { + type: "string", + label: "Extra 6", + description: "Extra field 6 for the contact.", + optional: true, + }, + extra7: { + type: "string", + label: "Extra 7", + description: "Extra field 7 for the contact.", + optional: true, + }, + extra8: { + type: "string", + label: "Extra 8", + description: "Extra field 8 for the contact.", + optional: true, + }, + unsubscribed: { + type: "boolean", + label: "Unsubscribed", + description: "Indicates if the contact is unsubscribed.", + optional: true, + }, + }, + async run({ $ }) { + try { + const { + smstools, + ...data + } = this; + + const response = await smstools.addContact({ + $, + data, + }); + + $.export("$summary", `Successfully added contact with ID: ${response.ID}`); + return response; + } catch (e) { + throw new ConfigurationError(e.response.data.errorMsg); + } + }, +}; diff --git a/components/smstools/actions/send-sms/send-sms.mjs b/components/smstools/actions/send-sms/send-sms.mjs new file mode 100644 index 0000000000000..aff1345e0b164 --- /dev/null +++ b/components/smstools/actions/send-sms/send-sms.mjs @@ -0,0 +1,72 @@ +import smstools from "../../smstools.app.mjs"; + +export default { + key: "smstools-send-sms", + name: "Send SMS or WhatsApp Message", + description: "Sends a SMS or WhatsApp message to a specified contact. [See the documentation](https://www.smstools.com/en/sms-gateway-api/send_message)", + version: "0.0.1", + type: "action", + props: { + smstools, + message: { + type: "string", + label: "Message", + description: "The message to be sent.", + }, + to: { + propDefinition: [ + smstools, + "contactNumber", + ], + type: "string[]", + description: "The contact(s) to send the message to.", + }, + sender: { + propDefinition: [ + smstools, + "sender", + ], + }, + date: { + type: "string", + label: "Scheduled Date", + description: "The date to send the message. **Format: yyyy-MM-dd HH:mm**. If not provided, the message will be sent as soon as possible.", + optional: true, + }, + reference: { + type: "string", + label: "Reference", + description: "Reference for the message.", + optional: true, + }, + test: { + type: "boolean", + label: "Test", + description: "Test mode for the message.", + optional: true, + }, + subId: { + propDefinition: [ + smstools, + "subId", + ], + optional: true, + }, + }, + async run({ $ }) { + const response = await this.smstools.sendMessage({ + $, + data: { + message: this.message, + to: this.to, + sender: this.sender, + date: this.date, + reference: this.reference, + test: this.test, + subId: this.subId, + }, + }); + $.export("$summary", `Message sent successfully with ID: ${response.messageid}`); + return response; + }, +}; diff --git a/components/smstools/package.json b/components/smstools/package.json index 787c8d992c03b..a69571462df76 100644 --- a/components/smstools/package.json +++ b/components/smstools/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/smstools", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream SMSTools Components", "main": "smstools.app.mjs", "keywords": [ @@ -11,5 +11,8 @@ "author": "Pipedream (https://pipedream.com/)", "publishConfig": { "access": "public" + }, + "dependencies": { + "@pipedream/platform": "^3.0.3" } -} \ No newline at end of file +} diff --git a/components/smstools/smstools.app.mjs b/components/smstools/smstools.app.mjs index 9a06528cfd4ce..5f2d0e4f35784 100644 --- a/components/smstools/smstools.app.mjs +++ b/components/smstools/smstools.app.mjs @@ -1,11 +1,165 @@ +import { axios } from "@pipedream/platform"; + export default { type: "app", app: "smstools", - propDefinitions: {}, + propDefinitions: { + groupId: { + type: "string", + label: "Group ID", + description: "The group ID where the contact should be added.", + async options({ page }) { + const groups = await this.getGroups({ + params: { + page: page + 1, + }, + }); + return groups.map(({ + ID: value, name: label, + }) => ({ + label, + value, + })); + }, + }, + contactNumber: { + type: "string", + label: "Contact Number", + description: "Select a contact number to add to the opt-out list.", + async options({ page }) { + const { contacts } = await this.getContactNumbers({ + params: { + page: page + 1, + }, + }); + return contacts.map(({ phone }) => phone); + }, + }, + sender: { + type: "string", + label: "Sender", + description: "The sender ID for the message.", + async options() { + const senders = await this.getSenderIds(); + return senders.map(({ + ID: value, name: label, + }) => ({ + label, + value, + })); + }, + }, + subId: { + type: "string", + label: "Sub ID", + description: "Subaccount ID from which the message is sent.", + async options() { + const subaccounts = await this.getSubAccounts(); + return subaccounts.map(({ + ID: value, username: label, + }) => ({ + label, + value, + })); + }, + optional: true, + }, + }, methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); + _baseUrl() { + return "https://api.smsgatewayapi.com/v1"; + }, + _params(params = {}) { + return { + client_id: `${this.$auth.client_id}`, + client_secret: `${this.$auth.client_secret}`, + ...params, + }; + }, + _makeRequest({ + $ = this, path, params, ...opts + }) { + return axios($, { + url: this._baseUrl() + path, + params: this._params(params), + ...opts, + }); + }, + getInboxMessages(opts = {}) { + return this._makeRequest({ + path: "/message/inbox", + ...opts, + }); + }, + getContactNumbers(opts = {}) { + return this._makeRequest({ + path: "/contact", + ...opts, + }); + }, + getGroups(opts = {}) { + return this._makeRequest({ + path: "/groups", + ...opts, + }); + }, + getSenderIds(opts = {}) { + return this._makeRequest({ + path: "/senderids", + ...opts, + }); + }, + getSubAccounts(opts = {}) { + return this._makeRequest({ + path: "/subaccount", + ...opts, + }); + }, + addContact(opts = {}) { + return this._makeRequest({ + method: "POST", + path: "/contact", + ...opts, + }); + }, + addOptOut(opts = {}) { + return this._makeRequest({ + method: "POST", + path: "/optouts", + ...opts, + }); + }, + sendMessage(opts = {}) { + return this._makeRequest({ + method: "POST", + path: "/message/send", + ...opts, + }); + }, + async *paginate({ + fn, params = {}, maxResults = null, ...opts + }) { + let hasMore = false; + let count = 0; + let page = 0; + + do { + params.page = ++page; + const { messages } = await fn({ + params, + ...opts, + }); + for (const d of messages) { + yield d; + + if (maxResults && ++count === maxResults) { + return count; + } + } + + hasMore = messages.length; + + } while (hasMore); }, }, -}; \ No newline at end of file +}; diff --git a/components/smstools/sources/new-inbound-message/new-inbound-message.mjs b/components/smstools/sources/new-inbound-message/new-inbound-message.mjs new file mode 100644 index 0000000000000..600a8616bdde3 --- /dev/null +++ b/components/smstools/sources/new-inbound-message/new-inbound-message.mjs @@ -0,0 +1,64 @@ +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; +import smstools from "../../smstools.app.mjs"; +import sampleEmit from "./test-event.mjs"; + +export default { + key: "smstools-new-inbound-message", + name: "New Inbound Message", + description: "Emit new event when a new inbound message is received.", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + smstools, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + }, + methods: { + _getLastId() { + return this.db.get("lastId") || 0; + }, + _setLastId(lastId) { + this.db.set("lastId", lastId); + }, + async emitEvent(maxResults = false) { + const lastId = this._getLastId(); + const response = this.smstools.paginate({ + fn: this.smstools.getInboxMessages, + maxResults, + }); + + let responseArray = []; + for await (const item of response) { + if (item.ID <= lastId) break; + responseArray.push(item); + } + + if (responseArray.length) { + this._setLastId(responseArray[0].ID); + } + + for (const item of responseArray.reverse()) { + this.$emit(item, { + id: item.ID, + summary: `New inbound message from ${item.sender}`, + ts: Date.parse(item.date), + }); + } + }, + }, + hooks: { + async deploy() { + await this.emitEvent(25); + }, + }, + async run() { + await this.emitEvent(); + }, + sampleEmit, +}; diff --git a/components/smstools/sources/new-inbound-message/test-event.mjs b/components/smstools/sources/new-inbound-message/test-event.mjs new file mode 100644 index 0000000000000..2fad07180fa26 --- /dev/null +++ b/components/smstools/sources/new-inbound-message/test-event.mjs @@ -0,0 +1,8 @@ +export default { + "ID": "{ID}", + "message": "Hello sms", + "sender": "{nr}", + "type": "sms", + "date": "2022-01-01 12:00:00", + "receiver": "{inbox_nr}" +} \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 119b497039009..d636d402b2bbe 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9418,7 +9418,10 @@ importers: '@pipedream/platform': 3.0.3 components/smstools: - specifiers: {} + specifiers: + '@pipedream/platform': ^3.0.3 + dependencies: + '@pipedream/platform': 3.0.3 components/smtp2go: specifiers: @@ -13186,55 +13189,6 @@ packages: - aws-crt dev: false - /@aws-sdk/client-sso-oidc/3.600.0_tdq3komn4zwyd65w7klbptsu34: - resolution: {integrity: sha512-7+I8RWURGfzvChyNQSyj5/tKrqRbzRl7H+BnTOf/4Vsw1nFOi5ROhlhD4X/Y0QCTacxnaoNcIrqnY7uGGvVRzw==} - engines: {node: '>=16.0.0'} - dependencies: - '@aws-crypto/sha256-browser': 5.2.0 - '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/client-sts': 3.600.0 - '@aws-sdk/core': 3.598.0 - '@aws-sdk/credential-provider-node': 3.600.0_f7n47caigsrjd2lr2szmwfuee4 - '@aws-sdk/middleware-host-header': 3.598.0 - '@aws-sdk/middleware-logger': 3.598.0 - '@aws-sdk/middleware-recursion-detection': 3.598.0 - '@aws-sdk/middleware-user-agent': 3.598.0 - '@aws-sdk/region-config-resolver': 3.598.0 - '@aws-sdk/types': 3.598.0 - '@aws-sdk/util-endpoints': 3.598.0 - '@aws-sdk/util-user-agent-browser': 3.598.0 - '@aws-sdk/util-user-agent-node': 3.598.0 - '@smithy/config-resolver': 3.0.3 - '@smithy/core': 2.2.3 - '@smithy/fetch-http-handler': 3.2.1 - '@smithy/hash-node': 3.0.2 - '@smithy/invalid-dependency': 3.0.2 - '@smithy/middleware-content-length': 3.0.2 - '@smithy/middleware-endpoint': 3.0.4 - '@smithy/middleware-retry': 3.0.6 - '@smithy/middleware-serde': 3.0.3 - '@smithy/middleware-stack': 3.0.3 - '@smithy/node-config-provider': 3.1.3 - '@smithy/node-http-handler': 3.1.2 - '@smithy/protocol-http': 4.0.3 - '@smithy/smithy-client': 3.1.6 - '@smithy/types': 3.3.0 - '@smithy/url-parser': 3.0.3 - '@smithy/util-base64': 3.0.0 - '@smithy/util-body-length-browser': 3.0.0 - '@smithy/util-body-length-node': 3.0.0 - '@smithy/util-defaults-mode-browser': 3.0.6 - '@smithy/util-defaults-mode-node': 3.0.6 - '@smithy/util-endpoints': 2.0.3 - '@smithy/util-middleware': 3.0.3 - '@smithy/util-retry': 3.0.2 - '@smithy/util-utf8': 3.0.0 - tslib: 2.6.3 - transitivePeerDependencies: - - '@aws-sdk/client-sts' - - aws-crt - dev: false - /@aws-sdk/client-sso/3.423.0: resolution: {integrity: sha512-znIufHkwhCIePgaYciIs3x/+BpzR57CZzbCKHR9+oOvGyufEPPpUT5bFLvbwTgfiVkTjuk6sG/ES3U5Bc+xtrA==} engines: {node: '>=14.0.0'} @@ -13470,7 +13424,7 @@ packages: dependencies: '@aws-crypto/sha256-browser': 5.2.0 '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/client-sso-oidc': 3.600.0_tdq3komn4zwyd65w7klbptsu34 + '@aws-sdk/client-sso-oidc': 3.600.0 '@aws-sdk/core': 3.598.0 '@aws-sdk/credential-provider-node': 3.600.0_f7n47caigsrjd2lr2szmwfuee4 '@aws-sdk/middleware-host-header': 3.598.0 @@ -13512,6 +13466,55 @@ packages: - aws-crt dev: false + /@aws-sdk/client-sts/3.600.0_dseaa2p5u2yk67qiepewcq3hkq: + resolution: {integrity: sha512-KQG97B7LvTtTiGmjlrG1LRAY8wUvCQzrmZVV5bjrJ/1oXAU7DITYwVbSJeX9NWg6hDuSk0VE3MFwIXS2SvfLIA==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/client-sso-oidc': 3.600.0 + '@aws-sdk/core': 3.598.0 + '@aws-sdk/credential-provider-node': 3.600.0_f7n47caigsrjd2lr2szmwfuee4 + '@aws-sdk/middleware-host-header': 3.598.0 + '@aws-sdk/middleware-logger': 3.598.0 + '@aws-sdk/middleware-recursion-detection': 3.598.0 + '@aws-sdk/middleware-user-agent': 3.598.0 + '@aws-sdk/region-config-resolver': 3.598.0 + '@aws-sdk/types': 3.598.0 + '@aws-sdk/util-endpoints': 3.598.0 + '@aws-sdk/util-user-agent-browser': 3.598.0 + '@aws-sdk/util-user-agent-node': 3.598.0 + '@smithy/config-resolver': 3.0.3 + '@smithy/core': 2.2.3 + '@smithy/fetch-http-handler': 3.2.1 + '@smithy/hash-node': 3.0.2 + '@smithy/invalid-dependency': 3.0.2 + '@smithy/middleware-content-length': 3.0.2 + '@smithy/middleware-endpoint': 3.0.4 + '@smithy/middleware-retry': 3.0.6 + '@smithy/middleware-serde': 3.0.3 + '@smithy/middleware-stack': 3.0.3 + '@smithy/node-config-provider': 3.1.3 + '@smithy/node-http-handler': 3.1.2 + '@smithy/protocol-http': 4.0.3 + '@smithy/smithy-client': 3.1.6 + '@smithy/types': 3.3.0 + '@smithy/url-parser': 3.0.3 + '@smithy/util-base64': 3.0.0 + '@smithy/util-body-length-browser': 3.0.0 + '@smithy/util-body-length-node': 3.0.0 + '@smithy/util-defaults-mode-browser': 3.0.6 + '@smithy/util-defaults-mode-node': 3.0.6 + '@smithy/util-endpoints': 2.0.3 + '@smithy/util-middleware': 3.0.3 + '@smithy/util-retry': 3.0.2 + '@smithy/util-utf8': 3.0.0 + tslib: 2.6.3 + transitivePeerDependencies: + - '@aws-sdk/client-sso-oidc' + - aws-crt + dev: false + /@aws-sdk/core/3.556.0: resolution: {integrity: sha512-vJaSaHw2kPQlo11j/Rzuz0gk1tEaKdz+2ser0f0qZ5vwFlANjt08m/frU17ctnVKC1s58bxpctO/1P894fHLrA==} engines: {node: '>=14.0.0'} @@ -17810,7 +17813,7 @@ packages: '@aws-sdk/client-sns': 3.423.0 '@aws-sdk/client-sqs': 3.423.0 '@aws-sdk/client-ssm': 3.423.0 - '@aws-sdk/client-sts': 3.600.0 + '@aws-sdk/client-sts': 3.600.0_dseaa2p5u2yk67qiepewcq3hkq '@aws-sdk/s3-request-presigner': 3.609.0 '@pipedream/helper_functions': 0.3.12 '@pipedream/platform': 1.6.6 From d7b33bc1b74be4b3fae129d9300d59aa56eac7be Mon Sep 17 00:00:00 2001 From: Jorge Cortes Date: Mon, 28 Oct 2024 09:47:01 -0500 Subject: [PATCH 14/18] roamresearch: new action components (#14385) --- .../add-content-to-daily-note-page.mjs | 63 ++++++++++++++++++ .../add-content-to-page.mjs | 61 +++++++++++++++++ .../add-content-underneath-block.mjs | 60 +++++++++++++++++ .../get-page-or-block-data.mjs | 53 +++++++++++++++ .../actions/search-title/search-title.mjs | 41 ++++++++++++ components/roamresearch/common/constants.mjs | 15 +++++ components/roamresearch/package.json | 7 +- components/roamresearch/roamresearch.app.mjs | 66 +++++++++++++++++-- pnpm-lock.yaml | 5 +- 9 files changed, 363 insertions(+), 8 deletions(-) create mode 100644 components/roamresearch/actions/add-content-to-daily-note-page/add-content-to-daily-note-page.mjs create mode 100644 components/roamresearch/actions/add-content-to-page/add-content-to-page.mjs create mode 100644 components/roamresearch/actions/add-content-underneath-block/add-content-underneath-block.mjs create mode 100644 components/roamresearch/actions/get-page-or-block-data/get-page-or-block-data.mjs create mode 100644 components/roamresearch/actions/search-title/search-title.mjs create mode 100644 components/roamresearch/common/constants.mjs diff --git a/components/roamresearch/actions/add-content-to-daily-note-page/add-content-to-daily-note-page.mjs b/components/roamresearch/actions/add-content-to-daily-note-page/add-content-to-daily-note-page.mjs new file mode 100644 index 0000000000000..1e896512c4df1 --- /dev/null +++ b/components/roamresearch/actions/add-content-to-daily-note-page/add-content-to-daily-note-page.mjs @@ -0,0 +1,63 @@ +import app from "../../roamresearch.app.mjs"; + +export default { + key: "roamresearch-add-content-to-daily-note-page", + name: "Add Content To Daily Note Page", + description: "Adds content as a child block to a daily note page in Roam Research (access to encrypted and non encrypted graphs). [See the documentation](https://roamresearch.com/#/app/developer-documentation/page/eb8OVhaFC).", + version: "0.0.1", + type: "action", + props: { + app, + dailyNoteTitle: { + type: "string", + label: "Date For Daily Note", + description: "The date for the daily note page title, formatted as `MM-DD-YYYY`. Keep in mind the Daily Note page should exist.", + }, + content: { + propDefinition: [ + app, + "content", + ], + }, + nestUnder: { + propDefinition: [ + app, + "nestUnder", + ], + }, + }, + async run({ $ }) { + const { + app, + dailyNoteTitle, + content, + nestUnder, + } = this; + + const response = app.appendBlocks({ + $, + data: { + "location": { + page: { + title: { + "daily-note-page": dailyNoteTitle, + }, + }, + ...(nestUnder && { + "nest-under": { + string: nestUnder, + }, + }), + }, + "append-data": [ + { + string: content, + }, + ], + }, + }); + + $.export("$summary", "Succesfully added content to daily note page."); + return response; + }, +}; diff --git a/components/roamresearch/actions/add-content-to-page/add-content-to-page.mjs b/components/roamresearch/actions/add-content-to-page/add-content-to-page.mjs new file mode 100644 index 0000000000000..bbf4fa000c582 --- /dev/null +++ b/components/roamresearch/actions/add-content-to-page/add-content-to-page.mjs @@ -0,0 +1,61 @@ +import app from "../../roamresearch.app.mjs"; + +export default { + key: "roamresearch-add-content-to-page", + name: "Add Content To Page", + description: "Add content as a child block to an existing or new page in Roam Research (access to encrypted and non encrypted graphs). [See the documentation](https://roamresearch.com/#/app/developer-documentation/page/eb8OVhaFC).", + version: "0.0.1", + type: "action", + props: { + app, + title: { + type: "string", + label: "Page Title", + description: "Title of the page to add content to.", + }, + content: { + propDefinition: [ + app, + "content", + ], + }, + nestUnder: { + propDefinition: [ + app, + "nestUnder", + ], + }, + }, + async run({ $ }) { + const { + app, + title, + content, + nestUnder, + } = this; + + const response = app.appendBlocks({ + $, + data: { + "location": { + page: { + title, + }, + ...(nestUnder && { + "nest-under": { + string: nestUnder, + }, + }), + }, + "append-data": [ + { + string: content, + }, + ], + }, + }); + + $.export("$summary", "Succesfully added content to page."); + return response; + }, +}; diff --git a/components/roamresearch/actions/add-content-underneath-block/add-content-underneath-block.mjs b/components/roamresearch/actions/add-content-underneath-block/add-content-underneath-block.mjs new file mode 100644 index 0000000000000..afc652916a26d --- /dev/null +++ b/components/roamresearch/actions/add-content-underneath-block/add-content-underneath-block.mjs @@ -0,0 +1,60 @@ +import app from "../../roamresearch.app.mjs"; + +export default { + key: "roamresearch-add-content-underneath-block", + name: "Add Content Underneath Block", + description: "Add content underneath an existing block in your Roam Research graph (access to encrypted and non encrypted graphs). [See the documentation](https://roamresearch.com/)", + version: "0.0.1", + type: "action", + props: { + app, + blockUid: { + type: "string", + label: "Block UID", + description: "The block UID in the Roam graph you want to append content to.", + }, + content: { + type: "string", + label: "Content", + description: "The content of the block to be added.", + }, + nestUnder: { + type: "string", + label: "Nest Under", + description: "Title of the block to nest the new block under.", + optional: true, + }, + }, + async run({ $ }) { + const { + app, + blockUid, + content, + nestUnder, + } = this; + + const response = app.appendBlocks({ + $, + data: { + "location": { + block: { + uid: blockUid, + }, + ...(nestUnder && { + "nest-under": { + string: nestUnder, + }, + }), + }, + "append-data": [ + { + string: content, + }, + ], + }, + }); + + $.export("$summary", "Succesfully added content underneath block."); + return response; + }, +}; diff --git a/components/roamresearch/actions/get-page-or-block-data/get-page-or-block-data.mjs b/components/roamresearch/actions/get-page-or-block-data/get-page-or-block-data.mjs new file mode 100644 index 0000000000000..6f2f3362bb46a --- /dev/null +++ b/components/roamresearch/actions/get-page-or-block-data/get-page-or-block-data.mjs @@ -0,0 +1,53 @@ +import app from "../../roamresearch.app.mjs"; + +export default { + key: "roamresearch-get-page-or-block-data", + name: "Get Page Or Block Data", + description: "Get the data for a page or block in Roam Research (access only to non ecrypted graphs). [See the documentation](https://roamresearch.com/#/app/developer-documentation/page/eb8OVhaFC).", + version: "0.0.1", + type: "action", + props: { + app, + resourceType: { + type: "string", + label: "Resource Type", + description: "The type of resource to get data for.", + options: [ + "page", + "block", + ], + }, + pageOrBlock: { + type: "string", + label: "Page Title Or Block UID", + description: "The page title of the block uid to get data for. Page title example: `My Page` and Block UID example: `ideWWvTgI`.", + }, + }, + async run({ $ }) { + const { + app, + resourceType, + pageOrBlock, + } = this; + + const attribute = resourceType === "page" + ? ":node/title" + : ":block/uid"; + + const response = await app.pull({ + $, + data: { + selector: `[${attribute} :block/string :block/order {:block/children ...}]`, + eid: `[${attribute} "${pageOrBlock}"]`, + }, + }); + + if (!response.result) { + $.export("$summary", `Failed to get data for ${resourceType}: \`${pageOrBlock}\`.`); + return response; + } + + $.export("$summary", `Succesfully got data for ${resourceType}: \`${pageOrBlock}\`.`); + return response; + }, +}; diff --git a/components/roamresearch/actions/search-title/search-title.mjs b/components/roamresearch/actions/search-title/search-title.mjs new file mode 100644 index 0000000000000..7d22344be6607 --- /dev/null +++ b/components/roamresearch/actions/search-title/search-title.mjs @@ -0,0 +1,41 @@ +import app from "../../roamresearch.app.mjs"; + +export default { + key: "roamresearch-search-title", + name: "Search Title", + description: "Search for a title in Roam Research pages (access only to non ecrypted graphs). [See the documentation](https://roamresearch.com/#/app/developer-documentation/page/eb8OVhaFC).", + version: "0.0.1", + type: "action", + props: { + app, + title: { + type: "string", + label: "Search Title", + description: "The title to search for.", + }, + }, + async run({ $ }) { + const { + app, + title, + } = this; + const response = await app.query({ + $, + data: { + query: `[ + :find (pull ?b [:block/uid :node/title]) + :in $ ?search-string + :where [?b :node/title ?page-title] [ + (clojure.string/includes? ?page-title ?search-string) + ] + ]`, + args: [ + title, + ], + }, + }); + + $.export("$summary", `Succesfully searched for title: \`${title}\`.`); + return response; + }, +}; diff --git a/components/roamresearch/common/constants.mjs b/components/roamresearch/common/constants.mjs new file mode 100644 index 0000000000000..e65123a69c092 --- /dev/null +++ b/components/roamresearch/common/constants.mjs @@ -0,0 +1,15 @@ +const SUBDOMAIN_PLACEHOLDER = "{subdomain}"; +const BASE_URL = `https://${SUBDOMAIN_PLACEHOLDER}.roamresearch.com`; +const VERSION_PATH = "/api/graph"; + +const API = { + DEFAULT: "api", + APPEND: "append-api", +}; + +export default { + SUBDOMAIN_PLACEHOLDER, + BASE_URL, + VERSION_PATH, + API, +}; diff --git a/components/roamresearch/package.json b/components/roamresearch/package.json index 2ba776959e91a..76ce74990ec36 100644 --- a/components/roamresearch/package.json +++ b/components/roamresearch/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/roamresearch", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream roamresearch Components", "main": "roamresearch.app.mjs", "keywords": [ @@ -11,5 +11,8 @@ "author": "Pipedream (https://pipedream.com/)", "publishConfig": { "access": "public" + }, + "dependencies": { + "@pipedream/platform": "3.0.3" } -} \ No newline at end of file +} diff --git a/components/roamresearch/roamresearch.app.mjs b/components/roamresearch/roamresearch.app.mjs index 5d9f7f5184551..16e77448428c1 100644 --- a/components/roamresearch/roamresearch.app.mjs +++ b/components/roamresearch/roamresearch.app.mjs @@ -1,11 +1,67 @@ +import { axios } from "@pipedream/platform"; +import constants from "./common/constants.mjs"; + export default { type: "app", app: "roamresearch", - propDefinitions: {}, + propDefinitions: { + content: { + type: "string", + label: "Content", + description: "The content of the block to be added.", + }, + nestUnder: { + type: "string", + label: "Nest Under", + description: "Title of the block to nest the new block under.", + optional: true, + }, + }, methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); + getUrl(path, api = constants.API.DEFAULT) { + const { graph_name: graphName } = this.$auth; + const baseUrl = constants.BASE_URL.replace(constants.SUBDOMAIN_PLACEHOLDER, api); + return `${baseUrl}${constants.VERSION_PATH}/${graphName}${path}`; + }, + getHeaders(headers) { + return { + ...headers, + "X-Authorization": `Bearer ${this.$auth.api_token}`, + }; + }, + _makeRequest({ + $ = this, path, headers, api, ...args + } = {}) { + return axios($, { + ...args, + url: this.getUrl(path, api), + headers: this.getHeaders(headers), + }); + }, + post(args = {}) { + return this._makeRequest({ + method: "POST", + ...args, + }); + }, + appendBlocks(args = {}) { + return this.post({ + api: constants.API.APPEND, + path: "/append-blocks", + ...args, + }); + }, + query(args = {}) { + return this.post({ + path: "/q", + ...args, + }); + }, + pull(args = {}) { + return this.post({ + path: "/pull", + ...args, + }); }, }, -}; \ No newline at end of file +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d636d402b2bbe..c5ccc4476dbf1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8436,7 +8436,10 @@ importers: specifiers: {} components/roamresearch: - specifiers: {} + specifiers: + '@pipedream/platform': 3.0.3 + dependencies: + '@pipedream/platform': 3.0.3 components/robocorp: specifiers: {} From cd3d56090a3e3367daca0d038c2e69621ce3c921 Mon Sep 17 00:00:00 2001 From: Luan Cazarine Date: Mon, 28 Oct 2024 12:31:56 -0300 Subject: [PATCH 15/18] New Components - everhour (#14307) * everhour init * init * pnpm update * [Components] everhour #13219 Sources - New Client (Instant) - New Task (Instant) - New Task Time Updated (Instant) Actions - Create Task - Start Timer - Stop Timer * [Components] everhour #13219 Sources - New Client (Instant) - New Task (Instant) - New Task Time Updated (Instant) Actions - Create Task - Start Timer - Stop Timer * fix status options --- .../actions/create-task/create-task.mjs | 84 +++++++++ .../actions/start-timer/start-timer.mjs | 52 ++++++ .../actions/stop-timer/stop-timer.mjs | 17 ++ components/everhour/common/constants.mjs | 12 ++ components/everhour/common/utils.mjs | 24 +++ components/everhour/everhour.app.mjs | 173 +++++++++++++++++- components/everhour/package.json | 18 ++ components/everhour/sources/common/base.mjs | 58 ++++++ .../new-client-instant/new-client-instant.mjs | 24 +++ .../sources/new-client-instant/test-event.mjs | 25 +++ .../new-task-instant/new-task-instant.mjs | 38 ++++ .../sources/new-task-instant/test-event.mjs | 24 +++ .../task-time-updated-instant.mjs | 38 ++++ .../task-time-updated-instant/test-event.mjs | 55 ++++++ pnpm-lock.yaml | 6 + 15 files changed, 644 insertions(+), 4 deletions(-) create mode 100644 components/everhour/actions/create-task/create-task.mjs create mode 100644 components/everhour/actions/start-timer/start-timer.mjs create mode 100644 components/everhour/actions/stop-timer/stop-timer.mjs create mode 100644 components/everhour/common/constants.mjs create mode 100644 components/everhour/common/utils.mjs create mode 100644 components/everhour/package.json create mode 100644 components/everhour/sources/common/base.mjs create mode 100644 components/everhour/sources/new-client-instant/new-client-instant.mjs create mode 100644 components/everhour/sources/new-client-instant/test-event.mjs create mode 100644 components/everhour/sources/new-task-instant/new-task-instant.mjs create mode 100644 components/everhour/sources/new-task-instant/test-event.mjs create mode 100644 components/everhour/sources/task-time-updated-instant/task-time-updated-instant.mjs create mode 100644 components/everhour/sources/task-time-updated-instant/test-event.mjs diff --git a/components/everhour/actions/create-task/create-task.mjs b/components/everhour/actions/create-task/create-task.mjs new file mode 100644 index 0000000000000..af6ce6cea5bdb --- /dev/null +++ b/components/everhour/actions/create-task/create-task.mjs @@ -0,0 +1,84 @@ +import { STATUS_OPTIONS } from "../../common/constants.mjs"; +import { parseObject } from "../../common/utils.mjs"; +import everhour from "../../everhour.app.mjs"; + +export default { + key: "everhour-create-task", + name: "Create Task", + description: "Creates a new task in Everhour. [See the documentation](https://everhour.docs.apiary.io/)", + version: "0.0.1", + type: "action", + props: { + everhour, + projectId: { + propDefinition: [ + everhour, + "projectId", + ], + }, + name: { + type: "string", + label: "Task Name", + description: "The name of the task to be created.", + }, + sectionId: { + propDefinition: [ + everhour, + "sectionId", + ({ projectId }) => ({ + projectId, + }), + ], + }, + tags: { + propDefinition: [ + everhour, + "tags", + ], + optional: true, + }, + position: { + type: "integer", + label: "Position", + description: "The position of the task", + optional: true, + }, + description: { + type: "string", + label: "Description", + description: "A description of the task", + optional: true, + }, + dueOn: { + type: "string", + label: "Due Date", + description: "The due date of the task. **Format: YYYY-MM-DD**", + optional: true, + }, + status: { + type: "string", + label: "Status", + description: "The status of the task", + options: STATUS_OPTIONS, + optional: true, + }, + }, + async run({ $ }) { + const response = await this.everhour.createTask({ + $, + projectId: this.projectId, + data: { + name: this.name, + section: this.sectionId, + tags: this.tags && parseObject(this.tags), + position: this.position, + description: this.description, + dueOn: this.dueOn, + status: this.status, + }, + }); + + $.export("$summary", `Successfully created task with ID: ${response.id}`); + return response; + }, +}; diff --git a/components/everhour/actions/start-timer/start-timer.mjs b/components/everhour/actions/start-timer/start-timer.mjs new file mode 100644 index 0000000000000..81d203782731c --- /dev/null +++ b/components/everhour/actions/start-timer/start-timer.mjs @@ -0,0 +1,52 @@ +import everhour from "../../everhour.app.mjs"; + +export default { + key: "everhour-start-timer", + name: "Start Timer", + description: "Begins a new timer for a task. [See the documentation](https://everhour.docs.apiary.io/#reference/0/timers/start-timer)", + version: "0.0.1", + type: "action", + props: { + everhour, + projectId: { + propDefinition: [ + everhour, + "projectId", + ], + }, + taskId: { + propDefinition: [ + everhour, + "taskId", + ({ projectId }) => ({ + projectId, + }), + ], + }, + userDate: { + type: "string", + label: "User Date", + description: "Date string to associate with the timer. Format as 'YYYY-MM-DD'", + optional: true, + }, + comment: { + type: "string", + label: "Comment", + description: "An optional comment to associate with the timer", + optional: true, + }, + }, + async run({ $ }) { + const response = await this.everhour.startTimer({ + $, + data: { + task: this.taskId, + userDate: this.userDate, + comment: this.comment, + }, + }); + + $.export("$summary", `Successfully started a timer for task ID: ${this.taskId}`); + return response; + }, +}; diff --git a/components/everhour/actions/stop-timer/stop-timer.mjs b/components/everhour/actions/stop-timer/stop-timer.mjs new file mode 100644 index 0000000000000..e533e1afe3c65 --- /dev/null +++ b/components/everhour/actions/stop-timer/stop-timer.mjs @@ -0,0 +1,17 @@ +import everhour from "../../everhour.app.mjs"; + +export default { + key: "everhour-stop-timer", + name: "Stop Timer", + description: "Halts the current running timer. [See the documentation](https://everhour.docs.apiary.io/#reference/timers/stop-timer)", + version: "0.0.1", + type: "action", + props: { + everhour, + }, + async run({ $ }) { + const response = await this.everhour.stopTimer(); + $.export("$summary", "Successfully stopped the timer"); + return response; + }, +}; diff --git a/components/everhour/common/constants.mjs b/components/everhour/common/constants.mjs new file mode 100644 index 0000000000000..5f6b8b3d178be --- /dev/null +++ b/components/everhour/common/constants.mjs @@ -0,0 +1,12 @@ +export const LIMIT = 100; + +export const STATUS_OPTIONS = [ + { + label: "Open", + value: "open", + }, + { + label: "Closed", + value: "closed", + }, +]; diff --git a/components/everhour/common/utils.mjs b/components/everhour/common/utils.mjs new file mode 100644 index 0000000000000..dcc9cc61f6f41 --- /dev/null +++ b/components/everhour/common/utils.mjs @@ -0,0 +1,24 @@ +export const parseObject = (obj) => { + if (!obj) return undefined; + + if (Array.isArray(obj)) { + return obj.map((item) => { + if (typeof item === "string") { + try { + return JSON.parse(item); + } catch (e) { + return item; + } + } + return item; + }); + } + if (typeof obj === "string") { + try { + return JSON.parse(obj); + } catch (e) { + return obj; + } + } + return obj; +}; diff --git a/components/everhour/everhour.app.mjs b/components/everhour/everhour.app.mjs index f17704bba377a..c87e3d01422f5 100644 --- a/components/everhour/everhour.app.mjs +++ b/components/everhour/everhour.app.mjs @@ -1,11 +1,176 @@ +import { axios } from "@pipedream/platform"; +import { LIMIT } from "./common/constants.mjs"; + export default { type: "app", app: "everhour", - propDefinitions: {}, + propDefinitions: { + projectId: { + type: "string", + label: "Project ID", + description: "The ID of the project", + async options({ page }) { + const projects = await this.listProjects({ + params: { + limit: LIMIT, + page: page + 1, + }, + }); + + return projects.map(({ + name: label, id: value, + }) => ({ + label, + value, + })); + }, + }, + sectionId: { + type: "string", + label: "Section ID", + description: "The section id of the task", + async options({ projectId }) { + const sections = await this.listSections({ + projectId, + }); + + return sections.map(({ + name: label, id: value, + }) => ({ + label, + value, + })); + }, + }, + tags: { + type: "string[]", + label: "Tag IDs", + description: "The tag ids of the task", + async options() { + const tags = await this.listTags(); + + return tags.map(({ + name: label, id: value, + }) => ({ + label, + value, + })); + }, + }, + labels: { + type: "string[]", + label: "Tags", + description: "An array of tags associated with the task", + async options({ projectId }) { + const sections = await this.listSections({ + projectId, + }); + + return sections.map(({ + name: label, id: value, + }) => ({ + label, + value, + })); + }, + }, + taskId: { + type: "string", + label: "Task ID", + description: "The ID of the task", + async options({ projectId }) { + const tasks = await this.getProjectTasks({ + projectId, + }); + return tasks.map(({ + name: label, id: value, + }) => ({ + label, + value, + })); + }, + }, + }, methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); + _baseUrl() { + return "https://api.everhour.com"; + }, + _headers() { + return { + "X-Api-Key": `${this.$auth.api_token}`, + }; + }, + _makeRequest({ + $ = this, path, ...opts + }) { + return axios($, { + url: this._baseUrl() + path, + headers: this._headers(), + ...opts, + }); + }, + listProjects(opts = {}) { + return this._makeRequest({ + path: "/projects", + ...opts, + }); + }, + listSections({ + projectId, opts, + }) { + return this._makeRequest({ + path: `/projects/${projectId}/sections`, + ...opts, + }); + }, + listTags() { + return this._makeRequest({ + path: "/tags", + }); + }, + getProjectTasks({ + projectId, ...opts + }) { + return this._makeRequest({ + path: `/projects/${projectId}/tasks`, + ...opts, + }); + }, + createTask({ + projectId, ...opts + }) { + return this._makeRequest({ + method: "POST", + path: `/projects/${projectId}/tasks`, + ...opts, + }); + }, + startTimer(opts = {}) { + return this._makeRequest({ + method: "POST", + path: "/timers", + ...opts, + }); + }, + stopTimer(opts = {}) { + return this._makeRequest({ + method: "DELETE", + path: "/timers/current", + ...opts, + }); + }, + createWebhook(opts = {}) { + return this._makeRequest({ + method: "POST", + path: "/hooks", + ...opts, + }); + }, + deleteWebhook(webhookId) { + return this._makeRequest({ + method: "DELETE", + path: `/hooks/${webhookId}`, + }); }, }, }; diff --git a/components/everhour/package.json b/components/everhour/package.json new file mode 100644 index 0000000000000..9093fbb1e87e5 --- /dev/null +++ b/components/everhour/package.json @@ -0,0 +1,18 @@ +{ + "name": "@pipedream/everhour", + "version": "0.1.0", + "description": "Pipedream Everhour Components", + "main": "everhour.app.mjs", + "keywords": [ + "pipedream", + "everhour" + ], + "homepage": "https://pipedream.com/apps/everhour", + "author": "Pipedream (https://pipedream.com/)", + "publishConfig": { + "access": "public" + }, + "dependencies": { + "@pipedream/platform": "^3.0.3" + } +} diff --git a/components/everhour/sources/common/base.mjs b/components/everhour/sources/common/base.mjs new file mode 100644 index 0000000000000..40eeb696c6bd3 --- /dev/null +++ b/components/everhour/sources/common/base.mjs @@ -0,0 +1,58 @@ +import everhour from "../../everhour.app.mjs"; + +export default { + props: { + everhour, + http: { + type: "$.interface.http", + customResponse: true, + }, + db: "$.service.db", + }, + methods: { + _getHookId() { + return this.db.get("hookId"); + }, + _setHookId(hookId) { + this.db.set("hookId", hookId); + }, + getExtraData() { + return {}; + }, + }, + hooks: { + async activate() { + const response = await this.everhour.createWebhook({ + data: { + targetUrl: this.http.endpoint, + events: this.getEventType(), + ...this.getExtraData(), + }, + }); + this._setHookId(response.id); + }, + async deactivate() { + const webhookId = this._getHookId(); + await this.everhour.deleteWebhook(webhookId); + }, + }, + async run({ + body, headers, + }) { + if (headers["x-hook-secret"]) { + return this.http.respond({ + status: 200, + headers: { + "X-Hook-Secret": headers["x-hook-secret"], + }, + }); + } + + const ts = Date.parse(new Date()); + this.$emit(body, { + id: `${body.resource}-${ts}`, + summary: this.getSummary(body), + ts: ts, + }); + }, +}; diff --git a/components/everhour/sources/new-client-instant/new-client-instant.mjs b/components/everhour/sources/new-client-instant/new-client-instant.mjs new file mode 100644 index 0000000000000..5ed199c80ee61 --- /dev/null +++ b/components/everhour/sources/new-client-instant/new-client-instant.mjs @@ -0,0 +1,24 @@ +import common from "../common/base.mjs"; +import sampleEmit from "./test-event.mjs"; + +export default { + ...common, + key: "everhour-new-client-instant", + name: "New Client (Instant)", + description: "Emit new event when a client is added.", + version: "0.0.1", + type: "source", + dedupe: "unique", + methods: { + ...common.methods, + getEventType() { + return [ + "api:client:created", + ]; + }, + getSummary(body) { + return `New Client: ${body.payload.data.name}`; + }, + }, + sampleEmit, +}; diff --git a/components/everhour/sources/new-client-instant/test-event.mjs b/components/everhour/sources/new-client-instant/test-event.mjs new file mode 100644 index 0000000000000..31104ffe31bf7 --- /dev/null +++ b/components/everhour/sources/new-client-instant/test-event.mjs @@ -0,0 +1,25 @@ +export default { + "event": "api:client:created", + "payload": { + "id": "9381500", + "data": { + "projects": [], + "id": 9381500, + "name": "Client Name", + "createdAt": "2024-10-22 14:25:29", + "lineItemMask": "%MEMBER% :: %PROJECT% :: for %PERIOD%", + "paymentDueDays": 0, + "reference": "", + "businessDetails": "", + "email": [ + "client@email.com" + ], + "invoicePublicNotes": "", + "excludedLabels": [], + "status": "active", + "enableResourcePlanner": false, + "favorite": false + } + }, + "createdAt": "2024-10-22 14:25:29" +} \ No newline at end of file diff --git a/components/everhour/sources/new-task-instant/new-task-instant.mjs b/components/everhour/sources/new-task-instant/new-task-instant.mjs new file mode 100644 index 0000000000000..385cc425bfb68 --- /dev/null +++ b/components/everhour/sources/new-task-instant/new-task-instant.mjs @@ -0,0 +1,38 @@ +import common from "../common/base.mjs"; +import sampleEmit from "./test-event.mjs"; + +export default { + ...common, + key: "everhour-new-task-instant", + name: "New Task Created (Instant)", + description: "Emit new event when a task is created.", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + ...common.props, + projectId: { + propDefinition: [ + common.props.everhour, + "projectId", + ], + }, + }, + methods: { + ...common.methods, + getExtraData() { + return { + project: this.projectId, + }; + }, + getEventType() { + return [ + "api:task:created", + ]; + }, + getSummary(body) { + return `New Task Created: ${body.payload.data.id}`; + }, + }, + sampleEmit, +}; diff --git a/components/everhour/sources/new-task-instant/test-event.mjs b/components/everhour/sources/new-task-instant/test-event.mjs new file mode 100644 index 0000000000000..c324485881340 --- /dev/null +++ b/components/everhour/sources/new-task-instant/test-event.mjs @@ -0,0 +1,24 @@ +export default { + "event": "api:task:created", + "payload": { + "id": "ev:188217209666811", + "data": { + "createdBy": 1362384, + "iteration": "Section Name", + "position": 5, + "projects": [ + "ev:188193235916605" + ], + "section": 1164091, + "comments": 0, + "completed": false, + "id": "ev:188217209666811", + "type": "task", + "name": "Task name", + "status": "open", + "labels": [], + "createdAt": "2024-10-18 14:01:36" + } + }, + "createdAt": "2024-10-18 14:01:36" +} \ No newline at end of file diff --git a/components/everhour/sources/task-time-updated-instant/task-time-updated-instant.mjs b/components/everhour/sources/task-time-updated-instant/task-time-updated-instant.mjs new file mode 100644 index 0000000000000..4bf9034b807b3 --- /dev/null +++ b/components/everhour/sources/task-time-updated-instant/task-time-updated-instant.mjs @@ -0,0 +1,38 @@ +import common from "../common/base.mjs"; +import sampleEmit from "./test-event.mjs"; + +export default { + ...common, + key: "everhour-task-time-updated-instant", + name: "New Task Time Updated (Instant)", + description: "Emit new event when a task's time spent is modified in Everhour.", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + ...common.props, + projectId: { + propDefinition: [ + common.props.everhour, + "projectId", + ], + }, + }, + methods: { + ...common.methods, + getExtraData() { + return { + project: this.projectId, + }; + }, + getEventType() { + return [ + "api:time:updated", + ]; + }, + getSummary(body) { + return `Task Time Updated: ${body.payload.data.id}`; + }, + }, + sampleEmit, +}; diff --git a/components/everhour/sources/task-time-updated-instant/test-event.mjs b/components/everhour/sources/task-time-updated-instant/test-event.mjs new file mode 100644 index 0000000000000..5d4caa2116255 --- /dev/null +++ b/components/everhour/sources/task-time-updated-instant/test-event.mjs @@ -0,0 +1,55 @@ +export default { + "event": "api:time:updated", + "payload": { + "id": "ev:188193235916608", + "data": { + "user": 1362384, + "history": [ + { + "id": 370578249, + "time": 60, + "previousTime": 0, + "action": "TIMER", + "source": "internal", + "createdAt": "2024-10-18 13:58:23", + "createdBy": 1362384 + } + ], + "lockReasons": [], + "cost": 42, + "isLocked": false, + "manualTime": 0, + "id": 214588860, + "date": "2024-10-18", + "time": 60, + "timerTime": 60, + "pastDateTime": 0, + "task": { + "createdBy": 1362384, + "position": 4, + "projects": [ + "ev:188193235916605" + ], + "section": 1163621, + "comments": 0, + "completed": false, + "id": "ev:188193235916608", + "type": "task", + "name": "Project Management", + "status": "open", + "labels": [], + "createdAt": "2024-10-15 19:25:59", + "time": { + "total": 60, + "users": { + "1362384": 60 + }, + "timerTime": 60 + } + }, + "createdAt": "2024-10-18 13:58:23", + "costRate": 2500 + } + }, + "createdAt": "2024-10-18 13:59:20" +} \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c5ccc4476dbf1..1de23b9ecd59a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3202,6 +3202,12 @@ importers: components/eventee: specifiers: {} + components/everhour: + specifiers: + '@pipedream/platform': ^3.0.3 + dependencies: + '@pipedream/platform': 3.0.3 + components/eversign: specifiers: '@pipedream/platform': ^1.1.1 From 9a16df6839d4ea8eeceb0420bbddd95b4b4a6785 Mon Sep 17 00:00:00 2001 From: Lucas Caresia Date: Mon, 28 Oct 2024 13:07:24 -0300 Subject: [PATCH 16/18] Added actions (#14426) --- .../create-datasheet/create-datasheet.mjs | 50 ++++++++ .../actions/create-field/create-field.mjs | 44 +++++++ .../actions/delete-field/delete-field.mjs | 34 +++++ components/aitable_ai/aitable_ai.app.mjs | 120 +++++++++++++++++- components/aitable_ai/common/constants.mjs | 11 ++ components/aitable_ai/package.json | 7 +- pnpm-lock.yaml | 5 +- 7 files changed, 263 insertions(+), 8 deletions(-) create mode 100644 components/aitable_ai/actions/create-datasheet/create-datasheet.mjs create mode 100644 components/aitable_ai/actions/create-field/create-field.mjs create mode 100644 components/aitable_ai/actions/delete-field/delete-field.mjs create mode 100644 components/aitable_ai/common/constants.mjs diff --git a/components/aitable_ai/actions/create-datasheet/create-datasheet.mjs b/components/aitable_ai/actions/create-datasheet/create-datasheet.mjs new file mode 100644 index 0000000000000..f89e7136903d1 --- /dev/null +++ b/components/aitable_ai/actions/create-datasheet/create-datasheet.mjs @@ -0,0 +1,50 @@ +import app from "../../aitable_ai.app.mjs"; + +export default { + key: "aitable_ai-create-datasheet", + name: "Create Datasheet", + description: "Create a datasheet in the specified space. [See the documentation](https://developers.aitable.ai/api/reference#tag/Datasheet/operation/create-datasheets)", + version: "0.0.1", + type: "action", + props: { + app, + spaceId: { + propDefinition: [ + app, + "spaceId", + ], + }, + name: { + propDefinition: [ + app, + "name", + ], + }, + description: { + propDefinition: [ + app, + "description", + ], + }, + folderId: { + propDefinition: [ + app, + "folderId", + ], + }, + }, + + async run({ $ }) { + const response = await this.app.createDatasheet({ + $, + spaceId: this.spaceId, + data: { + name: this.name, + description: this.description, + folderId: this.folderId, + }, + }); + $.export("$summary", `Successfully created Datasheet with ID '${response.data.id}'`); + return response; + }, +}; diff --git a/components/aitable_ai/actions/create-field/create-field.mjs b/components/aitable_ai/actions/create-field/create-field.mjs new file mode 100644 index 0000000000000..eadc1e7e58b40 --- /dev/null +++ b/components/aitable_ai/actions/create-field/create-field.mjs @@ -0,0 +1,44 @@ +import app from "../../aitable_ai.app.mjs"; + +export default { + key: "aitable_ai-create-field", + name: "Create Field", + description: "Create a new field in the specified datasheet. [See the documentation](https://developers.aitable.ai/api/reference#tag/Field/operation/create-fields)", + version: "0.0.1", + type: "action", + props: { + app, + spaceId: { + propDefinition: [ + app, + "spaceId", + ], + }, + type: { + propDefinition: [ + app, + "type", + ], + }, + name: { + propDefinition: [ + app, + "name", + ], + description: "Name of the Field", + }, + }, + + async run({ $ }) { + const response = await this.app.createField({ + $, + spaceId: this.spaceId, + data: { + type: this.type, + name: this.name, + }, + }); + $.export("$summary", `Successfully sent request to create field. Result: '${response.message}'`); + return response; + }, +}; diff --git a/components/aitable_ai/actions/delete-field/delete-field.mjs b/components/aitable_ai/actions/delete-field/delete-field.mjs new file mode 100644 index 0000000000000..04153ea8d7912 --- /dev/null +++ b/components/aitable_ai/actions/delete-field/delete-field.mjs @@ -0,0 +1,34 @@ +import app from "../../aitable_ai.app.mjs"; + +export default { + key: "aitable_ai-delete-field", + name: "Delete Field", + description: "Delete a field in the specified datasheet. [See the documentation](https://developers.aitable.ai/api/reference/#tag/Field/operation/delete-fields)", + version: "0.0.1", + type: "action", + props: { + app, + spaceId: { + propDefinition: [ + app, + "spaceId", + ], + }, + fieldId: { + propDefinition: [ + app, + "fieldId", + ], + }, + }, + + async run({ $ }) { + const response = await this.app.deleteField({ + $, + spaceId: this.spaceId, + fieldId: this.fieldId, + }); + $.export("$summary", `Successfully deleted the field with ID '${this.fieldId}'`); + return response; + }, +}; diff --git a/components/aitable_ai/aitable_ai.app.mjs b/components/aitable_ai/aitable_ai.app.mjs index 13a85025616ce..598c292151ff3 100644 --- a/components/aitable_ai/aitable_ai.app.mjs +++ b/components/aitable_ai/aitable_ai.app.mjs @@ -1,11 +1,121 @@ +import { axios } from "@pipedream/platform"; +import constants from "./common/constants.mjs"; + export default { type: "app", app: "aitable_ai", - propDefinitions: {}, + propDefinitions: { + spaceId: { + type: "string", + label: "Space ID", + description: "ID of the Space", + async options() { + const response = await this.getSpaces({}); + const spaceIds = response.data.spaces; + return spaceIds.map(({ + id, name, + }) => ({ + value: id, + label: name, + })); + }, + }, + fieldId: { + type: "string", + label: "Field ID", + description: "ID of the Field", + async options() { + const response = await this.getFields({}); + const fieldIds = response.data.fields; + return fieldIds.map(({ + id, name, + }) => ({ + value: id, + label: name, + })); + }, + }, + name: { + type: "string", + label: "Name", + description: "Name of the Datasheet", + }, + description: { + type: "string", + label: "Description", + description: "Description of the Datasheet", + }, + folderId: { + type: "string", + label: "Folder ID", + description: "The Folder ID is located in the `URL` when the folder is selected on the `Workbench page`, i.e.: if the URL is `https://aitable.ai/workbench/123456`, the `Folder ID` is 123456", + optional: true, + }, + type: { + type: "string", + label: "Type", + description: "Type of the Field", + options: constants.FIELD_TYPES, + }, + }, methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); + _baseUrl() { + return "https://aitable.ai/fusion/v1"; + }, + async _makeRequest(opts = {}) { + const { + $ = this, + path, + headers, + ...otherOpts + } = opts; + return axios($, { + ...otherOpts, + url: this._baseUrl() + path, + headers: { + ...headers, + Authorization: `Bearer ${this.$auth.api_token}`, + }, + }); + }, + async createDatasheet({ + spaceId, ...args + }) { + return this._makeRequest({ + path: `/spaces/${spaceId}/datasheets`, + method: "post", + ...args, + }); + }, + async createField({ + spaceId, ...args + }) { + return this._makeRequest({ + path: `/spaces/${spaceId}/datasheets/${this.$auth.datasheet_id}/fields`, + method: "post", + ...args, + }); + }, + async deleteField({ + spaceId, fieldId, ...args + }) { + return this._makeRequest({ + path: `/spaces/${spaceId}/datasheets/${this.$auth.datasheet_id}/fields/${fieldId}`, + method: "delete", + ...args, + }); + }, + async getSpaces(args = {}) { + return this._makeRequest({ + path: "/spaces", + ...args, + }); + }, + async getFields(args = {}) { + return this._makeRequest({ + path: `/datasheets/${this.$auth.datasheet_id}/fields`, + ...args, + }); }, }, -}; \ No newline at end of file +}; diff --git a/components/aitable_ai/common/constants.mjs b/components/aitable_ai/common/constants.mjs new file mode 100644 index 0000000000000..c4e7a5d13877e --- /dev/null +++ b/components/aitable_ai/common/constants.mjs @@ -0,0 +1,11 @@ +export default { + FIELD_TYPES: [ + "Text", + "URL", + "Phone", + "Email", + "WorkDoc", + "AutoNumber", + "CreatedBy", + ], +}; diff --git a/components/aitable_ai/package.json b/components/aitable_ai/package.json index cce6d9e1088a7..78fde63930a62 100644 --- a/components/aitable_ai/package.json +++ b/components/aitable_ai/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/aitable_ai", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream AITable.ai Components", "main": "aitable_ai.app.mjs", "keywords": [ @@ -11,5 +11,8 @@ "author": "Pipedream (https://pipedream.com/)", "publishConfig": { "access": "public" + }, + "dependencies": { + "@pipedream/platform": "^3.0.3" } -} \ No newline at end of file +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1de23b9ecd59a..6cff508484e5f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -434,7 +434,10 @@ importers: moment: 2.30.1 components/aitable_ai: - specifiers: {} + specifiers: + '@pipedream/platform': ^3.0.3 + dependencies: + '@pipedream/platform': 3.0.3 components/aivoov: specifiers: From 855a26657ac1a91cc7845be1eceb78142b794445 Mon Sep 17 00:00:00 2001 From: Lucas Caresia Date: Mon, 28 Oct 2024 13:07:36 -0300 Subject: [PATCH 17/18] [Components] liveswitch #13859 (#14427) * Added actions * Fixing action name --- .../actions/create-contact/create-contact.mjs | 52 +++++++++ .../create-conversation.mjs | 38 +++++++ .../actions/update-contact/update-contact.mjs | 59 ++++++++++ components/liveswitch/liveswitch.app.mjs | 103 ++++++++++++++++- components/liveswitch/package.json | 7 +- pnpm-lock.yaml | 107 +++++++++--------- 6 files changed, 307 insertions(+), 59 deletions(-) create mode 100644 components/liveswitch/actions/create-contact/create-contact.mjs create mode 100644 components/liveswitch/actions/create-conversation/create-conversation.mjs create mode 100644 components/liveswitch/actions/update-contact/update-contact.mjs diff --git a/components/liveswitch/actions/create-contact/create-contact.mjs b/components/liveswitch/actions/create-contact/create-contact.mjs new file mode 100644 index 0000000000000..b86efd667c144 --- /dev/null +++ b/components/liveswitch/actions/create-contact/create-contact.mjs @@ -0,0 +1,52 @@ +import app from "../../liveswitch.app.mjs"; + +export default { + key: "liveswitch-create-contact", + name: "Create Contact", + description: "Create a contact in LiveSwitch [See the documentation](https://developer.liveswitch.com/reference/post_v1-contacts)", + version: "0.0.1", + type: "action", + props: { + app, + phone: { + propDefinition: [ + app, + "phone", + ], + }, + firstName: { + propDefinition: [ + app, + "firstName", + ], + }, + lastName: { + propDefinition: [ + app, + "lastName", + ], + }, + email: { + propDefinition: [ + app, + "email", + ], + }, + }, + + async run({ $ }) { + const response = await this.app.createContact({ + $, + data: { + phone: this.phone, + firstName: this.firstName, + lastName: this.lastName, + email: this.email, + }, + }); + + $.export("$summary", `Successfully created Contact with ID: ${response.id}`); + + return response; + }, +}; diff --git a/components/liveswitch/actions/create-conversation/create-conversation.mjs b/components/liveswitch/actions/create-conversation/create-conversation.mjs new file mode 100644 index 0000000000000..da0428947a4e7 --- /dev/null +++ b/components/liveswitch/actions/create-conversation/create-conversation.mjs @@ -0,0 +1,38 @@ +import app from "../../liveswitch.app.mjs"; + +export default { + key: "liveswitch-create-conversation", + name: "Create Conversation", + description: "Create a conversation in LiveSwitch [See the documentation](https://developer.liveswitch.com/reference/post_v1-conversations)", + version: "0.0.1", + type: "action", + props: { + app, + contactId: { + propDefinition: [ + app, + "contactId", + ], + }, + message: { + propDefinition: [ + app, + "message", + ], + }, + }, + + async run({ $ }) { + const response = await this.app.createConversation({ + $, + data: { + contactId: this.contactId, + message: this.message, + }, + }); + + $.export("$summary", `Successfully created Conversation with ID: ${response.id}`); + + return response; + }, +}; diff --git a/components/liveswitch/actions/update-contact/update-contact.mjs b/components/liveswitch/actions/update-contact/update-contact.mjs new file mode 100644 index 0000000000000..593488339b62b --- /dev/null +++ b/components/liveswitch/actions/update-contact/update-contact.mjs @@ -0,0 +1,59 @@ +import app from "../../liveswitch.app.mjs"; + +export default { + key: "liveswitch-update-contact", + name: "Update Contact", + description: "Update a contact in LiveSwitch [See the documentation](https://developer.liveswitch.com/reference/post_v1-contacts)", + version: "0.0.1", + type: "action", + props: { + app, + contactId: { + propDefinition: [ + app, + "contactId", + ], + }, + phone: { + propDefinition: [ + app, + "phone", + ], + }, + firstName: { + propDefinition: [ + app, + "firstName", + ], + }, + lastName: { + propDefinition: [ + app, + "lastName", + ], + }, + email: { + propDefinition: [ + app, + "email", + ], + }, + }, + + async run({ $ }) { + const response = await this.app.updateContact({ + $, + contactId: this.contactId, + data: { + phone: this.phone, + firstName: this.firstName, + lastName: this.lastName, + email: this.email, + }, + }); + + $.export("$summary", `Successfully updated Contact with ID: ${this.contactId}`); + + return response; + }, +}; diff --git a/components/liveswitch/liveswitch.app.mjs b/components/liveswitch/liveswitch.app.mjs index 12b2910a584db..f1dd1ce333ed6 100644 --- a/components/liveswitch/liveswitch.app.mjs +++ b/components/liveswitch/liveswitch.app.mjs @@ -1,11 +1,104 @@ +import { axios } from "@pipedream/platform"; + export default { type: "app", app: "liveswitch", - propDefinitions: {}, + propDefinitions: { + firstName: { + type: "string", + label: "First Name", + description: "Contact's first name", + optional: true, + }, + lastName: { + type: "string", + label: "Last Name", + description: "Contact's last name", + optional: true, + }, + email: { + type: "string", + label: "Email", + description: "Contact's email", + optional: true, + }, + phone: { + type: "string", + label: "Phone", + description: "Contact's phone number, i.e.: `+1 407-982-1211`", + }, + message: { + type: "string", + label: "Message", + description: "The contents of the text message the user will receive", + }, + contactId: { + type: "string", + label: "Contact ID", + description: "Contact's ID", + async options() { + const response = await this.getContacts(); + return response.map(({ + id, firstName, lastName, + }) => ({ + value: id, + label: `${firstName} ${lastName}`, + })); + }, + }, + }, methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); + _baseUrl() { + return "https://public-api.production.liveswitch.com/v1"; + }, + async _makeRequest(opts = {}) { + const { + $ = this, + path, + headers, + ...otherOpts + } = opts; + return axios($, { + ...otherOpts, + url: this._baseUrl() + path, + headers: { + ...headers, + Authorization: `Bearer ${this.$auth.oauth_access_token}`, + }, + }); + }, + async createContact(args = {}) { + return this._makeRequest({ + path: "/contacts", + method: "post", + ...args, + }); + }, + async updateContact({ + contactId, ...args + }) { + return this._makeRequest({ + path: `/contacts/${contactId}`, + method: "put", + ...args, + }); + }, + async createConversation(args = {}) { + return this._makeRequest({ + path: "/conversations", + method: "post", + ...args, + }); + }, + async getContacts(args = {}) { + return this._makeRequest({ + path: "/contacts", + params: { + page: 1, + pageSize: 100, + }, + ...args, + }); }, }, -}; \ No newline at end of file +}; diff --git a/components/liveswitch/package.json b/components/liveswitch/package.json index 46f842a73af2f..ec7cde46285d3 100644 --- a/components/liveswitch/package.json +++ b/components/liveswitch/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/liveswitch", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream LiveSwitch Components", "main": "liveswitch.app.mjs", "keywords": [ @@ -11,5 +11,8 @@ "author": "Pipedream (https://pipedream.com/)", "publishConfig": { "access": "public" + }, + "dependencies": { + "@pipedream/platform": "^3.0.3" } -} \ No newline at end of file +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6cff508484e5f..f1c232d666f6c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5669,7 +5669,10 @@ importers: '@pipedream/platform': 1.5.1 components/liveswitch: - specifiers: {} + specifiers: + '@pipedream/platform': ^3.0.3 + dependencies: + '@pipedream/platform': 3.0.3 components/llama_ai: specifiers: @@ -13201,6 +13204,55 @@ packages: - aws-crt dev: false + /@aws-sdk/client-sso-oidc/3.600.0_tdq3komn4zwyd65w7klbptsu34: + resolution: {integrity: sha512-7+I8RWURGfzvChyNQSyj5/tKrqRbzRl7H+BnTOf/4Vsw1nFOi5ROhlhD4X/Y0QCTacxnaoNcIrqnY7uGGvVRzw==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/client-sts': 3.600.0 + '@aws-sdk/core': 3.598.0 + '@aws-sdk/credential-provider-node': 3.600.0_f7n47caigsrjd2lr2szmwfuee4 + '@aws-sdk/middleware-host-header': 3.598.0 + '@aws-sdk/middleware-logger': 3.598.0 + '@aws-sdk/middleware-recursion-detection': 3.598.0 + '@aws-sdk/middleware-user-agent': 3.598.0 + '@aws-sdk/region-config-resolver': 3.598.0 + '@aws-sdk/types': 3.598.0 + '@aws-sdk/util-endpoints': 3.598.0 + '@aws-sdk/util-user-agent-browser': 3.598.0 + '@aws-sdk/util-user-agent-node': 3.598.0 + '@smithy/config-resolver': 3.0.3 + '@smithy/core': 2.2.3 + '@smithy/fetch-http-handler': 3.2.1 + '@smithy/hash-node': 3.0.2 + '@smithy/invalid-dependency': 3.0.2 + '@smithy/middleware-content-length': 3.0.2 + '@smithy/middleware-endpoint': 3.0.4 + '@smithy/middleware-retry': 3.0.6 + '@smithy/middleware-serde': 3.0.3 + '@smithy/middleware-stack': 3.0.3 + '@smithy/node-config-provider': 3.1.3 + '@smithy/node-http-handler': 3.1.2 + '@smithy/protocol-http': 4.0.3 + '@smithy/smithy-client': 3.1.6 + '@smithy/types': 3.3.0 + '@smithy/url-parser': 3.0.3 + '@smithy/util-base64': 3.0.0 + '@smithy/util-body-length-browser': 3.0.0 + '@smithy/util-body-length-node': 3.0.0 + '@smithy/util-defaults-mode-browser': 3.0.6 + '@smithy/util-defaults-mode-node': 3.0.6 + '@smithy/util-endpoints': 2.0.3 + '@smithy/util-middleware': 3.0.3 + '@smithy/util-retry': 3.0.2 + '@smithy/util-utf8': 3.0.0 + tslib: 2.6.3 + transitivePeerDependencies: + - '@aws-sdk/client-sts' + - aws-crt + dev: false + /@aws-sdk/client-sso/3.423.0: resolution: {integrity: sha512-znIufHkwhCIePgaYciIs3x/+BpzR57CZzbCKHR9+oOvGyufEPPpUT5bFLvbwTgfiVkTjuk6sG/ES3U5Bc+xtrA==} engines: {node: '>=14.0.0'} @@ -13436,7 +13488,7 @@ packages: dependencies: '@aws-crypto/sha256-browser': 5.2.0 '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/client-sso-oidc': 3.600.0 + '@aws-sdk/client-sso-oidc': 3.600.0_tdq3komn4zwyd65w7klbptsu34 '@aws-sdk/core': 3.598.0 '@aws-sdk/credential-provider-node': 3.600.0_f7n47caigsrjd2lr2szmwfuee4 '@aws-sdk/middleware-host-header': 3.598.0 @@ -13478,55 +13530,6 @@ packages: - aws-crt dev: false - /@aws-sdk/client-sts/3.600.0_dseaa2p5u2yk67qiepewcq3hkq: - resolution: {integrity: sha512-KQG97B7LvTtTiGmjlrG1LRAY8wUvCQzrmZVV5bjrJ/1oXAU7DITYwVbSJeX9NWg6hDuSk0VE3MFwIXS2SvfLIA==} - engines: {node: '>=16.0.0'} - dependencies: - '@aws-crypto/sha256-browser': 5.2.0 - '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/client-sso-oidc': 3.600.0 - '@aws-sdk/core': 3.598.0 - '@aws-sdk/credential-provider-node': 3.600.0_f7n47caigsrjd2lr2szmwfuee4 - '@aws-sdk/middleware-host-header': 3.598.0 - '@aws-sdk/middleware-logger': 3.598.0 - '@aws-sdk/middleware-recursion-detection': 3.598.0 - '@aws-sdk/middleware-user-agent': 3.598.0 - '@aws-sdk/region-config-resolver': 3.598.0 - '@aws-sdk/types': 3.598.0 - '@aws-sdk/util-endpoints': 3.598.0 - '@aws-sdk/util-user-agent-browser': 3.598.0 - '@aws-sdk/util-user-agent-node': 3.598.0 - '@smithy/config-resolver': 3.0.3 - '@smithy/core': 2.2.3 - '@smithy/fetch-http-handler': 3.2.1 - '@smithy/hash-node': 3.0.2 - '@smithy/invalid-dependency': 3.0.2 - '@smithy/middleware-content-length': 3.0.2 - '@smithy/middleware-endpoint': 3.0.4 - '@smithy/middleware-retry': 3.0.6 - '@smithy/middleware-serde': 3.0.3 - '@smithy/middleware-stack': 3.0.3 - '@smithy/node-config-provider': 3.1.3 - '@smithy/node-http-handler': 3.1.2 - '@smithy/protocol-http': 4.0.3 - '@smithy/smithy-client': 3.1.6 - '@smithy/types': 3.3.0 - '@smithy/url-parser': 3.0.3 - '@smithy/util-base64': 3.0.0 - '@smithy/util-body-length-browser': 3.0.0 - '@smithy/util-body-length-node': 3.0.0 - '@smithy/util-defaults-mode-browser': 3.0.6 - '@smithy/util-defaults-mode-node': 3.0.6 - '@smithy/util-endpoints': 2.0.3 - '@smithy/util-middleware': 3.0.3 - '@smithy/util-retry': 3.0.2 - '@smithy/util-utf8': 3.0.0 - tslib: 2.6.3 - transitivePeerDependencies: - - '@aws-sdk/client-sso-oidc' - - aws-crt - dev: false - /@aws-sdk/core/3.556.0: resolution: {integrity: sha512-vJaSaHw2kPQlo11j/Rzuz0gk1tEaKdz+2ser0f0qZ5vwFlANjt08m/frU17ctnVKC1s58bxpctO/1P894fHLrA==} engines: {node: '>=14.0.0'} @@ -17825,7 +17828,7 @@ packages: '@aws-sdk/client-sns': 3.423.0 '@aws-sdk/client-sqs': 3.423.0 '@aws-sdk/client-ssm': 3.423.0 - '@aws-sdk/client-sts': 3.600.0_dseaa2p5u2yk67qiepewcq3hkq + '@aws-sdk/client-sts': 3.600.0 '@aws-sdk/s3-request-presigner': 3.609.0 '@pipedream/helper_functions': 0.3.12 '@pipedream/platform': 1.6.6 From d78b009ad167fe69c0919307f03a3339f008942c Mon Sep 17 00:00:00 2001 From: Michael Lim Date: Tue, 29 Oct 2024 13:37:49 -0700 Subject: [PATCH 18/18] Adding new validated list of evals. --- .../evals/component_retrieval/eval-10-29.json | 529 ++++++++++++++++++ 1 file changed, 529 insertions(+) create mode 100644 packages/evals/component_retrieval/eval-10-29.json diff --git a/packages/evals/component_retrieval/eval-10-29.json b/packages/evals/component_retrieval/eval-10-29.json new file mode 100644 index 0000000000000..7517b96cb745c --- /dev/null +++ b/packages/evals/component_retrieval/eval-10-29.json @@ -0,0 +1,529 @@ +{ + "evaluationTests": [ + { + "query": "When a lead is created in Salesforce, create a task in Asana and send a message in Slack", + "sources": [ + "salesforce_rest_api-new-record-instant", + "salesforce_rest_api-new-record" + ], + "actions": [ + "asana-create-task", + "slack-send-message", + "slack-send-message-advanced" + ], + "persona": "complex-workflow" + }, + { + "query": "add new hubspot contacts to mailchimp", + "sources": [ + "hubspot-new-contact", + "hubspot-new-contact-in-list" + ], + "actions": [ + "mailchimp-add-or-update-subscriber" + ], + "persona": "succinct" + }, + { + "query": "how do I get a phone number", + "sources": [], + "actions": [], + "persona": "invalid-query" + }, + { + "query": "when new emails arrive in Gmail, use OpenAI to summarize them and post to Slack", + "sources": [ + "gmail-new-email-received", + "gmail-new-email-matching-search" + ], + "actions": [ + "openai-chat", + "slack-send-message", + "slack-send-message-advanced" + ], + "persona": "complex-workflow" + }, + { + "query": "watch airtable for new records", + "sources": [ + "airtable_oauth-new-records", + "airtable_oauth-new-records-in-view" + ], + "actions": [], + "persona": "succinct" + }, + { + "query": "create jira ticket", + "sources": [], + "actions": [ + "jira-create-issue" + ], + "persona": "succinct" + }, + { + "query": "When new candidate appears in Lever, create user in JumpCloud", + "sources": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "Add new LeanIX components to our Architecture Handbook in Confluence", + "sources": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "When a deal closes in Close.com, create project in TeamGantt", + "sources": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "When new Zendesk tickets arrive, analyze sentiment with OpenAI and prioritize in Linear", + "sources": [ + "zendesk-new-ticket" + ], + "actions": [ + "openai-chat", + "linear-create-issue" + ], + "persona": "complex-workflow" + }, + { + "query": "when new issues are created in GitHub, analyze priority with chatgpt and create corresponding Trello cards", + "sources": [ + "github-new-or-updated-issue" + ], + "actions": [ + "openai-chat", + "trello-create-card" + ], + "persona": "complex-workflow" + }, + { + "query": "When someone books through Calendly, add event to Google Calendar", + "sources": [ + "calendly_v2-new-event-scheduled" + ], + "actions": [ + "google_calendar-create-event" + ], + "persona": "similar-app" + }, + { + "query": "When files are uploaded to Box, create Google Doc", + "sources": [ + "box-new-file" + ], + "actions": [ + "google_docs-create-document" + ], + "persona": "similar-app" + }, + { + "query": "When Box gets new file, upload to OneDrive and Google Drive", + "sources": [ + "box-new-file" + ], + "actions": [ + "microsoft_onedrive-upload-file", + "google_drive-upload-file" + ], + "persona": "similar-app" + }, + { + "query": "When files are added to OneDrive folder, save to Google Drive", + "sources": [ + "microsoft_onedrive-new-file-in-folder" + ], + "actions": [ + "google_drive-upload-file" + ], + "persona": "similar-app" + }, + { + "query": "create new task in Google Tasks when Todoist task is created", + "sources": [ + "todoist-new-task" + ], + "actions": [ + "google_tasks-create-task" + ], + "persona": "similar-app" + }, + { + "query": "create event in Zoho Calendar when Google Calendar event is updated", + "sources": [ + "google_calendar-new-or-updated-event-instant" + ], + "actions": [ + "zoho_calendar-create-or-update-event" + ], + "persona": "similar-app" + }, + { + "query": "send email through Gmail when Zoho Mail receives new message", + "sources": [ + "zoho_mail-new-email" + ], + "actions": [ + "gmail-send-email" + ], + "persona": "similar-app" + }, + { + "query": "When files are uploaded to Dropbox, copy them to Google Drive", + "sources": [ + "dropbox-new-file" + ], + "actions": [ + "google_drive-upload-file" + ], + "persona": "complex-workflow" + }, + { + "query": "upload file to Microsoft OneDrive when Google Drive gets new file", + "sources": [ + "google_drive-new-files-instant" + ], + "actions": [ + "microsoft_onedrive-upload-file" + ], + "persona": "similar-app" + }, + { + "query": "I need new RSS items to be posted to Discord", + "sources": [ + "rss-new-item-in-feed", + "rss-new-item-from-multiple-feeds" + ], + "actions": [ + "discord-send-message", + "discord_bot-send-message", + "discord_bot-send-message-to-forum-post" + ], + "persona": "verbose" + }, + { + "query": "When Box gets new file, copy to Dropbox", + "sources": [ + "box-new-file" + ], + "actions": [ + "dropbox-upload-file" + ], + "persona": "similar-app" + }, + { + "query": "Create a new project in Wrike when Typeform is submitted", + "sources": [ + "typeform-new-submission" + ], + "actions": [], + "persona": "no-component" + }, + { + "query": "When Gcal events are updated, post to Slack", + "sources": [ + "google_calendar-new-or-updated-event-instant" + ], + "actions": [ + "slack-send-message" + ], + "persona": "ambiguous" + }, + { + "query": "When customer churns in Chargebee, remove access in Auth0 and revoke Workato license", + "sources": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "When a new invoice is created in NetSuite, sync to Sage Intacct", + "sources": [], + "actions": [], + "persona": "no-component" + }, + { + "query": "When new document is uploaded to Box, scan with Nightfall", + "sources": [ + "box-new-file" + ], + "actions": [], + "persona": "no-component" + }, + { + "query": "When files change in shared Google Drive, notify in Slack", + "sources": [ + "google_drive-changes-to-specific-files-shared-drive" + ], + "actions": [ + "slack-send-message", + "slack-send-message-advanced" + ], + "persona": "complex-workflow" + }, + { + "query": "When worksheets are added to Google Sheets, create Notion pages", + "sources": [ + "google_sheets-new-worksheet" + ], + "actions": [ + "notion-create-page", + "notion-create-page-from-database" + ], + "persona": "complex-workflow" + }, + { + "query": "When documents are uploaded to Google Drive, use GPT to generate summaries in Notion", + "sources": [ + "google_drive-new-files-instant" + ], + "actions": [ + "openai-chat", + "notion-create-page", + "notion-create-page-from-database" + ], + "persona": "complex-workflow" + }, + { + "query": "When RingCentral records a call, transcribe with OpenAI", + "sources": [ + "ringcentral-new-call-recording" + ], + "actions": [ + "openai-create-transcription" + ], + "persona": "complex-workflow" + }, + { + "query": "When new article appears in Inoreader folder, save to Notion", + "sources": [ + "inoreader-new-article-in-folder" + ], + "actions": [ + "notion-create-page", + "notion-create-page-from-database" + ], + "persona": "complex-workflow" + }, + { + "query": "When deals change stage in HubSpot, update records in Salesforce", + "sources": [ + "hubspot-deal-updated" + ], + "actions": [ + "salesforce_rest_api-update-record" + ], + "persona": "complex-workflow" + }, + { + "query": "When someone stars our GitHub repo, send an email in Gmail", + "sources": [ + "github-new-stars", + "github-new-star-by-user" + ], + "actions": [ + "openai-chat", + "gmail-send-email" + ], + "persona": "complex-workflow" + }, + { + "query": "When a new form is answered in Google Forms, add to Airtable", + "sources": [ + "google_forms-new-form-answer" + ], + "actions": [ + "airtable_oauth-create-single-record" + ], + "persona": "complex-workflow" + }, + { + "query": "I need all Typeform submissions to be analyzed by ChatGPT and then posted to both Slack and Notion", + "sources": [ + "typeform-new-submission" + ], + "actions": [ + "openai-chat", + "slack-send-message", + "slack-send-message-advanced", + "notion-create-page", + "notion-create-page-from-database" + ], + "persona": "verbose" + }, + { + "query": "When a row is added to Google Sheets in a shared drive, create a Linear ticket", + "sources": [ + "google_sheets-new-row-added" + ], + "actions": [ + "linear-create-issue" + ], + "persona": "complex-workflow" + }, + { + "query": "When a message is posted to Discord, analyze with OpenAI", + "sources": [ + "discord-new-message", + "discord_bot-new-message-in-channel" + ], + "actions": [ + "openai-chat" + ], + "persona": "complex-workflow" + }, + { + "query": "When a new email is received in Gmail, create a task in Google Tasks, and append the email details to a Google Sheet", + "sources": [ + "gmail-new-email-received", + "gmail-new-email-matching-search" + ], + "actions": [ + "google_tasks-create-task", + "google_sheets-add-single-row" + ], + "persona": "complex-workflow" + }, + { + "query": "When a new event is added to Google Calendar, update a page in Notion, and notify the IT team in Slack", + "sources": [ + "google_calendar-new-event" + ], + "actions": [ + "notion-update-page", + "slack-send-message", + "slack-send-message-advanced" + ], + "persona": "complex-workflow" + }, + { + "query": "When a new order is created in Shopify, create a meeting in Zoom, and log the order details in Google Sheets", + "sources": [ + "shopify_developer_app-new-order-created" + ], + "actions": [ + "zoom-create-meeting", + "zoom_admin-create-meeting", + "google_sheets-add-single-row" + ], + "persona": "complex-workflow" + }, + { + "query": "when a new lead submits our Typeform, analyze their responses with OpenAI for sentiment and urgency, then alert the sales team in Slack", + "sources": [ + "typeform-new-submission" + ], + "actions": [ + "openai-chat", + "slack-send-message", + "slack-send-message-advanced" + ], + "persona": "complex-workflow" + }, + { + "query": "whenever someone mentions our company on Twitter, send the tweet to OpenAI for analysis and automatically create a Notion page to track our response", + "sources": [ + "twitter-new-mention-received-by-user" + ], + "actions": [ + "openai-chat", + "notion-create-page", + "notion-create-page-from-database" + ], + "persona": "complex-workflow" + }, + { + "query": "when new form submissions come in through Jotform, add them to our Google Sheet and notify the team in Discord", + "sources": [ + "jotform-new-submission" + ], + "actions": [ + "google_sheets-add-single-row", + "discord-send-message", + "discord_bot-send-message" + ], + "persona": "complex-workflow" + }, + { + "query": "when a new Google Calendar event is created, update our team availability spreadsheet and send a Telegram notification", + "sources": [ + "google_calendar-new-event" + ], + "actions": [ + "google_sheets-update-row", + "telegram_bot_api-send-text-message-or-reply" + ], + "persona": "complex-workflow" + }, + { + "query": "whenever someone books a meeting in Calendly, create a Notion page for notes and add a prep task in Todoist", + "sources": [ + "calendly_v2-new-event-scheduled" + ], + "actions": [ + "notion-create-page", + "notion-create-page-from-database", + "todoist-create-task" + ], + "persona": "complex-workflow" + }, + { + "query": "when a new file is uploaded to our Google Drive shared folder, create an entry in our asset tracking sheet and notify the team in Slack", + "sources": [ + "google_drive-new-files-instant" + ], + "actions": [ + "google_sheets-add-single-row", + "slack-send-message", + "slack-send-message-advanced" + ], + "persona": "complex-workflow" + }, + { + "query": "when we get a new YouTube video, transcribe it with OpenAI and create a Notion page with the content", + "sources": [ + "youtube_data_api-new-videos" + ], + "actions": [ + "openai-create-transcription", + "notion-create-page" + ], + "persona": "complex-workflow" + }, + { + "query": "whenever someone fills out our Typeform, add their info to Airtable and send them a welcome email", + "sources": [ + "typeform-new-submission" + ], + "actions": [ + "airtable_oauth-create-single-record", + "gmail-send-email" + ], + "persona": "complex-workflow" + }, + { + "query": "when new RSS items mention our company, analyze them with OpenAI and add important ones to our Google sheet", + "sources": [ + "rss-new-item-in-feed" + ], + "actions": [ + "openai-chat", + "google_sheets-add-single-row" + ], + "persona": "complex-workflow" + }, + { + "query": "whenever we get a new email in Gmail with attachments, save them to Google Drive and update our document log in Notion", + "sources": [ + "gmail-new-attachment-received" + ], + "actions": [ + "google_drive-upload-file", + "notion-update-page" + ], + "persona": "complex-workflow" + } + ] +} \ No newline at end of file