Component code gen: updating prompt, adding support for bug fixes

packages/component_code_gen/code_gen/generate_component_code.py

@@ -38,7 +38,7 @@ def generate_code(app, prompt, templates, parsed_common_files, urls_content, tri
     docs_meta = {}  # XXX - temporarily disable supabase docs
     results = []
 
-    auth_details = "## Auth details\n\nThese details come directly from the app configuration in Pipedream. Prioritize this information over any other information you find in the app's docs, since this section is specific to Pipedream."
+    auth_details = "<AuthDetails>\n\nThese details come directly from the app configuration in Pipedream. Prioritize this information over any other information you find in the app's docs, since this section is specific to Pipedream."
     auth_meta = db.get_app_auth_meta(app)
     auth_type = auth_meta.get('auth_type')
     if auth_type == "keys":
@@ -48,7 +48,9 @@ def generate_code(app, prompt, templates, parsed_common_files, urls_content, tri
     elif auth_type == "oauth":
         auth_details += f"{app} is an OAuth app. The `this` object exposes the OAuth access token in the variable `this.#{app.name_slug}.$auth.oauth_access_token`. When you make an API request, use the format from the \"Auth example\" section below."
     if auth_meta.get('component_code_scaffold_raw'):
-        auth_details = f"\n\n## Auth example\n\nHere's example Node.js code to show how authentication is done in {app}:\n\n{auth_meta['component_code_scaffold_raw']}\n\n"
+        auth_details += f"\n\n## Auth example\n\nHere's example Node.js code to show how authentication is done in {app}:\n\n{auth_meta['component_code_scaffold_raw']}\n\n"
+
+    auth_details += "</AuthDetails>"
 
     normal_order = False
     for i in range(tries):

packages/component_code_gen/templates/actions/additional_rules.py

@@ -1,36 +1,58 @@
-additional_rules = """## Additional rules for actions
+additional_rules = """<AdditionalRules>
 
 1. Always import the app file like this:
 
+```javascript
 import appName from "../../appName.app.mjs";
+```
+
+and pass the app file as a prop to the component:
+
+```javascript
+export default {
+  props: {
+    appName,
+  },
+  // rest of the component ...
+}
+```
 
 2. `return` the final value from the step. The data returned from steps must be JSON-serializable. The returned data is displayed in Pipedream. Think about it: if you don't return the data, the user won't see it.
 
 3. Always use this signature for the run method:
 
+```javascript
 async run({ $ }) {
-  // your code here
+  // you must fill in the actual code here
 }
+```
 
-Always pass { $ }, even if you don't use them in the code.
+Always pass `{ $ }` in the arguments to the `run` method, even if you don't use it in the code.
 
-4. Remember that `@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. Think about it: if you try to extract a data property that doesn't exist, the variable will hold the value `undefined`. You must return the data from the response directly and extract the proper data in the format provided by the API docs.
+4. `@pipedream/platform` axios returns a Promise that resolves to the HTTP response data. There is NO `data` property in the response that contains the data. The data from the HTTP response is returned directly in the response, not in the `data` property. Think about it: if you try to extract a data property that doesn't exist, the variable will hold the value `undefined`. You must return the data from the response directly and extract the proper data in the format provided by the API docs.
 
 For example, do this:
 
+```javascript
 const response = await axios(this, {
   url: `https://api.stability.ai/v1/engines/list`,
   headers: {
     Authorization: `Bearer ${this.dreamstudio.$auth.api_key}`,
   },
 });
-// process the response data. response.data is undefined
+// process the `response` as the return data. `response.data` is undefined, data is directly in `response`
+```
 
 not this:
 
+```javascript
+// this is incorrect — there is no `data` property in the response
 const { data } = await axios(this, {
   url: `https://api.stability.ai/v1/engines/list`,
   headers: {
     Authorization: `Bearer ${this.dreamstudio.$auth.api_key}`,
   },
-});"""
+});
+```
+
+</AdditionalRules>"""

packages/component_code_gen/templates/actions/export_summary.py

@@ -1,12 +1,15 @@
-export_summary = """## Export summary
+export_summary = """<ExportSummary>
 
-A short summary should be exported before the end so that the user can quickly read what has happened. This is done by calling `$.export("$summary", "Your summary here")`. This is not optional.
+You must call `$.export` to export a short text summary near the end of the `run` method so the user knows the call was successful. This is done by calling `$.export("$summary", "Your summary here")`.
 
-The summary should contain relevant metadata about the object that was created, updated, or deleted. For example, if you are creating a new issue, you should include the issue name or ID in the summary. 
+The summary should contain relevant metadata about the object that was created, updated, or deleted. For example, if you are creating a new issue, you should include the issue name or ID in the summary. Include information here that you think will be most relevant for the user reading the summary. 
 
-```
-`$.export("$summary", `Created issue ${name}`)
+Generally, this should happen immediately before you return the data at the end of the component's `run` method.
+
+```javascript
+$.export("$summary", `Created issue ${name}`)
+return data
 ```
 
-Include information here that you think will be most relevant for the user.
+</ExportSummary>
 """

packages/component_code_gen/templates/actions/introduction.py

@@ -1,9 +1,31 @@
-introduction = """## Instructions
+introduction = """<Instructions>
 
-Your goal is to create Pipedream Action Components. Your code should solve the requirements provided below.
+<Goal>
+Your goal is to create Pipedream action components, defined in the <Definitions> section below. 
 
-Other GPT agents will be reviewing your work, and will provide feedback on your code. I'll give you $500 for every rule you follow accurately, so you'll get a bigger tip if you follow all of the rules.
+Your code should solve the requirements provided below.
 
-## Pipedream components
+Think step by step:
 
-All Pipedream components are Node.js modules that have a default export: an javascript object - a Pipedream component - as its single argument."""
+1. Review the requirements
+2. Map out the `props`, `methods`, `run` method, and any other code you need to solve the requirements.
+3. Review whether you need async options for props (see the <AsyncOptions> section below)
+4. Review all of the rules carefully before producing code
+5. Produce full, complete, working code. This code is going straight to production.
+6. Review the code against the rules again, iterating or fixing items as necessary.
+7. Output the final code according to the rules of the <Output> section below.
+
+Other GPT agents will be reviewing your work, and will provide feedback on your code. Please review it before producing output.
+</Goal>
+
+<Defintions>
+<PipedreamActionComponents>
+
+All Pipedream components are Node.js modules that have a default export: an javascript object - a Pipedream component - as its single argument.
+
+See the <Rules>, <AsyncOptions>, <AdditionalRules>, and other sections below for details on how to structure components.
+
+</PipedreamActionComponents>
+</Definitions>
+
+</Instructions>"""

packages/component_code_gen/templates/actions/main_example.py

@@ -1,8 +1,9 @@
-main_example = """## OpenAI example component
+main_example = """<Example>
+<OpenAI>
 
-Here's an example component:
+Here's an example Pipedream action for OpenAI that lists all models available to the user:
 
-```
+```javascript
 import openai from "../../openai.app.mjs"
 import { axios } from "@pipedream/platform"
 
@@ -26,4 +27,7 @@
     return response
   },
 };
-```"""
+```
+</OpenAI>
+</Example>
+"""

packages/component_code_gen/templates/actions/other_example.py

@@ -1,8 +1,9 @@
-other_example = """## Slack API example component
+other_example = """<AdditionalExample>
+<Slack>
 
 Here's an example Pipedream component that makes a test request against the Slack API:
 
-```
+```javascript
 import slack from "../../slack.app.mjs"
 import { axios } from "@pipedream/platform"
 
@@ -41,18 +42,22 @@
     return response
   },
 };
+```
 
 Notice this section:
 
+```javascript
 data: {
   channel: this.channel,
   text: this.text,
 },
+```
 
-This shows you how to pass the values of props (e.g. this.channel and this.text) as params to the API. This is one of the most important things to know: you MUST generate code that adds inputs as props so that users can enter their own values when making the API request. You MUST NOT pass static values. See rule #2 below for more detail.
+This shows you how to pass the values of props (e.g. `this.channel` and `this.text`) as params to the API. This is one of the most important things to know: you MUST generate code that adds inputs as props so that users can enter their own values when making the API request. You MUST NOT pass static values. See the <Rules> for more information.
 
 The code you generate should be placed within the `run` method of the Pipedream component:
 
+```javascript
 import { axios } from "@pipedream/platform";
 
 export default {
@@ -69,4 +74,8 @@
     $.export("$summary", "Your summary here")
     return response
   },
-};"""
+};
+```
+
+</Slack>
+</AdditionalExample>"""

packages/component_code_gen/templates/apps/additional_rules.py

@@ -1,6 +1,7 @@
-additional_rules = """## Additional rules
+additional_rules = """<AdditionalRules>
 
-### Generate propDefinitions and methods for ALL requirements
+1. Generate `propDefinitions` and `methods` for ALL requirements
 
-The instructions should note the actions and source components that are required for the app. The code generator should generate the propDefinitions and methods for ALL requirements. Think about it: this way other agents will be able to use the shared props + methods in the action and source components. Double-check your output to make sure that the propDefinitions and methods are generated for ALL requirements.
+The instructions should note the actions and source components that are required for the app. The code generator should generate the `propDefinitions` and `methods` for ALL requirements. Think about it: this way other agents will be able to use the shared props + methods in the action and source components. Double-check your output to make sure that the propDefinitions and methods are generated for ALL requirements.
+</AdditionalRules>
 """

packages/component_code_gen/templates/apps/introduction.py

@@ -1,20 +1,42 @@
-introduction = """## Instructions
+introduction = """<Instructions>
 
-Your goal is to create Pipedream app files. Your code should solve the requirements provided below.
+<Goal>
+Your goal is to create Pipedream app files, defined in the Definition section below. 
 
-DO NOT remove any propDefinitions or methods that already exist. You can only write more, if required.
+Your code should solve the requirements provided below.
 
-Other GPT agents will be reviewing your work, and will provide feedback on your code. I'll give you $500 for every rule you follow accurately, so you'll get a bigger tip if you follow all of the rules.
+Think step by step:
 
-## Pipedream App files
+1. Review the requirements
+2. Map out the props and methods you need to solve the requirements
+3. Review whether you need async options for props (see the <AsyncOptions> section below)
+4. Review all of the rules carefully before producing code
+5. Produce full, complete, working code. This code is going straight to production.
+6. Review the code against the rules again, iterating or fixing items as necessary.
+7. Output the final code according to the rules of the <Output> section below.
+
+Other GPT agents will be reviewing your work, and will provide feedback on your code. Please review it before producing output.
+</Goal>
+
+<Definition>
+<PipedreamAppFiles>
 
 All Pipedream app files are Node.js modules that have a default export: a javascript object - a Pipedream app - as its single argument.
 
-All app objects have four properties: type, app, propDefinitions, and methods:
+All app objects have four properties: `type`, `app`, `propDefinitions`, and `methods`:
 
 - The `type` property is always set to "app".
 - The `app` property is the name of the app, e.g. "google_sheets".
 - The `propDefinitions` property is an object that contains the props for the app.
-- The methods property is an object that contains the methods for the app.
+- The `methods` property is an object that contains the methods for the app.
+
+These props and methods are shared across components for this app file. You'll need to generate props and methods for ALL requirements for all components that you're passed below.
+</PipedreamAppFiles>
+</Definition>
+<Output>
+Output Node.js code.
+
+DO NOT remove any `propDefinitions` or `methods` that already exist. You can only write more, if required.
+</Output>
 
-These props and methods are shared across components for this app file. You'll need to generate props and methods for ALL requirements for all components that you're passed below."""
+</Instructions>"""

packages/component_code_gen/templates/apps/main_example.py

@@ -1,4 +1,5 @@
-main_example = """## Example app file for Raindrop
+main_example = """<Examples>
+<Raindrop>
 
 Here's an example Pipedream app for Raindrop:
 
@@ -87,15 +88,17 @@
   },
 };
 ```
+</Raindrop>
+<GeneralExample>
+App files contain a `propDefinitions` property, which contains the definitions for the props of the app.
 
-This object contains a `propDefinitions` property, which contains the definitions for the props of the app.
-
-The propDefinitions object contains two props: collectionId and raindropId. The collectionId prop is a string prop. The raindropId prop is also a string prop. The propDefinitions object also contains an `options` method. The `options` method is an optional method that can be defined on a prop. It is used to dynamically generate the options for a prop and can return a static array of options or a Promise that resolves to an array of options.
+The `propDefinitions` object contains two props: `collectionId` and `raindropId`. The `collectionId` prop is a string prop. The `raindropId` prop is also a string prop. The `propDefinitions` object also contains an `options` method. The `options` method is an optional method that can be defined on a prop. It is used to dynamically generate the options for a prop and can return a static array of options or a Promise that resolves to an array of options.
 
 This object contains a `props` property, which defines a single prop of type "app":
 
 ```
 import { axios } from "@pipedream/platform";
+
 export default {
   type: "app",
   app: "the_app_name",
@@ -141,4 +144,7 @@
     },
   },
 }
-```"""
+```
+</GeneralExample>
+</Examples>
+"""

packages/component_code_gen/templates/apps/methods.py

@@ -1,10 +1,14 @@
-methods = """## Methods
+methods = """<Methods>
 
-The `methods` property contains helper methods. These methods can be called by other files.
+The `methods` property contains helper methods that compnents can use. These methods can be called by components that include the app file.
 
-A `_baseUrl` method is always required. It should return the base URL endpoint for the API.
+### _baseUrl method
 
-A `async _makeRequest` method is always required. It contains the code that makes the API request. It takes one argument, a single object named `opts`. 
+Always include a `_baseUrl` method. It should return the base URL endpoint for the API, defined by the OpenAPI spec or the API documentation.
+
+### _makeRequest method
+
+Always include an `async _makeRequest` method. It contains the code that makes the API request. It takes one argument, a single object named `opts`. 
 
 `opts` is an object that contains the parameters of the API request. When calling a component method with multiple parameters, you should pass them as a single object, using the Javascript spread syntax and destructuring when able to. 
 
@@ -32,7 +36,7 @@
 
 An example `_makeRequest` method is shown below. It is a simple GET request that returns the data from the response.
 
-```
+```javascript
 async _makeRequest(opts = {}) {
   const { $ = this, method = "GET", path = "/", headers, ...otherOpts } = opts;
   return axios($, {
@@ -48,7 +52,7 @@
 
 Auxiliary methods, usually for CRUD operations call `_makeRequest` with the appropriate parameters. Please add a few methods for common operations, e.g. get and list. You can also add other methods that you think are useful. Similar to the `_makeRequest` method, these auxiliary methods should have only one parameter, an object, `opts`. It should always destructure `...otherOpts`. Be sure to always add this parameter. `return` directly when calling the `_makeRequest` method. Here's an example:
 
-```
+```javascript
 async listObjects(opts = {}) {
   return this._makeRequest({
     path: "/objects",
@@ -70,4 +74,6 @@
 
 If we find more data, the method should call itself with the listing method and the parameters for fetching the next set of data. 
 
-If we don't find more data, you should return the array of results."""
+If we don't find more data, you should return the array of results.
+
+</Methods>"""

packages/component_code_gen/templates/apps/prop_definitions.py

@@ -1,6 +1,7 @@
-prop_definitions = """## Prop Definitions
+prop_definitions = """<PropDefinitions>
 
-The app code should contain a `propDefinitions` property, which are the definitions for the props. 
+The app code should contain a `propDefinitions` property, which define the props. 
 
-You'll find a "Props" section below that outlines the rules for generating props. All of those rules are valid, BUT instead of placing them in a `props` property, you MUST place props in the `propDefinitions` property. This is how Pipedream app files expect props to be declared.
+You'll find a <Props> section below that outlines the rules for generating props. All of those rules are valid, BUT instead of placing them in a `props` property, you MUST place props in the `propDefinitions` property. This is how Pipedream app files expect props to be declared.
+</PropDefinitions>
 """

packages/component_code_gen/templates/common/app_prop.py

@@ -1,8 +1,8 @@
-app_prop = """## App props
+app_prop = """<AppProp>
 
 The props object must contain a `props` property, which defines a single prop of type "app":
 
-```
+```javascript
 export default {
   props: {
     the_app_name: {
@@ -14,4 +14,6 @@
 }
 ```
 
-This lets the user connect their app account to the step, authorizing requests to the app API."""
+This lets the user connect their app account to the step, authorizing requests to the app API.
+
+</AppProp>"""