From 3788613084fbd473c7879d05248e313be53fda36 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Fri, 4 Oct 2024 12:58:55 -0400 Subject: [PATCH 1/8] wip --- .../actions/create-user/create-user.mjs | 47 +++++ .../actions/execute-query/execute-query.mjs | 28 +++ .../nile_database/nile_database.app.mjs | 166 +++++++++++++++++- components/nile_database/package.json | 8 +- .../sources/new-user-added/new-user-added.mjs | 0 5 files changed, 242 insertions(+), 7 deletions(-) create mode 100644 components/nile_database/actions/create-user/create-user.mjs create mode 100644 components/nile_database/actions/execute-query/execute-query.mjs create mode 100644 components/nile_database/sources/new-user-added/new-user-added.mjs diff --git a/components/nile_database/actions/create-user/create-user.mjs b/components/nile_database/actions/create-user/create-user.mjs new file mode 100644 index 0000000000000..1f9dd128326ab --- /dev/null +++ b/components/nile_database/actions/create-user/create-user.mjs @@ -0,0 +1,47 @@ +import nile from "../../nile_database.app.mjs"; + +export default { + key: "nile_database-create-user", + name: "Create User", + description: "Create a new database user by providing an email address and password. [See the documentation](https://www.thenile.dev/docs/reference/api-reference/users/create-user)", + version: "0.0.{{ts}}", + type: "action", + props: { + nile, + workspace: { + propDefinition: [ + nile, + "workspace", + ], + }, + email: { + type: "string", + label: "Email", + description: "Email address of the user", + }, + password: { + type: "string", + label: "Password", + description: "Password for the user", + }, + preferredName: { + type: "string", + label: "Preferred Name", + description: "The preferred name of the user", + optional: true, + }, + }, + async run({ $ }) { + const response = await this.nile.createUser({ + $, + workspace: this.workspace, + data: { + email: this.email, + password: this.password, + preferredName: this.preferredName, + }, + }); + $.export("$summary", `Successfully created user with ID: ${response.id}`); + return response; + }, +}; diff --git a/components/nile_database/actions/execute-query/execute-query.mjs b/components/nile_database/actions/execute-query/execute-query.mjs new file mode 100644 index 0000000000000..4264612bf966c --- /dev/null +++ b/components/nile_database/actions/execute-query/execute-query.mjs @@ -0,0 +1,28 @@ +import nile from "../../nile_database.app.mjs"; + +export default { + name: "Execute Query", + key: "nile_database-execute-query", + description: "Execute a custom PostgreSQL query. See [our docs](https://pipedream.com/docs/databases/working-with-sql) to learn more about working with SQL in Pipedream.", + version: "0.0.{{ts}}", + type: "action", + props: { + nile, + // eslint-disable-next-line pipedream/props-description + sql: { + type: "sql", + auth: { + app: "nile", + }, + label: "PostreSQL Query", + }, + }, + async run({ $ }) { + const args = this.nile.executeQueryAdapter(this.sql); + const data = await this.nile.executeQuery(args); + $.export("$summary", `Returned ${data.length} ${data.length === 1 + ? "row" + : "rows"}`); + return data; + }, +}; diff --git a/components/nile_database/nile_database.app.mjs b/components/nile_database/nile_database.app.mjs index cb85e6227fca2..04eb11d497167 100644 --- a/components/nile_database/nile_database.app.mjs +++ b/components/nile_database/nile_database.app.mjs @@ -1,11 +1,167 @@ +import { axios } from "@pipedream/platform"; +import pg from "pg"; + export default { type: "app", app: "nile_database", - propDefinitions: {}, + propDefinitions: { + workspace: { + type: "string", + label: "Workspace", + description: "Your workspace slug", + }, + }, methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); + _baseUrl(workspace) { + return `https://api.thenile.dev/workspaces/${workspace}/databases/${this.$auth.database}`; + }, + async _getHeaders($, useToken, workspace, email, password) { + const headers = { + "Content-Type": "application/json", + }; + if (useToken) { + headers.Authorization = `Bearer ${await this.getToken({ + $, + workspace, + email, + password, + })}`; + } + return headers; + }, + async getToken({ + $, workspace, email, password, + }) { + const { token: { jwt } } = await axios($, { + method: "POST", + url: `${this._baseUrl(workspace)}/users/login`, + data: { + email, + password, + }, + }); + return jwt; + }, + async _makeRequest({ + $ = this, + path, + workspace, + email, + password, + useToken = false, + ...args + }) { + return axios($, { + url: `${this._baseUrl(workspace)}${path}`, + headers: await this._getHeaders($, useToken, workspace, email, password), + ...args, + }); + }, + listUsers({ + workspace, email, password, ...args + }) { + return this._makeRequest({ + path: "/users", + workspace, + useToken: true, + email, + password, + ...args, + }); + }, + createUser({ + workspace, ...args + }) { + return this._makeRequest({ + method: "POST", + path: "/users", + workspace, + ...args, + }); + }, + getClientConfiguration() { + const { + username, + password, + host, + port, + database, + } = this.$auth; + + return { + user: username, + password, + host, + port, + database, + }; + }, + async _getClient() { + const config = this.getClientConfiguration(); + const pool = new pg.Pool(config); + const client = await pool.connect(); + return client; + }, + async _endClient(client) { + return client.release(); + }, + async executeQuery(query) { + const client = await this._getClient(); + + try { + const { rows } = await client.query(query); + return rows; + } finally { + await this._endClient(client); + } + }, + executeQueryAdapter(proxyArgs = {}) { + const { + query: text = "", + params: values = [], + } = proxyArgs; + return { + text, + values, + }; + }, + proxyAdapter(query) { + if (typeof query === "string") { + return { + query, + }; + } + + return { + query: query.text, + params: query.values, + }; + }, + async getSchema() { + const text = ` + SELECT table_name AS "tableName", + column_name AS "columnName", + is_nullable AS "isNullable", + data_type AS "dataType", + column_default AS "columnDefault" + FROM information_schema.columns + WHERE table_schema NOT IN ('pg_catalog', 'information_schema', 'users', 'auth') + ORDER BY table_name, + ordinal_position + `; + const rows = await this.executeQuery({ + text, + }); + return rows.reduce((acc, row) => { + acc[row.tableName] ??= { + metadata: {}, + schema: {}, + }; + acc[row.tableName].schema[row.columnName] = { + ...row, + }; + return acc; + }, {}); }, }, -}; \ No newline at end of file +}; diff --git a/components/nile_database/package.json b/components/nile_database/package.json index a93f49b6bc770..1d24c129fc4a7 100644 --- a/components/nile_database/package.json +++ b/components/nile_database/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/nile_database", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream Nile Database Components", "main": "nile_database.app.mjs", "keywords": [ @@ -11,5 +11,9 @@ "author": "Pipedream (https://pipedream.com/)", "publishConfig": { "access": "public" + }, + "dependencies": { + "@pipedream/platform": "^3.0.3", + "pg": "^8.13.0" } -} \ No newline at end of file +} diff --git a/components/nile_database/sources/new-user-added/new-user-added.mjs b/components/nile_database/sources/new-user-added/new-user-added.mjs new file mode 100644 index 0000000000000..e69de29bb2d1d From 1119cff9a6cd2d73b4c78b4892b34871b239a737 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Mon, 7 Oct 2024 12:22:19 -0400 Subject: [PATCH 2/8] execute-query-action --- .../actions/create-user/create-user.mjs | 47 ------------ .../actions/execute-query/execute-query.mjs | 2 +- .../nile_database/nile_database.app.mjs | 75 ------------------- components/nile_database/package.json | 1 - .../sources/new-user-added/new-user-added.mjs | 0 5 files changed, 1 insertion(+), 124 deletions(-) delete mode 100644 components/nile_database/actions/create-user/create-user.mjs delete mode 100644 components/nile_database/sources/new-user-added/new-user-added.mjs diff --git a/components/nile_database/actions/create-user/create-user.mjs b/components/nile_database/actions/create-user/create-user.mjs deleted file mode 100644 index 1f9dd128326ab..0000000000000 --- a/components/nile_database/actions/create-user/create-user.mjs +++ /dev/null @@ -1,47 +0,0 @@ -import nile from "../../nile_database.app.mjs"; - -export default { - key: "nile_database-create-user", - name: "Create User", - description: "Create a new database user by providing an email address and password. [See the documentation](https://www.thenile.dev/docs/reference/api-reference/users/create-user)", - version: "0.0.{{ts}}", - type: "action", - props: { - nile, - workspace: { - propDefinition: [ - nile, - "workspace", - ], - }, - email: { - type: "string", - label: "Email", - description: "Email address of the user", - }, - password: { - type: "string", - label: "Password", - description: "Password for the user", - }, - preferredName: { - type: "string", - label: "Preferred Name", - description: "The preferred name of the user", - optional: true, - }, - }, - async run({ $ }) { - const response = await this.nile.createUser({ - $, - workspace: this.workspace, - data: { - email: this.email, - password: this.password, - preferredName: this.preferredName, - }, - }); - $.export("$summary", `Successfully created user with ID: ${response.id}`); - return response; - }, -}; diff --git a/components/nile_database/actions/execute-query/execute-query.mjs b/components/nile_database/actions/execute-query/execute-query.mjs index 4264612bf966c..d32f2a3eba2eb 100644 --- a/components/nile_database/actions/execute-query/execute-query.mjs +++ b/components/nile_database/actions/execute-query/execute-query.mjs @@ -4,7 +4,7 @@ export default { name: "Execute Query", key: "nile_database-execute-query", description: "Execute a custom PostgreSQL query. See [our docs](https://pipedream.com/docs/databases/working-with-sql) to learn more about working with SQL in Pipedream.", - version: "0.0.{{ts}}", + version: "0.0.1", type: "action", props: { nile, diff --git a/components/nile_database/nile_database.app.mjs b/components/nile_database/nile_database.app.mjs index 04eb11d497167..b29170d753cb6 100644 --- a/components/nile_database/nile_database.app.mjs +++ b/components/nile_database/nile_database.app.mjs @@ -1,84 +1,9 @@ -import { axios } from "@pipedream/platform"; import pg from "pg"; export default { type: "app", app: "nile_database", - propDefinitions: { - workspace: { - type: "string", - label: "Workspace", - description: "Your workspace slug", - }, - }, methods: { - _baseUrl(workspace) { - return `https://api.thenile.dev/workspaces/${workspace}/databases/${this.$auth.database}`; - }, - async _getHeaders($, useToken, workspace, email, password) { - const headers = { - "Content-Type": "application/json", - }; - if (useToken) { - headers.Authorization = `Bearer ${await this.getToken({ - $, - workspace, - email, - password, - })}`; - } - return headers; - }, - async getToken({ - $, workspace, email, password, - }) { - const { token: { jwt } } = await axios($, { - method: "POST", - url: `${this._baseUrl(workspace)}/users/login`, - data: { - email, - password, - }, - }); - return jwt; - }, - async _makeRequest({ - $ = this, - path, - workspace, - email, - password, - useToken = false, - ...args - }) { - return axios($, { - url: `${this._baseUrl(workspace)}${path}`, - headers: await this._getHeaders($, useToken, workspace, email, password), - ...args, - }); - }, - listUsers({ - workspace, email, password, ...args - }) { - return this._makeRequest({ - path: "/users", - workspace, - useToken: true, - email, - password, - ...args, - }); - }, - createUser({ - workspace, ...args - }) { - return this._makeRequest({ - method: "POST", - path: "/users", - workspace, - ...args, - }); - }, getClientConfiguration() { const { username, diff --git a/components/nile_database/package.json b/components/nile_database/package.json index 1d24c129fc4a7..b14d7f86142a3 100644 --- a/components/nile_database/package.json +++ b/components/nile_database/package.json @@ -13,7 +13,6 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.3", "pg": "^8.13.0" } } diff --git a/components/nile_database/sources/new-user-added/new-user-added.mjs b/components/nile_database/sources/new-user-added/new-user-added.mjs deleted file mode 100644 index e69de29bb2d1d..0000000000000 From b173730e4f2b8ed8e34b359922f0d108eb33f8ca Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Mon, 7 Oct 2024 12:23:35 -0400 Subject: [PATCH 3/8] pnpm-lock.yaml --- pnpm-lock.yaml | 141 +++++++++++++++++++++++++++++++------------------ 1 file changed, 89 insertions(+), 52 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 93c26d6d11352..f2cda70f3e1bb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6438,7 +6438,10 @@ importers: specifiers: {} components/nile_database: - specifiers: {} + specifiers: + pg: ^8.13.0 + dependencies: + pg: 8.13.0 components/nimble: specifiers: @@ -12882,55 +12885,6 @@ packages: - aws-crt dev: false - /@aws-sdk/client-sso-oidc/3.600.0_tdq3komn4zwyd65w7klbptsu34: - resolution: {integrity: sha512-7+I8RWURGfzvChyNQSyj5/tKrqRbzRl7H+BnTOf/4Vsw1nFOi5ROhlhD4X/Y0QCTacxnaoNcIrqnY7uGGvVRzw==} - engines: {node: '>=16.0.0'} - dependencies: - '@aws-crypto/sha256-browser': 5.2.0 - '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/client-sts': 3.600.0 - '@aws-sdk/core': 3.598.0 - '@aws-sdk/credential-provider-node': 3.600.0_f7n47caigsrjd2lr2szmwfuee4 - '@aws-sdk/middleware-host-header': 3.598.0 - '@aws-sdk/middleware-logger': 3.598.0 - '@aws-sdk/middleware-recursion-detection': 3.598.0 - '@aws-sdk/middleware-user-agent': 3.598.0 - '@aws-sdk/region-config-resolver': 3.598.0 - '@aws-sdk/types': 3.598.0 - '@aws-sdk/util-endpoints': 3.598.0 - '@aws-sdk/util-user-agent-browser': 3.598.0 - '@aws-sdk/util-user-agent-node': 3.598.0 - '@smithy/config-resolver': 3.0.3 - '@smithy/core': 2.2.3 - '@smithy/fetch-http-handler': 3.2.1 - '@smithy/hash-node': 3.0.2 - '@smithy/invalid-dependency': 3.0.2 - '@smithy/middleware-content-length': 3.0.2 - '@smithy/middleware-endpoint': 3.0.4 - '@smithy/middleware-retry': 3.0.6 - '@smithy/middleware-serde': 3.0.3 - '@smithy/middleware-stack': 3.0.3 - '@smithy/node-config-provider': 3.1.3 - '@smithy/node-http-handler': 3.1.2 - '@smithy/protocol-http': 4.0.3 - '@smithy/smithy-client': 3.1.6 - '@smithy/types': 3.3.0 - '@smithy/url-parser': 3.0.3 - '@smithy/util-base64': 3.0.0 - '@smithy/util-body-length-browser': 3.0.0 - '@smithy/util-body-length-node': 3.0.0 - '@smithy/util-defaults-mode-browser': 3.0.6 - '@smithy/util-defaults-mode-node': 3.0.6 - '@smithy/util-endpoints': 2.0.3 - '@smithy/util-middleware': 3.0.3 - '@smithy/util-retry': 3.0.2 - '@smithy/util-utf8': 3.0.0 - tslib: 2.6.3 - transitivePeerDependencies: - - '@aws-sdk/client-sts' - - aws-crt - dev: false - /@aws-sdk/client-sso/3.423.0: resolution: {integrity: sha512-znIufHkwhCIePgaYciIs3x/+BpzR57CZzbCKHR9+oOvGyufEPPpUT5bFLvbwTgfiVkTjuk6sG/ES3U5Bc+xtrA==} engines: {node: '>=14.0.0'} @@ -13166,7 +13120,7 @@ packages: dependencies: '@aws-crypto/sha256-browser': 5.2.0 '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/client-sso-oidc': 3.600.0_tdq3komn4zwyd65w7klbptsu34 + '@aws-sdk/client-sso-oidc': 3.600.0 '@aws-sdk/core': 3.598.0 '@aws-sdk/credential-provider-node': 3.600.0_f7n47caigsrjd2lr2szmwfuee4 '@aws-sdk/middleware-host-header': 3.598.0 @@ -13208,6 +13162,55 @@ packages: - aws-crt dev: false + /@aws-sdk/client-sts/3.600.0_dseaa2p5u2yk67qiepewcq3hkq: + resolution: {integrity: sha512-KQG97B7LvTtTiGmjlrG1LRAY8wUvCQzrmZVV5bjrJ/1oXAU7DITYwVbSJeX9NWg6hDuSk0VE3MFwIXS2SvfLIA==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/client-sso-oidc': 3.600.0 + '@aws-sdk/core': 3.598.0 + '@aws-sdk/credential-provider-node': 3.600.0_f7n47caigsrjd2lr2szmwfuee4 + '@aws-sdk/middleware-host-header': 3.598.0 + '@aws-sdk/middleware-logger': 3.598.0 + '@aws-sdk/middleware-recursion-detection': 3.598.0 + '@aws-sdk/middleware-user-agent': 3.598.0 + '@aws-sdk/region-config-resolver': 3.598.0 + '@aws-sdk/types': 3.598.0 + '@aws-sdk/util-endpoints': 3.598.0 + '@aws-sdk/util-user-agent-browser': 3.598.0 + '@aws-sdk/util-user-agent-node': 3.598.0 + '@smithy/config-resolver': 3.0.3 + '@smithy/core': 2.2.3 + '@smithy/fetch-http-handler': 3.2.1 + '@smithy/hash-node': 3.0.2 + '@smithy/invalid-dependency': 3.0.2 + '@smithy/middleware-content-length': 3.0.2 + '@smithy/middleware-endpoint': 3.0.4 + '@smithy/middleware-retry': 3.0.6 + '@smithy/middleware-serde': 3.0.3 + '@smithy/middleware-stack': 3.0.3 + '@smithy/node-config-provider': 3.1.3 + '@smithy/node-http-handler': 3.1.2 + '@smithy/protocol-http': 4.0.3 + '@smithy/smithy-client': 3.1.6 + '@smithy/types': 3.3.0 + '@smithy/url-parser': 3.0.3 + '@smithy/util-base64': 3.0.0 + '@smithy/util-body-length-browser': 3.0.0 + '@smithy/util-body-length-node': 3.0.0 + '@smithy/util-defaults-mode-browser': 3.0.6 + '@smithy/util-defaults-mode-node': 3.0.6 + '@smithy/util-endpoints': 2.0.3 + '@smithy/util-middleware': 3.0.3 + '@smithy/util-retry': 3.0.2 + '@smithy/util-utf8': 3.0.0 + tslib: 2.6.3 + transitivePeerDependencies: + - '@aws-sdk/client-sso-oidc' + - aws-crt + dev: false + /@aws-sdk/core/3.556.0: resolution: {integrity: sha512-vJaSaHw2kPQlo11j/Rzuz0gk1tEaKdz+2ser0f0qZ5vwFlANjt08m/frU17ctnVKC1s58bxpctO/1P894fHLrA==} engines: {node: '>=14.0.0'} @@ -17534,7 +17537,7 @@ packages: '@aws-sdk/client-sns': 3.423.0 '@aws-sdk/client-sqs': 3.423.0 '@aws-sdk/client-ssm': 3.423.0 - '@aws-sdk/client-sts': 3.600.0 + '@aws-sdk/client-sts': 3.600.0_dseaa2p5u2yk67qiepewcq3hkq '@aws-sdk/s3-request-presigner': 3.609.0 '@pipedream/helper_functions': 0.3.12 '@pipedream/platform': 1.6.6 @@ -31381,6 +31384,10 @@ packages: resolution: {integrity: sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==} dev: false + /pg-connection-string/2.7.0: + resolution: {integrity: sha512-PI2W9mv53rXJQEOb8xNR8lH7Hr+EKa6oJa38zsK0S/ky2er16ios1wLKhZyxzD7jUReiWokc9WK5nxSnC7W1TA==} + dev: false + /pg-format/1.0.4: resolution: {integrity: sha512-YyKEF78pEA6wwTAqOUaHIN/rWpfzzIuMh9KdAhc3rSLQ/7zkRFcCgYBAEGatDstLyZw4g0s9SNICmaTGnBVeyw==} engines: {node: '>=4.0'} @@ -31399,10 +31406,22 @@ packages: pg: 8.11.3 dev: false + /pg-pool/3.7.0_pg@8.13.0: + resolution: {integrity: sha512-ZOBQForurqh4zZWjrgSwwAtzJ7QiRX0ovFkZr2klsen3Nm0aoh33Ls0fzfv3imeH/nw/O27cjdz5kzYJfeGp/g==} + peerDependencies: + pg: '>=8.0' + dependencies: + pg: 8.13.0 + dev: false + /pg-protocol/1.6.0: resolution: {integrity: sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==} dev: false + /pg-protocol/1.7.0: + resolution: {integrity: sha512-hTK/mE36i8fDDhgDFjy6xNOG+LCorxLG3WO17tku+ij6sVHXh1jQUJ8hYAnRhNla4QVD2H8er/FOjc/+EgC6yQ==} + dev: false + /pg-types/2.2.0: resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} engines: {node: '>=4'} @@ -31434,6 +31453,24 @@ packages: pg-cloudflare: 1.1.1 dev: false + /pg/8.13.0: + resolution: {integrity: sha512-34wkUTh3SxTClfoHB3pQ7bIMvw9dpFU1audQQeZG837fmHfHpr14n/AELVDoOYVDW2h5RDWU78tFjkD+erSBsw==} + engines: {node: '>= 8.0.0'} + peerDependencies: + pg-native: '>=3.0.1' + peerDependenciesMeta: + pg-native: + optional: true + dependencies: + pg-connection-string: 2.7.0 + pg-pool: 3.7.0_pg@8.13.0 + pg-protocol: 1.7.0 + pg-types: 2.2.0 + pgpass: 1.0.5 + optionalDependencies: + pg-cloudflare: 1.1.1 + dev: false + /pgpass/1.0.5: resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} dependencies: From e421de5a1fc4df9af3ed0dee210df1072645c71c Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Wed, 9 Oct 2024 11:24:06 -0400 Subject: [PATCH 4/8] execute-query --- .../actions/execute-query/execute-query.mjs | 48 +++++++++--- .../nile_database/nile_database.app.mjs | 73 +------------------ 2 files changed, 41 insertions(+), 80 deletions(-) diff --git a/components/nile_database/actions/execute-query/execute-query.mjs b/components/nile_database/actions/execute-query/execute-query.mjs index d32f2a3eba2eb..7a9eb807fccd3 100644 --- a/components/nile_database/actions/execute-query/execute-query.mjs +++ b/components/nile_database/actions/execute-query/execute-query.mjs @@ -3,23 +3,51 @@ import nile from "../../nile_database.app.mjs"; export default { name: "Execute Query", key: "nile_database-execute-query", - description: "Execute a custom PostgreSQL query. See [our docs](https://pipedream.com/docs/databases/working-with-sql) to learn more about working with SQL in Pipedream.", + description: "Execute a custom PostgreSQL query.", version: "0.0.1", type: "action", props: { nile, - // eslint-disable-next-line pipedream/props-description - sql: { - type: "sql", - auth: { - app: "nile", - }, - label: "PostreSQL Query", + user: { + type: "string", + label: "Username", + description: "The username of the database user", + }, + password: { + type: "string", + label: "Password", + description: "The password of the database user", + }, + host: { + type: "string", + label: "Host", + description: "The host of the database", + }, + port: { + type: "string", + label: "Port", + description: "The port to connect to the database. Example: `5432`", + }, + database: { + type: "string", + label: "Database", + description: "The name of the database to connect to", + }, + query: { + type: "string", + label: "Query", + description: "The PostgreSQL query to execute", }, }, async run({ $ }) { - const args = this.nile.executeQueryAdapter(this.sql); - const data = await this.nile.executeQuery(args); + const config = { + user: this.user, + password: this.password, + host: this.host, + port: this.port, + database: this.database, + }; + const data = await this.nile.executeQuery(config, this.query); $.export("$summary", `Returned ${data.length} ${data.length === 1 ? "row" : "rows"}`); diff --git a/components/nile_database/nile_database.app.mjs b/components/nile_database/nile_database.app.mjs index b29170d753cb6..ba07bdf0b8f3c 100644 --- a/components/nile_database/nile_database.app.mjs +++ b/components/nile_database/nile_database.app.mjs @@ -4,25 +4,7 @@ export default { type: "app", app: "nile_database", methods: { - getClientConfiguration() { - const { - username, - password, - host, - port, - database, - } = this.$auth; - - return { - user: username, - password, - host, - port, - database, - }; - }, - async _getClient() { - const config = this.getClientConfiguration(); + async _getClient(config) { const pool = new pg.Pool(config); const client = await pool.connect(); return client; @@ -30,9 +12,8 @@ export default { async _endClient(client) { return client.release(); }, - async executeQuery(query) { - const client = await this._getClient(); - + async executeQuery(config, query) { + const client = await this._getClient(config); try { const { rows } = await client.query(query); return rows; @@ -40,53 +21,5 @@ export default { await this._endClient(client); } }, - executeQueryAdapter(proxyArgs = {}) { - const { - query: text = "", - params: values = [], - } = proxyArgs; - return { - text, - values, - }; - }, - proxyAdapter(query) { - if (typeof query === "string") { - return { - query, - }; - } - - return { - query: query.text, - params: query.values, - }; - }, - async getSchema() { - const text = ` - SELECT table_name AS "tableName", - column_name AS "columnName", - is_nullable AS "isNullable", - data_type AS "dataType", - column_default AS "columnDefault" - FROM information_schema.columns - WHERE table_schema NOT IN ('pg_catalog', 'information_schema', 'users', 'auth') - ORDER BY table_name, - ordinal_position - `; - const rows = await this.executeQuery({ - text, - }); - return rows.reduce((acc, row) => { - acc[row.tableName] ??= { - metadata: {}, - schema: {}, - }; - acc[row.tableName].schema[row.columnName] = { - ...row, - }; - return acc; - }, {}); - }, }, }; From e3e859529b598ff246b3a012ad9e47868d118973 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Wed, 9 Oct 2024 13:30:09 -0400 Subject: [PATCH 5/8] user api components --- .../actions/create-user/create-user.mjs | 54 +++++++++++ .../actions/execute-query/execute-query.mjs | 7 +- .../nile_database/nile_database.app.mjs | 92 +++++++++++++++++++ components/nile_database/package.json | 1 + .../nile_database/sources/common/base.mjs | 48 ++++++++++ .../new-tenant-created/new-tenant-created.mjs | 24 +++++ .../new-user-created/new-user-created.mjs | 24 +++++ 7 files changed, 247 insertions(+), 3 deletions(-) create mode 100644 components/nile_database/actions/create-user/create-user.mjs create mode 100644 components/nile_database/sources/common/base.mjs create mode 100644 components/nile_database/sources/new-tenant-created/new-tenant-created.mjs create mode 100644 components/nile_database/sources/new-user-created/new-user-created.mjs diff --git a/components/nile_database/actions/create-user/create-user.mjs b/components/nile_database/actions/create-user/create-user.mjs new file mode 100644 index 0000000000000..dcaf2e06c2c02 --- /dev/null +++ b/components/nile_database/actions/create-user/create-user.mjs @@ -0,0 +1,54 @@ +import nile from "../../nile_database.app.mjs"; + +export default { + key: "nile_database-create-user", + name: "Create User", + description: "Create a new database user by providing an email address and password. [See the documentation](https://www.thenile.dev/docs/reference/api-reference/users/create-user)", + version: "0.0.1", + type: "action", + props: { + nile, + workspace: { + propDefinition: [ + nile, + "workspace", + ], + }, + database: { + propDefinition: [ + nile, + "database", + ], + }, + email: { + type: "string", + label: "Email", + description: "Email address of the user", + }, + password: { + type: "string", + label: "Password", + description: "Password for the user", + }, + preferredName: { + type: "string", + label: "Preferred Name", + description: "The preferred name of the user", + optional: true, + }, + }, + async run({ $ }) { + const response = await this.nile.createUser({ + $, + workspace: this.workspace, + database: this.database, + data: { + email: this.email, + password: this.password, + preferredName: this.preferredName, + }, + }); + $.export("$summary", `Successfully created user with ID: ${response.id}`); + return response; + }, +}; diff --git a/components/nile_database/actions/execute-query/execute-query.mjs b/components/nile_database/actions/execute-query/execute-query.mjs index 7a9eb807fccd3..c42bbe25b8ecc 100644 --- a/components/nile_database/actions/execute-query/execute-query.mjs +++ b/components/nile_database/actions/execute-query/execute-query.mjs @@ -29,9 +29,10 @@ export default { description: "The port to connect to the database. Example: `5432`", }, database: { - type: "string", - label: "Database", - description: "The name of the database to connect to", + propDefinition: [ + nile, + "database", + ], }, query: { type: "string", diff --git a/components/nile_database/nile_database.app.mjs b/components/nile_database/nile_database.app.mjs index ba07bdf0b8f3c..38ef3b7137b4d 100644 --- a/components/nile_database/nile_database.app.mjs +++ b/components/nile_database/nile_database.app.mjs @@ -1,9 +1,101 @@ +import { axios } from "@pipedream/platform"; import pg from "pg"; export default { type: "app", app: "nile_database", + propDefinitions: { + workspace: { + type: "string", + label: "Workspace", + description: "Your workspace slug", + async options() { + const { workspaces } = await this.getAuthenticatedUser(); + return workspaces?.map(({ slug }) => slug) || []; + }, + }, + database: { + type: "string", + label: "Database", + description: "The database name", + async options() { + const { databases } = await this.getAuthenticatedUser(); + return databases?.map(({ name }) => name) || []; + }, + }, + }, methods: { + _globalBaseUrl() { + return "https://global.thenile.dev"; + }, + async _getBaseUrl({ + workspace, database, ...opts + }) { + const { apiHost } = await this._makeRequest({ + url: `${this._globalBaseUrl()}/workspaces/${workspace}/databases/${database}`, + workspace, + database, + ...opts, + }); + return apiHost; + }, + async _makeRequest({ + $ = this, + workspace, + database, + url, + path, + ...opts + }) { + return axios($, { + url: url || `${await this._getBaseUrl({ + workspace, + database, + $, + })}${path}`, + headers: { + Authorization: `Bearer ${this.$auth.oauth_access_token}`, + }, + ...opts, + }); + }, + getAuthenticatedUser(opts = {}) { + return this._makeRequest({ + url: `${this._globalBaseUrl()}/developers/me/full`, + ...opts, + }); + }, + listUsers({ + workspace, database, ...opts + }) { + return this._makeRequest({ + path: "/users", + workspace, + database, + ...opts, + }); + }, + listTenants({ + workspace, database, ...opts + }) { + return this._makeRequest({ + path: "/tenants", + workspace, + database, + ...opts, + }); + }, + createUser({ + workspace, database, ...opts + }) { + return this._makeRequest({ + method: "POST", + path: "/users", + workspace, + database, + ...opts, + }); + }, async _getClient(config) { const pool = new pg.Pool(config); const client = await pool.connect(); diff --git a/components/nile_database/package.json b/components/nile_database/package.json index b14d7f86142a3..1d24c129fc4a7 100644 --- a/components/nile_database/package.json +++ b/components/nile_database/package.json @@ -13,6 +13,7 @@ "access": "public" }, "dependencies": { + "@pipedream/platform": "^3.0.3", "pg": "^8.13.0" } } diff --git a/components/nile_database/sources/common/base.mjs b/components/nile_database/sources/common/base.mjs new file mode 100644 index 0000000000000..0b45fedfa043c --- /dev/null +++ b/components/nile_database/sources/common/base.mjs @@ -0,0 +1,48 @@ +import nile from "../../nile_database.app.mjs"; +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; + +export default { + props: { + nile, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + workspace: { + propDefinition: [ + nile, + "workspace", + ], + }, + database: { + propDefinition: [ + nile, + "database", + ], + }, + }, + methods: { + getResourceFn() { + throw new Error("getResourceFn is not implemented"); + }, + generateMeta() { + throw new Error("generateMeta is not implemented"); + }, + }, + async run() { + const resourceFn = this.getResourceFn(); + + const results = await resourceFn({ + workspace: this.workspace, + database: this.database, + }); console.log(results); + + for (const item of results) { + const meta = this.generateMeta(item); + this.$emit(item, meta); + } + }, +}; diff --git a/components/nile_database/sources/new-tenant-created/new-tenant-created.mjs b/components/nile_database/sources/new-tenant-created/new-tenant-created.mjs new file mode 100644 index 0000000000000..fb0cd1c0249e9 --- /dev/null +++ b/components/nile_database/sources/new-tenant-created/new-tenant-created.mjs @@ -0,0 +1,24 @@ +import common from "../common/base.mjs"; + +export default { + ...common, + key: "nile_database-new-tenant-created", + name: "New Tenant Created", + description: "Emit new event when a new tenant is added to a Nile Database", + version: "0.0.1", + type: "source", + dedupe: "unique", + methods: { + ...common.methods, + getResourceFn() { + return this.nile.listTenants; + }, + generateMeta(tenant) { + return { + id: tenant.id, + summary: `New Tenant ID: ${tenant.id}`, + ts: Date.now(), + }; + }, + }, +}; diff --git a/components/nile_database/sources/new-user-created/new-user-created.mjs b/components/nile_database/sources/new-user-created/new-user-created.mjs new file mode 100644 index 0000000000000..2e7f3b6c29007 --- /dev/null +++ b/components/nile_database/sources/new-user-created/new-user-created.mjs @@ -0,0 +1,24 @@ +import common from "../common/base.mjs"; + +export default { + ...common, + key: "nile_database-new-user-created", + name: "New User Created", + description: "Emit new event when a new user is added in a Nile Database", + version: "0.0.1", + type: "source", + dedupe: "unique", + methods: { + ...common.methods, + getResourceFn() { + return this.nile.listUsers; + }, + generateMeta(user) { + return { + id: user.id, + summary: `New User ID: ${user.id}`, + ts: Date.parse(user.created), + }; + }, + }, +}; From aedfa14d607b4da78f3129afbb9e4429deaef3ce Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Wed, 9 Oct 2024 13:31:33 -0400 Subject: [PATCH 6/8] pnpm-lock.yaml --- pnpm-lock.yaml | 104 +++++++++++++++++++++++++------------------------ 1 file changed, 53 insertions(+), 51 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f2cda70f3e1bb..a52b9441bbaa3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6439,8 +6439,10 @@ importers: components/nile_database: specifiers: + '@pipedream/platform': ^3.0.3 pg: ^8.13.0 dependencies: + '@pipedream/platform': 3.0.3 pg: 8.13.0 components/nimble: @@ -12885,6 +12887,55 @@ packages: - aws-crt dev: false + /@aws-sdk/client-sso-oidc/3.600.0_tdq3komn4zwyd65w7klbptsu34: + resolution: {integrity: sha512-7+I8RWURGfzvChyNQSyj5/tKrqRbzRl7H+BnTOf/4Vsw1nFOi5ROhlhD4X/Y0QCTacxnaoNcIrqnY7uGGvVRzw==} + engines: {node: '>=16.0.0'} + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/client-sts': 3.600.0 + '@aws-sdk/core': 3.598.0 + '@aws-sdk/credential-provider-node': 3.600.0_f7n47caigsrjd2lr2szmwfuee4 + '@aws-sdk/middleware-host-header': 3.598.0 + '@aws-sdk/middleware-logger': 3.598.0 + '@aws-sdk/middleware-recursion-detection': 3.598.0 + '@aws-sdk/middleware-user-agent': 3.598.0 + '@aws-sdk/region-config-resolver': 3.598.0 + '@aws-sdk/types': 3.598.0 + '@aws-sdk/util-endpoints': 3.598.0 + '@aws-sdk/util-user-agent-browser': 3.598.0 + '@aws-sdk/util-user-agent-node': 3.598.0 + '@smithy/config-resolver': 3.0.3 + '@smithy/core': 2.2.3 + '@smithy/fetch-http-handler': 3.2.1 + '@smithy/hash-node': 3.0.2 + '@smithy/invalid-dependency': 3.0.2 + '@smithy/middleware-content-length': 3.0.2 + '@smithy/middleware-endpoint': 3.0.4 + '@smithy/middleware-retry': 3.0.6 + '@smithy/middleware-serde': 3.0.3 + '@smithy/middleware-stack': 3.0.3 + '@smithy/node-config-provider': 3.1.3 + '@smithy/node-http-handler': 3.1.2 + '@smithy/protocol-http': 4.0.3 + '@smithy/smithy-client': 3.1.6 + '@smithy/types': 3.3.0 + '@smithy/url-parser': 3.0.3 + '@smithy/util-base64': 3.0.0 + '@smithy/util-body-length-browser': 3.0.0 + '@smithy/util-body-length-node': 3.0.0 + '@smithy/util-defaults-mode-browser': 3.0.6 + '@smithy/util-defaults-mode-node': 3.0.6 + '@smithy/util-endpoints': 2.0.3 + '@smithy/util-middleware': 3.0.3 + '@smithy/util-retry': 3.0.2 + '@smithy/util-utf8': 3.0.0 + tslib: 2.6.3 + transitivePeerDependencies: + - '@aws-sdk/client-sts' + - aws-crt + dev: false + /@aws-sdk/client-sso/3.423.0: resolution: {integrity: sha512-znIufHkwhCIePgaYciIs3x/+BpzR57CZzbCKHR9+oOvGyufEPPpUT5bFLvbwTgfiVkTjuk6sG/ES3U5Bc+xtrA==} engines: {node: '>=14.0.0'} @@ -13120,55 +13171,7 @@ packages: dependencies: '@aws-crypto/sha256-browser': 5.2.0 '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/client-sso-oidc': 3.600.0 - '@aws-sdk/core': 3.598.0 - '@aws-sdk/credential-provider-node': 3.600.0_f7n47caigsrjd2lr2szmwfuee4 - '@aws-sdk/middleware-host-header': 3.598.0 - '@aws-sdk/middleware-logger': 3.598.0 - '@aws-sdk/middleware-recursion-detection': 3.598.0 - '@aws-sdk/middleware-user-agent': 3.598.0 - '@aws-sdk/region-config-resolver': 3.598.0 - '@aws-sdk/types': 3.598.0 - '@aws-sdk/util-endpoints': 3.598.0 - '@aws-sdk/util-user-agent-browser': 3.598.0 - '@aws-sdk/util-user-agent-node': 3.598.0 - '@smithy/config-resolver': 3.0.3 - '@smithy/core': 2.2.3 - '@smithy/fetch-http-handler': 3.2.1 - '@smithy/hash-node': 3.0.2 - '@smithy/invalid-dependency': 3.0.2 - '@smithy/middleware-content-length': 3.0.2 - '@smithy/middleware-endpoint': 3.0.4 - '@smithy/middleware-retry': 3.0.6 - '@smithy/middleware-serde': 3.0.3 - '@smithy/middleware-stack': 3.0.3 - '@smithy/node-config-provider': 3.1.3 - '@smithy/node-http-handler': 3.1.2 - '@smithy/protocol-http': 4.0.3 - '@smithy/smithy-client': 3.1.6 - '@smithy/types': 3.3.0 - '@smithy/url-parser': 3.0.3 - '@smithy/util-base64': 3.0.0 - '@smithy/util-body-length-browser': 3.0.0 - '@smithy/util-body-length-node': 3.0.0 - '@smithy/util-defaults-mode-browser': 3.0.6 - '@smithy/util-defaults-mode-node': 3.0.6 - '@smithy/util-endpoints': 2.0.3 - '@smithy/util-middleware': 3.0.3 - '@smithy/util-retry': 3.0.2 - '@smithy/util-utf8': 3.0.0 - tslib: 2.6.3 - transitivePeerDependencies: - - aws-crt - dev: false - - /@aws-sdk/client-sts/3.600.0_dseaa2p5u2yk67qiepewcq3hkq: - resolution: {integrity: sha512-KQG97B7LvTtTiGmjlrG1LRAY8wUvCQzrmZVV5bjrJ/1oXAU7DITYwVbSJeX9NWg6hDuSk0VE3MFwIXS2SvfLIA==} - engines: {node: '>=16.0.0'} - dependencies: - '@aws-crypto/sha256-browser': 5.2.0 - '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/client-sso-oidc': 3.600.0 + '@aws-sdk/client-sso-oidc': 3.600.0_tdq3komn4zwyd65w7klbptsu34 '@aws-sdk/core': 3.598.0 '@aws-sdk/credential-provider-node': 3.600.0_f7n47caigsrjd2lr2szmwfuee4 '@aws-sdk/middleware-host-header': 3.598.0 @@ -13207,7 +13210,6 @@ packages: '@smithy/util-utf8': 3.0.0 tslib: 2.6.3 transitivePeerDependencies: - - '@aws-sdk/client-sso-oidc' - aws-crt dev: false @@ -17537,7 +17539,7 @@ packages: '@aws-sdk/client-sns': 3.423.0 '@aws-sdk/client-sqs': 3.423.0 '@aws-sdk/client-ssm': 3.423.0 - '@aws-sdk/client-sts': 3.600.0_dseaa2p5u2yk67qiepewcq3hkq + '@aws-sdk/client-sts': 3.600.0 '@aws-sdk/s3-request-presigner': 3.609.0 '@pipedream/helper_functions': 0.3.12 '@pipedream/platform': 1.6.6 From 5127e5fc1d227450cc44e931b61f8fabdf90577a Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Wed, 9 Oct 2024 13:33:01 -0400 Subject: [PATCH 7/8] remove console.log --- components/nile_database/sources/common/base.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/nile_database/sources/common/base.mjs b/components/nile_database/sources/common/base.mjs index 0b45fedfa043c..9d43dfb69515b 100644 --- a/components/nile_database/sources/common/base.mjs +++ b/components/nile_database/sources/common/base.mjs @@ -38,7 +38,7 @@ export default { const results = await resourceFn({ workspace: this.workspace, database: this.database, - }); console.log(results); + }); for (const item of results) { const meta = this.generateMeta(item); From 9fb079c70b667d500f89879a5a36e7e0d379c980 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Wed, 9 Oct 2024 14:42:31 -0400 Subject: [PATCH 8/8] update execute-query --- .../actions/execute-query/execute-query.mjs | 37 +++++++++---------- .../nile_database/nile_database.app.mjs | 24 +++++++++++- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/components/nile_database/actions/execute-query/execute-query.mjs b/components/nile_database/actions/execute-query/execute-query.mjs index c42bbe25b8ecc..1b8c0ca926268 100644 --- a/components/nile_database/actions/execute-query/execute-query.mjs +++ b/components/nile_database/actions/execute-query/execute-query.mjs @@ -8,32 +8,28 @@ export default { type: "action", props: { nile, + workspace: { + propDefinition: [ + nile, + "workspace", + ], + }, + database: { + propDefinition: [ + nile, + "database", + ], + }, user: { type: "string", label: "Username", - description: "The username of the database user", + description: "The username or userId of the database user. Note: Credentials are generated in the Nile Dashboard under Settings -> Credentials", }, password: { type: "string", label: "Password", description: "The password of the database user", }, - host: { - type: "string", - label: "Host", - description: "The host of the database", - }, - port: { - type: "string", - label: "Port", - description: "The port to connect to the database. Example: `5432`", - }, - database: { - propDefinition: [ - nile, - "database", - ], - }, query: { type: "string", label: "Query", @@ -44,8 +40,11 @@ export default { const config = { user: this.user, password: this.password, - host: this.host, - port: this.port, + host: await this.nile.getHost({ + workspace: this.workspace, + database: this.database, + }), + port: "5432", database: this.database, }; const data = await this.nile.executeQuery(config, this.query); diff --git a/components/nile_database/nile_database.app.mjs b/components/nile_database/nile_database.app.mjs index 38ef3b7137b4d..0bad4b5983bc6 100644 --- a/components/nile_database/nile_database.app.mjs +++ b/components/nile_database/nile_database.app.mjs @@ -31,8 +31,7 @@ export default { async _getBaseUrl({ workspace, database, ...opts }) { - const { apiHost } = await this._makeRequest({ - url: `${this._globalBaseUrl()}/workspaces/${workspace}/databases/${database}`, + const { apiHost } = await this.getDatabase({ workspace, database, ...opts, @@ -59,12 +58,33 @@ export default { ...opts, }); }, + getDatabase({ + workspace, database, ...opts + }) { + return this._makeRequest({ + url: `${this._globalBaseUrl()}/workspaces/${workspace}/databases/${database}`, + workspace, + database, + ...opts, + }); + }, getAuthenticatedUser(opts = {}) { return this._makeRequest({ url: `${this._globalBaseUrl()}/developers/me/full`, ...opts, }); }, + async getHost({ + workspace, database, ...opts + }) { + const { dbHost } = await this.getDatabase({ + workspace, + database, + ...opts, + }); + const host = dbHost.match(/postgres:\/\/([^/]+)\//); + return host[1]; + }, listUsers({ workspace, database, ...opts }) {