From dbbb3b05f04fcf48296734e6aa70d2d84655b915 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Tue, 18 Mar 2025 11:09:16 -0400 Subject: [PATCH 1/9] new actions --- .../actions/create-event/create-event.mjs | 58 +++++ .../get-search-job-status.mjs | 36 +++ .../splunk/actions/run-search/run-search.mjs | 48 ++++ components/splunk/package.json | 8 +- .../new-alert-instant/new-alert-instant.mjs | 117 ++++++++++ .../new-event-instant/new-event-instant.mjs | 217 ++++++++++++++++++ .../new-search-result/new-search-result.mjs | 191 +++++++++++++++ components/splunk/splunk.app.mjs | 116 +++++++++- 8 files changed, 785 insertions(+), 6 deletions(-) create mode 100644 components/splunk/actions/create-event/create-event.mjs create mode 100644 components/splunk/actions/get-search-job-status/get-search-job-status.mjs create mode 100644 components/splunk/actions/run-search/run-search.mjs create mode 100644 components/splunk/sources/new-alert-instant/new-alert-instant.mjs create mode 100644 components/splunk/sources/new-event-instant/new-event-instant.mjs create mode 100644 components/splunk/sources/new-search-result/new-search-result.mjs diff --git a/components/splunk/actions/create-event/create-event.mjs b/components/splunk/actions/create-event/create-event.mjs new file mode 100644 index 0000000000000..52690136556ff --- /dev/null +++ b/components/splunk/actions/create-event/create-event.mjs @@ -0,0 +1,58 @@ +import splunk from "../../splunk.app.mjs"; + +export default { + key: "splunk-create-event", + name: "Create Event", + description: "Sends a new event to a specified Splunk index. [See the documentation](https://docs.splunk.com/Documentation/Splunk/9.4.1/RESTREF/RESTinput#receivers.2Fsimple)", + version: "0.0.1", + type: "action", + props: { + splunk, + selfSigned: { + propDefinition: [ + splunk, + "selfSigned", + ], + }, + indexName: { + propDefinition: [ + splunk, + "indexName", + (c) => ({ + selfSigned: c.selfSigned, + }), + ], + }, + eventData: { + type: "string", + label: "Event Data", + description: "The data of the event to send to the Splunk index. Raw event text. This is the entirety of the HTTP request body", + }, + source: { + type: "string", + label: "Source", + description: "The source value to fill in the metadata for this input's events", + optional: true, + }, + sourcetype: { + type: "string", + label: "Sourcetype", + description: "The sourcetype to apply to events from this input", + optional: true, + }, + }, + async run({ $ }) { + const response = await this.splunk.sendEvent({ + $, + selfSigned: this.selfSigned, + params: { + index: this.indexName, + source: this.source, + sourcetype: this.sourcetype, + }, + data: this.eventData, + }); + $.export("$summary", `Event sent to index ${this.indexName} successfully`); + return response; + }, +}; diff --git a/components/splunk/actions/get-search-job-status/get-search-job-status.mjs b/components/splunk/actions/get-search-job-status/get-search-job-status.mjs new file mode 100644 index 0000000000000..3bc17e07a54ae --- /dev/null +++ b/components/splunk/actions/get-search-job-status/get-search-job-status.mjs @@ -0,0 +1,36 @@ +import splunk from "../../splunk.app.mjs"; + +export default { + key: "splunk-get-search-job-status", + name: "Get Search Job Status", + description: "Retrieve the status of a previously executed Splunk search job. [See the documentation](https://docs.splunk.com/Documentation/Splunk/9.4.1/RESTREF/RESTsearch#search.2Fjobs)", + version: "0.0.1", + type: "action", + props: { + splunk, + selfSigned: { + propDefinition: [ + splunk, + "selfSigned", + ], + }, + searchId: { + propDefinition: [ + splunk, + "searchId", + (c) => ({ + selfSigned: c.selfSigned, + }), + ], + }, + }, + async run({ $ }) { + const response = await this.splunk.getSearchJobStatus({ + $, + selfSigned: this.selfSigned, + searchId: this.searchId, + }); + $.export("$summary", `Successfully retrieved status for job ID ${this.searchId}`); + return response; + }, +}; diff --git a/components/splunk/actions/run-search/run-search.mjs b/components/splunk/actions/run-search/run-search.mjs new file mode 100644 index 0000000000000..6ff3e18a87916 --- /dev/null +++ b/components/splunk/actions/run-search/run-search.mjs @@ -0,0 +1,48 @@ +import splunk from "../../splunk.app.mjs"; + +export default { + key: "splunk-run-search", + name: "Run Search", + description: "Executes a Splunk search query and returns the results. [See the documentation](https://docs.splunk.com/Documentation/Splunk/9.4.1/RESTREF/RESTsearch#search.2Fjobs)", + version: "0.0.1", + type: "action", + props: { + splunk, + selfSigned: { + propDefinition: [ + splunk, + "selfSigned", + ], + }, + query: { + type: "string", + label: "Search Query", + description: "The Splunk search query. Example: `search *`. [See the documentation](https://docs.splunk.com/Documentation/Splunk/9.4.1/SearchReference/Search) for more information about search command sytax.", + }, + earliestTime: { + type: "string", + label: "Earliest Time", + description: "Specify a time string. Sets the earliest (inclusive), respectively, time bounds for the search. The time string can be either a UTC time (with fractional seconds), a relative time specifier (to now) or a formatted time string. Refer to [Time modifiers](https://docs.splunk.com/Documentation/Splunk/9.4.1/SearchReference/SearchTimeModifiers) for search for information and examples of specifying a time string.", + optional: true, + }, + latestTime: { + type: "string", + label: "Latest Time", + description: " Specify a time string. Sets the latest (exclusive), respectively, time bounds for the search. The time string can be either a UTC time (with fractional seconds), a relative time specifier (to now) or a formatted time string. Refer to [Time modifiers](https://docs.splunk.com/Documentation/Splunk/9.4.1/SearchReference/SearchTimeModifiers) for search for information and examples of specifying a time string.", + optional: true, + }, + }, + async run({ $ }) { + const response = await this.splunk.executeSearchQuery({ + $, + selfSigned: this.selfSigned, + data: { + search: this.query, + earliest_time: this.earliestTime, + latest_time: this.latestTime, + }, + }); + $.export("$summary", `Executed Splunk search query: ${this.query}`); + return response; + }, +}; diff --git a/components/splunk/package.json b/components/splunk/package.json index 8fee8316f5794..19ffba9e752ce 100644 --- a/components/splunk/package.json +++ b/components/splunk/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/splunk", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream Splunk Components", "main": "splunk.app.mjs", "keywords": [ @@ -11,5 +11,9 @@ "author": "Pipedream (https://pipedream.com/)", "publishConfig": { "access": "public" + }, + "dependencies": { + "@pipedream/platform": "^3.0.3", + "https": "^1.0.0" } -} \ No newline at end of file +} diff --git a/components/splunk/sources/new-alert-instant/new-alert-instant.mjs b/components/splunk/sources/new-alert-instant/new-alert-instant.mjs new file mode 100644 index 0000000000000..b69dd340e2534 --- /dev/null +++ b/components/splunk/sources/new-alert-instant/new-alert-instant.mjs @@ -0,0 +1,117 @@ +import splunk from "../../splunk.app.mjs"; + +export default { + key: "splunk-new-alert-instant", + name: "Splunk New Alert (Instant)", + description: "Emit new event when a saved search alert is triggered in Splunk. [See the documentation]()", + version: "0.0.{{ts}}", + type: "source", + dedupe: "unique", + props: { + splunk, + alertName: { + propDefinition: [ + "splunk", + "alertName", + ], + }, + pollingInterval: { + propDefinition: [ + "splunk", + "pollingInterval", + ], + }, + http: { + type: "$.interface.http", + customResponse: false, + }, + db: "$.service.db", + }, + hooks: { + async deploy() { + const currentTime = Math.floor(Date.now() / 1000); + const lastRunTime = currentTime - (this.pollingInterval || 60); + await this.db.set("lastRunTime", lastRunTime); + }, + async activate() { + // No webhook creation needed for polling + }, + async deactivate() { + // No webhook deletion needed for polling + }, + }, + methods: { + async getSearchResults(jobId) { + let results = []; + let offset = 0; + const count = 100; + + while (true) { + const response = await this.splunk._makeRequest({ + method: "GET", + path: `search/jobs/${jobId}/results`, + params: { + count, + offset, + output_mode: "json", + }, + }); + + if (!response.results || response.results.length === 0) { + break; + } + + results = results.concat(response.results); + offset += count; + + if (response.results.length < count) { + break; + } + } + + return results; + }, + }, + async run() { + const lastRunTime = (await this.db.get("lastRunTime")) || 0; + const currentTime = Math.floor(Date.now() / 1000); + + let query = `_time > ${lastRunTime}`; + if (this.alertName) { + query = `savedsearch="${this.alertName}" AND _time > ${lastRunTime}`; + } + + const searchJob = await this.splunk.search({ + query, + earliestTime: lastRunTime, + latestTime: currentTime, + }); + + const jobId = searchJob.sid; + + let jobStatus; + do { + jobStatus = await this.splunk.getSearchJobStatus({ + jobId, + }); + if (jobStatus.status === "DONE") break; + await new Promise((resolve) => setTimeout(resolve, 1000)); + } while (true); + + const results = await this.methods.getSearchResults(jobId); + + for (const result of results) { + const eventData = result; + + this.$emit(eventData, { + id: result._raw || JSON.stringify(result), + summary: `Alert "${this.alertName || "All Alerts"}" triggered.`, + ts: result._time + ? result._time * 1000 + : Date.now(), + }); + } + + await this.db.set("lastRunTime", currentTime); + }, +}; diff --git a/components/splunk/sources/new-event-instant/new-event-instant.mjs b/components/splunk/sources/new-event-instant/new-event-instant.mjs new file mode 100644 index 0000000000000..5ca98118351ef --- /dev/null +++ b/components/splunk/sources/new-event-instant/new-event-instant.mjs @@ -0,0 +1,217 @@ +import splunk from "../../splunk.app.mjs"; +import crypto from "crypto"; + +export default { + key: "splunk-new-event-instant", + name: "Splunk New Event Instant", + description: "Emit a new event when a log event is added to a specified Splunk index. [See the documentation]()", + version: "0.0.{{ts}}", + type: "source", + dedupe: "unique", + props: { + splunk: { + type: "app", + app: "splunk", + }, + db: "$.service.db", + http: { + type: "$.interface.http", + customResponse: true, + }, + indexName: { + propDefinition: [ + splunk, + "indexName", + ], + }, + searchFilter: { + propDefinition: [ + splunk, + "searchFilter", + ], + optional: true, + }, + }, + hooks: { + async deploy() { + const query = `index=${this.indexName} ${this.searchFilter || ""} | head 50 | sort _time asc`; + const searchJob = await this.splunk.executeSearchQuery({ + query, + earliestTime: "-24h", + latestTime: "now", + }); + const jobId = searchJob.sid; + await this.db.set("searchJobId", jobId); + + // Poll for search completion + let jobStatus; + let attempts = 0; + const pollInterval = 5000; // 5 seconds + const maxAttempts = 10; + + while (attempts < maxAttempts) { + jobStatus = await this.splunk.getSearchJobStatus({ + jobId, + }); + if (jobStatus.entry[0].content.dispatchState === "DONE") { + break; + } + await new Promise((resolve) => setTimeout(resolve, pollInterval)); + attempts += 1; + } + + if (jobStatus.entry[0].content.dispatchState !== "DONE") { + throw new Error("Search job did not complete in time."); + } + + const results = await this.splunk._makeRequest({ + method: "GET", + path: `search/jobs/${jobId}/results`, + params: { + output_mode: "json", + }, + }); + + const events = results.results; + for (const event of events) { + const ts = Date.parse(event._time) || Date.now(); + this.$emit(event, { + id: event._time, + summary: `Historical event in index ${this.indexName}`, + ts, + }); + } + }, + async activate() { + const webhookUrl = this.http.endpoint; + const searchQuery = `search index=${this.indexName}${ + this.searchFilter + ? ` ${this.searchFilter}` + : "" + }`; + + const alertName = `Pipedream Alert ${Date.now()}`; + const savedSearch = await this.splunk._makeRequest({ + method: "POST", + path: "saved/searches", + data: { + "name": alertName, + "search": searchQuery, + "alert_type": "always", + "is_scheduled": true, + "dispatch.earliest_time": "-1m", + "dispatch.latest_time": "now", + "alert_actions": "webhook", + "action.webhook": true, + "action.webhook_url": webhookUrl, + }, + }); + + const webhookId = savedSearch.entry[0].name; + await this.db.set("webhookId", webhookId); + }, + async deactivate() { + const webhookId = await this.db.get("webhookId"); + if (webhookId) { + await this.splunk._makeRequest({ + method: "DELETE", + path: `saved/searches/${encodeURIComponent(webhookId)}`, + }); + await this.db.delete("webhookId"); + } + }, + }, + async run(event) { + const signature = event.headers["X-Splunk-Signature"]; + const secret = this.splunk.$auth.shared_secret; + const rawBody = event.body; + + if (signature && secret) { + const computedSignature = crypto + .createHmac("sha256", secret) + .update(rawBody) + .digest("hex"); + if (computedSignature !== signature) { + await this.http.respond({ + status: 401, + body: "Unauthorized", + }); + return; + } + } + + let eventData; + try { + eventData = JSON.parse(event.body); + } catch (error) { + await this.http.respond({ + status: 400, + body: "Invalid JSON", + }); + return; + } + + // Verify the index + if (eventData.index !== this.indexName) { + await this.http.respond({ + status: 200, + body: "Event does not match the specified index.", + }); + return; + } + + // Apply search filter if provided + if (this.searchFilter) { + const query = `index=${this.indexName} ${this.searchFilter} | head 1`; + const searchJob = await this.splunk.executeSearchQuery({ + query, + earliestTime: "-1m", + latestTime: "now", + }); + const jobId = searchJob.sid; + + // Poll for search result + let jobStatus; + let attempts = 0; + const pollInterval = 5000; // 5 seconds + const maxAttempts = 3; + let matchedEvent = null; + + while (attempts < maxAttempts) { + jobStatus = await this.splunk.getSearchJobStatus({ + jobId, + }); + if (jobStatus.entry[0].content.isDone) { + const results = jobStatus.entry[0].content.results; + if (results.length > 0) { + matchedEvent = results[0]; + } + break; + } + await new Promise((resolve) => setTimeout(resolve, pollInterval)); + attempts += 1; + } + + if (matchedEvent) { + const ts = Date.parse(matchedEvent._time) || Date.now(); + this.$emit(matchedEvent, { + id: matchedEvent._time, + summary: `New event in index ${this.indexName} matching filter`, + ts, + }); + } + } else { + const ts = Date.parse(eventData._time) || Date.now(); + this.$emit(eventData, { + id: eventData._time, + summary: `New event in index ${this.indexName}`, + ts, + }); + } + + await this.http.respond({ + status: 200, + body: "OK", + }); + }, +}; diff --git a/components/splunk/sources/new-search-result/new-search-result.mjs b/components/splunk/sources/new-search-result/new-search-result.mjs new file mode 100644 index 0000000000000..16d5a0f241b6f --- /dev/null +++ b/components/splunk/sources/new-search-result/new-search-result.mjs @@ -0,0 +1,191 @@ +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; +import splunkApp from "../../splunk.app.mjs"; + +export default { + key: "splunk-new-search-result", + name: "New Search Result in Splunk", + description: "Emit new events when a search query returns matching results in Splunk. [See the documentation]()", + version: "0.0.{{ts}}", + type: "source", + dedupe: "unique", + props: { + splunk: { + type: "app", + app: "splunk", + }, + db: { + type: "$.service.db", + }, + query: { + propDefinition: [ + splunkApp, + "query", + ], + }, + pollingInterval: { + propDefinition: [ + splunkApp, + "pollingInterval", + ], + optional: true, + default: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: "$pollingInterval", + }, + }, + }, + methods: { + /** + * Executes the Splunk search query within the specified time range. + * @param {string} query - The Splunk search query. + * @param {string} earliestTime - The earliest time for the search. + * @param {string} latestTime - The latest time for the search. + * @returns {Promise} - An array of search results. + */ + async executeSearch(query, earliestTime, latestTime) { + const searchResponse = await this.splunk.executeSearchQuery({ + query, + earliestTime, + latestTime, + }); + + const jobId = searchResponse.sid; + + let jobStatus = await this.splunk.getSearchJobStatus({ + jobId, + }); + const maxRetries = 30; + let retries = 0; + const pollInterval = 2000; // 2 seconds + + while (jobStatus.status !== "DONE" && retries < maxRetries) { + await this.$sleep(pollInterval); + jobStatus = await this.splunk.getSearchJobStatus({ + jobId, + }); + retries += 1; + } + + if (jobStatus.status !== "DONE") { + throw new Error(`Search job ${jobId} did not complete within the expected time.`); + } + + const results = await this.splunk.getSearchResults({ + jobId, + }); + return results.results; + }, + + /** + * Retrieves the last run timestamp from the database. + * @returns {string} - ISO timestamp. + */ + async getLastRunTime() { + const lastRun = await this.db.get("lastRunTime"); + if (lastRun) { + return lastRun; + } + const now = new Date(); + now.setSeconds(now.getSeconds() - ( + this.pollingInterval || DEFAULT_POLLING_SOURCE_TIMER_INTERVAL)); + return now.toISOString(); + }, + + /** + * Updates the last run timestamp in the database. + * @param {string} timestamp - ISO timestamp. + */ + async updateLastRunTime(timestamp) { + await this.db.set("lastRunTime", timestamp); + }, + }, + hooks: { + async deploy() { + const latestTime = new Date().toISOString(); + const earliestTime = await this.getLastRunTime(); + + try { + const results = await this.executeSearch(this.query, earliestTime, latestTime); + + for (const result of results.slice(-50).reverse()) { + const eventId = result._raw + ? result._raw.toString() + : latestTime; + const eventSummary = result._raw + ? result._raw.substring(0, 100) + : "New Splunk Search Result"; + const eventTimestamp = result._time + ? new Date(result._time).getTime() + : Date.now(); + + this.$emit(result, { + id: eventId, + summary: eventSummary, + ts: eventTimestamp, + }); + } + + await this.updateLastRunTime(latestTime); + } catch (error) { + this.$emit({ + error: error.message, + }, { + summary: "Error during deploy hook", + ts: Date.now(), + }); + throw error; + } + }, + + async activate() { + // No webhook setup required for polling + }, + + async deactivate() { + // No webhook teardown required for polling + }, + }, + async run() { + try { + const latestTime = new Date().toISOString(); + const earliestTime = await this.getLastRunTime(); + + const results = await this.executeSearch(this.query, earliestTime, latestTime); + + if (results.length === 0) { + return; + } + + for (const result of results) { + const eventId = result._raw + ? result._raw.toString() + : latestTime; + const eventSummary = result._raw + ? result._raw.substring(0, 100) + : "New Splunk Search Result"; + const eventTimestamp = result._time + ? new Date(result._time).getTime() + : Date.now(); + + this.$emit(result, { + id: eventId, + summary: eventSummary, + ts: eventTimestamp, + }); + } + + await this.updateLastRunTime(latestTime); + } catch (error) { + this.$emit({ + error: error.message, + }, { + summary: "Error executing Splunk search", + ts: Date.now(), + }); + throw error; + } + }, +}; diff --git a/components/splunk/splunk.app.mjs b/components/splunk/splunk.app.mjs index 13ba053e30d09..7b2f1a021a3a1 100644 --- a/components/splunk/splunk.app.mjs +++ b/components/splunk/splunk.app.mjs @@ -1,11 +1,119 @@ +import { axios } from "@pipedream/platform"; +import https from "https"; +const DEFAULT_LIMIT = 20; + export default { type: "app", app: "splunk", - propDefinitions: {}, + propDefinitions: { + searchId: { + type: "string", + label: "Search ID", + description: "The ID of the Splunk search job to retrieve status for", + async options({ + selfSigned, page, + }) { + const { entry } = await this.listSearches({ + selfSigned, + params: { + count: DEFAULT_LIMIT, + offset: DEFAULT_LIMIT * page, + }, + }); + return entry?.map(({ + name: label, content, + }) => ({ + label, + value: content.sid, + })) || []; + }, + }, + indexName: { + type: "string", + label: "Index Name", + description: "The name of the Splunk index", + async options({ + selfSigned, page, + }) { + const { entry } = await this.listIndexes({ + selfSigned, + params: { + count: DEFAULT_LIMIT, + offset: DEFAULT_LIMIT * page, + }, + }); + return entry?.map(({ name }) => name) || []; + }, + }, + selfSigned: { + type: "boolean", + label: "Self Signed", + description: "Set to `true` if your instance is using a self-signed certificate", + }, + }, methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); + _baseUrl() { + return `${this.$auth.api_url}:${this.$auth.api_port}`; + }, + _makeRequest({ + $ = this, + path, + params, + selfSigned = false, + ...otherOpts + }) { + const config = { + ...otherOpts, + debug: true, + url: `${this._baseUrl()}${path}`, + headers: { + Authorization: `Bearer ${this.$auth.api_token}`, + }, + params: { + ...params, + output_mode: "json", + }, + }; + if (selfSigned) { + config.httpsAgent = new https.Agent({ + rejectUnauthorized: false, + }); + } + return axios($, config); + }, + listSearches(opts = {}) { + return this._makeRequest({ + path: "/services/search/jobs", + ...opts, + }); + }, + listIndexes(opts = {}) { + return this._makeRequest({ + path: "/services/data/indexes", + ...opts, + }); + }, + executeSearchQuery(opts = {}) { + return this._makeRequest({ + method: "POST", + path: "/services/search/jobs", + ...opts, + }); + }, + getSearchJobStatus({ + searchId, ...opts + }) { + return this._makeRequest({ + path: `/services/search/jobs/${searchId}`, + ...opts, + }); + }, + sendEvent(opts = {}) { + return this._makeRequest({ + method: "POST", + path: "/services/receivers/simple", + ...opts, + }); }, }, }; From 008d4c7965a7592fd1afe2c7d937f2d00690e596 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Wed, 19 Mar 2025 13:35:08 -0400 Subject: [PATCH 2/9] new components --- .../get-search-job-status.mjs | 8 +- .../splunk/actions/run-search/run-search.mjs | 7 +- components/splunk/sources/common/base.mjs | 36 +++ .../new-alert-fired/new-alert-fired.mjs | 39 ++++ .../new-alert-instant/new-alert-instant.mjs | 117 ---------- .../new-event-instant/new-event-instant.mjs | 217 ------------------ .../new-search-event/new-search-event.mjs | 43 ++++ .../new-search-result/new-search-result.mjs | 205 +++-------------- components/splunk/splunk.app.mjs | 101 +++++++- 9 files changed, 243 insertions(+), 530 deletions(-) create mode 100644 components/splunk/sources/common/base.mjs create mode 100644 components/splunk/sources/new-alert-fired/new-alert-fired.mjs delete mode 100644 components/splunk/sources/new-alert-instant/new-alert-instant.mjs delete mode 100644 components/splunk/sources/new-event-instant/new-event-instant.mjs create mode 100644 components/splunk/sources/new-search-event/new-search-event.mjs diff --git a/components/splunk/actions/get-search-job-status/get-search-job-status.mjs b/components/splunk/actions/get-search-job-status/get-search-job-status.mjs index 3bc17e07a54ae..28feb49c398d8 100644 --- a/components/splunk/actions/get-search-job-status/get-search-job-status.mjs +++ b/components/splunk/actions/get-search-job-status/get-search-job-status.mjs @@ -14,10 +14,10 @@ export default { "selfSigned", ], }, - searchId: { + jobId: { propDefinition: [ splunk, - "searchId", + "jobId", (c) => ({ selfSigned: c.selfSigned, }), @@ -28,9 +28,9 @@ export default { const response = await this.splunk.getSearchJobStatus({ $, selfSigned: this.selfSigned, - searchId: this.searchId, + jobId: this.jobId, }); - $.export("$summary", `Successfully retrieved status for job ID ${this.searchId}`); + $.export("$summary", `Successfully retrieved status for job ID ${this.jobId}`); return response; }, }; diff --git a/components/splunk/actions/run-search/run-search.mjs b/components/splunk/actions/run-search/run-search.mjs index 6ff3e18a87916..a18504ee306f7 100644 --- a/components/splunk/actions/run-search/run-search.mjs +++ b/components/splunk/actions/run-search/run-search.mjs @@ -15,9 +15,10 @@ export default { ], }, query: { - type: "string", - label: "Search Query", - description: "The Splunk search query. Example: `search *`. [See the documentation](https://docs.splunk.com/Documentation/Splunk/9.4.1/SearchReference/Search) for more information about search command sytax.", + propDefinition: [ + splunk, + "query", + ], }, earliestTime: { type: "string", diff --git a/components/splunk/sources/common/base.mjs b/components/splunk/sources/common/base.mjs new file mode 100644 index 0000000000000..2e4dcda49dda0 --- /dev/null +++ b/components/splunk/sources/common/base.mjs @@ -0,0 +1,36 @@ +import splunk from "../../splunk.app.mjs"; +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; + +export default { + props: { + splunk, + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + db: "$.service.db", + selfSigned: { + propDefinition: [ + splunk, + "selfSigned", + ], + }, + }, + methods: { + async getRecentJobIds() { + const results = this.splunk.paginate({ + resourceFn: this.splunk.listJobs, + args: { + selfSigned: this.selfSigned, + }, + }); + const jobIds = []; + for await (const job of results) { + jobIds.push(job.content.sid); + } + return jobIds; + }, + }, +}; diff --git a/components/splunk/sources/new-alert-fired/new-alert-fired.mjs b/components/splunk/sources/new-alert-fired/new-alert-fired.mjs new file mode 100644 index 0000000000000..e2d8394ce636e --- /dev/null +++ b/components/splunk/sources/new-alert-fired/new-alert-fired.mjs @@ -0,0 +1,39 @@ +import common from "../common/base.mjs"; + +export default { + ...common, + key: "splunk-new-alert-fired", + name: "New Alert Fired", + description: "Emit new event when a new alert is triggered in Splunk. [See the documentation](https://docs.splunk.com/Documentation/Splunk/9.4.1/RESTREF/RESTsearch#alerts.2Ffired_alerts)", + version: "0.0.1", + type: "source", + dedupe: "unique", + methods: { + ...common.methods, + generateMeta(alert) { + return { + id: alert.id, + summary: `New Alert Fired: ${alert.name}`, + ts: Date.now(), + }; + }, + }, + async run() { + const results = this.splunk.paginate({ + resourceFn: this.splunk.listFiredAlerts, + args: { + selfSigned: this.selfSigned, + }, + }); + + const alerts = []; + for await (const item of results) { + alerts.push(item); + } + + alerts.forEach((alert) => { + const meta = this.generateMeta(alert); + this.$emit(alert, meta); + }); + }, +}; diff --git a/components/splunk/sources/new-alert-instant/new-alert-instant.mjs b/components/splunk/sources/new-alert-instant/new-alert-instant.mjs deleted file mode 100644 index b69dd340e2534..0000000000000 --- a/components/splunk/sources/new-alert-instant/new-alert-instant.mjs +++ /dev/null @@ -1,117 +0,0 @@ -import splunk from "../../splunk.app.mjs"; - -export default { - key: "splunk-new-alert-instant", - name: "Splunk New Alert (Instant)", - description: "Emit new event when a saved search alert is triggered in Splunk. [See the documentation]()", - version: "0.0.{{ts}}", - type: "source", - dedupe: "unique", - props: { - splunk, - alertName: { - propDefinition: [ - "splunk", - "alertName", - ], - }, - pollingInterval: { - propDefinition: [ - "splunk", - "pollingInterval", - ], - }, - http: { - type: "$.interface.http", - customResponse: false, - }, - db: "$.service.db", - }, - hooks: { - async deploy() { - const currentTime = Math.floor(Date.now() / 1000); - const lastRunTime = currentTime - (this.pollingInterval || 60); - await this.db.set("lastRunTime", lastRunTime); - }, - async activate() { - // No webhook creation needed for polling - }, - async deactivate() { - // No webhook deletion needed for polling - }, - }, - methods: { - async getSearchResults(jobId) { - let results = []; - let offset = 0; - const count = 100; - - while (true) { - const response = await this.splunk._makeRequest({ - method: "GET", - path: `search/jobs/${jobId}/results`, - params: { - count, - offset, - output_mode: "json", - }, - }); - - if (!response.results || response.results.length === 0) { - break; - } - - results = results.concat(response.results); - offset += count; - - if (response.results.length < count) { - break; - } - } - - return results; - }, - }, - async run() { - const lastRunTime = (await this.db.get("lastRunTime")) || 0; - const currentTime = Math.floor(Date.now() / 1000); - - let query = `_time > ${lastRunTime}`; - if (this.alertName) { - query = `savedsearch="${this.alertName}" AND _time > ${lastRunTime}`; - } - - const searchJob = await this.splunk.search({ - query, - earliestTime: lastRunTime, - latestTime: currentTime, - }); - - const jobId = searchJob.sid; - - let jobStatus; - do { - jobStatus = await this.splunk.getSearchJobStatus({ - jobId, - }); - if (jobStatus.status === "DONE") break; - await new Promise((resolve) => setTimeout(resolve, 1000)); - } while (true); - - const results = await this.methods.getSearchResults(jobId); - - for (const result of results) { - const eventData = result; - - this.$emit(eventData, { - id: result._raw || JSON.stringify(result), - summary: `Alert "${this.alertName || "All Alerts"}" triggered.`, - ts: result._time - ? result._time * 1000 - : Date.now(), - }); - } - - await this.db.set("lastRunTime", currentTime); - }, -}; diff --git a/components/splunk/sources/new-event-instant/new-event-instant.mjs b/components/splunk/sources/new-event-instant/new-event-instant.mjs deleted file mode 100644 index 5ca98118351ef..0000000000000 --- a/components/splunk/sources/new-event-instant/new-event-instant.mjs +++ /dev/null @@ -1,217 +0,0 @@ -import splunk from "../../splunk.app.mjs"; -import crypto from "crypto"; - -export default { - key: "splunk-new-event-instant", - name: "Splunk New Event Instant", - description: "Emit a new event when a log event is added to a specified Splunk index. [See the documentation]()", - version: "0.0.{{ts}}", - type: "source", - dedupe: "unique", - props: { - splunk: { - type: "app", - app: "splunk", - }, - db: "$.service.db", - http: { - type: "$.interface.http", - customResponse: true, - }, - indexName: { - propDefinition: [ - splunk, - "indexName", - ], - }, - searchFilter: { - propDefinition: [ - splunk, - "searchFilter", - ], - optional: true, - }, - }, - hooks: { - async deploy() { - const query = `index=${this.indexName} ${this.searchFilter || ""} | head 50 | sort _time asc`; - const searchJob = await this.splunk.executeSearchQuery({ - query, - earliestTime: "-24h", - latestTime: "now", - }); - const jobId = searchJob.sid; - await this.db.set("searchJobId", jobId); - - // Poll for search completion - let jobStatus; - let attempts = 0; - const pollInterval = 5000; // 5 seconds - const maxAttempts = 10; - - while (attempts < maxAttempts) { - jobStatus = await this.splunk.getSearchJobStatus({ - jobId, - }); - if (jobStatus.entry[0].content.dispatchState === "DONE") { - break; - } - await new Promise((resolve) => setTimeout(resolve, pollInterval)); - attempts += 1; - } - - if (jobStatus.entry[0].content.dispatchState !== "DONE") { - throw new Error("Search job did not complete in time."); - } - - const results = await this.splunk._makeRequest({ - method: "GET", - path: `search/jobs/${jobId}/results`, - params: { - output_mode: "json", - }, - }); - - const events = results.results; - for (const event of events) { - const ts = Date.parse(event._time) || Date.now(); - this.$emit(event, { - id: event._time, - summary: `Historical event in index ${this.indexName}`, - ts, - }); - } - }, - async activate() { - const webhookUrl = this.http.endpoint; - const searchQuery = `search index=${this.indexName}${ - this.searchFilter - ? ` ${this.searchFilter}` - : "" - }`; - - const alertName = `Pipedream Alert ${Date.now()}`; - const savedSearch = await this.splunk._makeRequest({ - method: "POST", - path: "saved/searches", - data: { - "name": alertName, - "search": searchQuery, - "alert_type": "always", - "is_scheduled": true, - "dispatch.earliest_time": "-1m", - "dispatch.latest_time": "now", - "alert_actions": "webhook", - "action.webhook": true, - "action.webhook_url": webhookUrl, - }, - }); - - const webhookId = savedSearch.entry[0].name; - await this.db.set("webhookId", webhookId); - }, - async deactivate() { - const webhookId = await this.db.get("webhookId"); - if (webhookId) { - await this.splunk._makeRequest({ - method: "DELETE", - path: `saved/searches/${encodeURIComponent(webhookId)}`, - }); - await this.db.delete("webhookId"); - } - }, - }, - async run(event) { - const signature = event.headers["X-Splunk-Signature"]; - const secret = this.splunk.$auth.shared_secret; - const rawBody = event.body; - - if (signature && secret) { - const computedSignature = crypto - .createHmac("sha256", secret) - .update(rawBody) - .digest("hex"); - if (computedSignature !== signature) { - await this.http.respond({ - status: 401, - body: "Unauthorized", - }); - return; - } - } - - let eventData; - try { - eventData = JSON.parse(event.body); - } catch (error) { - await this.http.respond({ - status: 400, - body: "Invalid JSON", - }); - return; - } - - // Verify the index - if (eventData.index !== this.indexName) { - await this.http.respond({ - status: 200, - body: "Event does not match the specified index.", - }); - return; - } - - // Apply search filter if provided - if (this.searchFilter) { - const query = `index=${this.indexName} ${this.searchFilter} | head 1`; - const searchJob = await this.splunk.executeSearchQuery({ - query, - earliestTime: "-1m", - latestTime: "now", - }); - const jobId = searchJob.sid; - - // Poll for search result - let jobStatus; - let attempts = 0; - const pollInterval = 5000; // 5 seconds - const maxAttempts = 3; - let matchedEvent = null; - - while (attempts < maxAttempts) { - jobStatus = await this.splunk.getSearchJobStatus({ - jobId, - }); - if (jobStatus.entry[0].content.isDone) { - const results = jobStatus.entry[0].content.results; - if (results.length > 0) { - matchedEvent = results[0]; - } - break; - } - await new Promise((resolve) => setTimeout(resolve, pollInterval)); - attempts += 1; - } - - if (matchedEvent) { - const ts = Date.parse(matchedEvent._time) || Date.now(); - this.$emit(matchedEvent, { - id: matchedEvent._time, - summary: `New event in index ${this.indexName} matching filter`, - ts, - }); - } - } else { - const ts = Date.parse(eventData._time) || Date.now(); - this.$emit(eventData, { - id: eventData._time, - summary: `New event in index ${this.indexName}`, - ts, - }); - } - - await this.http.respond({ - status: 200, - body: "OK", - }); - }, -}; diff --git a/components/splunk/sources/new-search-event/new-search-event.mjs b/components/splunk/sources/new-search-event/new-search-event.mjs new file mode 100644 index 0000000000000..fe7882a41d42d --- /dev/null +++ b/components/splunk/sources/new-search-event/new-search-event.mjs @@ -0,0 +1,43 @@ +import common from "../common/base.mjs"; +import md5 from "md5"; + +export default { + ...common, + key: "splunk-new-search-event", + name: "New Search Event", + description: "Emit new event when a new search event is created. [See the documentation](https://docs.splunk.com/Documentation/Splunk/9.4.1/RESTREF/RESTsearch#search.2Fv2.2Fjobs.2F.7Bsearch_id.7D.2Fevents)", + version: "0.0.1", + type: "source", + dedupe: "unique", + methods: { + ...common.methods, + generateMeta(event) { + return { + id: md5(JSON.stringify(event)), + summary: "New Search Event", + ts: Date.now(), + }; + }, + }, + async run() { + const jobIds = await this.getRecentJobIds(); + const events = []; + for (const jobId of jobIds) { + try { + const response = await this.splunk.getSearchEvents({ + selfSigned: this.selfSigned, + jobId, + }); + if (response?.results?.length) { + events.push(...response.results); + } + } catch { + console.log(`No events found for sid: ${jobId}`); + } + } + events.forEach((event) => { + const meta = this.generateMeta(event); + this.$emit(event, meta); + }); + }, +}; diff --git a/components/splunk/sources/new-search-result/new-search-result.mjs b/components/splunk/sources/new-search-result/new-search-result.mjs index 16d5a0f241b6f..8690c009262da 100644 --- a/components/splunk/sources/new-search-result/new-search-result.mjs +++ b/components/splunk/sources/new-search-result/new-search-result.mjs @@ -1,191 +1,42 @@ -import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; -import splunkApp from "../../splunk.app.mjs"; +import common from "../common/base.mjs"; export default { + ...common, key: "splunk-new-search-result", - name: "New Search Result in Splunk", - description: "Emit new events when a search query returns matching results in Splunk. [See the documentation]()", - version: "0.0.{{ts}}", + name: "New Search Result", + description: "Emit new events when a search returns results in Splunk. [See the documentation](https://docs.splunk.com/Documentation/Splunk/9.4.1/RESTREF/RESTsearch#saved.2Fsearches)", + version: "0.0.1", type: "source", dedupe: "unique", - props: { - splunk: { - type: "app", - app: "splunk", - }, - db: { - type: "$.service.db", - }, - query: { - propDefinition: [ - splunkApp, - "query", - ], - }, - pollingInterval: { - propDefinition: [ - splunkApp, - "pollingInterval", - ], - optional: true, - default: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, - }, - timer: { - type: "$.interface.timer", - default: { - intervalSeconds: "$pollingInterval", - }, - }, - }, methods: { - /** - * Executes the Splunk search query within the specified time range. - * @param {string} query - The Splunk search query. - * @param {string} earliestTime - The earliest time for the search. - * @param {string} latestTime - The latest time for the search. - * @returns {Promise} - An array of search results. - */ - async executeSearch(query, earliestTime, latestTime) { - const searchResponse = await this.splunk.executeSearchQuery({ - query, - earliestTime, - latestTime, - }); - - const jobId = searchResponse.sid; - - let jobStatus = await this.splunk.getSearchJobStatus({ - jobId, - }); - const maxRetries = 30; - let retries = 0; - const pollInterval = 2000; // 2 seconds - - while (jobStatus.status !== "DONE" && retries < maxRetries) { - await this.$sleep(pollInterval); - jobStatus = await this.splunk.getSearchJobStatus({ - jobId, - }); - retries += 1; - } - - if (jobStatus.status !== "DONE") { - throw new Error(`Search job ${jobId} did not complete within the expected time.`); - } - - const results = await this.splunk.getSearchResults({ - jobId, - }); - return results.results; - }, - - /** - * Retrieves the last run timestamp from the database. - * @returns {string} - ISO timestamp. - */ - async getLastRunTime() { - const lastRun = await this.db.get("lastRunTime"); - if (lastRun) { - return lastRun; - } - const now = new Date(); - now.setSeconds(now.getSeconds() - ( - this.pollingInterval || DEFAULT_POLLING_SOURCE_TIMER_INTERVAL)); - return now.toISOString(); - }, - - /** - * Updates the last run timestamp in the database. - * @param {string} timestamp - ISO timestamp. - */ - async updateLastRunTime(timestamp) { - await this.db.set("lastRunTime", timestamp); - }, - }, - hooks: { - async deploy() { - const latestTime = new Date().toISOString(); - const earliestTime = await this.getLastRunTime(); - - try { - const results = await this.executeSearch(this.query, earliestTime, latestTime); - - for (const result of results.slice(-50).reverse()) { - const eventId = result._raw - ? result._raw.toString() - : latestTime; - const eventSummary = result._raw - ? result._raw.substring(0, 100) - : "New Splunk Search Result"; - const eventTimestamp = result._time - ? new Date(result._time).getTime() - : Date.now(); - - this.$emit(result, { - id: eventId, - summary: eventSummary, - ts: eventTimestamp, - }); - } - - await this.updateLastRunTime(latestTime); - } catch (error) { - this.$emit({ - error: error.message, - }, { - summary: "Error during deploy hook", - ts: Date.now(), - }); - throw error; - } - }, - - async activate() { - // No webhook setup required for polling - }, - - async deactivate() { - // No webhook teardown required for polling + ...common.methods, + generateMeta(result) { + return { + id: result.sid, + summary: `New Search Results with SID: ${result.sid}`, + ts: Date.now(), + }; }, }, async run() { - try { - const latestTime = new Date().toISOString(); - const earliestTime = await this.getLastRunTime(); - - const results = await this.executeSearch(this.query, earliestTime, latestTime); - - if (results.length === 0) { - return; - } - - for (const result of results) { - const eventId = result._raw - ? result._raw.toString() - : latestTime; - const eventSummary = result._raw - ? result._raw.substring(0, 100) - : "New Splunk Search Result"; - const eventTimestamp = result._time - ? new Date(result._time).getTime() - : Date.now(); - - this.$emit(result, { - id: eventId, - summary: eventSummary, - ts: eventTimestamp, + const jobIds = await this.getRecentJobIds(); + const searchResults = []; + for (const jobId of jobIds) { + try { + const response = await this.splunk.getSearchResults({ + selfSigned: this.selfSigned, + jobId, }); + if (response?.results?.length) { + searchResults.push(...response.results); + } + } catch { + console.log(`No results found for sid: ${jobId}`); } - - await this.updateLastRunTime(latestTime); - } catch (error) { - this.$emit({ - error: error.message, - }, { - summary: "Error executing Splunk search", - ts: Date.now(), - }); - throw error; } + searchResults.forEach((result) => { + const meta = this.generateMeta(result); + this.$emit(result, meta); + }); }, }; diff --git a/components/splunk/splunk.app.mjs b/components/splunk/splunk.app.mjs index 7b2f1a021a3a1..a111e068a7221 100644 --- a/components/splunk/splunk.app.mjs +++ b/components/splunk/splunk.app.mjs @@ -1,19 +1,19 @@ import { axios } from "@pipedream/platform"; import https from "https"; -const DEFAULT_LIMIT = 20; +const DEFAULT_LIMIT = 50; export default { type: "app", app: "splunk", propDefinitions: { - searchId: { + jobId: { type: "string", label: "Search ID", description: "The ID of the Splunk search job to retrieve status for", async options({ selfSigned, page, }) { - const { entry } = await this.listSearches({ + const { entry } = await this.listJobs({ selfSigned, params: { count: DEFAULT_LIMIT, @@ -45,15 +45,37 @@ export default { return entry?.map(({ name }) => name) || []; }, }, + savedSearchName: { + type: "string", + label: "Saved Search Name", + description: "The name of a saved search", + async options({ + selfSigned, page, + }) { + const { entry } = await this.listSavedSearches({ + selfSigned, + params: { + count: DEFAULT_LIMIT, + offset: DEFAULT_LIMIT * page, + }, + }); + return entry?.map(({ name }) => name) || []; + }, + }, selfSigned: { type: "boolean", label: "Self Signed", description: "Set to `true` if your instance is using a self-signed certificate", }, + query: { + type: "string", + label: "Search Query", + description: "The Splunk search query. Example: `search *`. [See the documentation](https://docs.splunk.com/Documentation/Splunk/9.4.1/SearchReference/Search) for more information about search command sytax.", + }, }, methods: { _baseUrl() { - return `${this.$auth.api_url}:${this.$auth.api_port}`; + return `${this.$auth.api_url}:${this.$auth.api_port}/services`; }, _makeRequest({ $ = this, @@ -64,7 +86,6 @@ export default { }) { const config = { ...otherOpts, - debug: true, url: `${this._baseUrl()}${path}`, headers: { Authorization: `Bearer ${this.$auth.api_token}`, @@ -81,39 +102,95 @@ export default { } return axios($, config); }, - listSearches(opts = {}) { + listJobs(opts = {}) { return this._makeRequest({ - path: "/services/search/jobs", + path: "/search/jobs", ...opts, }); }, listIndexes(opts = {}) { return this._makeRequest({ - path: "/services/data/indexes", + path: "/data/indexes", + ...opts, + }); + }, + listSavedSearches(opts = {}) { + return this._makeRequest({ + path: "/saved/searches", + ...opts, + }); + }, + listFiredAlerts(opts = {}) { + return this._makeRequest({ + path: "/alerts/fired_alerts", ...opts, }); }, executeSearchQuery(opts = {}) { return this._makeRequest({ method: "POST", - path: "/services/search/jobs", + path: "/search/jobs", ...opts, }); }, getSearchJobStatus({ - searchId, ...opts + jobId, ...opts + }) { + return this._makeRequest({ + path: `/search/jobs/${jobId}`, + ...opts, + }); + }, + getSearchResults({ + jobId, ...opts }) { return this._makeRequest({ - path: `/services/search/jobs/${searchId}`, + path: `/search/v2/jobs/${jobId}/results`, + ...opts, + }); + }, + getSearchEvents({ + jobId, ...opts + }) { + return this._makeRequest({ + path: `/search/v2/jobs/${jobId}/events`, ...opts, }); }, sendEvent(opts = {}) { return this._makeRequest({ method: "POST", - path: "/services/receivers/simple", + path: "/receivers/simple", ...opts, }); }, + async *paginate({ + resourceFn, + args, + max, + }) { + args = { + ...args, + params: { + ...args?.params, + count: DEFAULT_LIMIT, + }, + }; + let hasMore, count = 0; + do { + const { + entry, paging, + } = await resourceFn(args); + for (const item of entry) { + yield item; + count++; + if (max && count >= max) { + return; + } + } + hasMore = paging.total > count; + args.params.offset += args.params.count; + } while (hasMore); + }, }, }; From c277a286816501715c1ab621a8cb8f66d0bee1cb Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Wed, 19 Mar 2025 13:38:03 -0400 Subject: [PATCH 3/9] package.json --- components/splunk/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/splunk/package.json b/components/splunk/package.json index 19ffba9e752ce..bc2b77303da01 100644 --- a/components/splunk/package.json +++ b/components/splunk/package.json @@ -14,6 +14,7 @@ }, "dependencies": { "@pipedream/platform": "^3.0.3", - "https": "^1.0.0" + "https": "^1.0.0", + "md5": "^2.3.0" } } From c6094bf7ccb774a0138f6ff3fcc42aebfd771400 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Wed, 19 Mar 2025 13:39:31 -0400 Subject: [PATCH 4/9] pnpm-lock.yaml --- pnpm-lock.yaml | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9a17809d5bc33..69e623534703b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7146,8 +7146,7 @@ importers: specifier: ^6.11.1 version: 6.13.1 - components/line_messaging_api: - specifiers: {} + components/line_messaging_api: {} components/linear: dependencies: @@ -12065,7 +12064,17 @@ importers: specifier: ^1.2.0 version: 1.6.6 - components/splunk: {} + components/splunk: + dependencies: + '@pipedream/platform': + specifier: ^3.0.3 + version: 3.0.3 + https: + specifier: ^1.0.0 + version: 1.0.0 + md5: + specifier: ^2.3.0 + version: 2.3.0 components/splunk_http_event_collector: {} @@ -23414,6 +23423,9 @@ packages: resolution: {integrity: sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==} engines: {node: '>= 14'} + https@1.0.0: + resolution: {integrity: sha512-4EC57ddXrkaF0x83Oj8sM6SLQHAWXw90Skqu2M4AEWENZ3F02dFJE/GARA8igO79tcgYqGrD7ae4f5L3um2lgg==} + human-signals@2.1.0: resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==} engines: {node: '>=10.17.0'} @@ -41015,6 +41027,8 @@ snapshots: transitivePeerDependencies: - supports-color + https@1.0.0: {} + human-signals@2.1.0: {} human-signals@5.0.0: {} From cddbf57ed1a772e3bf3657acc89dea954ecdce12 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Fri, 21 Mar 2025 17:44:12 -0400 Subject: [PATCH 5/9] use self_signed from $auth --- .../actions/create-event/create-event.mjs | 10 -------- .../get-search-job-status.mjs | 10 -------- .../splunk/actions/run-search/run-search.mjs | 7 ------ components/splunk/sources/common/base.mjs | 9 -------- .../new-alert-fired/new-alert-fired.mjs | 3 --- .../new-search-event/new-search-event.mjs | 1 - .../new-search-result/new-search-result.mjs | 1 - components/splunk/splunk.app.mjs | 23 ++++--------------- 8 files changed, 4 insertions(+), 60 deletions(-) diff --git a/components/splunk/actions/create-event/create-event.mjs b/components/splunk/actions/create-event/create-event.mjs index 52690136556ff..41871e1410cfd 100644 --- a/components/splunk/actions/create-event/create-event.mjs +++ b/components/splunk/actions/create-event/create-event.mjs @@ -8,19 +8,10 @@ export default { type: "action", props: { splunk, - selfSigned: { - propDefinition: [ - splunk, - "selfSigned", - ], - }, indexName: { propDefinition: [ splunk, "indexName", - (c) => ({ - selfSigned: c.selfSigned, - }), ], }, eventData: { @@ -44,7 +35,6 @@ export default { async run({ $ }) { const response = await this.splunk.sendEvent({ $, - selfSigned: this.selfSigned, params: { index: this.indexName, source: this.source, diff --git a/components/splunk/actions/get-search-job-status/get-search-job-status.mjs b/components/splunk/actions/get-search-job-status/get-search-job-status.mjs index 28feb49c398d8..ca089ae46a7de 100644 --- a/components/splunk/actions/get-search-job-status/get-search-job-status.mjs +++ b/components/splunk/actions/get-search-job-status/get-search-job-status.mjs @@ -8,26 +8,16 @@ export default { type: "action", props: { splunk, - selfSigned: { - propDefinition: [ - splunk, - "selfSigned", - ], - }, jobId: { propDefinition: [ splunk, "jobId", - (c) => ({ - selfSigned: c.selfSigned, - }), ], }, }, async run({ $ }) { const response = await this.splunk.getSearchJobStatus({ $, - selfSigned: this.selfSigned, jobId: this.jobId, }); $.export("$summary", `Successfully retrieved status for job ID ${this.jobId}`); diff --git a/components/splunk/actions/run-search/run-search.mjs b/components/splunk/actions/run-search/run-search.mjs index a18504ee306f7..ce3c4a01552e4 100644 --- a/components/splunk/actions/run-search/run-search.mjs +++ b/components/splunk/actions/run-search/run-search.mjs @@ -8,12 +8,6 @@ export default { type: "action", props: { splunk, - selfSigned: { - propDefinition: [ - splunk, - "selfSigned", - ], - }, query: { propDefinition: [ splunk, @@ -36,7 +30,6 @@ export default { async run({ $ }) { const response = await this.splunk.executeSearchQuery({ $, - selfSigned: this.selfSigned, data: { search: this.query, earliest_time: this.earliestTime, diff --git a/components/splunk/sources/common/base.mjs b/components/splunk/sources/common/base.mjs index 2e4dcda49dda0..a0088f7cdf538 100644 --- a/components/splunk/sources/common/base.mjs +++ b/components/splunk/sources/common/base.mjs @@ -11,20 +11,11 @@ export default { }, }, db: "$.service.db", - selfSigned: { - propDefinition: [ - splunk, - "selfSigned", - ], - }, }, methods: { async getRecentJobIds() { const results = this.splunk.paginate({ resourceFn: this.splunk.listJobs, - args: { - selfSigned: this.selfSigned, - }, }); const jobIds = []; for await (const job of results) { diff --git a/components/splunk/sources/new-alert-fired/new-alert-fired.mjs b/components/splunk/sources/new-alert-fired/new-alert-fired.mjs index e2d8394ce636e..044146b06d0f2 100644 --- a/components/splunk/sources/new-alert-fired/new-alert-fired.mjs +++ b/components/splunk/sources/new-alert-fired/new-alert-fired.mjs @@ -21,9 +21,6 @@ export default { async run() { const results = this.splunk.paginate({ resourceFn: this.splunk.listFiredAlerts, - args: { - selfSigned: this.selfSigned, - }, }); const alerts = []; diff --git a/components/splunk/sources/new-search-event/new-search-event.mjs b/components/splunk/sources/new-search-event/new-search-event.mjs index fe7882a41d42d..491e6ff9729fb 100644 --- a/components/splunk/sources/new-search-event/new-search-event.mjs +++ b/components/splunk/sources/new-search-event/new-search-event.mjs @@ -25,7 +25,6 @@ export default { for (const jobId of jobIds) { try { const response = await this.splunk.getSearchEvents({ - selfSigned: this.selfSigned, jobId, }); if (response?.results?.length) { diff --git a/components/splunk/sources/new-search-result/new-search-result.mjs b/components/splunk/sources/new-search-result/new-search-result.mjs index 8690c009262da..aacb6cf1ab795 100644 --- a/components/splunk/sources/new-search-result/new-search-result.mjs +++ b/components/splunk/sources/new-search-result/new-search-result.mjs @@ -24,7 +24,6 @@ export default { for (const jobId of jobIds) { try { const response = await this.splunk.getSearchResults({ - selfSigned: this.selfSigned, jobId, }); if (response?.results?.length) { diff --git a/components/splunk/splunk.app.mjs b/components/splunk/splunk.app.mjs index a111e068a7221..ce929d7354425 100644 --- a/components/splunk/splunk.app.mjs +++ b/components/splunk/splunk.app.mjs @@ -10,11 +10,8 @@ export default { type: "string", label: "Search ID", description: "The ID of the Splunk search job to retrieve status for", - async options({ - selfSigned, page, - }) { + async options({ page }) { const { entry } = await this.listJobs({ - selfSigned, params: { count: DEFAULT_LIMIT, offset: DEFAULT_LIMIT * page, @@ -32,11 +29,8 @@ export default { type: "string", label: "Index Name", description: "The name of the Splunk index", - async options({ - selfSigned, page, - }) { + async options({ page }) { const { entry } = await this.listIndexes({ - selfSigned, params: { count: DEFAULT_LIMIT, offset: DEFAULT_LIMIT * page, @@ -49,11 +43,8 @@ export default { type: "string", label: "Saved Search Name", description: "The name of a saved search", - async options({ - selfSigned, page, - }) { + async options({ page }) { const { entry } = await this.listSavedSearches({ - selfSigned, params: { count: DEFAULT_LIMIT, offset: DEFAULT_LIMIT * page, @@ -62,11 +53,6 @@ export default { return entry?.map(({ name }) => name) || []; }, }, - selfSigned: { - type: "boolean", - label: "Self Signed", - description: "Set to `true` if your instance is using a self-signed certificate", - }, query: { type: "string", label: "Search Query", @@ -81,7 +67,6 @@ export default { $ = this, path, params, - selfSigned = false, ...otherOpts }) { const config = { @@ -95,7 +80,7 @@ export default { output_mode: "json", }, }; - if (selfSigned) { + if (this.$auth.self_signed) { config.httpsAgent = new https.Agent({ rejectUnauthorized: false, }); From d0170bfe5ca32f4f3c11a99170949d56c06016b0 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Mon, 24 Mar 2025 16:19:13 -0400 Subject: [PATCH 6/9] updates --- .../splunk/actions/run-search/run-search.mjs | 24 ++---------- .../new-alert-fired/new-alert-fired.mjs | 37 +++++++++---------- .../sources/new-alert-fired/test-event.mjs | 30 +++++++++++++++ .../new-search-result/new-search-result.mjs | 33 ++++++++++------- components/splunk/splunk.app.mjs | 6 ++- 5 files changed, 76 insertions(+), 54 deletions(-) create mode 100644 components/splunk/sources/new-alert-fired/test-event.mjs diff --git a/components/splunk/actions/run-search/run-search.mjs b/components/splunk/actions/run-search/run-search.mjs index ce3c4a01552e4..52ad935fa0dfb 100644 --- a/components/splunk/actions/run-search/run-search.mjs +++ b/components/splunk/actions/run-search/run-search.mjs @@ -8,35 +8,19 @@ export default { type: "action", props: { splunk, - query: { + name: { propDefinition: [ splunk, - "query", + "savedSearchName", ], }, - earliestTime: { - type: "string", - label: "Earliest Time", - description: "Specify a time string. Sets the earliest (inclusive), respectively, time bounds for the search. The time string can be either a UTC time (with fractional seconds), a relative time specifier (to now) or a formatted time string. Refer to [Time modifiers](https://docs.splunk.com/Documentation/Splunk/9.4.1/SearchReference/SearchTimeModifiers) for search for information and examples of specifying a time string.", - optional: true, - }, - latestTime: { - type: "string", - label: "Latest Time", - description: " Specify a time string. Sets the latest (exclusive), respectively, time bounds for the search. The time string can be either a UTC time (with fractional seconds), a relative time specifier (to now) or a formatted time string. Refer to [Time modifiers](https://docs.splunk.com/Documentation/Splunk/9.4.1/SearchReference/SearchTimeModifiers) for search for information and examples of specifying a time string.", - optional: true, - }, }, async run({ $ }) { const response = await this.splunk.executeSearchQuery({ $, - data: { - search: this.query, - earliest_time: this.earliestTime, - latest_time: this.latestTime, - }, + name: this.name, }); - $.export("$summary", `Executed Splunk search query: ${this.query}`); + $.export("$summary", `Executed Splunk search: ${this.name}`); return response; }, }; diff --git a/components/splunk/sources/new-alert-fired/new-alert-fired.mjs b/components/splunk/sources/new-alert-fired/new-alert-fired.mjs index 044146b06d0f2..f78300605a6c8 100644 --- a/components/splunk/sources/new-alert-fired/new-alert-fired.mjs +++ b/components/splunk/sources/new-alert-fired/new-alert-fired.mjs @@ -1,36 +1,35 @@ -import common from "../common/base.mjs"; +import splunk from "../../splunk.app.mjs"; +import sampleEmit from "./test-event.mjs"; export default { - ...common, key: "splunk-new-alert-fired", - name: "New Alert Fired", + name: "New Alert Fired (Instant)", description: "Emit new event when a new alert is triggered in Splunk. [See the documentation](https://docs.splunk.com/Documentation/Splunk/9.4.1/RESTREF/RESTsearch#alerts.2Ffired_alerts)", version: "0.0.1", type: "source", dedupe: "unique", + props: { + splunk, + http: "$.interface.http", + }, methods: { - ...common.methods, generateMeta(alert) { + const ts = +alert.result._time; return { - id: alert.id, - summary: `New Alert Fired: ${alert.name}`, - ts: Date.now(), + id: ts, + summary: `New Alert Fired for Source: ${alert.result.source}`, + ts, }; }, }, - async run() { - const results = this.splunk.paginate({ - resourceFn: this.splunk.listFiredAlerts, - }); - - const alerts = []; - for await (const item of results) { - alerts.push(item); + async run(event) { + const { body } = event; + if (!body) { + return; } - alerts.forEach((alert) => { - const meta = this.generateMeta(alert); - this.$emit(alert, meta); - }); + const meta = this.generateMeta(body); + this.$emit(body, meta); }, + sampleEmit, }; diff --git a/components/splunk/sources/new-alert-fired/test-event.mjs b/components/splunk/sources/new-alert-fired/test-event.mjs new file mode 100644 index 0000000000000..d7c006b369832 --- /dev/null +++ b/components/splunk/sources/new-alert-fired/test-event.mjs @@ -0,0 +1,30 @@ +export default { + "sid": "", + "search_name": "", + "app": "search", + "owner": "", + "results_link": "https://splunk:8000/app/search/search?q=", + "result": { + "_confstr": "source::Source|host::44.210.81.125|webhook", + "_eventtype_color": "", + "_indextime": "1742843623", + "_raw": "{ \"name\": \"test\", \"value\": \"test\" }", + "_serial": "3", + "_si": [ + "main" + ], + "_sourcetype": "webhook", + "_time": "1742843623", + "eventtype": "", + "host": "44.210.81.125", + "index": "main", + "linecount": "", + "name": "test", + "punct": "{_\"\":_\"_\",_\"\":_\"\"_}", + "source": "Source", + "sourcetype": "webhook", + "splunk_server": "", + "timestamp": "none", + "value": "test" + } +} \ No newline at end of file diff --git a/components/splunk/sources/new-search-result/new-search-result.mjs b/components/splunk/sources/new-search-result/new-search-result.mjs index aacb6cf1ab795..b4fe08d6083c5 100644 --- a/components/splunk/sources/new-search-result/new-search-result.mjs +++ b/components/splunk/sources/new-search-result/new-search-result.mjs @@ -10,30 +10,37 @@ export default { dedupe: "unique", methods: { ...common.methods, + async getRecentJobs() { + const jobs = []; + const results = this.splunk.paginate({ + resourceFn: this.splunk.listJobs, + }); + for await (const job of results) { + jobs.push(job); + } + return jobs; + }, generateMeta(result) { return { - id: result.sid, - summary: `New Search Results with SID: ${result.sid}`, + id: result.id, + summary: `New Search with ID: ${result.id}`, ts: Date.now(), }; }, }, async run() { - const jobIds = await this.getRecentJobIds(); - const searchResults = []; - for (const jobId of jobIds) { - try { - const response = await this.splunk.getSearchResults({ - jobId, + const jobs = await this.getRecentJobs(); + for (const job of jobs) { + if (job.content?.resultCount > 0) { + const { results } = await this.splunk.getSearchResults({ + jobId: job.content.sid, }); - if (response?.results?.length) { - searchResults.push(...response.results); + if (results) { + job.results = results; } - } catch { - console.log(`No results found for sid: ${jobId}`); } } - searchResults.forEach((result) => { + jobs.forEach((result) => { const meta = this.generateMeta(result); this.$emit(result, meta); }); diff --git a/components/splunk/splunk.app.mjs b/components/splunk/splunk.app.mjs index ce929d7354425..4195aa22505c3 100644 --- a/components/splunk/splunk.app.mjs +++ b/components/splunk/splunk.app.mjs @@ -111,10 +111,12 @@ export default { ...opts, }); }, - executeSearchQuery(opts = {}) { + executeSearchQuery({ + name, ...opts + }) { return this._makeRequest({ method: "POST", - path: "/search/jobs", + path: `/saved/searches/${name}/dispatch`, ...opts, }); }, From c5d9a895f286d7b8d83c336bc185b22717b83470 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Tue, 25 Mar 2025 11:20:36 -0400 Subject: [PATCH 7/9] wip --- .../new-alert-fired/new-alert-fired.mjs | 25 ++++++++++++++++ components/splunk/splunk.app.mjs | 29 +++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/components/splunk/sources/new-alert-fired/new-alert-fired.mjs b/components/splunk/sources/new-alert-fired/new-alert-fired.mjs index f78300605a6c8..d29a611325595 100644 --- a/components/splunk/sources/new-alert-fired/new-alert-fired.mjs +++ b/components/splunk/sources/new-alert-fired/new-alert-fired.mjs @@ -11,6 +11,31 @@ export default { props: { splunk, http: "$.interface.http", + metricAlertName: { + propDefinition: [ + splunk, + "metricAlertName", + ], + }, + }, + hooks: { + async activate() { + await this.splunk.updateMetricAlert({ + name: this.metricAlertName, + data: { + "action.webhook": true, + "action.webhook.url": this.http.endpoint, + }, + }); + }, + async deactivate() { + await this.splunk.updateMetricAlert({ + name: this.metricAlertName, + data: { + "action.webhook": false, + }, + }); + }, }, methods: { generateMeta(alert) { diff --git a/components/splunk/splunk.app.mjs b/components/splunk/splunk.app.mjs index 4195aa22505c3..86edca178f288 100644 --- a/components/splunk/splunk.app.mjs +++ b/components/splunk/splunk.app.mjs @@ -53,6 +53,20 @@ export default { return entry?.map(({ name }) => name) || []; }, }, + metricAlertName: { + type: "string", + label: "Alert Name", + description: "The name of the alert to trigger events for", + async options({ page }) { + const { entry } = await this.listMetricAlerts({ + params: { + count: DEFAULT_LIMIT, + offset: DEFAULT_LIMIT * page, + }, + }); + return entry?.map(({ name }) => name) || []; + }, + }, query: { type: "string", label: "Search Query", @@ -111,6 +125,21 @@ export default { ...opts, }); }, + listMetricAlerts(opts = {}) { + return this._makeRequest({ + path: "/alerts/metric_alerts", + ...opts, + }); + }, + updateMetricAlert({ + name, ...opts + }) { + return this._makeRequest({ + method: "POST", + path: `/alerts/metric_alerts/${name}`, + ...opts, + }); + }, executeSearchQuery({ name, ...opts }) { From 8d7ccf4378a1292aba94312e431d1d19fba1849a Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Tue, 25 Mar 2025 15:04:34 -0400 Subject: [PATCH 8/9] wip --- .../new-alert-fired/new-alert-fired.mjs | 28 ++++++++----------- components/splunk/splunk.app.mjs | 26 +++++------------ 2 files changed, 19 insertions(+), 35 deletions(-) diff --git a/components/splunk/sources/new-alert-fired/new-alert-fired.mjs b/components/splunk/sources/new-alert-fired/new-alert-fired.mjs index d29a611325595..2f01b891d570d 100644 --- a/components/splunk/sources/new-alert-fired/new-alert-fired.mjs +++ b/components/splunk/sources/new-alert-fired/new-alert-fired.mjs @@ -1,43 +1,39 @@ import splunk from "../../splunk.app.mjs"; +import { exec } from "child_process"; import sampleEmit from "./test-event.mjs"; export default { key: "splunk-new-alert-fired", name: "New Alert Fired (Instant)", description: "Emit new event when a new alert is triggered in Splunk. [See the documentation](https://docs.splunk.com/Documentation/Splunk/9.4.1/RESTREF/RESTsearch#alerts.2Ffired_alerts)", - version: "0.0.1", + //version: "0.0.1", + version: "0.0.{{ts}}", type: "source", dedupe: "unique", props: { splunk, http: "$.interface.http", - metricAlertName: { + db: "$.service.db", + savedSearchName: { propDefinition: [ splunk, - "metricAlertName", + "savedSearchName", ], }, }, hooks: { async activate() { - await this.splunk.updateMetricAlert({ - name: this.metricAlertName, - data: { - "action.webhook": true, - "action.webhook.url": this.http.endpoint, - }, - }); + await this.updateSavedSearch(`-d action.webhook=1 -d action.webhook.param.url=${this.http.endpoint}`); }, async deactivate() { - await this.splunk.updateMetricAlert({ - name: this.metricAlertName, - data: { - "action.webhook": false, - }, - }); + await this.updateSavedSearch("-d action.webhook=0"); }, }, methods: { + async updateSavedSearch(data) { + const cmd = `curl -X POST ${this.splunk._baseUrl()}/saved/searches/${this.savedSearchName} -k -H "Authorization: Bearer ${this.splunk.$auth.api_token}" ${data}`; + await exec(cmd); + }, generateMeta(alert) { const ts = +alert.result._time; return { diff --git a/components/splunk/splunk.app.mjs b/components/splunk/splunk.app.mjs index 86edca178f288..730c4acc70cb3 100644 --- a/components/splunk/splunk.app.mjs +++ b/components/splunk/splunk.app.mjs @@ -53,20 +53,6 @@ export default { return entry?.map(({ name }) => name) || []; }, }, - metricAlertName: { - type: "string", - label: "Alert Name", - description: "The name of the alert to trigger events for", - async options({ page }) { - const { entry } = await this.listMetricAlerts({ - params: { - count: DEFAULT_LIMIT, - offset: DEFAULT_LIMIT * page, - }, - }); - return entry?.map(({ name }) => name) || []; - }, - }, query: { type: "string", label: "Search Query", @@ -125,18 +111,20 @@ export default { ...opts, }); }, - listMetricAlerts(opts = {}) { + updateSavedSearch({ + name, ...opts + }) { return this._makeRequest({ - path: "/alerts/metric_alerts", + method: "POST", + path: `/saved/searches/${name}`, ...opts, }); }, - updateMetricAlert({ + getSavedSearch({ name, ...opts }) { return this._makeRequest({ - method: "POST", - path: `/alerts/metric_alerts/${name}`, + path: `/saved/searches/${name}`, ...opts, }); }, From 061a60d68a4879e95e6f218c871adb36e9b99d87 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Tue, 25 Mar 2025 17:51:24 -0400 Subject: [PATCH 9/9] updates --- .../new-alert-fired/new-alert-fired.mjs | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/components/splunk/sources/new-alert-fired/new-alert-fired.mjs b/components/splunk/sources/new-alert-fired/new-alert-fired.mjs index 2f01b891d570d..9551057e47be0 100644 --- a/components/splunk/sources/new-alert-fired/new-alert-fired.mjs +++ b/components/splunk/sources/new-alert-fired/new-alert-fired.mjs @@ -1,13 +1,13 @@ import splunk from "../../splunk.app.mjs"; import { exec } from "child_process"; +import util from "util"; import sampleEmit from "./test-event.mjs"; export default { key: "splunk-new-alert-fired", name: "New Alert Fired (Instant)", description: "Emit new event when a new alert is triggered in Splunk. [See the documentation](https://docs.splunk.com/Documentation/Splunk/9.4.1/RESTREF/RESTsearch#alerts.2Ffired_alerts)", - //version: "0.0.1", - version: "0.0.{{ts}}", + version: "0.0.1", type: "source", dedupe: "unique", props: { @@ -23,16 +23,28 @@ export default { }, hooks: { async activate() { - await this.updateSavedSearch(`-d action.webhook=1 -d action.webhook.param.url=${this.http.endpoint}`); + const response = await this.updateSavedSearch(`-d action.webhook=1 -d action.webhook.param.url="${this.http.endpoint}"`); + if (!response) { + throw new Error("Error creating webhook"); + } }, async deactivate() { - await this.updateSavedSearch("-d action.webhook=0"); + const response = await this.updateSavedSearch("-d action.webhook=0"); + if (!response) { + throw new Error("Error disabling webhook"); + } }, }, methods: { async updateSavedSearch(data) { - const cmd = `curl -X POST ${this.splunk._baseUrl()}/saved/searches/${this.savedSearchName} -k -H "Authorization: Bearer ${this.splunk.$auth.api_token}" ${data}`; - await exec(cmd); + const cmd = `curl -X POST ${this.splunk._baseUrl()}/saved/searches/${encodeURIComponent(this.savedSearchName)}?output_mode=json -k -H "Authorization: Bearer ${this.splunk.$auth.api_token}" ${data}`; + const execPromise = util.promisify(exec); + try { + const { stdout } = await execPromise(cmd); + return stdout; + } catch (error) { + console.error("Error:", error.message); + } }, generateMeta(alert) { const ts = +alert.result._time;