[Components] docparser #13255

luancazarine · luancazarine · commit ac8140f3d89c · 2025-04-08T10:39:55.000-03:00
Sources
 - New Document Data Available

Actions
 - Fetch Document URL
 - Upload Document
diff --git a/components/docparser/actions/fetch-document-url/fetch-document-url.mjs b/components/docparser/actions/fetch-document-url/fetch-document-url.mjs
@@ -1,31 +1,35 @@
+import FormData from "form-data";
 import docparser from "../../docparser.app.mjs";
-import { axios } from "@pipedream/platform";
 
 export default {
   key: "docparser-fetch-document-url",
   name: "Fetch Document by URL",
   description: "Fetches a document from a provided URL and imports it to Docparser for parsing. [See the documentation](https://docparser.com/api/)",
-  version: "0.0.{{ts}}",
+  version: "0.0.1",
   type: "action",
   props: {
     docparser,
-    url: {
-      propDefinition: [
-        docparser,
-        "url",
-      ],
-    },
     parserId: {
       propDefinition: [
         docparser,
         "parserId",
       ],
     },
+    url: {
+      type: "string",
+      label: "Document URL",
+      description: "The URL of the document to be fetched and imported into Docparser.",
+    },
   },
   async run({ $ }) {
+    const data = new FormData();
+    data.append("url", this.url);
+
     const response = await this.docparser.fetchDocumentFromURL({
+      $,
       parserId: this.parserId,
-      url: this.url,
+      data,
+      headers: data.getHeaders(),
     });
 
     $.export("$summary", `Document is scheduled to be fetched and processed. Document ID: ${response.document_id}`);
diff --git a/components/docparser/actions/upload-document/upload-document.mjs b/components/docparser/actions/upload-document/upload-document.mjs
@@ -1,11 +1,13 @@
+import FormData from "form-data";
+import fs from "fs";
+import { checkTmp } from "../../common/utils.mjs";
 import docparser from "../../docparser.app.mjs";
-import { axios } from "@pipedream/platform";
 
 export default {
   key: "docparser-upload-document",
   name: "Upload Document",
-  description: "Uploads a document to docparser that initiates parsing immediately after reception. [See the documentation](https://docparser.com/api/)",
-  version: "0.0.{{ts}}",
+  description: "Uploads a document to docparser that initiates parsing immediately after reception. [See the documentation](https://docparser.com/api/#import-documents)",
+  version: "0.0.1",
   type: "action",
   props: {
     docparser,
@@ -16,16 +18,20 @@ export default {
       ],
     },
     file: {
-      propDefinition: [
-        docparser,
-        "file",
-      ],
+      type: "string",
+      label: "File",
+      description: "The path to a file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#writing-a-file-to-tmp)",
     },
   },
   async run({ $ }) {
+    const data = new FormData();
+    data.append("file", fs.createReadStream(checkTmp(this.file)));
+
     const response = await this.docparser.uploadDocument({
+      $,
       parserId: this.parserId,
-      file: this.file,
+      data,
+      headers: data.getHeaders(),
     });
 
     $.export("$summary", `Successfully uploaded document. Document ID: ${response.id}`);
diff --git a/components/docparser/common/utils.mjs b/components/docparser/common/utils.mjs
@@ -0,0 +1,6 @@
+export const checkTmp = (filename) => {
+  if (!filename.startsWith("/tmp")) {
+    return `/tmp/${filename}`;
+  }
+  return filename;
+};
diff --git a/components/docparser/docparser.app.mjs b/components/docparser/docparser.app.mjs
@@ -4,25 +4,17 @@ export default {
   type: "app",
   app: "docparser",
   propDefinitions: {
-    url: {
-      type: "string",
-      label: "Document URL",
-      description: "The URL of the document to be fetched and imported into Docparser.",
-    },
-    file: {
-      type: "string",
-      label: "File Content",
-      description: "The content of the file to be uploaded, encoded in base64.",
-    },
     parserId: {
       type: "string",
       label: "Parser ID",
       description: "The ID of the parser to be used.",
       async options() {
         const parsers = await this.listParsers();
-        return parsers.map((parser) => ({
-          label: parser.name,
-          value: parser.id,
+        return parsers.map(({
+          id: value, label,
+        }) => ({
+          label,
+          value,
         }));
       },
     },
@@ -31,53 +23,51 @@ export default {
     _baseUrl() {
       return "https://api.docparser.com";
     },
-    async _makeRequest(opts = {}) {
-      const {
-        $ = this, method = "GET", path, headers, ...otherOpts
-      } = opts;
+    _auth() {
+      return {
+        username: `${this.$auth.api_key}`,
+        password: "",
+      };
+    },
+    _makeRequest({
+      $ = this, path, ...opts
+    }) {
       return axios($, {
-        ...otherOpts,
-        method,
         url: this._baseUrl() + path,
-        headers: {
-          ...headers,
-          Authorization: `Bearer ${this.$auth.api_key}`,
-        },
+        auth: this._auth(),
+        ...opts,
       });
     },
-    async listParsers() {
+    listData({
+      parserId, ...opts
+    }) {
+      return this._makeRequest({
+        path: `/v1/results/${parserId}`,
+        ...opts,
+      });
+    },
+    listParsers() {
       return this._makeRequest({
-        path: "/v2/parsers",
+        path: "/v1/parsers",
       });
     },
-    async fetchDocumentFromURL({
-      parserId, url,
+    fetchDocumentFromURL({
+      parserId, ...opts
     }) {
       return this._makeRequest({
         method: "POST",
         path: `/v2/document/fetch/${parserId}`,
-        data: {
-          url,
-        },
+        ...opts,
       });
     },
-    async uploadDocument({
-      parserId, file,
+    uploadDocument({
+      parserId, ...opts
     }) {
       return this._makeRequest({
         method: "POST",
         path: `/v1/document/upload/${parserId}`,
-        data: {
-          file_content: file,
-        },
+        ...opts,
       });
     },
-    async pollParsedData() {
-      // Implement logic to emit new events when parsed data is available
-    },
-    async pollParsedTableRows() {
-      // Implement logic to emit new events when parsed table rows are available
-    },
   },
-  version: "0.0.{{{ts}}}", // Ensure the version is set as per requirements
 };
diff --git a/components/docparser/package.json b/components/docparser/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/docparser",
-  "version": "0.6.0",
+  "version": "0.1.0",
   "description": "Pipedream docparser Components",
   "main": "docparser.app.mjs",
   "keywords": [
@@ -13,6 +13,6 @@
     "access": "public"
   },
   "dependencies": {
-    "@pipedream/platform": "^3.0.0"
+    "@pipedream/platform": "^3.0.3"
   }
 }
diff --git a/components/docparser/sources/common/base.mjs b/components/docparser/sources/common/base.mjs
@@ -0,0 +1,69 @@
+import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
+import app from "../../docparser.app.mjs";
+
+export default {
+  props: {
+    app,
+    db: "$.service.db",
+    timer: {
+      type: "$.interface.timer",
+      default: {
+        intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
+      },
+    },
+    parserId: {
+      propDefinition: [
+        app,
+        "parserId",
+      ],
+    },
+  },
+  methods: {
+    _getLastDate() {
+      return this.db.get("lastDate") || "1970-01-01T00:00:00";
+    },
+    _setLastDate(lastDate) {
+      this.db.set("lastDate", lastDate);
+    },
+    async emitEvent(maxResults = false) {
+      const lastDate = this._getLastDate();
+      const fn = this.getFunction();
+      const params = {
+        sort_by: "parsed_at",
+        sort_order: "DESC",
+        list: "processed_after",
+        date: lastDate,
+      };
+
+      if (maxResults) {
+        params.limit = maxResults;
+      }
+
+      const response = await fn({
+        parserId: this.parserId,
+        params,
+      });
+
+      if (response.length) {
+        const dateTime = response[0].processed_at_utc;
+        this._setLastDate(dateTime.substring(0, dateTime.length - 6));
+      }
+
+      for (const item of response.reverse()) {
+        this.$emit(item, {
+          id: item.id,
+          summary: this.getSummary(item),
+          ts: Date.parse(item.created || new Date()),
+        });
+      }
+    },
+  },
+  hooks: {
+    async deploy() {
+      await this.emitEvent(25);
+    },
+  },
+  async run() {
+    await this.emitEvent();
+  },
+};
diff --git a/components/docparser/sources/new-document-data-available/new-document-data-available.mjs b/components/docparser/sources/new-document-data-available/new-document-data-available.mjs
@@ -1,84 +1,22 @@
-import docparser from "../../docparser.app.mjs";
-import { axios } from "@pipedream/platform";
+import common from "../common/base.mjs";
+import sampleEmit from "./test-event.mjs";
 
 export default {
+  ...common,
   key: "docparser-new-document-data-available",
   name: "New Document Data Available",
   description: "Emit new event every time a document is processed and parsed data is available. [See the documentation](https://docparser.com/api/)",
-  version: "0.0.{{ts}}",
+  version: "0.0.1",
   type: "source",
   dedupe: "unique",
-  props: {
-    docparser,
-    db: "$.service.db",
-    timer: {
-      type: "$.interface.timer",
-      default: {
-        intervalSeconds: 60,
-      },
-    },
-    parserId: {
-      propDefinition: [
-        docparser,
-        "parserId",
-      ],
-    },
-  },
   methods: {
-    _getLastFetchedId() {
-      return this.db.get("lastFetchedId") || null;
-    },
-    _setLastFetchedId(id) {
-      this.db.set("lastFetchedId", id);
+    ...common.methods,
+    getFunction() {
+      return this.app.listData;
     },
-    async getParsedData() {
-      return await this.docparser._makeRequest({
-        path: `/v2/results/${this.parserId}`,
-      });
+    getSummary(item) {
+      return `New Document Parsed: ${item.file_name}`;
     },
   },
-  hooks: {
-    async deploy() {
-      const parsedData = await this.getParsedData();
-      const eventsToEmit = parsedData.slice(-50);
-
-      for (const event of eventsToEmit) {
-        this.$emit(event, {
-          id: event.id,
-          summary: `New Document Parsed: ${event.file_name}`,
-          ts: Date.parse(event.parsed_at),
-        });
-      }
-
-      if (eventsToEmit.length > 0) {
-        const lastEvent = eventsToEmit[eventsToEmit.length - 1];
-        this._setLastFetchedId(lastEvent.id);
-      }
-    },
-    async activate() {
-      console.log("Component activated");
-    },
-    async deactivate() {
-      console.log("Component deactivated");
-    },
-  },
-  async run() {
-    const lastFetchedId = this._getLastFetchedId();
-    const parsedData = await this.getParsedData();
-
-    for (const event of parsedData) {
-      if (!lastFetchedId || event.id > lastFetchedId) {
-        this.$emit(event, {
-          id: event.id,
-          summary: `New Document Parsed: ${event.file_name}`,
-          ts: Date.parse(event.parsed_at),
-        });
-      }
-    }
-
-    if (parsedData.length > 0) {
-      const lastEvent = parsedData[parsedData.length - 1];
-      this._setLastFetchedId(lastEvent.id);
-    }
-  },
+  sampleEmit,
 };
diff --git a/components/docparser/sources/new-document-data-available/test-event.mjs b/components/docparser/sources/new-document-data-available/test-event.mjs
diff --git a/components/docparser/sources/new-table-row-available/new-table-row-available.mjs b/components/docparser/sources/new-table-row-available/new-table-row-available.mjs

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@pipedream/docparser",`
`3`		`- "version": "0.6.0",`
	`3`	`+ "version": "0.1.0",`
`4`	`4`	`"description": "Pipedream docparser Components",`
`5`	`5`	`"main": "docparser.app.mjs",`
`6`	`6`	`"keywords": [`
`@@ -13,6 +13,6 @@`
`13`	`13`	`"access": "public"`
`14`	`14`	`},`
`15`	`15`	`"dependencies": {`
`16`		`- "@pipedream/platform": "^3.0.0"`
	`16`	`+ "@pipedream/platform": "^3.0.3"`
`17`	`17`	`}`
`18`	`18`	`}`