wip

Author: michelle0927

components/oxylabs/actions/create-proxy-session/create-proxy-session.mjs

@@ -5,7 +5,7 @@ export default {
   key: "oxylabs-create-proxy-session",
   name: "Create Proxy Session",
   description: "Establish a proxy session using the Residential Proxy endpoint. [See the documentation](https://developers.oxylabs.io/proxies/residential-proxies/session-control#establishing-session)",
-  version: "0.0.{{ts}}",
+  version: "0.0.1",
   type: "action",
   props: {
     oxylabs,

components/oxylabs/actions/create-schedule/create-schedule.mjs

@@ -0,0 +1,44 @@
+import oxylabs from "../../oxylabs.app.mjs";
+import { parseObject } from "../../common/utils.mjs";
+
+export default {
+  key: "oxylabs-create-schedule",
+  name: "Create Schedule",
+  description: "Create a schedule for a scraping job. [See the documentation](https://developers.oxylabs.io/scraping-solutions/web-scraper-api/features/scheduler#create-a-new-schedule)",
+  version: "0.0.1",
+  type: "action",
+  props: {
+    oxylabs,
+    chron: {
+      type: "string",
+      label: "Cron Expression",
+      description: "Cron schedule expression. It determines how often the submitted schedule will run. E.g. `0 3 * * 1`. Read more [here](https://crontab.guru/) and [here](https://docs.oracle.com/cd/E12058_01/doc/doc.1014/e12030/cron_expressions.htm).",
+    },
+    items: {
+      type: "string[]",
+      label: "Items",
+      description: "List of Scraper APIs job parameter sets that should be executed as part of the schedule. E.g. `[{\"source\": \"universal\", \"url\": \"https://ip.oxylabs.io\"}]` [See the documentation](https://developers.oxylabs.io/scraping-solutions/web-scraper-api/features/scheduler#create-a-new-schedule) for more information.",
+      propDefinition: [
+        oxylabs,
+        "items",
+      ],
+    },
+    endTime: {
+      type: "string",
+      label: "End Time",
+      description: "The time at which the schedule should stop running. E.g. `2032-12-21 12:34:45`",
+    },
+  },
+  async run({ $ }) {
+    const response = await this.oxylabs.createSchedule({
+      $,
+      data: {
+        cron: this.chron,
+        items: parseObject(this.items),
+        end_time: this.endTime,
+      },
+    });
+    $.export("$summary", `Successfully created schedule: ${response.schedule_id}`);
+    return response;
+  },
+};

components/oxylabs/actions/scrape-url/scrape-url.mjs

@@ -1,27 +1,35 @@
 import oxylabs from "../../oxylabs.app.mjs";
+import constants from "../../common/constants.mjs";
 
 export default {
   key: "oxylabs-scrape-url",
   name: "Scrape URL",
   description: "Scrape a URL. [See the documentation](https://developers.oxylabs.io/scraping-solutions/web-scraper-api)",
-  version: "0.0.{{ts}}",
+  version: "0.0.1",
   type: "action",
   props: {
     oxylabs,
+    source: {
+      type: "string",
+      label: "Source",
+      description: "Sets the scraper that will be used to process your request",
+      options: constants.URL_SOURCES,
+      default: "universal",
+    },
     url: {
       type: "string",
       label: "URL",
       description: "The URL to scrape",
     },
     geoLocation: {
-      type: "string",
-      label: "Geo Location",
-      description: "The geo locatio to scrape from. E.g. `United States`",
-      optional: true,
+      propDefinition: [
+        oxylabs,
+        "geoLocation",
+      ],
     },
   },
   async run({ $ }) {
-    const response = await this.oxylabs.scrapeUrl({
+    const response = await this.oxylabs.scrape({
       $,
       data: {
         source: "universal",

components/oxylabs/actions/scrape-with-query/scrape-with-query.mjs

@@ -0,0 +1,42 @@
+import oxylabs from "../../oxylabs.app.mjs";
+import constants from "../../common/constants.mjs";
+
+export default {
+  key: "oxylabs-scrape-with-query",
+  name: "Scrape with Query",
+  description: "Extract data using a search query. [See the documentation](https://developers.oxylabs.io/scraping-solutions/web-scraper-api)",
+  version: "0.0.1",
+  type: "action",
+  props: {
+    oxylabs,
+    source: {
+      type: "string",
+      label: "Source",
+      description: "Sets the scraper that will be used to process your request",
+      options: constants.QUERY_SOURCES,
+    },
+    query: {
+      type: "string",
+      label: "Query",
+      description: "The query to search for. [See the documentation](https://developers.oxylabs.io/scraping-solutions/web-scraper-api/targets) for more information about specific sources/targets",
+    },
+    geoLocation: {
+      propDefinition: [
+        oxylabs,
+        "geoLocation",
+      ],
+    },
+  },
+  async run({ $ }) {
+    const response = await this.oxylabs.scrape({
+      $,
+      data: {
+        source: this.source,
+        query: this.query,
+        geo_location: this.geoLocation,
+      },
+    });
+    $.export("$summary", `Successfully scraped using query: ${this.query}`);
+    return response;
+  },
+};

components/oxylabs/common/constants.mjs

@@ -0,0 +1,37 @@
+const URL_SOURCES = [
+  "universal",
+  "amazon",
+  "google",
+  "bing",
+  "kroger",
+];
+
+const QUERY_SOURCES = [
+  "amazon_product",
+  "amazon_search",
+  "amazon_pricing",
+  "amazon_sellers",
+  "amazon_bestsellers",
+  "amazon_reviews",
+  "amazon_questions",
+  "google_search",
+  "google_ads",
+  "google_images",
+  "google_lens",
+  "google_maps",
+  "google_travel_hotels",
+  "google_suggest",
+  "google_trends_explore",
+  "google_shopping_product",
+  "google_shopping_search",
+  "google_shopping_pricing",
+  "bing_search",
+  "kroger_product",
+  "kroger_search",
+  "youtube_transcript",
+];
+
+export default {
+  URL_SOURCES,
+  QUERY_SOURCES,
+};

components/oxylabs/common/utils.mjs

@@ -0,0 +1,25 @@
+export const parseObject = (obj) => {
+  if (!obj) {
+    return undefined;
+  }
+  if (typeof obj === "string") {
+    try {
+      return JSON.parse(obj);
+    } catch (e) {
+      return obj;
+    }
+  }
+  if (Array.isArray(obj)) {
+    return obj.map(parseObject);
+  }
+  if (typeof obj === "object") {
+    return Object.fromEntries(Object.entries(obj).map(([
+      key,
+      value,
+    ]) => [
+      key,
+      parseObject(value),
+    ]));
+  }
+  return obj;
+};

components/oxylabs/oxylabs.app.mjs

@@ -4,7 +4,14 @@ import { HttpsProxyAgent } from "https-proxy-agent";
 export default {
   type: "app",
   app: "oxylabs",
-  propDefinitions: {},
+  propDefinitions: {
+    geoLocation: {
+      type: "string",
+      label: "Geo Location",
+      description: "The geo locatio to scrape from. E.g. `United States`",
+      optional: true,
+    },
+  },
   methods: {
     _getBaseUrl() {
       return `https://${this.$auth.api_name}.oxylabs.io/v1`;
@@ -24,7 +31,7 @@ export default {
         ...opts,
       });
     },
-    scrapeUrl(opts = {}) {
+    scrape(opts = {}) {
       return this._makeRequest({
         method: "POST",
         path: "/queries",
@@ -41,5 +48,12 @@ export default {
         ...opts,
       });
     },
+    createSchedule(opts = {}) {
+      return this._makeRequest({
+        method: "POST",
+        path: "/schedules",
+        ...opts,
+      });
+    },
   },
 };