add uploading to s3 buckets

Author: ScriptSmith

package-lock.json

@@ -40,6 +40,7 @@
     ],
     "dependencies": {
         "await-lock": "^2.0.1",
+        "aws-sdk": "^2.619.0",
         "axios": "^0.19.2",
         "chalk": "^3.0.0",
         "env-paths": "^2.2.0",
@@ -62,6 +63,7 @@
         "url": "[email protected]:ScriptSmith/instamancer.git"
     },
     "devDependencies": {
+        "@types/aws-sdk": "^2.7.0",
         "@types/concat-stream": "^1.6.0",
         "@types/express": "^4.17.2",
         "@types/jest": "^24.9.1",

package.json

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
 
+import * as aws from "aws-sdk";
 import * as fs from "fs";
 import * as readline from "readline";
 import * as winston from "winston";
@@ -10,8 +11,9 @@ import * as plugins from "../plugins";
 import {createApi, IOptions} from "./api/api";
 import {TFullApiPost, TPost} from "./api/types";
 import {GetPool} from "./getpool/getPool";
+import * as depotUpload from "./http/depot";
 import {download, toCSV, toJSON} from "./http/download";
-import * as upload from "./http/upload";
+import * as s3Upload from "./http/s3";
 
 const getLogger = (args) => {
     const transports = [];
@@ -217,13 +219,6 @@ function buildParser(args, callback) {
                 implies: "full",
                 group: "Download",
             },
-            upload: {
-                alias: ["u"],
-                string: true,
-                default: undefined,
-                describe: "Upload files to a URL with a PUT request",
-                group: "Download",
-            },
             sync: {
                 boolean: true,
                 default: false,
@@ -234,7 +229,7 @@ function buildParser(args, callback) {
                 alias: "k",
                 number: true,
                 default: 4,
-                describe: "Parallel download / upload threads",
+                describe: "Parallel download / depot threads",
                 group: "Download",
             },
             waitDownload: {
@@ -244,6 +239,18 @@ function buildParser(args, callback) {
                 describe: "Download media after scraping",
                 group: "Download",
             },
+            bucket: {
+                string: true,
+                default: undefined,
+                describe: "Upload files to an AWS S3 bucket",
+                group: "Upload",
+            },
+            depot: {
+                string: true,
+                default: undefined,
+                describe: "Upload files to a URL with a PUT request (depot)",
+                group: "Upload",
+            },
             file: {
                 alias: ["o"],
                 string: true,
@@ -351,33 +358,48 @@ async function spawn(args) {
         .replace("[id]", args["id"])
         .replace("[endpoint]", args["_"]);
 
-    // Replace upload url
-    let uploadUrl = args["upload"];
-    if (uploadUrl && uploadUrl.includes("[uuid]")) {
-        uploadUrl = uploadUrl.replace("[uuid]", uuid());
+    // Replace depot url
+    let depotUrl = args["depot"];
+    if (depotUrl && depotUrl.includes("[uuid]")) {
+        depotUrl = depotUrl.replace("[uuid]", uuid());
         if (!args["quiet"]) {
-            process.stdout.write(uploadUrl + "\n");
+            process.stdout.write(depotUrl + "\n");
         }
     }
 
+    // Get s3 bucket
+    const s3Bucket = args["bucket"];
+
     // Check if outputting to stdout
     const printOutput = args["file"] === "-";
 
     // Connect to object storage
     let downloadUpload;
     let toCSVFunc = toCSV;
     let toJSONFunc = toJSON;
-    if (uploadUrl) {
-        // Upload
-        const uploadConfig = {
+    if (depotUrl) {
+        // Depot
+        const depotConfig = {
+            directory: downdir,
+            url: depotUrl,
+            logger,
+        };
+
+        downloadUpload = depotUpload.depot.bind(depotConfig);
+        toCSVFunc = depotUpload.toCSV.bind(depotConfig);
+        toJSONFunc = depotUpload.toJSON.bind(depotConfig);
+    } else if (s3Bucket) {
+        // s3
+        const s3Config = {
+            bucket: s3Bucket,
             directory: downdir,
-            url: uploadUrl,
+            s3: new aws.S3(),
             logger,
         };
 
-        downloadUpload = upload.upload.bind(uploadConfig);
-        toCSVFunc = upload.toCSV.bind(uploadConfig);
-        toJSONFunc = upload.toJSON.bind(uploadConfig);
+        downloadUpload = s3Upload.s3.bind(s3Config);
+        toCSVFunc = s3Upload.toCSV.bind(s3Config);
+        toJSONFunc = s3Upload.toJSON.bind(s3Config);
     } else {
         // Download
         downloadUpload = download.bind({

src/cli.ts

@@ -12,7 +12,7 @@ interface IUpload {
     logger: winston.Logger;
 }
 
-export async function upload(
+export async function depot(
     this: IUpload,
     url: string,
     name: string,
@@ -34,7 +34,7 @@ export async function upload(
         const filePath = path.join(this.directory, name + "." + extension);
         const uploadUrl = resolve(this.url, filePath);
 
-        // Axios upload
+        // Axios depot
         await axios({
             data: downloadStream.data,
             headers: {
@@ -45,12 +45,10 @@ export async function upload(
             method: "PUT",
             ...authURL(uploadUrl),
         }).catch((error) => {
-            this.logger.info(`Downloading ${url} failed`);
-            this.logger.debug(error);
+            this.logger.error(`Uploading ${url} failed`, error);
         });
     } catch (e) {
-        this.logger.info(`Downloading ${url} failed`);
-        this.logger.debug(e);
+        this.logger.error(`Uploading ${url} failed`, e);
     }
 }