|
| 1 | +/** |
| 2 | + * External module Dependencies. |
| 3 | + */ |
| 4 | +const mkdirp = require("mkdirp"), |
| 5 | + path = require("path"), |
| 6 | + _ = require("lodash"), |
| 7 | + fs = require("fs"), |
| 8 | + axios = require("axios"); |
| 9 | + |
| 10 | +const chalk = require("chalk"); |
| 11 | +const config = require("../config"); |
| 12 | + |
| 13 | +/** |
| 14 | + * Internal module Dependencies . |
| 15 | + */ |
| 16 | +const helper = require("../utils/helper"); |
| 17 | + |
| 18 | +const { asset: assetConfig } = config.modules; |
| 19 | +const assetFolderPath = path.resolve(config.data, assetConfig.dirName); |
| 20 | +const assetMasterFolderPath = path.resolve(config.data, "logs", "assets"); |
| 21 | +const failedJSONFilePath = path.join(assetMasterFolderPath, "wp_failed.json"); |
| 22 | +const failedJSON = helper.readFile(failedJSONFilePath) || {}; |
| 23 | + |
| 24 | +var assetData = {}; |
| 25 | +function startingDir(){ |
| 26 | +if (!fs.existsSync(assetFolderPath)) { |
| 27 | + mkdirp.sync(assetFolderPath); |
| 28 | + helper.writeFile(path.join(assetFolderPath, assetConfig.fileName)); |
| 29 | + mkdirp.sync(assetMasterFolderPath); |
| 30 | + return true; |
| 31 | +} else { |
| 32 | + if (!fs.existsSync(path.join(assetFolderPath, assetConfig.fileName))) { |
| 33 | + helper.writeFile(path.join(assetFolderPath, assetConfig.fileName)); |
| 34 | + return true; |
| 35 | + }else{ |
| 36 | + assetData = helper.readFile(path.join(assetFolderPath, assetConfig.fileName)); |
| 37 | + |
| 38 | + } |
| 39 | + if (!fs.existsSync(assetMasterFolderPath)) { |
| 40 | + mkdirp.sync(assetMasterFolderPath); |
| 41 | + return true; |
| 42 | + } |
| 43 | +} |
| 44 | +} |
| 45 | +async function saveAsset (assets, retryCount, affix) { |
| 46 | + const url = encodeURI(assets["wp:attachment_url"]); |
| 47 | + const name = url.split("/").pop(); |
| 48 | + |
| 49 | + let description = |
| 50 | + assets["description"] || |
| 51 | + assets["content:encoded"] || |
| 52 | + assets["excerpt:encoded"] || |
| 53 | + ""; |
| 54 | + description = |
| 55 | + description.length > 255 ? description.slice(0, 255) : description; |
| 56 | + |
| 57 | + const parent_uid = affix ? "wordpressasset" : null; |
| 58 | + const assetPath = path.resolve( |
| 59 | + assetFolderPath, |
| 60 | + `assets_${assets["wp:post_id"].toString()}`, |
| 61 | + name |
| 62 | + ); |
| 63 | + |
| 64 | + if (fs.existsSync(assetPath)) { |
| 65 | + console.log(`Asset already present: ${assets["wp:post_id"]}`); |
| 66 | + return assets["wp:post_id"]; |
| 67 | + } |
| 68 | + |
| 69 | + try { |
| 70 | + const response = await axios.get(url, { responseType: "arraybuffer" }); |
| 71 | + mkdirp.sync( |
| 72 | + path.resolve( |
| 73 | + assetFolderPath, |
| 74 | + `assets_${assets["wp:post_id"].toString()}` |
| 75 | + ) |
| 76 | + ); |
| 77 | + fs.writeFileSync(assetPath, response.data); |
| 78 | + |
| 79 | + const stats = fs.lstatSync(assetPath); |
| 80 | + |
| 81 | + assetData[`assets_${assets["wp:post_id"]}`] = { |
| 82 | + uid: `assets_${assets["wp:post_id"]}`, |
| 83 | + urlPath: `/assets/assets_${assets["wp:post_id"]}`, |
| 84 | + status: true, |
| 85 | + file_size: `${stats.size}`, |
| 86 | + tag: [], |
| 87 | + filename: name, |
| 88 | + url, |
| 89 | + is_dir: false, |
| 90 | + parent_uid, |
| 91 | + _version: 1, |
| 92 | + title: assets["title"] || name.split(".").slice(0, -1).join("."), |
| 93 | + publish_details: [], |
| 94 | + description, |
| 95 | + }; |
| 96 | + |
| 97 | + if (failedJSON[assets["wp:post_id"]]) { |
| 98 | + // delete the assest entry from wp_failed log |
| 99 | + delete failedJSON[assets["wp:post_id"]]; |
| 100 | + helper.writeFile( |
| 101 | + failedJSONFilePath, |
| 102 | + JSON.stringify(failedJSON, null, 4) |
| 103 | + ); |
| 104 | + } |
| 105 | + |
| 106 | + helper.writeFile( |
| 107 | + path.join(assetFolderPath, assetConfig.fileName), |
| 108 | + JSON.stringify(assetData, null, 4) |
| 109 | + ); |
| 110 | + |
| 111 | + console.log( |
| 112 | + "An asset with id", |
| 113 | + chalk.green(`assets_${assets["wp:post_id"]}`), |
| 114 | + "and name", |
| 115 | + chalk.green(`${name}`), |
| 116 | + "downloaded successfully." |
| 117 | + ); |
| 118 | + |
| 119 | + return assets["wp:post_id"]; |
| 120 | + } catch (err) { |
| 121 | + const assetName = |
| 122 | + assets["title"] || name.split(".").slice(0, -1).join("."); |
| 123 | + failedJSON[assets["wp:post_id"]] = { |
| 124 | + failedUid: assets["wp:post_id"], |
| 125 | + name: assetName, |
| 126 | + url, |
| 127 | + reason_for_error: err?.message, |
| 128 | + }; |
| 129 | + |
| 130 | + helper.writeFile(failedJSONFilePath, JSON.stringify(failedJSON, null, 4)); |
| 131 | + |
| 132 | + if (retryCount === 0) { |
| 133 | + return saveAsset(assets, 1, affix); |
| 134 | + } else { |
| 135 | + console.log( |
| 136 | + chalk.red(`Failed to download asset with id `), |
| 137 | + assets["wp:post_id"], |
| 138 | + chalk.red("with error message"), |
| 139 | + err?.message |
| 140 | + ); |
| 141 | + return assets["wp:post_id"]; |
| 142 | + } |
| 143 | + } |
| 144 | +} |
| 145 | + |
| 146 | +async function getAsset(attachments, affix) { |
| 147 | + const BATCH_SIZE = 5; // 5 promises at a time |
| 148 | + const results = []; |
| 149 | + |
| 150 | + for (let i = 0; i < attachments.length; i += BATCH_SIZE) { |
| 151 | + const batch = attachments.slice(i, i + BATCH_SIZE); |
| 152 | + |
| 153 | + const batchResults = await Promise.allSettled( |
| 154 | + batch.map((data) => saveAsset(data, 0, affix)) |
| 155 | + ); |
| 156 | + results.push(...batchResults); |
| 157 | + } |
| 158 | + return results; |
| 159 | +} |
| 160 | + |
| 161 | +// Function to get all assets from the data source |
| 162 | +async function getAllAssets( affix) { |
| 163 | + const alldata = helper.readFile( |
| 164 | + path.join(config.data, config.json_filename) |
| 165 | + ); |
| 166 | + //const alldata = helper.readFile(newPath); |
| 167 | + const assets = alldata?.rss?.channel?.item ?? alldata?.channel?.item; |
| 168 | + |
| 169 | + if (!assets || assets.length === 0) { |
| 170 | + console.log(chalk.red("No assets found")); |
| 171 | + return; |
| 172 | + } |
| 173 | + |
| 174 | + const attachments = assets.filter( |
| 175 | + ({ "wp:post_type": postType }) => postType === "attachment" |
| 176 | + ); |
| 177 | + if (attachments.length > 0) { |
| 178 | + await getAsset(attachments, affix); |
| 179 | + } |
| 180 | + |
| 181 | + // if (!filePath) { |
| 182 | + // const attachments = assets.filter( |
| 183 | + // ({ "wp:post_type": postType }) => postType === "attachment" |
| 184 | + // ); |
| 185 | + // if (attachments.length > 0) { |
| 186 | + // await getAsset(attachments); |
| 187 | + // } |
| 188 | + // } else { |
| 189 | + // //if want to custom export |
| 190 | + // const assetIds = fs.existsSync(filePath) |
| 191 | + // ? fs.readFileSync(filePath, "utf-8").split(",") |
| 192 | + // : []; |
| 193 | + // if (assetIds.length > 0) { |
| 194 | + // const assetDetails = assetIds |
| 195 | + // .map((id) => assets.find(({ "wp:post_id": postId }) => postId === id)) |
| 196 | + // .filter((asset) => asset && asset["wp:post_type"] === "attachment"); |
| 197 | + // if (assetDetails.length > 0) { |
| 198 | + // await getAsset(assetDetails); |
| 199 | + // } else { |
| 200 | + // console.error("Please provide valid IDs for asset export"); |
| 201 | + // } |
| 202 | + // } else { |
| 203 | + // console.error("No asset IDs found"); |
| 204 | + // } |
| 205 | + // } |
| 206 | +} |
| 207 | + |
| 208 | +async function startAssetExport(affix) { |
| 209 | + console.log("Exporting assets..."); |
| 210 | + try { |
| 211 | + startingDir() |
| 212 | + await getAllAssets( affix); |
| 213 | + } catch (err) { |
| 214 | + console.error("Error in asset export:", err); |
| 215 | + } |
| 216 | +} |
| 217 | +module.exports = startAssetExport; |
0 commit comments