Merge pull request #173 from karyeet/file-ingest-dev

Improvements to file ingestion

Author: karyeet

.github/workflows/build.yml

@@ -7,6 +7,7 @@ on:
     branches:
       - 'main'
       - 'Dockerfile'
+      - 'file-ingest-dev'
   release:
     types:
       - created

Docker/docker-compose.yml

@@ -14,6 +14,5 @@ services:
       - ./playdlData:/Mandarine/.data
     environment:
       - "arl=5a2e"
-      - "runIndexer=True"
     restart: unless-stopped
 

README.md

@@ -55,7 +55,6 @@ services:
       - ./playdlData:/Mandarine/.data
     environment:
       - "arl=5a2e"
-      - "runIndexer=True"
     restart: unless-stopped
 ```
 

config.json.example

@@ -1,5 +1,7 @@
 {
 	"token":"abcdefghfdfdgsgfsgsdgds"
+	"additionalMusicDirs":["H:\\WindowsFolder\\Music","/linux/folder"],
+	"runIndexerOnStart": true,
 	"voiceCommands": {
 		"enabled": true
 		"picoAccessKey": "abcdefghijklmnopqrstuvwxyz=",

index.js

@@ -1,29 +1,36 @@
-if (process.env.runIndexer === "True") {
-	require("./music/indexer.js");
-}
-
 const { Intents, Client } = require("discord.js");
 
-const { reactions, guildsMeta } = require("./general.js");
-
 // Intents: Guilds, VoiceStates, GuildMesssages
 const intents = [Intents.FLAGS.GUILDS, Intents.FLAGS.GUILD_VOICE_STATES, Intents.FLAGS.GUILD_MESSAGES, Intents.FLAGS.GUILD_MESSAGE_REACTIONS, Intents.FLAGS.GUILD_PRESENCES];
 
-// Create Client with discord commando
+// Create Client
 const client = new Client({ intents });
 
+let DIY_COMMANDO;
+
+const { reactions, guildsMeta } = require("./general.js");
+
+const config = require("./config.json");
+
+// Login
+console.log("runIndexer is set to " + config.runIndexerOnStart);
+if (config.runIndexerOnStart == true) {
+	require("./music/indexer.js").index().then(() => {
+		client.login(config.token);
+	});
+}
+else {
+	client.login(config.token);
+}
+
 // table of command names and their corresponding files/functions
-const DIY_COMMANDO = require("./diy_commando.js");
 
 client.once("ready", () => {
 	console.log("Ready in " + client.guilds.cache.size + " guild(s).");
 	client.user.setActivity("Prefix: >", { type: "WATCHING" });
+	DIY_COMMANDO = require("./diy_commando.js");
 });
 
-const config = require("./config.json");
-
-// Login
-client.login(config.token);
 
 // check if prefix is ">", and if so return the command back
 function checkCommand(content) {

music/indexer.js

@@ -1,69 +1,104 @@
 /*
-	Only to be run seperatly when importing new music files
+	Only needs to be run seperatly when importing new music files
 	Very basic
 
 */
 
 
-const fs = require("fs");
-const localLibrary = require("./localLibrary.json");
+const fs = require("fs/promises");
+const localLibrary = {};
 
 const homedir = require("os").homedir();
-const pathToFiles = homedir + "/mandarineFiles/";
-
 const path = require("path");
 
+const pathToDLFiles = path.join(homedir, "mandarineFiles");
+
+const config = require("../config.json");
+
 let parseFile;
+let fileCount = 0;
+let dirCount = 0;
+
 // import('music-metadata').then((mmModule)=>{console.log("mm: " + mmModule); parseFile = mmModule.parseFile});
 
-fs.readdir(pathToFiles, async function(err, files) {
 
-	let filecount = 0;
-	if (err) throw err;
-	// console.log(files);
-	const musicmetadata = await import("music-metadata");
-	parseFile = musicmetadata.parseFile;
-	for (let i = 0; i < files.length; i++) {
-		const fileName = files[i];
-		// console.log(fileName);
-		// only support mp3
-		if (fileName.endsWith(".mp3")) {
-			await addFile(fileName);
-			filecount++;
+async function indexDirectory(directory) {
+	const files = await fs.readdir(directory, { withFileTypes: true });
+	for (const file of files) {
+		if (file.isDirectory()) {
+			await indexDirectory(path.join(directory, file.name));
+			dirCount++;
 		}
-		if (i == files.length - 1) {
-			console.log("write");
-			setTimeout(() => {
-				console.log("Indexed " + filecount + " files.");
-				write();
-			}, 3_000);
+		else if (file.name.endsWith(".mp3")) {
+			await addFile(path.join(directory, file.name));
+			fileCount++;
 		}
 	}
-});
+	return true;
+}
 
-async function addFile(fileName) {
+async function addFile(filePath) {
 
-	if (!localLibrary[fileName]) {
-		const metadata = await parseFile(path.join(pathToFiles, fileName));
+	if (!localLibrary[filePath]) {
+		const metadata = await parseFile(filePath);
 		// console.log("metadata:" + JSON.stringify(metadata.common.title));
 		const title = metadata.common.title;
 		const artist = metadata.common.artist;
-		localLibrary[fileName] = {
+		const search = [
+			(title).replace(/\.|'|-/g, ""),
+			(artist + " " + title).replace(/\.|'|-/g, ""),
+			(title + " " + artist).replace(/\.|'|-/g, ""),
+		];
+
+		if (artist.split(" ").length > 1) {
+			// if the artist has spaces in their names, chances are people will only add one part of their name
+			for (const artistNameSplit of artist.split(" ")) {
+				search.push((title + " " + artistNameSplit).replace(/\.|'|-/g, ""));
+				search.push((artistNameSplit + " " + title).replace(/\.|'|-/g, ""));
+			}
+		}
+
+		if (search[2] != (title).replace(/\(.*\)|\.|'|-/g, "")) {
+			search.push((title).replace(/\(.*\)|\.|'|-/g, ""));
+		}
+
+		localLibrary[filePath] = {
 			"title": title,
 			"artist": artist,
-			"search": [
-				(title + " " + artist).replace(/\.|'|-/g, ""),
-				(artist + " " + title).replace(/\.|'|-/g, ""),
-				(title).replace(/\.|'|-/g, ""),
-				(title).replace(/\(.*\)|\.|'|-/g, ""),
-			],
+			"search": search,
 		};
 
-
 	}
 }
 
-function write() {
+async function write() {
 	// console.log(localLibrary);
-	fs.writeFileSync(path.join(__dirname, "localLibrary.json"), JSON.stringify(localLibrary));
-}
+	await fs.writeFile(path.join(__dirname, "localLibrary.json"), JSON.stringify(localLibrary));
+	return true;
+}
+
+async function index() {
+	const musicmetadata = await import("music-metadata");
+	parseFile = musicmetadata.parseFile;
+
+	if (config.additionalMusicDirs) {
+		for (const dir of config.additionalMusicDirs) {
+			console.log("Indexing " + dir + " and its subdirectories.");
+			await indexDirectory(dir);
+		}
+	}
+	console.log("Indexing " + pathToDLFiles + " and its subdirectories.");
+	await indexDirectory(pathToDLFiles);
+	console.log("write");
+	console.log("Indexed " + fileCount + " files in " + dirCount + " folders.");
+	await write();
+	return 1;
+}
+
+
+if (require.main === module) {
+	// called directly
+	index();
+}
+
+module.exports = { index };

music/libraryManager.js

@@ -18,7 +18,7 @@ import("music-metadata").then((mmModule) => {
 const meezer = require("./meezer.js");
 const path = require("path");
 
-const pathToFiles = path.join(homedir, "/mandarineFiles/");
+const pathToDLFiles = path.join(homedir, "/mandarineFiles/");
 
 /*
 
@@ -47,21 +47,27 @@ function createFuzzySetArr() {
 	return fzpromise;
 }
 
+// get array of files and create new fuzzy object, only use search key
+let fuzzySet;
+createFuzzySetArr().then((arr) => {
+	fuzzySet = new FuzzySet(arr);
+});
+
+
 async function requestTrack(query) {
-	// get array of files and create new fuzzy object, only use search key
-	const fuzzyArr = await createFuzzySetArr();
-	const fuzzySet = new FuzzySet(fuzzyArr);
 
 	// perform search
-	const fuzzyResult = fuzzySet.get(query);
+	console.log("fuzzy search");
+	const fuzzyResult = await fuzzySet.get(query);
+	console.log(fuzzyResult);
 
 	// if valid match, return filename
-	if (fuzzyResult && fuzzyResult[0] && (console.log(fuzzyResult[0][0]) || true) && fuzzyResult[0][0] > 0.75) {
+	if (fuzzyResult && fuzzyResult[0] && (console.log(fuzzyResult[0][0]) || true) && fuzzyResult[0][0] > 0.77) {
 		console.log("fuzzy found");
-		console.log(fuzzyResult);
 		for (const key in localLibrary) {
 			if (localLibrary[key].search.find(pattern => pattern == fuzzyResult[0][1])) {
-				const fuzzyfilepath = path.join(pathToFiles, key);
+				// const fuzzyfilepath = path.join(pathToFiles, key);
+				const fuzzyfilepath = path.join(key);
 				console.log(fuzzyfilepath);
 				return {
 					"path":fuzzyfilepath,
@@ -79,7 +85,7 @@ async function requestTrack(query) {
 		console.log("Track not found");
 		return false;
 	}
-	// see if track exists and fuse missed it
+	//  before download, see if track exists and fuzzy missed it
 	const trackExists = meezer.trackExists(result.artist.name, result.title);
 	if (trackExists) {
 		console.log("track exist");
@@ -123,20 +129,38 @@ async function requestTrack(query) {
 	};*/
 
 function addToLibrary(artist, title, fileName) {
-	localLibrary[fileName] = {
+	const search = [
+		(title).replace(/\.|'|-/g, ""),
+		(artist + " " + title).replace(/\.|'|-/g, ""),
+		(title + " " + artist).replace(/\.|'|-/g, ""),
+	];
+
+	if (artist.split(" ").length > 1) {
+		// if the artist has spaces in their names, chances are people will only add one part of their name
+		for (const artistNameSplit of artist.split(" ")) {
+			search.push((title + " " + artistNameSplit).replace(/\.|'|-/g, ""));
+			search.push((artistNameSplit + " " + title).replace(/\.|'|-/g, ""));
+		}
+	}
+
+	if (search[2] != (title).replace(/\(.*\)|\.|'|-/g, "")) {
+		search.push((title).replace(/\(.*\)|\.|'|-/g, ""));
+	}
+
+	localLibrary[path.join(pathToDLFiles, fileName)] = {
 		"title": title,
 		"artist": artist,
-		"search": [
-			(title + " " + artist).replace(/\.|'|-/g, ""),
-			(artist + " " + title).replace(/\.|'|-/g, ""),
-			(title).replace(/\.|'|-/g, ""),
-			(title).replace(/\(.*\)|\.|'|-/g, ""),
-		],
+		"search": search,
 	};
+	// add to fuzzy set
+	for (const searchItem of search) {
+		fuzzySet.add(searchItem);
+	}
+
 	fs.writeFileSync(path.join(__dirname, "localLibrary.json"), JSON.stringify(localLibrary));
 }
 
-console.log(pathToFiles);
+console.log(pathToDLFiles);
 /* async function test() {
 	console.log(await requestTrack("metro boomin superhero"));
 }*/