Extend file system with owner keys

Author: Thomas Fuchs

docs/articles/Core/FileSystem.md

@@ -0,0 +1,22 @@
+---
+uid: FileSystem
+---
+# FileSystem
+
+````mermaid
+classDiagram
+    MoryxFile <|-- Blob
+    MoryxFile <|-- Tree
+    Tree --> MoryxFile    
+    OwnerFile --> Tree
+    class MoryxFileSystem{
+        -string Directory
+        +WriteBlob()
+        +WriteTree()
+        +ReadBlob()
+        +ReadTree()
+    }
+    class MoryxFile {
+        +String Hash
+    }
+````

src/Moryx.Runtime.Kernel/FileSystem/MoryxFileSystem.cs

@@ -1,8 +1,10 @@
 using Microsoft.Extensions.Logging;
 using Moryx.FileSystem;
+using Moryx.Logging;
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 
@@ -11,6 +13,7 @@ namespace Moryx.Runtime.Kernel.FileSystem
     internal class MoryxFileSystem : IMoryxFileSystem
     {
         private string _fsDirectory;
+        private string _ownerFilesDirectory;
         private readonly ILogger _logger;
 
         public MoryxFileSystem(ILoggerFactory loggerFactory)
@@ -21,15 +24,10 @@ public MoryxFileSystem(ILoggerFactory loggerFactory)
         public void SetBasePath(string basePath = "fs")
         {
             _fsDirectory = Path.Combine(Directory.GetCurrentDirectory(), basePath);
+            _ownerFilesDirectory = Path.Combine(_fsDirectory, "owners");
         }
 
-        public Stream ReadFile(string hash)
-        {
-            var path = HashPath.FromHash(hash).FilePath(_fsDirectory);           
-            return File.Exists(path) ? new FileStream(path, FileMode.Open, FileAccess.Read) : null;
-        }
-
-        public async Task<string> WriteFile(Stream stream)
+        public async Task<string> WriteBlob(Stream stream)
         {
             var hashPath = HashPath.FromStream(stream);
 
@@ -64,6 +62,139 @@ public async Task<string> WriteFile(Stream stream)
 
             return hashPath.Hash;
         }
+        public Task<string> WriteTree(IReadOnlyList<MoryxFileMetadata> metadata)
+        {
+            // Convert metadata to lines 
+            var lines = metadata.Select(md => md.ToString()).ToList();
+            var stream = new MemoryStream();
+            using (var sw = new StreamWriter(stream))
+            {
+                foreach (var line in lines)
+                    sw.WriteLine(line);
+                sw.Flush();
+            }
+
+            return WriteBlob(stream);
+        }
+
+        public async Task<string> WriteBlob(Stream fileStream, string ownerKey, MoryxFileMetadata metadata)
+        {
+            // Create file first
+            var hash = await WriteBlob(fileStream);
+            metadata.Hash = hash;
+
+            // Read current owner tree
+            var ownerFile = Path.Combine(_ownerFilesDirectory, ownerKey);
+            var ownerTree = File.ReadAllText(ownerFile);
+            var tree = ReadExtensibleTree(ownerTree);
+
+            // Add to owner tree and write new
+            tree.Add(metadata);
+            var treeHash = WriteTree(tree);
+
+            // Update owner file
+            var ownerFilePath = Path.Combine(_ownerFilesDirectory, ownerKey);
+            if (File.Exists(ownerFilePath))
+                await File.WriteAllLinesAsync(ownerFilePath, new[] { hash });
+            else
+                await File.AppendAllLinesAsync(ownerFilePath, new[] { hash });
+
+            return hash;
+        }
+
+        public Stream ReadBlob(string hash)
+        {
+            var path = HashPath.FromHash(hash).FilePath(_fsDirectory);
+            return File.Exists(path) ? new FileStream(path, FileMode.Open, FileAccess.Read) : null;
+        }
+
+        public IReadOnlyList<MoryxFileMetadata> ReadTree(string hash) => ReadExtensibleTree(hash);
+
+
+        public IReadOnlyList<MoryxFileMetadata> ReadTreeByOwner(string ownerKey)
+        {
+            // read hash from owner file
+            var ownerFile = Path.Combine(_ownerFilesDirectory, ownerKey);
+            var ownerTree = File.ReadAllText(ownerFile);
+
+            return ReadExtensibleTree(ownerTree);
+        }
+
+        private List<MoryxFileMetadata> ReadExtensibleTree(string hash)
+        {
+            // Read tree from hash
+            var stream = ReadBlob(hash);
+            var metadata = new List<MoryxFileMetadata>();
+            using (var sr = new StreamReader(stream))
+            {
+                var line = sr.ReadLine();
+                metadata.Add(MoryxFileMetadata.FromLine(line));
+            }
+
+            return metadata;
+        }
+
+        public bool RemoveFile(string hash, string ownerKey)
+        {
+            if (!IsOwner(hash, ownerKey))
+                return false;
+
+            // Delete file if found
+            var hashPath = HashPath.FromHash(hash);
+            var filePath = hashPath.FilePath(_fsDirectory);
+            if (!File.Exists(filePath))
+                return false;
+            RemoveFile(filePath, _logger);
+
+            // Check if subdirectory is empty and remove
+            var directory = hashPath.DirectoryPath(_fsDirectory);
+            CleanUpDirectory(directory, _logger);
+
+            // TODO: Remove file from owner list
+
+            return true;
+        }
+
+        private bool IsOwner(string hash, string ownerFile)
+        {
+            var ownerFilePath = Path.Combine(_ownerFilesDirectory, ownerFile);
+            using (var reader = new StreamReader(ownerFilePath))
+            {
+                string line;
+                while ((line = reader.ReadLine()) != null)
+                {
+                    if (line.Contains(searchLine))
+                        return true;
+                }
+            }
+            return false;
+        }
+
+        private void RemoveFile(string filePath, ILogger logger)
+        {
+            try
+            {
+                File.Delete(filePath);
+            }
+            catch (Exception e)
+            {
+                throw LoggedException(e, logger, filePath);
+            }
+        }
+
+        private void CleanUpDirectory(string directoryPath, ILogger logger)
+
+        {
+            try
+            {
+                if (Directory.GetFiles(directoryPath).Length == 0)
+                    Directory.Delete(directoryPath);
+            }
+            catch (Exception e)
+            {
+                throw LoggedException(e, logger, directoryPath);
+            }
+        }
 
         private Exception LoggedException(Exception e, ILogger logger, string cause)
         {

src/Moryx/FileSystem/IMoryxFileSystem.cs

@@ -14,13 +14,41 @@ public interface IMoryxFileSystem
         /// <summary>
         /// Write a file to the file system and receive the hash to access it later
         /// </summary>
-        Task<string> WriteFile(Stream fileStream);
+        Task<string> WriteBlob(Stream fileStream);
+
+        /// <summary>
+        /// Write a file to the file system and receive the hash to access it later
+        /// </summary>
+        Task<string> WriteBlob(Stream fileStream, string ownerKey, MoryxFileMetadata metadata);
+
+        /// <summary>
+        /// Write a file to the file system and receive the hash to access it later
+        /// </summary>
+        Task<string> WriteTree(IReadOnlyList<MoryxFileMetadata> metadata);
 
         /// <summary>
         /// Read the file by passing the file system hash
         /// </summary>
         /// <param name="hash"></param>
         /// <returns></returns>
-        Stream ReadFile(string hash);
+        Stream ReadBlob(string hash);
+
+        /// <summary>
+        /// Read a tree file and return the listed files
+        /// </summary>
+        IReadOnlyList<MoryxFileMetadata> ReadTree(string hash);
+
+        /// <summary>
+        /// Return all files stored under
+        /// </summary>
+        /// <param name="ownerKey"></param>
+        /// <returns></returns>
+        IReadOnlyList<MoryxFileMetadata> ReadTreeByOwner(string ownerKey);
+
+        /// <summary>
+        /// Remove a file by hash and provided owner key.
+        /// Files without owner key can not be removed
+        /// </summary>
+        bool RemoveFile(string hash, string ownerKey);
     }
 }

src/Moryx/FileSystem/MoryxFileMetadata.cs

@@ -0,0 +1,48 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Moryx.FileSystem
+{
+    public class MoryxFileMetadata
+    {
+        public int Mode { get; set; }
+
+        public FileType FileType { get; set; }
+
+        public string MimeType { get; set; }
+
+        public string Hash { get; set; }
+
+        public string FileName { get; set; }
+
+        public string ToLine()
+        {
+            return $"{Mode} {FileType.ToString().ToLower()} {MimeType} {Hash} {FileName}";
+        }
+
+        public static MoryxFileMetadata FromLine(string line)
+        {
+            var parts = line.Split(' ');
+            return new MoryxFileMetadata
+            {
+                Mode = int.Parse(parts[0]),
+                FileType = (FileType)Enum.Parse(typeof(FileType), parts[1]),
+                MimeType = parts[2],
+                Hash = parts[3],
+                FileName = string.Join(" ", parts.Skip(4))
+            };
+        }
+    }
+
+    /// <summary>
+    /// Moryx file types in owner tree file
+    /// </summary>
+    public enum FileType
+    {
+        Blob = 0,
+
+        Tree = 1,
+    }
+}

src/StartProject.Asp/Program.cs

@@ -27,6 +27,7 @@ public static void Main(string[] args)
                     webBuilder.UseStartup<Startup>();
                 }).Build();
 
+            host.Services.UseMoryxFileSystem("fs");
             host.Services.UseMoryxConfigurations("Config");
 
             var moduleManager = host.Services.GetRequiredService<IModuleManager>();