Skip to content

Commit a427627

Browse files
gfsgfs
authored
Add Archive Crawling powered by OSS Gadget MultiExtractor (#471)
* Add Archive Crawling powered by OSS Gadget MultiExtractor use --crawl-archives on file collector. * Catch all exceptions * Update FileSystemUtils.cs
1 parent 08f0c26 commit a427627

File tree

7 files changed

+186
-130
lines changed

7 files changed

+186
-130
lines changed

‎Lib/Collectors/FileSystemCollector.cs‎

Lines changed: 66 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// Copyright (c) Microsoft Corporation. All rights reserved.
22
// Licensed under the MIT License.
33
using AttackSurfaceAnalyzer.Objects;
4+
using AttackSurfaceAnalyzer.Types;
45
using AttackSurfaceAnalyzer.Utils;
56
using Microsoft.CodeAnalysis;
67
using Mono.Unix;
@@ -11,15 +12,14 @@
1112
using System.ComponentModel;
1213
using System.IO;
1314
using System.Linq;
14-
using System.Management;
1515
using System.Runtime.InteropServices;
1616
using System.Security;
1717
using System.Security.AccessControl;
1818
using System.Security.Cryptography.X509Certificates;
1919
using System.Security.Permissions;
2020
using System.Security.Principal;
21-
using System.Threading;
2221
using System.Threading.Tasks;
22+
using Microsoft.CST.OpenSource.MultiExtractor;
2323

2424
namespace AttackSurfaceAnalyzer.Collectors
2525
{
@@ -106,6 +106,16 @@ public override void ExecuteInternal()
106106
{
107107
HandleChange(obj);
108108

109+
// If we know how to handle this as an archive, and crawling archives is enabled
110+
if (opts.CrawlArchives && MiniMagic.DetectFileType(file) != ArchiveFileType.UNKNOWN)
111+
{
112+
Extractor extractor = new Extractor(new ExtractorOptions() { ExtractSelfOnFail = false });
113+
foreach (var fso in extractor.ExtractFile(file, !opts.SingleThread).Select(fileEntry => FileEntryToFileSystemObject(fileEntry)))
114+
{
115+
HandleChange(fso);
116+
}
117+
}
118+
109119
// TODO: Also try parse .DER as a key
110120
if (Path.EndsWith(".cer", StringComparison.CurrentCulture) ||
111121
Path.EndsWith(".der", StringComparison.CurrentCulture) ||
@@ -165,14 +175,42 @@ public override void ExecuteInternal()
165175
}
166176
}
167177

168-
/// <summary>
169-
/// Converts a FileSystemInfo into a FileSystemObject by reading in data about the file
170-
/// </summary>
171-
/// <param name="fileInfo">A reference to a file on disk.</param>
172-
/// <param name="downloadCloud">If the file is hosted in the cloud, the user has the option to include cloud files or not.</param>
173-
/// <param name="includeContentHash">If we should generate a hash of the file.</param>
174-
/// <returns></returns>
175-
public FileSystemObject FilePathToFileSystemObject(string path)
178+
private FileSystemObject FileEntryToFileSystemObject(FileEntry fileEntry)
179+
{
180+
var fso = new FileSystemObject(Path: fileEntry.FullPath)
181+
{
182+
Size = (ulong)fileEntry.Content.Length
183+
};
184+
185+
if (opts.GatherHashes == true)
186+
{
187+
fso.ContentHash = CryptoHelpers.CreateHash(fileEntry.Content);
188+
}
189+
190+
var exeType = FileSystemUtils.GetExecutableType(fileEntry.FullPath, fileEntry.Content);
191+
192+
if (exeType != EXECUTABLE_TYPE.NONE && exeType != EXECUTABLE_TYPE.UNKNOWN)
193+
{
194+
fso.IsExecutable = true;
195+
}
196+
197+
if (exeType == EXECUTABLE_TYPE.WINDOWS)
198+
{
199+
fso.SignatureStatus = WindowsFileSystemUtils.GetSignatureStatus(fileEntry.FullPath, fileEntry.Content);
200+
fso.Characteristics = WindowsFileSystemUtils.GetDllCharacteristics(fileEntry.FullPath, fileEntry.Content);
201+
}
202+
203+
return fso;
204+
}
205+
206+
/// <summary>
207+
/// Converts a FileSystemInfo into a FileSystemObject by reading in data about the file
208+
/// </summary>
209+
/// <param name="fileInfo">A reference to a file on disk.</param>
210+
/// <param name="downloadCloud">If the file is hosted in the cloud, the user has the option to include cloud files or not.</param>
211+
/// <param name="includeContentHash">If we should generate a hash of the file.</param>
212+
/// <returns></returns>
213+
public FileSystemObject FilePathToFileSystemObject(string path)
176214
{
177215
FileSystemObject obj = new FileSystemObject(path);
178216

@@ -318,14 +356,19 @@ e is ArgumentNullException
318356
obj.ContentHash = FileSystemUtils.GetFileHash(fileInfo);
319357
}
320358

321-
obj.IsExecutable = FileSystemUtils.IsExecutable(obj.Path, size);
359+
var exeType = FileSystemUtils.GetExecutableType(path);
360+
361+
if (exeType != EXECUTABLE_TYPE.NONE && exeType != EXECUTABLE_TYPE.UNKNOWN)
362+
{
363+
obj.IsExecutable = true;
364+
}
322365

323-
if (FileSystemUtils.IsWindowsExecutable(obj.Path, obj.Size))
366+
if (exeType == EXECUTABLE_TYPE.WINDOWS)
324367
{
325368
obj.SignatureStatus = WindowsFileSystemUtils.GetSignatureStatus(path);
326369
obj.Characteristics = WindowsFileSystemUtils.GetDllCharacteristics(path);
327370
}
328-
else if (FileSystemUtils.IsMacExecutable(obj.Path, obj.Size))
371+
else if (exeType == EXECUTABLE_TYPE.MACOS)
329372
{
330373
obj.MacSignatureStatus = FileSystemUtils.GetMacSignature(path);
331374
}
@@ -368,13 +411,20 @@ e is ArgumentNullException
368411
{
369412
obj.ContentHash = FileSystemUtils.GetFileHash(path);
370413
}
371-
obj.IsExecutable = FileSystemUtils.IsExecutable(obj.Path, obj.Size);
372-
if (FileSystemUtils.IsWindowsExecutable(obj.Path, obj.Size))
414+
415+
var exeType = FileSystemUtils.GetExecutableType(path);
416+
417+
if (exeType != EXECUTABLE_TYPE.NONE && exeType != EXECUTABLE_TYPE.UNKNOWN)
418+
{
419+
obj.IsExecutable = true;
420+
}
421+
422+
if (exeType == EXECUTABLE_TYPE.WINDOWS)
373423
{
374424
obj.SignatureStatus = WindowsFileSystemUtils.GetSignatureStatus(path);
375425
obj.Characteristics = WindowsFileSystemUtils.GetDllCharacteristics(path);
376426
}
377-
else if (FileSystemUtils.IsMacExecutable(obj.Path, obj.Size))
427+
else if (exeType == EXECUTABLE_TYPE.MACOS)
378428
{
379429
obj.MacSignatureStatus = FileSystemUtils.GetMacSignature(path);
380430
}

‎Lib/Collectors/FileSystemUtils.cs‎

Lines changed: 30 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// Copyright (c) Microsoft Corporation. All rights reserved.
22
// Licensed under the MIT License.
33
using AttackSurfaceAnalyzer.Objects;
4+
using AttackSurfaceAnalyzer.Types;
45
using AttackSurfaceAnalyzer.Utils;
56
using Mono.Unix;
67
using Serilog;
@@ -179,124 +180,52 @@ e is ArgumentException
179180
return signature;
180181
}
181182
}
182-
catch (Exception e) {
183+
catch (Exception e)
184+
{
183185
Log.Verbose("Failed to get Mac CodeSign information for {0} ({1}:{2})", Path, e.GetType(), e.Message);
184186
}
185187
return null;
186188
}
187189

188-
public static bool? IsExecutable(string? Path, ulong? Size)
190+
public static EXECUTABLE_TYPE GetExecutableType(string Path)
189191
{
190-
if (Path is null || Size is null) { return null; }
191-
if (Size < 4) { return false; }
192-
193-
// Shortcut to help with system files we can't read directly
194-
if (Path.EndsWith(".dll") || Path.EndsWith(".exe"))
195-
{
196-
return true;
197-
}
198-
199-
byte[] fourBytes = new byte[4];
200-
try
201-
{
202-
using (var fileStream = File.OpenRead(Path))
203-
{
204-
fileStream.Read(fourBytes, 0, 4);
205-
}
206-
}
207-
catch (Exception e) when (
208-
e is ArgumentException
209-
|| e is ArgumentNullException
210-
|| e is PathTooLongException
211-
|| e is DirectoryNotFoundException
212-
|| e is IOException
213-
|| e is UnauthorizedAccessException
214-
|| e is ArgumentOutOfRangeException
215-
|| e is FileNotFoundException
216-
|| e is NotSupportedException
217-
|| e is ObjectDisposedException)
218-
{
219-
Log.Verbose("Couldn't chomp 4 bytes of {0} ({1}:{2})", Path, e.GetType().ToString(), e.Message);
220-
return false;
221-
}
222-
223-
return fourBytes.SequenceEqual(ElfMagicNumber) || fourBytes.SequenceEqual(JavaMagicNumber) || MacMagicNumbers.Contains(fourBytes) || fourBytes[0..2].SequenceEqual(WindowsMagicNumber);
192+
using var fs = new FileStream(Path, FileMode.Open);
193+
return GetExecutableType(Path, fs);
224194
}
225195

226-
public static bool IsMacExecutable(string? Path, ulong? Size)
196+
public static EXECUTABLE_TYPE GetExecutableType(string? Path, Stream input)
227197
{
228-
if (Path is null) { return false; }
229-
if (Size < 4) { return false; }
198+
if (input == null) { return EXECUTABLE_TYPE.UNKNOWN; }
199+
if (input.Length < 4) { return EXECUTABLE_TYPE.NONE; }
230200

231-
// Shortcut to help with system files we can't read directly
232-
if (Path.EndsWith(".app"))
233-
{
234-
return true;
235-
}
201+
var fourBytes = new byte[4];
202+
var initialPosition = input.Position;
236203

237-
byte[] fourBytes = new byte[4];
238204
try
239205
{
240-
using (var fileStream = File.Open(Path, FileMode.Open))
241-
{
242-
fileStream.Read(fourBytes, 0, 4);
243-
}
206+
input.Read(fourBytes);
207+
input.Position = initialPosition;
244208
}
245-
catch (Exception e) when (
246-
e is ArgumentException
247-
|| e is ArgumentNullException
248-
|| e is PathTooLongException
249-
|| e is DirectoryNotFoundException
250-
|| e is IOException
251-
|| e is UnauthorizedAccessException
252-
|| e is ArgumentOutOfRangeException
253-
|| e is FileNotFoundException
254-
|| e is NotSupportedException
255-
|| e is ObjectDisposedException)
209+
catch (Exception e)
256210
{
257211
Log.Verbose("Couldn't chomp 4 bytes of {0} ({1}:{2})", Path, e.GetType().ToString(), e.Message);
258-
return false;
212+
return EXECUTABLE_TYPE.UNKNOWN;
259213
}
260214

261-
return MacMagicNumbers.Contains(fourBytes);
262-
}
263-
264-
public static bool IsWindowsExecutable(string? Path, ulong? Size)
265-
{
266-
if (Path is null) { return false; }
267-
if (Size < 4) { return false; }
268-
269-
// Shortcut to help with system files we can't read directly
270-
if (Path.EndsWith(".dll") || Path.EndsWith(".exe"))
215+
switch (fourBytes)
271216
{
272-
return true;
273-
}
217+
case var span when span.SequenceEqual(ElfMagicNumber):
218+
return EXECUTABLE_TYPE.LINUX;
219+
case var span when span.SequenceEqual(JavaMagicNumber):
220+
return EXECUTABLE_TYPE.JAVA;
221+
case var span when MacMagicNumbers.Contains(span):
222+
return EXECUTABLE_TYPE.MACOS;
223+
case var span when span[0..2].SequenceEqual(WindowsMagicNumber):
224+
return EXECUTABLE_TYPE.WINDOWS;
225+
default:
226+
return EXECUTABLE_TYPE.NONE;
274227

275-
byte[] fourBytes = new byte[4];
276-
try
277-
{
278-
using (var fileStream = File.OpenRead(Path))
279-
{
280-
fileStream.Read(fourBytes, 0, 4);
281-
}
282-
}
283-
catch (Exception e) when (
284-
e is ArgumentException
285-
|| e is ArgumentNullException
286-
|| e is PathTooLongException
287-
|| e is DirectoryNotFoundException
288-
|| e is IOException
289-
|| e is UnauthorizedAccessException
290-
|| e is ArgumentOutOfRangeException
291-
|| e is FileNotFoundException
292-
|| e is NotSupportedException
293-
|| e is ObjectDisposedException)
294-
{
295-
Log.Verbose("Couldn't chomp 4 bytes of {0} ({1}:{2})", Path, e.GetType().ToString(), e.Message);
296-
return false;
297228
}
298-
299-
return fourBytes[0..2].SequenceEqual(WindowsMagicNumber);
300229
}
301230

302231
public static string GetFileHash(string path)
@@ -313,35 +242,25 @@ public static string GetFileHash(string path)
313242

314243
public static string? GetFileHash(FileSystemInfo fileInfo)
315244
{
245+
string? hashValue = null;
246+
316247
if (fileInfo != null)
317248
{
318249
Log.Debug("{0} {1}", Strings.Get("FileHash"), fileInfo.FullName);
319250

320-
string? hashValue = null;
321251
try
322252
{
323253
using (var stream = File.OpenRead(fileInfo.FullName))
324254
{
325255
hashValue = CryptoHelpers.CreateHash(stream);
326256
}
327257
}
328-
catch (Exception e) when (
329-
e is ArgumentNullException
330-
|| e is ArgumentException
331-
|| e is NotSupportedException
332-
|| e is FileNotFoundException
333-
|| e is IOException
334-
|| e is System.Security.SecurityException
335-
|| e is DirectoryNotFoundException
336-
|| e is UnauthorizedAccessException
337-
|| e is PathTooLongException
338-
|| e is ArgumentOutOfRangeException)
258+
catch (Exception e)
339259
{
340260
Log.Verbose("{0}: {1} {2}", Strings.Get("Err_UnableToHash"), fileInfo.FullName, e.GetType().ToString());
341261
}
342-
return hashValue;
343262
}
344-
return null;
263+
return hashValue;
345264
}
346265
}
347266
}

0 commit comments

Comments
 (0)