add new tools, add new status checks, move from tabs to sidebar

Author: kevin-ctera

cloudfs.py

@@ -0,0 +1,114 @@
+#!/usr/bin/python
+import sys
+import csv
+import urllib3
+import logging
+
+from pathlib import Path
+from getpass import getpass
+from cterasdk import GlobalAdmin, portal_types, CTERAException, config, tojsonstr
+from cterasdk.lib.filesystem import FileSystem
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+def ask(prompt):
+    user_input = None
+    while not user_input:
+        user_input = input(prompt)
+    return user_input
+
+def create_folders(global_admin,filepath):
+    try:
+#        address = ask('Enter CTERA Portal address: ')
+#        username = ask('Username: ')
+#        password = getpass('Password: ')
+
+#        global_admin = GlobalAdmin(address)
+#        global_admin.login(username, password)
+        with open(filepath, 'r') as csv_file:
+            csv_reader = csv.DictReader(csv_file, delimiter=',')
+            for row in csv_reader:
+                try:
+                    folder_group = row['folder_group']
+                    cloud_drive_folder = row['cloud_drive_folder']
+                    cloud_drive_folder_description = row['cloud_drive_folder_description']
+                    user_owner = row['user_owner']
+                    zone_name = row['zone']
+                    zone_description = row['zone_description']
+                    deduplication_method_type = row['deduplication_method']
+                    idx = user_owner.rfind('\\')
+
+                    owner = None
+                    if idx > 0:
+                        domain, user = user_owner.split('\\')
+                        owner = portal_types.UserAccount(user, domain)
+                    else:
+                        owner = portal_types.UserAccount(user_owner)
+
+                    try:
+                        global_admin.cloudfs.mkfg(folder_group, owner, deduplication_method_type)
+                    except CTERAException as error:
+                        logging.getLogger().warn('Failed creating folder group. %s', {'name': folder_group, 'error': tojsonstr(error, False)})
+
+                    try:
+                        global_admin.cloudfs.mkdir(cloud_drive_folder, folder_group, owner, winacls=True, description=cloud_drive_folder_description)
+                    except CTERAException as error:
+                        logging.getLogger().warn('Failed creating cloud drive folder. %s', {'name': cloud_drive_folder, 'error': tojsonstr(error, False)})
+
+                    try:
+                        global_admin.zones.add(zone_name, description=zone_description)  # add zone
+                    except CTERAException as error:
+                        logging.getLogger().warn('Failed creating zone. %s', {'name': zone_name, 'error': tojsonstr(error, False)})
+
+                    try:
+                        cloudfs_folder_helper = portal_types.CloudFSFolderFindingHelper(cloud_drive_folder, owner)
+                        logging.getLogger().info('Adding cloud folders to zone. %s', {'zone': zone_name, 'cloud_drive_folders': [cloud_drive_folder]})
+                        global_admin.zones.add_folders(zone_name, [cloudfs_folder_helper])  # add folders
+                    except CTERAException as error:
+                        logging.getLogger().warn('Failed adding folders to zone. %s', {'zone': zone_name, 'error': tojsonstr(error, False)})
+                except CTERAException as error:
+                    print(error)
+                    input()
+        global_admin.logout()
+    except KeyboardInterrupt:
+        pass
+
+def usage():
+    print()
+    print('Usage: ' + sys.argv[0] + ' ' + '<csv>')
+
+if __name__ == "__main__":
+
+    config.http['ssl'] = 'Trust'  # ignore certificate errors connecting to CTERA Portal
+    config.Logging.get().setLevel(logging.INFO)  # enable debug: logging.DEBUG
+
+    args = sys.argv
+    if len(args) < 2:
+        logging.getLogger().error('You did not specify an input file. Exiting.')
+        usage()
+        quit()
+
+    if len(args) > 2:
+        logging.getLogger().error('Too many arguments.')
+        usage()
+        quit()
+
+    filesystem = FileSystem.instance()
+    try:
+        filepath = args[1]
+        logging.getLogger().debug('Looking for input file. %s', {'filepath': filepath})
+        info = FileSystem.get_local_file_info(filepath)  # look for config file
+        logging.getLogger().debug('Found input file. %s', {'name': info['name'], 'size': info['size']})
+    except CTERAException as error:
+        logging.getLogger().error('Could not find input file.', {'filepath': filepath})
+        usage()
+        quit()
+
+    try:
+        logging.getLogger().info('Populating CloudFS Structure on CTERA Portal.')
+        create_folders(filepath)
+        logging.getLogger().info('Completed.')
+    except CTERAException as error:
+        logging.getLogger().fatal(error.message)
+    except KeyboardInterrupt:
+        logging.getLogger().fatal('Cancelled by user.')

requirements.txt

@@ -1,2 +1,6 @@
-cterasdk 
+cterasdk
 Gooey
+itsdangerous==2.0.1
+Flask==2.1.0
+Werkzeug==2.0.1
+flask-socketio==4.3.2

smb_audit.py

@@ -0,0 +1,154 @@
+import sys
+import os
+import pandas as pd
+from pandas.api.types import CategoricalDtype
+import matplotlib.pyplot as plt
+from datetime import datetime
+import feather
+import math
+import glob
+import logging
+
+def convert_size(size_bytes):
+   if size_bytes == 0:
+       return "0B"
+   size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
+   i = int(math.floor(math.log(size_bytes, 1024)))
+   p = math.pow(1024, i)
+   s = round(size_bytes / p, 2)
+   return "%s %s" % (s, size_name[i])
+
+def extract_epoch(seq):
+    seq_type= type(seq)
+    return int(seq_type().join(filter(seq_type.isdigit, seq)))
+
+def convert_time(epoch_time):
+    return datetime.fromtimestamp(extract_epoch(epoch_time))
+
+def show_ftr_details(df):
+    logging.info('Most recent time of dataset: ' + str(df.local_time.max()))
+    logging.info('Oldest time of dataset: ' + str(df.local_time.min()))
+    logging.info('Time between start and end of dataset: ' + str(df.local_time.max() - df.local_time.min()))
+    logging.info("=====================================================") 
+    
+
+def parse_audit(source_directory, output_file):
+    pd.set_option('display.max_rows', None)
+    col_names = ["col0", "col1", "col2", "col3", "result", "col5", "user", "col7", "smb_operation_type", "utc_time", "local_time", "col11", "share", "path", "col14", "col15", "col16", "col17"]
+    small_dfs = [] 
+
+    all_files = glob.glob(os.path.join(source_directory, "audit.*.log"))
+    total_size = 0
+
+    for file in all_files:
+        total_size += os.path.getsize(file)
+    logging.info(all_files)
+    df_from_each_file = (pd.read_csv(f, sep='|', names=col_names, index_col=False, low_memory=False, parse_dates=['local_time'], date_parser=convert_time, usecols=[4, 6, 8, 10, 12, 13]) for f in all_files)
+
+    concatenated_df  = pd.concat(df_from_each_file, copy=False)
+    #concatenated_df.info()
+    #logging.info(concatenated_df.memory_usage(deep=True) / 1e6)
+
+    concatenated_df.smb_operation_type  = concatenated_df.smb_operation_type.astype('category')
+    concatenated_df.info()
+    logging.info(concatenated_df.memory_usage(deep=True) / 1e6)
+
+
+    logging.info('Creating feather file at: ' + str(output_file) + ".ftr")
+    concatenated_df.reset_index().to_feather(output_file + ".ftr")
+    #if (make_csv):
+    #    csv_file_name = output_file + ".csv"
+    #    logging.info('Creating output file at: ' + csv_file_name)
+    #    concatenated_df.reset_index().to_csv(csv_file_name, index = False, compression = 'gzip')
+    #    os.rename(csv_file_name, csv_file_name + ".gz")
+                
+    ftr_size = os.path.getsize(output_file + ".ftr")
+    logging.info("Completed parsing " + str(convert_size(total_size)) + " to " + str(convert_size(ftr_size)))
+    logging.info("=====================================================")
+    logging.info('Most recent time of dataset: ' + str(concatenated_df.local_time.max()))
+    logging.info('Oldest time of dataset: ' + str(concatenated_df.local_time.min()))
+    logging.info('Time between start and end of dataset: ' + str(concatenated_df.local_time.max() - concatenated_df.local_time.min()))
+    logging.info("=====================================================")  
+
+def summarize_audit(ftr_file, time_interval):
+    df = pd.read_feather(ftr_file)
+    logging.info('Loading FTR file at: ' + str(ftr_file))
+    logging.info("=====================================================")
+    show_ftr_details(df)
+    logging.info('Totals per SMB operation type for this dataset:\n' + str(df['smb_operation_type'].value_counts()))
+    logging.info("=====================================================")
+    logging.info('Top 10 users for this dataset:\n' + str(df['user'].value_counts().nlargest(10)))
+    logging.info("=====================================================")
+    logging.info('Top 10 shares for this dataset:\n' + str(df['share'].value_counts().nlargest(10)))
+    logging.info("=====================================================")
+    logging.info('Top 10 paths for this dataset:\n' + str(df['path'].value_counts().nlargest(10)))
+    logging.info("=====================================================")
+    #logging.info(str(df.groupby([df.local_time.dt.floor(time_interval), 'smb_operation_type']).size()))
+    
+    plt.rc('legend',fontsize=6)
+    #df.groupby([df.local_time.dt.floor('60min'), 'smb_operation_type']).size().plot()
+    #df.groupby([df.local_time.dt.floor(time_interval), 'smb_operation_type']).size().unstack().plot(colormap='nipy_spectral').legend(loc='center left',bbox_to_anchor=(1.0, 0.5))
+    df.groupby([df.local_time.dt.floor(time_interval), 'smb_operation_type']).size().unstack().plot(colormap='nipy_spectral', x_compat=True).legend(loc='best')
+    #df.groupby([df.local_time.dt.floor(time_interval), 'smb_operation_type']).size().unstack().plot(colormap='nipy_spectral', x_compat=True).legend(loc='best')
+    plt.show()
+    
+def search_audit(ftr_file, search_field, search_string, show_smb_ops):
+    df = pd.read_feather(ftr_file)
+    logging.info('Loading FTR file at: ' + str(ftr_file))
+    logging.info("=====================================================")
+    show_ftr_details(df)
+    #logging.info(df[df[arguments.search_field].str.contains(arguments.search_string)].to_string())
+    search_results = df[(df[search_field].str.contains(search_string)) & (df['smb_operation_type'].isin(show_smb_ops))].to_string()
+    logging.info(search_results)
+
+def smb_audit(args):
+    try:
+        if (args.is_debug):
+            logging.debug("Usage:\n{0}\n".format(" ".join([x for x in sys.argv])))
+            logging.debug("")
+            logging.debug("All settings used:")
+            for k,v in sorted(vars(args).items()):
+                logging.debug("{0}: {1}".format(k,v))
+        
+        if (args.function == 'Parse'):
+            parse_audit(args.source_directory, args.output_file)
+        
+        if (args.function == "Summarize"):
+            summarize_audit(args.ftr_file, args.time_interval)
+   
+        if (args.function == "Search"): 
+            #Create List of SMB Operations to show in Search    
+            show_smb_ops = []
+            if (not args.ACEChanged): 
+                show_smb_ops.append('op=ACEChanged')
+            if (not args.ACLAdded):
+                show_smb_ops.append('op=ACLAdded')
+            if (not args.ACLDeleted):
+                show_smb_ops.append('op=ACLDeleted')   
+            if (not args.AclDenied):
+                show_smb_ops.append('op=AclDenied')        
+            if (not args.chown):
+                show_smb_ops.append('op=chown')           
+            if (not args.create):
+                show_smb_ops.append('op=create')            
+            if (not args.createDenied):
+                show_smb_ops.append('op=createDenied')
+            if (not args.delete):
+                show_smb_ops.append('op=delete')
+            if (not args.deleteDenied):
+                show_smb_ops.append('op=deleteDenied')
+            if (not args.move):
+                show_smb_ops.append('op=move')
+            if (not args.open):
+                show_smb_ops.append('op=open')
+            if (not args.OpenDenied):
+                show_smb_ops.append('op=OpenDenied')
+            if (not args.setattrib):
+                show_smb_ops.append('op=setattrib')
+            if (not args.setdacl):
+                show_smb_ops.append('op=setdacl')
+            if (not args.write):
+                show_smb_ops.append('op=write')
+            search_audit(args.ftr_file, args.search_field, args.search_string, show_smb_ops)
+    except KeyboardInterrupt:
+        logging.getLogger().fatal('Cancelled by user.')