Skip to content

Commit c2280b8

Browse files
rohitcoderrohitcoder
committed
Added quiet and stdout flags
1 parent 6c19c2c commit c2280b8

File tree

19 files changed

+636
-671
lines changed

19 files changed

+636
-671
lines changed

‎Dockerfile‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ WORKDIR /app
66

77
# Copy the local requirements.txt file to the container at /app
88
COPY requirements.txt /app/
9+
RUN apt-get update && apt-get install ffmpeg libsm6 libxext6 -y
910

1011
# Install the dependencies from requirements.txt
1112
RUN pip3 install --no-cache-dir -r requirements.txt
@@ -16,5 +17,5 @@ COPY . /app/
1617
# Install the Python package (assuming it contains a setup.py file)
1718
RUN pip3 install .
1819

19-
# Set the entrypoint to hawk_scanner
20-
ENTRYPOINT ["hawk_scanner"]
20+
# Run hawk_Scanner from python3 main.py
21+
ENTRYPOINT ["python3", "main.py"]

‎connection.yml.sample‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,9 @@ sources:
4242
tables:
4343
- table1
4444
- table2
45+
text:
46+
profile1:
47+
text: "Hello World HHWPK6943Q"
4548
postgresql:
4649
postgresql_example:
4750
host: YOUR_POSTGRESQL_HOST

‎hawk_scanner/commands/couchdb.py‎

Lines changed: 27 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ def check_data_patterns(db, patterns, profile_name, database_name):
2525
for field_name, field_value in document.items():
2626
if field_value:
2727
value_str = str(field_value)
28-
matches = system.analyze_strings(value_str, 'couchdb')
28+
matches = system.match_strings(value_str)
2929
if matches:
3030
for match in matches:
3131
results.append({
@@ -42,39 +42,36 @@ def check_data_patterns(db, patterns, profile_name, database_name):
4242

4343
return results
4444

45-
def execute(args, programmatic=False):
46-
try:
47-
results = []
48-
system.print_info(f"Running Checks for CouchDB Sources")
49-
connections = system.get_connection(args, programmatic)
45+
def execute(args):
46+
results = []
47+
system.print_info(f"Running Checks for CouchDB Sources")
48+
connections = system.get_connection()
5049

51-
if 'sources' in connections:
52-
sources_config = connections['sources']
53-
couchdb_config = sources_config.get('couchdb')
50+
if 'sources' in connections:
51+
sources_config = connections['sources']
52+
couchdb_config = sources_config.get('couchdb')
5453

55-
if couchdb_config:
56-
patterns = system.get_fingerprint_file(args, programmatic)
54+
if couchdb_config:
55+
patterns = system.get_fingerprint_file()
5756

58-
for key, config in couchdb_config.items():
59-
host = config.get('host')
60-
port = config.get('port', 5984) # default CouchDB port
61-
username = config.get('username')
62-
password = config.get('password')
63-
database = config.get('database')
57+
for key, config in couchdb_config.items():
58+
host = config.get('host')
59+
port = config.get('port', 5984) # default CouchDB port
60+
username = config.get('username')
61+
password = config.get('password')
62+
database = config.get('database')
6463

65-
if host and username and password and database:
66-
system.print_info(f"Checking CouchDB Profile {key} with host and authentication")
67-
else:
68-
system.print_error(f"Incomplete CouchDB configuration for key: {key}")
69-
continue
64+
if host and username and password and database:
65+
system.print_info(f"Checking CouchDB Profile {key} with host and authentication")
66+
else:
67+
system.print_error(f"Incomplete CouchDB configuration for key: {key}")
68+
continue
7069

71-
db = connect_couchdb(host, port, username, password, database)
72-
if db:
73-
results += check_data_patterns(db, patterns, key, database)
74-
else:
75-
system.print_error("No CouchDB connection details found in connection.yml")
70+
db = connect_couchdb(host, port, username, password, database)
71+
if db:
72+
results += check_data_patterns(db, patterns, key, database)
7673
else:
77-
system.print_error("No 'sources' section found in connection.yml")
78-
except Exception as e:
79-
system.print_error(f"Failed to execute CouchDB checks with error: {e}")
74+
system.print_error("No CouchDB connection details found in connection.yml")
75+
else:
76+
system.print_error("No 'sources' section found in connection.yml")
8077
return results

‎hawk_scanner/commands/firebase.py‎

Lines changed: 61 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -15,78 +15,75 @@ def connect_firebase(credentials_file, bucket_name):
1515
except Exception as e:
1616
print(f"Failed to connect to Firebase bucket: {e}")
1717

18-
def execute(args, programmatic=False):
19-
try:
20-
results = []
21-
shouldDownload = True
22-
connections = system.get_connection(args, programmatic)
23-
fingerprints = system.get_fingerprint_file(args, programmatic)
18+
def execute(args):
19+
results = []
20+
shouldDownload = True
21+
connections = system.get_connection()
2422

25-
if 'sources' in connections:
26-
sources_config = connections['sources']
27-
firebase_config = sources_config.get('firebase')
23+
if 'sources' in connections:
24+
sources_config = connections['sources']
25+
firebase_config = sources_config.get('firebase')
2826

29-
if firebase_config:
30-
for key, config in firebase_config.items():
31-
credentials_file = config.get('credentials_file')
32-
bucket_name = config.get('bucket_name')
33-
exclude_patterns = config.get(key, {}).get('exclude_patterns', [])
27+
if firebase_config:
28+
for key, config in firebase_config.items():
29+
credentials_file = config.get('credentials_file')
30+
bucket_name = config.get('bucket_name')
31+
exclude_patterns = config.get(key, {}).get('exclude_patterns', [])
3432

35-
if credentials_file and bucket_name:
36-
bucket = connect_firebase(credentials_file, bucket_name)
37-
if bucket:
38-
for blob in bucket.list_blobs():
39-
file_name = blob.name
40-
## get unique etag or hash of file
41-
remote_etag = blob.etag
42-
system.print_debug(f"Remote etag: {remote_etag}")
33+
if credentials_file and bucket_name:
34+
bucket = connect_firebase(credentials_file, bucket_name)
35+
if bucket:
36+
for blob in bucket.list_blobs():
37+
file_name = blob.name
38+
## get unique etag or hash of file
39+
remote_etag = blob.etag
40+
system.print_debug(f"Remote etag: {remote_etag}")
4341

44-
if system.should_exclude_file(file_name, exclude_patterns):
45-
continue
42+
if system.should_exclude_file(file_name, exclude_patterns):
43+
continue
4644

47-
file_path = f"data/firebase/{remote_etag}-{file_name}"
48-
os.makedirs(os.path.dirname(file_path), exist_ok=True)
45+
file_path = f"data/firebase/{remote_etag}-{file_name}"
46+
os.makedirs(os.path.dirname(file_path), exist_ok=True)
4947

50-
if config.get("cache") == True:
51-
if os.path.exists(file_path):
48+
if config.get("cache") == True:
49+
if os.path.exists(file_path):
50+
shouldDownload = False
51+
local_etag = file_path.split('/')[-1].split('-')[0]
52+
system.print_debug(f"Local etag: {local_etag}")
53+
system.print_debug(f"File already exists in cache, using it. You can disable cache by setting 'cache: false' in connection.yml")
54+
if remote_etag != local_etag:
55+
system.print_debug(f"File in firebase bucket has changed, downloading it again...")
56+
shouldDownload = True
57+
else:
5258
shouldDownload = False
53-
local_etag = file_path.split('/')[-1].split('-')[0]
54-
system.print_debug(f"Local etag: {local_etag}")
55-
system.print_debug(f"File already exists in cache, using it. You can disable cache by setting 'cache: false' in connection.yml")
56-
if remote_etag != local_etag:
57-
system.print_debug(f"File in firebase bucket has changed, downloading it again...")
58-
shouldDownload = True
59-
else:
60-
shouldDownload = False
6159

62-
if shouldDownload:
63-
file_path = f"data/firebase/{remote_etag}-{file_name}"
64-
system.print_debug(f"Downloading file: {file_name} to {file_path}...")
65-
blob.download_to_filename(file_path)
66-
67-
matches = system.analyze_file(file_path, 'google_cloud_storage', connections, fingerprints, programmatic=programmatic)
68-
if matches:
69-
for match in matches:
70-
results.append({
71-
'bucket': bucket_name,
72-
'file_path': file_name,
73-
'pattern_name': match['pattern_name'],
74-
'matches': match['matches'],
75-
'sample_text': match['sample_text'],
76-
'profile': key,
77-
'data_source': 'firebase'
78-
})
79-
if config.get("cache") == False:
80-
os.system("rm -rf data/firebase")
81-
else:
82-
system.print_error(f"Failed to connect to Firebase bucket: {bucket_name}")
60+
if shouldDownload:
61+
file_path = f"data/firebase/{remote_etag}-{file_name}"
62+
system.print_debug(f"Downloading file: {file_name} to {file_path}...")
63+
blob.download_to_filename(file_path)
64+
65+
matches = system.read_match_strings(file_path, 'google_cloud_storage')
66+
if matches:
67+
for match in matches:
68+
results.append({
69+
'bucket': bucket_name,
70+
'file_path': file_name,
71+
'pattern_name': match['pattern_name'],
72+
'matches': match['matches'],
73+
'sample_text': match['sample_text'],
74+
'profile': key,
75+
'data_source': 'firebase'
76+
})
77+
8378
else:
84-
system.print_error(f"Incomplete Firebase configuration for key: {key}")
85-
else:
86-
system.print_error("No Firebase connection details found in connection file")
79+
system.print_error(f"Failed to connect to Firebase bucket: {bucket_name}")
80+
else:
81+
system.print_error(f"Incomplete Firebase configuration for key: {key}")
8782
else:
88-
system.print_error("No 'sources' section found in connection.yml")
89-
90-
except Exception as e:
91-
print(f"Failed to connect to Firebase bucket: {e}")
83+
system.print_error("No Firebase connection details found in connection file")
84+
else:
85+
system.print_error("No 'sources' section found in connection.yml")
86+
87+
if config.get("cache") == False:
88+
os.system("rm -rf data/firebase")
9289
return results

‎hawk_scanner/commands/fs.py‎

Lines changed: 39 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
1-
import argparse, os, concurrent.futures, time
1+
import argparse
22
from google.cloud import storage
33
from rich.console import Console
44
from hawk_scanner.internals import system
5+
import os
6+
import concurrent.futures
7+
import time
58

6-
def process_file(file_path, key, connections, fingerprints, programmatic=False):
7-
matches = system.analyze_file(file_path, 'fs', connections, fingerprints, programmatic=programmatic)
9+
def process_file(file_path, key, results):
10+
matches = system.read_match_strings(file_path, 'fs')
811
file_data = system.getFileData(file_path)
9-
results = []
1012
if matches:
1113
for match in matches:
1214
results.append({
@@ -19,49 +21,42 @@ def process_file(file_path, key, connections, fingerprints, programmatic=False):
1921
'data_source': 'fs',
2022
'file_data': file_data
2123
})
22-
return results
23-
24-
def execute(args, programmatic=False):
25-
try:
26-
results = []
27-
connections = system.get_connection(args, programmatic)
28-
fingerprints = system.get_fingerprint_file(args, programmatic)
2924

30-
if 'sources' in connections:
31-
sources_config = connections['sources']
32-
fs_config = sources_config.get('fs')
33-
if fs_config:
34-
for key, config in fs_config.items():
35-
if 'path' not in config:
36-
system.print_error(f"Path not found in fs profile '{key}'")
37-
continue
38-
path = config.get('path')
39-
if not os.path.exists(path):
40-
system.print_error(f"Path '{path}' does not exist")
41-
42-
exclude_patterns = fs_config.get(key, {}).get('exclude_patterns', [])
43-
start_time = time.time()
44-
files = system.list_all_files_iteratively(path, exclude_patterns)
25+
def execute(args):
26+
results = []
27+
connections = system.get_connection()
28+
if 'sources' in connections:
29+
sources_config = connections['sources']
30+
fs_config = sources_config.get('fs')
31+
if fs_config:
32+
for key, config in fs_config.items():
33+
if 'path' not in config:
34+
system.print_error(f"Path not found in fs profile '{key}'")
35+
continue
36+
path = config.get('path')
37+
if not os.path.exists(path):
38+
system.print_error(f"Path '{path}' does not exist")
39+
40+
exclude_patterns = fs_config.get(key, {}).get('exclude_patterns', [])
41+
start_time = time.time()
42+
files = system.list_all_files_iteratively(path, exclude_patterns)
43+
44+
# Use ThreadPoolExecutor for parallel processing
45+
file_count = 0
46+
with concurrent.futures.ThreadPoolExecutor() as executor:
47+
futures = []
48+
for file_path in files:
49+
file_count += 1
50+
futures.append(executor.submit(process_file, file_path, key, results))
4551

46-
# Use ThreadPoolExecutor for parallel processing
47-
file_count = 0
48-
with concurrent.futures.ThreadPoolExecutor() as executor:
49-
futures = []
50-
for file_path in files:
51-
file_count += 1
52-
results += process_file(file_path, key, connections, fingerprints, programmatic=programmatic)
53-
54-
# Wait for all tasks to complete
55-
concurrent.futures.wait(futures)
56-
end_time = time.time()
57-
elapsed_time = round(end_time - start_time, 2)
58-
system.print_info(f"Time taken to analyze {file_count} files: {elapsed_time} seconds")
59-
else:
60-
system.print_error("No filesystem 'fs' connection details found in connection.yml")
52+
# Wait for all tasks to complete
53+
concurrent.futures.wait(futures)
54+
end_time = time.time()
55+
system.print_info(f"Time taken to analyze {file_count} files: {end_time - start_time} seconds")
6156
else:
62-
system.print_error("No 'sources' section found in connection.yml")
63-
except Exception as e:
64-
system.print_error(f"Error in executing filesystem checks: {e}")
57+
system.print_error("No filesystem 'fs' connection details found in connection.yml")
58+
else:
59+
system.print_error("No 'sources' section found in connection.yml")
6560
return results
6661

6762
if __name__ == "__main__":

0 commit comments

Comments
 (0)