Skip to content

Commit 2d1b25e

Browse files
rachejazzrachejazz
authored
[URGENT] fixes empty data loading (#226)
* [URGENT] fixes empty data loading Signed-off-by: rachejazz <[email protected]> * changed credentials Signed-off-by: rachejazz <[email protected]> * removed json files references from classes Signed-off-by: rachejazz <[email protected]> * removed json files references from config Signed-off-by: rachejazz <[email protected]> * cleanup Signed-off-by: rachejazz <[email protected]> * cleanup Signed-off-by: rachejazz <[email protected]> * same host allthrought Signed-off-by: rachejazz <[email protected]> * Update database_build.py Signed-off-by: Divya Goswami <[email protected]> * Update database_build.py Signed-off-by: Divya Goswami <[email protected]> --------- Signed-off-by: rachejazz <[email protected]> Signed-off-by: Divya Goswami <[email protected]>
1 parent 3b84f8c commit 2d1b25e

File tree

4 files changed

+15
-134
lines changed

4 files changed

+15
-134
lines changed

‎.env.example‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@
22
DB_HOST=localhost
33
DB_USER=sdtreaduser
44
DB_PASSWORD=UPDATEME
5-
DB_NAME=sdtDB
5+
DB_NAME=sdtDB

‎bin/database_build.py‎

Lines changed: 12 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,20 @@
44
import pymysql
55
import sys
66
import os
7+
from dotenv import load_dotenv
8+
load_dotenv() # Load database name from the .env file
79

810
sys.path.append('/opt/software-discovery-tool/src/config')
911
import supported_distros
1012
SUPPORTED_DISTROS = supported_distros.SUPPORTED_DISTROS
1113
SDT_BASE = '/opt/software-discovery-tool'
1214
DATA_FILE_LOCATION = '%s/distro_data/data_files' % SDT_BASE
13-
HOST = 'localhost'
14-
DB_NAME = 'sdtDB'
15+
16+
HOST = os.environ.get('DB_HOST')
17+
USER = ''
18+
PASSWORD = ''
19+
DB_NAME = os.environ.get('DB_NAME')
20+
1521
def connectdb(username,password,database):
1622
conn = pymysql.connect(host=HOST,user=username,password=password)
1723
cur = conn.cursor()
@@ -21,20 +27,12 @@ def connectdb(username,password,database):
2127
print("DB INITIATILIZED SUCCESSFULLY")
2228

2329
def db_init():
24-
username = ""
30+
username = USER
31+
password = PASSWORD
2532
table_name = ""
2633

27-
if len(sys.argv)==2 and sys.argv[1]=='root':
28-
username = sys.argv[1]
29-
elif len(sys.argv)==2:
30-
username = input("Enter username to use for connecting to MariaDB server : ")
31-
table_name = sys.argv[1]
32-
elif len(sys.argv)==3 and sys.argv[1]=='root':
33-
username = sys.argv[1]
34-
table_name = sys.argv[2]
35-
else:
36-
username = input("Enter username to use for connecting to MariaDB server : ")
37-
password = input("Enter password for connecting to MariaDB server : ")
34+
username = input("Enter privileged username to create/update SQL tables: ")
35+
password = input("Enter password for privileged username: ")
3836
dbName = DB_NAME
3937

4038
if table_name == "" or table_name == "all" or table_name == "All":
@@ -46,9 +44,7 @@ def db_init():
4644
def jsontosql(db,table,file,os,user,password):
4745
filepath = f'{DATA_FILE_LOCATION}/{file}.json'
4846
jsonFile = open(file=filepath)
49-
#print(jsonFile)
5047
data = json.load(jsonFile)
51-
#final_data = [dict(item, osName=os) for item in data]
5248
final_data = []
5349
for item in data :
5450
if item :

‎src/classes/package_search.py‎

Lines changed: 0 additions & 115 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,6 @@
1919

2020
class PackageSearch:
2121
package_data = {}
22-
local_cache ={}
23-
cache_keys = []
2422
DISTRO_BIT_MAP = {}
2523
INSTANCE = None
2624

@@ -52,84 +50,12 @@ def loadSupportedDistros(cls):
5250
bitFlag += bitFlag
5351
return cls.DISTRO_BIT_MAP
5452

55-
@classmethod
56-
def loadPackageData(cls):
57-
'''
58-
Returns list of Packages in software-discovery-tool
59-
'''
60-
61-
LOGGER.debug('loadPackageData: In loadSupportedDistros')
62-
distro_data_file = '%s/cached_data.json' % cls.getDataFilePath()
63-
try:
64-
json_data = json.load(open(distro_data_file))
65-
except:
66-
LOGGER.warn('loadPackageData: Loading cached distros data failed generating from scratch')
67-
LOGGER.debug('loadPackageData: start writing distros data')
68-
json_data = cls.preparePackageData()
69-
cached_file = open(distro_data_file, 'w')
70-
cached_file.write(json.dumps(json_data, indent=2, separators=(',', ': ')))
71-
cached_file.close()
72-
LOGGER.debug('loadPackageData: end writing distros data')
73-
74-
LOGGER.debug('loadPackageData: Loading supported distros data')
75-
76-
return json_data
77-
78-
@classmethod
79-
def preparePackageData(cls):
80-
data_dir = cls.getDataFilePath()
81-
package_info = [];
82-
package_data = {};
83-
cachedPackage = {}
84-
85-
for distroName in list(SUPPORTED_DISTROS.keys()):
86-
for distroVersion in sorted(SUPPORTED_DISTROS[distroName].keys()):
87-
distro_file = SUPPORTED_DISTROS[distroName][distroVersion]
88-
89-
package_info = json.load(open('%s/%s' % (data_dir, distro_file)))
90-
distro_file_name = distro_file
91-
92-
for pkg in package_info:
93-
try:
94-
pkg_key = pkg["packageName"] + '_' + pkg["version"]
95-
except Exception as ex:
96-
LOGGER.error('preparePackageData: key not found for package %s' % str(ex))
97-
if pkg_key not in package_data:
98-
cachedPackage = {}
99-
cachedPackage["P"] = pkg["packageName"]
100-
cachedPackage["S"] = cachedPackage["P"].lower().upper()
101-
cachedPackage["V"] = pkg["version"]
102-
if "description" in pkg:
103-
cachedPackage["D"] = pkg["description"]
104-
try:
105-
cachedPackage["B"] = cls.DISTRO_BIT_MAP[distroName][distroVersion]
106-
except Exception as e:
107-
raise #This occurrs only if there is a problem with how SUPPORTED_DISTROS is configured in config py
108-
109-
cachedPackage[distroName] = [distroVersion]
110-
package_data[pkg_key] = cachedPackage
111-
else:
112-
if distroName not in package_data[pkg_key]:
113-
package_data[pkg_key][distroName] = [distroVersion]
114-
package_data[pkg_key]['B'] += cls.DISTRO_BIT_MAP[distroName][distroVersion]
115-
else:
116-
if distroVersion not in package_data[pkg_key][distroName]:
117-
package_data[pkg_key][distroName].append(distroVersion)
118-
package_data[pkg_key]['B'] += cls.DISTRO_BIT_MAP[distroName][distroVersion]
119-
120-
json_data = list(package_data.values())
121-
122-
return json_data
123-
12453
@classmethod
12554
def get_instance(cls):
12655
LOGGER.debug('get_instance: In get_instance')
12756
if not cls.INSTANCE:
12857
cls.INSTANCE = PackageSearch()
12958
cls.INSTANCE.DISTRO_BIT_MAP = cls.loadSupportedDistros()
130-
cls.INSTANCE.package_data = cls.loadPackageData()
131-
cls.INSTANCE.local_cache = {}
132-
cls.INSTANCE.cache_keys = []
13359
LOGGER.debug('get_instance: Creating singleton instance in get_instance')
13460
return cls.INSTANCE
13561

@@ -170,50 +96,9 @@ def searchPackages(self, search_term, exact_match, search_bit_flag, page_number
17096
LOGGER.debug('searchPackages: search_packages_end_with : %s' % (search_packages_end_with))
17197
LOGGER.debug('searchPackages: search_anywhere_in_packages : %s' % (search_anywhere_in_packages))
17298

173-
cache_key = 'ck_%s_%s_%s' % (search_term, exact_match, search_bit_flag)
174-
LOGGER.debug('searchPackages: Cache Key is : %s' % (cache_key))
175-
17699
search_term = search_term.replace('*', '')
177100
search_term_ucase = search_term.upper()
178101

179-
preliminary_results = {}
180-
if( (cache_key in self.INSTANCE.local_cache) == False ):
181-
LOGGER.debug('searchPackages: Not available in cache, so make fresh search')
182-
LOGGER.debug(self.INSTANCE.package_data)
183-
if (exact_match == True):
184-
LOGGER.debug('searchPackages: Doing exact search')
185-
preliminary_results = [s for s in self.INSTANCE.package_data if s['P'] == search_term and (s['B'] & search_bit_flag) > 0]
186-
elif search_anywhere_in_packages:
187-
LOGGER.debug('searchPackages: Doing Anywhere Search')
188-
preliminary_results = [s for s in self.INSTANCE.package_data if search_term_ucase in s['S'] and (s['B'] & search_bit_flag) > 0]
189-
elif search_packages_begin_with:
190-
LOGGER.debug('searchPackages: Find names that begin with')
191-
preliminary_results = [s for s in self.INSTANCE.package_data if str(s['S']).startswith(search_term_ucase) and (s['B'] & search_bit_flag) > 0]
192-
elif search_packages_end_with:
193-
LOGGER.debug('searchPackages: Find names that end with')
194-
preliminary_results = [s for s in self.INSTANCE.package_data if str(s['S']).endswith(search_term_ucase) and (s['B'] & search_bit_flag) > 0]
195-
196-
final_results = copy.deepcopy(preliminary_results); #Deep Copy is required since we just need to remove the "S" field from returnable result
197-
for pkg in final_results:
198-
del pkg['S']
199-
200-
LOGGER.debug('searchPackages: Search Results Length : %s' % (len(final_results)))
201-
202-
if(len(final_results) > MAX_RECORDS_TO_SEND): #This is a large result set so add it to cache
203-
LOGGER.debug('searchPackages: Add results to cache')
204-
if(len(list(self.INSTANCE.local_cache.keys())) >= CACHE_SIZE): #CACHE_SIZE is breached so remove oldest cached object
205-
#LOGGER.debug('searchPackages: Cache full. So remove the oldest item. Total of Cached Items: %s' % (len(self.INSTANCE.local_cache.keys()))
206-
self.INSTANCE.local_cache.pop(self.INSTANCE.cache_keys[0],None) #self.INSTANCE.cache_keys[0] has the Oldest Cache Key
207-
self.INSTANCE.cache_keys.remove(self.INSTANCE.cache_keys[0]) #Remoe the cache_key from cache_keys for it is removed from local_cache
208-
209-
LOGGER.debug('searchPackages: Add new Key to cache_keys for indexing.')
210-
self.INSTANCE.cache_keys.append(cache_key) #append the new key to the list of cache_keys
211-
self.INSTANCE.local_cache[cache_key] = final_results
212-
else:
213-
LOGGER.debug('searchPackages: Getting from cache')
214-
final_results = self.INSTANCE.local_cache[cache_key];
215-
216-
LOGGER.debug('searchPackages: Cache Keys: %s' %(json.dumps(self.INSTANCE.cache_keys)))
217102
totalLength = len(final_results)
218103

219104
last_page = math.ceil(totalLength/float(MAX_RECORDS_TO_SEND))

‎src/main.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@ def searchPackages():
4646
search_bit_flag = int(request.args.get('search_bit_flag', '0'))
4747
page_number = int(request.args.get('page_number', '0'))
4848

49-
json_data = package_search.searchSQLPackages(search_term,exact_match,search_bit_flag,page_number)
50-
resp = Response(json_data,mimetype="application/json")
49+
sql_data = package_search.searchSQLPackages(search_term,exact_match,search_bit_flag,page_number)
50+
resp = Response(sql_data,mimetype="application/json")
5151
resp.headers.set('Cache-Control','no-cache, no-store, must-revalidate')
5252
resp.headers.set('Pragma','no-cache')
5353
resp.headers.set('Expires','0')

0 commit comments

Comments
 (0)