modified script

Author: jeevangelista

script/ingestion/centers.py

@@ -7,42 +7,41 @@
 from ingest_common import connection
 import io
 import csv
+import json
 dccs = pd.read_csv('https://cfde-drc.s3.amazonaws.com/database/files/current_dccs.tsv', sep="\t", index_col=0, header=0)
 # map dcc names to their respective ids
 dcc_mapper = {}
 for i, v in dccs.loc[:,'short_label'].items():
 		dcc_mapper[v] = i
 data = {}
-center_publication = []
+# center_publication = []
 for filename in glob('../../src/pages/centers/*.md'):
 	with open(filename) as o:
 		markdown = o.read()
 		m = markdown.split("---")
 		row = yaml.safe_load(m[1])
 		if "label" in row:
 			label = row['label']
+			description = m[-1].strip()
 			uid = str(uuid5(NAMESPACE_URL, label))
-			data[uid] = {
-				"label": row["label"],
-				"short_label": row.get("short_label"),
-				"short_description": row.get("short_description"),
-				"description": row.get("description"),
-				"homepage": row.get("homepage"),
-				"icon": row.get("icon"),
-				"grant_num": row.get("grant_num"),
-				"active": row.get("active"),
-			}
-			if row.get("publications"):
-				for pub in set(row["publications"]):
-					center_publication.append({"center_id": uid, "publication_id": pub})
+			data[uid] = {"description": description}
+			for k,v in row.items():
+				if not k == "layout":
+					if type(v) == dict or type(v) == list:
+						data[uid][k] = json.dumps(v)
+					else:	
+						data[uid][k] = v
+			# if row.get("publications"):
+			# 	for pub in set(row["publications"]):
+			# 		center_publication.append({"center_id": uid, "publication_id": pub})
 
 center_df = pd.DataFrame.from_dict(data, orient="index").fillna('')
 center_df.index.name = "id"
-center_publication_df = pd.DataFrame.from_records(center_publication, columns=['center_id', 'publication_id'])
+# center_publication_df = pd.DataFrame.from_records(center_publication, columns=['center_id', 'publication_id'])
 
 ## Update S3
 backup_file(center_df, "centers", quoting=False)
-backup_file(center_publication_df, "center_publication", False)
+# backup_file(center_publication_df, "center_publication", False)
 
 ## ingest
 
@@ -65,7 +64,7 @@
 ''')
 
 p_buf = io.StringIO()
-center_df.to_csv(p_buf, header=True, quoting=csv.QUOTE_NONE, sep="\t")
+center_df.to_csv(p_buf, header=True, quoting=csv.QUOTE_NONE, sep="\t", escapechar='\\')
 p_buf.seek(0)
 columns = next(p_buf).strip().split('\t')
 cur.copy_from(p_buf, 'centers_tmp',
@@ -87,35 +86,35 @@
 cur.execute('drop table centers_tmp;')
 
 
-cur = connection.cursor()
-cur.execute('''
-	create table center_publications_tmp
-	as table center_publications
-	with no data;
-''')
-
-
-cp_buf = io.StringIO()
-center_publication_df.to_csv(cp_buf, header=True, sep="\t", index=None)
-cp_buf.seek(0)
-columns = next(cp_buf).strip().split('\t')
-cur.copy_from(cp_buf, 'center_publications_tmp',
-	columns=columns,
-	null='',
-	sep='\t',
-)
-
-column_string = ", ".join(columns)
-
-cur.execute('''
-		insert into center_publications (%s)
-			select %s
-			from center_publications_tmp
-			on conflict 
-				do nothing
-		;
-	'''%(column_string, column_string))
-cur.execute('drop table center_publications_tmp;')
+# cur = connection.cursor()
+# cur.execute('''
+# 	create table center_publications_tmp
+# 	as table center_publications
+# 	with no data;
+# ''')
+
+
+# cp_buf = io.StringIO()
+# center_publication_df.to_csv(cp_buf, header=True, sep="\t", index=None)
+# cp_buf.seek(0)
+# columns = next(cp_buf).strip().split('\t')
+# cur.copy_from(cp_buf, 'center_publications_tmp',
+# 	columns=columns,
+# 	null='',
+# 	sep='\t',
+# )
+
+# column_string = ", ".join(columns)
+
+# cur.execute('''
+# 		insert into center_publications (%s)
+# 			select %s
+# 			from center_publications_tmp
+# 			on conflict 
+# 				do nothing
+# 		;
+# 	'''%(column_string, column_string))
+# cur.execute('drop table center_publications_tmp;')
 connection.commit()
 
 print("Ingested centers")

script/ingestion/publications.py

@@ -13,6 +13,13 @@
 dcc_mapper = {}
 for k,v in dccs.iterrows():
 	dcc_mapper[v["short_label"]] = k
+	
+
+centers = pd.read_csv('https://cfde-drc.s3.amazonaws.com/database/files/current_centers.tsv', sep="\t", index_col=0, header=0)
+# map center names to their respective ids
+center_mapper = {}
+for k,v in centers.iterrows():
+	center_mapper[v["short_label"]] = k
 
 partnerships = pd.read_csv('https://cfde-drc.s3.amazonaws.com/database/files/current_partnerships.tsv', sep="\t", index_col=0)
 partnership_mapper = {}
@@ -26,12 +33,14 @@
 
 publication_columns = ["title", "journal", "authors", "year", "page", "volume", "issue", "pmid", "pmcid", "doi", "landmark", "tool_id", "carousel", "carousel_title", "carousel_link", "carousel_description", "image", "featured", "keywords" ]
 dcc_publication_columns = ["publication_id", "dcc_id"]
+center_publication_columns = ["publication_id", "center_id"]
 partnership_publication_columns = ["publication_id", "partnership_id"]
 r03_publication_columns = ["publication_id", "r03_id"]
 
 publication_df = pd.DataFrame("-", index=[], columns=publication_columns)
 publication_df.index.name = "id"
 dcc_publication_df = pd.DataFrame("-", index=[], columns=dcc_publication_columns)
+center_publication_df = pd.DataFrame("-", index=[], columns=center_publication_columns)
 partnership_publication_df = pd.DataFrame("-", index=[], columns=partnership_publication_columns)
 r03_publication_df = pd.DataFrame("-", index=[], columns=r03_publication_columns)
 ind = 0
@@ -54,6 +63,13 @@
 					dcc_id = dcc_mapper[dcc]
 					dcc_publication_df.loc[ind] = [uid, dcc_mapper[dcc]]
 					ind += 1
+			if "centers" in yml:
+				for center in yml["centers"]:
+					center = center.strip()
+					center_id = center_mapper[center]
+					center_publication_df.loc[ind] = [uid, center_mapper[center]]
+					ind += 1
+			
 			if "partnerships" in yml:
 				partnership = yml["partnerships"]
 				partnership = partnership.strip()
@@ -71,6 +87,7 @@
 ## Update S3
 backup_file(publication_df, "publications", quoting=False)
 backup_file(dcc_publication_df, "dcc_publications", False)
+backup_file(center_publication_df, "center_publications", False)
 backup_file(partnership_publication_df, "partnership_publications", False)
 backup_file(r03_publication_df, "r03_publications", False)
 
@@ -151,6 +168,30 @@
   ''')
 cur.execute('drop table dcc_publication_tmp;')
 
+cur.execute('''
+  create table center_publication_tmp
+  as table center_publications
+  with no data;
+''')
+center_buf = io.StringIO()
+center_publication_df.to_csv(center_buf, header=True, sep="\t", index=None)
+center_buf.seek(0)
+columns = next(center_buf).strip().split('\t')
+cur.copy_from(center_buf, 'center_publication_tmp',
+	columns=center_publication_columns,
+	null='',
+	sep='\t',
+)
+cur.execute('''
+    insert into center_publications (publication_id, center_id)
+      select publication_id, center_id
+      from center_publication_tmp
+      on conflict 
+        do nothing
+    ;
+  ''')
+cur.execute('drop table center_publication_tmp;')
+
 
 cur = connection.cursor()
 cur.execute('''

script/ingestion/s3_update.py

@@ -42,7 +42,7 @@ def backup_file(df, suffix, include_index=True, quoting=True):
 		df.to_csv(s_buf, header=True, sep="\t", quoting=csv.QUOTE_NONE)
 	else:
 		df.to_csv(s_buf, header=True, sep="\t",  index=None)
-	object_name = "database/test/%s_%s.tsv"%(now, suffix)
+	object_name = "database/files/%s_%s.tsv"%(now, suffix)
 	upload_file(s_buf.getvalue(), bucket, object_name)
-	object_name = "database/test/current_%s.tsv"%(suffix)
+	object_name = "database/files/current_%s.tsv"%(suffix)
 	upload_file(s_buf.getvalue(), bucket, object_name)