merge old vs new check into master shipping

Author: chenandrewy

Portfolios/Code/master.R

@@ -27,7 +27,7 @@
 rm(list = ls())
 # ENTER PROJECT PATH HERE (i.e. this should be the path to your local repo folder & location of SignalDoc.csv)
 # if using Rstudio, pathProject = paste0(getwd(), '/') should work
-pathProject = paste0(getwd(), '/')
+pathProject = 'd:/Dropbox/Open-AP-ac/CrossSection/'
 
 quickrun =  F # use T if you want to run quickly for testing
 quickrunlist = c('Accruals','AM') # list of signals to use for quickrun

Shipping/Code/old_vs_new_returns_check.R Shipping/Code/4_old_vs_new_check.RShipping/Code/old_vs_new_returns_check.R renamed to Shipping/Code/4_old_vs_new_check.R

@@ -5,39 +5,33 @@
 
 # Prints simple tables to terminal 
 
-# ENVIRONMENT ====
-rm(list = ls())
-library(tidyverse)
-library(data.table)
-library(googledrive)
-library(gridExtra)
-
-pathProject = 'D:/Dropbox/AC-OPENAP/CrossSection/' 
+# # ENVIRONMENT ====
+# rm(list = ls())
+# library(tidyverse)
+# library(data.table)
+# library(googledrive)
+# library(gridExtra)
+
+# pathProject = 'C:/Dropbox/OPEN-AP-ac/CrossSection/' 
 setwd(paste0(pathProject,'Shipping/Code/'))
 
 dir.create('../Data/temp')
 
-# root of August 2023 release
-OLD_PATH_RELEASES = 'https://drive.google.com/drive/folders/1EP6oEabyZRamveGNyzYU0u6qJ-N43Qfq'
+# # root of August 2023 release
+# OLD_PATH_RELEASES = 'https://drive.google.com/drive/folders/1EP6oEabyZRamveGNyzYU0u6qJ-N43Qfq'
 
-# root of August 2024
-NEW_PATH_RELEASES = 'https://drive.google.com/drive/folders/1-PqsR-tOjv3-U9DRHw85X-VznYlu-Sfc'
-
-# use this for original papers
-SUBDIR = 'Full Sets OP'; FILENAME = 'PredictorPortsFull.csv'
-
-# use this for VW or whatever else
-# SUBDIR = 'Full Sets Alt'; FILENAME = 'PredictorAltPorts_QuintilesVW.zip'
-# SUBDIR = 'Full Sets Alt'; FILENAME = 'PredictorAltPorts_LiqScreen_VWforce.zip'
+# root of October 2024
+# NEW_PATH_RELEASES = 'https://drive.google.com/drive/folders/1SSoHGbwgyhRwUCzLE0YWvUlS0DjLCd4k'
 
 #=====================================================================#
-# Download files                                                  ====
+# Load data                                                  ====
 #=====================================================================#
 
 # download old data
+FILENAME = 'PredictorPortsFull.csv'
 OLD_PATH_RELEASES %>% drive_ls() %>%
   filter(name == "Portfolios") %>% drive_ls() %>% 
-  filter(name == SUBDIR) %>% drive_ls() %>% 
+  filter(name == 'Full Sets OP') %>% drive_ls() %>% 
   filter(name == FILENAME) %>% 
   drive_download(path = paste0("../Data/temp/",FILENAME), overwrite = TRUE)
 
@@ -51,30 +45,32 @@ if (grepl('.csv',FILENAME)){
   )
 }
 
-# download new data
-id <-  NEW_PATH_RELEASES %>% drive_ls() %>%
-  filter(name == "Portfolios") %>% drive_ls() %>% 
-  filter(name == SUBDIR) %>% drive_ls() %>% 
-  filter(name == FILENAME) %>% 
-  drive_download(path = paste0("../Data/temp/",FILENAME), overwrite = TRUE)
-
-# import
-if (grepl('.csv',FILENAME)){
-  new_PredictorPortsFull <- fread(paste0("../Data/temp/",FILENAME))
-} else{
-  unzip(zipfile = paste0('../Data/temp',FILENAME), exdir = 'temp')
-  new_PredictorPortsFull <- fread(
-    paste0("../Data/temp/",substr(FILENAME, 1,(nchar(FILENAME)-4)),'.csv')
-  )
-}
-
-# download signal doc
-NEW_PATH_RELEASES %>% drive_ls() %>% 
-  filter(name == "SignalDoc.csv") %>% 
-  drive_download(path = "../Data/temp/SignalDoc.csv", overwrite = TRUE)
-
-SignalDoc <- fread("../Data/temp/SignalDoc.csv")
-
+# # download new data
+# id <-  NEW_PATH_RELEASES %>% drive_ls() %>%
+#   filter(name == "Portfolios") %>% drive_ls() %>% 
+#   filter(name == 'Full Sets OP') %>% drive_ls() %>% 
+#   filter(name == FILENAME) %>% 
+#   drive_download(path = paste0("../Data/temp/",FILENAME), overwrite = TRUE)
+
+# # import
+# if (grepl('.csv',FILENAME)){
+#   new_PredictorPortsFull <- fread(paste0("../Data/temp/",FILENAME))
+# } else{
+#   unzip(zipfile = paste0('../Data/temp',FILENAME), exdir = 'temp')
+#   new_PredictorPortsFull <- fread(
+#     paste0("../Data/temp/",substr(FILENAME, 1,(nchar(FILENAME)-4)),'.csv')
+#   )
+# }
+
+# load new data
+new_PredictorPortsFull <- fread(
+  paste0(pathStorage,'Portfolios/Full Sets OP/',FILENAME)
+)
+
+# load signal doc
+SignalDoc <- fread(
+  paste0(pathStorage,'SignalDoc.csv')
+)
 
 
 #=====================================================================#
@@ -92,6 +88,9 @@ PredictorPortsFull <- inner_join(
     rename(new_ret = ret) %>% 
     mutate(port = if_else(nchar(port)==2, port, paste0('0',port)))
   , by = c("signalname", "port", "date")
+  ) %>% 
+  mutate(
+    date = as.Date(date)
   )
 
 # Keep only relevant variables
@@ -151,9 +150,10 @@ write.csv(check, "../Data/temp/PredictorPortsCheck.csv", row.names = FALSE)
 
 
 #=====================================================================#
-# Summary stats output to console ====
+# Summary stats output to pathStorage/storage_checks_part2.txt ====
 #=====================================================================#
 
+sink(paste0(pathStorage,'storage_checks_part2.txt'))
 check_ls = check %>% 
   filter(port == 'LS', !is.na(samptype), !is.na(slope)) 
 

Shipping/Code/master_shipping.r

@@ -7,9 +7,12 @@
 library(tidyverse)
 library(readxl)
 library(data.table) # for speed
+library(googledrive)
+
+pathProject = 'C:/Dropbox/Open-AP-ac/CrossSection/' # local code base directory
+pathStorage = 'C:/Gdrive/My Drive/Work/Public/Open AP/Data Release 2024.10/' # a place to 
+OLD_PATH_RELEASES = 'https://drive.google.com/drive/folders/1EP6oEabyZRamveGNyzYU0u6qJ-N43Qfq' # location of previous release for comparison (here August 2023)
 
-pathProject = 'D:/Dropbox/AC-OPENAP/CrossSection/' # local code base directory
-pathStorage = 'D:/Gdrive/Work/Public/Open AP/Data Release 2024.08/' # a place to store copies for uploading
 
 pathShipping = paste0(pathProject,'Shipping/') # where Code/master_shipping.r is
 pathPredictors = paste0(pathProject, 'Signals/Data/Predictors/')
@@ -25,6 +28,8 @@ dir.create('../Data/Portfolios/')
 dir.create('../Data/Portfolios/Individual')
 dir.create('../Data/temp')
 
+# trigger googledrive auth
+drive_auth()
 
 # function for reading in documentation, copied for Portfolios/Code/
 readdocumentation = function(){
@@ -84,7 +89,7 @@ file.copy(
   , to = paste0(pathStorage)
 )
 
-source('1_pack_signals.r')
-source('2_pack_portfolios_and_results.r')
-source('3_check_storage.r')
+# source('1_pack_signals.r')
+# source('2_pack_portfolios_and_results.r')
+# source('3_check_storage.r')