From a02f7acea6ddc97d11c6dd3dc29efe29ec0d4011 Mon Sep 17 00:00:00 2001 From: arndtupb Date: Tue, 7 Dec 2021 17:22:17 +0100 Subject: [PATCH 1/3] changestowrdsconnection --- code/R/connect_WRDS.R | 48 +++++++++++++++++++++++++++++++++++++++++ code/R/pull_wrds_data.R | 27 +++++++---------------- 2 files changed, 56 insertions(+), 19 deletions(-) create mode 100644 code/R/connect_WRDS.R diff --git a/code/R/connect_WRDS.R b/code/R/connect_WRDS.R new file mode 100644 index 0000000..aebcfe6 --- /dev/null +++ b/code/R/connect_WRDS.R @@ -0,0 +1,48 @@ +# --- Header ------------------------------------------------------------------- +# See LICENSE file for details +# +# This code establishes a connection to WRDS +# ------------------------------------------------------------------------------ +# +# +library(RPostgres) +library(DBI) +# +# +message("Trying to establish connection to WRDS") +# +# +# --- WRDS Credentials ---------------------------------------------------------- +if(file.exists("config.csv")){ + df <- readr::read_csv("config.csv", col_type = readr::cols(), comment = "#") + cfg <- as.list(df$value) + names(cfg) <- df$variable + rm(df) + wrds_user <- cfg$wrds_user + wrds_pwd <- cfg$wrds_pwd + rm(cfg) +} else if(rstudioapi::isAvailable()) { + wrds_user <- rstudioapi::askForPassword(prompt="Enter your WRDS username") + wrds_pwd <- rstudioapi::askForPassword() +} else if(!rstudioapi::isAvailable() & !exists("wrds_user") & !exists("wrds_pw")) { + stop("missing WRDS credentials. see _config.csv for troubleshooting!") +} else if(!rstudioapi::isAvailable() & exists("wrds_user") & exists("wrds_pw")) { + warning("WRDS credentials wrds_user & wrds_pw were manually set. see _config.csv if connection fails") +} +# +# +# --- WRDS Connection ---------------------------------------------------------- +wrds <- dbConnect( + Postgres(), + host = 'wrds-pgdata.wharton.upenn.edu', + port = 9737, + user = wrds_user, + password = wrds_pwd, + sslmode = 'require', + dbname = 'wrds' +) +rm(wrds_user) +rm(wrds_pwd) +message("Logged on to WRDS!") +# +# diff --git a/code/R/pull_wrds_data.R b/code/R/pull_wrds_data.R index a55d8d7..6c650a0 100644 --- a/code/R/pull_wrds_data.R +++ b/code/R/pull_wrds_data.R @@ -3,12 +3,10 @@ # # This code pulls data from WRDS # ------------------------------------------------------------------------------ - +# library(RPostgres) library(DBI) - -if (!exists("cfg")) source("code/R/read_config.R") - +# save_wrds_data <- function(df, fname) { if(file.exists(fname)) { file.rename( @@ -21,21 +19,11 @@ save_wrds_data <- function(df, fname) { } saveRDS(df, fname) } - -# --- Connect to WRDS ---------------------------------------------------------- - -wrds <- dbConnect( - Postgres(), - host = 'wrds-pgdata.wharton.upenn.edu', - port = 9737, - user = cfg$wrds_user, - password = cfg$wrds_pwd, - sslmode = 'require', - dbname = 'wrds' -) - -message("Logged on to WRDS ...") - +# +# --- Connection to WRDS ------------------------------------------------------- +source("code/R/connect_WRDS.R") +# +# # --- Specify filters and variables -------------------------------------------- dyn_vars <- c( @@ -85,4 +73,5 @@ wrds_us <- merge(wrds_us_static, wrds_us_dynamic, by="gvkey") save_wrds_data(wrds_us, "data/pulled/cstat_us_sample.rds") dbDisconnect(wrds) +rm(wrds) message("Disconnected from WRDS") From 3e8cc49a466a02d9a6bc8d856218931850a7b9eb Mon Sep 17 00:00:00 2001 From: arndtupb Date: Thu, 9 Dec 2021 18:15:22 +0100 Subject: [PATCH 2/3] connect_WRDS_pull_data --- code/R/connect_WRDS.R | 11 ++++++----- code/R/pull_wrds_data.R | 27 ++++++++++++++------------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/code/R/connect_WRDS.R b/code/R/connect_WRDS.R index aebcfe6..73878cb 100644 --- a/code/R/connect_WRDS.R +++ b/code/R/connect_WRDS.R @@ -24,11 +24,12 @@ if(file.exists("config.csv")){ } else if(rstudioapi::isAvailable()) { wrds_user <- rstudioapi::askForPassword(prompt="Enter your WRDS username") wrds_pwd <- rstudioapi::askForPassword() -} else if(!rstudioapi::isAvailable() & !exists("wrds_user") & !exists("wrds_pw")) { - stop("missing WRDS credentials. see _config.csv for troubleshooting!") -} else if(!rstudioapi::isAvailable() & exists("wrds_user") & exists("wrds_pw")) { - warning("WRDS credentials wrds_user & wrds_pw were manually set. see _config.csv if connection fails") -} +} else if(!rstudioapi::isAvailable()) { + stop(" + cannot establish connection to WRDS (connect_WRDS.R) + - missing WRDS credentials (wrds_user & wrds_pwd) + - see _config.csv for troubleshooting") +} # # # --- WRDS Connection ---------------------------------------------------------- diff --git a/code/R/pull_wrds_data.R b/code/R/pull_wrds_data.R index 6c650a0..09dfe05 100644 --- a/code/R/pull_wrds_data.R +++ b/code/R/pull_wrds_data.R @@ -7,7 +7,7 @@ library(RPostgres) library(DBI) # -save_wrds_data <- function(df, fname) { +save_rds_versionctrl <- function(df, fname) { if(file.exists(fname)) { file.rename( fname, @@ -25,7 +25,7 @@ source("code/R/connect_WRDS.R") # # # --- Specify filters and variables -------------------------------------------- - +# dyn_vars <- c( "gvkey", "conm", "cik", "fyear", "datadate", "indfmt", "sich", "consol", "popsrc", "datafmt", "curcd", "curuscn", "fyr", @@ -37,17 +37,17 @@ dyn_vars <- c( "recch", "invch", "apalch", "txach", "aoloch", "gdwlip", "spi", "wdp", "rcp" ) - +# dyn_var_str <- paste(dyn_vars, collapse = ", ") - +# stat_vars <- c("gvkey", "loc", "sic", "spcindcd", "ipodate", "fic") stat_var_str <- paste(stat_vars, collapse = ", ") - +# cs_filter <- "consol='C' and (indfmt='INDL' or indfmt='FS') and datafmt='STD' and popsrc='D'" - - +# +# # --- Pull Compustat data ------------------------------------------------------ - +# message("Pulling dynamic Compustat data ... ", appendLF = FALSE) res <- dbSendQuery(wrds, paste( "select", @@ -55,7 +55,7 @@ res <- dbSendQuery(wrds, paste( "from COMP.FUNDA", "where", cs_filter )) - +# wrds_us_dynamic <- dbFetch(res, n=-1) dbClearResult(res) message("done!") @@ -64,14 +64,15 @@ message("Pulling static Compustat data ... ", appendLF = FALSE) res2<-dbSendQuery(wrds, paste( "select ", stat_var_str, "from COMP.COMPANY" )) - +# wrds_us_static <- dbFetch(res2,n=-1) dbClearResult(res2) message("done!") - +# wrds_us <- merge(wrds_us_static, wrds_us_dynamic, by="gvkey") -save_wrds_data(wrds_us, "data/pulled/cstat_us_sample.rds") - +save_rds_versionctrl(wrds_us, "data/pulled/cstat_us_sample.rds") +# dbDisconnect(wrds) rm(wrds) message("Disconnected from WRDS") +# From 748015331f161b0732d62a97cdf231fb2a019287 Mon Sep 17 00:00:00 2001 From: arndtupb Date: Sun, 12 Dec 2021 15:08:16 +0100 Subject: [PATCH 3/3] readconfigreplaced --- code/R/read_config.R | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 code/R/read_config.R diff --git a/code/R/read_config.R b/code/R/read_config.R deleted file mode 100644 index 7bc05bf..0000000 --- a/code/R/read_config.R +++ /dev/null @@ -1,17 +0,0 @@ -# ------------------------------------------------------------------------------ -# This reads the config file in project root and stores all variables in a list -# named cfg. -# -# (c) TRR 266 - Read LICENSE for details -# ------------------------------------------------------------------------------ - -if (!file.exists("config.csv")) stop(paste( - "File 'config.csv' not found in project root. Please edit", - "'_config.csv' and safe it as 'config.csv'." -)) - -df <- readr::read_csv("config.csv", col_type = readr::cols(), comment = "#") - -cfg <- as.list(df$value) -names(cfg) <- df$variable -rm(df) \ No newline at end of file