diff --git a/DESCRIPTION b/DESCRIPTION index e5939b3..80fba25 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -48,7 +48,7 @@ Suggests: covr Encoding: UTF-8 LazyData: true -RoxygenNote: 7.3.1 +RoxygenNote: 7.3.2 Config/testthat/edition: 3 Depends: R (>= 2.10) diff --git a/R/oa_fetch.R b/R/oa_fetch.R index 0a9a230..43aa890 100644 --- a/R/oa_fetch.R +++ b/R/oa_fetch.R @@ -187,6 +187,9 @@ oa_fetch <- function(entity = if (is.null(identifier)) NULL else id_type(shorten #' Your OpenAlex Premium API key, if available. #' @param verbose Logical. #' If TRUE, print information about the querying process. Defaults to TRUE. +#' @param json_dir existing directory +#' The directory to which the raw json files returned from OpenAlex will be written. If set, +#' no processing of the jsons will be done and the function returns the json_dir path. #' #' @return a data.frame or a list of bibliographic records. #' @@ -294,14 +297,16 @@ oa_fetch <- function(entity = if (is.null(identifier)) NULL else id_type(shorten #' } #' @export #' -oa_request <- function(query_url, - per_page = 200, - paging = "cursor", - pages = NULL, - count_only = FALSE, - mailto = oa_email(), - api_key = oa_apikey(), - verbose = FALSE) { +oa_request <- function( + query_url, + per_page = 200, + paging = "cursor", + pages = NULL, + count_only = FALSE, + mailto = oa_email(), + api_key = oa_apikey(), + verbose = FALSE, + json_dir = NULL) { # https://httr.r-lib.org/articles/api-packages.html#set-a-user-agent ua <- httr::user_agent("https://github.com/ropensci/openalexR/") @@ -348,8 +353,10 @@ oa_request <- function(query_url, if (verbose) cat("=") Sys.sleep(1 / 10) query_ls[[paging]] <- next_page - res <- api_request(query_url, ua, query = query_ls) - data <- c(data, res[[result_name]]) + res <- api_request(query_url, ua, query = query_ls, json_dir = json_dir) + if (is.null(json_dir)) { + data <- c(data, res[[result_name]]) + } i <- i + 1 next_page <- get_next_page("cursor", i, res) } @@ -394,8 +401,10 @@ oa_request <- function(query_url, Sys.sleep(1 / 10) next_page <- get_next_page(paging, i, res) query_ls[[paging]] <- next_page - res <- api_request(query_url, ua, query = query_ls) - if (!is.null(res[[result_name]])) data[[i]] <- res[[result_name]] + res <- api_request(query_url, ua, query = query_ls, json_dir = json_dir) + if (is.null(json_dir)) { + if (!is.null(res[[result_name]])) data[[i]] <- res[[result_name]] + } } data <- unlist(data, recursive = FALSE) @@ -418,14 +427,17 @@ oa_request <- function(query_url, ) } } - - data + if (!is.null(json_dir)) { + data <- json_dir + } + return(data) } + truncated_authors <- function(list_result) { lapply( list_result, - function(x){ + function(x) { trunc <- x$is_authors_truncated if (!is.null(trunc) && trunc) x$id else NULL } @@ -683,7 +695,12 @@ oa_random <- function(entity = oa_entities(), final_res } -api_request <- function(query_url, ua, query, api_key = oa_apikey()) { +api_request <- function( + query_url, + ua, + query, + api_key = oa_apikey(), + json_dir = NULL) { res <- httr::GET(query_url, ua, query = query, httr::add_headers(api_key = api_key)) if (httr::status_code(res) == 400) { @@ -709,6 +726,35 @@ api_request <- function(query_url, ua, query, api_key = oa_apikey()) { if (httr::http_type(res) != "application/json") { stop("API did not return json", call. = FALSE) } + if (!is.null(json_dir)) { + suppressWarnings( + last_num <- list.files( + json_dir, + pattern = "*.json$", + recursive = FALSE, + full.names = FALSE + ) |> + basename() |> + gsub( + pattern = ".json|result_", + replacement = "" + ) |> + as.numeric() |> + max( + na.rm = TRUE + ) + ) + + if (is.infinite(last_num)) { + last_num <- 0 + } + json_name <- file.path(json_dir, paste0("result_", last_num + 1, ".json")) + + writeLines( + m, + json_name + ) + } return(parsed) } diff --git a/man/oa_request.Rd b/man/oa_request.Rd index 9e982b5..01e8577 100644 --- a/man/oa_request.Rd +++ b/man/oa_request.Rd @@ -12,7 +12,8 @@ oa_request( count_only = FALSE, mailto = oa_email(), api_key = oa_apikey(), - verbose = FALSE + verbose = FALSE, + json_dir = NULL ) } \arguments{ @@ -46,6 +47,10 @@ Your OpenAlex Premium API key, if available.} \item{verbose}{Logical. If TRUE, print information about the querying process. Defaults to TRUE.} + +\item{json_dir}{existing directory +The directory to which the raw json files returned from OpenAlex will be written. If set, +no processing of the jsons will be done and the function returns the json_dir path.} } \value{ a data.frame or a list of bibliographic records.