Save raw json to enable download of large number of records

DESCRIPTION

@@ -48,7 +48,7 @@ Suggests:
     covr
 Encoding: UTF-8
 LazyData: true
-RoxygenNote: 7.3.1
+RoxygenNote: 7.3.2
 Config/testthat/edition: 3
 Depends: 
     R (>= 2.10)

R/oa_fetch.R

@@ -187,6 +187,9 @@ oa_fetch <- function(entity = if (is.null(identifier)) NULL else id_type(shorten
 #' Your OpenAlex Premium API key, if available.
 #' @param verbose Logical.
 #' If TRUE, print information about the querying process. Defaults to TRUE.
+#' @param json_dir existing directory
+#' The directory to which the raw json files returned from OpenAlex will be written. If set,
+#' no processing of the jsons will be done and the function returns the json_dir path.
 #'
 #' @return a data.frame or a list of bibliographic records.
 #'
@@ -294,14 +297,16 @@ oa_fetch <- function(entity = if (is.null(identifier)) NULL else id_type(shorten
 #' }
 #' @export
 #'
-oa_request <- function(query_url,
-                       per_page = 200,
-                       paging = "cursor",
-                       pages = NULL,
-                       count_only = FALSE,
-                       mailto = oa_email(),
-                       api_key = oa_apikey(),
-                       verbose = FALSE) {
+oa_request <- function(
+    query_url,
+    per_page = 200,
+    paging = "cursor",
+    pages = NULL,
+    count_only = FALSE,
+    mailto = oa_email(),
+    api_key = oa_apikey(),
+    verbose = FALSE,
+    json_dir = NULL) {
   # https://httr.r-lib.org/articles/api-packages.html#set-a-user-agent
   ua <- httr::user_agent("https://github.com/ropensci/openalexR/")
 
@@ -348,8 +353,10 @@ oa_request <- function(query_url,
       if (verbose) cat("=")
       Sys.sleep(1 / 10)
       query_ls[[paging]] <- next_page
-      res <- api_request(query_url, ua, query = query_ls)
-      data <- c(data, res[[result_name]])
+      res <- api_request(query_url, ua, query = query_ls, json_dir = json_dir)
+      if (is.null(json_dir)) {
+        data <- c(data, res[[result_name]])
+      }
       i <- i + 1
       next_page <- get_next_page("cursor", i, res)
     }
@@ -394,8 +401,10 @@ oa_request <- function(query_url,
     Sys.sleep(1 / 10)
     next_page <- get_next_page(paging, i, res)
     query_ls[[paging]] <- next_page
-    res <- api_request(query_url, ua, query = query_ls)
-    if (!is.null(res[[result_name]])) data[[i]] <- res[[result_name]]
+    res <- api_request(query_url, ua, query = query_ls, json_dir = json_dir)
+    if (is.null(json_dir)) {
+      if (!is.null(res[[result_name]])) data[[i]] <- res[[result_name]]
+    }
   }
 
   data <- unlist(data, recursive = FALSE)
@@ -418,14 +427,17 @@ oa_request <- function(query_url,
       )
     }
   }
-
-  data
+  if (!is.null(json_dir)) {
+    data <- json_dir
+  }
+  return(data)
 }
 
+
 truncated_authors <- function(list_result) {
   lapply(
     list_result,
-    function(x){
+    function(x) {
       trunc <- x$is_authors_truncated
       if (!is.null(trunc) && trunc) x$id else NULL
     }
@@ -683,7 +695,12 @@ oa_random <- function(entity = oa_entities(),
   final_res
 }
 
-api_request <- function(query_url, ua, query, api_key = oa_apikey()) {
+api_request <- function(
+    query_url,
+    ua,
+    query,
+    api_key = oa_apikey(),
+    json_dir = NULL) {
   res <- httr::GET(query_url, ua, query = query, httr::add_headers(api_key = api_key))
 
   if (httr::status_code(res) == 400) {
@@ -709,6 +726,35 @@ api_request <- function(query_url, ua, query, api_key = oa_apikey()) {
     if (httr::http_type(res) != "application/json") {
       stop("API did not return json", call. = FALSE)
     }
+    if (!is.null(json_dir)) {
+      suppressWarnings(
+        last_num <- list.files(
+          json_dir,
+          pattern = "*.json$",
+          recursive = FALSE,
+          full.names = FALSE
+        ) |>
+          basename() |>
+          gsub(
+            pattern = ".json|result_",
+            replacement = ""
+          ) |>
+          as.numeric() |>
+          max(
+            na.rm = TRUE
+          )
+      )
+
+      if (is.infinite(last_num)) {
+        last_num <- 0
+      }
+      json_name <- file.path(json_dir, paste0("result_", last_num + 1, ".json"))
+
+      writeLines(
+        m,
+        json_name
+      )
+    }
     return(parsed)
   }