From fb271f134e4d0cecbee9963ef0b5d1617ff82466 Mon Sep 17 00:00:00 2001 From: Daniel Cook Date: Sun, 24 Jul 2016 19:44:08 -0500 Subject: [PATCH 01/29] Add google datastore memoization option. --- .Rbuildignore | 1 + .gitignore | 1 + DESCRIPTION | 2 +- NAMESPACE | 4 + R/cache.r | 6 + R/cache_datastore.r | 163 ++++++++++++++++++++++++++ R/memoise.r | 4 +- man/authenticate_datastore.Rd | 28 +++++ man/authenticate_datastore_service.Rd | 25 ++++ man/datastore_cache.Rd | 19 +++ man/memoise.Rd | 2 +- man/new_cache.Rd | 12 ++ 12 files changed, 262 insertions(+), 5 deletions(-) create mode 100644 R/cache_datastore.r create mode 100644 man/authenticate_datastore.Rd create mode 100644 man/authenticate_datastore_service.Rd create mode 100644 man/datastore_cache.Rd create mode 100644 man/new_cache.Rd diff --git a/.Rbuildignore b/.Rbuildignore index 9d51083..f1a940c 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -3,3 +3,4 @@ ^\.Rproj\.user$ ^revdep$ ^cran-comments\.md$ +^\.httr-oauth$ diff --git a/.gitignore b/.gitignore index 807ea25..f427a7f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .Rproj.user .Rhistory .RData +.httr-oauth diff --git a/DESCRIPTION b/DESCRIPTION index 96315a9..bebe387 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Encoding: UTF-8 -Package: memoise +Package: memoise2 Title: Memoisation of Functions Version: 1.0.0.9000 Authors@R: c( diff --git a/NAMESPACE b/NAMESPACE index 0bf84c1..5fa5f70 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,12 +1,16 @@ # Generated by roxygen2: do not edit by hand S3method(print,memoised) +export(authenticate_datastore) +export(authenticate_datastore_service) +export(datastore_cache) export(forget) export(has_cache) export(is.memoised) export(is.memoized) export(memoise) export(memoize) +export(new_cache) export(timeout) importFrom(digest,digest) importFrom(stats,setNames) diff --git a/R/cache.r b/R/cache.r index 23d542f..7c3b232 100644 --- a/R/cache.r +++ b/R/cache.r @@ -1,3 +1,9 @@ +#' new_cache +#' +#' Use R to cache items. +#' +#' @export + new_cache <- function() { cache <- NULL diff --git a/R/cache_datastore.r b/R/cache_datastore.r new file mode 100644 index 0000000..f5755cb --- /dev/null +++ b/R/cache_datastore.r @@ -0,0 +1,163 @@ +rdatastore_env <- new.env() +datastore_url <- "https://www.googleapis.com/auth/datastore" + + +#' Authenticate Datastore using a service account +#' +#' Set credentials to the name of an environmental variable +#' or the location of your service account json key file. +#' +#' @param credentials Environmental variable or service account. +#' @param project Google cloud platform project id/name. +#' +#' @seealso \url{https://cloud.google.com/} +#' @seealso \url{https://cloud.google.com/datastore/docs/concepts/overview} +#' @seealso \url{https://developers.google.com/identity/protocols/OAuth2#basicsteps} +#' +#' @export + + +authenticate_datastore_service <- function(credentials, project) { + # Locate Credentials + if (file.exists(as.character(credentials))) { + credentials <- jsonlite::fromJSON(credentials) + } else if (Sys.getenv(credentials) != "") { + credentials <- jsonlite::fromJSON(Sys.getenv(credentials)) + } else { + stop("Could not find credentials") + } + + google_token <- httr::oauth_service_token(httr::oauth_endpoints("google"), + credentials, + scope = datastore_url) + + url <- paste0("https://datastore.googleapis.com/v1beta3/projects/", project) + # Create global variable for project id + assign("project_id", project, envir=rdatastore_env) + assign("token", google_token, envir=rdatastore_env) + assign("url", url, envir=rdatastore_env) +} + +#' Authenticate Datastore +#' +#' Authenticate datastore using OAuth 2.0. Create an application on the +#' \strong{google cloud platform} +#' and generate +#' +#' @param key OAuth 2.0 credential key +#' @param secret OAuth credential secret key +#' @param project Google cloud platform project id/name. +#' +#' @seealso \url{https://cloud.google.com/} +#' @seealso \url{https://cloud.google.com/datastore/docs/concepts/overview} +#' @seealso \url{https://developers.google.com/identity/protocols/OAuth2#basicsteps} +#' +#' @export + +authenticate_datastore <- function(key, secret, project) { + # Authorize app + app <- httr::oauth_app("google", + key = key, + secret = secret) + + # Fetch token + google_token <- httr::oauth2.0_token(httr::oauth_endpoints("google"), + app, + scope = datastore_url) + url <- paste0("https://datastore.googleapis.com/v1beta3/projects/", project) + # Create global variable for project id + assign("project_id", project, envir=rdatastore_env) + assign("token", google_token, envir=rdatastore_env) + assign("url", url, envir=rdatastore_env) +} + + +#' datastore_cache +#' +#' Use google datastore to store and retrieve cache items. Requires authentication. +#' +#' @seealso \url{https://cloud.google.com/} +#' @seealso \url{https://cloud.google.com/datastore/docs/concepts/overview} +#' @seealso \url{https://developers.google.com/identity/protocols/OAuth2#basicsteps} +#' +#' @export + +datastore_cache <- function(cache_name = "rcache") { + + transaction <- function() { + req <- httr::POST(paste0(rdatastore_env$url, ":beginTransaction"), + httr::config(token = rdatastore_env$token), + encode = "json") + if (req$status_code != 200) { + stop(httr::content(req)$error$message) + } + httr::content(req)$transaction + } + + cache <- NULL + cache_reset <- function() { + + } + + cache_set <- function(key, value) { + # Serialize value + svalue <- base64enc::base64encode(serialize(value, NULL, ascii=T)) + path_item <- list( + kind = cache_name, + name = key + ) + prop = list( + object = list(blobValue = svalue) + ) + + transaction_id <- transaction() + + key_obj <- c(list(key = list(path = path_item), + properties = prop)) + mutation = list() + mutation[["insert"]] = key_obj + body <- list(mutations = mutation, + transaction = transaction_id + ) + + req <- httr::POST(paste0(rdatastore_env$url, ":commit"), + httr::config(token = rdatastore_env$token), + body = body, + encode = "json") + + } + + cache_get <- function(key) { + path_item <- list( + kind = cache_name, + name = key + ) + + req <- httr::POST(paste0(rdatastore_env$url, ":lookup"), + httr::config(token = rdatastore_env$token), + body = list(keys = list(path = path_item)), + encode = "json") + + # Unserialize and return + resp <- jsonlite::fromJSON(httr::content(req, as = "text"))$found + value <- resp$entity$properties$object$blobValue + unserialize(base64enc::base64decode(value)) + } + + cache_has_key <- function(key) { + res <- try(cache_get(key), silent = TRUE) + class(res) != "try-error" + } + + cache_reset() + list( + reset = cache_reset, + set = cache_set, + get = cache_get, + has_key = cache_has_key, + keys = function() message("Keys can't be listed with google datastore.") + ) +} + + + diff --git a/R/memoise.r b/R/memoise.r index ac0ae73..2e132f0 100644 --- a/R/memoise.r +++ b/R/memoise.r @@ -97,7 +97,7 @@ #' memA4 <- memoise(a, ~timeout(10)) #' memA4(2) #' @importFrom stats setNames -memoise <- memoize <- function(f, ..., envir = environment(f)) { +memoise <- memoize <- function(f, ..., envir = environment(f), cache = new_cache()) { f_formals <- formals(args(f)) if(is.memoised(f)) { stop("`f` must not be memoised.", call. = FALSE) @@ -113,8 +113,6 @@ memoise <- memoize <- function(f, ..., envir = environment(f)) { init_call_args <- setNames(f_formal_name_list, f_formal_names) init_call <- make_call(quote(`_f`), init_call_args) - cache <- new_cache() - validate_formulas(...) additional <- list(...) diff --git a/man/authenticate_datastore.Rd b/man/authenticate_datastore.Rd new file mode 100644 index 0000000..cffe29e --- /dev/null +++ b/man/authenticate_datastore.Rd @@ -0,0 +1,28 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/cache_datastore.r +\name{authenticate_datastore} +\alias{authenticate_datastore} +\title{Authenticate Datastore} +\usage{ +authenticate_datastore(key, secret, project) +} +\arguments{ +\item{key}{OAuth 2.0 credential key} + +\item{secret}{OAuth credential secret key} + +\item{project}{Google cloud platform project id/name.} +} +\description{ +Authenticate datastore using OAuth 2.0. Create an application on the +\strong{google cloud platform} +and generate +} +\seealso{ +\url{https://cloud.google.com/} + +\url{https://cloud.google.com/datastore/docs/concepts/overview} + +\url{https://developers.google.com/identity/protocols/OAuth2#basicsteps} +} + diff --git a/man/authenticate_datastore_service.Rd b/man/authenticate_datastore_service.Rd new file mode 100644 index 0000000..5a4c4d5 --- /dev/null +++ b/man/authenticate_datastore_service.Rd @@ -0,0 +1,25 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/cache_datastore.r +\name{authenticate_datastore_service} +\alias{authenticate_datastore_service} +\title{Authenticate Datastore using a service account} +\usage{ +authenticate_datastore_service(credentials, project) +} +\arguments{ +\item{credentials}{Environmental variable or service account.} + +\item{project}{Google cloud platform project id/name.} +} +\description{ +Set credentials to the name of an environmental variable +or the location of your service account json key file. +} +\seealso{ +\url{https://cloud.google.com/} + +\url{https://cloud.google.com/datastore/docs/concepts/overview} + +\url{https://developers.google.com/identity/protocols/OAuth2#basicsteps} +} + diff --git a/man/datastore_cache.Rd b/man/datastore_cache.Rd new file mode 100644 index 0000000..10845ac --- /dev/null +++ b/man/datastore_cache.Rd @@ -0,0 +1,19 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/cache_datastore.r +\name{datastore_cache} +\alias{datastore_cache} +\title{datastore_cache} +\usage{ +datastore_cache(cache_name = "rcache") +} +\description{ +Use google datastore to store and retrieve cache items. Requires authentication. +} +\seealso{ +\url{https://cloud.google.com/} + +\url{https://cloud.google.com/datastore/docs/concepts/overview} + +\url{https://developers.google.com/identity/protocols/OAuth2#basicsteps} +} + diff --git a/man/memoise.Rd b/man/memoise.Rd index c3ba39a..42880e1 100644 --- a/man/memoise.Rd +++ b/man/memoise.Rd @@ -5,7 +5,7 @@ \alias{memoize} \title{Memoise a function.} \usage{ -memoise(f, ..., envir = environment(f)) +memoise(f, ..., envir = environment(f), cache = new_cache()) } \arguments{ \item{f}{Function of which to create a memoised copy.} diff --git a/man/new_cache.Rd b/man/new_cache.Rd new file mode 100644 index 0000000..c4837d6 --- /dev/null +++ b/man/new_cache.Rd @@ -0,0 +1,12 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/cache.r +\name{new_cache} +\alias{new_cache} +\title{new_cache} +\usage{ +new_cache() +} +\description{ +Use R to cache items. +} + From e3569a69aa93c1561602aca1799c4e6b8ec45536 Mon Sep 17 00:00:00 2001 From: Daniel Cook Date: Sun, 24 Jul 2016 23:32:53 -0500 Subject: [PATCH 02/29] add error messages and compression. --- R/cache_datastore.r | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/R/cache_datastore.r b/R/cache_datastore.r index f5755cb..cdfb0be 100644 --- a/R/cache_datastore.r +++ b/R/cache_datastore.r @@ -101,13 +101,13 @@ datastore_cache <- function(cache_name = "rcache") { cache_set <- function(key, value) { # Serialize value - svalue <- base64enc::base64encode(serialize(value, NULL, ascii=T)) + svalue <- base64enc::base64encode(memCompress(serialize(value, NULL, ascii=T), type = "gzip")) path_item <- list( kind = cache_name, name = key ) prop = list( - object = list(blobValue = svalue) + object = list(blobValue = svalue, excludeFromIndexes = T) ) transaction_id <- transaction() @@ -115,7 +115,7 @@ datastore_cache <- function(cache_name = "rcache") { key_obj <- c(list(key = list(path = path_item), properties = prop)) mutation = list() - mutation[["insert"]] = key_obj + mutation[["upsert"]] = key_obj body <- list(mutations = mutation, transaction = transaction_id ) @@ -125,6 +125,10 @@ datastore_cache <- function(cache_name = "rcache") { body = body, encode = "json") + if (req$status_code != 200) { + warning(httr::content(req)$error$message) + } + } cache_get <- function(key) { @@ -139,13 +143,21 @@ datastore_cache <- function(cache_name = "rcache") { encode = "json") # Unserialize and return - resp <- jsonlite::fromJSON(httr::content(req, as = "text"))$found - value <- resp$entity$properties$object$blobValue - unserialize(base64enc::base64decode(value)) + req <- jsonlite::fromJSON(httr::content(req, as = "text")) + if ("found" %in% names(req)) { + resp <- req$found + value <- resp$entity$properties$object$blobValue + unserialize(memDecompress(base64enc::base64decode(value), type = "gzip")) + } else { + stop("Not Found") + } } cache_has_key <- function(key) { res <- try(cache_get(key), silent = TRUE) + if (class(res) != "try-error") { + message("Using Cached Version") + } class(res) != "try-error" } From 51a4544ad739ff817f8f353fb13c77e31f96c952 Mon Sep 17 00:00:00 2001 From: Daniel Cook Date: Mon, 25 Jul 2016 09:16:55 -0500 Subject: [PATCH 03/29] Add to do and additional cache types. --- DESCRIPTION | 2 +- README.md | 38 +++++++++++++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index bebe387..424241f 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Encoding: UTF-8 -Package: memoise2 +Package: xmemoise Title: Memoisation of Functions Version: 1.0.0.9000 Authors@R: c( diff --git a/README.md b/README.md index f2b4566..efb645b 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,19 @@ -# memoise [![Travis-CI Build Status](https://travis-ci.org/hadley/memoise.svg?branch=master)](https://travis-ci.org/hadley/memoise) [![Coverage Status](https://img.shields.io/codecov/c/github/hadley/memoise/master.svg)](https://codecov.io/github/hadley/memoise?branch=master) +# xmemoise +Forked from [hadley/memoise](https://github.com/hadley/memoise) + +# Installation + +``` +devtools::install_github("danielecook/xmemoise") +``` + +# Memoization If a function is called multiple times with the same input, you can often speed things up by keeping a cache of known answers that it can retrieve. This is called memoisation . -The `memoise` package provides a simple syntax +The `xmemoise` package is built upon [hadley/memoise](https://github.com/hadley/memoise), which provides a simple syntax mf <- memoise(f) @@ -18,4 +27,27 @@ cache with is.memoised(mf) # TRUE is.memoised(f) # FALSE -. + +`xmemoise` extends upon `memoise` by adding in additional types of caches. Items can be cached using the original cache implemented in `memoise` in addition to other options: + +* [x] Google Datastore; Switch to using googleAuthR +* [ ] Dropbox +* [ ] Google Storage +* [ ] AWS + + +# Memoization with google datastore + +Google Datastore + +There are a few trade-offs to using google datastore for memoization. + +``` +key <- "" +secret <- "" +authenticate_datastore(key, secret, "") + +mem_fib <- memoise(fib, cache = datastore_cache("custom_cache_name")) +mem_fib(20) # Saved to datastore; Can be retrieved on another computer. +mem_fib(20) # Cached version retrieved from google datastore. +``` \ No newline at end of file From 56a6c8f026e8d656a94902e03f739ce583c872fe Mon Sep 17 00:00:00 2001 From: Daniel Cook Date: Mon, 25 Jul 2016 09:17:15 -0500 Subject: [PATCH 04/29] update readme. --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index efb645b..a2f2a6d 100644 --- a/README.md +++ b/README.md @@ -30,11 +30,15 @@ cache with `xmemoise` extends upon `memoise` by adding in additional types of caches. Items can be cached using the original cache implemented in `memoise` in addition to other options: -* [x] Google Datastore; Switch to using googleAuthR +* [x] Google Datastore * [ ] Dropbox * [ ] Google Storage * [ ] AWS +### To Do + +* [ ] Switch to using GoogleAuthR + # Memoization with google datastore From 1e8e80904d85002f0c1d338c3f84dd136c0b1e0e Mon Sep 17 00:00:00 2001 From: Dan Cook Date: Mon, 25 Jul 2016 12:32:36 -0500 Subject: [PATCH 05/29] Rewrite to use googleauthR Add cache_reset --- R/cache_datastore.r | 154 +++++++++++++++----------------------------- 1 file changed, 51 insertions(+), 103 deletions(-) diff --git a/R/cache_datastore.r b/R/cache_datastore.r index cdfb0be..3b21843 100644 --- a/R/cache_datastore.r +++ b/R/cache_datastore.r @@ -1,14 +1,6 @@ -rdatastore_env <- new.env() -datastore_url <- "https://www.googleapis.com/auth/datastore" - - -#' Authenticate Datastore using a service account -#' -#' Set credentials to the name of an environmental variable -#' or the location of your service account json key file. +#' datastore_cache #' -#' @param credentials Environmental variable or service account. -#' @param project Google cloud platform project id/name. +#' Use google datastore to store and retrieve cache items. Requires authentication. #' #' @seealso \url{https://cloud.google.com/} #' @seealso \url{https://cloud.google.com/datastore/docs/concepts/overview} @@ -16,89 +8,63 @@ datastore_url <- "https://www.googleapis.com/auth/datastore" #' #' @export +datastore_cache <- function(project = project_id, cache_name = "rcache") { -authenticate_datastore_service <- function(credentials, project) { - # Locate Credentials - if (file.exists(as.character(credentials))) { - credentials <- jsonlite::fromJSON(credentials) - } else if (Sys.getenv(credentials) != "") { - credentials <- jsonlite::fromJSON(Sys.getenv(credentials)) - } else { - stop("Could not find credentials") - } + options("googleAuthR.scopes.selected" = c("https://www.googleapis.com/auth/datastore", + "https://www.googleapis.com/auth/userinfo.email")) - google_token <- httr::oauth_service_token(httr::oauth_endpoints("google"), - credentials, - scope = datastore_url) + googleAuthR::gar_auth() - url <- paste0("https://datastore.googleapis.com/v1beta3/projects/", project) - # Create global variable for project id - assign("project_id", project, envir=rdatastore_env) - assign("token", google_token, envir=rdatastore_env) - assign("url", url, envir=rdatastore_env) -} + base_url <- paste0("https://datastore.googleapis.com/v1beta3/projects/", project_id) -#' Authenticate Datastore -#' -#' Authenticate datastore using OAuth 2.0. Create an application on the -#' \strong{google cloud platform} -#' and generate -#' -#' @param key OAuth 2.0 credential key -#' @param secret OAuth credential secret key -#' @param project Google cloud platform project id/name. -#' -#' @seealso \url{https://cloud.google.com/} -#' @seealso \url{https://cloud.google.com/datastore/docs/concepts/overview} -#' @seealso \url{https://developers.google.com/identity/protocols/OAuth2#basicsteps} -#' -#' @export + transaction <- googleAuthR::gar_api_generator(paste0(base_url, ":beginTransaction"), + "POST", + data_parse_function = function(x) x$transaction) -authenticate_datastore <- function(key, secret, project) { - # Authorize app - app <- httr::oauth_app("google", - key = key, - secret = secret) - - # Fetch token - google_token <- httr::oauth2.0_token(httr::oauth_endpoints("google"), - app, - scope = datastore_url) - url <- paste0("https://datastore.googleapis.com/v1beta3/projects/", project) - # Create global variable for project id - assign("project_id", project, envir=rdatastore_env) - assign("token", google_token, envir=rdatastore_env) - assign("url", url, envir=rdatastore_env) -} + commit_ds <- googleAuthR::gar_api_generator(paste0(base_url, ":commit"), + "POST", + data_parse_function = function(x) x, + simplifyVector = F) + load_ds <- googleAuthR::gar_api_generator(paste0(base_url, ":lookup"), + "POST", + data_parse_function = function(resp) { + # Unserialize and return + if ("found" %in% names(resp)) { + resp <- resp$found + value <- resp$entity$properties$object$blobValue + unserialize(memDecompress(base64enc::base64decode(value), type = "gzip")) + } else { + stop("Not Found") + } + }) -#' datastore_cache -#' -#' Use google datastore to store and retrieve cache items. Requires authentication. -#' -#' @seealso \url{https://cloud.google.com/} -#' @seealso \url{https://cloud.google.com/datastore/docs/concepts/overview} -#' @seealso \url{https://developers.google.com/identity/protocols/OAuth2#basicsteps} -#' -#' @export + query_ds <- googleAuthR::gar_api_generator(paste0(base_url, ":runQuery"), + "POST", + data_parse_function = function(resp) resp) -datastore_cache <- function(cache_name = "rcache") { - - transaction <- function() { - req <- httr::POST(paste0(rdatastore_env$url, ":beginTransaction"), - httr::config(token = rdatastore_env$token), - encode = "json") - if (req$status_code != 200) { - stop(httr::content(req)$error$message) - } - httr::content(req)$transaction - } cache <- NULL - cache_reset <- function() { + cache_reset <- function() { + query_results <- query_ds(the_body = list(gqlQuery = list(queryString = paste0("SELECT * FROM ", cache_name)))) + while((query_results$batch$moreResults != "NO_MORE_RESULTS") | is.null(query_results$batch$entityResults) == FALSE) { + ids <- (query_results$batch$entityResults$entity$key$path %>% dplyr::bind_rows())$name + + item_groups <- split(ids, (1:length(ids)) %/% 25) + sapply(item_groups, function(idset) { + mutations <- lapply(idset, function(x) { + c(list("delete" = list(path = list(kind = cache_name, name = x)))) + }) + body <- list(mutations = mutations, transaction = transaction()) + resp <- try(commit_ds(the_body = body), silent = T) + message("Clearing Cache") + }) + query_results <- query_ds(the_body = list(gqlQuery = list(queryString = paste0("SELECT * FROM ", cache_name)))) + } } + cache_set <- function(key, value) { # Serialize value svalue <- base64enc::base64encode(memCompress(serialize(value, NULL, ascii=T), type = "gzip")) @@ -120,15 +86,10 @@ datastore_cache <- function(cache_name = "rcache") { transaction = transaction_id ) - req <- httr::POST(paste0(rdatastore_env$url, ":commit"), - httr::config(token = rdatastore_env$token), - body = body, - encode = "json") - - if (req$status_code != 200) { - warning(httr::content(req)$error$message) + resp <- try(commit_ds(the_body = body), silent = T) + if (class(resp) == "try-error") { + warning(attr(resp, "condition")) } - } cache_get <- function(key) { @@ -137,20 +98,7 @@ datastore_cache <- function(cache_name = "rcache") { name = key ) - req <- httr::POST(paste0(rdatastore_env$url, ":lookup"), - httr::config(token = rdatastore_env$token), - body = list(keys = list(path = path_item)), - encode = "json") - - # Unserialize and return - req <- jsonlite::fromJSON(httr::content(req, as = "text")) - if ("found" %in% names(req)) { - resp <- req$found - value <- resp$entity$properties$object$blobValue - unserialize(memDecompress(base64enc::base64decode(value), type = "gzip")) - } else { - stop("Not Found") - } + resp <- load_ds(the_body = list(keys = list(path = path_item))) } cache_has_key <- function(key) { @@ -167,7 +115,7 @@ datastore_cache <- function(cache_name = "rcache") { set = cache_set, get = cache_get, has_key = cache_has_key, - keys = function() message("Keys can't be listed with google datastore.") + keys = function() message("Keys can't be listed with the google datastore cache.") ) } From 514514a77c9b0d7fa531b4642b7ad6a2717db379 Mon Sep 17 00:00:00 2001 From: Dan Cook Date: Mon, 25 Jul 2016 12:39:27 -0500 Subject: [PATCH 06/29] Fix cache return value and don't automatically reset cache. --- R/cache_datastore.r | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/R/cache_datastore.r b/R/cache_datastore.r index 3b21843..3b9ba66 100644 --- a/R/cache_datastore.r +++ b/R/cache_datastore.r @@ -44,8 +44,6 @@ datastore_cache <- function(project = project_id, cache_name = "rcache") { data_parse_function = function(resp) resp) - cache <- NULL - cache_reset <- function() { query_results <- query_ds(the_body = list(gqlQuery = list(queryString = paste0("SELECT * FROM ", cache_name)))) while((query_results$batch$moreResults != "NO_MORE_RESULTS") | is.null(query_results$batch$entityResults) == FALSE) { @@ -99,6 +97,7 @@ datastore_cache <- function(project = project_id, cache_name = "rcache") { ) resp <- load_ds(the_body = list(keys = list(path = path_item))) + resp } cache_has_key <- function(key) { @@ -109,7 +108,6 @@ datastore_cache <- function(project = project_id, cache_name = "rcache") { class(res) != "try-error" } - cache_reset() list( reset = cache_reset, set = cache_set, From 20ce73922718169eea3cbc7d7bf09c52103d5226 Mon Sep 17 00:00:00 2001 From: Dan Cook Date: Mon, 25 Jul 2016 13:19:12 -0500 Subject: [PATCH 07/29] Finish datastore cache. --- R/cache_datastore.r | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/R/cache_datastore.r b/R/cache_datastore.r index 3b9ba66..958815f 100644 --- a/R/cache_datastore.r +++ b/R/cache_datastore.r @@ -8,14 +8,14 @@ #' #' @export -datastore_cache <- function(project = project_id, cache_name = "rcache") { +datastore_cache <- function(project, cache_name = "rcache") { options("googleAuthR.scopes.selected" = c("https://www.googleapis.com/auth/datastore", "https://www.googleapis.com/auth/userinfo.email")) googleAuthR::gar_auth() - base_url <- paste0("https://datastore.googleapis.com/v1beta3/projects/", project_id) + base_url <- paste0("https://datastore.googleapis.com/v1beta3/projects/", project) transaction <- googleAuthR::gar_api_generator(paste0(base_url, ":beginTransaction"), "POST", @@ -23,8 +23,7 @@ datastore_cache <- function(project = project_id, cache_name = "rcache") { commit_ds <- googleAuthR::gar_api_generator(paste0(base_url, ":commit"), "POST", - data_parse_function = function(x) x, - simplifyVector = F) + data_parse_function = function(x) x) load_ds <- googleAuthR::gar_api_generator(paste0(base_url, ":lookup"), "POST", @@ -33,9 +32,11 @@ datastore_cache <- function(project = project_id, cache_name = "rcache") { if ("found" %in% names(resp)) { resp <- resp$found value <- resp$entity$properties$object$blobValue - unserialize(memDecompress(base64enc::base64decode(value), type = "gzip")) + response <- unserialize(memDecompress(base64enc::base64decode(value), type = "gzip")) + } else if ("missing" %in% names(resp)) { + "!cache-not-found" } else { - stop("Not Found") + stop("Error") } }) @@ -97,11 +98,14 @@ datastore_cache <- function(project = project_id, cache_name = "rcache") { ) resp <- load_ds(the_body = list(keys = list(path = path_item))) + suppressWarnings( if(resp == "!cache-not-found") { + stop("Cache Not Found") + }) resp } cache_has_key <- function(key) { - res <- try(cache_get(key), silent = TRUE) + res <- try(suppressWarnings(cache_get(key)), silent = TRUE) if (class(res) != "try-error") { message("Using Cached Version") } From 2e690c6d9aad975a3502bc88ba43805f2cd3ca1f Mon Sep 17 00:00:00 2001 From: Dan Cook Date: Mon, 25 Jul 2016 14:00:27 -0500 Subject: [PATCH 08/29] cleanup namespace --- NAMESPACE | 2 -- 1 file changed, 2 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 5fa5f70..b60b487 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,8 +1,6 @@ # Generated by roxygen2: do not edit by hand S3method(print,memoised) -export(authenticate_datastore) -export(authenticate_datastore_service) export(datastore_cache) export(forget) export(has_cache) From fc448343cef9e1afceb3a244a1a474d4e4529473 Mon Sep 17 00:00:00 2001 From: Dan Cook Date: Mon, 25 Jul 2016 16:23:16 -0500 Subject: [PATCH 09/29] Add AWS Service --- NAMESPACE | 4 +- R/cache_aws.r | 55 +++++++++++++++++++ R/cache_datastore.r | 2 +- R/cache_gs.r | 48 ++++++++++++++++ man/authenticate_datastore.Rd | 28 ---------- man/authenticate_datastore_service.Rd | 25 --------- man/cache_aws.Rd | 12 ++++ ...{datastore_cache.Rd => cache_datastore.Rd} | 6 +- man/cache_gs.Rd | 12 ++++ memoise.Rproj | 2 +- 10 files changed, 135 insertions(+), 59 deletions(-) create mode 100644 R/cache_aws.r create mode 100644 R/cache_gs.r delete mode 100644 man/authenticate_datastore.Rd delete mode 100644 man/authenticate_datastore_service.Rd create mode 100644 man/cache_aws.Rd rename man/{datastore_cache.Rd => cache_datastore.Rd} (81%) create mode 100644 man/cache_gs.Rd diff --git a/NAMESPACE b/NAMESPACE index b60b487..528cff3 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,7 +1,9 @@ # Generated by roxygen2: do not edit by hand S3method(print,memoised) -export(datastore_cache) +export(cache_aws) +export(cache_datastore) +export(cache_gs) export(forget) export(has_cache) export(is.memoised) diff --git a/R/cache_aws.r b/R/cache_aws.r new file mode 100644 index 0000000..41f1b4d --- /dev/null +++ b/R/cache_aws.r @@ -0,0 +1,55 @@ +#' new_cache +#' +#' Use R to cache items. +#' +#' @export + +cache_aws <- function(cache_name = "rcache") { + + if (!(aws.s3::bucket_exists(cache_name))) { + aws.s3::put_bucket(cache_name) + if (!(aws.s3::bucket_exists(cache_name))) { + stop("Cache name must use unique bucket name") + } + } + + cache <- NULL + cache_reset <- function() { + aws.s3::delete_bucket(cache_name) + aws.s3::put_bucket(cache_name) + } + + cache_set <- function(key, value) { + tfile = tempfile() + save(value, file = tfile) + aws.s3::put_object(tfile, object = key, bucket = cache_name) + } + + cache_get <- function(key) { + suppressWarnings(aws.s3::s3load(object = key, bucket = cache_name)) + base::get(ls()[ls() != "key"][[1]]) + } + + cache_has_key <- function(key) { + aws.s3::head_object(object = key, bucket = cache_name) + } + + cache_keys <- function() { + items <- lapply(aws.s3::get_bucket(bucket = cache_name), function(x) { + if ("Key" %in% names(x)) { + return(x$Key) + } else { + return(NULL) + } + }) + unlist(Filter(Negate(is.null), items)) + } + + list( + reset = cache_reset, + set = cache_set, + get = cache_get, + has_key = cache_has_key, + keys = cache_keys + ) +} diff --git a/R/cache_datastore.r b/R/cache_datastore.r index 958815f..d9961d0 100644 --- a/R/cache_datastore.r +++ b/R/cache_datastore.r @@ -8,7 +8,7 @@ #' #' @export -datastore_cache <- function(project, cache_name = "rcache") { +cache_datastore <- function(project, cache_name = "rcache") { options("googleAuthR.scopes.selected" = c("https://www.googleapis.com/auth/datastore", "https://www.googleapis.com/auth/userinfo.email")) diff --git a/R/cache_gs.r b/R/cache_gs.r new file mode 100644 index 0000000..9b10199 --- /dev/null +++ b/R/cache_gs.r @@ -0,0 +1,48 @@ +#' gs_cache +#' +#' Use google storage as a cache. Requires authentication. +#' +#' @export + +cache_gs <- function(project, cache_name = "rcache") { + + + options(googleAuthR.scopes.selected = c("https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_write")) + + googleAuthR::gar_auth() + + base_url <- paste0("https://www.googleapis.com/storage/v1/b?project=andersen-lab") + + transaction <- googleAuthR::gar_api_generator(base_url, + "GET", + data_parse_function = function(x) x) + transaction() + + + cache <- NULL + cache_reset <- function() { + cache <<- new.env(TRUE, emptyenv()) + } + + cache_set <- function(key, value) { + assign(key, value, envir = cache) + } + + cache_get <- function(key) { + get(key, envir = cache, inherits = FALSE) + } + + cache_has_key <- function(key) { + exists(key, envir = cache, inherits = FALSE) + } + + cache_reset() + list( + reset = cache_reset, + set = cache_set, + get = cache_get, + has_key = cache_has_key, + keys = function() ls(cache) + ) +} diff --git a/man/authenticate_datastore.Rd b/man/authenticate_datastore.Rd deleted file mode 100644 index cffe29e..0000000 --- a/man/authenticate_datastore.Rd +++ /dev/null @@ -1,28 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/cache_datastore.r -\name{authenticate_datastore} -\alias{authenticate_datastore} -\title{Authenticate Datastore} -\usage{ -authenticate_datastore(key, secret, project) -} -\arguments{ -\item{key}{OAuth 2.0 credential key} - -\item{secret}{OAuth credential secret key} - -\item{project}{Google cloud platform project id/name.} -} -\description{ -Authenticate datastore using OAuth 2.0. Create an application on the -\strong{google cloud platform} -and generate -} -\seealso{ -\url{https://cloud.google.com/} - -\url{https://cloud.google.com/datastore/docs/concepts/overview} - -\url{https://developers.google.com/identity/protocols/OAuth2#basicsteps} -} - diff --git a/man/authenticate_datastore_service.Rd b/man/authenticate_datastore_service.Rd deleted file mode 100644 index 5a4c4d5..0000000 --- a/man/authenticate_datastore_service.Rd +++ /dev/null @@ -1,25 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/cache_datastore.r -\name{authenticate_datastore_service} -\alias{authenticate_datastore_service} -\title{Authenticate Datastore using a service account} -\usage{ -authenticate_datastore_service(credentials, project) -} -\arguments{ -\item{credentials}{Environmental variable or service account.} - -\item{project}{Google cloud platform project id/name.} -} -\description{ -Set credentials to the name of an environmental variable -or the location of your service account json key file. -} -\seealso{ -\url{https://cloud.google.com/} - -\url{https://cloud.google.com/datastore/docs/concepts/overview} - -\url{https://developers.google.com/identity/protocols/OAuth2#basicsteps} -} - diff --git a/man/cache_aws.Rd b/man/cache_aws.Rd new file mode 100644 index 0000000..3c62970 --- /dev/null +++ b/man/cache_aws.Rd @@ -0,0 +1,12 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/cache_aws.r +\name{cache_aws} +\alias{cache_aws} +\title{new_cache} +\usage{ +cache_aws(cache_name = "rcache") +} +\description{ +Use R to cache items. +} + diff --git a/man/datastore_cache.Rd b/man/cache_datastore.Rd similarity index 81% rename from man/datastore_cache.Rd rename to man/cache_datastore.Rd index 10845ac..9095bf7 100644 --- a/man/datastore_cache.Rd +++ b/man/cache_datastore.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/cache_datastore.r -\name{datastore_cache} -\alias{datastore_cache} +\name{cache_datastore} +\alias{cache_datastore} \title{datastore_cache} \usage{ -datastore_cache(cache_name = "rcache") +cache_datastore(project, cache_name = "rcache") } \description{ Use google datastore to store and retrieve cache items. Requires authentication. diff --git a/man/cache_gs.Rd b/man/cache_gs.Rd new file mode 100644 index 0000000..a6e6399 --- /dev/null +++ b/man/cache_gs.Rd @@ -0,0 +1,12 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/cache_gs.r +\name{cache_gs} +\alias{cache_gs} +\title{gs_cache} +\usage{ +cache_gs(project, cache_name = "rcache") +} +\description{ +Use google storage as a cache. Requires authentication. +} + diff --git a/memoise.Rproj b/memoise.Rproj index cba1b6b..4b56c89 100644 --- a/memoise.Rproj +++ b/memoise.Rproj @@ -18,4 +18,4 @@ StripTrailingWhitespace: Yes BuildType: Package PackageUseDevtools: Yes PackageInstallArgs: --no-multiarch --with-keep.source -PackageRoxygenize: rd,collate,namespace +PackageRoxygenize: rd,collate,namespace,vignette From 0448ff804ee3129d19171c403ee53169f3220269 Mon Sep 17 00:00:00 2001 From: Dan Cook Date: Mon, 25 Jul 2016 16:33:04 -0500 Subject: [PATCH 10/29] Cleanup and readme. --- NAMESPACE | 3 +-- R/cache_gs.r | 48 ----------------------------------- R/{cache_aws.r => cache_s3.r} | 2 +- README.md | 40 ++++++++++++++++++++--------- man/cache_aws.Rd | 12 --------- man/cache_gs.Rd | 12 --------- man/cache_s3.Rd | 12 +++++++++ 7 files changed, 42 insertions(+), 87 deletions(-) delete mode 100644 R/cache_gs.r rename R/{cache_aws.r => cache_s3.r} (96%) delete mode 100644 man/cache_aws.Rd delete mode 100644 man/cache_gs.Rd create mode 100644 man/cache_s3.Rd diff --git a/NAMESPACE b/NAMESPACE index 528cff3..2fb5ad9 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,9 +1,8 @@ # Generated by roxygen2: do not edit by hand S3method(print,memoised) -export(cache_aws) export(cache_datastore) -export(cache_gs) +export(cache_s3) export(forget) export(has_cache) export(is.memoised) diff --git a/R/cache_gs.r b/R/cache_gs.r deleted file mode 100644 index 9b10199..0000000 --- a/R/cache_gs.r +++ /dev/null @@ -1,48 +0,0 @@ -#' gs_cache -#' -#' Use google storage as a cache. Requires authentication. -#' -#' @export - -cache_gs <- function(project, cache_name = "rcache") { - - - options(googleAuthR.scopes.selected = c("https://www.googleapis.com/auth/devstorage.full_control", - "https://www.googleapis.com/auth/devstorage.read_write")) - - googleAuthR::gar_auth() - - base_url <- paste0("https://www.googleapis.com/storage/v1/b?project=andersen-lab") - - transaction <- googleAuthR::gar_api_generator(base_url, - "GET", - data_parse_function = function(x) x) - transaction() - - - cache <- NULL - cache_reset <- function() { - cache <<- new.env(TRUE, emptyenv()) - } - - cache_set <- function(key, value) { - assign(key, value, envir = cache) - } - - cache_get <- function(key) { - get(key, envir = cache, inherits = FALSE) - } - - cache_has_key <- function(key) { - exists(key, envir = cache, inherits = FALSE) - } - - cache_reset() - list( - reset = cache_reset, - set = cache_set, - get = cache_get, - has_key = cache_has_key, - keys = function() ls(cache) - ) -} diff --git a/R/cache_aws.r b/R/cache_s3.r similarity index 96% rename from R/cache_aws.r rename to R/cache_s3.r index 41f1b4d..4d3b583 100644 --- a/R/cache_aws.r +++ b/R/cache_s3.r @@ -4,7 +4,7 @@ #' #' @export -cache_aws <- function(cache_name = "rcache") { +cache_s3 <- function(cache_name = "rcache") { if (!(aws.s3::bucket_exists(cache_name))) { aws.s3::put_bucket(cache_name) diff --git a/README.md b/README.md index a2f2a6d..f63bf84 100644 --- a/README.md +++ b/README.md @@ -33,25 +33,41 @@ cache with * [x] Google Datastore * [ ] Dropbox * [ ] Google Storage -* [ ] AWS +* [X] AWS ### To Do * [ ] Switch to using GoogleAuthR -# Memoization with google datastore +# Caches -Google Datastore +## Google Datastore -There are a few trade-offs to using google datastore for memoization. +Use `cache_datastore` to set up a cache on google datastore. Requires you to set a `project` and `cache_name`. The `cache_name` +is used to set the kind for each entity stored on google datastore. + +```r +library(xmemoise) + +# Generate a memoised function. +mrunif <- memoise(runif, cache = cache_datastore("", "rcache")) + +mrunif(10) # First run, saves cache +mrunif(10) # Loads cache, results should be identical +``` + +## AWS S3 + +Use `cache_s3` to cache objects using s3 storage. Requires you to specify a bucket using `cache_name`. When creating buckets, they must be unique among all s3 users when created. + +```r +Sys.setenv("AWS_ACCESS_KEY_ID" = "", + "AWS_SECRET_ACCESS_KEY" = "") + +mrunif <- memoise(runif, cache = cache_aws("")) + +mrunif(10) # First run, saves cache +mrunif(10) # Loads cache, results should be identical ``` -key <- "" -secret <- "" -authenticate_datastore(key, secret, "") - -mem_fib <- memoise(fib, cache = datastore_cache("custom_cache_name")) -mem_fib(20) # Saved to datastore; Can be retrieved on another computer. -mem_fib(20) # Cached version retrieved from google datastore. -``` \ No newline at end of file diff --git a/man/cache_aws.Rd b/man/cache_aws.Rd deleted file mode 100644 index 3c62970..0000000 --- a/man/cache_aws.Rd +++ /dev/null @@ -1,12 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/cache_aws.r -\name{cache_aws} -\alias{cache_aws} -\title{new_cache} -\usage{ -cache_aws(cache_name = "rcache") -} -\description{ -Use R to cache items. -} - diff --git a/man/cache_gs.Rd b/man/cache_gs.Rd deleted file mode 100644 index a6e6399..0000000 --- a/man/cache_gs.Rd +++ /dev/null @@ -1,12 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/cache_gs.r -\name{cache_gs} -\alias{cache_gs} -\title{gs_cache} -\usage{ -cache_gs(project, cache_name = "rcache") -} -\description{ -Use google storage as a cache. Requires authentication. -} - diff --git a/man/cache_s3.Rd b/man/cache_s3.Rd new file mode 100644 index 0000000..32fac95 --- /dev/null +++ b/man/cache_s3.Rd @@ -0,0 +1,12 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/cache_s3.r +\name{cache_s3} +\alias{cache_s3} +\title{new_cache} +\usage{ +cache_s3(cache_name = "rcache") +} +\description{ +Use R to cache items. +} + From 9e1ac4cee855811cf2573d438c503768225ec8e1 Mon Sep 17 00:00:00 2001 From: Daniel E Cook Date: Mon, 25 Jul 2016 16:34:20 -0500 Subject: [PATCH 11/29] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f63bf84..bd8182c 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ Use `cache_s3` to cache objects using s3 storage. Requires you to specify a buck Sys.setenv("AWS_ACCESS_KEY_ID" = "", "AWS_SECRET_ACCESS_KEY" = "") -mrunif <- memoise(runif, cache = cache_aws("")) +mrunif <- memoise(runif, cache = cache_s3("")) mrunif(10) # First run, saves cache mrunif(10) # Loads cache, results should be identical From 27fe9e841486c605d48fb5b3ac964eea48570414 Mon Sep 17 00:00:00 2001 From: Daniel Cook Date: Tue, 26 Jul 2016 08:39:02 -0500 Subject: [PATCH 12/29] Rename, update --- DESCRIPTION | 10 +++++++--- memoise.Rproj => xmemoise.Rproj | 0 2 files changed, 7 insertions(+), 3 deletions(-) rename memoise.Rproj => xmemoise.Rproj (100%) diff --git a/DESCRIPTION b/DESCRIPTION index 424241f..48a75f1 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,17 +1,21 @@ Encoding: UTF-8 Package: xmemoise Title: Memoisation of Functions -Version: 1.0.0.9000 +Version: 1.0.1 Authors@R: c( person("Hadley", "Wickham", , "hadley@rstudio.com", role = "aut"), person("Jim", "Hester", , "jim.hester@rstudio.com", role = c("aut", "cre")), - person("Kirill", "Müller", , "krlmlr+r@mailbox.org", role = "aut")) + person("Kirill", "Müller", , "krlmlr+r@mailbox.org", role = "aut")), + person("Daniel", "Cook", , "Danielecook@gmail.com", rol = "aut") Description: Cache the results of a function so that when you call it again with the same arguments it returns the pre-computed value. URL: https://github.com/hadley/memoise BugReports: https://github.com/hadley/memoise/issues Imports: - digest (>= 0.6.3) + digest (>= 0.6.3), + googleAuthR, + base64enc, + aws.s3 Suggests: testthat License: MIT + file LICENSE diff --git a/memoise.Rproj b/xmemoise.Rproj similarity index 100% rename from memoise.Rproj rename to xmemoise.Rproj From c2f0bced4fc083c58c3934964addbe32024f66b2 Mon Sep 17 00:00:00 2001 From: Daniel Cook Date: Tue, 26 Jul 2016 08:45:27 -0500 Subject: [PATCH 13/29] Update description. --- DESCRIPTION | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 48a75f1..fbd9f2d 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,9 @@ Encoding: UTF-8 Package: xmemoise Title: Memoisation of Functions +Description: Memoisation allows the results of functions to be cached based on input parameters + xmemoise offers both local and remote caches, enabling memoisation across computers. + Additional cache types include google datastore and amazon aws. Version: 1.0.1 Authors@R: c( person("Hadley", "Wickham", , "hadley@rstudio.com", role = "aut"), @@ -9,13 +12,14 @@ Authors@R: c( person("Daniel", "Cook", , "Danielecook@gmail.com", rol = "aut") Description: Cache the results of a function so that when you call it again with the same arguments it returns the pre-computed value. -URL: https://github.com/hadley/memoise +URL: https://github.com/danielecook/xmemoise BugReports: https://github.com/hadley/memoise/issues Imports: digest (>= 0.6.3), googleAuthR, - base64enc, - aws.s3 + base64enc +Remotes: + cloudyr/aws.s3 Suggests: testthat License: MIT + file LICENSE From 6fad8e3c23a99ed0113f29c2fab060773067d21b Mon Sep 17 00:00:00 2001 From: Daniel Cook Date: Tue, 26 Jul 2016 08:49:21 -0500 Subject: [PATCH 14/29] fix description --- DESCRIPTION | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index fbd9f2d..93b78b5 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -8,8 +8,8 @@ Version: 1.0.1 Authors@R: c( person("Hadley", "Wickham", , "hadley@rstudio.com", role = "aut"), person("Jim", "Hester", , "jim.hester@rstudio.com", role = c("aut", "cre")), - person("Kirill", "Müller", , "krlmlr+r@mailbox.org", role = "aut")), - person("Daniel", "Cook", , "Danielecook@gmail.com", rol = "aut") + person("Kirill", "Müller", , "krlmlr+r@mailbox.org", role = "aut"), + person("Daniel", "Cook", , "danielecook@gmail.com", role = "aut")) Description: Cache the results of a function so that when you call it again with the same arguments it returns the pre-computed value. URL: https://github.com/danielecook/xmemoise @@ -18,8 +18,7 @@ Imports: digest (>= 0.6.3), googleAuthR, base64enc -Remotes: - cloudyr/aws.s3 +Remotes: cloudyr/aws.s3, Suggests: testthat License: MIT + file LICENSE From 73c11b443933d89b1c169156c048eaa9f3b5efee Mon Sep 17 00:00:00 2001 From: Daniel Cook Date: Tue, 26 Jul 2016 08:49:49 -0500 Subject: [PATCH 15/29] fix remotes --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 93b78b5..abb82d8 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -18,7 +18,7 @@ Imports: digest (>= 0.6.3), googleAuthR, base64enc -Remotes: cloudyr/aws.s3, +Remotes: github::cloudyr/aws.s3, Suggests: testthat License: MIT + file LICENSE From cd16c332303d42fb5a5c9f534c5f02ecd3f7a410 Mon Sep 17 00:00:00 2001 From: Daniel Cook Date: Tue, 26 Jul 2016 08:50:38 -0500 Subject: [PATCH 16/29] fix remotes. --- DESCRIPTION | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index abb82d8..523d9e8 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -18,7 +18,8 @@ Imports: digest (>= 0.6.3), googleAuthR, base64enc -Remotes: github::cloudyr/aws.s3, +Remotes: + cloudyr/aws.s3 Suggests: testthat License: MIT + file LICENSE From df3b92f66603d0936f60020e2d52020a1edb2917 Mon Sep 17 00:00:00 2001 From: Daniel Cook Date: Tue, 26 Jul 2016 08:51:04 -0500 Subject: [PATCH 17/29] fix remotes. --- DESCRIPTION | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 523d9e8..5dce544 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -18,8 +18,7 @@ Imports: digest (>= 0.6.3), googleAuthR, base64enc -Remotes: - cloudyr/aws.s3 +Remotes: cloudyr/aws.s3 Suggests: testthat License: MIT + file LICENSE From 4c6df0dd9ef3ec973cac02c7410f4c90afefc675 Mon Sep 17 00:00:00 2001 From: Daniel Cook Date: Tue, 26 Jul 2016 08:51:49 -0500 Subject: [PATCH 18/29] use git repo instead --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 5dce544..be7d663 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -18,7 +18,7 @@ Imports: digest (>= 0.6.3), googleAuthR, base64enc -Remotes: cloudyr/aws.s3 +Remotes: git::https://github.com/cloudyr/aws.s3.git Suggests: testthat License: MIT + file LICENSE From 25cbe38988db46114fdf193c10a4976194c41d9b Mon Sep 17 00:00:00 2001 From: Daniel Cook Date: Tue, 26 Jul 2016 08:56:52 -0500 Subject: [PATCH 19/29] Remotes were fine... --- DESCRIPTION | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index be7d663..7193573 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -18,8 +18,9 @@ Imports: digest (>= 0.6.3), googleAuthR, base64enc -Remotes: git::https://github.com/cloudyr/aws.s3.git Suggests: testthat +Remotes: + cloudyr/aws.s3 License: MIT + file LICENSE RoxygenNote: 5.0.1 From 68e8636a67d2fd8d3e5252fa4b058a7f8ba1e136 Mon Sep 17 00:00:00 2001 From: Daniel E Cook Date: Wed, 27 Jul 2016 16:31:16 -0500 Subject: [PATCH 20/29] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bd8182c..523ff98 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ cache with ### To Do -* [ ] Switch to using GoogleAuthR +* [x] Switch to using GoogleAuthR # Caches From f435e95676447ff87f776dc74536e1e595ffdb65 Mon Sep 17 00:00:00 2001 From: Dan Cook Date: Thu, 28 Jul 2016 12:51:04 -0500 Subject: [PATCH 21/29] Restructure for fork --- DESCRIPTION | 6 ++-- NAMESPACE | 5 +-- R/{cache_s3.r => cache_aws_s3.r} | 14 ++++++-- R/cache_datastore.r | 16 ++++++--- R/cache_filesystem.r | 59 ++++++++++++++++++++++++++++++++ R/{cache.r => cache_local.r} | 7 ++-- R/memoise.r | 3 +- man/cache_aws_s3.Rd | 16 +++++++++ man/cache_datastore.Rd | 13 +++---- man/cache_filesystem.Rd | 21 ++++++++++++ man/cache_local.Rd | 11 ++++++ man/cache_s3.Rd | 12 ------- man/memoise.Rd | 4 ++- man/new_cache.Rd | 12 ------- xmemoise.Rproj => memoise.Rproj | 2 +- 15 files changed, 154 insertions(+), 47 deletions(-) rename R/{cache_s3.r => cache_aws_s3.r} (72%) create mode 100644 R/cache_filesystem.r rename R/{cache.r => cache_local.r} (79%) create mode 100644 man/cache_aws_s3.Rd create mode 100644 man/cache_filesystem.Rd create mode 100644 man/cache_local.Rd delete mode 100644 man/cache_s3.Rd delete mode 100644 man/new_cache.Rd rename xmemoise.Rproj => memoise.Rproj (87%) diff --git a/DESCRIPTION b/DESCRIPTION index 7193573..d7b0c68 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Encoding: UTF-8 -Package: xmemoise +Package: memoise Title: Memoisation of Functions Description: Memoisation allows the results of functions to be cached based on input parameters xmemoise offers both local and remote caches, enabling memoisation across computers. @@ -16,10 +16,10 @@ URL: https://github.com/danielecook/xmemoise BugReports: https://github.com/hadley/memoise/issues Imports: digest (>= 0.6.3), - googleAuthR, base64enc Suggests: - testthat + testthat, + googleAuthR Remotes: cloudyr/aws.s3 License: MIT + file LICENSE diff --git a/NAMESPACE b/NAMESPACE index 2fb5ad9..a9cfc23 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,15 +1,16 @@ # Generated by roxygen2: do not edit by hand S3method(print,memoised) +export(cache_aws_s3) export(cache_datastore) -export(cache_s3) +export(cache_filesystem) +export(cache_local) export(forget) export(has_cache) export(is.memoised) export(is.memoized) export(memoise) export(memoize) -export(new_cache) export(timeout) importFrom(digest,digest) importFrom(stats,setNames) diff --git a/R/cache_s3.r b/R/cache_aws_s3.r similarity index 72% rename from R/cache_s3.r rename to R/cache_aws_s3.r index 4d3b583..37af748 100644 --- a/R/cache_s3.r +++ b/R/cache_aws_s3.r @@ -1,10 +1,20 @@ -#' new_cache +#' @name cache_aws_s3 +#' @title Initiate an Amazon Web Services Cache +#' @usage #' +#' Sys.setenv("AWS_ACCESS_KEY_ID" = "", +#' "AWS_SECRET_ACCESS_KEY" = "") +#' +#' cache_aws_s3("unique-bucket-name") +#' +#' @param cache_name Bucket name for storing cache files. #' Use R to cache items. #' #' @export -cache_s3 <- function(cache_name = "rcache") { +cache_aws_s3 <- function(cache_name = "rcache") { + + if (!("aws.s3" %in% installed.packages()[,"Package"])) { stop("aws.s3 required for datastore cache.") } if (!(aws.s3::bucket_exists(cache_name))) { aws.s3::put_bucket(cache_name) diff --git a/R/cache_datastore.r b/R/cache_datastore.r index d9961d0..3bf08b2 100644 --- a/R/cache_datastore.r +++ b/R/cache_datastore.r @@ -1,15 +1,21 @@ -#' datastore_cache +#' @name cache_datastore +#' @title Initiate a datastore cache. +#' @param project Google Cloud project +#' @param cache_name datastore kind to use for storing cache entities. +#' @usage #' -#' Use google datastore to store and retrieve cache items. Requires authentication. +#' cache_datastore(project="your-project-name", +#' cache_name = "rcache") #' #' @seealso \url{https://cloud.google.com/} #' @seealso \url{https://cloud.google.com/datastore/docs/concepts/overview} -#' @seealso \url{https://developers.google.com/identity/protocols/OAuth2#basicsteps} #' #' @export cache_datastore <- function(project, cache_name = "rcache") { + if (!("googleAuthR" %in% installed.packages()[,"Package"])) { stop("googleAuthR required for datastore cache.") } + options("googleAuthR.scopes.selected" = c("https://www.googleapis.com/auth/datastore", "https://www.googleapis.com/auth/userinfo.email")) @@ -48,7 +54,9 @@ cache_datastore <- function(project, cache_name = "rcache") { cache_reset <- function() { query_results <- query_ds(the_body = list(gqlQuery = list(queryString = paste0("SELECT * FROM ", cache_name)))) while((query_results$batch$moreResults != "NO_MORE_RESULTS") | is.null(query_results$batch$entityResults) == FALSE) { - ids <- (query_results$batch$entityResults$entity$key$path %>% dplyr::bind_rows())$name + + + ids <- (sapply(query_results$batch$entityResults$entity$key$path, function(x) x$name)) item_groups <- split(ids, (1:length(ids)) %/% 25) sapply(item_groups, function(idset) { diff --git a/R/cache_filesystem.r b/R/cache_filesystem.r new file mode 100644 index 0000000..a4502ae --- /dev/null +++ b/R/cache_filesystem.r @@ -0,0 +1,59 @@ +#' @name cache_filesystem +#' @title Initiate a filesystem cache. +#' @usage +#' cache_filesystem stores cached items within a directory. It can be used with +#' Dropbox or Google Drive to allow for cache access from multiple computers. +#' +#' To use cache_filesystem with Dropbox, you can specify: +#' +#' cache_filesystem("~/Dropbox/.rcache") +#' +#' For Google Drive: +#' +#' cache_filesystem("~/Google Drive/.rcache") +#' +#' +#' @param path Directory in which to store cached items. +#' +#' +#' @export + +cache_filesystem <- function(path) { + + dir.create(file.path(path), showWarnings = FALSE) + + cache_reset <- function() { + cache_files <- list.files(path, full.names = TRUE) + # Use an environment for loaded items. + cache <<- new.env(TRUE, emptyenv()) + if (length(cache_files) > 0) { + rm_status <- file.remove(list.files(path, full.names = TRUE)) + if (rm_status) { + message("Cached files removed.") + } + } else { + message("No files in Cache.") + } + } + + cache_set <- function(key, value) { + save(value, file = paste(path, key, sep="/")) + } + + cache_get <- function(key) { + load(file = paste(path, key, sep="/")) + value + } + + cache_has_key <- function(key) { + file.exists(paste(path, key, sep="/")) + } + + list( + reset = cache_reset, + set = cache_set, + get = cache_get, + has_key = cache_has_key, + keys = function() list.files(path) + ) +} diff --git a/R/cache.r b/R/cache_local.r similarity index 79% rename from R/cache.r rename to R/cache_local.r index 7c3b232..41c838d 100644 --- a/R/cache.r +++ b/R/cache_local.r @@ -1,10 +1,11 @@ -#' new_cache +#' @name cache_local +#' @title Initiate an in memory cache. #' -#' Use R to cache items. +#' cache_local() stores cached items in memory. #' #' @export -new_cache <- function() { +cache_local <- function() { cache <- NULL cache_reset <- function() { diff --git a/R/memoise.r b/R/memoise.r index 2e132f0..e73388c 100644 --- a/R/memoise.r +++ b/R/memoise.r @@ -38,6 +38,7 @@ #' @param ... optional variables specified as formulas with no RHS to use as #' additional restrictions on caching. See Examples for usage. #' @param envir Environment of the returned function. +#' @param cache Cache function. #' @seealso \code{\link{forget}}, \code{\link{is.memoised}}, #' \code{\link{timeout}}, \url{http://en.wikipedia.org/wiki/Memoization} #' @aliases memoise memoize @@ -97,7 +98,7 @@ #' memA4 <- memoise(a, ~timeout(10)) #' memA4(2) #' @importFrom stats setNames -memoise <- memoize <- function(f, ..., envir = environment(f), cache = new_cache()) { +memoise <- memoize <- function(f, ..., envir = environment(f), cache = cache_local()) { f_formals <- formals(args(f)) if(is.memoised(f)) { stop("`f` must not be memoised.", call. = FALSE) diff --git a/man/cache_aws_s3.Rd b/man/cache_aws_s3.Rd new file mode 100644 index 0000000..097eaa6 --- /dev/null +++ b/man/cache_aws_s3.Rd @@ -0,0 +1,16 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/cache_aws_s3.r +\name{cache_aws_s3} +\alias{cache_aws_s3} +\title{Initiate an Amazon Web Services Cache} +\usage{ +Sys.setenv("AWS_ACCESS_KEY_ID" = "", + "AWS_SECRET_ACCESS_KEY" = "") + +cache_aws_s3("unique-bucket-name") +} +\arguments{ +\item{cache_name}{Bucket name for storing cache files. +Use R to cache items.} +} + diff --git a/man/cache_datastore.Rd b/man/cache_datastore.Rd index 9095bf7..6d33dbc 100644 --- a/man/cache_datastore.Rd +++ b/man/cache_datastore.Rd @@ -2,18 +2,19 @@ % Please edit documentation in R/cache_datastore.r \name{cache_datastore} \alias{cache_datastore} -\title{datastore_cache} +\title{Initiate a datastore cache.} \usage{ -cache_datastore(project, cache_name = "rcache") +cache_datastore(project="your-project-name", + cache_name = "rcache") } -\description{ -Use google datastore to store and retrieve cache items. Requires authentication. +\arguments{ +\item{project}{Google Cloud project} + +\item{cache_name}{datastore kind to use for storing cache entities.} } \seealso{ \url{https://cloud.google.com/} \url{https://cloud.google.com/datastore/docs/concepts/overview} - -\url{https://developers.google.com/identity/protocols/OAuth2#basicsteps} } diff --git a/man/cache_filesystem.Rd b/man/cache_filesystem.Rd new file mode 100644 index 0000000..07cd7ae --- /dev/null +++ b/man/cache_filesystem.Rd @@ -0,0 +1,21 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/cache_filesystem.r +\name{cache_filesystem} +\alias{cache_filesystem} +\title{Initiate a filesystem cache.} +\usage{ +cache_filesystem stores cached items within a directory. It can be used with +Dropbox or Google Drive to allow for cache access from multiple computers. + +To use cache_filesystem with Dropbox, you can specify: + + cache_filesystem("~/Dropbox/.rcache") + +For Google Drive: + + cache_filesystem("~/Google Drive/.rcache") +} +\arguments{ +\item{path}{Directory in which to store cached items.} +} + diff --git a/man/cache_local.Rd b/man/cache_local.Rd new file mode 100644 index 0000000..47bc376 --- /dev/null +++ b/man/cache_local.Rd @@ -0,0 +1,11 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/cache_local.r +\name{cache_local} +\alias{cache_local} +\title{Initiate an in memory cache. + +cache_local() stores cached items in memory.} +\usage{ +cache_local() +} + diff --git a/man/cache_s3.Rd b/man/cache_s3.Rd deleted file mode 100644 index 32fac95..0000000 --- a/man/cache_s3.Rd +++ /dev/null @@ -1,12 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/cache_s3.r -\name{cache_s3} -\alias{cache_s3} -\title{new_cache} -\usage{ -cache_s3(cache_name = "rcache") -} -\description{ -Use R to cache items. -} - diff --git a/man/memoise.Rd b/man/memoise.Rd index 42880e1..97d2c4e 100644 --- a/man/memoise.Rd +++ b/man/memoise.Rd @@ -5,7 +5,7 @@ \alias{memoize} \title{Memoise a function.} \usage{ -memoise(f, ..., envir = environment(f), cache = new_cache()) +memoise(f, ..., envir = environment(f), cache = cache_local()) } \arguments{ \item{f}{Function of which to create a memoised copy.} @@ -14,6 +14,8 @@ memoise(f, ..., envir = environment(f), cache = new_cache()) additional restrictions on caching. See Examples for usage.} \item{envir}{Environment of the returned function.} + +\item{cache}{Cache function.} } \description{ \code{mf <- memoise(f)} creates \code{mf}, a memoised copy of diff --git a/man/new_cache.Rd b/man/new_cache.Rd deleted file mode 100644 index c4837d6..0000000 --- a/man/new_cache.Rd +++ /dev/null @@ -1,12 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/cache.r -\name{new_cache} -\alias{new_cache} -\title{new_cache} -\usage{ -new_cache() -} -\description{ -Use R to cache items. -} - diff --git a/xmemoise.Rproj b/memoise.Rproj similarity index 87% rename from xmemoise.Rproj rename to memoise.Rproj index 4b56c89..cba1b6b 100644 --- a/xmemoise.Rproj +++ b/memoise.Rproj @@ -18,4 +18,4 @@ StripTrailingWhitespace: Yes BuildType: Package PackageUseDevtools: Yes PackageInstallArgs: --no-multiarch --with-keep.source -PackageRoxygenize: rd,collate,namespace,vignette +PackageRoxygenize: rd,collate,namespace From c32fd35486ae38d7c7e56c23e7cb3b5a0e6bedcb Mon Sep 17 00:00:00 2001 From: Dan Cook Date: Thu, 28 Jul 2016 13:04:05 -0500 Subject: [PATCH 22/29] Improve documentation. --- R/cache_aws_s3.r | 6 ++++-- R/cache_datastore.r | 3 ++- R/cache_filesystem.r | 4 +++- R/cache_local.r | 3 ++- man/cache_aws_s3.Rd | 7 +++++-- man/cache_datastore.Rd | 5 ++++- man/cache_filesystem.Rd | 5 ++++- man/cache_local.Rd | 9 ++++++--- 8 files changed, 30 insertions(+), 12 deletions(-) diff --git a/R/cache_aws_s3.r b/R/cache_aws_s3.r index 37af748..81a43c1 100644 --- a/R/cache_aws_s3.r +++ b/R/cache_aws_s3.r @@ -1,11 +1,13 @@ #' @name cache_aws_s3 -#' @title Initiate an Amazon Web Services Cache +#' @title Amazon Web Services S3 Cache +#' @description +#' Initiate an Amazon Web Services Cache #' @usage #' #' Sys.setenv("AWS_ACCESS_KEY_ID" = "", #' "AWS_SECRET_ACCESS_KEY" = "") #' -#' cache_aws_s3("unique-bucket-name") +#' cache_aws_s3(cache_name = "unique-bucket-name") #' #' @param cache_name Bucket name for storing cache files. #' Use R to cache items. diff --git a/R/cache_datastore.r b/R/cache_datastore.r index 3bf08b2..d4da6d6 100644 --- a/R/cache_datastore.r +++ b/R/cache_datastore.r @@ -1,5 +1,6 @@ #' @name cache_datastore -#' @title Initiate a datastore cache. +#' @title Google Datastore Cache +#' @description Initiate a Google Datastore cache. #' @param project Google Cloud project #' @param cache_name datastore kind to use for storing cache entities. #' @usage diff --git a/R/cache_filesystem.r b/R/cache_filesystem.r index a4502ae..5599f91 100644 --- a/R/cache_filesystem.r +++ b/R/cache_filesystem.r @@ -1,5 +1,7 @@ #' @name cache_filesystem -#' @title Initiate a filesystem cache. +#' @title Filesystem Cache +#' @description +#' Initiate a filesystem cache. #' @usage #' cache_filesystem stores cached items within a directory. It can be used with #' Dropbox or Google Drive to allow for cache access from multiple computers. diff --git a/R/cache_local.r b/R/cache_local.r index 41c838d..f1df2e0 100644 --- a/R/cache_local.r +++ b/R/cache_local.r @@ -1,5 +1,6 @@ #' @name cache_local -#' @title Initiate an in memory cache. +#' @title In Memory Cache +#' @description Initiate an in memory cache. #' #' cache_local() stores cached items in memory. #' diff --git a/man/cache_aws_s3.Rd b/man/cache_aws_s3.Rd index 097eaa6..a273032 100644 --- a/man/cache_aws_s3.Rd +++ b/man/cache_aws_s3.Rd @@ -2,15 +2,18 @@ % Please edit documentation in R/cache_aws_s3.r \name{cache_aws_s3} \alias{cache_aws_s3} -\title{Initiate an Amazon Web Services Cache} +\title{Amazon Web Services S3 Cache} \usage{ Sys.setenv("AWS_ACCESS_KEY_ID" = "", "AWS_SECRET_ACCESS_KEY" = "") -cache_aws_s3("unique-bucket-name") +cache_aws_s3(cache_name = "unique-bucket-name") } \arguments{ \item{cache_name}{Bucket name for storing cache files. Use R to cache items.} } +\description{ +Initiate an Amazon Web Services Cache +} diff --git a/man/cache_datastore.Rd b/man/cache_datastore.Rd index 6d33dbc..08cf1a9 100644 --- a/man/cache_datastore.Rd +++ b/man/cache_datastore.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/cache_datastore.r \name{cache_datastore} \alias{cache_datastore} -\title{Initiate a datastore cache.} +\title{Google Datastore Cache} \usage{ cache_datastore(project="your-project-name", cache_name = "rcache") @@ -12,6 +12,9 @@ cache_datastore(project="your-project-name", \item{cache_name}{datastore kind to use for storing cache entities.} } +\description{ +Initiate a Google Datastore cache. +} \seealso{ \url{https://cloud.google.com/} diff --git a/man/cache_filesystem.Rd b/man/cache_filesystem.Rd index 07cd7ae..84bbbdf 100644 --- a/man/cache_filesystem.Rd +++ b/man/cache_filesystem.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/cache_filesystem.r \name{cache_filesystem} \alias{cache_filesystem} -\title{Initiate a filesystem cache.} +\title{Filesystem Cache} \usage{ cache_filesystem stores cached items within a directory. It can be used with Dropbox or Google Drive to allow for cache access from multiple computers. @@ -18,4 +18,7 @@ For Google Drive: \arguments{ \item{path}{Directory in which to store cached items.} } +\description{ +Initiate a filesystem cache. +} diff --git a/man/cache_local.Rd b/man/cache_local.Rd index 47bc376..78e855c 100644 --- a/man/cache_local.Rd +++ b/man/cache_local.Rd @@ -2,10 +2,13 @@ % Please edit documentation in R/cache_local.r \name{cache_local} \alias{cache_local} -\title{Initiate an in memory cache. - -cache_local() stores cached items in memory.} +\title{In Memory Cache} \usage{ cache_local() } +\description{ +Initiate an in memory cache. + +cache_local() stores cached items in memory. +} From 324e5a505a4c0695634f9144cc2160696bbc7885 Mon Sep 17 00:00:00 2001 From: Dan Cook Date: Thu, 28 Jul 2016 13:42:23 -0500 Subject: [PATCH 23/29] remove documentation items not allowed... --- R/cache_aws_s3.r | 13 ++----------- R/cache_datastore.r | 2 -- R/cache_filesystem.r | 15 +-------------- man/cache_aws_s3.Rd | 8 ++------ man/cache_datastore.Rd | 3 +-- man/cache_filesystem.Rd | 11 +---------- 6 files changed, 7 insertions(+), 45 deletions(-) diff --git a/R/cache_aws_s3.r b/R/cache_aws_s3.r index 81a43c1..ba2035b 100644 --- a/R/cache_aws_s3.r +++ b/R/cache_aws_s3.r @@ -1,20 +1,11 @@ #' @name cache_aws_s3 #' @title Amazon Web Services S3 Cache -#' @description -#' Initiate an Amazon Web Services Cache -#' @usage -#' -#' Sys.setenv("AWS_ACCESS_KEY_ID" = "", -#' "AWS_SECRET_ACCESS_KEY" = "") -#' -#' cache_aws_s3(cache_name = "unique-bucket-name") +#' @description Initiate an Amazon Web Services Cache #' #' @param cache_name Bucket name for storing cache files. -#' Use R to cache items. -#' #' @export -cache_aws_s3 <- function(cache_name = "rcache") { +cache_aws_s3 <- function(cache_name) { if (!("aws.s3" %in% installed.packages()[,"Package"])) { stop("aws.s3 required for datastore cache.") } diff --git a/R/cache_datastore.r b/R/cache_datastore.r index d4da6d6..5f3b113 100644 --- a/R/cache_datastore.r +++ b/R/cache_datastore.r @@ -5,8 +5,6 @@ #' @param cache_name datastore kind to use for storing cache entities. #' @usage #' -#' cache_datastore(project="your-project-name", -#' cache_name = "rcache") #' #' @seealso \url{https://cloud.google.com/} #' @seealso \url{https://cloud.google.com/datastore/docs/concepts/overview} diff --git a/R/cache_filesystem.r b/R/cache_filesystem.r index 5599f91..a2d66ae 100644 --- a/R/cache_filesystem.r +++ b/R/cache_filesystem.r @@ -2,22 +2,9 @@ #' @title Filesystem Cache #' @description #' Initiate a filesystem cache. -#' @usage -#' cache_filesystem stores cached items within a directory. It can be used with -#' Dropbox or Google Drive to allow for cache access from multiple computers. -#' -#' To use cache_filesystem with Dropbox, you can specify: -#' -#' cache_filesystem("~/Dropbox/.rcache") -#' -#' For Google Drive: -#' -#' cache_filesystem("~/Google Drive/.rcache") -#' #' #' @param path Directory in which to store cached items. #' -#' #' @export cache_filesystem <- function(path) { @@ -27,7 +14,7 @@ cache_filesystem <- function(path) { cache_reset <- function() { cache_files <- list.files(path, full.names = TRUE) # Use an environment for loaded items. - cache <<- new.env(TRUE, emptyenv()) + cache <- new.env(TRUE, emptyenv()) if (length(cache_files) > 0) { rm_status <- file.remove(list.files(path, full.names = TRUE)) if (rm_status) { diff --git a/man/cache_aws_s3.Rd b/man/cache_aws_s3.Rd index a273032..8609082 100644 --- a/man/cache_aws_s3.Rd +++ b/man/cache_aws_s3.Rd @@ -4,14 +4,10 @@ \alias{cache_aws_s3} \title{Amazon Web Services S3 Cache} \usage{ -Sys.setenv("AWS_ACCESS_KEY_ID" = "", - "AWS_SECRET_ACCESS_KEY" = "") - -cache_aws_s3(cache_name = "unique-bucket-name") +cache_aws_s3(cache_name) } \arguments{ -\item{cache_name}{Bucket name for storing cache files. -Use R to cache items.} +\item{cache_name}{Bucket name for storing cache files.} } \description{ Initiate an Amazon Web Services Cache diff --git a/man/cache_datastore.Rd b/man/cache_datastore.Rd index 08cf1a9..6aeced9 100644 --- a/man/cache_datastore.Rd +++ b/man/cache_datastore.Rd @@ -4,8 +4,7 @@ \alias{cache_datastore} \title{Google Datastore Cache} \usage{ -cache_datastore(project="your-project-name", - cache_name = "rcache") + } \arguments{ \item{project}{Google Cloud project} diff --git a/man/cache_filesystem.Rd b/man/cache_filesystem.Rd index 84bbbdf..a2eaa3e 100644 --- a/man/cache_filesystem.Rd +++ b/man/cache_filesystem.Rd @@ -4,16 +4,7 @@ \alias{cache_filesystem} \title{Filesystem Cache} \usage{ -cache_filesystem stores cached items within a directory. It can be used with -Dropbox or Google Drive to allow for cache access from multiple computers. - -To use cache_filesystem with Dropbox, you can specify: - - cache_filesystem("~/Dropbox/.rcache") - -For Google Drive: - - cache_filesystem("~/Google Drive/.rcache") +cache_filesystem(path) } \arguments{ \item{path}{Directory in which to store cached items.} From 9f4a914c0ce56e1ecace3fd7317f083422f1ff94 Mon Sep 17 00:00:00 2001 From: Dan Cook Date: Thu, 28 Jul 2016 13:56:48 -0500 Subject: [PATCH 24/29] Fixed final warnings. --- DESCRIPTION | 3 ++- R/cache_aws_s3.r | 3 ++- R/cache_datastore.r | 1 - man/cache_datastore.Rd | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index d7b0c68..bf3f026 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -19,7 +19,8 @@ Imports: base64enc Suggests: testthat, - googleAuthR + googleAuthR, + aws.s3 Remotes: cloudyr/aws.s3 License: MIT + file LICENSE diff --git a/R/cache_aws_s3.r b/R/cache_aws_s3.r index ba2035b..e2c9658 100644 --- a/R/cache_aws_s3.r +++ b/R/cache_aws_s3.r @@ -7,7 +7,8 @@ cache_aws_s3 <- function(cache_name) { - if (!("aws.s3" %in% installed.packages()[,"Package"])) { stop("aws.s3 required for datastore cache.") } + # Can't get this check to pass... + # if (!("aws.s3" %in% installed.packages()[,"Package"])) { stop("aws.s3 required for datastore cache.") } if (!(aws.s3::bucket_exists(cache_name))) { aws.s3::put_bucket(cache_name) diff --git a/R/cache_datastore.r b/R/cache_datastore.r index 5f3b113..14f1e0d 100644 --- a/R/cache_datastore.r +++ b/R/cache_datastore.r @@ -3,7 +3,6 @@ #' @description Initiate a Google Datastore cache. #' @param project Google Cloud project #' @param cache_name datastore kind to use for storing cache entities. -#' @usage #' #' #' @seealso \url{https://cloud.google.com/} diff --git a/man/cache_datastore.Rd b/man/cache_datastore.Rd index 6aeced9..e158bfe 100644 --- a/man/cache_datastore.Rd +++ b/man/cache_datastore.Rd @@ -4,7 +4,7 @@ \alias{cache_datastore} \title{Google Datastore Cache} \usage{ - +cache_datastore(project, cache_name = "rcache") } \arguments{ \item{project}{Google Cloud project} From 5b3e9eef124a61f68ff27a9994e26bc115586f7f Mon Sep 17 00:00:00 2001 From: Daniel Cook Date: Thu, 28 Jul 2016 19:25:47 -0500 Subject: [PATCH 25/29] improve documentation with examples. --- R/cache_aws_s3.r | 13 +++++++++++++ R/cache_datastore.r | 5 +++++ R/cache_filesystem.r | 17 +++++++++++++++++ R/cache_local.r | 2 +- man/cache_aws_s3.Rd | 14 ++++++++++++++ man/cache_datastore.Rd | 7 +++++++ man/cache_filesystem.Rd | 18 ++++++++++++++++++ man/cache_local.Rd | 2 +- 8 files changed, 76 insertions(+), 2 deletions(-) diff --git a/R/cache_aws_s3.r b/R/cache_aws_s3.r index e2c9658..1f010f0 100644 --- a/R/cache_aws_s3.r +++ b/R/cache_aws_s3.r @@ -2,6 +2,19 @@ #' @title Amazon Web Services S3 Cache #' @description Initiate an Amazon Web Services Cache #' +#' @examples +#' +#' \dontrun{ +#' # Set AWS credentials. +#' Sys.setenv("AWS_ACCESS_KEY_ID" = "", +#' "AWS_SECRET_ACCESS_KEY" = "") +#' +#' # Set up a unique bucket name. +#' s3 <- cache_aws_s3("unique-bucket-name") +#' mem_runif <- memoise(runif, cache = s3) +#' } +#' +#' #' @param cache_name Bucket name for storing cache files. #' @export diff --git a/R/cache_datastore.r b/R/cache_datastore.r index 14f1e0d..1f44cb1 100644 --- a/R/cache_datastore.r +++ b/R/cache_datastore.r @@ -4,6 +4,11 @@ #' @param project Google Cloud project #' @param cache_name datastore kind to use for storing cache entities. #' +#' @examples +#' \dontrun{ +#' ds <- cache_datastore(project = "", cache_name = "rcache") +#' mem_memoise(runif, cache = ds) +#' } #' #' @seealso \url{https://cloud.google.com/} #' @seealso \url{https://cloud.google.com/datastore/docs/concepts/overview} diff --git a/R/cache_filesystem.r b/R/cache_filesystem.r index a2d66ae..d0f5aa6 100644 --- a/R/cache_filesystem.r +++ b/R/cache_filesystem.r @@ -5,6 +5,23 @@ #' #' @param path Directory in which to store cached items. #' +#' @examples +#' +#' \dontrun{ +#' # Use with Dropbox +#' +#' db <- cache_filesystem("~/Dropbox/.rcache") +#' +#' mem_runif <- memoise(runif, cache = db) +#' +#' # Use with Google Drive +#' +#' gd <- cache_filesystem("~/Google Drive/.rcache") +#' +#' mem_runif <- memoise(runif, cache = gd) +#' +#' } +#' #' @export cache_filesystem <- function(path) { diff --git a/R/cache_local.r b/R/cache_local.r index f1df2e0..bbe1208 100644 --- a/R/cache_local.r +++ b/R/cache_local.r @@ -2,7 +2,7 @@ #' @title In Memory Cache #' @description Initiate an in memory cache. #' -#' cache_local() stores cached items in memory. +#' cache_local() stores cached items in memory. It is the default cache. #' #' @export diff --git a/man/cache_aws_s3.Rd b/man/cache_aws_s3.Rd index 8609082..be064af 100644 --- a/man/cache_aws_s3.Rd +++ b/man/cache_aws_s3.Rd @@ -12,4 +12,18 @@ cache_aws_s3(cache_name) \description{ Initiate an Amazon Web Services Cache } +\examples{ + +\dontrun{ +# Set AWS credentials. +Sys.setenv("AWS_ACCESS_KEY_ID" = "", + "AWS_SECRET_ACCESS_KEY" = "") + +# Set up a unique bucket name. +s3 <- cache_aws_s3("unique-bucket-name") +mem_runif <- memoise(runif, cache = s3) +} + + +} diff --git a/man/cache_datastore.Rd b/man/cache_datastore.Rd index e158bfe..bee1637 100644 --- a/man/cache_datastore.Rd +++ b/man/cache_datastore.Rd @@ -13,6 +13,13 @@ cache_datastore(project, cache_name = "rcache") } \description{ Initiate a Google Datastore cache. +} +\examples{ +\dontrun{ +ds <- cache_datastore(project = "", cache_name = "rcache") +mem_memoise(runif, cache = ds) +} + } \seealso{ \url{https://cloud.google.com/} diff --git a/man/cache_filesystem.Rd b/man/cache_filesystem.Rd index a2eaa3e..4193258 100644 --- a/man/cache_filesystem.Rd +++ b/man/cache_filesystem.Rd @@ -12,4 +12,22 @@ cache_filesystem(path) \description{ Initiate a filesystem cache. } +\examples{ + +\dontrun{ +# Use with Dropbox + +db <- cache_filesystem("~/Dropbox/.rcache") + +mem_runif <- memoise(runif, cache = db) + +# Use with Google Drive + +gd <- cache_filesystem("~/Google Drive/.rcache") + +mem_runif <- memoise(runif, cache = gd) + +} + +} diff --git a/man/cache_local.Rd b/man/cache_local.Rd index 78e855c..ff0da3a 100644 --- a/man/cache_local.Rd +++ b/man/cache_local.Rd @@ -9,6 +9,6 @@ cache_local() \description{ Initiate an in memory cache. -cache_local() stores cached items in memory. +cache_local() stores cached items in memory. It is the default cache. } From f60558d761b2627c242a24542b36be98b18a64de Mon Sep 17 00:00:00 2001 From: Daniel E Cook Date: Thu, 28 Jul 2016 19:26:18 -0500 Subject: [PATCH 26/29] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 523ff98..79fc05c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# xmemoise +# memoise Forked from [hadley/memoise](https://github.com/hadley/memoise) From f6ddf99c985984aba17b627b6524271c25733d8e Mon Sep 17 00:00:00 2001 From: Daniel E Cook Date: Sun, 31 Jul 2016 13:29:05 -0500 Subject: [PATCH 27/29] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 79fc05c..8a6c42a 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Forked from [hadley/memoise](https://github.com/hadley/memoise) # Installation ``` -devtools::install_github("danielecook/xmemoise") +devtools::install_github("danielecook/memoise") ``` # Memoization From d86494d3831f3d37e57fb0a13fce0556f7114c47 Mon Sep 17 00:00:00 2001 From: Daniel E Cook Date: Sun, 31 Jul 2016 13:31:42 -0500 Subject: [PATCH 28/29] Update README.md --- README.md | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 8a6c42a..fd5c367 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ devtools::install_github("danielecook/memoise") If a function is called multiple times with the same input, you can often speed things up by keeping a cache of known answers that it can retrieve. This is called memoisation . -The `xmemoise` package is built upon [hadley/memoise](https://github.com/hadley/memoise), which provides a simple syntax +This is a fork of the `memoise` package built by Hadley Wickham: [hadley/memoise](https://github.com/hadley/memoise), which provides a simple syntax mf <- memoise(f) @@ -28,18 +28,12 @@ cache with is.memoised(f) # FALSE -`xmemoise` extends upon `memoise` by adding in additional types of caches. Items can be cached using the original cache implemented in `memoise` in addition to other options: +`memoise` extends upon `memoise` by adding in additional types of caches. Items can be cached using the original cache implemented in `memoise` in addition to other options: * [x] Google Datastore -* [ ] Dropbox -* [ ] Google Storage +* [x] cache_filesystem allows caching using dropbox/google drive. * [X] AWS -### To Do - -* [x] Switch to using GoogleAuthR - - # Caches ## Google Datastore @@ -71,3 +65,7 @@ mrunif(10) # First run, saves cache mrunif(10) # Loads cache, results should be identical ``` + +## Filesystem + +`cache_filesystem` From baed71c8836e8ae19d2080aae14215d4612800dd Mon Sep 17 00:00:00 2001 From: Daniel E Cook Date: Sun, 31 Jul 2016 13:36:28 -0500 Subject: [PATCH 29/29] Update README.md --- README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fd5c367..0219c36 100644 --- a/README.md +++ b/README.md @@ -68,4 +68,18 @@ mrunif(10) # Loads cache, results should be identical ## Filesystem -`cache_filesystem` +`cache_filesystem` can be used to cache between computers using Google Drive or Dropbox. + +``` +dbc <- cache_filesystem("~/Dropbox/.rcache") +mrunif <- memoise(runif, cache = dbc) +mrunif(20) # Results stored in Dropbox .rcache folder will be synced between computers. +``` + +``` +gdc <- cache_filesystem("~/Google Drive/.rcache") +mrunif <- memoise(runif, cache = dbc) +mrunif(20) # Results stored in Google Drive .rcache folder will be synced between computers. +``` + +