Cache engine for reticulate using dill

NAMESPACE

@@ -106,6 +106,7 @@ export("%as%")
 export(PyClass)
 export(array_reshape)
 export(as_iterator)
+export(cache_eng_python)
 export(conda_binary)
 export(conda_clone)
 export(conda_create)

R/knitr-engine.R

@@ -244,6 +244,10 @@ eng_python <- function(options) {
     outputs$push(output)
   }
 
+  if(options$cache > 0) {
+    save_python_session(options$hash)
+  }
+
   # if we had held outputs, add those in now (merging text output as appropriate)
   text_output <- character()
 
@@ -292,6 +296,16 @@ eng_python_initialize <- function(options, envir) {
   ensure_python_initialized()
   eng_python_initialize_hooks(options, envir)
 
+  if (options$cache > 0) {
+    module <- tryCatch(import("dill"), error = identity)
+    if (inherits(module, "error")) {
+      if (module$message == "ImportError: No module named dill") {
+        warning("The Python module dill was not found. This module is needed for full cache functionality.")
+      } else {
+        stop(module$message)
+      }
+    }
+  } 
 }
 
 eng_python_knit_figure_path <- function(options, suffix = NULL) {
@@ -628,3 +642,54 @@ eng_python_autoprint <- function(captured, options, autoshow) {
   }
 
 }
+
+save_python_session <- function(cache_path) {
+  module <- tryCatch(import("dill"), error = identity)
+  if (inherits(module, "error")) {
+    if (module$message == "ImportError: No module named dill") return()
+    signalCondition(module$message)
+  }
+
+  r_obj_exists <- "'r' in globals()"
+  r_is_R <- "type(r).__module__ == '__main__' and type(r).__name__ == 'R'"
+  if (py_eval(r_obj_exists) && py_eval(r_is_R)) {
+    py_run_string("del globals()['r']")
+  }
+
+  cache_path <- file.path(knitr::opts_knit$get("output.dir"), cache_path)
+  module$dump_session(filename = paste0(cache_path, ".pkl"), byref = TRUE)
+}
+
+#' A reticulate cache engine for Knitr
+#' 
+#' This provides a `reticulate` cache engine for `knitr`. The cache engine
+#' allows `knitr` to save and load Python sessions between cached chunks. The
+#' cache engine depends on the `dill` Python module. Therefore, you must have
+#' `dill` installed in your Python environment.
+#' 
+#' The engine can be activated by setting (for example)
+#' 
+#' ```
+#' knitr::cache_engines$set(python = reticulate::cache_eng_python)
+#' ```
+#' 
+#' Typically, this will be set within a document's setup chunk, or by the
+#' environment requesting that Python chunks be processed by this engine.
+#' 
+#' @param options
+#'   List of chunk options provided by `knitr` during chunk execution. 
+#'   Contains the caching path.
+#'   
+#' @export
+cache_eng_python <- function(options) {
+  module <- tryCatch(import("dill"), error = identity)
+  if (inherits(module, "error")) {
+    if (module$message == "ImportError: No module named dill") return()
+    stop(module$message)
+  }
+
+  cache_path <- normalizePath(paste0(options$hash, ".pkl"), mustWork = TRUE)
+  knitr:::in_input_dir(module$load_session(filename = cache_path))
+}
+
+

R/python.R

@@ -88,6 +88,66 @@ summary.python.builtin.object <- function(object, ...) {
   str(object)
 }
 
+
+#' Convert between Python and R objects
+#' 
+#' @inheritParams import
+#' @param x Object to convert
+#' 
+#' @return Converted object
+#' 
+#' @name r-py-conversion
+#' @export
+py_to_r <- function(x) {
+
+  ensure_python_initialized()
+
+  if (!inherits(x, "python.builtin.object"))
+    stop("Object to convert is not a Python object")
+
+  # get the default wrapper
+  x <- py_ref_to_r(x)
+
+  # allow customization of the wrapper
+  wrapper <- py_to_r_wrapper(x)
+  attributes(wrapper) <- attributes(x)
+
+  # return the wrapper
+  wrapper 
+}
+
+#' R wrapper for Python objects 
+#' 
+#' S3 method to create a custom R wrapper for a Python object.
+#' The default wrapper is either an R environment or an R function
+#' (for callable python objects).
+#' 
+#' @param x Python object 
+#' 
+#' @export
+py_to_r_wrapper <- function(x) {
+  UseMethod("py_to_r_wrapper")
+}
+
+#' @export
+py_to_r_wrapper.default <- function(x) {
+  x
+}
+
+
+
+
+
+#' @rdname r-py-conversion
+#' @export
+r_to_py <- function(x, convert = FALSE) {
+
+  ensure_python_initialized()
+
+  r_to_py_impl(x, convert = convert)
+}
+
+
 #' @export
 `$.python.builtin.module` <- function(x, name) {
 

tests/testthat/resources/eng-reticulate-cache-test.Rmd

@@ -0,0 +1,20 @@
+---
+title: "Using reticulate's Python Engine with knitr"
+---
+
+```{r setup, include = FALSE}
+library(reticulate)
+knitr::opts_chunk$set(cache=TRUE)
+```
+
+Cache can handle changes to second chunk:
+
+```{python}
+x = 1
+```
+
+```{python}
+print(x + 1)
+```
+
+

tests/testthat/test-python-cache-engine.R

@@ -0,0 +1,46 @@
+context("knitr-cache")
+
+test_that("An R Markdown document can be rendered with cache using reticulate", {
+
+  skip_on_cran()
+  skip_if_not_installed("rmarkdown")
+  skip_if_not_installed("callr")
+
+  unlink("resources/eng-reticulate-cache-test_cache/", recursive = TRUE)
+
+  path <- callr::r(
+    function() {
+      rmarkdown::render("resources/eng-reticulate-cache-test.Rmd", quiet = TRUE, envir = new.env())
+    })
+  expect_true(file.exists(path))
+  on.exit(unlink(path), add = TRUE)
+})
+
+test_that("An R Markdown document builds if a cache is modified", {
+
+  skip_on_cran()
+  skip_if_not_installed("rmarkdown")
+  skip_if_not_installed("callr")
+
+  old_var <- "1"
+  new_var <- "0"
+  mutate_chunk <- function(x) {
+    print_line <- 17
+    file_text <- readLines("resources/eng-reticulate-cache-test.Rmd")
+    file_text[print_line] <- paste0("print(x + ", x, ")")
+    writeLines(file_text, "resources/eng-reticulate-cache-test.Rmd")
+  }
+  mutate_chunk(old_var)
+  mutate_chunk(new_var)
+  path <- callr::r(
+    function() {
+      rmarkdown::render("resources/eng-reticulate-cache-test.Rmd", quiet = TRUE, envir = new.env())
+    })
+  mutate_chunk(old_var)
+  expect_true(file.exists(path))
+  on.exit(unlink(path), add = TRUE)
+  on.exit(unlink("resources/eng-reticulate-cache-test_cache/", recursive = TRUE), add = TRUE)
+})
+
+
+

tests/testthat/test-python-dill.R

@@ -0,0 +1,37 @@
+context("dill")
+
+source("helper-utils.R")
+
+test_that("Interpreter sessions can be saved and loaded with dill", {
+  skip_if_no_python()
+  skip_if_not_installed("callr")
+
+  session_one_vars <- callr::r(
+    function() {
+      module_load <- tryCatch(
+        dill <- reticulate::import("dill"),
+        error = function(c) {
+          py_error <- reticulate::py_last_error()
+          if(py_error$type == "ImportError" && py_error$value == "No module named dill") {
+            "No dill"
+          }})
+      if (module_load == "No dill") return(module_load)
+      main <- reticulate::py_run_string("x = 1")
+      reticulate::py_run_string("y = x + 1")
+      dill$dump_session(filename = "x.dill", byref = TRUE)
+      c(main$x, main$y)
+    })
+  if (session_one_vars[1] == "No dill")
+    skip("The dill Python module is not installed")
+
+  session_two_vars <- callr::r(
+    function() {
+      dill <- reticulate::import("dill")
+      dill$load_session(filename = "x.dill")
+      main <- reticulate::py_run_string("pass")
+      c(main$x, main$y)
+    })
+  on.exit(unlink("x.dill"), add = TRUE)
+  expect_equal(session_one_vars, session_two_vars)
+})
+

tests/testthat/test-python-globals.R

@@ -0,0 +1,27 @@
+context("globals")
+
+source("utils.R")
+
+test_that("Interpreter sessions can be saved and loaded with dill", {
+  skip_if_no_python()
+
+  py_run_string("x = 1")
+  py_run_string("y = 1")
+  py_run_string("[globals().pop(i) for i in ['x', 'y']]")
+
+  test_x <- tryCatch(
+    py_run_string("x = x + 1"),
+    error = function(e) {
+      py_last_error()$value
+    }
+  )
+  test_y <- tryCatch(
+    py_run_string("y = y + 1"),
+    error = function(e) {
+      py_last_error()$value
+    }
+  )
+  expect_equal(test_x, "name 'x' is not defined")
+  expect_equal(test_y, "name 'y' is not defined")
+})
+