Skip to content

Commit 364c134

Browse files
rhijmansrhijmans
committed
mem mngmt
1 parent 5e6b782 commit 364c134

File tree

5 files changed

+61
-56
lines changed

5 files changed

+61
-56
lines changed

‎ChangeLog

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ Fix lines falling between the cracks in extract (calling rasterzeLines): https:/
99
Better handling of NA values in 1 byte raster files
1010

1111
--- current version
12-
Improved estimation of available RAM with contributions by Lorenzo Busetto. The current situation is that RAM available is computed on windows and linux; but not yet on mac. Not more that 60% of available RAM is used (if the estimated RAM needed is not too low). Not more than raster:::.maxmemory() is used; but it should now be safe to set this to Inf (windows and linux only). If canProcessInMemory returns FALSE, chunksize is set 25% of available RAM, or raster:::.chunksize(), whichever is lower. (set options via rasterOptions())
12+
Improved estimation of available RAM with contributions by Lorenzo Busetto. RAM available is now computed on windows, linux and mac. No more that 60% of available RAM is used (if the estimated RAM needed is not too low). Not more than raster:::.maxmemory() is used; but it should now be safe to set this to Inf. If canProcessInMemory() returns FALSE, chunksize is set 25% of available RAM, or raster:::.chunksize(), whichever is lower. (set options via rasterOptions())
1313

1414
testthat unit tests introduced by Mike Sumner and Jakub Nowosad
1515
as.character() for Raster objects to create the R code to re-create the Raster skeleton in examples.

‎R/canProcessInMemory.R

Lines changed: 33 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -4,28 +4,29 @@
44
# Licence GPL v3
55

66

7-
.RAMavailable <- function(defmem=.maxmemory(), useC=TRUE) {
8-
if (useC) {
9-
memavail <- .availableRAM(defmem)
10-
} else {
7+
#.RAMavailable <- function(defmem=.maxmemory()) {
8+
#
9+
# if (useC) {
10+
# .availableRAM(defmem)
11+
# } else {
1112
# essentially the same results as above, but slower
1213

13-
if ( .Platform$OS.type == "windows" ) {
14-
mem <- system2("wmic", args = "OS get FreePhysicalMemory /Value", stdout = TRUE)
15-
mem3 <- gsub("\r", "", mem[3])
16-
mem3 <- gsub("FreePhysicalMemory=", "", mem3)
17-
memavail <- as.numeric(mem3) * 1024
14+
# if ( .Platform$OS.type == "windows" ) {
15+
# mem <- system2("wmic", args = "OS get FreePhysicalMemory /Value", stdout = TRUE)
16+
# mem3 <- gsub("\r", "", mem[3])
17+
# mem3 <- gsub("FreePhysicalMemory=", "", mem3)
18+
# memavail <- as.numeric(mem3) * 1024
1819
#memavail <- 0.5 * (utils::memory.size(NA) - utils::memory.size(FALSE))
19-
} else if ( .Platform$OS.type == "unix" ) {
20-
memavail <- as.numeric(system("awk '/MemFree/ {print $2}' /proc/meminfo", intern=TRUE))
21-
} else {
22-
#don't know how to do it for mac
23-
memavail <- defmem
24-
}
25-
}
26-
memavail
27-
28-
}
20+
# } else if ( .Platform$OS.type == "unix" ) {
21+
# mac is also "unix" and this does not work on mac
22+
# memavail <- as.numeric(system("awk '/MemFree/ {print $2}' /proc/meminfo", intern=TRUE))
23+
# } else {
24+
#don't know how to do this on a mac
25+
# memavail <- defmem
26+
# }
27+
# }
28+
# memavail
29+
#}
2930

3031

3132
canProcessInMemory <- function(x, n=4, verbose=FALSE) {
@@ -45,23 +46,27 @@ canProcessInMemory <- function(x, n=4, verbose=FALSE) {
4546

4647
n <- n * nlayers(x)
4748
memneed <- ncell(x) * n * 8
48-
4949
maxmem <- .maxmemory()
50-
memavail <- .RAMavailable(maxmem, TRUE)
50+
memavail <- .availableRAM(maxmem)
5151
if (verbose) {
52-
print(paste("mem available:", memavail))
53-
print(paste("mem needed:", memneed))
52+
gb <- 1073741824
53+
cat("memory stats in GB")
54+
cat(paste("\nmem available:", round(memavail / gb, 2)))
55+
cat(paste0("\n ", round(100*.memfrac()) , "% : ", round(.memfrac() * memavail / gb, 2)))
56+
cat(paste("\nmem needed :", round(memneed / gb, 2)))
57+
cat(paste("\nmax allowed :", round(maxmem / gb, 2), " (if available)\n"))
5458
}
55-
# the below should not be needed, but
56-
# this allows you to safely set a high maxmem
59+
60+
# can't use all of it; default is 60%
61+
memavail <- .memfrac() * memavail
62+
63+
# the below allows you to safely set a high maxmem
5764
# but still limit total mem use
5865
memavail <- min(memavail, maxmem)
5966

60-
# can't use all of it; the 0.6 could be an option
61-
memavail <- 0.6 * memavail
62-
6367
if (memneed > memavail) {
6468
# new (hidden) option; the 0.25 could be another option
69+
# now you can only make it lower via chunksize
6570
options(rasterChunk = min(.chunksize(), memavail * 0.25))
6671
return(FALSE)
6772
} else {

‎R/rasterOptions.R

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
# Licence GPL v3
55

66

7-
rasterOptions <- function(format, overwrite, datatype, tmpdir, tmptime, progress, timer, chunksize, maxmemory, todisk, setfileext, tolerance, standardnames, depracatedwarnings, addheader, default=FALSE) {
7+
rasterOptions <- function(format, overwrite, datatype, tmpdir, tmptime, progress, timer, chunksize, maxmemory, memfrac, todisk, setfileext, tolerance, standardnames, depracatedwarnings, addheader, default=FALSE) {
88

99
setFiletype <- function(format) {
1010
if (.isSupportedFormat(format)) {
@@ -109,13 +109,13 @@ rasterOptions <- function(format, overwrite, datatype, tmpdir, tmptime, progress
109109
options(rasterMaxMemory = maxmemory )
110110
}
111111

112-
# setEstimateMem <- function(estimatemem) {
113-
# if (is.logical(estimatemem)) {
114-
# options(rasterEstimateMem = estimatemem )
115-
# } else {
116-
# warning(paste('estimateMem argument must be a logical value'))
117-
# }
118-
# }
112+
setMemfrac <- function(memfrac) {
113+
if (memfrac >= 0.1 & memfrac <= 0.9) {
114+
options(rasterMemfrac = memfrac )
115+
} else {
116+
warning(paste('memfrac argument must be a value between 0.1 and 0.9'))
117+
}
118+
}
119119

120120

121121
setTolerance <- function(x) {
@@ -169,7 +169,7 @@ rasterOptions <- function(format, overwrite, datatype, tmpdir, tmptime, progress
169169
options(rasterChunkSize = 10^8)
170170
options(rasterChunk = 10^8)
171171
options(rasterMaxMemory = 10^9)
172-
# options(rasterEstimateMem = TRUE)
172+
options(rasterMemfrac = 0.6)
173173
options(rasterTolerance = 0.1)
174174
options(rasterStandardNames = TRUE)
175175
options(rasterDepracatedWarnings = TRUE)
@@ -190,7 +190,7 @@ rasterOptions <- function(format, overwrite, datatype, tmpdir, tmptime, progress
190190
if (!missing(todisk)) { setToDisk(todisk); cnt <- cnt+1 }
191191
if (!missing(setfileext)) { setFileExt(setfileext); cnt <- cnt+1 }
192192
if (!missing(maxmemory)) { setMaxMemorySize(maxmemory); cnt <- cnt+1 }
193-
# if (!missing(estimatemem)) { setEstimateMem(estimatemem); cnt <- cnt+1 }
193+
if (!missing(memfrac)) { setMemfrac(memfrac); cnt <- cnt+1 }
194194
if (!missing(chunksize)) { setChunksize(chunksize); cnt <- cnt+1 }
195195
if (!missing(tolerance)) { setTolerance(tolerance); cnt <- cnt+1 }
196196
if (!missing(standardnames)) { setStandardNames(standardnames); cnt <- cnt+1 }
@@ -208,13 +208,13 @@ rasterOptions <- function(format, overwrite, datatype, tmpdir, tmptime, progress
208208
timer=.timer(),
209209
chunksize=.chunksize(),
210210
maxmemory=.maxmemory(),
211+
memfrac = .memfrac(),
211212
todisk=.toDisk(),
212213
setfileext=.setfileext(),
213214
tolerance=.tolerance(),
214215
standardnames=.standardnames(),
215216
depwarning=.depracatedwarnings(),
216217
addheader=.addHeader()
217-
# estimatemem = .estimateMem()
218218
)
219219

220220
save <- FALSE
@@ -232,7 +232,7 @@ rasterOptions <- function(format, overwrite, datatype, tmpdir, tmptime, progress
232232
oplst <- c(oplst, paste("rasterTimer=", lst$timer, sep=''))
233233
oplst <- c(oplst, paste("rasterChunkSize=", lst$chunksize, sep=''))
234234
oplst <- c(oplst, paste("rasterMaxMemory=", lst$maxmemory, sep=''))
235-
# oplst <- c(oplst, paste("rasterEstimateMem=", lst$estimatemem, sep=''))
235+
oplst <- c(oplst, paste("rasterMemfrac=", lst$memfrac, sep=''))
236236
oplst <- c(oplst, paste("rasterSetFileExt=", lst$setfileext, sep=''))
237237
oplst <- c(oplst, paste("rasterTolerance=", lst$tolerance, sep=''))
238238
oplst <- c(oplst, paste("rasterStandardNames=", lst$standardnames, sep=''))
@@ -253,7 +253,7 @@ rasterOptions <- function(format, overwrite, datatype, tmpdir, tmptime, progress
253253
cat('timer :', lst$timer, '\n')
254254
cat('chunksize :', lst$chunksize, '\n')
255255
cat('maxmemory :', lst$maxmemory, '\n')
256-
# cat('estimatemem :', lst$estimatemem, '\n')
256+
cat('memfrac :', lst$memfrac, '\n')
257257
cat('tmpdir :', lst$tmpdir, '\n')
258258
cat('tmptime :', lst$tmptime, '\n')
259259
cat('setfileext :', lst$setfileext, '\n')
@@ -365,15 +365,15 @@ tmpDir <- function(create=TRUE) {
365365
}
366366

367367

368-
#.estimateMem <- function() {
369-
# default <- FALSE
370-
# d <- getOption('rasterEstimateMem')
371-
# if (is.null(d)) {
372-
# return( default )
373-
# } else {
374-
# return(isTRUE(d))
375-
# }
376-
#}
368+
.memfrac <- function() {
369+
default <- 0.6
370+
d <- getOption('rasterMemfrac')
371+
if (is.null(d)) {
372+
return( default )
373+
} else {
374+
return(d)
375+
}
376+
}
377377

378378

379379
.maxmemory <- function() {

‎man/rasterOptions.Rd

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ Function \code{tmpDir} returns the location of the temporary files
2020

2121
\usage{
2222
rasterOptions(format, overwrite, datatype, tmpdir, tmptime, progress,
23-
timer, chunksize, maxmemory, todisk, setfileext, tolerance,
23+
timer, chunksize, maxmemory, memfrac, todisk, setfileext, tolerance,
2424
standardnames, depracatedwarnings, addheader, default=FALSE)
2525

2626

@@ -37,7 +37,8 @@ tmpDir(create=TRUE)
3737
\item{progress}{character. Valid values are "text", "window" and "" (the default in most functions, no progress bar)}
3838
\item{timer}{Logical. If \code{TRUE}, the time it took to complete the function is printed}
3939
\item{chunksize}{integer. Maximum number of bytes to read/write in a single chunk while processing (chunk by chunk) disk based Raster* objects}
40-
\item{maxmemory}{integer. Maximum number of bytes to read into memory. I.e., if a Raster* function is expected to require more than this value, \code{\link[raster]{canProcessInMemory}} will return \code{FALSE} }
40+
\item{maxmemory}{numeric. Maximum number of bytes to read into memory. If a process is expected to require more than this value, \code{\link[raster]{canProcessInMemory}} will return \code{FALSE} }
41+
\item{memfrac}{numeric. Fraction of available RAM that may be used by a process}
4142
\item{todisk}{logical. For debugging only. Default is \code{FALSE} and should normally not be changed. If \code{TRUE}, results are always written to disk, even if no filename is supplied (a temporary filename is used)}
4243
\item{setfileext}{logical. Default is \code{TRUE}. If \code{TRUE}, the file extension will be changed when writing (if known for the file type). E.g. GTiff files will be saved with the .tif extension }
4344
\item{tolerance}{numeric. The tolerance used when comparing the origin and resolution of Raster* objects. Expressed as the fraction of a single cell. This should be a number between 0 and 0.5 }

‎src/memory.cpp

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ double availableRAM(double ram) {
2323
struct sysinfo memInfo;
2424
sysinfo (&memInfo);
2525
ram = memInfo.freeram;
26-
#else
26+
#elif __APPLE__
2727
vm_size_t page_size;
2828
mach_port_t mach_port;
2929
mach_msg_type_number_t count;
@@ -40,9 +40,8 @@ double availableRAM(double ram) {
4040
// (int64_t)vm_stats.inactive_count +
4141
// (int64_t)vm_stats.wire_count) * (int64_t)page_size;
4242
ram = free_memory;
43-
}
44-
// mac
4543
//https://stackoverflow.com/questions/63166/how-to-determine-cpu-and-memory-consumption-from-inside-a-process
44+
}
4645
#endif
4746

4847
return ram;

0 commit comments

Comments
 (0)