setting strategy to mimic rmpi/dompi

[yonicd]

is there an analogous setting in future::plan for rmpi?

library(parallel) 
hello_world <- function() { 
## Print the hostname and MPI worker rank. 
paste(Sys.info()["nodename"],Rmpi::mpi.comm.rank(), sep = ":") 
} 

cl <- makeCluster(Sys.getenv()["SLURM_NTASKS"], type = "MPI") 
clusterCall(cl, hello_world) 
stopCluster(cl)

where the plan is set on the workers and the call to slurm/grid is done via bash template.

[yonicd]

I think this is the equiv

cl <- parallel::makeCluster(Sys.getenv()["SLURM_NTASKS"], type = "MPI") 

future::plan(future::cluster, workers = cl)

furrr::future_map(1:4,.f=function(i){
paste(i,Sys.info()["nodename"], sep = ":") 
})

parallel::stopCluster(cl)

not sure how to mimic Rmpi::mpi.comm.rank()

[yonicd]

I have this version of multiprocess/sge/mpi working, which somewhat answers my initial query. Is there any way to float up the cl <- parallel::makeCluster(2, type = "MPI") into the plan step?

the sge template is one that is referenced in issue #39 (comment)

library(future)
library(batchtools)
library(future.batchtools)

sge <- future::tweak(
 future.batchtools::batchtools_sge,
 label = 'test',
 template = 'batchtools.sge-mrg.tmpl',
 resources = list(slots = 4)
)

future::plan(list(future::multiprocess,sge))

f <- future::future({
 future.apply::future_lapply(1:4,FUN=function(i){
 
 hello_world <- function() {
 ## Print the hostname and MPI worker rank.
 paste(i,Sys.info()["nodename"],Rmpi::mpi.comm.rank(), sep = ":")
 }

 cl <- parallel::makeCluster(2, type = "MPI")
 on.exit(parallel::stopCluster(cl),add = TRUE)

 parallel::clusterCall(cl, hello_world)
 
 
 })
})

future::value(f)

# 2 slaves are spawned successfully. 0 failed.
# 2 slaves are spawned successfully. 0 failed.
# 2 slaves are spawned successfully. 0 failed.
# 2 slaves are spawned successfully. 0 failed.
# Loading required namespace: Rmpi
# Loading required namespace: Rmpi
# Loading required namespace: Rmpi
# Loading required namespace: Rmpi
# [[1]]
# [[1]][[1]]
# [1] "1:ip-172-16-2-22:1"
# 
# [[1]][[2]]
# [1] "1:ip-172-16-2-22:2"
# 
# 
# [[2]]
# [[2]][[1]]
# [1] "2:ip-172-16-2-129:1"
# 
# [[2]][[2]]
# [1] "2:ip-172-16-2-129:2"
# 
# 
# [[3]]
# [[3]][[1]]
# [1] "3:ip-172-16-2-39:1"
# 
# [[3]][[2]]
# [1] "3:ip-172-16-2-39:2"
# 
# 
# [[4]]
# [[4]][[1]]
# [1] "4:ip-172-16-2-22:1"
# 
# [[4]][[2]]
# [1] "4:ip-172-16-2-22:2"