Introduction

Here is an explanation of the different objects:

  1. GSA: gres

  2. LAT: lacres

  3. LTT: lactopres

  4. LAW: nlacres

  5. LAM: lacmres

  6. LRM: lrmres

  7. ALT: alcres

  8. ALW: nalcres

  9. ALM: almres

suppressPackageStartupMessages({
  library("stringi")
  library("limma")
  library("missMethyl")
  library("IlluminaHumanMethylation450kmanifest")
  library("IlluminaHumanMethylationEPICanno.ilm10b4.hg19")
  library('org.Hs.eg.db')
  library("psych")
  library("mitch")
  library("kableExtra")
})

Functions

F1 <- function(x,y) {
  ( 2 * x * y ) / ( x + y )
}

summarise <- function(x) {
  PREC <- sum(x[,"TP"]) / ( sum(x[,"TP"]) + sum(x[,"FP"]) )
  REC <- sum(x[,"TP"]) / ( sum(x[,"TP"]) + sum(x[,"FN"]) )
  F <- F1(PREC,REC)
  return(c("PRECISION"=PREC,"RECALL"=REC,"F1"=F))
}

Overall summary

Make a set of three barplots which summarise method performance over the range of group size and delta values, for gene set sizes of 20, 50 and 200.

mar = c(5.1, 4.1, 5.1, 2.1)

load("GSE158422_simulate020.Rdata")

l1 <- list("GSA"=gres2,"LAT"=lacres2,"LTT"=lactopres2,"LAW"=nlacres2,
  "LAM"=lacmres2, "LRM"=lrmres2, "ALT"=alcres2,"ALW"=nalcres2,"ALM"=almres2)

x1 <- do.call(rbind,lapply(l1,summarise))

x1 %>% kbl(caption="sim 20 res") %>% kable_paper("hover", full_width = F)
sim 20 res
PRECISION RECALL F1
GSA 0.9713446 0.06610 0.1237770
LAT 0.9473684 0.00090 0.0017983
LTT 0.2995122 0.01535 0.0292033
LAW 0.9596977 0.01905 0.0373584
LAM 0.9601990 0.01930 0.0378394
LRM 0.9323308 0.00620 0.0123181
ALT 0.5000000 0.00070 0.0013980
ALW 0.9287671 0.01695 0.0332924
ALM 0.9327957 0.01735 0.0340664 
cols <- c("white","gray","black")

barplot(t(x1),beside=TRUE,ylim=c(0,1),ylab="index", col=cols, cex.names=0.7, cex.axis=0.7, main="20 genes per set")
grid()

legend("topleft", inset=.1, c("Precision","Recall","F1"), fill=cols, cex=0.7)

load("GSE158422_simulate050.Rdata")

l2 <- list("GSA"=gres2,"LAT"=lacres2,"LTT"=lactopres2,"LAW"=nlacres2,
  "LAM"=lacmres2, "LRM"=lrmres2, "ALT"=alcres2,"ALW"=nalcres2,"ALM"=almres2)

x2 <- do.call(rbind,lapply(l2,summarise))

x2 %>% kbl(caption="sim 50 res") %>% kable_paper("hover", full_width = F)
sim 50 res
PRECISION RECALL F1
GSA 0.9360848 0.16330 0.2780876
LAT 0.9363280 0.10735 0.1926165
LTT 0.9126838 0.14895 0.2561039
LAW 0.8966298 0.17825 0.2973807
LAM 0.9349553 0.18830 0.3134676
LRM 0.9217527 0.08835 0.1612447
ALT 0.9384929 0.11520 0.2052104
ALW 0.9035656 0.16725 0.2822547
ALM 0.9388135 0.17645 0.2970664 
barplot(t(x2),beside=TRUE,ylim=c(0,1),ylab="index", col=cols, cex.names=0.7, cex.axis=0.7, main="50 genes per set")
grid()

load("GSE158422_simulate100.Rdata")

l3 <- list("GSA"=gres2,"LAT"=lacres2,"LTT"=lactopres2,"LAW"=nlacres2,
  "LAM"=lacmres2, "LRM"=lrmres2, "ALT"=alcres2,"ALW"=nalcres2,"ALM"=almres2)

x3 <- do.call(rbind,lapply(l3,summarise))

x3 %>% kbl(caption="sim 100 res") %>% kable_paper("hover", full_width = F)
sim 100 res
PRECISION RECALL F1
GSA 0.7688263 0.22808 0.3517962
LAT 0.8807878 0.37208 0.5231574
LTT 0.9160032 0.36816 0.5252226
LAW 0.8385550 0.40576 0.5468906
LAM 0.9353216 0.45726 0.6142335
LRM 0.9375987 0.32064 0.4778611
ALT 0.8682377 0.40380 0.5512327
ALW 0.8407552 0.39724 0.5395523
ALM 0.9341152 0.44604 0.6037766 
barplot(t(x3),beside=TRUE,ylim=c(0,1),ylab="index", col=cols, cex.names=0.7, cex.axis=0.7, main="100 genes per set")
grid()

Make a PNG copy of the data.

png(filename="fig1.png",height=70,width=210,units="mm",res=200)

par(mar = c(2.5, 4.1, 2.1, 0.5))

par(mfrow=c(1,3))

cols <- c("white","gray","black")

barplot(t(x1),beside=TRUE,ylim=c(0,1),ylab="index", col=cols, cex.names=0.75, cex.axis=0.75, main="20 genes per set")
grid()

legend("topleft", inset=.1, c("Precision","Recall","F1"), fill=cols, cex=0.75)

barplot(t(x2),beside=TRUE,ylim=c(0,1),ylab="index", col=cols, cex.names=0.75, cex.axis=0.75, main="50 genes per set")
grid()

barplot(t(x3),beside=TRUE,ylim=c(0,1),ylab="index", col=cols, cex.names=0.75, cex.axis=0.75, main="100 genes per set")
grid()

dev.off()
## png 
##   2
par(mfrow=c(1,1))

In depth analysis of GSA vs LAM

As GSA appears to be the best-performing existing method for enrichment analysis, it is a good idea to compare it to LAM which is the best one out of the new methods evaluated here. The following charts show the performance of GSA and LAM including precision and recall over the range of group sizes and delta values when the sets contain 20, 50 and 100 genes each.

# 20
load("GSE158422_simulate020.Rdata")

par(mfrow=c(2,3))

gres2[,"PREC"] <- gres2[,"TP"] / ( gres2[,"TP"] + gres2[,"FP"] )
gres3p <- do.call(rbind,lapply(groupsizes, function (g) { gres2[params$groupsizes==g,"PREC"] }))
colnames(gres3p) <- deltas
rownames(gres3p) <- groupsizes
gres3p %>% kbl(caption="GSAmeth precision") %>% kable_paper("hover", full_width = F)
GSAmeth precision
0.1 0.2 0.3 0.4 0.5
3 NaN NaN 1.0000000 1.0000000 0.8000000
5 NaN NaN 0.7500000 0.8750000 0.9230769
8 NaN NaN 1.0000000 1.0000000 0.9691358
12 NaN 1 0.9285714 0.9768638 0.9729207 
gres3r <- do.call(rbind,lapply(groupsizes, function (g) { gres2[params$groupsizes==g,"REC"] }))
colnames(gres3r) <- deltas
rownames(gres3r) <- groupsizes
gres3r %>% kbl(caption="GSAmeth recall") %>% kable_paper("hover", full_width = F)
GSAmeth recall
0.1 0.2 0.3 0.4 0.5
3 0 0.00 0.001 0.002 0.004
5 0 0.00 0.003 0.007 0.012
8 0 0.00 0.009 0.051 0.314
12 0 0.01 0.026 0.380 0.503 
gres3f <- F1(gres3p,gres3r)
gres3f %>% kbl(caption="GSAmeth F1") %>% kable_paper("hover", full_width = F)
GSAmeth F1
0.1 0.2 0.3 0.4 0.5
3 NaN NaN 0.0019980 0.0039920 0.0079602
5 NaN NaN 0.0059761 0.0138889 0.0236920
8 NaN NaN 0.0178394 0.0970504 0.4743202
12 NaN 0.019802 0.0505837 0.5471562 0.6631510 
plot(rownames(gres3p),gres3p[,"0.5"],type="b",las=1,ylab="precision",xlab="",
  main="",ylim=c(0,1))
lines(rownames(gres3p),gres3p[,"0.4"],type="b",col="red")
lines(rownames(gres3p),gres3p[,"0.3"],type="b",col="blue")
lines(rownames(gres3p),gres3p[,"0.2"],type="b",col="darkgreen")
grid()

legend("bottomright", legend=c("0.5", "0.4", "0.3", "0.2"),title="delta",
  lty=1, lwd=2, col=c("black", "red", "blue", "darkgreen"), cex=1)

plot(rownames(gres3r),gres3r[,"0.5"],type="b",las=1,ylab="recall",xlab="group size",
  main="GSA",ylim=c(0,1))
lines(rownames(gres3r),gres3r[,"0.4"],type="b",col="red")
lines(rownames(gres3r),gres3r[,"0.3"],type="b",col="blue")
lines(rownames(gres3r),gres3r[,"0.2"],type="b",col="darkgreen")
grid()

plot(rownames(gres3r),gres3f[,"0.5"],type="b",las=1,ylab="F1",xlab="",
  main="",ylim=c(0,1))
lines(rownames(gres3r),gres3f[,"0.4"],type="b",col="red")
lines(rownames(gres3r),gres3f[,"0.3"],type="b",col="blue")
lines(rownames(gres3r),gres3f[,"0.2"],type="b",col="darkgreen")
grid()

lacmres2[,"PREC"] <- lacmres2[,"TP"] / ( lacmres2[,"TP"] + lacmres2[,"FP"] )
lacmres3p <- do.call(rbind,lapply(groupsizes, function (g) { lacmres2[params$groupsizes==g,"PREC"] }))
colnames(lacmres3p) <- deltas
rownames(lacmres3p) <- groupsizes
lacmres3p %>% kbl(caption="LAM precision") %>% kable_paper("hover", full_width = F)
LAM precision
0.1 0.2 0.3 0.4 0.5
3 1 1.0000000 1.0000000 1.0000000 1.0000000
5 NaN 1.0000000 1.0000000 1.0000000 1.0000000
8 0 0.8750000 0.9583333 0.9534884 0.9531250
12 0 0.9230769 0.9714286 0.9629630 0.9589041 
lacmres3r <- do.call(rbind,lapply(groupsizes, function (g) { lacmres2[params$groupsizes==g,"REC"] }))
colnames(lacmres3r) <- deltas
rownames(lacmres3r) <- groupsizes
lacmres3r %>% kbl(caption="LAM recall") %>% kable_paper("hover", full_width = F)
LAM recall
0.1 0.2 0.3 0.4 0.5
3 0.001 0.001 0.004 0.006 0.018
5 0.000 0.005 0.010 0.015 0.026
8 0.000 0.007 0.023 0.041 0.061
12 0.000 0.012 0.034 0.052 0.070 
lacmres3f <- F1(lacmres3p,lacmres3r)
lacmres3f %>% kbl(caption="LAM F1") %>% kable_paper("hover", full_width = F)
LAM F1
0.1 0.2 0.3 0.4 0.5
3 0.001998 0.0019980 0.0079681 0.0119284 0.0353635
5 NaN 0.0099502 0.0198020 0.0295567 0.0506823
8 NaN 0.0138889 0.0449219 0.0786194 0.1146617
12 NaN 0.0236920 0.0657005 0.0986717 0.1304753 
plot(rownames(lacmres3p),lacmres3p[,"0.5"],type="b",las=1,ylab="precision",xlab="",
  main="",ylim=c(0,1))
lines(rownames(lacmres3p),lacmres3p[,"0.4"],type="b",col="red")
lines(rownames(lacmres3p),lacmres3p[,"0.3"],type="b",col="blue")
lines(rownames(lacmres3p),lacmres3p[,"0.2"],type="b",col="darkgreen")
grid()

legend("bottomright", legend=c("0.5", "0.4", "0.3", "0.2"),title="delta",
  lty=1, lwd=2, col=c("black", "red", "blue", "darkgreen"), cex=1)

plot(rownames(lacmres3r),lacmres3r[,"0.5"],type="b",las=1,ylab="recall",xlab="group size",
  main="LAM",ylim=c(0,1))
lines(rownames(lacmres3r),lacmres3r[,"0.4"],type="b",col="red")
lines(rownames(lacmres3r),lacmres3r[,"0.3"],type="b",col="blue")
lines(rownames(lacmres3r),lacmres3r[,"0.2"],type="b",,col="darkgreen")
grid()

plot(rownames(lacmres3r),lacmres3f[,"0.5"],type="b",las=1,ylab="F1",xlab="",
  main="",ylim=c(0,1))
lines(rownames(lacmres3r),lacmres3f[,"0.4"],type="b",col="red")
lines(rownames(lacmres3r),lacmres3f[,"0.3"],type="b",col="blue")
lines(rownames(lacmres3r),lacmres3f[,"0.2"],type="b",col="darkgreen")
grid()

# 50
load("GSE158422_simulate050.Rdata")

gres2[,"PREC"] <- gres2[,"TP"] / ( gres2[,"TP"] + gres2[,"FP"] )
gres3p <- do.call(rbind,lapply(groupsizes, function (g) { gres2[params$groupsizes==g,"PREC"] }))
colnames(gres3p) <- deltas
rownames(gres3p) <- groupsizes
gres3p %>% kbl(caption="GSAmeth precision") %>% kable_paper("hover", full_width = F)
GSAmeth precision
0.1 0.2 0.3 0.4 0.5
3 0 0 NaN 1.0000000 1.0000000
5 NaN NaN 1.0000000 0.9729730 0.9275362
8 NaN 1 0.8727273 0.9585253 0.9343148
12 NaN 1 0.9588235 0.9358842 0.9302326 
gres3r <- do.call(rbind,lapply(groupsizes, function (g) { gres2[params$groupsizes==g,"REC"] }))
colnames(gres3r) <- deltas
rownames(gres3r) <- groupsizes
gres3r %>% kbl(caption="GSAmeth recall") %>% kable_paper("hover", full_width = F)
GSAmeth recall
0.1 0.2 0.3 0.4 0.5
3 0 0.000 0.000 0.001 0.010
5 0 0.000 0.002 0.036 0.064
8 0 0.003 0.048 0.208 0.825
12 0 0.041 0.163 0.905 0.960 
gres3f <- F1(gres3p,gres3r)
gres3f %>% kbl(caption="GSAmeth F1") %>% kable_paper("hover", full_width = F)
GSAmeth F1
0.1 0.2 0.3 0.4 0.5
3 NaN NaN NaN 0.0019980 0.0198020
5 NaN NaN 0.0039920 0.0694311 0.1197381
8 NaN 0.0059821 0.0909953 0.3418242 0.8762613
12 NaN 0.0787704 0.2786325 0.9201830 0.9448819 
plot(rownames(gres3p),gres3p[,"0.5"],type="b",las=1,ylab="precision",xlab="",
  main="",ylim=c(0,1))
lines(rownames(gres3p),gres3p[,"0.4"],type="b",col="red")
lines(rownames(gres3p),gres3p[,"0.3"],type="b",col="blue")
lines(rownames(gres3p),gres3p[,"0.2"],type="b",col="darkgreen")
grid()

legend("bottomright", legend=c("0.5", "0.4", "0.3", "0.2"),title="delta",
  lty=1, lwd=2, col=c("black", "red", "blue", "darkgreen"), cex=1)

plot(rownames(gres3r),gres3r[,"0.5"],type="b",las=1,ylab="recall",xlab="group size",
  main="GSA",ylim=c(0,1))
lines(rownames(gres3r),gres3r[,"0.4"],type="b",col="red")
lines(rownames(gres3r),gres3r[,"0.3"],type="b",col="blue")
lines(rownames(gres3r),gres3r[,"0.2"],type="b",col="darkgreen")
grid()

plot(rownames(gres3r),gres3f[,"0.5"],type="b",las=1,ylab="F1",xlab="",
  main="",ylim=c(0,1))
lines(rownames(gres3r),gres3f[,"0.4"],type="b",col="red")
lines(rownames(gres3r),gres3f[,"0.3"],type="b",col="blue")
lines(rownames(gres3r),gres3f[,"0.2"],type="b",col="darkgreen")
grid()

lacmres2[,"PREC"] <- lacmres2[,"TP"] / ( lacmres2[,"TP"] + lacmres2[,"FP"] )
lacmres3p <- do.call(rbind,lapply(groupsizes, function (g) { lacmres2[params$groupsizes==g,"PREC"] }))
colnames(lacmres3p) <- deltas
rownames(lacmres3p) <- groupsizes
lacmres3p %>% kbl(caption="LAM precision") %>% kable_paper("hover", full_width = F)
LAM precision
0.1 0.2 0.3 0.4 0.5
3 0.0000000 0.5555556 0.8846154 0.8985507 0.9080460
5 0.7500000 0.9361702 0.9490446 0.9520548 0.9522546
8 0.8571429 0.9363636 0.9372694 0.9342105 0.9417040
12 0.9285714 0.9382022 0.9365559 0.9362187 0.9395712 
lacmres3r <- do.call(rbind,lapply(groupsizes, function (g) { lacmres2[params$groupsizes==g,"REC"] }))
colnames(lacmres3r) <- deltas
rownames(lacmres3r) <- groupsizes
lacmres3r %>% kbl(caption="LAM recall") %>% kable_paper("hover", full_width = F)
LAM recall
0.1 0.2 0.3 0.4 0.5
3 0.000 0.005 0.046 0.124 0.237
5 0.003 0.044 0.149 0.278 0.359
8 0.006 0.103 0.254 0.355 0.420
12 0.013 0.167 0.310 0.411 0.482 
lacmres3f <- F1(lacmres3p,lacmres3r)
lacmres3f %>% kbl(caption="LAM F1") %>% kable_paper("hover", full_width = F)
LAM F1
0.1 0.2 0.3 0.4 0.5
3 NaN 0.0099108 0.0874525 0.2179262 0.3758921
5 0.0059761 0.0840497 0.2575627 0.4303406 0.5214234
8 0.0119166 0.1855856 0.3996853 0.5144928 0.5809129
12 0.0256410 0.2835314 0.4658152 0.5712300 0.6371447 
plot(rownames(lacmres3p),lacmres3p[,"0.5"],type="b",las=1,ylab="precision",xlab="",
  main="",ylim=c(0,1))
lines(rownames(lacmres3p),lacmres3p[,"0.4"],type="b",col="red")
lines(rownames(lacmres3p),lacmres3p[,"0.3"],type="b",col="blue")
lines(rownames(lacmres3p),lacmres3p[,"0.2"],type="b",col="darkgreen")
grid()

legend("bottomright", legend=c("0.5", "0.4", "0.3", "0.2"),title="delta",
  lty=1, lwd=2, col=c("black", "red", "blue", "darkgreen"), cex=1)

plot(rownames(lacmres3r),lacmres3r[,"0.5"],type="b",las=1,ylab="recall",xlab="group size",
  main="LAM",ylim=c(0,1))
lines(rownames(lacmres3r),lacmres3r[,"0.4"],type="b",col="red")
lines(rownames(lacmres3r),lacmres3r[,"0.3"],type="b",col="blue")
lines(rownames(lacmres3r),lacmres3r[,"0.2"],type="b",,col="darkgreen")
grid()

plot(rownames(lacmres3r),lacmres3f[,"0.5"],type="b",las=1,ylab="F1",xlab="",
  main="",ylim=c(0,1))
lines(rownames(lacmres3r),lacmres3f[,"0.4"],type="b",col="red")
lines(rownames(lacmres3r),lacmres3f[,"0.3"],type="b",col="blue")
lines(rownames(lacmres3r),lacmres3f[,"0.2"],type="b",col="darkgreen")
grid()

# 100
load("GSE158422_simulate100.Rdata")

gres2[,"PREC"] <- gres2[,"TP"] / ( gres2[,"TP"] + gres2[,"FP"] )
gres3p <- do.call(rbind,lapply(groupsizes, function (g) { gres2[params$groupsizes==g,"PREC"] }))
colnames(gres3p) <- deltas
rownames(gres3p) <- groupsizes
gres3p %>% kbl(caption="GSAmeth precision") %>% kable_paper("hover", full_width = F)
GSAmeth precision
0.1 0.2 0.3 0.4 0.5
3 0 NaN 0.5000000 0.7500000 0.8867925
5 NaN 1.0000000 0.8695652 0.9409449 0.9027237
8 NaN 0.8636364 0.9195980 0.8445099 0.7548960
12 1 0.9009288 0.8762609 0.7223188 0.6898455 
gres3r <- do.call(rbind,lapply(groupsizes, function (g) { gres2[params$groupsizes==g,"REC"] }))
colnames(gres3r) <- deltas
rownames(gres3r) <- groupsizes
gres3r %>% kbl(caption="GSAmeth recall") %>% kable_paper("hover", full_width = F)
GSAmeth recall
0.1 0.2 0.3 0.4 0.5
3 0.0000 0.0000 0.0004 0.0024 0.0188
5 0.0000 0.0004 0.0160 0.0956 0.1856
8 0.0000 0.0152 0.1464 0.4584 0.9868
12 0.0012 0.1164 0.5212 0.9968 1.0000 
gres3f <- F1(gres3p,gres3r)
gres3f %>% kbl(caption="GSAmeth F1") %>% kable_paper("hover", full_width = F)
GSAmeth F1
0.1 0.2 0.3 0.4 0.5
3 NaN NaN 0.0007994 0.0047847 0.0368194
5 NaN 0.0007997 0.0314218 0.1735657 0.3078965
8 NaN 0.0298742 0.2525880 0.5942442 0.8554092
12 0.0023971 0.2061637 0.6536243 0.8376471 0.8164598 
plot(rownames(gres3p),gres3p[,"0.5"],type="b",las=1,ylab="precision",xlab="",
  main="",ylim=c(0,1))
lines(rownames(gres3p),gres3p[,"0.4"],type="b",col="red")
lines(rownames(gres3p),gres3p[,"0.3"],type="b",col="blue")
lines(rownames(gres3p),gres3p[,"0.2"],type="b",col="darkgreen")
grid()

legend("bottomright", legend=c("0.5", "0.4", "0.3", "0.2"),title="delta",
  lty=1, lwd=2, col=c("black", "red", "blue", "darkgreen"), cex=1)

plot(rownames(gres3r),gres3r[,"0.5"],type="b",las=1,ylab="recall",xlab="group size",
  main="GSA",ylim=c(0,1))
lines(rownames(gres3r),gres3r[,"0.4"],type="b",col="red")
lines(rownames(gres3r),gres3r[,"0.3"],type="b",col="blue")
lines(rownames(gres3r),gres3r[,"0.2"],type="b",col="darkgreen")
grid()

plot(rownames(gres3r),gres3f[,"0.5"],type="b",las=1,ylab="F1",xlab="",
  main="",ylim=c(0,1))
lines(rownames(gres3r),gres3f[,"0.4"],type="b",col="red")
lines(rownames(gres3r),gres3f[,"0.3"],type="b",col="blue")
lines(rownames(gres3r),gres3f[,"0.2"],type="b",col="darkgreen")
grid()

lacmres2[,"PREC"] <- lacmres2[,"TP"] / ( lacmres2[,"TP"] + lacmres2[,"FP"] )
lacmres3p <- do.call(rbind,lapply(groupsizes, function (g) { lacmres2[params$groupsizes==g,"PREC"] }))
colnames(lacmres3p) <- deltas
rownames(lacmres3p) <- groupsizes
lacmres3p %>% kbl(caption="LAM precision") %>% kable_paper("hover", full_width = F)
LAM precision
0.1 0.2 0.3 0.4 0.5
3 0.5833333 0.8956044 0.9379043 0.9413469 0.9342481
5 0.8571429 0.9250000 0.9335394 0.9300226 0.9286417
8 0.9384615 0.9426752 0.9414303 0.9425229 0.9423168
12 0.9033816 0.9371302 0.9352882 0.9333638 0.9343612 
lacmres3r <- do.call(rbind,lapply(groupsizes, function (g) { lacmres2[params$groupsizes==g,"REC"] }))
colnames(lacmres3r) <- deltas
rownames(lacmres3r) <- groupsizes
lacmres3r %>% kbl(caption="LAM recall") %>% kable_paper("hover", full_width = F)
LAM recall
0.1 0.2 0.3 0.4 0.5
3 0.0028 0.0652 0.2900 0.5200 0.6536
5 0.0072 0.1924 0.4832 0.6592 0.7548
8 0.0244 0.3552 0.6108 0.7412 0.7972
12 0.0748 0.5068 0.7400 0.8180 0.8484 
lacmres3f <- F1(lacmres3p,lacmres3r)
lacmres3f %>% kbl(caption="LAM F1") %>% kable_paper("hover", full_width = F)
LAM F1
0.1 0.2 0.3 0.4 0.5
3 0.0055732 0.1215511 0.4430186 0.6699304 0.7691221
5 0.0142800 0.3185430 0.6367949 0.7715356 0.8327449
8 0.0475634 0.5159791 0.7409025 0.8298253 0.8637053
12 0.1381603 0.6578401 0.8262617 0.8718823 0.8893082 
plot(rownames(lacmres3p),lacmres3p[,"0.5"],type="b",las=1,ylab="precision",xlab="",
  main="",ylim=c(0,1))
lines(rownames(lacmres3p),lacmres3p[,"0.4"],type="b",col="red")
lines(rownames(lacmres3p),lacmres3p[,"0.3"],type="b",col="blue")
lines(rownames(lacmres3p),lacmres3p[,"0.2"],type="b",col="darkgreen")
grid()

legend("bottomright", legend=c("0.5", "0.4", "0.3", "0.2"),title="delta",
  lty=1, lwd=2, col=c("black", "red", "blue", "darkgreen"), cex=1)

plot(rownames(lacmres3r),lacmres3r[,"0.5"],type="b",las=1,ylab="recall",xlab="group size",
  main="LAM",ylim=c(0,1))
lines(rownames(lacmres3r),lacmres3r[,"0.4"],type="b",col="red")
lines(rownames(lacmres3r),lacmres3r[,"0.3"],type="b",col="blue")
lines(rownames(lacmres3r),lacmres3r[,"0.2"],type="b",,col="darkgreen")
grid()

plot(rownames(lacmres3r),lacmres3f[,"0.5"],type="b",las=1,ylab="F1",xlab="",
  main="",ylim=c(0,1))
lines(rownames(lacmres3r),lacmres3f[,"0.4"],type="b",col="red")
lines(rownames(lacmres3r),lacmres3f[,"0.3"],type="b",col="blue")
lines(rownames(lacmres3r),lacmres3f[,"0.2"],type="b",col="darkgreen")
grid()

# png
png(filename="fig2.png",height=150,width=210,units="mm",res=200)

par(mar = c(4.1, 4.1, 2.1, 0.5))

par(mfrow=c(2,3))

plot(rownames(gres3p),gres3p[,"0.5"],type="b",las=1,ylab="precision",xlab="",
  main="",ylim=c(0,1))
lines(rownames(gres3p),gres3p[,"0.4"],type="b",col="red")
lines(rownames(gres3p),gres3p[,"0.3"],type="b",col="blue")
lines(rownames(gres3p),gres3p[,"0.2"],type="b",col="darkgreen")
grid()

legend("bottomright", legend=c("0.5", "0.4", "0.3", "0.2"),title="delta",
  lty=1, lwd=2, col=c("black", "red", "blue", "darkgreen"), cex=1)

plot(rownames(gres3r),gres3r[,"0.5"],type="b",las=1,ylab="recall",xlab="group size",
  main="GSA",ylim=c(0,1))
lines(rownames(gres3r),gres3r[,"0.4"],type="b",col="red")
lines(rownames(gres3r),gres3r[,"0.3"],type="b",col="blue")
lines(rownames(gres3r),gres3r[,"0.2"],type="b",col="darkgreen")
grid()

plot(rownames(gres3r),gres3f[,"0.5"],type="b",las=1,ylab="F1",xlab="",
  main="",ylim=c(0,1))
lines(rownames(gres3r),gres3f[,"0.4"],type="b",col="red")
lines(rownames(gres3r),gres3f[,"0.3"],type="b",col="blue")
lines(rownames(gres3r),gres3f[,"0.2"],type="b",col="darkgreen")
grid()

plot(rownames(lacmres3p),lacmres3p[,"0.5"],type="b",las=1,ylab="precision",xlab="",
  main="",ylim=c(0,1))
lines(rownames(lacmres3p),lacmres3p[,"0.4"],type="b",col="red")
lines(rownames(lacmres3p),lacmres3p[,"0.3"],type="b",col="blue")
lines(rownames(lacmres3p),lacmres3p[,"0.2"],type="b",col="darkgreen")
grid()

legend("bottomright", legend=c("0.5", "0.4", "0.3", "0.2"),title="delta",
  lty=1, lwd=2, col=c("black", "red", "blue", "darkgreen"), cex=1)

plot(rownames(lacmres3r),lacmres3r[,"0.5"],type="b",las=1,ylab="recall",xlab="group size",
  main="LAM",ylim=c(0,1))
lines(rownames(lacmres3r),lacmres3r[,"0.4"],type="b",col="red")
lines(rownames(lacmres3r),lacmres3r[,"0.3"],type="b",col="blue")
lines(rownames(lacmres3r),lacmres3r[,"0.2"],type="b",,col="darkgreen")
grid()

plot(rownames(lacmres3r),lacmres3f[,"0.5"],type="b",las=1,ylab="F1",xlab="",
  main="",ylim=c(0,1))
lines(rownames(lacmres3r),lacmres3f[,"0.4"],type="b",col="red")
lines(rownames(lacmres3r),lacmres3f[,"0.3"],type="b",col="blue")
lines(rownames(lacmres3r),lacmres3f[,"0.2"],type="b",col="darkgreen")
grid()

dev.off()
## png 
##   2
par(mfrow=c(1,1))

Session information

sessionInfo()
## R version 4.3.2 (2023-10-31)
## Platform: x86_64-pc-linux-gnu (64-bit)
## Running under: Ubuntu 22.04.3 LTS
## 
## Matrix products: default
## BLAS:   /usr/lib/x86_64-linux-gnu/openblas-pthread/libblas.so.3 
## LAPACK: /usr/lib/x86_64-linux-gnu/openblas-pthread/libopenblasp-r0.3.20.so;  LAPACK version 3.10.0
## 
## locale:
##  [1] LC_CTYPE=en_US.UTF-8       LC_NUMERIC=C              
##  [3] LC_TIME=en_US.UTF-8        LC_COLLATE=en_US.UTF-8    
##  [5] LC_MONETARY=en_US.UTF-8    LC_MESSAGES=en_US.UTF-8   
##  [7] LC_PAPER=en_US.UTF-8       LC_NAME=C                 
##  [9] LC_ADDRESS=C               LC_TELEPHONE=C            
## [11] LC_MEASUREMENT=en_US.UTF-8 LC_IDENTIFICATION=C       
## 
## time zone: Etc/UTC
## tzcode source: system (glibc)
## 
## attached base packages:
## [1] parallel  stats4    stats     graphics  grDevices utils     datasets 
## [8] methods   base     
## 
## other attached packages:
##  [1] kableExtra_1.3.4                                   
##  [2] mitch_1.15.0                                       
##  [3] psych_2.3.9                                        
##  [4] org.Hs.eg.db_3.18.0                                
##  [5] AnnotationDbi_1.64.1                               
##  [6] IlluminaHumanMethylation450kmanifest_0.4.0         
##  [7] missMethyl_1.36.0                                  
##  [8] IlluminaHumanMethylationEPICanno.ilm10b4.hg19_0.6.0
##  [9] IlluminaHumanMethylation450kanno.ilmn12.hg19_0.6.1 
## [10] minfi_1.48.0                                       
## [11] bumphunter_1.44.0                                  
## [12] locfit_1.5-9.8                                     
## [13] iterators_1.0.14                                   
## [14] foreach_1.5.2                                      
## [15] Biostrings_2.70.1                                  
## [16] XVector_0.42.0                                     
## [17] SummarizedExperiment_1.32.0                        
## [18] Biobase_2.62.0                                     
## [19] MatrixGenerics_1.14.0                              
## [20] matrixStats_1.2.0                                  
## [21] GenomicRanges_1.54.1                               
## [22] GenomeInfoDb_1.38.2                                
## [23] IRanges_2.36.0                                     
## [24] S4Vectors_0.40.2                                   
## [25] BiocGenerics_0.48.1                                
## [26] limma_3.58.1                                       
## [27] stringi_1.8.3                                      
## 
## loaded via a namespace (and not attached):
##   [1] splines_4.3.2             later_1.3.2              
##   [3] BiocIO_1.12.0             bitops_1.0-7             
##   [5] filelock_1.0.3            tibble_3.2.1             
##   [7] preprocessCore_1.64.0     XML_3.99-0.16            
##   [9] lifecycle_1.0.4           lattice_0.22-5           
##  [11] MASS_7.3-60               base64_2.0.1             
##  [13] scrime_1.3.5              magrittr_2.0.3           
##  [15] sass_0.4.8                rmarkdown_2.25           
##  [17] jquerylib_0.1.4           yaml_2.3.8               
##  [19] httpuv_1.6.13             doRNG_1.8.6              
##  [21] askpass_1.2.0             DBI_1.1.3                
##  [23] RColorBrewer_1.1-3        abind_1.4-5              
##  [25] zlibbioc_1.48.0           rvest_1.0.3              
##  [27] quadprog_1.5-8            purrr_1.0.2              
##  [29] RCurl_1.98-1.13           rappdirs_0.3.3           
##  [31] GenomeInfoDbData_1.2.11   genefilter_1.84.0        
##  [33] annotate_1.80.0           svglite_2.1.3            
##  [35] DelayedMatrixStats_1.24.0 codetools_0.2-19         
##  [37] DelayedArray_0.28.0       xml2_1.3.6               
##  [39] tidyselect_1.2.0          beanplot_1.3.1           
##  [41] BiocFileCache_2.10.1      webshot_0.5.5            
##  [43] illuminaio_0.44.0         GenomicAlignments_1.38.0 
##  [45] jsonlite_1.8.8            multtest_2.58.0          
##  [47] ellipsis_0.3.2            survival_3.5-7           
##  [49] systemfonts_1.0.5         tools_4.3.2              
##  [51] progress_1.2.3            Rcpp_1.0.11              
##  [53] glue_1.6.2                gridExtra_2.3            
##  [55] mnormt_2.1.1              SparseArray_1.2.2        
##  [57] xfun_0.41                 dplyr_1.1.4              
##  [59] HDF5Array_1.30.0          fastmap_1.1.1            
##  [61] GGally_2.2.0              rhdf5filters_1.14.1      
##  [63] fansi_1.0.6               openssl_2.1.1            
##  [65] caTools_1.18.2            digest_0.6.33            
##  [67] R6_2.5.1                  mime_0.12                
##  [69] colorspace_2.1-0          gtools_3.9.5             
##  [71] biomaRt_2.58.0            RSQLite_2.3.4            
##  [73] utf8_1.2.4                tidyr_1.3.0              
##  [75] generics_0.1.3            data.table_1.14.10       
##  [77] rtracklayer_1.62.0        prettyunits_1.2.0        
##  [79] httr_1.4.7                htmlwidgets_1.6.4        
##  [81] S4Arrays_1.2.0            ggstats_0.5.1            
##  [83] pkgconfig_2.0.3           gtable_0.3.4             
##  [85] blob_1.2.4                siggenes_1.76.0          
##  [87] htmltools_0.5.7           echarts4r_0.4.5          
##  [89] scales_1.3.0              png_0.1-8                
##  [91] rstudioapi_0.15.0         knitr_1.45               
##  [93] reshape2_1.4.4            tzdb_0.4.0               
##  [95] rjson_0.2.21              nlme_3.1-163             
##  [97] curl_5.2.0                cachem_1.0.8             
##  [99] rhdf5_2.46.1              stringr_1.5.1            
## [101] KernSmooth_2.23-22        restfulr_0.0.15          
## [103] GEOquery_2.70.0           pillar_1.9.0             
## [105] grid_4.3.2                reshape_0.8.9            
## [107] vctrs_0.6.5               gplots_3.1.3             
## [109] promises_1.2.1            dbplyr_2.4.0             
## [111] xtable_1.8-4              beeswarm_0.4.0           
## [113] evaluate_0.23             readr_2.1.4              
## [115] GenomicFeatures_1.54.1    cli_3.6.2                
## [117] compiler_4.3.2            Rsamtools_2.18.0         
## [119] rlang_1.1.2               crayon_1.5.2             
## [121] rngtools_1.5.2            nor1mix_1.3-2            
## [123] mclust_6.0.1              plyr_1.8.9               
## [125] viridisLite_0.4.2         BiocParallel_1.36.0      
## [127] munsell_0.5.0             Matrix_1.6-1.1           
## [129] hms_1.1.3                 sparseMatrixStats_1.14.0 
## [131] bit64_4.0.5               ggplot2_3.4.4            
## [133] Rhdf5lib_1.24.1           KEGGREST_1.42.0          
## [135] statmod_1.5.0             shiny_1.8.0              
## [137] highr_0.10                memoise_2.0.1            
## [139] bslib_0.6.1               bit_4.0.5

Notes

LA

  • simla: parametric self-contained

  • simlac: parametric competitive

  • simnla: nonparametric self-contained

  • simnlac: nonparametric competitive

AL

  • simal: parametric self-contained

  • simalc: parametric competitive

  • simnal: nonparametric self-contained

  • simnalc: nonparametric competitive