diversity-maximization/experiments.sh at master · Cecca/diversity-maximization · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
#!/bin/bash

LIB=$HOME/lib/diversity-maximization-experiments-assembly-0.1.0.jar
DATASETS_DIR=~/Datasets

function spark () {
  CLASS=$1
  EXECS=$2
  shift
  shift
  ARGS=$@
  echo "Running Spark with class $CLASS, library $LIB and arguments $ARGS on $EXECS executors"
  spark-submit \
       --num-executors $EXECS --executor-memory 8G --executor-cores 1 \
       --properties-file ~/Configurations/standalone-128.properties \
       --class $CLASS $LIB $ARGS
}

function streaming() {
  DATASET=$1
  FACTORS="1 2 4 8 16 32"
  TAUS="8 16 32 64 128 256"
  if [[ $DATASET =~ "glove" ]]
  then
    local SPECIAL_ARGS="--uniform glove"
    local TARGETS="100 25"
    local TARGETS="25"
  elif [[ $DATASET =~ "mxm" ]]
  then
    local SPECIAL_ARGS="--genres genres.rank88.txt"
    local TARGETS="88 22"
    local TARGETS="22"
  elif [[ $DATASET =~ "wiki" ]]
  then
    local SPECIAL_ARGS="--topics"
    local TARGETS="100 25"
    local TARGETS="25"
  fi
  for TARGET in $TARGETS
  do
    for TAU in $TAUS
    do
      spark it.unipd.dei.diversity.MainMatroid 9 \
        --algorithm streaming \
        --input $DATASET \
        --target $TARGET \
        --sparsify \
        --tau $TAU \
        $SPECIAL_ARGS
    done
  done
}

function mapreduce() {
  DATASET=$1
  FACTORS="1 2 4 8"
  if [[ $DATASET =~ "glove" ]]
  then
    local SPECIAL_ARGS="--uniform glove"
    local TARGETS="25 100"
  elif [[ $DATASET =~ "mxm" ]]
  then
    local SPECIAL_ARGS="--genres genres.rank88.txt"
    local TARGETS="22 88"
  elif [[ $DATASET =~ "wiki" ]]
  then
    local SPECIAL_ARGS="--topics"
    local TARGETS="100 25"
  fi
  for TARGET in $TARGETS
  do
    for PARALLELISM in 1 2 4 8 16
    do
      local TAU=64
      local TAU_P=$(( 64 / $PARALLELISM ))
      spark it.unipd.dei.diversity.MainMatroid $PARALLELISM \
        --algorithm mapreduce \
        --parallelism $PARALLELISM \
        --input $DATASET \
        --target $TARGET \
        --sparsify \
        --tau $TAU \
        --tau-p $TAU_P \
        $SPECIAL_ARGS
    done
  done
}

function sequential() {
  DATASET=$1
  TAUS="8 16 32 128 256"
  if [[ $DATASET =~ "glove" ]]
  then
    local SPECIAL_ARGS="--uniform glove"
    local TARGETS="25 100"
  elif [[ $DATASET =~ "mxm" ]]
  then
    local SPECIAL_ARGS="--genres genres.rank88.txt"
    local TARGETS="22 88"
  elif [[ $DATASET =~ "wiki" ]]
  then
    local SPECIAL_ARGS="--topics"
    local TARGETS="100 25"
  fi
  for TARGET in $TARGETS
  do
    for TAU in $TAUS
    do
      spark it.unipd.dei.diversity.MainMatroid 9 \
        --algorithm sequential-coreset \
        --input $DATASET \
        --target $TARGET \
        --sparsify \
        --tau $TAU \
        $SPECIAL_ARGS
    done
  done
}


streaming mxm.pq
streaming glove.twitter.27B.25d.pq
streaming wikipedia-25.json.bz2

mapreduce mxm.pq
mapreduce glove.twitter.27B.25d.pq
mapreduce wikipedia-25.json.bz2

sequential mxm-5000.pq
sequential glove-5000.pq
sequential wikipedia-5000.json