|
jpayne@69
|
1 #!/bin/bash
|
|
jpayne@69
|
2
|
|
jpayne@69
|
3 usage(){
|
|
jpayne@69
|
4 echo "
|
|
jpayne@69
|
5 Written by Brian Bushnell
|
|
jpayne@69
|
6 Last modified August 11, 2021
|
|
jpayne@69
|
7
|
|
jpayne@69
|
8 Description: Reorders reads randomly, keeping pairs together.
|
|
jpayne@69
|
9 Unlike Shuffle, Shuffle2 can write temp files to handle large datasets.
|
|
jpayne@69
|
10
|
|
jpayne@69
|
11 Usage: shuffle2.sh in=<file> out=<file>
|
|
jpayne@69
|
12
|
|
jpayne@69
|
13 Standard parameters:
|
|
jpayne@69
|
14 in=<file> The 'in=' flag is needed if the input file is not the first parameter. 'in=stdin' will pipe from standard in.
|
|
jpayne@69
|
15 in2=<file> Use this if 2nd read of pairs are in a different file.
|
|
jpayne@69
|
16 out=<file> The 'out=' flag is needed if the output file is not the second parameter. 'out=stdout' will pipe to standard out.
|
|
jpayne@69
|
17 out2=<file> Use this to write 2nd read of pairs to a different file.
|
|
jpayne@69
|
18 overwrite=t (ow) Set to false to force the program to abort rather than overwrite an existing file.
|
|
jpayne@69
|
19 ziplevel=2 (zl) Set to 1 (lowest) through 9 (max) to change compression level; lower compression is faster.
|
|
jpayne@69
|
20 int=auto (interleaved) Set to t or f to override interleaving autodetection.
|
|
jpayne@69
|
21
|
|
jpayne@69
|
22 Processing parameters:
|
|
jpayne@69
|
23 shuffle Randomly reorders reads (default).
|
|
jpayne@69
|
24 seed=-1 Set to a positive number for deterministic shuffling.
|
|
jpayne@69
|
25
|
|
jpayne@69
|
26 Java Parameters:
|
|
jpayne@69
|
27 -Xmx This will set Java's memory usage, overriding autodetection.
|
|
jpayne@69
|
28 -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will specify 200 megs.
|
|
jpayne@69
|
29 The max is typically 85% of physical memory.
|
|
jpayne@69
|
30 -eoom This flag will cause the process to exit if an out-of-memory
|
|
jpayne@69
|
31 exception occurs. Requires Java 8u92+.
|
|
jpayne@69
|
32 -da Disable assertions.
|
|
jpayne@69
|
33
|
|
jpayne@69
|
34 Please contact Brian Bushnell at bbushnell@lbl.gov if you encounter any problems.
|
|
jpayne@69
|
35 "
|
|
jpayne@69
|
36 }
|
|
jpayne@69
|
37
|
|
jpayne@69
|
38 #This block allows symlinked shellscripts to correctly set classpath.
|
|
jpayne@69
|
39 pushd . > /dev/null
|
|
jpayne@69
|
40 DIR="${BASH_SOURCE[0]}"
|
|
jpayne@69
|
41 while [ -h "$DIR" ]; do
|
|
jpayne@69
|
42 cd "$(dirname "$DIR")"
|
|
jpayne@69
|
43 DIR="$(readlink "$(basename "$DIR")")"
|
|
jpayne@69
|
44 done
|
|
jpayne@69
|
45 cd "$(dirname "$DIR")"
|
|
jpayne@69
|
46 DIR="$(pwd)/"
|
|
jpayne@69
|
47 popd > /dev/null
|
|
jpayne@69
|
48
|
|
jpayne@69
|
49 #DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/"
|
|
jpayne@69
|
50 CP="$DIR""current/"
|
|
jpayne@69
|
51
|
|
jpayne@69
|
52 z="-Xmx2g"
|
|
jpayne@69
|
53 z2="-Xms2g"
|
|
jpayne@69
|
54 set=0
|
|
jpayne@69
|
55
|
|
jpayne@69
|
56 if [ -z "$1" ] || [[ $1 == -h ]] || [[ $1 == --help ]]; then
|
|
jpayne@69
|
57 usage
|
|
jpayne@69
|
58 exit
|
|
jpayne@69
|
59 fi
|
|
jpayne@69
|
60
|
|
jpayne@69
|
61 calcXmx () {
|
|
jpayne@69
|
62 source "$DIR""/calcmem.sh"
|
|
jpayne@69
|
63 setEnvironment
|
|
jpayne@69
|
64 parseXmx "$@"
|
|
jpayne@69
|
65 if [[ $set == 1 ]]; then
|
|
jpayne@69
|
66 return
|
|
jpayne@69
|
67 fi
|
|
jpayne@69
|
68 freeRam 2000m 84
|
|
jpayne@69
|
69 z="-Xmx${RAM}m"
|
|
jpayne@69
|
70 z2="-Xms${RAM}m"
|
|
jpayne@69
|
71 }
|
|
jpayne@69
|
72 calcXmx "$@"
|
|
jpayne@69
|
73
|
|
jpayne@69
|
74 shuffle() {
|
|
jpayne@69
|
75 local CMD="java $EA $EOOM $z -cp $CP sort.Shuffle2 $@"
|
|
jpayne@69
|
76 echo $CMD >&2
|
|
jpayne@69
|
77 eval $CMD
|
|
jpayne@69
|
78 }
|
|
jpayne@69
|
79
|
|
jpayne@69
|
80 shuffle "$@"
|
