Mercurial > repos > rliterman > csp2

comparison CSP2/CSP2_env/env-d9b9114564458d9d-741b3de822f2aaca6c6caa4325c4afce/opt/bbmap-39.01-1/current/fun/DiskBench.java @ 68:5028fdace37b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 2e9511a184a1ca667c7be0c6321a36dc4e3d116d
author jpayne
date Tue, 18 Mar 2025 16:23:26 -0400
parents
children
comparison
equal deleted inserted replaced
67:0e9998148a16 68:5028fdace37b
1 package fun;
2
3 import java.io.File;
4 import java.io.IOException;
5 import java.io.InputStream;
6 import java.io.PrintStream;
7 import java.util.ArrayList;
8 import java.util.Arrays;
9 import java.util.Locale;
10 import java.util.Random;
11
12 import fileIO.ByteFile;
13 import fileIO.ByteStreamWriter;
14 import fileIO.FileFormat;
15 import fileIO.QuickFile;
16 import fileIO.ReadWrite;
17 import fileIO.TextFile;
18 import shared.Parse;
19 import shared.Parser;
20 import shared.PreParser;
21 import shared.Shared;
22 import shared.Timer;
23 import shared.Tools;
24 import stream.FastaReadInputStream;
25 import structures.ByteBuilder;
26
27 /**
28 * @author Brian Bushnell
29 * @date December 6, 2017
30 *
31 */
32 public class DiskBench {
33
34 public static void main(String[] args){
35 //Start a timer immediately upon code entrance.
36 Timer t=new Timer();
37
38 //Create an instance of this class
39 DiskBench x=new DiskBench(args);
40
41 //Run the object
42 x.process(t);
43
44 //Close the print stream if it was redirected
45 Shared.closeStream(x.outstream);
46 }
47
48 public DiskBench(String[] args){
49
50 {//Preparse block for help, config files, and outstream
51 PreParser pp=new PreParser(args, getClass(), false);
52 args=pp.args;
53 outstream=pp.outstream;
54 }
55
56 ReadWrite.USE_PIGZ=ReadWrite.USE_UNPIGZ=true;
57 ReadWrite.MAX_ZIP_THREADS=Shared.threads();
58
59 Parser parser=new Parser();
60 parser.overwrite=true;
61 for(int i=0; i<args.length; i++){
62 String arg=args[i];
63 String[] split=arg.split("=");
64 String a=split[0].toLowerCase();
65 String b=split.length>1 ? split[1] : null;
66
67 if(a.equals("path")){
68 path=b;
69 if(path==null){path="";}
70 else if(!path.endsWith("/")){path=path+"/";}
71 }else if(a.equals("lines")){
72 maxLines=Long.parseLong(b);
73 if(maxLines<0){maxLines=Long.MAX_VALUE;}
74 }else if(a.equals("data") || a.equals("size")){
75 data=Parse.parseKMG(b);
76 }else if(a.equals("passes")){
77 passes=Integer.parseInt(b);
78 }else if(a.equals("verbose")){
79 verbose=Parse.parseBoolean(b);
80 }else if(a.equals("gzip")){
81 verbose=Parse.parseBoolean(b);
82 }else if(a.equals("mode")){
83 assert(b!=null) : "Bad parameter: "+arg;
84 if(Tools.isDigit(b.charAt(0))){mode=Integer.parseInt(b);}
85 else if("read".equalsIgnoreCase(b) || "r".equalsIgnoreCase(b)){mode=READ;}
86 else if("write".equalsIgnoreCase(b) || "w".equalsIgnoreCase(b)){mode=WRITE;}
87 else if("readwrite".equalsIgnoreCase(b) || "rw".equalsIgnoreCase(b)){mode=READWRITE;}
88 else{assert(false) : "Bad mode: "+arg;}
89 }else if(a.equals("read") || a.equals("r")){
90 mode=READ;
91 }else if(a.equals("write") || a.equals("w")){
92 mode=WRITE;
93 }else if(a.equals("readwrite") || a.equals("rw")){
94 mode=READWRITE;
95 }else if(a.equals("printtid")){
96 printTid=Parse.parseBoolean(b);
97 }else if(a.equals("processbis")){
98 processBis=Parse.parseBoolean(b);
99 }else if(a.equals("preread")){
100 preRead=Parse.parseBoolean(b);
101 }
102
103 else if(a.equals("method")){
104 assert(b!=null) : "Bad parameter: "+arg;
105 if(Tools.isDigit(b.charAt(0))){method=Integer.parseInt(b);}
106 else if("BYTEFILE".equalsIgnoreCase(b) || "bf".equalsIgnoreCase(b)){method=BYTEFILE;}
107 else if("TEXTFILE".equalsIgnoreCase(b) || "tf".equalsIgnoreCase(b)){method=TEXTFILE;}
108 else if("QUICKFILE".equalsIgnoreCase(b) || "qf".equalsIgnoreCase(b)){method=QUICKFILE;}
109 else if("BUFFEREDINPUTSTREAM".equalsIgnoreCase(b) || "bis".equalsIgnoreCase(b)){method=BUFFEREDINPUTSTREAM;}
110 else if("FILEINPUTSTREAM".equalsIgnoreCase(b) || "fis".equalsIgnoreCase(b)){method=FILEINPUTSTREAM;}
111 else if("BUFFEREDINPUTSTREAM2".equalsIgnoreCase(b) || "bis2".equalsIgnoreCase(b)){method=BUFFEREDINPUTSTREAM2;}
112 else if("FILEINPUTSTREAM2".equalsIgnoreCase(b) || "fis2".equalsIgnoreCase(b)){method=FILEINPUTSTREAM2;}
113 else{assert(false) : "Bad mode: "+arg;}
114 }
115 else if("BYTEFILE".equalsIgnoreCase(a) || "bf".equalsIgnoreCase(a)){method=BYTEFILE;}
116 else if("TEXTFILE".equalsIgnoreCase(a) || "tf".equalsIgnoreCase(a)){method=TEXTFILE;}
117 else if("QUICKFILE".equalsIgnoreCase(a) || "qf".equalsIgnoreCase(a)){method=QUICKFILE;}
118 else if("BUFFEREDINPUTSTREAM".equalsIgnoreCase(a) || "bis".equalsIgnoreCase(a)){method=BUFFEREDINPUTSTREAM;}
119 else if("FILEINPUTSTREAM".equalsIgnoreCase(a) || "fis".equalsIgnoreCase(a)){method=FILEINPUTSTREAM;}
120 else if("BUFFEREDINPUTSTREAM2".equalsIgnoreCase(a) || "bis2".equalsIgnoreCase(a)){method=BUFFEREDINPUTSTREAM2;}
121 else if("FILEINPUTSTREAM2".equalsIgnoreCase(a) || "fis2".equalsIgnoreCase(a)){method=FILEINPUTSTREAM2;}
122 else if("buffer".equalsIgnoreCase(a) || "bufferlen".equalsIgnoreCase(a)){
123 bufferlen=(int)Parse.parseKMGBinary(b);
124 }
125
126 else if(parser.parse(arg, a, b)){
127 //do nothing
128 }else{
129 outstream.println("Unknown parameter "+args[i]);
130 assert(false) : "Unknown parameter "+args[i];
131 // throw new RuntimeException("Unknown parameter "+args[i]);
132 }
133 }
134
135 {//Process parser fields
136 overwrite=parser.overwrite;
137 threads=Shared.threads();
138 }
139
140 assert(FastaReadInputStream.settingsOK());
141
142 if(!ByteFile.FORCE_MODE_BF2){
143 ByteFile.FORCE_MODE_BF2=false;
144 ByteFile.FORCE_MODE_BF1=true;
145 }
146
147 File pfile=new File(path);
148 if(!pfile.exists()){pfile.mkdirs();}
149 }
150
151 class WriteThread extends Thread{
152
153 public WriteThread(String fname_, long size_){
154 fname=fname_;
155 size=size_;
156 }
157
158 @Override
159 public void run(){
160 t=new Timer();
161 written=writeRandomData(fname, size, t, overwrite);
162 }
163
164 String fname;
165 long size;
166 long written=0;
167 Timer t;
168
169 }
170
171 public static long writeRandomData(final String fname, final long size, final Timer t, final boolean overwrite){
172 if(t!=null){t.start();}
173 long written=0;
174 final Random randy=Shared.threadLocalRandom();
175 FileFormat ffout=FileFormat.testOutput(fname, FileFormat.TEXT, null, true, overwrite, false, false);
176 ByteStreamWriter bsw=new ByteStreamWriter(ffout);
177 bsw.start();
178 final ByteBuilder bb=new ByteBuilder(66000);
179 final int shift=6;
180 final int shiftsPerRand=32/shift;
181 assert(shiftsPerRand>0);
182 final long limit=size-20-shiftsPerRand*1000;
183 while(written<limit){
184 for(int i=0; i<1000; i+=shiftsPerRand){
185 int x=randy.nextInt();
186 for(int j=0; j<shiftsPerRand; j++){
187 byte b=(byte)(33+x&63);
188 bb.append(b);
189 x>>=shift;
190 }
191 }
192 // for(int i=0; i<1000; i+=shiftsPerRand){
193 // long x=randy.nextLong();
194 // for(int j=0; j<shiftsPerRand; j++){
195 // byte b=(byte)(33+x&63);
196 // bb.append(b);
197 // x>>=shift;
198 // }
199 // }
200 bb.nl();
201 written+=bb.length;
202 bsw.print(bb);
203 bb.clear();
204 }
205 while(written<size-1){
206 bb.append((byte)(33+(randy.nextInt()&63)));
207 written++;
208 }
209 bb.nl();
210 written+=bb.length;
211 bsw.print(bb);
212 bb.clear();
213 bsw.poisonAndWait();
214 File f=new File(fname);
215 long diskSize=(f.length());
216 if(t!=null){t.stop();}
217 return diskSize;
218 }
219
220 class ReadThread extends Thread{
221
222 public ReadThread(String fname_, int tid_){
223 fname=fname_;
224 tid=tid_;
225 }
226
227 @Override
228 public void run(){
229 t=new Timer();
230 FileFormat ffin=FileFormat.testInput(fname, FileFormat.TEXT, null, false, false, false);
231
232 if(method==BYTEFILE){
233 runBf(ffin);
234 }else if(method==QUICKFILE){
235 runQf(ffin);
236 }else if(method==TEXTFILE){
237 runTf(ffin);
238 }else if(method==BUFFEREDINPUTSTREAM){
239 runBis(ffin, true);
240 }else if(method==FILEINPUTSTREAM){
241 runBis(ffin, false);
242 }else if(method==BUFFEREDINPUTSTREAM2){
243 runBis2(ffin, true);
244 }else if(method==FILEINPUTSTREAM2){
245 runBis2(ffin, false);
246 }
247 if(printTid){System.err.print(tid+",");}
248 t.stop();
249 }
250
251 private void runBf(FileFormat ffin){
252 ByteFile bf=ByteFile.makeByteFile(ffin);
253 for(byte[] line=bf.nextLine(); line!=null; line=bf.nextLine()){
254 read+=line.length+1;
255 }
256 }
257
258 private void runQf(FileFormat ffin){
259 QuickFile qf=new QuickFile(ffin);
260 for(byte[] line=qf.nextLine(); line!=null; line=qf.nextLine()){
261 read+=line.length+1;
262 }
263 }
264
265 private void runTf(FileFormat ffin){
266 TextFile tf=new TextFile(ffin);
267 for(String line=tf.nextLine(); line!=null; line=tf.nextLine()){
268 read+=line.length()+1;
269 }
270 }
271
272 private void runBis(FileFormat ffin, boolean bufferedStream){
273
274 final byte[] buffer=new byte[bufferlen];
275 InputStream is=ReadWrite.getInputStream(ffin.name(), bufferedStream, false);
276
277 for(int r=1; r>0; ){
278 r=0;
279 try {
280 r=is.read(buffer);
281 if(r>0){read+=r;}
282
283 if(processBis){
284 int last=0;
285 for(int i=1; i<r; i++){
286 byte b=buffer[i];
287 if(b=='\n'){
288 cache=Arrays.copyOfRange(buffer, last, i);
289 last=i+1;
290 }
291 }
292 }
293
294 } catch (IOException e) {
295 e.printStackTrace();
296 }
297 }
298
299 try {
300 is.close();
301 } catch (IOException e) {
302 // TODO Auto-generated catch block
303 e.printStackTrace();
304 }
305 }
306
307 private void runBis2(FileFormat ffin, boolean bufferedStream){
308
309 final byte[] buffer=new byte[bufferlen];
310 InputStream is=ReadWrite.getInputStream(ffin.name(), bufferedStream, false);
311
312 list=new ArrayList<byte[]>(800);
313 for(int r=1; r>0; ){
314 r=0;
315 try {
316 r=is.read(buffer);
317 if(r>0){read+=r;}
318
319 if(processBis){
320 int last=0;
321 for(int i=1; i<r; i++){
322 byte b=buffer[i];
323 if(b=='\n'){
324 byte[] line=Arrays.copyOfRange(buffer, last, i);
325 list.add(line);
326 if(list.size()>=800){
327 list=new ArrayList<byte[]>(800);
328 }
329 last=i+1;
330 }
331 }
332 }
333
334 } catch (IOException e) {
335 e.printStackTrace();
336 }
337 }
338
339 try {
340 is.close();
341 } catch (IOException e) {
342 // TODO Auto-generated catch block
343 e.printStackTrace();
344 }
345 }
346
347 byte[] cache;
348 ArrayList<byte[]> list;
349 String fname;
350 long read=0;
351 long lines=0;
352 Timer t;
353 final int tid;
354
355 }
356
357 String[] makeFnames(int pass){
358 String[] fnames=new String[threads];
359 Random randy=new Random();
360 for(int i=0; i<threads; i++){
361 fnames[i]=path+pass+"_"+i+"_"+(System.nanoTime()&0xFFFF)+"_"+randy.nextInt(4096);
362 }
363 return fnames;
364 }
365
366 Timer readWrite(String[] fnamesW, String[] fnamesR){
367 Timer t=new Timer();
368
369 WriteThread[] wta=new WriteThread[threads];
370 long size=data/threads;
371 for(int i=0; i<threads; i++){
372 wta[i]=new WriteThread(fnamesW[i], size);
373 }
374 for(int i=0; i<threads; i++){
375 wta[i].start();
376 }
377
378 ReadThread[] rta=new ReadThread[threads];
379 for(int i=0; i<threads; i++){
380 rta[i]=new ReadThread(fnamesR[i], i);
381 }
382 for(int i=0; i<threads; i++){
383 rta[i].start();
384 }
385
386 for(int i=0; i<threads; i++){
387 while(wta[i].getState()!=Thread.State.TERMINATED){
388 try {
389 wta[i].join();
390 } catch (InterruptedException e) {
391 // TODO Auto-generated catch block
392 e.printStackTrace();
393 }
394 }
395 }
396
397 for(int i=0; i<threads; i++){
398 while(rta[i].getState()!=Thread.State.TERMINATED){
399 try {
400 rta[i].join();
401 } catch (InterruptedException e) {
402 // TODO Auto-generated catch block
403 e.printStackTrace();
404 }
405 }
406 }
407
408 t.stop();
409 return t;
410 }
411
412 Timer write(String[] fnames){
413 Timer t=new Timer();
414 WriteThread[] wta=new WriteThread[threads];
415 long size=data/threads;
416 for(int i=0; i<threads; i++){
417 wta[i]=new WriteThread(fnames[i], size);
418 }
419 for(int i=0; i<threads; i++){
420 wta[i].start();
421 }
422 for(int i=0; i<threads; i++){
423 while(wta[i].getState()!=Thread.State.TERMINATED){
424 try {
425 wta[i].join();
426 } catch (InterruptedException e) {
427 // TODO Auto-generated catch block
428 e.printStackTrace();
429 }
430 }
431 }
432 t.stop();
433 return t;
434 }
435
436 Timer read(String[] fnames){
437
438 Timer t=new Timer();
439
440 if(preRead){
441 ReadThread rt=new ReadThread(fnames[0], 0);
442 rt.start();
443 while(rt.getState()!=Thread.State.TERMINATED){
444 try {
445 rt.join();
446 } catch (InterruptedException e) {
447 // TODO Auto-generated catch block
448 e.printStackTrace();
449 }
450 }
451 }
452
453 ReadThread[] rta=new ReadThread[threads];
454 for(int i=0; i<threads; i++){
455 rta[i]=new ReadThread(fnames[i], i);
456 }
457 for(int i=0; i<threads; i++){
458 rta[i].start();
459 }
460 for(int i=0; i<threads; i++){
461 while(rta[i].getState()!=Thread.State.TERMINATED){
462 try {
463 rta[i].join();
464 } catch (InterruptedException e) {
465 // TODO Auto-generated catch block
466 e.printStackTrace();
467 }
468 }
469 linesInternal+=(rta[i].list==null ? 0 : rta[i].list.size());
470 }
471 t.stop();
472 return t;
473 }
474
475 void delete(String[] fnames){
476 for(String s : fnames){
477 File f=new File(s);
478 if(f.exists()){
479 f.delete();
480 }
481 }
482 }
483
484 void process(Timer t0){
485
486 t0.start();
487 String[] fnamesW=makeFnames(0);
488
489 Timer t=write(fnamesW);
490 String[] fnamesR=fnamesW;
491 fnamesW=null;
492
493 final long initialWriteElapsed=t.elapsed;
494
495 System.err.println("Initial write: \t"+t.toString()+" \t"+String.format(Locale.ROOT, "%.3f MB/s", (1000.0*data)/t.elapsed));
496
497 for(int pass=0; pass<passes; pass++){
498 if(mode==READWRITE){
499 fnamesW=makeFnames(pass);
500 t=readWrite(fnamesW, fnamesR);
501 delete(fnamesR);
502 fnamesR=fnamesW;
503 fnamesW=null;
504 }else if(mode==READ){
505 t=read(fnamesR);
506 }else{
507 delete(fnamesR);
508 fnamesW=makeFnames(pass);
509 t=write(fnamesW);
510 fnamesR=fnamesW;
511 fnamesW=null;
512 }
513 System.err.println("Pass "+pass+": \t"+t.toString()+" \t"+String.format(Locale.ROOT, "%.3f MB/s", (1000.0*data)/t.elapsed));
514 }
515 delete(fnamesR);
516
517 t0.stop();
518 System.err.println("Overall: \t"+t0.toString()+" \t"+String.format(Locale.ROOT, "%.3f MB/s", (1000.0*(data*passes))/(t0.elapsed-initialWriteElapsed)));
519
520 if(errorState){
521 throw new RuntimeException(getClass().getName()+" terminated in an error state; the output may be corrupt.");
522 }
523 }
524
525 /*--------------------------------------------------------------*/
526
527
528 /*--------------------------------------------------------------*/
529
530 private String path="";
531
532 /*--------------------------------------------------------------*/
533
534 private int bufferlen=4096;
535 private long data=8000000000L;
536 private int passes=2;
537
538 public int linesInternal;
539
540 private int threads;
541
542 private long maxLines=Long.MAX_VALUE;
543
544 int mode=READWRITE;
545 static final int READWRITE=1, READ=2, WRITE=3;
546
547 boolean printTid=false;
548 boolean processBis=false;
549 boolean preRead=false;
550
551 int method=BYTEFILE;
552 static final int BYTEFILE=1;
553 static final int TEXTFILE=2;
554 static final int BUFFEREDINPUTSTREAM=3;
555 static final int FILEINPUTSTREAM=4;
556 static final int BUFFEREDINPUTSTREAM2=5;
557 static final int FILEINPUTSTREAM2=6;
558 static final int QUICKFILE=7;
559
560 /*--------------------------------------------------------------*/
561
562 /*--------------------------------------------------------------*/
563
564 private PrintStream outstream=System.err;
565 public static boolean verbose=false;
566 public boolean errorState=false;
567 private boolean overwrite=true;
568
569 }