Mercurial > repos > rliterman > csp2

comparison CSP2/CSP2_env/env-d9b9114564458d9d-741b3de822f2aaca6c6caa4325c4afce/opt/bbmap-39.01-1/current/fileIO/ReadWrite.java @ 68:5028fdace37b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 2e9511a184a1ca667c7be0c6321a36dc4e3d116d
author jpayne
date Tue, 18 Mar 2025 16:23:26 -0400
parents
children
comparison
equal deleted inserted replaced
67:0e9998148a16 68:5028fdace37b
1 package fileIO;
2
3 import java.io.BufferedInputStream;
4 import java.io.BufferedOutputStream;
5 import java.io.BufferedReader;
6 import java.io.File;
7 import java.io.FileInputStream;
8 import java.io.FileNotFoundException;
9 import java.io.FileOutputStream;
10 import java.io.IOException;
11 import java.io.InputStream;
12 import java.io.InputStreamReader;
13 import java.io.ObjectInputStream;
14 import java.io.ObjectOutputStream;
15 import java.io.OutputStream;
16 import java.io.PrintWriter;
17 import java.io.Reader;
18 import java.lang.ProcessBuilder.Redirect;
19 import java.net.MalformedURLException;
20 import java.net.URL;
21 import java.util.ArrayList;
22 import java.util.Arrays;
23 import java.util.HashMap;
24 import java.util.HashSet;
25 import java.util.Locale;
26 import java.util.zip.GZIPInputStream;
27 import java.util.zip.GZIPOutputStream;
28 import java.util.zip.ZipEntry;
29 import java.util.zip.ZipInputStream;
30 import java.util.zip.ZipOutputStream;
31
32 import dna.Data;
33 import shared.KillSwitch;
34 import shared.Shared;
35 import shared.Tools;
36 import stream.ConcurrentReadOutputStream;
37 import stream.ConcurrentReadStreamInterface;
38 import stream.MultiCros;
39 import structures.ByteBuilder;
40
41 public class ReadWrite {
42
43
44 public static void main(String[] args){
45 File f=new File(args[1]);
46 assert(!f.exists()) : "Destination file already exists.";
47 copyFile(args[0], args[1]);
48 }
49
50 public static void writeStringInThread(CharSequence x, String fname){
51 writeStringInThread(x, fname, false);
52 }
53
54 public static void writeStringInThread(CharSequence x, String fname, boolean append){
55 addThread(1);
56 new Thread(new WriteStringThread(x, fname, append)).start();
57 }
58
59 public static void writeObjectInThread(Object x, String fname, boolean allowSubprocess){
60 addThread(1);
61 new Thread(new WriteObjectThread(x, fname, allowSubprocess)).start();
62 }
63
64 private static class WriteStringThread implements Runnable{
65
66 private final CharSequence x;
67 private final String fname;
68 private final boolean append;
69 WriteStringThread(CharSequence x_, String fname_, boolean append_){
70 x=x_;
71 fname=fname_;
72 append=append_;
73 }
74
75 @Override
76 public void run() {
77 if(verbose){System.err.println("WriteStringThread.run() started for fname "+fname);}
78 addRunningThread(1);
79 writeStringAsync(x, fname, append);
80 addThread(-1);
81 if(verbose){System.err.println("WriteStringThread.run() finished for fname "+fname);}
82 }
83
84 }
85
86 private static class WriteObjectThread implements Runnable{
87
88 private final Object x;
89 private final String fname;
90 private final boolean allowSubprocess;
91 WriteObjectThread(Object x_, String fname_, boolean allowSubprocess_){
92 x=x_;
93 fname=fname_;
94 allowSubprocess=allowSubprocess_;
95 }
96
97 @Override
98 public void run() {
99 if(verbose){System.err.println("WriteObjectThread.run() started for fname "+fname);}
100 addRunningThread(1);
101 // System.out.println(fname+" began writing.");
102 writeAsync(x, fname, allowSubprocess);
103 // System.out.println(fname+" finished writing.");
104 addThread(-1);
105 // System.out.println(fname+" reports "+countActiveThreads()+" active threads.");
106 if(verbose){System.err.println("WriteObjectThread.run() finished for fname "+fname);}
107 }
108
109 }
110
111 public static boolean setPermissions(String fname, boolean read, boolean write, boolean execute, boolean ownerOnly){
112 File f=new File(fname);
113 if(!f.exists()){return false;}
114 try {
115 f.setReadable(read, ownerOnly);
116 f.setWritable(write, ownerOnly);
117 f.setExecutable(execute, ownerOnly);
118 } catch (Exception e) {
119 return false;
120 }
121 return true;
122 }
123
124 public static void writeString(CharSequence x, String fname){writeString(x, fname, false);}
125 public static void writeString(CharSequence x, String fname, boolean append){
126 if(verbose){System.err.println("writeString(x, "+fname+", "+append+")");}
127 OutputStream os=getOutputStream(fname, append, true, false);
128
129 try {
130
131 synchronized(diskSync){
132 PrintWriter out=new PrintWriter(os);
133 out.print(x);
134 out.flush();
135
136 if(os.getClass()==ZipOutputStream.class){
137 ZipOutputStream zos=(ZipOutputStream)os;
138 zos.closeEntry();
139 zos.finish();
140 }
141 // else if(PROCESS_XZ && os.getClass()==org.tukaani.xz.XZOutputStream.class){
142 // org.tukaani.xz.XZOutputStream zos=(org.tukaani.xz.XZOutputStream)os;
143 // zos.finish();
144 // }
145 out.close();
146 }
147 // System.out.println("Wrote to "+fname);
148
149 // String read=readString(fname);
150 // assert(x.equals(read)) : x.length()+", "+read.length();
151
152 } catch (FileNotFoundException e) {
153 throw new RuntimeException(e);
154 } catch (IOException e) {
155 throw new RuntimeException(e);
156 } catch (OutOfMemoryError e) {
157 KillSwitch.memKill(e);
158 }
159 }
160
161 public static void writeStringAsync(CharSequence x, String fname){writeStringAsync(x, fname, false);}
162 public static void writeStringAsync(CharSequence x, String fname, boolean append){
163 if(verbose){System.err.println("writeStringAsync(x, "+fname+", "+append+")");}
164
165 OutputStream os=getOutputStream(fname, append, true, false);
166
167 try {
168
169 synchronized(diskSync){
170 PrintWriter out=new PrintWriter(os);
171 out.print(x);
172 out.flush();
173
174 if(os.getClass()==ZipOutputStream.class){
175 ZipOutputStream zos=(ZipOutputStream)os;
176 zos.closeEntry();
177 zos.finish();
178 }
179 // else if(PROCESS_XZ && os.getClass()==org.tukaani.xz.XZOutputStream.class){
180 // org.tukaani.xz.XZOutputStream zos=(org.tukaani.xz.XZOutputStream)os;
181 // zos.finish();
182 // }
183 out.close();
184 }
185 // System.out.println("Wrote to "+fname);
186
187 // String read=readString(fname);
188 // assert(x.equals(read)) : x.length()+", "+read.length();
189
190 } catch (FileNotFoundException e) {
191 throw new RuntimeException(e);
192 } catch (IOException e) {
193 throw new RuntimeException(e);
194 } catch (OutOfMemoryError e) {
195 KillSwitch.memKill(e);
196 }
197 }
198
199 public static <X> void write(X x, String fname, boolean allowSubprocess){
200 if(verbose){System.err.println("write(x, "+fname+", "+allowSubprocess+")");}
201
202 OutputStream os=getOutputStream(fname, false, true, allowSubprocess);
203
204 try {
205
206 synchronized(diskSync){
207 ObjectOutputStream out=new ObjectOutputStream(os);
208 out.writeObject(x);
209 close(out);
210 }
211
212 } catch (FileNotFoundException e) {
213 throw new RuntimeException(e);
214 } catch (IOException e) {
215 throw new RuntimeException(e);
216 } catch (OutOfMemoryError e) {
217 KillSwitch.memKill(e);
218 }
219 }
220
221 public static <X> void writeAsync(X x, String fname, boolean allowSubprocess){
222 if(verbose){System.err.println("writeAsync(x, "+fname+", "+allowSubprocess+")");}
223
224 OutputStream os=getOutputStream(fname, false, true, allowSubprocess);
225
226 try {
227
228 ObjectOutputStream out=new ObjectOutputStream(os);
229 out.writeObject(x);
230 close(out);
231
232 } catch (FileNotFoundException e) {
233 throw new RuntimeException(e);
234 } catch (IOException e) {
235 throw new RuntimeException(e);
236 } catch (OutOfMemoryError e) {
237 KillSwitch.memKill(e);
238 }
239 }
240
241 public static final boolean finishReading(InputStream is, String fname, boolean killProcess, Reader...ra){
242 if(verbose){System.err.println("finishReading("+is+", "+fname+", "+killProcess+", "+ra.length+")");}
243 boolean error=false;
244 if(ra!=null){
245 for(Reader r : ra){
246 try {
247 r.close();
248 } catch (IOException e) {
249 error=true;
250 e.printStackTrace();
251 }
252 }
253 }
254 error|=finishReading(is, fname, killProcess);
255 if(verbose){System.err.println("finishReading("+is+", "+fname+", "+killProcess+", "+ra.length+") returned "+error);}
256 return error;
257 }
258
259 public static final boolean finishReading(InputStream is, String fname, boolean killProcess){
260 if(verbose){System.err.println("finishReading("+is+", "+fname+", "+killProcess+")");}
261 boolean error=false;
262 if(is!=System.in){
263 try {
264 is.close();
265 } catch (IOException e) {
266 error=true;
267 // TODO Auto-generated catch block
268 e.printStackTrace();
269 }
270 }
271 if(killProcess && fname!=null && is!=System.in){error|=ReadWrite.killProcess(fname);}
272 if(verbose){System.err.println("finishReading("+is+", "+fname+", "+killProcess+") returned "+error);}
273 return error;
274 }
275
276 // public static final boolean finishWriting(PrintWriter writer, OutputStream outStream, String fname){
277 // return finishWriting(writer, outStream, fname, fname!=null);
278 // }
279
280 public static final boolean finishWriting(PrintWriter writer, OutputStream outStream, String fname, boolean killProcess){
281 if(verbose){System.err.println("finishWriting("+writer+", "+outStream+" , "+fname+", "+killProcess+")");}
282 boolean error=false;
283 if(writer!=null){writer.flush();}
284 close(outStream);
285 if(writer!=null && outStream!=System.out && outStream!=System.err){writer.close();}
286 if(killProcess && fname!=null && outStream!=System.err && outStream!=System.out){error|=ReadWrite.killProcess(fname);}
287 if(verbose){System.err.println("finishWriting("+writer+", "+outStream+" , "+fname+", "+killProcess+") returned "+error);}
288 return error;
289 }
290
291 public static final boolean close(OutputStream os, String fname){
292 if(verbose){System.err.println("close("+os+", "+fname+")");}
293 boolean error=false;
294 if(os!=null){error|=close(os);}
295 if(fname!=null && os!=System.err && os!=System.out){error|=killProcess(fname);}
296 if(verbose){System.err.println("close("+os+", "+fname+") returned "+error);}
297 return error;
298 }
299
300 public static final boolean close(OutputStream os){
301 if(verbose){System.err.println("close("+os+")");}
302 boolean error=false;
303 try {
304 os.flush();
305 } catch (IOException e1) {
306 // TODO Auto-generated catch block
307 e1.printStackTrace();
308 error=true;
309 }
310 if(os.getClass()==ZipOutputStream.class){
311 ZipOutputStream zos=(ZipOutputStream)os;
312 try {
313 zos.closeEntry();
314 zos.finish();
315 } catch (IOException e) {
316 // TODO Auto-generated catch block
317 e.printStackTrace();
318 error=true;
319 }
320 }
321 // else if(PROCESS_XZ && os.getClass()==org.tukaani.xz.XZOutputStream.class){
322 // org.tukaani.xz.XZOutputStream zos=(org.tukaani.xz.XZOutputStream)os;
323 // try {
324 // zos.finish();
325 // } catch (IOException e) {
326 // // TODO Auto-generated catch block
327 // e.printStackTrace();
328 // }
329 // }
330 if(os!=System.out && os!=System.err){
331 try {
332 os.close();
333 } catch (IOException e) {
334 // TODO Auto-generated catch block
335 e.printStackTrace();
336 error=true;
337 }
338 }
339 if(verbose){System.err.println("close("+os+") returned "+error);}
340 return error;
341 }
342
343 public static OutputStream getOutputStream(FileFormat ff, boolean buffered){
344 return getOutputStream(ff.name(), ff.append(), buffered, ff.allowSubprocess());
345 }
346
347 public static OutputStream getOutputStream(String fname, boolean append, boolean buffered, boolean allowSubprocess){
348
349 if(verbose){
350 System.err.println("getOutputStream("+fname+", "+append+", "+buffered+", "+allowSubprocess+")");
351 new Exception().printStackTrace(System.err);
352 }
353
354 // assert(false) : fname; //TODO: for testing
355 // fname=fname.replaceAll("\\\\", "/");
356 fname=fname.replace('\\', '/');
357 assert(fname.indexOf('\\')<0);
358 // assert(!fname.contains("//"));
359
360 {//Create directories if needed.
361 final int index=fname.lastIndexOf('/');
362 if(index>0){
363 File f=new File(fname.substring(0, index+1));
364 if(!f.exists()){f.mkdirs();}
365 }
366 }
367
368 boolean gzipped=fname.endsWith(".gz") || fname.endsWith(".gzip");
369 boolean zipped=fname.endsWith(".zip");
370 boolean bzipped=PROCESS_BZ2 && fname.endsWith(".bz2");
371 boolean xz=PROCESS_XZ && fname.endsWith(".xz");
372 boolean dsrced=fname.endsWith(".dsrc");
373 boolean fqz=USE_FQZ && fname.endsWith(".fqz");
374 boolean alapy=USE_ALAPY && fname.endsWith(".ac");
375
376 // assert(false) : fname;
377
378 allowSubprocess=(allowSubprocess && Shared.threads()>1);
379
380 if(gzipped){
381 // assert(!append);
382 return getGZipOutputStream(fname, append, allowSubprocess);
383 }else if(zipped){
384 assert(!append) : "Append is not allowed for zip archives.";
385 return getZipOutputStream(fname, buffered, allowSubprocess);
386 }else if(bzipped){
387 assert(!append) : "Append is not allowed for bz2 archives.";//TODO: This might be OK; try it.
388 return getBZipOutputStream(fname, buffered, append, allowSubprocess);
389 }else if(xz){
390 assert(!append) : "Append is not allowed for xz archives.";
391 return getXZOutputStream(fname, buffered, allowSubprocess);
392 }else if(dsrced){
393 assert(!append) : "Append is not allowed for dsrc archives.";
394 return getDsrcOutputStream(fname, buffered, allowSubprocess);
395 }else if(fqz){
396 assert(!append) : "Append is not allowed for fqz archives.";
397 return getFqzStream(fname);
398 }else if(alapy){
399 assert(!append) : "Append is not allowed for alapy archives.";
400 return getAlapyStream(fname);
401 }
402 return getRawOutputStream(fname, append, buffered);
403 }
404
405 public static OutputStream getRawOutputStream(String fname, boolean append, boolean buffered){
406
407 if(verbose){System.err.println("getRawOutputStream("+fname+", "+append+", "+buffered+")");}
408
409 if(fname.equals("stdout") || fname.startsWith("stdout.")){
410 return System.out;
411 }else if(fname.equals("stderr") || fname.startsWith("stderr.")){
412 return System.err;
413 }else if(fname.startsWith("/dev/null/")){
414 fname="/dev/null/";
415 }
416
417 if(fname.indexOf('|')>=0){fname=fname.replace('|', '_');}
418
419 FileOutputStream fos=null;
420 try {
421 fos = new FileOutputStream(fname, append);
422 } catch (FileNotFoundException e) {
423 synchronized(ReadWrite.class){
424 try {
425 File f=new File(fname);
426 String parent=f.getParent();
427
428 if(parent!=null){
429 f=new File(parent);
430 if(!f.exists()){
431 boolean b=f.mkdirs();
432 if(!b){System.err.println("Warning - could not create directory "+f.getAbsolutePath());}
433 }
434 }
435 fos = new FileOutputStream(fname, append);
436 } catch (Exception e2) {
437 throw new RuntimeException(e2);
438 }
439 }
440 }
441 assert(fos!=null);
442 if(buffered){return new BufferedOutputStream(fos);}
443 return fos;
444 }
445
446 public static OutputStream getXZOutputStream(String fname, boolean buffered, boolean allowSubprocess){
447 final OutputStream raw=getRawOutputStream(fname, false, buffered);
448 if(RAWMODE){return raw;}
449 throw new RuntimeException("Unsupported format: XZ");
450 // try {
451 // org.tukaani.xz.LZMA2Options options = new org.tukaani.xz.LZMA2Options();
452 // options.setPreset(ZIPLEVEL);
453 // org.tukaani.xz.XZOutputStream out=new org.tukaani.xz.XZOutputStream(raw, options);
454 // return out;
455 // } catch (IOException e) {
456 // // TODO Auto-generated catch block
457 // e.printStackTrace();
458 // }
459 // assert(false);
460 // return null;
461 }
462
463 public static OutputStream getBZipOutputStream(String fname, boolean buffered, boolean append, boolean allowSubprocess){
464 if(verbose){System.err.println("getBZipOutputStream("+fname+", "+buffered+", "+append+", "+allowSubprocess+")");}
465 // assert(false) : ReadWrite.ZIPLEVEL+", "+Shared.threads()+", "+MAX_ZIP_THREADS+", "+ZIP_THREAD_MULT+", "+allowSubprocess+", "+USE_PIGZ+", "+Data.PIGZ();
466
467 if(RAWMODE){
468 final OutputStream raw=getRawOutputStream(fname, false, buffered);
469 return raw;
470 }
471
472 if(USE_LBZIP2 && Data.LBZIP2()){return getLbzip2Stream(fname, append);}
473 if(USE_PBZIP2 && Data.PBZIP2()){return getPbzip2Stream(fname, append);}
474 if(USE_BZIP2 && Data.BZIP2()){return getBzip2Stream(fname, append);}
475
476 throw new RuntimeException("bz2 compression not supported in this version, unless lbzip2, pbzip2 or bzip2 is installed.");
477
478
479 // getBzip2Stream
480
481 // {//comment to disable BZip2
482 // try {
483 // raw.write('B');
484 // raw.write('Z');
485 // CBZip2OutputStream out=new CBZip2OutputStream(raw, 8192);
486 // return out;
487 // } catch (IOException e) {
488 // // TODO Auto-generated catch block
489 // e.printStackTrace();
490 // }
491 // assert(false);
492 // return null;
493 // }
494 }
495
496 public static OutputStream getDsrcOutputStream(String fname, boolean buffered, boolean append){
497 if(verbose){System.err.println("getDsrcOutputStream("+fname+", "+buffered+", "+append+")");}
498 if(RAWMODE){
499 final OutputStream raw=getRawOutputStream(fname, false, buffered);
500 return raw;
501 }
502
503 if(USE_DSRC && Data.DSRC() /*&& (Data.SH() || fname.equals("stdout") || fname.startsWith("stdout."))*/){return getDsrcOutputStream2(fname, append);}
504
505 throw new RuntimeException("dsrc compression requires dsrc in the path.");
506 }
507
508 public static OutputStream getZipOutputStream(String fname, boolean buffered, boolean allowSubprocess){
509 if(verbose){System.err.println("getZipOutputStream("+fname+", "+buffered+", "+allowSubprocess+")");}
510 final OutputStream raw=getRawOutputStream(fname, false, buffered);
511 if(RAWMODE){return raw;}
512 try {
513 ZipOutputStream out=new ZipOutputStream(raw);
514 out.setLevel(Tools.min(ZIPLEVEL, 9));
515 final String basename=basename(fname);
516 out.putNextEntry(new ZipEntry(basename));
517 return out;
518 } catch (IOException e) {
519 // TODO Auto-generated catch block
520 e.printStackTrace();
521 }
522 assert(false);
523 return null;
524 }
525
526 public static OutputStream getGZipOutputStream(String fname, boolean append, boolean allowSubprocess){
527 if(verbose){System.err.println("getGZipOutputStream("+fname+", "+append+", "+allowSubprocess+"); "+USE_BGZIP+", "+USE_PIGZ+", "+USE_GZIP+", "+RAWMODE);}
528 // assert(false) : ReadWrite.ZIPLEVEL+", "+Shared.threads()+", "+MAX_ZIP_THREADS+", "+ZIP_THREAD_MULT+", "+allowSubprocess+", "+USE_PIGZ+", "+Data.PIGZ();
529
530 if(FORCE_BGZIP && USE_BGZIP && Data.BGZIP()){return getBgzipStream(fname, append);}
531
532 if(FORCE_PIGZ || (allowSubprocess && Shared.threads()>=2)){
533 if((fname.endsWith(".vcf.gz") || fname.endsWith(".sam.gz") || (PREFER_BGZIP && ZIPLEVEL<10)) && USE_BGZIP && Data.BGZIP()){return getBgzipStream(fname, append);}
534 if(USE_PIGZ && Data.PIGZ()){return getPigzStream(fname, append);}
535 if(USE_BGZIP && Data.BGZIP()){return getBgzipStream(fname, append);}
536 if(USE_GZIP && Data.GZIP()/* && (Data.SH() /*|| fname.equals("stdout") || fname.startsWith("stdout."))*/){return getGzipStream(fname, append);}
537 }
538
539 final OutputStream raw=getRawOutputStream(fname, append, false);
540 if(RAWMODE){return raw;}
541 try {
542 final GZIPOutputStream out=new GZIPOutputStream(raw, 8192){
543 {
544 // def.setLevel(Deflater.DEFAULT_COMPRESSION);
545 def.setLevel(Tools.min(ZIPLEVEL, 9));
546 }
547 };
548 return out;
549 } catch (IOException e) {
550 // TODO Auto-generated catch block
551 e.printStackTrace();
552 }
553 assert(false);
554 return null;
555 }
556
557 public static OutputStream getPigzStream(String fname, boolean append){
558 if(verbose){System.err.println("getPigzStream("+fname+")");}
559 // System.err.println(MAX_ZIP_THREADS); //123
560 int threads=Tools.min(MAX_ZIP_THREADS, Tools.max((int)((Shared.threads()+1)*ZIP_THREAD_MULT), 1));
561 // System.err.println(threads); //123
562 threads=Tools.max(1, Tools.min(Shared.threads(), threads));
563 // System.err.println(threads); //123
564 int zl=(ZIPLEVEL<10 ? ZIPLEVEL : Data.PIGZ_VERSION_23plus ? 11 : 9);
565 if(ALLOW_ZIPLEVEL_CHANGE && threads>=4 && zl>0 && zl<4){zl=4;}
566 if(zl<3){threads=Tools.min(threads, 12);}
567 else if(zl<5){threads=Tools.min(threads, 24);}
568 else if(zl<7){threads=Tools.min(threads, 40);}
569 // System.err.println(threads); //123
570 OutputStream out;
571 String command="pigz -c -p "+threads+" -"+zl;
572 if(PIGZ_BLOCKSIZE!=128){
573 command=command+" -b "+PIGZ_BLOCKSIZE;
574 }
575 if(PIGZ_ITERATIONS>0 && Data.PIGZ_VERSION_231plus){
576 command=command+" -I "+PIGZ_ITERATIONS;
577 }
578
579 // System.err.println("*** "+command);
580
581 //Sample command on command line, without piping: pigz -11 -k -f -b 256 -I 25000 file.fa
582 // assert(false) : MAX_ZIP_THREADS+", "+Shared.threads()+", "+ZIP_THREAD_MULT+", "+ZIPLEVEL+", "+command;
583 out=getOutputStreamFromProcess(fname, command, true, append, true, true);
584
585 return out;
586 }
587
588 public static OutputStream getFqzStream(String fname){
589 if(verbose){System.err.println("getFqzStream("+fname+")");}
590 String command="fqz_comp -s"+Tools.mid(1, ZIPLEVEL, 8)+"+"; //9 gives bad compression
591 if(ZIPLEVEL>5){command=command+" -q3";}
592 //if(ZIPLEVEL>5){command=command+" -b -q3";} //b does not seem to work
593 OutputStream out=getOutputStreamFromProcess(fname, command, true, false, true, true);
594 return out;
595 }
596
597 public static OutputStream getAlapyStream(String fname){
598 if(verbose){System.err.println("getAlapyStream("+fname+")");}
599 String compression=(ZIPLEVEL>6 ? "-l best" : ZIPLEVEL<4 ? "-l fast" : "-l medium");
600 // String compression="";
601 String command="alapy_arc "+compression+" -n "+fname+" -q -c - ";
602 OutputStream out=getOutputStreamFromProcess(fname, command, true, false, true, false);
603 return out;
604 }
605
606 public static OutputStream getGzipStream(String fname, boolean append){
607 if(verbose){System.err.println("getGzipStream("+fname+")");}
608 OutputStream out=getOutputStreamFromProcess(fname, "gzip -c -"+Tools.min(ZIPLEVEL, 9), true, append, true, true);
609 return out;
610 }
611
612 public static OutputStream getBgzipStream(String fname, boolean append){
613 if(verbose){System.err.println("getBgzipStream("+fname+")");}
614
615 int threads=Tools.min(MAX_ZIP_THREADS, Tools.max((int)((Shared.threads()+1)*ZIP_THREAD_MULT), 1));
616 // System.err.println(threads); //123
617 threads=Tools.max(1, Tools.min(Shared.threads(), threads));
618 // System.err.println(threads); //123
619 int zl=Tools.mid(ZIPLEVEL, 1, 9);
620 if(ALLOW_ZIPLEVEL_CHANGE && threads>=4 && zl>0 && zl<4){zl=4;}
621 if(zl<3){threads=Tools.min(threads, 12);}
622 else if(zl<5){threads=Tools.min(threads, 16);}
623 else if(zl<7){threads=Tools.min(threads, 40);}
624
625 // assert(false) : Data.BGZIP()+", "+Data.PIGZ();
626 String command="bgzip -c "+(append ? "" : "-f ")+(Data.BGZIP_VERSION_levelFlag ? "-l "+zl+" " : "")+(Data.BGZIP_VERSION_threadsFlag ? "-@ "+threads+" " : "");
627 if(verbose){System.err.println(command);}
628 OutputStream out=getOutputStreamFromProcess(fname, command, true, append, true, true);
629 if(verbose){System.err.println("fetched bgzip stream.");}
630 return out;
631 }
632
633 public static OutputStream getBzip2Stream(String fname, boolean append){
634 if(verbose){System.err.println("getBzip2Stream("+fname+")");}
635 String command="bzip2 -c -"+Tools.min(BZIPLEVEL, 9);
636 OutputStream out=getOutputStreamFromProcess(fname, command, true, append, true, true);
637 return out;
638 }
639
640 public static OutputStream getPbzip2Stream(String fname, boolean append){
641 if(verbose){System.err.println("getPbzip2Stream("+fname+")");}
642 int threads=Tools.min(MAX_ZIP_THREADS, Tools.max((int)((Shared.threads()+1)*ZIP_THREAD_MULT), 1));
643 threads=Tools.max(1, Tools.min(Shared.threads(), threads));
644 String command="pbzip2 -c -p"+threads+" -"+Tools.min(BZIPLEVEL, 9);
645 OutputStream out=getOutputStreamFromProcess(fname, command, true, append, true, true);
646 return out;
647 }
648
649 public static OutputStream getLbzip2Stream(String fname, boolean append){
650 if(verbose){System.err.println("getLbzip2Stream("+fname+")");}
651 String command="lbzip2 -"+Tools.min(BZIPLEVEL, 9);
652 OutputStream out=getOutputStreamFromProcess(fname, command, true, append, true, true);
653 return out;
654 }
655
656 public static OutputStream getDsrcOutputStream2(String fname, boolean append){
657 if(verbose){System.err.println("getDsrcOutpustream2("+fname+")");}
658 int threads=Tools.min(MAX_ZIP_THREADS, Tools.max((int)((Shared.threads()+1)*ZIP_THREAD_MULT), 1));
659 threads=Tools.max(1, Tools.min(Shared.threads()-1, threads));
660 String params=null;
661 if(ZIPLEVEL<=2){
662 params="-d0 -q0 -b8";
663 }else if(ZIPLEVEL<=4){
664 params="-d1 -q1 -b16";
665 }else if(ZIPLEVEL<=8){
666 params="-d2 -q2 -b32";
667 }else{
668 params="-d3 -q2 -b64";
669 }
670 String command="dsrc c -t"+threads+" "+params+" -s";
671 if(fname.equals("stdout") || fname.startsWith("stdout.")){
672 //???
673 assert(false) : "Undefined dsrc option.";
674 }else{
675 command+=" "+fname;
676 }
677 System.err.println(command);//123
678 // OutputStream out=getOutputStreamFromProcess(fname, command, true, append, true);
679 OutputStream out=getOutputStreamFromProcess(fname, command+" "+fname, true, append, true, false);
680 return out;
681 }
682
683 public static OutputStream getOutputStreamFromProcess(final String fname, final String command, boolean sh, boolean append, boolean useProcessBuilder, boolean useFname){
684 if(verbose){System.err.println("getOutputStreamFromProcess("+fname+", "+command+", "+sh+", "+useProcessBuilder+")");}
685
686 OutputStream out=null;
687 Process p=null;
688 if(useProcessBuilder){
689 ProcessBuilder pb=new ProcessBuilder();
690 pb.redirectError(Redirect.INHERIT);
691
692 if(fname.equals("stdout") || fname.startsWith("stdout.")){
693 pb.redirectOutput(Redirect.INHERIT);
694 pb.command(command.split(" "));
695 }else{
696
697 if(useFname){
698 if(append){
699 pb.redirectOutput(ProcessBuilder.Redirect.appendTo(new File(fname)));
700 }else{
701 pb.redirectOutput(new File(fname));
702 }
703 }
704
705 pb.command(command.split(" "));
706 }
707 try {
708 p=pb.start();
709 } catch (IOException e) {
710 // TODO Auto-generated catch block
711 e.printStackTrace();
712 }
713 assert(p!=null) : "Could not execute "+command;
714 addProcess(fname, p);
715 out=p.getOutputStream();
716 {
717 out=p.getOutputStream();
718 InputStream es=p.getErrorStream();
719 assert(es!=null);
720 PipeThread et=new PipeThread(es, System.err);
721 addPipeThread(fname, et);
722 et.start();
723 }
724 return out;
725 }
726
727 if(fname.equals("stdout") || fname.startsWith("stdout.")){
728 try {
729 p = Runtime.getRuntime().exec(command);
730 } catch (IOException e) {
731 // TODO Auto-generated catch block
732 e.printStackTrace();
733 }
734 assert(p!=null) : "Could not execute "+command;
735 InputStream is=p.getInputStream();
736 PipeThread it=new PipeThread(is, System.out);
737 addPipeThread(fname, it);
738 it.start();
739 // }else if(fname.equals("stderr") || fname.startsWith("stderr.")){
740 // try {
741 // p = Runtime.getRuntime().exec(command);
742 // } catch (IOException e) {
743 // // TODO Auto-generated catch block
744 // e.printStackTrace();
745 // }
746 // InputStream is=p.getErrorStream();
747 // PipeThread it=new PipeThread(is, System.err);
748 // it.start();
749 }else{
750 try {
751 if(sh){
752 String[] cmd = {
753 "sh",
754 "-c",
755 command+(useFname ? " 1"+(append ? ">>" : ">")+fname : "")
756 };
757 p=Runtime.getRuntime().exec(cmd);
758 }else{
759 //TODO: append won't work here...
760 assert(false) : command;
761 p=Runtime.getRuntime().exec(command);
762 }
763 } catch (IOException e) {
764 // TODO Auto-generated catch block
765 e.printStackTrace();
766 }
767 }
768 assert(p!=null) : "Could not execute "+command;
769 addProcess(fname, p);
770 out=p.getOutputStream();
771 InputStream es=p.getErrorStream();
772 assert(es!=null);
773 PipeThread et=new PipeThread(es, System.err);
774 addPipeThread(fname, et);
775 et.start();
776
777 return out;
778 }
779
780 public static String readString(String fname){
781 if(verbose){System.err.println("readString("+fname+")");}
782 String x=null;
783 InputStream is=getInputStream(fname, false, false);
784
785 try {
786
787 StringBuilder sb=new StringBuilder();
788
789 // synchronized(diskSync){
790 BufferedReader in=new BufferedReader(new InputStreamReader(is), INBUF);
791 String temp=in.readLine();
792 while(temp!=null){
793 sb.append(temp).append('\n');
794 temp=in.readLine();
795 }
796 in.close();
797 // }
798
799 x=sb.toString();
800 } catch (FileNotFoundException e) {
801 throw new RuntimeException(e);
802 } catch (IOException e) {
803 throw new RuntimeException(e);
804 } catch (OutOfMemoryError e) {
805 KillSwitch.memKill(e);
806 }
807
808 return x;
809 }
810
811 public static Object readObject(String fname, boolean allowSubprocess){
812 if(verbose){System.err.println("readObject("+fname+")");}
813 Object x=null;
814 InputStream is=getInputStream(fname, true, allowSubprocess);
815
816 try {
817 // synchronized(diskSync){
818 ObjectInputStream in=new ObjectInputStream(is);
819 x=in.readObject();
820 in.close();
821 // }
822 } catch (IOException e) {
823 throw new RuntimeException(e);
824 } catch (ClassNotFoundException e) {
825 throw new RuntimeException(e);
826 } catch (OutOfMemoryError e) {
827 KillSwitch.memKill(e);
828 }
829
830 return x;
831 }
832
833 public static InputStream getInputStream(String fname, boolean buffer, boolean allowSubprocess){
834 if(verbose){System.err.println("getInputStream("+fname+", "+buffer+", "+allowSubprocess+")");}
835 boolean xz=fname.endsWith(".xz");
836 boolean gzipped=fname.endsWith(".gz") || fname.endsWith(".gzip");
837 boolean zipped=fname.endsWith(".zip");
838 boolean bzipped=PROCESS_BZ2 && fname.endsWith(".bz2");
839 boolean dsrced=fname.endsWith(".dsrc");
840 boolean bam=fname.endsWith(".bam") && (SAMBAMBA() || Data.SAMTOOLS());
841 boolean fqz=fname.endsWith(".fqz");
842 boolean alapy=fname.endsWith(".ac");
843
844 allowSubprocess=(allowSubprocess && Shared.threads()>1);
845
846 if(!RAWMODE){
847 if(zipped){return getZipInputStream(fname);}
848 if(gzipped){return getGZipInputStream(fname, allowSubprocess, false);}
849 if(bzipped){return getBZipInputStream(fname, allowSubprocess);}
850 if(dsrced){return getDsrcInputStream(fname);}
851 if(bam){
852 if(SAMBAMBA()){
853 String command="sambamba -q view -h";
854 // new Exception().printStackTrace(); //123
855 if(SAMTOOLS_IGNORE_FLAG!=0){
856 command=command+" --num-filter=0/"+SAMTOOLS_IGNORE_FLAG;
857 }
858 return getInputStreamFromProcess(fname, command, false, true, true);
859 }else{
860 String command="samtools view -h";
861 // new Exception().printStackTrace(); //123
862 if(SAMTOOLS_IGNORE_FLAG!=0){
863 // command=command+" -F 4";
864 command=command+" -F 0x"+Integer.toHexString(SAMTOOLS_IGNORE_FLAG);
865 }
866 String version=Data.SAMTOOLS_VERSION;
867 if(Shared.threads()>1 && version!=null && version.startsWith("1.") && version.length()>2){
868 try {
869 String[] split=version.split("\\.");
870 int number=-1;
871 try {
872 number=Integer.parseInt(split[1]);
873 } catch (Exception e) {}
874 if(number<0){
875 try {
876 number=Integer.parseInt(split[1].substring(0, 1));
877 } catch (Exception e1) {}
878 }
879 if(number>3){
880 command=command+" -@ 2";
881 }
882 } catch (NumberFormatException e) {
883 // TODO Auto-generated catch block
884 e.printStackTrace();
885 }
886 }
887 // System.err.println(command);
888 return getInputStreamFromProcess(fname, command, false, true, true);
889 }
890 }
891
892 if(fqz){return getInputStreamFromProcess(fname, "fqz_comp -d ", false, true, true);}
893 if(alapy){
894 return getInputStreamFromProcess(fname, "alapy_arc -q -d "+fname+" -", false, false, true);
895 }
896 }
897
898 return getRawInputStream(fname, buffer);
899 }
900
901 public static InputStream getRawInputStream(String fname, boolean buffer){
902 if(verbose){System.err.println("getRawInputStream("+fname+", "+buffer+")");}
903
904 assert(fname!=null);
905 fname=fname.replace('\\', '/');
906 assert(fname.indexOf('\\')<0);
907 assert(!fname.contains("\\\\"));
908 // assert(!fname.contains("//")) : fname;
909
910 final boolean jar=fname.startsWith("jar:");
911
912 if(!jar){
913 boolean failed=false;
914 File f=new File(fname);
915 if(!f.exists()){
916 String f2=fname.toLowerCase();
917 if(f2.equals("stdin") || f2.startsWith("stdin.")){
918 // System.err.println("Returning stdin: A");
919 return System.in;
920 }
921
922 if(fname.indexOf('/')<0){
923 f2=Data.ROOT_CURRENT+"/"+fname;
924 if(!new File(f2).exists()){
925 failed=true;
926 }else{
927 fname=f2;
928 }
929 }else{
930 failed=true;
931 }
932 }
933 // if(failed){throw new RuntimeException("Can't find file "+fname);}
934 if(failed){
935 shared.KillSwitch.exceptionKill(new RuntimeException("Can't find file "+fname));
936 }
937 }
938
939 // System.err.println("Getting input stream for "+fname);
940 // assert(!fname.contains("\\"));
941 // assert(!loadedFiles.contains(fname)) : "Already loaded "+fname;
942 // loadedFiles.add(fname);
943
944 InputStream in=null;
945 if(jar){
946 try {
947
948 URL url=new URL(fname);
949
950 InputStream is=url.openStream();
951
952 if(buffer){
953 BufferedInputStream bis=new BufferedInputStream(is, INBUF);
954 in=bis;
955 }else{
956 in=is;
957 }
958
959 } catch (FileNotFoundException e) {
960 System.err.println("Error when attempting to read "+fname);
961 throw new RuntimeException(e);
962 } catch (MalformedURLException e) {
963 System.err.println("Error when attempting to read "+fname);
964 throw new RuntimeException(e);
965 } catch (IOException e) {
966 System.err.println("Error when attempting to read "+fname);
967 throw new RuntimeException(e);
968 }
969 }else{
970 try {
971
972 FileInputStream fis=new FileInputStream(fname);
973
974 if(buffer){
975 BufferedInputStream bis=new BufferedInputStream(fis, INBUF);
976 in=bis;
977 }else{
978 in=fis;
979 }
980
981 } catch (FileNotFoundException e) {
982 throw new RuntimeException(e);
983 }
984 }
985
986 return in;
987 }
988
989 public static InputStream getZipInputStream(String fname){return getZipInputStream(fname, true);}
990 public static InputStream getZipInputStream(String fname, boolean buffer){
991 if(verbose){System.err.println("getZipInputStream("+fname+", "+buffer+")");}
992 InputStream raw=getRawInputStream(fname, buffer);
993 InputStream in=null;
994
995 final String basename=basename(fname);
996
997 try {
998
999 ZipInputStream zis=new ZipInputStream(raw);
1000 ZipEntry ze=zis.getNextEntry();
1001 assert(ze!=null);
1002 assert(basename.equals(ze.getName())) : basename+" != "+ze.getName();
1003 in=zis;
1004
1005 } catch (FileNotFoundException e) {
1006 System.err.println("Error when attempting to read "+fname);
1007 throw new RuntimeException(e);
1008 } catch (IOException e) {
1009 System.err.println("Error when attempting to read "+fname);
1010 throw new RuntimeException(e);
1011 }
1012
1013 return in;
1014 }
1015
1016 public static InputStream getGZipInputStream(String fname, boolean allowSubprocess, boolean buffer){
1017 if(verbose){
1018 System.err.println("getGZipInputStream("+fname+", "+allowSubprocess+")");
1019 // new Exception().printStackTrace(System.err);
1020 }
1021 // assert(!fname.contains("temp")) : fname+", "+USE_UNBGZIP+", "+allowSubprocess;
1022 if(allowSubprocess && Shared.threads()>2){
1023 if(!fname.startsWith("jar:")){
1024 if(verbose){
1025 System.err.println("Fetching gzip input stream: "+fname+", allowSubprocess="+allowSubprocess+", USE_UNPIGZ="+USE_UNPIGZ+", Data.PIGZ()="+Data.PIGZ());
1026 }
1027 if((PREFER_UNBGZIP || fname.endsWith(".vcf.gz")) && USE_UNBGZIP && Data.BGZIP()){
1028 if(!fname.contains("stdin") && new File(fname).exists()){
1029 int magicNumber=getMagicNumber(fname);
1030 if(magicNumber==529205252){return getUnbgzipStream(fname);}
1031 // System.err.println(magicNumber);
1032 }
1033 }
1034 if(USE_UNPIGZ && Data.PIGZ()){return getUnpigzStream(fname);}
1035 // if(USE_UNBGZIP && Data.BGZIP()){return getUnbgzipStream(fname);}
1036 if(USE_GUNZIP && Data.GUNZIP()){return getGunzipStream(fname);}
1037 }
1038 }
1039 // assert(false) : "allowSubprocess="+allowSubprocess+", Shared.threads()="+Shared.threads()+", fname="+fname+"\n"
1040 // +"PREFER_UNBGZIP="+PREFER_UNBGZIP+", "+"USE_UNBGZIP="+USE_UNBGZIP+", "+"Data.BGZIP()="+Data.BGZIP()+"\n"
1041 // +"USE_UNPIGZ="+USE_UNPIGZ+", "+"Data.PIGZ()="+Data.PIGZ()+"\n";
1042 InputStream raw=getRawInputStream(fname, buffer);//123
1043 InputStream in=null;
1044 try {
1045 in=new GZIPInputStream(raw, INBUF);
1046 } catch (FileNotFoundException e) {
1047 System.err.println("Error when attempting to read "+fname);
1048 throw new RuntimeException(e);
1049 } catch (IOException e) {
1050 System.err.println("Error when attempting to read "+fname);
1051 throw new RuntimeException(e);
1052 }
1053
1054 return in;
1055 }
1056
1057 public static InputStream getGunzipStream(String fname){
1058 if(verbose){System.err.println("getGunzipStream("+fname+")");}
1059 return getInputStreamFromProcess(fname, "gzip -c -d", false, true, true);
1060 }
1061
1062 public static InputStream getUnpigzStream(String fname){
1063 if(verbose){System.err.println("getUnpigzStream("+fname+")");}
1064 return getInputStreamFromProcess(fname, "pigz -c -d", false, true, true);
1065 }
1066
1067 public static InputStream getUnbgzipStream(String fname){
1068 if(verbose){System.err.println("getUnbgzipStream("+fname+")");}
1069 int threads=Tools.mid(4, 1, Shared.threads());
1070 return getInputStreamFromProcess(fname, "bgzip -c -d"+(Data.BGZIP_VERSION_threadsFlag ? " -@ "+threads : ""), false, true, true);
1071 }
1072
1073 public static InputStream getUnpbzip2Stream(String fname){
1074 if(verbose){System.err.println("getUnpbzip2Stream("+fname+")");}
1075 return getInputStreamFromProcess(fname, "pbzip2 -c -d", false, true, true);
1076 }
1077
1078 public static InputStream getUnlbzip2Stream(String fname){
1079 if(verbose){System.err.println("getUnlbzip2Stream("+fname+")");}
1080 return getInputStreamFromProcess(fname, "lbzip2 -c -d", false, true, true);
1081 }
1082
1083 public static InputStream getUnbzip2Stream(String fname){
1084 if(verbose){System.err.println("getUnbzip2Stream("+fname+")");}
1085 return getInputStreamFromProcess(fname, "bzip2 -c -d", false, true, true);
1086 }
1087
1088 public static InputStream getUnDsrcStream(String fname){
1089 if(verbose){System.err.println("getUnDsrcStream("+fname+")");}
1090 int threads=Tools.min(MAX_ZIP_THREADS, Tools.max((int)((Shared.threads()+1)*ZIP_THREAD_MULT), 1));
1091 threads=Tools.max(1, Tools.min(Shared.threads()-1, threads));
1092 return getInputStreamFromProcess(fname, "dsrc d -s -t"+threads, false, true, true);
1093 }
1094
1095
1096 public static InputStream getInputStreamFromProcess(final String fname, String command, boolean cat, final boolean appendFname, final boolean useProcessBuilder){
1097 if(verbose){System.err.println("getInputStreamFromProcess("+fname+", "+command+", "+cat+")");}
1098
1099 //InputStream raw=getRawInputStream(fname, false);
1100 InputStream in=null;
1101
1102 Process p=null;
1103
1104 if(useProcessBuilder){
1105 ProcessBuilder pb=new ProcessBuilder();
1106 pb.redirectError(Redirect.INHERIT);
1107
1108 if(fname.equals("stdin") || fname.startsWith("stdin.")){
1109 pb.redirectInput(Redirect.INHERIT);
1110 pb.command(command.split(" "));
1111 }else{
1112 if(appendFname){
1113 command=command+" "+fname;
1114 }else{
1115 pb.redirectInput(new File(fname));
1116 }
1117 // System.err.println(command+", "+appendFname);
1118 pb.command(command.split(" "));
1119 }
1120 try {
1121 p=pb.start();
1122 } catch (IOException e) {
1123 // TODO Auto-generated catch block
1124 e.printStackTrace();
1125 }
1126 assert(p!=null) : "Could not execute "+command;
1127 addProcess(fname, p);
1128 in=p.getInputStream();
1129 // {
1130 // out=p.getOutputStream();
1131 // InputStream es=p.getErrorStream();
1132 // assert(es!=null);
1133 // PipeThread et=new PipeThread(es, System.err);
1134 // addPipeThread(fname, et);
1135 // et.start();
1136 // }
1137 return in;
1138 }
1139
1140 if(!appendFname){
1141 try {
1142 p=Runtime.getRuntime().exec(command);
1143 } catch (IOException e) {
1144 // TODO Auto-generated catch block
1145 e.printStackTrace();
1146 }
1147 }else if(fname.equals("stdin") || fname.startsWith("stdin.")){
1148 try {
1149 if(cat){
1150 throw new RuntimeException();
1151 }else{
1152 p=Runtime.getRuntime().exec(command);
1153 }
1154 } catch (IOException e) {
1155 // TODO Auto-generated catch block
1156 e.printStackTrace();
1157 }
1158 assert(p!=null) : "Could not execute "+command;
1159 OutputStream os=p.getOutputStream();
1160 PipeThread it=new PipeThread(System.in, os);
1161 addPipeThread(fname, it);
1162 it.start();
1163 }else{
1164 try {
1165 if(cat){
1166 assert(false) : "This mode is untested.";
1167 String[] cmd = {
1168 "sh","cat "+fname,
1169 " | "+command
1170 };
1171 p=Runtime.getRuntime().exec(cmd);
1172 }else{
1173 p = Runtime.getRuntime().exec(command+" "+fname);
1174 }
1175 } catch (IOException e) {
1176 // TODO Auto-generated catch block
1177 e.printStackTrace();
1178 }
1179 }
1180 assert(p!=null) : "Could not execute "+command;
1181
1182 addProcess(fname, p);
1183 in=p.getInputStream();
1184 InputStream es=p.getErrorStream();
1185 assert(es!=null);
1186 PipeThread et=new PipeThread(es, System.err);
1187 addPipeThread(fname, et);
1188 et.start();
1189
1190 return in;
1191 }
1192
1193
1194 public static InputStream getBZipInputStream(String fname, boolean allowSubprocess){
1195 if(verbose){System.err.println("getBZipInputStream("+fname+")");}
1196 InputStream in=null;
1197
1198 try {in=getBZipInputStream2(fname, allowSubprocess);}
1199 catch (IOException e) {
1200 System.err.println("Error when attempting to read "+fname);
1201 throw new RuntimeException(e);
1202 }catch (NullPointerException e) {
1203 System.err.println("Error when attempting to read "+fname);
1204 throw new RuntimeException(e);
1205 }
1206
1207 assert(in!=null);
1208 return in;
1209 }
1210
1211 private static InputStream getBZipInputStream2(String fname, boolean allowSubprocess) throws IOException{
1212 if(verbose){
1213 if(verbose){System.err.println("getBZipInputStream("+fname+")");}
1214 }
1215
1216 if(!fname.startsWith("jar:")){
1217 if(verbose){System.err.println("Fetching bz2 input stream: "+fname+", "+USE_PBZIP2+", "+USE_BZIP2+", "+Data.PBZIP2()+Data.BZIP2());}
1218 if(USE_LBZIP2 && Data.LBZIP2()){return getUnlbzip2Stream(fname);}
1219 if(USE_PBZIP2 && Data.PBZIP2()){return getUnpbzip2Stream(fname);}
1220 if(USE_BZIP2 && Data.BZIP2()){return getUnbzip2Stream(fname);}
1221 }
1222
1223 throw new IOException("\nlbzip2, pbzip2, or bzip2 must be in the path to read bz2 files:\n"+fname+"\n");
1224 }
1225
1226 public static InputStream getDsrcInputStream(String fname){
1227 if(verbose){System.err.println("getDsrcInputStream("+fname+")");}
1228 InputStream in=null;
1229
1230 try {in=getDsrcInputStream2(fname);}
1231 catch (IOException e) {
1232 System.err.println("Error when attempting to read "+fname);
1233 throw new RuntimeException(e);
1234 }catch (NullPointerException e) {
1235 System.err.println("Error when attempting to read "+fname);
1236 throw new RuntimeException(e);
1237 }
1238
1239 assert(in!=null);
1240 return in;
1241 }
1242
1243 private static InputStream getDsrcInputStream2(String fname) throws IOException{
1244 if(verbose){
1245 if(verbose){System.err.println("getDsrcInputStream2("+fname+")");}
1246 }
1247
1248 if(USE_DSRC && Data.DSRC()){return getUnDsrcStream(fname);}
1249
1250 throw new IOException("\nDsrc must be in the path to read Dsrc files:\n"+fname+"\n");
1251 }
1252
1253 public static InputStream getXZInputStream(String fname){
1254
1255 InputStream in=null;
1256
1257 // if(PROCESS_XZ){
1258 // InputStream raw=getRawInputStream(fname, true);
1259 // try {
1260 // in=new org.tukaani.xz.XZInputStream(raw);
1261 // } catch (FileNotFoundException e) {
1262 // throw new RuntimeException(e);
1263 // } catch (IOException e) {
1264 // throw new RuntimeException(e);
1265 // }
1266 // }
1267
1268 return in;
1269 }
1270
1271 public static byte[] readRaw(String fname) throws IOException{
1272 InputStream ris=getRawInputStream(fname, false);
1273 ByteBuilder bb=new ByteBuilder();
1274 byte[] buffer=new byte[16384];
1275 int x=ris.read(buffer);
1276 while(x>0){
1277 bb.append(buffer, x);
1278 x=ris.read(buffer);
1279 }
1280 ris.close();
1281 return bb.toBytes();
1282 }
1283
1284 public static <X> X read(Class<X> cx, String fname, boolean allowSubprocess){
1285 X x=(X)readObject(fname, allowSubprocess);
1286 return x;
1287 }
1288
1289 public static <X> X[] readArray(Class<X> cx, String fname, boolean allowSubprocess){
1290 X[] x=(X[])readObject(fname, allowSubprocess);
1291 return x;
1292 }
1293
1294 public static <X> X[][] readArray2(Class<X> cx, String fname, boolean allowSubprocess){
1295 X[][] x=(X[][])readObject(fname, allowSubprocess);
1296 return x;
1297 }
1298
1299 public static <X> X[][][] readArray3(Class<X> cx, String fname, boolean allowSubprocess){
1300 X[][][] x=(X[][][])readObject(fname, allowSubprocess);
1301 return x;
1302 }
1303
1304
1305 public static String basename(String fname){
1306 fname=fname.replace('\\', '/');
1307 boolean xz=fname.endsWith(".xz");
1308 boolean gzipped=fname.endsWith(".gz");
1309 boolean zipped=fname.endsWith(".zip");
1310 boolean bzipped=PROCESS_BZ2 && fname.endsWith(".bz2");
1311 boolean dsrced=fname.endsWith(".dsrc");
1312 String basename=fname;
1313 // if(basename.contains("\\")){basename=basename.substring(basename.lastIndexOf("\\")+1);}
1314 if(basename.contains("/")){basename=basename.substring(basename.lastIndexOf('/')+1);}
1315 if(zipped || bzipped){basename=basename.substring(0, basename.length()-4);}
1316 else if(gzipped){basename=basename.substring(0, basename.length()-3);}
1317 else if(dsrced){basename=basename.substring(0, basename.length()-5);}
1318 return basename;
1319 }
1320
1321 public static String rawName(String fname){
1322 for(String s : compressedExtensions){
1323 while(fname.endsWith(s)){fname=fname.substring(0, fname.length()-s.length());}
1324 }
1325 return fname;
1326 }
1327
1328 /**
1329 * Returns the path without the file extension.
1330 * Only strips known extensions. */
1331 public static String stripExtension(String fname){
1332 if(fname==null){return null;}
1333 for(String ext : FileFormat.EXTENSION_LIST){
1334 String s="."+ext;
1335 if(fname.endsWith(s)){return stripExtension(fname.substring(0, fname.length()-s.length()));}
1336 }
1337 return fname;
1338 }
1339
1340 /** Returns the whole extension, include compression and raw type */
1341 public static String getExtension(String fname){
1342 if(fname==null){return null;}
1343 String stripped=stripExtension(fname);
1344 if(stripped==null){return fname;}
1345 if(stripped.length()==fname.length()){return "";}
1346 return fname.substring(stripped.length());
1347 }
1348
1349 public static String stripToCore(String fname){
1350 fname=stripPath(fname);
1351 return stripExtension(fname);
1352 }
1353
1354 /**
1355 * Strips the directories, leaving only a filename
1356 * @param fname
1357 * @return File name without directories
1358 */
1359 public static String stripPath(String fname){
1360 if(fname==null){return null;}
1361 fname=fname.replace('\\', '/');
1362 int idx=fname.lastIndexOf('/');
1363 if(idx>=0){fname=fname.substring(idx+1);}
1364 return fname;
1365 }
1366
1367 public static String getPath(String fname){
1368 if(fname==null){return null;}
1369 fname=fname.replace('\\', '/');
1370 int idx=fname.lastIndexOf('/');
1371 if(idx>=0){return fname.substring(0, idx+1);}
1372 return "";
1373 }
1374
1375 public static String compressionType(String fname){
1376 fname=fname.toLowerCase(Locale.ENGLISH);
1377 for(int i=0; i<compressedExtensions.length; i++){
1378 if(fname.endsWith(compressedExtensions[i])){return compressedExtensionMap[i];}
1379 }
1380 return null;
1381 }
1382
1383 public static boolean isCompressed(String fname){
1384 return compressionType(fname)!=null;
1385 }
1386
1387 public static boolean isSam(String fname){
1388 fname=fname.toLowerCase(Locale.ENGLISH);
1389 if(fname.endsWith(".sam")){return true;}
1390 String s=compressionType(fname);
1391 if(s==null){return false;}
1392 return fname.substring(0, fname.lastIndexOf('.')).endsWith(".sam");
1393 }
1394
1395 /** Returns extension, lower-case, without a period */
1396 public static String rawExtension(String fname){
1397 fname=rawName(fname);
1398 int x=fname.lastIndexOf('.');
1399 //if(x<0){return "";}
1400 return fname.substring(x+1).toLowerCase(Locale.ENGLISH);
1401 }
1402
1403 public static String parseRoot(String path){
1404 File f=new File(path);
1405 if(f.isDirectory()){
1406 if(!path.endsWith(FILESEP)){
1407 path=path+FILESEP;
1408 }
1409 return path;
1410 }else if(f.isFile()){
1411 int slash=path.lastIndexOf(FILESEP);
1412 if(slash<0){
1413 return "";
1414 }else{
1415 return path.substring(0, slash+1);
1416 }
1417 }else{
1418 throw new RuntimeException("Can't find "+path); //Try using parseRoot2 instead.
1419 }
1420 }
1421
1422 /** This one does not throw an exception for non-existing paths */
1423 public static String parseRoot2(String path){
1424 File f=new File(path);
1425
1426 if(!f.exists()){
1427 if(path.endsWith(FILESEP)){return path;}
1428 int slash=path.lastIndexOf(FILESEP);
1429 if(slash<0){
1430 return "";
1431 }else{
1432 return path.substring(0, slash+1);
1433 }
1434 }
1435
1436 if(f.isDirectory()){
1437 if(!path.endsWith(FILESEP)){
1438 path=path+FILESEP;
1439 }
1440 return path;
1441 }else if(f.isFile()){
1442 int slash=path.lastIndexOf(FILESEP);
1443 if(slash<0){
1444 return "";
1445 }else{
1446 return path.substring(0, slash+1);
1447 }
1448 }else{
1449 throw new RuntimeException("Can't find "+path);
1450 }
1451 }
1452
1453 public static String findFileExtension(final String fname){
1454
1455 File file=new File(fname);
1456 if(file.exists()){return fname;}
1457
1458 String basename=fname, temp;
1459 if(fname.endsWith(".zip") || fname.endsWith(".gz") || (PROCESS_BZ2 && fname.endsWith(".bz2")) || (PROCESS_XZ && fname.endsWith(".xz"))){
1460 basename=fname.substring(0, fname.lastIndexOf('.'));
1461 }
1462 temp=basename;
1463 file=new File(temp);
1464 if(!file.exists()){
1465 temp=basename+".gz";
1466 file=new File(temp);
1467 }
1468 // System.err.println(temp+" "+(file.exists() ? " exists" : " does not exist"));
1469 if(!file.exists()){
1470 temp=basename+".zip";
1471 file=new File(temp);
1472 }
1473 // System.err.println(temp+" "+(file.exists() ? " exists" : " does not exist"));
1474 if(!file.exists() && PROCESS_BZ2){
1475 temp=basename+".bz2";
1476 file=new File(temp);
1477 }
1478 // System.err.println(temp+" "+(file.exists() ? " exists" : " does not exist"));
1479 if(!file.exists() && PROCESS_XZ){
1480 temp=basename+".xz";
1481 file=new File(temp);
1482 }
1483 // System.err.println(temp+" "+(file.exists() ? " exists" : " does not exist"));
1484 if(!file.exists()){temp=fname;}
1485
1486 return temp;
1487 }
1488
1489 /**
1490 * Delete a file.
1491 */
1492 public static boolean delete(String path, boolean verbose){
1493 if(path==null){return false;}
1494 if(verbose){System.err.println("Trying to delete "+path);}
1495 File f=new File(path);
1496 if(f.exists()){
1497 try {
1498 f.delete();
1499 return true;
1500 } catch (Exception e) {
1501 // TODO Auto-generated catch block
1502 e.printStackTrace();
1503 }
1504 }
1505 return false;
1506 }
1507
1508 public static synchronized void copyFile(String source, String dest){copyFile(source, dest, false);}
1509 public static synchronized void copyFile(String source, String dest, boolean createPathIfNeeded){
1510
1511 assert(!new File(dest).exists()) : "Destination file already exists: "+dest;
1512 if(createPathIfNeeded){
1513 File parent=new File(dest).getParentFile();
1514 if(parent!=null && !parent.exists()){
1515 parent.mkdirs();
1516 }
1517 }
1518
1519 final boolean oldRawmode=RAWMODE;
1520 if((source.endsWith(".zip") && dest.endsWith(".zip"))
1521 || (source.endsWith(".gz") && dest.endsWith(".gz")
1522 || (source.endsWith(".bz2") && dest.endsWith(".bz2"))
1523 || (source.endsWith(".xz") && dest.endsWith(".xz")))){
1524 RAWMODE=true;
1525 }
1526
1527 try{
1528 InputStream in=getInputStream(source, false, false);
1529 OutputStream out=getOutputStream(dest, false, false, true);
1530
1531 byte[] buffer=new byte[INBUF];
1532 int len;
1533
1534 while((len = in.read(buffer)) > 0){
1535 out.write(buffer, 0, len);
1536 }
1537
1538 in.close();
1539 out.flush();
1540 if(out.getClass()==ZipOutputStream.class){
1541 ZipOutputStream zos=(ZipOutputStream)out;
1542 zos.closeEntry();
1543 zos.finish();
1544 }
1545 // else if(PROCESS_XZ && out.getClass()==org.tukaani.xz.XZOutputStream.class){
1546 // org.tukaani.xz.XZOutputStream zos=(org.tukaani.xz.XZOutputStream)out;
1547 // zos.finish();
1548 // }
1549 out.close();
1550
1551 }catch(FileNotFoundException e){
1552 RAWMODE=oldRawmode;
1553 throw new RuntimeException(e);
1554 }catch(IOException e){
1555 RAWMODE=oldRawmode;
1556 throw new RuntimeException(e);
1557 }
1558
1559 RAWMODE=oldRawmode;
1560 }
1561
1562 public static void copyDirectoryContents(String from, String to){
1563 assert(!from.equalsIgnoreCase(to));
1564
1565 if(to.indexOf('\\')>0){to=to.replace('\\', '/');}
1566
1567 File d1=new File(from);
1568 assert(d1.exists());
1569 assert(d1.isDirectory());
1570
1571 File d2=new File(to);
1572 assert(!d1.equals(d2));
1573 if(d2.exists()){
1574 assert(d2.isDirectory());
1575 }else{
1576 d2.mkdirs();
1577 }
1578 if(!to.endsWith("/")){to=to+"/";}
1579
1580 File[] array=d1.listFiles();
1581
1582 for(File f : array){
1583 String name=f.getName();
1584 String dest=to+name;
1585 if(f.isFile()){
1586 copyFile(f.getAbsolutePath(), dest);
1587 }else{
1588 assert(f.isDirectory());
1589 File f2=new File(dest);
1590 if(!f2.exists()){
1591 f2.mkdir();
1592 }else{
1593 assert(f2.isDirectory());
1594 }
1595 copyDirectoryContents(f.getAbsolutePath(), f2.getAbsolutePath());
1596 }
1597 }
1598
1599 }
1600
1601
1602 static final int addThread(int x){
1603 if(verbose){System.err.println("addThread("+x+")");}
1604 synchronized(activeThreads){
1605 assert(x!=0);
1606 if(x>0){
1607 activeThreads[0]+=x;
1608 activeThreads[1]+=x;
1609 }else{
1610 addRunningThread(x);
1611 }
1612 assert(activeThreads[0]==(activeThreads[1]+activeThreads[2]) && activeThreads[0]>=0 && activeThreads[1]>=0 &&
1613 activeThreads[2]>=0 && activeThreads[2]<=maxWriteThreads) : Arrays.toString(activeThreads);
1614
1615 return activeThreads[0];
1616 }
1617 }
1618
1619 static final int addRunningThread(int x){
1620 if(verbose){System.err.println("addRunningThread("+x+")");}
1621 final int max=(Shared.LOW_MEMORY ? 1 : maxWriteThreads);
1622 synchronized(activeThreads){
1623 assert(x!=0);
1624 if(x>0){
1625 assert(activeThreads[1]>=x);
1626 while(activeThreads[2]>=max){
1627 try {
1628 activeThreads.wait();
1629 } catch (InterruptedException e) {
1630 // TODO Auto-generated catch block
1631 e.printStackTrace();
1632 }
1633 }
1634 activeThreads[1]-=x; //Remove from waiting
1635 }else{
1636 activeThreads[0]+=x; //Remove from active
1637 }
1638 activeThreads[2]+=x; //Change number running
1639
1640 assert(activeThreads[0]==(activeThreads[1]+activeThreads[2]) && activeThreads[0]>=0 && activeThreads[1]>=0 &&
1641 activeThreads[2]>=0 && activeThreads[2]<=max) : Arrays.toString(activeThreads);
1642
1643 if(activeThreads[2]==0 || (activeThreads[2]<max && activeThreads[1]>0)){activeThreads.notify();}
1644 return activeThreads[2];
1645 }
1646 }
1647
1648 public static final int countActiveThreads(){
1649 if(verbose){System.err.println("countActiveThreads()");}
1650 synchronized(activeThreads){
1651 assert(activeThreads[0]==(activeThreads[1]+activeThreads[2]) && activeThreads[0]>=0 && activeThreads[1]>=0 &&
1652 activeThreads[2]>=0 && activeThreads[2]<=maxWriteThreads) : Arrays.toString(activeThreads);
1653 return activeThreads[0];
1654 }
1655 }
1656
1657 public static final void waitForWritingToFinish(){
1658 if(verbose){System.err.println("waitForWritingToFinish()");}
1659 synchronized(activeThreads){
1660 while(activeThreads[0]>0){
1661 assert(activeThreads[0]==(activeThreads[1]+activeThreads[2]) && activeThreads[0]>=0 && activeThreads[1]>=0 &&
1662 activeThreads[2]>=0 && activeThreads[2]<=maxWriteThreads) : Arrays.toString(activeThreads);
1663 try {
1664 activeThreads.wait(8000);
1665 } catch (InterruptedException e) {
1666 // TODO Auto-generated catch block
1667 e.printStackTrace();
1668 }
1669 if(activeThreads[2]==0 || (activeThreads[2]<maxWriteThreads && activeThreads[1]>0)){activeThreads.notify();}
1670 }
1671 }
1672 }
1673
1674
1675 public static final boolean closeStream(ConcurrentReadStreamInterface cris){return closeStreams(cris, (ConcurrentReadOutputStream[])null);}
1676 public static final boolean closeStream(ConcurrentReadOutputStream ross){return closeStreams((ConcurrentReadStreamInterface)null, ross);}
1677 public static final boolean closeOutputStreams(ConcurrentReadOutputStream...ross){return closeStreams(null, ross);}
1678
1679 public static final boolean closeStreams(MultiCros mc){
1680 if(mc==null){return false;}
1681 return closeStreams(null, mc.streamList.toArray(new ConcurrentReadOutputStream[0]));
1682 }
1683
1684 /**
1685 * Close these streams and wait for them to finish.
1686 * @param cris An input stream. May be null.
1687 * @param ross Zero or more output streams.
1688 * @return True if an error was encountered.
1689 */
1690 public static final boolean closeStreams(ConcurrentReadStreamInterface cris, ConcurrentReadOutputStream...ross){
1691 if(verbose){
1692 System.err.println("closeStreams("+cris+", "+(ross==null ? "null" : ross.length)+")");
1693 new Exception().printStackTrace(System.err);
1694 }
1695 boolean errorState=false;
1696 if(cris!=null){
1697 if(verbose){System.err.println("Closing cris; error="+errorState);}
1698 cris.close();
1699 errorState|=cris.errorState();
1700 // Object[] prods=cris.producers();
1701 // for(Object o : prods){
1702 // if(o!=null && o.getClass()==ReadInputStream.class){
1703 // ReadInputStream ris=(ReadInputStream)o;
1704 // ris.
1705 // }
1706 // }
1707 if(verbose){System.err.println("Closed cris; error="+errorState);}
1708 }
1709 if(ross!=null){
1710 for(ConcurrentReadOutputStream ros : ross){
1711 if(ros!=null){
1712 if(verbose){System.err.println("Closing ros "+ros+"; error="+errorState);}
1713 ros.close();
1714 ros.join();
1715 errorState|=(ros.errorState() || !ros.finishedSuccessfully());
1716 if(verbose){System.err.println("Closed ros; error="+errorState);}
1717 }
1718 }
1719 }
1720 return errorState;
1721 }
1722
1723 public static boolean killProcess(String fname){
1724 if(verbose){
1725 System.err.println("killProcess("+fname+")");
1726 new Exception().printStackTrace(System.err);
1727 System.err.println("processMap before: "+processMap.keySet());
1728 }
1729 if(fname==null || (!isCompressed(fname) && !fname.endsWith(".bam") && !FORCE_KILL)){return false;}
1730
1731 boolean error=false;
1732 synchronized(processMap){
1733 Process p=processMap.remove(fname);
1734 if(p!=null){
1735 if(verbose){System.err.println("Found Process for "+fname);}
1736 int x=-1, tries=0;
1737 for(; tries<20; tries++){
1738 if(verbose){System.err.println("Trying p.waitFor()");}
1739 try {
1740 // long t=System.nanoTime();
1741 // Thread.sleep(4000);
1742 if(verbose){System.err.println("p.isAlive()="+p.isAlive());}
1743 x=p.waitFor();
1744 // if(verbose){System.err.println(System.nanoTime()-t+" ns");}
1745 if(verbose){System.err.println("success; return="+x);}
1746 break;
1747 } catch (InterruptedException e) {
1748 if(verbose){System.err.println("Failed.");}
1749 e.printStackTrace();
1750 }
1751 }
1752 error|=(tries>=20 || (x!=0 && x!=141));//141 is sigpipe and appears to be OK when forcibly closing a pipe.
1753 if(verbose){System.err.println("killProcess("+fname+") returned "+error+"; tries="+tries+", code="+x);}
1754 if(tries>=20){
1755 if(verbose){System.err.println("Calling p.destroy because tries=="+tries+"; error="+error);}
1756 p.destroy();
1757 if(verbose){System.err.println("destroyed");}
1758 }
1759 }else{
1760 if(verbose){System.err.println("WARNING: Could not find process for "+fname);}
1761 }
1762 if(verbose){
1763 System.err.println("processMap after: "+processMap.keySet());
1764 }
1765 }
1766 synchronized(pipeThreadMap){
1767 if(verbose){System.err.println("pipeMap before: "+processMap.keySet());}
1768 ArrayList<PipeThread> atp=pipeThreadMap.remove(fname);
1769 if(atp!=null){
1770 for(PipeThread p : atp){
1771 if(p!=null){
1772 if(verbose){System.err.println("Found PipeThread for "+fname);}
1773 p.terminate();
1774 if(verbose){System.err.println("Terminated PipeThread");}
1775 }else{
1776 if(verbose){System.err.println("WARNING: Could not find process for "+fname);}
1777 }
1778 }
1779 }
1780 if(verbose){System.err.println("pipeMap after: "+processMap.keySet());}
1781 }
1782 if(verbose){System.err.println("killProcess("+fname+") returned "+error);}
1783 return error;
1784 }
1785
1786 private static void addProcess(String fname, Process p){
1787 if(verbose){
1788 System.err.println("addProcess("+fname+", "+p+")");
1789 new Exception().printStackTrace();
1790 }
1791 synchronized(processMap){
1792 Process old=processMap.put(fname, p);
1793 if(old!=null){
1794 old.destroy();
1795 // throw new RuntimeException("Duplicate process for file "+fname);
1796 KillSwitch.kill("Duplicate process for file "+fname);
1797 }
1798 }
1799 }
1800
1801 private static void addPipeThread(String fname, PipeThread pt){
1802 if(verbose){System.err.println("addPipeThread("+fname+", "+pt+")");}
1803 synchronized(pipeThreadMap){
1804 // System.err.println("Adding PipeThread for "+fname);
1805 ArrayList<PipeThread> atp=pipeThreadMap.get(fname);
1806 if(atp==null){
1807 atp=new ArrayList<PipeThread>(2);
1808 pipeThreadMap.put(fname, atp);
1809 }
1810 atp.add(pt);
1811 }
1812 }
1813
1814 /**
1815 * Note:
1816 * Magic number of bgzip files is (first 4 bytes):
1817 * 1f 8b 08 04
1818 * 31 139 8 4
1819 * = 529205252
1820 *
1821 * gzip/pigz:
1822 * 1f 8b 08 00
1823 * 31 139 8 0
1824 * = 529205248
1825 *
1826 * od --format=x1 --read-bytes=16 names.txt_gzip.gz
1827 */
1828 public static int getMagicNumber(String fname) {
1829 InputStream is=null;
1830 try {
1831 FileInputStream fis=new FileInputStream(fname);
1832 is=new BufferedInputStream(fis);
1833 } catch (FileNotFoundException e) {
1834 // TODO Auto-generated catch block
1835 e.printStackTrace();
1836 }
1837
1838 //This is fine but uses Java11 methods
1839 // byte[] array=new byte[4];
1840 // int read=0;
1841 // try {
1842 //// read=is.readNBytes(array, 0, 4);
1843 // read=is.readNBytes(array, 0, 4);
1844 // } catch (IOException e) {
1845 // // TODO Auto-generated catch block
1846 // e.printStackTrace();
1847 // }
1848 // assert(read==4) : read;
1849 // int x=0;
1850 // for(int i=0; i<read; i++){
1851 // int b=((int)array[i])&255;
1852 // x=(x<<8)|b;
1853 // }
1854
1855 int x=0;
1856 for(int i=0; i<4; i++){
1857 try {
1858 x=(x<<8)|(is.read()&255);
1859 } catch (IOException e) {
1860 // TODO Auto-generated catch block
1861 e.printStackTrace();
1862 }
1863 }
1864
1865 return x;
1866 }
1867
1868 /** {active, waiting, running} <br>
1869 * Active means running or waiting.
1870 */
1871 public static int[] activeThreads={0, 0, 0};
1872 public static int maxWriteThreads=Shared.threads();
1873
1874 public static boolean verbose=false;
1875
1876 public static boolean RAWMODE=false; //Does not automatically compress and decompress when true
1877
1878 //For killing subprocesses that are neither compression nor samtools
1879 public static boolean FORCE_KILL=false;
1880
1881 public static boolean USE_GZIP=false;
1882 public static boolean USE_BGZIP=true;
1883 public static boolean USE_PIGZ=true;
1884 public static boolean USE_GUNZIP=false;
1885 public static boolean USE_UNBGZIP=true;
1886 public static boolean USE_UNPIGZ=true;
1887
1888 public static boolean FORCE_PIGZ=false;
1889 public static boolean FORCE_BGZIP=false;
1890
1891 public static boolean PREFER_BGZIP=true;
1892 public static boolean PREFER_UNBGZIP=true;
1893
1894 public static boolean USE_BZIP2=true;
1895 public static boolean USE_PBZIP2=true;
1896 public static boolean USE_LBZIP2=true;
1897 public static boolean USE_DSRC=true;
1898 public static boolean USE_FQZ=true;
1899 public static boolean USE_ALAPY=true;
1900 public static boolean USE_SAMBAMBA=true;
1901 public static boolean SAMBAMBA(){return USE_SAMBAMBA && Data.SAMBAMBA();}
1902
1903 // public static boolean SAMTOOLS_IGNORE_UNMAPPED_INPUT=false;
1904 public static int SAMTOOLS_IGNORE_FLAG=0;
1905 public static final int SAM_UNMAPPED=0x4;
1906 public static final int SAM_DUPLICATE=0x400;
1907 public static final int SAM_SUPPLIMENTARY=0x800;
1908 public static final int SAM_SECONDARY=0x100;
1909 public static final int SAM_QFAIL=0x200;
1910
1911 public static boolean PROCESS_BZ2=true;
1912 public static final boolean PROCESS_XZ=false;
1913
1914 public static final int INBUF=16384;
1915 public static final int OUTBUF=16384;
1916
1917 /** Gzip compression level */
1918 public static int ZIPLEVEL=4;
1919 /** Bzip2 compression level */
1920 public static int BZIPLEVEL=9;
1921 public static int MAX_ZIP_THREADS=96;
1922 public static int MAX_SAMTOOLS_THREADS=64;
1923 public static int PIGZ_BLOCKSIZE=128;
1924 public static int PIGZ_ITERATIONS=-1;
1925
1926 public static void setZipThreadMult(float x){
1927 ZIP_THREAD_MULT=Tools.min(1, Tools.max(0.125f, x));
1928 }
1929 public static float ZIP_THREAD_MULT=1f;
1930 public static boolean ALLOW_ZIPLEVEL_CHANGE=true;
1931
1932 public static final String FILESEP=System.getProperty("file.separator");
1933
1934 private static final String diskSync=new String("DISKSYNC");
1935
1936 public static final HashSet<String> loadedFiles=new HashSet<String>();
1937
1938 private static final String[] compressedExtensions=new String[] {".gz", ".gzip", ".zip", ".bz2", ".xz", ".dsrc", ".fqz", ".ac"};
1939 private static final String[] compressedExtensionMap=new String[] {"gz", "gz", "zip", "bz2", "xz", "dsrc", "fqz", "ac"};
1940
1941 // private static HashMap<String, Process> inputProcesses=new HashMap<String, Process>(8);
1942 // private static HashMap<String, Process> outputProcesses=new HashMap<String, Process>(8);
1943 private static HashMap<String, Process> processMap=new HashMap<String, Process>(8);
1944 private static HashMap<String, ArrayList<PipeThread>> pipeThreadMap=new HashMap<String, ArrayList<PipeThread>>(8);
1945
1946 }