jpayne@68: package clump;
jpayne@68: 
jpayne@68: import java.io.File;
jpayne@68: import java.io.PrintStream;
jpayne@68: import java.util.ArrayList;
jpayne@68: import java.util.Random;
jpayne@68: 
jpayne@68: import fileIO.FileFormat;
jpayne@68: import fileIO.ReadWrite;
jpayne@68: import jgi.BBMerge;
jpayne@68: import shared.Parse;
jpayne@68: import shared.Parser;
jpayne@68: import shared.PreParser;
jpayne@68: import shared.Shared;
jpayne@68: import shared.Timer;
jpayne@68: import shared.Tools;
jpayne@68: import sort.SortByName;
jpayne@68: import stream.FASTQ;
jpayne@68: import stream.Read;
jpayne@68: import structures.ByteBuilder;
jpayne@68: import structures.Quantizer;
jpayne@68: 
jpayne@68: /**
jpayne@68:  * @author Brian Bushnell
jpayne@68:  * @date Nov 6, 2015
jpayne@68:  *
jpayne@68:  */
jpayne@68: public class Clumpify {
jpayne@68: 
jpayne@68: 	/**
jpayne@68: 	 * Code entrance from the command line.
jpayne@68: 	 * @param args Command line arguments
jpayne@68: 	 */
jpayne@68: 	public static void main(String[] args){
jpayne@68: 		Timer t=new Timer();
jpayne@68: 		ReadWrite.ZIPLEVEL=Tools.max(ReadWrite.ZIPLEVEL, 6);
jpayne@68: 		
jpayne@68: 		//Capture values of static variables that might be modified in case this is called by another class.
jpayne@68: 		final boolean oldCQ=Read.CHANGE_QUALITY;
jpayne@68: 		final boolean oldBgzip=ReadWrite.USE_BGZIP, oldPreferBgzip=ReadWrite.PREFER_BGZIP;
jpayne@68: 		
jpayne@68: 		BBMerge.changeQuality=Read.CHANGE_QUALITY=false;
jpayne@68: 		ReadWrite.USE_BGZIP=true;
jpayne@68: 		ReadWrite.PREFER_BGZIP=true;
jpayne@68: 		
jpayne@68: 		Clumpify x=new Clumpify(args);
jpayne@68: 		x.process(t);
jpayne@68: 		
jpayne@68: 		//Restore values of static variables.
jpayne@68: //		Shared.setBuffers(oldCap);
jpayne@68: //		ReadWrite.ZIPLEVEL=oldZl;
jpayne@68: //		ReadWrite.USE_PIGZ=oldPigz;
jpayne@68: 		ReadWrite.USE_BGZIP=oldBgzip;
jpayne@68: 		ReadWrite.PREFER_BGZIP=oldPreferBgzip;
jpayne@68: //		ReadWrite.USE_UNPIGZ=oldUnpigz;
jpayne@68: //		ReadWrite.MAX_ZIP_THREADS=oldZipThreads;
jpayne@68: 		BBMerge.changeQuality=Read.CHANGE_QUALITY=oldCQ;
jpayne@68: 		
jpayne@68: 		//Close the print stream if it was redirected
jpayne@68: 		Shared.closeStream(x.outstream);
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	/**
jpayne@68: 	 * Constructor.
jpayne@68: 	 * @param args Command line arguments
jpayne@68: 	 */
jpayne@68: 	public Clumpify(String[] args){
jpayne@68: 		
jpayne@68: 		{//Preparse block for help, config files, and outstream
jpayne@68: 			PreParser pp=new PreParser(args, getClass(), true);
jpayne@68: 			args=pp.args;
jpayne@68: 			outstream=pp.outstream;
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		Read.VALIDATE_IN_CONSTRUCTOR=Shared.threads()<4;
jpayne@68: 		
jpayne@68: 		args2=new ArrayList<String>();
jpayne@68: 		args2.add("in1");
jpayne@68: 		args2.add("in2");
jpayne@68: 		args2.add("out1");
jpayne@68: 		args2.add("out2");
jpayne@68: 		args2.add("groups");
jpayne@68: 		args2.add("ecco=f");
jpayne@68: 		args2.add("rename=f");
jpayne@68: 		args2.add("shortname=f");
jpayne@68: 		args2.add("unpair=f");
jpayne@68: 		args2.add("repair=f");
jpayne@68: 		args2.add("namesort=f");
jpayne@68: 		args2.add("overwrite=t");
jpayne@68: 		
jpayne@68: 		String gString="auto";
jpayne@68: 		for(int i=0; i<args.length; i++){
jpayne@68: 			String arg=args[i];
jpayne@68: 			String[] split=arg.split("=");
jpayne@68: 			String a=split[0].toLowerCase();
jpayne@68: 			String b=split.length>1 ? split[1] : null;
jpayne@68: 
jpayne@68: 			if(a.equals("in") || a.equals("in1")){
jpayne@68: 				in1=b;
jpayne@68: 			}else if(a.equals("in2")){
jpayne@68: 				in2=b;
jpayne@68: 			}else if(a.equals("out") || a.equals("out1")){
jpayne@68: 				out1=b;
jpayne@68: 			}else if(a.equals("out2")){
jpayne@68: 				out2=b;
jpayne@68: 			}else if(a.equals("groups") || a.equals("g") || a.equals("sets") || a.equals("ways")){
jpayne@68: 				gString=b;
jpayne@68: 			}else if(a.equals("delete") || a.equals("deletetemp")){
jpayne@68: 				delete=Parse.parseBoolean(b);
jpayne@68: 			}else if(a.equals("deleteinput")){
jpayne@68: 				deleteInput=Parse.parseBoolean(b);
jpayne@68: 			}else if(a.equals("usetmpdir")){
jpayne@68: 				useTmpdir=Parse.parseBoolean(b);
jpayne@68: 			}else if(a.equals("ecco")){
jpayne@68: 				ecco=Parse.parseBoolean(b);
jpayne@68: 			}else if(a.equals("compresstemp") || a.equals("ct")){
jpayne@68: 				if(b!=null && b.equalsIgnoreCase("auto")){forceCompressTemp=forceRawTemp=false;}
jpayne@68: 				else{
jpayne@68: 					forceCompressTemp=Parse.parseBoolean(b);
jpayne@68: 					forceRawTemp=!forceCompressTemp;
jpayne@68: 				}
jpayne@68: 			}else if(a.equals("tmpdir")){
jpayne@68: 				Shared.setTmpdir(b);
jpayne@68: 			}else if(a.equals("rename") || a.equals("addname")){
jpayne@68: 				addName=Parse.parseBoolean(b);
jpayne@68: 			}else if(a.equals("shortname") || a.equals("shortnames")){
jpayne@68: 				shortName=b;
jpayne@68: 			}else if(a.equals("seed")){
jpayne@68: 				KmerComparator.defaultSeed=Long.parseLong(b);
jpayne@68: 			}else if(a.equals("hashes")){
jpayne@68: 				KmerComparator.setHashes(Integer.parseInt(b));
jpayne@68: 			}else if(a.equals("passes")){
jpayne@68: 				passes=Integer.parseInt(b);
jpayne@68: 				args2.add(arg);
jpayne@68: //			}else if(a.equals("k")){
jpayne@68: //				k=Integer.parseInt(b);
jpayne@68: //				args2.add(arg);
jpayne@68: 			}else if(a.equals("border")){
jpayne@68: 				KmerComparator.defaultBorder=Integer.parseInt(b);
jpayne@68: 			}
jpayne@68: 
jpayne@68: 			else if(a.equals("unpair")){
jpayne@68: 				unpair=Parse.parseBoolean(b);
jpayne@68: 			}else if(a.equals("repair")){
jpayne@68: 				repair=Parse.parseBoolean(b);
jpayne@68: 			}else if(a.equals("namesort") || a.equals("sort")){
jpayne@68: 				namesort=Parse.parseBoolean(b);
jpayne@68: 			}else if(a.equals("overwrite")){
jpayne@68: 				overwrite=Parse.parseBoolean(b);
jpayne@68: 			}else if(a.equals("v1") || a.equals("kmersort1")){
jpayne@68: 				boolean x=Parse.parseBoolean(b);
jpayne@68: 				if(x){V2=V3=false;}
jpayne@68: 			}else if(a.equals("v2") || a.equals("kmersort2")){
jpayne@68: 				V2=Parse.parseBoolean(b);
jpayne@68: 				if(V2){V3=false;}
jpayne@68: 			}else if(a.equals("v3") || a.equals("kmersort3")){
jpayne@68: 				V3=Parse.parseBoolean(b);
jpayne@68: 				if(V3){V2=false;}
jpayne@68: 			}else if(a.equals("fetchthreads")){
jpayne@68: 				KmerSort3.fetchThreads=Integer.parseInt(b);
jpayne@68: 				assert(KmerSort3.fetchThreads>0) : KmerSort3.fetchThreads+"\nFetch threads must be at least 1.";
jpayne@68: 			}
jpayne@68: 			
jpayne@68: 			else if(a.equals("comparesequence")){
jpayne@68: 				KmerComparator.compareSequence=Parse.parseBoolean(b);
jpayne@68: 			}else if(a.equals("allowadjacenttiles") || a.equals("spantiles")){
jpayne@68: 				ReadKey.spanTilesX=ReadKey.spanTilesY=Parse.parseBoolean(b);
jpayne@68: 			}else if(a.equals("spanx") || a.equals("spantilesx")){
jpayne@68: 				ReadKey.spanTilesX=Parse.parseBoolean(b);
jpayne@68: 			}else if(a.equals("spany") || a.equals("spantilesy")){
jpayne@68: 				ReadKey.spanTilesY=Parse.parseBoolean(b);
jpayne@68: 			}else if(a.equals("spanadjacent") || a.equals("spanadjacentonly") || a.equals("adjacentonly") || a.equals("adjacent")){
jpayne@68: 				ReadKey.spanAdjacentOnly=Parse.parseBoolean(b);
jpayne@68: 			}
jpayne@68: 			
jpayne@68: //			else if(a.equals("repair")){
jpayne@68: //				repair=Parse.parseBoolean(b);
jpayne@68: //			}else if(a.equals("namesort") || a.equals("sort")){
jpayne@68: //				namesort=Parse.parseBoolean(b);
jpayne@68: //			}
jpayne@68: 			
jpayne@68: 			else if(a.equals("interleaved") || a.equals("int")){
jpayne@68: 				if("auto".equalsIgnoreCase(b)){FASTQ.FORCE_INTERLEAVED=!(FASTQ.TEST_INTERLEAVED=true);}
jpayne@68: 				else{
jpayne@68: 					FASTQ.FORCE_INTERLEAVED=FASTQ.TEST_INTERLEAVED=Parse.parseBoolean(b);
jpayne@68: 					System.err.println("Set INTERLEAVED to "+FASTQ.FORCE_INTERLEAVED);
jpayne@68: 				}
jpayne@68: 			}else if(a.equals("cq") || a.equals("changequality")){
jpayne@68: 				BBMerge.changeQuality=Read.CHANGE_QUALITY=Parse.parseBoolean(b);
jpayne@68: 			}else if(a.equals("quantize") || a.equals("quantizesticky")){
jpayne@68: 				quantizeQuality=Quantizer.parse(arg, a, b);
jpayne@68: 			}else if(a.equals("lowcomplexity")){
jpayne@68: 				lowComplexity=Parse.parseBoolean(b);
jpayne@68: 			}
jpayne@68: 			
jpayne@68: 			else if(Clump.parseStatic(arg, a, b)){
jpayne@68: 				//Do nothing
jpayne@68: 			}else if(Parser.parseQuality(arg, a, b)){
jpayne@68: 				//Do nothing
jpayne@68: 			}
jpayne@68: 			
jpayne@68: 			else{
jpayne@68: 				args2.add(arg);
jpayne@68: 			}
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		Clump.setXY();
jpayne@68: 		
jpayne@68: 		KmerSplit.quantizeQuality=KmerSort1.quantizeQuality=quantizeQuality;
jpayne@68: 		
jpayne@68: 		Parser.processQuality();
jpayne@68: 		
jpayne@68: 		assert(!unpair || !KmerComparator.mergeFirst) : "Unpair and mergefirst may not be used together.";
jpayne@68: 		
jpayne@68: 		if(in1==null){throw new RuntimeException("\nOne input file is required.\n");}
jpayne@68: 		
jpayne@68: 		if(in1!=null && in2==null && in1.indexOf('#')>-1 && !new File(in1).exists()){
jpayne@68: 			in2=in1.replace("#", "2");
jpayne@68: 			in1=in1.replace("#", "1");
jpayne@68: 		}
jpayne@68: 		if(out1!=null && out2==null && out1.indexOf('#')>-1){
jpayne@68: 			out2=out1.replace("#", "2");
jpayne@68: 			out1=out1.replace("#", "1");
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		//Ensure input files can be read
jpayne@68: 		if(!Tools.testInputFiles(false, true, in1)){
jpayne@68: 			throw new RuntimeException("\nCan't read some input files.\n");  
jpayne@68: 		}
jpayne@68: 		
jpayne@68: //		assert(false) : ReadKey.spanTiles()+", "+ReadKey.spanTilesX+", "+ReadKey.spanTilesY+", "+Clump.sortX+", "+Clump.sortY;
jpayne@68: 		
jpayne@68: 		autoSetGroups(gString);
jpayne@68: 		
jpayne@68: 		if((in2!=null || out2!=null) && groups>1){FASTQ.FORCE_INTERLEAVED=true;} //Fix for crash with twin fasta files
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	/*----------------         Outer Methods        ----------------*/
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 
jpayne@68: 	/** Create read streams and process all data */
jpayne@68: 	public void process(Timer t){
jpayne@68: 		String[] args=args2.toArray(new String[0]);
jpayne@68: 		args[4]="groups="+groups;
jpayne@68: 		
jpayne@68: 		useSharedHeader=(FileFormat.hasSamOrBamExtension(in1) && out1!=null
jpayne@68: 				&& FileFormat.hasSamOrBamExtension(out1));
jpayne@68: 		
jpayne@68: 		if(groups==1){
jpayne@68: 			args[0]="in1="+in1;
jpayne@68: 			args[1]="in2="+in2;
jpayne@68: 			args[2]="out1="+out1;
jpayne@68: 			args[3]="out2="+out2;
jpayne@68: 			args[5]="ecco="+ecco;
jpayne@68: 			args[6]="rename="+addName;
jpayne@68: 			args[7]="shortname="+shortName;
jpayne@68: 			args[8]="unpair="+unpair;
jpayne@68: 			args[9]="repair="+repair;
jpayne@68: 			args[10]="namesort="+namesort;
jpayne@68: 			args[11]="ow="+overwrite;
jpayne@68: 			KmerSort1.main(args);
jpayne@68: 		}else{
jpayne@68: 			String pin1=in1, pin2=in2, temp;
jpayne@68: 			final int conservativePasses=Clump.conservativeFlag ? passes : Tools.max(1, passes/2);
jpayne@68: 			if(passes>1){Clump.setConservative(true);}
jpayne@68: 			long fileMem=-1;
jpayne@68: 			for(int pass=1; pass<=passes; pass++){
jpayne@68: 				if(/*passes>1 &&*/ (V2 || V3)){
jpayne@68: //					System.err.println("Running pass with fileMem="+fileMem);
jpayne@68: //					out=(pass==passes ? out1 : getTempFname("clumpify_p"+(pass+1)+"_temp%_"));
jpayne@68: 					temp=getTempFname("clumpify_p"+(pass+1)+"_temp%_");
jpayne@68: 					if(pass==passes){
jpayne@68: 						fileMem=runOnePass_v2(args, pass, pin1, pin2, out1, out2, fileMem);
jpayne@68: 					}else{
jpayne@68: 						fileMem=runOnePass_v2(args, pass, pin1, pin2, temp, null, fileMem);
jpayne@68: 					}
jpayne@68: //					System.err.println("New fileMem="+fileMem);
jpayne@68: 				}else{
jpayne@68: //					out=(pass==passes ? out1 : getTempFname("clumpify_temp_pass"+pass+"_"));
jpayne@68: 					temp=getTempFname("clumpify_temp_pass"+pass+"_");
jpayne@68: 					if(pass==passes){
jpayne@68: 						runOnePass(args, pass, pin1, pin2, out1, out2);
jpayne@68: 					}else{
jpayne@68: 						runOnePass(args, pass, pin1, pin2, temp, null);
jpayne@68: 					}
jpayne@68: 				}
jpayne@68: 				pin1=temp;
jpayne@68: 				pin2=null;
jpayne@68: 				KmerComparator.defaultBorder=Tools.max(0, KmerComparator.defaultBorder-1);
jpayne@68: 				KmerComparator.defaultSeed++;
jpayne@68: 				if(pass>=conservativePasses){Clump.setConservative(false);}
jpayne@68: 			}
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		if(deleteInput && !sharedErrorState && out1!=null && in1!=null){
jpayne@68: 			try {
jpayne@68: 				new File(in1).delete();
jpayne@68: 				if(in2!=null){new File(in2).delete();}
jpayne@68: 			} catch (Exception e) {
jpayne@68: 				System.err.println("WARNING: Failed to delete input files.");
jpayne@68: 			}
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		t.stop();
jpayne@68: 		System.err.println("Total time: \t"+t);
jpayne@68: 		
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	private void runOnePass(String[] args, int pass, String in1, String in2, String out1, String out2){
jpayne@68: 		assert(groups>1);
jpayne@68: 		if(pass>1){
jpayne@68: 			ecco=false;
jpayne@68: 			shortName="f";
jpayne@68: 			addName=false;
jpayne@68: 		}
jpayne@68: 
jpayne@68: 		String temp=getTempFname("clumpify_p"+pass+"_temp%_");
jpayne@68: 		
jpayne@68: 		String temp2=temp.replace("%", "FINAL");
jpayne@68: 		final boolean externalSort=(pass==passes && (repair || namesort));
jpayne@68: 
jpayne@68: 		args[0]="in1="+in1;
jpayne@68: 		args[1]="in2="+in2;
jpayne@68: 		args[2]="out="+temp;
jpayne@68: 		args[3]="out2="+null;
jpayne@68: 		args[5]="ecco="+ecco;
jpayne@68: 		args[6]="addname=f";
jpayne@68: 		args[7]="shortname="+shortName;
jpayne@68: 		args[8]="unpair="+unpair;
jpayne@68: 		args[9]="repair=f";
jpayne@68: 		args[10]="namesort=f";
jpayne@68: 		args[11]="ow="+overwrite;
jpayne@68: 		KmerSplit.maxZipLevel=2;
jpayne@68: 		KmerSplit.main(args);
jpayne@68: 		
jpayne@68: 		FASTQ.DETECT_QUALITY=FASTQ.DETECT_QUALITY_OUT=false;
jpayne@68: 		FASTQ.ASCII_OFFSET=FASTQ.ASCII_OFFSET_OUT;
jpayne@68: 
jpayne@68: 		args[0]="in="+temp;
jpayne@68: 		args[1]="in2="+null;
jpayne@68: 		args[2]="out="+(externalSort ? temp2 : out1);
jpayne@68: 		args[3]="out2="+(externalSort ? "null" : out2);
jpayne@68: 		args[5]="ecco=f";
jpayne@68: 		args[6]="addname="+addName;
jpayne@68: 		args[7]="shortname=f";
jpayne@68: 		args[8]="unpair=f";
jpayne@68: 		args[9]="repair="+(repair && externalSort);
jpayne@68: 		args[10]="namesort="+(namesort && externalSort);
jpayne@68: 		args[11]="ow="+overwrite;
jpayne@68: 		if(unpair){
jpayne@68: 			FASTQ.FORCE_INTERLEAVED=FASTQ.TEST_INTERLEAVED=false;
jpayne@68: 		}
jpayne@68: 		KmerSort1.main(args);
jpayne@68: 		
jpayne@68: 		if(delete){
jpayne@68: 			for(int i=0; i<groups; i++){
jpayne@68: 				new File(temp.replaceFirst("%", ""+i)).delete();
jpayne@68: 			}
jpayne@68: 			if(pass>1){
jpayne@68: 				assert(in2==null);
jpayne@68: 				new File(in1).delete();
jpayne@68: 			}
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		if(externalSort){
jpayne@68: 			outstream.println();
jpayne@68: 			String[] sortArgs=new String[] {"in="+temp2, "out="+out1, "ow="+overwrite};
jpayne@68: 			if(out2!=null){sortArgs=new String[] {"in="+temp2, "out="+out1, "out2="+out2, "ow="+overwrite};}
jpayne@68: 			SortByName.main(sortArgs);
jpayne@68: 			if(delete){new File(temp2).delete();}
jpayne@68: 		}
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	private long runOnePass_v2(String[] args, int pass, String in1, String in2, String out1, String out2, long fileMem){
jpayne@68: 		assert(groups>1);
jpayne@68: 		if(pass>1){
jpayne@68: 			ecco=false;
jpayne@68: 			shortName="f";
jpayne@68: 			addName=false;
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		String temp=getTempFname("clumpify_p"+pass+"_temp%_");
jpayne@68: 		
jpayne@68: //		String temp2=temp.replace("%", "FINAL");
jpayne@68: 		String namesorted=temp.replace("%", "namesorted_%");
jpayne@68: 		final boolean externalSort=(pass==passes && (repair || namesort));
jpayne@68: 		
jpayne@68: 		if(pass==1){
jpayne@68: 			args[0]="in1="+in1;
jpayne@68: 			args[1]="in2="+in2;
jpayne@68: 			args[2]="out="+temp;
jpayne@68: 			args[3]="out2="+null;
jpayne@68: 			args[5]="ecco="+ecco;
jpayne@68: 			args[6]="addname=f";
jpayne@68: 			args[7]="shortname="+shortName;
jpayne@68: 			args[8]="unpair="+unpair;
jpayne@68: 			args[9]="repair=f";
jpayne@68: 			args[10]="namesort=f";
jpayne@68: 			args[11]="ow="+overwrite;
jpayne@68: 			KmerSplit.maxZipLevel=2;
jpayne@68: 			KmerSplit.main(args);
jpayne@68: 			fileMem=KmerSplit.lastMemProcessed;
jpayne@68: 			
jpayne@68: 			FASTQ.DETECT_QUALITY=FASTQ.DETECT_QUALITY_OUT=false;
jpayne@68: 			FASTQ.ASCII_OFFSET=FASTQ.ASCII_OFFSET_OUT;
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		args[0]="in1="+(pass==1 ? temp : in1);
jpayne@68: 		args[1]="in2="+null;
jpayne@68: 		args[2]="out="+(externalSort ? namesorted : out1);
jpayne@68: 		args[3]="out2="+(externalSort ? "null" : out2);
jpayne@68: 		args[5]="ecco=f";
jpayne@68: 		args[6]="addname="+addName;
jpayne@68: 		args[7]="shortname=f";
jpayne@68: 		args[8]="unpair=f";
jpayne@68: 		args[9]="repair="+(repair && externalSort);
jpayne@68: 		args[10]="namesort="+(namesort && externalSort);
jpayne@68: 		args[11]="ow="+overwrite;
jpayne@68: 		if(unpair){
jpayne@68: 			FASTQ.FORCE_INTERLEAVED=FASTQ.TEST_INTERLEAVED=false;
jpayne@68: 		}
jpayne@68: 		if(externalSort){
jpayne@68: 			KmerSort.doHashAndSplit=false;
jpayne@68: 		}
jpayne@68: 		if(V3){
jpayne@68: 			KmerSort3.main(fileMem, pass, passes, args);
jpayne@68: 			if(fileMem<1){fileMem=KmerSort3.lastMemProcessed;}
jpayne@68: 		}else{KmerSort2.main(args);}
jpayne@68: 		
jpayne@68: 		if(delete){
jpayne@68: 			for(int i=0; i<groups; i++){
jpayne@68: 				new File((pass==1 ? temp : in1).replaceFirst("%", ""+i)).delete();
jpayne@68: 			}
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		if(externalSort){
jpayne@68: 			outstream.println();
jpayne@68: 			
jpayne@68: 			ArrayList<String> names=new ArrayList<String>();
jpayne@68: 			for(int i=0; i<groups; i++){
jpayne@68: 				names.add(namesorted.replaceFirst("%", ""+i));
jpayne@68: 			}
jpayne@68: 			ReadWrite.MAX_ZIP_THREADS=Shared.threads();
jpayne@68: 			
jpayne@68: 			ReadWrite.USE_PIGZ=true;
jpayne@68: 			ReadWrite.ZIPLEVEL=Tools.max(ReadWrite.ZIPLEVEL, 6);
jpayne@68: 			FASTQ.TEST_INTERLEAVED=FASTQ.FORCE_INTERLEAVED=false;
jpayne@68: 			FileFormat dest=FileFormat.testOutput(out1, FileFormat.FASTQ, null, true, overwrite, false, false);
jpayne@68: 			FileFormat dest2=FileFormat.testOutput(out2, FileFormat.FASTQ, null, true, overwrite, false, false);
jpayne@68: 			SortByName.mergeAndDump(names, /*null, */dest, dest2, delete, useSharedHeader, false, outstream, 1000);
jpayne@68: 		}
jpayne@68: 		
jpayne@68: //		if(externalSort){
jpayne@68: //			outstream.println();
jpayne@68: //			SortByName.main(new String[] {"in="+temp2, "out="+out, "ow="+overwrite});
jpayne@68: //			if(delete){new File(temp2).delete();}
jpayne@68: //		}
jpayne@68: 		return fileMem;
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	/*----------------         Inner Methods        ----------------*/
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	
jpayne@68: 	private void autoSetGroups(String s) {
jpayne@68: 		if(s==null || s.equalsIgnoreCase("null")){return;}
jpayne@68: 		if(Tools.isDigit(s.charAt(0))){
jpayne@68: 			groups=Integer.parseInt(s);
jpayne@68: 			return;
jpayne@68: 		}
jpayne@68: 		assert(s.equalsIgnoreCase("auto")) : "Unknown groups setting: "+s;
jpayne@68: 		
jpayne@68: 		final long maxMem=Shared.memAvailable(1);
jpayne@68: 		FileFormat ff1=FileFormat.testInput(in1, FileFormat.FASTQ, null, false, false);
jpayne@68: 		if(ff1==null || ff1.stdio()){return;}
jpayne@68: 		
jpayne@68: //		outstream.println("in1="+in1+", overhead="+(0.5*(ReadKey.overhead+Clump.overhead)));
jpayne@68: 		
jpayne@68: 		double[] estimates=Tools.estimateFileMemory(in1, 1000, 0.5*(ReadKey.overhead+Clump.overhead), true, lowComplexity);
jpayne@68: 		if(in2!=null){
jpayne@68: 			double[] estimates2=Tools.estimateFileMemory(in2, 1000, 0.5*(ReadKey.overhead+Clump.overhead), true, lowComplexity);
jpayne@68: 			estimates[0]+=estimates2[0];
jpayne@68: 			estimates[1]+=estimates2[1];
jpayne@68: 			estimates[4]+=estimates2[4];
jpayne@68: 		}
jpayne@68: 		
jpayne@68: //		outstream.println(Arrays.toString(estimates));
jpayne@68: 		
jpayne@68: 		double memEstimate=estimates==null ? 0 : estimates[0];
jpayne@68: 		double diskEstimate=estimates==null ? 0 : estimates[1];
jpayne@68: 		double readEstimate=estimates==null ? 0 : estimates[4];
jpayne@68: 		double worstCase=memEstimate*1.5;
jpayne@68: 
jpayne@68: //		outstream.println("Raw Disk Size Estimate: "+(long)(diskEstimate/(1024*1024))+" MB");
jpayne@68: 		outstream.println("Read Estimate:          "+(long)(readEstimate));
jpayne@68: 		outstream.println("Memory Estimate:        "+(long)(memEstimate/(1024*1024))+" MB");
jpayne@68: 		outstream.println("Memory Available:       "+(maxMem/(1024*1024))+" MB");
jpayne@68: 		
jpayne@68: 		if(maxMem>worstCase && readEstimate<Integer.MAX_VALUE){
jpayne@68: 			groups=1;
jpayne@68: 		}else{
jpayne@68: 			groups=Tools.max(11, (int)(3+(3*worstCase/maxMem)*(V3 ? KmerSort3.fetchThreads : 2)), (int)((2*readEstimate)/Integer.MAX_VALUE))|1;
jpayne@68: 		}
jpayne@68: 		outstream.println("Set groups to "+groups);
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	private String getTempFname(String core){
jpayne@68: //		outstream.println(core);
jpayne@68: 		String temp;
jpayne@68: 		String path="", extension=".fq";
jpayne@68: 		if(out1!=null){
jpayne@68: 			core=ReadWrite.stripToCore(out1)+"_"+core;
jpayne@68: 			path=ReadWrite.getPath(out1);
jpayne@68: 			extension=ReadWrite.getExtension(out1);
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		if(useTmpdir && Shared.tmpdir()!=null){
jpayne@68: 			temp=Shared.tmpdir()+core+Long.toHexString((randy.nextLong()&Long.MAX_VALUE))+extension;
jpayne@68: 		}else{
jpayne@68: 			temp=path+core+Long.toHexString((randy.nextLong()&Long.MAX_VALUE))+extension;
jpayne@68: 		}
jpayne@68: //		assert(false) : path+", "+temp+", "+core+", "+out1;
jpayne@68: 		
jpayne@68: 		String comp=ReadWrite.compressionType(temp);
jpayne@68: 		if(comp!=null){comp=".gz";} //Prevent bz2 temp files which cause a crash
jpayne@68: 		
jpayne@68: 		if(forceCompressTemp && comp==null){
jpayne@68: 			temp+=".gz";
jpayne@68: 		}else if(comp!=null && forceRawTemp){
jpayne@68: 			temp=temp.substring(0, temp.lastIndexOf('.'));
jpayne@68: 		}
jpayne@68: 		if(temp.endsWith(".bz2")){temp=temp.substring(0, temp.length()-4);} //Prevent bz2 temp files which cause a crash
jpayne@68: 
jpayne@68: //		outstream.println(temp);
jpayne@68: 		return temp;
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	public static void shrinkName(Read r) {
jpayne@68: 		if(r==null){return;}
jpayne@68: 		String s=r.id;
jpayne@68: 		if(s.contains("HISEQ")){s=s.replace("HISEQ", "H");}
jpayne@68: 		if(s.contains("MISEQ")){
jpayne@68: 			s=s.replace("MISEQ", "M");
jpayne@68: 		}
jpayne@68: 		if(s.contains(":000000000-")){
jpayne@68: 			s=s.replace(":000000000-", ":");
jpayne@68: 		}
jpayne@68: 		r.id=s;
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	public static void shortName(Read r) {
jpayne@68: 		ByteBuilder sb=new ByteBuilder(14);
jpayne@68: 		long x=r.numericID|1;
jpayne@68: 		
jpayne@68: 		while(x<1000000000L){
jpayne@68: 			x*=10;
jpayne@68: 			sb.append('0');
jpayne@68: 		}
jpayne@68: 		sb.append(r.numericID);
jpayne@68: 		
jpayne@68: //		while(x<0x10000000L){
jpayne@68: //			x*=16;
jpayne@68: //			sb.append('0');
jpayne@68: //		}
jpayne@68: //		sb.append(Long.toHexString(r.numericID));
jpayne@68: 		
jpayne@68: 		sb.append(r.pairnum()==0 ? " 1:" : " 2:");
jpayne@68: 		r.id=sb.toString();
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	/*----------------            Fields            ----------------*/
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	
jpayne@68: 	private boolean lowComplexity=false;
jpayne@68: 	
jpayne@68: 	private boolean quantizeQuality=false;
jpayne@68: 	private Random randy=new Random();
jpayne@68: 	private int groups=31;
jpayne@68: 	private int passes=1;
jpayne@68: 	private boolean ecco=false;
jpayne@68: 	private boolean addName=false;
jpayne@68: 	private String shortName="f";
jpayne@68: 	private boolean useTmpdir=false;
jpayne@68: 	private boolean delete=true;
jpayne@68: 	private boolean deleteInput=false;
jpayne@68: 	private boolean useSharedHeader=false;
jpayne@68: 	private boolean forceCompressTemp=false;
jpayne@68: 	private boolean forceRawTemp=false;
jpayne@68: 	private boolean overwrite=true;
jpayne@68: 
jpayne@68: 	private boolean unpair=false;
jpayne@68: 	private boolean repair=false;
jpayne@68: 	private boolean namesort=false;
jpayne@68: 	private boolean V2=false;
jpayne@68: 	private boolean V3=true;
jpayne@68: 
jpayne@68: 	private String in1=null;
jpayne@68: 	private String in2=null;
jpayne@68: 	private String out1=null;
jpayne@68: 	private String out2=null;
jpayne@68: 	
jpayne@68: 	ArrayList<String> args2=new ArrayList<String>();
jpayne@68: 	private PrintStream outstream=System.err;
jpayne@68: 
jpayne@68: 	public static boolean sharedErrorState=false;
jpayne@68: 	
jpayne@68: }