jpayne@68: package gff;
jpayne@68: 
jpayne@68: import java.io.File;
jpayne@68: import java.io.PrintStream;
jpayne@68: import java.util.ArrayList;
jpayne@68: import java.util.Arrays;
jpayne@68: import java.util.HashMap;
jpayne@68: import java.util.concurrent.atomic.AtomicInteger;
jpayne@68: import java.util.concurrent.atomic.AtomicLong;
jpayne@68: 
jpayne@68: import aligner.Alignment;
jpayne@68: import fileIO.ByteFile;
jpayne@68: import fileIO.ByteStreamWriter;
jpayne@68: import fileIO.FileFormat;
jpayne@68: import fileIO.ReadWrite;
jpayne@68: import prok.PGMTools;
jpayne@68: import prok.ProkObject;
jpayne@68: import shared.Parse;
jpayne@68: import shared.Parser;
jpayne@68: import shared.PreParser;
jpayne@68: import shared.ReadStats;
jpayne@68: import shared.Shared;
jpayne@68: import shared.Timer;
jpayne@68: import shared.Tools;
jpayne@68: import stream.ConcurrentReadOutputStream;
jpayne@68: import stream.Read;
jpayne@68: import stream.ReadInputStream;
jpayne@68: import structures.ByteBuilder;
jpayne@68: import tax.GiToTaxid;
jpayne@68: import tax.TaxClient;
jpayne@68: import tax.TaxTree;
jpayne@68: import template.Accumulator;
jpayne@68: import template.ThreadWaiter;
jpayne@68: 
jpayne@68: public class CutGff implements Accumulator<CutGff.ProcessThread>  {
jpayne@68: 	
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	/*----------------        Initialization        ----------------*/
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	
jpayne@68: 	/**
jpayne@68: 	 * Code entrance from the command line.
jpayne@68: 	 * @param args Command line arguments
jpayne@68: 	 */
jpayne@68: 	public static void main(String[] args){
jpayne@68: 		//Start a timer immediately upon code entrance.
jpayne@68: 		Timer t=new Timer();
jpayne@68: 		
jpayne@68: 		//Create an instance of this class
jpayne@68: 		CutGff x=new CutGff(args);
jpayne@68: 		
jpayne@68: 		//Run the object
jpayne@68: 		x.process(t);
jpayne@68: 		
jpayne@68: 		//Close the print stream if it was redirected
jpayne@68: 		Shared.closeStream(x.outstream);
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	/**
jpayne@68: 	 * Constructor.
jpayne@68: 	 * @param args Command line arguments
jpayne@68: 	 */
jpayne@68: 	public CutGff(String[] args){
jpayne@68: 		{//Preparse block for help, config files, and outstream
jpayne@68: 			PreParser pp=new PreParser(args, null/*getClass()*/, false);
jpayne@68: 			args=pp.args;
jpayne@68: 			outstream=pp.outstream;
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		//Set shared static variables prior to parsing
jpayne@68: 		ReadWrite.USE_PIGZ=ReadWrite.USE_UNPIGZ=true;
jpayne@68: 		ReadWrite.MAX_ZIP_THREADS=Shared.threads();
jpayne@68: 		
jpayne@68: 		Shared.TRIM_READ_COMMENTS=Shared.TRIM_RNAME=true;
jpayne@68: 		Read.TO_UPPER_CASE=true;
jpayne@68: 		Read.VALIDATE_IN_CONSTRUCTOR=true;
jpayne@68: 		GffLine.parseAttributes=true;
jpayne@68: 		
jpayne@68: 		{//Parse the arguments
jpayne@68: 			final Parser parser=parse(args);
jpayne@68: 			overwrite=parser.overwrite;
jpayne@68: 			append=parser.append;
jpayne@68: 
jpayne@68: 			out=parser.out1;
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		if(alignRibo){
jpayne@68: 			//Load sequences
jpayne@68: 			ProkObject.loadConsensusSequenceFromFile(false, false);
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		fixExtensions(); //Add or remove .gz or .bz2 as needed
jpayne@68: 		checkFileExistence(); //Ensure files can be read and written
jpayne@68: 		checkStatics(); //Adjust file-related static fields as needed for this program
jpayne@68: 
jpayne@68: 		ffout=FileFormat.testOutput(out, FileFormat.PGM, null, true, overwrite, append, false);
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	/*----------------    Initialization Helpers    ----------------*/
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	
jpayne@68: 	/** Parse arguments from the command line */
jpayne@68: 	private Parser parse(String[] args){
jpayne@68: 		
jpayne@68: 		Parser parser=new Parser();
jpayne@68: 		parser.overwrite=overwrite;
jpayne@68: 		for(int i=0; i<args.length; i++){
jpayne@68: 			String arg=args[i];
jpayne@68: 			String[] split=arg.split("=");
jpayne@68: 			String a=split[0].toLowerCase();
jpayne@68: 			String b=split.length>1 ? split[1] : null;
jpayne@68: 			if(b!=null && b.equalsIgnoreCase("null")){b=null;}
jpayne@68: 
jpayne@68: //			outstream.println(arg+", "+a+", "+b);
jpayne@68: 			if(PGMTools.parseStatic(arg, a, b)){
jpayne@68: 				//do nothing
jpayne@68: 			}else if(a.equals("in") || a.equals("infna") || a.equals("fnain") || a.equals("fna") || a.equals("ref")){
jpayne@68: 				assert(b!=null);
jpayne@68: 				Tools.addFiles(b, fnaList);
jpayne@68: 			}else if(a.equals("gff") || a.equals("ingff") || a.equals("gffin")){
jpayne@68: 				assert(b!=null);
jpayne@68: 				Tools.addFiles(b, gffList);
jpayne@68: 			}else if(a.equals("verbose")){
jpayne@68: 				verbose=Parse.parseBoolean(b);
jpayne@68: 				ReadWrite.verbose=verbose;
jpayne@68: 			}else if(a.equals("alignribo") || a.equals("align")){
jpayne@68: 				alignRibo=Parse.parseBoolean(b);
jpayne@68: 			}else if(a.equals("adjustendpoints")){
jpayne@68: 				adjustEndpoints=Parse.parseBoolean(b);
jpayne@68: 			}else if(a.equalsIgnoreCase("slop16s") || a.equalsIgnoreCase("16sslop") || a.equalsIgnoreCase("ssuslop")){
jpayne@68: 				ssuSlop=Integer.parseInt(b);
jpayne@68: 			}else if(a.equalsIgnoreCase("slop23s") || a.equalsIgnoreCase("23sslop") || a.equalsIgnoreCase("lsuslop")){
jpayne@68: 				lsuSlop=Integer.parseInt(b);
jpayne@68: 			}else if(a.equalsIgnoreCase("maxns") || a.equalsIgnoreCase("maxundefined")){
jpayne@68: 				maxNs=Integer.parseInt(b);
jpayne@68: 			}else if(a.equalsIgnoreCase("maxnrate") || a.equalsIgnoreCase("maxnfraction")){
jpayne@68: 				maxNFraction=Integer.parseInt(b);
jpayne@68: 			}else if(a.equals("invert")){
jpayne@68: 				invert=Parse.parseBoolean(b);
jpayne@68: 			}else if(a.equals("type") || a.equals("types")){
jpayne@68: 				types=b;
jpayne@68: 			}else if(a.equals("attributes") || a.equals("requiredattributes")){
jpayne@68: 				requiredAttributes=b.split(",");
jpayne@68: 			}else if(a.equals("banattributes") || a.equals("bannedattributes")){
jpayne@68: 				bannedAttributes=b.split(",");
jpayne@68: 			}else if(a.equals("banpartial")){
jpayne@68: 				banPartial=Parse.parseBoolean(b);
jpayne@68: 			}
jpayne@68: 			
jpayne@68: 			else if(a.equalsIgnoreCase("renameByTaxID")){
jpayne@68: 				renameByTaxID=Parse.parseBoolean(b);
jpayne@68: 			}else if(a.equals("taxmode")){
jpayne@68: 				if("accession".equalsIgnoreCase(b)){
jpayne@68: 					taxMode=ACCESSION_MODE;
jpayne@68: 				}else if("header".equalsIgnoreCase(b)){
jpayne@68: 					taxMode=HEADER_MODE;
jpayne@68: 				}else if("gi".equalsIgnoreCase(b)){
jpayne@68: 					taxMode=GI_MODE;
jpayne@68: 				}else if("taxid".equalsIgnoreCase(b)){
jpayne@68: 					taxMode=TAXID_MODE;
jpayne@68: 				}else{
jpayne@68: 					assert(false) : "Bad tax mode: "+b;
jpayne@68: 				}
jpayne@68: 			}else if(a.equals("requirepresent")){
jpayne@68: 				requirePresent=Parse.parseBoolean(b);
jpayne@68: 			}else if(a.equalsIgnoreCase("onePerFile")){
jpayne@68: 				onePerFile=Parse.parseBoolean(b);
jpayne@68: 			}else if(a.equalsIgnoreCase("pickBest") || a.equalsIgnoreCase("findBest") || a.equalsIgnoreCase("keepBest")){
jpayne@68: 				pickBest=Parse.parseBoolean(b);
jpayne@68: 			}
jpayne@68: 			
jpayne@68: 			else if(a.equals("minlen")){
jpayne@68: 				minLen=Integer.parseInt(b);
jpayne@68: 			}else if(a.equals("maxlen")){
jpayne@68: 				maxLen=Integer.parseInt(b);
jpayne@68: 			}
jpayne@68: 			
jpayne@68: 			else if(ProkObject.parse(arg, a, b)){
jpayne@68: 				//do nothing
jpayne@68: 			}else if(parser.parse(arg, a, b)){
jpayne@68: 				//do nothing
jpayne@68: 			}else if(arg.indexOf('=')<0 && new File(arg).exists() && FileFormat.isFastaFile(arg)){
jpayne@68: 				fnaList.add(arg);
jpayne@68: 			}else{
jpayne@68: 				outstream.println("Unknown parameter "+args[i]);
jpayne@68: 				assert(false) : "Unknown parameter "+args[i];
jpayne@68: 				//				throw new RuntimeException("Unknown parameter "+args[i]);
jpayne@68: 			}
jpayne@68: 		}
jpayne@68: 
jpayne@68: 		ArrayList<String> banned=new ArrayList<String>();
jpayne@68: 		if(banPartial){banned.add("partial=true");}
jpayne@68: 		if(bannedAttributes!=null){
jpayne@68: 			for(String s : bannedAttributes){banned.add(s);}
jpayne@68: 		}
jpayne@68: 		bannedAttributes=banned.isEmpty() ? null : banned.toArray(new String[0]);
jpayne@68: 		
jpayne@68: 		if(gffList.isEmpty()){
jpayne@68: 			for(String s : fnaList){
jpayne@68: 				String prefix=ReadWrite.stripExtension(s);
jpayne@68: 				String gff=prefix+".gff";
jpayne@68: 				File f=new File(gff);
jpayne@68: 				if(!f.exists()){
jpayne@68: 					String gz=gff+".gz";
jpayne@68: 					f=new File(gz);
jpayne@68: 					assert(f.exists() && f.canRead()) : "Can't read file "+gff;
jpayne@68: 					gff=gz;
jpayne@68: 				}
jpayne@68: 				gffList.add(gff);
jpayne@68: 			}
jpayne@68: 		}
jpayne@68: 		assert(gffList.size()==fnaList.size()) : "Number of fna and gff files do not match: "+fnaList.size()+", "+gffList.size();
jpayne@68: 		return parser;
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	/** Add or remove .gz or .bz2 as needed */
jpayne@68: 	private void fixExtensions(){
jpayne@68: 		fnaList=Tools.fixExtension(fnaList);
jpayne@68: 		gffList=Tools.fixExtension(gffList);
jpayne@68: 		if(fnaList.isEmpty()){throw new RuntimeException("Error - at least one input file is required.");}
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	/** Ensure files can be read and written */
jpayne@68: 	private void checkFileExistence(){
jpayne@68: 		//Ensure output files can be written
jpayne@68: 		if(!Tools.testOutputFiles(overwrite, append, false, out)){
jpayne@68: 			outstream.println((out==null)+", "+out);
jpayne@68: 			throw new RuntimeException("\n\noverwrite="+overwrite+"; Can't write to output file "+out+"\n");
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		//Ensure input files can be read
jpayne@68: 		ArrayList<String> foo=new ArrayList<String>();
jpayne@68: 		foo.addAll(fnaList);
jpayne@68: 		foo.addAll(gffList);
jpayne@68: 		if(!Tools.testInputFiles(false, true, foo.toArray(new String[0]))){
jpayne@68: 			throw new RuntimeException("\nCan't read some input files.\n");  
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		//Ensure that no file was specified multiple times
jpayne@68: 		foo.add(out);
jpayne@68: 		if(!Tools.testForDuplicateFiles(true, foo.toArray(new String[0]))){
jpayne@68: 			throw new RuntimeException("\nSome file names were specified multiple times.\n");
jpayne@68: 		}
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	/** Adjust file-related static fields as needed for this program */
jpayne@68: 	private static void checkStatics(){
jpayne@68: 		//Adjust the number of threads for input file reading
jpayne@68: 		if(!ByteFile.FORCE_MODE_BF1 && !ByteFile.FORCE_MODE_BF2 && Shared.threads()>2){
jpayne@68: 			ByteFile.FORCE_MODE_BF2=true;
jpayne@68: 		}
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	/*----------------         Actual Code          ----------------*/
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	
jpayne@68: 	
jpayne@68: 	
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	
jpayne@68: 	public void process(Timer t){
jpayne@68: 		if(Shared.threads()>2 && fnaList.size()>1){
jpayne@68: 			processMT(t);
jpayne@68: 		}else{
jpayne@68: 			processST(t);
jpayne@68: 		}
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	public void processST(Timer t){
jpayne@68: 		ByteStreamWriter bsw=(ffout==null ? null : new ByteStreamWriter(ffout));
jpayne@68: 		if(bsw!=null){bsw.start();}
jpayne@68: 		
jpayne@68: 		for(int i=0; i<fnaList.size(); i++){
jpayne@68: 			processFileST(fnaList.get(i), gffList.get(i), types, bsw);
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		if(bsw!=null){bsw.poisonAndWait();}
jpayne@68: 		t.stop();
jpayne@68: 		if(ffout!=null){outstream.println("Wrote "+out);}
jpayne@68: 		outstream.println(Tools.timeReadsBasesProcessed(t, readsProcessed, basesProcessed, 8));
jpayne@68: 		outstream.println(Tools.readsBasesOut(readsProcessed, basesProcessed, readsOut, basesOut, 8, false));
jpayne@68: 		if(alignRibo){
jpayne@68: 			outstream.println(Tools.number("Flipped:           ", flipped.get(), 8));
jpayne@68: 			outstream.println(Tools.number("Failed Alignment:  ", failed.get(), 8));
jpayne@68: 		}
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	public void processMT(Timer t){
jpayne@68: 		
jpayne@68: 		//Optionally create a read output stream
jpayne@68: 		final ConcurrentReadOutputStream ros=makeCros();
jpayne@68: 		
jpayne@68: 		//Reset counters
jpayne@68: 		readsProcessed=readsOut=0;
jpayne@68: 		basesProcessed=basesOut=0;
jpayne@68: 		
jpayne@68: 		//Process the reads in separate threads
jpayne@68: 		spawnThreads(ros);
jpayne@68: 		
jpayne@68: 		if(verbose){outstream.println("Finished; closing streams.");}
jpayne@68: 		
jpayne@68: 		//Write anything that was accumulated by ReadStats
jpayne@68: 		errorState|=ReadStats.writeAll();
jpayne@68: 		//Close the read streams
jpayne@68: 		errorState|=ReadWrite.closeStream(ros);
jpayne@68: 		
jpayne@68: 		//Report timing and results
jpayne@68: 		t.stop();
jpayne@68: 		if(ffout!=null){outstream.println("Wrote "+out);}
jpayne@68: 		outstream.println(Tools.timeReadsBasesProcessed(t, readsProcessed, basesProcessed, 8));
jpayne@68: 		outstream.println(Tools.readsBasesOut(readsProcessed, basesProcessed, readsOut, basesOut, 8, false));
jpayne@68: 		if(alignRibo){
jpayne@68: 			outstream.println(Tools.number("Flipped:           ", flipped.get(), 8));
jpayne@68: 			outstream.println(Tools.number("Failed Alignment:  ", failed.get(), 8));
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		//Throw an exception of there was an error in a thread
jpayne@68: 		if(errorState){
jpayne@68: 			throw new RuntimeException(getClass().getName()+" terminated in an error state; the output may be corrupt.");
jpayne@68: 		}
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	
jpayne@68: 	private boolean hasAttributes(GffLine gline){
jpayne@68: 		int len=gline.length();
jpayne@68: 		if(len<minLen || len>maxLen){return false;}
jpayne@68: 		if(hasAttributes(gline, bannedAttributes)){return false;}
jpayne@68: 		return requiredAttributes==null || hasAttributes(gline, requiredAttributes);
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	private static boolean hasAttributes(GffLine gline, String[] attributes){
jpayne@68: 		if(attributes==null){return false;}
jpayne@68: 		for(String s : attributes){
jpayne@68: 			if(gline.attributes.contains(s)){
jpayne@68: 				return true;
jpayne@68: 			}
jpayne@68: 		}
jpayne@68: 		return false;
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	private void processFileST(String fna, String gff, String types, ByteStreamWriter bsw){
jpayne@68: 		ArrayList<Read> reads=processFile(fna, gff, types);
jpayne@68: 		if(reads!=null){
jpayne@68: 			for(Read r : reads){
jpayne@68: 				if(bsw!=null){bsw.println(r);}
jpayne@68: 			}
jpayne@68: 		}
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	private ArrayList<Read> processFile(String fna, String gff, String types){
jpayne@68: 		ArrayList<GffLine> lines=GffLine.loadGffFile(gff, types, false);
jpayne@68: 		
jpayne@68: 		ArrayList<Read> list=ReadInputStream.toReads(fna, FileFormat.FA, -1);
jpayne@68: 		HashMap<String, Read> map=new HashMap<String, Read>();
jpayne@68: 		
jpayne@68: 		for(Read r : list){
jpayne@68: 			readsProcessed++;
jpayne@68: 			basesProcessed+=r.length();
jpayne@68: 			map.put(r.id, r);
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		if(renameByTaxID){//Note this must be AFTER adding to the hashmap.
jpayne@68: 			renameByTaxID(list);
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		ArrayList<Read> outList=processLines(lines, map, invert);
jpayne@68: 		
jpayne@68: 		if(invert){
jpayne@68: 			for(Read r : list){
jpayne@68: 				readsOut++;
jpayne@68: 				basesOut+=r.length();
jpayne@68: 			}
jpayne@68: 			return list;
jpayne@68: 		}else{
jpayne@68: 			if(outList!=null){
jpayne@68: 				for(Read r : outList){
jpayne@68: 					readsOut++;
jpayne@68: 					basesOut+=r.length();
jpayne@68: 				}
jpayne@68: 			}
jpayne@68: 			return outList;
jpayne@68: 		}
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	private void renameByTaxID(ArrayList<Read> list){
jpayne@68: 		ByteBuilder bb=new ByteBuilder();
jpayne@68: 		for(Read r : list){
jpayne@68: 			if(bb.length>0){bb.comma();}
jpayne@68: 			bb.append(taxMode==HEADER_MODE ? r.id : taxMode==ACCESSION_MODE ? parseAccession(r.id) : parseGi(r.id));
jpayne@68: 		}
jpayne@68: 		final int[] ids;
jpayne@68: 		if(taxMode==ACCESSION_MODE){
jpayne@68: 			ids=TaxClient.accessionToTaxidArray(bb.toString());
jpayne@68: 		}else if(taxMode==GI_MODE){
jpayne@68: 			ids=TaxClient.giToTaxidArray(bb.toString());
jpayne@68: 		}else if(taxMode==HEADER_MODE){
jpayne@68: 			ids=TaxClient.headerToTaxidArray(bb.toString());
jpayne@68: 		}else if(taxMode==TAXID_MODE){
jpayne@68: 			ids=parseTaxIds(list);
jpayne@68: 		}else{
jpayne@68: 			throw new RuntimeException("Bad mode: "+TAXID_MODE);
jpayne@68: 		}
jpayne@68: 		assert(ids.length==list.size()) : ids.length+", "+list.size();
jpayne@68: 		for(int i=0; i<ids.length; i++){
jpayne@68: 			Read r=list.get(i);
jpayne@68: 			int id=ids[i];
jpayne@68: 			
jpayne@68: 			if(r.id!=null && r.id.startsWith("tid|")){
jpayne@68: 				id=TaxTree.parseHeaderStatic(r.id);
jpayne@68: 				r.obj=id;
jpayne@68: 			}else {
jpayne@68: 				assert(id>=0 || !requirePresent) : "Can't find taxID for header: "+id+", "+r.name();
jpayne@68: 
jpayne@68: 				r.obj=id;
jpayne@68: 				r.id="tid|"+id+"|"+id;
jpayne@68: 			}
jpayne@68: 		}
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	private int[] parseTaxIds(ArrayList<Read> list){
jpayne@68: 		int[] ids=new int[list.size()];
jpayne@68: 		for(int i=0; i<list.size(); i++){
jpayne@68: 			Read r=list.get(i);
jpayne@68: 			int x=GiToTaxid.parseTaxidNumber(r.id, '|');
jpayne@68: 			ids[i]=x;
jpayne@68: 		}
jpayne@68: 		return ids;
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	private String parseAccession(String id){
jpayne@68: 		int dot=id.indexOf('.');
jpayne@68: 		int space=id.indexOf(' ');
jpayne@68: //		assert(dot>0 && space>0) : "Unexpected header format: "+id+"\nTry header mode instead of accession mode.";
jpayne@68: 		int limit=Tools.min((dot<0 ? id.length() : dot), (space<0 ? id.length() : space));
jpayne@68: 		return id.substring(0, limit);
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	private String parseGi(String id){
jpayne@68: 		assert(id.startsWith("gi|"));
jpayne@68: 		String[] split=id.split("|");
jpayne@68: 		return split[1];
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	private ArrayList<Read> processLines(ArrayList<GffLine> lines, HashMap<String, Read> map, boolean invertSelection){
jpayne@68: 		ArrayList<Read> list=null; 
jpayne@68: 		for(GffLine gline : lines){
jpayne@68: 			if(hasAttributes(gline)){
jpayne@68: 				Read scaf=map.get(gline.seqid);
jpayne@68: 				assert(scaf!=null) : "Can't find "+gline.seqid+" in "+map.keySet();
jpayne@68: 				
jpayne@68: 				boolean pass=true;
jpayne@68: 				Float identity=null;
jpayne@68: 				if(alignRibo && gline.inbounds(scaf.length())){
jpayne@68: 					int type=gline.prokType();
jpayne@68: 					identity=align(gline, scaf.bases, type);
jpayne@68: 					if(identity==null){pass=false;}
jpayne@68: 				}
jpayne@68: 				
jpayne@68: 				if(pass){
jpayne@68: 					final int start=gline.start-1;
jpayne@68: 					final int stop=gline.stop-1;
jpayne@68: 
jpayne@68: 					if(invertSelection){
jpayne@68: 						byte[] bases=scaf.bases;
jpayne@68: 						for(int i=start; i<=stop; i++){
jpayne@68: 							if(i>=0 && i<bases.length){
jpayne@68: 								bases[i]='N';
jpayne@68: 							}
jpayne@68: 						}
jpayne@68: 					}else{
jpayne@68: 						if(start>=0 && stop<scaf.length()){
jpayne@68: 							String id=gline.attributes;
jpayne@68: 							if(renameByTaxID){
jpayne@68: 								id="tid|"+scaf.obj+"|"+id;
jpayne@68: 							}
jpayne@68: 							Read r=new Read(Arrays.copyOfRange(scaf.bases, start, stop+1), null, id, 1);
jpayne@68: 							r.obj=identity;
jpayne@68: 							
jpayne@68: 							assert(!r.containsLowercase()) : r.toFasta()+"\n"
jpayne@68: 							+ "validated="+r.validated()+", scaf.validated="+scaf.validated()+", tuc="+Read.TO_UPPER_CASE+", vic="+Read.VALIDATE_IN_CONSTRUCTOR;
jpayne@68: 							if(maxNs>=0 || maxNFraction>=0){
jpayne@68: 								long allowed=Tools.min(maxNs>=0 ? maxNs : r.length(), (long)(r.length()*(maxNFraction>=0 ? maxNFraction : 1)));
jpayne@68: 								if(r.countUndefined()>allowed){r=null;}
jpayne@68: 							}
jpayne@68: 							
jpayne@68: 							if(r!=null){
jpayne@68: 								if(gline.strand==1){r.reverseComplement();}
jpayne@68: 								if(list==null){list=new ArrayList<Read>(8);}
jpayne@68: 								list.add(r);
jpayne@68: 								if(onePerFile && !pickBest){break;}
jpayne@68: 							}
jpayne@68: 						}
jpayne@68: 					}
jpayne@68: 				}
jpayne@68: 			}
jpayne@68: 		}
jpayne@68: 		if(pickBest && list!=null && list.size()>1){
jpayne@68: 			Read best=null;
jpayne@68: 			float bestID=0;
jpayne@68: 			for(Read r : list){
jpayne@68: 				float identity=(r.obj==null ? 0.001f : (Float)r.obj);
jpayne@68: 				if(identity>bestID){
jpayne@68: 					bestID=identity;
jpayne@68: 					best=r;
jpayne@68: 				}
jpayne@68: 			}
jpayne@68: 			assert(best!=null);
jpayne@68: 			list.clear();
jpayne@68: 			list.add(best);
jpayne@68: 		}
jpayne@68: 		return list;
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	private Float align(GffLine gline, byte[] scaf, int type){
jpayne@68: 		Read[] consensusReads=ProkObject.consensusReads(type);
jpayne@68: 		if(consensusReads==null || consensusReads.length==0){
jpayne@68: 			assert(false) : type+"\n"+gline.toString();
jpayne@68: 			return null;
jpayne@68: 		}
jpayne@68: 		byte[] universal=consensusReads[0].bases;
jpayne@68: 		float minIdentity=ProkObject.minID(type)*ID_MULT;
jpayne@68: 		if(universal==null){assert(false); return 1F;}
jpayne@68: 		
jpayne@68: 		int start=gline.start-1;
jpayne@68: 		int stop=gline.stop-1;
jpayne@68: 		assert(start<=stop) : start+", "+stop+", "+scaf.length;
jpayne@68: 		assert(start>=0 && start<scaf.length) : start+", "+stop+", "+scaf.length;
jpayne@68: 		assert(stop>=0 && stop<scaf.length) : start+", "+stop+", "+scaf.length;
jpayne@68: //		final int a=Tools.max(0, start-(adjustEndpoints ? 100 : 20));
jpayne@68: //		final int b=Tools.min(scaf.length-1, stop+(adjustEndpoints ? 100 : 20));
jpayne@68: 		final int a=Tools.max(0, start);
jpayne@68: 		final int b=Tools.min(scaf.length-1, stop);
jpayne@68: 		
jpayne@68: 		byte[] ref=Arrays.copyOfRange(scaf, a, b+1);
jpayne@68: 		Read r=new Read(ref, null, 0);
jpayne@68: 		if(gline.strand==GffLine.MINUS){r.reverseComplement();}
jpayne@68: 
jpayne@68: 		Alignment plus=new Alignment(r);
jpayne@68: 		plus.align(universal);
jpayne@68: //		if(plus.id>=minIdentity){return true;}
jpayne@68: 
jpayne@68: 		r.reverseComplement();
jpayne@68: 		Alignment minus=new Alignment(r);
jpayne@68: 		minus.align(universal);
jpayne@68: 		
jpayne@68: 		Alignment best=null;
jpayne@68: 		if(plus.id>=minus.id){
jpayne@68: 			best=plus;
jpayne@68: //			System.err.println("Kept:    "+plus.id+" \t"+minus.id);
jpayne@68: 		}else{
jpayne@68: 			best=minus;
jpayne@68: 			if(minus.id>=minIdentity){
jpayne@68: 				if(verbose) {System.err.println("Flipped: "+plus.id+" \t"+minus.id+"");}
jpayne@68: 				flipped.incrementAndGet();
jpayne@68: 				gline.strand=Shared.MINUS;
jpayne@68: 			}
jpayne@68: 		}
jpayne@68: 		if(best.id>=minIdentity){
jpayne@68: 			return best.id;
jpayne@68: 		}else{
jpayne@68: 			if(verbose) {System.err.println("Failed alignment: "+plus.id+" \t"+minus.id);}
jpayne@68: 			failed.incrementAndGet();
jpayne@68: 			return null;
jpayne@68: 		}
jpayne@68: //		
jpayne@68: //		int[] coords=KillSwitch.allocInt1D(2);
jpayne@68: //		float id1=align(universal, scaf, a, b, minIdentity, coords);
jpayne@68: //		final int rstart=coords[0], rstop=coords[1];
jpayne@68: //		//				assert(false) : rstart+", "+rstop+", "+(rstop-rstart+1)+", "+start+", "+stop;
jpayne@68: //		if(id1<minIdentity){
jpayne@68: //			System.err.println("Low identity: "+String.format("%.2s", 100*id1));
jpayne@68: //			return false;
jpayne@68: //		}
jpayne@68: //		if(adjustEndpoints){
jpayne@68: //			int slop=(flag==4 ? AnalyzeGenes.lsuSlop : AnalyzeGenes.ssuSlop);
jpayne@68: //			if(Tools.absdif(start, rstart)>slop){
jpayne@68: //				//						System.err.println("rstart:\t"+start+" -> "+rstart);
jpayne@68: //				start=rstart;
jpayne@68: //			}
jpayne@68: //			if(Tools.absdif(stop, rstop)>slop){
jpayne@68: //				//						System.err.println("rstop: \t"+stop+" -> "+rstop);
jpayne@68: //				stop=rstop;
jpayne@68: //			}
jpayne@68: //		}
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	/*----------------       Thread Management      ----------------*/
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	
jpayne@68: 	private ConcurrentReadOutputStream makeCros(){
jpayne@68: 		if(ffout==null){return null;}
jpayne@68: 
jpayne@68: 		//Select output buffer size based on whether it needs to be ordered
jpayne@68: 		final int buff=(ordered ? Tools.mid(16, 128, (Shared.threads()*2)/3) : 8);
jpayne@68: 
jpayne@68: 		final ConcurrentReadOutputStream ros=ConcurrentReadOutputStream.getStream(ffout, null, buff, null, false);
jpayne@68: 		ros.start(); //Start the stream
jpayne@68: 		return ros;
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	/** Spawn process threads */
jpayne@68: 	private void spawnThreads(ConcurrentReadOutputStream ros){
jpayne@68: 		//Do anything necessary prior to processing
jpayne@68: 		
jpayne@68: 		//Determine how many threads may be used
jpayne@68: 		final int threads=Tools.min(Shared.threads(), fnaList.size());
jpayne@68: 		
jpayne@68: 		//Controls access to input files
jpayne@68: 		AtomicInteger atom=new AtomicInteger(0);
jpayne@68: 		
jpayne@68: 		//Fill a list with ProcessThreads
jpayne@68: 		ArrayList<ProcessThread> alpt=new ArrayList<ProcessThread>(threads);
jpayne@68: 		for(int i=0; i<threads; i++){
jpayne@68: 			alpt.add(new ProcessThread(atom, ros, i));
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		//Start the threads and wait for them to finish
jpayne@68: 		boolean success=ThreadWaiter.startAndWait(alpt, this);
jpayne@68: 		errorState&=!success;
jpayne@68: 		
jpayne@68: 		//Do anything necessary after processing
jpayne@68: 		
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	@Override
jpayne@68: 	public final void accumulate(ProcessThread pt){
jpayne@68: 		readsProcessed+=pt.readsProcessedT;
jpayne@68: 		basesProcessed+=pt.basesProcessedT;
jpayne@68: 		readsOut+=pt.readsOutT;
jpayne@68: 		basesOut+=pt.basesOutT;
jpayne@68: 		errorState|=(!pt.success);
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	@Override
jpayne@68: 	public final boolean success(){return !errorState;}
jpayne@68: 	
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	/*----------------         Inner Classes        ----------------*/
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	
jpayne@68: 	/** This class is static to prevent accidental writing to shared variables.
jpayne@68: 	 * It is safe to remove the static modifier. */
jpayne@68: 	class ProcessThread extends Thread {
jpayne@68: 		
jpayne@68: 		//Constructor
jpayne@68: 		ProcessThread(final AtomicInteger atom_, ConcurrentReadOutputStream ros_, final int tid_){
jpayne@68: 			atom=atom_;
jpayne@68: 			ros=ros_;
jpayne@68: 			tid=tid_;
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		//Called by start()
jpayne@68: 		@Override
jpayne@68: 		public void run(){
jpayne@68: 			//Do anything necessary prior to processing
jpayne@68: 			
jpayne@68: 			for(int fnum=atom.getAndIncrement(), lim=fnaList.size(); fnum<lim; fnum=atom.getAndIncrement()) {
jpayne@68: 				String fna=fnaList.get(fnum);
jpayne@68: 				String gff=gffList.get(fnum);
jpayne@68: 				//Process the reads
jpayne@68: 				ArrayList<Read> list=processFileT(fna, gff, types);
jpayne@68: 				if(ros!=null){
jpayne@68: 					if(list==null){list=dummy;}
jpayne@68: 					ros.add(list, fnum);
jpayne@68: 				}
jpayne@68: 			}
jpayne@68: 			
jpayne@68: 			//Do anything necessary after processing
jpayne@68: 			
jpayne@68: 			//Indicate successful exit status
jpayne@68: 			success=true;
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		//Duplicated
jpayne@68: 		private ArrayList<Read> processFileT(String fna, String gff, String types){
jpayne@68: 			ArrayList<GffLine> lines=GffLine.loadGffFile(gff, types, false);
jpayne@68: 			
jpayne@68: 			ArrayList<Read> list=ReadInputStream.toReads(fna, FileFormat.FA, -1);
jpayne@68: 			HashMap<String, Read> map=new HashMap<String, Read>();
jpayne@68: 			
jpayne@68: 			for(Read r : list){
jpayne@68: 				readsProcessedT++;
jpayne@68: 				basesProcessedT+=r.length();
jpayne@68: 				map.put(r.id, r);
jpayne@68: 			}
jpayne@68: 			
jpayne@68: 			if(renameByTaxID){//Note this must be AFTER adding to the hashmap.
jpayne@68: 				renameByTaxID(list);
jpayne@68: 			}
jpayne@68: //			assert(false) : renameByTaxID+", "+list.size();
jpayne@68: 			
jpayne@68: 			ArrayList<Read> outList=processLines(lines, map, invert);
jpayne@68: 			
jpayne@68: 			if(invert){
jpayne@68: 				for(Read r : list){
jpayne@68: 					readsOutT++;
jpayne@68: 					basesOutT+=r.length();
jpayne@68: 				}
jpayne@68: 				return list;
jpayne@68: 			}else{
jpayne@68: 				if(outList!=null){
jpayne@68: 					for(Read r : outList){
jpayne@68: 						readsOutT++;
jpayne@68: 						basesOutT+=r.length();
jpayne@68: 					}
jpayne@68: 				}
jpayne@68: 				return outList;
jpayne@68: 			}
jpayne@68: 		}
jpayne@68: 
jpayne@68: 		/** Number of reads processed by this thread */
jpayne@68: 		protected long readsProcessedT=0;
jpayne@68: 		/** Number of bases processed by this thread */
jpayne@68: 		protected long basesProcessedT=0;
jpayne@68: 		
jpayne@68: 		/** Number of reads retained by this thread */
jpayne@68: 		protected long readsOutT=0;
jpayne@68: 		/** Number of bases retained by this thread */
jpayne@68: 		protected long basesOutT=0;
jpayne@68: 		
jpayne@68: 		/** True only if this thread has completed successfully */
jpayne@68: 		boolean success=false;
jpayne@68: 		
jpayne@68: 		/** Shared output stream */
jpayne@68: 		private final ConcurrentReadOutputStream ros;
jpayne@68: 		/** Thread ID */
jpayne@68: 		final int tid;
jpayne@68: 		/** Next file ID */
jpayne@68: 		final AtomicInteger atom;
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	/*----------------            Fields            ----------------*/
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 
jpayne@68: 	private ArrayList<String> fnaList=new ArrayList<String>();
jpayne@68: 	private ArrayList<String> gffList=new ArrayList<String>();
jpayne@68: 	private String out=null;
jpayne@68: 	private String types="CDS";
jpayne@68: 	private boolean invert=false;
jpayne@68: 	private boolean banPartial=true;
jpayne@68: 	private int minLen=1;
jpayne@68: 	private int maxLen=Integer.MAX_VALUE;
jpayne@68: 
jpayne@68: 	private String[] requiredAttributes;
jpayne@68: 	private String[] bannedAttributes;
jpayne@68: 	
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	
jpayne@68: 	private long bytesOut=0;
jpayne@68: 	private boolean renameByTaxID=false;
jpayne@68: 	private int taxMode=ACCESSION_MODE;
jpayne@68: 	private boolean requirePresent=false;
jpayne@68: 	private boolean alignRibo=false;
jpayne@68: 	private boolean adjustEndpoints=false;
jpayne@68: 	private boolean onePerFile=false;
jpayne@68: 	private boolean pickBest=false;
jpayne@68: 	private int ssuSlop=999;
jpayne@68: 	private int lsuSlop=999;
jpayne@68: 	
jpayne@68: 	private float ID_MULT=0.96f;
jpayne@68: 	
jpayne@68: 	private int maxNs=-1;
jpayne@68: 	private double maxNFraction=-1;
jpayne@68: 	
jpayne@68: 	private static int ACCESSION_MODE=0, GI_MODE=1, HEADER_MODE=2, TAXID_MODE=3;
jpayne@68: 	
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 
jpayne@68: 	/** Number of reads processed */
jpayne@68: 	protected long readsProcessed=0;
jpayne@68: 	/** Number of bases processed */
jpayne@68: 	protected long basesProcessed=0;
jpayne@68: 
jpayne@68: 	/** Number of reads retained */
jpayne@68: 	protected long readsOut=0;
jpayne@68: 	/** Number of bases retained */
jpayne@68: 	protected long basesOut=0;
jpayne@68: 
jpayne@68: 	protected AtomicLong flipped=new AtomicLong(0);
jpayne@68: 	protected AtomicLong failed=new AtomicLong(0);
jpayne@68: 
jpayne@68: 	/** Quit after processing this many input reads; -1 means no limit */
jpayne@68: 	private long maxReads=-1;
jpayne@68: 	
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	/*----------------         Final Fields         ----------------*/
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	
jpayne@68: 	private final FileFormat ffout;
jpayne@68: 	private final ArrayList<Read> dummy=new ArrayList<Read>();
jpayne@68: 	
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	/*----------------        Common Fields         ----------------*/
jpayne@68: 	/*--------------------------------------------------------------*/
jpayne@68: 	
jpayne@68: 	private PrintStream outstream=System.err;
jpayne@68: 	public static boolean verbose=false;
jpayne@68: 	public boolean errorState=false;
jpayne@68: 	public boolean ordered=true;
jpayne@68: 	private boolean overwrite=true;
jpayne@68: 	private boolean append=false;
jpayne@68: 	
jpayne@68: }