jpayne@68: package fileIO;
jpayne@68: import java.io.File;
jpayne@68: import java.io.IOException;
jpayne@68: import java.io.InputStream;
jpayne@68: import java.util.ArrayList;
jpayne@68: 
jpayne@68: import shared.KillSwitch;
jpayne@68: import shared.Shared;
jpayne@68: import shared.Timer;
jpayne@68: import shared.Tools;
jpayne@68: import structures.ListNum;
jpayne@68: 
jpayne@68: 
jpayne@68: /**
jpayne@68:  * Written for testing a NERSC slowdown in multithreaded file reading.
jpayne@68:  * The problem was sometimes sidestepped by eliminating "if(pushBack!=null){" and reimplementing pushback.
jpayne@68:  * However, that does not address the cause, so is not an overall solution; the cause remains a mystery.
jpayne@68:  * This class may safely be deleted.
jpayne@68:  * 
jpayne@68:  * @author Brian Bushnell
jpayne@68:  *
jpayne@68:  */
jpayne@68: public class QuickFile {
jpayne@68: 	
jpayne@68: 	
jpayne@68: 	public static void main(String[] args){
jpayne@68: 		QuickFile tf=new QuickFile(args.length>0 ? args[0] : "stdin", true);
jpayne@68: 		long first=0, last=100;
jpayne@68: 		boolean speedtest=false;
jpayne@68: 		if(args.length>1){
jpayne@68: 			if(args[1].equalsIgnoreCase("speedtest")){
jpayne@68: 				speedtest=true;
jpayne@68: 				first=0;
jpayne@68: 				last=Long.MAX_VALUE;
jpayne@68: 			}else{
jpayne@68: 				first=Integer.parseInt(args[1]);
jpayne@68: 				last=first+100;
jpayne@68: 			}
jpayne@68: 		}
jpayne@68: 		if(args.length>2){
jpayne@68: 			last=Integer.parseInt(args[2]);
jpayne@68: 		}
jpayne@68: 		speedtest(tf, first, last, !speedtest);
jpayne@68: 		
jpayne@68: 		tf.close();
jpayne@68: 		tf.reset();
jpayne@68: 		tf.close();
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	private static void speedtest(QuickFile tf, long first, long last, boolean reprint){
jpayne@68: 		Timer t=new Timer();
jpayne@68: 		long lines=0;
jpayne@68: 		long bytes=0;
jpayne@68: 		for(long i=0; i<first; i++){tf.nextLine();}
jpayne@68: 		if(reprint){
jpayne@68: 			for(long i=first; i<last; i++){
jpayne@68: 				byte[] s=tf.nextLine();
jpayne@68: 				if(s==null){break;}
jpayne@68: 
jpayne@68: 				lines++;
jpayne@68: 				bytes+=s.length;
jpayne@68: 				System.out.println(new String(s));
jpayne@68: 			}
jpayne@68: 			
jpayne@68: 			System.err.println("\n");
jpayne@68: 			System.err.println("Lines: "+lines);
jpayne@68: 			System.err.println("Bytes: "+bytes);
jpayne@68: 		}else{
jpayne@68: 			for(long i=first; i<last; i++){
jpayne@68: 				byte[] s=tf.nextLine();
jpayne@68: 				if(s==null){break;}
jpayne@68: 				lines++;
jpayne@68: 				bytes+=s.length;
jpayne@68: 			}
jpayne@68: 		}
jpayne@68: 		t.stop();
jpayne@68: 		
jpayne@68: 		if(!reprint){
jpayne@68: 			System.err.println(Tools.timeLinesBytesProcessed(t, lines, bytes, 8));
jpayne@68: 		}
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	public QuickFile(String fname, boolean allowSubprocess_){
jpayne@68: 		this(FileFormat.testInput(fname, FileFormat.TEXT, null, allowSubprocess_, false));
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	public QuickFile(FileFormat ff_){
jpayne@68: 		ff=ff_;
jpayne@68: 		assert(ff.read()) : ff;
jpayne@68: 		if(verbose){System.err.println("ByteFile1("+ff+")");}
jpayne@68: 		is=open();
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	public final void reset(){
jpayne@68: 		close();
jpayne@68: 		is=open();
jpayne@68: 
jpayne@68: 		pushBack=null;
jpayne@68: 		nextID=0;
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	public synchronized final boolean close(){
jpayne@68: 		if(verbose){System.err.println("Closing "+this.getClass().getName()+" for "+name()+"; open="+open+"; errorState="+errorState);}
jpayne@68: 		if(!open){return errorState;}
jpayne@68: 		open=false;
jpayne@68: 		assert(is!=null);
jpayne@68: //		assert(false) : name()+","+allowSubprocess();
jpayne@68: 		errorState|=ReadWrite.finishReading(is, name(), (allowSubprocess() || FileFormat.isBamFile(name())));
jpayne@68: 		
jpayne@68: 		is=null;
jpayne@68: 		lineNum=-1;
jpayne@68: 		pushBack=null;
jpayne@68: 		if(verbose){System.err.println("Closed "+this.getClass().getName()+" for "+name()+"; open="+open+"; errorState="+errorState);}
jpayne@68: 		return errorState;
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	public byte[] nextLine(){
jpayne@68: //		if(pushBack!=null){
jpayne@68: //			byte[] temp=pushBack;
jpayne@68: //			pushBack=null;
jpayne@68: //			return temp;
jpayne@68: //		}
jpayne@68: 		
jpayne@68: 		if(verbose){System.err.println("Reading line "+this.getClass().getName()+" for "+name()+"; open="+open+"; errorState="+errorState);}
jpayne@68: 		
jpayne@68: 		if(!open || is==null){
jpayne@68: 			if(Shared.WINDOWS){System.err.println("Attempting to read from a closed file: "+name());}
jpayne@68: 			return null;
jpayne@68: 		}
jpayne@68: 
jpayne@68: //		System.out.println("\nCalled nextLine() for line "+lineNum);
jpayne@68: //		System.out.println("A: bstart="+bstart+", bstop="+bstop);
jpayne@68: 		
jpayne@68: 		//if(bstart<bstop && lasteol==slasher && buffer[bstart]==slashn){bstart++;}
jpayne@68: //		assert(bstart>=bstop || (buffer[bstart]!=slashn)/*buffer[bstart]>slasher || buffer[bstart]==slashn*/);
jpayne@68: 		int nlpos=bstart;
jpayne@68: 		
jpayne@68: //		System.out.println("B: bstart="+bstart+", bstop="+bstop+", nlpos="+nlpos);
jpayne@68: //		while(nlpos<bstop && (buffer[nlpos]>slasher || buffer[nlpos]==tab)){nlpos++;}
jpayne@68: 		while(nlpos<bstop && buffer[nlpos]!=slashn){nlpos++;}
jpayne@68: //		System.out.println("C: bstart="+bstart+", bstop="+bstop+", nlpos="+nlpos);
jpayne@68: 		if(nlpos>=bstop){
jpayne@68: 			nlpos=fillBuffer();
jpayne@68: //			System.out.println("Filled buffer.");
jpayne@68: 		}
jpayne@68: //		System.out.println("D: bstart="+bstart+", bstop="+bstop+", nlpos="+nlpos);
jpayne@68: 		
jpayne@68: 		if(nlpos<0 || bstop<1){
jpayne@68: 			close();
jpayne@68: 			return null;
jpayne@68: 		}
jpayne@68: 
jpayne@68: 		lineNum++;
jpayne@68: 		//Limit is the position after the last position to copy.
jpayne@68: 		//Limit equals nlpos unless there was a \r before the \n.
jpayne@68: 		final int limit=(nlpos>bstart && buffer[nlpos-1]==slashr) ? nlpos-1 : nlpos;
jpayne@68: 		if(bstart==limit){//Empty line.
jpayne@68: 			bstart=nlpos+1;
jpayne@68: //			System.out.println("E: bstart="+bstart+", bstop="+bstop+", nlpos="+nlpos+", returning='"+printNL(blankLine)+"'");
jpayne@68: 			return blankLine;
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		byte[] line=KillSwitch.copyOfRange(buffer, bstart, limit);
jpayne@68: 		
jpayne@68: 		assert(line.length>0) : bstart+", "+nlpos+", "+limit;
jpayne@68: 		bstart=nlpos+1;
jpayne@68: //		System.out.println("F: bstart="+bstart+", bstop="+bstop+", nlpos="+nlpos+", returning='"+printNL(line)+"'");
jpayne@68: 		return line;
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	final byte[] dummy=new byte[100];
jpayne@68: 	
jpayne@68: 	private int fillBuffer(){
jpayne@68: 		if(bstart<bstop){ //Shift end bytes to beginning
jpayne@68: //			System.err.println("Shift: "+bstart+", "+bstop);
jpayne@68: 			assert(bstart>0);
jpayne@68: //			assert(bstop==buffer.length);
jpayne@68: 			int extra=bstop-bstart;
jpayne@68: 			for(int i=0; i<extra; i++, bstart++){
jpayne@68: //				System.err.print((char)buffer[bstart]);
jpayne@68: 				//System.err.print('.');
jpayne@68: 				buffer[i]=buffer[bstart];
jpayne@68: //				assert(buffer[i]>=slasher || buffer[i]==tab);
jpayne@68: 				assert(buffer[i]!=slashn);
jpayne@68: 			}
jpayne@68: 			bstop=extra;
jpayne@68: //			System.err.println();
jpayne@68: 
jpayne@68: //			{//for debugging only
jpayne@68: //				buffer=new byte[bufferlen];
jpayne@68: //				bstop=0;
jpayne@68: //				bstart=0;
jpayne@68: //			}
jpayne@68: 		}else{
jpayne@68: 			bstop=0;
jpayne@68: 		}
jpayne@68: 
jpayne@68: 		bstart=0;
jpayne@68: 		int len=bstop;
jpayne@68: 		int r=-1;
jpayne@68: 		while(len==bstop){//hit end of input without encountering a newline
jpayne@68: 			if(bstop==buffer.length){
jpayne@68: //				assert(false) : len+", "+bstop;
jpayne@68: 				buffer=KillSwitch.copyOf(buffer, buffer.length*2);
jpayne@68: 			}
jpayne@68: 			try {
jpayne@68: 				r=is.read(buffer, bstop, buffer.length-bstop);
jpayne@68: //				byte[] x=new byte[buffer.length-bstop];
jpayne@68: //				r=is.read(x);
jpayne@68: //				if(r>0){
jpayne@68: //					for(int i=0, j=bstop; i<r; i++, j++){
jpayne@68: //						buffer[j]=x[i];
jpayne@68: //					}
jpayne@68: //				}
jpayne@68: 			} catch (IOException e) {
jpayne@68: 				e.printStackTrace();
jpayne@68: 				System.err.println("open="+open);
jpayne@68: 			}
jpayne@68: 			if(r>0){
jpayne@68: 				bstop=bstop+r;
jpayne@68: //				while(len<bstop && (buffer[len]>slasher || buffer[len]==tab)){len++;}
jpayne@68: 				while(len<bstop && buffer[len]!=slashn){len++;}
jpayne@68: 			}else{
jpayne@68: 				len=bstop;
jpayne@68: 				break;
jpayne@68: 			}
jpayne@68: 		}
jpayne@68: 		
jpayne@68: //		System.err.println("After Fill: ");
jpayne@68: //		printBuffer();
jpayne@68: //		System.err.println();
jpayne@68: 		
jpayne@68: //		System.out.println("Filled buffer; r="+r+", returning "+len);
jpayne@68: 		assert(r==-1 || buffer[len]==slashn);
jpayne@68: 		
jpayne@68: //		System.err.println("lasteol="+(lasteol=='\n' ? "\\n" : lasteol==slashr ? "\\r" : ""+(int)lasteol));
jpayne@68: //		System.err.println("First="+(int)buffer[0]+"\nLastEOL="+(int)lasteol);
jpayne@68: 		
jpayne@68: 		return len;
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	private final synchronized InputStream open(){
jpayne@68: 		if(open){
jpayne@68: 			throw new RuntimeException("Attempt to open already-opened TextFile "+name());
jpayne@68: 		}
jpayne@68: 		open=true;
jpayne@68: 		is=ReadWrite.getInputStream(name(), BUFFERED, allowSubprocess());
jpayne@68: 		bstart=-1;
jpayne@68: 		bstop=-1;
jpayne@68: 		return is;
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	public final ArrayList<byte[]> toByteLines(){
jpayne@68: 		
jpayne@68: 		byte[] s=null;
jpayne@68: 		ArrayList<byte[]> list=new ArrayList<byte[]>(4096);
jpayne@68: 		
jpayne@68: 		for(s=nextLine(); s!=null; s=nextLine()){
jpayne@68: 			list.add(s);
jpayne@68: 		}
jpayne@68: 		
jpayne@68: 		return list;
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	public final long countLines(){
jpayne@68: 		byte[] s=null;
jpayne@68: 		long count=0;
jpayne@68: 		for(s=nextLine(); s!=null; s=nextLine()){count++;}
jpayne@68: 		reset();
jpayne@68: 		
jpayne@68: 		return count;
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	public synchronized final ListNum<byte[]> nextList(){
jpayne@68: 		byte[] line=nextLine();
jpayne@68: 		if(line==null){return null;}
jpayne@68: 		ArrayList<byte[]> list=new ArrayList<byte[]>(200);
jpayne@68: 		list.add(line);
jpayne@68: 		for(int i=1; i<200; i++){
jpayne@68: 			line=nextLine();
jpayne@68: 			if(line==null){break;}
jpayne@68: 			list.add(line);
jpayne@68: 		}
jpayne@68: 		ListNum<byte[]> ln=new ListNum<byte[]>(list, nextID);
jpayne@68: 		nextID++;
jpayne@68: 		return ln;
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	public final boolean exists(){
jpayne@68: 		return name().equals("stdin") || name().startsWith("stdin.") || name().startsWith("jar:") || new File(name()).exists(); //TODO Ugly and unsafe hack for files in jars
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	public final void pushBack(byte[] line){
jpayne@68: 		assert(pushBack==null);
jpayne@68: 		pushBack=line;
jpayne@68: 	}
jpayne@68: 	
jpayne@68: 	public final String name(){return ff.name();}
jpayne@68: 	public final boolean allowSubprocess(){return ff.allowSubprocess();}
jpayne@68: 	
jpayne@68: 	public final FileFormat ff;
jpayne@68: 
jpayne@68: 	public static boolean FORCE_MODE_BF1=false;//!(Shared.GENEPOOL || Shared.DENOVO || Shared.CORI || Shared.WINDOWS);
jpayne@68: 	public static boolean FORCE_MODE_BF2=false;
jpayne@68: 	public static boolean FORCE_MODE_BF3=false;
jpayne@68: 	
jpayne@68: 	protected final static byte slashr='\r', slashn='\n', carrot='>', plus='+', at='@';//, tab='\t';
jpayne@68: 
jpayne@68: 	long a=1;
jpayne@68: 	long b=2;
jpayne@68: 	long c=3;
jpayne@68: 	long d=4;
jpayne@68: 	byte[] p0=null;
jpayne@68: 	byte[] p1=null;
jpayne@68: 	byte[] p2=null;
jpayne@68: 	byte[] p3=null;
jpayne@68: 	private byte[] pushBack=null;
jpayne@68: 	private long nextID=0;
jpayne@68: 	
jpayne@68: 	private boolean open=false;
jpayne@68: 	private byte[] buffer=new byte[bufferlen];
jpayne@68: 	private static final byte[] blankLine=new byte[0];
jpayne@68: 	private int bstart=0, bstop=0;
jpayne@68: 	public InputStream is;
jpayne@68: 	public long lineNum=-1;
jpayne@68: 	
jpayne@68: 	public static boolean verbose=false;
jpayne@68: 	public static boolean BUFFERED=false;
jpayne@68: 	public static int bufferlen=16384;
jpayne@68: 
jpayne@68: 	private boolean errorState=false;
jpayne@68: 	
jpayne@68: }