Mercurial > repos > rliterman > csp2

annotate CSP2/CSP2_env/env-d9b9114564458d9d-741b3de822f2aaca6c6caa4325c4afce/opt/bbmap-39.01-1/current/fileIO/QuickFile.java @ 68:5028fdace37b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 2e9511a184a1ca667c7be0c6321a36dc4e3d116d
author jpayne
date Tue, 18 Mar 2025 16:23:26 -0400
parents
children
rev   line source
jpayne@68 1 package fileIO;
jpayne@68 2 import java.io.File;
jpayne@68 3 import java.io.IOException;
jpayne@68 4 import java.io.InputStream;
jpayne@68 5 import java.util.ArrayList;
jpayne@68 6
jpayne@68 7 import shared.KillSwitch;
jpayne@68 8 import shared.Shared;
jpayne@68 9 import shared.Timer;
jpayne@68 10 import shared.Tools;
jpayne@68 11 import structures.ListNum;
jpayne@68 12
jpayne@68 13
jpayne@68 14 /**
jpayne@68 15 * Written for testing a NERSC slowdown in multithreaded file reading.
jpayne@68 16 * The problem was sometimes sidestepped by eliminating "if(pushBack!=null){" and reimplementing pushback.
jpayne@68 17 * However, that does not address the cause, so is not an overall solution; the cause remains a mystery.
jpayne@68 18 * This class may safely be deleted.
jpayne@68 19 *
jpayne@68 20 * @author Brian Bushnell
jpayne@68 21 *
jpayne@68 22 */
jpayne@68 23 public class QuickFile {
jpayne@68 24
jpayne@68 25
jpayne@68 26 public static void main(String[] args){
jpayne@68 27 QuickFile tf=new QuickFile(args.length>0 ? args[0] : "stdin", true);
jpayne@68 28 long first=0, last=100;
jpayne@68 29 boolean speedtest=false;
jpayne@68 30 if(args.length>1){
jpayne@68 31 if(args[1].equalsIgnoreCase("speedtest")){
jpayne@68 32 speedtest=true;
jpayne@68 33 first=0;
jpayne@68 34 last=Long.MAX_VALUE;
jpayne@68 35 }else{
jpayne@68 36 first=Integer.parseInt(args[1]);
jpayne@68 37 last=first+100;
jpayne@68 38 }
jpayne@68 39 }
jpayne@68 40 if(args.length>2){
jpayne@68 41 last=Integer.parseInt(args[2]);
jpayne@68 42 }
jpayne@68 43 speedtest(tf, first, last, !speedtest);
jpayne@68 44
jpayne@68 45 tf.close();
jpayne@68 46 tf.reset();
jpayne@68 47 tf.close();
jpayne@68 48 }
jpayne@68 49
jpayne@68 50 private static void speedtest(QuickFile tf, long first, long last, boolean reprint){
jpayne@68 51 Timer t=new Timer();
jpayne@68 52 long lines=0;
jpayne@68 53 long bytes=0;
jpayne@68 54 for(long i=0; i<first; i++){tf.nextLine();}
jpayne@68 55 if(reprint){
jpayne@68 56 for(long i=first; i<last; i++){
jpayne@68 57 byte[] s=tf.nextLine();
jpayne@68 58 if(s==null){break;}
jpayne@68 59
jpayne@68 60 lines++;
jpayne@68 61 bytes+=s.length;
jpayne@68 62 System.out.println(new String(s));
jpayne@68 63 }
jpayne@68 64
jpayne@68 65 System.err.println("\n");
jpayne@68 66 System.err.println("Lines: "+lines);
jpayne@68 67 System.err.println("Bytes: "+bytes);
jpayne@68 68 }else{
jpayne@68 69 for(long i=first; i<last; i++){
jpayne@68 70 byte[] s=tf.nextLine();
jpayne@68 71 if(s==null){break;}
jpayne@68 72 lines++;
jpayne@68 73 bytes+=s.length;
jpayne@68 74 }
jpayne@68 75 }
jpayne@68 76 t.stop();
jpayne@68 77
jpayne@68 78 if(!reprint){
jpayne@68 79 System.err.println(Tools.timeLinesBytesProcessed(t, lines, bytes, 8));
jpayne@68 80 }
jpayne@68 81 }
jpayne@68 82
jpayne@68 83 public QuickFile(String fname, boolean allowSubprocess_){
jpayne@68 84 this(FileFormat.testInput(fname, FileFormat.TEXT, null, allowSubprocess_, false));
jpayne@68 85 }
jpayne@68 86
jpayne@68 87 public QuickFile(FileFormat ff_){
jpayne@68 88 ff=ff_;
jpayne@68 89 assert(ff.read()) : ff;
jpayne@68 90 if(verbose){System.err.println("ByteFile1("+ff+")");}
jpayne@68 91 is=open();
jpayne@68 92 }
jpayne@68 93
jpayne@68 94 public final void reset(){
jpayne@68 95 close();
jpayne@68 96 is=open();
jpayne@68 97
jpayne@68 98 pushBack=null;
jpayne@68 99 nextID=0;
jpayne@68 100 }
jpayne@68 101
jpayne@68 102 public synchronized final boolean close(){
jpayne@68 103 if(verbose){System.err.println("Closing "+this.getClass().getName()+" for "+name()+"; open="+open+"; errorState="+errorState);}
jpayne@68 104 if(!open){return errorState;}
jpayne@68 105 open=false;
jpayne@68 106 assert(is!=null);
jpayne@68 107 // assert(false) : name()+","+allowSubprocess();
jpayne@68 108 errorState|=ReadWrite.finishReading(is, name(), (allowSubprocess() || FileFormat.isBamFile(name())));
jpayne@68 109
jpayne@68 110 is=null;
jpayne@68 111 lineNum=-1;
jpayne@68 112 pushBack=null;
jpayne@68 113 if(verbose){System.err.println("Closed "+this.getClass().getName()+" for "+name()+"; open="+open+"; errorState="+errorState);}
jpayne@68 114 return errorState;
jpayne@68 115 }
jpayne@68 116
jpayne@68 117 public byte[] nextLine(){
jpayne@68 118 // if(pushBack!=null){
jpayne@68 119 // byte[] temp=pushBack;
jpayne@68 120 // pushBack=null;
jpayne@68 121 // return temp;
jpayne@68 122 // }
jpayne@68 123
jpayne@68 124 if(verbose){System.err.println("Reading line "+this.getClass().getName()+" for "+name()+"; open="+open+"; errorState="+errorState);}
jpayne@68 125
jpayne@68 126 if(!open || is==null){
jpayne@68 127 if(Shared.WINDOWS){System.err.println("Attempting to read from a closed file: "+name());}
jpayne@68 128 return null;
jpayne@68 129 }
jpayne@68 130
jpayne@68 131 // System.out.println("\nCalled nextLine() for line "+lineNum);
jpayne@68 132 // System.out.println("A: bstart="+bstart+", bstop="+bstop);
jpayne@68 133
jpayne@68 134 //if(bstart<bstop && lasteol==slasher && buffer[bstart]==slashn){bstart++;}
jpayne@68 135 // assert(bstart>=bstop || (buffer[bstart]!=slashn)/*buffer[bstart]>slasher || buffer[bstart]==slashn*/);
jpayne@68 136 int nlpos=bstart;
jpayne@68 137
jpayne@68 138 // System.out.println("B: bstart="+bstart+", bstop="+bstop+", nlpos="+nlpos);
jpayne@68 139 // while(nlpos<bstop && (buffer[nlpos]>slasher || buffer[nlpos]==tab)){nlpos++;}
jpayne@68 140 while(nlpos<bstop && buffer[nlpos]!=slashn){nlpos++;}
jpayne@68 141 // System.out.println("C: bstart="+bstart+", bstop="+bstop+", nlpos="+nlpos);
jpayne@68 142 if(nlpos>=bstop){
jpayne@68 143 nlpos=fillBuffer();
jpayne@68 144 // System.out.println("Filled buffer.");
jpayne@68 145 }
jpayne@68 146 // System.out.println("D: bstart="+bstart+", bstop="+bstop+", nlpos="+nlpos);
jpayne@68 147
jpayne@68 148 if(nlpos<0 || bstop<1){
jpayne@68 149 close();
jpayne@68 150 return null;
jpayne@68 151 }
jpayne@68 152
jpayne@68 153 lineNum++;
jpayne@68 154 //Limit is the position after the last position to copy.
jpayne@68 155 //Limit equals nlpos unless there was a \r before the \n.
jpayne@68 156 final int limit=(nlpos>bstart && buffer[nlpos-1]==slashr) ? nlpos-1 : nlpos;
jpayne@68 157 if(bstart==limit){//Empty line.
jpayne@68 158 bstart=nlpos+1;
jpayne@68 159 // System.out.println("E: bstart="+bstart+", bstop="+bstop+", nlpos="+nlpos+", returning='"+printNL(blankLine)+"'");
jpayne@68 160 return blankLine;
jpayne@68 161 }
jpayne@68 162
jpayne@68 163 byte[] line=KillSwitch.copyOfRange(buffer, bstart, limit);
jpayne@68 164
jpayne@68 165 assert(line.length>0) : bstart+", "+nlpos+", "+limit;
jpayne@68 166 bstart=nlpos+1;
jpayne@68 167 // System.out.println("F: bstart="+bstart+", bstop="+bstop+", nlpos="+nlpos+", returning='"+printNL(line)+"'");
jpayne@68 168 return line;
jpayne@68 169 }
jpayne@68 170
jpayne@68 171 final byte[] dummy=new byte[100];
jpayne@68 172
jpayne@68 173 private int fillBuffer(){
jpayne@68 174 if(bstart<bstop){ //Shift end bytes to beginning
jpayne@68 175 // System.err.println("Shift: "+bstart+", "+bstop);
jpayne@68 176 assert(bstart>0);
jpayne@68 177 // assert(bstop==buffer.length);
jpayne@68 178 int extra=bstop-bstart;
jpayne@68 179 for(int i=0; i<extra; i++, bstart++){
jpayne@68 180 // System.err.print((char)buffer[bstart]);
jpayne@68 181 //System.err.print('.');
jpayne@68 182 buffer[i]=buffer[bstart];
jpayne@68 183 // assert(buffer[i]>=slasher || buffer[i]==tab);
jpayne@68 184 assert(buffer[i]!=slashn);
jpayne@68 185 }
jpayne@68 186 bstop=extra;
jpayne@68 187 // System.err.println();
jpayne@68 188
jpayne@68 189 // {//for debugging only
jpayne@68 190 // buffer=new byte[bufferlen];
jpayne@68 191 // bstop=0;
jpayne@68 192 // bstart=0;
jpayne@68 193 // }
jpayne@68 194 }else{
jpayne@68 195 bstop=0;
jpayne@68 196 }
jpayne@68 197
jpayne@68 198 bstart=0;
jpayne@68 199 int len=bstop;
jpayne@68 200 int r=-1;
jpayne@68 201 while(len==bstop){//hit end of input without encountering a newline
jpayne@68 202 if(bstop==buffer.length){
jpayne@68 203 // assert(false) : len+", "+bstop;
jpayne@68 204 buffer=KillSwitch.copyOf(buffer, buffer.length*2);
jpayne@68 205 }
jpayne@68 206 try {
jpayne@68 207 r=is.read(buffer, bstop, buffer.length-bstop);
jpayne@68 208 // byte[] x=new byte[buffer.length-bstop];
jpayne@68 209 // r=is.read(x);
jpayne@68 210 // if(r>0){
jpayne@68 211 // for(int i=0, j=bstop; i<r; i++, j++){
jpayne@68 212 // buffer[j]=x[i];
jpayne@68 213 // }
jpayne@68 214 // }
jpayne@68 215 } catch (IOException e) {
jpayne@68 216 e.printStackTrace();
jpayne@68 217 System.err.println("open="+open);
jpayne@68 218 }
jpayne@68 219 if(r>0){
jpayne@68 220 bstop=bstop+r;
jpayne@68 221 // while(len<bstop && (buffer[len]>slasher || buffer[len]==tab)){len++;}
jpayne@68 222 while(len<bstop && buffer[len]!=slashn){len++;}
jpayne@68 223 }else{
jpayne@68 224 len=bstop;
jpayne@68 225 break;
jpayne@68 226 }
jpayne@68 227 }
jpayne@68 228
jpayne@68 229 // System.err.println("After Fill: ");
jpayne@68 230 // printBuffer();
jpayne@68 231 // System.err.println();
jpayne@68 232
jpayne@68 233 // System.out.println("Filled buffer; r="+r+", returning "+len);
jpayne@68 234 assert(r==-1 || buffer[len]==slashn);
jpayne@68 235
jpayne@68 236 // System.err.println("lasteol="+(lasteol=='\n' ? "\\n" : lasteol==slashr ? "\\r" : ""+(int)lasteol));
jpayne@68 237 // System.err.println("First="+(int)buffer[0]+"\nLastEOL="+(int)lasteol);
jpayne@68 238
jpayne@68 239 return len;
jpayne@68 240 }
jpayne@68 241
jpayne@68 242 private final synchronized InputStream open(){
jpayne@68 243 if(open){
jpayne@68 244 throw new RuntimeException("Attempt to open already-opened TextFile "+name());
jpayne@68 245 }
jpayne@68 246 open=true;
jpayne@68 247 is=ReadWrite.getInputStream(name(), BUFFERED, allowSubprocess());
jpayne@68 248 bstart=-1;
jpayne@68 249 bstop=-1;
jpayne@68 250 return is;
jpayne@68 251 }
jpayne@68 252
jpayne@68 253 public final ArrayList<byte[]> toByteLines(){
jpayne@68 254
jpayne@68 255 byte[] s=null;
jpayne@68 256 ArrayList<byte[]> list=new ArrayList<byte[]>(4096);
jpayne@68 257
jpayne@68 258 for(s=nextLine(); s!=null; s=nextLine()){
jpayne@68 259 list.add(s);
jpayne@68 260 }
jpayne@68 261
jpayne@68 262 return list;
jpayne@68 263 }
jpayne@68 264
jpayne@68 265 public final long countLines(){
jpayne@68 266 byte[] s=null;
jpayne@68 267 long count=0;
jpayne@68 268 for(s=nextLine(); s!=null; s=nextLine()){count++;}
jpayne@68 269 reset();
jpayne@68 270
jpayne@68 271 return count;
jpayne@68 272 }
jpayne@68 273
jpayne@68 274 public synchronized final ListNum<byte[]> nextList(){
jpayne@68 275 byte[] line=nextLine();
jpayne@68 276 if(line==null){return null;}
jpayne@68 277 ArrayList<byte[]> list=new ArrayList<byte[]>(200);
jpayne@68 278 list.add(line);
jpayne@68 279 for(int i=1; i<200; i++){
jpayne@68 280 line=nextLine();
jpayne@68 281 if(line==null){break;}
jpayne@68 282 list.add(line);
jpayne@68 283 }
jpayne@68 284 ListNum<byte[]> ln=new ListNum<byte[]>(list, nextID);
jpayne@68 285 nextID++;
jpayne@68 286 return ln;
jpayne@68 287 }
jpayne@68 288
jpayne@68 289 public final boolean exists(){
jpayne@68 290 return name().equals("stdin") || name().startsWith("stdin.") || name().startsWith("jar:") || new File(name()).exists(); //TODO Ugly and unsafe hack for files in jars
jpayne@68 291 }
jpayne@68 292
jpayne@68 293 public final void pushBack(byte[] line){
jpayne@68 294 assert(pushBack==null);
jpayne@68 295 pushBack=line;
jpayne@68 296 }
jpayne@68 297
jpayne@68 298 public final String name(){return ff.name();}
jpayne@68 299 public final boolean allowSubprocess(){return ff.allowSubprocess();}
jpayne@68 300
jpayne@68 301 public final FileFormat ff;
jpayne@68 302
jpayne@68 303 public static boolean FORCE_MODE_BF1=false;//!(Shared.GENEPOOL || Shared.DENOVO || Shared.CORI || Shared.WINDOWS);
jpayne@68 304 public static boolean FORCE_MODE_BF2=false;
jpayne@68 305 public static boolean FORCE_MODE_BF3=false;
jpayne@68 306
jpayne@68 307 protected final static byte slashr='\r', slashn='\n', carrot='>', plus='+', at='@';//, tab='\t';
jpayne@68 308
jpayne@68 309 long a=1;
jpayne@68 310 long b=2;
jpayne@68 311 long c=3;
jpayne@68 312 long d=4;
jpayne@68 313 byte[] p0=null;
jpayne@68 314 byte[] p1=null;
jpayne@68 315 byte[] p2=null;
jpayne@68 316 byte[] p3=null;
jpayne@68 317 private byte[] pushBack=null;
jpayne@68 318 private long nextID=0;
jpayne@68 319
jpayne@68 320 private boolean open=false;
jpayne@68 321 private byte[] buffer=new byte[bufferlen];
jpayne@68 322 private static final byte[] blankLine=new byte[0];
jpayne@68 323 private int bstart=0, bstop=0;
jpayne@68 324 public InputStream is;
jpayne@68 325 public long lineNum=-1;
jpayne@68 326
jpayne@68 327 public static boolean verbose=false;
jpayne@68 328 public static boolean BUFFERED=false;
jpayne@68 329 public static int bufferlen=16384;
jpayne@68 330
jpayne@68 331 private boolean errorState=false;
jpayne@68 332
jpayne@68 333 }