Mercurial > repos > rliterman > csp2

annotate CSP2/CSP2_env/env-d9b9114564458d9d-741b3de822f2aaca6c6caa4325c4afce/opt/bbmap-39.01-1/current/sketch/CompareSketch.java @ 68:5028fdace37b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 2e9511a184a1ca667c7be0c6321a36dc4e3d116d
author jpayne
date Tue, 18 Mar 2025 16:23:26 -0400
parents
children
rev   line source
jpayne@68 1 package sketch;
jpayne@68 2
jpayne@68 3 import java.io.File;
jpayne@68 4 import java.io.PrintStream;
jpayne@68 5 import java.util.ArrayList;
jpayne@68 6 import java.util.Collection;
jpayne@68 7 import java.util.LinkedHashSet;
jpayne@68 8 import java.util.concurrent.ConcurrentHashMap;
jpayne@68 9 import java.util.concurrent.atomic.AtomicInteger;
jpayne@68 10
jpayne@68 11 import fileIO.ByteFile;
jpayne@68 12 import fileIO.ByteStreamWriter;
jpayne@68 13 import fileIO.FileFormat;
jpayne@68 14 import fileIO.ReadWrite;
jpayne@68 15 import kmer.AbstractKmerTableSet;
jpayne@68 16 import shared.Parse;
jpayne@68 17 import shared.Parser;
jpayne@68 18 import shared.PreParser;
jpayne@68 19 import shared.ReadStats;
jpayne@68 20 import shared.Shared;
jpayne@68 21 import shared.Timer;
jpayne@68 22 import shared.Tools;
jpayne@68 23 import structures.ByteBuilder;
jpayne@68 24 import tax.TaxFilter;
jpayne@68 25 import tax.TaxTree;
jpayne@68 26
jpayne@68 27 /**
jpayne@68 28 * Compares one or more input sketches to a set of reference sketches.
jpayne@68 29 *
jpayne@68 30 * @author Brian Bushnell
jpayne@68 31 * @date July 29, 2016
jpayne@68 32 *
jpayne@68 33 */
jpayne@68 34 public class CompareSketch extends SketchObject {
jpayne@68 35
jpayne@68 36
jpayne@68 37
jpayne@68 38 /*--------------------------------------------------------------*/
jpayne@68 39 /*---------------- Initialization ----------------*/
jpayne@68 40 /*--------------------------------------------------------------*/
jpayne@68 41
jpayne@68 42 /**
jpayne@68 43 * Code entrance from the command line.
jpayne@68 44 * @param args Command line arguments
jpayne@68 45 */
jpayne@68 46 public static void main(String[] args){
jpayne@68 47
jpayne@68 48 //Start a timer immediately upon code entrance.
jpayne@68 49 Timer t=new Timer();
jpayne@68 50
jpayne@68 51 final int oldBufLen=Shared.bufferLen();
jpayne@68 52
jpayne@68 53 //Create an instance of this class
jpayne@68 54 CompareSketch x=new CompareSketch(args);
jpayne@68 55
jpayne@68 56 //Run the object
jpayne@68 57 x.process(t);
jpayne@68 58
jpayne@68 59 Shared.setBufferLen(oldBufLen);
jpayne@68 60
jpayne@68 61 //Close the print stream if it was redirected
jpayne@68 62 Shared.closeStream(x.outstream);
jpayne@68 63
jpayne@68 64 alignerPool.poison();
jpayne@68 65 }
jpayne@68 66
jpayne@68 67 /**
jpayne@68 68 * Constructor.
jpayne@68 69 * @param args Command line arguments
jpayne@68 70 */
jpayne@68 71 public CompareSketch(String[] args){
jpayne@68 72
jpayne@68 73 {//Preparse block for help, config files, and outstream
jpayne@68 74 PreParser pp=new PreParser(args, null, false);
jpayne@68 75 // PreParser pp=new PreParser(args, getClass(), false);
jpayne@68 76 args=pp.args;
jpayne@68 77 outstream=pp.outstream;
jpayne@68 78 silent=PreParser.silent;
jpayne@68 79 if(silent){AbstractKmerTableSet.DISPLAY_PROGRESS=false;}
jpayne@68 80 }
jpayne@68 81
jpayne@68 82 //Set shared static variables
jpayne@68 83 ReadWrite.USE_PIGZ=ReadWrite.USE_UNPIGZ=true;
jpayne@68 84 ReadWrite.MAX_ZIP_THREADS=Shared.threads();
jpayne@68 85 KILL_OK=true;
jpayne@68 86 TaxFilter.REQUIRE_PRESENT=false;
jpayne@68 87 defaultParams.mode=PER_FILE;
jpayne@68 88
jpayne@68 89 //Create a parser object
jpayne@68 90 Parser parser=new Parser();
jpayne@68 91 parser.out1="stdout.txt";
jpayne@68 92
jpayne@68 93 //Parse each argument
jpayne@68 94 for(int i=0; i<args.length; i++){
jpayne@68 95 String arg=args[i];
jpayne@68 96
jpayne@68 97 //Break arguments into their constituent parts, in the form of "a=b"
jpayne@68 98 String[] split=arg.split("=");
jpayne@68 99 String a=split[0].toLowerCase();
jpayne@68 100 String b=split.length>1 ? split[1] : null;
jpayne@68 101
jpayne@68 102 if(a.equals("verbose")){
jpayne@68 103 verbose=Parse.parseBoolean(b);
jpayne@68 104 }else if(a.equals("in")){
jpayne@68 105 addFiles(b, in);
jpayne@68 106 }else if(parseSketchFlags(arg, a, b)){
jpayne@68 107 //Do nothing
jpayne@68 108 }else if(a.equals("parse_flag_goes_here")){
jpayne@68 109 long fake_variable=Parse.parseKMG(b);
jpayne@68 110 //Set a variable here
jpayne@68 111 }else if(a.equals("ordered")){
jpayne@68 112 ordered=Parse.parseBoolean(b);
jpayne@68 113 }else if(a.equals("alltoall") || a.equals("ata")){
jpayne@68 114 allToAll=Parse.parseBoolean(b);
jpayne@68 115 }else if(a.equals("skipcompare") || a.equals("sketchonly")){
jpayne@68 116 skipCompare=Parse.parseBoolean(b);
jpayne@68 117 }else if(a.equals("compareself") || a.equals("includeself")){
jpayne@68 118 compareSelf=Parse.parseBoolean(b);
jpayne@68 119 }else if(a.equals("printmemory")){
jpayne@68 120 printMemory=Parse.parseBoolean(b);
jpayne@68 121 }else if(a.equals("parsesubunit")){
jpayne@68 122 SketchMaker.parseSubunit=Parse.parseBoolean(b);
jpayne@68 123 }
jpayne@68 124
jpayne@68 125 else if(a.equals("taxtree") || a.equals("tree")){
jpayne@68 126 taxTreeFile=b;
jpayne@68 127 }
jpayne@68 128
jpayne@68 129 else if(a.equals("name") || a.equals("taxname")){
jpayne@68 130 outTaxName=b;
jpayne@68 131 }else if(a.equals("name0")){
jpayne@68 132 outName0=b;
jpayne@68 133 }else if(a.equals("fname")){
jpayne@68 134 outFname=b;
jpayne@68 135 }else if(a.equals("outsketch") || a.equals("sketchout") || a.equals("outs") || a.equals("sketch")){
jpayne@68 136 outSketch=b;
jpayne@68 137 }else if(a.equals("files")){
jpayne@68 138 sketchFiles=Integer.parseInt(b);
jpayne@68 139 }else if(a.equals("taxid") || a.equals("tid")){
jpayne@68 140 outTaxID=Integer.parseInt(b);
jpayne@68 141 }else if(a.equals("spid")){
jpayne@68 142 outSpid=Integer.parseInt(b);
jpayne@68 143 }else if(a.equals("imgid")){
jpayne@68 144 outImgID=Integer.parseInt(b);
jpayne@68 145 }else if((a.startsWith("meta_") || a.startsWith("mt_")) && b!=null){
jpayne@68 146 if(outMeta==null){outMeta=new ArrayList<String>();}
jpayne@68 147 int underscore=a.indexOf('_', 0);
jpayne@68 148 outMeta.add(a.substring(underscore+1)+":"+b);
jpayne@68 149 }
jpayne@68 150
jpayne@68 151 else if(searcher.parse(arg, a, b, false)){
jpayne@68 152 // System.err.println("*"+arg);
jpayne@68 153 parser.parse(arg, a, b); //Catches shared flags like "threads"
jpayne@68 154 Blacklist.parseBlacklist(arg, a, b); //Catches flags like "nt" or "refseq"
jpayne@68 155 }
jpayne@68 156
jpayne@68 157 else if(parser.parse(arg, a, b)){//Parse standard flags in the parser
jpayne@68 158 //do nothing
jpayne@68 159 }
jpayne@68 160
jpayne@68 161 else if(searcher.parse(arg, a, b, true)){
jpayne@68 162 // System.err.println("**"+arg);
jpayne@68 163 //do nothing
jpayne@68 164 }
jpayne@68 165
jpayne@68 166 else{
jpayne@68 167 outstream.println("Unknown parameter "+args[i]);
jpayne@68 168 assert(false) : "Unknown parameter "+args[i];
jpayne@68 169 }
jpayne@68 170 }
jpayne@68 171 if("auto".equalsIgnoreCase(taxTreeFile)){taxTreeFile=TaxTree.defaultTreeFile();}
jpayne@68 172
jpayne@68 173 outMeta=SketchObject.fixMeta(outMeta);
jpayne@68 174 SketchObject.postParse();
jpayne@68 175
jpayne@68 176 if(skipCompare){
jpayne@68 177 allToAll=false;
jpayne@68 178 searcher.autoIndex=false;
jpayne@68 179 makeIndex=false;
jpayne@68 180 in.addAll(searcher.refFiles);
jpayne@68 181 searcher.refFiles.clear();
jpayne@68 182 }else if(in.isEmpty() && args.length>0 && !allToAll){ //Allows first argument to be used as the input file without in= flag
jpayne@68 183 String x=args[0];
jpayne@68 184 if(x.indexOf('=')<0 && new File(x).exists() && searcher.refFiles.contains(x)){
jpayne@68 185 searcher.refFiles.remove(x);
jpayne@68 186 in.add(x);
jpayne@68 187 }
jpayne@68 188 }
jpayne@68 189
jpayne@68 190 {//Process parser fields
jpayne@68 191 overwrite=ReadStats.overwrite=parser.overwrite;
jpayne@68 192 append=ReadStats.append=parser.append;
jpayne@68 193
jpayne@68 194 out=parser.out1;
jpayne@68 195 }
jpayne@68 196
jpayne@68 197 // assert(false) : in+"\n"+searcher.refFiles;
jpayne@68 198
jpayne@68 199 if(allToAll){
jpayne@68 200 LinkedHashSet<String> set=new LinkedHashSet<String>();
jpayne@68 201 set.addAll(in);
jpayne@68 202 set.addAll(searcher.refFiles);
jpayne@68 203 in.clear();
jpayne@68 204 searcher.refFiles.clear();
jpayne@68 205 in.addAll(set);
jpayne@68 206 searcher.refFiles.addAll(set);
jpayne@68 207 }
jpayne@68 208
jpayne@68 209 //Ensure there is an input file
jpayne@68 210 if(in.isEmpty() && !skipCompare){throw new RuntimeException("Error - at least one input file is required.");}
jpayne@68 211
jpayne@68 212 //Ensure there is an ref file
jpayne@68 213 if(searcher.refFiles.isEmpty() && !skipCompare){
jpayne@68 214 if(outSketch==null){throw new RuntimeException("Error - at least one reference file is required.");}
jpayne@68 215 }
jpayne@68 216
jpayne@68 217 //Adjust the number of threads for input file reading
jpayne@68 218 if(!ByteFile.FORCE_MODE_BF1 && !ByteFile.FORCE_MODE_BF2 && Shared.threads()>2){
jpayne@68 219 ByteFile.FORCE_MODE_BF2=true;
jpayne@68 220 }
jpayne@68 221
jpayne@68 222 ffout=FileFormat.testOutput(out, FileFormat.TEXT, null, false, overwrite, append, ordered);
jpayne@68 223 if(!ffout.stdio() && !defaultParams.setColors){defaultParams.printColors=false;}
jpayne@68 224
jpayne@68 225 //Ensure input files can be read
jpayne@68 226 if(!Tools.testInputFiles(false, true, taxTreeFile)){
jpayne@68 227 throw new RuntimeException("\nCan't read some input files.\n");
jpayne@68 228 }
jpayne@68 229 if(!Tools.testInputFiles(true, false, in.toArray(new String[0]))){
jpayne@68 230 if(in.size()==1){
jpayne@68 231 String s=in.get(0);
jpayne@68 232 String s1=s.replaceFirst("#", "1"), s2=s.replaceFirst("#", "2");
jpayne@68 233 Tools.testInputFiles(true, false, s1, s2);
jpayne@68 234 }else{
jpayne@68 235 throw new RuntimeException("\nCan't read some input files.\n");
jpayne@68 236 }
jpayne@68 237 }
jpayne@68 238
jpayne@68 239 // assert(makeIndex || defaultParams.printContam2==false) : "Contam2 requires the flag index=t";
jpayne@68 240
jpayne@68 241 SSUMap.load(outstream);
jpayne@68 242 if(taxTreeFile!=null){setTaxtree(taxTreeFile, silent ? null : outstream);}
jpayne@68 243 defaultParams.postParse(true, true);
jpayne@68 244 if(!defaultParams.printSSU){processSSU=false;}
jpayne@68 245 allowMultithreadedFastq=in.size()<2 && !allToAll;
jpayne@68 246 if(!allowMultithreadedFastq){Shared.capBufferLen(40);}
jpayne@68 247 // assert(defaultParams.checkValid());
jpayne@68 248 }
jpayne@68 249
jpayne@68 250 /*--------------------------------------------------------------*/
jpayne@68 251 /*---------------- Outer Methods ----------------*/
jpayne@68 252 /*--------------------------------------------------------------*/
jpayne@68 253
jpayne@68 254 public void process(Timer t){
jpayne@68 255 Timer ttotal=new Timer();
jpayne@68 256
jpayne@68 257 t.start();
jpayne@68 258
jpayne@68 259 if(!silent){outstream.println("Loading sketches.");}
jpayne@68 260 searcher.makeTool(1, false, defaultParams.mergePairs);
jpayne@68 261 SketchTool tool=new SketchTool(targetSketchSize, defaultParams);
jpayne@68 262
jpayne@68 263 final int mode2=(defaultParams.mode==PER_FILE ? PER_FILE : PER_TAXA);
jpayne@68 264 if(skipCompare){
jpayne@68 265 makeIndex=false;
jpayne@68 266 inSketches=tool.loadSketches_MT(defaultParams, in);
jpayne@68 267 }else if(!useWhitelist || allToAll){
jpayne@68 268 if(allToAll){
jpayne@68 269 makeIndex=searcher.refFileCount()>0 && (makeIndex || defaultParams.needIndex() || searcher.autoIndex);
jpayne@68 270 searcher.loadReferences(mode2, defaultParams);
jpayne@68 271 inSketches=(ArrayList<Sketch>) searcher.refSketches.clone();
jpayne@68 272 }else{
jpayne@68 273 inSketches=tool.loadSketches_MT(defaultParams, in);
jpayne@68 274
jpayne@68 275 for(Sketch sk : inSketches){
jpayne@68 276 if(sk.taxID<1 || sk.taxID>=minFakeID || outTaxID>0){sk.taxID=outTaxID;}
jpayne@68 277 if(outSpid>0){sk.spid=outSpid;}
jpayne@68 278 if(outImgID>0){sk.imgID=outImgID;}
jpayne@68 279 if(outTaxName!=null){sk.setTaxName(outTaxName);}
jpayne@68 280 if(outFname!=null){sk.setFname(outFname);}
jpayne@68 281 if(outName0!=null){sk.setName0(outName0);}
jpayne@68 282 if(SketchMaker.parseSubunit && sk.name0()!=null){
jpayne@68 283 if(outMeta!=null){
jpayne@68 284 sk.meta=(ArrayList<String>)sk.meta.clone();
jpayne@68 285 }else if(sk.meta==null){
jpayne@68 286 if(sk.name0().contains("SSU_")){
jpayne@68 287 sk.addMeta("subunit:ssu");
jpayne@68 288 }else if(sk.name0().contains("LSU_")){
jpayne@68 289 sk.addMeta("subunit:lsu");
jpayne@68 290 }
jpayne@68 291 }
jpayne@68 292 }
jpayne@68 293 sk.setMeta(outMeta);
jpayne@68 294 if(defaultParams.printSSU()){sk.loadSSU();}//since taxID was just set
jpayne@68 295 }
jpayne@68 296
jpayne@68 297 if(outTaxID>0){
jpayne@68 298 for(Sketch sk : inSketches){
jpayne@68 299 if(sk.taxID<1 || sk.taxID>=minFakeID){sk.taxID=outTaxID;}
jpayne@68 300 }
jpayne@68 301 }
jpayne@68 302 makeIndex=searcher.refFileCount()>0 && ((searcher.autoIndex && inSketches.size()>8) || defaultParams.needIndex() || (makeIndex && !searcher.autoIndex));
jpayne@68 303 searcher.loadReferences(mode2, defaultParams);
jpayne@68 304 if(mode2==PER_FILE){
jpayne@68 305 int max=inSketches.size();
jpayne@68 306 for(int i=0; i<searcher.refSketches.size(); i++){
jpayne@68 307 searcher.refSketches.get(i).sketchID=max+i+1;
jpayne@68 308 }
jpayne@68 309 }
jpayne@68 310 }
jpayne@68 311 }else{
jpayne@68 312 //assert(searcher.makeIndex && !searcher.autoIndex) : "whitelist=t requires index=t";
jpayne@68 313 makeIndex=true; //(searcher.refFileCount()>0); //Index is required in whitelist mode.
jpayne@68 314 searcher.loadReferences(mode2, defaultParams);
jpayne@68 315 inSketches=tool.loadSketches_MT(defaultParams, in);
jpayne@68 316 }
jpayne@68 317
jpayne@68 318 if(outSketch!=null){
jpayne@68 319 writeSketches(outSketch, sketchFiles);
jpayne@68 320 }
jpayne@68 321
jpayne@68 322 final int numLoaded=(inSketches.size()+searcher.refSketchCount())/(allToAll ? 2 : 1);
jpayne@68 323 t.stop();
jpayne@68 324 if(!silent){outstream.println("Loaded "+numLoaded+" sketch"+(numLoaded==1 ? "" : "es")+" in "+t.toString());}
jpayne@68 325 if(printMemory){
jpayne@68 326 System.gc();
jpayne@68 327 Shared.printMemory();
jpayne@68 328 }
jpayne@68 329
jpayne@68 330 if(skipCompare) {
jpayne@68 331 ttotal.stop("Total Time: \t");
jpayne@68 332 return;
jpayne@68 333 }
jpayne@68 334
jpayne@68 335 t.start();
jpayne@68 336
jpayne@68 337
jpayne@68 338 ByteStreamWriter tsw=(ffout==null ? null : new ByteStreamWriter(ffout));
jpayne@68 339 if(tsw!=null){
jpayne@68 340 tsw.start();
jpayne@68 341 if(defaultParams.format==DisplayParams.FORMAT_QUERY_REF_ANI || defaultParams.format==DisplayParams.FORMAT_CONSTELLATION){
jpayne@68 342 String s=defaultParams.header()+"\n";
jpayne@68 343 tsw.forcePrint(s.getBytes());
jpayne@68 344 }
jpayne@68 345 }
jpayne@68 346
jpayne@68 347 boolean success=true;
jpayne@68 348 final int inSize=inSketches.size();
jpayne@68 349 if(inSize==1 || Shared.threads()<2 || inSize<4){
jpayne@68 350 ByteBuilder sb=new ByteBuilder();
jpayne@68 351 success=searcher.compare(inSketches, sb, defaultParams, Shared.threads());
jpayne@68 352 success&=(!searcher.errorState);
jpayne@68 353 if(tsw!=null){
jpayne@68 354 sb.append('\n');
jpayne@68 355 if(ordered){
jpayne@68 356 tsw.addJob(sb);
jpayne@68 357 }else{
jpayne@68 358 tsw.println(sb);
jpayne@68 359 }
jpayne@68 360 }
jpayne@68 361 }else{//More sketches than threads, and more than one thread
jpayne@68 362 final int threads=Tools.min(Shared.threads(), inSize);
jpayne@68 363
jpayne@68 364 ArrayList<CompareThread> alct=new ArrayList<CompareThread>(threads);
jpayne@68 365 AtomicInteger next=new AtomicInteger(0);
jpayne@68 366 for(int i=0; i<threads; i++){
jpayne@68 367 alct.add(new CompareThread(i, next, tsw));
jpayne@68 368 }
jpayne@68 369 for(CompareThread ct : alct){ct.start();}
jpayne@68 370 for(CompareThread ct : alct){
jpayne@68 371
jpayne@68 372 //Wait until this thread has terminated
jpayne@68 373 while(ct.getState()!=Thread.State.TERMINATED){
jpayne@68 374 try {
jpayne@68 375 //Attempt a join operation
jpayne@68 376 ct.join();
jpayne@68 377 } catch (InterruptedException e) {
jpayne@68 378 e.printStackTrace();
jpayne@68 379 }
jpayne@68 380 }
jpayne@68 381
jpayne@68 382 synchronized(ct){
jpayne@68 383 success&=ct.success;
jpayne@68 384 }
jpayne@68 385 }
jpayne@68 386 alct=null;
jpayne@68 387 }
jpayne@68 388
jpayne@68 389 //Track whether any threads failed
jpayne@68 390 if(!success){errorState=true;}
jpayne@68 391 if(tsw!=null){errorState|=tsw.poisonAndWait();}
jpayne@68 392
jpayne@68 393 t.stop();
jpayne@68 394 // long comparisons=(makeIndex ? searcher.comparisons.get() :
jpayne@68 395 // allToAll ? (inSketches.size()*(long)(inSketches.size()-(compareSelf ? 0 : 1)))
jpayne@68 396 // : inSketches.size()*(long)searcher.refSketchCount());
jpayne@68 397 long comparisons=searcher.comparisons.get();
jpayne@68 398 if(!skipCompare && !silent) {outstream.println("\nRan "+comparisons+" comparison"+(comparisons==1 ? "" : "s")+" in "+t);}
jpayne@68 399 ttotal.stop();
jpayne@68 400 if(!silent){outstream.println("Total Time: \t"+ttotal);}
jpayne@68 401 }
jpayne@68 402
jpayne@68 403 void writeSketches(String fname, int files){
jpayne@68 404 if(fname==null){return;}
jpayne@68 405 if(files==1 || fname.indexOf('#')<0){
jpayne@68 406 writeOneSketchFile(fname);
jpayne@68 407 }else{
jpayne@68 408 writeManySketchFiles(fname, files);
jpayne@68 409 }
jpayne@68 410 }
jpayne@68 411
jpayne@68 412 void writeOneSketchFile(String fname){
jpayne@68 413 if(fname==null){return;}
jpayne@68 414 ByteBuilder bb=new ByteBuilder();
jpayne@68 415 ByteStreamWriter bsw=new ByteStreamWriter(outSketch, overwrite, append, true, FileFormat.SKETCH);
jpayne@68 416 bsw.start();
jpayne@68 417 for(Sketch sk : inSketches){
jpayne@68 418 sk.toBytes(bb);
jpayne@68 419 bsw.print(bb);
jpayne@68 420 bb.clear();
jpayne@68 421 }
jpayne@68 422 bsw.poisonAndWait();
jpayne@68 423 errorState|=bsw.errorState;
jpayne@68 424 }
jpayne@68 425
jpayne@68 426 void writeManySketchFiles(String fname, int files){
jpayne@68 427 if(fname==null){return;}
jpayne@68 428 assert(fname.indexOf('#')>=0) : fname;
jpayne@68 429 assert(files>0) : files;
jpayne@68 430
jpayne@68 431 ByteStreamWriter[] bswa=new ByteStreamWriter[files];
jpayne@68 432 for(int i=0; i<files; i++){
jpayne@68 433 ByteStreamWriter bsw=new ByteStreamWriter(outSketch.replaceFirst("#", ""+i), overwrite, append, true, FileFormat.SKETCH);
jpayne@68 434 bsw.start();
jpayne@68 435 bswa[i]=bsw;
jpayne@68 436 }
jpayne@68 437 for(Sketch sk : inSketches){
jpayne@68 438 ByteBuilder bb=new ByteBuilder(4096);
jpayne@68 439 sk.toBytes(bb);
jpayne@68 440 bswa[sk.sketchID%files].addJob(bb);
jpayne@68 441 }
jpayne@68 442 for(ByteStreamWriter bsw : bswa){
jpayne@68 443 bsw.poisonAndWait();
jpayne@68 444 errorState|=bsw.errorState;
jpayne@68 445 }
jpayne@68 446 }
jpayne@68 447
jpayne@68 448 /*--------------------------------------------------------------*/
jpayne@68 449 /*---------------- Inner Methods ----------------*/
jpayne@68 450 /*--------------------------------------------------------------*/
jpayne@68 451
jpayne@68 452 private static void addFiles(String a, Collection<String> list){
jpayne@68 453 if(a==null){return;}
jpayne@68 454 File f=null;
jpayne@68 455 if(a.indexOf(',')>=0){f=new File(a);}
jpayne@68 456 if(f==null || f.exists()){
jpayne@68 457 list.add(a);
jpayne@68 458 }else{
jpayne@68 459 for(String s : a.split(",")){list.add(s);}
jpayne@68 460 }
jpayne@68 461 }
jpayne@68 462
jpayne@68 463 /*--------------------------------------------------------------*/
jpayne@68 464 /*---------------- Inner Classes ----------------*/
jpayne@68 465 /*--------------------------------------------------------------*/
jpayne@68 466
jpayne@68 467 private class CompareThread extends Thread {
jpayne@68 468
jpayne@68 469 CompareThread(final int tid_, final AtomicInteger nextSketch_, ByteStreamWriter tsw_){
jpayne@68 470 tid=tid_;
jpayne@68 471 nextSketch=nextSketch_;
jpayne@68 472 tsw=tsw_;
jpayne@68 473 }
jpayne@68 474
jpayne@68 475 @Override
jpayne@68 476 public void run(){
jpayne@68 477 success=false;
jpayne@68 478 final int inLim=inSketches.size();
jpayne@68 479 final boolean json=defaultParams.json();
jpayne@68 480
jpayne@68 481 for(int inNum=nextSketch.getAndIncrement(); inNum<inLim; inNum=nextSketch.getAndIncrement()){
jpayne@68 482 Sketch a=inSketches.get(inNum);
jpayne@68 483 assert(buffer.cbs==null); //Because this sketch will only be used by one thread at a time, so per-buffer bitsets are not needed.
jpayne@68 484 SketchResults sr=searcher.processSketch(a, buffer, fakeID, map, defaultParams, 1);
jpayne@68 485 a.clearRefHitCounts();
jpayne@68 486
jpayne@68 487 if(tsw!=null){
jpayne@68 488 ByteBuilder sb=sr.toText(defaultParams);
jpayne@68 489 synchronized(tsw){
jpayne@68 490 if(ordered){
jpayne@68 491 if(json){
jpayne@68 492 if(inNum==0){
jpayne@68 493 sb.insert(0, (byte)'[');//Rare, slow case
jpayne@68 494 }
jpayne@68 495 if(inNum<inLim-1){
jpayne@68 496 sb.append(',');
jpayne@68 497 }else{
jpayne@68 498 sb.append(']');
jpayne@68 499 }
jpayne@68 500 }
jpayne@68 501 tsw.add(sb, inNum);
jpayne@68 502 }else{
jpayne@68 503 if(json){
jpayne@68 504 if(resultsPrinted==0){
jpayne@68 505 tsw.print('[');
jpayne@68 506 }else{
jpayne@68 507 sb.insert(0, (byte)',');
jpayne@68 508 }
jpayne@68 509 }
jpayne@68 510 tsw.print(sb);
jpayne@68 511 }
jpayne@68 512 resultsPrinted++;
jpayne@68 513 }
jpayne@68 514 }
jpayne@68 515 }
jpayne@68 516 synchronized(this){success=true;}
jpayne@68 517 }
jpayne@68 518
jpayne@68 519 private final int tid;
jpayne@68 520 private final CompareBuffer buffer=new CompareBuffer(false);
jpayne@68 521
jpayne@68 522 private final AtomicInteger nextSketch;
jpayne@68 523 private final AtomicInteger fakeID=new AtomicInteger(minFakeID);
jpayne@68 524 private ConcurrentHashMap<Integer, Comparison> map=new ConcurrentHashMap<Integer, Comparison>(101);
jpayne@68 525 final ByteStreamWriter tsw;
jpayne@68 526
jpayne@68 527 boolean success=false;
jpayne@68 528
jpayne@68 529 }
jpayne@68 530
jpayne@68 531 /*--------------------------------------------------------------*/
jpayne@68 532 /*---------------- Fields ----------------*/
jpayne@68 533 /*--------------------------------------------------------------*/
jpayne@68 534
jpayne@68 535 private ArrayList<String> in=new ArrayList<String>();
jpayne@68 536
jpayne@68 537 private String out="stdout.txt";
jpayne@68 538
jpayne@68 539 private String taxTreeFile=null;
jpayne@68 540
jpayne@68 541 private ArrayList<Sketch> inSketches;
jpayne@68 542
jpayne@68 543 public final SketchSearcher searcher=new SketchSearcher();
jpayne@68 544
jpayne@68 545 private boolean printMemory=false;
jpayne@68 546 private boolean silent=false;
jpayne@68 547
jpayne@68 548 /*Override metadata */
jpayne@68 549 private String outTaxName=null;
jpayne@68 550 private String outFname=null;
jpayne@68 551 private String outName0=null;
jpayne@68 552 private String outSketch=null;
jpayne@68 553 private int sketchFiles=1;
jpayne@68 554 private int outTaxID=-1;
jpayne@68 555 private long outSpid=-1;
jpayne@68 556 private long outImgID=-1;
jpayne@68 557 private ArrayList<String> outMeta=null;
jpayne@68 558 private long resultsPrinted=0;
jpayne@68 559
jpayne@68 560 /*--------------------------------------------------------------*/
jpayne@68 561 /*---------------- Final Fields ----------------*/
jpayne@68 562 /*--------------------------------------------------------------*/
jpayne@68 563
jpayne@68 564 /** Primary output file */
jpayne@68 565 private final FileFormat ffout;
jpayne@68 566
jpayne@68 567 /*--------------------------------------------------------------*/
jpayne@68 568 /*---------------- Common Fields ----------------*/
jpayne@68 569 /*--------------------------------------------------------------*/
jpayne@68 570
jpayne@68 571 /** Print status messages to this output stream */
jpayne@68 572 private PrintStream outstream=System.err;
jpayne@68 573 /** Print verbose messages */
jpayne@68 574 public static boolean verbose=false;
jpayne@68 575 /** True if an error was encountered */
jpayne@68 576 public boolean errorState=false;
jpayne@68 577 /** Overwrite existing output files */
jpayne@68 578 private boolean overwrite=false;
jpayne@68 579 /** Append to existing output files */
jpayne@68 580 private boolean append=false;
jpayne@68 581 private boolean ordered=true;
jpayne@68 582
jpayne@68 583 }