Mercurial > repos > rliterman > csp2

annotate CSP2/CSP2_env/env-d9b9114564458d9d-741b3de822f2aaca6c6caa4325c4afce/opt/bbmap-39.01-1/current/sketch/SendSketch.java @ 68:5028fdace37b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 2e9511a184a1ca667c7be0c6321a36dc4e3d116d
author jpayne
date Tue, 18 Mar 2025 16:23:26 -0400
parents
children
rev   line source
jpayne@68 1 package sketch;
jpayne@68 2
jpayne@68 3 import java.io.File;
jpayne@68 4 import java.io.PrintStream;
jpayne@68 5 import java.util.ArrayList;
jpayne@68 6 import java.util.Collection;
jpayne@68 7
jpayne@68 8 import fileIO.ByteFile;
jpayne@68 9 import fileIO.ByteStreamWriter;
jpayne@68 10 import fileIO.FileFormat;
jpayne@68 11 import fileIO.ReadWrite;
jpayne@68 12 import fileIO.TextStreamWriter;
jpayne@68 13 import kmer.AbstractKmerTableSet;
jpayne@68 14 import server.ServerTools;
jpayne@68 15 import shared.KillSwitch;
jpayne@68 16 import shared.Parse;
jpayne@68 17 import shared.Parser;
jpayne@68 18 import shared.PreParser;
jpayne@68 19 import shared.ReadStats;
jpayne@68 20 import shared.Shared;
jpayne@68 21 import shared.Timer;
jpayne@68 22 import shared.Tools;
jpayne@68 23 import structures.ByteBuilder;
jpayne@68 24 import structures.StringNum;
jpayne@68 25 import tax.TaxTree;
jpayne@68 26
jpayne@68 27 /**
jpayne@68 28 * Compares one or more input sketches to a set of reference sketches.
jpayne@68 29 *
jpayne@68 30 * @author Brian Bushnell
jpayne@68 31 * @date July 29, 2016
jpayne@68 32 *
jpayne@68 33 */
jpayne@68 34 public class SendSketch extends SketchObject {
jpayne@68 35
jpayne@68 36 /*--------------------------------------------------------------*/
jpayne@68 37 /*---------------- Initialization ----------------*/
jpayne@68 38 /*--------------------------------------------------------------*/
jpayne@68 39
jpayne@68 40 /**
jpayne@68 41 * Code entrance from the command line.
jpayne@68 42 * @param args Command line arguments
jpayne@68 43 */
jpayne@68 44 public static void main(String[] args){
jpayne@68 45 //Start a timer immediately upon code entrance.
jpayne@68 46 Timer t=new Timer();
jpayne@68 47
jpayne@68 48 final boolean oldUnpigz=ReadWrite.USE_UNPIGZ;
jpayne@68 49 final int oldBufLen=Shared.bufferLen();
jpayne@68 50
jpayne@68 51 //Create an instance of this class
jpayne@68 52 SendSketch x=new SendSketch(args);
jpayne@68 53
jpayne@68 54 //Run the object
jpayne@68 55 x.process(t);
jpayne@68 56
jpayne@68 57 ReadWrite.USE_UNPIGZ=oldUnpigz;
jpayne@68 58 Shared.setBufferLen(oldBufLen);
jpayne@68 59
jpayne@68 60 //Close the print stream if it was redirected
jpayne@68 61 Shared.closeStream(x.outstream);
jpayne@68 62
jpayne@68 63 assert(!x.errorState) : "This program ended in an error state.";
jpayne@68 64 }
jpayne@68 65
jpayne@68 66 /**
jpayne@68 67 * Constructor.
jpayne@68 68 * @param args Command line arguments
jpayne@68 69 */
jpayne@68 70 public SendSketch(String[] args){
jpayne@68 71
jpayne@68 72 {//Preparse block for help, config files, and outstream
jpayne@68 73 PreParser pp=new PreParser(args, null, false);
jpayne@68 74 args=pp.args;
jpayne@68 75 outstream=pp.outstream;
jpayne@68 76 silent=PreParser.silent;
jpayne@68 77 if(silent){AbstractKmerTableSet.DISPLAY_PROGRESS=false;}
jpayne@68 78 }
jpayne@68 79
jpayne@68 80 //Set shared static variables
jpayne@68 81 ReadWrite.USE_UNPIGZ=true;
jpayne@68 82 KILL_OK=true;
jpayne@68 83
jpayne@68 84 //Create a parser object
jpayne@68 85 Parser parser=new Parser();
jpayne@68 86 parser.out1="stdout.txt";
jpayne@68 87
jpayne@68 88 defaultParams.inputVersion=Shared.BBMAP_VERSION_STRING;
jpayne@68 89 defaultParams.mode=PER_FILE;
jpayne@68 90 boolean setBlacklist=false;
jpayne@68 91 boolean setLocal=false;
jpayne@68 92 boolean setPrintDepth=false;
jpayne@68 93
jpayne@68 94 //Parse each argument
jpayne@68 95 for(int i=0; i<args.length; i++){
jpayne@68 96 String arg=args[i];
jpayne@68 97
jpayne@68 98 //Break arguments into their constituent parts, in the form of "a=b"
jpayne@68 99 String[] split=arg.split("=");
jpayne@68 100 String a=split[0].toLowerCase();
jpayne@68 101 String b=split.length>1 ? split[1] : null;
jpayne@68 102
jpayne@68 103 if(a.equals("verbose")){
jpayne@68 104 verbose=Parse.parseBoolean(b);
jpayne@68 105 }else if(a.equals("in")){
jpayne@68 106 addFiles(b, in);
jpayne@68 107 }else if(a.equals("blacklist") || a.equalsIgnoreCase("bl")){
jpayne@68 108 setBlacklist=true;
jpayne@68 109 parseSketchFlags(arg, a, b);
jpayne@68 110 }else if(parseSketchFlags(arg, a, b)){
jpayne@68 111 //Do nothing
jpayne@68 112 }else if(defaultParams.parse(arg, a, b)){
jpayne@68 113 //Do nothing
jpayne@68 114 }else if(a.equals("local")){
jpayne@68 115 local=Parse.parseBoolean(b);
jpayne@68 116 setLocal=true;
jpayne@68 117 }else if(a.equals("refid") || a.equals("refids") || a.equals("refname") || a.equals("refnames")){
jpayne@68 118 refNames=b;
jpayne@68 119 }else if(a.equals("parse_flag_goes_here")){
jpayne@68 120 long fake_variable=Parse.parseKMG(b);
jpayne@68 121 //Set a variable here
jpayne@68 122 }else if(a.equals("address")){
jpayne@68 123 assert(b!=null) : "Bad parameter: "+arg;
jpayne@68 124 address=b;
jpayne@68 125 }
jpayne@68 126 // else if(a.equals("alternateaddress") || a.equals("altaddress") || a.equalsIgnoreCase("vm")){
jpayne@68 127 // boolean alt=Parse.parseBoolean(b);
jpayne@68 128 // switchDefaultAddresses(alt);
jpayne@68 129 // }
jpayne@68 130
jpayne@68 131
jpayne@68 132 else if(a.equalsIgnoreCase("nt") || a.equalsIgnoreCase("silva") || a.equalsIgnoreCase("ribo") ||
jpayne@68 133 a.equalsIgnoreCase("refseq") || a.equalsIgnoreCase("img") || a.equalsIgnoreCase("nr") ||
jpayne@68 134 a.equalsIgnoreCase("refseqprot") || a.equalsIgnoreCase("prokprot") || a.equalsIgnoreCase("protein") ||
jpayne@68 135 a.equalsIgnoreCase("protien") || a.equalsIgnoreCase("prot") || a.equalsIgnoreCase("mito") || a.equalsIgnoreCase("fungi")){
jpayne@68 136 address=a;
jpayne@68 137 blacklist=a;
jpayne@68 138 }
jpayne@68 139
jpayne@68 140 else if(a.equals("taxtree") || a.equals("tree")){
jpayne@68 141 taxTreeFile=b;
jpayne@68 142 }
jpayne@68 143
jpayne@68 144 else if(a.equals("name") || a.equals("taxname")){
jpayne@68 145 outTaxName=b;
jpayne@68 146 }else if(a.equals("name0")){
jpayne@68 147 outName0=b;
jpayne@68 148 }else if(a.equals("fname")){
jpayne@68 149 outFname=b;
jpayne@68 150 }else if(a.equals("taxid") || a.equals("tid")){
jpayne@68 151 outTaxID=Integer.parseInt(b);
jpayne@68 152 }else if(a.equals("spid")){
jpayne@68 153 outSpid=Integer.parseInt(b);
jpayne@68 154 }else if(a.equals("imgid")){
jpayne@68 155 outImgID=Integer.parseInt(b);
jpayne@68 156 }else if((a.startsWith("meta_") || a.startsWith("mt_")) && b!=null){
jpayne@68 157 if(outMeta==null){outMeta=new ArrayList<String>();}
jpayne@68 158 int underscore=a.indexOf('_', 0);
jpayne@68 159 outMeta.add(a.substring(underscore+1)+":"+b);
jpayne@68 160 }
jpayne@68 161
jpayne@68 162 else if(a.equals("outsketch") || a.equals("outs") || a.equals("sketchout") || a.equals("sketch")){
jpayne@68 163 outSketch=b;
jpayne@68 164 }
jpayne@68 165
jpayne@68 166 else if(parser.parse(arg, a, b)){//Parse standard flags in the parser
jpayne@68 167 //do nothing
jpayne@68 168 }
jpayne@68 169
jpayne@68 170 else if(b==null && in.isEmpty() && new File(arg).exists()){
jpayne@68 171 in.add(arg);
jpayne@68 172 }
jpayne@68 173
jpayne@68 174 else{
jpayne@68 175 outstream.println("Unknown parameter "+args[i]);
jpayne@68 176 assert(false) : "Unknown parameter "+args[i];
jpayne@68 177 }
jpayne@68 178 }
jpayne@68 179 if("auto".equalsIgnoreCase(taxTreeFile)){taxTreeFile=TaxTree.defaultTreeFile();}
jpayne@68 180
jpayne@68 181 address=toAddress(address);
jpayne@68 182 outMeta=SketchObject.fixMeta(outMeta);
jpayne@68 183
jpayne@68 184 if(address!=null && !SET_AUTOSIZE_FACTOR){
jpayne@68 185 if(address.equals(refseqAddress())){
jpayne@68 186 AUTOSIZE_FACTOR=2.0f;
jpayne@68 187 }else if(address.equals(prokProtAddress())){
jpayne@68 188 AUTOSIZE_FACTOR=3.0f;
jpayne@68 189 }
jpayne@68 190 }
jpayne@68 191
jpayne@68 192 while(address!=null && address.endsWith("/")){
jpayne@68 193 address=address.substring(0, address.length()-1);
jpayne@68 194 }
jpayne@68 195
jpayne@68 196 setFromAddress(address, setBlacklist);
jpayne@68 197
jpayne@68 198 if(local){blacklist=null;}
jpayne@68 199
jpayne@68 200 postParse();
jpayne@68 201
jpayne@68 202 {//Process parser fields
jpayne@68 203 overwrite=ReadStats.overwrite=parser.overwrite;
jpayne@68 204 append=ReadStats.append=parser.append;
jpayne@68 205
jpayne@68 206 out=parser.out1;
jpayne@68 207 }
jpayne@68 208
jpayne@68 209 //Ensure there is an input file
jpayne@68 210 if(in.isEmpty() && refNames==null){throw new RuntimeException("Error - at least one input file is required.");}
jpayne@68 211
jpayne@68 212 //Adjust the number of threads for input file reading
jpayne@68 213 if(!ByteFile.FORCE_MODE_BF1 && !ByteFile.FORCE_MODE_BF2 && Shared.threads()>2){
jpayne@68 214 ByteFile.FORCE_MODE_BF2=true;
jpayne@68 215 }
jpayne@68 216
jpayne@68 217 ffout=FileFormat.testOutput(out, FileFormat.TEXT, null, false, overwrite, append, false);
jpayne@68 218 if(!ffout.stdio() && !defaultParams.setColors){defaultParams.printColors=false;}
jpayne@68 219
jpayne@68 220 //Ensure input files can be read
jpayne@68 221 if(!Tools.testInputFiles(false, true, taxTreeFile)){
jpayne@68 222 throw new RuntimeException("\nCan't read some input files.\n");
jpayne@68 223 }
jpayne@68 224
jpayne@68 225 if(!local && refNames==null){
jpayne@68 226 if(!Tools.testInputFiles(true, false, in.toArray(new String[0]))){
jpayne@68 227 if(in.size()==1){
jpayne@68 228 String s=in.get(0);
jpayne@68 229 String s1=s.replaceFirst("#", "1"), s2=s.replaceFirst("#", "2");
jpayne@68 230 Tools.testInputFiles(true, false, s1, s2);
jpayne@68 231 }else{
jpayne@68 232 throw new RuntimeException("\nCan't read some input files.\n");
jpayne@68 233 }
jpayne@68 234 }
jpayne@68 235 }
jpayne@68 236
jpayne@68 237 if(!Tools.testOutputFiles(overwrite, append, false, out, outSketch)){
jpayne@68 238 throw new RuntimeException("\n\noverwrite="+overwrite+"; Can't write to output files "+
jpayne@68 239 out+", "+outSketch+"\n");
jpayne@68 240 }
jpayne@68 241
jpayne@68 242 //Ensure that no file was specified multiple times
jpayne@68 243 if(!Tools.testForDuplicateFiles(true, in.toArray(new String[0]))){
jpayne@68 244 throw new RuntimeException("\nSome file names were specified multiple times.\n");
jpayne@68 245 }
jpayne@68 246
jpayne@68 247 SSUMap.load(outstream);
jpayne@68 248 tool=new SketchTool(targetSketchSize, defaultParams);
jpayne@68 249
jpayne@68 250 // assert(false) : defaultParams.toString()+"\n"+k+", "+amino+", "+HASH_VERSION;
jpayne@68 251 if(verbose){
jpayne@68 252 if(local){outstream.println("Running in local mode.");}
jpayne@68 253 if(useWhitelist){outstream.println("Using a whitelist.");}
jpayne@68 254 if(blacklist!=null){outstream.println("Using a blacklist.");}
jpayne@68 255 }
jpayne@68 256
jpayne@68 257 if(taxTreeFile!=null){setTaxtree(taxTreeFile, silent ? null : outstream);}
jpayne@68 258 defaultParams.postParse(false, false);
jpayne@68 259 if(!defaultParams.printSSU){processSSU=false;}
jpayne@68 260 allowMultithreadedFastq=(in.size()==1 && Shared.threads()>2);
jpayne@68 261 if(!allowMultithreadedFastq){Shared.capBufferLen(40);}
jpayne@68 262 }
jpayne@68 263
jpayne@68 264 private void setFromAddress(String address, boolean setBlacklist){
jpayne@68 265 if(address.equals(nrAddress())){
jpayne@68 266 amino=true;
jpayne@68 267 if(!setK){k=defaultKAmino; k2=defaultK2Amino;}
jpayne@68 268 // defaultParams.dbName="nr";
jpayne@68 269 assert(false) : "Need to set K.";
jpayne@68 270 }else if(address.equals(prokProtAddress())){
jpayne@68 271 translate=true;
jpayne@68 272 if(!setK){k=defaultKAmino; k2=defaultK2Amino;}
jpayne@68 273 // defaultParams.dbName="nr";
jpayne@68 274 if(blacklist==null && !setBlacklist){blacklist=Blacklist.prokProtBlacklist();}
jpayne@68 275 }else if(address.equals(ntAddress())){
jpayne@68 276 if(!setK){k=defaultK; k2=defaultK2;}
jpayne@68 277 // defaultParams.dbName="nt";
jpayne@68 278 if(blacklist==null && !setBlacklist){blacklist=Blacklist.ntBlacklist();}
jpayne@68 279 }else if(address.equals(refseqAddress())){
jpayne@68 280 if(!setK){k=defaultK; k2=defaultK2;}
jpayne@68 281 // defaultParams.dbName="RefSeq";
jpayne@68 282 if(blacklist==null && !setBlacklist){blacklist=Blacklist.refseqBlacklist();}
jpayne@68 283 }else if(address.equals(silvaAddress())){
jpayne@68 284 if(!setK){k=defaultK; k2=defaultK2;}
jpayne@68 285 // defaultParams.dbName="Silva";
jpayne@68 286 if(blacklist==null && !setBlacklist){blacklist=Blacklist.silvaBlacklist();}
jpayne@68 287 }else if(address.equals(imgAddress())){
jpayne@68 288 if(!setK){k=defaultK; k2=defaultK2;}
jpayne@68 289 // defaultParams.dbName="IMG";
jpayne@68 290 if(blacklist==null && !setBlacklist){blacklist=Blacklist.imgBlacklist();}
jpayne@68 291 }else if(address.equals(mitoAddress())){
jpayne@68 292 if(!setK){k=defaultK; k2=defaultK2;}
jpayne@68 293 if(blacklist==null && !setBlacklist){blacklist=Blacklist.mitoBlacklist();}
jpayne@68 294 }else if(address.equals(fungiAddress())){
jpayne@68 295 if(!setK){k=defaultK; k2=defaultK2;}
jpayne@68 296 if(blacklist==null && !setBlacklist){blacklist=Blacklist.fungiBlacklist();}
jpayne@68 297 }
jpayne@68 298 }
jpayne@68 299
jpayne@68 300 /*--------------------------------------------------------------*/
jpayne@68 301 /*---------------- Outer Methods ----------------*/
jpayne@68 302 /*--------------------------------------------------------------*/
jpayne@68 303
jpayne@68 304 public void process(Timer t){
jpayne@68 305 if(local){processLocal(t);}
jpayne@68 306 else if(refNames!=null){processRefMode(t);}
jpayne@68 307 else{processRemote(t);}
jpayne@68 308 }
jpayne@68 309
jpayne@68 310 @Deprecated
jpayne@68 311 private void processRemote_old(Timer t){
jpayne@68 312 Timer ttotal=new Timer();
jpayne@68 313
jpayne@68 314 t.start();
jpayne@68 315 inSketches=tool.loadSketches_MT(defaultParams, in);
jpayne@68 316 final int numLoaded=(inSketches.size());
jpayne@68 317 if(numLoaded>1 && defaultParams.mode==PER_FILE){
jpayne@68 318 inSketches.sort(SketchIdComparator.comparator);//Otherwise they come out disordered
jpayne@68 319 }
jpayne@68 320
jpayne@68 321 t.stop();
jpayne@68 322 if(!silent){outstream.println("Loaded "+numLoaded+" sketch"+(numLoaded==1 ? "" : "es")+" in "+t);}
jpayne@68 323 assert(numLoaded<=MAX_ALLOWED_SKETCHES) : "\nSendSketch is configured to send at most "+MAX_ALLOWED_SKETCHES+" to prevent overwhelming the server.\n"
jpayne@68 324 + "If you need to compare more than that, please use CompareSketch locally instead.\n"
jpayne@68 325 + "References can be downloaded at http://portal.nersc.gov/dna/microbial/assembly/bushnell/\n";
jpayne@68 326 t.start();
jpayne@68 327 // outstream.println(inSketches.get(0));
jpayne@68 328
jpayne@68 329 if(numLoaded>4000){
jpayne@68 330 SEND_BUFFER_MAX_BYTES*=4;
jpayne@68 331 SEND_BUFFER_MAX_SKETCHES*=4;
jpayne@68 332 }else if(numLoaded>1000){
jpayne@68 333 SEND_BUFFER_MAX_BYTES*=2;
jpayne@68 334 SEND_BUFFER_MAX_SKETCHES*=2;
jpayne@68 335 }
jpayne@68 336
jpayne@68 337 if(ffout==null){return;}
jpayne@68 338 TextStreamWriter tsw=new TextStreamWriter(ffout);
jpayne@68 339 tsw.start();
jpayne@68 340 if(defaultParams.format==DisplayParams.FORMAT_QUERY_REF_ANI || defaultParams.format==DisplayParams.FORMAT_CONSTELLATION){tsw.println(defaultParams.header());}
jpayne@68 341
jpayne@68 342 ByteBuilder bb=new ByteBuilder();
jpayne@68 343
jpayne@68 344 int cntr=0;
jpayne@68 345 int chunks=0;
jpayne@68 346 for(Sketch sk : inSketches){
jpayne@68 347
jpayne@68 348 if(sk.taxID<1 || sk.taxID>=minFakeID || outTaxID>0){sk.taxID=outTaxID;}
jpayne@68 349 if(defaultParams.printSSU()){sk.loadSSU();}
jpayne@68 350
jpayne@68 351 if(outSpid>0){sk.spid=outSpid;}
jpayne@68 352 if(outImgID>0){sk.imgID=outImgID;}
jpayne@68 353 if(outTaxName!=null){sk.setTaxName(outTaxName);}
jpayne@68 354 if(outFname!=null){sk.setFname(outFname);}
jpayne@68 355 if(outName0!=null){sk.setName0(outName0);}
jpayne@68 356 sk.setMeta(outMeta);
jpayne@68 357
jpayne@68 358 if(bb.length==0){
jpayne@68 359 bb.append(defaultParams.toString(chunks));
jpayne@68 360 chunks++;
jpayne@68 361 }
jpayne@68 362 sk.toBytes(bb);
jpayne@68 363 cntr++;
jpayne@68 364 if(cntr>=SEND_BUFFER_MAX_SKETCHES || bb.length>SEND_BUFFER_MAX_BYTES){ //Don't allow too much data in a single transaction
jpayne@68 365 if(verbose){outstream.println("Sending:\n"+bb);}
jpayne@68 366 // outstream.println(cntr+", "+bb.length);
jpayne@68 367
jpayne@68 368 byte[] message=bb.toBytes();
jpayne@68 369 bb.clear();
jpayne@68 370 try {
jpayne@68 371 // outstream.println("Sending to "+address+"\n"+message+"\n"); //123
jpayne@68 372 StringNum result=ServerTools.sendAndReceive(message, address);
jpayne@68 373 if(!ServerTools.suppressErrors && (result.n<200 || result.n>299)){
jpayne@68 374 System.err.println("ERROR: Server returned code "+result.n+" and this message:\n"+result.s);
jpayne@68 375 KillSwitch.kill();
jpayne@68 376 }
jpayne@68 377 errorState|=checkForError(result.s);
jpayne@68 378 tsw.print(result.s);
jpayne@68 379 } catch (Exception e) {
jpayne@68 380 // TODO Auto-generated catch block
jpayne@68 381 e.printStackTrace();
jpayne@68 382 }
jpayne@68 383 cntr=0;
jpayne@68 384 }
jpayne@68 385 }
jpayne@68 386
jpayne@68 387 if(bb.length>0){
jpayne@68 388 if(verbose){outstream.println("Sending:\n"+bb);}
jpayne@68 389 byte[] message=bb.toBytes();
jpayne@68 390 bb.clear();
jpayne@68 391 try {
jpayne@68 392 StringNum result=ServerTools.sendAndReceive(message, address);
jpayne@68 393 if(!ServerTools.suppressErrors && (result.n<200 || result.n>299)){
jpayne@68 394 System.err.println("ERROR: Server returned code "+result.n+" and this message:\n"+result.s);
jpayne@68 395 KillSwitch.kill();
jpayne@68 396 }
jpayne@68 397 errorState|=checkForError(result.s);
jpayne@68 398 tsw.print(result.s);
jpayne@68 399 } catch (Exception e) {
jpayne@68 400 // TODO Auto-generated catch block
jpayne@68 401 e.printStackTrace();
jpayne@68 402 }
jpayne@68 403 }
jpayne@68 404 if(!silent){tsw.println();}
jpayne@68 405 tsw.poison();
jpayne@68 406
jpayne@68 407 // outstream.println("sending "+bb.toString());
jpayne@68 408
jpayne@68 409 if(outSketch!=null){
jpayne@68 410 ByteStreamWriter bsw=new ByteStreamWriter(outSketch, overwrite, append, true, FileFormat.SKETCH);
jpayne@68 411 bsw.start();
jpayne@68 412 for(Sketch sk : inSketches){
jpayne@68 413 sk.toBytes(bb);
jpayne@68 414 bsw.print(bb);
jpayne@68 415 bb.clear();
jpayne@68 416 }
jpayne@68 417 bsw.poisonAndWait();
jpayne@68 418 errorState|=bsw.errorState;
jpayne@68 419 }
jpayne@68 420
jpayne@68 421 tsw.waitForFinish();
jpayne@68 422 errorState|=tsw.errorState;
jpayne@68 423
jpayne@68 424 t.stop();
jpayne@68 425 // outstream.println("\nRan "+(inSketches.size()*refSketches.size())+" comparisons in \t"+t);
jpayne@68 426 ttotal.stop();
jpayne@68 427 if(!silent){outstream.println("Total Time: \t"+ttotal);}
jpayne@68 428 }
jpayne@68 429
jpayne@68 430
jpayne@68 431 private void processRemote(Timer t){
jpayne@68 432 Timer ttotal=new Timer();
jpayne@68 433
jpayne@68 434 t.start();
jpayne@68 435 inSketches=tool.loadSketches_MT(defaultParams, in);
jpayne@68 436 final int numLoaded=(inSketches.size());
jpayne@68 437 if(numLoaded>1 && defaultParams.mode==PER_FILE){
jpayne@68 438 inSketches.sort(SketchIdComparator.comparator);//Otherwise they come out disordered
jpayne@68 439 }
jpayne@68 440
jpayne@68 441 t.stop();
jpayne@68 442 if(!silent){outstream.println("Loaded "+numLoaded+" sketch"+(numLoaded==1 ? "" : "es")+" in "+t);}
jpayne@68 443 assert(numLoaded<=MAX_ALLOWED_SKETCHES) : "\nSendSketch is configured to send at most "+MAX_ALLOWED_SKETCHES+" to prevent overwhelming the server.\n"
jpayne@68 444 + "If you need to compare more than that, please use CompareSketch locally instead.\n"
jpayne@68 445 + "References can be downloaded at http://portal.nersc.gov/dna/microbial/assembly/bushnell/\n";
jpayne@68 446 t.start();
jpayne@68 447 // outstream.println(inSketches.get(0));
jpayne@68 448
jpayne@68 449 if(numLoaded>4000){
jpayne@68 450 SEND_BUFFER_MAX_BYTES*=4;
jpayne@68 451 SEND_BUFFER_MAX_SKETCHES*=4;
jpayne@68 452 }else if(numLoaded>1000){
jpayne@68 453 SEND_BUFFER_MAX_BYTES*=2;
jpayne@68 454 SEND_BUFFER_MAX_SKETCHES*=2;
jpayne@68 455 }
jpayne@68 456 // SEND_BUFFER_MAX_SKETCHES=1;//This is for testing json array format.
jpayne@68 457
jpayne@68 458 if(ffout==null){return;}
jpayne@68 459 TextStreamWriter tsw=new TextStreamWriter(ffout);
jpayne@68 460 tsw.start();
jpayne@68 461 if(defaultParams.format==DisplayParams.FORMAT_QUERY_REF_ANI || defaultParams.format==DisplayParams.FORMAT_CONSTELLATION){tsw.println(defaultParams.header());}
jpayne@68 462
jpayne@68 463 ByteBuilder bb=new ByteBuilder();
jpayne@68 464
jpayne@68 465 int cntr=0;
jpayne@68 466 int chunks=0;
jpayne@68 467 boolean firstBlock=true; //Set to false after printing has started
jpayne@68 468 final Sketch lastSketch=Tools.getLast(inSketches); //Last input sketch to process
jpayne@68 469 for(Sketch sk : inSketches){
jpayne@68 470
jpayne@68 471 if(sk.taxID<1 || sk.taxID>=minFakeID || outTaxID>0){sk.taxID=outTaxID;}
jpayne@68 472 if(defaultParams.printSSU()){sk.loadSSU();}
jpayne@68 473
jpayne@68 474 if(outSpid>0){sk.spid=outSpid;}
jpayne@68 475 if(outImgID>0){sk.imgID=outImgID;}
jpayne@68 476 if(outTaxName!=null){sk.setTaxName(outTaxName);}
jpayne@68 477 if(outFname!=null){sk.setFname(outFname);}
jpayne@68 478 if(outName0!=null){sk.setName0(outName0);}
jpayne@68 479 sk.setMeta(outMeta);
jpayne@68 480
jpayne@68 481 if(bb.length==0){
jpayne@68 482 bb.append(defaultParams.toString(chunks));
jpayne@68 483 chunks++;
jpayne@68 484 }
jpayne@68 485 sk.toBytes(bb);
jpayne@68 486 cntr++;
jpayne@68 487 if(cntr>=SEND_BUFFER_MAX_SKETCHES || bb.length>SEND_BUFFER_MAX_BYTES || sk==lastSketch){ //Don't allow too much data in a single transaction
jpayne@68 488 if(verbose){outstream.println("Sending:\n"+bb);}
jpayne@68 489 // outstream.println(cntr+", "+bb.length);
jpayne@68 490
jpayne@68 491 byte[] message=bb.toBytes();
jpayne@68 492 bb.clear();
jpayne@68 493 try {
jpayne@68 494 // outstream.println("Sending to "+address+"\n"+message+"\n"); //123
jpayne@68 495 StringNum result=ServerTools.sendAndReceive(message, address);
jpayne@68 496 if(!ServerTools.suppressErrors && (result.n<200 || result.n>299)){
jpayne@68 497 System.err.println("ERROR: Server returned code "+result.n+" and this message:\n"+result.s);
jpayne@68 498 KillSwitch.kill();
jpayne@68 499 }
jpayne@68 500 errorState|=checkForError(result.s);
jpayne@68 501 if(!defaultParams.json()){
jpayne@68 502 tsw.print(result.s);
jpayne@68 503 }else{
jpayne@68 504 String s=result.s;
jpayne@68 505
jpayne@68 506 {//Fixes rare case of multiple sketches broken into single sketches per block
jpayne@68 507 if(!firstBlock || sk!=lastSketch){
jpayne@68 508 if(s.charAt(0)!='['){s="["+s;}
jpayne@68 509 if(!s.endsWith("]")){;s=s+"]";}
jpayne@68 510 }
jpayne@68 511 }
jpayne@68 512
jpayne@68 513 {//Case where there are more than one block of sketches, breaking JSON array format
jpayne@68 514 if(!firstBlock){
jpayne@68 515 s=","+s.substring(1);
jpayne@68 516 }
jpayne@68 517 if(sk!=lastSketch){
jpayne@68 518 s=s.substring(0, s.length()-1);
jpayne@68 519 }
jpayne@68 520 }
jpayne@68 521
jpayne@68 522 tsw.print(s);
jpayne@68 523 firstBlock=false;
jpayne@68 524 }
jpayne@68 525 } catch (Exception e) {
jpayne@68 526 // TODO Auto-generated catch block
jpayne@68 527 e.printStackTrace();
jpayne@68 528 }
jpayne@68 529 cntr=0;
jpayne@68 530 }
jpayne@68 531 }
jpayne@68 532
jpayne@68 533 if(bb.length>0){//This should no longer trigger unless there are no sketches since it's handled above.
jpayne@68 534 if(verbose){outstream.println("Sending:\n"+bb);}
jpayne@68 535 byte[] message=bb.toBytes();
jpayne@68 536 bb.clear();
jpayne@68 537 try {
jpayne@68 538 StringNum result=ServerTools.sendAndReceive(message, address);
jpayne@68 539 if(!ServerTools.suppressErrors && (result.n<200 || result.n>299)){
jpayne@68 540 System.err.println("ERROR: Server returned code "+result.n+" and this message:\n"+result.s);
jpayne@68 541 KillSwitch.kill();
jpayne@68 542 }
jpayne@68 543 errorState|=checkForError(result.s);
jpayne@68 544 tsw.print(result.s);
jpayne@68 545 } catch (Exception e) {
jpayne@68 546 // TODO Auto-generated catch block
jpayne@68 547 e.printStackTrace();
jpayne@68 548 }
jpayne@68 549 }
jpayne@68 550 if(!silent){tsw.println();}
jpayne@68 551 tsw.poison();
jpayne@68 552
jpayne@68 553 // outstream.println("sending "+bb.toString());
jpayne@68 554
jpayne@68 555 if(outSketch!=null){
jpayne@68 556 ByteStreamWriter bsw=new ByteStreamWriter(outSketch, overwrite, append, true, FileFormat.SKETCH);
jpayne@68 557 bsw.start();
jpayne@68 558 for(Sketch sk : inSketches){
jpayne@68 559 sk.toBytes(bb);
jpayne@68 560 bsw.print(bb);
jpayne@68 561 bb.clear();
jpayne@68 562 }
jpayne@68 563 bsw.poisonAndWait();
jpayne@68 564 errorState|=bsw.errorState;
jpayne@68 565 }
jpayne@68 566
jpayne@68 567 tsw.waitForFinish();
jpayne@68 568 errorState|=tsw.errorState;
jpayne@68 569
jpayne@68 570 t.stop();
jpayne@68 571 // outstream.println("\nRan "+(inSketches.size()*refSketches.size())+" comparisons in \t"+t);
jpayne@68 572 ttotal.stop();
jpayne@68 573 if(!silent){outstream.println("Total Time: \t"+ttotal);}
jpayne@68 574 }
jpayne@68 575
jpayne@68 576 /** For programmatic use */
jpayne@68 577 public static String sendSketch(Sketch sk, String address, int format, int chunkNum){
jpayne@68 578 address=toAddress(address);
jpayne@68 579 ByteBuilder bb=new ByteBuilder();
jpayne@68 580
jpayne@68 581 DisplayParams params=defaultParams;
jpayne@68 582 if(format>=0){
jpayne@68 583 new DisplayParams();
jpayne@68 584 params.format=format;
jpayne@68 585 }
jpayne@68 586 if(bb.length==0){bb.append(params.toString(chunkNum));}
jpayne@68 587 sk.toBytes(bb);
jpayne@68 588
jpayne@68 589 byte[] message=bb.toBytes();
jpayne@68 590 try {
jpayne@68 591 // System.err.println("Sending to "+address+"\n"+new String(message)+"\n"); //123
jpayne@68 592 StringNum result=ServerTools.sendAndReceive(message, address);
jpayne@68 593 if(!ServerTools.suppressErrors && (result.n<200 || result.n>299)){
jpayne@68 594 System.err.println("ERROR: Server returned code "+result.n+" and this message:\n"+result.s);
jpayne@68 595 KillSwitch.kill();
jpayne@68 596 }
jpayne@68 597 return result.s;
jpayne@68 598 } catch (Exception e) {
jpayne@68 599 // TODO Auto-generated catch block
jpayne@68 600 e.printStackTrace();
jpayne@68 601 }
jpayne@68 602 return null;
jpayne@68 603 }
jpayne@68 604
jpayne@68 605 private static boolean checkForError(String s){
jpayne@68 606 if(s==null){return false;}
jpayne@68 607 return s.contains("java.io.IOException: Server returned HTTP response code:");
jpayne@68 608 }
jpayne@68 609
jpayne@68 610 private void processLocal(Timer t){
jpayne@68 611 Timer ttotal=new Timer();
jpayne@68 612
jpayne@68 613 t.start();
jpayne@68 614
jpayne@68 615 if(ffout==null){return;}
jpayne@68 616 TextStreamWriter tsw=new TextStreamWriter(ffout);
jpayne@68 617 tsw.start();
jpayne@68 618
jpayne@68 619 final String message=defaultParams.toString(0);
jpayne@68 620 for(String fname : in){
jpayne@68 621 String address2=address+"/file/"+new File(fname).getAbsolutePath();
jpayne@68 622
jpayne@68 623 if(verbose){outstream.println("Sending:\n"+message+"\nto "+address2);}
jpayne@68 624 try {
jpayne@68 625 // outstream.println("Sending to "+address2+"\n"+message+"\n"); //123
jpayne@68 626 StringNum result=ServerTools.sendAndReceive(message.getBytes(), address2);
jpayne@68 627 if(!ServerTools.suppressErrors && (result.n<200 || result.n>299)){
jpayne@68 628 System.err.println("ERROR: Server returned code "+result.n+" and this message:\n"+result.s);
jpayne@68 629 KillSwitch.kill();
jpayne@68 630 }
jpayne@68 631 tsw.print(result.s);
jpayne@68 632 } catch (Exception e) {
jpayne@68 633 // TODO Auto-generated catch block
jpayne@68 634 if(!suppressErrors){e.printStackTrace();}
jpayne@68 635 }
jpayne@68 636 }
jpayne@68 637 tsw.println();
jpayne@68 638
jpayne@68 639 // outstream.println("sending "+bb.toString());
jpayne@68 640
jpayne@68 641 tsw.poisonAndWait();
jpayne@68 642 errorState|=tsw.errorState;
jpayne@68 643
jpayne@68 644 t.stop();
jpayne@68 645 // outstream.println("\nRan "+(inSketches.size()*refSketches.size())+" comparisons in \t"+t);
jpayne@68 646 ttotal.stop();
jpayne@68 647 outstream.println("Total Time: \t"+ttotal);
jpayne@68 648 }
jpayne@68 649
jpayne@68 650 private void processRefMode(Timer t){
jpayne@68 651 Timer ttotal=new Timer();
jpayne@68 652
jpayne@68 653 t.start();
jpayne@68 654
jpayne@68 655 if(ffout==null){return;}
jpayne@68 656 TextStreamWriter tsw=new TextStreamWriter(ffout);
jpayne@68 657 tsw.start();
jpayne@68 658
jpayne@68 659 final String message=defaultParams.toString(0);
jpayne@68 660 {
jpayne@68 661 String address2=address+"/ref/"+refNames;
jpayne@68 662
jpayne@68 663 if(verbose){outstream.println("Sending:\n"+message+"\nto "+address2);}
jpayne@68 664 try {
jpayne@68 665 // outstream.println("Sending to "+address2+"\n"+message+"\n"); //123
jpayne@68 666 StringNum result=ServerTools.sendAndReceive(message.getBytes(), address2);
jpayne@68 667 if(!ServerTools.suppressErrors && (result.n<200 || result.n>299)){
jpayne@68 668 System.err.println("ERROR: Server returned code "+result.n+" and this message:\n"+result.s);
jpayne@68 669 KillSwitch.kill();
jpayne@68 670 }
jpayne@68 671 tsw.print(result.s);
jpayne@68 672 } catch (Exception e) {
jpayne@68 673 // TODO Auto-generated catch block
jpayne@68 674 if(!suppressErrors){e.printStackTrace();}
jpayne@68 675 }
jpayne@68 676 }
jpayne@68 677 tsw.println();
jpayne@68 678
jpayne@68 679 // outstream.println("sending "+bb.toString());
jpayne@68 680
jpayne@68 681 tsw.poisonAndWait();
jpayne@68 682 errorState|=tsw.errorState;
jpayne@68 683
jpayne@68 684 t.stop();
jpayne@68 685 // outstream.println("\nRan "+(inSketches.size()*refSketches.size())+" comparisons in \t"+t);
jpayne@68 686 ttotal.stop();
jpayne@68 687 outstream.println("Total Time: \t"+ttotal);
jpayne@68 688 }
jpayne@68 689
jpayne@68 690
jpayne@68 691 /*--------------------------------------------------------------*/
jpayne@68 692 /*---------------- Inner Methods ----------------*/
jpayne@68 693 /*--------------------------------------------------------------*/
jpayne@68 694
jpayne@68 695 private static boolean addFiles(String a, Collection<String> list){
jpayne@68 696 int initial=list.size();
jpayne@68 697 if(a==null){return false;}
jpayne@68 698 File f=null;
jpayne@68 699 if(a.indexOf(',')>=0){f=new File(a);}
jpayne@68 700 if(f==null || f.exists()){
jpayne@68 701 list.add(a);
jpayne@68 702 }else{
jpayne@68 703 for(String s : a.split(",")){
jpayne@68 704 list.add(s);
jpayne@68 705 }
jpayne@68 706 }
jpayne@68 707 return list.size()>initial;
jpayne@68 708 }
jpayne@68 709
jpayne@68 710 /*--------------------------------------------------------------*/
jpayne@68 711 /*---------------- Fields ----------------*/
jpayne@68 712 /*--------------------------------------------------------------*/
jpayne@68 713
jpayne@68 714 private ArrayList<String> in=new ArrayList<String>();
jpayne@68 715
jpayne@68 716 private String out="stdout.txt";
jpayne@68 717 private String outSketch=null;
jpayne@68 718
jpayne@68 719 private String taxTreeFile=null;
jpayne@68 720
jpayne@68 721 private final SketchTool tool;
jpayne@68 722
jpayne@68 723 private ArrayList<Sketch> inSketches;
jpayne@68 724
jpayne@68 725 private String address=null;
jpayne@68 726 private boolean local=false;
jpayne@68 727 /** List of reference names or TaxIDs to use as queries */
jpayne@68 728 private String refNames=null;
jpayne@68 729
jpayne@68 730 /*Override metadata */
jpayne@68 731 private String outTaxName=null;
jpayne@68 732 private String outFname=null;
jpayne@68 733 private String outName0=null;
jpayne@68 734 private int outTaxID=-1;
jpayne@68 735 private long outSpid=-1;
jpayne@68 736 private long outImgID=-1;
jpayne@68 737 private ArrayList<String> outMeta=null;
jpayne@68 738
jpayne@68 739 /*--------------------------------------------------------------*/
jpayne@68 740 /*---------------- Final Fields ----------------*/
jpayne@68 741 /*--------------------------------------------------------------*/
jpayne@68 742
jpayne@68 743 /** Primary output file */
jpayne@68 744 private final FileFormat ffout;
jpayne@68 745
jpayne@68 746 /*--------------------------------------------------------------*/
jpayne@68 747 /*---------------- Common Fields ----------------*/
jpayne@68 748 /*--------------------------------------------------------------*/
jpayne@68 749
jpayne@68 750 /** Print status messages to this output stream */
jpayne@68 751 private PrintStream outstream=System.err;
jpayne@68 752 /** Print verbose messages */
jpayne@68 753 public static boolean verbose=false;
jpayne@68 754 /** True if an error was encountered */
jpayne@68 755 public boolean errorState=false;
jpayne@68 756 /** Overwrite existing output files */
jpayne@68 757 private boolean overwrite=false;
jpayne@68 758 /** Append to existing output files */
jpayne@68 759 private boolean append=false;
jpayne@68 760
jpayne@68 761 private boolean silent=false;
jpayne@68 762
jpayne@68 763 /*--------------------------------------------------------------*/
jpayne@68 764 /*---------------- Static Fields ----------------*/
jpayne@68 765 /*--------------------------------------------------------------*/
jpayne@68 766
jpayne@68 767 public static final String toAddress(String b){
jpayne@68 768 String address=b;
jpayne@68 769 if(b==null){
jpayne@68 770 address=refseqAddress();//default
jpayne@68 771 }else if(b.equalsIgnoreCase("nt")){
jpayne@68 772 address=ntAddress();
jpayne@68 773 }else if(b.equalsIgnoreCase("refseq")){
jpayne@68 774 address=refseqAddress();
jpayne@68 775 }else if(b.equalsIgnoreCase("silva") || b.equalsIgnoreCase("ribo")){
jpayne@68 776 address=silvaAddress();
jpayne@68 777 }else if(b.equalsIgnoreCase("img")){
jpayne@68 778 address=imgAddress();
jpayne@68 779 }else if(b.equalsIgnoreCase("refseqprot") || b.equalsIgnoreCase("prokprot")
jpayne@68 780 || b.equalsIgnoreCase("protein") || b.equalsIgnoreCase("protien") || b.equalsIgnoreCase("prot")){
jpayne@68 781 address=prokProtAddress();
jpayne@68 782 }else if(b.equalsIgnoreCase("refseqmito") || b.equalsIgnoreCase("mito")){
jpayne@68 783 address=mitoAddress();
jpayne@68 784 }else if(b.equalsIgnoreCase("refseqfungi") || b.equalsIgnoreCase("fungi")){
jpayne@68 785 address=fungiAddress();
jpayne@68 786 }
jpayne@68 787 return address;
jpayne@68 788 }
jpayne@68 789
jpayne@68 790 public int SEND_BUFFER_MAX_BYTES=8000000;
jpayne@68 791 public int SEND_BUFFER_MAX_SKETCHES=400;
jpayne@68 792 private static final int MAX_ALLOWED_SKETCHES=100000;
jpayne@68 793
jpayne@68 794 /** Don't print caught exceptions */
jpayne@68 795 public static boolean suppressErrors=false;
jpayne@68 796
jpayne@68 797 private static String ntAddress(){return Shared.ntSketchServer()+"sketch";}
jpayne@68 798 private static String refseqAddress(){return Shared.refseqSketchServer()+"sketch";}
jpayne@68 799 private static String silvaAddress(){return Shared.riboSketchServer()+"sketch";}
jpayne@68 800 private static String imgAddress(){return null;}//Shared.SketchServer()+"sketch";
jpayne@68 801 private static String nrAddress(){return null;}//Shared.SketchServer()+"sketch";
jpayne@68 802 private static String prokProtAddress(){return Shared.proteinSketchServer()+"sketch";}
jpayne@68 803 private static String mitoAddress(){return null;}//Shared.SketchServer()+"sketch";
jpayne@68 804 private static String fungiAddress(){return null;}//Shared.SketchServer()+"sketch";
jpayne@68 805
jpayne@68 806 }