jpayne@68: package sketch; jpayne@68: jpayne@68: import java.io.File; jpayne@68: import java.io.PrintStream; jpayne@68: import java.util.ArrayList; jpayne@68: import java.util.Collection; jpayne@68: jpayne@68: import fileIO.ByteFile; jpayne@68: import fileIO.ByteStreamWriter; jpayne@68: import fileIO.FileFormat; jpayne@68: import fileIO.ReadWrite; jpayne@68: import fileIO.TextStreamWriter; jpayne@68: import kmer.AbstractKmerTableSet; jpayne@68: import server.ServerTools; jpayne@68: import shared.KillSwitch; jpayne@68: import shared.Parse; jpayne@68: import shared.Parser; jpayne@68: import shared.PreParser; jpayne@68: import shared.ReadStats; jpayne@68: import shared.Shared; jpayne@68: import shared.Timer; jpayne@68: import shared.Tools; jpayne@68: import structures.ByteBuilder; jpayne@68: import structures.StringNum; jpayne@68: import tax.TaxTree; jpayne@68: jpayne@68: /** jpayne@68: * Compares one or more input sketches to a set of reference sketches. jpayne@68: * jpayne@68: * @author Brian Bushnell jpayne@68: * @date July 29, 2016 jpayne@68: * jpayne@68: */ jpayne@68: public class SendSketch extends SketchObject { jpayne@68: jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: /*---------------- Initialization ----------------*/ jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: jpayne@68: /** jpayne@68: * Code entrance from the command line. jpayne@68: * @param args Command line arguments jpayne@68: */ jpayne@68: public static void main(String[] args){ jpayne@68: //Start a timer immediately upon code entrance. jpayne@68: Timer t=new Timer(); jpayne@68: jpayne@68: final boolean oldUnpigz=ReadWrite.USE_UNPIGZ; jpayne@68: final int oldBufLen=Shared.bufferLen(); jpayne@68: jpayne@68: //Create an instance of this class jpayne@68: SendSketch x=new SendSketch(args); jpayne@68: jpayne@68: //Run the object jpayne@68: x.process(t); jpayne@68: jpayne@68: ReadWrite.USE_UNPIGZ=oldUnpigz; jpayne@68: Shared.setBufferLen(oldBufLen); jpayne@68: jpayne@68: //Close the print stream if it was redirected jpayne@68: Shared.closeStream(x.outstream); jpayne@68: jpayne@68: assert(!x.errorState) : "This program ended in an error state."; jpayne@68: } jpayne@68: jpayne@68: /** jpayne@68: * Constructor. jpayne@68: * @param args Command line arguments jpayne@68: */ jpayne@68: public SendSketch(String[] args){ jpayne@68: jpayne@68: {//Preparse block for help, config files, and outstream jpayne@68: PreParser pp=new PreParser(args, null, false); jpayne@68: args=pp.args; jpayne@68: outstream=pp.outstream; jpayne@68: silent=PreParser.silent; jpayne@68: if(silent){AbstractKmerTableSet.DISPLAY_PROGRESS=false;} jpayne@68: } jpayne@68: jpayne@68: //Set shared static variables jpayne@68: ReadWrite.USE_UNPIGZ=true; jpayne@68: KILL_OK=true; jpayne@68: jpayne@68: //Create a parser object jpayne@68: Parser parser=new Parser(); jpayne@68: parser.out1="stdout.txt"; jpayne@68: jpayne@68: defaultParams.inputVersion=Shared.BBMAP_VERSION_STRING; jpayne@68: defaultParams.mode=PER_FILE; jpayne@68: boolean setBlacklist=false; jpayne@68: boolean setLocal=false; jpayne@68: boolean setPrintDepth=false; jpayne@68: jpayne@68: //Parse each argument jpayne@68: for(int i=0; i1 ? split[1] : null; jpayne@68: jpayne@68: if(a.equals("verbose")){ jpayne@68: verbose=Parse.parseBoolean(b); jpayne@68: }else if(a.equals("in")){ jpayne@68: addFiles(b, in); jpayne@68: }else if(a.equals("blacklist") || a.equalsIgnoreCase("bl")){ jpayne@68: setBlacklist=true; jpayne@68: parseSketchFlags(arg, a, b); jpayne@68: }else if(parseSketchFlags(arg, a, b)){ jpayne@68: //Do nothing jpayne@68: }else if(defaultParams.parse(arg, a, b)){ jpayne@68: //Do nothing jpayne@68: }else if(a.equals("local")){ jpayne@68: local=Parse.parseBoolean(b); jpayne@68: setLocal=true; jpayne@68: }else if(a.equals("refid") || a.equals("refids") || a.equals("refname") || a.equals("refnames")){ jpayne@68: refNames=b; jpayne@68: }else if(a.equals("parse_flag_goes_here")){ jpayne@68: long fake_variable=Parse.parseKMG(b); jpayne@68: //Set a variable here jpayne@68: }else if(a.equals("address")){ jpayne@68: assert(b!=null) : "Bad parameter: "+arg; jpayne@68: address=b; jpayne@68: } jpayne@68: // else if(a.equals("alternateaddress") || a.equals("altaddress") || a.equalsIgnoreCase("vm")){ jpayne@68: // boolean alt=Parse.parseBoolean(b); jpayne@68: // switchDefaultAddresses(alt); jpayne@68: // } jpayne@68: jpayne@68: jpayne@68: else if(a.equalsIgnoreCase("nt") || a.equalsIgnoreCase("silva") || a.equalsIgnoreCase("ribo") || jpayne@68: a.equalsIgnoreCase("refseq") || a.equalsIgnoreCase("img") || a.equalsIgnoreCase("nr") || jpayne@68: a.equalsIgnoreCase("refseqprot") || a.equalsIgnoreCase("prokprot") || a.equalsIgnoreCase("protein") || jpayne@68: a.equalsIgnoreCase("protien") || a.equalsIgnoreCase("prot") || a.equalsIgnoreCase("mito") || a.equalsIgnoreCase("fungi")){ jpayne@68: address=a; jpayne@68: blacklist=a; jpayne@68: } jpayne@68: jpayne@68: else if(a.equals("taxtree") || a.equals("tree")){ jpayne@68: taxTreeFile=b; jpayne@68: } jpayne@68: jpayne@68: else if(a.equals("name") || a.equals("taxname")){ jpayne@68: outTaxName=b; jpayne@68: }else if(a.equals("name0")){ jpayne@68: outName0=b; jpayne@68: }else if(a.equals("fname")){ jpayne@68: outFname=b; jpayne@68: }else if(a.equals("taxid") || a.equals("tid")){ jpayne@68: outTaxID=Integer.parseInt(b); jpayne@68: }else if(a.equals("spid")){ jpayne@68: outSpid=Integer.parseInt(b); jpayne@68: }else if(a.equals("imgid")){ jpayne@68: outImgID=Integer.parseInt(b); jpayne@68: }else if((a.startsWith("meta_") || a.startsWith("mt_")) && b!=null){ jpayne@68: if(outMeta==null){outMeta=new ArrayList();} jpayne@68: int underscore=a.indexOf('_', 0); jpayne@68: outMeta.add(a.substring(underscore+1)+":"+b); jpayne@68: } jpayne@68: jpayne@68: else if(a.equals("outsketch") || a.equals("outs") || a.equals("sketchout") || a.equals("sketch")){ jpayne@68: outSketch=b; jpayne@68: } jpayne@68: jpayne@68: else if(parser.parse(arg, a, b)){//Parse standard flags in the parser jpayne@68: //do nothing jpayne@68: } jpayne@68: jpayne@68: else if(b==null && in.isEmpty() && new File(arg).exists()){ jpayne@68: in.add(arg); jpayne@68: } jpayne@68: jpayne@68: else{ jpayne@68: outstream.println("Unknown parameter "+args[i]); jpayne@68: assert(false) : "Unknown parameter "+args[i]; jpayne@68: } jpayne@68: } jpayne@68: if("auto".equalsIgnoreCase(taxTreeFile)){taxTreeFile=TaxTree.defaultTreeFile();} jpayne@68: jpayne@68: address=toAddress(address); jpayne@68: outMeta=SketchObject.fixMeta(outMeta); jpayne@68: jpayne@68: if(address!=null && !SET_AUTOSIZE_FACTOR){ jpayne@68: if(address.equals(refseqAddress())){ jpayne@68: AUTOSIZE_FACTOR=2.0f; jpayne@68: }else if(address.equals(prokProtAddress())){ jpayne@68: AUTOSIZE_FACTOR=3.0f; jpayne@68: } jpayne@68: } jpayne@68: jpayne@68: while(address!=null && address.endsWith("/")){ jpayne@68: address=address.substring(0, address.length()-1); jpayne@68: } jpayne@68: jpayne@68: setFromAddress(address, setBlacklist); jpayne@68: jpayne@68: if(local){blacklist=null;} jpayne@68: jpayne@68: postParse(); jpayne@68: jpayne@68: {//Process parser fields jpayne@68: overwrite=ReadStats.overwrite=parser.overwrite; jpayne@68: append=ReadStats.append=parser.append; jpayne@68: jpayne@68: out=parser.out1; jpayne@68: } jpayne@68: jpayne@68: //Ensure there is an input file jpayne@68: if(in.isEmpty() && refNames==null){throw new RuntimeException("Error - at least one input file is required.");} jpayne@68: jpayne@68: //Adjust the number of threads for input file reading jpayne@68: if(!ByteFile.FORCE_MODE_BF1 && !ByteFile.FORCE_MODE_BF2 && Shared.threads()>2){ jpayne@68: ByteFile.FORCE_MODE_BF2=true; jpayne@68: } jpayne@68: jpayne@68: ffout=FileFormat.testOutput(out, FileFormat.TEXT, null, false, overwrite, append, false); jpayne@68: if(!ffout.stdio() && !defaultParams.setColors){defaultParams.printColors=false;} jpayne@68: jpayne@68: //Ensure input files can be read jpayne@68: if(!Tools.testInputFiles(false, true, taxTreeFile)){ jpayne@68: throw new RuntimeException("\nCan't read some input files.\n"); jpayne@68: } jpayne@68: jpayne@68: if(!local && refNames==null){ jpayne@68: if(!Tools.testInputFiles(true, false, in.toArray(new String[0]))){ jpayne@68: if(in.size()==1){ jpayne@68: String s=in.get(0); jpayne@68: String s1=s.replaceFirst("#", "1"), s2=s.replaceFirst("#", "2"); jpayne@68: Tools.testInputFiles(true, false, s1, s2); jpayne@68: }else{ jpayne@68: throw new RuntimeException("\nCan't read some input files.\n"); jpayne@68: } jpayne@68: } jpayne@68: } jpayne@68: jpayne@68: if(!Tools.testOutputFiles(overwrite, append, false, out, outSketch)){ jpayne@68: throw new RuntimeException("\n\noverwrite="+overwrite+"; Can't write to output files "+ jpayne@68: out+", "+outSketch+"\n"); jpayne@68: } jpayne@68: jpayne@68: //Ensure that no file was specified multiple times jpayne@68: if(!Tools.testForDuplicateFiles(true, in.toArray(new String[0]))){ jpayne@68: throw new RuntimeException("\nSome file names were specified multiple times.\n"); jpayne@68: } jpayne@68: jpayne@68: SSUMap.load(outstream); jpayne@68: tool=new SketchTool(targetSketchSize, defaultParams); jpayne@68: jpayne@68: // assert(false) : defaultParams.toString()+"\n"+k+", "+amino+", "+HASH_VERSION; jpayne@68: if(verbose){ jpayne@68: if(local){outstream.println("Running in local mode.");} jpayne@68: if(useWhitelist){outstream.println("Using a whitelist.");} jpayne@68: if(blacklist!=null){outstream.println("Using a blacklist.");} jpayne@68: } jpayne@68: jpayne@68: if(taxTreeFile!=null){setTaxtree(taxTreeFile, silent ? null : outstream);} jpayne@68: defaultParams.postParse(false, false); jpayne@68: if(!defaultParams.printSSU){processSSU=false;} jpayne@68: allowMultithreadedFastq=(in.size()==1 && Shared.threads()>2); jpayne@68: if(!allowMultithreadedFastq){Shared.capBufferLen(40);} jpayne@68: } jpayne@68: jpayne@68: private void setFromAddress(String address, boolean setBlacklist){ jpayne@68: if(address.equals(nrAddress())){ jpayne@68: amino=true; jpayne@68: if(!setK){k=defaultKAmino; k2=defaultK2Amino;} jpayne@68: // defaultParams.dbName="nr"; jpayne@68: assert(false) : "Need to set K."; jpayne@68: }else if(address.equals(prokProtAddress())){ jpayne@68: translate=true; jpayne@68: if(!setK){k=defaultKAmino; k2=defaultK2Amino;} jpayne@68: // defaultParams.dbName="nr"; jpayne@68: if(blacklist==null && !setBlacklist){blacklist=Blacklist.prokProtBlacklist();} jpayne@68: }else if(address.equals(ntAddress())){ jpayne@68: if(!setK){k=defaultK; k2=defaultK2;} jpayne@68: // defaultParams.dbName="nt"; jpayne@68: if(blacklist==null && !setBlacklist){blacklist=Blacklist.ntBlacklist();} jpayne@68: }else if(address.equals(refseqAddress())){ jpayne@68: if(!setK){k=defaultK; k2=defaultK2;} jpayne@68: // defaultParams.dbName="RefSeq"; jpayne@68: if(blacklist==null && !setBlacklist){blacklist=Blacklist.refseqBlacklist();} jpayne@68: }else if(address.equals(silvaAddress())){ jpayne@68: if(!setK){k=defaultK; k2=defaultK2;} jpayne@68: // defaultParams.dbName="Silva"; jpayne@68: if(blacklist==null && !setBlacklist){blacklist=Blacklist.silvaBlacklist();} jpayne@68: }else if(address.equals(imgAddress())){ jpayne@68: if(!setK){k=defaultK; k2=defaultK2;} jpayne@68: // defaultParams.dbName="IMG"; jpayne@68: if(blacklist==null && !setBlacklist){blacklist=Blacklist.imgBlacklist();} jpayne@68: }else if(address.equals(mitoAddress())){ jpayne@68: if(!setK){k=defaultK; k2=defaultK2;} jpayne@68: if(blacklist==null && !setBlacklist){blacklist=Blacklist.mitoBlacklist();} jpayne@68: }else if(address.equals(fungiAddress())){ jpayne@68: if(!setK){k=defaultK; k2=defaultK2;} jpayne@68: if(blacklist==null && !setBlacklist){blacklist=Blacklist.fungiBlacklist();} jpayne@68: } jpayne@68: } jpayne@68: jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: /*---------------- Outer Methods ----------------*/ jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: jpayne@68: public void process(Timer t){ jpayne@68: if(local){processLocal(t);} jpayne@68: else if(refNames!=null){processRefMode(t);} jpayne@68: else{processRemote(t);} jpayne@68: } jpayne@68: jpayne@68: @Deprecated jpayne@68: private void processRemote_old(Timer t){ jpayne@68: Timer ttotal=new Timer(); jpayne@68: jpayne@68: t.start(); jpayne@68: inSketches=tool.loadSketches_MT(defaultParams, in); jpayne@68: final int numLoaded=(inSketches.size()); jpayne@68: if(numLoaded>1 && defaultParams.mode==PER_FILE){ jpayne@68: inSketches.sort(SketchIdComparator.comparator);//Otherwise they come out disordered jpayne@68: } jpayne@68: jpayne@68: t.stop(); jpayne@68: if(!silent){outstream.println("Loaded "+numLoaded+" sketch"+(numLoaded==1 ? "" : "es")+" in "+t);} jpayne@68: assert(numLoaded<=MAX_ALLOWED_SKETCHES) : "\nSendSketch is configured to send at most "+MAX_ALLOWED_SKETCHES+" to prevent overwhelming the server.\n" jpayne@68: + "If you need to compare more than that, please use CompareSketch locally instead.\n" jpayne@68: + "References can be downloaded at http://portal.nersc.gov/dna/microbial/assembly/bushnell/\n"; jpayne@68: t.start(); jpayne@68: // outstream.println(inSketches.get(0)); jpayne@68: jpayne@68: if(numLoaded>4000){ jpayne@68: SEND_BUFFER_MAX_BYTES*=4; jpayne@68: SEND_BUFFER_MAX_SKETCHES*=4; jpayne@68: }else if(numLoaded>1000){ jpayne@68: SEND_BUFFER_MAX_BYTES*=2; jpayne@68: SEND_BUFFER_MAX_SKETCHES*=2; jpayne@68: } jpayne@68: jpayne@68: if(ffout==null){return;} jpayne@68: TextStreamWriter tsw=new TextStreamWriter(ffout); jpayne@68: tsw.start(); jpayne@68: if(defaultParams.format==DisplayParams.FORMAT_QUERY_REF_ANI || defaultParams.format==DisplayParams.FORMAT_CONSTELLATION){tsw.println(defaultParams.header());} jpayne@68: jpayne@68: ByteBuilder bb=new ByteBuilder(); jpayne@68: jpayne@68: int cntr=0; jpayne@68: int chunks=0; jpayne@68: for(Sketch sk : inSketches){ jpayne@68: jpayne@68: if(sk.taxID<1 || sk.taxID>=minFakeID || outTaxID>0){sk.taxID=outTaxID;} jpayne@68: if(defaultParams.printSSU()){sk.loadSSU();} jpayne@68: jpayne@68: if(outSpid>0){sk.spid=outSpid;} jpayne@68: if(outImgID>0){sk.imgID=outImgID;} jpayne@68: if(outTaxName!=null){sk.setTaxName(outTaxName);} jpayne@68: if(outFname!=null){sk.setFname(outFname);} jpayne@68: if(outName0!=null){sk.setName0(outName0);} jpayne@68: sk.setMeta(outMeta); jpayne@68: jpayne@68: if(bb.length==0){ jpayne@68: bb.append(defaultParams.toString(chunks)); jpayne@68: chunks++; jpayne@68: } jpayne@68: sk.toBytes(bb); jpayne@68: cntr++; jpayne@68: if(cntr>=SEND_BUFFER_MAX_SKETCHES || bb.length>SEND_BUFFER_MAX_BYTES){ //Don't allow too much data in a single transaction jpayne@68: if(verbose){outstream.println("Sending:\n"+bb);} jpayne@68: // outstream.println(cntr+", "+bb.length); jpayne@68: jpayne@68: byte[] message=bb.toBytes(); jpayne@68: bb.clear(); jpayne@68: try { jpayne@68: // outstream.println("Sending to "+address+"\n"+message+"\n"); //123 jpayne@68: StringNum result=ServerTools.sendAndReceive(message, address); jpayne@68: if(!ServerTools.suppressErrors && (result.n<200 || result.n>299)){ jpayne@68: System.err.println("ERROR: Server returned code "+result.n+" and this message:\n"+result.s); jpayne@68: KillSwitch.kill(); jpayne@68: } jpayne@68: errorState|=checkForError(result.s); jpayne@68: tsw.print(result.s); jpayne@68: } catch (Exception e) { jpayne@68: // TODO Auto-generated catch block jpayne@68: e.printStackTrace(); jpayne@68: } jpayne@68: cntr=0; jpayne@68: } jpayne@68: } jpayne@68: jpayne@68: if(bb.length>0){ jpayne@68: if(verbose){outstream.println("Sending:\n"+bb);} jpayne@68: byte[] message=bb.toBytes(); jpayne@68: bb.clear(); jpayne@68: try { jpayne@68: StringNum result=ServerTools.sendAndReceive(message, address); jpayne@68: if(!ServerTools.suppressErrors && (result.n<200 || result.n>299)){ jpayne@68: System.err.println("ERROR: Server returned code "+result.n+" and this message:\n"+result.s); jpayne@68: KillSwitch.kill(); jpayne@68: } jpayne@68: errorState|=checkForError(result.s); jpayne@68: tsw.print(result.s); jpayne@68: } catch (Exception e) { jpayne@68: // TODO Auto-generated catch block jpayne@68: e.printStackTrace(); jpayne@68: } jpayne@68: } jpayne@68: if(!silent){tsw.println();} jpayne@68: tsw.poison(); jpayne@68: jpayne@68: // outstream.println("sending "+bb.toString()); jpayne@68: jpayne@68: if(outSketch!=null){ jpayne@68: ByteStreamWriter bsw=new ByteStreamWriter(outSketch, overwrite, append, true, FileFormat.SKETCH); jpayne@68: bsw.start(); jpayne@68: for(Sketch sk : inSketches){ jpayne@68: sk.toBytes(bb); jpayne@68: bsw.print(bb); jpayne@68: bb.clear(); jpayne@68: } jpayne@68: bsw.poisonAndWait(); jpayne@68: errorState|=bsw.errorState; jpayne@68: } jpayne@68: jpayne@68: tsw.waitForFinish(); jpayne@68: errorState|=tsw.errorState; jpayne@68: jpayne@68: t.stop(); jpayne@68: // outstream.println("\nRan "+(inSketches.size()*refSketches.size())+" comparisons in \t"+t); jpayne@68: ttotal.stop(); jpayne@68: if(!silent){outstream.println("Total Time: \t"+ttotal);} jpayne@68: } jpayne@68: jpayne@68: jpayne@68: private void processRemote(Timer t){ jpayne@68: Timer ttotal=new Timer(); jpayne@68: jpayne@68: t.start(); jpayne@68: inSketches=tool.loadSketches_MT(defaultParams, in); jpayne@68: final int numLoaded=(inSketches.size()); jpayne@68: if(numLoaded>1 && defaultParams.mode==PER_FILE){ jpayne@68: inSketches.sort(SketchIdComparator.comparator);//Otherwise they come out disordered jpayne@68: } jpayne@68: jpayne@68: t.stop(); jpayne@68: if(!silent){outstream.println("Loaded "+numLoaded+" sketch"+(numLoaded==1 ? "" : "es")+" in "+t);} jpayne@68: assert(numLoaded<=MAX_ALLOWED_SKETCHES) : "\nSendSketch is configured to send at most "+MAX_ALLOWED_SKETCHES+" to prevent overwhelming the server.\n" jpayne@68: + "If you need to compare more than that, please use CompareSketch locally instead.\n" jpayne@68: + "References can be downloaded at http://portal.nersc.gov/dna/microbial/assembly/bushnell/\n"; jpayne@68: t.start(); jpayne@68: // outstream.println(inSketches.get(0)); jpayne@68: jpayne@68: if(numLoaded>4000){ jpayne@68: SEND_BUFFER_MAX_BYTES*=4; jpayne@68: SEND_BUFFER_MAX_SKETCHES*=4; jpayne@68: }else if(numLoaded>1000){ jpayne@68: SEND_BUFFER_MAX_BYTES*=2; jpayne@68: SEND_BUFFER_MAX_SKETCHES*=2; jpayne@68: } jpayne@68: // SEND_BUFFER_MAX_SKETCHES=1;//This is for testing json array format. jpayne@68: jpayne@68: if(ffout==null){return;} jpayne@68: TextStreamWriter tsw=new TextStreamWriter(ffout); jpayne@68: tsw.start(); jpayne@68: if(defaultParams.format==DisplayParams.FORMAT_QUERY_REF_ANI || defaultParams.format==DisplayParams.FORMAT_CONSTELLATION){tsw.println(defaultParams.header());} jpayne@68: jpayne@68: ByteBuilder bb=new ByteBuilder(); jpayne@68: jpayne@68: int cntr=0; jpayne@68: int chunks=0; jpayne@68: boolean firstBlock=true; //Set to false after printing has started jpayne@68: final Sketch lastSketch=Tools.getLast(inSketches); //Last input sketch to process jpayne@68: for(Sketch sk : inSketches){ jpayne@68: jpayne@68: if(sk.taxID<1 || sk.taxID>=minFakeID || outTaxID>0){sk.taxID=outTaxID;} jpayne@68: if(defaultParams.printSSU()){sk.loadSSU();} jpayne@68: jpayne@68: if(outSpid>0){sk.spid=outSpid;} jpayne@68: if(outImgID>0){sk.imgID=outImgID;} jpayne@68: if(outTaxName!=null){sk.setTaxName(outTaxName);} jpayne@68: if(outFname!=null){sk.setFname(outFname);} jpayne@68: if(outName0!=null){sk.setName0(outName0);} jpayne@68: sk.setMeta(outMeta); jpayne@68: jpayne@68: if(bb.length==0){ jpayne@68: bb.append(defaultParams.toString(chunks)); jpayne@68: chunks++; jpayne@68: } jpayne@68: sk.toBytes(bb); jpayne@68: cntr++; jpayne@68: if(cntr>=SEND_BUFFER_MAX_SKETCHES || bb.length>SEND_BUFFER_MAX_BYTES || sk==lastSketch){ //Don't allow too much data in a single transaction jpayne@68: if(verbose){outstream.println("Sending:\n"+bb);} jpayne@68: // outstream.println(cntr+", "+bb.length); jpayne@68: jpayne@68: byte[] message=bb.toBytes(); jpayne@68: bb.clear(); jpayne@68: try { jpayne@68: // outstream.println("Sending to "+address+"\n"+message+"\n"); //123 jpayne@68: StringNum result=ServerTools.sendAndReceive(message, address); jpayne@68: if(!ServerTools.suppressErrors && (result.n<200 || result.n>299)){ jpayne@68: System.err.println("ERROR: Server returned code "+result.n+" and this message:\n"+result.s); jpayne@68: KillSwitch.kill(); jpayne@68: } jpayne@68: errorState|=checkForError(result.s); jpayne@68: if(!defaultParams.json()){ jpayne@68: tsw.print(result.s); jpayne@68: }else{ jpayne@68: String s=result.s; jpayne@68: jpayne@68: {//Fixes rare case of multiple sketches broken into single sketches per block jpayne@68: if(!firstBlock || sk!=lastSketch){ jpayne@68: if(s.charAt(0)!='['){s="["+s;} jpayne@68: if(!s.endsWith("]")){;s=s+"]";} jpayne@68: } jpayne@68: } jpayne@68: jpayne@68: {//Case where there are more than one block of sketches, breaking JSON array format jpayne@68: if(!firstBlock){ jpayne@68: s=","+s.substring(1); jpayne@68: } jpayne@68: if(sk!=lastSketch){ jpayne@68: s=s.substring(0, s.length()-1); jpayne@68: } jpayne@68: } jpayne@68: jpayne@68: tsw.print(s); jpayne@68: firstBlock=false; jpayne@68: } jpayne@68: } catch (Exception e) { jpayne@68: // TODO Auto-generated catch block jpayne@68: e.printStackTrace(); jpayne@68: } jpayne@68: cntr=0; jpayne@68: } jpayne@68: } jpayne@68: jpayne@68: if(bb.length>0){//This should no longer trigger unless there are no sketches since it's handled above. jpayne@68: if(verbose){outstream.println("Sending:\n"+bb);} jpayne@68: byte[] message=bb.toBytes(); jpayne@68: bb.clear(); jpayne@68: try { jpayne@68: StringNum result=ServerTools.sendAndReceive(message, address); jpayne@68: if(!ServerTools.suppressErrors && (result.n<200 || result.n>299)){ jpayne@68: System.err.println("ERROR: Server returned code "+result.n+" and this message:\n"+result.s); jpayne@68: KillSwitch.kill(); jpayne@68: } jpayne@68: errorState|=checkForError(result.s); jpayne@68: tsw.print(result.s); jpayne@68: } catch (Exception e) { jpayne@68: // TODO Auto-generated catch block jpayne@68: e.printStackTrace(); jpayne@68: } jpayne@68: } jpayne@68: if(!silent){tsw.println();} jpayne@68: tsw.poison(); jpayne@68: jpayne@68: // outstream.println("sending "+bb.toString()); jpayne@68: jpayne@68: if(outSketch!=null){ jpayne@68: ByteStreamWriter bsw=new ByteStreamWriter(outSketch, overwrite, append, true, FileFormat.SKETCH); jpayne@68: bsw.start(); jpayne@68: for(Sketch sk : inSketches){ jpayne@68: sk.toBytes(bb); jpayne@68: bsw.print(bb); jpayne@68: bb.clear(); jpayne@68: } jpayne@68: bsw.poisonAndWait(); jpayne@68: errorState|=bsw.errorState; jpayne@68: } jpayne@68: jpayne@68: tsw.waitForFinish(); jpayne@68: errorState|=tsw.errorState; jpayne@68: jpayne@68: t.stop(); jpayne@68: // outstream.println("\nRan "+(inSketches.size()*refSketches.size())+" comparisons in \t"+t); jpayne@68: ttotal.stop(); jpayne@68: if(!silent){outstream.println("Total Time: \t"+ttotal);} jpayne@68: } jpayne@68: jpayne@68: /** For programmatic use */ jpayne@68: public static String sendSketch(Sketch sk, String address, int format, int chunkNum){ jpayne@68: address=toAddress(address); jpayne@68: ByteBuilder bb=new ByteBuilder(); jpayne@68: jpayne@68: DisplayParams params=defaultParams; jpayne@68: if(format>=0){ jpayne@68: new DisplayParams(); jpayne@68: params.format=format; jpayne@68: } jpayne@68: if(bb.length==0){bb.append(params.toString(chunkNum));} jpayne@68: sk.toBytes(bb); jpayne@68: jpayne@68: byte[] message=bb.toBytes(); jpayne@68: try { jpayne@68: // System.err.println("Sending to "+address+"\n"+new String(message)+"\n"); //123 jpayne@68: StringNum result=ServerTools.sendAndReceive(message, address); jpayne@68: if(!ServerTools.suppressErrors && (result.n<200 || result.n>299)){ jpayne@68: System.err.println("ERROR: Server returned code "+result.n+" and this message:\n"+result.s); jpayne@68: KillSwitch.kill(); jpayne@68: } jpayne@68: return result.s; jpayne@68: } catch (Exception e) { jpayne@68: // TODO Auto-generated catch block jpayne@68: e.printStackTrace(); jpayne@68: } jpayne@68: return null; jpayne@68: } jpayne@68: jpayne@68: private static boolean checkForError(String s){ jpayne@68: if(s==null){return false;} jpayne@68: return s.contains("java.io.IOException: Server returned HTTP response code:"); jpayne@68: } jpayne@68: jpayne@68: private void processLocal(Timer t){ jpayne@68: Timer ttotal=new Timer(); jpayne@68: jpayne@68: t.start(); jpayne@68: jpayne@68: if(ffout==null){return;} jpayne@68: TextStreamWriter tsw=new TextStreamWriter(ffout); jpayne@68: tsw.start(); jpayne@68: jpayne@68: final String message=defaultParams.toString(0); jpayne@68: for(String fname : in){ jpayne@68: String address2=address+"/file/"+new File(fname).getAbsolutePath(); jpayne@68: jpayne@68: if(verbose){outstream.println("Sending:\n"+message+"\nto "+address2);} jpayne@68: try { jpayne@68: // outstream.println("Sending to "+address2+"\n"+message+"\n"); //123 jpayne@68: StringNum result=ServerTools.sendAndReceive(message.getBytes(), address2); jpayne@68: if(!ServerTools.suppressErrors && (result.n<200 || result.n>299)){ jpayne@68: System.err.println("ERROR: Server returned code "+result.n+" and this message:\n"+result.s); jpayne@68: KillSwitch.kill(); jpayne@68: } jpayne@68: tsw.print(result.s); jpayne@68: } catch (Exception e) { jpayne@68: // TODO Auto-generated catch block jpayne@68: if(!suppressErrors){e.printStackTrace();} jpayne@68: } jpayne@68: } jpayne@68: tsw.println(); jpayne@68: jpayne@68: // outstream.println("sending "+bb.toString()); jpayne@68: jpayne@68: tsw.poisonAndWait(); jpayne@68: errorState|=tsw.errorState; jpayne@68: jpayne@68: t.stop(); jpayne@68: // outstream.println("\nRan "+(inSketches.size()*refSketches.size())+" comparisons in \t"+t); jpayne@68: ttotal.stop(); jpayne@68: outstream.println("Total Time: \t"+ttotal); jpayne@68: } jpayne@68: jpayne@68: private void processRefMode(Timer t){ jpayne@68: Timer ttotal=new Timer(); jpayne@68: jpayne@68: t.start(); jpayne@68: jpayne@68: if(ffout==null){return;} jpayne@68: TextStreamWriter tsw=new TextStreamWriter(ffout); jpayne@68: tsw.start(); jpayne@68: jpayne@68: final String message=defaultParams.toString(0); jpayne@68: { jpayne@68: String address2=address+"/ref/"+refNames; jpayne@68: jpayne@68: if(verbose){outstream.println("Sending:\n"+message+"\nto "+address2);} jpayne@68: try { jpayne@68: // outstream.println("Sending to "+address2+"\n"+message+"\n"); //123 jpayne@68: StringNum result=ServerTools.sendAndReceive(message.getBytes(), address2); jpayne@68: if(!ServerTools.suppressErrors && (result.n<200 || result.n>299)){ jpayne@68: System.err.println("ERROR: Server returned code "+result.n+" and this message:\n"+result.s); jpayne@68: KillSwitch.kill(); jpayne@68: } jpayne@68: tsw.print(result.s); jpayne@68: } catch (Exception e) { jpayne@68: // TODO Auto-generated catch block jpayne@68: if(!suppressErrors){e.printStackTrace();} jpayne@68: } jpayne@68: } jpayne@68: tsw.println(); jpayne@68: jpayne@68: // outstream.println("sending "+bb.toString()); jpayne@68: jpayne@68: tsw.poisonAndWait(); jpayne@68: errorState|=tsw.errorState; jpayne@68: jpayne@68: t.stop(); jpayne@68: // outstream.println("\nRan "+(inSketches.size()*refSketches.size())+" comparisons in \t"+t); jpayne@68: ttotal.stop(); jpayne@68: outstream.println("Total Time: \t"+ttotal); jpayne@68: } jpayne@68: jpayne@68: jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: /*---------------- Inner Methods ----------------*/ jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: jpayne@68: private static boolean addFiles(String a, Collection list){ jpayne@68: int initial=list.size(); jpayne@68: if(a==null){return false;} jpayne@68: File f=null; jpayne@68: if(a.indexOf(',')>=0){f=new File(a);} jpayne@68: if(f==null || f.exists()){ jpayne@68: list.add(a); jpayne@68: }else{ jpayne@68: for(String s : a.split(",")){ jpayne@68: list.add(s); jpayne@68: } jpayne@68: } jpayne@68: return list.size()>initial; jpayne@68: } jpayne@68: jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: /*---------------- Fields ----------------*/ jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: jpayne@68: private ArrayList in=new ArrayList(); jpayne@68: jpayne@68: private String out="stdout.txt"; jpayne@68: private String outSketch=null; jpayne@68: jpayne@68: private String taxTreeFile=null; jpayne@68: jpayne@68: private final SketchTool tool; jpayne@68: jpayne@68: private ArrayList inSketches; jpayne@68: jpayne@68: private String address=null; jpayne@68: private boolean local=false; jpayne@68: /** List of reference names or TaxIDs to use as queries */ jpayne@68: private String refNames=null; jpayne@68: jpayne@68: /*Override metadata */ jpayne@68: private String outTaxName=null; jpayne@68: private String outFname=null; jpayne@68: private String outName0=null; jpayne@68: private int outTaxID=-1; jpayne@68: private long outSpid=-1; jpayne@68: private long outImgID=-1; jpayne@68: private ArrayList outMeta=null; jpayne@68: jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: /*---------------- Final Fields ----------------*/ jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: jpayne@68: /** Primary output file */ jpayne@68: private final FileFormat ffout; jpayne@68: jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: /*---------------- Common Fields ----------------*/ jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: jpayne@68: /** Print status messages to this output stream */ jpayne@68: private PrintStream outstream=System.err; jpayne@68: /** Print verbose messages */ jpayne@68: public static boolean verbose=false; jpayne@68: /** True if an error was encountered */ jpayne@68: public boolean errorState=false; jpayne@68: /** Overwrite existing output files */ jpayne@68: private boolean overwrite=false; jpayne@68: /** Append to existing output files */ jpayne@68: private boolean append=false; jpayne@68: jpayne@68: private boolean silent=false; jpayne@68: jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: /*---------------- Static Fields ----------------*/ jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: jpayne@68: public static final String toAddress(String b){ jpayne@68: String address=b; jpayne@68: if(b==null){ jpayne@68: address=refseqAddress();//default jpayne@68: }else if(b.equalsIgnoreCase("nt")){ jpayne@68: address=ntAddress(); jpayne@68: }else if(b.equalsIgnoreCase("refseq")){ jpayne@68: address=refseqAddress(); jpayne@68: }else if(b.equalsIgnoreCase("silva") || b.equalsIgnoreCase("ribo")){ jpayne@68: address=silvaAddress(); jpayne@68: }else if(b.equalsIgnoreCase("img")){ jpayne@68: address=imgAddress(); jpayne@68: }else if(b.equalsIgnoreCase("refseqprot") || b.equalsIgnoreCase("prokprot") jpayne@68: || b.equalsIgnoreCase("protein") || b.equalsIgnoreCase("protien") || b.equalsIgnoreCase("prot")){ jpayne@68: address=prokProtAddress(); jpayne@68: }else if(b.equalsIgnoreCase("refseqmito") || b.equalsIgnoreCase("mito")){ jpayne@68: address=mitoAddress(); jpayne@68: }else if(b.equalsIgnoreCase("refseqfungi") || b.equalsIgnoreCase("fungi")){ jpayne@68: address=fungiAddress(); jpayne@68: } jpayne@68: return address; jpayne@68: } jpayne@68: jpayne@68: public int SEND_BUFFER_MAX_BYTES=8000000; jpayne@68: public int SEND_BUFFER_MAX_SKETCHES=400; jpayne@68: private static final int MAX_ALLOWED_SKETCHES=100000; jpayne@68: jpayne@68: /** Don't print caught exceptions */ jpayne@68: public static boolean suppressErrors=false; jpayne@68: jpayne@68: private static String ntAddress(){return Shared.ntSketchServer()+"sketch";} jpayne@68: private static String refseqAddress(){return Shared.refseqSketchServer()+"sketch";} jpayne@68: private static String silvaAddress(){return Shared.riboSketchServer()+"sketch";} jpayne@68: private static String imgAddress(){return null;}//Shared.SketchServer()+"sketch"; jpayne@68: private static String nrAddress(){return null;}//Shared.SketchServer()+"sketch"; jpayne@68: private static String prokProtAddress(){return Shared.proteinSketchServer()+"sketch";} jpayne@68: private static String mitoAddress(){return null;}//Shared.SketchServer()+"sketch"; jpayne@68: private static String fungiAddress(){return null;}//Shared.SketchServer()+"sketch"; jpayne@68: jpayne@68: }