jpayne@68: package tax; jpayne@68: jpayne@68: import java.io.File; jpayne@68: import java.io.IOException; jpayne@68: import java.io.InputStream; jpayne@68: import java.io.PrintStream; jpayne@68: import java.net.InetSocketAddress; jpayne@68: import java.util.ArrayList; jpayne@68: import java.util.Arrays; jpayne@68: import java.util.Collections; jpayne@68: import java.util.Date; jpayne@68: import java.util.HashMap; jpayne@68: import java.util.List; jpayne@68: import java.util.Locale; jpayne@68: import java.util.Map.Entry; jpayne@68: import java.util.concurrent.ConcurrentHashMap; jpayne@68: import java.util.concurrent.atomic.AtomicInteger; jpayne@68: import java.util.concurrent.atomic.AtomicLong; jpayne@68: import java.util.concurrent.atomic.AtomicLongArray; jpayne@68: jpayne@68: import com.sun.net.httpserver.HttpExchange; jpayne@68: import com.sun.net.httpserver.HttpHandler; jpayne@68: import com.sun.net.httpserver.HttpServer; jpayne@68: import com.sun.net.httpserver.HttpsServer; jpayne@68: jpayne@68: import dna.Data; jpayne@68: import fileIO.ReadWrite; jpayne@68: import json.JsonObject; jpayne@68: import server.PercentEncoding; jpayne@68: import server.ServerTools; jpayne@68: import shared.KillSwitch; jpayne@68: import shared.Parse; jpayne@68: import shared.Parser; jpayne@68: import shared.PreParser; jpayne@68: import shared.Shared; jpayne@68: import shared.Timer; jpayne@68: import shared.Tools; jpayne@68: import sketch.CompareBuffer; jpayne@68: import sketch.Comparison; jpayne@68: import sketch.DisplayParams; jpayne@68: import sketch.Sketch; jpayne@68: import sketch.SketchMakerMini; jpayne@68: import sketch.SketchObject; jpayne@68: import sketch.SketchResults; jpayne@68: import sketch.SketchSearcher; jpayne@68: import sketch.SketchTool; jpayne@68: import sketch.Whitelist; jpayne@68: import stream.Read; jpayne@68: import structures.ByteBuilder; jpayne@68: import structures.IntList; jpayne@68: import structures.StringNum; jpayne@68: jpayne@68: /** jpayne@68: * Server for taxonomy or Sketch queries. jpayne@68: * @author Shijie Yao, Brian Bushnell jpayne@68: * @date Dec 13, 2016 jpayne@68: * jpayne@68: */ jpayne@68: public class TaxServer { jpayne@68: jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: /*---------------- Startup ----------------*/ jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: jpayne@68: /** Command line entrance */ jpayne@68: public static void main(String[] args) throws Exception { jpayne@68: Timer t=new Timer(); jpayne@68: @SuppressWarnings("unused") jpayne@68: TaxServer ts=new TaxServer(args); jpayne@68: jpayne@68: t.stop("Time: "); jpayne@68: jpayne@68: System.err.println("Ready!"); jpayne@68: jpayne@68: //ts.begin(); jpayne@68: } jpayne@68: jpayne@68: /** Constructor */ jpayne@68: public TaxServer(String[] args) throws Exception { jpayne@68: jpayne@68: {//Preparse block for help, config files, and outstream jpayne@68: PreParser pp=new PreParser(args, getClass(), false); jpayne@68: args=pp.args; jpayne@68: outstream=pp.outstream; jpayne@68: } jpayne@68: jpayne@68: ReadWrite.USE_UNPIGZ=true; jpayne@68: TaxFilter.printNodesAdded=false; jpayne@68: TaxFilter.REQUIRE_PRESENT=false; //Due to missing entries in TaxDump. jpayne@68: Read.JUNK_MODE=Read.FIX_JUNK; jpayne@68: SketchObject.compareSelf=true; jpayne@68: jpayne@68: int port_=3068; //Taxonomy server jpayne@68: String killCode_=null; jpayne@68: boolean allowRemoteFileAccess_=false; jpayne@68: boolean allowLocalHost_=false; jpayne@68: String addressPrefix_="128."; //LBL jpayne@68: long defaultSketchReads=200000; jpayne@68: boolean https=false; jpayne@68: jpayne@68: int serverNum_=0; jpayne@68: int serverCount_=1; jpayne@68: jpayne@68: //Create a parser object jpayne@68: Parser parser=new Parser(); jpayne@68: jpayne@68: //Parse each argument jpayne@68: for(int i=0; i1 ? split[1] : null; jpayne@68: jpayne@68: if(a.equals("verbose")){ jpayne@68: verbose=Parse.parseBoolean(b); jpayne@68: }else if(a.equals("verbose2")){ jpayne@68: verbose2=SketchObject.verbose2=Parse.parseBoolean(b); jpayne@68: }else if(a.equals("html")){ jpayne@68: useHtml=Parse.parseBoolean(b); jpayne@68: }else if(a.equals("https")){ jpayne@68: https=Parse.parseBoolean(b); jpayne@68: }else if(a.equals("http")){ jpayne@68: https=!Parse.parseBoolean(b); jpayne@68: }else if(a.equals("servers") || a.equals("numservers") || a.equals("servercount")){ jpayne@68: serverCount_=Integer.parseInt(b); jpayne@68: assert(serverCount_>0) : arg; jpayne@68: }else if(a.equals("servernum")){ jpayne@68: serverNum_=Integer.parseInt(b); jpayne@68: assert(serverNum_>=0) : arg; jpayne@68: }else if(a.startsWith("slave") && Tools.isDigit(a.charAt(a.length()-1))){ jpayne@68: int num=Integer.parseInt(a.substring(5)); jpayne@68: if(slaveAddress==null){slaveAddress=new ArrayList(serverCount_);} jpayne@68: while(slaveAddress.size()<=num){slaveAddress.add(null);} jpayne@68: slaveAddress.set(num, b); jpayne@68: }else if(a.equals("table") || a.equals("gi") || a.equals("gitable")){ jpayne@68: giTableFile=b; jpayne@68: }else if(a.equals("tree") || a.equals("taxtree")){ jpayne@68: taxTreeFile=b; jpayne@68: }else if(a.equals("accession")){ jpayne@68: accessionFile=b; jpayne@68: }else if(a.equals("pattern")){ jpayne@68: patternFile=b; jpayne@68: }else if(a.equals("size") || a.equals("sizefile")){ jpayne@68: sizeFile=b; jpayne@68: }else if(a.equalsIgnoreCase("img")){ jpayne@68: imgFile=b; jpayne@68: }else if(a.equals("domain")){ jpayne@68: domain=b; jpayne@68: while(domain!=null && domain.endsWith("/")){domain=domain.substring(0, domain.length()-1);} jpayne@68: }else if(a.equals("port")){ jpayne@68: port_=Integer.parseInt(b); jpayne@68: }else if(a.equals("kill") || a.equals("killcode")){ jpayne@68: killCode_=b; jpayne@68: }else if(a.equals("oldcode")){ jpayne@68: oldKillCode=b; jpayne@68: }else if(a.equals("oldaddress")){ jpayne@68: oldAddress=b; jpayne@68: }else if(a.equals("sketchonly")){ jpayne@68: sketchOnly=Parse.parseBoolean(b); jpayne@68: }else if(a.equals("sketchreads")){ jpayne@68: defaultSketchReads=Parse.parseKMG(b); jpayne@68: }else if(a.equals("handlerthreads")){ jpayne@68: handlerThreads=Integer.parseInt(b); jpayne@68: }else if(a.equals("sketchthreads") || a.equals("sketchcomparethreads")){ jpayne@68: maxConcurrentSketchCompareThreads=Integer.parseInt(b); jpayne@68: }else if(a.equals("sketchloadthreads")){ jpayne@68: maxConcurrentSketchLoadThreads=Integer.parseInt(b); jpayne@68: }else if(a.equals("hashnames")){ jpayne@68: hashNames=Parse.parseBoolean(b); jpayne@68: }else if(a.equals("hashdotformat")){ jpayne@68: hashDotFormat=Parse.parseBoolean(b); jpayne@68: }else if(a.equals("printip")){ jpayne@68: printIP=Parse.parseBoolean(b); jpayne@68: }else if(a.equals("printheaders")){ jpayne@68: printHeaders=Parse.parseBoolean(b); jpayne@68: }else if(a.equals("countqueries")){ jpayne@68: countQueries=Parse.parseBoolean(b); jpayne@68: }else if(a.equals("clear") || a.equals("clearmem")){ jpayne@68: clearMem=Parse.parseBoolean(b); jpayne@68: }else if(a.equals("dbname")){ jpayne@68: SketchObject.defaultParams.dbName=b; jpayne@68: }else if(a.equals("allowremotefileaccess")){ jpayne@68: allowRemoteFileAccess_=Parse.parseBoolean(b); jpayne@68: }else if(a.equals("allowlocalhost")){ jpayne@68: allowLocalHost_=Parse.parseBoolean(b); jpayne@68: }else if(a.equals("addressprefix")){ jpayne@68: addressPrefix_=b; jpayne@68: }else if(a.equals("maxpigzprocesses")){ jpayne@68: AccessionToTaxid.maxPigzProcesses=Integer.parseInt(b); jpayne@68: }else if(a.equals("path") || a.equals("treepath") || a.equals("basepath")){ jpayne@68: basePath=b; jpayne@68: }else if(a.equalsIgnoreCase("prealloc")){ jpayne@68: if(b==null || Character.isLetter(b.charAt(0))){ jpayne@68: if(Parse.parseBoolean(b)){ jpayne@68: prealloc=0.78f; jpayne@68: }else{ jpayne@68: prealloc=0; jpayne@68: } jpayne@68: }else{ jpayne@68: prealloc=Float.parseFloat(b); jpayne@68: } jpayne@68: SketchObject.prealloc=prealloc; jpayne@68: }else if(searcher.parse(arg, a, b, true)){ jpayne@68: //do nothing jpayne@68: }else if(parser.parse(arg, a, b)){//Parse standard flags in the parser jpayne@68: //do nothing jpayne@68: }else{ jpayne@68: throw new RuntimeException(arg); jpayne@68: } jpayne@68: } jpayne@68: if("auto".equalsIgnoreCase(imgFile)){imgFile=TaxTree.defaultImgFile();} jpayne@68: if("auto".equalsIgnoreCase(taxTreeFile)){taxTreeFile=TaxTree.defaultTreeFile();} jpayne@68: if("auto".equalsIgnoreCase(giTableFile)){giTableFile=TaxTree.defaultTableFile();} jpayne@68: if("auto".equalsIgnoreCase(accessionFile)){accessionFile=TaxTree.defaultAccessionFile();} jpayne@68: if("auto".equalsIgnoreCase(patternFile)){patternFile=TaxTree.defaultPatternFile();} jpayne@68: if("auto".equalsIgnoreCase(sizeFile)){sizeFile=TaxTree.defaultSizeFile();} jpayne@68: jpayne@68: serverNum=AccessionToTaxid.serverNum=serverNum_; jpayne@68: serverCount=AccessionToTaxid.serverCount=serverCount_; jpayne@68: distributed=AccessionToTaxid.distributed=serverCount>1; jpayne@68: assert(serverNum=0); jpayne@68: if(distributed && serverNum==0){ jpayne@68: assert(slaveAddress!=null); jpayne@68: assert(slaveAddress.size()==serverCount); jpayne@68: for(int i=1; i=1); jpayne@68: assert(maxConcurrentSketchLoadThreads>=1); jpayne@68: jpayne@68: if(basePath==null || basePath.trim().length()==0){basePath="";} jpayne@68: else{ jpayne@68: basePath=basePath.trim().replace('\\', '/').replaceAll("/+", "/"); jpayne@68: if(!basePath.endsWith("/")){basePath=basePath+"/";} jpayne@68: } jpayne@68: jpayne@68: //Adjust SketchSearch rcomp and amino flags jpayne@68: SketchObject.postParse(); jpayne@68: jpayne@68: if(sketchOnly){ jpayne@68: // hashNames=false; jpayne@68: giTableFile=null; jpayne@68: accessionFile=null; jpayne@68: imgFile=null; jpayne@68: patternFile=null; jpayne@68: } jpayne@68: jpayne@68: port=port_; jpayne@68: killCode=killCode_; jpayne@68: allowRemoteFileAccess=allowRemoteFileAccess_; jpayne@68: allowLocalHost=allowLocalHost_; jpayne@68: addressPrefix=addressPrefix_; jpayne@68: jpayne@68: //Fill some data objects jpayne@68: USAGE=makeUsagePrefix(); jpayne@68: rawHtml=(useHtml ? loadRawHtml() : null); jpayne@68: typeMap=makeTypeMap(); jpayne@68: commonMap=makeCommonMap(); jpayne@68: jpayne@68: //Load the GI table jpayne@68: if(giTableFile!=null){ jpayne@68: outstream.println("Loading gi table."); jpayne@68: GiToTaxid.initialize(giTableFile); jpayne@68: } jpayne@68: jpayne@68: //Load the taxTree jpayne@68: if(taxTreeFile!=null){ jpayne@68: tree=TaxTree.loadTaxTree(taxTreeFile, outstream, hashNames, hashDotFormat); jpayne@68: if(hashNames){tree.hashChildren();} jpayne@68: assert(tree.nameMap!=null || sketchOnly); jpayne@68: }else{//The tree is required jpayne@68: tree=null; jpayne@68: throw new RuntimeException("No tree specified."); jpayne@68: } jpayne@68: //Set a default taxtree for sketch-related usage jpayne@68: SketchObject.taxtree=tree; jpayne@68: jpayne@68: if(sizeFile!=null){ jpayne@68: Timer t=new Timer(); jpayne@68: outstream.println("Loading size file."); jpayne@68: tree.loadSizeFile(sizeFile); jpayne@68: t.stopAndPrint(); jpayne@68: } jpayne@68: jpayne@68: if(imgFile!=null){ jpayne@68: TaxTree.loadIMG(imgFile, false, outstream); jpayne@68: } jpayne@68: jpayne@68: if(patternFile!=null){ jpayne@68: Timer t=new Timer(); jpayne@68: AnalyzeAccession.loadCodeMap(patternFile); jpayne@68: outstream.println("Loading pattern table."); jpayne@68: t.stopAndPrint(); jpayne@68: } jpayne@68: jpayne@68: //Load accession files jpayne@68: if(accessionFile!=null){ jpayne@68: Timer t=new Timer(); jpayne@68: AccessionToTaxid.tree=tree; jpayne@68: AccessionToTaxid.prealloc=prealloc; jpayne@68: outstream.println("Loading accession table."); jpayne@68: AccessionToTaxid.load(accessionFile); jpayne@68: t.stopAndPrint(); jpayne@68: // if(searcher.refFiles.isEmpty()){System.gc();} jpayne@68: } jpayne@68: jpayne@68: // assert(false) : searcher.refFileCount(); jpayne@68: jpayne@68: //Load reference sketches jpayne@68: hasSketches=searcher.refFileCount()>0; jpayne@68: if(hasSketches){ jpayne@68: outstream.println("Loading sketches."); jpayne@68: Timer t=new Timer(); jpayne@68: searcher.loadReferences(SketchObject.PER_TAXA, SketchObject.defaultParams); jpayne@68: t.stopAndPrint(); jpayne@68: // System.gc(); jpayne@68: } jpayne@68: jpayne@68: SketchObject.allowMultithreadedFastq=(maxConcurrentSketchLoadThreads>1); jpayne@68: SketchObject.defaultParams.maxReads=defaultSketchReads; jpayne@68: ReadWrite.USE_UNPIGZ=false; jpayne@68: // ReadWrite.USE_UNBGZIP=false; jpayne@68: jpayne@68: if(clearMem){ jpayne@68: System.err.println("Clearing memory."); jpayne@68: System.gc(); jpayne@68: Shared.printMemory(); jpayne@68: } jpayne@68: jpayne@68: //If there is a kill code, kill the old instance jpayne@68: if(oldKillCode!=null && oldAddress!=null){ jpayne@68: outstream.println("Killing old instance."); jpayne@68: killOldInstance(); jpayne@68: } jpayne@68: jpayne@68: //Wait for server initialization jpayne@68: httpServer=initializeServer(1000, 8, https); jpayne@68: assert(httpServer!=null); jpayne@68: jpayne@68: //Initialize handlers jpayne@68: if(!sketchOnly){ jpayne@68: httpServer.createContext("/", new TaxHandler(false)); jpayne@68: httpServer.createContext("/tax", new TaxHandler(false)); jpayne@68: httpServer.createContext("/stax", new TaxHandler(true)); jpayne@68: httpServer.createContext("/simpletax", new TaxHandler(true)); jpayne@68: }else{ jpayne@68: httpServer.createContext("/", new SketchHandler()); jpayne@68: } jpayne@68: httpServer.createContext("/sketch", new SketchHandler()); jpayne@68: if(killCode!=null){ jpayne@68: httpServer.createContext("/kill", new KillHandler()); jpayne@68: } jpayne@68: jpayne@68: httpServer.createContext("/help", new HelpHandler()); jpayne@68: httpServer.createContext("/usage", new HelpHandler()); jpayne@68: httpServer.createContext("/stats", new StatsHandler()); jpayne@68: httpServer.createContext("/favicon.ico", new IconHandler()); jpayne@68: jpayne@68: handlerThreads=handlerThreads>0 ? handlerThreads : Tools.max(2, Shared.threads()); jpayne@68: httpServer.setExecutor(java.util.concurrent.Executors.newFixedThreadPool(handlerThreads)); // Creates a multithreaded executor jpayne@68: // httpServer.setExecutor(java.util.concurrent.Executors.newCachedThreadPool()); // Creates a multithreaded executor jpayne@68: // httpServer.setExecutor(null); // Creates a singlethreaded executor jpayne@68: jpayne@68: //Start the server jpayne@68: httpServer.start(); jpayne@68: } jpayne@68: jpayne@68: /** Kill a prior server instance */ jpayne@68: private void killOldInstance(){ jpayne@68: StringNum result=null; jpayne@68: try { jpayne@68: result=ServerTools.sendAndReceive(oldKillCode.getBytes(), oldAddress); jpayne@68: } catch (Exception e) { jpayne@68: // TODO Auto-generated catch block jpayne@68: if(e!=null){e.printStackTrace();} jpayne@68: System.err.println("\nException suppressed; continuing.\n"); jpayne@68: return; jpayne@68: } jpayne@68: if(result==null || result.s==null || !"Success.".equals(result.s)){ jpayne@68: // KillSwitch.kill("Bad kill result: "+result+"\nQuitting.\n"); jpayne@68: System.err.println("Bad kill result: "+result+"\nContinuing.\n"); jpayne@68: } jpayne@68: ServerTools.pause(1000); jpayne@68: } jpayne@68: jpayne@68: /** Iterative wait for server initialization */ jpayne@68: private HttpServer initializeServer(int millis0, int iterations, boolean https){ jpayne@68: HttpServer server=null; jpayne@68: InetSocketAddress isa=new InetSocketAddress(port); jpayne@68: Exception ee=null; jpayne@68: for(int i=0, millis=millis0; i1){//URL mode jpayne@68: return (params[1].equals(killCode)); jpayne@68: }else{//Body mode jpayne@68: try { jpayne@68: String code=ServerTools.receive(t); jpayne@68: return (code!=null && code.equals(killCode)); jpayne@68: } catch (Exception e) { jpayne@68: // TODO Auto-generated catch block jpayne@68: e.printStackTrace(); jpayne@68: } jpayne@68: } jpayne@68: } jpayne@68: return false; jpayne@68: } jpayne@68: } jpayne@68: jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: jpayne@68: /** Listens for sketch comparison requests */ jpayne@68: class SketchHandler implements HttpHandler { jpayne@68: jpayne@68: @Override jpayne@68: public void handle(HttpExchange t) throws IOException { jpayne@68: if(verbose2){outstream.println("Got a request.");} jpayne@68: SketchInstance si=new SketchInstance(t); jpayne@68: if(verbose2){outstream.println("Made si.");} jpayne@68: si.handleInner(); jpayne@68: if(verbose2){outstream.println("Done.");} jpayne@68: } jpayne@68: jpayne@68: private ArrayList loadSketchesFromBody(String body){ jpayne@68: //List of query sketches jpayne@68: ArrayList sketches=null; jpayne@68: jpayne@68: if(body!=null && body.length()>0){ jpayne@68: sketches=searcher.loadSketchesFromString(body); jpayne@68: if(Whitelist.exists()){ jpayne@68: for(Sketch sk : sketches){ jpayne@68: Whitelist.apply(sk); jpayne@68: } jpayne@68: } jpayne@68: } jpayne@68: return sketches; jpayne@68: } jpayne@68: jpayne@68: private ArrayList loadSketchesFromFile(String fname, DisplayParams params){ jpayne@68: //List of query sketches jpayne@68: ArrayList sketches=null; jpayne@68: jpayne@68: SketchTool tool=searcher.tool; jpayne@68: if(tool.minKeyOccuranceCount!=params.minKeyOccuranceCount || params.trackCounts()){ jpayne@68: tool=new SketchTool(SketchObject.targetSketchSize, params); jpayne@68: } jpayne@68: jpayne@68: if(verbose2){System.err.println("Loading sketches from file "+fname);} jpayne@68: sketches=tool.loadSketchesFromFile(fname, (SketchMakerMini)null, maxConcurrentSketchLoadThreads, params.maxReads, params.mode, params, true); jpayne@68: if(verbose2){System.err.println("Loaded "+(sketches==null ? "null" : sketches.size())+" sketches from file "+fname);} jpayne@68: return sketches; jpayne@68: } jpayne@68: jpayne@68: //Created in handle() jpayne@68: private class SketchInstance { jpayne@68: jpayne@68: SketchInstance(HttpExchange t_){ jpayne@68: t=t_; jpayne@68: instanceStartTime=System.nanoTime(); jpayne@68: } jpayne@68: jpayne@68: void handleInner(){ jpayne@68: jpayne@68: if(!hasSketches){ jpayne@68: if(verbose2){System.err.println("No sketches.");} jpayne@68: ServerTools.reply("\nERROR: This server has no sketches loaded.\n" jpayne@68: + "Please download the latest BBTools version to use SendSketch.\n", "text/plain", t, verbose2, 400, true); jpayne@68: return; jpayne@68: } jpayne@68: jpayne@68: String rparam=parseRparamSketch(t); jpayne@68: if(verbose2){System.err.println("Parsed rparam.");} jpayne@68: if(rparam==null){ jpayne@68: returnUsage(instanceStartTime, t); jpayne@68: return; jpayne@68: } jpayne@68: final boolean internal=incrementQueries(t, fileMode, refMode, false, false, false, false, false, false, false, false, -1); jpayne@68: if(verbose2){System.err.println("Incremented queries rparam.");} jpayne@68: jpayne@68: if(verbose2){System.err.println(rparam);} jpayne@68: if(verbose2){System.err.println("fileMode="+fileMode+", refMode="+refMode);} jpayne@68: jpayne@68: if(fileMode && !internal && !allowRemoteFileAccess){ jpayne@68: if(verbose){System.err.println("Illegal file query from "+ServerTools.getClientAddress(t));} jpayne@68: malformedQueries.incrementAndGet(); jpayne@68: // if(verbose){System.err.println("test1");} jpayne@68: // String body=getBody(t);//123 jpayne@68: ServerTools.reply("\nERROR: This server does not allow remote file access. " jpayne@68: + "You may only use the 'local' flag from with the local intranet.\n", "text/plain", t, verbose2, 400, true); jpayne@68: // if(verbose){System.err.println("test2");} jpayne@68: return; jpayne@68: } jpayne@68: jpayne@68: String body=getBody(t); jpayne@68: if(verbose2){System.err.println("Got body.");} jpayne@68: jpayne@68: if(body!=null){bytesIn.addAndGet(body.length());} jpayne@68: jpayne@68: if(verbose2){System.err.println("Found body: "+body);} jpayne@68: if(body!=null && body.length()>0){ jpayne@68: if((fileMode || refMode) && !body.startsWith("##")){ jpayne@68: body="##"+body; jpayne@68: } jpayne@68: try { jpayne@68: params=params.parseDoubleHeader(body); jpayne@68: if(verbose2){System.err.println("Passed parse params.");} jpayne@68: jpayne@68: } catch (Throwable e) { jpayne@68: String s=Tools.toString(e); jpayne@68: ServerTools.reply("\nERROR: \n"+ s, jpayne@68: "text/plain", t, verbose2, 400, true); jpayne@68: return; jpayne@68: } jpayne@68: if(!params.compatible()){ jpayne@68: ServerTools.reply("\nERROR: The sketch is not compatible with this server.\n" jpayne@68: + "Server settings: k="+SketchObject.k+(SketchObject.k2>0 ? ","+SketchObject.k2 : "") jpayne@68: +" amino="+SketchObject.amino+" hash_version="+SketchObject.HASH_VERSION+"\n" jpayne@68: + "You may need to download a newer version of BBTools; this server is running version "+Shared.BBMAP_VERSION_STRING, jpayne@68: "text/plain", t, verbose2, 400, true); jpayne@68: return; jpayne@68: } jpayne@68: } jpayne@68: if(params.trackCounts()){ jpayne@68: depthQueries.incrementAndGet(); jpayne@68: } jpayne@68: jpayne@68: if(verbose2){System.err.println("Parsed params: "+params.toString());} jpayne@68: jpayne@68: //List of query sketches jpayne@68: ArrayList sketches; jpayne@68: jpayne@68: if(fileMode){ jpayne@68: File f=new File(rparam); jpayne@68: if(!f.exists() && !rparam.startsWith("/")){ jpayne@68: String temp="/"+rparam; jpayne@68: f=new File(temp); jpayne@68: if(f.exists()){rparam=temp;} jpayne@68: } jpayne@68: // if(f.exists()){ jpayne@68: // if(f.length()>100000000L){ jpayne@68: // if(params.reads<0){ jpayne@68: // params.reads=200000;//Cap default number of reads at 200000 jpayne@68: // } jpayne@68: // } jpayne@68: // } jpayne@68: sketches=loadSketchesFromFile(rparam, params); jpayne@68: }else if(refMode){ jpayne@68: String[] split=rparam.split(","); jpayne@68: sketches=new ArrayList(split.length); jpayne@68: for(String s : split){ jpayne@68: Sketch sk=findRefSketch(s); jpayne@68: if(sk!=null){sketches.add(sk);} jpayne@68: } jpayne@68: }else{ jpayne@68: sketches=loadSketchesFromBody(body); jpayne@68: } jpayne@68: if(verbose2){System.err.println("Loaded "+sketches.size()+" sketches.");} jpayne@68: jpayne@68: final int numSketches=sketches==null ? 0 : sketches.size(); jpayne@68: if(params.chunkNum<0){ jpayne@68: if(numSketches<2){ jpayne@68: unknownChunkSingle.incrementAndGet(); jpayne@68: }else{ jpayne@68: unknownChunkMulti.incrementAndGet(); jpayne@68: } jpayne@68: }else if(params.chunkNum==0){ jpayne@68: if(numSketches<2){ jpayne@68: firstChunkSingle.incrementAndGet(); jpayne@68: }else{ jpayne@68: firstChunkMulti.incrementAndGet(); jpayne@68: } jpayne@68: }else{ jpayne@68: if(numSketches<2){ jpayne@68: nthChunkSingle.incrementAndGet(); jpayne@68: }else{ jpayne@68: nthChunkMulti.incrementAndGet(); jpayne@68: } jpayne@68: } jpayne@68: bulkCount.addAndGet(numSketches); jpayne@68: jpayne@68: if(params.inputVersion==null){params.inputVersion="unknown";} jpayne@68: synchronized(versionMap){ jpayne@68: StringNum sn=versionMap.get(params.inputVersion); jpayne@68: if(sn==null){versionMap.put(params.inputVersion, new StringNum(params.inputVersion, 1));} jpayne@68: else{sn.increment();} jpayne@68: } jpayne@68: jpayne@68: String response=null; jpayne@68: if(sketches==null || sketches.isEmpty()){ jpayne@68: malformedQueries.incrementAndGet(); jpayne@68: response="Error."; jpayne@68: if(verbose){ jpayne@68: StringBuilder sb=new StringBuilder(); jpayne@68: sb.append("Malformed query from ").append(ServerTools.getClientAddress(t)).append(". body:"); jpayne@68: if(body==null){ jpayne@68: sb.append(" null"); jpayne@68: }else{ jpayne@68: String[] split = body.split("\n"); jpayne@68: sb.append(" ").append(split.length).append(" lines total, displaying ").append(Tools.min(3, split.length)).append('.'); jpayne@68: for(int i=0; i<3 && i1000){s=s.substring(0, 1000)+" [truncated, "+len+" total]";} jpayne@68: sb.append('\n'); jpayne@68: sb.append(s); jpayne@68: } jpayne@68: } jpayne@68: System.err.println(sb); jpayne@68: } jpayne@68: }else{ jpayne@68: if(verbose2){ jpayne@68: System.err.println("Received "+sketches.get(0).name()+", size "+sketches.get(0).keys.length); jpayne@68: System.err.println("params: "+params); jpayne@68: System.err.println("postparsed: "+params.postParsed()); jpayne@68: System.err.println("taxwhitelist: "+params.taxFilterWhite); jpayne@68: } jpayne@68: response=compare(sketches, params); jpayne@68: // searcher.compare(sketches, response, params, maxConcurrentSketchCompareThreads); //This is where it gets stuck if comparing takes too long jpayne@68: if(verbose2){System.err.println("Result: '"+response+"'");} jpayne@68: } jpayne@68: jpayne@68: bytesOut.addAndGet(response.length()); jpayne@68: ServerTools.reply(response, "text/plain", t, verbose2, 200, true); jpayne@68: jpayne@68: final long stopTime=System.nanoTime(); jpayne@68: final long elapsed=stopTime-instanceStartTime; jpayne@68: if(fileMode){ jpayne@68: timeMeasurementsLocal.incrementAndGet(); jpayne@68: elapsedTimeLocal.addAndGet(elapsed); jpayne@68: lastTimeLocal.set(elapsed); jpayne@68: }else if(refMode){ jpayne@68: timeMeasurementsReference.incrementAndGet(); jpayne@68: elapsedTimeReference.addAndGet(elapsed); jpayne@68: lastTimeReference.set(elapsed); jpayne@68: }else{ jpayne@68: timeMeasurementsRemote.incrementAndGet(); jpayne@68: elapsedTimeRemote.addAndGet(elapsed); jpayne@68: lastTimeRemote.set(elapsed); jpayne@68: jpayne@68: queryCounts.incrementAndGet(Tools.min(numSketches, queryCounts.length()-1)); jpayne@68: timesByCount.addAndGet(Tools.min(numSketches, queryCounts.length()-1), elapsed); jpayne@68: } jpayne@68: } jpayne@68: jpayne@68: private String parseRparamSketch(HttpExchange t){ jpayne@68: //Parse the query from the URL jpayne@68: String rparam=getRParam(t, false); jpayne@68: if(rparam!=null){bytesIn.addAndGet(rparam.length());} jpayne@68: jpayne@68: if(rparam.length()<1 || rparam.equalsIgnoreCase("help") || rparam.equalsIgnoreCase("usage") || rparam.equalsIgnoreCase("help/") || rparam.equalsIgnoreCase("usage/")){ jpayne@68: return null; jpayne@68: } jpayne@68: jpayne@68: if(rparam.startsWith("sketch/")){rparam=rparam.substring(7);} jpayne@68: else if(rparam.equals("sketch")){rparam="";} jpayne@68: while(rparam.startsWith("/")){rparam=rparam.substring(1);} jpayne@68: jpayne@68: //Toggle between local files and sketch transmission jpayne@68: jpayne@68: if(rparam.length()<2){ jpayne@68: params=SketchObject.defaultParams; jpayne@68: }else{ jpayne@68: params=SketchObject.defaultParams.clone(); jpayne@68: String[] args=rparam.split("/"); jpayne@68: int trimmed=0; jpayne@68: for(int i=0; i0){ jpayne@68: String[] split=arg.split("="); jpayne@68: String a=split[0].toLowerCase(); jpayne@68: String b=split.length>1 ? split[1] : null; jpayne@68: jpayne@68: if(a.equals("file")){ jpayne@68: fileMode=true; jpayne@68: trimmed+=5; jpayne@68: break; jpayne@68: }else if(a.equals("ref") || a.equals("taxid") || a.equals("tid")){ jpayne@68: refMode=true; jpayne@68: trimmed+=4; jpayne@68: break; jpayne@68: }else if(params.parse(arg, a, b)){ jpayne@68: trimmed+=arg.length()+1; jpayne@68: }else{ jpayne@68: assert(false) : "Bad argument:'"+arg+"'"+"\n"+Arrays.toString(args)+"\n"+rparam; jpayne@68: } jpayne@68: } jpayne@68: } jpayne@68: params.postParse(true, true); jpayne@68: // System.err.println("Trimmed="+trimmed+", rparam="+rparam); jpayne@68: if(trimmed>0){ jpayne@68: rparam=rparam.substring(Tools.min(trimmed, rparam.length())); jpayne@68: } jpayne@68: // System.err.println("rparam="+rparam); jpayne@68: } jpayne@68: jpayne@68: if(verbose2){ jpayne@68: System.err.println(rparam); jpayne@68: System.err.println("rparam.startsWith(\"file/\"):"+rparam.startsWith("file/")); jpayne@68: } jpayne@68: jpayne@68: return rparam; jpayne@68: } jpayne@68: jpayne@68: private final HttpExchange t; jpayne@68: private DisplayParams params; jpayne@68: private final long instanceStartTime; jpayne@68: private boolean fileMode=false; jpayne@68: private boolean refMode=false; jpayne@68: } jpayne@68: jpayne@68: } jpayne@68: jpayne@68: private Sketch findRefSketch(String s){ jpayne@68: assert(s!=null); jpayne@68: if(s==null || s.length()<1){return null;} jpayne@68: int tid=-1; jpayne@68: if(Tools.isDigit(s.charAt(0))){tid=Integer.parseInt(s);} jpayne@68: else{ jpayne@68: TaxNode tn=getTaxNodeByName(s); jpayne@68: tid=tn==null ? -1 : tn.id; jpayne@68: } jpayne@68: Sketch sk=tid<0 ? null : searcher.findReferenceSketch(tid); jpayne@68: if(sk!=null){sk=(Sketch)sk.clone();} jpayne@68: return sk; jpayne@68: } jpayne@68: jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: jpayne@68: /** Handles taxonomy lookups */ jpayne@68: class TaxHandler implements HttpHandler { jpayne@68: jpayne@68: public TaxHandler(boolean skipNonCanonical_){ jpayne@68: skipNonCanonical=skipNonCanonical_; jpayne@68: } jpayne@68: jpayne@68: @Override jpayne@68: public void handle(HttpExchange t) throws IOException { jpayne@68: if(verbose2){System.err.println("Tax handler");} jpayne@68: final long startTime=System.nanoTime(); jpayne@68: jpayne@68: if(sketchOnly){ jpayne@68: ServerTools.reply("\nERROR: This server is tunning in sketch mode and should not be used for taxonomic lookups.\n" jpayne@68: + "The taxonomy server is at "+Shared.taxServer()+"\n", "text/plain", t, verbose2, 400, true); jpayne@68: return; jpayne@68: } jpayne@68: jpayne@68: //Parse the query from the URL jpayne@68: String rparam=getRParam(t, true); jpayne@68: jpayne@68: jpayne@68: boolean simple=skipNonCanonical; jpayne@68: jpayne@68: {//Legacy support for old style of invoking simple jpayne@68: if(rparam.startsWith("simpletax/")){rparam=rparam.substring(7); simple=true;} jpayne@68: else if(rparam.startsWith("stax/")){rparam=rparam.substring(5); simple=true;} jpayne@68: else if(rparam.startsWith("tax/")){rparam=rparam.substring(4);} jpayne@68: else if(rparam.equals("simpletax") || rparam.equals("stax")){rparam=""; simple=true;} jpayne@68: else if(rparam.equals("tax")){rparam="";} jpayne@68: } jpayne@68: while(rparam.startsWith("/")){rparam=rparam.substring(1);} jpayne@68: if(rparam.length()<1 || rparam.equalsIgnoreCase("help") || rparam.equalsIgnoreCase("usage")){ jpayne@68: returnUsage(startTime, t); jpayne@68: return; jpayne@68: } jpayne@68: jpayne@68: String[] params = rparam.split("/"); jpayne@68: if(verbose2){System.err.println(Arrays.toString(params));} jpayne@68: jpayne@68: final String response=toResponse(simple, params, t); jpayne@68: final String type=response.startsWith("{") ? "application/json" : "text/plain"; jpayne@68: jpayne@68: ServerTools.reply(response, type, t, verbose2, 200, true); jpayne@68: jpayne@68: final long stopTime=System.nanoTime(); jpayne@68: final long elapsed=stopTime-startTime; jpayne@68: if(response.startsWith("Welcome to ")){ jpayne@68: timeMeasurementsUsage.incrementAndGet(); jpayne@68: elapsedTimeUsage.addAndGet(elapsed); jpayne@68: lastTimeUsage.set(elapsed); jpayne@68: }else{ jpayne@68: timeMeasurementsRemote.incrementAndGet(); jpayne@68: elapsedTimeRemote.addAndGet(elapsed); jpayne@68: lastTimeRemote.set(elapsed); jpayne@68: } jpayne@68: } jpayne@68: jpayne@68: //TODO: Integrate something like this to improve parsing jpayne@68: // String parse(String rparam){ jpayne@68: // jpayne@68: // if(rparam.length()<2){return rparam;} jpayne@68: // jpayne@68: // String[] args=rparam.split("/"); jpayne@68: // int trimmed=0; jpayne@68: // for(int i=0; i0){ jpayne@68: // String[] split=arg.split("="); jpayne@68: // String a=split[0].toLowerCase(); jpayne@68: // String b=split.length>1 ? split[1] : null; jpayne@68: // jpayne@68: // if(a.equals("file")){ jpayne@68: // fileMode=true; jpayne@68: // trimmed+=5; jpayne@68: // break; jpayne@68: // }else if(params.parse(arg, a, b)){ jpayne@68: // trimmed+=arg.length()+1; jpayne@68: // }else{ jpayne@68: // assert(false) : "Bad argument:'"+arg+"'"+"\n"+Arrays.toString(args)+"\n"+rparam; jpayne@68: // } jpayne@68: // } jpayne@68: // } jpayne@68: // params.postParse(true); jpayne@68: // // System.err.println("Trimmed="+trimmed+", rparam="+rparam); jpayne@68: // if(trimmed>0){ jpayne@68: // rparam=rparam.substring(Tools.min(trimmed, rparam.length())); jpayne@68: // } jpayne@68: // } jpayne@68: jpayne@68: /** Only print nodes at canonical tax levels */ jpayne@68: public final boolean skipNonCanonical; jpayne@68: } jpayne@68: jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: /*---------------- Helpers ----------------*/ jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: jpayne@68: static String getBody(HttpExchange t){ jpayne@68: if(verbose2){System.err.println("getBody");} jpayne@68: InputStream is=t.getRequestBody(); jpayne@68: String s=ServerTools.readStream(is); jpayne@68: return s; jpayne@68: } jpayne@68: jpayne@68: /** Parse the query from the URL */ jpayne@68: static String getRParam(HttpExchange t, boolean allowPost){ jpayne@68: if(verbose2){System.err.println("getRParam");} jpayne@68: String rparam = t.getRequestURI().toString(); jpayne@68: jpayne@68: //Trim leading slashes jpayne@68: while(rparam.startsWith("/")){ jpayne@68: rparam = rparam.substring(1); jpayne@68: } jpayne@68: jpayne@68: //Trim trailing slashes jpayne@68: while(rparam.endsWith("/")){ jpayne@68: rparam = rparam.substring(0, rparam.length()-1); jpayne@68: } jpayne@68: jpayne@68: if(allowPost && ("$POST".equalsIgnoreCase(rparam) || "POST".equalsIgnoreCase(rparam))){ jpayne@68: String body=getBody(t); jpayne@68: rparam=body; jpayne@68: } jpayne@68: jpayne@68: if(verbose){System.err.println(rparam==null || rparam.trim().length()<1 ? "usage" : rparam+"\t"+System.currentTimeMillis());} jpayne@68: return rparam; jpayne@68: } jpayne@68: jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: /*---------------- Taxonomy Formatting ----------------*/ jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: jpayne@68: /** All tax queries enter here from the handler */ jpayne@68: String toResponse(boolean simple, String[] params, HttpExchange t){ jpayne@68: if(verbose2){System.err.println("toResponse");} jpayne@68: jpayne@68: boolean printNumChildren=false; jpayne@68: boolean printChildren=false; jpayne@68: boolean printPath=false; jpayne@68: boolean printSize=false; jpayne@68: boolean printRange=false; jpayne@68: boolean silvaHeader=false; jpayne@68: boolean plaintext=false, semicolon=false, path=false; jpayne@68: boolean mononomial=false; jpayne@68: boolean ancestor=false; jpayne@68: int source=SOURCE_REFSEQ; jpayne@68: jpayne@68: ArrayList newParams=new ArrayList(params.length); jpayne@68: for(int i=0; i3){ jpayne@68: if(logUsage){System.err.println("usage");} jpayne@68: return USAGE(USAGE); jpayne@68: } jpayne@68: jpayne@68: final String query=params[params.length-1]; jpayne@68: final String[] names=query.split(","); jpayne@68: if(names==null){return USAGE(USAGE);} jpayne@68: jpayne@68: if(names==null || names.length<2){ jpayne@68: firstChunkSingle.incrementAndGet(); jpayne@68: }else{ jpayne@68: firstChunkMulti.incrementAndGet(); jpayne@68: } jpayne@68: bulkCount.addAndGet(names.length); jpayne@68: // System.err.println(params[2]+", "+ancestor); jpayne@68: jpayne@68: //Raw query type code jpayne@68: final int type; jpayne@68: //Type code excluding formatting jpayne@68: final int type2; jpayne@68: { jpayne@68: String typeS=params[0]; jpayne@68: Integer value=typeMap.get(typeS); jpayne@68: if(value==null){ jpayne@68: if(typeS.equalsIgnoreCase("advice")){ jpayne@68: return TAX_ADVICE; jpayne@68: }else{ jpayne@68: return "{\"error\": \"Bad type ("+typeS+"); should be gi, taxid, or name.\"}"; jpayne@68: } jpayne@68: } jpayne@68: int x=value.intValue(); jpayne@68: if((x&15)==HEADER && silvaHeader){x=SILVAHEADER;} jpayne@68: type=x; jpayne@68: type2=x&15; jpayne@68: if(type2==IMG){source=SOURCE_IMG;} jpayne@68: } jpayne@68: jpayne@68: plaintext=(type>=PT_BIT || plaintext); jpayne@68: semicolon=(type>=SC_BIT || semicolon); jpayne@68: path=(type>=PA_BIT || path); jpayne@68: if(semicolon || path){plaintext=false;} jpayne@68: if(path){semicolon=false;} jpayne@68: jpayne@68: final boolean internal=incrementQueries(t, false, false, simple, ancestor, jpayne@68: plaintext, semicolon, path, printChildren, printPath, printSize, type); //Ignores usage information. jpayne@68: jpayne@68: // if(type2==GI){//123 jpayne@68: // return "{\"error\": \"GI number support is temporarily suspended due to conflicts in NCBI databases. " jpayne@68: // + "It may come back split into nucleotide and protein GI numbers, which currently are not exclusive.\"}"; jpayne@68: // } jpayne@68: jpayne@68: if(!internal && !allowRemoteFileAccess){ jpayne@68: path=printPath=false; jpayne@68: } jpayne@68: jpayne@68: if(verbose2){System.err.println("Type: "+type);} jpayne@68: if(type2==NAME || type2==HEADER || type2==SILVAHEADER){ jpayne@68: for(int i=0; i")){names[i]=names[i].substring(1);} jpayne@68: } jpayne@68: } jpayne@68: if(verbose2){System.err.println("Revised: "+Arrays.toString(names));} jpayne@68: } jpayne@68: jpayne@68: if(ancestor){ jpayne@68: if(verbose2){System.err.println("toAncestor: "+Arrays.toString(names));} jpayne@68: return toAncestor(type, names, plaintext, semicolon, path, query, simple, !simple, printNumChildren, printChildren, printPath, printSize, printRange, mononomial, source); jpayne@68: } jpayne@68: jpayne@68: if(semicolon){ jpayne@68: return toSemicolon(type, names, simple, mononomial); jpayne@68: }else if(plaintext){ jpayne@68: return toText(type, names); jpayne@68: }else if(path){ jpayne@68: return toPath(type, names, source); jpayne@68: } jpayne@68: jpayne@68: JsonObject j=new JsonObject(); jpayne@68: for(String name : names){ jpayne@68: j.add(name, toJson(type, name, simple, !simple, printNumChildren, printChildren, jpayne@68: printPath, printSize, printRange, mononomial, source)); jpayne@68: } jpayne@68: return j.toString(); jpayne@68: } jpayne@68: jpayne@68: /** Look up common ancestor of terms */ jpayne@68: String toAncestor(final int type, final String[] names, boolean plaintext, boolean semicolon, boolean path, jpayne@68: String query, final boolean skipNonCanonical, boolean originalLevel, boolean printNumChildren, jpayne@68: boolean printChildren, boolean printPath, boolean printSize, boolean printRange, boolean mononomial, jpayne@68: int source){ jpayne@68: IntList ilist=toIntList(type, names); jpayne@68: int id=FindAncestor.findAncestor(tree, ilist); jpayne@68: TaxNode tn=(id>-1 ? tree.getNode(id) : null); jpayne@68: if(tn==null){ jpayne@68: return new JsonObject("error","Not found.").toString(query); jpayne@68: } jpayne@68: if(semicolon){ jpayne@68: return tree.toSemicolon(tn, skipNonCanonical, mononomial); jpayne@68: }else if(plaintext){ jpayne@68: return ""+id; jpayne@68: }else if(path){ jpayne@68: return toPath(tn, source); jpayne@68: } jpayne@68: jpayne@68: JsonObject j=new JsonObject(); jpayne@68: // j.add("name", mononomial ? tree.mononomial(tn) : tn.name); jpayne@68: j.add("name", tn.name); jpayne@68: if(mononomial || true){ jpayne@68: String mono=tree.mononomial(tn); jpayne@68: if(tn.name!=mono){j.add("mononomial", mono);} jpayne@68: } jpayne@68: j.add("tax_id", tn.id); jpayne@68: if(printNumChildren){j.add("num_children", tn.numChildren);} jpayne@68: if(printPath){j.add("path", toPath(tn, source));} jpayne@68: if(printSize){ jpayne@68: j.add("size", tree.toSize(tn)); jpayne@68: j.add("cumulative_size", tree.toSizeC(tn)); jpayne@68: j.add("seqs", tree.toSeqs(tn)); jpayne@68: j.add("cumulative_seqs", tree.toSeqsC(tn)); jpayne@68: j.add("cumulative_nodes", tree.toNodes(tn)); jpayne@68: } jpayne@68: j.add("level", tn.levelStringExtended(originalLevel)); jpayne@68: if(tn.levelExtended<1 && printRange){ jpayne@68: j.add("maxDescendent", TaxTree.levelToStringExtended(tn.maxChildLevelExtended)); jpayne@68: j.add("minAncestor", TaxTree.levelToStringExtended(tn.minParentLevelExtended)); jpayne@68: } jpayne@68: // if(printChildren){j.add(getChildren(id, originalLevel, printRange));} jpayne@68: while(tn!=null && tn.levelExtended!=TaxTree.LIFE_E && tn.id!=TaxTree.CELLULAR_ORGANISMS_ID){ jpayne@68: if(!skipNonCanonical || tn.isSimple()){ jpayne@68: j.addAndRename(tn.levelStringExtended(originalLevel), toJson(tn, originalLevel, printNumChildren, printChildren, printPath, printSize, printRange, mononomial, source, -1)); jpayne@68: } jpayne@68: if(tn.pid==tn.id){break;} jpayne@68: tn=tree.getNode(tn.pid); jpayne@68: } jpayne@68: return j.toString(); jpayne@68: } jpayne@68: jpayne@68: JsonObject getChildren(final int id, boolean originalLevel, boolean printRange, boolean mononomial){ jpayne@68: TaxNode x=tree.getNode(id); jpayne@68: if(x==null || x.numChildren==0){return null;} jpayne@68: ArrayList list=tree.getChildren(x); jpayne@68: return makeChildrenObject(list, originalLevel, printRange, mononomial); jpayne@68: } jpayne@68: jpayne@68: JsonObject makeChildrenObject(ArrayList list, boolean originalLevel, boolean printRange, boolean mononomial){ jpayne@68: if(list==null || list.isEmpty()){return null;} jpayne@68: JsonObject j=new JsonObject(); jpayne@68: for(TaxNode tn : list){ jpayne@68: JsonObject child=new JsonObject(); jpayne@68: // child.add("name", mononomial ? tree.mononomial(tn) : tn.name); jpayne@68: child.add("name", tn.name); jpayne@68: if(mononomial || true){ jpayne@68: String mono=tree.mononomial(tn); jpayne@68: if(tn.name!=mono){child.add("mononomial", mono);} jpayne@68: } jpayne@68: child.add("tax_id", tn.id); jpayne@68: child.add("num_children", tn.numChildren); jpayne@68: child.add("level", tn.levelStringExtended(originalLevel)); jpayne@68: if(tn.levelExtended<1 && printRange){ jpayne@68: child.add("maxDescendent", TaxTree.levelToStringExtended(tn.maxChildLevelExtended)); jpayne@68: child.add("minAncestor", TaxTree.levelToStringExtended(tn.minParentLevelExtended)); jpayne@68: } jpayne@68: j.add(tn.id+"", child); jpayne@68: } jpayne@68: return j; jpayne@68: } jpayne@68: jpayne@68: /** Format a reply as plaintext, comma-delimited, TaxID only */ jpayne@68: String toText(final int type, final String[] names){ jpayne@68: jpayne@68: StringBuilder sb=new StringBuilder(); jpayne@68: String comma=""; jpayne@68: jpayne@68: int type2=type&15; jpayne@68: if(type2==GI){ jpayne@68: for(String name : names){ jpayne@68: sb.append(comma); jpayne@68: TaxNode tn=getTaxNodeGi(Long.parseLong(name)); jpayne@68: if(tn==null){sb.append("-1");} jpayne@68: else{sb.append(tn.id);} jpayne@68: comma=","; jpayne@68: } jpayne@68: }else if(type2==NAME){ jpayne@68: for(String name : names){ jpayne@68: sb.append(comma); jpayne@68: TaxNode tn=getTaxNodeByName(name); jpayne@68: if(tn==null){sb.append("-1");} jpayne@68: else{sb.append(tn.id);} jpayne@68: comma=","; jpayne@68: } jpayne@68: }else if(type2==TAXID){ jpayne@68: for(String name : names){ jpayne@68: sb.append(comma); jpayne@68: TaxNode tn=getTaxNodeTaxid(Integer.parseInt(name)); jpayne@68: if(tn==null){sb.append("-1");} jpayne@68: else{sb.append(tn.id);} jpayne@68: comma=","; jpayne@68: } jpayne@68: }else if(type2==ACCESSION){ jpayne@68: for(String name : names){ jpayne@68: sb.append(comma); jpayne@68: int ncbi=accessionToTaxid(name); jpayne@68: sb.append(ncbi); jpayne@68: comma=","; jpayne@68: } jpayne@68: }else if(type2==HEADER || type2==SILVAHEADER){ jpayne@68: for(String name : names){ jpayne@68: sb.append(comma); jpayne@68: TaxNode tn=getTaxNodeHeader(name, type2==SILVAHEADER); jpayne@68: if(tn==null){sb.append("-1");} jpayne@68: else{sb.append(tn.id);} jpayne@68: comma=","; jpayne@68: } jpayne@68: }else if(type2==IMG){ jpayne@68: for(String name : names){ jpayne@68: sb.append(comma); jpayne@68: int ncbi=TaxTree.imgToTaxid(Long.parseLong(name)); jpayne@68: sb.append(ncbi); jpayne@68: comma=","; jpayne@68: } jpayne@68: }else{ jpayne@68: return "Bad type; should be pt_gi or pt_name; e.g. /pt_gi/1234"; jpayne@68: } jpayne@68: jpayne@68: return sb.toString(); jpayne@68: } jpayne@68: jpayne@68: private TaxNode toNode(final int type, final String name){ jpayne@68: int type2=type&15; jpayne@68: final TaxNode tn; jpayne@68: if(type2==GI){ jpayne@68: tn=getTaxNodeGi(Long.parseLong(name)); jpayne@68: }else if(type2==NAME){ jpayne@68: tn=getTaxNodeByName(name); jpayne@68: }else if(type2==TAXID){ jpayne@68: tn=getTaxNodeTaxid(Integer.parseInt(name)); jpayne@68: }else if(type2==ACCESSION){ jpayne@68: int ncbi=accessionToTaxid(name); jpayne@68: tn=(ncbi<0 ? null : tree.getNode(ncbi)); jpayne@68: }else if(type2==HEADER || type2==SILVAHEADER){ jpayne@68: tn=getTaxNodeHeader(name, type2==SILVAHEADER); jpayne@68: }else if(type2==IMG){ jpayne@68: int ncbi=TaxTree.imgToTaxid(Long.parseLong(name)); jpayne@68: tn=(ncbi<0 ? null : tree.getNode(ncbi)); jpayne@68: }else{ jpayne@68: tn=null; jpayne@68: } jpayne@68: return tn; jpayne@68: } jpayne@68: jpayne@68: /** Format a reply as paths, comma-delimited*/ jpayne@68: String toPath(final int type, final String[] names, final int source){ jpayne@68: jpayne@68: StringBuilder sb=new StringBuilder(); jpayne@68: String comma=""; jpayne@68: jpayne@68: int type2=type&15; jpayne@68: jpayne@68: for(String name : names){ jpayne@68: sb.append(comma); jpayne@68: if(type2==IMG){ jpayne@68: sb.append(toPathIMG(Long.parseLong(name))); jpayne@68: }else{ jpayne@68: TaxNode tn=toNode(type2, name); jpayne@68: sb.append(toPath(tn, source)); jpayne@68: } jpayne@68: comma=","; jpayne@68: } jpayne@68: jpayne@68: return sb.toString(); jpayne@68: } jpayne@68: jpayne@68: /** Format a reply as plaintext, semicolon-delimited, full lineage */ jpayne@68: String toSemicolon(final int type, final String[] names, boolean skipNonCanonical, boolean mononomial){ jpayne@68: jpayne@68: StringBuilder sb=new StringBuilder(); jpayne@68: String comma=""; jpayne@68: jpayne@68: int type2=type&15; jpayne@68: if(type2==GI){ jpayne@68: for(String name : names){ jpayne@68: sb.append(comma); jpayne@68: TaxNode tn=getTaxNodeGi(Long.parseLong(name)); jpayne@68: if(tn==null){sb.append("Not found");} jpayne@68: else{sb.append(tree.toSemicolon(tn, skipNonCanonical, mononomial));} jpayne@68: comma=","; jpayne@68: } jpayne@68: }else if(type2==NAME){ jpayne@68: for(String name : names){ jpayne@68: sb.append(comma); jpayne@68: TaxNode tn=getTaxNodeByName(name); jpayne@68: if(tn==null){sb.append("Not found");} jpayne@68: else{sb.append(tree.toSemicolon(tn, skipNonCanonical, mononomial));} jpayne@68: comma=","; jpayne@68: } jpayne@68: }else if(type2==TAXID){ jpayne@68: for(String name : names){ jpayne@68: sb.append(comma); jpayne@68: TaxNode tn=getTaxNodeTaxid(Integer.parseInt(name)); jpayne@68: // if(verbose2){outstream.println("name="+name+", tn="+tn);} jpayne@68: if(tn==null){sb.append("Not found");} jpayne@68: else{sb.append(tree.toSemicolon(tn, skipNonCanonical, mononomial));} jpayne@68: comma=","; jpayne@68: } jpayne@68: }else if(type2==ACCESSION){ jpayne@68: for(String name : names){ jpayne@68: sb.append(comma); jpayne@68: final int tid=accessionToTaxid(name); jpayne@68: TaxNode tn=tree.getNode(tid, true); jpayne@68: if(tn==null){sb.append("Not found");} jpayne@68: else{sb.append(tree.toSemicolon(tn, skipNonCanonical, mononomial));} jpayne@68: comma=","; jpayne@68: } jpayne@68: }else if(type2==HEADER || type2==SILVAHEADER){ jpayne@68: for(String name : names){ jpayne@68: sb.append(comma); jpayne@68: TaxNode tn=getTaxNodeHeader(name, type2==SILVAHEADER); jpayne@68: if(tn==null){sb.append("Not found");} jpayne@68: else{sb.append(tree.toSemicolon(tn, skipNonCanonical, mononomial));} jpayne@68: comma=","; jpayne@68: } jpayne@68: }else if(type2==IMG){ jpayne@68: for(String name : names){ jpayne@68: sb.append(comma); jpayne@68: final int tid=TaxTree.imgToTaxid(Long.parseLong(name)); jpayne@68: TaxNode tn=tree.getNode(tid, true); jpayne@68: if(tn==null){sb.append("Not found");} jpayne@68: else{sb.append(tree.toSemicolon(tn, skipNonCanonical, mononomial));} jpayne@68: comma=","; jpayne@68: } jpayne@68: }else{ jpayne@68: return "Bad type; should be sc_gi or sc_name; e.g. /sc_gi/1234"; jpayne@68: } jpayne@68: jpayne@68: // if(verbose2){outstream.println("In toSemicolon; type="+type+", type2="+type2+", made "+sb);} jpayne@68: jpayne@68: return sb.toString(); jpayne@68: } jpayne@68: jpayne@68: /** Create a JsonObject from a String, including full lineage */ jpayne@68: JsonObject toJson(final int type, final String name, boolean skipNonCanonical, boolean originalLevel, jpayne@68: boolean printNumChildren, boolean printChildren, boolean printPath, boolean printSize, jpayne@68: boolean printRange, boolean mononomial, int source){ jpayne@68: final TaxNode tn0; jpayne@68: TaxNode tn; jpayne@68: jpayne@68: long img=-1; jpayne@68: if(type==GI){ jpayne@68: tn0=getTaxNodeGi(Long.parseLong(name)); jpayne@68: }else if(type==NAME){ jpayne@68: tn0=getTaxNodeByName(name); jpayne@68: }else if(type==TAXID){ jpayne@68: tn0=getTaxNodeTaxid(Integer.parseInt(name)); jpayne@68: }else if(type==ACCESSION){ jpayne@68: int ncbi=accessionToTaxid(name); jpayne@68: tn0=(ncbi>=0 ? tree.getNode(ncbi) : null); jpayne@68: }else if(type==HEADER || type==SILVAHEADER){ jpayne@68: tn0=getTaxNodeHeader(name, type==SILVAHEADER); jpayne@68: }else if(type==IMG){ jpayne@68: img=Long.parseLong(name); jpayne@68: final int tid=TaxTree.imgToTaxid(img); jpayne@68: tn0=tree.getNode(tid, true); jpayne@68: }else{ jpayne@68: JsonObject j=new JsonObject("error","Bad type; should be gi, taxid, or name; e.g. /name/homo_sapiens"); jpayne@68: j.add("name", name); jpayne@68: j.add("type", type); jpayne@68: return j; jpayne@68: } jpayne@68: tn=tn0; jpayne@68: if(verbose2){System.err.println("Got node: "+tn);} jpayne@68: jpayne@68: if(tn!=null){ jpayne@68: JsonObject j=new JsonObject(); jpayne@68: // j.add("name", mononomial ? tree.mononomial(tn) : tn.name); jpayne@68: j.add("name", tn.name); jpayne@68: if(mononomial || true){ jpayne@68: String mono=tree.mononomial(tn); jpayne@68: if(tn.name!=mono){j.add("mononomial", mono);} jpayne@68: } jpayne@68: j.add("tax_id", tn.id); jpayne@68: if(printNumChildren){j.add("num_children", tn.numChildren);} jpayne@68: if(printPath){j.add("path", type==IMG ? toPathIMG(img) : toPath(tn, source));} jpayne@68: if(printSize){ jpayne@68: j.add("size", tree.toSize(tn)); jpayne@68: j.add("cumulative_size", tree.toSizeC(tn)); jpayne@68: j.add("seqs", tree.toSeqs(tn)); jpayne@68: j.add("cumulative_seqs", tree.toSeqsC(tn)); jpayne@68: j.add("cumulative_nodes", tree.toNodes(tn)); jpayne@68: } jpayne@68: j.add("level", tn.levelStringExtended(originalLevel)); jpayne@68: if(tn.levelExtended<1 && printRange){ jpayne@68: j.add("maxDescendent", TaxTree.levelToStringExtended(tn.maxChildLevelExtended)); jpayne@68: j.add("minAncestor", TaxTree.levelToStringExtended(tn.minParentLevelExtended)); jpayne@68: } jpayne@68: if(printChildren && (tn.id==tn.pid || tn.id==TaxTree.CELLULAR_ORGANISMS_ID)){j.add("children", getChildren(tn.id, originalLevel, printRange, mononomial));} jpayne@68: while(tn!=null && tn.levelExtended!=TaxTree.LIFE_E && tn.id!=TaxTree.CELLULAR_ORGANISMS_ID){ jpayne@68: // System.err.println(tn+", "+(!skipNonCanonical)+", "+tn.isSimple()); jpayne@68: if(!skipNonCanonical || tn.isSimple()){ jpayne@68: j.addAndRename(tn.levelStringExtended(originalLevel), toJson(tn, originalLevel, printNumChildren, printChildren, printPath && tn==tn0, printSize, printRange, mononomial, source, img)); jpayne@68: // System.err.println(j); jpayne@68: } jpayne@68: if(tn.pid==tn.id){break;} jpayne@68: tn=tree.getNode(tn.pid); jpayne@68: } jpayne@68: return j; jpayne@68: } jpayne@68: { jpayne@68: JsonObject j=new JsonObject("error","Not found."); jpayne@68: j.add("name", name); jpayne@68: j.add("type", type); jpayne@68: return j; jpayne@68: } jpayne@68: } jpayne@68: jpayne@68: /** Create a JsonObject from a TaxNode, at that level only */ jpayne@68: JsonObject toJson(TaxNode tn, boolean originalLevel, boolean printNumChildren, jpayne@68: boolean printChildren, boolean printPath, boolean printSize, boolean printRange, jpayne@68: boolean mononomial, int source, long img){ jpayne@68: JsonObject j=new JsonObject(); jpayne@68: // j.add("name", mononomial ? tree.mononomial(tn) : tn.name); jpayne@68: j.add("name", tn.name); jpayne@68: if(mononomial || true){ jpayne@68: String mono=tree.mononomial(tn); jpayne@68: if(tn.name!=mono){j.add("mononomial", mono);} jpayne@68: } jpayne@68: j.add("tax_id", tn.id); jpayne@68: if(printNumChildren){j.add("num_children", tn.numChildren);} jpayne@68: if(printPath){j.add("path", source==SOURCE_IMG ? toPathIMG(img) : toPath(tn, source));} jpayne@68: if(printSize){ jpayne@68: j.add("size", tree.toSize(tn)); jpayne@68: j.add("cumulative_size", tree.toSizeC(tn)); jpayne@68: j.add("seqs", tree.toSeqs(tn)); jpayne@68: j.add("cumulative_seqs", tree.toSeqsC(tn)); jpayne@68: j.add("cumulative_nodes", tree.toNodes(tn)); jpayne@68: } jpayne@68: if(tn.levelExtended<1 && printRange){ jpayne@68: j.add("maxDescendent", TaxTree.levelToStringExtended(tn.maxChildLevelExtended)); jpayne@68: j.add("minAncestor", TaxTree.levelToStringExtended(tn.minParentLevelExtended)); jpayne@68: } jpayne@68: if(printChildren){ jpayne@68: JsonObject children=getChildren(tn.id, originalLevel, printRange, mononomial); jpayne@68: if(children!=null){j.add("children", children);} jpayne@68: } jpayne@68: return j; jpayne@68: } jpayne@68: jpayne@68: String toPath(TaxNode tn, int source){ jpayne@68: if(tn==null){return "null";} jpayne@68: String path; jpayne@68: if(source==SOURCE_REFSEQ){ jpayne@68: path=tree.toDir(tn, basePath)+"refseq_"+tn.id+".fa.gz"; jpayne@68: }else if(source==SOURCE_SILVA){ jpayne@68: path=tree.toDir(tn, basePath)+"silva_"+tn.id+".fa.gz"; jpayne@68: }else if(source==SOURCE_IMG){ jpayne@68: assert(false); jpayne@68: path="null"; jpayne@68: }else{ jpayne@68: assert(false); jpayne@68: path="null"; jpayne@68: } jpayne@68: if(!path.equals("null") && !new File(path).exists()){path="null";} jpayne@68: return path; jpayne@68: } jpayne@68: jpayne@68: String toPathIMG(long imgID){ jpayne@68: String path="/global/dna/projectdirs/microbial/img_web_data/taxon.fna/"+imgID+".fna"; jpayne@68: if(!new File(path).exists()){path="null";} jpayne@68: return path; jpayne@68: } jpayne@68: jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: /*---------------- Taxonomy Lookup ----------------*/ jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: jpayne@68: /** Convert a list of terms to a list of TaxIDs */ jpayne@68: IntList toIntList(final int type, final String[] names){ jpayne@68: IntList list=new IntList(names.length); jpayne@68: int type2=type&15; jpayne@68: if(type2==GI){ jpayne@68: for(String name : names){ jpayne@68: TaxNode tn=getTaxNodeGi(Long.parseLong(name)); jpayne@68: if(tn!=null){list.add(tn.id);} jpayne@68: else{notFound.incrementAndGet();} jpayne@68: } jpayne@68: }else if(type2==NAME){ jpayne@68: for(String name : names){ jpayne@68: TaxNode tn=getTaxNodeByName(name); jpayne@68: if(tn!=null){list.add(tn.id);} jpayne@68: else{notFound.incrementAndGet();} jpayne@68: } jpayne@68: }else if(type2==TAXID){ jpayne@68: for(String name : names){ jpayne@68: TaxNode tn=getTaxNodeTaxid(Integer.parseInt(name)); jpayne@68: if(tn!=null){list.add(tn.id);} jpayne@68: else{notFound.incrementAndGet();} jpayne@68: } jpayne@68: }else if(type2==ACCESSION){ jpayne@68: for(String name : names){ jpayne@68: int ncbi=accessionToTaxid(name); jpayne@68: if(ncbi>=0){list.add(ncbi);} jpayne@68: else{notFound.incrementAndGet();} jpayne@68: } jpayne@68: }else if(type2==IMG){ jpayne@68: for(String name : names){ jpayne@68: final int tid=TaxTree.imgToTaxid(Long.parseLong(name)); jpayne@68: if(tid>=0){list.add(tid);} jpayne@68: else{notFound.incrementAndGet();} jpayne@68: } jpayne@68: }else{ jpayne@68: throw new RuntimeException("{\"error\": \"Bad type\"}"); jpayne@68: } jpayne@68: return list; jpayne@68: } jpayne@68: jpayne@68: public static final String stripAccession(String s){ jpayne@68: if(s==null){return null;} jpayne@68: s=s.toUpperCase(); jpayne@68: for(int i=0; i list=tree.getNodesByNameExtended(name); jpayne@68: if(verbose2){System.err.println("Fetched "+list);} jpayne@68: if(list==null){ jpayne@68: if(verbose2){System.err.println("Fetched in common map "+name);} jpayne@68: String name2=commonMap.get(name); jpayne@68: if(verbose2){System.err.println("Fetched "+name2);} jpayne@68: if(name2!=null){list=tree.getNodesByName(name2);} jpayne@68: } jpayne@68: if(list==null){notFound.incrementAndGet();} jpayne@68: return list==null ? null : list.get(0); jpayne@68: } jpayne@68: jpayne@68: /** Look up a TaxNode from the gi number */ jpayne@68: TaxNode getTaxNodeGi(long gi){ jpayne@68: int ncbi=-1; jpayne@68: try { jpayne@68: ncbi=GiToTaxid.getID(gi); jpayne@68: } catch (Throwable e) { jpayne@68: if(verbose){e.printStackTrace();} jpayne@68: } jpayne@68: if(ncbi<0){notFound.incrementAndGet();} jpayne@68: return ncbi<0 ? null : getTaxNodeTaxid(ncbi); jpayne@68: } jpayne@68: jpayne@68: /** Look up a TaxNode by parsing the full header */ jpayne@68: TaxNode getTaxNodeHeader(String header, boolean silvaMode){ jpayne@68: TaxNode tn=silvaMode ? tree.getNodeSilva(header, true) : tree.parseNodeFromHeader(header, true); jpayne@68: if(tn==null){notFound.incrementAndGet();} jpayne@68: return tn; jpayne@68: } jpayne@68: jpayne@68: /** Look up a TaxNode from the ncbi TaxID */ jpayne@68: TaxNode getTaxNodeTaxid(int ncbi){ jpayne@68: TaxNode tn=null; jpayne@68: try { jpayne@68: tn=tree.getNode(ncbi); jpayne@68: } catch (Throwable e) { jpayne@68: if(verbose){e.printStackTrace();} jpayne@68: } jpayne@68: if(tn==null){notFound.incrementAndGet();} jpayne@68: return tn; jpayne@68: } jpayne@68: jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: /*---------------- Data Initialization ----------------*/ jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: jpayne@68: private static HashMap makeTypeMap() { jpayne@68: HashMap map=new HashMap(63); jpayne@68: map.put("gi", GI); jpayne@68: // map.put("ngi", NGI); jpayne@68: // map.put("pgi", PGI); jpayne@68: map.put("name", NAME); jpayne@68: map.put("tax_id", TAXID); jpayne@68: map.put("ncbi", TAXID); jpayne@68: map.put("taxid", TAXID); jpayne@68: map.put("id", TAXID); jpayne@68: map.put("tid", TAXID); jpayne@68: map.put("header", HEADER); jpayne@68: map.put("accession", ACCESSION); jpayne@68: map.put("img", IMG); jpayne@68: map.put("silvaheader", SILVAHEADER); jpayne@68: jpayne@68: map.put("pt_gi", PT_GI); jpayne@68: // map.put("pt_ngi", PT_NGI); jpayne@68: // map.put("pt_pgi", PT_PGI); jpayne@68: map.put("pt_name", PT_NAME); jpayne@68: map.put("pt_tax_id", PT_TAXID); jpayne@68: map.put("pt_id", PT_TAXID); jpayne@68: map.put("pt_tid", PT_TAXID); jpayne@68: map.put("pt_ncbi", PT_TAXID); jpayne@68: map.put("pt_taxid", PT_TAXID); jpayne@68: map.put("pt_header", PT_HEADER); jpayne@68: map.put("pt_header", PT_HEADER); jpayne@68: map.put("pt_accession", PT_ACCESSION); jpayne@68: map.put("pt_img", PT_IMG); jpayne@68: map.put("pt_silvaheader", PT_SILVAHEADER); jpayne@68: jpayne@68: map.put("sc_gi", SC_GI); jpayne@68: // map.put("sc_ngi", SC_NGI); jpayne@68: // map.put("sc_pgi", SC_PGI); jpayne@68: map.put("sc_name", SC_NAME); jpayne@68: map.put("sc_tax_id", SC_TAXID); jpayne@68: map.put("sc_id", SC_TAXID); jpayne@68: map.put("sc_tid", SC_TAXID); jpayne@68: map.put("sc_ncbi", SC_TAXID); jpayne@68: map.put("sc_taxid", SC_TAXID); jpayne@68: map.put("sc_header", SC_HEADER); jpayne@68: map.put("sc_header", SC_HEADER); jpayne@68: map.put("sc_accession", SC_ACCESSION); jpayne@68: map.put("sc_silvaheader", SC_SILVAHEADER); jpayne@68: jpayne@68: return map; jpayne@68: } jpayne@68: jpayne@68: public static HashMap makeCommonMap(){ jpayne@68: HashMap map=new HashMap(); jpayne@68: map.put("human", "homo sapiens"); jpayne@68: map.put("cat", "felis catus"); jpayne@68: map.put("dog", "canis lupus familiaris"); jpayne@68: map.put("mouse", "mus musculus"); jpayne@68: map.put("cow", "bos taurus"); jpayne@68: map.put("bull", "bos taurus"); jpayne@68: map.put("horse", "Equus ferus"); jpayne@68: map.put("pig", "Sus scrofa domesticus"); jpayne@68: map.put("sheep", "Ovis aries"); jpayne@68: map.put("goat", "Capra aegagrus"); jpayne@68: map.put("turkey", "Meleagris gallopavo"); jpayne@68: map.put("fox", "Vulpes vulpes"); jpayne@68: map.put("chicken", "Gallus gallus domesticus"); jpayne@68: map.put("wolf", "canis lupus"); jpayne@68: map.put("fruitfly", "drosophila melanogaster"); jpayne@68: map.put("zebrafish", "Danio rerio"); jpayne@68: map.put("catfish", "Ictalurus punctatus"); jpayne@68: map.put("trout", "Oncorhynchus mykiss"); jpayne@68: map.put("salmon", "Salmo salar"); jpayne@68: map.put("tilapia", "Oreochromis niloticus"); jpayne@68: map.put("e coli", "Escherichia coli"); jpayne@68: map.put("e.coli", "Escherichia coli"); jpayne@68: jpayne@68: map.put("lion", "Panthera leo"); jpayne@68: map.put("tiger", "Panthera tigris"); jpayne@68: map.put("bear", "Ursus arctos"); jpayne@68: map.put("deer", "Odocoileus virginianus"); jpayne@68: map.put("coyote", "Canis latrans"); jpayne@68: jpayne@68: map.put("corn", "Zea mays subsp. mays"); jpayne@68: map.put("maize", "Zea mays subsp. mays"); jpayne@68: map.put("oat", "Avena sativa"); jpayne@68: map.put("wheat", "Triticum aestivum"); jpayne@68: map.put("rice", "Oryza sativa"); jpayne@68: map.put("potato", "Solanum tuberosum"); jpayne@68: map.put("barley", "Hordeum vulgare"); jpayne@68: map.put("poplar", "Populus alba"); jpayne@68: map.put("lettuce", "Lactuca sativa"); jpayne@68: map.put("beet", "Beta vulgaris"); jpayne@68: map.put("strawberry", "Fragaria x ananassa"); jpayne@68: map.put("orange", "Citrus sinensis"); jpayne@68: map.put("lemon", "Citrus limon"); jpayne@68: map.put("soy", "Glycine max"); jpayne@68: map.put("soybean", "Glycine max"); jpayne@68: map.put("grape", "Vitis vinifera"); jpayne@68: map.put("olive", "Olea europaea"); jpayne@68: map.put("cotton", "Gossypium hirsutum"); jpayne@68: map.put("apple", "Malus pumila"); jpayne@68: map.put("bannana", "Musa acuminata"); jpayne@68: map.put("tomato", "Solanum lycopersicum"); jpayne@68: map.put("sugarcane", "Saccharum officinarum"); jpayne@68: map.put("bean", "Phaseolus vulgaris"); jpayne@68: map.put("onion", "Allium cepa"); jpayne@68: map.put("garlic", "Allium sativum"); jpayne@68: jpayne@68: map.put("pichu", "mus musculus"); jpayne@68: map.put("pikachu", "mus musculus"); jpayne@68: map.put("vulpix", "Vulpes vulpes"); jpayne@68: map.put("ninetails", "Vulpes vulpes"); jpayne@68: map.put("mareep", "Ovis aries"); jpayne@68: jpayne@68: return map; jpayne@68: } jpayne@68: jpayne@68: //Customize usage message to include domain jpayne@68: private String makeUsagePrefix(){ jpayne@68: if(!sketchOnly){ jpayne@68: return "Welcome to the JGI taxonomy server!\n" jpayne@68: + "This service provides taxonomy information from NCBI taxID numbers, gi numbers, organism names, and accessions.\n" jpayne@68: + "The output is formatted as a Json object.\n\n" jpayne@68: + "Usage:\n\n" jpayne@68: + "All addresses below are assumed to be prefixed by "+domain+", e.g. /name/homo_sapiens implies a full URL of:\n" jpayne@68: + domain+"/name/homo_sapiens\n" jpayne@68: + "\n" jpayne@68: + "/name/homo_sapiens will give taxonomy information for an organism name.\n" jpayne@68: + "Names are case-insensitive and underscores are equivalent to spaces.\n" jpayne@68: + "/id/9606 will give taxonomy information for an NCBI taxID.\n" jpayne@68: + "/gi/1234 will give taxonomy information from an NCBI gi number.\n" jpayne@68: jpayne@68: // + "\n****NOTICE**** gi number support is temporarily suspended due to conflicts in NCBI data.\n" jpayne@68: // + "Support may be restored, altered, or discontinued pending a response from NCBI.\n" jpayne@68: // + "Currently, it is not possible to ensure correct results when looking up a GI number, because some map to multiple organisms.\n\n" jpayne@68: jpayne@68: + "/accession/NZ_AAAA01000057.1 will give taxonomy information from an accession.\n" jpayne@68: + "/header/ will accept an NCBI sequence header such as gi|7|emb|X51700.1| Bos taurus\n" jpayne@68: + "/silvaheader/ will accept a Silva sequence header such as KC415233.1.1497 Bacteria;Spirochaetae;Spirochaetes\n" jpayne@68: + "/img/ will accept an IMG id such as 2724679250\n" jpayne@68: + "Vertical bars (|) may cause problems on the command line and can be replaced by tilde (~).\n" jpayne@68: + "\nComma-delimited lists are accepted for bulk queries, such as tax/gi/1234,7000,42\n" jpayne@68: + "For plaintext (non-Json) results, add the term /pt/ or /sc/.\n" jpayne@68: + "pt will give just the taxID, while sc will give the whole lineage, semicolon-delimited. For example:\n" jpayne@68: + "/pt/name/homo_sapiens\n" jpayne@68: + "/sc/gi/1234\n\n" jpayne@68: + "Additional supported display options are children, numchildren, range, simple, path, size, and ancestor.\n" jpayne@68: + "The order is not important but they need to come before the query term. For example:\n" jpayne@68: + "/children/numchildren/range/gi/1234\n" jpayne@68: + "\nTo find the common ancestor of multiple organisms, add /ancestor/. For example:\n" jpayne@68: + "/id/ancestor/1234,5678,42\n" jpayne@68: + "/name/ancestor/homo_sapiens,canis_lupus,bos_taurus\n" jpayne@68: + "\nFor a simplified taxonomic tree, add simple.\n" jpayne@68: + "This will ignore unranked or uncommon levels like tribe and parvorder, and only display the following levels:\n" jpayne@68: + "SUBSPECIES, SPECIES, GENUS, FAMILY, ORDER, CLASS, PHYLUM, KINGDOM, SUPERKINGDOM, DOMAIN\n" jpayne@68: + "For example:\n" jpayne@68: + "/simple/id/1234\n" jpayne@68: + "\nTo print taxonomy from the command line in Linux, use curl:\n" jpayne@68: + "curl https://taxonomy.jgi.doe.gov/id/9606\n" jpayne@68: + "\nQueries longer than around 8kB can be sent via POST: curl https://taxonomy..doe.gov/POST" jpayne@68: + "\n...where the data sent is, for example: name/e.coli,h.sapiens,c.lupus\n" jpayne@68: + "\nLast restarted "+startTime+"\n" jpayne@68: + "Running BBMap version "+Shared.BBMAP_VERSION_STRING+"\n"; jpayne@68: }else{ jpayne@68: StringBuilder sb=new StringBuilder(); jpayne@68: sb.append("Welcome to the JGI"+(SketchObject.defaultParams.dbName==null ? "" : " "+SketchObject.defaultParams.dbName)+" sketch server!\n"); jpayne@68: // if(dbName!=null){ jpayne@68: // sb.append("This server has the "+dbName+ " database loaded.\n"); jpayne@68: // } jpayne@68: sb.append("\nUsage:\n\n"); jpayne@68: sb.append("sendsketch.sh in=file.fasta"+(SketchObject.defaultParams.dbName==null ? "" : " "+SketchObject.defaultParams.dbName.toLowerCase())+"\n\n"); jpayne@68: sb.append("SendSketch creates a sketch from a local sequence file, and sends the sketch to this server.\n"); jpayne@68: sb.append("The server receives the sketch, compares it to all sketches in memory, and returns the results.\n"); jpayne@68: sb.append("For files on the same system as the server, the 'local' flag may be used to offload sketch creation to the server.\n"); jpayne@68: sb.append("For more details and parameters please run sendsketch.sh with no arguments.\n"); jpayne@68: sb.append("\n"); jpayne@68: if(SketchObject.useWhitelist()){ jpayne@68: sb.append("This server is running in whitelist mode; for best results, use local queries.\n"); jpayne@68: sb.append("Remote queries should specify a larger-than-normal sketch size.\n\n"); jpayne@68: }else if(SketchObject.blacklist()!=null){ jpayne@68: sb.append("This server is running in blacklist mode, using "+new File(SketchObject.blacklist()).getName()+".\n\n"); jpayne@68: } jpayne@68: sb.append("Last restarted "+startTime+"\n"); jpayne@68: sb.append("Running BBMap version "+Shared.BBMAP_VERSION_STRING+"\n"); jpayne@68: sb.append("Settings:\tk="+SketchObject.k+(SketchObject.k2>0 ? ","+SketchObject.k2 : "")); jpayne@68: if(SketchObject.amino){sb.append(" amino");} jpayne@68: if(SketchObject.makeIndex){sb.append(" index");} jpayne@68: if(SketchObject.useWhitelist()){sb.append(" whitelist");} jpayne@68: if(SketchObject.blacklist()!=null){sb.append(" blacklist="+new File(SketchObject.blacklist()).getName());} jpayne@68: sb.append('\n'); jpayne@68: return sb.toString(); jpayne@68: } jpayne@68: } jpayne@68: jpayne@68: private String makeUsageHtml(){ jpayne@68: String html=rawHtml; jpayne@68: html=html.replace("STATISTICSSTRING", makeStats()); jpayne@68: // html=html.replace("TIMESTAMPSTRING", startTime); jpayne@68: // html=html.replace("VERSIONSTRING", "Running BBMap version "+Shared.BBMAP_VERSION_STRING); jpayne@68: return html; jpayne@68: } jpayne@68: jpayne@68: private String loadRawHtml(){ jpayne@68: String path=Data.findPath("?tax_server.html"); jpayne@68: String html=ReadWrite.readString(path); jpayne@68: return html; jpayne@68: } jpayne@68: jpayne@68: private String makeStats(){ jpayne@68: ByteBuilder sb=new ByteBuilder(); jpayne@68: jpayne@68: if(!sketchOnly){ jpayne@68: sb.append("JGI taxonomy server stats:\n" jpayne@68: + "\nLast restarted "+startTime+"\n" jpayne@68: + "Running BBMap version "+Shared.BBMAP_VERSION_STRING+"\n"); jpayne@68: }else{ jpayne@68: sb.append("JGI"+(SketchObject.defaultParams.dbName==null ? "" : " "+SketchObject.defaultParams.dbName)+" sketch server stats:\n\n"); jpayne@68: jpayne@68: if(domain!=null) {sb.append("Domain: "+domain+"\n");} jpayne@68: if(SketchObject.useWhitelist()){ jpayne@68: sb.append("This server is running in whitelist mode; for best results, use local queries.\n"); jpayne@68: sb.append("Remote queries should specify a larger-than-normal sketch size.\n\n"); jpayne@68: }else if(SketchObject.blacklist()!=null){ jpayne@68: sb.append("This server is running in blacklist mode, using "+new File(SketchObject.blacklist()).getName()+".\n\n"); jpayne@68: } jpayne@68: sb.append("Last restarted "+startTime+"\n"); jpayne@68: sb.append("Running BBMap version "+Shared.BBMAP_VERSION_STRING+"\n"); jpayne@68: sb.append("Settings: k="+SketchObject.k+(SketchObject.k2>0 ? ","+SketchObject.k2 : "")); jpayne@68: if(SketchObject.amino){sb.append(" amino");} jpayne@68: if(SketchObject.makeIndex){sb.append(" index");} jpayne@68: if(SketchObject.useWhitelist()){sb.append(" whitelist");} jpayne@68: if(SketchObject.blacklist()!=null){sb.append(" blacklist="+new File(SketchObject.blacklist()).getName());} jpayne@68: } jpayne@68: sb.nl().nl(); jpayne@68: sb.append(basicStats()); jpayne@68: if(sketchOnly){sb.append(makeExtendedStats());} jpayne@68: jpayne@68: return sb.toString(); jpayne@68: } jpayne@68: jpayne@68: public String makeExtendedStats(){ jpayne@68: ByteBuilder sb=new ByteBuilder(); jpayne@68: sb.append('\n'); jpayne@68: jpayne@68: { jpayne@68: sb.append("\nVersion\tCount\n"); jpayne@68: ArrayList list=new ArrayList(); jpayne@68: for(Entry e : versionMap.entrySet()){ jpayne@68: list.add(e.getValue().toString()); jpayne@68: } jpayne@68: Collections.sort(list); jpayne@68: for(String s : list){ jpayne@68: sb.append(s).append('\n'); jpayne@68: } jpayne@68: } jpayne@68: jpayne@68: { jpayne@68: sb.append("\nSketchs\tCount\tAvgTime\n"); jpayne@68: for(int i=0; i0){ jpayne@68: sb.append(i).append('\t').append(b).append('\t').append(a/b, 3).append('\n'); jpayne@68: } jpayne@68: } jpayne@68: sb.append('\n'); jpayne@68: } jpayne@68: return sb.toString(); jpayne@68: } jpayne@68: jpayne@68: public String USAGE(String prefix){ jpayne@68: if(!countQueries){return prefix;} jpayne@68: String basicStats=basicStats(); jpayne@68: return (prefix==null ? basicStats : prefix+"\n"+basicStats); jpayne@68: } jpayne@68: jpayne@68: public String basicStats(){ jpayne@68: if(!countQueries){return "";} jpayne@68: StringBuilder sb=new StringBuilder(500); jpayne@68: jpayne@68: final long uq=usageQueries.getAndIncrement(); jpayne@68: final long mq=malformedQueries.get(); jpayne@68: final long pt=plaintextQueries.get(), sc=semicolonQueries.get(), pa=pathQueries.get(), pp=printPathQueries.get(), ps=printSizeQueries.get(); jpayne@68: final long iq=internalQueries.get(); jpayne@68: final long lq=localQueries.get(); jpayne@68: final long rfq=refQueries.get(); jpayne@68: final long q=queries.get(); jpayne@68: final long nf=notFound.get(); jpayne@68: final double avgTimeDL=.000001*(elapsedTimeLocal.get()/(Tools.max(1.0, timeMeasurementsLocal.get())));//in milliseconds jpayne@68: final double lastTimeDL=.000001*lastTimeLocal.get(); jpayne@68: final double avgTimeDR=.000001*(elapsedTimeRemote.get()/(Tools.max(1.0, timeMeasurementsRemote.get())));//in milliseconds jpayne@68: final double lastTimeDR=.000001*lastTimeRemote.get(); jpayne@68: final double avgTimeDRF=.000001*(elapsedTimeReference.get()/(Tools.max(1.0, timeMeasurementsReference.get())));//in milliseconds jpayne@68: final double lastTimeDRF=.000001*lastTimeReference.get(); jpayne@68: final double avgTimeDU=.000001*(elapsedTimeUsage.get()/(Tools.max(1.0, timeMeasurementsUsage.get())));//in milliseconds jpayne@68: final double lastTimeDU=.000001*lastTimeUsage.get(); jpayne@68: final long exq=q-iq; jpayne@68: final long rmq=q-lq; jpayne@68: jpayne@68: sb.append('\n').append("Queries: ").append(q); jpayne@68: sb.append('\n').append("Usage: ").append(uq); jpayne@68: if(sketchOnly){ jpayne@68: sb.append('\n').append("Invalid: ").append(mq); jpayne@68: sb.append('\n').append("Avg time: ").append(String.format(Locale.ROOT, "%.3f ms (local queries)", avgTimeDL)); jpayne@68: sb.append('\n').append("Last time: ").append(String.format(Locale.ROOT, "%.3f ms (local queries)", lastTimeDL)); jpayne@68: sb.append('\n').append("Avg time: ").append(String.format(Locale.ROOT, "%.3f ms (remote queries)", avgTimeDR)); jpayne@68: sb.append('\n').append("Last time: ").append(String.format(Locale.ROOT, "%.3f ms (remote queries)", lastTimeDR)); jpayne@68: sb.append('\n').append("Avg time: ").append(String.format(Locale.ROOT, "%.3f ms (ref queries)", avgTimeDRF)); jpayne@68: sb.append('\n').append("Last time: ").append(String.format(Locale.ROOT, "%.3f ms (ref queries)", lastTimeDRF)); jpayne@68: }else{ jpayne@68: sb.append('\n').append("Avg time: ").append(String.format(Locale.ROOT, "%.3f ms", avgTimeDR)); jpayne@68: sb.append('\n').append("Last time: ").append(String.format(Locale.ROOT, "%.3f ms", lastTimeDR)); jpayne@68: sb.append('\n').append("Avg time: ").append(String.format(Locale.ROOT, "%.3f ms (usage queries)", avgTimeDU)); jpayne@68: sb.append('\n').append("Last time: ").append(String.format(Locale.ROOT, "%.3f ms (usage queries)", lastTimeDU)); jpayne@68: } jpayne@68: sb.append('\n'); jpayne@68: sb.append('\n').append("Internal: ").append(iq); jpayne@68: sb.append('\n').append("External: ").append(exq); jpayne@68: if(!sketchOnly){sb.append('\n').append("NotFound: ").append(nf);} jpayne@68: sb.append('\n'); jpayne@68: jpayne@68: if(sketchOnly){ jpayne@68: sb.append('\n').append("Local: ").append(lq); jpayne@68: sb.append('\n').append("Remote: ").append(rmq); jpayne@68: sb.append('\n').append("Reference: ").append(rfq); jpayne@68: sb.append('\n'); jpayne@68: sb.append('\n').append("Depth: ").append(depthQueries.get()); jpayne@68: sb.append('\n'); jpayne@68: sb.append('\n').append("Sketches: ").append(querySketches.get()); jpayne@68: sb.append('\n').append("BytesIn: ").append(bytesIn.get()); jpayne@68: sb.append('\n').append("BytesOut: ").append(bytesOut.get()); jpayne@68: sb.append('\n'); jpayne@68: sb.append('\n').append("Single: ").append(firstChunkSingle.get()); jpayne@68: sb.append('\n').append("Bulk: ").append(firstChunkMulti.get()); jpayne@68: sb.append('\n').append("UnknownS: ").append(unknownChunkSingle.get()); jpayne@68: sb.append('\n').append("UnknownB: ").append(unknownChunkMulti.get()); jpayne@68: sb.append('\n').append("Total: ").append(bulkCount.get()); jpayne@68: }else{ jpayne@68: sb.append('\n').append("gi: ").append(giQueries.get()); jpayne@68: sb.append('\n').append("Name: ").append(nameQueries.get()); jpayne@68: sb.append('\n').append("TaxID: ").append(taxidQueries.get()); jpayne@68: sb.append('\n').append("Header: ").append(headerQueries.get()); jpayne@68: sb.append('\n').append("Accession: ").append(accessionQueries.get()); jpayne@68: sb.append('\n').append("IMG: ").append(imgQueries.get()); jpayne@68: sb.append('\n').append("Silva: ").append(silvaHeaderQueries.get()); jpayne@68: sb.append('\n'); jpayne@68: sb.append('\n').append("Simple: ").append(simpleQueries.get()); jpayne@68: sb.append('\n').append("Ancestor: ").append(ancestorQueries.get()); jpayne@68: sb.append('\n').append("Children: ").append(childrenQueries.get()); jpayne@68: sb.append('\n'); jpayne@68: sb.append('\n').append("Json: ").append(q-pt-sc-pa); jpayne@68: sb.append('\n').append("Plaintext: ").append(pt); jpayne@68: sb.append('\n').append("Semicolon: ").append(sc); jpayne@68: sb.append('\n').append("Path: ").append(pa+pp); jpayne@68: sb.append('\n').append("Size: ").append(ps); jpayne@68: sb.append('\n').append("Single: ").append(firstChunkSingle.get()); jpayne@68: sb.append('\n').append("Bulk: ").append(firstChunkMulti.get()); jpayne@68: sb.append('\n').append("Total: ").append(bulkCount.get()); jpayne@68: } jpayne@68: sb.append('\n'); jpayne@68: return sb.toString(); jpayne@68: } jpayne@68: jpayne@68: public boolean incrementQueries(HttpExchange t, boolean local, boolean refMode, boolean simple, boolean ancestor, jpayne@68: boolean plaintext, boolean semicolon, boolean path, boolean printChildren, boolean printPath, boolean printSize, int type){ jpayne@68: final boolean internal=ServerTools.isInternalQuery(t, addressPrefix, allowLocalHost, printIP, printHeaders); jpayne@68: jpayne@68: if(!countQueries){return internal;} jpayne@68: queries.incrementAndGet(); jpayne@68: if(local){localQueries.incrementAndGet();} jpayne@68: else if(refMode){localQueries.incrementAndGet();} jpayne@68: jpayne@68: if(type>=0){ jpayne@68: int type2=type&15; jpayne@68: if(type2==GI){ jpayne@68: giQueries.incrementAndGet(); jpayne@68: }else if(type2==NAME){ jpayne@68: nameQueries.incrementAndGet(); jpayne@68: }else if(type2==TAXID){ jpayne@68: taxidQueries.incrementAndGet(); jpayne@68: }else if(type2==ACCESSION){ jpayne@68: accessionQueries.incrementAndGet(); jpayne@68: }else if(type2==IMG){ jpayne@68: imgQueries.incrementAndGet(); jpayne@68: }else if(type2==HEADER){ jpayne@68: headerQueries.incrementAndGet(); jpayne@68: }else if(type2==UNKNOWN){ jpayne@68: unknownQueries.incrementAndGet(); jpayne@68: }else if(type2==SILVAHEADER){ jpayne@68: silvaHeaderQueries.incrementAndGet(); jpayne@68: } jpayne@68: jpayne@68: if(simple){simpleQueries.incrementAndGet();} jpayne@68: if(ancestor){ancestorQueries.incrementAndGet();} jpayne@68: jpayne@68: if(plaintext){plaintextQueries.incrementAndGet();} jpayne@68: else if(semicolon){semicolonQueries.incrementAndGet();} jpayne@68: else if(path){pathQueries.incrementAndGet();} jpayne@68: jpayne@68: if(printChildren){childrenQueries.incrementAndGet();} jpayne@68: if(printPath){printPathQueries.incrementAndGet();} jpayne@68: if(printSize){printSizeQueries.incrementAndGet();} jpayne@68: } jpayne@68: jpayne@68: if(internal){internalQueries.incrementAndGet();} jpayne@68: jpayne@68: return internal; jpayne@68: } jpayne@68: jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: jpayne@68: String compare(ArrayList inSketches, DisplayParams params){ jpayne@68: boolean success=true; jpayne@68: final int inSize=inSketches.size(); jpayne@68: querySketches.addAndGet(inSize); jpayne@68: if(Shared.threads()<2 || maxConcurrentSketchCompareThreads<2 || inSize<4){ jpayne@68: ByteBuilder sb=new ByteBuilder(); jpayne@68: success=searcher.compare(inSketches, sb, params, maxConcurrentSketchCompareThreads); jpayne@68: return sb.toString(); jpayne@68: }else{//More sketches than threads, and more than one thread jpayne@68: final int threads=Tools.min(maxConcurrentSketchCompareThreads, (inSize+4)/4); jpayne@68: jpayne@68: ByteBuilder[] out=new ByteBuilder[inSize]; jpayne@68: ArrayList alct=new ArrayList(threads); jpayne@68: AtomicInteger next=new AtomicInteger(0); jpayne@68: for(int i=0; i inSketches_, final int tid_, final AtomicInteger nextSketch_, ByteBuilder[] out_, DisplayParams params_){ jpayne@68: inSketches=inSketches_; jpayne@68: tid=tid_; jpayne@68: nextSketch=nextSketch_; jpayne@68: out=out_; jpayne@68: params=params_; jpayne@68: } jpayne@68: jpayne@68: @Override jpayne@68: public void run(){ jpayne@68: success=false; jpayne@68: final int inLim=inSketches.size(); jpayne@68: final boolean json=params.json(); jpayne@68: jpayne@68: for(int inNum=nextSketch.getAndIncrement(); inNum1){ jpayne@68: if(inNum==0){ jpayne@68: bb.insert(0, (byte)'['); jpayne@68: } jpayne@68: if(inNum inSketches; jpayne@68: private final int tid; jpayne@68: private final CompareBuffer buffer=new CompareBuffer(false); jpayne@68: private final DisplayParams params; jpayne@68: private final ByteBuilder[] out; jpayne@68: jpayne@68: private final AtomicInteger nextSketch; jpayne@68: private final AtomicInteger fakeID=new AtomicInteger(SketchObject.minFakeID); jpayne@68: private ConcurrentHashMap map=new ConcurrentHashMap(101); jpayne@68: jpayne@68: boolean success=false; jpayne@68: jpayne@68: } jpayne@68: jpayne@68: jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: /*---------------- Fields ----------------*/ jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: jpayne@68: public boolean sketchOnly=false; jpayne@68: jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: /*---------------- Counters ----------------*/ jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: jpayne@68: private HashMap versionMap=new HashMap(); jpayne@68: private AtomicLongArray timesByCount=new AtomicLongArray(10000); jpayne@68: private AtomicLongArray queryCounts=new AtomicLongArray(10000); jpayne@68: jpayne@68: private AtomicLong notFound=new AtomicLong(0); jpayne@68: private AtomicLong queries=new AtomicLong(0); jpayne@68: /** Same IP address mask */ jpayne@68: private AtomicLong internalQueries=new AtomicLong(0); jpayne@68: /** Local filesystem sketch */ jpayne@68: private AtomicLong localQueries=new AtomicLong(0); jpayne@68: private AtomicLong refQueries=new AtomicLong(0); jpayne@68: jpayne@68: private AtomicLong depthQueries=new AtomicLong(0); jpayne@68: jpayne@68: private AtomicLong iconQueries=new AtomicLong(0); jpayne@68: jpayne@68: private AtomicLong querySketches=new AtomicLong(0); jpayne@68: jpayne@68: private AtomicLong unknownChunkSingle=new AtomicLong(0); jpayne@68: private AtomicLong unknownChunkMulti=new AtomicLong(0); jpayne@68: private AtomicLong firstChunkSingle=new AtomicLong(0); jpayne@68: private AtomicLong firstChunkMulti=new AtomicLong(0); jpayne@68: private AtomicLong nthChunkSingle=new AtomicLong(0); jpayne@68: private AtomicLong nthChunkMulti=new AtomicLong(0); jpayne@68: jpayne@68: private AtomicLong singleQueries=new AtomicLong(0); jpayne@68: private AtomicLong bulkQueries=new AtomicLong(0); jpayne@68: private AtomicLong bulkCount=new AtomicLong(0); jpayne@68: jpayne@68: private AtomicLong giQueries=new AtomicLong(0); jpayne@68: private AtomicLong nameQueries=new AtomicLong(0); jpayne@68: private AtomicLong taxidQueries=new AtomicLong(0); jpayne@68: private AtomicLong headerQueries=new AtomicLong(0); jpayne@68: private AtomicLong accessionQueries=new AtomicLong(0); jpayne@68: private AtomicLong imgQueries=new AtomicLong(0); jpayne@68: private AtomicLong unknownQueries=new AtomicLong(0); jpayne@68: private AtomicLong silvaHeaderQueries=new AtomicLong(0); jpayne@68: jpayne@68: private AtomicLong plaintextQueries=new AtomicLong(0); jpayne@68: private AtomicLong semicolonQueries=new AtomicLong(0); jpayne@68: private AtomicLong pathQueries=new AtomicLong(0); jpayne@68: private AtomicLong printPathQueries=new AtomicLong(0); jpayne@68: private AtomicLong printSizeQueries=new AtomicLong(0); jpayne@68: private AtomicLong childrenQueries=new AtomicLong(0); jpayne@68: jpayne@68: private AtomicLong simpleQueries=new AtomicLong(0); jpayne@68: private AtomicLong ancestorQueries=new AtomicLong(0); jpayne@68: jpayne@68: private AtomicLong usageQueries=new AtomicLong(0); jpayne@68: private AtomicLong bytesIn=new AtomicLong(0); jpayne@68: private AtomicLong bytesOut=new AtomicLong(0); jpayne@68: jpayne@68: // private AtomicLong elapsedTime=new AtomicLong(0); jpayne@68: // private AtomicLong timeMeasurements=new AtomicLong(0); jpayne@68: // private AtomicLong lastTime=new AtomicLong(0); jpayne@68: jpayne@68: private AtomicLong elapsedTimeUsage=new AtomicLong(0); jpayne@68: private AtomicLong timeMeasurementsUsage=new AtomicLong(0); jpayne@68: private AtomicLong lastTimeUsage=new AtomicLong(0); jpayne@68: jpayne@68: private AtomicLong elapsedTimeRemote=new AtomicLong(0); jpayne@68: private AtomicLong timeMeasurementsRemote=new AtomicLong(0); jpayne@68: private AtomicLong lastTimeRemote=new AtomicLong(0); jpayne@68: jpayne@68: private AtomicLong elapsedTimeLocal=new AtomicLong(0); jpayne@68: private AtomicLong timeMeasurementsLocal=new AtomicLong(0); jpayne@68: private AtomicLong lastTimeLocal=new AtomicLong(0); jpayne@68: jpayne@68: private AtomicLong elapsedTimeReference=new AtomicLong(0); jpayne@68: private AtomicLong timeMeasurementsReference=new AtomicLong(0); jpayne@68: private AtomicLong lastTimeReference=new AtomicLong(0); jpayne@68: jpayne@68: private AtomicLong malformedQueries=new AtomicLong(0); jpayne@68: jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: /*---------------- Params ----------------*/ jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: jpayne@68: public boolean printIP=false; jpayne@68: public boolean printHeaders=false; jpayne@68: public boolean countQueries=true; jpayne@68: public float prealloc=0; jpayne@68: public boolean useHtml=false; jpayne@68: jpayne@68: /** Location of GiTable file */ jpayne@68: private String giTableFile=null; jpayne@68: /** Location of TaxTree file */ jpayne@68: private String taxTreeFile="auto"; jpayne@68: /** Comma-delimited locations of Accession files */ jpayne@68: private String accessionFile=null; jpayne@68: /** Location of IMG dump file */ jpayne@68: private String imgFile=null; jpayne@68: /** Location of accession pattern file */ jpayne@68: private String patternFile=null; jpayne@68: jpayne@68: private String sizeFile=null; jpayne@68: jpayne@68: /** Location of sequence directory tree */ jpayne@68: private String basePath="/global/cfs/cdirs/bbtools/tree/"; jpayne@68: jpayne@68: /** Used for taxonomic tree traversal */ jpayne@68: private final TaxTree tree; jpayne@68: jpayne@68: /** Maps URL Strings to numeric query types */ jpayne@68: private final HashMap typeMap; jpayne@68: /** Maps common organism names to scientific names */ jpayne@68: private final HashMap commonMap; jpayne@68: jpayne@68: /** Hash taxonomic names for lookup */ jpayne@68: private boolean hashNames=true; jpayne@68: private boolean hashDotFormat=true; jpayne@68: jpayne@68: /** Kill code of prior server instance (optional) */ jpayne@68: private String oldKillCode=null; jpayne@68: /** Address of prior server instance (optional) */ jpayne@68: private String oldAddress=null; jpayne@68: jpayne@68: /** Address of current server instance (optional) */ jpayne@68: public String domain=null; jpayne@68: jpayne@68: public int maxConcurrentSketchCompareThreads=8;//TODO: This might be too high when lots of concurrent sessions are active jpayne@68: public int maxConcurrentSketchLoadThreads=4;//TODO: This might be too high when lots of concurrent sessions are active jpayne@68: public int handlerThreads=-1; jpayne@68: jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: /*---------------- Final Fields ----------------*/ jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: jpayne@68: private final boolean distributed; jpayne@68: private final int serverNum; jpayne@68: private final int serverCount; jpayne@68: private ArrayList slaveAddress; jpayne@68: jpayne@68: public final String favIconPath=Data.findPath("?favicon.ico"); jpayne@68: public final byte[] favIcon=ReadWrite.readRaw(favIconPath); jpayne@68: jpayne@68: private final String startTime=new Date().toString(); jpayne@68: jpayne@68: /** Listen on this port */ jpayne@68: public final int port; jpayne@68: /** Code to validate kill requests */ jpayne@68: public final String killCode; jpayne@68: jpayne@68: public final HttpServer httpServer; jpayne@68: jpayne@68: /** Bit to set for plaintext query types */ jpayne@68: public static final int PT_BIT=16; jpayne@68: /** Bit to set for semicolon-delimited query types */ jpayne@68: public static final int SC_BIT=32; jpayne@68: /** Bit to set for path query types */ jpayne@68: public static final int PA_BIT=64; jpayne@68: /** Request query types */ jpayne@68: public static final int UNKNOWN=0, GI=1, NAME=2, TAXID=3, HEADER=4, ACCESSION=5, IMG=6, SILVAHEADER=7; jpayne@68: /** Plaintext-response query types */ jpayne@68: public static final int PT_GI=GI+PT_BIT, PT_NAME=NAME+PT_BIT, PT_TAXID=TAXID+PT_BIT, jpayne@68: PT_HEADER=HEADER+PT_BIT, PT_ACCESSION=ACCESSION+PT_BIT, PT_IMG=IMG+PT_BIT, PT_SILVAHEADER=SILVAHEADER+PT_BIT; jpayne@68: /** Semicolon-response query types */ jpayne@68: public static final int SC_GI=GI+SC_BIT, SC_NAME=NAME+SC_BIT, SC_TAXID=TAXID+SC_BIT, jpayne@68: SC_HEADER=HEADER+SC_BIT, SC_ACCESSION=ACCESSION+SC_BIT, SC_IMG=IMG+SC_BIT, SC_SILVAHEADER=SILVAHEADER+PT_BIT; jpayne@68: jpayne@68: public static final int SOURCE_REFSEQ=1, SOURCE_SILVA=2, SOURCE_IMG=3; jpayne@68: jpayne@68: /** Generic response when asking for tax advice */ jpayne@68: public static final String TAX_ADVICE="This site does not give tax advice."; jpayne@68: /** Generic response for incorrect kill code */ jpayne@68: public static final String BAD_CODE="Incorrect code."; jpayne@68: /** Generic response for badly-formatted queries */ jpayne@68: public final String USAGE; jpayne@68: /** HTML version */ jpayne@68: // public final String USAGE_HTML; jpayne@68: public final String rawHtml; jpayne@68: jpayne@68: /** Tool for comparing query sketches to reference sketches */ jpayne@68: public final SketchSearcher searcher=new SketchSearcher(); jpayne@68: jpayne@68: public final boolean hasSketches; jpayne@68: jpayne@68: final boolean allowRemoteFileAccess; jpayne@68: final boolean allowLocalHost; jpayne@68: final String addressPrefix; jpayne@68: private boolean clearMem=true; jpayne@68: jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: /*---------------- Common Fields ----------------*/ jpayne@68: /*--------------------------------------------------------------*/ jpayne@68: jpayne@68: /** Print status messages to this output stream */ jpayne@68: private PrintStream outstream=System.err; jpayne@68: /** Print verbose messages */ jpayne@68: public static boolean verbose=false, verbose2=false, logUsage=false; jpayne@68: /** True if an error was encountered */ jpayne@68: public boolean errorState=false; jpayne@68: jpayne@68: }