Mercurial > repos > rliterman > csp2

annotate CSP2/CSP2_env/env-d9b9114564458d9d-741b3de822f2aaca6c6caa4325c4afce/opt/bbmap-39.01-1/current/tax/TaxServer.java @ 68:5028fdace37b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 2e9511a184a1ca667c7be0c6321a36dc4e3d116d
author jpayne
date Tue, 18 Mar 2025 16:23:26 -0400
parents
children
rev   line source
jpayne@68 1 package tax;
jpayne@68 2
jpayne@68 3 import java.io.File;
jpayne@68 4 import java.io.IOException;
jpayne@68 5 import java.io.InputStream;
jpayne@68 6 import java.io.PrintStream;
jpayne@68 7 import java.net.InetSocketAddress;
jpayne@68 8 import java.util.ArrayList;
jpayne@68 9 import java.util.Arrays;
jpayne@68 10 import java.util.Collections;
jpayne@68 11 import java.util.Date;
jpayne@68 12 import java.util.HashMap;
jpayne@68 13 import java.util.List;
jpayne@68 14 import java.util.Locale;
jpayne@68 15 import java.util.Map.Entry;
jpayne@68 16 import java.util.concurrent.ConcurrentHashMap;
jpayne@68 17 import java.util.concurrent.atomic.AtomicInteger;
jpayne@68 18 import java.util.concurrent.atomic.AtomicLong;
jpayne@68 19 import java.util.concurrent.atomic.AtomicLongArray;
jpayne@68 20
jpayne@68 21 import com.sun.net.httpserver.HttpExchange;
jpayne@68 22 import com.sun.net.httpserver.HttpHandler;
jpayne@68 23 import com.sun.net.httpserver.HttpServer;
jpayne@68 24 import com.sun.net.httpserver.HttpsServer;
jpayne@68 25
jpayne@68 26 import dna.Data;
jpayne@68 27 import fileIO.ReadWrite;
jpayne@68 28 import json.JsonObject;
jpayne@68 29 import server.PercentEncoding;
jpayne@68 30 import server.ServerTools;
jpayne@68 31 import shared.KillSwitch;
jpayne@68 32 import shared.Parse;
jpayne@68 33 import shared.Parser;
jpayne@68 34 import shared.PreParser;
jpayne@68 35 import shared.Shared;
jpayne@68 36 import shared.Timer;
jpayne@68 37 import shared.Tools;
jpayne@68 38 import sketch.CompareBuffer;
jpayne@68 39 import sketch.Comparison;
jpayne@68 40 import sketch.DisplayParams;
jpayne@68 41 import sketch.Sketch;
jpayne@68 42 import sketch.SketchMakerMini;
jpayne@68 43 import sketch.SketchObject;
jpayne@68 44 import sketch.SketchResults;
jpayne@68 45 import sketch.SketchSearcher;
jpayne@68 46 import sketch.SketchTool;
jpayne@68 47 import sketch.Whitelist;
jpayne@68 48 import stream.Read;
jpayne@68 49 import structures.ByteBuilder;
jpayne@68 50 import structures.IntList;
jpayne@68 51 import structures.StringNum;
jpayne@68 52
jpayne@68 53 /**
jpayne@68 54 * Server for taxonomy or Sketch queries.
jpayne@68 55 * @author Shijie Yao, Brian Bushnell
jpayne@68 56 * @date Dec 13, 2016
jpayne@68 57 *
jpayne@68 58 */
jpayne@68 59 public class TaxServer {
jpayne@68 60
jpayne@68 61 /*--------------------------------------------------------------*/
jpayne@68 62 /*---------------- Startup ----------------*/
jpayne@68 63 /*--------------------------------------------------------------*/
jpayne@68 64
jpayne@68 65 /** Command line entrance */
jpayne@68 66 public static void main(String[] args) throws Exception {
jpayne@68 67 Timer t=new Timer();
jpayne@68 68 @SuppressWarnings("unused")
jpayne@68 69 TaxServer ts=new TaxServer(args);
jpayne@68 70
jpayne@68 71 t.stop("Time: ");
jpayne@68 72
jpayne@68 73 System.err.println("Ready!");
jpayne@68 74
jpayne@68 75 //ts.begin();
jpayne@68 76 }
jpayne@68 77
jpayne@68 78 /** Constructor */
jpayne@68 79 public TaxServer(String[] args) throws Exception {
jpayne@68 80
jpayne@68 81 {//Preparse block for help, config files, and outstream
jpayne@68 82 PreParser pp=new PreParser(args, getClass(), false);
jpayne@68 83 args=pp.args;
jpayne@68 84 outstream=pp.outstream;
jpayne@68 85 }
jpayne@68 86
jpayne@68 87 ReadWrite.USE_UNPIGZ=true;
jpayne@68 88 TaxFilter.printNodesAdded=false;
jpayne@68 89 TaxFilter.REQUIRE_PRESENT=false; //Due to missing entries in TaxDump.
jpayne@68 90 Read.JUNK_MODE=Read.FIX_JUNK;
jpayne@68 91 SketchObject.compareSelf=true;
jpayne@68 92
jpayne@68 93 int port_=3068; //Taxonomy server
jpayne@68 94 String killCode_=null;
jpayne@68 95 boolean allowRemoteFileAccess_=false;
jpayne@68 96 boolean allowLocalHost_=false;
jpayne@68 97 String addressPrefix_="128."; //LBL
jpayne@68 98 long defaultSketchReads=200000;
jpayne@68 99 boolean https=false;
jpayne@68 100
jpayne@68 101 int serverNum_=0;
jpayne@68 102 int serverCount_=1;
jpayne@68 103
jpayne@68 104 //Create a parser object
jpayne@68 105 Parser parser=new Parser();
jpayne@68 106
jpayne@68 107 //Parse each argument
jpayne@68 108 for(int i=0; i<args.length; i++){
jpayne@68 109 String arg=args[i];
jpayne@68 110
jpayne@68 111 //Break arguments into their constituent parts, in the form of "a=b"
jpayne@68 112 String[] split=arg.split("=");
jpayne@68 113 String a=split[0].toLowerCase();
jpayne@68 114 String b=split.length>1 ? split[1] : null;
jpayne@68 115
jpayne@68 116 if(a.equals("verbose")){
jpayne@68 117 verbose=Parse.parseBoolean(b);
jpayne@68 118 }else if(a.equals("verbose2")){
jpayne@68 119 verbose2=SketchObject.verbose2=Parse.parseBoolean(b);
jpayne@68 120 }else if(a.equals("html")){
jpayne@68 121 useHtml=Parse.parseBoolean(b);
jpayne@68 122 }else if(a.equals("https")){
jpayne@68 123 https=Parse.parseBoolean(b);
jpayne@68 124 }else if(a.equals("http")){
jpayne@68 125 https=!Parse.parseBoolean(b);
jpayne@68 126 }else if(a.equals("servers") || a.equals("numservers") || a.equals("servercount")){
jpayne@68 127 serverCount_=Integer.parseInt(b);
jpayne@68 128 assert(serverCount_>0) : arg;
jpayne@68 129 }else if(a.equals("servernum")){
jpayne@68 130 serverNum_=Integer.parseInt(b);
jpayne@68 131 assert(serverNum_>=0) : arg;
jpayne@68 132 }else if(a.startsWith("slave") && Tools.isDigit(a.charAt(a.length()-1))){
jpayne@68 133 int num=Integer.parseInt(a.substring(5));
jpayne@68 134 if(slaveAddress==null){slaveAddress=new ArrayList<String>(serverCount_);}
jpayne@68 135 while(slaveAddress.size()<=num){slaveAddress.add(null);}
jpayne@68 136 slaveAddress.set(num, b);
jpayne@68 137 }else if(a.equals("table") || a.equals("gi") || a.equals("gitable")){
jpayne@68 138 giTableFile=b;
jpayne@68 139 }else if(a.equals("tree") || a.equals("taxtree")){
jpayne@68 140 taxTreeFile=b;
jpayne@68 141 }else if(a.equals("accession")){
jpayne@68 142 accessionFile=b;
jpayne@68 143 }else if(a.equals("pattern")){
jpayne@68 144 patternFile=b;
jpayne@68 145 }else if(a.equals("size") || a.equals("sizefile")){
jpayne@68 146 sizeFile=b;
jpayne@68 147 }else if(a.equalsIgnoreCase("img")){
jpayne@68 148 imgFile=b;
jpayne@68 149 }else if(a.equals("domain")){
jpayne@68 150 domain=b;
jpayne@68 151 while(domain!=null && domain.endsWith("/")){domain=domain.substring(0, domain.length()-1);}
jpayne@68 152 }else if(a.equals("port")){
jpayne@68 153 port_=Integer.parseInt(b);
jpayne@68 154 }else if(a.equals("kill") || a.equals("killcode")){
jpayne@68 155 killCode_=b;
jpayne@68 156 }else if(a.equals("oldcode")){
jpayne@68 157 oldKillCode=b;
jpayne@68 158 }else if(a.equals("oldaddress")){
jpayne@68 159 oldAddress=b;
jpayne@68 160 }else if(a.equals("sketchonly")){
jpayne@68 161 sketchOnly=Parse.parseBoolean(b);
jpayne@68 162 }else if(a.equals("sketchreads")){
jpayne@68 163 defaultSketchReads=Parse.parseKMG(b);
jpayne@68 164 }else if(a.equals("handlerthreads")){
jpayne@68 165 handlerThreads=Integer.parseInt(b);
jpayne@68 166 }else if(a.equals("sketchthreads") || a.equals("sketchcomparethreads")){
jpayne@68 167 maxConcurrentSketchCompareThreads=Integer.parseInt(b);
jpayne@68 168 }else if(a.equals("sketchloadthreads")){
jpayne@68 169 maxConcurrentSketchLoadThreads=Integer.parseInt(b);
jpayne@68 170 }else if(a.equals("hashnames")){
jpayne@68 171 hashNames=Parse.parseBoolean(b);
jpayne@68 172 }else if(a.equals("hashdotformat")){
jpayne@68 173 hashDotFormat=Parse.parseBoolean(b);
jpayne@68 174 }else if(a.equals("printip")){
jpayne@68 175 printIP=Parse.parseBoolean(b);
jpayne@68 176 }else if(a.equals("printheaders")){
jpayne@68 177 printHeaders=Parse.parseBoolean(b);
jpayne@68 178 }else if(a.equals("countqueries")){
jpayne@68 179 countQueries=Parse.parseBoolean(b);
jpayne@68 180 }else if(a.equals("clear") || a.equals("clearmem")){
jpayne@68 181 clearMem=Parse.parseBoolean(b);
jpayne@68 182 }else if(a.equals("dbname")){
jpayne@68 183 SketchObject.defaultParams.dbName=b;
jpayne@68 184 }else if(a.equals("allowremotefileaccess")){
jpayne@68 185 allowRemoteFileAccess_=Parse.parseBoolean(b);
jpayne@68 186 }else if(a.equals("allowlocalhost")){
jpayne@68 187 allowLocalHost_=Parse.parseBoolean(b);
jpayne@68 188 }else if(a.equals("addressprefix")){
jpayne@68 189 addressPrefix_=b;
jpayne@68 190 }else if(a.equals("maxpigzprocesses")){
jpayne@68 191 AccessionToTaxid.maxPigzProcesses=Integer.parseInt(b);
jpayne@68 192 }else if(a.equals("path") || a.equals("treepath") || a.equals("basepath")){
jpayne@68 193 basePath=b;
jpayne@68 194 }else if(a.equalsIgnoreCase("prealloc")){
jpayne@68 195 if(b==null || Character.isLetter(b.charAt(0))){
jpayne@68 196 if(Parse.parseBoolean(b)){
jpayne@68 197 prealloc=0.78f;
jpayne@68 198 }else{
jpayne@68 199 prealloc=0;
jpayne@68 200 }
jpayne@68 201 }else{
jpayne@68 202 prealloc=Float.parseFloat(b);
jpayne@68 203 }
jpayne@68 204 SketchObject.prealloc=prealloc;
jpayne@68 205 }else if(searcher.parse(arg, a, b, true)){
jpayne@68 206 //do nothing
jpayne@68 207 }else if(parser.parse(arg, a, b)){//Parse standard flags in the parser
jpayne@68 208 //do nothing
jpayne@68 209 }else{
jpayne@68 210 throw new RuntimeException(arg);
jpayne@68 211 }
jpayne@68 212 }
jpayne@68 213 if("auto".equalsIgnoreCase(imgFile)){imgFile=TaxTree.defaultImgFile();}
jpayne@68 214 if("auto".equalsIgnoreCase(taxTreeFile)){taxTreeFile=TaxTree.defaultTreeFile();}
jpayne@68 215 if("auto".equalsIgnoreCase(giTableFile)){giTableFile=TaxTree.defaultTableFile();}
jpayne@68 216 if("auto".equalsIgnoreCase(accessionFile)){accessionFile=TaxTree.defaultAccessionFile();}
jpayne@68 217 if("auto".equalsIgnoreCase(patternFile)){patternFile=TaxTree.defaultPatternFile();}
jpayne@68 218 if("auto".equalsIgnoreCase(sizeFile)){sizeFile=TaxTree.defaultSizeFile();}
jpayne@68 219
jpayne@68 220 serverNum=AccessionToTaxid.serverNum=serverNum_;
jpayne@68 221 serverCount=AccessionToTaxid.serverCount=serverCount_;
jpayne@68 222 distributed=AccessionToTaxid.distributed=serverCount>1;
jpayne@68 223 assert(serverNum<serverCount && serverNum>=0);
jpayne@68 224 if(distributed && serverNum==0){
jpayne@68 225 assert(slaveAddress!=null);
jpayne@68 226 assert(slaveAddress.size()==serverCount);
jpayne@68 227 for(int i=1; i<slaveAddress.size(); i++){
jpayne@68 228 assert(slaveAddress.get(i)!=null);
jpayne@68 229 }
jpayne@68 230 }
jpayne@68 231
jpayne@68 232 maxConcurrentSketchCompareThreads=Tools.mid(1, maxConcurrentSketchCompareThreads, Shared.threads());
jpayne@68 233 maxConcurrentSketchLoadThreads=Tools.mid(1, maxConcurrentSketchLoadThreads, Shared.threads());
jpayne@68 234 assert(maxConcurrentSketchCompareThreads>=1);
jpayne@68 235 assert(maxConcurrentSketchLoadThreads>=1);
jpayne@68 236
jpayne@68 237 if(basePath==null || basePath.trim().length()==0){basePath="";}
jpayne@68 238 else{
jpayne@68 239 basePath=basePath.trim().replace('\\', '/').replaceAll("/+", "/");
jpayne@68 240 if(!basePath.endsWith("/")){basePath=basePath+"/";}
jpayne@68 241 }
jpayne@68 242
jpayne@68 243 //Adjust SketchSearch rcomp and amino flags
jpayne@68 244 SketchObject.postParse();
jpayne@68 245
jpayne@68 246 if(sketchOnly){
jpayne@68 247 // hashNames=false;
jpayne@68 248 giTableFile=null;
jpayne@68 249 accessionFile=null;
jpayne@68 250 imgFile=null;
jpayne@68 251 patternFile=null;
jpayne@68 252 }
jpayne@68 253
jpayne@68 254 port=port_;
jpayne@68 255 killCode=killCode_;
jpayne@68 256 allowRemoteFileAccess=allowRemoteFileAccess_;
jpayne@68 257 allowLocalHost=allowLocalHost_;
jpayne@68 258 addressPrefix=addressPrefix_;
jpayne@68 259
jpayne@68 260 //Fill some data objects
jpayne@68 261 USAGE=makeUsagePrefix();
jpayne@68 262 rawHtml=(useHtml ? loadRawHtml() : null);
jpayne@68 263 typeMap=makeTypeMap();
jpayne@68 264 commonMap=makeCommonMap();
jpayne@68 265
jpayne@68 266 //Load the GI table
jpayne@68 267 if(giTableFile!=null){
jpayne@68 268 outstream.println("Loading gi table.");
jpayne@68 269 GiToTaxid.initialize(giTableFile);
jpayne@68 270 }
jpayne@68 271
jpayne@68 272 //Load the taxTree
jpayne@68 273 if(taxTreeFile!=null){
jpayne@68 274 tree=TaxTree.loadTaxTree(taxTreeFile, outstream, hashNames, hashDotFormat);
jpayne@68 275 if(hashNames){tree.hashChildren();}
jpayne@68 276 assert(tree.nameMap!=null || sketchOnly);
jpayne@68 277 }else{//The tree is required
jpayne@68 278 tree=null;
jpayne@68 279 throw new RuntimeException("No tree specified.");
jpayne@68 280 }
jpayne@68 281 //Set a default taxtree for sketch-related usage
jpayne@68 282 SketchObject.taxtree=tree;
jpayne@68 283
jpayne@68 284 if(sizeFile!=null){
jpayne@68 285 Timer t=new Timer();
jpayne@68 286 outstream.println("Loading size file.");
jpayne@68 287 tree.loadSizeFile(sizeFile);
jpayne@68 288 t.stopAndPrint();
jpayne@68 289 }
jpayne@68 290
jpayne@68 291 if(imgFile!=null){
jpayne@68 292 TaxTree.loadIMG(imgFile, false, outstream);
jpayne@68 293 }
jpayne@68 294
jpayne@68 295 if(patternFile!=null){
jpayne@68 296 Timer t=new Timer();
jpayne@68 297 AnalyzeAccession.loadCodeMap(patternFile);
jpayne@68 298 outstream.println("Loading pattern table.");
jpayne@68 299 t.stopAndPrint();
jpayne@68 300 }
jpayne@68 301
jpayne@68 302 //Load accession files
jpayne@68 303 if(accessionFile!=null){
jpayne@68 304 Timer t=new Timer();
jpayne@68 305 AccessionToTaxid.tree=tree;
jpayne@68 306 AccessionToTaxid.prealloc=prealloc;
jpayne@68 307 outstream.println("Loading accession table.");
jpayne@68 308 AccessionToTaxid.load(accessionFile);
jpayne@68 309 t.stopAndPrint();
jpayne@68 310 // if(searcher.refFiles.isEmpty()){System.gc();}
jpayne@68 311 }
jpayne@68 312
jpayne@68 313 // assert(false) : searcher.refFileCount();
jpayne@68 314
jpayne@68 315 //Load reference sketches
jpayne@68 316 hasSketches=searcher.refFileCount()>0;
jpayne@68 317 if(hasSketches){
jpayne@68 318 outstream.println("Loading sketches.");
jpayne@68 319 Timer t=new Timer();
jpayne@68 320 searcher.loadReferences(SketchObject.PER_TAXA, SketchObject.defaultParams);
jpayne@68 321 t.stopAndPrint();
jpayne@68 322 // System.gc();
jpayne@68 323 }
jpayne@68 324
jpayne@68 325 SketchObject.allowMultithreadedFastq=(maxConcurrentSketchLoadThreads>1);
jpayne@68 326 SketchObject.defaultParams.maxReads=defaultSketchReads;
jpayne@68 327 ReadWrite.USE_UNPIGZ=false;
jpayne@68 328 // ReadWrite.USE_UNBGZIP=false;
jpayne@68 329
jpayne@68 330 if(clearMem){
jpayne@68 331 System.err.println("Clearing memory.");
jpayne@68 332 System.gc();
jpayne@68 333 Shared.printMemory();
jpayne@68 334 }
jpayne@68 335
jpayne@68 336 //If there is a kill code, kill the old instance
jpayne@68 337 if(oldKillCode!=null && oldAddress!=null){
jpayne@68 338 outstream.println("Killing old instance.");
jpayne@68 339 killOldInstance();
jpayne@68 340 }
jpayne@68 341
jpayne@68 342 //Wait for server initialization
jpayne@68 343 httpServer=initializeServer(1000, 8, https);
jpayne@68 344 assert(httpServer!=null);
jpayne@68 345
jpayne@68 346 //Initialize handlers
jpayne@68 347 if(!sketchOnly){
jpayne@68 348 httpServer.createContext("/", new TaxHandler(false));
jpayne@68 349 httpServer.createContext("/tax", new TaxHandler(false));
jpayne@68 350 httpServer.createContext("/stax", new TaxHandler(true));
jpayne@68 351 httpServer.createContext("/simpletax", new TaxHandler(true));
jpayne@68 352 }else{
jpayne@68 353 httpServer.createContext("/", new SketchHandler());
jpayne@68 354 }
jpayne@68 355 httpServer.createContext("/sketch", new SketchHandler());
jpayne@68 356 if(killCode!=null){
jpayne@68 357 httpServer.createContext("/kill", new KillHandler());
jpayne@68 358 }
jpayne@68 359
jpayne@68 360 httpServer.createContext("/help", new HelpHandler());
jpayne@68 361 httpServer.createContext("/usage", new HelpHandler());
jpayne@68 362 httpServer.createContext("/stats", new StatsHandler());
jpayne@68 363 httpServer.createContext("/favicon.ico", new IconHandler());
jpayne@68 364
jpayne@68 365 handlerThreads=handlerThreads>0 ? handlerThreads : Tools.max(2, Shared.threads());
jpayne@68 366 httpServer.setExecutor(java.util.concurrent.Executors.newFixedThreadPool(handlerThreads)); // Creates a multithreaded executor
jpayne@68 367 // httpServer.setExecutor(java.util.concurrent.Executors.newCachedThreadPool()); // Creates a multithreaded executor
jpayne@68 368 // httpServer.setExecutor(null); // Creates a singlethreaded executor
jpayne@68 369
jpayne@68 370 //Start the server
jpayne@68 371 httpServer.start();
jpayne@68 372 }
jpayne@68 373
jpayne@68 374 /** Kill a prior server instance */
jpayne@68 375 private void killOldInstance(){
jpayne@68 376 StringNum result=null;
jpayne@68 377 try {
jpayne@68 378 result=ServerTools.sendAndReceive(oldKillCode.getBytes(), oldAddress);
jpayne@68 379 } catch (Exception e) {
jpayne@68 380 // TODO Auto-generated catch block
jpayne@68 381 if(e!=null){e.printStackTrace();}
jpayne@68 382 System.err.println("\nException suppressed; continuing.\n");
jpayne@68 383 return;
jpayne@68 384 }
jpayne@68 385 if(result==null || result.s==null || !"Success.".equals(result.s)){
jpayne@68 386 // KillSwitch.kill("Bad kill result: "+result+"\nQuitting.\n");
jpayne@68 387 System.err.println("Bad kill result: "+result+"\nContinuing.\n");
jpayne@68 388 }
jpayne@68 389 ServerTools.pause(1000);
jpayne@68 390 }
jpayne@68 391
jpayne@68 392 /** Iterative wait for server initialization */
jpayne@68 393 private HttpServer initializeServer(int millis0, int iterations, boolean https){
jpayne@68 394 HttpServer server=null;
jpayne@68 395 InetSocketAddress isa=new InetSocketAddress(port);
jpayne@68 396 Exception ee=null;
jpayne@68 397 for(int i=0, millis=millis0; i<iterations && server==null; i++){
jpayne@68 398 try {
jpayne@68 399 if(https){
jpayne@68 400 server=HttpsServer.create(isa, 0);
jpayne@68 401 }else{
jpayne@68 402 server=HttpServer.create(isa, 0);
jpayne@68 403 }
jpayne@68 404 } catch (java.net.BindException e) {//Expected
jpayne@68 405 System.err.println(e);
jpayne@68 406 System.err.println("\nWaiting "+millis+" ms");
jpayne@68 407 ee=e;
jpayne@68 408 ServerTools.pause(millis);
jpayne@68 409 millis=millis*2;
jpayne@68 410 } catch (IOException e) {//Not sure when this would occur... it would be unexpected
jpayne@68 411 System.err.println(e);
jpayne@68 412 System.err.println("\nWaiting "+millis+" ms");
jpayne@68 413 ee=e;
jpayne@68 414 ServerTools.pause(millis);
jpayne@68 415 millis=millis*2;
jpayne@68 416 }
jpayne@68 417 }
jpayne@68 418 if(server==null){throw new RuntimeException(ee);}
jpayne@68 419 return server;
jpayne@68 420 }
jpayne@68 421
jpayne@68 422 public void returnUsage(long startTime, HttpExchange t){
jpayne@68 423 if(useHtml){
jpayne@68 424 returnUsageHtml(startTime, t);
jpayne@68 425 return;
jpayne@68 426 }
jpayne@68 427 if(logUsage){System.err.println("usage");}
jpayne@68 428 // String usage=USAGE(USAGE);
jpayne@68 429 bytesOut.addAndGet(USAGE.length());
jpayne@68 430 ServerTools.reply(USAGE, "text/plain", t, verbose2, 200, true);
jpayne@68 431 final long stopTime=System.nanoTime();
jpayne@68 432 final long elapsed=stopTime-startTime;
jpayne@68 433 timeMeasurementsUsage.incrementAndGet();
jpayne@68 434 elapsedTimeUsage.addAndGet(elapsed);
jpayne@68 435 lastTimeUsage.set(elapsed);
jpayne@68 436 }
jpayne@68 437
jpayne@68 438 public void returnUsageHtml(long startTime, HttpExchange t){
jpayne@68 439 if(logUsage){System.err.println("usage");}
jpayne@68 440 String s=makeUsageHtml();
jpayne@68 441 bytesOut.addAndGet(s.length());
jpayne@68 442 ServerTools.reply(s, "html", t, verbose2, 200, true);
jpayne@68 443 final long stopTime=System.nanoTime();
jpayne@68 444 final long elapsed=stopTime-startTime;
jpayne@68 445 timeMeasurementsUsage.incrementAndGet();
jpayne@68 446 elapsedTimeUsage.addAndGet(elapsed);
jpayne@68 447 lastTimeUsage.set(elapsed);
jpayne@68 448 }
jpayne@68 449
jpayne@68 450 public void returnStats(long startTime, HttpExchange t){
jpayne@68 451 if(logUsage){System.err.println("stats");}
jpayne@68 452 String stats=makeStats();
jpayne@68 453 bytesOut.addAndGet(stats.length());
jpayne@68 454 ServerTools.reply(stats, "text/plain", t, verbose2, 200, true);
jpayne@68 455 final long stopTime=System.nanoTime();
jpayne@68 456 final long elapsed=stopTime-startTime;
jpayne@68 457 timeMeasurementsUsage.incrementAndGet();
jpayne@68 458 elapsedTimeUsage.addAndGet(elapsed);
jpayne@68 459 lastTimeUsage.set(elapsed);
jpayne@68 460 }
jpayne@68 461
jpayne@68 462 /*--------------------------------------------------------------*/
jpayne@68 463 /*---------------- Handlers ----------------*/
jpayne@68 464 /*--------------------------------------------------------------*/
jpayne@68 465
jpayne@68 466 /** Handles queries for favicon.ico */
jpayne@68 467 class IconHandler implements HttpHandler {
jpayne@68 468
jpayne@68 469 @Override
jpayne@68 470 public void handle(HttpExchange t) throws IOException {
jpayne@68 471 if(verbose2){System.err.println("Icon handler");}
jpayne@68 472 iconQueries.incrementAndGet();
jpayne@68 473 ServerTools.reply(favIcon, "image/x-icon", t, verbose2, 200, true);
jpayne@68 474 }
jpayne@68 475
jpayne@68 476 }
jpayne@68 477
jpayne@68 478 /*--------------------------------------------------------------*/
jpayne@68 479
jpayne@68 480 /** Handles queries that fall through other handlers */
jpayne@68 481 class HelpHandler implements HttpHandler {
jpayne@68 482
jpayne@68 483 @Override
jpayne@68 484 public void handle(HttpExchange t) throws IOException {
jpayne@68 485 if(verbose2){System.err.println("Help handler");}
jpayne@68 486 final long startTime=System.nanoTime();
jpayne@68 487 returnUsage(startTime, t);
jpayne@68 488 }
jpayne@68 489
jpayne@68 490 }
jpayne@68 491
jpayne@68 492 /*--------------------------------------------------------------*/
jpayne@68 493
jpayne@68 494 /** Handles queries that fall through other handlers */
jpayne@68 495 class StatsHandler implements HttpHandler {
jpayne@68 496
jpayne@68 497 @Override
jpayne@68 498 public void handle(HttpExchange t) throws IOException {
jpayne@68 499 if(verbose2){System.err.println("Http handler");}
jpayne@68 500 final long startTime=System.nanoTime();
jpayne@68 501 returnStats(startTime, t);
jpayne@68 502 }
jpayne@68 503
jpayne@68 504 }
jpayne@68 505
jpayne@68 506 /*--------------------------------------------------------------*/
jpayne@68 507
jpayne@68 508 /** Handles requests to kill the server */
jpayne@68 509 class KillHandler implements HttpHandler {
jpayne@68 510
jpayne@68 511 @Override
jpayne@68 512 public void handle(HttpExchange t) throws IOException {
jpayne@68 513 if(verbose2){System.err.println("Kill handler");}
jpayne@68 514
jpayne@68 515 //Parse the query from the URL
jpayne@68 516 String rparam=getRParam(t, false);
jpayne@68 517 InetSocketAddress remote=t.getRemoteAddress();
jpayne@68 518
jpayne@68 519 if(testCode(t, rparam)){
jpayne@68 520 ServerTools.reply("Success.", "text/plain", t, verbose2, 200, true);
jpayne@68 521 System.err.println("Killed by remote address "+remote);
jpayne@68 522 //TODO: Perhaps try to close open resources such as the server
jpayne@68 523 KillSwitch.killSilent();
jpayne@68 524 }
jpayne@68 525
jpayne@68 526 if(verbose){System.err.println("Bad kill from address "+remote);}
jpayne@68 527 ServerTools.reply(BAD_CODE, "text/plain", t, verbose2, 403, true);
jpayne@68 528 }
jpayne@68 529
jpayne@68 530 /** Determines whether kill code was correct */
jpayne@68 531 private boolean testCode(HttpExchange t, String rparam){
jpayne@68 532 String[] params = rparam.split("/");
jpayne@68 533 if(verbose2){System.err.println(Arrays.toString(params));}
jpayne@68 534
jpayne@68 535 if(killCode!=null){
jpayne@68 536 if(params.length>1){//URL mode
jpayne@68 537 return (params[1].equals(killCode));
jpayne@68 538 }else{//Body mode
jpayne@68 539 try {
jpayne@68 540 String code=ServerTools.receive(t);
jpayne@68 541 return (code!=null && code.equals(killCode));
jpayne@68 542 } catch (Exception e) {
jpayne@68 543 // TODO Auto-generated catch block
jpayne@68 544 e.printStackTrace();
jpayne@68 545 }
jpayne@68 546 }
jpayne@68 547 }
jpayne@68 548 return false;
jpayne@68 549 }
jpayne@68 550 }
jpayne@68 551
jpayne@68 552 /*--------------------------------------------------------------*/
jpayne@68 553
jpayne@68 554 /** Listens for sketch comparison requests */
jpayne@68 555 class SketchHandler implements HttpHandler {
jpayne@68 556
jpayne@68 557 @Override
jpayne@68 558 public void handle(HttpExchange t) throws IOException {
jpayne@68 559 if(verbose2){outstream.println("Got a request.");}
jpayne@68 560 SketchInstance si=new SketchInstance(t);
jpayne@68 561 if(verbose2){outstream.println("Made si.");}
jpayne@68 562 si.handleInner();
jpayne@68 563 if(verbose2){outstream.println("Done.");}
jpayne@68 564 }
jpayne@68 565
jpayne@68 566 private ArrayList<Sketch> loadSketchesFromBody(String body){
jpayne@68 567 //List of query sketches
jpayne@68 568 ArrayList<Sketch> sketches=null;
jpayne@68 569
jpayne@68 570 if(body!=null && body.length()>0){
jpayne@68 571 sketches=searcher.loadSketchesFromString(body);
jpayne@68 572 if(Whitelist.exists()){
jpayne@68 573 for(Sketch sk : sketches){
jpayne@68 574 Whitelist.apply(sk);
jpayne@68 575 }
jpayne@68 576 }
jpayne@68 577 }
jpayne@68 578 return sketches;
jpayne@68 579 }
jpayne@68 580
jpayne@68 581 private ArrayList<Sketch> loadSketchesFromFile(String fname, DisplayParams params){
jpayne@68 582 //List of query sketches
jpayne@68 583 ArrayList<Sketch> sketches=null;
jpayne@68 584
jpayne@68 585 SketchTool tool=searcher.tool;
jpayne@68 586 if(tool.minKeyOccuranceCount!=params.minKeyOccuranceCount || params.trackCounts()){
jpayne@68 587 tool=new SketchTool(SketchObject.targetSketchSize, params);
jpayne@68 588 }
jpayne@68 589
jpayne@68 590 if(verbose2){System.err.println("Loading sketches from file "+fname);}
jpayne@68 591 sketches=tool.loadSketchesFromFile(fname, (SketchMakerMini)null, maxConcurrentSketchLoadThreads, params.maxReads, params.mode, params, true);
jpayne@68 592 if(verbose2){System.err.println("Loaded "+(sketches==null ? "null" : sketches.size())+" sketches from file "+fname);}
jpayne@68 593 return sketches;
jpayne@68 594 }
jpayne@68 595
jpayne@68 596 //Created in handle()
jpayne@68 597 private class SketchInstance {
jpayne@68 598
jpayne@68 599 SketchInstance(HttpExchange t_){
jpayne@68 600 t=t_;
jpayne@68 601 instanceStartTime=System.nanoTime();
jpayne@68 602 }
jpayne@68 603
jpayne@68 604 void handleInner(){
jpayne@68 605
jpayne@68 606 if(!hasSketches){
jpayne@68 607 if(verbose2){System.err.println("No sketches.");}
jpayne@68 608 ServerTools.reply("\nERROR: This server has no sketches loaded.\n"
jpayne@68 609 + "Please download the latest BBTools version to use SendSketch.\n", "text/plain", t, verbose2, 400, true);
jpayne@68 610 return;
jpayne@68 611 }
jpayne@68 612
jpayne@68 613 String rparam=parseRparamSketch(t);
jpayne@68 614 if(verbose2){System.err.println("Parsed rparam.");}
jpayne@68 615 if(rparam==null){
jpayne@68 616 returnUsage(instanceStartTime, t);
jpayne@68 617 return;
jpayne@68 618 }
jpayne@68 619 final boolean internal=incrementQueries(t, fileMode, refMode, false, false, false, false, false, false, false, false, -1);
jpayne@68 620 if(verbose2){System.err.println("Incremented queries rparam.");}
jpayne@68 621
jpayne@68 622 if(verbose2){System.err.println(rparam);}
jpayne@68 623 if(verbose2){System.err.println("fileMode="+fileMode+", refMode="+refMode);}
jpayne@68 624
jpayne@68 625 if(fileMode && !internal && !allowRemoteFileAccess){
jpayne@68 626 if(verbose){System.err.println("Illegal file query from "+ServerTools.getClientAddress(t));}
jpayne@68 627 malformedQueries.incrementAndGet();
jpayne@68 628 // if(verbose){System.err.println("test1");}
jpayne@68 629 // String body=getBody(t);//123
jpayne@68 630 ServerTools.reply("\nERROR: This server does not allow remote file access. "
jpayne@68 631 + "You may only use the 'local' flag from with the local intranet.\n", "text/plain", t, verbose2, 400, true);
jpayne@68 632 // if(verbose){System.err.println("test2");}
jpayne@68 633 return;
jpayne@68 634 }
jpayne@68 635
jpayne@68 636 String body=getBody(t);
jpayne@68 637 if(verbose2){System.err.println("Got body.");}
jpayne@68 638
jpayne@68 639 if(body!=null){bytesIn.addAndGet(body.length());}
jpayne@68 640
jpayne@68 641 if(verbose2){System.err.println("Found body: "+body);}
jpayne@68 642 if(body!=null && body.length()>0){
jpayne@68 643 if((fileMode || refMode) && !body.startsWith("##")){
jpayne@68 644 body="##"+body;
jpayne@68 645 }
jpayne@68 646 try {
jpayne@68 647 params=params.parseDoubleHeader(body);
jpayne@68 648 if(verbose2){System.err.println("Passed parse params.");}
jpayne@68 649
jpayne@68 650 } catch (Throwable e) {
jpayne@68 651 String s=Tools.toString(e);
jpayne@68 652 ServerTools.reply("\nERROR: \n"+ s,
jpayne@68 653 "text/plain", t, verbose2, 400, true);
jpayne@68 654 return;
jpayne@68 655 }
jpayne@68 656 if(!params.compatible()){
jpayne@68 657 ServerTools.reply("\nERROR: The sketch is not compatible with this server.\n"
jpayne@68 658 + "Server settings: k="+SketchObject.k+(SketchObject.k2>0 ? ","+SketchObject.k2 : "")
jpayne@68 659 +" amino="+SketchObject.amino+" hash_version="+SketchObject.HASH_VERSION+"\n"
jpayne@68 660 + "You may need to download a newer version of BBTools; this server is running version "+Shared.BBMAP_VERSION_STRING,
jpayne@68 661 "text/plain", t, verbose2, 400, true);
jpayne@68 662 return;
jpayne@68 663 }
jpayne@68 664 }
jpayne@68 665 if(params.trackCounts()){
jpayne@68 666 depthQueries.incrementAndGet();
jpayne@68 667 }
jpayne@68 668
jpayne@68 669 if(verbose2){System.err.println("Parsed params: "+params.toString());}
jpayne@68 670
jpayne@68 671 //List of query sketches
jpayne@68 672 ArrayList<Sketch> sketches;
jpayne@68 673
jpayne@68 674 if(fileMode){
jpayne@68 675 File f=new File(rparam);
jpayne@68 676 if(!f.exists() && !rparam.startsWith("/")){
jpayne@68 677 String temp="/"+rparam;
jpayne@68 678 f=new File(temp);
jpayne@68 679 if(f.exists()){rparam=temp;}
jpayne@68 680 }
jpayne@68 681 // if(f.exists()){
jpayne@68 682 // if(f.length()>100000000L){
jpayne@68 683 // if(params.reads<0){
jpayne@68 684 // params.reads=200000;//Cap default number of reads at 200000
jpayne@68 685 // }
jpayne@68 686 // }
jpayne@68 687 // }
jpayne@68 688 sketches=loadSketchesFromFile(rparam, params);
jpayne@68 689 }else if(refMode){
jpayne@68 690 String[] split=rparam.split(",");
jpayne@68 691 sketches=new ArrayList<Sketch>(split.length);
jpayne@68 692 for(String s : split){
jpayne@68 693 Sketch sk=findRefSketch(s);
jpayne@68 694 if(sk!=null){sketches.add(sk);}
jpayne@68 695 }
jpayne@68 696 }else{
jpayne@68 697 sketches=loadSketchesFromBody(body);
jpayne@68 698 }
jpayne@68 699 if(verbose2){System.err.println("Loaded "+sketches.size()+" sketches.");}
jpayne@68 700
jpayne@68 701 final int numSketches=sketches==null ? 0 : sketches.size();
jpayne@68 702 if(params.chunkNum<0){
jpayne@68 703 if(numSketches<2){
jpayne@68 704 unknownChunkSingle.incrementAndGet();
jpayne@68 705 }else{
jpayne@68 706 unknownChunkMulti.incrementAndGet();
jpayne@68 707 }
jpayne@68 708 }else if(params.chunkNum==0){
jpayne@68 709 if(numSketches<2){
jpayne@68 710 firstChunkSingle.incrementAndGet();
jpayne@68 711 }else{
jpayne@68 712 firstChunkMulti.incrementAndGet();
jpayne@68 713 }
jpayne@68 714 }else{
jpayne@68 715 if(numSketches<2){
jpayne@68 716 nthChunkSingle.incrementAndGet();
jpayne@68 717 }else{
jpayne@68 718 nthChunkMulti.incrementAndGet();
jpayne@68 719 }
jpayne@68 720 }
jpayne@68 721 bulkCount.addAndGet(numSketches);
jpayne@68 722
jpayne@68 723 if(params.inputVersion==null){params.inputVersion="unknown";}
jpayne@68 724 synchronized(versionMap){
jpayne@68 725 StringNum sn=versionMap.get(params.inputVersion);
jpayne@68 726 if(sn==null){versionMap.put(params.inputVersion, new StringNum(params.inputVersion, 1));}
jpayne@68 727 else{sn.increment();}
jpayne@68 728 }
jpayne@68 729
jpayne@68 730 String response=null;
jpayne@68 731 if(sketches==null || sketches.isEmpty()){
jpayne@68 732 malformedQueries.incrementAndGet();
jpayne@68 733 response="Error.";
jpayne@68 734 if(verbose){
jpayne@68 735 StringBuilder sb=new StringBuilder();
jpayne@68 736 sb.append("Malformed query from ").append(ServerTools.getClientAddress(t)).append(". body:");
jpayne@68 737 if(body==null){
jpayne@68 738 sb.append(" null");
jpayne@68 739 }else{
jpayne@68 740 String[] split = body.split("\n");
jpayne@68 741 sb.append(" ").append(split.length).append(" lines total, displaying ").append(Tools.min(3, split.length)).append('.');
jpayne@68 742 for(int i=0; i<3 && i<split.length; i++){
jpayne@68 743 String s=split[i];
jpayne@68 744 int len=s.length();
jpayne@68 745 if(s.length()>1000){s=s.substring(0, 1000)+" [truncated, "+len+" total]";}
jpayne@68 746 sb.append('\n');
jpayne@68 747 sb.append(s);
jpayne@68 748 }
jpayne@68 749 }
jpayne@68 750 System.err.println(sb);
jpayne@68 751 }
jpayne@68 752 }else{
jpayne@68 753 if(verbose2){
jpayne@68 754 System.err.println("Received "+sketches.get(0).name()+", size "+sketches.get(0).keys.length);
jpayne@68 755 System.err.println("params: "+params);
jpayne@68 756 System.err.println("postparsed: "+params.postParsed());
jpayne@68 757 System.err.println("taxwhitelist: "+params.taxFilterWhite);
jpayne@68 758 }
jpayne@68 759 response=compare(sketches, params);
jpayne@68 760 // searcher.compare(sketches, response, params, maxConcurrentSketchCompareThreads); //This is where it gets stuck if comparing takes too long
jpayne@68 761 if(verbose2){System.err.println("Result: '"+response+"'");}
jpayne@68 762 }
jpayne@68 763
jpayne@68 764 bytesOut.addAndGet(response.length());
jpayne@68 765 ServerTools.reply(response, "text/plain", t, verbose2, 200, true);
jpayne@68 766
jpayne@68 767 final long stopTime=System.nanoTime();
jpayne@68 768 final long elapsed=stopTime-instanceStartTime;
jpayne@68 769 if(fileMode){
jpayne@68 770 timeMeasurementsLocal.incrementAndGet();
jpayne@68 771 elapsedTimeLocal.addAndGet(elapsed);
jpayne@68 772 lastTimeLocal.set(elapsed);
jpayne@68 773 }else if(refMode){
jpayne@68 774 timeMeasurementsReference.incrementAndGet();
jpayne@68 775 elapsedTimeReference.addAndGet(elapsed);
jpayne@68 776 lastTimeReference.set(elapsed);
jpayne@68 777 }else{
jpayne@68 778 timeMeasurementsRemote.incrementAndGet();
jpayne@68 779 elapsedTimeRemote.addAndGet(elapsed);
jpayne@68 780 lastTimeRemote.set(elapsed);
jpayne@68 781
jpayne@68 782 queryCounts.incrementAndGet(Tools.min(numSketches, queryCounts.length()-1));
jpayne@68 783 timesByCount.addAndGet(Tools.min(numSketches, queryCounts.length()-1), elapsed);
jpayne@68 784 }
jpayne@68 785 }
jpayne@68 786
jpayne@68 787 private String parseRparamSketch(HttpExchange t){
jpayne@68 788 //Parse the query from the URL
jpayne@68 789 String rparam=getRParam(t, false);
jpayne@68 790 if(rparam!=null){bytesIn.addAndGet(rparam.length());}
jpayne@68 791
jpayne@68 792 if(rparam.length()<1 || rparam.equalsIgnoreCase("help") || rparam.equalsIgnoreCase("usage") || rparam.equalsIgnoreCase("help/") || rparam.equalsIgnoreCase("usage/")){
jpayne@68 793 return null;
jpayne@68 794 }
jpayne@68 795
jpayne@68 796 if(rparam.startsWith("sketch/")){rparam=rparam.substring(7);}
jpayne@68 797 else if(rparam.equals("sketch")){rparam="";}
jpayne@68 798 while(rparam.startsWith("/")){rparam=rparam.substring(1);}
jpayne@68 799
jpayne@68 800 //Toggle between local files and sketch transmission
jpayne@68 801
jpayne@68 802 if(rparam.length()<2){
jpayne@68 803 params=SketchObject.defaultParams;
jpayne@68 804 }else{
jpayne@68 805 params=SketchObject.defaultParams.clone();
jpayne@68 806 String[] args=rparam.split("/");
jpayne@68 807 int trimmed=0;
jpayne@68 808 for(int i=0; i<args.length; i++){//parse rparam
jpayne@68 809 String arg=args[i];
jpayne@68 810 if(arg.length()>0){
jpayne@68 811 String[] split=arg.split("=");
jpayne@68 812 String a=split[0].toLowerCase();
jpayne@68 813 String b=split.length>1 ? split[1] : null;
jpayne@68 814
jpayne@68 815 if(a.equals("file")){
jpayne@68 816 fileMode=true;
jpayne@68 817 trimmed+=5;
jpayne@68 818 break;
jpayne@68 819 }else if(a.equals("ref") || a.equals("taxid") || a.equals("tid")){
jpayne@68 820 refMode=true;
jpayne@68 821 trimmed+=4;
jpayne@68 822 break;
jpayne@68 823 }else if(params.parse(arg, a, b)){
jpayne@68 824 trimmed+=arg.length()+1;
jpayne@68 825 }else{
jpayne@68 826 assert(false) : "Bad argument:'"+arg+"'"+"\n"+Arrays.toString(args)+"\n"+rparam;
jpayne@68 827 }
jpayne@68 828 }
jpayne@68 829 }
jpayne@68 830 params.postParse(true, true);
jpayne@68 831 // System.err.println("Trimmed="+trimmed+", rparam="+rparam);
jpayne@68 832 if(trimmed>0){
jpayne@68 833 rparam=rparam.substring(Tools.min(trimmed, rparam.length()));
jpayne@68 834 }
jpayne@68 835 // System.err.println("rparam="+rparam);
jpayne@68 836 }
jpayne@68 837
jpayne@68 838 if(verbose2){
jpayne@68 839 System.err.println(rparam);
jpayne@68 840 System.err.println("rparam.startsWith(\"file/\"):"+rparam.startsWith("file/"));
jpayne@68 841 }
jpayne@68 842
jpayne@68 843 return rparam;
jpayne@68 844 }
jpayne@68 845
jpayne@68 846 private final HttpExchange t;
jpayne@68 847 private DisplayParams params;
jpayne@68 848 private final long instanceStartTime;
jpayne@68 849 private boolean fileMode=false;
jpayne@68 850 private boolean refMode=false;
jpayne@68 851 }
jpayne@68 852
jpayne@68 853 }
jpayne@68 854
jpayne@68 855 private Sketch findRefSketch(String s){
jpayne@68 856 assert(s!=null);
jpayne@68 857 if(s==null || s.length()<1){return null;}
jpayne@68 858 int tid=-1;
jpayne@68 859 if(Tools.isDigit(s.charAt(0))){tid=Integer.parseInt(s);}
jpayne@68 860 else{
jpayne@68 861 TaxNode tn=getTaxNodeByName(s);
jpayne@68 862 tid=tn==null ? -1 : tn.id;
jpayne@68 863 }
jpayne@68 864 Sketch sk=tid<0 ? null : searcher.findReferenceSketch(tid);
jpayne@68 865 if(sk!=null){sk=(Sketch)sk.clone();}
jpayne@68 866 return sk;
jpayne@68 867 }
jpayne@68 868
jpayne@68 869 /*--------------------------------------------------------------*/
jpayne@68 870
jpayne@68 871 /** Handles taxonomy lookups */
jpayne@68 872 class TaxHandler implements HttpHandler {
jpayne@68 873
jpayne@68 874 public TaxHandler(boolean skipNonCanonical_){
jpayne@68 875 skipNonCanonical=skipNonCanonical_;
jpayne@68 876 }
jpayne@68 877
jpayne@68 878 @Override
jpayne@68 879 public void handle(HttpExchange t) throws IOException {
jpayne@68 880 if(verbose2){System.err.println("Tax handler");}
jpayne@68 881 final long startTime=System.nanoTime();
jpayne@68 882
jpayne@68 883 if(sketchOnly){
jpayne@68 884 ServerTools.reply("\nERROR: This server is tunning in sketch mode and should not be used for taxonomic lookups.\n"
jpayne@68 885 + "The taxonomy server is at "+Shared.taxServer()+"\n", "text/plain", t, verbose2, 400, true);
jpayne@68 886 return;
jpayne@68 887 }
jpayne@68 888
jpayne@68 889 //Parse the query from the URL
jpayne@68 890 String rparam=getRParam(t, true);
jpayne@68 891
jpayne@68 892
jpayne@68 893 boolean simple=skipNonCanonical;
jpayne@68 894
jpayne@68 895 {//Legacy support for old style of invoking simple
jpayne@68 896 if(rparam.startsWith("simpletax/")){rparam=rparam.substring(7); simple=true;}
jpayne@68 897 else if(rparam.startsWith("stax/")){rparam=rparam.substring(5); simple=true;}
jpayne@68 898 else if(rparam.startsWith("tax/")){rparam=rparam.substring(4);}
jpayne@68 899 else if(rparam.equals("simpletax") || rparam.equals("stax")){rparam=""; simple=true;}
jpayne@68 900 else if(rparam.equals("tax")){rparam="";}
jpayne@68 901 }
jpayne@68 902 while(rparam.startsWith("/")){rparam=rparam.substring(1);}
jpayne@68 903 if(rparam.length()<1 || rparam.equalsIgnoreCase("help") || rparam.equalsIgnoreCase("usage")){
jpayne@68 904 returnUsage(startTime, t);
jpayne@68 905 return;
jpayne@68 906 }
jpayne@68 907
jpayne@68 908 String[] params = rparam.split("/");
jpayne@68 909 if(verbose2){System.err.println(Arrays.toString(params));}
jpayne@68 910
jpayne@68 911 final String response=toResponse(simple, params, t);
jpayne@68 912 final String type=response.startsWith("{") ? "application/json" : "text/plain";
jpayne@68 913
jpayne@68 914 ServerTools.reply(response, type, t, verbose2, 200, true);
jpayne@68 915
jpayne@68 916 final long stopTime=System.nanoTime();
jpayne@68 917 final long elapsed=stopTime-startTime;
jpayne@68 918 if(response.startsWith("Welcome to ")){
jpayne@68 919 timeMeasurementsUsage.incrementAndGet();
jpayne@68 920 elapsedTimeUsage.addAndGet(elapsed);
jpayne@68 921 lastTimeUsage.set(elapsed);
jpayne@68 922 }else{
jpayne@68 923 timeMeasurementsRemote.incrementAndGet();
jpayne@68 924 elapsedTimeRemote.addAndGet(elapsed);
jpayne@68 925 lastTimeRemote.set(elapsed);
jpayne@68 926 }
jpayne@68 927 }
jpayne@68 928
jpayne@68 929 //TODO: Integrate something like this to improve parsing
jpayne@68 930 // String parse(String rparam){
jpayne@68 931 //
jpayne@68 932 // if(rparam.length()<2){return rparam;}
jpayne@68 933 //
jpayne@68 934 // String[] args=rparam.split("/");
jpayne@68 935 // int trimmed=0;
jpayne@68 936 // for(int i=0; i<args.length; i++){//parse rparam
jpayne@68 937 // String arg=args[i];
jpayne@68 938 // if(arg.length()>0){
jpayne@68 939 // String[] split=arg.split("=");
jpayne@68 940 // String a=split[0].toLowerCase();
jpayne@68 941 // String b=split.length>1 ? split[1] : null;
jpayne@68 942 //
jpayne@68 943 // if(a.equals("file")){
jpayne@68 944 // fileMode=true;
jpayne@68 945 // trimmed+=5;
jpayne@68 946 // break;
jpayne@68 947 // }else if(params.parse(arg, a, b)){
jpayne@68 948 // trimmed+=arg.length()+1;
jpayne@68 949 // }else{
jpayne@68 950 // assert(false) : "Bad argument:'"+arg+"'"+"\n"+Arrays.toString(args)+"\n"+rparam;
jpayne@68 951 // }
jpayne@68 952 // }
jpayne@68 953 // }
jpayne@68 954 // params.postParse(true);
jpayne@68 955 // // System.err.println("Trimmed="+trimmed+", rparam="+rparam);
jpayne@68 956 // if(trimmed>0){
jpayne@68 957 // rparam=rparam.substring(Tools.min(trimmed, rparam.length()));
jpayne@68 958 // }
jpayne@68 959 // }
jpayne@68 960
jpayne@68 961 /** Only print nodes at canonical tax levels */
jpayne@68 962 public final boolean skipNonCanonical;
jpayne@68 963 }
jpayne@68 964
jpayne@68 965 /*--------------------------------------------------------------*/
jpayne@68 966 /*---------------- Helpers ----------------*/
jpayne@68 967 /*--------------------------------------------------------------*/
jpayne@68 968
jpayne@68 969 static String getBody(HttpExchange t){
jpayne@68 970 if(verbose2){System.err.println("getBody");}
jpayne@68 971 InputStream is=t.getRequestBody();
jpayne@68 972 String s=ServerTools.readStream(is);
jpayne@68 973 return s;
jpayne@68 974 }
jpayne@68 975
jpayne@68 976 /** Parse the query from the URL */
jpayne@68 977 static String getRParam(HttpExchange t, boolean allowPost){
jpayne@68 978 if(verbose2){System.err.println("getRParam");}
jpayne@68 979 String rparam = t.getRequestURI().toString();
jpayne@68 980
jpayne@68 981 //Trim leading slashes
jpayne@68 982 while(rparam.startsWith("/")){
jpayne@68 983 rparam = rparam.substring(1);
jpayne@68 984 }
jpayne@68 985
jpayne@68 986 //Trim trailing slashes
jpayne@68 987 while(rparam.endsWith("/")){
jpayne@68 988 rparam = rparam.substring(0, rparam.length()-1);
jpayne@68 989 }
jpayne@68 990
jpayne@68 991 if(allowPost && ("$POST".equalsIgnoreCase(rparam) || "POST".equalsIgnoreCase(rparam))){
jpayne@68 992 String body=getBody(t);
jpayne@68 993 rparam=body;
jpayne@68 994 }
jpayne@68 995
jpayne@68 996 if(verbose){System.err.println(rparam==null || rparam.trim().length()<1 ? "usage" : rparam+"\t"+System.currentTimeMillis());}
jpayne@68 997 return rparam;
jpayne@68 998 }
jpayne@68 999
jpayne@68 1000 /*--------------------------------------------------------------*/
jpayne@68 1001 /*---------------- Taxonomy Formatting ----------------*/
jpayne@68 1002 /*--------------------------------------------------------------*/
jpayne@68 1003
jpayne@68 1004 /** All tax queries enter here from the handler */
jpayne@68 1005 String toResponse(boolean simple, String[] params, HttpExchange t){
jpayne@68 1006 if(verbose2){System.err.println("toResponse");}
jpayne@68 1007
jpayne@68 1008 boolean printNumChildren=false;
jpayne@68 1009 boolean printChildren=false;
jpayne@68 1010 boolean printPath=false;
jpayne@68 1011 boolean printSize=false;
jpayne@68 1012 boolean printRange=false;
jpayne@68 1013 boolean silvaHeader=false;
jpayne@68 1014 boolean plaintext=false, semicolon=false, path=false;
jpayne@68 1015 boolean mononomial=false;
jpayne@68 1016 boolean ancestor=false;
jpayne@68 1017 int source=SOURCE_REFSEQ;
jpayne@68 1018
jpayne@68 1019 ArrayList<String> newParams=new ArrayList<String>(params.length);
jpayne@68 1020 for(int i=0; i<params.length-1; i++){
jpayne@68 1021 String s=params[i];
jpayne@68 1022 if(s.length()==0 || s.equals("tax")){
jpayne@68 1023 //Do nothing
jpayne@68 1024 }else if(s.equals("printnumchildren") || s.equals("numchildren")){
jpayne@68 1025 printNumChildren=true;
jpayne@68 1026 }else if(s.equals("printchildren") || s.equals("children")){
jpayne@68 1027 printChildren=true;
jpayne@68 1028 }else if(s.equals("mononomial") || s.equals("cn")
jpayne@68 1029 || s.equals("fixname") || s.equals("fn") || s.equals("mono") || s.equals("mononomial")){
jpayne@68 1030 mononomial=true;
jpayne@68 1031 }else if(s.equals("printpath") || s.equals("pp")){
jpayne@68 1032 printPath=true;
jpayne@68 1033 }else if(s.equals("printsize") || s.equals("size") || s.equals("ps")){
jpayne@68 1034 printSize=true;
jpayne@68 1035 }else if(s.equals("printrange") || s.equals("range")){
jpayne@68 1036 printRange=true;
jpayne@68 1037 }else if(s.equals("plaintext") || s.equals("pt")){
jpayne@68 1038 plaintext=true; semicolon=false; path=false;
jpayne@68 1039 }else if(s.equals("semicolon") || s.equals("sc")){
jpayne@68 1040 semicolon=true; plaintext=false; path=false;
jpayne@68 1041 }else if(s.equals("path") || s.equals("pa")){
jpayne@68 1042 path=true; semicolon=false; plaintext=false;
jpayne@68 1043 }else if(s.equals("silva")){
jpayne@68 1044 silvaHeader=true;
jpayne@68 1045 source=SOURCE_SILVA;
jpayne@68 1046 }else if(s.equals("refseq")){
jpayne@68 1047 source=SOURCE_REFSEQ;
jpayne@68 1048 }else if(s.equals("ancestor")){
jpayne@68 1049 ancestor=true;
jpayne@68 1050 }else if(s.equals("simple")){
jpayne@68 1051 simple=true;
jpayne@68 1052 }else{
jpayne@68 1053 newParams.add(s);
jpayne@68 1054 }
jpayne@68 1055 }
jpayne@68 1056 newParams.add(params[params.length-1]);
jpayne@68 1057 params=newParams.toArray(new String[newParams.size()]);
jpayne@68 1058
jpayne@68 1059 // System.err.println("mononomial="+mononomial);
jpayne@68 1060
jpayne@68 1061 if(params.length<2){
jpayne@68 1062 if(params.length==1 && "advice".equalsIgnoreCase(params[0])){return TAX_ADVICE;}
jpayne@68 1063 if(logUsage){System.err.println("usage");}
jpayne@68 1064 return USAGE(USAGE);
jpayne@68 1065 }
jpayne@68 1066 if(params.length>3){
jpayne@68 1067 if(logUsage){System.err.println("usage");}
jpayne@68 1068 return USAGE(USAGE);
jpayne@68 1069 }
jpayne@68 1070
jpayne@68 1071 final String query=params[params.length-1];
jpayne@68 1072 final String[] names=query.split(",");
jpayne@68 1073 if(names==null){return USAGE(USAGE);}
jpayne@68 1074
jpayne@68 1075 if(names==null || names.length<2){
jpayne@68 1076 firstChunkSingle.incrementAndGet();
jpayne@68 1077 }else{
jpayne@68 1078 firstChunkMulti.incrementAndGet();
jpayne@68 1079 }
jpayne@68 1080 bulkCount.addAndGet(names.length);
jpayne@68 1081 // System.err.println(params[2]+", "+ancestor);
jpayne@68 1082
jpayne@68 1083 //Raw query type code
jpayne@68 1084 final int type;
jpayne@68 1085 //Type code excluding formatting
jpayne@68 1086 final int type2;
jpayne@68 1087 {
jpayne@68 1088 String typeS=params[0];
jpayne@68 1089 Integer value=typeMap.get(typeS);
jpayne@68 1090 if(value==null){
jpayne@68 1091 if(typeS.equalsIgnoreCase("advice")){
jpayne@68 1092 return TAX_ADVICE;
jpayne@68 1093 }else{
jpayne@68 1094 return "{\"error\": \"Bad type ("+typeS+"); should be gi, taxid, or name.\"}";
jpayne@68 1095 }
jpayne@68 1096 }
jpayne@68 1097 int x=value.intValue();
jpayne@68 1098 if((x&15)==HEADER && silvaHeader){x=SILVAHEADER;}
jpayne@68 1099 type=x;
jpayne@68 1100 type2=x&15;
jpayne@68 1101 if(type2==IMG){source=SOURCE_IMG;}
jpayne@68 1102 }
jpayne@68 1103
jpayne@68 1104 plaintext=(type>=PT_BIT || plaintext);
jpayne@68 1105 semicolon=(type>=SC_BIT || semicolon);
jpayne@68 1106 path=(type>=PA_BIT || path);
jpayne@68 1107 if(semicolon || path){plaintext=false;}
jpayne@68 1108 if(path){semicolon=false;}
jpayne@68 1109
jpayne@68 1110 final boolean internal=incrementQueries(t, false, false, simple, ancestor,
jpayne@68 1111 plaintext, semicolon, path, printChildren, printPath, printSize, type); //Ignores usage information.
jpayne@68 1112
jpayne@68 1113 // if(type2==GI){//123
jpayne@68 1114 // return "{\"error\": \"GI number support is temporarily suspended due to conflicts in NCBI databases. "
jpayne@68 1115 // + "It may come back split into nucleotide and protein GI numbers, which currently are not exclusive.\"}";
jpayne@68 1116 // }
jpayne@68 1117
jpayne@68 1118 if(!internal && !allowRemoteFileAccess){
jpayne@68 1119 path=printPath=false;
jpayne@68 1120 }
jpayne@68 1121
jpayne@68 1122 if(verbose2){System.err.println("Type: "+type);}
jpayne@68 1123 if(type2==NAME || type2==HEADER || type2==SILVAHEADER){
jpayne@68 1124 for(int i=0; i<names.length; i++){
jpayne@68 1125 names[i]=PercentEncoding.codeToSymbol(names[i]);
jpayne@68 1126 if(type2==HEADER || type2==SILVAHEADER){
jpayne@68 1127 if(names[i].startsWith("@") || names[i].startsWith(">")){names[i]=names[i].substring(1);}
jpayne@68 1128 }
jpayne@68 1129 }
jpayne@68 1130 if(verbose2){System.err.println("Revised: "+Arrays.toString(names));}
jpayne@68 1131 }
jpayne@68 1132
jpayne@68 1133 if(ancestor){
jpayne@68 1134 if(verbose2){System.err.println("toAncestor: "+Arrays.toString(names));}
jpayne@68 1135 return toAncestor(type, names, plaintext, semicolon, path, query, simple, !simple, printNumChildren, printChildren, printPath, printSize, printRange, mononomial, source);
jpayne@68 1136 }
jpayne@68 1137
jpayne@68 1138 if(semicolon){
jpayne@68 1139 return toSemicolon(type, names, simple, mononomial);
jpayne@68 1140 }else if(plaintext){
jpayne@68 1141 return toText(type, names);
jpayne@68 1142 }else if(path){
jpayne@68 1143 return toPath(type, names, source);
jpayne@68 1144 }
jpayne@68 1145
jpayne@68 1146 JsonObject j=new JsonObject();
jpayne@68 1147 for(String name : names){
jpayne@68 1148 j.add(name, toJson(type, name, simple, !simple, printNumChildren, printChildren,
jpayne@68 1149 printPath, printSize, printRange, mononomial, source));
jpayne@68 1150 }
jpayne@68 1151 return j.toString();
jpayne@68 1152 }
jpayne@68 1153
jpayne@68 1154 /** Look up common ancestor of terms */
jpayne@68 1155 String toAncestor(final int type, final String[] names, boolean plaintext, boolean semicolon, boolean path,
jpayne@68 1156 String query, final boolean skipNonCanonical, boolean originalLevel, boolean printNumChildren,
jpayne@68 1157 boolean printChildren, boolean printPath, boolean printSize, boolean printRange, boolean mononomial,
jpayne@68 1158 int source){
jpayne@68 1159 IntList ilist=toIntList(type, names);
jpayne@68 1160 int id=FindAncestor.findAncestor(tree, ilist);
jpayne@68 1161 TaxNode tn=(id>-1 ? tree.getNode(id) : null);
jpayne@68 1162 if(tn==null){
jpayne@68 1163 return new JsonObject("error","Not found.").toString(query);
jpayne@68 1164 }
jpayne@68 1165 if(semicolon){
jpayne@68 1166 return tree.toSemicolon(tn, skipNonCanonical, mononomial);
jpayne@68 1167 }else if(plaintext){
jpayne@68 1168 return ""+id;
jpayne@68 1169 }else if(path){
jpayne@68 1170 return toPath(tn, source);
jpayne@68 1171 }
jpayne@68 1172
jpayne@68 1173 JsonObject j=new JsonObject();
jpayne@68 1174 // j.add("name", mononomial ? tree.mononomial(tn) : tn.name);
jpayne@68 1175 j.add("name", tn.name);
jpayne@68 1176 if(mononomial || true){
jpayne@68 1177 String mono=tree.mononomial(tn);
jpayne@68 1178 if(tn.name!=mono){j.add("mononomial", mono);}
jpayne@68 1179 }
jpayne@68 1180 j.add("tax_id", tn.id);
jpayne@68 1181 if(printNumChildren){j.add("num_children", tn.numChildren);}
jpayne@68 1182 if(printPath){j.add("path", toPath(tn, source));}
jpayne@68 1183 if(printSize){
jpayne@68 1184 j.add("size", tree.toSize(tn));
jpayne@68 1185 j.add("cumulative_size", tree.toSizeC(tn));
jpayne@68 1186 j.add("seqs", tree.toSeqs(tn));
jpayne@68 1187 j.add("cumulative_seqs", tree.toSeqsC(tn));
jpayne@68 1188 j.add("cumulative_nodes", tree.toNodes(tn));
jpayne@68 1189 }
jpayne@68 1190 j.add("level", tn.levelStringExtended(originalLevel));
jpayne@68 1191 if(tn.levelExtended<1 && printRange){
jpayne@68 1192 j.add("maxDescendent", TaxTree.levelToStringExtended(tn.maxChildLevelExtended));
jpayne@68 1193 j.add("minAncestor", TaxTree.levelToStringExtended(tn.minParentLevelExtended));
jpayne@68 1194 }
jpayne@68 1195 // if(printChildren){j.add(getChildren(id, originalLevel, printRange));}
jpayne@68 1196 while(tn!=null && tn.levelExtended!=TaxTree.LIFE_E && tn.id!=TaxTree.CELLULAR_ORGANISMS_ID){
jpayne@68 1197 if(!skipNonCanonical || tn.isSimple()){
jpayne@68 1198 j.addAndRename(tn.levelStringExtended(originalLevel), toJson(tn, originalLevel, printNumChildren, printChildren, printPath, printSize, printRange, mononomial, source, -1));
jpayne@68 1199 }
jpayne@68 1200 if(tn.pid==tn.id){break;}
jpayne@68 1201 tn=tree.getNode(tn.pid);
jpayne@68 1202 }
jpayne@68 1203 return j.toString();
jpayne@68 1204 }
jpayne@68 1205
jpayne@68 1206 JsonObject getChildren(final int id, boolean originalLevel, boolean printRange, boolean mononomial){
jpayne@68 1207 TaxNode x=tree.getNode(id);
jpayne@68 1208 if(x==null || x.numChildren==0){return null;}
jpayne@68 1209 ArrayList<TaxNode> list=tree.getChildren(x);
jpayne@68 1210 return makeChildrenObject(list, originalLevel, printRange, mononomial);
jpayne@68 1211 }
jpayne@68 1212
jpayne@68 1213 JsonObject makeChildrenObject(ArrayList<TaxNode> list, boolean originalLevel, boolean printRange, boolean mononomial){
jpayne@68 1214 if(list==null || list.isEmpty()){return null;}
jpayne@68 1215 JsonObject j=new JsonObject();
jpayne@68 1216 for(TaxNode tn : list){
jpayne@68 1217 JsonObject child=new JsonObject();
jpayne@68 1218 // child.add("name", mononomial ? tree.mononomial(tn) : tn.name);
jpayne@68 1219 child.add("name", tn.name);
jpayne@68 1220 if(mononomial || true){
jpayne@68 1221 String mono=tree.mononomial(tn);
jpayne@68 1222 if(tn.name!=mono){child.add("mononomial", mono);}
jpayne@68 1223 }
jpayne@68 1224 child.add("tax_id", tn.id);
jpayne@68 1225 child.add("num_children", tn.numChildren);
jpayne@68 1226 child.add("level", tn.levelStringExtended(originalLevel));
jpayne@68 1227 if(tn.levelExtended<1 && printRange){
jpayne@68 1228 child.add("maxDescendent", TaxTree.levelToStringExtended(tn.maxChildLevelExtended));
jpayne@68 1229 child.add("minAncestor", TaxTree.levelToStringExtended(tn.minParentLevelExtended));
jpayne@68 1230 }
jpayne@68 1231 j.add(tn.id+"", child);
jpayne@68 1232 }
jpayne@68 1233 return j;
jpayne@68 1234 }
jpayne@68 1235
jpayne@68 1236 /** Format a reply as plaintext, comma-delimited, TaxID only */
jpayne@68 1237 String toText(final int type, final String[] names){
jpayne@68 1238
jpayne@68 1239 StringBuilder sb=new StringBuilder();
jpayne@68 1240 String comma="";
jpayne@68 1241
jpayne@68 1242 int type2=type&15;
jpayne@68 1243 if(type2==GI){
jpayne@68 1244 for(String name : names){
jpayne@68 1245 sb.append(comma);
jpayne@68 1246 TaxNode tn=getTaxNodeGi(Long.parseLong(name));
jpayne@68 1247 if(tn==null){sb.append("-1");}
jpayne@68 1248 else{sb.append(tn.id);}
jpayne@68 1249 comma=",";
jpayne@68 1250 }
jpayne@68 1251 }else if(type2==NAME){
jpayne@68 1252 for(String name : names){
jpayne@68 1253 sb.append(comma);
jpayne@68 1254 TaxNode tn=getTaxNodeByName(name);
jpayne@68 1255 if(tn==null){sb.append("-1");}
jpayne@68 1256 else{sb.append(tn.id);}
jpayne@68 1257 comma=",";
jpayne@68 1258 }
jpayne@68 1259 }else if(type2==TAXID){
jpayne@68 1260 for(String name : names){
jpayne@68 1261 sb.append(comma);
jpayne@68 1262 TaxNode tn=getTaxNodeTaxid(Integer.parseInt(name));
jpayne@68 1263 if(tn==null){sb.append("-1");}
jpayne@68 1264 else{sb.append(tn.id);}
jpayne@68 1265 comma=",";
jpayne@68 1266 }
jpayne@68 1267 }else if(type2==ACCESSION){
jpayne@68 1268 for(String name : names){
jpayne@68 1269 sb.append(comma);
jpayne@68 1270 int ncbi=accessionToTaxid(name);
jpayne@68 1271 sb.append(ncbi);
jpayne@68 1272 comma=",";
jpayne@68 1273 }
jpayne@68 1274 }else if(type2==HEADER || type2==SILVAHEADER){
jpayne@68 1275 for(String name : names){
jpayne@68 1276 sb.append(comma);
jpayne@68 1277 TaxNode tn=getTaxNodeHeader(name, type2==SILVAHEADER);
jpayne@68 1278 if(tn==null){sb.append("-1");}
jpayne@68 1279 else{sb.append(tn.id);}
jpayne@68 1280 comma=",";
jpayne@68 1281 }
jpayne@68 1282 }else if(type2==IMG){
jpayne@68 1283 for(String name : names){
jpayne@68 1284 sb.append(comma);
jpayne@68 1285 int ncbi=TaxTree.imgToTaxid(Long.parseLong(name));
jpayne@68 1286 sb.append(ncbi);
jpayne@68 1287 comma=",";
jpayne@68 1288 }
jpayne@68 1289 }else{
jpayne@68 1290 return "Bad type; should be pt_gi or pt_name; e.g. /pt_gi/1234";
jpayne@68 1291 }
jpayne@68 1292
jpayne@68 1293 return sb.toString();
jpayne@68 1294 }
jpayne@68 1295
jpayne@68 1296 private TaxNode toNode(final int type, final String name){
jpayne@68 1297 int type2=type&15;
jpayne@68 1298 final TaxNode tn;
jpayne@68 1299 if(type2==GI){
jpayne@68 1300 tn=getTaxNodeGi(Long.parseLong(name));
jpayne@68 1301 }else if(type2==NAME){
jpayne@68 1302 tn=getTaxNodeByName(name);
jpayne@68 1303 }else if(type2==TAXID){
jpayne@68 1304 tn=getTaxNodeTaxid(Integer.parseInt(name));
jpayne@68 1305 }else if(type2==ACCESSION){
jpayne@68 1306 int ncbi=accessionToTaxid(name);
jpayne@68 1307 tn=(ncbi<0 ? null : tree.getNode(ncbi));
jpayne@68 1308 }else if(type2==HEADER || type2==SILVAHEADER){
jpayne@68 1309 tn=getTaxNodeHeader(name, type2==SILVAHEADER);
jpayne@68 1310 }else if(type2==IMG){
jpayne@68 1311 int ncbi=TaxTree.imgToTaxid(Long.parseLong(name));
jpayne@68 1312 tn=(ncbi<0 ? null : tree.getNode(ncbi));
jpayne@68 1313 }else{
jpayne@68 1314 tn=null;
jpayne@68 1315 }
jpayne@68 1316 return tn;
jpayne@68 1317 }
jpayne@68 1318
jpayne@68 1319 /** Format a reply as paths, comma-delimited*/
jpayne@68 1320 String toPath(final int type, final String[] names, final int source){
jpayne@68 1321
jpayne@68 1322 StringBuilder sb=new StringBuilder();
jpayne@68 1323 String comma="";
jpayne@68 1324
jpayne@68 1325 int type2=type&15;
jpayne@68 1326
jpayne@68 1327 for(String name : names){
jpayne@68 1328 sb.append(comma);
jpayne@68 1329 if(type2==IMG){
jpayne@68 1330 sb.append(toPathIMG(Long.parseLong(name)));
jpayne@68 1331 }else{
jpayne@68 1332 TaxNode tn=toNode(type2, name);
jpayne@68 1333 sb.append(toPath(tn, source));
jpayne@68 1334 }
jpayne@68 1335 comma=",";
jpayne@68 1336 }
jpayne@68 1337
jpayne@68 1338 return sb.toString();
jpayne@68 1339 }
jpayne@68 1340
jpayne@68 1341 /** Format a reply as plaintext, semicolon-delimited, full lineage */
jpayne@68 1342 String toSemicolon(final int type, final String[] names, boolean skipNonCanonical, boolean mononomial){
jpayne@68 1343
jpayne@68 1344 StringBuilder sb=new StringBuilder();
jpayne@68 1345 String comma="";
jpayne@68 1346
jpayne@68 1347 int type2=type&15;
jpayne@68 1348 if(type2==GI){
jpayne@68 1349 for(String name : names){
jpayne@68 1350 sb.append(comma);
jpayne@68 1351 TaxNode tn=getTaxNodeGi(Long.parseLong(name));
jpayne@68 1352 if(tn==null){sb.append("Not found");}
jpayne@68 1353 else{sb.append(tree.toSemicolon(tn, skipNonCanonical, mononomial));}
jpayne@68 1354 comma=",";
jpayne@68 1355 }
jpayne@68 1356 }else if(type2==NAME){
jpayne@68 1357 for(String name : names){
jpayne@68 1358 sb.append(comma);
jpayne@68 1359 TaxNode tn=getTaxNodeByName(name);
jpayne@68 1360 if(tn==null){sb.append("Not found");}
jpayne@68 1361 else{sb.append(tree.toSemicolon(tn, skipNonCanonical, mononomial));}
jpayne@68 1362 comma=",";
jpayne@68 1363 }
jpayne@68 1364 }else if(type2==TAXID){
jpayne@68 1365 for(String name : names){
jpayne@68 1366 sb.append(comma);
jpayne@68 1367 TaxNode tn=getTaxNodeTaxid(Integer.parseInt(name));
jpayne@68 1368 // if(verbose2){outstream.println("name="+name+", tn="+tn);}
jpayne@68 1369 if(tn==null){sb.append("Not found");}
jpayne@68 1370 else{sb.append(tree.toSemicolon(tn, skipNonCanonical, mononomial));}
jpayne@68 1371 comma=",";
jpayne@68 1372 }
jpayne@68 1373 }else if(type2==ACCESSION){
jpayne@68 1374 for(String name : names){
jpayne@68 1375 sb.append(comma);
jpayne@68 1376 final int tid=accessionToTaxid(name);
jpayne@68 1377 TaxNode tn=tree.getNode(tid, true);
jpayne@68 1378 if(tn==null){sb.append("Not found");}
jpayne@68 1379 else{sb.append(tree.toSemicolon(tn, skipNonCanonical, mononomial));}
jpayne@68 1380 comma=",";
jpayne@68 1381 }
jpayne@68 1382 }else if(type2==HEADER || type2==SILVAHEADER){
jpayne@68 1383 for(String name : names){
jpayne@68 1384 sb.append(comma);
jpayne@68 1385 TaxNode tn=getTaxNodeHeader(name, type2==SILVAHEADER);
jpayne@68 1386 if(tn==null){sb.append("Not found");}
jpayne@68 1387 else{sb.append(tree.toSemicolon(tn, skipNonCanonical, mononomial));}
jpayne@68 1388 comma=",";
jpayne@68 1389 }
jpayne@68 1390 }else if(type2==IMG){
jpayne@68 1391 for(String name : names){
jpayne@68 1392 sb.append(comma);
jpayne@68 1393 final int tid=TaxTree.imgToTaxid(Long.parseLong(name));
jpayne@68 1394 TaxNode tn=tree.getNode(tid, true);
jpayne@68 1395 if(tn==null){sb.append("Not found");}
jpayne@68 1396 else{sb.append(tree.toSemicolon(tn, skipNonCanonical, mononomial));}
jpayne@68 1397 comma=",";
jpayne@68 1398 }
jpayne@68 1399 }else{
jpayne@68 1400 return "Bad type; should be sc_gi or sc_name; e.g. /sc_gi/1234";
jpayne@68 1401 }
jpayne@68 1402
jpayne@68 1403 // if(verbose2){outstream.println("In toSemicolon; type="+type+", type2="+type2+", made "+sb);}
jpayne@68 1404
jpayne@68 1405 return sb.toString();
jpayne@68 1406 }
jpayne@68 1407
jpayne@68 1408 /** Create a JsonObject from a String, including full lineage */
jpayne@68 1409 JsonObject toJson(final int type, final String name, boolean skipNonCanonical, boolean originalLevel,
jpayne@68 1410 boolean printNumChildren, boolean printChildren, boolean printPath, boolean printSize,
jpayne@68 1411 boolean printRange, boolean mononomial, int source){
jpayne@68 1412 final TaxNode tn0;
jpayne@68 1413 TaxNode tn;
jpayne@68 1414
jpayne@68 1415 long img=-1;
jpayne@68 1416 if(type==GI){
jpayne@68 1417 tn0=getTaxNodeGi(Long.parseLong(name));
jpayne@68 1418 }else if(type==NAME){
jpayne@68 1419 tn0=getTaxNodeByName(name);
jpayne@68 1420 }else if(type==TAXID){
jpayne@68 1421 tn0=getTaxNodeTaxid(Integer.parseInt(name));
jpayne@68 1422 }else if(type==ACCESSION){
jpayne@68 1423 int ncbi=accessionToTaxid(name);
jpayne@68 1424 tn0=(ncbi>=0 ? tree.getNode(ncbi) : null);
jpayne@68 1425 }else if(type==HEADER || type==SILVAHEADER){
jpayne@68 1426 tn0=getTaxNodeHeader(name, type==SILVAHEADER);
jpayne@68 1427 }else if(type==IMG){
jpayne@68 1428 img=Long.parseLong(name);
jpayne@68 1429 final int tid=TaxTree.imgToTaxid(img);
jpayne@68 1430 tn0=tree.getNode(tid, true);
jpayne@68 1431 }else{
jpayne@68 1432 JsonObject j=new JsonObject("error","Bad type; should be gi, taxid, or name; e.g. /name/homo_sapiens");
jpayne@68 1433 j.add("name", name);
jpayne@68 1434 j.add("type", type);
jpayne@68 1435 return j;
jpayne@68 1436 }
jpayne@68 1437 tn=tn0;
jpayne@68 1438 if(verbose2){System.err.println("Got node: "+tn);}
jpayne@68 1439
jpayne@68 1440 if(tn!=null){
jpayne@68 1441 JsonObject j=new JsonObject();
jpayne@68 1442 // j.add("name", mononomial ? tree.mononomial(tn) : tn.name);
jpayne@68 1443 j.add("name", tn.name);
jpayne@68 1444 if(mononomial || true){
jpayne@68 1445 String mono=tree.mononomial(tn);
jpayne@68 1446 if(tn.name!=mono){j.add("mononomial", mono);}
jpayne@68 1447 }
jpayne@68 1448 j.add("tax_id", tn.id);
jpayne@68 1449 if(printNumChildren){j.add("num_children", tn.numChildren);}
jpayne@68 1450 if(printPath){j.add("path", type==IMG ? toPathIMG(img) : toPath(tn, source));}
jpayne@68 1451 if(printSize){
jpayne@68 1452 j.add("size", tree.toSize(tn));
jpayne@68 1453 j.add("cumulative_size", tree.toSizeC(tn));
jpayne@68 1454 j.add("seqs", tree.toSeqs(tn));
jpayne@68 1455 j.add("cumulative_seqs", tree.toSeqsC(tn));
jpayne@68 1456 j.add("cumulative_nodes", tree.toNodes(tn));
jpayne@68 1457 }
jpayne@68 1458 j.add("level", tn.levelStringExtended(originalLevel));
jpayne@68 1459 if(tn.levelExtended<1 && printRange){
jpayne@68 1460 j.add("maxDescendent", TaxTree.levelToStringExtended(tn.maxChildLevelExtended));
jpayne@68 1461 j.add("minAncestor", TaxTree.levelToStringExtended(tn.minParentLevelExtended));
jpayne@68 1462 }
jpayne@68 1463 if(printChildren && (tn.id==tn.pid || tn.id==TaxTree.CELLULAR_ORGANISMS_ID)){j.add("children", getChildren(tn.id, originalLevel, printRange, mononomial));}
jpayne@68 1464 while(tn!=null && tn.levelExtended!=TaxTree.LIFE_E && tn.id!=TaxTree.CELLULAR_ORGANISMS_ID){
jpayne@68 1465 // System.err.println(tn+", "+(!skipNonCanonical)+", "+tn.isSimple());
jpayne@68 1466 if(!skipNonCanonical || tn.isSimple()){
jpayne@68 1467 j.addAndRename(tn.levelStringExtended(originalLevel), toJson(tn, originalLevel, printNumChildren, printChildren, printPath && tn==tn0, printSize, printRange, mononomial, source, img));
jpayne@68 1468 // System.err.println(j);
jpayne@68 1469 }
jpayne@68 1470 if(tn.pid==tn.id){break;}
jpayne@68 1471 tn=tree.getNode(tn.pid);
jpayne@68 1472 }
jpayne@68 1473 return j;
jpayne@68 1474 }
jpayne@68 1475 {
jpayne@68 1476 JsonObject j=new JsonObject("error","Not found.");
jpayne@68 1477 j.add("name", name);
jpayne@68 1478 j.add("type", type);
jpayne@68 1479 return j;
jpayne@68 1480 }
jpayne@68 1481 }
jpayne@68 1482
jpayne@68 1483 /** Create a JsonObject from a TaxNode, at that level only */
jpayne@68 1484 JsonObject toJson(TaxNode tn, boolean originalLevel, boolean printNumChildren,
jpayne@68 1485 boolean printChildren, boolean printPath, boolean printSize, boolean printRange,
jpayne@68 1486 boolean mononomial, int source, long img){
jpayne@68 1487 JsonObject j=new JsonObject();
jpayne@68 1488 // j.add("name", mononomial ? tree.mononomial(tn) : tn.name);
jpayne@68 1489 j.add("name", tn.name);
jpayne@68 1490 if(mononomial || true){
jpayne@68 1491 String mono=tree.mononomial(tn);
jpayne@68 1492 if(tn.name!=mono){j.add("mononomial", mono);}
jpayne@68 1493 }
jpayne@68 1494 j.add("tax_id", tn.id);
jpayne@68 1495 if(printNumChildren){j.add("num_children", tn.numChildren);}
jpayne@68 1496 if(printPath){j.add("path", source==SOURCE_IMG ? toPathIMG(img) : toPath(tn, source));}
jpayne@68 1497 if(printSize){
jpayne@68 1498 j.add("size", tree.toSize(tn));
jpayne@68 1499 j.add("cumulative_size", tree.toSizeC(tn));
jpayne@68 1500 j.add("seqs", tree.toSeqs(tn));
jpayne@68 1501 j.add("cumulative_seqs", tree.toSeqsC(tn));
jpayne@68 1502 j.add("cumulative_nodes", tree.toNodes(tn));
jpayne@68 1503 }
jpayne@68 1504 if(tn.levelExtended<1 && printRange){
jpayne@68 1505 j.add("maxDescendent", TaxTree.levelToStringExtended(tn.maxChildLevelExtended));
jpayne@68 1506 j.add("minAncestor", TaxTree.levelToStringExtended(tn.minParentLevelExtended));
jpayne@68 1507 }
jpayne@68 1508 if(printChildren){
jpayne@68 1509 JsonObject children=getChildren(tn.id, originalLevel, printRange, mononomial);
jpayne@68 1510 if(children!=null){j.add("children", children);}
jpayne@68 1511 }
jpayne@68 1512 return j;
jpayne@68 1513 }
jpayne@68 1514
jpayne@68 1515 String toPath(TaxNode tn, int source){
jpayne@68 1516 if(tn==null){return "null";}
jpayne@68 1517 String path;
jpayne@68 1518 if(source==SOURCE_REFSEQ){
jpayne@68 1519 path=tree.toDir(tn, basePath)+"refseq_"+tn.id+".fa.gz";
jpayne@68 1520 }else if(source==SOURCE_SILVA){
jpayne@68 1521 path=tree.toDir(tn, basePath)+"silva_"+tn.id+".fa.gz";
jpayne@68 1522 }else if(source==SOURCE_IMG){
jpayne@68 1523 assert(false);
jpayne@68 1524 path="null";
jpayne@68 1525 }else{
jpayne@68 1526 assert(false);
jpayne@68 1527 path="null";
jpayne@68 1528 }
jpayne@68 1529 if(!path.equals("null") && !new File(path).exists()){path="null";}
jpayne@68 1530 return path;
jpayne@68 1531 }
jpayne@68 1532
jpayne@68 1533 String toPathIMG(long imgID){
jpayne@68 1534 String path="/global/dna/projectdirs/microbial/img_web_data/taxon.fna/"+imgID+".fna";
jpayne@68 1535 if(!new File(path).exists()){path="null";}
jpayne@68 1536 return path;
jpayne@68 1537 }
jpayne@68 1538
jpayne@68 1539 /*--------------------------------------------------------------*/
jpayne@68 1540 /*---------------- Taxonomy Lookup ----------------*/
jpayne@68 1541 /*--------------------------------------------------------------*/
jpayne@68 1542
jpayne@68 1543 /** Convert a list of terms to a list of TaxIDs */
jpayne@68 1544 IntList toIntList(final int type, final String[] names){
jpayne@68 1545 IntList list=new IntList(names.length);
jpayne@68 1546 int type2=type&15;
jpayne@68 1547 if(type2==GI){
jpayne@68 1548 for(String name : names){
jpayne@68 1549 TaxNode tn=getTaxNodeGi(Long.parseLong(name));
jpayne@68 1550 if(tn!=null){list.add(tn.id);}
jpayne@68 1551 else{notFound.incrementAndGet();}
jpayne@68 1552 }
jpayne@68 1553 }else if(type2==NAME){
jpayne@68 1554 for(String name : names){
jpayne@68 1555 TaxNode tn=getTaxNodeByName(name);
jpayne@68 1556 if(tn!=null){list.add(tn.id);}
jpayne@68 1557 else{notFound.incrementAndGet();}
jpayne@68 1558 }
jpayne@68 1559 }else if(type2==TAXID){
jpayne@68 1560 for(String name : names){
jpayne@68 1561 TaxNode tn=getTaxNodeTaxid(Integer.parseInt(name));
jpayne@68 1562 if(tn!=null){list.add(tn.id);}
jpayne@68 1563 else{notFound.incrementAndGet();}
jpayne@68 1564 }
jpayne@68 1565 }else if(type2==ACCESSION){
jpayne@68 1566 for(String name : names){
jpayne@68 1567 int ncbi=accessionToTaxid(name);
jpayne@68 1568 if(ncbi>=0){list.add(ncbi);}
jpayne@68 1569 else{notFound.incrementAndGet();}
jpayne@68 1570 }
jpayne@68 1571 }else if(type2==IMG){
jpayne@68 1572 for(String name : names){
jpayne@68 1573 final int tid=TaxTree.imgToTaxid(Long.parseLong(name));
jpayne@68 1574 if(tid>=0){list.add(tid);}
jpayne@68 1575 else{notFound.incrementAndGet();}
jpayne@68 1576 }
jpayne@68 1577 }else{
jpayne@68 1578 throw new RuntimeException("{\"error\": \"Bad type\"}");
jpayne@68 1579 }
jpayne@68 1580 return list;
jpayne@68 1581 }
jpayne@68 1582
jpayne@68 1583 public static final String stripAccession(String s){
jpayne@68 1584 if(s==null){return null;}
jpayne@68 1585 s=s.toUpperCase();
jpayne@68 1586 for(int i=0; i<s.length(); i++){
jpayne@68 1587 char c=s.charAt(i);
jpayne@68 1588 if(c=='.' || c==':'){return s.substring(0, i);}
jpayne@68 1589 }
jpayne@68 1590 return s;
jpayne@68 1591 }
jpayne@68 1592
jpayne@68 1593 private int accessionToTaxid(String accession){
jpayne@68 1594 if(accession==null){return -1;}
jpayne@68 1595 int tid=AccessionToTaxid.get(accession);
jpayne@68 1596 if(tid<0 && distributed && serverNum==0){
jpayne@68 1597 accession=stripAccession(accession);
jpayne@68 1598 int slaveNum=accession.hashCode()%serverCount;
jpayne@68 1599 if(slaveNum!=serverNum){
jpayne@68 1600 String path=slaveAddress.get(slaveNum);
jpayne@68 1601 tid=TaxClient.accessionToTaxidSpecificServer(path, accession);
jpayne@68 1602 }
jpayne@68 1603 }
jpayne@68 1604 return tid;
jpayne@68 1605 }
jpayne@68 1606
jpayne@68 1607 /** Look up a TaxNode by parsing the organism name */
jpayne@68 1608 TaxNode getTaxNodeByName(String name){
jpayne@68 1609 if(verbose2){System.err.println("Fetching node for "+name);}
jpayne@68 1610 List<TaxNode> list=tree.getNodesByNameExtended(name);
jpayne@68 1611 if(verbose2){System.err.println("Fetched "+list);}
jpayne@68 1612 if(list==null){
jpayne@68 1613 if(verbose2){System.err.println("Fetched in common map "+name);}
jpayne@68 1614 String name2=commonMap.get(name);
jpayne@68 1615 if(verbose2){System.err.println("Fetched "+name2);}
jpayne@68 1616 if(name2!=null){list=tree.getNodesByName(name2);}
jpayne@68 1617 }
jpayne@68 1618 if(list==null){notFound.incrementAndGet();}
jpayne@68 1619 return list==null ? null : list.get(0);
jpayne@68 1620 }
jpayne@68 1621
jpayne@68 1622 /** Look up a TaxNode from the gi number */
jpayne@68 1623 TaxNode getTaxNodeGi(long gi){
jpayne@68 1624 int ncbi=-1;
jpayne@68 1625 try {
jpayne@68 1626 ncbi=GiToTaxid.getID(gi);
jpayne@68 1627 } catch (Throwable e) {
jpayne@68 1628 if(verbose){e.printStackTrace();}
jpayne@68 1629 }
jpayne@68 1630 if(ncbi<0){notFound.incrementAndGet();}
jpayne@68 1631 return ncbi<0 ? null : getTaxNodeTaxid(ncbi);
jpayne@68 1632 }
jpayne@68 1633
jpayne@68 1634 /** Look up a TaxNode by parsing the full header */
jpayne@68 1635 TaxNode getTaxNodeHeader(String header, boolean silvaMode){
jpayne@68 1636 TaxNode tn=silvaMode ? tree.getNodeSilva(header, true) : tree.parseNodeFromHeader(header, true);
jpayne@68 1637 if(tn==null){notFound.incrementAndGet();}
jpayne@68 1638 return tn;
jpayne@68 1639 }
jpayne@68 1640
jpayne@68 1641 /** Look up a TaxNode from the ncbi TaxID */
jpayne@68 1642 TaxNode getTaxNodeTaxid(int ncbi){
jpayne@68 1643 TaxNode tn=null;
jpayne@68 1644 try {
jpayne@68 1645 tn=tree.getNode(ncbi);
jpayne@68 1646 } catch (Throwable e) {
jpayne@68 1647 if(verbose){e.printStackTrace();}
jpayne@68 1648 }
jpayne@68 1649 if(tn==null){notFound.incrementAndGet();}
jpayne@68 1650 return tn;
jpayne@68 1651 }
jpayne@68 1652
jpayne@68 1653 /*--------------------------------------------------------------*/
jpayne@68 1654 /*---------------- Data Initialization ----------------*/
jpayne@68 1655 /*--------------------------------------------------------------*/
jpayne@68 1656
jpayne@68 1657 private static HashMap<String, Integer> makeTypeMap() {
jpayne@68 1658 HashMap<String, Integer> map=new HashMap<String, Integer>(63);
jpayne@68 1659 map.put("gi", GI);
jpayne@68 1660 // map.put("ngi", NGI);
jpayne@68 1661 // map.put("pgi", PGI);
jpayne@68 1662 map.put("name", NAME);
jpayne@68 1663 map.put("tax_id", TAXID);
jpayne@68 1664 map.put("ncbi", TAXID);
jpayne@68 1665 map.put("taxid", TAXID);
jpayne@68 1666 map.put("id", TAXID);
jpayne@68 1667 map.put("tid", TAXID);
jpayne@68 1668 map.put("header", HEADER);
jpayne@68 1669 map.put("accession", ACCESSION);
jpayne@68 1670 map.put("img", IMG);
jpayne@68 1671 map.put("silvaheader", SILVAHEADER);
jpayne@68 1672
jpayne@68 1673 map.put("pt_gi", PT_GI);
jpayne@68 1674 // map.put("pt_ngi", PT_NGI);
jpayne@68 1675 // map.put("pt_pgi", PT_PGI);
jpayne@68 1676 map.put("pt_name", PT_NAME);
jpayne@68 1677 map.put("pt_tax_id", PT_TAXID);
jpayne@68 1678 map.put("pt_id", PT_TAXID);
jpayne@68 1679 map.put("pt_tid", PT_TAXID);
jpayne@68 1680 map.put("pt_ncbi", PT_TAXID);
jpayne@68 1681 map.put("pt_taxid", PT_TAXID);
jpayne@68 1682 map.put("pt_header", PT_HEADER);
jpayne@68 1683 map.put("pt_header", PT_HEADER);
jpayne@68 1684 map.put("pt_accession", PT_ACCESSION);
jpayne@68 1685 map.put("pt_img", PT_IMG);
jpayne@68 1686 map.put("pt_silvaheader", PT_SILVAHEADER);
jpayne@68 1687
jpayne@68 1688 map.put("sc_gi", SC_GI);
jpayne@68 1689 // map.put("sc_ngi", SC_NGI);
jpayne@68 1690 // map.put("sc_pgi", SC_PGI);
jpayne@68 1691 map.put("sc_name", SC_NAME);
jpayne@68 1692 map.put("sc_tax_id", SC_TAXID);
jpayne@68 1693 map.put("sc_id", SC_TAXID);
jpayne@68 1694 map.put("sc_tid", SC_TAXID);
jpayne@68 1695 map.put("sc_ncbi", SC_TAXID);
jpayne@68 1696 map.put("sc_taxid", SC_TAXID);
jpayne@68 1697 map.put("sc_header", SC_HEADER);
jpayne@68 1698 map.put("sc_header", SC_HEADER);
jpayne@68 1699 map.put("sc_accession", SC_ACCESSION);
jpayne@68 1700 map.put("sc_silvaheader", SC_SILVAHEADER);
jpayne@68 1701
jpayne@68 1702 return map;
jpayne@68 1703 }
jpayne@68 1704
jpayne@68 1705 public static HashMap<String, String> makeCommonMap(){
jpayne@68 1706 HashMap<String, String> map=new HashMap<String, String>();
jpayne@68 1707 map.put("human", "homo sapiens");
jpayne@68 1708 map.put("cat", "felis catus");
jpayne@68 1709 map.put("dog", "canis lupus familiaris");
jpayne@68 1710 map.put("mouse", "mus musculus");
jpayne@68 1711 map.put("cow", "bos taurus");
jpayne@68 1712 map.put("bull", "bos taurus");
jpayne@68 1713 map.put("horse", "Equus ferus");
jpayne@68 1714 map.put("pig", "Sus scrofa domesticus");
jpayne@68 1715 map.put("sheep", "Ovis aries");
jpayne@68 1716 map.put("goat", "Capra aegagrus");
jpayne@68 1717 map.put("turkey", "Meleagris gallopavo");
jpayne@68 1718 map.put("fox", "Vulpes vulpes");
jpayne@68 1719 map.put("chicken", "Gallus gallus domesticus");
jpayne@68 1720 map.put("wolf", "canis lupus");
jpayne@68 1721 map.put("fruitfly", "drosophila melanogaster");
jpayne@68 1722 map.put("zebrafish", "Danio rerio");
jpayne@68 1723 map.put("catfish", "Ictalurus punctatus");
jpayne@68 1724 map.put("trout", "Oncorhynchus mykiss");
jpayne@68 1725 map.put("salmon", "Salmo salar");
jpayne@68 1726 map.put("tilapia", "Oreochromis niloticus");
jpayne@68 1727 map.put("e coli", "Escherichia coli");
jpayne@68 1728 map.put("e.coli", "Escherichia coli");
jpayne@68 1729
jpayne@68 1730 map.put("lion", "Panthera leo");
jpayne@68 1731 map.put("tiger", "Panthera tigris");
jpayne@68 1732 map.put("bear", "Ursus arctos");
jpayne@68 1733 map.put("deer", "Odocoileus virginianus");
jpayne@68 1734 map.put("coyote", "Canis latrans");
jpayne@68 1735
jpayne@68 1736 map.put("corn", "Zea mays subsp. mays");
jpayne@68 1737 map.put("maize", "Zea mays subsp. mays");
jpayne@68 1738 map.put("oat", "Avena sativa");
jpayne@68 1739 map.put("wheat", "Triticum aestivum");
jpayne@68 1740 map.put("rice", "Oryza sativa");
jpayne@68 1741 map.put("potato", "Solanum tuberosum");
jpayne@68 1742 map.put("barley", "Hordeum vulgare");
jpayne@68 1743 map.put("poplar", "Populus alba");
jpayne@68 1744 map.put("lettuce", "Lactuca sativa");
jpayne@68 1745 map.put("beet", "Beta vulgaris");
jpayne@68 1746 map.put("strawberry", "Fragaria x ananassa");
jpayne@68 1747 map.put("orange", "Citrus sinensis");
jpayne@68 1748 map.put("lemon", "Citrus limon");
jpayne@68 1749 map.put("soy", "Glycine max");
jpayne@68 1750 map.put("soybean", "Glycine max");
jpayne@68 1751 map.put("grape", "Vitis vinifera");
jpayne@68 1752 map.put("olive", "Olea europaea");
jpayne@68 1753 map.put("cotton", "Gossypium hirsutum");
jpayne@68 1754 map.put("apple", "Malus pumila");
jpayne@68 1755 map.put("bannana", "Musa acuminata");
jpayne@68 1756 map.put("tomato", "Solanum lycopersicum");
jpayne@68 1757 map.put("sugarcane", "Saccharum officinarum");
jpayne@68 1758 map.put("bean", "Phaseolus vulgaris");
jpayne@68 1759 map.put("onion", "Allium cepa");
jpayne@68 1760 map.put("garlic", "Allium sativum");
jpayne@68 1761
jpayne@68 1762 map.put("pichu", "mus musculus");
jpayne@68 1763 map.put("pikachu", "mus musculus");
jpayne@68 1764 map.put("vulpix", "Vulpes vulpes");
jpayne@68 1765 map.put("ninetails", "Vulpes vulpes");
jpayne@68 1766 map.put("mareep", "Ovis aries");
jpayne@68 1767
jpayne@68 1768 return map;
jpayne@68 1769 }
jpayne@68 1770
jpayne@68 1771 //Customize usage message to include domain
jpayne@68 1772 private String makeUsagePrefix(){
jpayne@68 1773 if(!sketchOnly){
jpayne@68 1774 return "Welcome to the JGI taxonomy server!\n"
jpayne@68 1775 + "This service provides taxonomy information from NCBI taxID numbers, gi numbers, organism names, and accessions.\n"
jpayne@68 1776 + "The output is formatted as a Json object.\n\n"
jpayne@68 1777 + "Usage:\n\n"
jpayne@68 1778 + "All addresses below are assumed to be prefixed by "+domain+", e.g. /name/homo_sapiens implies a full URL of:\n"
jpayne@68 1779 + domain+"/name/homo_sapiens\n"
jpayne@68 1780 + "\n"
jpayne@68 1781 + "/name/homo_sapiens will give taxonomy information for an organism name.\n"
jpayne@68 1782 + "Names are case-insensitive and underscores are equivalent to spaces.\n"
jpayne@68 1783 + "/id/9606 will give taxonomy information for an NCBI taxID.\n"
jpayne@68 1784 + "/gi/1234 will give taxonomy information from an NCBI gi number.\n"
jpayne@68 1785
jpayne@68 1786 // + "\n****NOTICE**** gi number support is temporarily suspended due to conflicts in NCBI data.\n"
jpayne@68 1787 // + "Support may be restored, altered, or discontinued pending a response from NCBI.\n"
jpayne@68 1788 // + "Currently, it is not possible to ensure correct results when looking up a GI number, because some map to multiple organisms.\n\n"
jpayne@68 1789
jpayne@68 1790 + "/accession/NZ_AAAA01000057.1 will give taxonomy information from an accession.\n"
jpayne@68 1791 + "/header/ will accept an NCBI sequence header such as gi|7|emb|X51700.1| Bos taurus\n"
jpayne@68 1792 + "/silvaheader/ will accept a Silva sequence header such as KC415233.1.1497 Bacteria;Spirochaetae;Spirochaetes\n"
jpayne@68 1793 + "/img/ will accept an IMG id such as 2724679250\n"
jpayne@68 1794 + "Vertical bars (|) may cause problems on the command line and can be replaced by tilde (~).\n"
jpayne@68 1795 + "\nComma-delimited lists are accepted for bulk queries, such as tax/gi/1234,7000,42\n"
jpayne@68 1796 + "For plaintext (non-Json) results, add the term /pt/ or /sc/.\n"
jpayne@68 1797 + "pt will give just the taxID, while sc will give the whole lineage, semicolon-delimited. For example:\n"
jpayne@68 1798 + "/pt/name/homo_sapiens\n"
jpayne@68 1799 + "/sc/gi/1234\n\n"
jpayne@68 1800 + "Additional supported display options are children, numchildren, range, simple, path, size, and ancestor.\n"
jpayne@68 1801 + "The order is not important but they need to come before the query term. For example:\n"
jpayne@68 1802 + "/children/numchildren/range/gi/1234\n"
jpayne@68 1803 + "\nTo find the common ancestor of multiple organisms, add /ancestor/. For example:\n"
jpayne@68 1804 + "/id/ancestor/1234,5678,42\n"
jpayne@68 1805 + "/name/ancestor/homo_sapiens,canis_lupus,bos_taurus\n"
jpayne@68 1806 + "\nFor a simplified taxonomic tree, add simple.\n"
jpayne@68 1807 + "This will ignore unranked or uncommon levels like tribe and parvorder, and only display the following levels:\n"
jpayne@68 1808 + "SUBSPECIES, SPECIES, GENUS, FAMILY, ORDER, CLASS, PHYLUM, KINGDOM, SUPERKINGDOM, DOMAIN\n"
jpayne@68 1809 + "For example:\n"
jpayne@68 1810 + "/simple/id/1234\n"
jpayne@68 1811 + "\nTo print taxonomy from the command line in Linux, use curl:\n"
jpayne@68 1812 + "curl https://taxonomy.jgi.doe.gov/id/9606\n"
jpayne@68 1813 + "\nQueries longer than around 8kB can be sent via POST: curl https://taxonomy..doe.gov/POST"
jpayne@68 1814 + "\n...where the data sent is, for example: name/e.coli,h.sapiens,c.lupus\n"
jpayne@68 1815 + "\nLast restarted "+startTime+"\n"
jpayne@68 1816 + "Running BBMap version "+Shared.BBMAP_VERSION_STRING+"\n";
jpayne@68 1817 }else{
jpayne@68 1818 StringBuilder sb=new StringBuilder();
jpayne@68 1819 sb.append("Welcome to the JGI"+(SketchObject.defaultParams.dbName==null ? "" : " "+SketchObject.defaultParams.dbName)+" sketch server!\n");
jpayne@68 1820 // if(dbName!=null){
jpayne@68 1821 // sb.append("This server has the "+dbName+ " database loaded.\n");
jpayne@68 1822 // }
jpayne@68 1823 sb.append("\nUsage:\n\n");
jpayne@68 1824 sb.append("sendsketch.sh in=file.fasta"+(SketchObject.defaultParams.dbName==null ? "" : " "+SketchObject.defaultParams.dbName.toLowerCase())+"\n\n");
jpayne@68 1825 sb.append("SendSketch creates a sketch from a local sequence file, and sends the sketch to this server.\n");
jpayne@68 1826 sb.append("The server receives the sketch, compares it to all sketches in memory, and returns the results.\n");
jpayne@68 1827 sb.append("For files on the same system as the server, the 'local' flag may be used to offload sketch creation to the server.\n");
jpayne@68 1828 sb.append("For more details and parameters please run sendsketch.sh with no arguments.\n");
jpayne@68 1829 sb.append("\n");
jpayne@68 1830 if(SketchObject.useWhitelist()){
jpayne@68 1831 sb.append("This server is running in whitelist mode; for best results, use local queries.\n");
jpayne@68 1832 sb.append("Remote queries should specify a larger-than-normal sketch size.\n\n");
jpayne@68 1833 }else if(SketchObject.blacklist()!=null){
jpayne@68 1834 sb.append("This server is running in blacklist mode, using "+new File(SketchObject.blacklist()).getName()+".\n\n");
jpayne@68 1835 }
jpayne@68 1836 sb.append("Last restarted "+startTime+"\n");
jpayne@68 1837 sb.append("Running BBMap version "+Shared.BBMAP_VERSION_STRING+"\n");
jpayne@68 1838 sb.append("Settings:\tk="+SketchObject.k+(SketchObject.k2>0 ? ","+SketchObject.k2 : ""));
jpayne@68 1839 if(SketchObject.amino){sb.append(" amino");}
jpayne@68 1840 if(SketchObject.makeIndex){sb.append(" index");}
jpayne@68 1841 if(SketchObject.useWhitelist()){sb.append(" whitelist");}
jpayne@68 1842 if(SketchObject.blacklist()!=null){sb.append(" blacklist="+new File(SketchObject.blacklist()).getName());}
jpayne@68 1843 sb.append('\n');
jpayne@68 1844 return sb.toString();
jpayne@68 1845 }
jpayne@68 1846 }
jpayne@68 1847
jpayne@68 1848 private String makeUsageHtml(){
jpayne@68 1849 String html=rawHtml;
jpayne@68 1850 html=html.replace("STATISTICSSTRING", makeStats());
jpayne@68 1851 // html=html.replace("TIMESTAMPSTRING", startTime);
jpayne@68 1852 // html=html.replace("VERSIONSTRING", "Running BBMap version "+Shared.BBMAP_VERSION_STRING);
jpayne@68 1853 return html;
jpayne@68 1854 }
jpayne@68 1855
jpayne@68 1856 private String loadRawHtml(){
jpayne@68 1857 String path=Data.findPath("?tax_server.html");
jpayne@68 1858 String html=ReadWrite.readString(path);
jpayne@68 1859 return html;
jpayne@68 1860 }
jpayne@68 1861
jpayne@68 1862 private String makeStats(){
jpayne@68 1863 ByteBuilder sb=new ByteBuilder();
jpayne@68 1864
jpayne@68 1865 if(!sketchOnly){
jpayne@68 1866 sb.append("JGI taxonomy server stats:\n"
jpayne@68 1867 + "\nLast restarted "+startTime+"\n"
jpayne@68 1868 + "Running BBMap version "+Shared.BBMAP_VERSION_STRING+"\n");
jpayne@68 1869 }else{
jpayne@68 1870 sb.append("JGI"+(SketchObject.defaultParams.dbName==null ? "" : " "+SketchObject.defaultParams.dbName)+" sketch server stats:\n\n");
jpayne@68 1871
jpayne@68 1872 if(domain!=null) {sb.append("Domain: "+domain+"\n");}
jpayne@68 1873 if(SketchObject.useWhitelist()){
jpayne@68 1874 sb.append("This server is running in whitelist mode; for best results, use local queries.\n");
jpayne@68 1875 sb.append("Remote queries should specify a larger-than-normal sketch size.\n\n");
jpayne@68 1876 }else if(SketchObject.blacklist()!=null){
jpayne@68 1877 sb.append("This server is running in blacklist mode, using "+new File(SketchObject.blacklist()).getName()+".\n\n");
jpayne@68 1878 }
jpayne@68 1879 sb.append("Last restarted "+startTime+"\n");
jpayne@68 1880 sb.append("Running BBMap version "+Shared.BBMAP_VERSION_STRING+"\n");
jpayne@68 1881 sb.append("Settings: k="+SketchObject.k+(SketchObject.k2>0 ? ","+SketchObject.k2 : ""));
jpayne@68 1882 if(SketchObject.amino){sb.append(" amino");}
jpayne@68 1883 if(SketchObject.makeIndex){sb.append(" index");}
jpayne@68 1884 if(SketchObject.useWhitelist()){sb.append(" whitelist");}
jpayne@68 1885 if(SketchObject.blacklist()!=null){sb.append(" blacklist="+new File(SketchObject.blacklist()).getName());}
jpayne@68 1886 }
jpayne@68 1887 sb.nl().nl();
jpayne@68 1888 sb.append(basicStats());
jpayne@68 1889 if(sketchOnly){sb.append(makeExtendedStats());}
jpayne@68 1890
jpayne@68 1891 return sb.toString();
jpayne@68 1892 }
jpayne@68 1893
jpayne@68 1894 public String makeExtendedStats(){
jpayne@68 1895 ByteBuilder sb=new ByteBuilder();
jpayne@68 1896 sb.append('\n');
jpayne@68 1897
jpayne@68 1898 {
jpayne@68 1899 sb.append("\nVersion\tCount\n");
jpayne@68 1900 ArrayList<String> list=new ArrayList<String>();
jpayne@68 1901 for(Entry<String, StringNum> e : versionMap.entrySet()){
jpayne@68 1902 list.add(e.getValue().toString());
jpayne@68 1903 }
jpayne@68 1904 Collections.sort(list);
jpayne@68 1905 for(String s : list){
jpayne@68 1906 sb.append(s).append('\n');
jpayne@68 1907 }
jpayne@68 1908 }
jpayne@68 1909
jpayne@68 1910 {
jpayne@68 1911 sb.append("\nSketchs\tCount\tAvgTime\n");
jpayne@68 1912 for(int i=0; i<timesByCount.length(); i++){
jpayne@68 1913 double a=timesByCount.get(i)/1000000.0;
jpayne@68 1914 long b=queryCounts.get(i);
jpayne@68 1915 if(b>0){
jpayne@68 1916 sb.append(i).append('\t').append(b).append('\t').append(a/b, 3).append('\n');
jpayne@68 1917 }
jpayne@68 1918 }
jpayne@68 1919 sb.append('\n');
jpayne@68 1920 }
jpayne@68 1921 return sb.toString();
jpayne@68 1922 }
jpayne@68 1923
jpayne@68 1924 public String USAGE(String prefix){
jpayne@68 1925 if(!countQueries){return prefix;}
jpayne@68 1926 String basicStats=basicStats();
jpayne@68 1927 return (prefix==null ? basicStats : prefix+"\n"+basicStats);
jpayne@68 1928 }
jpayne@68 1929
jpayne@68 1930 public String basicStats(){
jpayne@68 1931 if(!countQueries){return "";}
jpayne@68 1932 StringBuilder sb=new StringBuilder(500);
jpayne@68 1933
jpayne@68 1934 final long uq=usageQueries.getAndIncrement();
jpayne@68 1935 final long mq=malformedQueries.get();
jpayne@68 1936 final long pt=plaintextQueries.get(), sc=semicolonQueries.get(), pa=pathQueries.get(), pp=printPathQueries.get(), ps=printSizeQueries.get();
jpayne@68 1937 final long iq=internalQueries.get();
jpayne@68 1938 final long lq=localQueries.get();
jpayne@68 1939 final long rfq=refQueries.get();
jpayne@68 1940 final long q=queries.get();
jpayne@68 1941 final long nf=notFound.get();
jpayne@68 1942 final double avgTimeDL=.000001*(elapsedTimeLocal.get()/(Tools.max(1.0, timeMeasurementsLocal.get())));//in milliseconds
jpayne@68 1943 final double lastTimeDL=.000001*lastTimeLocal.get();
jpayne@68 1944 final double avgTimeDR=.000001*(elapsedTimeRemote.get()/(Tools.max(1.0, timeMeasurementsRemote.get())));//in milliseconds
jpayne@68 1945 final double lastTimeDR=.000001*lastTimeRemote.get();
jpayne@68 1946 final double avgTimeDRF=.000001*(elapsedTimeReference.get()/(Tools.max(1.0, timeMeasurementsReference.get())));//in milliseconds
jpayne@68 1947 final double lastTimeDRF=.000001*lastTimeReference.get();
jpayne@68 1948 final double avgTimeDU=.000001*(elapsedTimeUsage.get()/(Tools.max(1.0, timeMeasurementsUsage.get())));//in milliseconds
jpayne@68 1949 final double lastTimeDU=.000001*lastTimeUsage.get();
jpayne@68 1950 final long exq=q-iq;
jpayne@68 1951 final long rmq=q-lq;
jpayne@68 1952
jpayne@68 1953 sb.append('\n').append("Queries: ").append(q);
jpayne@68 1954 sb.append('\n').append("Usage: ").append(uq);
jpayne@68 1955 if(sketchOnly){
jpayne@68 1956 sb.append('\n').append("Invalid: ").append(mq);
jpayne@68 1957 sb.append('\n').append("Avg time: ").append(String.format(Locale.ROOT, "%.3f ms (local queries)", avgTimeDL));
jpayne@68 1958 sb.append('\n').append("Last time: ").append(String.format(Locale.ROOT, "%.3f ms (local queries)", lastTimeDL));
jpayne@68 1959 sb.append('\n').append("Avg time: ").append(String.format(Locale.ROOT, "%.3f ms (remote queries)", avgTimeDR));
jpayne@68 1960 sb.append('\n').append("Last time: ").append(String.format(Locale.ROOT, "%.3f ms (remote queries)", lastTimeDR));
jpayne@68 1961 sb.append('\n').append("Avg time: ").append(String.format(Locale.ROOT, "%.3f ms (ref queries)", avgTimeDRF));
jpayne@68 1962 sb.append('\n').append("Last time: ").append(String.format(Locale.ROOT, "%.3f ms (ref queries)", lastTimeDRF));
jpayne@68 1963 }else{
jpayne@68 1964 sb.append('\n').append("Avg time: ").append(String.format(Locale.ROOT, "%.3f ms", avgTimeDR));
jpayne@68 1965 sb.append('\n').append("Last time: ").append(String.format(Locale.ROOT, "%.3f ms", lastTimeDR));
jpayne@68 1966 sb.append('\n').append("Avg time: ").append(String.format(Locale.ROOT, "%.3f ms (usage queries)", avgTimeDU));
jpayne@68 1967 sb.append('\n').append("Last time: ").append(String.format(Locale.ROOT, "%.3f ms (usage queries)", lastTimeDU));
jpayne@68 1968 }
jpayne@68 1969 sb.append('\n');
jpayne@68 1970 sb.append('\n').append("Internal: ").append(iq);
jpayne@68 1971 sb.append('\n').append("External: ").append(exq);
jpayne@68 1972 if(!sketchOnly){sb.append('\n').append("NotFound: ").append(nf);}
jpayne@68 1973 sb.append('\n');
jpayne@68 1974
jpayne@68 1975 if(sketchOnly){
jpayne@68 1976 sb.append('\n').append("Local: ").append(lq);
jpayne@68 1977 sb.append('\n').append("Remote: ").append(rmq);
jpayne@68 1978 sb.append('\n').append("Reference: ").append(rfq);
jpayne@68 1979 sb.append('\n');
jpayne@68 1980 sb.append('\n').append("Depth: ").append(depthQueries.get());
jpayne@68 1981 sb.append('\n');
jpayne@68 1982 sb.append('\n').append("Sketches: ").append(querySketches.get());
jpayne@68 1983 sb.append('\n').append("BytesIn: ").append(bytesIn.get());
jpayne@68 1984 sb.append('\n').append("BytesOut: ").append(bytesOut.get());
jpayne@68 1985 sb.append('\n');
jpayne@68 1986 sb.append('\n').append("Single: ").append(firstChunkSingle.get());
jpayne@68 1987 sb.append('\n').append("Bulk: ").append(firstChunkMulti.get());
jpayne@68 1988 sb.append('\n').append("UnknownS: ").append(unknownChunkSingle.get());
jpayne@68 1989 sb.append('\n').append("UnknownB: ").append(unknownChunkMulti.get());
jpayne@68 1990 sb.append('\n').append("Total: ").append(bulkCount.get());
jpayne@68 1991 }else{
jpayne@68 1992 sb.append('\n').append("gi: ").append(giQueries.get());
jpayne@68 1993 sb.append('\n').append("Name: ").append(nameQueries.get());
jpayne@68 1994 sb.append('\n').append("TaxID: ").append(taxidQueries.get());
jpayne@68 1995 sb.append('\n').append("Header: ").append(headerQueries.get());
jpayne@68 1996 sb.append('\n').append("Accession: ").append(accessionQueries.get());
jpayne@68 1997 sb.append('\n').append("IMG: ").append(imgQueries.get());
jpayne@68 1998 sb.append('\n').append("Silva: ").append(silvaHeaderQueries.get());
jpayne@68 1999 sb.append('\n');
jpayne@68 2000 sb.append('\n').append("Simple: ").append(simpleQueries.get());
jpayne@68 2001 sb.append('\n').append("Ancestor: ").append(ancestorQueries.get());
jpayne@68 2002 sb.append('\n').append("Children: ").append(childrenQueries.get());
jpayne@68 2003 sb.append('\n');
jpayne@68 2004 sb.append('\n').append("Json: ").append(q-pt-sc-pa);
jpayne@68 2005 sb.append('\n').append("Plaintext: ").append(pt);
jpayne@68 2006 sb.append('\n').append("Semicolon: ").append(sc);
jpayne@68 2007 sb.append('\n').append("Path: ").append(pa+pp);
jpayne@68 2008 sb.append('\n').append("Size: ").append(ps);
jpayne@68 2009 sb.append('\n').append("Single: ").append(firstChunkSingle.get());
jpayne@68 2010 sb.append('\n').append("Bulk: ").append(firstChunkMulti.get());
jpayne@68 2011 sb.append('\n').append("Total: ").append(bulkCount.get());
jpayne@68 2012 }
jpayne@68 2013 sb.append('\n');
jpayne@68 2014 return sb.toString();
jpayne@68 2015 }
jpayne@68 2016
jpayne@68 2017 public boolean incrementQueries(HttpExchange t, boolean local, boolean refMode, boolean simple, boolean ancestor,
jpayne@68 2018 boolean plaintext, boolean semicolon, boolean path, boolean printChildren, boolean printPath, boolean printSize, int type){
jpayne@68 2019 final boolean internal=ServerTools.isInternalQuery(t, addressPrefix, allowLocalHost, printIP, printHeaders);
jpayne@68 2020
jpayne@68 2021 if(!countQueries){return internal;}
jpayne@68 2022 queries.incrementAndGet();
jpayne@68 2023 if(local){localQueries.incrementAndGet();}
jpayne@68 2024 else if(refMode){localQueries.incrementAndGet();}
jpayne@68 2025
jpayne@68 2026 if(type>=0){
jpayne@68 2027 int type2=type&15;
jpayne@68 2028 if(type2==GI){
jpayne@68 2029 giQueries.incrementAndGet();
jpayne@68 2030 }else if(type2==NAME){
jpayne@68 2031 nameQueries.incrementAndGet();
jpayne@68 2032 }else if(type2==TAXID){
jpayne@68 2033 taxidQueries.incrementAndGet();
jpayne@68 2034 }else if(type2==ACCESSION){
jpayne@68 2035 accessionQueries.incrementAndGet();
jpayne@68 2036 }else if(type2==IMG){
jpayne@68 2037 imgQueries.incrementAndGet();
jpayne@68 2038 }else if(type2==HEADER){
jpayne@68 2039 headerQueries.incrementAndGet();
jpayne@68 2040 }else if(type2==UNKNOWN){
jpayne@68 2041 unknownQueries.incrementAndGet();
jpayne@68 2042 }else if(type2==SILVAHEADER){
jpayne@68 2043 silvaHeaderQueries.incrementAndGet();
jpayne@68 2044 }
jpayne@68 2045
jpayne@68 2046 if(simple){simpleQueries.incrementAndGet();}
jpayne@68 2047 if(ancestor){ancestorQueries.incrementAndGet();}
jpayne@68 2048
jpayne@68 2049 if(plaintext){plaintextQueries.incrementAndGet();}
jpayne@68 2050 else if(semicolon){semicolonQueries.incrementAndGet();}
jpayne@68 2051 else if(path){pathQueries.incrementAndGet();}
jpayne@68 2052
jpayne@68 2053 if(printChildren){childrenQueries.incrementAndGet();}
jpayne@68 2054 if(printPath){printPathQueries.incrementAndGet();}
jpayne@68 2055 if(printSize){printSizeQueries.incrementAndGet();}
jpayne@68 2056 }
jpayne@68 2057
jpayne@68 2058 if(internal){internalQueries.incrementAndGet();}
jpayne@68 2059
jpayne@68 2060 return internal;
jpayne@68 2061 }
jpayne@68 2062
jpayne@68 2063 /*--------------------------------------------------------------*/
jpayne@68 2064
jpayne@68 2065 String compare(ArrayList<Sketch> inSketches, DisplayParams params){
jpayne@68 2066 boolean success=true;
jpayne@68 2067 final int inSize=inSketches.size();
jpayne@68 2068 querySketches.addAndGet(inSize);
jpayne@68 2069 if(Shared.threads()<2 || maxConcurrentSketchCompareThreads<2 || inSize<4){
jpayne@68 2070 ByteBuilder sb=new ByteBuilder();
jpayne@68 2071 success=searcher.compare(inSketches, sb, params, maxConcurrentSketchCompareThreads);
jpayne@68 2072 return sb.toString();
jpayne@68 2073 }else{//More sketches than threads, and more than one thread
jpayne@68 2074 final int threads=Tools.min(maxConcurrentSketchCompareThreads, (inSize+4)/4);
jpayne@68 2075
jpayne@68 2076 ByteBuilder[] out=new ByteBuilder[inSize];
jpayne@68 2077 ArrayList<CompareThread> alct=new ArrayList<CompareThread>(threads);
jpayne@68 2078 AtomicInteger next=new AtomicInteger(0);
jpayne@68 2079 for(int i=0; i<threads; i++){
jpayne@68 2080 alct.add(new CompareThread(inSketches, i, next, out, params));
jpayne@68 2081 }
jpayne@68 2082 for(CompareThread ct : alct){ct.start();}
jpayne@68 2083 for(CompareThread ct : alct){
jpayne@68 2084
jpayne@68 2085 //Wait until this thread has terminated
jpayne@68 2086 while(ct.getState()!=Thread.State.TERMINATED){
jpayne@68 2087 try {
jpayne@68 2088 //Attempt a join operation
jpayne@68 2089 ct.join();
jpayne@68 2090 } catch (InterruptedException e) {
jpayne@68 2091 e.printStackTrace();
jpayne@68 2092 }
jpayne@68 2093 }
jpayne@68 2094
jpayne@68 2095 synchronized(ct){
jpayne@68 2096 success&=ct.success;
jpayne@68 2097 }
jpayne@68 2098 }
jpayne@68 2099 alct=null;
jpayne@68 2100
jpayne@68 2101 int len=0;
jpayne@68 2102 for(ByteBuilder bb : out){len=len+bb.length;}
jpayne@68 2103 ByteBuilder bb2=new ByteBuilder(len);
jpayne@68 2104 for(int i=0; i<out.length; i++){
jpayne@68 2105 ByteBuilder bb=out[i];
jpayne@68 2106 bb2.append(bb);
jpayne@68 2107 out[i]=null;
jpayne@68 2108 }
jpayne@68 2109 return bb2.toString();
jpayne@68 2110 }
jpayne@68 2111 }
jpayne@68 2112
jpayne@68 2113 private class CompareThread extends Thread {
jpayne@68 2114
jpayne@68 2115 CompareThread(final ArrayList<Sketch> inSketches_, final int tid_, final AtomicInteger nextSketch_, ByteBuilder[] out_, DisplayParams params_){
jpayne@68 2116 inSketches=inSketches_;
jpayne@68 2117 tid=tid_;
jpayne@68 2118 nextSketch=nextSketch_;
jpayne@68 2119 out=out_;
jpayne@68 2120 params=params_;
jpayne@68 2121 }
jpayne@68 2122
jpayne@68 2123 @Override
jpayne@68 2124 public void run(){
jpayne@68 2125 success=false;
jpayne@68 2126 final int inLim=inSketches.size();
jpayne@68 2127 final boolean json=params.json();
jpayne@68 2128
jpayne@68 2129 for(int inNum=nextSketch.getAndIncrement(); inNum<inLim; inNum=nextSketch.getAndIncrement()){
jpayne@68 2130 Sketch a=inSketches.get(inNum);
jpayne@68 2131 assert(buffer.cbs==null); //Because this sketch will only be used by one thread at a time, so per-buffer bitsets are not needed.
jpayne@68 2132 SketchResults sr=searcher.processSketch(a, buffer, fakeID, map, params, 1);
jpayne@68 2133 a.clearRefHitCounts();
jpayne@68 2134
jpayne@68 2135 ByteBuilder bb=sr.toText(params);
jpayne@68 2136 if(out!=null){
jpayne@68 2137 if(json && inLim>1){
jpayne@68 2138 if(inNum==0){
jpayne@68 2139 bb.insert(0, (byte)'[');
jpayne@68 2140 }
jpayne@68 2141 if(inNum<inLim-1){
jpayne@68 2142 bb.append(',');
jpayne@68 2143 }else{
jpayne@68 2144 bb.append(']');
jpayne@68 2145 }
jpayne@68 2146 }
jpayne@68 2147 synchronized(out){
jpayne@68 2148 out[inNum]=bb;
jpayne@68 2149 }
jpayne@68 2150 }
jpayne@68 2151 }
jpayne@68 2152 synchronized(this){success=true;}
jpayne@68 2153 }
jpayne@68 2154
jpayne@68 2155 private final ArrayList<Sketch> inSketches;
jpayne@68 2156 private final int tid;
jpayne@68 2157 private final CompareBuffer buffer=new CompareBuffer(false);
jpayne@68 2158 private final DisplayParams params;
jpayne@68 2159 private final ByteBuilder[] out;
jpayne@68 2160
jpayne@68 2161 private final AtomicInteger nextSketch;
jpayne@68 2162 private final AtomicInteger fakeID=new AtomicInteger(SketchObject.minFakeID);
jpayne@68 2163 private ConcurrentHashMap<Integer, Comparison> map=new ConcurrentHashMap<Integer, Comparison>(101);
jpayne@68 2164
jpayne@68 2165 boolean success=false;
jpayne@68 2166
jpayne@68 2167 }
jpayne@68 2168
jpayne@68 2169
jpayne@68 2170 /*--------------------------------------------------------------*/
jpayne@68 2171 /*---------------- Fields ----------------*/
jpayne@68 2172 /*--------------------------------------------------------------*/
jpayne@68 2173
jpayne@68 2174 public boolean sketchOnly=false;
jpayne@68 2175
jpayne@68 2176 /*--------------------------------------------------------------*/
jpayne@68 2177 /*---------------- Counters ----------------*/
jpayne@68 2178 /*--------------------------------------------------------------*/
jpayne@68 2179
jpayne@68 2180 private HashMap<String, StringNum> versionMap=new HashMap<String, StringNum>();
jpayne@68 2181 private AtomicLongArray timesByCount=new AtomicLongArray(10000);
jpayne@68 2182 private AtomicLongArray queryCounts=new AtomicLongArray(10000);
jpayne@68 2183
jpayne@68 2184 private AtomicLong notFound=new AtomicLong(0);
jpayne@68 2185 private AtomicLong queries=new AtomicLong(0);
jpayne@68 2186 /** Same IP address mask */
jpayne@68 2187 private AtomicLong internalQueries=new AtomicLong(0);
jpayne@68 2188 /** Local filesystem sketch */
jpayne@68 2189 private AtomicLong localQueries=new AtomicLong(0);
jpayne@68 2190 private AtomicLong refQueries=new AtomicLong(0);
jpayne@68 2191
jpayne@68 2192 private AtomicLong depthQueries=new AtomicLong(0);
jpayne@68 2193
jpayne@68 2194 private AtomicLong iconQueries=new AtomicLong(0);
jpayne@68 2195
jpayne@68 2196 private AtomicLong querySketches=new AtomicLong(0);
jpayne@68 2197
jpayne@68 2198 private AtomicLong unknownChunkSingle=new AtomicLong(0);
jpayne@68 2199 private AtomicLong unknownChunkMulti=new AtomicLong(0);
jpayne@68 2200 private AtomicLong firstChunkSingle=new AtomicLong(0);
jpayne@68 2201 private AtomicLong firstChunkMulti=new AtomicLong(0);
jpayne@68 2202 private AtomicLong nthChunkSingle=new AtomicLong(0);
jpayne@68 2203 private AtomicLong nthChunkMulti=new AtomicLong(0);
jpayne@68 2204
jpayne@68 2205 private AtomicLong singleQueries=new AtomicLong(0);
jpayne@68 2206 private AtomicLong bulkQueries=new AtomicLong(0);
jpayne@68 2207 private AtomicLong bulkCount=new AtomicLong(0);
jpayne@68 2208
jpayne@68 2209 private AtomicLong giQueries=new AtomicLong(0);
jpayne@68 2210 private AtomicLong nameQueries=new AtomicLong(0);
jpayne@68 2211 private AtomicLong taxidQueries=new AtomicLong(0);
jpayne@68 2212 private AtomicLong headerQueries=new AtomicLong(0);
jpayne@68 2213 private AtomicLong accessionQueries=new AtomicLong(0);
jpayne@68 2214 private AtomicLong imgQueries=new AtomicLong(0);
jpayne@68 2215 private AtomicLong unknownQueries=new AtomicLong(0);
jpayne@68 2216 private AtomicLong silvaHeaderQueries=new AtomicLong(0);
jpayne@68 2217
jpayne@68 2218 private AtomicLong plaintextQueries=new AtomicLong(0);
jpayne@68 2219 private AtomicLong semicolonQueries=new AtomicLong(0);
jpayne@68 2220 private AtomicLong pathQueries=new AtomicLong(0);
jpayne@68 2221 private AtomicLong printPathQueries=new AtomicLong(0);
jpayne@68 2222 private AtomicLong printSizeQueries=new AtomicLong(0);
jpayne@68 2223 private AtomicLong childrenQueries=new AtomicLong(0);
jpayne@68 2224
jpayne@68 2225 private AtomicLong simpleQueries=new AtomicLong(0);
jpayne@68 2226 private AtomicLong ancestorQueries=new AtomicLong(0);
jpayne@68 2227
jpayne@68 2228 private AtomicLong usageQueries=new AtomicLong(0);
jpayne@68 2229 private AtomicLong bytesIn=new AtomicLong(0);
jpayne@68 2230 private AtomicLong bytesOut=new AtomicLong(0);
jpayne@68 2231
jpayne@68 2232 // private AtomicLong elapsedTime=new AtomicLong(0);
jpayne@68 2233 // private AtomicLong timeMeasurements=new AtomicLong(0);
jpayne@68 2234 // private AtomicLong lastTime=new AtomicLong(0);
jpayne@68 2235
jpayne@68 2236 private AtomicLong elapsedTimeUsage=new AtomicLong(0);
jpayne@68 2237 private AtomicLong timeMeasurementsUsage=new AtomicLong(0);
jpayne@68 2238 private AtomicLong lastTimeUsage=new AtomicLong(0);
jpayne@68 2239
jpayne@68 2240 private AtomicLong elapsedTimeRemote=new AtomicLong(0);
jpayne@68 2241 private AtomicLong timeMeasurementsRemote=new AtomicLong(0);
jpayne@68 2242 private AtomicLong lastTimeRemote=new AtomicLong(0);
jpayne@68 2243
jpayne@68 2244 private AtomicLong elapsedTimeLocal=new AtomicLong(0);
jpayne@68 2245 private AtomicLong timeMeasurementsLocal=new AtomicLong(0);
jpayne@68 2246 private AtomicLong lastTimeLocal=new AtomicLong(0);
jpayne@68 2247
jpayne@68 2248 private AtomicLong elapsedTimeReference=new AtomicLong(0);
jpayne@68 2249 private AtomicLong timeMeasurementsReference=new AtomicLong(0);
jpayne@68 2250 private AtomicLong lastTimeReference=new AtomicLong(0);
jpayne@68 2251
jpayne@68 2252 private AtomicLong malformedQueries=new AtomicLong(0);
jpayne@68 2253
jpayne@68 2254 /*--------------------------------------------------------------*/
jpayne@68 2255 /*---------------- Params ----------------*/
jpayne@68 2256 /*--------------------------------------------------------------*/
jpayne@68 2257
jpayne@68 2258 public boolean printIP=false;
jpayne@68 2259 public boolean printHeaders=false;
jpayne@68 2260 public boolean countQueries=true;
jpayne@68 2261 public float prealloc=0;
jpayne@68 2262 public boolean useHtml=false;
jpayne@68 2263
jpayne@68 2264 /** Location of GiTable file */
jpayne@68 2265 private String giTableFile=null;
jpayne@68 2266 /** Location of TaxTree file */
jpayne@68 2267 private String taxTreeFile="auto";
jpayne@68 2268 /** Comma-delimited locations of Accession files */
jpayne@68 2269 private String accessionFile=null;
jpayne@68 2270 /** Location of IMG dump file */
jpayne@68 2271 private String imgFile=null;
jpayne@68 2272 /** Location of accession pattern file */
jpayne@68 2273 private String patternFile=null;
jpayne@68 2274
jpayne@68 2275 private String sizeFile=null;
jpayne@68 2276
jpayne@68 2277 /** Location of sequence directory tree */
jpayne@68 2278 private String basePath="/global/cfs/cdirs/bbtools/tree/";
jpayne@68 2279
jpayne@68 2280 /** Used for taxonomic tree traversal */
jpayne@68 2281 private final TaxTree tree;
jpayne@68 2282
jpayne@68 2283 /** Maps URL Strings to numeric query types */
jpayne@68 2284 private final HashMap<String, Integer> typeMap;
jpayne@68 2285 /** Maps common organism names to scientific names */
jpayne@68 2286 private final HashMap<String, String> commonMap;
jpayne@68 2287
jpayne@68 2288 /** Hash taxonomic names for lookup */
jpayne@68 2289 private boolean hashNames=true;
jpayne@68 2290 private boolean hashDotFormat=true;
jpayne@68 2291
jpayne@68 2292 /** Kill code of prior server instance (optional) */
jpayne@68 2293 private String oldKillCode=null;
jpayne@68 2294 /** Address of prior server instance (optional) */
jpayne@68 2295 private String oldAddress=null;
jpayne@68 2296
jpayne@68 2297 /** Address of current server instance (optional) */
jpayne@68 2298 public String domain=null;
jpayne@68 2299
jpayne@68 2300 public int maxConcurrentSketchCompareThreads=8;//TODO: This might be too high when lots of concurrent sessions are active
jpayne@68 2301 public int maxConcurrentSketchLoadThreads=4;//TODO: This might be too high when lots of concurrent sessions are active
jpayne@68 2302 public int handlerThreads=-1;
jpayne@68 2303
jpayne@68 2304 /*--------------------------------------------------------------*/
jpayne@68 2305 /*---------------- Final Fields ----------------*/
jpayne@68 2306 /*--------------------------------------------------------------*/
jpayne@68 2307
jpayne@68 2308 private final boolean distributed;
jpayne@68 2309 private final int serverNum;
jpayne@68 2310 private final int serverCount;
jpayne@68 2311 private ArrayList<String> slaveAddress;
jpayne@68 2312
jpayne@68 2313 public final String favIconPath=Data.findPath("?favicon.ico");
jpayne@68 2314 public final byte[] favIcon=ReadWrite.readRaw(favIconPath);
jpayne@68 2315
jpayne@68 2316 private final String startTime=new Date().toString();
jpayne@68 2317
jpayne@68 2318 /** Listen on this port */
jpayne@68 2319 public final int port;
jpayne@68 2320 /** Code to validate kill requests */
jpayne@68 2321 public final String killCode;
jpayne@68 2322
jpayne@68 2323 public final HttpServer httpServer;
jpayne@68 2324
jpayne@68 2325 /** Bit to set for plaintext query types */
jpayne@68 2326 public static final int PT_BIT=16;
jpayne@68 2327 /** Bit to set for semicolon-delimited query types */
jpayne@68 2328 public static final int SC_BIT=32;
jpayne@68 2329 /** Bit to set for path query types */
jpayne@68 2330 public static final int PA_BIT=64;
jpayne@68 2331 /** Request query types */
jpayne@68 2332 public static final int UNKNOWN=0, GI=1, NAME=2, TAXID=3, HEADER=4, ACCESSION=5, IMG=6, SILVAHEADER=7;
jpayne@68 2333 /** Plaintext-response query types */
jpayne@68 2334 public static final int PT_GI=GI+PT_BIT, PT_NAME=NAME+PT_BIT, PT_TAXID=TAXID+PT_BIT,
jpayne@68 2335 PT_HEADER=HEADER+PT_BIT, PT_ACCESSION=ACCESSION+PT_BIT, PT_IMG=IMG+PT_BIT, PT_SILVAHEADER=SILVAHEADER+PT_BIT;
jpayne@68 2336 /** Semicolon-response query types */
jpayne@68 2337 public static final int SC_GI=GI+SC_BIT, SC_NAME=NAME+SC_BIT, SC_TAXID=TAXID+SC_BIT,
jpayne@68 2338 SC_HEADER=HEADER+SC_BIT, SC_ACCESSION=ACCESSION+SC_BIT, SC_IMG=IMG+SC_BIT, SC_SILVAHEADER=SILVAHEADER+PT_BIT;
jpayne@68 2339
jpayne@68 2340 public static final int SOURCE_REFSEQ=1, SOURCE_SILVA=2, SOURCE_IMG=3;
jpayne@68 2341
jpayne@68 2342 /** Generic response when asking for tax advice */
jpayne@68 2343 public static final String TAX_ADVICE="This site does not give tax advice.";
jpayne@68 2344 /** Generic response for incorrect kill code */
jpayne@68 2345 public static final String BAD_CODE="Incorrect code.";
jpayne@68 2346 /** Generic response for badly-formatted queries */
jpayne@68 2347 public final String USAGE;
jpayne@68 2348 /** HTML version */
jpayne@68 2349 // public final String USAGE_HTML;
jpayne@68 2350 public final String rawHtml;
jpayne@68 2351
jpayne@68 2352 /** Tool for comparing query sketches to reference sketches */
jpayne@68 2353 public final SketchSearcher searcher=new SketchSearcher();
jpayne@68 2354
jpayne@68 2355 public final boolean hasSketches;
jpayne@68 2356
jpayne@68 2357 final boolean allowRemoteFileAccess;
jpayne@68 2358 final boolean allowLocalHost;
jpayne@68 2359 final String addressPrefix;
jpayne@68 2360 private boolean clearMem=true;
jpayne@68 2361
jpayne@68 2362 /*--------------------------------------------------------------*/
jpayne@68 2363 /*---------------- Common Fields ----------------*/
jpayne@68 2364 /*--------------------------------------------------------------*/
jpayne@68 2365
jpayne@68 2366 /** Print status messages to this output stream */
jpayne@68 2367 private PrintStream outstream=System.err;
jpayne@68 2368 /** Print verbose messages */
jpayne@68 2369 public static boolean verbose=false, verbose2=false, logUsage=false;
jpayne@68 2370 /** True if an error was encountered */
jpayne@68 2371 public boolean errorState=false;
jpayne@68 2372
jpayne@68 2373 }