Mercurial > repos > galaxytrakr > data_manager_mapseq

annotate data_manager/data_manager_fetch_mapseq_db.py @ 0:f99c1cc0b190 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
author galaxytrakr
date Thu, 26 Mar 2026 20:22:36 +0000
parents
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
1 #!/usr/bin/env python
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
2 # Python 3.6 compatible
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
3
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
4 import argparse
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
5 import json
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
6 import os
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
7 import shutil
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
8 import tarfile
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
9 import tempfile
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
10 from datetime import datetime
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
11
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
12 import wget
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
13
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
14 DB_paths = {
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
15 "mgnify_v5_lsu": "ftp://ftp.ebi.ac.uk/pub/databases/metagenomics/pipeline-5.0/ref-dbs/silva_lsu-20200130.tar.gz",
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
16 "mgnify_v5_ssu": "ftp://ftp.ebi.ac.uk/pub/databases/metagenomics/pipeline-5.0/ref-dbs/silva_ssu-20200130.tar.gz",
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
17 "mgnify_v5_its_unite": "ftp://ftp.ebi.ac.uk/pub/databases/metagenomics/pipeline-5.0/ref-dbs/UNITE-20200214.tar.gz",
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
18 "mgnify_v5_its_itsonedb": "ftp://ftp.ebi.ac.uk/pub/databases/metagenomics/pipeline-5.0/ref-dbs/ITSoneDB-20200214.tar.gz",
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
19 "mgnify_v6_lsu": "ftp://ftp.ebi.ac.uk/pub/databases/metagenomics/pipelines/tool-dbs/silva-lsu/silva-lsu_138.1.tar.gz",
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
20 "mgnify_v6_ssu": "ftp://ftp.ebi.ac.uk/pub/databases/metagenomics/pipelines/tool-dbs/silva-ssu/silva-ssu_138.1.tar.gz",
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
21 "mgnify_v6_its_unite": "ftp://ftp.ebi.ac.uk/pub/databases/metagenomics/pipelines/tool-dbs/unite/unite_9.0.tar.gz",
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
22 "mgnify_v6_its_itsonedb": "ftp://ftp.ebi.ac.uk/pub/databases/metagenomics/pipelines/tool-dbs/itsonedb/itsonedb_1.141.tar.gz",
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
23 "mgnify_v6_pr2": "ftp://ftp.ebi.ac.uk/pub/databases/metagenomics/pipelines/tool-dbs/pr2/pr2_5.0.0.tar.gz",
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
24 "test_lsu": "https://zenodo.org/record/8205348/files/test_lsu.tar.gz",
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
25 }
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
26
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
27 DB_names = {
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
28 "mgnify_v5_lsu": "MGnify LSU (v5.0.7) - silva_lsu-20200130",
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
29 "mgnify_v5_ssu": "MGnify SSU (v5.0.7) - silva_ssu-20200130",
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
30 "mgnify_v5_its_unite": "MGnify ITS UNITE (v5.0.7) - UNITE-20200214",
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
31 "mgnify_v5_its_itsonedb": "MGnify ITS ITSonedb (v5.0.7) - ITSoneDB-20200214",
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
32 "mgnify_v6_lsu": "MGnify LSU (v6.0) - silva_lsu-20240702",
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
33 "mgnify_v6_ssu": "MGnify SSU (v6.0) - silva_ssu-20240701",
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
34 "mgnify_v6_its_unite": "MGnify ITS UNITE (v6.0) - UNITE-20240702",
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
35 "mgnify_v6_its_itsonedb": "MGnify ITS ITSonedb (v6.0) - ITSoneDB-20240702",
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
36 "mgnify_v6_pr2": "MGnify PR2 (v6.0) - PR2-20240702",
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
37 "test_lsu": "Trimmed LSU Test DB",
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
38 }
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
39
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
40 def _empty_dir(path):
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
41 if not os.path.isdir(path):
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
42 os.makedirs(path)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
43 return
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
44 for name in os.listdir(path):
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
45 p = os.path.join(path, name)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
46 if os.path.isdir(p) and not os.path.islink(p):
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
47 shutil.rmtree(p)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
48 else:
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
49 try:
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
50 os.remove(p)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
51 except OSError:
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
52 pass
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
53
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
54 def _copy_all(src_dir, dest_dir):
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
55 for name in os.listdir(src_dir):
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
56 s = os.path.join(src_dir, name)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
57 d = os.path.join(dest_dir, name)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
58 if os.path.isdir(s) and not os.path.islink(s):
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
59 if os.path.exists(d):
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
60 shutil.rmtree(d)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
61 shutil.copytree(s, d)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
62 else:
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
63 shutil.copy2(s, dest_dir)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
64
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
65 def _safe_members(members):
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
66 for m in members:
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
67 mpath = m.name
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
68 norm = mpath.replace("\\", "/")
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
69 if os.path.isabs(mpath) or ".." in norm.split("/"):
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
70 continue
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
71 yield m
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
72
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
73 def materialize_db_into(output_dir, url):
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
74 # Work entirely under /tmp to avoid job-dir races
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
75 tmp_root = tempfile.mkdtemp(prefix="mapseq_dm_", dir="/tmp")
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
76 temp_extract = tempfile.mkdtemp(prefix="extract_", dir=tmp_root)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
77
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
78 tar_path = wget.download(url, out=tmp_root)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
79 with tarfile.open(tar_path, "r:*") as tar:
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
80 tar.extractall(temp_extract, members=_safe_members(tar))
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
81
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
82 _empty_dir(output_dir)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
83
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
84 entries = os.listdir(temp_extract)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
85 if len(entries) == 1 and os.path.isdir(os.path.join(temp_extract, entries[0])):
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
86 inner = os.path.join(temp_extract, entries[0])
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
87 vf = os.path.join(inner, "VERSION.txt")
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
88 if os.path.exists(vf):
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
89 try:
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
90 os.remove(vf)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
91 except OSError:
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
92 pass
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
93 _copy_all(inner, output_dir)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
94 else:
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
95 vf = os.path.join(temp_extract, "VERSION.txt")
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
96 if os.path.exists(vf):
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
97 try:
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
98 os.remove(vf)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
99 except OSError:
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
100 pass
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
101 _copy_all(temp_extract, output_dir)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
102
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
103 try:
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
104 shutil.rmtree(tmp_root)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
105 except Exception:
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
106 pass
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
107
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
108 def main():
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
109 parser = argparse.ArgumentParser(description="Fetch and register MAPseq DB")
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
110 parser.add_argument("--out", dest="output", required=True, help="Galaxy params/DM JSON path")
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
111 parser.add_argument("--version", dest="version", required=False, help="DB version label (e.g., 6.0)")
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
112 parser.add_argument("--database-type", dest="db_type", required=False, help="DB key (e.g., mgnify_v6_lsu)")
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
113 parser.add_argument("--test", action="store_true", help="Use small test DB")
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
114 args = parser.parse_args()
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
115
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
116 # Read Galaxy params to get extra_files_path
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
117 with open(args.output) as fh:
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
118 params = json.load(fh)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
119
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
120 output_dir = params["output_data"][0]["extra_files_path"]
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
121 if not os.path.isdir(output_dir):
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
122 os.makedirs(output_dir)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
123
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
124 # Resolve db_type/version: prefer CLI, else try params.select_version
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
125 db_type = args.db_type
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
126 version = args.version
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
127 sel = params.get("select_version") or {}
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
128 if not db_type:
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
129 db_type = sel.get("database_type")
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
130 if not version:
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
131 version = sel.get("version")
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
132
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
133 if not db_type:
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
134 raise RuntimeError("Missing --database-type and no params['select_version']['database_type'] provided.")
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
135 if db_type not in DB_paths:
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
136 raise KeyError("Unknown database type: {}. Valid keys: {}".format(db_type, ", ".join(sorted(DB_paths.keys()))))
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
137
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
138 url = DB_paths["test_lsu"] if args.test else DB_paths[db_type]
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
139 materialize_db_into(output_dir, url)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
140
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
141 # Build DM JSON (value/name pattern you used earlier)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
142 date_str = datetime.utcnow().strftime("%Y-%m-%d")
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
143 db_value = "{}_from_{}".format(db_type, date_str)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
144 db_name = DB_names.get(db_type, db_type)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
145 entry_name = "{} downloaded at {}".format(db_name, date_str)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
146
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
147 data_manager_entry = {
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
148 "data_tables": {
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
149 "mapseq_db": [
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
150 {
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
151 "value": db_value,
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
152 "name": entry_name,
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
153 "path": output_dir,
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
154 "version": version,
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
155 }
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
156 ]
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
157 }
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
158 }
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
159
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
160 with open(args.output, "w") as fh:
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
161 json.dump(data_manager_entry, fh, indent=2, sort_keys=True)
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
162
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
163 try:
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
164 count = len(os.listdir(output_dir))
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
165 print("[INFO] Wrote {} items into {}".format(count, output_dir))
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
166 except Exception:
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
167 pass
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
168
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
169 print("[SUCCESS] Data Manager JSON written to {}".format(args.output))
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
170
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
171 if __name__ == "__main__":
f99c1cc0b190 planemo upload commit bae430588c393222b68c5a7aa47f57bffee99475
galaxytrakr
parents:
diff changeset
172 main()