Mercurial > repos > jpayne > snp_pipeline

comparison snp-cache.py @ 60:0b0e3e4376a7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload
author jpayne
date Mon, 24 Jun 2024 05:10:35 -0400
parents 11296a86e01b
children fb44b003e29b
comparison
equal deleted inserted replaced
59:49d672730d3e 60:0b0e3e4376a7
1 #! /usr/bin/env python 1 #! /usr/bin/env python
2 2
3 from __future__ import print_function 3 from __future__ import print_function
4 4
5 import boto3 5
6 from botocore.exceptions import ClientError, DataNotFoundError, NoCredentialsError, BotoCoreError
7 6
8 import argparse 7 import argparse
9 import subprocess 8 import subprocess
10 import contextlib 9 import contextlib
11 import logging 10 import logging
99 table = table.strip() 98 table = table.strip()
100 name = "{table}/{id}".format(**locals()) 99 name = "{table}/{id}".format(**locals())
101 with open(output, 'wb') as output_f: 100 with open(output, 'wb') as output_f:
102 #lookup ID in table and get a FH to the resource 101 #lookup ID in table and get a FH to the resource
103 try: 102 try:
103 import boto3
104 api_key = os.environ.get('AWS_API_KEY', '') 104 api_key = os.environ.get('AWS_API_KEY', '')
105 s3 = boto3.resource('s3').Bucket(CACHE_NAMESPACE) 105 s3 = boto3.resource('s3').Bucket(CACHE_NAMESPACE)
106 s3.download_fileobj(name, output_f) 106 s3.download_fileobj(name, output_f)
107 logging.getLogger('snp-cache.cache').info("cache hit on {name}, retrieved.".format(**locals())) 107 logging.getLogger('snp-cache.cache').info("cache hit on {name}, retrieved.".format(**locals()))
108 except (DataNotFoundError, NoCredentialsError, BotoCoreError, ClientError) as e: 108 except Exception as e:
109 if type(e) is DataNotFoundError: 109 if type(e) is not ImportError:
110 logging.getLogger('snp-cache.cache').info("cache miss on {name}".format(**locals())) 110 logging.getLogger('snp-cache.cache').info("cache miss on {name}".format(**locals()))
111 else: 111 else:
112 logging.getLogger('snp-cache.cache').error(e) 112 logging.getLogger('snp-cache.cache').error(e)
113 #if we couldn't find the data, we need to run the command to generate it 113 #if we couldn't find the data, we need to run the command to generate it
114 if not command: 114 if not command:
119 # cached, err = io.BytesIO(cached), io.BytesIO(err) 119 # cached, err = io.BytesIO(cached), io.BytesIO(err)
120 try: 120 try:
121 cached = subprocess.check_output(command, shell=True) 121 cached = subprocess.check_output(command, shell=True)
122 try: 122 try:
123 s3.upload_fileobj(BytesIO(cached), name) 123 s3.upload_fileobj(BytesIO(cached), name)
124 except (ClientError, BotoCoreError) as e: 124 except Exception as e:
125 logging.getLogger('snp-cache.cache').error('Error writing to cache:') 125 logging.getLogger('snp-cache.cache').error('Error writing to cache:')
126 logging.getLogger('snp-cache.cache').error(e) 126 logging.getLogger('snp-cache.cache').error(e)
127 finally: 127 finally:
128 #stream_to(cached, output_f) #stream FROM cached TO output_f 128 #stream_to(cached, output_f) #stream FROM cached TO output_f
129 output_f.write(cached) 129 output_f.write(cached)