jpayne@1: #!/usr/bin/env python3 jpayne@1: jpayne@1: jpayne@1: import gzip jpayne@1: import io jpayne@1: import pickle jpayne@1: import os jpayne@1: import sys jpayne@1: jpayne@1: from argparse import ArgumentParser jpayne@1: try: jpayne@1: from version import SalmID_version jpayne@1: except: jpayne@1: SalmID_version = "version unknown" jpayne@1: jpayne@1: jpayne@1: def reverse_complement(sequence): jpayne@1: complement = {'A': 'T', 'C': 'G', 'G': 'C', 'T': 'A', 'N': 'N', 'M': 'K', 'R': 'Y', 'W': 'W', jpayne@1: 'S': 'S', 'Y': 'R', 'K': 'M', 'V': 'B', 'H': 'D', 'D': 'H', 'B': 'V'} jpayne@1: return "".join(complement[base] for base in reversed(sequence)) jpayne@1: jpayne@1: jpayne@1: def parse_args(): jpayne@1: "Parse the input arguments, use '-h' for help." jpayne@1: parser = ArgumentParser(description='SalmID - rapid Kmer based Salmonella identifier from sequence data') jpayne@1: # inputs jpayne@1: parser.add_argument('-v','--version', action='version', version='%(prog)s ' + SalmID_version) jpayne@1: parser.add_argument( jpayne@1: '-i','--input_file', type=str, required=False, default= 'None', metavar = 'your_fastqgz', jpayne@1: help='Single fastq.gz file input, include path to file if file is not in same directory ') jpayne@1: parser.add_argument( jpayne@1: '-e', '--extension', type=str, required=False, default= '.fastq.gz', metavar = 'file_extension', jpayne@1: help='File extension, if specified without "--input_dir", SalmID will attempt to ID all files\n' + jpayne@1: ' with this extension in current directory, otherwise files in input directory') jpayne@1: jpayne@1: parser.add_argument( jpayne@1: '-d','--input_dir', type=str, required=False, default='.', metavar = 'directory', jpayne@1: help='Directory which contains data for identification, when not specified files in current directory will be analyzed.') jpayne@1: parser.add_argument( jpayne@1: '-r', '--report', type=str, required=False, default='percentage', metavar = 'percentage, coverage or taxonomy', jpayne@1: help='Report either percentage ("percentage") of clade specific kmers recovered, average kmer-coverage ("cov"), or ' jpayne@1: 'taxonomy (taxonomic species ID, plus observed mean k-mer coverages and expected coverage).') jpayne@1: parser.add_argument( jpayne@1: '-m', '--mode', type=str, required=False, default='quick', metavar = 'quick or thorough', jpayne@1: help='Quick [quick] or thorough [thorough] mode') jpayne@1: if len(sys.argv)==1: jpayne@1: parser.print_help(sys.stderr) jpayne@1: sys.exit(1) jpayne@1: return parser.parse_args() jpayne@1: jpayne@1: def get_av_read_length(file): jpayne@1: i = 1 jpayne@1: n_reads = 0 jpayne@1: total_length = 0 jpayne@1: if file.endswith(".gz"): jpayne@1: file_content=io.BufferedReader(gzip.open(file)) jpayne@1: else: jpayne@1: file_content=open(file,"r").readlines() jpayne@1: for line in file_content: jpayne@1: if i % 4 == 2: jpayne@1: total_length += len(line.strip()) jpayne@1: n_reads +=1 jpayne@1: i += 1 jpayne@1: if n_reads == 100: jpayne@1: break jpayne@1: return total_length/100 jpayne@1: jpayne@1: jpayne@1: def createKmerDict_reads(list_of_strings, kmer): jpayne@1: kmer_table = {} jpayne@1: for string in list_of_strings: jpayne@1: sequence = string.strip('\n') jpayne@1: for i in range(len(sequence)-kmer+1): jpayne@1: new_mer =sequence[i:i+kmer] jpayne@1: new_mer_rc = reverse_complement(new_mer) jpayne@1: if new_mer in kmer_table: jpayne@1: kmer_table[new_mer.upper()] += 1 jpayne@1: else: jpayne@1: kmer_table[new_mer.upper()] = 1 jpayne@1: if new_mer_rc in kmer_table: jpayne@1: kmer_table[new_mer_rc.upper()] += 1 jpayne@1: else: jpayne@1: kmer_table[new_mer_rc.upper()] = 1 jpayne@1: return kmer_table jpayne@1: jpayne@1: jpayne@1: def target_read_kmerizer_multi(file, k, kmerDict_1, kmerDict_2, mode): jpayne@1: mean_1 = None jpayne@1: mean_2 = None jpayne@1: i = 1 jpayne@1: n_reads_1 = 0 jpayne@1: n_reads_2 = 0 jpayne@1: total_coverage_1 = 0 jpayne@1: total_coverage_2 = 0 jpayne@1: reads_1 = [] jpayne@1: reads_2 = [] jpayne@1: total_reads = 0 jpayne@1: if file.endswith(".gz"): jpayne@1: file_content=io.BufferedReader(gzip.open(file)) jpayne@1: else: jpayne@1: file_content=open(file,"r").readlines() jpayne@1: for line in file_content: jpayne@1: start = int((len(line) - k) // 2) jpayne@1: if i % 4 == 2: jpayne@1: total_reads += 1 jpayne@1: if file.endswith(".gz"): jpayne@1: s1 = line[start:k + start].decode() jpayne@1: line=line.decode() jpayne@1: else: jpayne@1: s1 = line[start:k + start] jpayne@1: if s1 in kmerDict_1: jpayne@1: n_reads_1 += 1 jpayne@1: total_coverage_1 += len(line) jpayne@1: reads_1.append(line) jpayne@1: if s1 in kmerDict_2: jpayne@1: n_reads_2 += 1 jpayne@1: total_coverage_2 += len(line) jpayne@1: reads_2.append(line) jpayne@1: i += 1 jpayne@1: if mode == 'quick': jpayne@1: if total_coverage_2 >= 800000: jpayne@1: break jpayne@1: jpayne@1: if len(reads_1) == 0: jpayne@1: kmer_Dict1 = {} jpayne@1: else: jpayne@1: kmer_Dict1 = createKmerDict_reads(reads_1, k) jpayne@1: mers_1 = set([key for key in kmer_Dict1]) jpayne@1: mean_1 = sum([kmer_Dict1[key] for key in kmer_Dict1])/len(mers_1) jpayne@1: if len(reads_2) == 0: jpayne@1: kmer_Dict2 = {} jpayne@1: else: jpayne@1: kmer_Dict2 = createKmerDict_reads(reads_2, k) jpayne@1: mers_2 = set([key for key in kmer_Dict2]) jpayne@1: mean_2 = sum([kmer_Dict2[key] for key in kmer_Dict2])/len(mers_2) jpayne@1: return kmer_Dict1, kmer_Dict2, mean_1, mean_2, total_reads jpayne@1: jpayne@1: def mean_cov_selected_kmers(iterable, kmer_dict, clade_specific_kmers): jpayne@1: ''' jpayne@1: Given an iterable (list, set, dictrionary) returns mean coverage for the kmers in iterable jpayne@1: :param iterable: set, list or dictionary containing kmers jpayne@1: :param kmer_dict: dictionary with kmers as keys, kmer-frequency as value jpayne@1: :param clade_specific_kmers: list, dict or set of clade specific kmers jpayne@1: :return: mean frequency as float jpayne@1: ''' jpayne@1: if len(iterable) == 0: jpayne@1: return 0 jpayne@1: return sum([kmer_dict[value] for value in iterable])/len(clade_specific_kmers) jpayne@1: jpayne@1: def kmer_lists(query_fastq_gz, k, jpayne@1: allmers,allmers_rpoB, jpayne@1: uniqmers_bongori, jpayne@1: uniqmers_I, jpayne@1: uniqmers_IIa, jpayne@1: uniqmers_IIb, jpayne@1: uniqmers_IIIa, jpayne@1: uniqmers_IIIb, jpayne@1: uniqmers_IV, jpayne@1: uniqmers_VI, jpayne@1: uniqmers_VII, jpayne@1: uniqmers_VIII, jpayne@1: uniqmers_bongori_rpoB, jpayne@1: uniqmers_S_enterica_rpoB, jpayne@1: uniqmers_Escherichia_rpoB, jpayne@1: uniqmers_Listeria_ss_rpoB, jpayne@1: uniqmers_Lmono_rpoB, jpayne@1: mode): jpayne@1: dict_invA, dict_rpoB, mean_invA, mean_rpoB , total_reads = target_read_kmerizer_multi(query_fastq_gz, k, allmers, jpayne@1: allmers_rpoB, mode) jpayne@1: target_mers_invA = set([key for key in dict_invA]) jpayne@1: target_mers_rpoB = set([key for key in dict_rpoB]) jpayne@1: if target_mers_invA == 0: jpayne@1: print('No reads found matching invA, no Salmonella in sample?') jpayne@1: else: jpayne@1: p_bongori = (len(uniqmers_bongori & target_mers_invA) / len(uniqmers_bongori)) * 100 jpayne@1: p_I = (len(uniqmers_I & target_mers_invA) / len(uniqmers_I)) * 100 jpayne@1: p_IIa = (len(uniqmers_IIa & target_mers_invA) / len(uniqmers_IIa)) * 100 jpayne@1: p_IIb = (len(uniqmers_IIb & target_mers_invA) / len(uniqmers_IIb)) * 100 jpayne@1: p_IIIa = (len(uniqmers_IIIa & target_mers_invA) / len(uniqmers_IIIa)) * 100 jpayne@1: p_IIIb = (len(uniqmers_IIIb & target_mers_invA) / len(uniqmers_IIIb)) * 100 jpayne@1: p_VI = (len(uniqmers_VI & target_mers_invA) / len(uniqmers_VI)) * 100 jpayne@1: p_IV = (len(uniqmers_IV & target_mers_invA) / len(uniqmers_IV)) * 100 jpayne@1: p_VII = (len(uniqmers_VII & target_mers_invA) / len(uniqmers_VII)) * 100 jpayne@1: p_VIII = (len(uniqmers_VIII & target_mers_invA) / len(uniqmers_VIII)) * 100 jpayne@1: p_bongori_rpoB = (len(uniqmers_bongori_rpoB & target_mers_rpoB) / len(uniqmers_bongori_rpoB)) * 100 jpayne@1: p_Senterica = (len(uniqmers_S_enterica_rpoB & target_mers_rpoB) / len(uniqmers_S_enterica_rpoB)) * 100 jpayne@1: p_Escherichia = (len(uniqmers_Escherichia_rpoB & target_mers_rpoB) / len(uniqmers_Escherichia_rpoB)) * 100 jpayne@1: p_Listeria_ss = (len(uniqmers_Listeria_ss_rpoB & target_mers_rpoB) / len(uniqmers_Listeria_ss_rpoB)) * 100 jpayne@1: p_Lmono = (len(uniqmers_Lmono_rpoB & target_mers_rpoB) / len(uniqmers_Lmono_rpoB)) * 100 jpayne@1: bongori_invA_cov = mean_cov_selected_kmers(uniqmers_bongori & target_mers_invA, dict_invA, uniqmers_bongori) jpayne@1: I_invA_cov = mean_cov_selected_kmers(uniqmers_I & target_mers_invA, dict_invA, uniqmers_I) jpayne@1: IIa_invA_cov = mean_cov_selected_kmers(uniqmers_IIa & target_mers_invA, dict_invA, uniqmers_IIa) jpayne@1: IIb_invA_cov = mean_cov_selected_kmers(uniqmers_IIb & target_mers_invA, dict_invA, uniqmers_IIb) jpayne@1: IIIa_invA_cov = mean_cov_selected_kmers(uniqmers_IIIa & target_mers_invA, dict_invA, uniqmers_IIIa) jpayne@1: IIIb_invA_cov = mean_cov_selected_kmers(uniqmers_IIIb & target_mers_invA, dict_invA, uniqmers_IIIb) jpayne@1: IV_invA_cov = mean_cov_selected_kmers(uniqmers_IV & target_mers_invA, dict_invA, uniqmers_IV) jpayne@1: VI_invA_cov = mean_cov_selected_kmers(uniqmers_VI & target_mers_invA, dict_invA, uniqmers_VI) jpayne@1: VII_invA_cov = mean_cov_selected_kmers(uniqmers_VII & target_mers_invA, dict_invA, uniqmers_VII) jpayne@1: VIII_invA_cov = mean_cov_selected_kmers(uniqmers_VIII & target_mers_invA, dict_invA, uniqmers_VIII) jpayne@1: S_enterica_rpoB_cov = mean_cov_selected_kmers((uniqmers_S_enterica_rpoB & target_mers_rpoB), dict_rpoB, jpayne@1: uniqmers_S_enterica_rpoB) jpayne@1: S_bongori_rpoB_cov = mean_cov_selected_kmers((uniqmers_bongori_rpoB & target_mers_rpoB), dict_rpoB, jpayne@1: uniqmers_bongori_rpoB) jpayne@1: Escherichia_rpoB_cov = mean_cov_selected_kmers((uniqmers_Escherichia_rpoB & target_mers_rpoB), dict_rpoB, jpayne@1: uniqmers_Escherichia_rpoB) jpayne@1: Listeria_ss_rpoB_cov = mean_cov_selected_kmers((uniqmers_Listeria_ss_rpoB & target_mers_rpoB), dict_rpoB, jpayne@1: uniqmers_Listeria_ss_rpoB) jpayne@1: Lmono_rpoB_cov = mean_cov_selected_kmers((uniqmers_Lmono_rpoB & target_mers_rpoB), dict_rpoB, jpayne@1: uniqmers_Lmono_rpoB) jpayne@1: coverages = [Listeria_ss_rpoB_cov, Lmono_rpoB_cov, Escherichia_rpoB_cov, S_bongori_rpoB_cov, jpayne@1: S_enterica_rpoB_cov, bongori_invA_cov, I_invA_cov, IIa_invA_cov, IIb_invA_cov, jpayne@1: IIIa_invA_cov, IIIb_invA_cov, IV_invA_cov, VI_invA_cov, VII_invA_cov, VIII_invA_cov] jpayne@1: locus_scores = [p_Listeria_ss, p_Lmono, p_Escherichia, p_bongori_rpoB, p_Senterica, p_bongori, jpayne@1: p_I, p_IIa,p_IIb, p_IIIa, p_IIIb, p_IV, p_VI, p_VII, p_VIII] jpayne@1: return locus_scores, coverages, total_reads jpayne@1: jpayne@1: def report_taxon(locus_covs, average_read_length, number_of_reads): jpayne@1: list_taxa = [ 'Listeria ss', 'Listeria monocytogenes', 'Escherichia sp.', jpayne@1: 'Salmonella bongori (rpoB)', 'Salmonella enterica (rpoB)', jpayne@1: 'Salmonella bongori (invA)', 'S. enterica subsp. enterica (invA)', jpayne@1: 'S. enterica subsp. salamae (invA: clade a)','S. enterica subsp. salamae (invA: clade b)', jpayne@1: 'S. enterica subsp. arizonae (invA)', 'S. enterica subsp. diarizonae (invA)', jpayne@1: 'S. enterica subsp. houtenae (invA)', 'S. enterica subsp. indica (invA)', jpayne@1: 'S. enterica subsp. VII (invA)', 'S. enterica subsp. salamae (invA: clade VIII)'] jpayne@1: if sum(locus_covs) < 1: jpayne@1: rpoB = ('No rpoB matches!', 0) jpayne@1: invA = ('No invA matches!', 0) jpayne@1: return rpoB, invA, 0.0 jpayne@1: else: jpayne@1: # given list of scores get taxon jpayne@1: if sum(locus_covs[0:5]) > 0: jpayne@1: best_rpoB = max(range(len(locus_covs[1:5])), key=lambda x: locus_covs[1:5][x])+1 jpayne@1: all_rpoB = max(range(len(locus_covs[0:5])), key=lambda x: locus_covs[0:5][x]) jpayne@1: if (locus_covs[best_rpoB] != 0) & (all_rpoB == 0): jpayne@1: rpoB = (list_taxa[best_rpoB], locus_covs[best_rpoB]) jpayne@1: elif (all_rpoB == 0) & (round(sum(locus_covs[1:5]),1) < 1): jpayne@1: rpoB = (list_taxa[0], locus_covs[0]) jpayne@1: else: jpayne@1: rpoB = (list_taxa[best_rpoB], locus_covs[best_rpoB]) jpayne@1: else: jpayne@1: rpoB = ('No rpoB matches!', 0) jpayne@1: if sum(locus_covs[5:]) > 0: jpayne@1: best_invA = max(range(len(locus_covs[5:])), key=lambda x: locus_covs[5:][x])+5 jpayne@1: invA = (list_taxa[best_invA], locus_covs[best_invA]) jpayne@1: else: jpayne@1: invA = ('No invA matches!', 0) jpayne@1: if 'Listeria' in rpoB[0]: jpayne@1: return rpoB, invA, (average_read_length * number_of_reads) / 3000000 jpayne@1: else: jpayne@1: return rpoB, invA, (average_read_length * number_of_reads) / 5000000 jpayne@1: jpayne@1: jpayne@1: jpayne@1: def main(): jpayne@1: ex_dir = os.path.dirname(os.path.realpath(__file__)) jpayne@1: args = parse_args() jpayne@1: input_file = args.input_file jpayne@1: if input_file != 'None': jpayne@1: files = [input_file] jpayne@1: else: jpayne@1: extension = args.extension jpayne@1: inputdir = args.input_dir jpayne@1: files = [inputdir + '/'+ f for f in os.listdir(inputdir) if f.endswith(extension)] jpayne@1: report = args.report jpayne@1: mode = args.mode jpayne@1: f_invA = open(ex_dir + "/invA_mers_dict", "rb") jpayne@1: sets_dict_invA = pickle.load(f_invA) jpayne@1: f_invA.close() jpayne@1: allmers = sets_dict_invA['allmers'] jpayne@1: uniqmers_I = sets_dict_invA['uniqmers_I'] jpayne@1: uniqmers_IIa = sets_dict_invA['uniqmers_IIa'] jpayne@1: uniqmers_IIb = sets_dict_invA['uniqmers_IIb'] jpayne@1: uniqmers_IIIa = sets_dict_invA['uniqmers_IIIa'] jpayne@1: uniqmers_IIIb = sets_dict_invA['uniqmers_IIIb'] jpayne@1: uniqmers_IV = sets_dict_invA['uniqmers_IV'] jpayne@1: uniqmers_VI = sets_dict_invA['uniqmers_VI'] jpayne@1: uniqmers_VII = sets_dict_invA['uniqmers_VII'] jpayne@1: uniqmers_VIII = sets_dict_invA['uniqmers_VIII'] jpayne@1: uniqmers_bongori = sets_dict_invA['uniqmers_bongori'] jpayne@1: jpayne@1: f = open(ex_dir + "/rpoB_mers_dict", "rb") jpayne@1: sets_dict = pickle.load(f) jpayne@1: f.close() jpayne@1: jpayne@1: allmers_rpoB = sets_dict['allmers'] jpayne@1: uniqmers_bongori_rpoB = sets_dict['uniqmers_bongori'] jpayne@1: uniqmers_S_enterica_rpoB = sets_dict['uniqmers_S_enterica'] jpayne@1: uniqmers_Escherichia_rpoB = sets_dict['uniqmers_Escherichia'] jpayne@1: uniqmers_Listeria_ss_rpoB = sets_dict['uniqmers_Listeria_ss'] jpayne@1: uniqmers_Lmono_rpoB = sets_dict['uniqmers_L_mono'] jpayne@1: #todo: run kmer_lists() once, create list of tuples containing data to be used fro different reports jpayne@1: if report == 'taxonomy': jpayne@1: print('file\trpoB\tinvA\texpected coverage') jpayne@1: for f in files: jpayne@1: locus_scores, coverages, reads = kmer_lists(f, 27, jpayne@1: allmers, allmers_rpoB, jpayne@1: uniqmers_bongori, jpayne@1: uniqmers_I, jpayne@1: uniqmers_IIa, jpayne@1: uniqmers_IIb, jpayne@1: uniqmers_IIIa, jpayne@1: uniqmers_IIIb, jpayne@1: uniqmers_IV, jpayne@1: uniqmers_VI, jpayne@1: uniqmers_VII, jpayne@1: uniqmers_VIII, jpayne@1: uniqmers_bongori_rpoB, jpayne@1: uniqmers_S_enterica_rpoB, jpayne@1: uniqmers_Escherichia_rpoB, jpayne@1: uniqmers_Listeria_ss_rpoB, jpayne@1: uniqmers_Lmono_rpoB, jpayne@1: mode) jpayne@1: pretty_covs = [round(cov, 1) for cov in coverages] jpayne@1: report = report_taxon(pretty_covs, get_av_read_length(f), reads) jpayne@1: print(f.split('/')[-1] + '\t' + report[0][0] + '[' + str(report[0][1]) + ']' + '\t' + report[1][0] + jpayne@1: '[' + str(report[1][1]) + ']' + jpayne@1: '\t' + str(round(report[2], 1))) jpayne@1: else: jpayne@1: print( jpayne@1: 'file\tListeria sensu stricto (rpoB)\tL. monocytogenes (rpoB)\tEscherichia spp. (rpoB)\tS. bongori (rpoB)\tS. enterica' + jpayne@1: '(rpoB)\tS. bongori (invA)\tsubsp. I (invA)\tsubsp. II (clade a: invA)\tsubsp. II' + jpayne@1: ' (clade b: invA)\tsubsp. IIIa (invA)\tsubsp. IIIb (invA)\tsubsp.IV (invA)\tsubsp. VI (invA)\tsubsp. VII (invA)' + jpayne@1: '\tsubsp. II (clade VIII : invA)') jpayne@1: if report == 'percentage': jpayne@1: for f in files: jpayne@1: locus_scores, coverages , reads = kmer_lists( f, 27, jpayne@1: allmers,allmers_rpoB, jpayne@1: uniqmers_bongori, jpayne@1: uniqmers_I, jpayne@1: uniqmers_IIa, jpayne@1: uniqmers_IIb, jpayne@1: uniqmers_IIIa, jpayne@1: uniqmers_IIIb, jpayne@1: uniqmers_IV, jpayne@1: uniqmers_VI, jpayne@1: uniqmers_VII, jpayne@1: uniqmers_VIII, jpayne@1: uniqmers_bongori_rpoB, jpayne@1: uniqmers_S_enterica_rpoB, jpayne@1: uniqmers_Escherichia_rpoB, jpayne@1: uniqmers_Listeria_ss_rpoB, jpayne@1: uniqmers_Lmono_rpoB, jpayne@1: mode) jpayne@1: pretty_scores = [str(round(score)) for score in locus_scores] jpayne@1: print(f.split('/')[-1] +'\t' + '\t'.join(pretty_scores)) jpayne@1: else: jpayne@1: for f in files: jpayne@1: locus_scores, coverages , reads = kmer_lists( f, 27, jpayne@1: allmers,allmers_rpoB, jpayne@1: uniqmers_bongori, jpayne@1: uniqmers_I, jpayne@1: uniqmers_IIa, jpayne@1: uniqmers_IIb, jpayne@1: uniqmers_IIIa, jpayne@1: uniqmers_IIIb, jpayne@1: uniqmers_IV, jpayne@1: uniqmers_VI, jpayne@1: uniqmers_VII, jpayne@1: uniqmers_VIII, jpayne@1: uniqmers_bongori_rpoB, jpayne@1: uniqmers_S_enterica_rpoB, jpayne@1: uniqmers_Escherichia_rpoB, jpayne@1: uniqmers_Listeria_ss_rpoB, jpayne@1: uniqmers_Lmono_rpoB, jpayne@1: mode) jpayne@1: pretty_covs = [str(round(cov, 1)) for cov in coverages] jpayne@1: print(f.split('/')[-1] + '\t' + '\t'.join(pretty_covs)) jpayne@1: jpayne@1: if __name__ == '__main__': jpayne@1: main() jpayne@1: