|
cstrittmatter@0
|
1 """Cache, generate and load resources"""
|
|
cstrittmatter@0
|
2
|
|
cstrittmatter@0
|
3 import csv, json, os, sys
|
|
cstrittmatter@0
|
4 from nltk import word_tokenize
|
|
cstrittmatter@0
|
5 from itertools import permutations
|
|
cstrittmatter@0
|
6 from collections import OrderedDict
|
|
cstrittmatter@0
|
7 from lexmapr.definitions import ROOT
|
|
cstrittmatter@0
|
8 from lexmapr.pipeline_helpers import punctuation_treatment
|
|
cstrittmatter@0
|
9
|
|
cstrittmatter@0
|
10
|
|
cstrittmatter@0
|
11 def _create_lookup_table_skeleton():
|
|
cstrittmatter@0
|
12 '''Generate an empty lookup table'''
|
|
cstrittmatter@0
|
13 return({'abbreviations':{}, 'inflection_exceptions':{}, 'non_english_words':{},
|
|
cstrittmatter@0
|
14 'spelling_mistakes':{}, 'stop_words':{}, 'suffixes':{}})
|
|
cstrittmatter@0
|
15
|
|
cstrittmatter@0
|
16
|
|
cstrittmatter@0
|
17 def _get_resource_dict(resource_file_name):
|
|
cstrittmatter@0
|
18 '''Get dictionary of resources from CSV file'''
|
|
cstrittmatter@0
|
19 ret_dic = {}
|
|
cstrittmatter@0
|
20 with open(os.path.join(ROOT, 'predefined_resources', resource_file_name)) as RES_file:
|
|
cstrittmatter@0
|
21 next(RES_file)
|
|
cstrittmatter@0
|
22 for row in csv.reader(RES_file, delimiter=','):
|
|
cstrittmatter@0
|
23 try:
|
|
cstrittmatter@0
|
24 ret_dic[punctuation_treatment(row[0].strip())] = \
|
|
cstrittmatter@0
|
25 punctuation_treatment(row[1].strip())
|
|
cstrittmatter@0
|
26 except IndexError:
|
|
cstrittmatter@0
|
27 ret_dic[punctuation_treatment(row[0].strip())] = ''
|
|
cstrittmatter@0
|
28 return(ret_dic)
|
|
cstrittmatter@0
|
29
|
|
cstrittmatter@0
|
30
|
|
cstrittmatter@0
|
31 def _add_predefined_resources_to_lookup_table(lookup_table):
|
|
cstrittmatter@0
|
32 '''Adds elements from lexmapr/predefined_resources to lookup table'''
|
|
cstrittmatter@0
|
33 lookup_table['abbreviations'] = _get_resource_dict('abbrv.csv')
|
|
cstrittmatter@0
|
34 lookup_table['inflection_exceptions'] = _get_resource_dict('inflection_exceptions.csv')
|
|
cstrittmatter@0
|
35 lookup_table['non_english_words'] = _get_resource_dict('nengwords.csv')
|
|
cstrittmatter@0
|
36 lookup_table['spelling_mistakes'] = _get_resource_dict('ScorLex.csv') #only from v 0.7
|
|
cstrittmatter@0
|
37 lookup_table['spelling_mistakes'].update(_get_resource_dict('misspellings.csv'))
|
|
cstrittmatter@0
|
38 lookup_table['stop_words'] = _get_resource_dict('mining_stopwords.csv')
|
|
cstrittmatter@0
|
39 lookup_table['suffixes'] = _get_resource_dict('suffixes.csv')
|
|
cstrittmatter@0
|
40 return(lookup_table)
|
|
cstrittmatter@0
|
41
|
|
cstrittmatter@0
|
42
|
|
cstrittmatter@0
|
43 def get_predefined_resources():
|
|
cstrittmatter@0
|
44 '''Creates lookup table'''
|
|
cstrittmatter@0
|
45 lookup_table_path = os.path.join(ROOT, 'predefined_resources', 'lookup_table.json')
|
|
cstrittmatter@0
|
46 if os.path.exists(lookup_table_path):
|
|
cstrittmatter@0
|
47 with open(lookup_table_path) as LT_file:
|
|
cstrittmatter@0
|
48 lookup_table = json.load(LT_file)
|
|
cstrittmatter@0
|
49 else:
|
|
cstrittmatter@0
|
50 lookup_table = _create_lookup_table_skeleton()
|
|
cstrittmatter@0
|
51 lookup_table = _add_predefined_resources_to_lookup_table(lookup_table)
|
|
cstrittmatter@0
|
52 with open(lookup_table_path, 'w') as LT_file:
|
|
cstrittmatter@0
|
53 json.dump(lookup_table, LT_file)
|
|
cstrittmatter@0
|
54 return(lookup_table)
|
|
cstrittmatter@0
|
55
|
|
cstrittmatter@0
|
56
|
|
cstrittmatter@0
|
57 def get_resource_label_permutations(resource_label):
|
|
cstrittmatter@0
|
58 '''Get permutations of some term'''
|
|
cstrittmatter@0
|
59 permutations_set = list(OrderedDict.fromkeys(permutations(resource_label.split())))
|
|
cstrittmatter@0
|
60 ret_list = []
|
|
cstrittmatter@0
|
61 for permutation_tuple in permutations_set:
|
|
cstrittmatter@0
|
62 ret_list.append(' '.join(permutation_tuple))
|
|
cstrittmatter@0
|
63 return(ret_list)
|
