Coverage for runmacs/processor/fixmeta.py : 86%

# -*- coding: utf-8 -*- 

# Filename: fixmeta.py 

""" 

metadata additions for macsProcessor 

==================================== 

This is part of the macsProcessor suite. 

Copyright (C) 2014 Tobias Kölling 

general metadata info 

--------------------- 

Important user defined metadata may be: 

* intent -> measurement intent 

* campaign 

* mountOptions -> e.g. servoCAT, macsDrive, HALO, CHB 

Important autogenerated metadata may be: 

* date/time 

* quality 

""" 

from copy import deepcopy 

import dateutil.parser 

import logging 

logger = logging.getLogger('__name__') 

campaignByTime = [ 

('nawdex_2016', '2016-09-10T00:00:00Z', '2016-10-20T23:59:59Z'), 

('narval_ii_2016', '2016-08-01T00:00:00Z', '2016-09-01T23:59:59Z'), 

('acridicon2014', '2014-08-21T00:00:00Z', '2014-10-05T23:59:59Z'), 

('mlcirrus2014', '2014-02-21T00:00:00Z', '2014-04-20T23:59:59Z'), 

('chb1_2014', '2014-01-16T00:00:00Z', '2014-02-14T23:59:59Z'), 

('hopeMelpitz2013', '2013-09-10T00:00:00Z', '2013-09-22T23:59:59Z'), 

('hopeJuelich2013', '2013-04-17T00:00:00Z', '2013-04-25T23:59:59Z'), 

('acridiconZugspitze2012', '2012-09-27T00:00:00Z', '2012-10-03T23:59:59Z'), 

] 

intentByTime = [ 

('c++test', '2013-12-10T00:00:00Z', '2013-12-31T23:59:59Z'), 

('cirrus', '2013-09-14T00:00:00Z', '2013-09-14T12:30Z'), #cirrus in melpitz 

('cirrus', '2013-07-09T00:00:00Z', '2013-07-09T23:59:59Z'), 

('halo', '2013-07-09T00:00:00Z', '2013-07-09T23:59:59Z'), 

('cloudSides', '2013-07-24T00:00:00Z', '2013-08-05T23:59:59Z'), 

('cloudSides', '2013-05-05T00:00:00Z', '2013-07-05T23:59:59Z'), 

('grill', '2015-06-03T00:00:00Z', '2015-06-03T23:59:59Z'), 

] 

intentByCampaign = [ 

('nawdex_2016', ['cloudtop', '-narval_ii']), 

('narval_ii_2016', ['cloudtop', '-narval_ii']), 

('acridicon2014', ['cloudSides']), 

('mlcirrus2014', ['cirrus']), 

('chb1_2014', ['labCalibration']), 

('hopeMelpitz2013', ['compareToACTOS', 'cloudSides']), 

('hopeJuelich2013', ['cloudSides']), 

] 

mountByTime = [ 

('macsDrive', '2015-06-03T00:00:00Z', '2015-12-31T23:59:59Z'), 

('macsDrive', '2015-01-01T00:00:00Z', '2015-03-24T23:59:59Z'), 

('macsDrive', '2014-02-21T00:00:00Z', '2014-07-31T23:59:59Z'), 

('servoCAT', '2011-01-01T00:00:00Z', '2013-12-31T23:59:59Z'), 

] 

mountByCampaign = [ 

('nawdex_2016', ['HALO']), 

('narval_ii_2016', ['HALO']), 

('acridicon2014', ['HALO']), 

('chb1_2014', ['CHB']), 

('hopeMelpitz2013', ['servoCAT']), 

('hopeJuelich2013', ['servoCAT']), 

('acridiconZugspitze2012', ['servoCAT']), 

] 

opticsByTime = [ 

('plain', '2012-01-01T00:00:00Z', '2014-01-27T00:00:00Z'), 

('baffle', '2014-01-27T00:00:00Z', '2018-12-31T23:59:59Z'), 

] 

#overwrites time 

opticsByCampaign = [ 

('nawdex_2016', 'plain+HALO_HSM'), 

('narval_ii_2016', 'plain+HALO_HSM'), 

('acridicon2014', 'baffle+HALO_SVP'), 

] 

def parseDates(evlist): 

for elem in evlist: 

yield elem[:-2] + tuple(map(dateutil.parser.parse, elem[-2:])) 

def foldDict(l): 

d = {} 

for key, parts in l: 

try: 

d[key] += parts 

except KeyError: 

d[key] = parts 

return d 

campaignByTime = list(parseDates(campaignByTime)) 

intentByTime = list(parseDates(intentByTime)) 

mountByTime = list(parseDates(mountByTime)) 

opticsByTime = list(parseDates(opticsByTime)) 

intentByCampaign = foldDict(intentByCampaign) 

mountByCampaign = foldDict(mountByCampaign) 

opticsByCampaign = foldDict(opticsByCampaign) 

def fixSingleByTimeAndCampaign(l, d, c, byTime, byCampaign): 

if d is not None: 

for elem, start, stop in byTime: 

if d >= start and d <= stop: 

l = elem 

try: 

l = byCampaign[c] 

except KeyError: 

pass 

return l 

def fixByTimeAndCampaign(l, d, c, byTime, byCampaign): 

126 ↛ 127line 126 didn't jump to line 127, because the condition on line 126 was never true if isinstance(l, (unicode, str)): 

l = [l] 

else: 

l = list(l) 

if d is not None: 

for elem, start, stop in byTime: 

132 ↛ 133line 132 didn't jump to line 133, because the condition on line 132 was never true if d >= start and d <= stop: 

l.append(elem) 

try: 

l += byCampaign[c] 

except KeyError: 

pass 

bad_entries = set(e[1:] for e in l if e.startswith("-")) 

return sorted(list(set(e for e in l if not e.startswith("-") and e not in bad_entries))) 

def improveUserMeta(product): 

try: 

userMeta = deepcopy(product.userMeta) 

except AttributeError: 

userMeta = {} 

try: 

date = product.date 

except AttributeError: 

date = None 

#fix campaign 

152 ↛ 153line 152 didn't jump to line 153, because the condition on line 152 was never true if 'campaign' in userMeta and date is not None: 

for campaign, start, stop in campaignByTime: 

if campaign == userMeta['campaign'] and (date < start or date > stop): 

del userMeta['campaign'] 

break 

if 'campaign' not in userMeta and date is not None: 

for campaign, start, stop in campaignByTime: 

if date >= start and date <= stop: 

userMeta['campaign'] = campaign 

break 

#fix intent 

userMeta['intent'] = fixByTimeAndCampaign(userMeta.get('intent', []), 

date, 

userMeta.get('campaign', None), 

intentByTime, 

intentByCampaign) 

#fix mountOptions 

userMeta['mountOptions'] = fixByTimeAndCampaign(userMeta.get('mountOptions', []), 

date, 

userMeta.get('campaign', None), 

mountByTime, 

mountByCampaign) 

userMeta['optics'] = fixSingleByTimeAndCampaign(userMeta.get('optics', ''), 

date, 

userMeta.get('campaign', None), 

opticsByTime, 

opticsByCampaign) 

userMeta['fixups'] = userMeta.get('fixups', []) 

# special handling section --------- 

if userMeta.get('campaign', '') == 'narval_ii_2016' and product.sensorId == '550008': 

# bad wire on VNIR CameraLink Grabber. Only odd pixels are valid 

userMeta['fixups'].append('odd_spatial_only') 

userMeta['optics'] = "plain+odd_spatial+HALO_HSM" 

# end special handling section ----- 

191 ↛ 192line 191 didn't jump to line 192, because the condition on line 191 was never true if userMeta['optics'] == '': 

del userMeta['optics'] 

return userMeta 

def mergeSingleAttr(attr, a, b, meta): 

""" 

merges attribute ``attr`` from dicts ``a`` and ``b`` to dict ``meta``. 

""" 

if attr not in a: 

try: 

meta[attr] = b[attr] 

except KeyError: 

return 

else: 

205 ↛ 206line 205 didn't jump to line 206, because the condition on line 205 was never true if attr not in b: 

meta[attr] = a[attr] 

else: 

208 ↛ 209line 208 didn't jump to line 209, because the condition on line 208 was never true if a[attr] != b[attr]: 

logger.warning('mergeUserMeta %s missmatch: %s != %s', attr, a[attr], b[attr]) 

meta[attr] = a[attr] 

def mergeUserMeta(a, b): 

meta = {} 

mergeSingleAttr('campaign', a, b, meta) 

mergeSingleAttr('optics', a, b, meta) 

meta['intent'] = list(sorted(set(a.get('intent',[]) + b.get('intent',[])))) 

meta['mountOptions'] = list(sorted(set(a.get('mountOptions',[]) + b.get('mountOptions',[])))) 

meta['fixups'] = list(sorted(set(a.get('fixups',[]) + b.get('fixups',[])))) 

return meta