1 """!Defines the Revital class which manipulates tcvitals files.
3 This module deals with rewriting TCVitals files to remove errors,
4 change Invests to storms, and other such operations. """
8 __all__=[
'Revital',
'RevitalError',
'RevitalInitError']
10 import logging, datetime, getopt, sys, os.path,re, math, errno, collections
14 from hwrf.numerics import great_arc_dist, to_fraction, to_timedelta
19 """!Base class of errors related to rewriting vitals."""
21 """!This exception is raised when an argument to the Revital
22 constructor is invalid."""
28 zero_time=to_timedelta(0)
32 two_days=to_timedelta(3600*24*2)
36 six_hours=to_timedelta(3600*6)
39 """!This class reads one or more tcvitals files and rewrites them
41 def __init__(self,logger=None,invest_number_name=False,stormid=None,
42 adeckdir=
None,renumberlog=
None,
43 search_dx=200e3, search_dt=
None, debug=
True,copy=
None):
44 """!Creates a Revital object:
46 @param logger A logging.Logger object for logging or None to
47 disable. Default: None
48 @param invest_number_name Rename storms to have the last
50 @param stormid Ignored.
51 @param adeckdir Directory with A deck files. This is used to
52 read the storm type information from CARQ entries and
53 append them to produce vitals with the storm type from
55 @param renumberlog Ignored.
56 @param search_dx Search radius in km for deciding whether two
58 @param search_dt Search timespan for finding two storms that are
60 @param debug If True, enables DEBUG level logging.
61 @param copy Used by copy() to make a shallow copy of a
62 Revital. If specified, the other arguments are ignored, and
63 the copy's contents are copied. Do not use this argument.
64 If you need a copy, use copy() instead."""
67 ( copy.search_dx, copy.search_dt, copy.logger, copy.debug )
69 ( copy.invest_number_name, copy.adeckdir, copy.is_cleaned )
71 for key,cdat
in copy.carqdat.iteritems():
73 for ymdh,card
in cdat.iteritems():
74 self.
carqdat[key][ymdh]=card.copy()
76 self.
vitals=[ v.copy()
for v
in copy.vitals ]
79 self.
search_dt=six_hours if(search_dt
is None)
else search_dt
81 self.
debug=debug
and logger
is not None
84 if adeckdir
is not None:
85 if not os.path.isdir(adeckdir):
87 'Specified directory %s is not a directory.'
131 """!Appends a vital entry to self.vitals.
132 @param vital an hwrf.storminfo.StormInfo to append"""
134 raise TypeError(
'The argument to Revital.append must be an hwrf.storminfo.StormInfo')
135 self.vitals.append(vital)
137 """!Given the specified iterable object, appends its contents
139 @param vitals an iterable object filled with hwrf.storminfo.StormInfo"""
140 self.vitals.extend(vitals)
143 """!Returns a deep copy of this Revital. Modifying the copy
144 will not modify the original."""
148 """!Same as readfiles except the tcvitals have already been
149 parsed and the vitals are being passed in, not the
150 filelist. This was created to handle the multistorm fake
151 storm. This is being used to validate any self generated
152 vitals by the fake storm.
153 @param vitalslist a list of strings with tcvitals data
154 @param raise_all if True, raise exceptions for any parsing errors.
157 if len(vitalslist) == 0:
158 self.logger.critical(
'No parsed tcvitals provided to revital.readvitals')
160 self.logger.info(
'Processing parsed tcvitals, line1: %s'%(vitalslist[:1]))
163 vitalslist,raise_all=raise_all,logger=self.
logger))
166 self.logger.debug(
'line count: %d'%(len(self.
vitals),))
170 """!Reads the list of files and parses them as tcvitals files.
171 @param filelist a list of string filenames
172 @param raise_all if True, all exceptions are raised. If False,
173 then exceptions are ignored, and the function will attempt to
174 process all files, even if earlier ones failed."""
175 if isinstance(filelist,basestring):
179 for tcvitals
in filelist:
180 if self.
logger is not None:
181 self.logger.info(
'read file: %s'%(tcvitals,))
183 with open(tcvitals,
'rt')
as f:
184 lines.extend(f.readlines())
186 except EnvironmentError
as e:
187 if e.errno==errno.ENOENT
or e.errno==errno.EISDIR:
188 self.logger.warning(tcvitals+
': cannot open: '+str(e))
191 self.logger.warning(tcvitals+
': cannot open: '+str(e))
194 self.logger.critical(
'No message files or tcvitals files '
195 'provided to revital.readfiles.')
197 lines,raise_all=raise_all,logger=self.
logger))
200 self.logger.debug(
'line count: %d'%(len(self.
vitals),))
203 """!Returns a tuple containing the latitude and longitude of
204 the storm at a different position according to the storm
206 @param vital the hwrf.storminfo.StormInfo for the storm fix being extrapolated
207 @param dt the time difference in hours"""
208 stormspeed=getattr(vital,
'stormspeed',
None)
209 stormdir=getattr(vital,
'stormdir',
None)
210 if stormspeed
is None or stormspeed<0
or stormdir
is None or \
211 stormdir<0
or stormspeed==0:
216 k=stormspeed*dt/Rearth / pi180
217 moveangle=pi180*(90.0-stormdir)
218 dlat=k*math.sin(moveangle)
219 dlon=k*math.cos(moveangle)/math.cos(vital.lat*pi180)
220 return (vital.lat+dlat, vital.lon+dlon)
223 """!Tries to find the CARQ data for the specified storm. Reads
224 it into the self.carqdat array, or adds the stormid to
225 self.carqfail if the data cannot be read in.
226 @param longstormid the long stormid of the storm to read
227 @post the self.carqfail will contain longstormid OR
228 self.carqdat[longstormid] will contain data for that storm"""
229 if longstormid
in self.
carqfail:
return
230 filename=os.path.join(self.
adeckdir,
'a%s.dat'%(longstormid,))
231 if not isnonempty(filename):
232 self.carqfail.add(longstormid)
234 with open(filename,
'rt')
as f:
235 data=[ line
for line
in f.readlines() ]
243 basin_center_checker=
None,vitals_cmp=
None):
244 """!Calls the hwrf.storminfo.clean_up_vitals on this object's
245 vitals. The optional arguments are passed to
246 hwrf.storminfo.clean_up_vitals.
248 @param name_number_checker a function like
249 hwrf.storminfo.name_number_okay() for validating the storm
251 @param vitals_cmp a cmp-like function for ordering hwrf.storminfo.StormInfo objects
252 @param basin_center_checker a function like
253 hwrf.storminfo.basin_center_okay() for checking the storm
254 basin and forecast center (RSMC)
256 @post is_cleaned=True"""
258 self.
vitals,name_number_checker=name_number_checker,
259 basin_center_checker=basin_center_checker,
260 vitals_cmp=vitals_cmp)
265 def renumber_one(self,vital,lastvit,vit_motion,other_motion,threshold):
266 """!Internal function that handles renumbering of storms.
269 This is an internal implementation function that should
270 never be called directly. It handles part of the work of
271 renumbering storms in the list of vitals. You should call
273 @param vital the vital being renumbered
274 @param lastvit the last vital seen
275 @param vit_motion time since vital
276 @param other_motion time since other storms vital
277 @param threshold cold start threshold, used to decide when to
278 stop connecting an invest to a non-invest"""
283 assert(vit_motion>=0)
284 assert(other_motion<=0)
291 (lat,lon)=self.
move_latlon(vital,3600.0*vit_motion)
292 if self.
debug and lat
is not None and lon
is not None:
293 self.logger.debug(
' -- lat=%.3f lon=%.3f'%(lat,lon))
295 (lat,lon)=(vital.lat,vital.lon)
296 if debug
and lat
is None or lon
is None:
297 if debug: logger.debug(
' -- no lat,lon for search')
301 debug=self.
debug and logger
is not None
303 for stormid
in lastvit.keys():
304 othervit=lastvit[stormid]
306 old_id=getattr(othervit,
'old_stnum',0)
307 if old_id>=90
and othervit.wmax<threshold:
310 'Old %s vit was a low intensity invest, so '
311 'not considering it: %s'%(
312 othervit.old_stormid3,othervit.line))
314 if debug: logger.debug(
' vs. %s'%(othervit.line,))
315 dt=othervit.when-vital.when
318 if dt==zero_time:
continue
321 if debug: logger.debug(
' -- age out othervit')
325 if debug: logger.debug(
' -- dt is too large')
327 if debug: logger.debug(
' -- within dt')
328 if othervit.has_old_stnum:
329 if othervit.old_stnum==vital.stnum
and \
330 othervit.basin1==vital.basin1:
332 logger.debug(
' -- continue renumbering (%s) %s'
333 %(vital.stormid3,vital.line))
335 vital.rename_storm(
'INVEST%02d%1s'%
336 (int(vital.stnum),vital.basin1))
337 vital.change_basin(othervit.basin1,othervit.pubbasin2)
338 vital.renumber_storm(int(othervit.stormid3[0:2]))
341 logger.debug(
' NOW %s'%(vital.line,))
342 lastvit[othervit.stormid3]=vital
347 othervit,3600.0*other_motion)
348 if debug
and otherlat
is not None and otherlon
is not None:
349 logger.debug(
' -- vs lat=%.3f lon=%.3f'
350 %(otherlat,otherlon))
352 (otherlat,otherlon)=(othervit.lat,othervit.lon)
353 if otherlat
is None or otherlon
is None:
356 ' -- cannot get other vitals location; moving on.')
359 dist=great_arc_dist(lon,lat,otherlon,otherlat)
360 if not (dist<self.
search_dx and dt==six_hours):
363 ' -- not kinda near (distance %f km)'
368 logger.debug(
' -- within dx: renumber to %s and store'%\
369 (othervit.stormid3,))
371 vital.rename_storm(
'INVEST%02d%1s'%
372 (int(vital.stnum),vital.basin1))
373 vital.change_basin(othervit.basin1,othervit.pubbasin2)
374 vital.renumber_storm(int(othervit.stormid3[0:2]))
376 if debug: logger.debug(
' NOW %s'%(vital.line,))
377 lastvit[othervit.stormid3]=vital
378 if debug: logger.debug(
' - renumbered = %s'%(repr(renumbered),))
381 def renumber(self,unrenumber=False,clean=True,threshold=0,
382 discard_duplicates=
True):
383 """!Renumbers storms with numbers 90-99, if possible, to have
384 the same number as later 1-49 numbered storms.
386 Loops over all vitals from last to first, renumbering 90-99
387 storms to have the same storm number as later 1-49 storms.
389 @param threshold If a threshold is given, then a cycle will
390 only be considered for renumbering if it is either above that
391 threshold, or is not an Invest.
392 @param unrenumber If unrenumber is True, the original storm
393 numbers are restored after renumbering.
394 @param discard_duplicate If True, discard invests that are
395 duplicates of non-invests. This feature is disabled if
396 unrenumber is enabled or cleaning is disabled.
397 @param clean If clean is True (the default), then
398 self.clean_up_vitals is called, which will (among other
399 things) delete vitals lines that have the same time and storm
400 ID. The cleaning is done after unrenumbering, so if both
401 options are turned on, the result will contain only one entry
402 per storm ID per time, but with all storm IDs that are
403 available for a given storm at any one time."""
408 if not threshold: threshold=0
409 threshold=int(threshold)
415 for vital
in reversed(self.
vitals):
416 if debug: logger.debug(
'VITAL %s'%(vital.line,))
418 if vital.stnum>=50
and vital.stnum<90:
421 lastvit[vital.stormid3]=vital
424 if debug: logger.debug(
' -- done renumbering this one')
428 ' - SEARCH AGAIN: subtract storm motion from later cycle')
433 ' - SEARCH AGAIN: add storm motion to earlier cycle')
438 ' - SEARCH AGAIN: add half motion to later and '
445 if not unrenumber
and discard_duplicates:
446 if logger
is not None:
447 logger.info(
'Delete Invests that are duplicates of non-Invests.')
449 if logger
is not None:
450 logger.info(
'Clean up the vitals again after renumbering...')
454 """!Deletes Invest entries that have the same location and time
455 as non-invest entries."""
456 o=collections.defaultdict(dict)
464 k=(int(v.lat*5), int(v.lon*5))
470 k=(int(v.lat*5), int(v.lon*5))
471 if k
not in o[v.YMDH]:
475 for yv
in o.itervalues():
476 for v
in yv.itervalues():
481 """!Calls swap_numbers on all vitals to swap old and new storm IDs."""
487 """!Duplicates all vitals that have been renumbered, creating
488 one StormInfo with the old number and one with the new
493 if 'old_stormid3' in vital.__dict__:
494 newvit.append(vital.old())
499 """!Discards all vitals except those for which the
500 keep_condition function returns True.
502 @param keep_condition A function that receives a StormInfo
503 object as its only argument, returning True if the vital
504 should be kept and False if not.
505 @note The list will be unmodified if an exception is thrown."""
508 if keep_condition(vit):
515 """!This subroutine undoes the effect of renaming by swapping
518 if hasattr(vital,
'old_stormname'):
519 (vital.old_stormname,vital.stormname) = \
520 (vital.stormname,vital.old_stormname)
523 """!This subroutine renames storms so that they have the last name seen
524 for their storm number."""
527 debug=self.
debug and logger
is not None
529 for vital
in reversed(self.
vitals):
533 if vital.stormname!=name:
535 logger.debug(
'Rename to %s: %s'%(name,vital.line))
536 vital.rename_storm(name)
537 if debug: logger.debug(
'Now: %s'%(vital.line,))
539 lastname[key]=vital.stormname[0:9]
542 """!Add the storm type parameter from the CARQ entries in the A
545 debug=self.
debug and logger
is not None
547 lsid=vital.longstormid.lower()
552 if logger
is not None:
553 logger.warning(
'storm %s: no CARQ data found. '
554 'Using stormtype XX.'%(lsid,))
555 vital.set_stormtype(
'XX')
559 if logger
is not None:
560 logger.warning(
'storm %s cycle %s: no CARQ data for '
561 'this cycle. Using stormtype XX.'
563 vital.set_stormtype(
'XX')
565 vital.set_stormtype(carq[when])
568 """!Resorts the vitals using the specified cmp-like function.
569 @param cmpfun a cmp-like function for comparing hwrf.storminfo.StormInfo objects"""
573 """!Resorts the vitals by storm instead of date. See
574 hwrf.storminfo.vit_cmp_by_storm for details."""
575 self.
vitals=sorted(self.
vitals,cmp=hwrf.storminfo.vit_cmp_by_storm)
577 """!Iterates over all vitals, yielding StormInfo objects for
579 for x
in self.
vitals:
yield x
580 def each(self,stormid=None,old=False):
581 """!Iterates over all vitals that match the specified stormid.
582 If no stormid is given, iterates over all vitals.
584 @param stormid the storm ID to search for. This can be
585 a stormid3, stormid4 or longstormid.
586 @param old If old=True, also searches the old_ copy of the
587 stormid. Any of stormid3, stormid4 or longstormid are
592 def selected(vital):
return True
594 stormid=str(stormid).upper()
595 if re.search(
'\A\d\d[a-zA-Z]\Z',stormid):
596 def selected(vital):
return vital.stormid3==stormid
598 def old_selected(vital):
599 return 'old_stormid3' in vital.__dict__
and \
600 vital.old_stormid3==stormid
601 elif re.search(
'\A[a-zA-Z]{2}\d\d\Z',stormid):
602 def selected(vital):
return vital.stormid4==stormid
604 def old_selected(vital):
605 return 'old_stormid4' in vital.__dict__
and \
606 vital.old_stormid4==stormid
607 elif re.search(
'\A[a-zA-Z]{2}\d{6}\Z',stormid):
608 def selected(vital):
return vital.longstormid==stormid
610 def old_selected(vital):
611 return 'old_longstormid' in vital.__dict__
and \
612 vital.old_longstormid==stormid
615 'one of these three formats: 04L '
616 'AL04 AL042013'%(str(val),))
623 if old_selected(vit):
625 def print_vitals(self,stream,renumberlog=None,format='line',stormid=None,
627 """!Print the vitals to the given stream in a specified format.
629 @param stream The stream (eg.: opened file) to receive the vitals.
630 @param format Either "tcvitals" to reformat as tcvitals (cleaning up
631 any errors); or "line" to simply print the original data for each
632 line; or "HHS" to use the HHS output format. (Do not use the "HHS"
633 option unless you are HHS.)
634 @param renumberlog If given, sends information about renaming and
635 renumbering of the vitals to a second stream.
636 @param stormid The "stormid" argument is used to restrict
637 printing to only a certain stormid.
638 @param old If True, then vitals with an old_stormid that matches are
640 for vit
in self.
each(stormid=stormid,old=old):
641 if renumberlog
is not None:
642 xstormid3=vit.stormid3
643 oldid=getattr(vit,
'old_stormid3',xstormid3)
645 oldname=getattr(vit,
'old_stormname',name)
646 renumberlog.write(
'%10s %3s %3s %-9s %-9s\n'%
647 (vit.YMDH,oldid,xstormid3,oldname[0:9],name[0:9]))
648 if format==
'tcvitals':
649 print>>stream, vit.as_tcvitals()
650 elif format==
'renumbering':
652 oldid=getattr(vit,
'old_stormid3',vit.stormid3)
653 oldname=getattr(vit,
'old_stormname',vit.stormname)
654 print>>stream,
'%3s %9s => %s'%(oldid,oldname,s)
656 print>>stream,
'%s %s "TCVT"'%(vit.longstormid.lower(),vit.YMDH)
658 print>>stream, vit.line
661 """!A drop-in replacement for "cmp" that can be used for sorting or
662 comparison. Returns -1 if a<b, 1 if a>b or 0 if a=b. Decision is
665 1. User priority (a.userprio): lower (priority 1) is "more
666 important" than higher numbers (priority 9999 is fill value).
668 2. Invest vs. non-invest: invest is less important
670 3. wind: stronger wind is more important than weaker wind
672 4. North Atlantic (L) storms: farther west is more important
674 5. North East Pacific (E) storms: farther East is more important
676 If all of the above values are equal, 0 is returned.
678 @param a,b the vitals to compare"""
679 a_userprio=getattr(a,
'userprio',9999)
680 b_userprio=getattr(b,
'userprio',9999)
681 a_invest=1
if (a.stormname==
'INVEST')
else 0
682 b_invest=1
if (b.stormname==
'INVEST')
else 0
684 c = cmp(a_userprio,b_userprio)
or cmp(a_invest,b_invest)
or\
685 -cmp(a.wmax,b.wmax)
or\
686 (a.basin1==
'L' and b.basin1==
'L' and cmp(a.lon,b.lon))
or \
687 (a.basin1==
'E' and b.basin1==
'E' and -cmp(a.lon,b.lon))
def copy(self)
Returns a deep copy of this Revital.
def clean_up_vitals
Given a list of StormInfo, sorts using the vitcmp comparison, discards suspect storm names and number...
This module provides a set of utility functions to do filesystem operations.
def swap_names(self)
This subroutine undoes the effect of renaming by swapping old and new names.
vitals
The list of hwrf.storminfo.StormInfo objects being revitalized.
def move_latlon(self, vital, dt)
Returns a tuple containing the latitude and longitude of the storm at a different position according ...
def readfiles
Reads the list of files and parses them as tcvitals files.
def rename(self)
This subroutine renames storms so that they have the last name seen for their storm number...
def renumber
Renumbers storms with numbers 90-99, if possible, to have the same number as later 1-49 numbered stor...
def renumber_one(self, vital, lastvit, vit_motion, other_motion, threshold)
Internal function that handles renumbering of storms.
def print_vitals
Print the vitals to the given stream in a specified format.
def mirror_renumbered_vitals(self)
Duplicates all vitals that have been renumbered, creating one StormInfo with the old number and one w...
Sets up signal handlers to ensure a clean exit.
debug
If True, send numerous extra log messages at DEBUG level.
def __init__
Creates a Revital object:
logger
The logging.Logger to use for log messages.
search_dx
Search radius in km for deciding whether two storms are the same.
def parse_tcvitals
Reads data from a tcvitals file.
def append(self, vital)
Appends a vital entry to self.vitals.
Defines StormInfo and related functions for interacting with vitals ATCF data.
Base class of errors related to rewriting vitals.
This exception is raised when an argument to the Revital constructor is invalid.
def readvitals
Same as readfiles except the tcvitals have already been parsed and the vitals are being passed in...
def multistorm_priority(self)
Does nothing.
def readcarq(self, longstormid)
Tries to find the CARQ data for the specified storm.
def clean_up_vitals
Calls the hwrf.storminfo.clean_up_vitals on this object's vitals.
Time manipulation and other numerical routines.
Base class of all exceptions in this module.
carqfail
Set of longstormid entries that had no CARQ data.
def extend(self, vitals)
Given the specified iterable object, appends its contents to my own.
def delete_invest_duplicates(self)
Deletes Invest entries that have the same location and time as non-invest entries.
is_cleaned
Has clean_up_vitals() been called since the last operation that modified the vitals?
def each
Iterates over all vitals that match the specified stormid.
def discard_except(self, keep_condition)
Discards all vitals except those for which the keep_condition function returns True.
def parse_carq
Scans an A deck file connected to stream-like object fd, reading it into a list of StormInfo objects...
adeckdir
Directory with A deck files.
Exceptions raised by the hwrf package.
carqdat
Contains CARQ entries read from the adeckdir A deck files.
def add_stormtype(self)
Add the storm type parameter from the CARQ entries in the A deck.
def sort_by_function(self, cmpfun)
Resorts the vitals using the specified cmp-like function.
def hrd_multistorm_sorter(self, a, b)
A drop-in replacement for "cmp" that can be used for sorting or comparison.
invest_number_name
Rename stores to have the last non-INVEST name seen.
def sort_by_storm(self)
Resorts the vitals by storm instead of date.
search_dt
Search timespan for finding two storms that are the same.
def __iter__(self)
Iterates over all vitals, yielding StormInfo objects for each one.
Storm vitals information from ATCF, B-deck, tcvitals or message files.
def swap_numbers(self)
Calls swap_numbers on all vitals to swap old and new storm IDs.
This class reads one or more tcvitals files and rewrites them as requested.