#!/opt/imh-python/bin/python3
"""Automatic resource overage suspension script"""
from collections import defaultdict
from multiprocessing import cpu_count
from functools import partial
from rads.shared import (
Gathers current and historic user cp usage data and determines
whether or not to enact an account suspension, send a warning
or pass over system users
'/opt/sharedrads/etc/autosuspend.cfg',
'/opt/sharedrads/etc/autosuspend.cfg.local',
brand = ('imh', 'hub')['hub' in socket.gethostname()]
Initializes an instance of Autosuspend, including a logging
object, parsed config from Autosuspend.config_files and
various information on system users
self.config = configparser.ConfigParser(allow_no_value=False)
self.config.read(self.config_files)
format=f'%(asctime)s {sys.argv[0]}: %(message)s',
datefmt='%Y-%m-%d:%H:%M:%S %Z',
filename=self.suspension_log,
self.logger = logging.getLogger('suspension_logger')
self.priors = prior_events(
data_file=self.data_file,
log_offset=self.suspension_log_offset,
self.suspensions_enabled = self.config.getboolean(
self.warnings_enabled = self.config.getboolean(
self.freepass_enabled = self.config.getboolean(
'suspension': enact_suspension,
'freepass': give_free_pass,
self.server_overloaded = server_overloaded()
interval_file=self.sa_interval_file,
max_age=self.sa_interval_file_max_age,
suspensions=self.priors.get(name, {}).get('suspensions', []),
warnings=self.priors.get(name, {}).get('warnings', []),
freepasses=self.priors.get(name, {}).get('freepasses', []),
for name, delta in _users
if not is_suspended(name)
Returns a representation of an Autosuspend object
'disruptive_action_interval',
repr_str = '<Autosuspend {}>'.format(
' '.join([f'{i}:{getattr(self, i)}' for i in repr_data])
def suspension_critera_met(self, user):
Tests a User object to see if it meets suspension criteria
if not user.warned_within(self.disruptive_action_interval):
f'{user.name} not warned within {self.disruptive_action_interval}, not suspending'
# double check this logic - if user was suspended longer ago than action_interval they should be elligible
if not user.suspended_longer_ago(self.disruptive_action_interval):
f'{user.name} not suspended within {self.disruptive_action_interval}, not suspending'
if user.num_warnings <= self.warning_count:
f'Not suspended; only {user.num_warnings} warnings, need {self.warning_count}'
if float(user.delta) >= float(self.suspensions['max_delta']):
def warning_critera_met(self, user):
Tests a User object to see if it meets warning criteria
if user.warned_within(self.disruptive_action_interval):
f'{user.name} warned within {self.disruptive_action_interval}, not warning'
if float(user.delta) >= float(self.warnings['max_delta']):
'{} has not consumed more than {}cp in the last {}'.format(
self.warnings['max_delta'],
self.disruptive_action_interval,
def freepass_criteria_met(self, user):
Tests a user to see if it meets freepass criteria
self.logger.debug(f'Testing {user.name} for freepass...')
if float(user.delta) >= float(self.freepass['max_delta']):
f'{user.name} has a delta of {user.delta}, which is above the threshold.'
if not user.given_freepass_within(self.time_between_free_passes):
f'{user.name} was not given a free pass within {self.time_between_free_passes} so they get one'
f'{user.name} got a free pass within the last {self.time_between_free_passes} days, not sending another'
Loops through Autosuspend.users, calling Autosuspend.test for each
self.logger.info(f'Autosuspend run starting {repr(self)}')
for user in self.users.values():
action_func = self.actions.get(
action_func, email_template=getattr(self, f'{action}_template')
wrapper(user=user.name, comment=user.note)
self.logger.info('Autosuspend run complete')
Determines what action, if any to take against an individual
user.suspend = self.suspension_critera_met(user)
user.warn = self.warning_critera_met(user)
user.freepass = self.freepass_criteria_met(user)
if user.suspend and self.suspensions_enabled and self.server_overloaded:
'AUTO SUSPENSION: Consumed {:.2f}cp within '
'the last measured interval.'.format(user.delta)
self.logger.debug(f'Suspending {user}')
f'{user.delta} [AUTO_SUSPENSION] ra - root "{user.note}"'
elif user.freepass and self.freepass_enabled:
'AUTO RA FREEPASS: Consumed {:.2f}cp within '
'the last measured interval.'.format(user.delta)
self.logger.debug(f'Freepassing {user}')
self.logger.info(f'{user.name} [FREEPASS] ra - root "{user.note}"')
elif user.warn and self.warnings_enabled:
'AUTO RA WARNING: Consumed {:.2f}cp within '
'the last measured interval.'.format(user.delta)
self.logger.debug(f'Warning {user}')
self.logger.info(f'{user.name} [WARNING] ra - root "{user.note}"')
self.logger.debug(f'Skipping {user}')
def __getattr__(self, item):
Returns items as strings from settings and brand-specific settings
sections or entire config sections as a dict
e.g. <Autosuspend instance>.suspension_log;
<Autosuspend instance>.settings['suspension_log']
if item in self.config.sections():
return dict(self.config.items(item))
# See if a given key is present in the settings section
for section in (f'settings_{self.brand}', 'settings'):
if self.config.has_option(section, item):
return self.config.get(section, item)
Instantiated to represent a system user.
def __init__(self, **args):
Initializes the User object
self.num_suspensions = len(args['suspensions'])
self.num_warnings = len(args['warnings'])
self.num_freepasses = len(args['freepasses'])
def __getattr__(self, item):
Returns an item from self.data_dict or None in the event of a KeyError
return self.data_dict[item]
Returns a useful representation of a User object
repr_str = '<User {}>'.format(
' '.join([f'{i}:{getattr(self, i)}' for i in repr_data])
def warned_within(self, delta):
Returns True if the user's last warning was sent within the current
time - delta, False otherwise
if not isinstance(delta, datetime.timedelta):
delta = str_to_timedelta(delta)
return datetime.datetime.now() < (self.last_warning + delta)
def suspended_longer_ago(self, delta):
Returns True if the user's last suspension was longer ago
than the current time - delta, False otherwise
if not isinstance(delta, datetime.timedelta):
delta = str_to_timedelta(delta)
return datetime.datetime.now() > (self.last_suspension + delta)
def given_freepass_within(self, delta):
In the case self.last_freepass is None, we return false.
if not self.last_freepass:
if not isinstance(delta, datetime.timedelta):
delta = str_to_timedelta(delta)
return datetime.datetime.now() < (self.last_freepass + delta)
def last_suspension(self):
returns a datetime object which represents the last time
the user was suspended or None
return self._last_suspension
def last_suspension(self):
returns a datetime object which represents the last time
the user was suspended or None
return self._nth_date('suspensions', -1)
returns a datetime object which represents the last time
the user was warned or None
return self._last_warning
returns a datetime object which represents the last time
the user was warned or None
return self._nth_date('warnings', -1)
return self._last_freepass
return self._nth_date('freepasses', -1)
def _nth_date(self, attr, index):
Return a datetime object representation of a date from
suspension or warning lists
items = getattr(self, attr)
return datetime.datetime.fromtimestamp(
sorted(map(float, items))[index]
except (TypeError, IndexError):
def str_to_timedelta(time_str):
Munges strings into timedelta objects
(:?(?P<minutes>\d+)[Mm])?
(:?(?P<seconds>\d+)[Ss])?
''.join(time_str.split()),
groups = {k: float(v) for k, v in match.groupdict().items() if v}
return datetime.timedelta(**groups)
def server_overloaded(factor=1.5):
Determines whether or not the sever is unduly stressed by comparing the
15-minute load average and the product of number of cores and 'factor'.
return (cpu_count() * factor) <= os.getloadavg()[-1]
def try_open_yaml(yaml_path):
"""Try to read a yaml file. If impossible, return an empty dict"""
data = yaml.load(file(yaml_path, 'r'))
except (OSError, yaml.error.MarkedYAMLError):
if not isinstance(data, dict):
def get_log_data(logfile, offsetfile, ignore_offset=False):
Reads and offset from the offset file, returns data from the offset to
# try to read the offset from the offset file
with open(offsetfile) as offset_fh:
offset = int(offset_fh.readline())
# Set offset to 0 if the offset can't be converted to an integer or the
except (OSError, ValueError):
with open(logfile) as logfile_fh:
logfile_length = logfile_fh.tell()
if offset > logfile_length:
output = logfile_fh.readlines()
newoffset = logfile_fh.tell()
# If the file can't be opened return an empty string
# Write the new offset to the offset file
with open(offsetfile, 'w') as offset_fh:
offset_fh.write(str(newoffset))
def prior_events(log=None, log_offset=None, data_file=None):
'''Returns a dict that contains account suspension times'''
suspension_re = re.compile(
r"""(?P<time>\d{4}-\d{2}-\d{2}:\d{2}:\d{2}:\d{2})
\s\w+\s+[\w/\.-]+:\s+(?P<user>\w+)\s+\[
(?P<suspensions>(:?AUTO_)?SUSPENSION)|
It is recommended that you Edit text format, this type of Fix handles quite a lot in one request
