#!/opt/imh-python/bin/python3

[0] Fix | Delete

"""

[1] Fix | Delete

Aggregate worker information from Apache's Full Status output and determines

[2] Fix | Delete

whether the processes need to be ended.

[3] Fix | Delete

"""

[4] Fix | Delete

import argparse

[5] Fix | Delete

import logging

[6] Fix | Delete

import signal

[7] Fix | Delete

import subprocess

[8] Fix | Delete

import re

[9] Fix | Delete

import os

[10] Fix | Delete

import sys

[11] Fix | Delete

from rads import setup_logging

[12] Fix | Delete

[13] Fix | Delete

THREAD_RE = re.compile(r'^(?P<worker>\d{1,3})\-(?P<generation>\d{1,3})\s+(?P<pid>[0-9\-]+)\s+(?P<acc>\d+\/\d+\/\d+)\s+(?P<status>[A-Z\.])\s+(?P<cpu>\d+\.\d+)\s+(?P<lastreqsecs>\d+)\s+(?P<req>\d+)\s+(?P<dur>\d+)\s+([0-9\.]+)\s+([0-9\.]+)\s+([0-9\.]+)\s+(?P<ipaddr>(\d+\.){3}\d+)\s+(?P<protocol>http\/[0-9\.]{3})\s+(?P<domain>[a-z\:0-9\.\-]+)\s+(?P<method>[A-Z]+)\s+(?P<path>.*$)')

[14] Fix | Delete

[15] Fix | Delete

def parse_args():

[16] Fix | Delete

parser = argparse.ArgumentParser(description="anaylze apache workers")

[17] Fix | Delete

[18] Fix | Delete

parser.add_argument(

[19] Fix | Delete

'--clean', action='store_true', default=False, dest='clean'

[20] Fix | Delete

)

[21] Fix | Delete

[22] Fix | Delete

parser.add_argument(

[23] Fix | Delete

'--debug', action='store_true', default=False, dest='debug'

[24] Fix | Delete

)

[25] Fix | Delete

[26] Fix | Delete

return parser.parse_args()

[27] Fix | Delete

[28] Fix | Delete

def get_apache_status():

[29] Fix | Delete

"""Run apachectl fullstatus and return its output in a list line by line.

[30] Fix | Delete

[31] Fix | Delete

Returns:

[32] Fix | Delete

list: apachectl fullstatus output in a list

[33] Fix | Delete

"""

[34] Fix | Delete

[35] Fix | Delete

proc = subprocess.run(

[36] Fix | Delete

['/usr/sbin/apachectl', 'fullstatus'],

[37] Fix | Delete

stdout=subprocess.PIPE,

[38] Fix | Delete

stderr=subprocess.DEVNULL,

[39] Fix | Delete

encoding='utf-8',

[40] Fix | Delete

check=True

[41] Fix | Delete

)

[42] Fix | Delete

[43] Fix | Delete

return proc.stdout.splitlines()

[44] Fix | Delete

[45] Fix | Delete

def check_apache(clean_procs = False):

[46] Fix | Delete

"""Aggregate worker information from Apache's Full Status output.

[47] Fix | Delete

[48] Fix | Delete

Args:

[49] Fix | Delete

clean_procs (bool): clean/kill processes deemed to be killed

[50] Fix | Delete

"""

[51] Fix | Delete

logging.debug("Scanning Apache workers for stuck workers")

[52] Fix | Delete

graceful_workers = {}

[53] Fix | Delete

[54] Fix | Delete

domains = {}

[55] Fix | Delete

domain_paths = {}

[56] Fix | Delete

[57] Fix | Delete

for line in get_apache_status():

[58] Fix | Delete

if proc_match := THREAD_RE.match(line):

[59] Fix | Delete

thread_data = proc_match.groupdict()

[60] Fix | Delete

worker_status = thread_data.get('status')

[61] Fix | Delete

if worker_status == 'G':

[62] Fix | Delete

worker_id = thread_data.get('worker')

[63] Fix | Delete

worker_pid = thread_data.get('pid')

[64] Fix | Delete

last_req = int(thread_data.get('lastreqsecs'))

[65] Fix | Delete

req_domain = thread_data.get('domain').split(':')[0]

[66] Fix | Delete

req_method = thread_data.get('method')

[67] Fix | Delete

req_path = thread_data.get('path').split('HTTP/1')[0]

[68] Fix | Delete

domains.setdefault(req_domain, 0)

[69] Fix | Delete

domain_paths.setdefault(req_domain + req_path, 0)

[70] Fix | Delete

domains[req_domain] += 1

[71] Fix | Delete

domain_paths[req_domain + req_path] += 1

[72] Fix | Delete

graceful_workers.setdefault(worker_pid, last_req)

[73] Fix | Delete

if last_req < graceful_workers[worker_pid]:

[74] Fix | Delete

graceful_workers[worker_pid] = last_req

[75] Fix | Delete

[76] Fix | Delete

logging.info(

[77] Fix | Delete

"G state process: %s %s",

[78] Fix | Delete

f"{worker_pid=} {worker_id=} {last_req=}",

[79] Fix | Delete

f"{req_domain=} {req_method=} {req_path=}"

[80] Fix | Delete

)

[81] Fix | Delete

[82] Fix | Delete

if len(graceful_workers) == 0:

[83] Fix | Delete

return

[84] Fix | Delete

[85] Fix | Delete

reclaim_workers = set()

[86] Fix | Delete

for worker_pid, last_request in graceful_workers.items():

[87] Fix | Delete

# final sanity check, if well above timeout then we can kill

[88] Fix | Delete

# let Apache at least try to properly close workers

[89] Fix | Delete

if last_request > 120:

[90] Fix | Delete

reclaim_workers.add(worker_pid)

[91] Fix | Delete

else:

[92] Fix | Delete

logging.info(

[93] Fix | Delete

"G state worker with request under 120 seconds: %s -> %s",

[94] Fix | Delete

worker_pid,

[95] Fix | Delete

last_request

[96] Fix | Delete

)

[97] Fix | Delete

[98] Fix | Delete

if len(reclaim_workers) == 0:

[99] Fix | Delete

return

[100] Fix | Delete

[101] Fix | Delete

logging.info("Top six domains with G processes:")

[102] Fix | Delete

for domain in sorted(domains, key=domains.get, reverse=True)[:6]:

[103] Fix | Delete

logging.info("%s: %s", domain, domains[domain])

[104] Fix | Delete

[105] Fix | Delete

[106] Fix | Delete

logging.info("Top six domain paths with G processes:")

[107] Fix | Delete

for domain in sorted(domain_paths, key=domain_paths.get, reverse=True)[:6]:

[108] Fix | Delete

logging.info("%s: %s", domain, domain_paths[domain])

[109] Fix | Delete

[110] Fix | Delete

[111] Fix | Delete

if clean_procs:

[112] Fix | Delete

logging.warning(

[113] Fix | Delete

"Sending kills to the following PIDs: %s",

[114] Fix | Delete

', '.join(reclaim_workers)

[115] Fix | Delete

)

[116] Fix | Delete

for proc in reclaim_workers:

[117] Fix | Delete

logging.debug("Sending SIGABRT to %s", proc)

[118] Fix | Delete

# Nothing but SIGABRT or SIGKILL works here

[119] Fix | Delete

# SIGABRT seems more sane, since it can be caught and handled

[120] Fix | Delete

os.kill(int(proc), signal.SIGABRT)

[121] Fix | Delete

else:

[122] Fix | Delete

logging.info(

[123] Fix | Delete

"Would kill the following PIDs: %s",

[124] Fix | Delete

', '.join(reclaim_workers)

[125] Fix | Delete

)

[126] Fix | Delete

[127] Fix | Delete

if __name__ == '__main__':

[128] Fix | Delete

args = parse_args()

[129] Fix | Delete

setup_logging(

[130] Fix | Delete

path="/var/log/check_apache.log",

[131] Fix | Delete

loglevel=logging.DEBUG if args.debug else logging.INFO,

[132] Fix | Delete

print_out=sys.stdout if args.debug else None

[133] Fix | Delete

)

[134] Fix | Delete

[135] Fix | Delete

check_apache(clean_procs=args.clean)

[136] Fix | Delete

[137] Fix | Delete