1#@!#!123s

: / opt / imh-python / lib / python3.9 / site-packages / cproc /
Filename : monitor.py
back
"""cproc module process monitoring threads"""
from typing import TYPE_CHECKING
import shlex
import os
import time
import threading
import signal
import ctypes
import psutil

if TYPE_CHECKING:
    from cproc.process import Proc as ProcType
else:
    ProcType = 'Proc'  # pylint: disable=invalid-name

POLL_INTERVAL = 1


class ProcMonitor(threading.Thread):
    """Thread that monitors server load and pauses/resumes a subprocess
    accordingly. You should not need to instantiate this class manually"""

    __module__ = 'cproc'

    try:
        libcap_prctl = ctypes.CDLL('libcap.so.2').prctl
    except OSError:
        libcap_prctl = None

    def __init__(self, subproc: ProcType):
        super().__init__(target=self._mainloop, daemon=True, name='ProcMonitor')
        self.subproc = subproc
        try:
            self.ps_proc = psutil.Process(subproc.pid)
        except (OSError, psutil.NoSuchProcess):  # process probably died
            return  # do not .start()
        # last_time is the time last_state was changed
        self.last_time = time.time()
        # last_state holds the last state change made to the process.
        # This is not necessarily the current state.
        self.last_state = psutil.STATUS_RUNNING
        self.start()

    def _bootstrap(self):
        # PR_SET_NAME = 15 in Linux kernel uapi/linux/prctl.h
        if ProcMonitor.libcap_prctl is not None:
            ProcMonitor.libcap_prctl(15, b'ProcMonitor')
        super()._bootstrap()

    def _mainloop(self) -> None:
        """Runs and monitors load until the process ends"""
        while self.subproc.returncode is None:
            time.sleep(POLL_INTERVAL)
            try:
                if self.subproc.lim.max_mem is not None:
                    self._poll_memory()
                self._poll_load()
            except (OSError, psutil.NoSuchProcess):
                return

    def _change_state(self, state: str, signum: int) -> None:
        """Send a signal to the process and update last_state/last_time

        Args:
            state (str): psutil state
            signum (int): signal number

        Raises:
            OSError: tried to send a signal to an already ended process
        """
        self.subproc.send_signal(signum)
        self.last_state = state
        self.last_time = time.time()

    def _poll_memory(self) -> None:
        mem = self.ps_proc.memory_info()
        if mem.rss <= self.subproc.lim.max_mem:
            return  # memory is okay
        if self.subproc.lim.mem_log_func is not None:
            self.subproc.lim.mem_log_func(
                'Sending SIGTERM to PID %d due to high '
                'memory usage. rss=%d, vms=%d cmd=%s',
                self.subproc.pid,
                mem.rss,
                mem.vms,
                shlex.join(self.subproc.args),
            )
        self.subproc.send_signal(self.subproc.lim.mem_signal)

    def _poll_load(self) -> None:
        """Repeatedly runs to pause/resume the process

        Raises:
            OSError: tried to send a signal to an already ended process
            psutil.NoSuchProcess: psutil operation on an already ended process
        """
        # if load is not too high
        if self.subproc.lim.value > os.getloadavg()[0]:
            # if the process is currently paused
            if self.ps_proc.status() == psutil.STATUS_STOPPED:
                self._change_state(psutil.STATUS_RUNNING, signal.SIGCONT)
            return
        # if we reach here, load is too high
        if self.subproc.lim.grace is None:  # using simple pausing behavior
            self._change_state(psutil.STATUS_STOPPED, signal.SIGSTOP)
            return
        # if we reach here, load is high and we're using grace periods
        sleep_secs, run_secs = self.subproc.lim.grace
        secs = time.time() - self.last_time  # since last state change
        # if we recently sent kill -STOP
        if self.last_state == psutil.STATUS_STOPPED:
            # if it's been sleeping as long as allowed
            if secs >= sleep_secs:
                self._change_state(psutil.STATUS_RUNNING, signal.SIGCONT)
                return
            # It can sleep longer. If not already sleeping (sometimes D-state
            # eats SIGSTOP) then send SIGSTOP again, but don't update last_time
            if self.ps_proc.status() != psutil.STATUS_STOPPED:
                self.subproc.send_signal(signal.SIGSTOP)
        elif secs >= run_secs:
            # self.last_state == psutil.STATUS_RUNNING and it's been running
            # at high load as long as allowed
            self._change_state(psutil.STATUS_STOPPED, signal.SIGSTOP)