进程冻结技术：深入探究 Linux 内核中的 cgroup freezer 子系统

struct task_struct {

...

#ifdef CONFIG_CGROUPS
        /* Control Group info protected by css_set_lock: */
        struct css_set __rcu            *cgroups;
        /* cg_list protected by css_set_lock and tsk->alloc_lock: */
        struct list_head                cg_list;
#endif

...

}

/*
 * A css_set is a structure holding pointers to a set of
 * cgroup_subsys_state objects. This saves space in the task struct
 * object and speeds up fork()/exit(), since a single inc/dec and a
 * list_add()/del() can bump the reference count on the entire cgroup
 * set for a task.
 */

struct css_set {
        /*
         * Set of subsystem states, one for each subsystem. This array is
         * immutable after creation apart from the init_css_set during
         * subsystem registration (at boot time).
         */
        struct cgroup_subsys_state *subsys[CGROUP_SUBSYS_COUNT]; //保存子系统状态的集合，初始化后不可修改
                                                        
        ...
        
        struct list_head tasks;         //用来链接所有使用此css_set的task_struct集合

        /*
         * List running through all cgroup groups in the same hash
         * slot. Protected by css_set_lock
         */
        struct hlist_node hlist;         //将所有的css_set组成一个hash表

        /*
         * List of cgrp_cset_links pointing at cgroups referenced from this
         * css_set.  Protected by css_set_lock.
         */
        struct list_head cgrp_links;    //将这个css_set对应的cgroup链接到一起

     ...
};

/*
 * Per-subsystem/per-cgroup state maintained by the system.  This is the
 * fundamental structural building block that controllers deal with.
 *
 * Fields marked with "PI:" are public and immutable and may be accessed
 * directly without synchronization.
 */
struct cgroup_subsys_state {
        /* PI: the cgroup that this css is attached to */
        struct cgroup *cgroup;    //连接到对应的cgroup

        /* PI: the cgroup subsystem that this css is attached to */
        struct cgroup_subsys *ss;   //归属于哪个子系统
        
        /*
         * PI: Subsys-unique ID.  0 is unused and root is always 1.  The
         * matching css can be looked up using css_from_id().
         */
        int id;

        unsigned int flags;

        /* percpu_ref killing and RCU release */
        struct work_struct destroy_work;
        struct rcu_work destroy_rwork;

        /*
         * PI: the parent css.  Placed here for cache proximity to following
         * fields of the containing structure.
         */
        struct cgroup_subsys_state *parent;   父css
};

struct cgroup {
        /* self css with NULL ->ss, points back to this cgroup */
        struct cgroup_subsys_state self;

        unsigned long flags;            /* "unsigned long" so bitops work */

        /*
         * The depth this cgroup is at.  The root is at depth zero and each
         * step down the hierarchy increments the level.  This along with
         * ancestor_ids[] can determine whether a given cgroup is a
         * descendant of another without traversing the hierarchy.
         */
        int level;

        /* Maximum allowed descent tree depth */
        int max_depth;

...

        struct cgroup_root *root;               //

        /*
         * List of cgrp_cset_links pointing at css_sets with tasks in this
         * cgroup.  Protected by css_set_lock.
         */
        struct list_head cset_links; //指向 css_sets 的 cgrp_cset_links 列表，其中包含此 cgroup 中的任务。

        /*
         * On the default hierarchy, a css_set for a cgroup with some
         * susbsys disabled will point to css's which are associated with
         * the closest ancestor which has the subsys enabled.  The
         * following lists all css_sets which point to this cgroup's css
         * for the given subsystem.
         */
        struct list_head e_csets[CGROUP_SUBSYS_COUNT];

        /*
         * If !threaded, self.  If threaded, it points to the nearest
         * domain ancestor.  Inside a threaded subtree, cgroups are exempt
         * from process granularity and no-internal-task constraint.
         * Domain level resource consumptions which aren't tied to a
         * specific task are charged to the dom_cgrp.
         */

...

        /* Used to store internal freezer state */
        struct cgroup_freezer_state freezer;        //用于存储内部freezer状态

};

/* cgroup core interface files for the default hierarchy */
static struct cftype cgroup_base_files[] = {
        {
                .name = "cgroup.procs",
                .flags = CFTYPE_NS_DELEGATABLE,
                .file_offset = offsetof(struct cgroup, procs_file),
                .release = cgroup_procs_release,
                .seq_start = cgroup_procs_start,
                .seq_next = cgroup_procs_next,
                .seq_show = cgroup_procs_show,
                .write = cgroup_procs_write,
        },
        
        ...

        {
                .name = "cgroup.freeze",
                .flags = CFTYPE_NOT_ON_ROOT,
                .seq_show = cgroup_freeze_show,
                .write = cgroup_freeze_write,
        },
        ...
}

void cgroup_freeze(struct cgroup *cgrp, bool freeze)
{
     ...
     
       /*
        * Propagate changes downwards the cgroup tree.
        */
        css_for_each_descendant_pre(css, &cgrp->self) {
                dsct = css->cgroup;
     ...
     
                /*
                 * Do change actual state: freeze or unfreeze.
                 */
                cgroup_do_freeze(dsct, freeze);
                applied = true;
        }

     ...
}

/*
 * Freeze or unfreeze all tasks in the given cgroup.
 */
static void cgroup_do_freeze(struct cgroup *cgrp, bool freeze)
{
     ...
        css_task_iter_start(&cgrp->self, 0, &it);
        while ((task = css_task_iter_next(&it))) {
                /*
                 * Ignore kernel threads here. Freezing cgroups containing
                 * kthreads isn't supported.
                 */
                if (task->flags & PF_KTHREAD)   //遇到内核线程直接跳出该次循环       
                        continue;
                cgroup_freeze_task(task, freeze);
        }
        css_task_iter_end(&it);
        
      ...
}

/*
 * Freeze or unfreeze the task by setting or clearing the JOBCTL_TRAP_FREEZE
 * jobctl bit.
 */
static void cgroup_freeze_task(struct task_struct *task, bool freeze)
{
     ...
     
        if (freeze) {
                task->jobctl |= JOBCTL_TRAP_FREEZE;    //设置冻结标志位
                signal_wake_up(task, false);
        } else {
                task->jobctl &= ~JOBCTL_TRAP_FREEZE;   //清除冻结标志位
                wake_up_process(task);                 //将已经冻结的进程唤醒，选择合适的CPU来运行，这里就是解冻
        }

        unlock_task_sighand(task, &flags);
}

/*
 * Tell a process that it has a new active signal..
 *
 * NOTE! we rely on the previous spin_lock to
 * lock interrupts for us! We can only be called with
 * "siglock" held, and the local interrupt must
 * have been disabled when that got acquired!
 *
 * No need to set need_resched since signal event passing
 * goes through ->blocked
 */
void signal_wake_up_state(struct task_struct *t, unsigned int state)
{
        set_tsk_thread_flag(t, TIF_SIGPENDING);
        /*
         * TASK_WAKEKILL also means wake it up in the stopped/traced/killable
         * case. We don't check t->state here because there is a race with it
         * executing another processor and just now entering stopped state.
         * By using wake_up_state, we ensure the process will wake up and
         * handle its death signal.
         */
        if (!wake_up_state(t, state | TASK_INTERRUPTIBLE))
                kick_process(t);
}

/***
 * kick_process - kick a running thread to enter/exit the kernel
 * @p: the to-be-kicked thread
 *
 * Cause a process which is running on another CPU to enter
 * kernel-mode, without any delay. (to get signals handled.)
 *
 * NOTE: this function doesn't have to take the runqueue lock,
 * because all it wants to ensure is that the remote task enters
 * the kernel. If the IPI races and the task has been migrated
 * to another CPU then no harm is done and the purpose has been
 * achieved as well.
 */
void kick_process(struct task_struct *p)
{
        int cpu;

        preempt_disable();
        cpu = task_cpu(p);
        if ((cpu != smp_processor_id()) && task_curr(p))
                smp_send_reschedule(cpu);
        preempt_enable();
}

void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags)
{
       ...
       
                        if (thread_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL))
                                do_signal(regs);
       ...
}

bool get_signal(struct ksignal *ksig)
{
...
if (unlikely(uprobe_deny_signal()))
return false;

/*
* Do this once, we can't return to user-mode if freezing() == T.
* do_signal_stop() and ptrace_stop() do freezable_schedule() and
* thus do not need another check after return.
*/

// 这里的freeze系统在suspend时会调用suspend_freeze_processes()来freeze线程
// 实际上也是唤醒线程，让线程在ret_to_user时刻去freeze自己，这个跟cgroup中的freeze无关
try_to_freeze();

relock:
spin_lock_irq(&sighand->siglock);

/*
* Every stopped thread goes here after wakeup. Check to see if
* we should notify the parent, prepare_signal(SIGCONT) encodes
* the CLD_ si_code into SIGNAL_CLD_MASK bits.
*/
if (unlikely(signal->flags & SIGNAL_CLD_MASK)) {
int why;

if (signal->flags & SIGNAL_CLD_CONTINUED)
why = CLD_CONTINUED;
else
why = CLD_STOPPED;

signal->flags &= ~SIGNAL_CLD_MASK;

spin_unlock_irq(&sighand->siglock);

/*
* Notify the parent that we're continuing.  This event is
* always per-process and doesn't make whole lot of sense
* for ptracers, who shouldn't consume the state via
* wait(2) either, but, for backward compatibility, notify
* the ptracer of the group leader too unless it's gonna be
* a duplicate.
*/
read_lock(&tasklist_lock);
do_notify_parent_cldstop(current, false, why);

if (ptrace_reparented(current->group_leader))
do_notify_parent_cldstop(current->group_leader,
true, why);
read_unlock(&tasklist_lock);

goto relock;
}

for (;;) {
struct k_sigaction *ka;

/* Has this task already been marked for death? */
if (signal_group_exit(signal)) {
ksig->info.si_signo = signr = SIGKILL;
sigdelset(&current->pending.signal, SIGKILL);
trace_signal_deliver(SIGKILL, SEND_SIG_NOINFO,
&sighand->action[SIGKILL - 1]);
recalc_sigpending();
goto fatal;
}

if (unlikely(current->jobctl & JOBCTL_STOP_PENDING) &&
do_signal_stop(0))
goto relock;

if (unlikely(current->jobctl &
(JOBCTL_TRAP_MASK | JOBCTL_TRAP_FREEZE))) {
if (current->jobctl & JOBCTL_TRAP_MASK) {
do_jobctl_trap();
spin_unlock_irq(&sighand->siglock);
} else if (current->jobctl & JOBCTL_TRAP_FREEZE) {
do_freezer_trap();      //cgroup freeze处理，开始真正执行进程冻结挂起操作，在ret_to_user时刻去执行
}

goto relock;
}

/*
* If the task is leaving the frozen state, let's update
* cgroup counters and reset the frozen bit.
*/
if (unlikely(cgroup_task_frozen(current))) {
spin_unlock_irq(&sighand->siglock);
cgroup_leave_frozen(false);
goto relock;
}

/*
* Signals generated by the execution of an instruction
* need to be delivered before any other pending signals
* so that the instruction pointer in the signal stack
* frame points to the faulting instruction.
*/
signr = dequeue_synchronous_signal(&ksig->info);
if (!signr)
signr = dequeue_signal(current, &current->blocked, &ksig->info);  //从信号pending队列中，取出优先级最高的信号

if (!signr)
break; /* will return 0 */

if (unlikely(current->ptrace) && (signr != SIGKILL) &&
!(sighand->action[signr -1].sa.sa_flags & SA_IMMUTABLE)) {
signr = ptrace_signal(signr, &ksig->info);
if (!signr)
continue;
}
                                       //下面这些逻辑跟冻结没什么关联，备注一下以便日后分析signal机制
ka = &sighand->action[signr-1];   //从信号处理数组sighand中，取出信号对应的处理函数

/* Trace actually delivered signals. */
trace_signal_deliver(signr, &ksig->info, ka);

if (ka->sa.sa_handler == SIG_IGN) /* Do nothing.  */      //信号处理的第一种方法：忽略
continue;
if (ka->sa.sa_handler != SIG_DFL) {    //信号处理的第二种方法，调用用户态注册的处理函数  
/* Run the handler.  */              //获取到用户态的处理函数指针，返回调用handle_signal来执行
ksig->ka = *ka;

if (ka->sa.sa_flags & SA_ONESHOT)
ka->sa.sa_handler = SIG_DFL;

break; /* will return non-zero "signr" value */
}

/*
* Now we are doing the default action for this signal.     //信号处理的第三种方法，调用默认的内核态处理函数
*/
if (sig_kernel_ignore(signr)) /* Default is nothing. */
continue;

/*
* Global init gets no signals it doesn't want.
* Container-init gets no signals it doesn't want from same
* container.
*
* Note that if global/container-init sees a sig_kernel_only()
* signal here, the signal must have been generated internally
* or must have come from an ancestor namespace. In either
* case, the signal cannot be dropped.
*/
if (unlikely(signal->flags & SIGNAL_UNKILLABLE) &&
!sig_kernel_only(signr))
continue;


}
spin_unlock_irq(&sighand->siglock);
  
...
}

/**
 * do_freezer_trap - handle the freezer jobctl trap
 *
 * Puts the task into frozen state, if only the task is not about to quit.
 * In this case it drops JOBCTL_TRAP_FREEZE.
 *
 * CONTEXT:
 * Must be called with @current->sighand->siglock held,
 * which is always released before returning.
 */
static void do_freezer_trap(void)
        __releases(&current->sighand->siglock)
{
        /*
         * If there are other trap bits pending except JOBCTL_TRAP_FREEZE,
         * let's make another loop to give it a chance to be handled.
         * In any case, we'll return back.
         */

        if ((current->jobctl & (JOBCTL_PENDING_MASK | JOBCTL_TRAP_FREEZE)) !=
             JOBCTL_TRAP_FREEZE) {
                spin_unlock_irq(&current->sighand->siglock);
                return;
        }

        /*
         * Now we're sure that there is no pending fatal signal and no
         * pending traps. Clear TIF_SIGPENDING to not get out of schedule()
         * immediately (if there is a non-fatal signal pending), and
         * put the task into sleep.
         */

        pr_info("current_jobctl_value: %d", current->jobctl);
        __set_current_state(TASK_INTERRUPTIBLE);
        clear_thread_flag(TIF_SIGPENDING);
        spin_unlock_irq(&current->sighand->siglock);
        cgroup_enter_frozen();
        freezable_schedule();  //进程挂起，主动调度出cpu
}

/* Like schedule(), but should not block the freezer. */
static inline void freezable_schedule(void)
{
        freezer_do_not_count();
        schedule();
        freezer_count();
}