Hardware/Software IRQs, tasklets and wait queues

// kernel/irq/handle.c
struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = {
  [0 ... NR_IRQS-1] = {
    .status = IRQ_DISABLED,
    .chip = &no_irq_chip,
    .handle_irq = handle_bad_irq,
    .depth = 1,
    // ...
  }
};

// <irq.h>
struct irq_desc {
  irq_flow_handler_t handle_irq;
  struct irq_chip *chip;
  void *handler_data;
  void *chip_data;
  struct irqaction *action; /* IRQ action list */
  unsigned int status; /* IRQ status */
  unsigned int depth; /* nested irq disables */
  unsigned int irq_count; /* For detecting broken IRQs */
  unsigned int irqs_unhandled;
  ...
  const char *name;
} ____cacheline_internodealigned_in_smp;

// kernel/irq/handle.c
irqreturn_t handle_IRQ_event(unsigned int irq, struct irqaction *action);

// kernel/softirq.c
static int ksoftirqd(void * __bind_cpu)
// ...
    while (!kthread_should_stop()) {
      if (!local_softirq_pending()) {
        schedule();
      }
      __set_current_state(TASK_RUNNING);
      while (local_softirq_pending()) {
        do_softirq();
        cond_resched();
      }
      set_current_state(TASK_INTERRUPTIBLE);
    }
// ...
}

// <interrupt.h>
struct tasklet_struct
{
  struct tasklet_struct *next;
  unsigned long state;
  atomic_t count;
  void (*func)(unsigned long);
  unsigned long data;
};

// <interrupt.h>
static inline void tasklet_schedule(struct tasklet_struct *t);

// kernel/softirq.c
static void tasklet_action(struct softirq_action *a)
// ...
    while (list) {
      struct tasklet_struct *t = list;
      list = list->next;
      if (tasklet_trylock(t)) {
        if (!atomic_read(&t->count)) {
          if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
            BUG();
          t->func(t->data);
          tasklet_unlock(t);
          continue;
        }
        tasklet_unlock(t);
      }
      // ...
    }
// ...
}

// <interrupt.h>
static inline int tasklet_trylock(struct tasklet_struct *t)
{
  return !test_and_set_bit(TASKLET_STATE_RUN, &(t)->state);
}

#include <linux/module.h>       
#include <linux/kernel.h>       
#include <linux/init.h>
#include <linux/sched.h>
#include <linux/interrupt.h>

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("tasklet examples");

void tasklet_func(unsigned long arg) {
  printk(KERN_INFO "tasklet function %lu", arg);
}
DECLARE_TASKLET(task_test1, tasklet_func, 1);
DECLARE_TASKLET_DISABLED(task_test2, tasklet_func, 2);

static int __init tasklet_example_init(void) {
  printk(KERN_INFO "the state of task1 is %ld and count is %d\n",
         task_test1.state, task_test1.count);
  printk(KERN_INFO "the state of task2 is %ld and count is %d\n",
         task_test2.state, task_test2.count);
  printk(KERN_INFO "enable task2\n");
  tasklet_enable(&task_test2);
  printk(KERN_INFO "after enable, the state of task2 is %ld and count is %d\n",
         task_test2.state, task_test2.count);
  printk(KERN_INFO "schedule tasklets\n");
  tasklet_schedule(&task_test1);
  tasklet_schedule(&task_test2);
  printk(KERN_INFO "the state of task1 is %ld and count is %d\n",
         task_test1.state, task_test1.count);
  printk(KERN_INFO "the state of task2 is %ld and count is %d\n",
         task_test2.state, task_test2.count);
  return 0;
}

static void __exit tasklet_example_exit(void) {
  printk(KERN_INFO "tasklet exit\n");
}
module_init(tasklet_example_init);
module_exit(tasklet_example_exit);

obj-m += tasklet_example.o

all:
        make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules

clean:
        make -C /lib/modules/$(shell uname -r)/build M=$(PWD) clean

$ make
$ sudo insmod ./tasklet_example.ko 
$ dmesg | tail
[758618.602354] the state of task1 is 0 and count is 0
[758618.602356] the state of task2 is 0 and count is 1
[758618.602357] enable task2
[758618.602358] after enable, the state of task2 is 0 and count is 0
[758618.602359] schedule tasklets
[758618.602362] the state of task1 is 1 and count is 0
[758618.602363] the state of task2 is 1 and count is 0
[758618.602426] tasklet function 1
[758618.602427] tasklet function 2

// <completion.h>
struct completion {
  unsigned int done;
  wait_queue_head_t wait;
};

// <completion.h>
void wait_for_completion(struct completion *);
int wait_for_completion_interruptible(struct completion *x);
unsigned long wait_for_completion_timeout(struct completion *x,
                                          unsigned long timeout);
unsigned long wait_for_completion_interruptible_timeout(
    struct completion *x, unsigned long timeout);

// <completion.h>
void complete(struct completion *);
void complete_all(struct completion *);
// kernel/exit.c
NORET_TYPE void complete_and_exit(struct completion *comp, long code);

// kernel/workqueue.c
#define __create_workqueue(name, singlethread, freezeable, rt)  \
        __create_workqueue_key((name), (singlethread), (freezeable), (rt), \
                               NULL, NULL)

// <workqueue.h>
struct work_struct;
typedef void (*work_func_t)(struct work_struct *work);
struct work_struct {
  atomic_long_t data;
  struct list_head entry;
  work_func_t func;
}

// <workqueue.h>
struct work_struct {
  ...
  void (*func)(void *);
  void *data;
  ...
};

// <workqueue.h>
#define WORK_STRUCT_FLAG_MASK (3UL)
#define WORK_STRUCT_WQ_DATA_MASK (~WORK_STRUCT_FLAG_MASK)

// <workqueue.h>
struct delayed_work {
  struct work_struct work;
  struct timer_list timer;
};

// kernel/workqueue.c
int fastcall queue_work(struct workqueue_struct *wq, struct work_struct *work)
int fastcall queue_delayed_work(struct workqueue_struct *wq,
                                struct delayed_work *dwork, unsigned long delay)

static struct workqueue_struct *keventd_wq __read_mostly;
// kernel/workqueue.c
int schedule_work(struct work_struct *work)
int schedule_delayed_work(struct delay_work *dwork, unsigned long delay)

/**
 * struct irq_desc - interrupt descriptor
 * @irq:                interrupt number for this descriptor
 * @timer_rand_state:   pointer to timer rand state struct
 * @kstat_irqs:         irq stats per cpu
 * @irq_2_iommu:        iommu with this irq
 * @handle_irq:         highlevel irq-events handler [if NULL, __do_IRQ()]
 * @chip:               low level interrupt hardware access
 * @msi_desc:           MSI descriptor
 * @handler_data:       per-IRQ data for the irq_chip methods
 * @chip_data:          platform-specific per-chip private data for the chip
 *                      methods, to allow shared chip implementations
 * @action:             the irq action chain
 * @status:             status information
 * @depth:              disable-depth, for nested irq_disable() calls
 * @wake_depth:         enable depth, for multiple set_irq_wake() callers
 * @irq_count:          stats field to detect stalled irqs
 * @last_unhandled:     aging timer for unhandled count
 * @irqs_unhandled:     stats field for spurious unhandled interrupts
 * @lock:               locking for SMP
 * @affinity:           IRQ affinity on SMP
 * @node:               node index useful for balancing
 * @pending_mask:       pending rebalanced interrupts
 * @threads_active:     number of irqaction threads currently running
 * @wait_for_threads:   wait queue for sync_irq to wait for threaded handlers
 * @dir:                /proc/irq/ procfs entry
 * @name:               flow handler name for /proc/interrupts output
 */
struct irq_desc {
        unsigned int            irq;
        struct timer_rand_state *timer_rand_state;
        unsigned int            *kstat_irqs;
#ifdef CONFIG_INTR_REMAP
        struct irq_2_iommu      *irq_2_iommu;
#endif
        irq_flow_handler_t      handle_irq;
        struct irq_chip         *chip;
        struct msi_desc         *msi_desc;
        void                    *handler_data;
        void                    *chip_data;
        struct irqaction        *action;        /* IRQ action list */
        unsigned int            status;         /* IRQ status */

        unsigned int            depth;          /* nested irq disables */
        unsigned int            wake_depth;     /* nested wake enables */
        unsigned int            irq_count;      /* For detecting broken IRQs */
        unsigned long           last_unhandled; /* Aging timer for unhandled count */
        unsigned int            irqs_unhandled;
        spinlock_t              lock;
#ifdef CONFIG_SMP
        cpumask_var_t           affinity;
        unsigned int            node;
#ifdef CONFIG_GENERIC_PENDING_IRQ
        cpumask_var_t           pending_mask;
#endif
#endif
        atomic_t                threads_active;
        wait_queue_head_t       wait_for_threads;
#ifdef CONFIG_PROC_FS
        struct proc_dir_entry   *dir;
#endif
        const char              *name;
} ____cacheline_internodealigned_in_s

/**
 *      set_irq_chip - set the irq chip for an irq
 *      @irq:   irq number
 *      @chip:  pointer to irq chip description structure
 */
int set_irq_chip(unsigned int irq, struct irq_chip *chip)
{
        struct irq_desc *desc = irq_to_desc(irq);
        unsigned long flags;

        if (!desc) {
                WARN(1, KERN_ERR "Trying to install chip for IRQ%d\n", irq);
                return -EINVAL;
        }

        if (!chip)
                chip = &no_irq_chip;

        spin_lock_irqsave(&desc->lock, flags);
        irq_chip_set_defaults(chip);
        desc->chip = chip;
        spin_unlock_irqrestore(&desc->lock, flags);

        return 0;
}

void
handle_edge_irq(unsigned int irq, struct irq_desc *desc)
{
        spin_lock(&desc->lock);

        desc->status &= ~(IRQ_REPLAY | IRQ_WAITING);

        /*
         * If we're currently running this IRQ, or its disabled,
         * we shouldn't process the IRQ. Mark it pending, handle
         * the necessary masking and go out
         */
        if (unlikely((desc->status & (IRQ_INPROGRESS | IRQ_DISABLED)) ||
                    !desc->action)) {
                desc->status |= (IRQ_PENDING | IRQ_MASKED);
                mask_ack_irq(desc, irq);
                goto out_unlock;
        }
        kstat_incr_irqs_this_cpu(irq, desc);

        /* Start handling the irq */
        if (desc->chip->ack)
                desc->chip->ack(irq);

        /* Mark the IRQ currently in progress.*/
        desc->status |= IRQ_INPROGRESS;

        do {
                struct irqaction *action = desc->action;
                irqreturn_t action_ret;

                if (unlikely(!action)) {
                        desc->chip->mask(irq);
                        goto out_unlock;
                }

                /*
                 * When another irq arrived while we were handling
                 * one, we could have masked the irq.
                 * Renable it, if it was not disabled in meantime.
                 */
                if (unlikely((desc->status &
                               (IRQ_PENDING | IRQ_MASKED | IRQ_DISABLED)) ==
                              (IRQ_PENDING | IRQ_MASKED))) {
                        desc->chip->unmask(irq);
                        desc->status &= ~IRQ_MASKED;
                }

                desc->status &= ~IRQ_PENDING;
                spin_unlock(&desc->lock);
                action_ret = handle_IRQ_event(irq, action);
                if (!noirqdebug)
                        note_interrupt(irq, desc, action_ret);
                spin_lock(&desc->lock);

        } while ((desc->status & (IRQ_PENDING | IRQ_DISABLED)) == IRQ_PENDING);

        desc->status &= ~IRQ_INPROGRESS;
out_unlock:
        spin_unlock(&desc->lock);
}

void
handle_level_irq(unsigned int irq, struct irq_desc *desc)
{
        struct irqaction *action;
        irqreturn_t action_ret;

        spin_lock(&desc->lock);
        mask_ack_irq(desc, irq);

        if (unlikely(desc->status & IRQ_INPROGRESS))
                goto out_unlock;
        desc->status &= ~(IRQ_REPLAY | IRQ_WAITING);
        kstat_incr_irqs_this_cpu(irq, desc);

        /*
         * If its disabled or no action available
         * keep it masked and get out of here
         */
        action = desc->action;
        if (unlikely(!action || (desc->status & IRQ_DISABLED)))
                goto out_unlock;

        desc->status |= IRQ_INPROGRESS;
        spin_unlock(&desc->lock);

        action_ret = handle_IRQ_event(irq, action);
        if (!noirqdebug)
                note_interrupt(irq, desc, action_ret);

        spin_lock(&desc->lock);
        desc->status &= ~IRQ_INPROGRESS;

        if (unlikely(desc->status & IRQ_ONESHOT))
                desc->status |= IRQ_MASKED;
        else if (!(desc->status & IRQ_DISABLED) && desc->chip->unmask)
                desc->chip->unmask(irq);
out_unlock:
        spin_unlock(&desc->lock);
}

irqreturn_t handle_IRQ_event(unsigned int irq, struct irqaction *action)
{
        irqreturn_t ret, retval = IRQ_NONE;
        unsigned int status = 0;

        if (!(action->flags & IRQF_DISABLED))
                local_irq_enable_in_hardirq();

        do {
                trace_irq_handler_entry(irq, action);
                ret = action->handler(irq, action->dev_id);
                trace_irq_handler_exit(irq, action, ret);

                switch (ret) {
                case IRQ_WAKE_THREAD:
                        /*
                         * Set result to handled so the spurious check
                         * does not trigger.
                         */
                        ret = IRQ_HANDLED;
                        /*
                         * Catch drivers which return WAKE_THREAD but
                         * did not set up a thread function
                         */
                        if (unlikely(!action->thread_fn)) {
                                warn_no_thread(irq, action);
                                break;
                        }
                        /*
                         * Wake up the handler thread for this
                         * action. In case the thread crashed and was
                         * killed we just pretend that we handled the
                         * interrupt. The hardirq handler above has
                         * disabled the device interrupt, so no irq
                         * storm is lurking.
                         */
                        if (likely(!test_bit(IRQTF_DIED,
                                             &action->thread_flags))) {
                                set_bit(IRQTF_RUNTHREAD, &action->thread_flags);
                                wake_up_process(action->thread);
                        }
                        /* Fall through to add to randomness */
                case IRQ_HANDLED:
                        status |= action->flags;
                        break;

                default:
                        break;
                }

                retval |= ret;
                action = action->next;
        } while (action);

        if (status & IRQF_SAMPLE_RANDOM)
                add_interrupt_randomness(irq);
        local_irq_disable();

        return retval;
}

#define MAX_SOFTIRQ_RESTART 10

asmlinkage void __do_softirq(void)
{
        struct softirq_action *h;
        __u32 pending;
        int max_restart = MAX_SOFTIRQ_RESTART;
        int cpu;

        pending = local_softirq_pending();
        account_system_vtime(current);

        __local_bh_disable((unsigned long)__builtin_return_address(0));
        lockdep_softirq_enter();

        cpu = smp_processor_id();
restart:
        /* Reset the pending bitmask before enabling irqs */
        set_softirq_pending(0);

        local_irq_enable();

        h = softirq_vec;

        do {
                if (pending & 1) {
                        int prev_count = preempt_count();
                        kstat_incr_softirqs_this_cpu(h - softirq_vec);

                        trace_softirq_entry(h, softirq_vec);
                        h->action(h);
                        trace_softirq_exit(h, softirq_vec);
                        if (unlikely(prev_count != preempt_count())) {
                                printk(KERN_ERR "huh, entered softirq %td %s %p"
                                       "with preempt_count %08x,"
                                       " exited with %08x?\n", h - softirq_vec,
                                       softirq_to_name[h - softirq_vec],
                                       h->action, prev_count, preempt_count());
                                preempt_count() = prev_count;
                        }

                        rcu_bh_qs(cpu);
                }
                h++;
                pending >>= 1;
        } while (pending);

        local_irq_disable();

        pending = local_softirq_pending();
        if (pending && --max_restart)
                goto restart;

        if (pending)
                wakeup_softirqd();

        lockdep_softirq_exit();

        account_system_vtime(current);
        _local_bh_enable();
}

static int ksoftirqd(void * __bind_cpu)
{
        set_current_state(TASK_INTERRUPTIBLE);

        while (!kthread_should_stop()) {
                preempt_disable();
                if (!local_softirq_pending()) {
                        preempt_enable_no_resched();
                        schedule();
                        preempt_disable();
                }

                __set_current_state(TASK_RUNNING);

                while (local_softirq_pending()) {
                        /* Preempt disable stops cpu going offline.
                           If already offline, we'll be on wrong CPU:
                           don't process */
                        if (cpu_is_offline((long)__bind_cpu))
                                goto wait_to_die;
                        do_softirq();
                        preempt_enable_no_resched();
                        cond_resched();
                        preempt_disable();
                        rcu_sched_qs((long)__bind_cpu);
                }
                preempt_enable();
                set_current_state(TASK_INTERRUPTIBLE);
        }
        __set_current_state(TASK_RUNNING);
        return 0;

wait_to_die:
        preempt_enable();
        /* Wait for kthread_stop */
        set_current_state(TASK_INTERRUPTIBLE);
        while (!kthread_should_stop()) {
                schedule();
                set_current_state(TASK_INTERRUPTIBLE);
        }
        __set_current_state(TASK_RUNNING);
        return 0;
}