diff -ruN linux-2.2.18/Documentation/Configure.help linux-2.2.18-hrtime/Documentation/Configure.help --- linux-2.2.18/Documentation/Configure.help Sun Dec 10 18:49:41 2000 +++ linux-2.2.18-hrtime/Documentation/Configure.help Tue Jan 30 16:19:22 2001 @@ -108,6 +108,23 @@ If you don't know what to do here, say N. +High-resolution process virtual time +CONFIG_HRTIME + This enables cycle-counter precision for process virtual time. + A timing page per task is allocated where the values are kept. + /proc/PID/hrtime can then be mmapped to access these values from + user space without having to do a syscall. + + If you don't know what to do here, say N. + +High-resolution process user and system time +CONFIG_HRUSTIME + This enables cycle-counter precision for virtualized user and system + time. Warning: Turning this option on will add code to the system + call path. + + If you don't know what to do here, say N. + Kernel math emulation CONFIG_MATH_EMULATION Linux can emulate a math coprocessor (used for floating point diff -ruN linux-2.2.18/arch/i386/config.in linux-2.2.18-hrtime/arch/i386/config.in --- linux-2.2.18/arch/i386/config.in Sun Dec 10 18:49:41 2000 +++ linux-2.2.18-hrtime/arch/i386/config.in Tue Jan 30 16:19:22 2001 @@ -28,6 +28,8 @@ fi if [ "$CONFIG_M686" = "y" -o "$CONFIG_M586TSC" = "y" ]; then define_bool CONFIG_X86_TSC y +else + define_bool CONFIG_X86_TSC n fi if [ "$CONFIG_M686" = "y" ]; then define_bool CONFIG_X86_GOOD_APIC y @@ -44,6 +46,14 @@ bool 'Math emulation' CONFIG_MATH_EMULATION bool 'MTRR (Memory Type Range Register) support' CONFIG_MTRR bool 'Symmetric multi-processing support' CONFIG_SMP + +if [ "$CONFIG_X86_TSC" = "y" -a "$CONFIG_EXPERIMENTAL" = "y" ]; then + bool 'High-resolution process virtual time support' CONFIG_HRTIME + if [ "$CONFIG_HRTIME" = "y" ]; then + bool 'High-resolution process utime/stime support' CONFIG_HRUSTIME + fi +fi + endmenu mainmenu_option next_comment diff -ruN linux-2.2.18/arch/i386/defconfig linux-2.2.18-hrtime/arch/i386/defconfig --- linux-2.2.18/arch/i386/defconfig Sun Dec 10 18:49:41 2000 +++ linux-2.2.18-hrtime/arch/i386/defconfig Tue Jan 30 16:19:22 2001 @@ -26,6 +26,8 @@ # CONFIG_MATH_EMULATION is not set # CONFIG_MTRR is not set CONFIG_SMP=y +# CONFIG_HRTIME is not set +# CONFIG_HRUSTIME is not set # # Loadable module support diff -ruN linux-2.2.18/arch/i386/kernel/entry.S linux-2.2.18-hrtime/arch/i386/kernel/entry.S --- linux-2.2.18/arch/i386/kernel/entry.S Mon Sep 4 12:39:16 2000 +++ linux-2.2.18-hrtime/arch/i386/kernel/entry.S Tue Jan 30 16:19:22 2001 @@ -43,6 +43,7 @@ #include #include #include +#include #define ASSEMBLY #include @@ -76,6 +77,16 @@ addr_limit = 12 exec_domain = 16 need_resched = 20 +hrtime = 24 + +/* The following are offsets into the hrtime_struct, not the task_struct */ +last_us_dispatch_low = 0 +last_us_dispatch_high = 4 +utime_low = 8 +utime_high = 12 +stime_low = 16 +stime_high = 20 +in_system = 24 ENOSYS = 38 @@ -176,8 +187,34 @@ jae badsys testb $0x20,flags(%ebx) # PF_TRACESYS jne tracesys +#ifdef CONFIG_HRUSTIME + pushl %eax + rdtsc # edx:eax has current timestamp value + movl hrtime(%ebx), %ebx # ebx now points to the htrime_struct + subl last_us_dispatch_low(%ebx), %eax + sbbl last_us_dispatch_high(%ebx), %edx # edx:eax has number of tics since last dispatch + addl %eax, utime_low(%ebx) + adcl %edx, utime_high(%ebx) + addl %eax, last_us_dispatch_low(%ebx) + adcl %edx, last_us_dispatch_high(%ebx) + addl $1, in_system(%ebx) + GET_CURRENT(%ebx) # reload the task_struct pointer + popl %eax # reload the system call number +#endif call *SYMBOL_NAME(sys_call_table)(,%eax,4) movl %eax,EAX(%esp) # save the return value +#ifdef CONFIG_HRUSTIME + rdtsc # edx:eax has current timestamp value + movl hrtime(%ebx), %ebx # ebx now points to the htrime_struct + subl last_us_dispatch_low(%ebx), %eax + sbbl last_us_dispatch_high(%ebx), %edx # edx:eax has number of tics since last dispatch + addl %eax, stime_low(%ebx) + adcl %edx, stime_high(%ebx) + addl %eax, last_us_dispatch_low(%ebx) + adcl %edx, last_us_dispatch_high(%ebx) + subl $1, in_system(%ebx) + GET_CURRENT(%ebx) +#endif ALIGN .globl ret_from_sys_call .globl ret_from_intr diff -ruN linux-2.2.18/arch/i386/kernel/init_task.c linux-2.2.18-hrtime/arch/i386/kernel/init_task.c --- linux-2.2.18/arch/i386/kernel/init_task.c Wed Aug 25 19:29:46 1999 +++ linux-2.2.18-hrtime/arch/i386/kernel/init_task.c Tue Jan 30 16:19:22 2001 @@ -1,5 +1,7 @@ #include #include +#include +#include #include #include @@ -10,6 +12,9 @@ static struct files_struct init_files = INIT_FILES; static struct signal_struct init_signals = INIT_SIGNALS; struct mm_struct init_mm = INIT_MM; +#ifdef CONFIG_HRTIME +static struct hrtime_struct init_hrtime = INIT_HRTIME; +#endif /* * Initial task structure. diff -ruN linux-2.2.18/fs/proc/Makefile linux-2.2.18-hrtime/fs/proc/Makefile --- linux-2.2.18/fs/proc/Makefile Wed Jun 24 16:30:10 1998 +++ linux-2.2.18-hrtime/fs/proc/Makefile Tue Jan 30 16:19:22 2001 @@ -13,6 +13,9 @@ ifdef CONFIG_OMIRR O_OBJS := $(O_OBJS) omirr.o endif +ifdef CONFIG_HRTIME +O_OBJS := $(O_OBJS) hrtime.o +endif OX_OBJS := procfs_syms.o M_OBJS := diff -ruN linux-2.2.18/fs/proc/base.c linux-2.2.18-hrtime/fs/proc/base.c --- linux-2.2.18/fs/proc/base.c Mon Aug 24 15:02:43 1998 +++ linux-2.2.18-hrtime/fs/proc/base.c Tue Jan 30 16:19:22 2001 @@ -175,6 +175,15 @@ }; #endif +#ifdef CONFIG_HRTIME +static struct proc_dir_entry proc_pid_hrtime = { + PROC_PID_HRTIME, 6, "hrtime", + S_IFREG | S_IRUSR, 1, 0, 0, + 0, &proc_hrtime_inode_operations, + NULL, proc_pid_fill_inode, +}; +#endif + __initfunc(void proc_base_init(void)) { #if CONFIG_AP1000 @@ -193,6 +202,9 @@ proc_register(&proc_pid, &proc_pid_maps); #ifdef __SMP__ proc_register(&proc_pid, &proc_pid_cpu); +#endif +#ifdef CONFIG_HRTIME + proc_register(&proc_pid, &proc_pid_hrtime); #endif }; diff -ruN linux-2.2.18/fs/proc/hrtime.c linux-2.2.18-hrtime/fs/proc/hrtime.c --- linux-2.2.18/fs/proc/hrtime.c Wed Dec 31 18:00:00 1969 +++ linux-2.2.18-hrtime/fs/proc/hrtime.c Tue Jan 30 16:27:15 2001 @@ -0,0 +1,142 @@ +/* + * linux/fs/proc/hrtime.c + * + * Copyright (C) 1999, Nick Rasmussen + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#define PROC_BLOCK_SIZE (3*1024) + +static void hrtime_vm_open(struct vm_area_struct *area) +{ + ref_hrtime_struct((struct hrtime_struct *) area->vm_file->private_data); +} + +static void hrtime_vm_close(struct vm_area_struct *area) +{ + unref_hrtime_struct((struct hrtime_struct *) area->vm_file->private_data); +} + +static struct vm_operations_struct hrtime_vm_ops = { + open: hrtime_vm_open, + close: hrtime_vm_close, +}; + +/* + * hrtime_mmap will map another (or your own) process' hrtime_struct + * into your address space. + */ +int hrtime_mmap(struct file *file, struct vm_area_struct * vma) +{ + int error = 0; + + vma->vm_ops = &hrtime_vm_ops; + + read_lock(&tasklist_lock); + + /* The timing page can only be mapped readonly */ + if ((pgprot_val(vma->vm_page_prot) & PROT_WRITE) != 0) { + error = -EPERM; + goto exit_with_unlock; + } + + /* We only allow mapping exactly one page -- the hrtime page */ + if (vma->vm_end - vma->vm_start != PAGE_SIZE) { + error = -EINVAL; + goto exit_with_unlock; + } + + error = remap_page_range(vma->vm_start, + virt_to_phys(file->private_data), + PAGE_SIZE, vma->vm_page_prot); + + if (error == 0) + ref_hrtime_struct((struct hrtime_struct *) file->private_data); + +exit_with_unlock: + + read_unlock(&tasklist_lock); + return error; +} + +static int hrtime_open(struct inode *inode, struct file *file) +{ + unsigned int pid; + int error = 0; + struct task_struct * tsk; + + pid = inode->i_ino >> 16; + + if (!pid) { + return -EBADF; + } + + file->private_data = 0; + + read_lock(&tasklist_lock); + + tsk = current; + + if (pid != tsk->pid) + tsk = find_task_by_pid(pid); + + if (tsk == NULL) + { + error = -ESRCH; + goto exit_with_unlock; + } + + file->private_data = tsk->hrtime; + + ref_hrtime_struct(file->private_data); + +exit_with_unlock: + + read_unlock(&tasklist_lock); + return error; +} + +/* + * When closing /proc/PID/hrtime, unref the respective processes hrtime field + */ +static int hrtime_release(struct inode *inode, struct file *file) +{ + /* Do I really need to grab the tasklist lock here? I am changing + * fields inside a potential task. + */ + read_lock(&tasklist_lock); + + if (file->private_data != 0) { + unref_hrtime_struct(file->private_data); + } + + read_unlock(&tasklist_lock); + + return 0; +} + +static struct file_operations proc_hrtime_operations = { + mmap: hrtime_mmap, + open: hrtime_open, + release: hrtime_release, +}; + +struct inode_operations proc_hrtime_inode_operations = { + default_file_ops: &proc_hrtime_operations, + permission: proc_permission, +}; + diff -ruN linux-2.2.18/include/asm-i386/hrtime.h linux-2.2.18-hrtime/include/asm-i386/hrtime.h --- linux-2.2.18/include/asm-i386/hrtime.h Wed Dec 31 18:00:00 1969 +++ linux-2.2.18-hrtime/include/asm-i386/hrtime.h Tue Jan 30 16:19:22 2001 @@ -0,0 +1,23 @@ +#ifndef _ASM_HRTIME_H +#define _ASM_HRTIME_H +/* + * linux/include/asm-i386/hrtime.h + * + * Copyright (C) 1999, Nick Rasmussen + */ + +#include + +#include + +typedef unsigned long long hrtime_t; + +static inline void get_current_hrtime(volatile hrtime_t *t) +{ + volatile unsigned long *parts = (volatile unsigned long *) t; + + rdtsc(parts[0], parts[1]); + return; +} + +#endif diff -ruN linux-2.2.18/include/linux/hrtime.h linux-2.2.18-hrtime/include/linux/hrtime.h --- linux-2.2.18/include/linux/hrtime.h Wed Dec 31 18:00:00 1969 +++ linux-2.2.18-hrtime/include/linux/hrtime.h Tue Jan 30 16:31:33 2001 @@ -0,0 +1,91 @@ +#ifndef _LINUX_HRTIME_H +#define _LINUX_HRTIME_H +/* + * include/linux/hrtime.h + * + * High-resolution process timing + * + * Copyright (C) 1999, Nick Rasmussen + */ + +#include + +#include +#include + +/* Will be 1 once the cpu-cpu offset code for smp boxes is written */ +#define HRTIME_HAS_OFFSETS 0 + +/* + * Careful if you add fields to this struct - the offsets to utime and + * stime are hard-coded in arch/i386/kernel/entry.S + */ +struct hrtime_struct +{ + /* These fields are only used if CONFIG_USTIME is enabled */ + /* dispatch times are non-zero only when the process is executing */ + volatile hrtime_t last_us_dispatch; + volatile hrtime_t utime; + volatile hrtime_t stime; + volatile long in_system; + + /* A flag as to wether CONFIG_USTIME is enabled. This is so that + * the hrtime interface can be binary compatible between kernels + * compiled with and without CONFIG_USTIME + */ + long has_ustime; + + /* start_time's value is with reference to cpu0 */ + hrtime_t start_time; + + /* dispatch times are non-zero only when the process is executing */ + volatile hrtime_t last_dispatch; + volatile hrtime_t vtime; + + volatile long offset_to_cpu0; + + long refcount; + spinlock_t reflock; +}; + +#ifdef __KERNEL__ + +#ifdef CONFIG_HRTIME + +# ifdef CONFIG_HRUSTIME +# define HRTIME_HAS_USTIME 1 +# else +# define HRTIME_HAS_USTIME 0 +# endif + +#define INIT_HRTIME { \ +/* last_us_dispatch */ 0, \ +/* utime */ 0, \ +/* stime */ 0, \ +/* in_system */ 1, \ +/* has_ustime */ HRTIME_HAS_USTIME, \ +/* start_time */ 0, \ +/* last_dispatch */ 0, \ +/* vtime */ 0, \ +/* offset_to_cpu0 */ 0, \ +/* refcount */ 1, \ +/* reflock */ SPIN_LOCK_UNLOCKED } + +extern struct hrtime_struct init_hrtime; +#define INIT_HRTIME_ADDR &init_hrtime + +#else /* CONFIG_HRTIME */ + +#define INIT_HRTIME_ADDR NULL + +#endif /* CONFIG_HRTIME */ + + +extern struct hrtime_struct *alloc_hrtime_struct(void); +extern void init_hrtime_struct(struct hrtime_struct *s); +extern void ref_hrtime_struct(struct hrtime_struct *s); +extern void unref_hrtime_struct(struct hrtime_struct *s); + +#endif /* __KERNEL__ */ + +#endif diff -ruN linux-2.2.18/include/linux/proc_fs.h linux-2.2.18-hrtime/include/linux/proc_fs.h --- linux-2.2.18/include/linux/proc_fs.h Sun Dec 10 18:49:44 2000 +++ linux-2.2.18-hrtime/include/linux/proc_fs.h Tue Jan 30 16:31:34 2001 @@ -72,6 +72,9 @@ PROC_PID_RINGBUF, #endif PROC_PID_CPU, +#ifdef CONFIG_HRTIME + PROC_PID_HRTIME, +#endif }; enum pid_subdirectory_inos { @@ -429,6 +432,9 @@ #endif extern struct inode_operations proc_omirr_inode_operations; extern struct inode_operations proc_ppc_htab_inode_operations; +#ifdef CONFIG_HRTIME +extern struct inode_operations proc_hrtime_inode_operations; +#endif /* * generic.c diff -ruN linux-2.2.18/include/linux/sched.h linux-2.2.18-hrtime/include/linux/sched.h --- linux-2.2.18/include/linux/sched.h Sun Dec 10 18:49:44 2000 +++ linux-2.2.18-hrtime/include/linux/sched.h Tue Jan 30 16:31:33 2001 @@ -22,6 +22,7 @@ #include #include #include +#include /* * cloning flags: @@ -239,6 +240,8 @@ struct exec_domain *exec_domain; long need_resched; + struct hrtime_struct *hrtime; + /* various fields */ long counter; long priority; @@ -367,6 +370,7 @@ */ #define INIT_TASK \ /* state etc */ { 0,0,0,KERNEL_DS,&default_exec_domain,0, \ +/* hrtime */ INIT_HRTIME_ADDR, \ /* counter */ DEF_PRIORITY,DEF_PRIORITY,0, \ /* SMP */ 0,0,0,-1, \ /* schedlink */ &init_task,&init_task, &init_task, &init_task, \ diff -ruN linux-2.2.18/kernel/Makefile linux-2.2.18-hrtime/kernel/Makefile --- linux-2.2.18/kernel/Makefile Wed May 6 13:01:46 1998 +++ linux-2.2.18-hrtime/kernel/Makefile Tue Jan 30 16:19:22 2001 @@ -25,6 +25,10 @@ OX_OBJS += ksyms.o endif +ifeq ($(CONFIG_HRTIME),y) +OX_OBJS += hrtime.o +endif + CFLAGS_sched.o := $(PROFILING) -fno-omit-frame-pointer include $(TOPDIR)/Rules.make diff -ruN linux-2.2.18/kernel/exit.c linux-2.2.18-hrtime/kernel/exit.c --- linux-2.2.18/kernel/exit.c Tue Jan 4 12:12:25 2000 +++ linux-2.2.18-hrtime/kernel/exit.c Tue Jan 30 16:19:22 2001 @@ -56,6 +56,10 @@ current->cmin_flt += p->min_flt + p->cmin_flt; current->cmaj_flt += p->maj_flt + p->cmaj_flt; current->cnswap += p->nswap + p->cnswap; +#ifdef CONFIG_HRTIME + unref_hrtime_struct(p->hrtime); + p->hrtime = 0; +#endif free_task_struct(p); } else { printk("task releasing itself\n"); diff -ruN linux-2.2.18/kernel/fork.c linux-2.2.18-hrtime/kernel/fork.c --- linux-2.2.18/kernel/fork.c Tue Oct 26 19:53:42 1999 +++ linux-2.2.18-hrtime/kernel/fork.c Tue Jan 30 16:19:22 2001 @@ -573,6 +573,9 @@ int nr; int retval = -ENOMEM; struct task_struct *p; +#ifdef CONFIG_HRTIME + struct hrtime_struct *h; +#endif struct semaphore sem = MUTEX_LOCKED; if(clone_flags & CLONE_PID) @@ -588,6 +591,15 @@ if (!p) goto fork_out; +#ifdef CONFIG_HRTIME + h = alloc_hrtime_struct(); + if (!h) + { + free_task_struct(p); + goto fork_out; + } +#endif + *p = *current; down(¤t->mm->mmap_sem); @@ -651,6 +663,14 @@ p->tty_old_pgrp = 0; p->times.tms_utime = p->times.tms_stime = 0; p->times.tms_cutime = p->times.tms_cstime = 0; + +#ifdef CONFIG_HRTIME + p->hrtime = h; + /* re-zero on fork */ + init_hrtime_struct(p->hrtime); + ref_hrtime_struct(p->hrtime); +#endif + #ifdef __SMP__ { int i; diff -ruN linux-2.2.18/kernel/hrtime.c linux-2.2.18-hrtime/kernel/hrtime.c --- linux-2.2.18/kernel/hrtime.c Wed Dec 31 18:00:00 1969 +++ linux-2.2.18-hrtime/kernel/hrtime.c Tue Jan 30 16:19:22 2001 @@ -0,0 +1,119 @@ +/* + * linux/kernel/hrtime.c + * + * Copyright (C) 1999, Nick Rasmussen + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#define HRTIME_DEBUG_ALLOC 0 + +#if HRTIME_DEBUG_ALLOC +static volatile atomic_t num_hrtime_structs = ATOMIC_INIT(1); /* due to init_hrtime */ +#endif + +struct hrtime_struct *alloc_hrtime_struct() +{ + unsigned long page; + + /* Should this be using vmalloc etc. as does drivers/char/bttv.c? */ + struct hrtime_struct *hr = (struct hrtime_struct *) __get_free_page(GFP_KERNEL); + if (hr != 0) + { + page = virt_to_phys(hr); + mem_map_reserve(MAP_NR(__va(page))); +#if HRTIME_DEBUG_ALLOC + atomic_inc(&num_hrtime_structs); + printk("alloc_hrtime_struct: num_hrtime_structs are %d\n", atomic_read(&num_hrtime_structs)); +#endif + } + return hr; +} + +static void free_hrtime_struct(struct hrtime_struct *s) +{ + unsigned long page; + if (s == 0) + return; + + page = virt_to_phys(s); + mem_map_unreserve(MAP_NR(__va(page))); + free_page((unsigned long) s); + +#if HRTIME_DEBUG_ALLOC + atomic_dec(&num_hrtime_structs); + printk("free_hrtime_struct: num_hrtime_structs are %d\n", atomic_read(&num_hrtime_structs)); +#endif +} + +void init_hrtime_struct(struct hrtime_struct *s) +{ + get_current_hrtime(&s->start_time); + + /* These are meaningless when CONFIG_HRUSTIME is not defined */ + s->last_us_dispatch = s->start_time; + s->utime = 0; + s->stime = 0; + + /* in_system starts out as zero since processes exiting from fork + * do not leave on the system call return path. */ + s->in_system = 0; + +#ifdef CONFIG_HRUSTIME + s->has_ustime = 1; +#else + s->has_ustime = 0; +#endif + + s->last_dispatch = s->start_time; + s->vtime = 0; + + /* XXX - FIXME when we do the cpu offset timings */ + s->offset_to_cpu0 = 0; + + s->refcount = 0; + s->reflock = SPIN_LOCK_UNLOCKED; +} + +void ref_hrtime_struct(struct hrtime_struct *s) +{ + unsigned long flags; + + if (s == 0) + return; + + spin_lock_irqsave(&s->reflock, flags); + s->refcount += 1; + spin_unlock_irqrestore(&s->reflock, flags); +} + +void unref_hrtime_struct(struct hrtime_struct *s) +{ + unsigned long flags; + + if (s == 0) + return; + + spin_lock_irqsave(&s->reflock, flags); + s->refcount -= 1; + + if (s->refcount == 0) + free_hrtime_struct(s); + + if (s->refcount < 0) + printk("unref_hrtime_struct: unreffing s@0x%08lx, " + "refcount %ld is now negative\n", + (unsigned long) s, s->refcount); + + spin_unlock_irqrestore(&s->reflock, flags); +} diff -ruN linux-2.2.18/kernel/sched.c linux-2.2.18-hrtime/kernel/sched.c --- linux-2.2.18/kernel/sched.c Sun Dec 10 18:49:44 2000 +++ linux-2.2.18-hrtime/kernel/sched.c Tue Jan 30 16:19:22 2001 @@ -39,6 +39,7 @@ #include #include +#include /* * kernel variables @@ -826,11 +827,51 @@ #endif /* __SMP__ */ +#ifdef CONFIG_HRTIME + { + hrtime_t t; + get_current_hrtime(&t); + + prev->hrtime->vtime += t - prev->hrtime->last_dispatch; + + /* A poor man's interprocess synchronization - by overloading + * last_dispatch to also indicate that the process is currently + * executing, the spin-loop in the get_hr[usv]time to make sure + * that this process' hrtime_struct is self-consistent is + * simplified a bit. With no good way of doing atomic operations + * between the two processes, having last_dispatch change both + * when the process is swapped out and swapped in is an almost + * foolproof way of doing this. + */ + prev->hrtime->last_dispatch = 0; +#ifdef CONFIG_HRUSTIME + t -= prev->hrtime->last_us_dispatch; + if (prev->hrtime->in_system) + prev->hrtime->stime += t; + else + prev->hrtime->utime += t; + + /* ditto for last_us_dispatch */ + prev->hrtime->last_us_dispatch = 0; +#endif + } +#endif + kstat.context_swtch++; get_mmu_context(next); switch_to(prev, next, prev); __schedule_tail(prev); +#ifdef CONFIG_HRTIME +#if HRTIME_HAS_OFFSETS + /* XXX - FIXME - find CPU and insert offset */ +#endif + get_current_hrtime(¤t->hrtime->last_dispatch); +#ifdef CONFIG_HRUSTIME + get_current_hrtime(¤t->hrtime->last_us_dispatch); +#endif +#endif + same_process: reacquire_kernel_lock(current);