From 22001821d9cb6ddb83ee4e1f81e6b905de623165 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 11 Jun 2014 23:59:12 +0000 Subject: [PATCH 001/114] acct: Use ktime_get_ts() do_posix_clock_monotonic_gettime() is a leftover from the initial posix timer implementation which maps to ktime_get_ts() Signed-off-by: Thomas Gleixner Cc: John Stultz Cc: Peter Zijlstra Link: http://lkml.kernel.org/r/20140611234606.764810535@linutronix.de Signed-off-by: Thomas Gleixner --- kernel/acct.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/acct.c b/kernel/acct.c index 808a86ff229d..1be013c6053e 100644 --- a/kernel/acct.c +++ b/kernel/acct.c @@ -484,7 +484,7 @@ static void do_acct_process(struct bsd_acct_struct *acct, strlcpy(ac.ac_comm, current->comm, sizeof(ac.ac_comm)); /* calculate run_time in nsec*/ - do_posix_clock_monotonic_gettime(&uptime); + ktime_get_ts(&uptime); run_time = (u64)uptime.tv_sec*NSEC_PER_SEC + uptime.tv_nsec; run_time -= (u64)current->group_leader->start_time.tv_sec * NSEC_PER_SEC + current->group_leader->start_time.tv_nsec; From b5d7682533941edb121f7495bdb2a17abac03ff3 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 11 Jun 2014 23:59:13 +0000 Subject: [PATCH 002/114] delayacct: Use ktime_get_ts() do_posix_clock_monotonic_gettime() is a leftover from the initial posix timer implementation which maps to ktime_get_ts(). Remove the silly wrapper while at it. Signed-off-by: Thomas Gleixner Cc: John Stultz Cc: Peter Zijlstra Link: http://lkml.kernel.org/r/20140611234606.931409215@linutronix.de Signed-off-by: Thomas Gleixner --- kernel/delayacct.c | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/kernel/delayacct.c b/kernel/delayacct.c index 54996b71e66d..de699f42f9bc 100644 --- a/kernel/delayacct.c +++ b/kernel/delayacct.c @@ -45,16 +45,6 @@ void __delayacct_tsk_init(struct task_struct *tsk) spin_lock_init(&tsk->delays->lock); } -/* - * Start accounting for a delay statistic using - * its starting timestamp (@start) - */ - -static inline void delayacct_start(struct timespec *start) -{ - do_posix_clock_monotonic_gettime(start); -} - /* * Finish delay accounting for a statistic using * its timestamps (@start, @end), accumalator (@total) and @count @@ -67,7 +57,7 @@ static void delayacct_end(struct timespec *start, struct timespec *end, s64 ns; unsigned long flags; - do_posix_clock_monotonic_gettime(end); + ktime_get_ts(end); ts = timespec_sub(*end, *start); ns = timespec_to_ns(&ts); if (ns < 0) @@ -81,7 +71,7 @@ static void delayacct_end(struct timespec *start, struct timespec *end, void __delayacct_blkio_start(void) { - delayacct_start(¤t->delays->blkio_start); + ktime_get_ts(¤t->delays->blkio_start); } void __delayacct_blkio_end(void) @@ -169,7 +159,7 @@ __u64 __delayacct_blkio_ticks(struct task_struct *tsk) void __delayacct_freepages_start(void) { - delayacct_start(¤t->delays->freepages_start); + ktime_get_ts(¤t->delays->freepages_start); } void __delayacct_freepages_end(void) From 4e8c5847d1c55efed896508fb769f78ab07b968a Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 11 Jun 2014 23:59:13 +0000 Subject: [PATCH 003/114] tsacct: Use ktime_get_ts() do_posix_clock_monotonic_gettime() is a leftover from the initial posix timer implementation which maps to ktime_get_ts() Signed-off-by: Thomas Gleixner Cc: John Stultz Cc: Peter Zijlstra Link: http://lkml.kernel.org/r/20140611234606.840900621@linutronix.de Signed-off-by: Thomas Gleixner --- kernel/tsacct.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/tsacct.c b/kernel/tsacct.c index a1dd9a1b1327..ea6d170452c4 100644 --- a/kernel/tsacct.c +++ b/kernel/tsacct.c @@ -38,7 +38,7 @@ void bacct_add_tsk(struct user_namespace *user_ns, BUILD_BUG_ON(TS_COMM_LEN < TASK_COMM_LEN); /* calculate task elapsed time in timespec */ - do_posix_clock_monotonic_gettime(&uptime); + ktime_get_ts(&uptime); ts = timespec_sub(uptime, tsk->start_time); /* rebase elapsed time to usec (should never be negative) */ ac_etime = timespec_to_ns(&ts); From a9821c741c960a77a7f08491883f9cc4bffd2279 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 11 Jun 2014 23:59:16 +0000 Subject: [PATCH 004/114] kdb: Use ktime_get_ts() do_posix_clock_monotonic_gettime() is a leftover from the initial posix timer implementation which maps to ktime_get_ts(). Signed-off-by: Thomas Gleixner Cc: John Stultz Cc: Peter Zijlstra Cc: Jason Wessel Link: http://lkml.kernel.org/r/20140611234607.261629142@linutronix.de Signed-off-by: Thomas Gleixner --- kernel/debug/kdb/kdb_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/debug/kdb/kdb_main.c b/kernel/debug/kdb/kdb_main.c index 2f7c760305ca..379650b984f8 100644 --- a/kernel/debug/kdb/kdb_main.c +++ b/kernel/debug/kdb/kdb_main.c @@ -2472,7 +2472,7 @@ static void kdb_gmtime(struct timespec *tv, struct kdb_tm *tm) static void kdb_sysinfo(struct sysinfo *val) { struct timespec uptime; - do_posix_clock_monotonic_gettime(&uptime); + ktime_get_ts(&uptime); memset(val, 0, sizeof(*val)); val->uptime = uptime.tv_sec; val->loads[0] = avenrun[0]; From 889235ce2b26cdd231c7a1d6005928ec42f11637 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 11 Jun 2014 23:59:16 +0000 Subject: [PATCH 005/114] firewire: Use ktime_get_ts() do_posix_clock_monotonic_gettime() is a leftover from the initial posix timer implementation which maps to ktime_get_ts() Signed-off-by: Thomas Gleixner Cc: John Stultz Cc: Peter Zijlstra Acked-by: Stefan Richter Link: http://lkml.kernel.org/r/20140611234607.351283464@linutronix.de Signed-off-by: Thomas Gleixner --- drivers/firewire/core-cdev.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/firewire/core-cdev.c b/drivers/firewire/core-cdev.c index d7d5c8af92b9..5d997a33907e 100644 --- a/drivers/firewire/core-cdev.c +++ b/drivers/firewire/core-cdev.c @@ -1214,9 +1214,9 @@ static int ioctl_get_cycle_timer2(struct client *client, union ioctl_arg *arg) cycle_time = card->driver->read_csr(card, CSR_CYCLE_TIME); switch (a->clk_id) { - case CLOCK_REALTIME: getnstimeofday(&ts); break; - case CLOCK_MONOTONIC: do_posix_clock_monotonic_gettime(&ts); break; - case CLOCK_MONOTONIC_RAW: getrawmonotonic(&ts); break; + case CLOCK_REALTIME: getnstimeofday(&ts); break; + case CLOCK_MONOTONIC: ktime_get_ts(&ts); break; + case CLOCK_MONOTONIC_RAW: getrawmonotonic(&ts); break; default: ret = -EINVAL; } From f037c1171db79be2a047b1a5aafa2fd1f05051cb Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 11 Jun 2014 23:59:17 +0000 Subject: [PATCH 006/114] fork: Use ktime_get_ts() do_posix_clock_monotonic_gettime() is a leftover from the initial posix timer implementation which maps to ktime_get_ts(). Signed-off-by: Thomas Gleixner Cc: John Stultz Cc: Peter Zijlstra Cc: Oleg Nesterov Link: http://lkml.kernel.org/r/20140611234607.427408044@linutronix.de Signed-off-by: Thomas Gleixner Cc: Oleg Nesterov --- kernel/fork.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/fork.c b/kernel/fork.c index d2799d1fc952..ea0dd7075543 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1262,7 +1262,7 @@ static struct task_struct *copy_process(unsigned long clone_flags, posix_cpu_timers_init(p); - do_posix_clock_monotonic_gettime(&p->start_time); + ktime_get_ts(&p->start_time); p->real_start_time = p->start_time; monotonic_to_bootbased(&p->real_start_time); p->io_context = NULL; From 77f4fa089c724adc3a87c10eb031bca91b144ac0 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 11 Jun 2014 23:59:19 +0000 Subject: [PATCH 007/114] tomoyo: Use sensible time interface There is no point in calling gettimeofday if only the seconds part of the timespec is used. Use get_seconds() instead. It's not only the proper interface it's also faster. Signed-off-by: Thomas Gleixner Acked-by: Tetsuo Handa Cc: John Stultz Cc: Peter Zijlstra Cc: Kentaro Takeda Cc: linux-security-module@vger.kernel.org Link: http://lkml.kernel.org/r/20140611234607.775273584@linutronix.de --- security/tomoyo/audit.c | 8 +++----- security/tomoyo/common.c | 4 +--- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/security/tomoyo/audit.c b/security/tomoyo/audit.c index c1b00375c9ad..3ffa4f5509d8 100644 --- a/security/tomoyo/audit.c +++ b/security/tomoyo/audit.c @@ -155,11 +155,9 @@ static char *tomoyo_print_header(struct tomoyo_request_info *r) u8 i; if (!buffer) return NULL; - { - struct timeval tv; - do_gettimeofday(&tv); - tomoyo_convert_time(tv.tv_sec, &stamp); - } + + tomoyo_convert_time(get_seconds(), &stamp); + pos = snprintf(buffer, tomoyo_buffer_len - 1, "#%04u/%02u/%02u %02u:%02u:%02u# profile=%u mode=%s " "granted=%s (global-pid=%u) task={ pid=%u ppid=%u " diff --git a/security/tomoyo/common.c b/security/tomoyo/common.c index 283862aebdc8..e0fb75052550 100644 --- a/security/tomoyo/common.c +++ b/security/tomoyo/common.c @@ -2267,13 +2267,11 @@ static unsigned int tomoyo_stat_modified[TOMOYO_MAX_POLICY_STAT]; */ void tomoyo_update_stat(const u8 index) { - struct timeval tv; - do_gettimeofday(&tv); /* * I don't use atomic operations because race condition is not fatal. */ tomoyo_stat_updated[index]++; - tomoyo_stat_modified[index] = tv.tv_sec; + tomoyo_stat_modified[index] = get_seconds(); } /** From 58394271c610e9c65dd0165a1c1f6dec75dc5f3e Mon Sep 17 00:00:00 2001 From: Jean Delvare Date: Mon, 16 Jun 2014 11:48:45 +0200 Subject: [PATCH 008/114] clocksource: Add a Kconfig menu Move the clocksource Kconfig entries into their own menu, so that they don't pollute the main device driver menu. Signed-off-by: Jean Delvare Cc: Daniel Lezcano Link: http://lkml.kernel.org/r/20140616114845.343e9960@endymion.delvare Signed-off-by: Thomas Gleixner --- drivers/clocksource/Kconfig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index 065131cbfcc0..6a1151840510 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -1,3 +1,5 @@ +menu "Clock Source drivers" + config CLKSRC_OF bool @@ -207,3 +209,5 @@ config CLKSRC_VERSATILE counter available in the "System Registers" block of ARM Versatile, RealView and Versatile Express reference platforms. + +endmenu From 5cee964597260237dd2cabb3ec22bba0da24b25d Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sun, 22 Jun 2014 12:06:40 +0200 Subject: [PATCH 009/114] time/timers: Move all time(r) related files into kernel/time Except for Kconfig.HZ. That needs a separate treatment. Signed-off-by: Thomas Gleixner --- MAINTAINERS | 6 +++--- kernel/Makefile | 25 ++++--------------------- kernel/time/Makefile | 17 +++++++++++++++++ kernel/{ => time}/hrtimer.c | 0 kernel/{ => time}/itimer.c | 0 kernel/{ => time}/posix-cpu-timers.c | 0 kernel/{ => time}/posix-timers.c | 0 kernel/{ => time}/time.c | 0 kernel/{ => time}/timeconst.bc | 0 kernel/{ => time}/timer.c | 0 10 files changed, 24 insertions(+), 24 deletions(-) rename kernel/{ => time}/hrtimer.c (100%) rename kernel/{ => time}/itimer.c (100%) rename kernel/{ => time}/posix-cpu-timers.c (100%) rename kernel/{ => time}/posix-timers.c (100%) rename kernel/{ => time}/time.c (100%) rename kernel/{ => time}/timeconst.bc (100%) rename kernel/{ => time}/timer.c (100%) diff --git a/MAINTAINERS b/MAINTAINERS index 1b22565c59ac..970c4a07a9ab 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4147,7 +4147,7 @@ L: linux-kernel@vger.kernel.org T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git timers/core S: Maintained F: Documentation/timers/ -F: kernel/hrtimer.c +F: kernel/time/hrtimer.c F: kernel/time/clockevents.c F: kernel/time/tick*.* F: kernel/time/timer_*.c @@ -6945,10 +6945,10 @@ POSIX CLOCKS and TIMERS M: Thomas Gleixner L: linux-kernel@vger.kernel.org T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git timers/core -S: Supported +S: Maintained F: fs/timerfd.c F: include/linux/timer* -F: kernel/*timer* +F: kernel/time/*timer* POWER SUPPLY CLASS/SUBSYSTEM and DRIVERS M: Dmitry Eremin-Solenikov diff --git a/kernel/Makefile b/kernel/Makefile index f2a8b6246ce9..973a40cf8068 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -3,12 +3,11 @@ # obj-y = fork.o exec_domain.o panic.o \ - cpu.o exit.o itimer.o time.o softirq.o resource.o \ - sysctl.o sysctl_binary.o capability.o ptrace.o timer.o user.o \ + cpu.o exit.o softirq.o resource.o \ + sysctl.o sysctl_binary.o capability.o ptrace.o user.o \ signal.o sys.o kmod.o workqueue.o pid.o task_work.o \ - extable.o params.o posix-timers.o \ - kthread.o sys_ni.o posix-cpu-timers.o \ - hrtimer.o nsproxy.o \ + extable.o params.o \ + kthread.o sys_ni.o nsproxy.o \ notifier.o ksysfs.o cred.o reboot.o \ async.o range.o groups.o smpboot.o @@ -110,22 +109,6 @@ targets += config_data.h $(obj)/config_data.h: $(obj)/config_data.gz FORCE $(call filechk,ikconfiggz) -$(obj)/time.o: $(obj)/timeconst.h - -quiet_cmd_hzfile = HZFILE $@ - cmd_hzfile = echo "hz=$(CONFIG_HZ)" > $@ - -targets += hz.bc -$(obj)/hz.bc: $(objtree)/include/config/hz.h FORCE - $(call if_changed,hzfile) - -quiet_cmd_bc = BC $@ - cmd_bc = bc -q $(filter-out FORCE,$^) > $@ - -targets += timeconst.h -$(obj)/timeconst.h: $(obj)/hz.bc $(src)/timeconst.bc FORCE - $(call if_changed,bc) - ############################################################################### # # Roll all the X.509 certificates that we can find together and pull them into diff --git a/kernel/time/Makefile b/kernel/time/Makefile index 57a413fd0ebf..e59ce8b1b550 100644 --- a/kernel/time/Makefile +++ b/kernel/time/Makefile @@ -1,3 +1,4 @@ +obj-y += time.o timer.o hrtimer.o itimer.o posix-timers.o posix-cpu-timers.o obj-y += timekeeping.o ntp.o clocksource.o jiffies.o timer_list.o obj-y += timeconv.o posix-clock.o alarmtimer.o @@ -12,3 +13,19 @@ obj-$(CONFIG_TICK_ONESHOT) += tick-oneshot.o obj-$(CONFIG_TICK_ONESHOT) += tick-sched.o obj-$(CONFIG_TIMER_STATS) += timer_stats.o obj-$(CONFIG_DEBUG_FS) += timekeeping_debug.o + +$(obj)/time.o: $(obj)/timeconst.h + +quiet_cmd_hzfile = HZFILE $@ + cmd_hzfile = echo "hz=$(CONFIG_HZ)" > $@ + +targets += hz.bc +$(obj)/hz.bc: $(objtree)/include/config/hz.h FORCE + $(call if_changed,hzfile) + +quiet_cmd_bc = BC $@ + cmd_bc = bc -q $(filter-out FORCE,$^) > $@ + +targets += timeconst.h +$(obj)/timeconst.h: $(obj)/hz.bc $(src)/timeconst.bc FORCE + $(call if_changed,bc) diff --git a/kernel/hrtimer.c b/kernel/time/hrtimer.c similarity index 100% rename from kernel/hrtimer.c rename to kernel/time/hrtimer.c diff --git a/kernel/itimer.c b/kernel/time/itimer.c similarity index 100% rename from kernel/itimer.c rename to kernel/time/itimer.c diff --git a/kernel/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c similarity index 100% rename from kernel/posix-cpu-timers.c rename to kernel/time/posix-cpu-timers.c diff --git a/kernel/posix-timers.c b/kernel/time/posix-timers.c similarity index 100% rename from kernel/posix-timers.c rename to kernel/time/posix-timers.c diff --git a/kernel/time.c b/kernel/time/time.c similarity index 100% rename from kernel/time.c rename to kernel/time/time.c diff --git a/kernel/timeconst.bc b/kernel/time/timeconst.bc similarity index 100% rename from kernel/timeconst.bc rename to kernel/time/timeconst.bc diff --git a/kernel/timer.c b/kernel/time/timer.c similarity index 100% rename from kernel/timer.c rename to kernel/time/timer.c From d6f93829811a3e74f58e3c3823d507411eed651a Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Sun, 22 Jun 2014 01:29:13 +0200 Subject: [PATCH 010/114] timer: Store cpu-number in struct tvec_base Timers are serviced by the tick. But when a timer is enqueued on a dynticks target, we need to kick it in order to make it reconsider the next tick to schedule to correctly handle the timer's expiring time. Now while this kick is correctly performed for add_timer_on(), the mod_timer*() family has been a bit neglected. To prepare for fixing this, we need internal_add_timer() to be able to resolve the CPU target associated to a timer's object 'base' so that the kick can be centralized there. This can't be passed as an argument as not all the callers know the CPU number of a timer's base. So lets store it in the struct tvec_base to resolve the CPU without much overhead. It is set once for good at every CPU's first boot. Signed-off-by: Viresh Kumar Signed-off-by: Frederic Weisbecker Link: http://lkml.kernel.org/r/1403393357-2070-2-git-send-email-fweisbec@gmail.com Signed-off-by: Thomas Gleixner --- kernel/time/timer.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 3bb01a323b2a..9e5f4f25dcc0 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -82,6 +82,7 @@ struct tvec_base { unsigned long next_timer; unsigned long active_timers; unsigned long all_timers; + int cpu; struct tvec_root tv1; struct tvec tv2; struct tvec tv3; @@ -1568,6 +1569,7 @@ static int init_timers_cpu(int cpu) } spin_lock_init(&base->lock); tvec_base_done[cpu] = 1; + base->cpu = cpu; } else { base = per_cpu(tvec_bases, cpu); } From 9f6d9baaa8ca94b48aea495261cadaf2967c7784 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Sun, 22 Jun 2014 01:29:14 +0200 Subject: [PATCH 011/114] timer: Kick dynticks targets on mod_timer*() calls When a timer is enqueued or modified on a dynticks target, that CPU must re-evaluate the next tick to service that timer. The tick re-evaluation is performed by an IPI kick on the target. Now while we correctly call wake_up_nohz_cpu() from add_timer_on(), the mod_timer*() API family doesn't support so well dynticks targets. The reason for this is likely that __mod_timer() isn't supposed to select an idle target for a timer, unless that target is the current CPU, in which case a dynticks idle kick isn't actually needed. But there is a small race window lurking behind that assumption: the elected target has all the time to turn dynticks idle between the call to get_nohz_timer_target() and the locking of its base. Hence a risk that we enqueue a timer on a dynticks idle destination without kicking it. As a result, the timer might be serviced too late in the future. Also a target elected by __mod_timer() can be in full dynticks mode and thus require to be kicked as well. And unlike idle dynticks, this concern both local and remote targets. To fix this whole issue, lets centralize the dynticks kick to internal_add_timer() so that it is well handled for all sort of timer enqueue. Even timer migration is concerned so that a full dynticks target is correctly kicked as needed when timers are migrating to it. Signed-off-by: Viresh Kumar Signed-off-by: Frederic Weisbecker Link: http://lkml.kernel.org/r/1403393357-2070-3-git-send-email-fweisbec@gmail.com Signed-off-by: Thomas Gleixner --- kernel/time/timer.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 9e5f4f25dcc0..aca5dfe2fa3d 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -410,6 +410,22 @@ static void internal_add_timer(struct tvec_base *base, struct timer_list *timer) base->next_timer = timer->expires; } base->all_timers++; + + /* + * Check whether the other CPU is in dynticks mode and needs + * to be triggered to reevaluate the timer wheel. + * We are protected against the other CPU fiddling + * with the timer by holding the timer base lock. This also + * makes sure that a CPU on the way to stop its tick can not + * evaluate the timer wheel. + * + * Spare the IPI for deferrable timers on idle targets though. + * The next busy ticks will take care of it. Except full dynticks + * require special care against races with idle_cpu(), lets deal + * with that later. + */ + if (!tbase_get_deferrable(base) || tick_nohz_full_cpu(base->cpu)) + wake_up_nohz_cpu(base->cpu); } #ifdef CONFIG_TIMER_STATS @@ -949,22 +965,6 @@ void add_timer_on(struct timer_list *timer, int cpu) timer_set_base(timer, base); debug_activate(timer, timer->expires); internal_add_timer(base, timer); - /* - * Check whether the other CPU is in dynticks mode and needs - * to be triggered to reevaluate the timer wheel. - * We are protected against the other CPU fiddling - * with the timer by holding the timer base lock. This also - * makes sure that a CPU on the way to stop its tick can not - * evaluate the timer wheel. - * - * Spare the IPI for deferrable timers on idle targets though. - * The next busy ticks will take care of it. Except full dynticks - * require special care against races with idle_cpu(), lets deal - * with that later. - */ - if (!tbase_get_deferrable(timer->base) || tick_nohz_full_cpu(cpu)) - wake_up_nohz_cpu(cpu); - spin_unlock_irqrestore(&base->lock, flags); } EXPORT_SYMBOL_GPL(add_timer_on); From cddd02489f52ccf635ed65931214729a23b93cd6 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Sun, 22 Jun 2014 01:29:15 +0200 Subject: [PATCH 012/114] hrtimer: Store cpu-number in struct hrtimer_cpu_base In lowres mode, hrtimers are serviced by the tick instead of a clock event. Now it works well as long as the tick stays periodic but we must also make sure that the hrtimers are serviced in dynticks mode. Part of that job consist in kicking a dynticks hrtimer target in order to make it reconsider the next tick to schedule to correctly handle the hrtimer's expiring time. And that part isn't handled by the hrtimers subsystem. To prepare for fixing this, we need __hrtimer_start_range_ns() to be able to resolve the CPU target associated to a hrtimer's object 'cpu_base' so that the kick can be centralized there. So lets store it in the 'struct hrtimer_cpu_base' to resolve the CPU without overhead. It is set once at CPU's online notification. Signed-off-by: Viresh Kumar Signed-off-by: Frederic Weisbecker Link: http://lkml.kernel.org/r/1403393357-2070-4-git-send-email-fweisbec@gmail.com Signed-off-by: Thomas Gleixner --- include/linux/hrtimer.h | 2 ++ kernel/time/hrtimer.c | 1 + 2 files changed, 3 insertions(+) diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index e7a8d3fa91d5..bb4ffff31c69 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -165,6 +165,7 @@ enum hrtimer_base_type { * struct hrtimer_cpu_base - the per cpu clock bases * @lock: lock protecting the base and associated clock bases * and timers + * @cpu: cpu number * @active_bases: Bitfield to mark bases with active timers * @clock_was_set: Indicates that clock was set from irq context. * @expires_next: absolute time of the next event which was scheduled @@ -179,6 +180,7 @@ enum hrtimer_base_type { */ struct hrtimer_cpu_base { raw_spinlock_t lock; + unsigned int cpu; unsigned int active_bases; unsigned int clock_was_set; #ifdef CONFIG_HIGH_RES_TIMERS diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 3ab28993f6e0..0e32d4e7583f 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1680,6 +1680,7 @@ static void init_hrtimers_cpu(int cpu) timerqueue_init_head(&cpu_base->clock_base[i].active); } + cpu_base->cpu = cpu; hrtimer_init_hres(cpu_base); } From 49a2a07514a3a2ea4a02482fa60575e106d960f9 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Mon, 23 Jun 2014 13:39:37 +0530 Subject: [PATCH 013/114] hrtimer: Kick lowres dynticks targets on timer enqueue In lowres mode, hrtimers are serviced by the tick instead of a clock event. It works well as long as the tick stays periodic but we must also make sure that the hrtimers are serviced in dynticks mode targets, pretty much like timer list timers do. Note that all dynticks modes are concerned: get_nohz_timer_target() tries not to return remote idle CPUs but there is nothing to prevent the elected target from entering dynticks idle mode until we lock its base. It's also prefectly legal to enqueue hrtimers on full dynticks CPU. So there are two requirements to correctly handle dynticks: 1) On target's tick stop time, we must not delay the next tick further the next hrtimer. 2) On hrtimer queue time. If the tick of the target is stopped, we must wake up that CPU such that it sees the new hrtimer and recalculate the next tick accordingly. The point 1 is well handled currently through get_nohz_timer_interrupt() and cmp_next_hrtimer_event(). But the point 2 isn't handled at all. Fixing this is easy though as we have the necessary API ready for that. All we need is to call wake_up_nohz_cpu() on a target when a newly enqueued hrtimer requires tick rescheduling, like timer list timer do. Signed-off-by: Viresh Kumar Signed-off-by: Frederic Weisbecker Link: http://lkml.kernel.org/r/3d7ea08ce008698e26bd39fe10f55949391073ab.1403507178.git.viresh.kumar@linaro.org Signed-off-by: Thomas Gleixner --- kernel/time/hrtimer.c | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 0e32d4e7583f..f9007478fcce 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1013,14 +1013,25 @@ int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, leftmost = enqueue_hrtimer(timer, new_base); - /* - * Only allow reprogramming if the new base is on this CPU. - * (it might still be on another CPU if the timer was pending) - * - * XXX send_remote_softirq() ? - */ - if (leftmost && new_base->cpu_base == &__get_cpu_var(hrtimer_bases) - && hrtimer_enqueue_reprogram(timer, new_base)) { + if (!leftmost) { + unlock_hrtimer_base(timer, &flags); + return ret; + } + + if (!hrtimer_is_hres_active(timer)) { + /* + * Kick to reschedule the next tick to handle the new timer + * on dynticks target. + */ + wake_up_nohz_cpu(new_base->cpu_base->cpu); + } else if (new_base->cpu_base == &__get_cpu_var(hrtimer_bases) && + hrtimer_enqueue_reprogram(timer, new_base)) { + /* + * Only allow reprogramming if the new base is on this CPU. + * (it might still be on another CPU if the timer was pending) + * + * XXX send_remote_softirq() ? + */ if (wakeup) { /* * We need to drop cpu_base->lock to avoid a From 9e1e01dd79ac4cf936623399abe57dfba4528ae6 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Sun, 22 Jun 2014 01:29:17 +0200 Subject: [PATCH 014/114] hrtimer: Remove hrtimer_enqueue_reprogram() We call hrtimer_enqueue_reprogram() only when we are in high resolution mode now so we don't need to check that again in hrtimer_enqueue_reprogram(). Once the check is removed, hrtimer_enqueue_reprogram() turns to be an useless wrapper over hrtimer_reprogram() and can be dropped. Signed-off-by: Viresh Kumar Signed-off-by: Frederic Weisbecker Link: http://lkml.kernel.org/r/1403393357-2070-6-git-send-email-fweisbec@gmail.com Signed-off-by: Thomas Gleixner --- kernel/time/hrtimer.c | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index f9007478fcce..66a6dc1075ad 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -602,6 +602,11 @@ hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal) * timers, we have to check, whether it expires earlier than the timer for * which the clock event device was armed. * + * Note, that in case the state has HRTIMER_STATE_CALLBACK set, no reprogramming + * and no expiry check happens. The timer gets enqueued into the rbtree. The + * reprogramming and expiry check is done in the hrtimer_interrupt or in the + * softirq. + * * Called with interrupts disabled and base->cpu_base.lock held */ static int hrtimer_reprogram(struct hrtimer *timer, @@ -662,18 +667,6 @@ static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) base->hres_active = 0; } -/* - * When High resolution timers are active, try to reprogram. Note, that in case - * the state has HRTIMER_STATE_CALLBACK set, no reprogramming and no expiry - * check happens. The timer gets enqueued into the rbtree. The reprogramming - * and expiry check is done in the hrtimer_interrupt or in the softirq. - */ -static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, - struct hrtimer_clock_base *base) -{ - return base->cpu_base->hres_active && hrtimer_reprogram(timer, base); -} - static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base) { ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset; @@ -755,8 +748,8 @@ static inline int hrtimer_is_hres_enabled(void) { return 0; } static inline int hrtimer_switch_to_hres(void) { return 0; } static inline void hrtimer_force_reprogram(struct hrtimer_cpu_base *base, int skip_equal) { } -static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, - struct hrtimer_clock_base *base) +static inline int hrtimer_reprogram(struct hrtimer *timer, + struct hrtimer_clock_base *base) { return 0; } @@ -1025,7 +1018,7 @@ int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, */ wake_up_nohz_cpu(new_base->cpu_base->cpu); } else if (new_base->cpu_base == &__get_cpu_var(hrtimer_bases) && - hrtimer_enqueue_reprogram(timer, new_base)) { + hrtimer_reprogram(timer, new_base)) { /* * Only allow reprogramming if the new base is on this CPU. * (it might still be on another CPU if the timer was pending) From 31e912f598371bcfdffc990289029e1110f8b3f9 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Tue, 28 Jan 2014 15:52:46 +0100 Subject: [PATCH 015/114] clocksource: sh_cmt: Drop support for legacy platform data Now that all platforms have switched to the new-style platform data, drop support for the legacy version. Signed-off-by: Laurent Pinchart Tested-by: Simon Horman --- drivers/clocksource/sh_cmt.c | 172 ++++++++--------------------------- 1 file changed, 40 insertions(+), 132 deletions(-) diff --git a/drivers/clocksource/sh_cmt.c b/drivers/clocksource/sh_cmt.c index dfa780396b91..fcd38db9ce5c 100644 --- a/drivers/clocksource/sh_cmt.c +++ b/drivers/clocksource/sh_cmt.c @@ -114,9 +114,7 @@ struct sh_cmt_device { struct platform_device *pdev; const struct sh_cmt_info *info; - bool legacy; - void __iomem *mapbase_ch; void __iomem *mapbase; struct clk *clk; @@ -792,7 +790,7 @@ static int sh_cmt_register_clockevent(struct sh_cmt_channel *ch, int irq; int ret; - irq = platform_get_irq(ch->cmt->pdev, ch->cmt->legacy ? 0 : ch->index); + irq = platform_get_irq(ch->cmt->pdev, ch->index); if (irq < 0) { dev_err(&ch->cmt->pdev->dev, "ch%u: failed to get irq\n", ch->index); @@ -863,33 +861,26 @@ static int sh_cmt_setup_channel(struct sh_cmt_channel *ch, unsigned int index, * Compute the address of the channel control register block. For the * timers with a per-channel start/stop register, compute its address * as well. - * - * For legacy configuration the address has been mapped explicitly. */ - if (cmt->legacy) { - ch->ioctrl = cmt->mapbase_ch; - } else { - switch (cmt->info->model) { - case SH_CMT_16BIT: - ch->ioctrl = cmt->mapbase + 2 + ch->hwidx * 6; - break; - case SH_CMT_32BIT: - case SH_CMT_48BIT: - ch->ioctrl = cmt->mapbase + 0x10 + ch->hwidx * 0x10; - break; - case SH_CMT_32BIT_FAST: - /* - * The 32-bit "fast" timer has a single channel at hwidx - * 5 but is located at offset 0x40 instead of 0x60 for - * some reason. - */ - ch->ioctrl = cmt->mapbase + 0x40; - break; - case SH_CMT_48BIT_GEN2: - ch->iostart = cmt->mapbase + ch->hwidx * 0x100; - ch->ioctrl = ch->iostart + 0x10; - break; - } + switch (cmt->info->model) { + case SH_CMT_16BIT: + ch->ioctrl = cmt->mapbase + 2 + ch->hwidx * 6; + break; + case SH_CMT_32BIT: + case SH_CMT_48BIT: + ch->ioctrl = cmt->mapbase + 0x10 + ch->hwidx * 0x10; + break; + case SH_CMT_32BIT_FAST: + /* + * The 32-bit "fast" timer has a single channel at hwidx 5 but + * is located at offset 0x40 instead of 0x60 for some reason. + */ + ch->ioctrl = cmt->mapbase + 0x40; + break; + case SH_CMT_48BIT_GEN2: + ch->iostart = cmt->mapbase + ch->hwidx * 0x100; + ch->ioctrl = ch->iostart + 0x10; + break; } if (cmt->info->width == (sizeof(ch->max_match_value) * 8)) @@ -900,12 +891,7 @@ static int sh_cmt_setup_channel(struct sh_cmt_channel *ch, unsigned int index, ch->match_value = ch->max_match_value; raw_spin_lock_init(&ch->lock); - if (cmt->legacy) { - ch->timer_bit = ch->hwidx; - } else { - ch->timer_bit = cmt->info->model == SH_CMT_48BIT_GEN2 - ? 0 : ch->hwidx; - } + ch->timer_bit = cmt->info->model == SH_CMT_48BIT_GEN2 ? 0 : ch->hwidx; ret = sh_cmt_register(ch, dev_name(&cmt->pdev->dev), clockevent, clocksource); @@ -938,60 +924,12 @@ static int sh_cmt_map_memory(struct sh_cmt_device *cmt) return 0; } -static int sh_cmt_map_memory_legacy(struct sh_cmt_device *cmt) -{ - struct sh_timer_config *cfg = cmt->pdev->dev.platform_data; - struct resource *res, *res2; - - /* map memory, let mapbase_ch point to our channel */ - res = platform_get_resource(cmt->pdev, IORESOURCE_MEM, 0); - if (!res) { - dev_err(&cmt->pdev->dev, "failed to get I/O memory\n"); - return -ENXIO; - } - - cmt->mapbase_ch = ioremap_nocache(res->start, resource_size(res)); - if (cmt->mapbase_ch == NULL) { - dev_err(&cmt->pdev->dev, "failed to remap I/O memory\n"); - return -ENXIO; - } - - /* optional resource for the shared timer start/stop register */ - res2 = platform_get_resource(cmt->pdev, IORESOURCE_MEM, 1); - - /* map second resource for CMSTR */ - cmt->mapbase = ioremap_nocache(res2 ? res2->start : - res->start - cfg->channel_offset, - res2 ? resource_size(res2) : 2); - if (cmt->mapbase == NULL) { - dev_err(&cmt->pdev->dev, "failed to remap I/O second memory\n"); - iounmap(cmt->mapbase_ch); - return -ENXIO; - } - - /* identify the model based on the resources */ - if (resource_size(res) == 6) - cmt->info = &sh_cmt_info[SH_CMT_16BIT]; - else if (res2 && (resource_size(res2) == 4)) - cmt->info = &sh_cmt_info[SH_CMT_48BIT_GEN2]; - else - cmt->info = &sh_cmt_info[SH_CMT_32BIT]; - - return 0; -} - -static void sh_cmt_unmap_memory(struct sh_cmt_device *cmt) -{ - iounmap(cmt->mapbase); - if (cmt->mapbase_ch) - iounmap(cmt->mapbase_ch); -} - static int sh_cmt_setup(struct sh_cmt_device *cmt, struct platform_device *pdev) { struct sh_timer_config *cfg = pdev->dev.platform_data; const struct platform_device_id *id = pdev->id_entry; - unsigned int hw_channels; + unsigned int mask; + unsigned int i; int ret; memset(cmt, 0, sizeof(*cmt)); @@ -1003,10 +941,9 @@ static int sh_cmt_setup(struct sh_cmt_device *cmt, struct platform_device *pdev) } cmt->info = (const struct sh_cmt_info *)id->driver_data; - cmt->legacy = cmt->info ? false : true; /* Get hold of clock. */ - cmt->clk = clk_get(&cmt->pdev->dev, cmt->legacy ? "cmt_fck" : "fck"); + cmt->clk = clk_get(&cmt->pdev->dev, "fck"); if (IS_ERR(cmt->clk)) { dev_err(&cmt->pdev->dev, "cannot get clock\n"); return PTR_ERR(cmt->clk); @@ -1016,27 +953,13 @@ static int sh_cmt_setup(struct sh_cmt_device *cmt, struct platform_device *pdev) if (ret < 0) goto err_clk_put; - /* - * Map the memory resource(s). We need to support both the legacy - * platform device configuration (with one device per channel) and the - * new version (with multiple channels per device). - */ - if (cmt->legacy) - ret = sh_cmt_map_memory_legacy(cmt); - else - ret = sh_cmt_map_memory(cmt); - + /* Map the memory resource(s). */ + ret = sh_cmt_map_memory(cmt); if (ret < 0) goto err_clk_unprepare; /* Allocate and setup the channels. */ - if (cmt->legacy) { - cmt->num_channels = 1; - hw_channels = 0; - } else { - cmt->num_channels = hweight8(cfg->channels_mask); - hw_channels = cfg->channels_mask; - } + cmt->num_channels = hweight8(cfg->channels_mask); cmt->channels = kzalloc(cmt->num_channels * sizeof(*cmt->channels), GFP_KERNEL); @@ -1045,35 +968,21 @@ static int sh_cmt_setup(struct sh_cmt_device *cmt, struct platform_device *pdev) goto err_unmap; } - if (cmt->legacy) { - ret = sh_cmt_setup_channel(&cmt->channels[0], - cfg->timer_bit, cfg->timer_bit, - cfg->clockevent_rating != 0, - cfg->clocksource_rating != 0, cmt); + /* + * Use the first channel as a clock event device and the second channel + * as a clock source. If only one channel is available use it for both. + */ + for (i = 0, mask = cfg->channels_mask; i < cmt->num_channels; ++i) { + unsigned int hwidx = ffs(mask) - 1; + bool clocksource = i == 1 || cmt->num_channels == 1; + bool clockevent = i == 0; + + ret = sh_cmt_setup_channel(&cmt->channels[i], i, hwidx, + clockevent, clocksource, cmt); if (ret < 0) goto err_unmap; - } else { - unsigned int mask = hw_channels; - unsigned int i; - /* - * Use the first channel as a clock event device and the second - * channel as a clock source. If only one channel is available - * use it for both. - */ - for (i = 0; i < cmt->num_channels; ++i) { - unsigned int hwidx = ffs(mask) - 1; - bool clocksource = i == 1 || cmt->num_channels == 1; - bool clockevent = i == 0; - - ret = sh_cmt_setup_channel(&cmt->channels[i], i, hwidx, - clockevent, clocksource, - cmt); - if (ret < 0) - goto err_unmap; - - mask &= ~(1 << hwidx); - } + mask &= ~(1 << hwidx); } platform_set_drvdata(pdev, cmt); @@ -1082,7 +991,7 @@ static int sh_cmt_setup(struct sh_cmt_device *cmt, struct platform_device *pdev) err_unmap: kfree(cmt->channels); - sh_cmt_unmap_memory(cmt); + iounmap(cmt->mapbase); err_clk_unprepare: clk_unprepare(cmt->clk); err_clk_put: @@ -1133,7 +1042,6 @@ static int sh_cmt_remove(struct platform_device *pdev) } static const struct platform_device_id sh_cmt_id_table[] = { - { "sh_cmt", 0 }, { "sh-cmt-16", (kernel_ulong_t)&sh_cmt_info[SH_CMT_16BIT] }, { "sh-cmt-32", (kernel_ulong_t)&sh_cmt_info[SH_CMT_32BIT] }, { "sh-cmt-32-fast", (kernel_ulong_t)&sh_cmt_info[SH_CMT_32BIT_FAST] }, From de599c8843ebbdfc29a119c94af481b1de76700e Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Mon, 17 Feb 2014 16:49:05 +0100 Subject: [PATCH 016/114] clocksource: sh_cmt: Replace global spinlock with a per-device spinlock The global spinlock is used to protect the shared start/stop register. Now that all CMT channels are handled by a single device instance, use a per-device spinlock. Signed-off-by: Laurent Pinchart Tested-by: Simon Horman --- drivers/clocksource/sh_cmt.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/clocksource/sh_cmt.c b/drivers/clocksource/sh_cmt.c index fcd38db9ce5c..190c655d8352 100644 --- a/drivers/clocksource/sh_cmt.c +++ b/drivers/clocksource/sh_cmt.c @@ -118,6 +118,8 @@ struct sh_cmt_device { void __iomem *mapbase; struct clk *clk; + raw_spinlock_t lock; /* Protect the shared start/stop register */ + struct sh_cmt_channel *channels; unsigned int num_channels; @@ -299,14 +301,12 @@ static unsigned long sh_cmt_get_counter(struct sh_cmt_channel *ch, return v2; } -static DEFINE_RAW_SPINLOCK(sh_cmt_lock); - static void sh_cmt_start_stop_ch(struct sh_cmt_channel *ch, int start) { unsigned long flags, value; /* start stop register shared by multiple timer channels */ - raw_spin_lock_irqsave(&sh_cmt_lock, flags); + raw_spin_lock_irqsave(&ch->cmt->lock, flags); value = sh_cmt_read_cmstr(ch); if (start) @@ -315,7 +315,7 @@ static void sh_cmt_start_stop_ch(struct sh_cmt_channel *ch, int start) value &= ~(1 << ch->timer_bit); sh_cmt_write_cmstr(ch, value); - raw_spin_unlock_irqrestore(&sh_cmt_lock, flags); + raw_spin_unlock_irqrestore(&ch->cmt->lock, flags); } static int sh_cmt_enable(struct sh_cmt_channel *ch, unsigned long *rate) @@ -934,6 +934,7 @@ static int sh_cmt_setup(struct sh_cmt_device *cmt, struct platform_device *pdev) memset(cmt, 0, sizeof(*cmt)); cmt->pdev = pdev; + raw_spin_lock_init(&cmt->lock); if (!cfg) { dev_err(&cmt->pdev->dev, "missing platform data\n"); From 681b9e852cf2d48ebd63954028814c7cece71945 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Tue, 28 Jan 2014 15:52:46 +0100 Subject: [PATCH 017/114] clocksource: sh_tmu: Drop support for legacy platform data Now that all platforms have switched to the new-style platform data, drop support for the legacy version. Signed-off-by: Laurent Pinchart Tested-by: Simon Horman --- drivers/clocksource/sh_tmu.c | 82 +++++++----------------------------- 1 file changed, 15 insertions(+), 67 deletions(-) diff --git a/drivers/clocksource/sh_tmu.c b/drivers/clocksource/sh_tmu.c index 6bd17a8f3dd4..3eee5c8c26aa 100644 --- a/drivers/clocksource/sh_tmu.c +++ b/drivers/clocksource/sh_tmu.c @@ -32,7 +32,6 @@ #include enum sh_tmu_model { - SH_TMU_LEGACY, SH_TMU, SH_TMU_SH3, }; @@ -91,8 +90,6 @@ static inline unsigned long sh_tmu_read(struct sh_tmu_channel *ch, int reg_nr) if (reg_nr == TSTR) { switch (ch->tmu->model) { - case SH_TMU_LEGACY: - return ioread8(ch->tmu->mapbase); case SH_TMU_SH3: return ioread8(ch->tmu->mapbase + 2); case SH_TMU: @@ -115,8 +112,6 @@ static inline void sh_tmu_write(struct sh_tmu_channel *ch, int reg_nr, if (reg_nr == TSTR) { switch (ch->tmu->model) { - case SH_TMU_LEGACY: - return iowrite8(value, ch->tmu->mapbase); case SH_TMU_SH3: return iowrite8(value, ch->tmu->mapbase + 2); case SH_TMU: @@ -476,27 +471,12 @@ static int sh_tmu_channel_setup(struct sh_tmu_channel *ch, unsigned int index, return 0; ch->tmu = tmu; + ch->index = index; - if (tmu->model == SH_TMU_LEGACY) { - struct sh_timer_config *cfg = tmu->pdev->dev.platform_data; - - /* - * The SH3 variant (SH770x, SH7705, SH7710 and SH7720) maps - * channel registers blocks at base + 2 + 12 * index, while all - * other variants map them at base + 4 + 12 * index. We can - * compute the index by just dividing by 12, the 2 bytes or 4 - * bytes offset being hidden by the integer division. - */ - ch->index = cfg->channel_offset / 12; - ch->base = tmu->mapbase + cfg->channel_offset; - } else { - ch->index = index; - - if (tmu->model == SH_TMU_SH3) - ch->base = tmu->mapbase + 4 + ch->index * 12; - else - ch->base = tmu->mapbase + 8 + ch->index * 12; - } + if (tmu->model == SH_TMU_SH3) + ch->base = tmu->mapbase + 4 + ch->index * 12; + else + ch->base = tmu->mapbase + 8 + ch->index * 12; ch->irq = platform_get_irq(tmu->pdev, index); if (ch->irq < 0) { @@ -526,28 +506,9 @@ static int sh_tmu_map_memory(struct sh_tmu_device *tmu) if (tmu->mapbase == NULL) return -ENXIO; - /* - * In legacy platform device configuration (with one device per channel) - * the resource points to the channel base address. - */ - if (tmu->model == SH_TMU_LEGACY) { - struct sh_timer_config *cfg = tmu->pdev->dev.platform_data; - tmu->mapbase -= cfg->channel_offset; - } - return 0; } -static void sh_tmu_unmap_memory(struct sh_tmu_device *tmu) -{ - if (tmu->model == SH_TMU_LEGACY) { - struct sh_timer_config *cfg = tmu->pdev->dev.platform_data; - tmu->mapbase += cfg->channel_offset; - } - - iounmap(tmu->mapbase); -} - static int sh_tmu_setup(struct sh_tmu_device *tmu, struct platform_device *pdev) { struct sh_timer_config *cfg = pdev->dev.platform_data; @@ -564,8 +525,7 @@ static int sh_tmu_setup(struct sh_tmu_device *tmu, struct platform_device *pdev) tmu->model = id->driver_data; /* Get hold of clock. */ - tmu->clk = clk_get(&tmu->pdev->dev, - tmu->model == SH_TMU_LEGACY ? "tmu_fck" : "fck"); + tmu->clk = clk_get(&tmu->pdev->dev, "fck"); if (IS_ERR(tmu->clk)) { dev_err(&tmu->pdev->dev, "cannot get clock\n"); return PTR_ERR(tmu->clk); @@ -583,10 +543,7 @@ static int sh_tmu_setup(struct sh_tmu_device *tmu, struct platform_device *pdev) } /* Allocate and setup the channels. */ - if (tmu->model == SH_TMU_LEGACY) - tmu->num_channels = 1; - else - tmu->num_channels = hweight8(cfg->channels_mask); + tmu->num_channels = hweight8(cfg->channels_mask); tmu->channels = kzalloc(sizeof(*tmu->channels) * tmu->num_channels, GFP_KERNEL); @@ -595,23 +552,15 @@ static int sh_tmu_setup(struct sh_tmu_device *tmu, struct platform_device *pdev) goto err_unmap; } - if (tmu->model == SH_TMU_LEGACY) { - ret = sh_tmu_channel_setup(&tmu->channels[0], 0, - cfg->clockevent_rating != 0, - cfg->clocksource_rating != 0, tmu); + /* + * Use the first channel as a clock event device and the second channel + * as a clock source. + */ + for (i = 0; i < tmu->num_channels; ++i) { + ret = sh_tmu_channel_setup(&tmu->channels[i], i, + i == 0, i == 1, tmu); if (ret < 0) goto err_unmap; - } else { - /* - * Use the first channel as a clock event device and the second - * channel as a clock source. - */ - for (i = 0; i < tmu->num_channels; ++i) { - ret = sh_tmu_channel_setup(&tmu->channels[i], i, - i == 0, i == 1, tmu); - if (ret < 0) - goto err_unmap; - } } platform_set_drvdata(pdev, tmu); @@ -620,7 +569,7 @@ static int sh_tmu_setup(struct sh_tmu_device *tmu, struct platform_device *pdev) err_unmap: kfree(tmu->channels); - sh_tmu_unmap_memory(tmu); + iounmap(tmu->mapbase); err_clk_unprepare: clk_unprepare(tmu->clk); err_clk_put: @@ -671,7 +620,6 @@ static int sh_tmu_remove(struct platform_device *pdev) } static const struct platform_device_id sh_tmu_id_table[] = { - { "sh_tmu", SH_TMU_LEGACY }, { "sh-tmu", SH_TMU }, { "sh-tmu-sh3", SH_TMU_SH3 }, { } From 2b027f1f0f887097b4140a71b5c1e878da1e2fd9 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Mon, 17 Feb 2014 16:49:05 +0100 Subject: [PATCH 018/114] clocksource: sh_tmu: Replace global spinlock with a per-device spinlock The global spinlock is used to protect the shared start/stop register. Now that all TMU channels are handled by a single device instance, use a per-device spinlock. Signed-off-by: Laurent Pinchart Tested-by: Simon Horman --- drivers/clocksource/sh_tmu.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/clocksource/sh_tmu.c b/drivers/clocksource/sh_tmu.c index 3eee5c8c26aa..560a31acbc9c 100644 --- a/drivers/clocksource/sh_tmu.c +++ b/drivers/clocksource/sh_tmu.c @@ -61,6 +61,8 @@ struct sh_tmu_device { enum sh_tmu_model model; + raw_spinlock_t lock; /* Protect the shared start/stop register */ + struct sh_tmu_channel *channels; unsigned int num_channels; @@ -68,8 +70,6 @@ struct sh_tmu_device { bool has_clocksource; }; -static DEFINE_RAW_SPINLOCK(sh_tmu_lock); - #define TSTR -1 /* shared register */ #define TCOR 0 /* channel register */ #define TCNT 1 /* channel register */ @@ -132,7 +132,7 @@ static void sh_tmu_start_stop_ch(struct sh_tmu_channel *ch, int start) unsigned long flags, value; /* start stop register shared by multiple timer channels */ - raw_spin_lock_irqsave(&sh_tmu_lock, flags); + raw_spin_lock_irqsave(&ch->tmu->lock, flags); value = sh_tmu_read(ch, TSTR); if (start) @@ -141,7 +141,7 @@ static void sh_tmu_start_stop_ch(struct sh_tmu_channel *ch, int start) value &= ~(1 << ch->index); sh_tmu_write(ch, TSTR, value); - raw_spin_unlock_irqrestore(&sh_tmu_lock, flags); + raw_spin_unlock_irqrestore(&ch->tmu->lock, flags); } static int __sh_tmu_enable(struct sh_tmu_channel *ch) @@ -524,6 +524,8 @@ static int sh_tmu_setup(struct sh_tmu_device *tmu, struct platform_device *pdev) tmu->pdev = pdev; tmu->model = id->driver_data; + raw_spin_lock_init(&tmu->lock); + /* Get hold of clock. */ tmu->clk = clk_get(&tmu->pdev->dev, "fck"); if (IS_ERR(tmu->clk)) { From 1a5da0e43be0c07462e445549dbdd4a1731a3e11 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Tue, 4 Mar 2014 18:13:57 +0100 Subject: [PATCH 019/114] clocksource: sh_mtu2: Drop support for legacy platform data Now that all platforms have switched to the new-style platform data, drop support for the legacy version. Signed-off-by: Laurent Pinchart Tested-by: Wolfram Sang --- drivers/clocksource/sh_mtu2.c | 132 +++++++++------------------------- 1 file changed, 32 insertions(+), 100 deletions(-) diff --git a/drivers/clocksource/sh_mtu2.c b/drivers/clocksource/sh_mtu2.c index 188d4e092efc..0342e4a01c9e 100644 --- a/drivers/clocksource/sh_mtu2.c +++ b/drivers/clocksource/sh_mtu2.c @@ -37,7 +37,6 @@ struct sh_mtu2_channel { unsigned int index; void __iomem *base; - int irq; struct clock_event_device ced; }; @@ -51,7 +50,6 @@ struct sh_mtu2_device { struct sh_mtu2_channel *channels; unsigned int num_channels; - bool legacy; bool has_clockevent; }; @@ -162,12 +160,8 @@ static inline unsigned long sh_mtu2_read(struct sh_mtu2_channel *ch, int reg_nr) { unsigned long offs; - if (reg_nr == TSTR) { - if (ch->mtu->legacy) - return ioread8(ch->mtu->mapbase); - else - return ioread8(ch->mtu->mapbase + 0x280); - } + if (reg_nr == TSTR) + return ioread8(ch->mtu->mapbase + 0x280); offs = mtu2_reg_offs[reg_nr]; @@ -182,12 +176,8 @@ static inline void sh_mtu2_write(struct sh_mtu2_channel *ch, int reg_nr, { unsigned long offs; - if (reg_nr == TSTR) { - if (ch->mtu->legacy) - return iowrite8(value, ch->mtu->mapbase); - else - return iowrite8(value, ch->mtu->mapbase + 0x280); - } + if (reg_nr == TSTR) + return iowrite8(value, ch->mtu->mapbase + 0x280); offs = mtu2_reg_offs[reg_nr]; @@ -331,7 +321,6 @@ static void sh_mtu2_register_clockevent(struct sh_mtu2_channel *ch, const char *name) { struct clock_event_device *ced = &ch->ced; - int ret; ced->name = name; ced->features = CLOCK_EVT_FEAT_PERIODIC; @@ -344,24 +333,12 @@ static void sh_mtu2_register_clockevent(struct sh_mtu2_channel *ch, dev_info(&ch->mtu->pdev->dev, "ch%u: used for clock events\n", ch->index); clockevents_register_device(ced); - - ret = request_irq(ch->irq, sh_mtu2_interrupt, - IRQF_TIMER | IRQF_IRQPOLL | IRQF_NOBALANCING, - dev_name(&ch->mtu->pdev->dev), ch); - if (ret) { - dev_err(&ch->mtu->pdev->dev, "ch%u: failed to request irq %d\n", - ch->index, ch->irq); - return; - } } -static int sh_mtu2_register(struct sh_mtu2_channel *ch, const char *name, - bool clockevent) +static int sh_mtu2_register(struct sh_mtu2_channel *ch, const char *name) { - if (clockevent) { - ch->mtu->has_clockevent = true; - sh_mtu2_register_clockevent(ch, name); - } + ch->mtu->has_clockevent = true; + sh_mtu2_register_clockevent(ch, name); return 0; } @@ -372,40 +349,32 @@ static int sh_mtu2_setup_channel(struct sh_mtu2_channel *ch, unsigned int index, static const unsigned int channel_offsets[] = { 0x300, 0x380, 0x000, }; - bool clockevent; + char name[6]; + int irq; + int ret; ch->mtu = mtu; - if (mtu->legacy) { - struct sh_timer_config *cfg = mtu->pdev->dev.platform_data; - - clockevent = cfg->clockevent_rating != 0; - - ch->irq = platform_get_irq(mtu->pdev, 0); - ch->base = mtu->mapbase - cfg->channel_offset; - ch->index = cfg->timer_bit; - } else { - char name[6]; - - clockevent = true; - - sprintf(name, "tgi%ua", index); - ch->irq = platform_get_irq_byname(mtu->pdev, name); - ch->base = mtu->mapbase + channel_offsets[index]; - ch->index = index; - } - - if (ch->irq < 0) { + sprintf(name, "tgi%ua", index); + irq = platform_get_irq_byname(mtu->pdev, name); + if (irq < 0) { /* Skip channels with no declared interrupt. */ - if (!mtu->legacy) - return 0; - - dev_err(&mtu->pdev->dev, "ch%u: failed to get irq\n", - ch->index); - return ch->irq; + return 0; } - return sh_mtu2_register(ch, dev_name(&mtu->pdev->dev), clockevent); + ret = request_irq(irq, sh_mtu2_interrupt, + IRQF_TIMER | IRQF_IRQPOLL | IRQF_NOBALANCING, + dev_name(&ch->mtu->pdev->dev), ch); + if (ret) { + dev_err(&ch->mtu->pdev->dev, "ch%u: failed to request irq %d\n", + index, irq); + return ret; + } + + ch->base = mtu->mapbase + channel_offsets[index]; + ch->index = index; + + return sh_mtu2_register(ch, dev_name(&mtu->pdev->dev)); } static int sh_mtu2_map_memory(struct sh_mtu2_device *mtu) @@ -422,46 +391,19 @@ static int sh_mtu2_map_memory(struct sh_mtu2_device *mtu) if (mtu->mapbase == NULL) return -ENXIO; - /* - * In legacy platform device configuration (with one device per channel) - * the resource points to the channel base address. - */ - if (mtu->legacy) { - struct sh_timer_config *cfg = mtu->pdev->dev.platform_data; - mtu->mapbase += cfg->channel_offset; - } - return 0; } -static void sh_mtu2_unmap_memory(struct sh_mtu2_device *mtu) -{ - if (mtu->legacy) { - struct sh_timer_config *cfg = mtu->pdev->dev.platform_data; - mtu->mapbase -= cfg->channel_offset; - } - - iounmap(mtu->mapbase); -} - static int sh_mtu2_setup(struct sh_mtu2_device *mtu, struct platform_device *pdev) { - struct sh_timer_config *cfg = pdev->dev.platform_data; - const struct platform_device_id *id = pdev->id_entry; unsigned int i; int ret; mtu->pdev = pdev; - mtu->legacy = id->driver_data; - - if (mtu->legacy && !cfg) { - dev_err(&mtu->pdev->dev, "missing platform data\n"); - return -ENXIO; - } /* Get hold of clock. */ - mtu->clk = clk_get(&mtu->pdev->dev, mtu->legacy ? "mtu2_fck" : "fck"); + mtu->clk = clk_get(&mtu->pdev->dev, "fck"); if (IS_ERR(mtu->clk)) { dev_err(&mtu->pdev->dev, "cannot get clock\n"); return PTR_ERR(mtu->clk); @@ -479,10 +421,7 @@ static int sh_mtu2_setup(struct sh_mtu2_device *mtu, } /* Allocate and setup the channels. */ - if (mtu->legacy) - mtu->num_channels = 1; - else - mtu->num_channels = 3; + mtu->num_channels = 3; mtu->channels = kzalloc(sizeof(*mtu->channels) * mtu->num_channels, GFP_KERNEL); @@ -491,16 +430,10 @@ static int sh_mtu2_setup(struct sh_mtu2_device *mtu, goto err_unmap; } - if (mtu->legacy) { - ret = sh_mtu2_setup_channel(&mtu->channels[0], 0, mtu); + for (i = 0; i < mtu->num_channels; ++i) { + ret = sh_mtu2_setup_channel(&mtu->channels[i], i, mtu); if (ret < 0) goto err_unmap; - } else { - for (i = 0; i < mtu->num_channels; ++i) { - ret = sh_mtu2_setup_channel(&mtu->channels[i], i, mtu); - if (ret < 0) - goto err_unmap; - } } platform_set_drvdata(pdev, mtu); @@ -509,7 +442,7 @@ static int sh_mtu2_setup(struct sh_mtu2_device *mtu, err_unmap: kfree(mtu->channels); - sh_mtu2_unmap_memory(mtu); + iounmap(mtu->mapbase); err_clk_unprepare: clk_unprepare(mtu->clk); err_clk_put: @@ -560,7 +493,6 @@ static int sh_mtu2_remove(struct platform_device *pdev) } static const struct platform_device_id sh_mtu2_id_table[] = { - { "sh_mtu2", 1 }, { "sh-mtu2", 0 }, { }, }; From 8b2463d8cae2dda0c98ab5a15f25a0350a0e998d Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Tue, 4 Mar 2014 15:25:56 +0100 Subject: [PATCH 020/114] clocksource: sh_mtu2: Replace global spinlock with a per-device spinlock The global spinlock is used to protect the shared start/stop register. Now that all MTU2 channels are handled by a single device instance, use a per-device spinlock. Signed-off-by: Laurent Pinchart Tested-by: Wolfram Sang --- drivers/clocksource/sh_mtu2.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/clocksource/sh_mtu2.c b/drivers/clocksource/sh_mtu2.c index 0342e4a01c9e..b0c229f4b4c6 100644 --- a/drivers/clocksource/sh_mtu2.c +++ b/drivers/clocksource/sh_mtu2.c @@ -47,14 +47,14 @@ struct sh_mtu2_device { void __iomem *mapbase; struct clk *clk; + raw_spinlock_t lock; /* Protect the shared registers */ + struct sh_mtu2_channel *channels; unsigned int num_channels; bool has_clockevent; }; -static DEFINE_RAW_SPINLOCK(sh_mtu2_lock); - #define TSTR -1 /* shared register */ #define TCR 0 /* channel register */ #define TMDR 1 /* channel register */ @@ -192,7 +192,7 @@ static void sh_mtu2_start_stop_ch(struct sh_mtu2_channel *ch, int start) unsigned long flags, value; /* start stop register shared by multiple timer channels */ - raw_spin_lock_irqsave(&sh_mtu2_lock, flags); + raw_spin_lock_irqsave(&ch->mtu->lock, flags); value = sh_mtu2_read(ch, TSTR); if (start) @@ -201,7 +201,7 @@ static void sh_mtu2_start_stop_ch(struct sh_mtu2_channel *ch, int start) value &= ~(1 << ch->index); sh_mtu2_write(ch, TSTR, value); - raw_spin_unlock_irqrestore(&sh_mtu2_lock, flags); + raw_spin_unlock_irqrestore(&ch->mtu->lock, flags); } static int sh_mtu2_enable(struct sh_mtu2_channel *ch) @@ -402,6 +402,8 @@ static int sh_mtu2_setup(struct sh_mtu2_device *mtu, mtu->pdev = pdev; + raw_spin_lock_init(&mtu->lock); + /* Get hold of clock. */ mtu->clk = clk_get(&mtu->pdev->dev, "fck"); if (IS_ERR(mtu->clk)) { From 628627bfd943c077c65489acd8b23c7bb14eb0e2 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Tue, 4 Mar 2014 18:50:53 +0100 Subject: [PATCH 021/114] clocksource: shmobile: Remove unused sh_timer_config members The name, channel_offset, timer_bit, clockevent_rating and clocksource_rating members are unused. Remove them. Signed-off-by: Laurent Pinchart Tested-by: Simon Horman --- include/linux/sh_timer.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/include/linux/sh_timer.h b/include/linux/sh_timer.h index 8e1e036d6d45..64638b058076 100644 --- a/include/linux/sh_timer.h +++ b/include/linux/sh_timer.h @@ -2,11 +2,6 @@ #define __SH_TIMER_H__ struct sh_timer_config { - char *name; - long channel_offset; - int timer_bit; - unsigned long clockevent_rating; - unsigned long clocksource_rating; unsigned int channels_mask; }; From 1768aa2f4c1248051013282c6cf63b368016cb53 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Wed, 12 Feb 2014 17:12:40 +0100 Subject: [PATCH 022/114] clocksource: sh_cmt: Add DT support Document DT bindings and parse them in the CMT driver. Signed-off-by: Laurent Pinchart Tested-by: Simon Horman --- .../devicetree/bindings/timer/renesas,cmt.txt | 47 +++++++++++++ drivers/clocksource/sh_cmt.c | 66 ++++++++++++++----- 2 files changed, 95 insertions(+), 18 deletions(-) create mode 100644 Documentation/devicetree/bindings/timer/renesas,cmt.txt diff --git a/Documentation/devicetree/bindings/timer/renesas,cmt.txt b/Documentation/devicetree/bindings/timer/renesas,cmt.txt new file mode 100644 index 000000000000..a17418b0ece3 --- /dev/null +++ b/Documentation/devicetree/bindings/timer/renesas,cmt.txt @@ -0,0 +1,47 @@ +* Renesas R-Car Compare Match Timer (CMT) + +The CMT is a multi-channel 16/32/48-bit timer/counter with configurable clock +inputs and programmable compare match. + +Channels share hardware resources but their counter and compare match value +are independent. A particular CMT instance can implement only a subset of the +channels supported by the CMT model. Channel indices represent the hardware +position of the channel in the CMT and don't match the channel numbers in the +datasheets. + +Required Properties: + + - compatible: must contain one of the following. + - "renesas,cmt-32" for the 32-bit CMT + (CMT0 on sh7372, sh73a0 and r8a7740) + - "renesas,cmt-32-fast" for the 32-bit CMT with fast clock support + (CMT[234] on sh7372, sh73a0 and r8a7740) + - "renesas,cmt-48" for the 48-bit CMT + (CMT1 on sh7372, sh73a0 and r8a7740) + - "renesas,cmt-48-gen2" for the second generation 48-bit CMT + (CMT[01] on r8a73a4, r8a7790 and r8a7791) + + - reg: base address and length of the registers block for the timer module. + - interrupts: interrupt-specifier for the timer, one per channel. + - clocks: a list of phandle + clock-specifier pairs, one for each entry + in clock-names. + - clock-names: must contain "fck" for the functional clock. + + - renesas,channels-mask: bitmask of the available channels. + + +Example: R8A7790 (R-Car H2) CMT0 node + + CMT0 on R8A7790 implements hardware channels 5 and 6 only and names + them channels 0 and 1 in the documentation. + + cmt0: timer@ffca0000 { + compatible = "renesas,cmt-48-gen2"; + reg = <0 0xffca0000 0 0x1004>; + interrupts = <0 142 IRQ_TYPE_LEVEL_HIGH>, + <0 142 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&mstp1_clks R8A7790_CLK_CMT0>; + clock-names = "fck"; + + renesas,channels-mask = <0x60>; + }; diff --git a/drivers/clocksource/sh_cmt.c b/drivers/clocksource/sh_cmt.c index 190c655d8352..2bd13b53b727 100644 --- a/drivers/clocksource/sh_cmt.c +++ b/drivers/clocksource/sh_cmt.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -122,6 +123,7 @@ struct sh_cmt_device { struct sh_cmt_channel *channels; unsigned int num_channels; + unsigned int hw_channels; bool has_clockevent; bool has_clocksource; @@ -924,10 +926,35 @@ static int sh_cmt_map_memory(struct sh_cmt_device *cmt) return 0; } +static const struct platform_device_id sh_cmt_id_table[] = { + { "sh-cmt-16", (kernel_ulong_t)&sh_cmt_info[SH_CMT_16BIT] }, + { "sh-cmt-32", (kernel_ulong_t)&sh_cmt_info[SH_CMT_32BIT] }, + { "sh-cmt-32-fast", (kernel_ulong_t)&sh_cmt_info[SH_CMT_32BIT_FAST] }, + { "sh-cmt-48", (kernel_ulong_t)&sh_cmt_info[SH_CMT_48BIT] }, + { "sh-cmt-48-gen2", (kernel_ulong_t)&sh_cmt_info[SH_CMT_48BIT_GEN2] }, + { } +}; +MODULE_DEVICE_TABLE(platform, sh_cmt_id_table); + +static const struct of_device_id sh_cmt_of_table[] __maybe_unused = { + { .compatible = "renesas,cmt-32", .data = &sh_cmt_info[SH_CMT_32BIT] }, + { .compatible = "renesas,cmt-32-fast", .data = &sh_cmt_info[SH_CMT_32BIT_FAST] }, + { .compatible = "renesas,cmt-48", .data = &sh_cmt_info[SH_CMT_48BIT] }, + { .compatible = "renesas,cmt-48-gen2", .data = &sh_cmt_info[SH_CMT_48BIT_GEN2] }, + { } +}; +MODULE_DEVICE_TABLE(of, sh_cmt_of_table); + +static int sh_cmt_parse_dt(struct sh_cmt_device *cmt) +{ + struct device_node *np = cmt->pdev->dev.of_node; + + return of_property_read_u32(np, "renesas,channels-mask", + &cmt->hw_channels); +} + static int sh_cmt_setup(struct sh_cmt_device *cmt, struct platform_device *pdev) { - struct sh_timer_config *cfg = pdev->dev.platform_data; - const struct platform_device_id *id = pdev->id_entry; unsigned int mask; unsigned int i; int ret; @@ -936,13 +963,26 @@ static int sh_cmt_setup(struct sh_cmt_device *cmt, struct platform_device *pdev) cmt->pdev = pdev; raw_spin_lock_init(&cmt->lock); - if (!cfg) { + if (IS_ENABLED(CONFIG_OF) && pdev->dev.of_node) { + const struct of_device_id *id; + + id = of_match_node(sh_cmt_of_table, pdev->dev.of_node); + cmt->info = id->data; + + ret = sh_cmt_parse_dt(cmt); + if (ret < 0) + return ret; + } else if (pdev->dev.platform_data) { + struct sh_timer_config *cfg = pdev->dev.platform_data; + const struct platform_device_id *id = pdev->id_entry; + + cmt->info = (const struct sh_cmt_info *)id->driver_data; + cmt->hw_channels = cfg->channels_mask; + } else { dev_err(&cmt->pdev->dev, "missing platform data\n"); return -ENXIO; } - cmt->info = (const struct sh_cmt_info *)id->driver_data; - /* Get hold of clock. */ cmt->clk = clk_get(&cmt->pdev->dev, "fck"); if (IS_ERR(cmt->clk)) { @@ -960,8 +1000,7 @@ static int sh_cmt_setup(struct sh_cmt_device *cmt, struct platform_device *pdev) goto err_clk_unprepare; /* Allocate and setup the channels. */ - cmt->num_channels = hweight8(cfg->channels_mask); - + cmt->num_channels = hweight8(cmt->hw_channels); cmt->channels = kzalloc(cmt->num_channels * sizeof(*cmt->channels), GFP_KERNEL); if (cmt->channels == NULL) { @@ -973,7 +1012,7 @@ static int sh_cmt_setup(struct sh_cmt_device *cmt, struct platform_device *pdev) * Use the first channel as a clock event device and the second channel * as a clock source. If only one channel is available use it for both. */ - for (i = 0, mask = cfg->channels_mask; i < cmt->num_channels; ++i) { + for (i = 0, mask = cmt->hw_channels; i < cmt->num_channels; ++i) { unsigned int hwidx = ffs(mask) - 1; bool clocksource = i == 1 || cmt->num_channels == 1; bool clockevent = i == 0; @@ -1042,21 +1081,12 @@ static int sh_cmt_remove(struct platform_device *pdev) return -EBUSY; /* cannot unregister clockevent and clocksource */ } -static const struct platform_device_id sh_cmt_id_table[] = { - { "sh-cmt-16", (kernel_ulong_t)&sh_cmt_info[SH_CMT_16BIT] }, - { "sh-cmt-32", (kernel_ulong_t)&sh_cmt_info[SH_CMT_32BIT] }, - { "sh-cmt-32-fast", (kernel_ulong_t)&sh_cmt_info[SH_CMT_32BIT_FAST] }, - { "sh-cmt-48", (kernel_ulong_t)&sh_cmt_info[SH_CMT_48BIT] }, - { "sh-cmt-48-gen2", (kernel_ulong_t)&sh_cmt_info[SH_CMT_48BIT_GEN2] }, - { } -}; -MODULE_DEVICE_TABLE(platform, sh_cmt_id_table); - static struct platform_driver sh_cmt_device_driver = { .probe = sh_cmt_probe, .remove = sh_cmt_remove, .driver = { .name = "sh_cmt", + .of_match_table = of_match_ptr(sh_cmt_of_table), }, .id_table = sh_cmt_id_table, }; From 3e29b5543f9250bb358169cff0594f58284ece74 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Fri, 11 Apr 2014 16:23:40 +0200 Subject: [PATCH 023/114] clocksource: sh_tmu: Add DT support Document DT bindings and parse them in the TMU driver. Signed-off-by: Laurent Pinchart Tested-by: Simon Horman --- .../devicetree/bindings/timer/renesas,tmu.txt | 39 ++++++++++++++ drivers/clocksource/sh_tmu.c | 51 +++++++++++++++---- 2 files changed, 80 insertions(+), 10 deletions(-) create mode 100644 Documentation/devicetree/bindings/timer/renesas,tmu.txt diff --git a/Documentation/devicetree/bindings/timer/renesas,tmu.txt b/Documentation/devicetree/bindings/timer/renesas,tmu.txt new file mode 100644 index 000000000000..425d0c5f4aee --- /dev/null +++ b/Documentation/devicetree/bindings/timer/renesas,tmu.txt @@ -0,0 +1,39 @@ +* Renesas R-Car Timer Unit (TMU) + +The TMU is a 32-bit timer/counter with configurable clock inputs and +programmable compare match. + +Channels share hardware resources but their counter and compare match value +are independent. The TMU hardware supports up to three channels. + +Required Properties: + + - compatible: must contain "renesas,tmu" + + - reg: base address and length of the registers block for the timer module. + + - interrupts: interrupt-specifier for the timer, one per channel. + + - clocks: a list of phandle + clock-specifier pairs, one for each entry + in clock-names. + - clock-names: must contain "fck" for the functional clock. + +Optional Properties: + + - #renesas,channels: number of channels implemented by the timer, must be 2 + or 3 (if not specified the value defaults to 3). + + +Example: R8A7779 (R-Car H1) TMU0 node + + tmu0: timer@ffd80000 { + compatible = "renesas,tmu"; + reg = <0xffd80000 0x30>; + interrupts = <0 32 IRQ_TYPE_LEVEL_HIGH>, + <0 33 IRQ_TYPE_LEVEL_HIGH>, + <0 34 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&mstp0_clks R8A7779_CLK_TMU0>; + clock-names = "fck"; + + #renesas,channels = <3>; + }; diff --git a/drivers/clocksource/sh_tmu.c b/drivers/clocksource/sh_tmu.c index 560a31acbc9c..0f665b8f2461 100644 --- a/drivers/clocksource/sh_tmu.c +++ b/drivers/clocksource/sh_tmu.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -509,23 +510,48 @@ static int sh_tmu_map_memory(struct sh_tmu_device *tmu) return 0; } +static int sh_tmu_parse_dt(struct sh_tmu_device *tmu) +{ + struct device_node *np = tmu->pdev->dev.of_node; + + tmu->model = SH_TMU; + tmu->num_channels = 3; + + of_property_read_u32(np, "#renesas,channels", &tmu->num_channels); + + if (tmu->num_channels != 2 && tmu->num_channels != 3) { + dev_err(&tmu->pdev->dev, "invalid number of channels %u\n", + tmu->num_channels); + return -EINVAL; + } + + return 0; +} + static int sh_tmu_setup(struct sh_tmu_device *tmu, struct platform_device *pdev) { - struct sh_timer_config *cfg = pdev->dev.platform_data; - const struct platform_device_id *id = pdev->id_entry; unsigned int i; int ret; - if (!cfg) { + tmu->pdev = pdev; + + raw_spin_lock_init(&tmu->lock); + + if (IS_ENABLED(CONFIG_OF) && pdev->dev.of_node) { + ret = sh_tmu_parse_dt(tmu); + if (ret < 0) + return ret; + } else if (pdev->dev.platform_data) { + const struct platform_device_id *id = pdev->id_entry; + struct sh_timer_config *cfg = pdev->dev.platform_data; + + tmu->model = id->driver_data; + tmu->num_channels = hweight8(cfg->channels_mask); + } else { dev_err(&tmu->pdev->dev, "missing platform data\n"); return -ENXIO; } - tmu->pdev = pdev; - tmu->model = id->driver_data; - - raw_spin_lock_init(&tmu->lock); - /* Get hold of clock. */ tmu->clk = clk_get(&tmu->pdev->dev, "fck"); if (IS_ERR(tmu->clk)) { @@ -545,8 +571,6 @@ static int sh_tmu_setup(struct sh_tmu_device *tmu, struct platform_device *pdev) } /* Allocate and setup the channels. */ - tmu->num_channels = hweight8(cfg->channels_mask); - tmu->channels = kzalloc(sizeof(*tmu->channels) * tmu->num_channels, GFP_KERNEL); if (tmu->channels == NULL) { @@ -628,11 +652,18 @@ static const struct platform_device_id sh_tmu_id_table[] = { }; MODULE_DEVICE_TABLE(platform, sh_tmu_id_table); +static const struct of_device_id sh_tmu_of_table[] __maybe_unused = { + { .compatible = "renesas,tmu" }, + { } +}; +MODULE_DEVICE_TABLE(of, sh_tmu_of_table); + static struct platform_driver sh_tmu_device_driver = { .probe = sh_tmu_probe, .remove = sh_tmu_remove, .driver = { .name = "sh_tmu", + .of_match_table = of_match_ptr(sh_tmu_of_table), }, .id_table = sh_tmu_id_table, }; From cca8d0596c4c7acb371ea1bc5eee9b404b30516a Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Tue, 4 Mar 2014 18:28:26 +0100 Subject: [PATCH 024/114] clocksource: sh_mtu2: Add DT support Document DT bindings and parse them in the MTU2 driver. Signed-off-by: Laurent Pinchart Tested-by: Wolfram Sang --- .../bindings/timer/renesas,mtu2.txt | 39 +++++++++++++++++++ drivers/clocksource/sh_mtu2.c | 8 ++++ 2 files changed, 47 insertions(+) create mode 100644 Documentation/devicetree/bindings/timer/renesas,mtu2.txt diff --git a/Documentation/devicetree/bindings/timer/renesas,mtu2.txt b/Documentation/devicetree/bindings/timer/renesas,mtu2.txt new file mode 100644 index 000000000000..917453f826bc --- /dev/null +++ b/Documentation/devicetree/bindings/timer/renesas,mtu2.txt @@ -0,0 +1,39 @@ +* Renesas R-Car Multi-Function Timer Pulse Unit 2 (MTU2) + +The MTU2 is a multi-purpose, multi-channel timer/counter with configurable +clock inputs and programmable compare match. + +Channels share hardware resources but their counter and compare match value +are independent. The MTU2 hardware supports five channels indexed from 0 to 4. + +Required Properties: + + - compatible: must contain "renesas,mtu2" + + - reg: base address and length of the registers block for the timer module. + + - interrupts: interrupt specifiers for the timer, one for each entry in + interrupt-names. + - interrupt-names: must contain one entry named "tgi?a" for each enabled + channel, where "?" is the channel index expressed as one digit from "0" to + "4". + + - clocks: a list of phandle + clock-specifier pairs, one for each entry + in clock-names. + - clock-names: must contain "fck" for the functional clock. + + +Example: R7S72100 (RZ/A1H) MTU2 node + + mtu2: timer@fcff0000 { + compatible = "renesas,mtu2"; + reg = <0xfcff0000 0x400>; + interrupts = <0 139 IRQ_TYPE_LEVEL_HIGH>, + <0 146 IRQ_TYPE_LEVEL_HIGH>, + <0 150 IRQ_TYPE_LEVEL_HIGH>, + <0 154 IRQ_TYPE_LEVEL_HIGH>, + <0 159 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tgi0a", "tgi1a", "tgi2a", "tgi3a", "tgi4a"; + clocks = <&mstp3_clks R7S72100_CLK_MTU2>; + clock-names = "fck"; + }; diff --git a/drivers/clocksource/sh_mtu2.c b/drivers/clocksource/sh_mtu2.c index b0c229f4b4c6..3d88698cf2b8 100644 --- a/drivers/clocksource/sh_mtu2.c +++ b/drivers/clocksource/sh_mtu2.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -500,11 +501,18 @@ static const struct platform_device_id sh_mtu2_id_table[] = { }; MODULE_DEVICE_TABLE(platform, sh_mtu2_id_table); +static const struct of_device_id sh_mtu2_of_table[] __maybe_unused = { + { .compatible = "renesas,mtu2" }, + { } +}; +MODULE_DEVICE_TABLE(of, sh_mtu2_of_table); + static struct platform_driver sh_mtu2_device_driver = { .probe = sh_mtu2_probe, .remove = sh_mtu2_remove, .driver = { .name = "sh_mtu2", + .of_match_table = of_match_ptr(sh_mtu2_of_table), }, .id_table = sh_mtu2_id_table, }; From be11e6d86081aa6328eaa4fe6dd14ccf39a023c8 Mon Sep 17 00:00:00 2001 From: Masanari Iida Date: Fri, 11 Jul 2014 03:43:50 +0900 Subject: [PATCH 025/114] Documentation: Docbook: Fix files location change of kernel/[hr]timer.c make xmldocs failed because of file location change from kernel/[hr]timer.c to kernel/time/[hr]timer.c. Signed-off-by: Masanari Iida Cc: john.stultz@linaro.org Cc: rdunlap@infradead.org Link: http://lkml.kernel.org/r/1405017830-31194-1-git-send-email-standby24x7@gmail.com Signed-off-by: Thomas Gleixner --- Documentation/DocBook/device-drivers.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/DocBook/device-drivers.tmpl b/Documentation/DocBook/device-drivers.tmpl index cc63f30de166..6e06ebdbe0c7 100644 --- a/Documentation/DocBook/device-drivers.tmpl +++ b/Documentation/DocBook/device-drivers.tmpl @@ -54,7 +54,7 @@ !Ikernel/sched/cpupri.c !Ikernel/sched/fair.c !Iinclude/linux/completion.h -!Ekernel/timer.c +!Ekernel/time/timer.c Wait queues and Wake events !Iinclude/linux/wait.h @@ -63,7 +63,7 @@ High-resolution timers !Iinclude/linux/ktime.h !Iinclude/linux/hrtimer.h -!Ekernel/hrtimer.c +!Ekernel/time/hrtimer.c Workqueues and Kevents !Ekernel/workqueue.c From af9c4957cf212ad9cf0bee34c95cb11de5426e85 Mon Sep 17 00:00:00 2001 From: Cyrill Gorcunov Date: Wed, 16 Jul 2014 01:54:52 +0400 Subject: [PATCH 026/114] timerfd: Implement show_fdinfo method For checkpoint/restore of timerfd files we need to know how exactly the timer were armed, to be able to recreate it on restore stage. Thus implement show_fdinfo method which provides enough information for that. One of significant changes I think is the addition of @settime_flags member. Currently there are two flags TFD_TIMER_ABSTIME and TFD_TIMER_CANCEL_ON_SET, and the second can be found from @might_cancel variable but in case if the flags will be extended in future we most probably will have to somehow remember them explicitly anyway so I guss doing that right now won't hurt. To not bloat the timerfd_ctx structure I've converted @expired to short integer and defined @settime_flags as short too. v2 (by avagin@, vdavydov@ and tglx@): - Add it_value/it_interval fields - Save flags being used in timerfd_setup in context v3 (by tglx@): - don't forget to use CONFIG_PROC_FS v4 (by akpm@): -Use define timerfd_show NULL for non c/r config Signed-off-by: Cyrill Gorcunov Cc: Andrew Morton Cc: Michael Kerrisk Cc: Andrey Vagin Cc: Pavel Emelyanov Cc: Vladimir Davydov Link: http://lkml.kernel.org/r/20140715215703.114365649@openvz.org Signed-off-by: Thomas Gleixner --- fs/timerfd.c | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/fs/timerfd.c b/fs/timerfd.c index 0013142c0475..77183f047f65 100644 --- a/fs/timerfd.c +++ b/fs/timerfd.c @@ -35,8 +35,9 @@ struct timerfd_ctx { ktime_t moffs; wait_queue_head_t wqh; u64 ticks; - int expired; int clockid; + short unsigned expired; + short unsigned settime_flags; /* to show in fdinfo */ struct rcu_head rcu; struct list_head clist; bool might_cancel; @@ -196,6 +197,8 @@ static int timerfd_setup(struct timerfd_ctx *ctx, int flags, if (timerfd_canceled(ctx)) return -ECANCELED; } + + ctx->settime_flags = flags & TFD_SETTIME_FLAGS; return 0; } @@ -284,11 +287,40 @@ static ssize_t timerfd_read(struct file *file, char __user *buf, size_t count, return res; } +#ifdef CONFIG_PROC_FS +static int timerfd_show(struct seq_file *m, struct file *file) +{ + struct timerfd_ctx *ctx = file->private_data; + struct itimerspec t; + + spin_lock_irq(&ctx->wqh.lock); + t.it_value = ktime_to_timespec(timerfd_get_remaining(ctx)); + t.it_interval = ktime_to_timespec(ctx->tintv); + spin_unlock_irq(&ctx->wqh.lock); + + return seq_printf(m, + "clockid: %d\n" + "ticks: %llu\n" + "settime flags: 0%o\n" + "it_value: (%llu, %llu)\n" + "it_interval: (%llu, %llu)\n", + ctx->clockid, (unsigned long long)ctx->ticks, + ctx->settime_flags, + (unsigned long long)t.it_value.tv_sec, + (unsigned long long)t.it_value.tv_nsec, + (unsigned long long)t.it_interval.tv_sec, + (unsigned long long)t.it_interval.tv_nsec); +} +#else +#define timerfd_show NULL +#endif + static const struct file_operations timerfd_fops = { .release = timerfd_release, .poll = timerfd_poll, .read = timerfd_read, .llseek = noop_llseek, + .show_fdinfo = timerfd_show, }; static int timerfd_fget(int fd, struct fd *p) From 854d06d9fc350130617debdd9e9f49d1c4ba5206 Mon Sep 17 00:00:00 2001 From: Cyrill Gorcunov Date: Wed, 16 Jul 2014 01:54:53 +0400 Subject: [PATCH 027/114] docs: Procfs -- Document timerfd output Signed-off-by: Cyrill Gorcunov Cc: Andrew Morton Cc: Michael Kerrisk Cc: Andrey Vagin Cc: Pavel Emelyanov Cc: Vladimir Davydov Link: http://lkml.kernel.org/r/20140715215703.199905126@openvz.org Signed-off-by: Thomas Gleixner --- Documentation/filesystems/proc.txt | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Documentation/filesystems/proc.txt b/Documentation/filesystems/proc.txt index ddc531a74d04..eb8a10e22f7c 100644 --- a/Documentation/filesystems/proc.txt +++ b/Documentation/filesystems/proc.txt @@ -1743,6 +1743,25 @@ pair provide additional information particular to the objects they represent. While the first three lines are mandatory and always printed, the rest is optional and may be omitted if no marks created yet. + Timerfd files + ~~~~~~~~~~~~~ + + pos: 0 + flags: 02 + mnt_id: 9 + clockid: 0 + ticks: 0 + settime flags: 01 + it_value: (0, 49406829) + it_interval: (1, 0) + + where 'clockid' is the clock type and 'ticks' is the number of the timer expirations + that have occurred [see timerfd_create(2) for details]. 'settime flags' are + flags in octal form been used to setup the timer [see timerfd_settime(2) for + details]. 'it_value' is remaining time until the timer exiration. + 'it_interval' is the interval for the timer. Note the timer might be set up + with TIMER_ABSTIME option which will be shown in 'settime flags', but 'it_value' + still exhibits timer's remaining time. ------------------------------------------------------------------------------ Configuring procfs From 5442e9fbd7c23172a1c9bc736629cd123a9923f0 Mon Sep 17 00:00:00 2001 From: Cyrill Gorcunov Date: Wed, 16 Jul 2014 01:54:54 +0400 Subject: [PATCH 028/114] timerfd: Implement timerfd_ioctl method to restore timerfd_ctx::ticks, v3 The read() of timerfd files allows to fetch the number of timer ticks while there is no way to set it back from userspace. To restore the timer's state as it was at checkpoint moment we need a path to bring @ticks back. Initially I thought about writing ticks back via write() interface but it seems such API is somehow obscure. Instead implement timerfd_ioctl() method with TFD_IOC_SET_TICKS command which allows to adjust @ticks into non-zero value waking up the waiters. I wrapped code with CONFIG_CHECKPOINT_RESTORE which can be dropped off if there users except c/r camp appear. v2 (by akpm@): - Use define timerfd_ioctl NULL for non c/r config v3: - Use copy_from_user for @ticks fetching since not all arch support get_user for 8 byte argument Signed-off-by: Cyrill Gorcunov Cc: Andrew Morton Cc: Michael Kerrisk Cc: Andrey Vagin Cc: Arnd Bergmann Cc: Christopher Covington Cc: Pavel Emelyanov Cc: Vladimir Davydov Link: http://lkml.kernel.org/r/20140715215703.285617923@openvz.org Signed-off-by: Thomas Gleixner --- fs/timerfd.c | 37 +++++++++++++++++++++++++++++++++++++ include/linux/timerfd.h | 5 +++++ 2 files changed, 42 insertions(+) diff --git a/fs/timerfd.c b/fs/timerfd.c index 77183f047f65..709603cac9e6 100644 --- a/fs/timerfd.c +++ b/fs/timerfd.c @@ -315,12 +315,49 @@ static int timerfd_show(struct seq_file *m, struct file *file) #define timerfd_show NULL #endif +#ifdef CONFIG_CHECKPOINT_RESTORE +static long timerfd_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + struct timerfd_ctx *ctx = file->private_data; + int ret = 0; + + switch (cmd) { + case TFD_IOC_SET_TICKS: { + u64 ticks; + + if (copy_from_user(&ticks, (u64 __user *)arg, sizeof(ticks))) + return -EFAULT; + if (!ticks) + return -EINVAL; + + spin_lock_irq(&ctx->wqh.lock); + if (!timerfd_canceled(ctx)) { + ctx->ticks = ticks; + if (ticks) + wake_up_locked(&ctx->wqh); + } else + ret = -ECANCELED; + spin_unlock_irq(&ctx->wqh.lock); + break; + } + default: + ret = -ENOTTY; + break; + } + + return ret; +} +#else +#define timerfd_ioctl NULL +#endif + static const struct file_operations timerfd_fops = { .release = timerfd_release, .poll = timerfd_poll, .read = timerfd_read, .llseek = noop_llseek, .show_fdinfo = timerfd_show, + .unlocked_ioctl = timerfd_ioctl, }; static int timerfd_fget(int fd, struct fd *p) diff --git a/include/linux/timerfd.h b/include/linux/timerfd.h index d3b57fa12225..bd36ce431e32 100644 --- a/include/linux/timerfd.h +++ b/include/linux/timerfd.h @@ -11,6 +11,9 @@ /* For O_CLOEXEC and O_NONBLOCK */ #include +/* For _IO helpers */ +#include + /* * CAREFUL: Check include/asm-generic/fcntl.h when defining * new flags, since they might collide with O_* ones. We want @@ -29,4 +32,6 @@ /* Flags for timerfd_settime. */ #define TFD_SETTIME_FLAGS (TFD_TIMER_ABSTIME | TFD_TIMER_CANCEL_ON_SET) +#define TFD_IOC_SET_TICKS _IOW('T', 0, u64) + #endif /* _LINUX_TIMERFD_H */ From 7e1391876c8f95df9241deb78fa98e799e7bb05e Mon Sep 17 00:00:00 2001 From: Kuninori Morimoto Date: Fri, 18 Jul 2014 11:36:36 +0200 Subject: [PATCH 029/114] clocksource: sh_mtu2: Tidy up Kconfig typo for MTU2 It should be "MTU2" instead of "TMU2" Signed-off-by: Kuninori Morimoto Acked-by: Wolfram Sang Acked-by: Simon Horman Signed-off-by: Daniel Lezcano --- drivers/clocksource/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index 6a1151840510..b17c9098ba36 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -175,7 +175,7 @@ config SH_TIMER_MTU2 default SYS_SUPPORTS_SH_MTU2 help This enables build of a clockevent driver for the Multi-Function - Timer Pulse Unit 2 (TMU2) hardware available on SoCs from Renesas. + Timer Pulse Unit 2 (MTU2) hardware available on SoCs from Renesas. This hardware comes with 16 bit-timer registers. config SH_TIMER_TMU From efd342fb0031a17758571dce42e3f373d94e2fec Mon Sep 17 00:00:00 2001 From: Matthias Brugger Date: Fri, 18 Jul 2014 11:36:39 +0200 Subject: [PATCH 030/114] of: Provide a function to request and map memory A call to of_iomap does not request the memory region. This patch adds the function of_io_request_and_map which requests the memory region before mapping it. Signed-off-by: Matthias Brugger Suggested-by: Thomas Gleixner Suggested-by: Rob Herring Acked-by: Rob Herring Signed-off-by: Daniel Lezcano --- drivers/of/address.c | 36 ++++++++++++++++++++++++++++++++++++ include/linux/io.h | 2 ++ include/linux/of_address.h | 11 +++++++++++ lib/devres.c | 2 -- 4 files changed, 49 insertions(+), 2 deletions(-) diff --git a/drivers/of/address.c b/drivers/of/address.c index 5edfcb0da37d..e3718250d66e 100644 --- a/drivers/of/address.c +++ b/drivers/of/address.c @@ -702,6 +702,42 @@ void __iomem *of_iomap(struct device_node *np, int index) } EXPORT_SYMBOL(of_iomap); +/* + * of_io_request_and_map - Requests a resource and maps the memory mapped IO + * for a given device_node + * @device: the device whose io range will be mapped + * @index: index of the io range + * @name: name of the resource + * + * Returns a pointer to the requested and mapped memory or an ERR_PTR() encoded + * error code on failure. Usage example: + * + * base = of_io_request_and_map(node, 0, "foo"); + * if (IS_ERR(base)) + * return PTR_ERR(base); + */ +void __iomem *of_io_request_and_map(struct device_node *np, int index, + char *name) +{ + struct resource res; + void __iomem *mem; + + if (of_address_to_resource(np, index, &res)) + return IOMEM_ERR_PTR(-EINVAL); + + if (!request_mem_region(res.start, resource_size(&res), name)) + return IOMEM_ERR_PTR(-EBUSY); + + mem = ioremap(res.start, resource_size(&res)); + if (!mem) { + release_mem_region(res.start, resource_size(&res)); + return IOMEM_ERR_PTR(-ENOMEM); + } + + return mem; +} +EXPORT_SYMBOL(of_io_request_and_map); + /** * of_dma_get_range - Get DMA range info * @np: device node to get DMA range info diff --git a/include/linux/io.h b/include/linux/io.h index b76e6e545806..d5fc9b8d8b03 100644 --- a/include/linux/io.h +++ b/include/linux/io.h @@ -58,6 +58,8 @@ static inline void devm_ioport_unmap(struct device *dev, void __iomem *addr) } #endif +#define IOMEM_ERR_PTR(err) (__force void __iomem *)ERR_PTR(err) + void __iomem *devm_ioremap(struct device *dev, resource_size_t offset, unsigned long size); void __iomem *devm_ioremap_nocache(struct device *dev, resource_size_t offset, diff --git a/include/linux/of_address.h b/include/linux/of_address.h index c13b8782a4eb..fb7b7221e063 100644 --- a/include/linux/of_address.h +++ b/include/linux/of_address.h @@ -109,7 +109,12 @@ static inline bool of_dma_is_coherent(struct device_node *np) extern int of_address_to_resource(struct device_node *dev, int index, struct resource *r); void __iomem *of_iomap(struct device_node *node, int index); +void __iomem *of_io_request_and_map(struct device_node *device, + int index, char *name); #else + +#include + static inline int of_address_to_resource(struct device_node *dev, int index, struct resource *r) { @@ -120,6 +125,12 @@ static inline void __iomem *of_iomap(struct device_node *device, int index) { return NULL; } + +static inline void __iomem *of_io_request_and_map(struct device_node *device, + int index, char *name) +{ + return IOMEM_ERR_PTR(-EINVAL); +} #endif #if defined(CONFIG_OF_ADDRESS) && defined(CONFIG_PCI) diff --git a/lib/devres.c b/lib/devres.c index f562bf6ff71d..bb632484a860 100644 --- a/lib/devres.c +++ b/lib/devres.c @@ -86,8 +86,6 @@ void devm_iounmap(struct device *dev, void __iomem *addr) } EXPORT_SYMBOL(devm_iounmap); -#define IOMEM_ERR_PTR(err) (__force void __iomem *)ERR_PTR(err) - /** * devm_ioremap_resource() - check, request region, and ioremap resource * @dev: generic device to handle the resource for From ecb3530dd59ddd5158bcc507d634e698130fd937 Mon Sep 17 00:00:00 2001 From: Matthias Brugger Date: Fri, 18 Jul 2014 11:36:43 +0200 Subject: [PATCH 031/114] clocksource: Add support for the Mediatek SoCs This patch adds a clock source and clock event for the timer found on the Mediatek SoCs. The Mediatek General Purpose Timer block provides five 32 bit timers and one 64 bit timer. Two 32 bit timers are used by this driver: TIMER1: clock events supporting periodic and oneshot events TIMER2: clock source configured as a free running counter The General Purpose Timer block can be run with two clocks. A 13 MHz system clock and the RTC clock running at 32 KHz. This implementation uses the system clock with no clock source divider. The interrupts are shared between the different timers and have to be read back from a register. We just enable one interrupt for the clock event. The clock event timer is used by all cores. Signed-off-by: Matthias Brugger Acked-by: Thomas Gleixner Signed-off-by: Daniel Lezcano --- drivers/clocksource/Kconfig | 5 + drivers/clocksource/Makefile | 1 + drivers/clocksource/mtk_timer.c | 261 ++++++++++++++++++++++++++++++++ 3 files changed, 267 insertions(+) create mode 100644 drivers/clocksource/mtk_timer.c diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index b17c9098ba36..497356d4dc70 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -151,6 +151,11 @@ config VF_PIT_TIMER config SYS_SUPPORTS_SH_CMT bool +config MTK_TIMER + select CLKSRC_OF + select CLKSRC_MMIO + bool + config SYS_SUPPORTS_SH_MTU2 bool diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile index 800b1303c236..da1ac09dc771 100644 --- a/drivers/clocksource/Makefile +++ b/drivers/clocksource/Makefile @@ -34,6 +34,7 @@ obj-$(CONFIG_CLKSRC_SAMSUNG_PWM) += samsung_pwm_timer.o obj-$(CONFIG_FSL_FTM_TIMER) += fsl_ftm_timer.o obj-$(CONFIG_VF_PIT_TIMER) += vf_pit_timer.o obj-$(CONFIG_CLKSRC_QCOM) += qcom-timer.o +obj-$(CONFIG_MTK_TIMER) += mtk_timer.o obj-$(CONFIG_ARM_ARCH_TIMER) += arm_arch_timer.o obj-$(CONFIG_ARM_GLOBAL_TIMER) += arm_global_timer.o diff --git a/drivers/clocksource/mtk_timer.c b/drivers/clocksource/mtk_timer.c new file mode 100644 index 000000000000..32a3d25795d3 --- /dev/null +++ b/drivers/clocksource/mtk_timer.c @@ -0,0 +1,261 @@ +/* + * Mediatek SoCs General-Purpose Timer handling. + * + * Copyright (C) 2014 Matthias Brugger + * + * Matthias Brugger + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define GPT_IRQ_EN_REG 0x00 +#define GPT_IRQ_ENABLE(val) BIT((val) - 1) +#define GPT_IRQ_ACK_REG 0x08 +#define GPT_IRQ_ACK(val) BIT((val) - 1) + +#define TIMER_CTRL_REG(val) (0x10 * (val)) +#define TIMER_CTRL_OP(val) (((val) & 0x3) << 4) +#define TIMER_CTRL_OP_ONESHOT (0) +#define TIMER_CTRL_OP_REPEAT (1) +#define TIMER_CTRL_OP_FREERUN (3) +#define TIMER_CTRL_CLEAR (2) +#define TIMER_CTRL_ENABLE (1) +#define TIMER_CTRL_DISABLE (0) + +#define TIMER_CLK_REG(val) (0x04 + (0x10 * (val))) +#define TIMER_CLK_SRC(val) (((val) & 0x1) << 4) +#define TIMER_CLK_SRC_SYS13M (0) +#define TIMER_CLK_SRC_RTC32K (1) +#define TIMER_CLK_DIV1 (0x0) +#define TIMER_CLK_DIV2 (0x1) + +#define TIMER_CNT_REG(val) (0x08 + (0x10 * (val))) +#define TIMER_CMP_REG(val) (0x0C + (0x10 * (val))) + +#define GPT_CLK_EVT 1 +#define GPT_CLK_SRC 2 + +struct mtk_clock_event_device { + void __iomem *gpt_base; + u32 ticks_per_jiffy; + struct clock_event_device dev; +}; + +static inline struct mtk_clock_event_device *to_mtk_clk( + struct clock_event_device *c) +{ + return container_of(c, struct mtk_clock_event_device, dev); +} + +static void mtk_clkevt_time_stop(struct mtk_clock_event_device *evt, u8 timer) +{ + u32 val; + + val = readl(evt->gpt_base + TIMER_CTRL_REG(timer)); + writel(val & ~TIMER_CTRL_ENABLE, evt->gpt_base + + TIMER_CTRL_REG(timer)); +} + +static void mtk_clkevt_time_setup(struct mtk_clock_event_device *evt, + unsigned long delay, u8 timer) +{ + writel(delay, evt->gpt_base + TIMER_CMP_REG(timer)); +} + +static void mtk_clkevt_time_start(struct mtk_clock_event_device *evt, + bool periodic, u8 timer) +{ + u32 val; + + /* Acknowledge interrupt */ + writel(GPT_IRQ_ACK(timer), evt->gpt_base + GPT_IRQ_ACK_REG); + + val = readl(evt->gpt_base + TIMER_CTRL_REG(timer)); + + /* Clear 2 bit timer operation mode field */ + val &= ~TIMER_CTRL_OP(0x3); + + if (periodic) + val |= TIMER_CTRL_OP(TIMER_CTRL_OP_REPEAT); + else + val |= TIMER_CTRL_OP(TIMER_CTRL_OP_ONESHOT); + + writel(val | TIMER_CTRL_ENABLE | TIMER_CTRL_CLEAR, + evt->gpt_base + TIMER_CTRL_REG(timer)); +} + +static void mtk_clkevt_mode(enum clock_event_mode mode, + struct clock_event_device *clk) +{ + struct mtk_clock_event_device *evt = to_mtk_clk(clk); + + mtk_clkevt_time_stop(evt, GPT_CLK_EVT); + + switch (mode) { + case CLOCK_EVT_MODE_PERIODIC: + mtk_clkevt_time_setup(evt, evt->ticks_per_jiffy, GPT_CLK_EVT); + mtk_clkevt_time_start(evt, true, GPT_CLK_EVT); + break; + case CLOCK_EVT_MODE_ONESHOT: + /* Timer is enabled in set_next_event */ + break; + case CLOCK_EVT_MODE_UNUSED: + case CLOCK_EVT_MODE_SHUTDOWN: + default: + /* No more interrupts will occur as source is disabled */ + break; + } +} + +static int mtk_clkevt_next_event(unsigned long event, + struct clock_event_device *clk) +{ + struct mtk_clock_event_device *evt = to_mtk_clk(clk); + + mtk_clkevt_time_stop(evt, GPT_CLK_EVT); + mtk_clkevt_time_setup(evt, event, GPT_CLK_EVT); + mtk_clkevt_time_start(evt, false, GPT_CLK_EVT); + + return 0; +} + +static irqreturn_t mtk_timer_interrupt(int irq, void *dev_id) +{ + struct mtk_clock_event_device *evt = dev_id; + + /* Acknowledge timer0 irq */ + writel(GPT_IRQ_ACK(GPT_CLK_EVT), evt->gpt_base + GPT_IRQ_ACK_REG); + evt->dev.event_handler(&evt->dev); + + return IRQ_HANDLED; +} + +static void mtk_timer_global_reset(struct mtk_clock_event_device *evt) +{ + /* Disable all interrupts */ + writel(0x0, evt->gpt_base + GPT_IRQ_EN_REG); + /* Acknowledge all interrupts */ + writel(0x3f, evt->gpt_base + GPT_IRQ_ACK_REG); +} + +static void +mtk_timer_setup(struct mtk_clock_event_device *evt, u8 timer, u8 option) +{ + writel(TIMER_CTRL_CLEAR | TIMER_CTRL_DISABLE, + evt->gpt_base + TIMER_CTRL_REG(timer)); + + writel(TIMER_CLK_SRC(TIMER_CLK_SRC_SYS13M) | TIMER_CLK_DIV1, + evt->gpt_base + TIMER_CLK_REG(timer)); + + writel(0x0, evt->gpt_base + TIMER_CMP_REG(timer)); + + writel(TIMER_CTRL_OP(option) | TIMER_CTRL_ENABLE, + evt->gpt_base + TIMER_CTRL_REG(timer)); +} + +static void mtk_timer_enable_irq(struct mtk_clock_event_device *evt, u8 timer) +{ + u32 val; + + val = readl(evt->gpt_base + GPT_IRQ_EN_REG); + writel(val | GPT_IRQ_ENABLE(timer), + evt->gpt_base + GPT_IRQ_EN_REG); +} + +static void __init mtk_timer_init(struct device_node *node) +{ + struct mtk_clock_event_device *evt; + struct resource res; + unsigned long rate = 0; + struct clk *clk; + + evt = kzalloc(sizeof(*evt), GFP_KERNEL); + if (!evt) { + pr_warn("Can't allocate mtk clock event driver struct"); + return; + } + + evt->dev.name = "mtk_tick"; + evt->dev.rating = 300; + evt->dev.features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT; + evt->dev.set_mode = mtk_clkevt_mode; + evt->dev.set_next_event = mtk_clkevt_next_event; + evt->dev.cpumask = cpu_possible_mask; + + evt->gpt_base = of_io_request_and_map(node, 0, "mtk-timer"); + if (IS_ERR(evt->gpt_base)) { + pr_warn("Can't get resource\n"); + return; + } + + evt->dev.irq = irq_of_parse_and_map(node, 0); + if (evt->dev.irq <= 0) { + pr_warn("Can't parse IRQ"); + goto err_mem; + } + + clk = of_clk_get(node, 0); + if (IS_ERR(clk)) { + pr_warn("Can't get timer clock"); + goto err_irq; + } + + if (clk_prepare_enable(clk)) { + pr_warn("Can't prepare clock"); + goto err_clk_put; + } + rate = clk_get_rate(clk); + + if (request_irq(evt->dev.irq, mtk_timer_interrupt, + IRQF_TIMER | IRQF_IRQPOLL, "mtk_timer", evt)) { + pr_warn("failed to setup irq %d\n", evt->dev.irq); + goto err_clk_disable; + } + + evt->ticks_per_jiffy = DIV_ROUND_UP(rate, HZ); + + mtk_timer_global_reset(evt); + + /* Configure clock source */ + mtk_timer_setup(evt, GPT_CLK_SRC, TIMER_CTRL_OP_FREERUN); + clocksource_mmio_init(evt->gpt_base + TIMER_CNT_REG(GPT_CLK_SRC), + node->name, rate, 300, 32, clocksource_mmio_readl_up); + + /* Configure clock event */ + mtk_timer_setup(evt, GPT_CLK_EVT, TIMER_CTRL_OP_REPEAT); + mtk_timer_enable_irq(evt, GPT_CLK_EVT); + + clockevents_config_and_register(&evt->dev, rate, 0x3, + 0xffffffff); + return; + +err_clk_disable: + clk_disable_unprepare(clk); +err_clk_put: + clk_put(clk); +err_irq: + irq_dispose_mapping(evt->dev.irq); +err_mem: + iounmap(evt->gpt_base); + of_address_to_resource(node, 0, &res); + release_mem_region(res.start, resource_size(&res)); +} +CLOCKSOURCE_OF_DECLARE(mtk_mt6577, "mediatek,mt6577-timer", mtk_timer_init); From 98c6bf2a2d426d7c6a07997a48443b2f60190d71 Mon Sep 17 00:00:00 2001 From: Matthias Brugger Date: Fri, 18 Jul 2014 11:36:46 +0200 Subject: [PATCH 032/114] dt-bindings: Add mtk-timer bindings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add binding documentation for the General Purpose Timer driver of the Mediatek SoCs. Signed-off-by: Matthias Brugger Acked-by: Sören Brinkmann Acked-by: Rob Herring Signed-off-by: Daniel Lezcano --- .../bindings/timer/mediatek,mtk-timer.txt | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 Documentation/devicetree/bindings/timer/mediatek,mtk-timer.txt diff --git a/Documentation/devicetree/bindings/timer/mediatek,mtk-timer.txt b/Documentation/devicetree/bindings/timer/mediatek,mtk-timer.txt new file mode 100644 index 000000000000..7c4408ff4b83 --- /dev/null +++ b/Documentation/devicetree/bindings/timer/mediatek,mtk-timer.txt @@ -0,0 +1,17 @@ +Mediatek MT6577, MT6572 and MT6589 Timers +--------------------------------------- + +Required properties: +- compatible: Should be "mediatek,mt6577-timer" +- reg: Should contain location and length for timers register. +- clocks: Clocks driving the timer hardware. This list should include two + clocks. The order is system clock and as second clock the RTC clock. + +Examples: + + timer@10008000 { + compatible = "mediatek,mt6577-timer"; + reg = <0x10008000 0x80>; + interrupts = ; + clocks = <&system_clk>, <&rtc_clk>; + }; From 22c4e026d68eaa7cc539e94111835f7cd24edecd Mon Sep 17 00:00:00 2001 From: Matthias Brugger Date: Fri, 18 Jul 2014 11:36:49 +0200 Subject: [PATCH 033/114] vendor-prefixes: Add prefix for Mediatek Inc. Signed-off-by: Matthias Brugger Acked-by: Rob Herring Signed-off-by: Daniel Lezcano --- Documentation/devicetree/bindings/vendor-prefixes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/vendor-prefixes.txt b/Documentation/devicetree/bindings/vendor-prefixes.txt index 46a311e728a8..dd5bce848cef 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.txt +++ b/Documentation/devicetree/bindings/vendor-prefixes.txt @@ -77,6 +77,7 @@ lsi LSI Corp. (LSI Logic) lltc Linear Technology Corporation marvell Marvell Technology Group Ltd. maxim Maxim Integrated Products +mediatek MediaTek Inc. micrel Micrel Inc. microchip Microchip Technology Inc. mosaixtech Mosaix Technologies, Inc. From 40c96312dc3534d97c64d7d69acf1ea14ceff404 Mon Sep 17 00:00:00 2001 From: Chen Gang Date: Tue, 8 Jul 2014 20:39:40 +0800 Subject: [PATCH 034/114] clocksource: Kconfig: Let EM_TIMER_STI depend on HAS_IOMEM In 'em_sti.c', it will call devm_ioremap_resource() which need HAS_IOMEM. So need let EM_TIMER_STI depend on HAS_IOMEM, too. The related error (with allmodconfig under score): LD init/built-in.o em_sti.c:(.text.em_sti_probe+0x84): undefined reference to `devm_ioremap_resource' make: *** [vmlinux] Error 1 Signed-off-by: Chen Gang Signed-off-by: Daniel Lezcano --- drivers/clocksource/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index 497356d4dc70..7ea7d7c48d72 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -194,7 +194,7 @@ config SH_TIMER_TMU config EM_TIMER_STI bool "Renesas STI timer driver" if COMPILE_TEST - depends on GENERIC_CLOCKEVENTS + depends on GENERIC_CLOCKEVENTS && HAS_IOMEM default SYS_SUPPORTS_EM_STI help This enables build of a clocksource and clockevent driver for From 38941522ecbd2198694b742343a67ea128b44913 Mon Sep 17 00:00:00 2001 From: Zhiwu Song Date: Thu, 3 Jul 2014 20:52:51 +0800 Subject: [PATCH 035/114] clocksource: sirf: Fix incorrect clock enable counter for timer In the clocksource driver, we didn't explicitly enable the clock. it makes the clk reference counter wrong. We didn't encounter any hang issue because the tick's clock input has been open and is shared by some other hardware components, but if we don't enable those components in kernel, in the stage of disabling unused clk in kernel boot, Linux tick hangs. This patch fixes it. it does an explicit prepare and enable to the clock input, and increases the usage counter of the clk. Signed-off-by: Zhiwu Song Signed-off-by: Barry Song Signed-off-by: Daniel Lezcano --- drivers/clocksource/timer-marco.c | 3 +++ drivers/clocksource/timer-prima2.c | 3 +++ 2 files changed, 6 insertions(+) diff --git a/drivers/clocksource/timer-marco.c b/drivers/clocksource/timer-marco.c index dbd30398222a..330e93064692 100644 --- a/drivers/clocksource/timer-marco.c +++ b/drivers/clocksource/timer-marco.c @@ -260,6 +260,9 @@ static void __init sirfsoc_marco_timer_init(struct device_node *np) clk = of_clk_get(np, 0); BUG_ON(IS_ERR(clk)); + + BUG_ON(clk_prepare_enable(clk)); + rate = clk_get_rate(clk); BUG_ON(rate < MARCO_CLOCK_FREQ); diff --git a/drivers/clocksource/timer-prima2.c b/drivers/clocksource/timer-prima2.c index a722aac7ac02..ce18d570e1cd 100644 --- a/drivers/clocksource/timer-prima2.c +++ b/drivers/clocksource/timer-prima2.c @@ -200,6 +200,9 @@ static void __init sirfsoc_prima2_timer_init(struct device_node *np) clk = of_clk_get(np, 0); BUG_ON(IS_ERR(clk)); + + BUG_ON(clk_prepare_enable(clk)); + rate = clk_get_rate(clk); BUG_ON(rate < PRIMA2_CLOCK_FREQ); From f0b7fabec3273c85fa2c4714762177d04f64c08e Mon Sep 17 00:00:00 2001 From: Alexander Shiyan Date: Sun, 13 Jul 2014 08:45:13 +0400 Subject: [PATCH 036/114] clocksource: clps711x: Add CLPS711X clocksource driver This adds the clocksource driver for Cirrus Logic CLPS711X series SoCs. Designed primarily for migration CLPS711X subarch for multiplatform & DT, for this as the "OF" and "non-OF" calls implemented. Signed-off-by: Alexander Shiyan Acked-by: Arnd Bergmann Signed-off-by: Daniel Lezcano --- drivers/clocksource/Makefile | 1 + drivers/clocksource/clps711x-timer.c | 131 +++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 drivers/clocksource/clps711x-timer.c diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile index da1ac09dc771..e40435d271d8 100644 --- a/drivers/clocksource/Makefile +++ b/drivers/clocksource/Makefile @@ -16,6 +16,7 @@ obj-$(CONFIG_CLKSRC_DBX500_PRCMU) += clksrc-dbx500-prcmu.o obj-$(CONFIG_ARMADA_370_XP_TIMER) += time-armada-370-xp.o obj-$(CONFIG_ORION_TIMER) += time-orion.o obj-$(CONFIG_ARCH_BCM2835) += bcm2835_timer.o +obj-$(CONFIG_ARCH_CLPS711X) += clps711x-timer.o obj-$(CONFIG_ARCH_MARCO) += timer-marco.o obj-$(CONFIG_ARCH_MOXART) += moxart_timer.o obj-$(CONFIG_ARCH_MXS) += mxs_timer.o diff --git a/drivers/clocksource/clps711x-timer.c b/drivers/clocksource/clps711x-timer.c new file mode 100644 index 000000000000..d83ec1f2fddc --- /dev/null +++ b/drivers/clocksource/clps711x-timer.c @@ -0,0 +1,131 @@ +/* + * Cirrus Logic CLPS711X clocksource driver + * + * Copyright (C) 2014 Alexander Shiyan + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { + CLPS711X_CLKSRC_CLOCKSOURCE, + CLPS711X_CLKSRC_CLOCKEVENT, +}; + +static void __iomem *tcd; + +static u64 notrace clps711x_sched_clock_read(void) +{ + return ~readw(tcd); +} + +static int __init _clps711x_clksrc_init(struct clk *clock, void __iomem *base) +{ + unsigned long rate; + + if (!base) + return -ENOMEM; + if (IS_ERR(clock)) + return PTR_ERR(clock); + + rate = clk_get_rate(clock); + + tcd = base; + + clocksource_mmio_init(tcd, "clps711x-clocksource", rate, 300, 16, + clocksource_mmio_readw_down); + + sched_clock_register(clps711x_sched_clock_read, 16, rate); + + return 0; +} + +static irqreturn_t clps711x_timer_interrupt(int irq, void *dev_id) +{ + struct clock_event_device *evt = dev_id; + + evt->event_handler(evt); + + return IRQ_HANDLED; +} + +static void clps711x_clockevent_set_mode(enum clock_event_mode mode, + struct clock_event_device *evt) +{ +} + +static int __init _clps711x_clkevt_init(struct clk *clock, void __iomem *base, + unsigned int irq) +{ + struct clock_event_device *clkevt; + unsigned long rate; + + if (!irq) + return -EINVAL; + if (!base) + return -ENOMEM; + if (IS_ERR(clock)) + return PTR_ERR(clock); + + clkevt = kzalloc(sizeof(*clkevt), GFP_KERNEL); + if (!clkevt) + return -ENOMEM; + + rate = clk_get_rate(clock); + + /* Set Timer prescaler */ + writew(DIV_ROUND_CLOSEST(rate, HZ), base); + + clkevt->name = "clps711x-clockevent"; + clkevt->rating = 300; + clkevt->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_C3STOP; + clkevt->set_mode = clps711x_clockevent_set_mode; + clkevt->cpumask = cpumask_of(0); + clockevents_config_and_register(clkevt, HZ, 0, 0); + + return request_irq(irq, clps711x_timer_interrupt, IRQF_TIMER, + "clps711x-timer", clkevt); +} + +void __init clps711x_clksrc_init(void __iomem *tc1_base, void __iomem *tc2_base, + unsigned int irq) +{ + struct clk *tc1 = clk_get_sys("clps711x-timer.0", NULL); + struct clk *tc2 = clk_get_sys("clps711x-timer.1", NULL); + + BUG_ON(_clps711x_clksrc_init(tc1, tc1_base)); + BUG_ON(_clps711x_clkevt_init(tc2, tc2_base, irq)); +} + +#ifdef CONFIG_CLKSRC_OF +static void __init clps711x_timer_init(struct device_node *np) +{ + unsigned int irq = irq_of_parse_and_map(np, 0); + struct clk *clock = of_clk_get(np, 0); + void __iomem *base = of_iomap(np, 0); + + switch (of_alias_get_id(np, "timer")) { + case CLPS711X_CLKSRC_CLOCKSOURCE: + BUG_ON(_clps711x_clksrc_init(clock, base)); + break; + case CLPS711X_CLKSRC_CLOCKEVENT: + BUG_ON(_clps711x_clkevt_init(clock, base, irq)); + break; + default: + break; + } +} +CLOCKSOURCE_OF_DECLARE(clps711x, "cirrus,clps711x-timer", clps711x_timer_init); +#endif From fd944da37fab391cfd80d649b511d777123ee6f9 Mon Sep 17 00:00:00 2001 From: Alexander Shiyan Date: Sun, 13 Jul 2014 08:45:36 +0400 Subject: [PATCH 037/114] clocksource: clps711x: Add DT bindings documentation This patch adds DT binding documentation for the Cirrus Logic CLPS711X-based CPUs clocksource subsystem. Signed-off-by: Alexander Shiyan Acked-by: Arnd Bergmann Signed-off-by: Daniel Lezcano --- .../bindings/timer/cirrus,clps711x-timer.txt | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 Documentation/devicetree/bindings/timer/cirrus,clps711x-timer.txt diff --git a/Documentation/devicetree/bindings/timer/cirrus,clps711x-timer.txt b/Documentation/devicetree/bindings/timer/cirrus,clps711x-timer.txt new file mode 100644 index 000000000000..cd55b52548e4 --- /dev/null +++ b/Documentation/devicetree/bindings/timer/cirrus,clps711x-timer.txt @@ -0,0 +1,29 @@ +* Cirrus Logic CLPS711X Timer Counter + +Required properties: +- compatible: Shall contain "cirrus,clps711x-timer". +- reg : Address and length of the register set. +- interrupts: The interrupt number of the timer. +- clocks : phandle of timer reference clock. + +Note: Each timer should have an alias correctly numbered in "aliases" node. + +Example: + aliases { + timer0 = &timer1; + timer1 = &timer2; + }; + + timer1: timer@80000300 { + compatible = "cirrus,ep7312-timer", "cirrus,clps711x-timer"; + reg = <0x80000300 0x4>; + interrupts = <8>; + clocks = <&clks 5>; + }; + + timer2: timer@80000340 { + compatible = "cirrus,ep7312-timer", "cirrus,clps711x-timer"; + reg = <0x80000340 0x4>; + interrupts = <9>; + clocks = <&clks 6>; + }; From c5421d7aa40965b9527999e65a78f71aec48f19d Mon Sep 17 00:00:00 2001 From: Robert Jarzmik Date: Mon, 14 Jul 2014 18:52:01 +0200 Subject: [PATCH 038/114] clocksource: pxa: Move PXA timer to clocksource framework Move time.c from arch/arm/mach-pxa/time.c to drivers/clocksource/pxa_timer.c. Signed-off-by: Robert Jarzmik Signed-off-by: Daniel Lezcano --- arch/arm/mach-pxa/Makefile | 2 +- drivers/clocksource/Makefile | 1 + arch/arm/mach-pxa/time.c => drivers/clocksource/pxa_timer.c | 0 3 files changed, 2 insertions(+), 1 deletion(-) rename arch/arm/mach-pxa/time.c => drivers/clocksource/pxa_timer.c (100%) diff --git a/arch/arm/mach-pxa/Makefile b/arch/arm/mach-pxa/Makefile index 648867a8caa8..2fe1824c6dcb 100644 --- a/arch/arm/mach-pxa/Makefile +++ b/arch/arm/mach-pxa/Makefile @@ -4,7 +4,7 @@ # Common support (must be linked before board specific support) obj-y += clock.o devices.o generic.o irq.o \ - time.o reset.o + reset.o obj-$(CONFIG_PM) += pm.o sleep.o standby.o # Generic drivers that other drivers may depend upon diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile index e40435d271d8..7fd9fd1dff42 100644 --- a/drivers/clocksource/Makefile +++ b/drivers/clocksource/Makefile @@ -20,6 +20,7 @@ obj-$(CONFIG_ARCH_CLPS711X) += clps711x-timer.o obj-$(CONFIG_ARCH_MARCO) += timer-marco.o obj-$(CONFIG_ARCH_MOXART) += moxart_timer.o obj-$(CONFIG_ARCH_MXS) += mxs_timer.o +obj-$(CONFIG_ARCH_PXA) += pxa_timer.o obj-$(CONFIG_ARCH_PRIMA2) += timer-prima2.o obj-$(CONFIG_ARCH_U300) += timer-u300.o obj-$(CONFIG_SUN4I_TIMER) += sun4i_timer.o diff --git a/arch/arm/mach-pxa/time.c b/drivers/clocksource/pxa_timer.c similarity index 100% rename from arch/arm/mach-pxa/time.c rename to drivers/clocksource/pxa_timer.c From ab5354c48d58a6a72e70d37d2032206eba3b1dbb Mon Sep 17 00:00:00 2001 From: Robert Jarzmik Date: Mon, 14 Jul 2014 18:52:02 +0200 Subject: [PATCH 039/114] clocksource: pxa: Add device-tree support for PXA timer Add device-tree support to PXA platforms. The driver still needs to maintain backward non device-tree compatibility as well, which implies : - a non device-tree init function - a static registers base address in the driver Signed-off-by: Robert Jarzmik Signed-off-by: Daniel Lezcano --- drivers/clocksource/pxa_timer.c | 137 +++++++++++++++++++++++--------- 1 file changed, 101 insertions(+), 36 deletions(-) diff --git a/drivers/clocksource/pxa_timer.c b/drivers/clocksource/pxa_timer.c index fca174e3865d..941f3f344e08 100644 --- a/drivers/clocksource/pxa_timer.c +++ b/drivers/clocksource/pxa_timer.c @@ -15,14 +15,30 @@ #include #include #include +#include #include +#include +#include #include #include -#include -#include -#include -#include + +#define OSMR0 0x00 /* OS Timer 0 Match Register */ +#define OSMR1 0x04 /* OS Timer 1 Match Register */ +#define OSMR2 0x08 /* OS Timer 2 Match Register */ +#define OSMR3 0x0C /* OS Timer 3 Match Register */ + +#define OSCR 0x10 /* OS Timer Counter Register */ +#define OSSR 0x14 /* OS Timer Status Register */ +#define OWER 0x18 /* OS Timer Watchdog Enable Register */ +#define OIER 0x1C /* OS Timer Interrupt Enable Register */ + +#define OSSR_M3 (1 << 3) /* Match status channel 3 */ +#define OSSR_M2 (1 << 2) /* Match status channel 2 */ +#define OSSR_M1 (1 << 1) /* Match status channel 1 */ +#define OSSR_M0 (1 << 0) /* Match status channel 0 */ + +#define OIER_E0 (1 << 0) /* Interrupt enable channel 0 */ /* * This is PXA's sched_clock implementation. This has a resolution @@ -33,9 +49,14 @@ * calls to sched_clock() which should always be the case in practice. */ +#define timer_readl(reg) readl_relaxed(timer_base + (reg)) +#define timer_writel(val, reg) writel_relaxed((val), timer_base + (reg)) + +static void __iomem *timer_base; + static u64 notrace pxa_read_sched_clock(void) { - return readl_relaxed(OSCR); + return timer_readl(OSCR); } @@ -47,8 +68,8 @@ pxa_ost0_interrupt(int irq, void *dev_id) struct clock_event_device *c = dev_id; /* Disarm the compare/match, signal the event. */ - writel_relaxed(readl_relaxed(OIER) & ~OIER_E0, OIER); - writel_relaxed(OSSR_M0, OSSR); + timer_writel(timer_readl(OIER) & ~OIER_E0, OIER); + timer_writel(OSSR_M0, OSSR); c->event_handler(c); return IRQ_HANDLED; @@ -59,10 +80,10 @@ pxa_osmr0_set_next_event(unsigned long delta, struct clock_event_device *dev) { unsigned long next, oscr; - writel_relaxed(readl_relaxed(OIER) | OIER_E0, OIER); - next = readl_relaxed(OSCR) + delta; - writel_relaxed(next, OSMR0); - oscr = readl_relaxed(OSCR); + timer_writel(timer_readl(OIER) | OIER_E0, OIER); + next = timer_readl(OSCR) + delta; + timer_writel(next, OSMR0); + oscr = timer_readl(OSCR); return (signed)(next - oscr) <= MIN_OSCR_DELTA ? -ETIME : 0; } @@ -72,15 +93,15 @@ pxa_osmr0_set_mode(enum clock_event_mode mode, struct clock_event_device *dev) { switch (mode) { case CLOCK_EVT_MODE_ONESHOT: - writel_relaxed(readl_relaxed(OIER) & ~OIER_E0, OIER); - writel_relaxed(OSSR_M0, OSSR); + timer_writel(timer_readl(OIER) & ~OIER_E0, OIER); + timer_writel(OSSR_M0, OSSR); break; case CLOCK_EVT_MODE_UNUSED: case CLOCK_EVT_MODE_SHUTDOWN: /* initializing, released, or preparing for suspend */ - writel_relaxed(readl_relaxed(OIER) & ~OIER_E0, OIER); - writel_relaxed(OSSR_M0, OSSR); + timer_writel(timer_readl(OIER) & ~OIER_E0, OIER); + timer_writel(OSSR_M0, OSSR); break; case CLOCK_EVT_MODE_RESUME: @@ -94,12 +115,12 @@ static unsigned long osmr[4], oier, oscr; static void pxa_timer_suspend(struct clock_event_device *cedev) { - osmr[0] = readl_relaxed(OSMR0); - osmr[1] = readl_relaxed(OSMR1); - osmr[2] = readl_relaxed(OSMR2); - osmr[3] = readl_relaxed(OSMR3); - oier = readl_relaxed(OIER); - oscr = readl_relaxed(OSCR); + osmr[0] = timer_readl(OSMR0); + osmr[1] = timer_readl(OSMR1); + osmr[2] = timer_readl(OSMR2); + osmr[3] = timer_readl(OSMR3); + oier = timer_readl(OIER); + oscr = timer_readl(OSCR); } static void pxa_timer_resume(struct clock_event_device *cedev) @@ -113,12 +134,12 @@ static void pxa_timer_resume(struct clock_event_device *cedev) if (osmr[0] - oscr < MIN_OSCR_DELTA) osmr[0] += MIN_OSCR_DELTA; - writel_relaxed(osmr[0], OSMR0); - writel_relaxed(osmr[1], OSMR1); - writel_relaxed(osmr[2], OSMR2); - writel_relaxed(osmr[3], OSMR3); - writel_relaxed(oier, OIER); - writel_relaxed(oscr, OSCR); + timer_writel(osmr[0], OSMR0); + timer_writel(osmr[1], OSMR1); + timer_writel(osmr[2], OSMR2); + timer_writel(osmr[3], OSMR3); + timer_writel(oier, OIER); + timer_writel(oscr, OSCR); } #else #define pxa_timer_suspend NULL @@ -142,21 +163,65 @@ static struct irqaction pxa_ost0_irq = { .dev_id = &ckevt_pxa_osmr0, }; -void __init pxa_timer_init(void) +static void pxa_timer_common_init(int irq, unsigned long clock_tick_rate) { - unsigned long clock_tick_rate = get_clock_tick_rate(); - - writel_relaxed(0, OIER); - writel_relaxed(OSSR_M0 | OSSR_M1 | OSSR_M2 | OSSR_M3, OSSR); + timer_writel(0, OIER); + timer_writel(OSSR_M0 | OSSR_M1 | OSSR_M2 | OSSR_M3, OSSR); sched_clock_register(pxa_read_sched_clock, 32, clock_tick_rate); ckevt_pxa_osmr0.cpumask = cpumask_of(0); - setup_irq(IRQ_OST0, &pxa_ost0_irq); + setup_irq(irq, &pxa_ost0_irq); - clocksource_mmio_init(OSCR, "oscr0", clock_tick_rate, 200, 32, - clocksource_mmio_readl_up); + clocksource_mmio_init(timer_base + OSCR, "oscr0", clock_tick_rate, 200, + 32, clocksource_mmio_readl_up); clockevents_config_and_register(&ckevt_pxa_osmr0, clock_tick_rate, - MIN_OSCR_DELTA * 2, 0x7fffffff); + MIN_OSCR_DELTA * 2, 0x7fffffff); +} + +static void __init pxa_timer_dt_init(struct device_node *np) +{ + struct clk *clk; + int irq; + + /* timer registers are shared with watchdog timer */ + timer_base = of_iomap(np, 0); + if (!timer_base) + panic("%s: unable to map resource\n", np->name); + + clk = of_clk_get(np, 0); + if (IS_ERR(clk)) { + pr_crit("%s: unable to get clk\n", np->name); + return; + } + clk_prepare_enable(clk); + + /* we are only interested in OS-timer0 irq */ + irq = irq_of_parse_and_map(np, 0); + if (irq <= 0) { + pr_crit("%s: unable to parse OS-timer0 irq\n", np->name); + return; + } + + pxa_timer_common_init(irq, clk_get_rate(clk)); +} +CLOCKSOURCE_OF_DECLARE(pxa_timer, "marvell,pxa-timer", pxa_timer_dt_init); + +/* + * Legacy timer init for non device-tree boards. + */ +void __init pxa_timer_nodt_init(int irq, void __iomem *base, + unsigned long clock_tick_rate) +{ + struct clk *clk; + + timer_base = base; + clk = clk_get(NULL, "OSTIMER0"); + if (clk && !IS_ERR(clk)) + clk_prepare_enable(clk); + else + pr_crit("%s: unable to get clk\n", __func__); + + pxa_timer_common_init(irq, clock_tick_rate); } From 6f6caeaa9a27eb97a15a707192db601a8ef6c272 Mon Sep 17 00:00:00 2001 From: Robert Jarzmik Date: Mon, 14 Jul 2014 18:52:03 +0200 Subject: [PATCH 040/114] ARM: pxa: Add CLKSRC_OF dependency Select CLKSRC_OF for PXA architectures. Signed-off-by: Robert Jarzmik Signed-off-by: Daniel Lezcano --- arch/arm/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 245058b3b0ef..05a71511ab3c 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -634,6 +634,7 @@ config ARCH_PXA select AUTO_ZRELADDR select CLKDEV_LOOKUP select CLKSRC_MMIO + select CLKSRC_OF select GENERIC_CLOCKEVENTS select GPIO_PXA select HAVE_IDE From a38b1f60b5245a3f610baac2019c0ecd8abd8752 Mon Sep 17 00:00:00 2001 From: Robert Jarzmik Date: Mon, 14 Jul 2014 18:52:04 +0200 Subject: [PATCH 041/114] ARM: pxa: Add non device-tree timer link to clocksource As clocksource pxa_timer was moved to clocksource framework, the pxa_timer initialization needs to be a bit amended, to pass the necessary informations to clocksource, ie : - the timer interrupt (mach specific) - the timer registers base (ditto) - the timer clockrate Signed-off-by: Robert Jarzmik Signed-off-by: Daniel Lezcano --- arch/arm/mach-pxa/generic.c | 11 +++++++++++ include/clocksource/pxa.h | 18 ++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 include/clocksource/pxa.h diff --git a/arch/arm/mach-pxa/generic.c b/arch/arm/mach-pxa/generic.c index 42254175fcf4..6f38e1af45af 100644 --- a/arch/arm/mach-pxa/generic.c +++ b/arch/arm/mach-pxa/generic.c @@ -25,11 +25,13 @@ #include #include +#include #include #include #include #include "generic.h" +#include void clear_reset_status(unsigned int mask) { @@ -56,6 +58,15 @@ unsigned long get_clock_tick_rate(void) } EXPORT_SYMBOL(get_clock_tick_rate); +/* + * For non device-tree builds, keep legacy timer init + */ +void pxa_timer_init(void) +{ + pxa_timer_nodt_init(IRQ_OST0, io_p2v(0x40a00000), + get_clock_tick_rate()); +} + /* * Get the clock frequency as reflected by CCCR and the turbo flag. * We assume these values have been applied via a fcs. diff --git a/include/clocksource/pxa.h b/include/clocksource/pxa.h new file mode 100644 index 000000000000..1efbe5a66958 --- /dev/null +++ b/include/clocksource/pxa.h @@ -0,0 +1,18 @@ +/* + * PXA clocksource, clockevents, and OST interrupt handlers. + * + * Copyright (C) 2014 Robert Jarzmik + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + */ + +#ifndef _CLOCKSOURCE_PXA_H +#define _CLOCKSOURCE_PXA_H + +extern void pxa_timer_nodt_init(int irq, void __iomem *base, + unsigned long clock_tick_rate); + +#endif From fdb06f66d53e3c9ba7eeab3c0629c450aee76937 Mon Sep 17 00:00:00 2001 From: Doug Anderson Date: Sat, 5 Jul 2014 06:43:20 +0900 Subject: [PATCH 042/114] clocksource: exynos_mct: Use readl_relaxed/writel_relaxed Using the __raw functions is discouraged. Update the file to consistently use the proper functions. Signed-off-by: Doug Anderson Signed-off-by: Kukjin Kim Signed-off-by: Daniel Lezcano --- drivers/clocksource/exynos_mct.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/clocksource/exynos_mct.c b/drivers/clocksource/exynos_mct.c index ab51bf20a3ed..2df03e238c1b 100644 --- a/drivers/clocksource/exynos_mct.c +++ b/drivers/clocksource/exynos_mct.c @@ -94,7 +94,7 @@ static void exynos4_mct_write(unsigned int value, unsigned long offset) u32 mask; u32 i; - __raw_writel(value, reg_base + offset); + writel_relaxed(value, reg_base + offset); if (likely(offset >= EXYNOS4_MCT_L_BASE(0))) { stat_addr = (offset & ~EXYNOS4_MCT_L_MASK) + MCT_L_WSTAT_OFFSET; @@ -144,8 +144,8 @@ static void exynos4_mct_write(unsigned int value, unsigned long offset) /* Wait maximum 1 ms until written values are applied */ for (i = 0; i < loops_per_jiffy / 1000 * HZ; i++) - if (__raw_readl(reg_base + stat_addr) & mask) { - __raw_writel(mask, reg_base + stat_addr); + if (readl_relaxed(reg_base + stat_addr) & mask) { + writel_relaxed(mask, reg_base + stat_addr); return; } @@ -157,7 +157,7 @@ static void exynos4_mct_frc_start(void) { u32 reg; - reg = __raw_readl(reg_base + EXYNOS4_MCT_G_TCON); + reg = readl_relaxed(reg_base + EXYNOS4_MCT_G_TCON); reg |= MCT_G_TCON_START; exynos4_mct_write(reg, EXYNOS4_MCT_G_TCON); } @@ -165,12 +165,12 @@ static void exynos4_mct_frc_start(void) static cycle_t notrace _exynos4_frc_read(void) { unsigned int lo, hi; - u32 hi2 = __raw_readl(reg_base + EXYNOS4_MCT_G_CNT_U); + u32 hi2 = readl_relaxed(reg_base + EXYNOS4_MCT_G_CNT_U); do { hi = hi2; - lo = __raw_readl(reg_base + EXYNOS4_MCT_G_CNT_L); - hi2 = __raw_readl(reg_base + EXYNOS4_MCT_G_CNT_U); + lo = readl_relaxed(reg_base + EXYNOS4_MCT_G_CNT_L); + hi2 = readl_relaxed(reg_base + EXYNOS4_MCT_G_CNT_U); } while (hi != hi2); return ((cycle_t)hi << 32) | lo; @@ -225,7 +225,7 @@ static void exynos4_mct_comp0_stop(void) { unsigned int tcon; - tcon = __raw_readl(reg_base + EXYNOS4_MCT_G_TCON); + tcon = readl_relaxed(reg_base + EXYNOS4_MCT_G_TCON); tcon &= ~(MCT_G_TCON_COMP0_ENABLE | MCT_G_TCON_COMP0_AUTO_INC); exynos4_mct_write(tcon, EXYNOS4_MCT_G_TCON); @@ -238,7 +238,7 @@ static void exynos4_mct_comp0_start(enum clock_event_mode mode, unsigned int tcon; cycle_t comp_cycle; - tcon = __raw_readl(reg_base + EXYNOS4_MCT_G_TCON); + tcon = readl_relaxed(reg_base + EXYNOS4_MCT_G_TCON); if (mode == CLOCK_EVT_MODE_PERIODIC) { tcon |= MCT_G_TCON_COMP0_AUTO_INC; @@ -327,7 +327,7 @@ static void exynos4_mct_tick_stop(struct mct_clock_event_device *mevt) unsigned long mask = MCT_L_TCON_INT_START | MCT_L_TCON_TIMER_START; unsigned long offset = mevt->base + MCT_L_TCON_OFFSET; - tmp = __raw_readl(reg_base + offset); + tmp = readl_relaxed(reg_base + offset); if (tmp & mask) { tmp &= ~mask; exynos4_mct_write(tmp, offset); @@ -349,7 +349,7 @@ static void exynos4_mct_tick_start(unsigned long cycles, /* enable MCT tick interrupt */ exynos4_mct_write(0x1, mevt->base + MCT_L_INT_ENB_OFFSET); - tmp = __raw_readl(reg_base + mevt->base + MCT_L_TCON_OFFSET); + tmp = readl_relaxed(reg_base + mevt->base + MCT_L_TCON_OFFSET); tmp |= MCT_L_TCON_INT_START | MCT_L_TCON_TIMER_START | MCT_L_TCON_INTERVAL_MODE; exynos4_mct_write(tmp, mevt->base + MCT_L_TCON_OFFSET); @@ -401,7 +401,7 @@ static int exynos4_mct_tick_clear(struct mct_clock_event_device *mevt) exynos4_mct_tick_stop(mevt); /* Clear the MCT tick interrupt */ - if (__raw_readl(reg_base + mevt->base + MCT_L_INT_CSTAT_OFFSET) & 1) { + if (readl_relaxed(reg_base + mevt->base + MCT_L_INT_CSTAT_OFFSET) & 1) { exynos4_mct_write(0x1, mevt->base + MCT_L_INT_CSTAT_OFFSET); return 1; } else { From 3252a646aa2cf706b2a26433a8bd9cb2e5dce410 Mon Sep 17 00:00:00 2001 From: Doug Anderson Date: Sat, 5 Jul 2014 06:43:26 +0900 Subject: [PATCH 043/114] clocksource: exynos_mct: Only use 32-bits where possible The MCT has a nice 64-bit counter. That means that we _can_ register as a 64-bit clocksource and sched_clock. ...but that doesn't mean we should. The 64-bit counter is read by reading two 32-bit registers. That means reading needs to be something like: - Read upper half - Read lower half - Read upper half and confirm that it hasn't changed. That wouldn't be terrible, but: - THe MCT isn't very fast to access (hundreds of nanoseconds). - The clocksource is queried _all the time_. In total system profiles of real workloads on ChromeOS, we've seen exynos_frc_read() taking 2% or more of CPU time even after optimizing the 3 reads above to 2 (see below). The MCT is clocked at ~24MHz on all known systems. That means that the 32-bit half of the counter rolls over every ~178 seconds. This inspired an optimization in ChromeOS to cache the upper half between calls, moving 3 reads to 2. ...but we can do better! Having a 32-bit timer that flips every 178 seconds is more than sufficient for Linux. Let's just use the lower half of the MCT. Times on 5420 to do 1000000 gettimeofday() calls from userspace: * Original code: 1323852 us * ChromeOS cache upper half: 1173084 us * ChromeOS + ldmia to optimize: 1045674 us * Use lower 32-bit only (this code): 1014429 us As you can see, the time used doesn't increase linearly with the number of reads and we can make 64-bit work almost as fast as 32-bit with a bit of assembly code. But since there's no real gain for 64-bit, let's go with the simplest and fastest implementation. Note: with this change roughly half the time for gettimeofday() is spent in exynos_frc_read(). The rest is timer / system call overhead. Also note: this patch disables the use of the MCT on ARM64 systems until we've sorted out how to make "cycles_t" always 32-bit. Really ARM64 systems should be using arch timers anyway. Signed-off-by: Doug Anderson Acked-by Vincent Guittot Signed-off-by: Kukjin Kim Signed-off-by: Daniel Lezcano --- drivers/clocksource/Kconfig | 1 + drivers/clocksource/exynos_mct.c | 39 ++++++++++++++++++++++++++------ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index 7ea7d7c48d72..cfd6519df661 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -127,6 +127,7 @@ config CLKSRC_METAG_GENERIC config CLKSRC_EXYNOS_MCT def_bool y if ARCH_EXYNOS + depends on !ARM64 help Support for Multi Core Timer controller on Exynos SoCs. diff --git a/drivers/clocksource/exynos_mct.c b/drivers/clocksource/exynos_mct.c index 2df03e238c1b..9403061a2acc 100644 --- a/drivers/clocksource/exynos_mct.c +++ b/drivers/clocksource/exynos_mct.c @@ -162,7 +162,17 @@ static void exynos4_mct_frc_start(void) exynos4_mct_write(reg, EXYNOS4_MCT_G_TCON); } -static cycle_t notrace _exynos4_frc_read(void) +/** + * exynos4_read_count_64 - Read all 64-bits of the global counter + * + * This will read all 64-bits of the global counter taking care to make sure + * that the upper and lower half match. Note that reading the MCT can be quite + * slow (hundreds of nanoseconds) so you should use the 32-bit (lower half + * only) version when possible. + * + * Returns the number of cycles in the global counter. + */ +static u64 exynos4_read_count_64(void) { unsigned int lo, hi; u32 hi2 = readl_relaxed(reg_base + EXYNOS4_MCT_G_CNT_U); @@ -176,9 +186,22 @@ static cycle_t notrace _exynos4_frc_read(void) return ((cycle_t)hi << 32) | lo; } +/** + * exynos4_read_count_32 - Read the lower 32-bits of the global counter + * + * This will read just the lower 32-bits of the global counter. This is marked + * as notrace so it can be used by the scheduler clock. + * + * Returns the number of cycles in the global counter (lower 32 bits). + */ +static u32 notrace exynos4_read_count_32(void) +{ + return readl_relaxed(reg_base + EXYNOS4_MCT_G_CNT_L); +} + static cycle_t exynos4_frc_read(struct clocksource *cs) { - return _exynos4_frc_read(); + return exynos4_read_count_32(); } static void exynos4_frc_resume(struct clocksource *cs) @@ -190,21 +213,23 @@ struct clocksource mct_frc = { .name = "mct-frc", .rating = 400, .read = exynos4_frc_read, - .mask = CLOCKSOURCE_MASK(64), + .mask = CLOCKSOURCE_MASK(32), .flags = CLOCK_SOURCE_IS_CONTINUOUS, .resume = exynos4_frc_resume, }; static u64 notrace exynos4_read_sched_clock(void) { - return _exynos4_frc_read(); + return exynos4_read_count_32(); } static struct delay_timer exynos4_delay_timer; static cycles_t exynos4_read_current_timer(void) { - return _exynos4_frc_read(); + BUILD_BUG_ON_MSG(sizeof(cycles_t) != sizeof(u32), + "cycles_t needs to move to 32-bit for ARM64 usage"); + return exynos4_read_count_32(); } static void __init exynos4_clocksource_init(void) @@ -218,7 +243,7 @@ static void __init exynos4_clocksource_init(void) if (clocksource_register_hz(&mct_frc, clk_rate)) panic("%s: can't register clocksource\n", mct_frc.name); - sched_clock_register(exynos4_read_sched_clock, 64, clk_rate); + sched_clock_register(exynos4_read_sched_clock, 32, clk_rate); } static void exynos4_mct_comp0_stop(void) @@ -245,7 +270,7 @@ static void exynos4_mct_comp0_start(enum clock_event_mode mode, exynos4_mct_write(cycles, EXYNOS4_MCT_G_COMP0_ADD_INCR); } - comp_cycle = exynos4_frc_read(&mct_frc) + cycles; + comp_cycle = exynos4_read_count_64() + cycles; exynos4_mct_write((u32)comp_cycle, EXYNOS4_MCT_G_COMP0_L); exynos4_mct_write((u32)(comp_cycle >> 32), EXYNOS4_MCT_G_COMP0_U); From e704f93af5a083c07b8f722672d63a1d908daf55 Mon Sep 17 00:00:00 2001 From: David Riley Date: Mon, 16 Jun 2014 14:58:32 -0700 Subject: [PATCH 044/114] kernel: time: Add udelay_test module to validate udelay Create a module that allows udelay() to be executed to ensure that it is delaying at least as long as requested (with a little bit of error allowed). There are some configurations which don't have reliably udelay due to using a loop delay with cpufreq changes which should use a counter time based delay instead. This test aims to identify those configurations where timing is unreliable. Signed-off-by: David Riley Signed-off-by: John Stultz --- kernel/time/Makefile | 2 + kernel/time/udelay_test.c | 168 ++++++++++++++++++++++++++++++++++++++ lib/Kconfig.debug | 9 ++ 3 files changed, 179 insertions(+) create mode 100644 kernel/time/udelay_test.c diff --git a/kernel/time/Makefile b/kernel/time/Makefile index e59ce8b1b550..7347426fa68d 100644 --- a/kernel/time/Makefile +++ b/kernel/time/Makefile @@ -13,6 +13,7 @@ obj-$(CONFIG_TICK_ONESHOT) += tick-oneshot.o obj-$(CONFIG_TICK_ONESHOT) += tick-sched.o obj-$(CONFIG_TIMER_STATS) += timer_stats.o obj-$(CONFIG_DEBUG_FS) += timekeeping_debug.o +obj-$(CONFIG_TEST_UDELAY) += udelay_test.o $(obj)/time.o: $(obj)/timeconst.h @@ -29,3 +30,4 @@ quiet_cmd_bc = BC $@ targets += timeconst.h $(obj)/timeconst.h: $(obj)/hz.bc $(src)/timeconst.bc FORCE $(call if_changed,bc) + diff --git a/kernel/time/udelay_test.c b/kernel/time/udelay_test.c new file mode 100644 index 000000000000..e622ba365a13 --- /dev/null +++ b/kernel/time/udelay_test.c @@ -0,0 +1,168 @@ +/* + * udelay() test kernel module + * + * Test is executed by writing and reading to /sys/kernel/debug/udelay_test + * Tests are configured by writing: USECS ITERATIONS + * Tests are executed by reading from the same file. + * Specifying usecs of 0 or negative values will run multiples tests. + * + * Copyright (C) 2014 Google, Inc. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include + +#define DEFAULT_ITERATIONS 100 + +#define DEBUGFS_FILENAME "udelay_test" + +static DEFINE_MUTEX(udelay_test_lock); +static struct dentry *udelay_test_debugfs_file; +static int udelay_test_usecs; +static int udelay_test_iterations = DEFAULT_ITERATIONS; + +static int udelay_test_single(struct seq_file *s, int usecs, uint32_t iters) +{ + int min = 0, max = 0, fail_count = 0; + uint64_t sum = 0; + uint64_t avg; + int i; + /* Allow udelay to be up to 0.5% fast */ + int allowed_error_ns = usecs * 5; + + for (i = 0; i < iters; ++i) { + struct timespec ts1, ts2; + int time_passed; + + ktime_get_ts(&ts1); + udelay(usecs); + ktime_get_ts(&ts2); + time_passed = timespec_to_ns(&ts2) - timespec_to_ns(&ts1); + + if (i == 0 || time_passed < min) + min = time_passed; + if (i == 0 || time_passed > max) + max = time_passed; + if ((time_passed + allowed_error_ns) / 1000 < usecs) + ++fail_count; + WARN_ON(time_passed < 0); + sum += time_passed; + } + + avg = sum; + do_div(avg, iters); + seq_printf(s, "%d usecs x %d: exp=%d allowed=%d min=%d avg=%lld max=%d", + usecs, iters, usecs * 1000, + (usecs * 1000) - allowed_error_ns, min, avg, max); + if (fail_count) + seq_printf(s, " FAIL=%d", fail_count); + seq_puts(s, "\n"); + + return 0; +} + +static int udelay_test_show(struct seq_file *s, void *v) +{ + int usecs; + int iters; + int ret = 0; + + mutex_lock(&udelay_test_lock); + usecs = udelay_test_usecs; + iters = udelay_test_iterations; + mutex_unlock(&udelay_test_lock); + + if (usecs > 0 && iters > 0) { + return udelay_test_single(s, usecs, iters); + } else if (usecs == 0) { + struct timespec ts; + + ktime_get_ts(&ts); + seq_printf(s, "udelay() test (lpj=%ld kt=%ld.%09ld)\n", + loops_per_jiffy, ts.tv_sec, ts.tv_nsec); + seq_puts(s, "usage:\n"); + seq_puts(s, "echo USECS [ITERS] > " DEBUGFS_FILENAME "\n"); + seq_puts(s, "cat " DEBUGFS_FILENAME "\n"); + } + + return ret; +} + +static int udelay_test_open(struct inode *inode, struct file *file) +{ + return single_open(file, udelay_test_show, inode->i_private); +} + +static ssize_t udelay_test_write(struct file *file, const char __user *buf, + size_t count, loff_t *pos) +{ + char lbuf[32]; + int ret; + int usecs; + int iters; + + if (count >= sizeof(lbuf)) + return -EINVAL; + + if (copy_from_user(lbuf, buf, count)) + return -EFAULT; + lbuf[count] = '\0'; + + ret = sscanf(lbuf, "%d %d", &usecs, &iters); + if (ret < 1) + return -EINVAL; + else if (ret < 2) + iters = DEFAULT_ITERATIONS; + + mutex_lock(&udelay_test_lock); + udelay_test_usecs = usecs; + udelay_test_iterations = iters; + mutex_unlock(&udelay_test_lock); + + return count; +} + +static const struct file_operations udelay_test_debugfs_ops = { + .owner = THIS_MODULE, + .open = udelay_test_open, + .read = seq_read, + .write = udelay_test_write, + .llseek = seq_lseek, + .release = single_release, +}; + +static int __init udelay_test_init(void) +{ + mutex_lock(&udelay_test_lock); + udelay_test_debugfs_file = debugfs_create_file(DEBUGFS_FILENAME, + S_IRUSR, NULL, NULL, &udelay_test_debugfs_ops); + mutex_unlock(&udelay_test_lock); + + return 0; +} + +module_init(udelay_test_init); + +static void __exit udelay_test_exit(void) +{ + mutex_lock(&udelay_test_lock); + debugfs_remove(udelay_test_debugfs_file); + mutex_unlock(&udelay_test_lock); +} + +module_exit(udelay_test_exit); + +MODULE_AUTHOR("David Riley "); +MODULE_LICENSE("GPL"); diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 7a638aa3545b..24a26ad5c99d 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1649,6 +1649,15 @@ config TEST_BPF If unsure, say N. +config TEST_UDELAY + tristate "udelay test driver" + default n + help + This builds the "udelay_test" module that helps to make sure + that udelay() is working properly. + + If unsure, say N. + source "samples/Kconfig" source "lib/Kconfig.kgdb" From 988b0c541ed8b1c633c4d4df7169010635942e18 Mon Sep 17 00:00:00 2001 From: David Riley Date: Mon, 16 Jun 2014 14:58:33 -0700 Subject: [PATCH 045/114] tools: add script to test udelay This script makes use of the udelay_test module to exercise udelay() and ensure that it is delaying long enough (as compared to ktime). Signed-off-by: David Riley Signed-off-by: John Stultz --- tools/time/udelay_test.sh | 66 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100755 tools/time/udelay_test.sh diff --git a/tools/time/udelay_test.sh b/tools/time/udelay_test.sh new file mode 100755 index 000000000000..12d46b926917 --- /dev/null +++ b/tools/time/udelay_test.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# udelay() test script +# +# Test is executed by writing and reading to /sys/kernel/debug/udelay_test +# and exercises a variety of delays to ensure that udelay() is delaying +# at least as long as requested (as compared to ktime). +# +# Copyright (C) 2014 Google, Inc. +# +# This software is licensed under the terms of the GNU General Public +# License version 2, as published by the Free Software Foundation, and +# may be copied, distributed, and modified under those terms. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +MODULE_NAME=udelay_test +UDELAY_PATH=/sys/kernel/debug/udelay_test + +setup() +{ + /sbin/modprobe -q $MODULE_NAME + tmp_file=`mktemp` +} + +test_one() +{ + delay=$1 + echo $delay > $UDELAY_PATH + tee -a $tmp_file < $UDELAY_PATH +} + +cleanup() +{ + if [ -f $tmp_file ]; then + rm $tmp_file + fi + /sbin/modprobe -q -r $MODULE_NAME +} + +trap cleanup EXIT +setup + +# Delay for a variety of times. +# 1..200, 200..500 (by 10), 500..2000 (by 100) +for (( delay = 1; delay < 200; delay += 1 )); do + test_one $delay +done +for (( delay = 200; delay < 500; delay += 10 )); do + test_one $delay +done +for (( delay = 500; delay <= 2000; delay += 100 )); do + test_one $delay +done + +# Search for failures +count=`grep -c FAIL $tmp_file` +if [ $? -eq "0" ]; then + echo "ERROR: $count delays failed to delay long enough" + retcode=1 +fi + +exit $retcode From dc01c9fae1c5e40458c086a868d2028dfd6faebd Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:03:49 +0000 Subject: [PATCH 046/114] tile: Convert VDSO timekeeping to the precise mechanism The code was only halfarsed converted to the new VSDO update mechanism and still uses the inaccurate base value which lacks the fractional part of xtime_nsec. Fix it up. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- arch/tile/kernel/time.c | 9 ++++----- arch/tile/kernel/vdso/vgettimeofday.c | 7 ++++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/tile/kernel/time.c b/arch/tile/kernel/time.c index 462dcd0c1700..ae70155c2f16 100644 --- a/arch/tile/kernel/time.c +++ b/arch/tile/kernel/time.c @@ -260,7 +260,6 @@ void update_vsyscall_tz(void) void update_vsyscall(struct timekeeper *tk) { - struct timespec wall_time = tk_xtime(tk); struct timespec *wtm = &tk->wall_to_monotonic; struct clocksource *clock = tk->clock; @@ -271,12 +270,12 @@ void update_vsyscall(struct timekeeper *tk) ++vdso_data->tb_update_count; smp_wmb(); vdso_data->xtime_tod_stamp = clock->cycle_last; - vdso_data->xtime_clock_sec = wall_time.tv_sec; - vdso_data->xtime_clock_nsec = wall_time.tv_nsec; + vdso_data->xtime_clock_sec = tk->xtime_sec; + vdso_data->xtime_clock_nsec = tk->xtime_nsec; vdso_data->wtom_clock_sec = wtm->tv_sec; vdso_data->wtom_clock_nsec = wtm->tv_nsec; - vdso_data->mult = clock->mult; - vdso_data->shift = clock->shift; + vdso_data->mult = tk->mult; + vdso_data->shift = tk->shift; smp_wmb(); ++vdso_data->tb_update_count; } diff --git a/arch/tile/kernel/vdso/vgettimeofday.c b/arch/tile/kernel/vdso/vgettimeofday.c index 51ec8e46f5f9..e933fb9fbf5c 100644 --- a/arch/tile/kernel/vdso/vgettimeofday.c +++ b/arch/tile/kernel/vdso/vgettimeofday.c @@ -83,10 +83,11 @@ int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz) if (count & 1) continue; - cycles = (get_cycles() - vdso_data->xtime_tod_stamp); - ns = (cycles * vdso_data->mult) >> vdso_data->shift; sec = vdso_data->xtime_clock_sec; - ns += vdso_data->xtime_clock_nsec; + cycles = get_cycles() - vdso_data->xtime_tod_stamp; + ns = (cycles * vdso_data->mult) + vdso_data->xtime_clock_nsec; + ns >>= vdso_data->shift; + if (ns >= NSEC_PER_SEC) { ns -= NSEC_PER_SEC; sec += 1; From e06fde37b860f5030e93475a2a95857af7ad13e1 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:03:50 +0000 Subject: [PATCH 047/114] timekeeping: Simplify arch_gettimeoffset() Provide a default stub function instead of having the extra conditional. Cuts binary size on a m68k build by ~100 bytes. Signed-off-by: Thomas Gleixner Acked-by: Geert Uytterhoeven Signed-off-by: John Stultz --- kernel/time/timekeeping.c | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 32d8d6aaedb8..908861c58e62 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -153,16 +153,10 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) /* Timekeeper helper functions. */ #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET -u32 (*arch_gettimeoffset)(void); - -u32 get_arch_timeoffset(void) -{ - if (likely(arch_gettimeoffset)) - return arch_gettimeoffset(); - return 0; -} +static u32 default_arch_gettimeoffset(void) { return 0; } +u32 (*arch_gettimeoffset)(void) = default_arch_gettimeoffset; #else -static inline u32 get_arch_timeoffset(void) { return 0; } +static inline u32 arch_gettimeoffset(void) { return 0; } #endif static inline s64 timekeeping_get_ns(struct timekeeper *tk) @@ -182,7 +176,7 @@ static inline s64 timekeeping_get_ns(struct timekeeper *tk) nsec >>= tk->shift; /* If arch requires, add in get_arch_timeoffset() */ - return nsec + get_arch_timeoffset(); + return nsec + arch_gettimeoffset(); } static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk) @@ -202,7 +196,7 @@ static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk) nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift); /* If arch requires, add in get_arch_timeoffset() */ - return nsec + get_arch_timeoffset(); + return nsec + arch_gettimeoffset(); } static RAW_NOTIFIER_HEAD(pvclock_gtod_chain); @@ -282,7 +276,7 @@ static void timekeeping_forward_now(struct timekeeper *tk) tk->xtime_nsec += cycle_delta * tk->mult; /* If arch requires, add in get_arch_timeoffset() */ - tk->xtime_nsec += (u64)get_arch_timeoffset() << tk->shift; + tk->xtime_nsec += (u64)arch_gettimeoffset() << tk->shift; tk_normalize_xtime(tk); From 76f4108892d9a9e3408bba839914f97a54086a6f Mon Sep 17 00:00:00 2001 From: John Stultz Date: Wed, 16 Jul 2014 21:03:52 +0000 Subject: [PATCH 048/114] hrtimer: Cleanup hrtimer accessors to the timekepeing state Rather then having two similar but totally different implementations that provide timekeeping state to the hrtimer code, try to unify the two implementations to be more simliar. Thus this clarifies ktime_get_update_offsets to ktime_get_update_offsets_now and changes get_xtime... to ktime_get_update_offsets_tick. Signed-off-by: John Stultz Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/hrtimer.h | 9 ++++++--- include/linux/time.h | 2 -- kernel/time/hrtimer.c | 19 ++++++++----------- kernel/time/timekeeping.c | 36 +++++++++++++++++++++++------------- 4 files changed, 37 insertions(+), 29 deletions(-) diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index bb4ffff31c69..e84eb4f228cd 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -331,9 +331,12 @@ extern ktime_t ktime_get_real(void); extern ktime_t ktime_get_boottime(void); extern ktime_t ktime_get_monotonic_offset(void); extern ktime_t ktime_get_clocktai(void); -extern ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot, - ktime_t *offs_tai); - +extern ktime_t ktime_get_update_offsets_tick(ktime_t *offs_real, + ktime_t *offs_boot, + ktime_t *offs_tai); +extern ktime_t ktime_get_update_offsets_now(ktime_t *offs_real, + ktime_t *offs_boot, + ktime_t *offs_tai); DECLARE_PER_CPU(struct tick_device, tick_cpu_device); diff --git a/include/linux/time.h b/include/linux/time.h index d5d229b2e5af..f6d990d1c79a 100644 --- a/include/linux/time.h +++ b/include/linux/time.h @@ -133,8 +133,6 @@ unsigned long get_seconds(void); struct timespec current_kernel_time(void); struct timespec __current_kernel_time(void); /* does not take xtime_lock */ struct timespec get_monotonic_coarse(void); -void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, - struct timespec *wtom, struct timespec *sleep); void timekeeping_inject_sleeptime(struct timespec *delta); #define CURRENT_TIME (current_kernel_time()) diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 66a6dc1075ad..2f4ef8a1e5ff 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -114,21 +114,18 @@ static inline int hrtimer_clockid_to_base(clockid_t clock_id) */ static void hrtimer_get_softirq_time(struct hrtimer_cpu_base *base) { - ktime_t xtim, mono, boot; - struct timespec xts, tom, slp; - s32 tai_offset; + ktime_t xtim, mono, boot, tai; + ktime_t off_real, off_boot, off_tai; - get_xtime_and_monotonic_and_sleep_offset(&xts, &tom, &slp); - tai_offset = timekeeping_get_tai_offset(); + mono = ktime_get_update_offsets_tick(&off_real, &off_boot, &off_tai); + boot = ktime_add(mono, off_boot); + xtim = ktime_add(mono, off_real); + tai = ktime_add(xtim, off_tai); - xtim = timespec_to_ktime(xts); - mono = ktime_add(xtim, timespec_to_ktime(tom)); - boot = ktime_add(mono, timespec_to_ktime(slp)); base->clock_base[HRTIMER_BASE_REALTIME].softirq_time = xtim; base->clock_base[HRTIMER_BASE_MONOTONIC].softirq_time = mono; base->clock_base[HRTIMER_BASE_BOOTTIME].softirq_time = boot; - base->clock_base[HRTIMER_BASE_TAI].softirq_time = - ktime_add(xtim, ktime_set(tai_offset, 0)); + base->clock_base[HRTIMER_BASE_TAI].softirq_time = tai; } /* @@ -673,7 +670,7 @@ static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base) ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset; ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset; - return ktime_get_update_offsets(offs_real, offs_boot, offs_tai); + return ktime_get_update_offsets_now(offs_real, offs_boot, offs_tai); } /* diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 908861c58e62..b94fa3652aaa 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -1581,29 +1581,39 @@ void do_timer(unsigned long ticks) } /** - * get_xtime_and_monotonic_and_sleep_offset() - get xtime, wall_to_monotonic, - * and sleep offsets. - * @xtim: pointer to timespec to be set with xtime - * @wtom: pointer to timespec to be set with wall_to_monotonic - * @sleep: pointer to timespec to be set with time in suspend + * ktime_get_update_offsets_tick - hrtimer helper + * @offs_real: pointer to storage for monotonic -> realtime offset + * @offs_boot: pointer to storage for monotonic -> boottime offset + * @offs_tai: pointer to storage for monotonic -> clock tai offset + * + * Returns monotonic time at last tick and various offsets */ -void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, - struct timespec *wtom, struct timespec *sleep) +ktime_t ktime_get_update_offsets_tick(ktime_t *offs_real, ktime_t *offs_boot, + ktime_t *offs_tai) { struct timekeeper *tk = &timekeeper; - unsigned long seq; + struct timespec ts; + ktime_t now; + unsigned int seq; do { seq = read_seqcount_begin(&timekeeper_seq); - *xtim = tk_xtime(tk); - *wtom = tk->wall_to_monotonic; - *sleep = tk->total_sleep_time; + + ts = tk_xtime(tk); + + *offs_real = tk->offs_real; + *offs_boot = tk->offs_boot; + *offs_tai = tk->offs_tai; } while (read_seqcount_retry(&timekeeper_seq, seq)); + + now = ktime_set(ts.tv_sec, ts.tv_nsec); + now = ktime_sub(now, *offs_real); + return now; } #ifdef CONFIG_HIGH_RES_TIMERS /** - * ktime_get_update_offsets - hrtimer helper + * ktime_get_update_offsets_now - hrtimer helper * @offs_real: pointer to storage for monotonic -> realtime offset * @offs_boot: pointer to storage for monotonic -> boottime offset * @offs_tai: pointer to storage for monotonic -> clock tai offset @@ -1611,7 +1621,7 @@ void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, * Returns current monotonic time and updates the offsets * Called from hrtimer_interrupt() or retrigger_next_event() */ -ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot, +ktime_t ktime_get_update_offsets_now(ktime_t *offs_real, ktime_t *offs_boot, ktime_t *offs_tai) { struct timekeeper *tk = &timekeeper; From 24e4a8c3e8868874835b0f1ad6dd417341e99822 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Wed, 16 Jul 2014 21:03:53 +0000 Subject: [PATCH 049/114] ktime: Kill non-scalar ktime_t implementation for 2038 The non-scalar ktime_t implementation is basically a timespec which has to be changed to support dates past 2038 on 32bit systems. This patch removes the non-scalar ktime_t implementation, forcing the scalar s64 nanosecond version on all architectures. This may have additional performance overhead on some 32bit systems when converting between ktime_t and timespec structures, however the majority of 32bit systems (arm and i386) were already using scalar ktime_t, so no performance regressions will be seen on those platforms. On affected platforms, I'm open to finding optimizations, including avoiding converting to timespecs where possible. [ tglx: We can now cleanup the ktime_t.tv64 mess, but thats a different issue and we can throw a coccinelle script at it ] Signed-off-by: John Stultz Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- arch/arm/Kconfig | 1 - arch/hexagon/Kconfig | 1 - arch/s390/Kconfig | 1 - arch/x86/Kconfig | 1 - include/linux/ktime.h | 173 +------------------------------------- include/linux/time.h | 11 +-- kernel/time/Kconfig | 4 - kernel/time/hrtimer.c | 54 ------------ kernel/time/timekeeping.c | 7 +- 9 files changed, 7 insertions(+), 246 deletions(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 05a71511ab3c..b9f6728331c8 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -64,7 +64,6 @@ config ARM select HAVE_UID16 select HAVE_VIRT_CPU_ACCOUNTING_GEN select IRQ_FORCED_THREADING - select KTIME_SCALAR select MODULES_USE_ELF_REL select NO_BOOTMEM select OLD_SIGACTION diff --git a/arch/hexagon/Kconfig b/arch/hexagon/Kconfig index 0fd6138f6203..4dc89d1f9c48 100644 --- a/arch/hexagon/Kconfig +++ b/arch/hexagon/Kconfig @@ -23,7 +23,6 @@ config HEXAGON select GENERIC_IOMAP select GENERIC_SMP_IDLE_THREAD select STACKTRACE_SUPPORT - select KTIME_SCALAR select GENERIC_CLOCKEVENTS select GENERIC_CLOCKEVENTS_BROADCAST select MODULES_USE_ELF_RELA diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index bb63499fc5d3..1afc7a686702 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -137,7 +137,6 @@ config S390 select HAVE_SYSCALL_TRACEPOINTS select HAVE_UID16 if 32BIT select HAVE_VIRT_CPU_ACCOUNTING - select KTIME_SCALAR if 32BIT select MODULES_USE_ELF_RELA select NO_BOOTMEM select OLD_SIGACTION diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index a8f749ef0fdc..7fa17b5ce668 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -111,7 +111,6 @@ config X86 select ARCH_CLOCKSOURCE_DATA select GENERIC_CLOCKEVENTS_BROADCAST if X86_64 || (X86_32 && X86_LOCAL_APIC) select GENERIC_TIME_VSYSCALL - select KTIME_SCALAR if X86_32 select GENERIC_STRNCPY_FROM_USER select GENERIC_STRNLEN_USER select HAVE_CONTEXT_TRACKING if X86_64 diff --git a/include/linux/ktime.h b/include/linux/ktime.h index de9e46e6bcc9..fbc64f8481b7 100644 --- a/include/linux/ktime.h +++ b/include/linux/ktime.h @@ -27,43 +27,19 @@ /* * ktime_t: * - * On 64-bit CPUs a single 64-bit variable is used to store the hrtimers + * A single 64-bit variable is used to store the hrtimers * internal representation of time values in scalar nanoseconds. The * design plays out best on 64-bit CPUs, where most conversions are * NOPs and most arithmetic ktime_t operations are plain arithmetic * operations. * - * On 32-bit CPUs an optimized representation of the timespec structure - * is used to avoid expensive conversions from and to timespecs. The - * endian-aware order of the tv struct members is chosen to allow - * mathematical operations on the tv64 member of the union too, which - * for certain operations produces better code. - * - * For architectures with efficient support for 64/32-bit conversions the - * plain scalar nanosecond based representation can be selected by the - * config switch CONFIG_KTIME_SCALAR. */ union ktime { s64 tv64; -#if BITS_PER_LONG != 64 && !defined(CONFIG_KTIME_SCALAR) - struct { -# ifdef __BIG_ENDIAN - s32 sec, nsec; -# else - s32 nsec, sec; -# endif - } tv; -#endif }; typedef union ktime ktime_t; /* Kill this */ -/* - * ktime_t definitions when using the 64-bit scalar representation: - */ - -#if (BITS_PER_LONG == 64) || defined(CONFIG_KTIME_SCALAR) - /** * ktime_set - Set a ktime_t variable from a seconds/nanoseconds value * @secs: seconds to set @@ -123,153 +99,6 @@ static inline ktime_t timeval_to_ktime(struct timeval tv) /* Convert ktime_t to nanoseconds - NOP in the scalar storage format: */ #define ktime_to_ns(kt) ((kt).tv64) -#else /* !((BITS_PER_LONG == 64) || defined(CONFIG_KTIME_SCALAR)) */ - -/* - * Helper macros/inlines to get the ktime_t math right in the timespec - * representation. The macros are sometimes ugly - their actual use is - * pretty okay-ish, given the circumstances. We do all this for - * performance reasons. The pure scalar nsec_t based code was nice and - * simple, but created too many 64-bit / 32-bit conversions and divisions. - * - * Be especially aware that negative values are represented in a way - * that the tv.sec field is negative and the tv.nsec field is greater - * or equal to zero but less than nanoseconds per second. This is the - * same representation which is used by timespecs. - * - * tv.sec < 0 and 0 >= tv.nsec < NSEC_PER_SEC - */ - -/* Set a ktime_t variable to a value in sec/nsec representation: */ -static inline ktime_t ktime_set(const long secs, const unsigned long nsecs) -{ - return (ktime_t) { .tv = { .sec = secs, .nsec = nsecs } }; -} - -/** - * ktime_sub - subtract two ktime_t variables - * @lhs: minuend - * @rhs: subtrahend - * - * Return: The remainder of the subtraction. - */ -static inline ktime_t ktime_sub(const ktime_t lhs, const ktime_t rhs) -{ - ktime_t res; - - res.tv64 = lhs.tv64 - rhs.tv64; - if (res.tv.nsec < 0) - res.tv.nsec += NSEC_PER_SEC; - - return res; -} - -/** - * ktime_add - add two ktime_t variables - * @add1: addend1 - * @add2: addend2 - * - * Return: The sum of @add1 and @add2. - */ -static inline ktime_t ktime_add(const ktime_t add1, const ktime_t add2) -{ - ktime_t res; - - res.tv64 = add1.tv64 + add2.tv64; - /* - * performance trick: the (u32) -NSEC gives 0x00000000Fxxxxxxx - * so we subtract NSEC_PER_SEC and add 1 to the upper 32 bit. - * - * it's equivalent to: - * tv.nsec -= NSEC_PER_SEC - * tv.sec ++; - */ - if (res.tv.nsec >= NSEC_PER_SEC) - res.tv64 += (u32)-NSEC_PER_SEC; - - return res; -} - -/** - * ktime_add_ns - Add a scalar nanoseconds value to a ktime_t variable - * @kt: addend - * @nsec: the scalar nsec value to add - * - * Return: The sum of @kt and @nsec in ktime_t format. - */ -extern ktime_t ktime_add_ns(const ktime_t kt, u64 nsec); - -/** - * ktime_sub_ns - Subtract a scalar nanoseconds value from a ktime_t variable - * @kt: minuend - * @nsec: the scalar nsec value to subtract - * - * Return: The subtraction of @nsec from @kt in ktime_t format. - */ -extern ktime_t ktime_sub_ns(const ktime_t kt, u64 nsec); - -/** - * timespec_to_ktime - convert a timespec to ktime_t format - * @ts: the timespec variable to convert - * - * Return: A ktime_t variable with the converted timespec value. - */ -static inline ktime_t timespec_to_ktime(const struct timespec ts) -{ - return (ktime_t) { .tv = { .sec = (s32)ts.tv_sec, - .nsec = (s32)ts.tv_nsec } }; -} - -/** - * timeval_to_ktime - convert a timeval to ktime_t format - * @tv: the timeval variable to convert - * - * Return: A ktime_t variable with the converted timeval value. - */ -static inline ktime_t timeval_to_ktime(const struct timeval tv) -{ - return (ktime_t) { .tv = { .sec = (s32)tv.tv_sec, - .nsec = (s32)(tv.tv_usec * - NSEC_PER_USEC) } }; -} - -/** - * ktime_to_timespec - convert a ktime_t variable to timespec format - * @kt: the ktime_t variable to convert - * - * Return: The timespec representation of the ktime value. - */ -static inline struct timespec ktime_to_timespec(const ktime_t kt) -{ - return (struct timespec) { .tv_sec = (time_t) kt.tv.sec, - .tv_nsec = (long) kt.tv.nsec }; -} - -/** - * ktime_to_timeval - convert a ktime_t variable to timeval format - * @kt: the ktime_t variable to convert - * - * Return: The timeval representation of the ktime value. - */ -static inline struct timeval ktime_to_timeval(const ktime_t kt) -{ - return (struct timeval) { - .tv_sec = (time_t) kt.tv.sec, - .tv_usec = (suseconds_t) (kt.tv.nsec / NSEC_PER_USEC) }; -} - -/** - * ktime_to_ns - convert a ktime_t variable to scalar nanoseconds - * @kt: the ktime_t variable to convert - * - * Return: The scalar nanoseconds representation of @kt. - */ -static inline s64 ktime_to_ns(const ktime_t kt) -{ - return (s64) kt.tv.sec * NSEC_PER_SEC + kt.tv.nsec; -} - -#endif /* !((BITS_PER_LONG == 64) || defined(CONFIG_KTIME_SCALAR)) */ /** * ktime_equal - Compares two ktime_t variables to see if they are equal diff --git a/include/linux/time.h b/include/linux/time.h index f6d990d1c79a..129f0bd36a8d 100644 --- a/include/linux/time.h +++ b/include/linux/time.h @@ -19,6 +19,10 @@ extern struct timezone sys_tz; #define TIME_T_MAX (time_t)((1UL << ((sizeof(time_t) << 3) - 1)) - 1) +/* Located here for timespec_valid_strict */ +#define KTIME_MAX ((s64)~((u64)1 << 63)) +#define KTIME_SEC_MAX (KTIME_MAX / NSEC_PER_SEC) + static inline int timespec_equal(const struct timespec *a, const struct timespec *b) { @@ -84,13 +88,6 @@ static inline struct timespec timespec_sub(struct timespec lhs, return ts_delta; } -#define KTIME_MAX ((s64)~((u64)1 << 63)) -#if (BITS_PER_LONG == 64) -# define KTIME_SEC_MAX (KTIME_MAX / NSEC_PER_SEC) -#else -# define KTIME_SEC_MAX LONG_MAX -#endif - /* * Returns true if the timespec is norm, false if denorm: */ diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig index f448513a45ed..feccfd888732 100644 --- a/kernel/time/Kconfig +++ b/kernel/time/Kconfig @@ -20,10 +20,6 @@ config GENERIC_TIME_VSYSCALL config GENERIC_TIME_VSYSCALL_OLD bool -# ktime_t scalar 64bit nsec representation -config KTIME_SCALAR - bool - # Old style timekeeping config ARCH_USES_GETTIMEOFFSET bool diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 2f4ef8a1e5ff..19f211051c35 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -261,60 +261,6 @@ lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) * too large for inlining: */ #if BITS_PER_LONG < 64 -# ifndef CONFIG_KTIME_SCALAR -/** - * ktime_add_ns - Add a scalar nanoseconds value to a ktime_t variable - * @kt: addend - * @nsec: the scalar nsec value to add - * - * Returns the sum of kt and nsec in ktime_t format - */ -ktime_t ktime_add_ns(const ktime_t kt, u64 nsec) -{ - ktime_t tmp; - - if (likely(nsec < NSEC_PER_SEC)) { - tmp.tv64 = nsec; - } else { - unsigned long rem = do_div(nsec, NSEC_PER_SEC); - - /* Make sure nsec fits into long */ - if (unlikely(nsec > KTIME_SEC_MAX)) - return (ktime_t){ .tv64 = KTIME_MAX }; - - tmp = ktime_set((long)nsec, rem); - } - - return ktime_add(kt, tmp); -} - -EXPORT_SYMBOL_GPL(ktime_add_ns); - -/** - * ktime_sub_ns - Subtract a scalar nanoseconds value from a ktime_t variable - * @kt: minuend - * @nsec: the scalar nsec value to subtract - * - * Returns the subtraction of @nsec from @kt in ktime_t format - */ -ktime_t ktime_sub_ns(const ktime_t kt, u64 nsec) -{ - ktime_t tmp; - - if (likely(nsec < NSEC_PER_SEC)) { - tmp.tv64 = nsec; - } else { - unsigned long rem = do_div(nsec, NSEC_PER_SEC); - - tmp = ktime_set((long)nsec, rem); - } - - return ktime_sub(kt, tmp); -} - -EXPORT_SYMBOL_GPL(ktime_sub_ns); -# endif /* !CONFIG_KTIME_SCALAR */ - /* * Divide a ktime value by a nanosecond value */ diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index b94fa3652aaa..cafef242d8f9 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -344,11 +344,8 @@ ktime_t ktime_get(void) nsecs = timekeeping_get_ns(tk) + tk->wall_to_monotonic.tv_nsec; } while (read_seqcount_retry(&timekeeper_seq, seq)); - /* - * Use ktime_set/ktime_add_ns to create a proper ktime on - * 32-bit architectures without CONFIG_KTIME_SCALAR. - */ - return ktime_add_ns(ktime_set(secs, 0), nsecs); + + return ktime_set(secs, nsecs); } EXPORT_SYMBOL_GPL(ktime_get); From 166afb64511eef08e13331b970c44fe91cea45ef Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:03:55 +0000 Subject: [PATCH 050/114] ktime: Sanitize ktime_to_us/ms conversion With the plain nanoseconds based ktime_t we can simply use ktime_divns() instead of going through loops and hoops of timespec/timeval conversion. Reported-by: John Stultz Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/hrtimer.h | 6 ------ include/linux/ktime.h | 12 ++++++++---- kernel/time/hrtimer.c | 1 + 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index e84eb4f228cd..adf5056bd7b3 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -457,12 +457,6 @@ extern void hrtimer_run_pending(void); /* Bootup initialization: */ extern void __init hrtimers_init(void); -#if BITS_PER_LONG < 64 -extern u64 ktime_divns(const ktime_t kt, s64 div); -#else /* BITS_PER_LONG < 64 */ -# define ktime_divns(kt, div) (u64)((kt).tv64 / (div)) -#endif - /* Show pending timers: */ extern void sysrq_timer_list_show(void); diff --git a/include/linux/ktime.h b/include/linux/ktime.h index fbc64f8481b7..74eaba9b3569 100644 --- a/include/linux/ktime.h +++ b/include/linux/ktime.h @@ -157,16 +157,20 @@ static inline bool ktime_before(const ktime_t cmp1, const ktime_t cmp2) return ktime_compare(cmp1, cmp2) < 0; } +#if BITS_PER_LONG < 64 +extern u64 ktime_divns(const ktime_t kt, s64 div); +#else /* BITS_PER_LONG < 64 */ +# define ktime_divns(kt, div) (u64)((kt).tv64 / (div)) +#endif + static inline s64 ktime_to_us(const ktime_t kt) { - struct timeval tv = ktime_to_timeval(kt); - return (s64) tv.tv_sec * USEC_PER_SEC + tv.tv_usec; + return ktime_divns(kt, NSEC_PER_USEC); } static inline s64 ktime_to_ms(const ktime_t kt) { - struct timeval tv = ktime_to_timeval(kt); - return (s64) tv.tv_sec * MSEC_PER_SEC + tv.tv_usec / USEC_PER_MSEC; + return ktime_divns(kt, NSEC_PER_MSEC); } static inline s64 ktime_us_delta(const ktime_t later, const ktime_t earlier) diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 19f211051c35..64843a836637 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -280,6 +280,7 @@ u64 ktime_divns(const ktime_t kt, s64 div) return dclc; } +EXPORT_SYMBOL_GPL(ktime_divns); #endif /* BITS_PER_LONG >= 64 */ /* From b17b20d70dcbe48dd1aa6aba073a60ddfce5d7db Mon Sep 17 00:00:00 2001 From: John Stultz Date: Wed, 16 Jul 2014 21:03:56 +0000 Subject: [PATCH 051/114] ktime: Change ktime_set() to take 64bit seconds value In order to support dates past 2038 on 32bit systems, ktime_set() needs to handle 64bit second values. [ tglx: Removed the BITS_PER_LONG check ] Signed-off-by: John Stultz Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/ktime.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/include/linux/ktime.h b/include/linux/ktime.h index 74eaba9b3569..538c283714e1 100644 --- a/include/linux/ktime.h +++ b/include/linux/ktime.h @@ -47,13 +47,12 @@ typedef union ktime ktime_t; /* Kill this */ * * Return: The ktime_t representation of the value. */ -static inline ktime_t ktime_set(const long secs, const unsigned long nsecs) +static inline ktime_t ktime_set(const s64 secs, const unsigned long nsecs) { -#if (BITS_PER_LONG == 64) if (unlikely(secs >= KTIME_SEC_MAX)) return (ktime_t){ .tv64 = KTIME_MAX }; -#endif - return (ktime_t) { .tv64 = (s64)secs * NSEC_PER_SEC + (s64)nsecs }; + + return (ktime_t) { .tv64 = secs * NSEC_PER_SEC + (s64)nsecs }; } /* Subtract two ktime_t variables. rem = lhs -rhs: */ From 361a3bf00582469877f8d18ff20f1efa6b781274 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Wed, 16 Jul 2014 21:03:58 +0000 Subject: [PATCH 052/114] time64: Add time64.h header and define struct timespec64 Define the timespec64 structure and standard helper functions. [ tglx: Make it 32bit only. 64bit really can map timespec to timespec64 ] Signed-off-by: John Stultz Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/time.h | 15 +--- include/linux/time64.h | 162 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 14 deletions(-) create mode 100644 include/linux/time64.h diff --git a/include/linux/time.h b/include/linux/time.h index 129f0bd36a8d..234feac7f1c3 100644 --- a/include/linux/time.h +++ b/include/linux/time.h @@ -4,25 +4,12 @@ # include # include # include -#include +# include extern struct timezone sys_tz; -/* Parameters used to convert the timespec values: */ -#define MSEC_PER_SEC 1000L -#define USEC_PER_MSEC 1000L -#define NSEC_PER_USEC 1000L -#define NSEC_PER_MSEC 1000000L -#define USEC_PER_SEC 1000000L -#define NSEC_PER_SEC 1000000000L -#define FSEC_PER_SEC 1000000000000000LL - #define TIME_T_MAX (time_t)((1UL << ((sizeof(time_t) << 3) - 1)) - 1) -/* Located here for timespec_valid_strict */ -#define KTIME_MAX ((s64)~((u64)1 << 63)) -#define KTIME_SEC_MAX (KTIME_MAX / NSEC_PER_SEC) - static inline int timespec_equal(const struct timespec *a, const struct timespec *b) { diff --git a/include/linux/time64.h b/include/linux/time64.h new file mode 100644 index 000000000000..e7b499e1cd79 --- /dev/null +++ b/include/linux/time64.h @@ -0,0 +1,162 @@ +#ifndef _LINUX_TIME64_H +#define _LINUX_TIME64_H + +#include + +typedef __s64 time64_t; + +/* + * This wants to go into uapi/linux/time.h once we agreed about the + * userspace interfaces. + */ +#if __BITS_PER_LONG == 64 +# define timespec64 timespec +#else +struct timespec64 { + time64_t tv_sec; /* seconds */ + long tv_nsec; /* nanoseconds */ +}; +#endif + +/* Parameters used to convert the timespec values: */ +#define MSEC_PER_SEC 1000L +#define USEC_PER_MSEC 1000L +#define NSEC_PER_USEC 1000L +#define NSEC_PER_MSEC 1000000L +#define USEC_PER_SEC 1000000L +#define NSEC_PER_SEC 1000000000L +#define FSEC_PER_SEC 1000000000000000LL + +/* Located here for timespec[64]_valid_strict */ +#define KTIME_MAX ((s64)~((u64)1 << 63)) +#define KTIME_SEC_MAX (KTIME_MAX / NSEC_PER_SEC) + +#if __BITS_PER_LONG == 64 + +# define timespec64_equal timespec_equal +# define timespec64_compare timespec_compare +# define set_normalized_timespec64 set_normalized_timespec +# define timespec64_add_safe timespec_add_safe +# define timespec64_add timespec_add +# define timespec64_sub timespec_sub +# define timespec64_valid timespec_valid +# define timespec64_valid_strict timespec_valid_strict +# define timespec64_to_ns timespec_to_ns +# define ns_to_timespec64 ns_to_timespec +# define timespec64_add_ns timespec_add_ns + +#else + +static inline int timespec64_equal(const struct timespec64 *a, + const struct timespec64 *b) +{ + return (a->tv_sec == b->tv_sec) && (a->tv_nsec == b->tv_nsec); +} + +/* + * lhs < rhs: return <0 + * lhs == rhs: return 0 + * lhs > rhs: return >0 + */ +static inline int timespec64_compare(const struct timespec64 *lhs, const struct timespec64 *rhs) +{ + if (lhs->tv_sec < rhs->tv_sec) + return -1; + if (lhs->tv_sec > rhs->tv_sec) + return 1; + return lhs->tv_nsec - rhs->tv_nsec; +} + +extern void set_normalized_timespec64(struct timespec64 *ts, time64_t sec, s64 nsec); + +/* + * timespec64_add_safe assumes both values are positive and checks for + * overflow. It will return TIME_T_MAX if the returned value would be + * smaller then either of the arguments. + */ +extern struct timespec64 timespec64_add_safe(const struct timespec64 lhs, + const struct timespec64 rhs); + + +static inline struct timespec64 timespec64_add(struct timespec64 lhs, + struct timespec64 rhs) +{ + struct timespec64 ts_delta; + set_normalized_timespec64(&ts_delta, lhs.tv_sec + rhs.tv_sec, + lhs.tv_nsec + rhs.tv_nsec); + return ts_delta; +} + +/* + * sub = lhs - rhs, in normalized form + */ +static inline struct timespec64 timespec64_sub(struct timespec64 lhs, + struct timespec64 rhs) +{ + struct timespec64 ts_delta; + set_normalized_timespec64(&ts_delta, lhs.tv_sec - rhs.tv_sec, + lhs.tv_nsec - rhs.tv_nsec); + return ts_delta; +} + +/* + * Returns true if the timespec64 is norm, false if denorm: + */ +static inline bool timespec64_valid(const struct timespec64 *ts) +{ + /* Dates before 1970 are bogus */ + if (ts->tv_sec < 0) + return false; + /* Can't have more nanoseconds then a second */ + if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC) + return false; + return true; +} + +static inline bool timespec64_valid_strict(const struct timespec64 *ts) +{ + if (!timespec64_valid(ts)) + return false; + /* Disallow values that could overflow ktime_t */ + if ((unsigned long long)ts->tv_sec >= KTIME_SEC_MAX) + return false; + return true; +} + +/** + * timespec64_to_ns - Convert timespec64 to nanoseconds + * @ts: pointer to the timespec64 variable to be converted + * + * Returns the scalar nanosecond representation of the timespec64 + * parameter. + */ +static inline s64 timespec64_to_ns(const struct timespec64 *ts) +{ + return ((s64) ts->tv_sec * NSEC_PER_SEC) + ts->tv_nsec; +} + +/** + * ns_to_timespec64 - Convert nanoseconds to timespec64 + * @nsec: the nanoseconds value to be converted + * + * Returns the timespec64 representation of the nsec parameter. + */ +extern struct timespec64 ns_to_timespec64(const s64 nsec); + +/** + * timespec64_add_ns - Adds nanoseconds to a timespec64 + * @a: pointer to timespec64 to be incremented + * @ns: unsigned nanoseconds value to be added + * + * This must always be inlined because its used from the x86-64 vdso, + * which cannot call other kernel functions. + */ +static __always_inline void timespec64_add_ns(struct timespec64 *a, u64 ns) +{ + a->tv_sec += __iter_div_u64_rem(a->tv_nsec + ns, NSEC_PER_SEC, &ns); + a->tv_nsec = ns; +} + +#endif + +#endif /* _LINUX_TIME64_H */ From 49cd6f869984692547c57621bf42697aaa7f5622 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Wed, 16 Jul 2014 21:03:59 +0000 Subject: [PATCH 053/114] time: More core infrastructure for timespec64 Helper and conversion functions for timespec64. Signed-off-by: John Stultz Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/ktime.h | 28 +++++++++++++++++++ include/linux/time64.h | 28 +++++++++++++++++++ kernel/time/time.c | 62 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+) diff --git a/include/linux/ktime.h b/include/linux/ktime.h index 538c283714e1..da6b680c252b 100644 --- a/include/linux/ktime.h +++ b/include/linux/ktime.h @@ -83,6 +83,12 @@ static inline ktime_t timespec_to_ktime(struct timespec ts) return ktime_set(ts.tv_sec, ts.tv_nsec); } +/* convert a timespec64 to ktime_t format: */ +static inline ktime_t timespec64_to_ktime(struct timespec64 ts) +{ + return ktime_set(ts.tv_sec, ts.tv_nsec); +} + /* convert a timeval to ktime_t format: */ static inline ktime_t timeval_to_ktime(struct timeval tv) { @@ -92,6 +98,9 @@ static inline ktime_t timeval_to_ktime(struct timeval tv) /* Map the ktime_t to timespec conversion to ns_to_timespec function */ #define ktime_to_timespec(kt) ns_to_timespec((kt).tv64) +/* Map the ktime_t to timespec conversion to ns_to_timespec function */ +#define ktime_to_timespec64(kt) ns_to_timespec64((kt).tv64) + /* Map the ktime_t to timeval conversion to ns_to_timeval function */ #define ktime_to_timeval(kt) ns_to_timeval((kt).tv64) @@ -213,6 +222,25 @@ static inline __must_check bool ktime_to_timespec_cond(const ktime_t kt, } } +/** + * ktime_to_timespec64_cond - convert a ktime_t variable to timespec64 + * format only if the variable contains data + * @kt: the ktime_t variable to convert + * @ts: the timespec variable to store the result in + * + * Return: %true if there was a successful conversion, %false if kt was 0. + */ +static inline __must_check bool ktime_to_timespec64_cond(const ktime_t kt, + struct timespec64 *ts) +{ + if (kt.tv64) { + *ts = ktime_to_timespec64(kt); + return true; + } else { + return false; + } +} + /* * The resolution of the clocks. The resolution value is returned in * the clock_getres() system call to give application programmers an diff --git a/include/linux/time64.h b/include/linux/time64.h index e7b499e1cd79..a3831478d9cf 100644 --- a/include/linux/time64.h +++ b/include/linux/time64.h @@ -33,6 +33,16 @@ struct timespec64 { #if __BITS_PER_LONG == 64 +static inline struct timespec timespec64_to_timespec(const struct timespec64 ts64) +{ + return ts64; +} + +static inline struct timespec64 timespec_to_timespec64(const struct timespec ts) +{ + return ts; +} + # define timespec64_equal timespec_equal # define timespec64_compare timespec_compare # define set_normalized_timespec64 set_normalized_timespec @@ -47,6 +57,24 @@ struct timespec64 { #else +static inline struct timespec timespec64_to_timespec(const struct timespec64 ts64) +{ + struct timespec ret; + + ret.tv_sec = (time_t)ts64.tv_sec; + ret.tv_nsec = ts64.tv_nsec; + return ret; +} + +static inline struct timespec64 timespec_to_timespec64(const struct timespec ts) +{ + struct timespec64 ret; + + ret.tv_sec = ts.tv_sec; + ret.tv_nsec = ts.tv_nsec; + return ret; +} + static inline int timespec64_equal(const struct timespec64 *a, const struct timespec64 *b) { diff --git a/kernel/time/time.c b/kernel/time/time.c index 7c7964c33ae7..e8121a67fd74 100644 --- a/kernel/time/time.c +++ b/kernel/time/time.c @@ -420,6 +420,68 @@ struct timeval ns_to_timeval(const s64 nsec) } EXPORT_SYMBOL(ns_to_timeval); +#if BITS_PER_LONG == 32 +/** + * set_normalized_timespec - set timespec sec and nsec parts and normalize + * + * @ts: pointer to timespec variable to be set + * @sec: seconds to set + * @nsec: nanoseconds to set + * + * Set seconds and nanoseconds field of a timespec variable and + * normalize to the timespec storage format + * + * Note: The tv_nsec part is always in the range of + * 0 <= tv_nsec < NSEC_PER_SEC + * For negative values only the tv_sec field is negative ! + */ +void set_normalized_timespec64(struct timespec64 *ts, time64_t sec, s64 nsec) +{ + while (nsec >= NSEC_PER_SEC) { + /* + * The following asm() prevents the compiler from + * optimising this loop into a modulo operation. See + * also __iter_div_u64_rem() in include/linux/time.h + */ + asm("" : "+rm"(nsec)); + nsec -= NSEC_PER_SEC; + ++sec; + } + while (nsec < 0) { + asm("" : "+rm"(nsec)); + nsec += NSEC_PER_SEC; + --sec; + } + ts->tv_sec = sec; + ts->tv_nsec = nsec; +} +EXPORT_SYMBOL(set_normalized_timespec64); + +/** + * ns_to_timespec64 - Convert nanoseconds to timespec64 + * @nsec: the nanoseconds value to be converted + * + * Returns the timespec64 representation of the nsec parameter. + */ +struct timespec64 ns_to_timespec64(const s64 nsec) +{ + struct timespec64 ts; + s32 rem; + + if (!nsec) + return (struct timespec64) {0, 0}; + + ts.tv_sec = div_s64_rem(nsec, NSEC_PER_SEC, &rem); + if (unlikely(rem < 0)) { + ts.tv_sec--; + rem += NSEC_PER_SEC; + } + ts.tv_nsec = rem; + + return ts; +} +EXPORT_SYMBOL(ns_to_timespec64); +#endif /* * When we convert to jiffies then we interpret incoming values * the following way: From 7d489d15ce4be5310ca60e5896df833f9b3b4088 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Wed, 16 Jul 2014 21:04:01 +0000 Subject: [PATCH 054/114] timekeeping: Convert timekeeping core to use timespec64s Convert the core timekeeping logic to use timespec64s. This moves the 2038 issues out of the core logic and into all of the accessor functions. Future changes will need to push the timespec64s out to all timekeeping users, but that can be done interface by interface. Signed-off-by: John Stultz Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/timekeeper_internal.h | 10 +- kernel/time/ntp.c | 8 +- kernel/time/ntp_internal.h | 2 +- kernel/time/timekeeping.c | 172 ++++++++++++++++------------ kernel/time/timekeeping_debug.c | 2 +- kernel/time/timekeeping_internal.h | 2 +- 6 files changed, 109 insertions(+), 87 deletions(-) diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h index c1825eb436ed..1b05491e10f9 100644 --- a/include/linux/timekeeper_internal.h +++ b/include/linux/timekeeper_internal.h @@ -55,15 +55,15 @@ struct timekeeper { * - wall_to_monotonic is no longer the boot time, getboottime must be * used instead. */ - struct timespec wall_to_monotonic; + struct timespec64 wall_to_monotonic; /* Offset clock monotonic -> clock realtime */ ktime_t offs_real; /* time spent in suspend */ - struct timespec total_sleep_time; + struct timespec64 total_sleep_time; /* Offset clock monotonic -> clock boottime */ ktime_t offs_boot; /* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */ - struct timespec raw_time; + struct timespec64 raw_time; /* The current UTC to TAI offset in seconds */ s32 tai_offset; /* Offset clock monotonic -> clock tai */ @@ -71,9 +71,9 @@ struct timekeeper { }; -static inline struct timespec tk_xtime(struct timekeeper *tk) +static inline struct timespec64 tk_xtime(struct timekeeper *tk) { - struct timespec ts; + struct timespec64 ts; ts.tv_sec = tk->xtime_sec; ts.tv_nsec = (long)(tk->xtime_nsec >> tk->shift); diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c index 33db43a39515..6e87df94122f 100644 --- a/kernel/time/ntp.c +++ b/kernel/time/ntp.c @@ -531,7 +531,7 @@ void ntp_notify_cmos_timer(void) { } /* * Propagate a new txc->status value into the NTP state: */ -static inline void process_adj_status(struct timex *txc, struct timespec *ts) +static inline void process_adj_status(struct timex *txc, struct timespec64 *ts) { if ((time_status & STA_PLL) && !(txc->status & STA_PLL)) { time_state = TIME_OK; @@ -554,7 +554,7 @@ static inline void process_adj_status(struct timex *txc, struct timespec *ts) static inline void process_adjtimex_modes(struct timex *txc, - struct timespec *ts, + struct timespec64 *ts, s32 *time_tai) { if (txc->modes & ADJ_STATUS) @@ -640,7 +640,7 @@ int ntp_validate_timex(struct timex *txc) * adjtimex mainly allows reading (and writing, if superuser) of * kernel time-keeping variables. used by xntpd. */ -int __do_adjtimex(struct timex *txc, struct timespec *ts, s32 *time_tai) +int __do_adjtimex(struct timex *txc, struct timespec64 *ts, s32 *time_tai) { int result; @@ -684,7 +684,7 @@ int __do_adjtimex(struct timex *txc, struct timespec *ts, s32 *time_tai) /* fill PPS status fields */ pps_fill_timex(txc); - txc->time.tv_sec = ts->tv_sec; + txc->time.tv_sec = (time_t)ts->tv_sec; txc->time.tv_usec = ts->tv_nsec; if (!(time_status & STA_NANO)) txc->time.tv_usec /= NSEC_PER_USEC; diff --git a/kernel/time/ntp_internal.h b/kernel/time/ntp_internal.h index 1950cb4ca2a4..bbd102ad9df7 100644 --- a/kernel/time/ntp_internal.h +++ b/kernel/time/ntp_internal.h @@ -7,6 +7,6 @@ extern void ntp_clear(void); extern u64 ntp_tick_length(void); extern int second_overflow(unsigned long secs); extern int ntp_validate_timex(struct timex *); -extern int __do_adjtimex(struct timex *, struct timespec *, s32 *); +extern int __do_adjtimex(struct timex *, struct timespec64 *, s32 *); extern void __hardpps(const struct timespec *, const struct timespec *); #endif /* _LINUX_NTP_INTERNAL_H */ diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index cafef242d8f9..84a2075c3eb4 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -51,43 +51,43 @@ static inline void tk_normalize_xtime(struct timekeeper *tk) } } -static void tk_set_xtime(struct timekeeper *tk, const struct timespec *ts) +static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts) { tk->xtime_sec = ts->tv_sec; tk->xtime_nsec = (u64)ts->tv_nsec << tk->shift; } -static void tk_xtime_add(struct timekeeper *tk, const struct timespec *ts) +static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts) { tk->xtime_sec += ts->tv_sec; tk->xtime_nsec += (u64)ts->tv_nsec << tk->shift; tk_normalize_xtime(tk); } -static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec wtm) +static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm) { - struct timespec tmp; + struct timespec64 tmp; /* * Verify consistency of: offset_real = -wall_to_monotonic * before modifying anything */ - set_normalized_timespec(&tmp, -tk->wall_to_monotonic.tv_sec, + set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec, -tk->wall_to_monotonic.tv_nsec); - WARN_ON_ONCE(tk->offs_real.tv64 != timespec_to_ktime(tmp).tv64); + WARN_ON_ONCE(tk->offs_real.tv64 != timespec64_to_ktime(tmp).tv64); tk->wall_to_monotonic = wtm; - set_normalized_timespec(&tmp, -wtm.tv_sec, -wtm.tv_nsec); - tk->offs_real = timespec_to_ktime(tmp); + set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec); + tk->offs_real = timespec64_to_ktime(tmp); tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0)); } -static void tk_set_sleep_time(struct timekeeper *tk, struct timespec t) +static void tk_set_sleep_time(struct timekeeper *tk, struct timespec64 t) { /* Verify consistency before modifying */ - WARN_ON_ONCE(tk->offs_boot.tv64 != timespec_to_ktime(tk->total_sleep_time).tv64); + WARN_ON_ONCE(tk->offs_boot.tv64 != timespec64_to_ktime(tk->total_sleep_time).tv64); tk->total_sleep_time = t; - tk->offs_boot = timespec_to_ktime(t); + tk->offs_boot = timespec64_to_ktime(t); } /** @@ -281,7 +281,7 @@ static void timekeeping_forward_now(struct timekeeper *tk) tk_normalize_xtime(tk); nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift); - timespec_add_ns(&tk->raw_time, nsec); + timespec64_add_ns(&tk->raw_time, nsec); } /** @@ -360,7 +360,7 @@ EXPORT_SYMBOL_GPL(ktime_get); void ktime_get_ts(struct timespec *ts) { struct timekeeper *tk = &timekeeper; - struct timespec tomono; + struct timespec64 ts64, tomono; s64 nsec; unsigned int seq; @@ -368,15 +368,16 @@ void ktime_get_ts(struct timespec *ts) do { seq = read_seqcount_begin(&timekeeper_seq); - ts->tv_sec = tk->xtime_sec; + ts64.tv_sec = tk->xtime_sec; nsec = timekeeping_get_ns(tk); tomono = tk->wall_to_monotonic; } while (read_seqcount_retry(&timekeeper_seq, seq)); - ts->tv_sec += tomono.tv_sec; - ts->tv_nsec = 0; - timespec_add_ns(ts, nsec + tomono.tv_nsec); + ts64.tv_sec += tomono.tv_sec; + ts64.tv_nsec = 0; + timespec64_add_ns(&ts64, nsec + tomono.tv_nsec); + *ts = timespec64_to_timespec(ts64); } EXPORT_SYMBOL_GPL(ktime_get_ts); @@ -390,6 +391,7 @@ EXPORT_SYMBOL_GPL(ktime_get_ts); void timekeeping_clocktai(struct timespec *ts) { struct timekeeper *tk = &timekeeper; + struct timespec64 ts64; unsigned long seq; u64 nsecs; @@ -398,13 +400,14 @@ void timekeeping_clocktai(struct timespec *ts) do { seq = read_seqcount_begin(&timekeeper_seq); - ts->tv_sec = tk->xtime_sec + tk->tai_offset; + ts64.tv_sec = tk->xtime_sec + tk->tai_offset; nsecs = timekeeping_get_ns(tk); } while (read_seqcount_retry(&timekeeper_seq, seq)); - ts->tv_nsec = 0; - timespec_add_ns(ts, nsecs); + ts64.tv_nsec = 0; + timespec64_add_ns(&ts64, nsecs); + *ts = timespec64_to_timespec(ts64); } EXPORT_SYMBOL(timekeeping_clocktai); @@ -446,7 +449,7 @@ void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real) do { seq = read_seqcount_begin(&timekeeper_seq); - *ts_raw = tk->raw_time; + *ts_raw = timespec64_to_timespec(tk->raw_time); ts_real->tv_sec = tk->xtime_sec; ts_real->tv_nsec = 0; @@ -487,7 +490,7 @@ EXPORT_SYMBOL(do_gettimeofday); int do_settimeofday(const struct timespec *tv) { struct timekeeper *tk = &timekeeper; - struct timespec ts_delta, xt; + struct timespec64 ts_delta, xt, tmp; unsigned long flags; if (!timespec_valid_strict(tv)) @@ -502,9 +505,10 @@ int do_settimeofday(const struct timespec *tv) ts_delta.tv_sec = tv->tv_sec - xt.tv_sec; ts_delta.tv_nsec = tv->tv_nsec - xt.tv_nsec; - tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, ts_delta)); + tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts_delta)); - tk_set_xtime(tk, tv); + tmp = timespec_to_timespec64(*tv); + tk_set_xtime(tk, &tmp); timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); @@ -528,26 +532,28 @@ int timekeeping_inject_offset(struct timespec *ts) { struct timekeeper *tk = &timekeeper; unsigned long flags; - struct timespec tmp; + struct timespec64 ts64, tmp; int ret = 0; if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC) return -EINVAL; + ts64 = timespec_to_timespec64(*ts); + raw_spin_lock_irqsave(&timekeeper_lock, flags); write_seqcount_begin(&timekeeper_seq); timekeeping_forward_now(tk); /* Make sure the proposed value is valid */ - tmp = timespec_add(tk_xtime(tk), *ts); - if (!timespec_valid_strict(&tmp)) { + tmp = timespec64_add(tk_xtime(tk), ts64); + if (!timespec64_valid_strict(&tmp)) { ret = -EINVAL; goto error; } - tk_xtime_add(tk, ts); - tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, *ts)); + tk_xtime_add(tk, &ts64); + tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts64)); error: /* even if we error out, we forwarded the time, so call update */ timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); @@ -691,17 +697,19 @@ EXPORT_SYMBOL_GPL(ktime_get_real); void getrawmonotonic(struct timespec *ts) { struct timekeeper *tk = &timekeeper; + struct timespec64 ts64; unsigned long seq; s64 nsecs; do { seq = read_seqcount_begin(&timekeeper_seq); nsecs = timekeeping_get_ns_raw(tk); - *ts = tk->raw_time; + ts64 = tk->raw_time; } while (read_seqcount_retry(&timekeeper_seq, seq)); - timespec_add_ns(ts, nsecs); + timespec64_add_ns(&ts64, nsecs); + *ts = timespec64_to_timespec(ts64); } EXPORT_SYMBOL(getrawmonotonic); @@ -781,11 +789,12 @@ void __init timekeeping_init(void) struct timekeeper *tk = &timekeeper; struct clocksource *clock; unsigned long flags; - struct timespec now, boot, tmp; + struct timespec64 now, boot, tmp; + struct timespec ts; - read_persistent_clock(&now); - - if (!timespec_valid_strict(&now)) { + read_persistent_clock(&ts); + now = timespec_to_timespec64(ts); + if (!timespec64_valid_strict(&now)) { pr_warn("WARNING: Persistent clock returned invalid value!\n" " Check your CMOS/BIOS settings.\n"); now.tv_sec = 0; @@ -793,8 +802,9 @@ void __init timekeeping_init(void) } else if (now.tv_sec || now.tv_nsec) persistent_clock_exist = true; - read_boot_clock(&boot); - if (!timespec_valid_strict(&boot)) { + read_boot_clock(&ts); + boot = timespec_to_timespec64(ts); + if (!timespec64_valid_strict(&boot)) { pr_warn("WARNING: Boot clock returned invalid value!\n" " Check your CMOS/BIOS settings.\n"); boot.tv_sec = 0; @@ -816,7 +826,7 @@ void __init timekeeping_init(void) if (boot.tv_sec == 0 && boot.tv_nsec == 0) boot = tk_xtime(tk); - set_normalized_timespec(&tmp, -boot.tv_sec, -boot.tv_nsec); + set_normalized_timespec64(&tmp, -boot.tv_sec, -boot.tv_nsec); tk_set_wall_to_mono(tk, tmp); tmp.tv_sec = 0; @@ -830,7 +840,7 @@ void __init timekeeping_init(void) } /* time in seconds when suspend began */ -static struct timespec timekeeping_suspend_time; +static struct timespec64 timekeeping_suspend_time; /** * __timekeeping_inject_sleeptime - Internal function to add sleep interval @@ -840,17 +850,17 @@ static struct timespec timekeeping_suspend_time; * adds the sleep offset to the timekeeping variables. */ static void __timekeeping_inject_sleeptime(struct timekeeper *tk, - struct timespec *delta) + struct timespec64 *delta) { - if (!timespec_valid_strict(delta)) { + if (!timespec64_valid_strict(delta)) { printk_deferred(KERN_WARNING "__timekeeping_inject_sleeptime: Invalid " "sleep delta value!\n"); return; } tk_xtime_add(tk, delta); - tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, *delta)); - tk_set_sleep_time(tk, timespec_add(tk->total_sleep_time, *delta)); + tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta)); + tk_set_sleep_time(tk, timespec64_add(tk->total_sleep_time, *delta)); tk_debug_account_sleep_time(delta); } @@ -867,6 +877,7 @@ static void __timekeeping_inject_sleeptime(struct timekeeper *tk, void timekeeping_inject_sleeptime(struct timespec *delta) { struct timekeeper *tk = &timekeeper; + struct timespec64 tmp; unsigned long flags; /* @@ -881,7 +892,8 @@ void timekeeping_inject_sleeptime(struct timespec *delta) timekeeping_forward_now(tk); - __timekeeping_inject_sleeptime(tk, delta); + tmp = timespec_to_timespec64(*delta); + __timekeeping_inject_sleeptime(tk, &tmp); timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); @@ -904,11 +916,13 @@ static void timekeeping_resume(void) struct timekeeper *tk = &timekeeper; struct clocksource *clock = tk->clock; unsigned long flags; - struct timespec ts_new, ts_delta; + struct timespec64 ts_new, ts_delta; + struct timespec tmp; cycle_t cycle_now, cycle_delta; bool suspendtime_found = false; - read_persistent_clock(&ts_new); + read_persistent_clock(&tmp); + ts_new = timespec_to_timespec64(tmp); clockevents_resume(); clocksource_resume(); @@ -951,10 +965,10 @@ static void timekeeping_resume(void) } nsec += ((u64) cycle_delta * mult) >> shift; - ts_delta = ns_to_timespec(nsec); + ts_delta = ns_to_timespec64(nsec); suspendtime_found = true; - } else if (timespec_compare(&ts_new, &timekeeping_suspend_time) > 0) { - ts_delta = timespec_sub(ts_new, timekeeping_suspend_time); + } else if (timespec64_compare(&ts_new, &timekeeping_suspend_time) > 0) { + ts_delta = timespec64_sub(ts_new, timekeeping_suspend_time); suspendtime_found = true; } @@ -981,10 +995,12 @@ static int timekeeping_suspend(void) { struct timekeeper *tk = &timekeeper; unsigned long flags; - struct timespec delta, delta_delta; - static struct timespec old_delta; + struct timespec64 delta, delta_delta; + static struct timespec64 old_delta; + struct timespec tmp; - read_persistent_clock(&timekeeping_suspend_time); + read_persistent_clock(&tmp); + timekeeping_suspend_time = timespec_to_timespec64(tmp); /* * On some systems the persistent_clock can not be detected at @@ -1005,8 +1021,8 @@ static int timekeeping_suspend(void) * try to compensate so the difference in system time * and persistent_clock time stays close to constant. */ - delta = timespec_sub(tk_xtime(tk), timekeeping_suspend_time); - delta_delta = timespec_sub(delta, old_delta); + delta = timespec64_sub(tk_xtime(tk), timekeeping_suspend_time); + delta_delta = timespec64_sub(delta, old_delta); if (abs(delta_delta.tv_sec) >= 2) { /* * if delta_delta is too large, assume time correction @@ -1016,7 +1032,7 @@ static int timekeeping_suspend(void) } else { /* Otherwise try to adjust old_system to compensate */ timekeeping_suspend_time = - timespec_add(timekeeping_suspend_time, delta_delta); + timespec64_add(timekeeping_suspend_time, delta_delta); } timekeeping_update(tk, TK_MIRROR); @@ -1253,14 +1269,14 @@ static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk) /* Figure out if its a leap sec and apply if needed */ leap = second_overflow(tk->xtime_sec); if (unlikely(leap)) { - struct timespec ts; + struct timespec64 ts; tk->xtime_sec += leap; ts.tv_sec = leap; ts.tv_nsec = 0; tk_set_wall_to_mono(tk, - timespec_sub(tk->wall_to_monotonic, ts)); + timespec64_sub(tk->wall_to_monotonic, ts)); __timekeeping_set_tai_offset(tk, tk->tai_offset - leap); @@ -1469,7 +1485,7 @@ EXPORT_SYMBOL_GPL(getboottime); void get_monotonic_boottime(struct timespec *ts) { struct timekeeper *tk = &timekeeper; - struct timespec tomono, sleep; + struct timespec64 tomono, sleep, ret; s64 nsec; unsigned int seq; @@ -1477,16 +1493,17 @@ void get_monotonic_boottime(struct timespec *ts) do { seq = read_seqcount_begin(&timekeeper_seq); - ts->tv_sec = tk->xtime_sec; + ret.tv_sec = tk->xtime_sec; nsec = timekeeping_get_ns(tk); tomono = tk->wall_to_monotonic; sleep = tk->total_sleep_time; } while (read_seqcount_retry(&timekeeper_seq, seq)); - ts->tv_sec += tomono.tv_sec + sleep.tv_sec; - ts->tv_nsec = 0; - timespec_add_ns(ts, nsec + tomono.tv_nsec + sleep.tv_nsec); + ret.tv_sec += tomono.tv_sec + sleep.tv_sec; + ret.tv_nsec = 0; + timespec64_add_ns(&ret, nsec + tomono.tv_nsec + sleep.tv_nsec); + *ts = timespec64_to_timespec(ret); } EXPORT_SYMBOL_GPL(get_monotonic_boottime); @@ -1514,8 +1531,11 @@ EXPORT_SYMBOL_GPL(ktime_get_boottime); void monotonic_to_bootbased(struct timespec *ts) { struct timekeeper *tk = &timekeeper; + struct timespec64 ts64; - *ts = timespec_add(*ts, tk->total_sleep_time); + ts64 = timespec_to_timespec64(*ts); + ts64 = timespec64_add(ts64, tk->total_sleep_time); + *ts = timespec64_to_timespec(ts64); } EXPORT_SYMBOL_GPL(monotonic_to_bootbased); @@ -1531,13 +1551,13 @@ struct timespec __current_kernel_time(void) { struct timekeeper *tk = &timekeeper; - return tk_xtime(tk); + return timespec64_to_timespec(tk_xtime(tk)); } struct timespec current_kernel_time(void) { struct timekeeper *tk = &timekeeper; - struct timespec now; + struct timespec64 now; unsigned long seq; do { @@ -1546,14 +1566,14 @@ struct timespec current_kernel_time(void) now = tk_xtime(tk); } while (read_seqcount_retry(&timekeeper_seq, seq)); - return now; + return timespec64_to_timespec(now); } EXPORT_SYMBOL(current_kernel_time); struct timespec get_monotonic_coarse(void) { struct timekeeper *tk = &timekeeper; - struct timespec now, mono; + struct timespec64 now, mono; unsigned long seq; do { @@ -1563,9 +1583,10 @@ struct timespec get_monotonic_coarse(void) mono = tk->wall_to_monotonic; } while (read_seqcount_retry(&timekeeper_seq, seq)); - set_normalized_timespec(&now, now.tv_sec + mono.tv_sec, + set_normalized_timespec64(&now, now.tv_sec + mono.tv_sec, now.tv_nsec + mono.tv_nsec); - return now; + + return timespec64_to_timespec(now); } /* @@ -1589,7 +1610,7 @@ ktime_t ktime_get_update_offsets_tick(ktime_t *offs_real, ktime_t *offs_boot, ktime_t *offs_tai) { struct timekeeper *tk = &timekeeper; - struct timespec ts; + struct timespec64 ts; ktime_t now; unsigned int seq; @@ -1597,7 +1618,6 @@ ktime_t ktime_get_update_offsets_tick(ktime_t *offs_real, ktime_t *offs_boot, seq = read_seqcount_begin(&timekeeper_seq); ts = tk_xtime(tk); - *offs_real = tk->offs_real; *offs_boot = tk->offs_boot; *offs_tai = tk->offs_tai; @@ -1650,14 +1670,14 @@ ktime_t ktime_get_monotonic_offset(void) { struct timekeeper *tk = &timekeeper; unsigned long seq; - struct timespec wtom; + struct timespec64 wtom; do { seq = read_seqcount_begin(&timekeeper_seq); wtom = tk->wall_to_monotonic; } while (read_seqcount_retry(&timekeeper_seq, seq)); - return timespec_to_ktime(wtom); + return timespec64_to_ktime(wtom); } EXPORT_SYMBOL_GPL(ktime_get_monotonic_offset); @@ -1668,7 +1688,8 @@ int do_adjtimex(struct timex *txc) { struct timekeeper *tk = &timekeeper; unsigned long flags; - struct timespec ts; + struct timespec64 ts; + struct timespec tmp; s32 orig_tai, tai; int ret; @@ -1688,7 +1709,8 @@ int do_adjtimex(struct timex *txc) return ret; } - getnstimeofday(&ts); + getnstimeofday(&tmp); + ts = timespec_to_timespec64(tmp); raw_spin_lock_irqsave(&timekeeper_lock, flags); write_seqcount_begin(&timekeeper_seq); diff --git a/kernel/time/timekeeping_debug.c b/kernel/time/timekeeping_debug.c index 4d54f97558df..f6bd65236712 100644 --- a/kernel/time/timekeeping_debug.c +++ b/kernel/time/timekeeping_debug.c @@ -67,7 +67,7 @@ static int __init tk_debug_sleep_time_init(void) } late_initcall(tk_debug_sleep_time_init); -void tk_debug_account_sleep_time(struct timespec *t) +void tk_debug_account_sleep_time(struct timespec64 *t) { sleep_time_bin[fls(t->tv_sec)]++; } diff --git a/kernel/time/timekeeping_internal.h b/kernel/time/timekeeping_internal.h index 13323ea08ffa..e3d28ad236f9 100644 --- a/kernel/time/timekeeping_internal.h +++ b/kernel/time/timekeeping_internal.h @@ -6,7 +6,7 @@ #include #ifdef CONFIG_DEBUG_FS -extern void tk_debug_account_sleep_time(struct timespec *t); +extern void tk_debug_account_sleep_time(struct timespec64 *t); #else #define tk_debug_account_sleep_time(x) #endif From 8b094cd03b4a3793220d8d8d86a173bfea8c285b Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:02 +0000 Subject: [PATCH 055/114] time: Consolidate the time accessor prototypes Right now we have time related prototypes in 3 different header files. Move it to a single timekeeping header file and move the core internal stuff into a core private header. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/hrtimer.h | 11 ------ include/linux/ktime.h | 8 +--- include/linux/time.h | 45 ++------------------- include/linux/timekeeping.h | 78 +++++++++++++++++++++++++++++++++++++ kernel/time/hrtimer.c | 2 + kernel/time/posix-timers.c | 2 + kernel/time/tick-internal.h | 2 + kernel/time/time.c | 1 + kernel/time/timekeeping.h | 20 ++++++++++ 9 files changed, 111 insertions(+), 58 deletions(-) create mode 100644 include/linux/timekeeping.h create mode 100644 kernel/time/timekeeping.h diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index adf5056bd7b3..a036d058a249 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -326,17 +326,6 @@ static inline void timerfd_clock_was_set(void) { } #endif extern void hrtimers_resume(void); -extern ktime_t ktime_get(void); -extern ktime_t ktime_get_real(void); -extern ktime_t ktime_get_boottime(void); -extern ktime_t ktime_get_monotonic_offset(void); -extern ktime_t ktime_get_clocktai(void); -extern ktime_t ktime_get_update_offsets_tick(ktime_t *offs_real, - ktime_t *offs_boot, - ktime_t *offs_tai); -extern ktime_t ktime_get_update_offsets_now(ktime_t *offs_real, - ktime_t *offs_boot, - ktime_t *offs_tai); DECLARE_PER_CPU(struct tick_device, tick_cpu_device); diff --git a/include/linux/ktime.h b/include/linux/ktime.h index da6b680c252b..c9d645ad98ff 100644 --- a/include/linux/ktime.h +++ b/include/linux/ktime.h @@ -250,12 +250,6 @@ static inline __must_check bool ktime_to_timespec64_cond(const ktime_t kt, #define LOW_RES_NSEC TICK_NSEC #define KTIME_LOW_RES (ktime_t){ .tv64 = LOW_RES_NSEC } -/* Get the monotonic time in timespec format: */ -extern void ktime_get_ts(struct timespec *ts); - -/* Get the real (wall-) time in timespec format: */ -#define ktime_get_real_ts(ts) getnstimeofday(ts) - static inline ktime_t ns_to_ktime(u64 ns) { static const ktime_t ktime_zero = { .tv64 = 0 }; @@ -270,4 +264,6 @@ static inline ktime_t ms_to_ktime(u64 ms) return ktime_add_ms(ktime_zero, ms); } +# include + #endif diff --git a/include/linux/time.h b/include/linux/time.h index 234feac7f1c3..8c42cf8d2444 100644 --- a/include/linux/time.h +++ b/include/linux/time.h @@ -99,25 +99,7 @@ static inline bool timespec_valid_strict(const struct timespec *ts) return true; } -extern bool persistent_clock_exist; - -static inline bool has_persistent_clock(void) -{ - return persistent_clock_exist; -} - -extern void read_persistent_clock(struct timespec *ts); -extern void read_boot_clock(struct timespec *ts); -extern int persistent_clock_is_local; -extern int update_persistent_clock(struct timespec now); -void timekeeping_init(void); -extern int timekeeping_suspended; - -unsigned long get_seconds(void); -struct timespec current_kernel_time(void); -struct timespec __current_kernel_time(void); /* does not take xtime_lock */ -struct timespec get_monotonic_coarse(void); -void timekeeping_inject_sleeptime(struct timespec *delta); +extern struct timespec timespec_trunc(struct timespec t, unsigned gran); #define CURRENT_TIME (current_kernel_time()) #define CURRENT_TIME_SEC ((struct timespec) { get_seconds(), 0 }) @@ -135,33 +117,14 @@ void timekeeping_inject_sleeptime(struct timespec *delta); extern u32 (*arch_gettimeoffset)(void); #endif -extern void do_gettimeofday(struct timeval *tv); -extern int do_settimeofday(const struct timespec *tv); -extern int do_sys_settimeofday(const struct timespec *tv, - const struct timezone *tz); -#define do_posix_clock_monotonic_gettime(ts) ktime_get_ts(ts) -extern long do_utimes(int dfd, const char __user *filename, struct timespec *times, int flags); struct itimerval; extern int do_setitimer(int which, struct itimerval *value, struct itimerval *ovalue); -extern unsigned int alarm_setitimer(unsigned int seconds); extern int do_getitimer(int which, struct itimerval *value); -extern int __getnstimeofday(struct timespec *tv); -extern void getnstimeofday(struct timespec *tv); -extern void getrawmonotonic(struct timespec *ts); -extern void getnstime_raw_and_real(struct timespec *ts_raw, - struct timespec *ts_real); -extern void getboottime(struct timespec *ts); -extern void monotonic_to_bootbased(struct timespec *ts); -extern void get_monotonic_boottime(struct timespec *ts); -extern struct timespec timespec_trunc(struct timespec t, unsigned gran); -extern int timekeeping_valid_for_hres(void); -extern u64 timekeeping_max_deferment(void); -extern int timekeeping_inject_offset(struct timespec *ts); -extern s32 timekeeping_get_tai_offset(void); -extern void timekeeping_set_tai_offset(s32 tai_offset); -extern void timekeeping_clocktai(struct timespec *ts); +extern unsigned int alarm_setitimer(unsigned int seconds); + +extern long do_utimes(int dfd, const char __user *filename, struct timespec *times, int flags); struct tms; extern void do_sys_times(struct tms *); diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h new file mode 100644 index 000000000000..e81c35b71da7 --- /dev/null +++ b/include/linux/timekeeping.h @@ -0,0 +1,78 @@ +#ifndef _LINUX_TIMEKEEPING_H +#define _LINUX_TIMEKEEPING_H + +/* Included from linux/ktime.h */ + +void timekeeping_init(void); +extern int timekeeping_suspended; + +/* + * Get and set timeofday + */ +extern void do_gettimeofday(struct timeval *tv); +extern int do_settimeofday(const struct timespec *tv); +extern int do_sys_settimeofday(const struct timespec *tv, + const struct timezone *tz); + +/* + * Kernel time accessors + */ +unsigned long get_seconds(void); +struct timespec current_kernel_time(void); +/* does not take xtime_lock */ +struct timespec __current_kernel_time(void); + +/* + * timespec based interfaces + */ +struct timespec get_monotonic_coarse(void); +extern void getrawmonotonic(struct timespec *ts); +extern void monotonic_to_bootbased(struct timespec *ts); +extern void get_monotonic_boottime(struct timespec *ts); +extern void ktime_get_ts(struct timespec *ts); + +extern int __getnstimeofday(struct timespec *tv); +extern void getnstimeofday(struct timespec *tv); +extern void getboottime(struct timespec *ts); + +#define do_posix_clock_monotonic_gettime(ts) ktime_get_ts(ts) +#define ktime_get_real_ts(ts) getnstimeofday(ts) + + +/* + * ktime_t based interfaces + */ +extern ktime_t ktime_get(void); +extern ktime_t ktime_get_real(void); +extern ktime_t ktime_get_boottime(void); +extern ktime_t ktime_get_monotonic_offset(void); +extern ktime_t ktime_get_clocktai(void); + +/* + * RTC specific + */ +extern void timekeeping_inject_sleeptime(struct timespec *delta); + +/* + * PPS accessor + */ +extern void getnstime_raw_and_real(struct timespec *ts_raw, + struct timespec *ts_real); + +/* + * Persistent clock related interfaces + */ +extern bool persistent_clock_exist; +extern int persistent_clock_is_local; + +static inline bool has_persistent_clock(void) +{ + return persistent_clock_exist; +} + +extern void read_persistent_clock(struct timespec *ts); +extern void read_boot_clock(struct timespec *ts); +extern int update_persistent_clock(struct timespec now); + + +#endif diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 64843a836637..1c2fe7de2842 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -54,6 +54,8 @@ #include +#include "timekeeping.h" + /* * The timer bases: * diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c index 424c2d4265c9..42b463ad90f2 100644 --- a/kernel/time/posix-timers.c +++ b/kernel/time/posix-timers.c @@ -49,6 +49,8 @@ #include #include +#include "timekeeping.h" + /* * Management arrays for POSIX timers. Timers are now kept in static hash table * with 512 entries. diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h index 7ab92b19965a..c19c1d84b6f3 100644 --- a/kernel/time/tick-internal.h +++ b/kernel/time/tick-internal.h @@ -4,6 +4,8 @@ #include #include +#include "timekeeping.h" + extern seqlock_t jiffies_lock; #define CS_NAME_LEN 32 diff --git a/kernel/time/time.c b/kernel/time/time.c index e8121a67fd74..278c63cc8054 100644 --- a/kernel/time/time.c +++ b/kernel/time/time.c @@ -42,6 +42,7 @@ #include #include "timeconst.h" +#include "timekeeping.h" /* * The timezone where the local system is located. Used as a default by some diff --git a/kernel/time/timekeeping.h b/kernel/time/timekeeping.h new file mode 100644 index 000000000000..adc1fc98bde3 --- /dev/null +++ b/kernel/time/timekeeping.h @@ -0,0 +1,20 @@ +#ifndef _KERNEL_TIME_TIMEKEEPING_H +#define _KERNEL_TIME_TIMEKEEPING_H +/* + * Internal interfaces for kernel/time/ + */ +extern ktime_t ktime_get_update_offsets_tick(ktime_t *offs_real, + ktime_t *offs_boot, + ktime_t *offs_tai); +extern ktime_t ktime_get_update_offsets_now(ktime_t *offs_real, + ktime_t *offs_boot, + ktime_t *offs_tai); + +extern int timekeeping_valid_for_hres(void); +extern u64 timekeeping_max_deferment(void); +extern int timekeeping_inject_offset(struct timespec *ts); +extern s32 timekeeping_get_tai_offset(void); +extern void timekeeping_set_tai_offset(s32 tai_offset); +extern void timekeeping_clocktai(struct timespec *ts); + +#endif From d6d29896c665dfd50e6e0be7a9039901640433a3 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:04 +0000 Subject: [PATCH 056/114] timekeeping: Provide timespec64 based interfaces To convert callers of the core code to timespec64 we need to provide the proper interfaces. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/timekeeping.h | 66 ++++++++++++++++++++++++++++++++++--- kernel/time/ntp.c | 7 ++-- kernel/time/timekeeping.c | 47 +++++++++++++------------- 3 files changed, 87 insertions(+), 33 deletions(-) diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h index e81c35b71da7..3eb19e34cc20 100644 --- a/include/linux/timekeeping.h +++ b/include/linux/timekeeping.h @@ -29,15 +29,71 @@ struct timespec get_monotonic_coarse(void); extern void getrawmonotonic(struct timespec *ts); extern void monotonic_to_bootbased(struct timespec *ts); extern void get_monotonic_boottime(struct timespec *ts); -extern void ktime_get_ts(struct timespec *ts); +extern void ktime_get_ts64(struct timespec64 *ts); + +extern int __getnstimeofday64(struct timespec64 *tv); +extern void getnstimeofday64(struct timespec64 *tv); + +#if BITS_PER_LONG == 64 +static inline int __getnstimeofday(struct timespec *ts) +{ + return __getnstimeofday64(ts); +} + +static inline void getnstimeofday(struct timespec *ts) +{ + getnstimeofday64(ts); +} + +static inline void ktime_get_ts(struct timespec *ts) +{ + ktime_get_ts64(ts); +} + +static inline void ktime_get_real_ts(struct timespec *ts) +{ + getnstimeofday64(ts); +} + +#else +static inline int __getnstimeofday(struct timespec *ts) +{ + struct timespec64 ts64; + int ret = __getnstimeofday64(&ts64); + + *ts = timespec64_to_timespec(ts64); + return ret; +} + +static inline void getnstimeofday(struct timespec *ts) +{ + struct timespec64 ts64; + + getnstimeofday64(&ts64); + *ts = timespec64_to_timespec(ts64); +} + +static inline void ktime_get_ts(struct timespec *ts) +{ + struct timespec64 ts64; + + ktime_get_ts64(&ts64); + *ts = timespec64_to_timespec(ts64); +} + +static inline void ktime_get_real_ts(struct timespec *ts) +{ + struct timespec64 ts64; + + getnstimeofday64(&ts64); + *ts = timespec64_to_timespec(ts64); +} +#endif -extern int __getnstimeofday(struct timespec *tv); -extern void getnstimeofday(struct timespec *tv); extern void getboottime(struct timespec *ts); #define do_posix_clock_monotonic_gettime(ts) ktime_get_ts(ts) -#define ktime_get_real_ts(ts) getnstimeofday(ts) - +#define ktime_get_real_ts64(ts) getnstimeofday64(ts) /* * ktime_t based interfaces diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c index 6e87df94122f..87a346fd6d61 100644 --- a/kernel/time/ntp.c +++ b/kernel/time/ntp.c @@ -466,7 +466,8 @@ static DECLARE_DELAYED_WORK(sync_cmos_work, sync_cmos_clock); static void sync_cmos_clock(struct work_struct *work) { - struct timespec now, next; + struct timespec64 now; + struct timespec next; int fail = 1; /* @@ -485,9 +486,9 @@ static void sync_cmos_clock(struct work_struct *work) return; } - getnstimeofday(&now); + getnstimeofday64(&now); if (abs(now.tv_nsec - (NSEC_PER_SEC / 2)) <= tick_nsec * 5) { - struct timespec adjust = now; + struct timespec adjust = timespec64_to_timespec(now); fail = -ENODEV; if (persistent_clock_is_local) diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 84a2075c3eb4..3210c9e690c5 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -285,13 +285,13 @@ static void timekeeping_forward_now(struct timekeeper *tk) } /** - * __getnstimeofday - Returns the time of day in a timespec. + * __getnstimeofday64 - Returns the time of day in a timespec64. * @ts: pointer to the timespec to be set * * Updates the time of day in the timespec. * Returns 0 on success, or -ve when suspended (timespec will be undefined). */ -int __getnstimeofday(struct timespec *ts) +int __getnstimeofday64(struct timespec64 *ts) { struct timekeeper *tk = &timekeeper; unsigned long seq; @@ -306,7 +306,7 @@ int __getnstimeofday(struct timespec *ts) } while (read_seqcount_retry(&timekeeper_seq, seq)); ts->tv_nsec = 0; - timespec_add_ns(ts, nsecs); + timespec64_add_ns(ts, nsecs); /* * Do not bail out early, in case there were callers still using @@ -316,19 +316,19 @@ int __getnstimeofday(struct timespec *ts) return -EAGAIN; return 0; } -EXPORT_SYMBOL(__getnstimeofday); +EXPORT_SYMBOL(__getnstimeofday64); /** - * getnstimeofday - Returns the time of day in a timespec. + * getnstimeofday64 - Returns the time of day in a timespec64. * @ts: pointer to the timespec to be set * * Returns the time of day in a timespec (WARN if suspended). */ -void getnstimeofday(struct timespec *ts) +void getnstimeofday64(struct timespec64 *ts) { - WARN_ON(__getnstimeofday(ts)); + WARN_ON(__getnstimeofday64(ts)); } -EXPORT_SYMBOL(getnstimeofday); +EXPORT_SYMBOL(getnstimeofday64); ktime_t ktime_get(void) { @@ -350,17 +350,17 @@ ktime_t ktime_get(void) EXPORT_SYMBOL_GPL(ktime_get); /** - * ktime_get_ts - get the monotonic clock in timespec format + * ktime_get_ts64 - get the monotonic clock in timespec64 format * @ts: pointer to timespec variable * * The function calculates the monotonic clock from the realtime * clock and the wall_to_monotonic offset and stores the result * in normalized timespec format in the variable pointed to by @ts. */ -void ktime_get_ts(struct timespec *ts) +void ktime_get_ts64(struct timespec64 *ts) { struct timekeeper *tk = &timekeeper; - struct timespec64 ts64, tomono; + struct timespec64 tomono; s64 nsec; unsigned int seq; @@ -368,18 +368,17 @@ void ktime_get_ts(struct timespec *ts) do { seq = read_seqcount_begin(&timekeeper_seq); - ts64.tv_sec = tk->xtime_sec; + ts->tv_sec = tk->xtime_sec; nsec = timekeeping_get_ns(tk); tomono = tk->wall_to_monotonic; } while (read_seqcount_retry(&timekeeper_seq, seq)); - ts64.tv_sec += tomono.tv_sec; - ts64.tv_nsec = 0; - timespec64_add_ns(&ts64, nsec + tomono.tv_nsec); - *ts = timespec64_to_timespec(ts64); + ts->tv_sec += tomono.tv_sec; + ts->tv_nsec = 0; + timespec64_add_ns(ts, nsec + tomono.tv_nsec); } -EXPORT_SYMBOL_GPL(ktime_get_ts); +EXPORT_SYMBOL_GPL(ktime_get_ts64); /** @@ -473,9 +472,9 @@ EXPORT_SYMBOL(getnstime_raw_and_real); */ void do_gettimeofday(struct timeval *tv) { - struct timespec now; + struct timespec64 now; - getnstimeofday(&now); + getnstimeofday64(&now); tv->tv_sec = now.tv_sec; tv->tv_usec = now.tv_nsec/1000; } @@ -680,11 +679,11 @@ int timekeeping_notify(struct clocksource *clock) */ ktime_t ktime_get_real(void) { - struct timespec now; + struct timespec64 now; - getnstimeofday(&now); + getnstimeofday64(&now); - return timespec_to_ktime(now); + return timespec64_to_ktime(now); } EXPORT_SYMBOL_GPL(ktime_get_real); @@ -1689,7 +1688,6 @@ int do_adjtimex(struct timex *txc) struct timekeeper *tk = &timekeeper; unsigned long flags; struct timespec64 ts; - struct timespec tmp; s32 orig_tai, tai; int ret; @@ -1709,8 +1707,7 @@ int do_adjtimex(struct timex *txc) return ret; } - getnstimeofday(&tmp); - ts = timespec_to_timespec64(tmp); + getnstimeofday64(&ts); raw_spin_lock_irqsave(&timekeeper_lock, flags); write_seqcount_begin(&timekeeper_seq); From c905fae43f61c2b4508fc01722e8db61b6b8ac0b Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:05 +0000 Subject: [PATCH 057/114] timekeeper: Move tk_xtime to core code No users outside of the core. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/timekeeper_internal.h | 18 -------- kernel/time/timekeeping.c | 70 ++++++++++++++++++----------- 2 files changed, 43 insertions(+), 45 deletions(-) diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h index 1b05491e10f9..16de6d7c240a 100644 --- a/include/linux/timekeeper_internal.h +++ b/include/linux/timekeeper_internal.h @@ -71,16 +71,6 @@ struct timekeeper { }; -static inline struct timespec64 tk_xtime(struct timekeeper *tk) -{ - struct timespec64 ts; - - ts.tv_sec = tk->xtime_sec; - ts.tv_nsec = (long)(tk->xtime_nsec >> tk->shift); - return ts; -} - - #ifdef CONFIG_GENERIC_TIME_VSYSCALL extern void update_vsyscall(struct timekeeper *tk); @@ -92,14 +82,6 @@ extern void update_vsyscall_old(struct timespec *ts, struct timespec *wtm, struct clocksource *c, u32 mult); extern void update_vsyscall_tz(void); -static inline void update_vsyscall(struct timekeeper *tk) -{ - struct timespec xt; - - xt = tk_xtime(tk); - update_vsyscall_old(&xt, &tk->wall_to_monotonic, tk->clock, tk->mult); -} - #else static inline void update_vsyscall(struct timekeeper *tk) diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 3210c9e690c5..983d67b388d7 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -51,6 +51,15 @@ static inline void tk_normalize_xtime(struct timekeeper *tk) } } +static inline struct timespec64 tk_xtime(struct timekeeper *tk) +{ + struct timespec64 ts; + + ts.tv_sec = tk->xtime_sec; + ts.tv_nsec = (long)(tk->xtime_nsec >> tk->shift); + return ts; +} + static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts) { tk->xtime_sec = ts->tv_sec; @@ -199,6 +208,40 @@ static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk) return nsec + arch_gettimeoffset(); } +#ifdef CONFIG_GENERIC_TIME_VSYSCALL_OLD + +static inline void update_vsyscall(struct timekeeper *tk) +{ + struct timespec xt; + + xt = tk_xtime(tk); + update_vsyscall_old(&xt, &tk->wall_to_monotonic, tk->clock, tk->mult); +} + +static inline void old_vsyscall_fixup(struct timekeeper *tk) +{ + s64 remainder; + + /* + * Store only full nanoseconds into xtime_nsec after rounding + * it up and add the remainder to the error difference. + * XXX - This is necessary to avoid small 1ns inconsistnecies caused + * by truncating the remainder in vsyscalls. However, it causes + * additional work to be done in timekeeping_adjust(). Once + * the vsyscall implementations are converted to use xtime_nsec + * (shifted nanoseconds), and CONFIG_GENERIC_TIME_VSYSCALL_OLD + * users are removed, this can be killed. + */ + remainder = tk->xtime_nsec & ((1ULL << tk->shift) - 1); + tk->xtime_nsec -= remainder; + tk->xtime_nsec += 1ULL << tk->shift; + tk->ntp_error += remainder << tk->ntp_error_shift; + tk->ntp_error -= (1ULL << tk->shift) << tk->ntp_error_shift; +} +#else +#define old_vsyscall_fixup(tk) +#endif + static RAW_NOTIFIER_HEAD(pvclock_gtod_chain); static void update_pvclock_gtod(struct timekeeper *tk, bool was_set) @@ -1330,33 +1373,6 @@ static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset, return offset; } -#ifdef CONFIG_GENERIC_TIME_VSYSCALL_OLD -static inline void old_vsyscall_fixup(struct timekeeper *tk) -{ - s64 remainder; - - /* - * Store only full nanoseconds into xtime_nsec after rounding - * it up and add the remainder to the error difference. - * XXX - This is necessary to avoid small 1ns inconsistnecies caused - * by truncating the remainder in vsyscalls. However, it causes - * additional work to be done in timekeeping_adjust(). Once - * the vsyscall implementations are converted to use xtime_nsec - * (shifted nanoseconds), and CONFIG_GENERIC_TIME_VSYSCALL_OLD - * users are removed, this can be killed. - */ - remainder = tk->xtime_nsec & ((1ULL << tk->shift) - 1); - tk->xtime_nsec -= remainder; - tk->xtime_nsec += 1ULL << tk->shift; - tk->ntp_error += remainder << tk->ntp_error_shift; - tk->ntp_error -= (1ULL << tk->shift) << tk->ntp_error_shift; -} -#else -#define old_vsyscall_fixup(tk) -#endif - - - /** * update_wall_time - Uses the current clocksource to increment the wall time * From 3fdb14fd1df70325e1e91e1203a699a4803ed741 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:07 +0000 Subject: [PATCH 058/114] timekeeping: Cache optimize struct timekeeper struct timekeeper is quite badly sorted for the hot readout path. Most time access functions need to load two cache lines. Rearrange it so ktime_get() and getnstimeofday() are happy with a single cache line. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/timekeeper_internal.h | 86 +++++++------ kernel/time/timekeeping.c | 185 +++++++++++++++------------- 2 files changed, 144 insertions(+), 127 deletions(-) diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h index 16de6d7c240a..2cb96235c249 100644 --- a/include/linux/timekeeper_internal.h +++ b/include/linux/timekeeper_internal.h @@ -10,7 +10,22 @@ #include #include -/* Structure holding internal timekeeping values. */ +/* + * Structure holding internal timekeeping values. + * + * Note: wall_to_monotonic is what we need to add to xtime (or xtime + * corrected for sub jiffie times) to get to monotonic time. + * Monotonic is pegged at zero at system boot time, so + * wall_to_monotonic will be negative, however, we will ALWAYS keep + * the tv_nsec part positive so we can use the usual normalization. + * + * wall_to_monotonic is moved after resume from suspend for the + * monotonic time not to jump. We need to add total_sleep_time to + * wall_to_monotonic to get the real boot based time offset. + * + * - wall_to_monotonic is no longer the boot time, getboottime must be + * used instead. + */ struct timekeeper { /* Current clocksource used for timekeeping. */ struct clocksource *clock; @@ -18,6 +33,29 @@ struct timekeeper { u32 mult; /* The shift value of the current clocksource. */ u32 shift; + /* Clock shifted nano seconds */ + u64 xtime_nsec; + + /* Current CLOCK_REALTIME time in seconds */ + u64 xtime_sec; + /* CLOCK_REALTIME to CLOCK_MONOTONIC offset */ + struct timespec64 wall_to_monotonic; + + /* Offset clock monotonic -> clock realtime */ + ktime_t offs_real; + /* Offset clock monotonic -> clock boottime */ + ktime_t offs_boot; + /* Offset clock monotonic -> clock tai */ + ktime_t offs_tai; + + /* time spent in suspend */ + struct timespec64 total_sleep_time; + /* The current UTC to TAI offset in seconds */ + s32 tai_offset; + + /* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */ + struct timespec64 raw_time; + /* Number of clock cycles in one NTP interval. */ cycle_t cycle_interval; /* Last cycle value (also stored in clock->cycle_last) */ @@ -29,46 +67,16 @@ struct timekeeper { /* Raw nano seconds accumulated per NTP interval. */ u32 raw_interval; - /* Current CLOCK_REALTIME time in seconds */ - u64 xtime_sec; - /* Clock shifted nano seconds */ - u64 xtime_nsec; - - /* Difference between accumulated time and NTP time in ntp - * shifted nano seconds. */ - s64 ntp_error; - /* Shift conversion between clock shifted nano seconds and - * ntp shifted nano seconds. */ - u32 ntp_error_shift; - /* - * wall_to_monotonic is what we need to add to xtime (or xtime corrected - * for sub jiffie times) to get to monotonic time. Monotonic is pegged - * at zero at system boot time, so wall_to_monotonic will be negative, - * however, we will ALWAYS keep the tv_nsec part positive so we can use - * the usual normalization. - * - * wall_to_monotonic is moved after resume from suspend for the - * monotonic time not to jump. We need to add total_sleep_time to - * wall_to_monotonic to get the real boot based time offset. - * - * - wall_to_monotonic is no longer the boot time, getboottime must be - * used instead. + * Difference between accumulated time and NTP time in ntp + * shifted nano seconds. */ - struct timespec64 wall_to_monotonic; - /* Offset clock monotonic -> clock realtime */ - ktime_t offs_real; - /* time spent in suspend */ - struct timespec64 total_sleep_time; - /* Offset clock monotonic -> clock boottime */ - ktime_t offs_boot; - /* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */ - struct timespec64 raw_time; - /* The current UTC to TAI offset in seconds */ - s32 tai_offset; - /* Offset clock monotonic -> clock tai */ - ktime_t offs_tai; - + s64 ntp_error; + /* + * Shift conversion between clock shifted nano seconds and + * ntp shifted nano seconds. + */ + u32 ntp_error_shift; }; #ifdef CONFIG_GENERIC_TIME_VSYSCALL diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 983d67b388d7..7ca150ad387d 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -32,9 +32,16 @@ #define TK_MIRROR (1 << 1) #define TK_CLOCK_WAS_SET (1 << 2) -static struct timekeeper timekeeper; +/* + * The most important data for readout fits into a single 64 byte + * cache line. + */ +static struct { + seqcount_t seq; + struct timekeeper timekeeper; +} tk_core ____cacheline_aligned; + static DEFINE_RAW_SPINLOCK(timekeeper_lock); -static seqcount_t timekeeper_seq; static struct timekeeper shadow_timekeeper; /* flag for if timekeeping is suspended */ @@ -254,7 +261,7 @@ static void update_pvclock_gtod(struct timekeeper *tk, bool was_set) */ int pvclock_gtod_register_notifier(struct notifier_block *nb) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; unsigned long flags; int ret; @@ -295,7 +302,8 @@ static void timekeeping_update(struct timekeeper *tk, unsigned int action) update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET); if (action & TK_MIRROR) - memcpy(&shadow_timekeeper, &timekeeper, sizeof(timekeeper)); + memcpy(&shadow_timekeeper, &tk_core.timekeeper, + sizeof(tk_core.timekeeper)); } /** @@ -336,17 +344,17 @@ static void timekeeping_forward_now(struct timekeeper *tk) */ int __getnstimeofday64(struct timespec64 *ts) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; unsigned long seq; s64 nsecs = 0; do { - seq = read_seqcount_begin(&timekeeper_seq); + seq = read_seqcount_begin(&tk_core.seq); ts->tv_sec = tk->xtime_sec; nsecs = timekeeping_get_ns(tk); - } while (read_seqcount_retry(&timekeeper_seq, seq)); + } while (read_seqcount_retry(&tk_core.seq, seq)); ts->tv_nsec = 0; timespec64_add_ns(ts, nsecs); @@ -375,18 +383,18 @@ EXPORT_SYMBOL(getnstimeofday64); ktime_t ktime_get(void) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; unsigned int seq; s64 secs, nsecs; WARN_ON(timekeeping_suspended); do { - seq = read_seqcount_begin(&timekeeper_seq); + seq = read_seqcount_begin(&tk_core.seq); secs = tk->xtime_sec + tk->wall_to_monotonic.tv_sec; nsecs = timekeeping_get_ns(tk) + tk->wall_to_monotonic.tv_nsec; - } while (read_seqcount_retry(&timekeeper_seq, seq)); + } while (read_seqcount_retry(&tk_core.seq, seq)); return ktime_set(secs, nsecs); } @@ -402,7 +410,7 @@ EXPORT_SYMBOL_GPL(ktime_get); */ void ktime_get_ts64(struct timespec64 *ts) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; struct timespec64 tomono; s64 nsec; unsigned int seq; @@ -410,12 +418,12 @@ void ktime_get_ts64(struct timespec64 *ts) WARN_ON(timekeeping_suspended); do { - seq = read_seqcount_begin(&timekeeper_seq); + seq = read_seqcount_begin(&tk_core.seq); ts->tv_sec = tk->xtime_sec; nsec = timekeeping_get_ns(tk); tomono = tk->wall_to_monotonic; - } while (read_seqcount_retry(&timekeeper_seq, seq)); + } while (read_seqcount_retry(&tk_core.seq, seq)); ts->tv_sec += tomono.tv_sec; ts->tv_nsec = 0; @@ -432,7 +440,7 @@ EXPORT_SYMBOL_GPL(ktime_get_ts64); */ void timekeeping_clocktai(struct timespec *ts) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; struct timespec64 ts64; unsigned long seq; u64 nsecs; @@ -440,12 +448,12 @@ void timekeeping_clocktai(struct timespec *ts) WARN_ON(timekeeping_suspended); do { - seq = read_seqcount_begin(&timekeeper_seq); + seq = read_seqcount_begin(&tk_core.seq); ts64.tv_sec = tk->xtime_sec + tk->tai_offset; nsecs = timekeeping_get_ns(tk); - } while (read_seqcount_retry(&timekeeper_seq, seq)); + } while (read_seqcount_retry(&tk_core.seq, seq)); ts64.tv_nsec = 0; timespec64_add_ns(&ts64, nsecs); @@ -482,14 +490,14 @@ EXPORT_SYMBOL(ktime_get_clocktai); */ void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; unsigned long seq; s64 nsecs_raw, nsecs_real; WARN_ON_ONCE(timekeeping_suspended); do { - seq = read_seqcount_begin(&timekeeper_seq); + seq = read_seqcount_begin(&tk_core.seq); *ts_raw = timespec64_to_timespec(tk->raw_time); ts_real->tv_sec = tk->xtime_sec; @@ -498,7 +506,7 @@ void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real) nsecs_raw = timekeeping_get_ns_raw(tk); nsecs_real = timekeeping_get_ns(tk); - } while (read_seqcount_retry(&timekeeper_seq, seq)); + } while (read_seqcount_retry(&tk_core.seq, seq)); timespec_add_ns(ts_raw, nsecs_raw); timespec_add_ns(ts_real, nsecs_real); @@ -531,7 +539,7 @@ EXPORT_SYMBOL(do_gettimeofday); */ int do_settimeofday(const struct timespec *tv) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; struct timespec64 ts_delta, xt, tmp; unsigned long flags; @@ -539,7 +547,7 @@ int do_settimeofday(const struct timespec *tv) return -EINVAL; raw_spin_lock_irqsave(&timekeeper_lock, flags); - write_seqcount_begin(&timekeeper_seq); + write_seqcount_begin(&tk_core.seq); timekeeping_forward_now(tk); @@ -554,7 +562,7 @@ int do_settimeofday(const struct timespec *tv) timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); - write_seqcount_end(&timekeeper_seq); + write_seqcount_end(&tk_core.seq); raw_spin_unlock_irqrestore(&timekeeper_lock, flags); /* signal hrtimers about time change */ @@ -572,7 +580,7 @@ EXPORT_SYMBOL(do_settimeofday); */ int timekeeping_inject_offset(struct timespec *ts) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; unsigned long flags; struct timespec64 ts64, tmp; int ret = 0; @@ -583,7 +591,7 @@ int timekeeping_inject_offset(struct timespec *ts) ts64 = timespec_to_timespec64(*ts); raw_spin_lock_irqsave(&timekeeper_lock, flags); - write_seqcount_begin(&timekeeper_seq); + write_seqcount_begin(&tk_core.seq); timekeeping_forward_now(tk); @@ -600,7 +608,7 @@ int timekeeping_inject_offset(struct timespec *ts) error: /* even if we error out, we forwarded the time, so call update */ timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); - write_seqcount_end(&timekeeper_seq); + write_seqcount_end(&tk_core.seq); raw_spin_unlock_irqrestore(&timekeeper_lock, flags); /* signal hrtimers about time change */ @@ -617,14 +625,14 @@ EXPORT_SYMBOL(timekeeping_inject_offset); */ s32 timekeeping_get_tai_offset(void) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; unsigned int seq; s32 ret; do { - seq = read_seqcount_begin(&timekeeper_seq); + seq = read_seqcount_begin(&tk_core.seq); ret = tk->tai_offset; - } while (read_seqcount_retry(&timekeeper_seq, seq)); + } while (read_seqcount_retry(&tk_core.seq, seq)); return ret; } @@ -645,14 +653,14 @@ static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset) */ void timekeeping_set_tai_offset(s32 tai_offset) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; unsigned long flags; raw_spin_lock_irqsave(&timekeeper_lock, flags); - write_seqcount_begin(&timekeeper_seq); + write_seqcount_begin(&tk_core.seq); __timekeeping_set_tai_offset(tk, tai_offset); timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET); - write_seqcount_end(&timekeeper_seq); + write_seqcount_end(&tk_core.seq); raw_spin_unlock_irqrestore(&timekeeper_lock, flags); clock_was_set(); } @@ -664,14 +672,14 @@ void timekeeping_set_tai_offset(s32 tai_offset) */ static int change_clocksource(void *data) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; struct clocksource *new, *old; unsigned long flags; new = (struct clocksource *) data; raw_spin_lock_irqsave(&timekeeper_lock, flags); - write_seqcount_begin(&timekeeper_seq); + write_seqcount_begin(&tk_core.seq); timekeeping_forward_now(tk); /* @@ -691,7 +699,7 @@ static int change_clocksource(void *data) } timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); - write_seqcount_end(&timekeeper_seq); + write_seqcount_end(&tk_core.seq); raw_spin_unlock_irqrestore(&timekeeper_lock, flags); return 0; @@ -706,7 +714,7 @@ static int change_clocksource(void *data) */ int timekeeping_notify(struct clocksource *clock) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; if (tk->clock == clock) return 0; @@ -738,17 +746,17 @@ EXPORT_SYMBOL_GPL(ktime_get_real); */ void getrawmonotonic(struct timespec *ts) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; struct timespec64 ts64; unsigned long seq; s64 nsecs; do { - seq = read_seqcount_begin(&timekeeper_seq); + seq = read_seqcount_begin(&tk_core.seq); nsecs = timekeeping_get_ns_raw(tk); ts64 = tk->raw_time; - } while (read_seqcount_retry(&timekeeper_seq, seq)); + } while (read_seqcount_retry(&tk_core.seq, seq)); timespec64_add_ns(&ts64, nsecs); *ts = timespec64_to_timespec(ts64); @@ -760,16 +768,16 @@ EXPORT_SYMBOL(getrawmonotonic); */ int timekeeping_valid_for_hres(void) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; unsigned long seq; int ret; do { - seq = read_seqcount_begin(&timekeeper_seq); + seq = read_seqcount_begin(&tk_core.seq); ret = tk->clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; - } while (read_seqcount_retry(&timekeeper_seq, seq)); + } while (read_seqcount_retry(&tk_core.seq, seq)); return ret; } @@ -779,16 +787,16 @@ int timekeeping_valid_for_hres(void) */ u64 timekeeping_max_deferment(void) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; unsigned long seq; u64 ret; do { - seq = read_seqcount_begin(&timekeeper_seq); + seq = read_seqcount_begin(&tk_core.seq); ret = tk->clock->max_idle_ns; - } while (read_seqcount_retry(&timekeeper_seq, seq)); + } while (read_seqcount_retry(&tk_core.seq, seq)); return ret; } @@ -828,7 +836,7 @@ void __weak read_boot_clock(struct timespec *ts) */ void __init timekeeping_init(void) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; struct clocksource *clock; unsigned long flags; struct timespec64 now, boot, tmp; @@ -854,7 +862,7 @@ void __init timekeeping_init(void) } raw_spin_lock_irqsave(&timekeeper_lock, flags); - write_seqcount_begin(&timekeeper_seq); + write_seqcount_begin(&tk_core.seq); ntp_init(); clock = clocksource_default_clock(); @@ -875,9 +883,10 @@ void __init timekeeping_init(void) tmp.tv_nsec = 0; tk_set_sleep_time(tk, tmp); - memcpy(&shadow_timekeeper, &timekeeper, sizeof(timekeeper)); + memcpy(&shadow_timekeeper, &tk_core.timekeeper, + sizeof(tk_core.timekeeper)); - write_seqcount_end(&timekeeper_seq); + write_seqcount_end(&tk_core.seq); raw_spin_unlock_irqrestore(&timekeeper_lock, flags); } @@ -918,7 +927,7 @@ static void __timekeeping_inject_sleeptime(struct timekeeper *tk, */ void timekeeping_inject_sleeptime(struct timespec *delta) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; struct timespec64 tmp; unsigned long flags; @@ -930,7 +939,7 @@ void timekeeping_inject_sleeptime(struct timespec *delta) return; raw_spin_lock_irqsave(&timekeeper_lock, flags); - write_seqcount_begin(&timekeeper_seq); + write_seqcount_begin(&tk_core.seq); timekeeping_forward_now(tk); @@ -939,7 +948,7 @@ void timekeeping_inject_sleeptime(struct timespec *delta) timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); - write_seqcount_end(&timekeeper_seq); + write_seqcount_end(&tk_core.seq); raw_spin_unlock_irqrestore(&timekeeper_lock, flags); /* signal hrtimers about time change */ @@ -955,7 +964,7 @@ void timekeeping_inject_sleeptime(struct timespec *delta) */ static void timekeeping_resume(void) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; struct clocksource *clock = tk->clock; unsigned long flags; struct timespec64 ts_new, ts_delta; @@ -970,7 +979,7 @@ static void timekeeping_resume(void) clocksource_resume(); raw_spin_lock_irqsave(&timekeeper_lock, flags); - write_seqcount_begin(&timekeeper_seq); + write_seqcount_begin(&tk_core.seq); /* * After system resumes, we need to calculate the suspended time and @@ -1022,7 +1031,7 @@ static void timekeeping_resume(void) tk->ntp_error = 0; timekeeping_suspended = 0; timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET); - write_seqcount_end(&timekeeper_seq); + write_seqcount_end(&tk_core.seq); raw_spin_unlock_irqrestore(&timekeeper_lock, flags); touch_softlockup_watchdog(); @@ -1035,7 +1044,7 @@ static void timekeeping_resume(void) static int timekeeping_suspend(void) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; unsigned long flags; struct timespec64 delta, delta_delta; static struct timespec64 old_delta; @@ -1053,7 +1062,7 @@ static int timekeeping_suspend(void) persistent_clock_exist = true; raw_spin_lock_irqsave(&timekeeper_lock, flags); - write_seqcount_begin(&timekeeper_seq); + write_seqcount_begin(&tk_core.seq); timekeeping_forward_now(tk); timekeeping_suspended = 1; @@ -1078,7 +1087,7 @@ static int timekeeping_suspend(void) } timekeeping_update(tk, TK_MIRROR); - write_seqcount_end(&timekeeper_seq); + write_seqcount_end(&tk_core.seq); raw_spin_unlock_irqrestore(&timekeeper_lock, flags); clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL); @@ -1380,7 +1389,7 @@ static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset, void update_wall_time(void) { struct clocksource *clock; - struct timekeeper *real_tk = &timekeeper; + struct timekeeper *real_tk = &tk_core.timekeeper; struct timekeeper *tk = &shadow_timekeeper; cycle_t offset; int shift = 0, maxshift; @@ -1440,7 +1449,7 @@ void update_wall_time(void) */ clock_set |= accumulate_nsecs_to_secs(tk); - write_seqcount_begin(&timekeeper_seq); + write_seqcount_begin(&tk_core.seq); /* Update clock->cycle_last with the new value */ clock->cycle_last = tk->cycle_last; /* @@ -1450,12 +1459,12 @@ void update_wall_time(void) * requires changes to all other timekeeper usage sites as * well, i.e. move the timekeeper pointer getter into the * spinlocked/seqcount protected sections. And we trade this - * memcpy under the timekeeper_seq against one before we start + * memcpy under the tk_core.seq against one before we start * updating. */ memcpy(real_tk, tk, sizeof(*tk)); timekeeping_update(real_tk, clock_set); - write_seqcount_end(&timekeeper_seq); + write_seqcount_end(&tk_core.seq); out: raw_spin_unlock_irqrestore(&timekeeper_lock, flags); if (clock_set) @@ -1476,7 +1485,7 @@ out: */ void getboottime(struct timespec *ts) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; struct timespec boottime = { .tv_sec = tk->wall_to_monotonic.tv_sec + tk->total_sleep_time.tv_sec, @@ -1499,7 +1508,7 @@ EXPORT_SYMBOL_GPL(getboottime); */ void get_monotonic_boottime(struct timespec *ts) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; struct timespec64 tomono, sleep, ret; s64 nsec; unsigned int seq; @@ -1507,13 +1516,13 @@ void get_monotonic_boottime(struct timespec *ts) WARN_ON(timekeeping_suspended); do { - seq = read_seqcount_begin(&timekeeper_seq); + seq = read_seqcount_begin(&tk_core.seq); ret.tv_sec = tk->xtime_sec; nsec = timekeeping_get_ns(tk); tomono = tk->wall_to_monotonic; sleep = tk->total_sleep_time; - } while (read_seqcount_retry(&timekeeper_seq, seq)); + } while (read_seqcount_retry(&tk_core.seq, seq)); ret.tv_sec += tomono.tv_sec + sleep.tv_sec; ret.tv_nsec = 0; @@ -1545,7 +1554,7 @@ EXPORT_SYMBOL_GPL(ktime_get_boottime); */ void monotonic_to_bootbased(struct timespec *ts) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; struct timespec64 ts64; ts64 = timespec_to_timespec64(*ts); @@ -1556,7 +1565,7 @@ EXPORT_SYMBOL_GPL(monotonic_to_bootbased); unsigned long get_seconds(void) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; return tk->xtime_sec; } @@ -1564,22 +1573,22 @@ EXPORT_SYMBOL(get_seconds); struct timespec __current_kernel_time(void) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; return timespec64_to_timespec(tk_xtime(tk)); } struct timespec current_kernel_time(void) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; struct timespec64 now; unsigned long seq; do { - seq = read_seqcount_begin(&timekeeper_seq); + seq = read_seqcount_begin(&tk_core.seq); now = tk_xtime(tk); - } while (read_seqcount_retry(&timekeeper_seq, seq)); + } while (read_seqcount_retry(&tk_core.seq, seq)); return timespec64_to_timespec(now); } @@ -1587,16 +1596,16 @@ EXPORT_SYMBOL(current_kernel_time); struct timespec get_monotonic_coarse(void) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; struct timespec64 now, mono; unsigned long seq; do { - seq = read_seqcount_begin(&timekeeper_seq); + seq = read_seqcount_begin(&tk_core.seq); now = tk_xtime(tk); mono = tk->wall_to_monotonic; - } while (read_seqcount_retry(&timekeeper_seq, seq)); + } while (read_seqcount_retry(&tk_core.seq, seq)); set_normalized_timespec64(&now, now.tv_sec + mono.tv_sec, now.tv_nsec + mono.tv_nsec); @@ -1624,19 +1633,19 @@ void do_timer(unsigned long ticks) ktime_t ktime_get_update_offsets_tick(ktime_t *offs_real, ktime_t *offs_boot, ktime_t *offs_tai) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; struct timespec64 ts; ktime_t now; unsigned int seq; do { - seq = read_seqcount_begin(&timekeeper_seq); + seq = read_seqcount_begin(&tk_core.seq); ts = tk_xtime(tk); *offs_real = tk->offs_real; *offs_boot = tk->offs_boot; *offs_tai = tk->offs_tai; - } while (read_seqcount_retry(&timekeeper_seq, seq)); + } while (read_seqcount_retry(&tk_core.seq, seq)); now = ktime_set(ts.tv_sec, ts.tv_nsec); now = ktime_sub(now, *offs_real); @@ -1656,13 +1665,13 @@ ktime_t ktime_get_update_offsets_tick(ktime_t *offs_real, ktime_t *offs_boot, ktime_t ktime_get_update_offsets_now(ktime_t *offs_real, ktime_t *offs_boot, ktime_t *offs_tai) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; ktime_t now; unsigned int seq; u64 secs, nsecs; do { - seq = read_seqcount_begin(&timekeeper_seq); + seq = read_seqcount_begin(&tk_core.seq); secs = tk->xtime_sec; nsecs = timekeeping_get_ns(tk); @@ -1670,7 +1679,7 @@ ktime_t ktime_get_update_offsets_now(ktime_t *offs_real, ktime_t *offs_boot, *offs_real = tk->offs_real; *offs_boot = tk->offs_boot; *offs_tai = tk->offs_tai; - } while (read_seqcount_retry(&timekeeper_seq, seq)); + } while (read_seqcount_retry(&tk_core.seq, seq)); now = ktime_add_ns(ktime_set(secs, 0), nsecs); now = ktime_sub(now, *offs_real); @@ -1683,14 +1692,14 @@ ktime_t ktime_get_update_offsets_now(ktime_t *offs_real, ktime_t *offs_boot, */ ktime_t ktime_get_monotonic_offset(void) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; unsigned long seq; struct timespec64 wtom; do { - seq = read_seqcount_begin(&timekeeper_seq); + seq = read_seqcount_begin(&tk_core.seq); wtom = tk->wall_to_monotonic; - } while (read_seqcount_retry(&timekeeper_seq, seq)); + } while (read_seqcount_retry(&tk_core.seq, seq)); return timespec64_to_ktime(wtom); } @@ -1701,7 +1710,7 @@ EXPORT_SYMBOL_GPL(ktime_get_monotonic_offset); */ int do_adjtimex(struct timex *txc) { - struct timekeeper *tk = &timekeeper; + struct timekeeper *tk = &tk_core.timekeeper; unsigned long flags; struct timespec64 ts; s32 orig_tai, tai; @@ -1726,7 +1735,7 @@ int do_adjtimex(struct timex *txc) getnstimeofday64(&ts); raw_spin_lock_irqsave(&timekeeper_lock, flags); - write_seqcount_begin(&timekeeper_seq); + write_seqcount_begin(&tk_core.seq); orig_tai = tai = tk->tai_offset; ret = __do_adjtimex(txc, &ts, &tai); @@ -1735,7 +1744,7 @@ int do_adjtimex(struct timex *txc) __timekeeping_set_tai_offset(tk, tai); timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET); } - write_seqcount_end(&timekeeper_seq); + write_seqcount_end(&tk_core.seq); raw_spin_unlock_irqrestore(&timekeeper_lock, flags); if (tai != orig_tai) @@ -1755,11 +1764,11 @@ void hardpps(const struct timespec *phase_ts, const struct timespec *raw_ts) unsigned long flags; raw_spin_lock_irqsave(&timekeeper_lock, flags); - write_seqcount_begin(&timekeeper_seq); + write_seqcount_begin(&tk_core.seq); __hardpps(phase_ts, raw_ts); - write_seqcount_end(&timekeeper_seq); + write_seqcount_end(&tk_core.seq); raw_spin_unlock_irqrestore(&timekeeper_lock, flags); } EXPORT_SYMBOL(hardpps); From f111adfdd7ff7d9fe54b6efa440b80824984749c Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:09 +0000 Subject: [PATCH 059/114] timekeeping: Use timekeeping_update() instead of memcpy() We already have a function which does the right thing, that also makes sure that the coming ktime_t based cached values are getting updated. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- kernel/time/timekeeping.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 7ca150ad387d..bfe3ea09afc9 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -883,8 +883,7 @@ void __init timekeeping_init(void) tmp.tv_nsec = 0; tk_set_sleep_time(tk, tmp); - memcpy(&shadow_timekeeper, &tk_core.timekeeper, - sizeof(tk_core.timekeeper)); + timekeeping_update(tk, TK_MIRROR); write_seqcount_end(&tk_core.seq); raw_spin_unlock_irqrestore(&timekeeper_lock, flags); From 7c032df5570388044b4efda3d9f4d2ffb96a3116 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:10 +0000 Subject: [PATCH 060/114] timekeeping: Provide internal ktime_t based data The ktime_t based interfaces are used a lot in performance critical code pathes. Add ktime_t based data so the interfaces don't have to convert from the xtime/timespec based data. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/timekeeper_internal.h | 3 +++ kernel/time/timekeeping.c | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h index 2cb96235c249..87e0992564f2 100644 --- a/include/linux/timekeeper_internal.h +++ b/include/linux/timekeeper_internal.h @@ -36,6 +36,9 @@ struct timekeeper { /* Clock shifted nano seconds */ u64 xtime_nsec; + /* Monotonic base time */ + ktime_t base_mono; + /* Current CLOCK_REALTIME time in seconds */ u64 xtime_sec; /* CLOCK_REALTIME to CLOCK_MONOTONIC offset */ diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index bfe3ea09afc9..86a92476c027 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -291,6 +291,26 @@ int pvclock_gtod_unregister_notifier(struct notifier_block *nb) } EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier); +/* + * Update the ktime_t based scalar nsec members of the timekeeper + */ +static inline void tk_update_ktime_data(struct timekeeper *tk) +{ + s64 nsec; + + /* + * The xtime based monotonic readout is: + * nsec = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec + now(); + * The ktime based monotonic readout is: + * nsec = base_mono + now(); + * ==> base_mono = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec + */ + nsec = (s64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec); + nsec *= NSEC_PER_SEC; + nsec += tk->wall_to_monotonic.tv_nsec; + tk->base_mono = ns_to_ktime(nsec); +} + /* must hold timekeeper_lock */ static void timekeeping_update(struct timekeeper *tk, unsigned int action) { @@ -301,6 +321,8 @@ static void timekeeping_update(struct timekeeper *tk, unsigned int action) update_vsyscall(tk); update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET); + tk_update_ktime_data(tk); + if (action & TK_MIRROR) memcpy(&shadow_timekeeper, &tk_core.timekeeper, sizeof(tk_core.timekeeper)); From a016a5bd62e29a738531d9d4d925037a1fdb52f5 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:12 +0000 Subject: [PATCH 061/114] timekeeping: Use ktime_t based data for ktime_get() Speed up ktime_get() by using ktime_t based data. Text size shrinks by 64 bytes on x8664. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- kernel/time/timekeeping.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 86a92476c027..d5be1425cc03 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -407,18 +407,19 @@ ktime_t ktime_get(void) { struct timekeeper *tk = &tk_core.timekeeper; unsigned int seq; - s64 secs, nsecs; + ktime_t base; + s64 nsecs; WARN_ON(timekeeping_suspended); do { seq = read_seqcount_begin(&tk_core.seq); - secs = tk->xtime_sec + tk->wall_to_monotonic.tv_sec; - nsecs = timekeeping_get_ns(tk) + tk->wall_to_monotonic.tv_nsec; + base = tk->base_mono; + nsecs = timekeeping_get_ns(tk); } while (read_seqcount_retry(&tk_core.seq, seq)); - return ktime_set(secs, nsecs); + return ktime_add_ns(base, nsecs); } EXPORT_SYMBOL_GPL(ktime_get); From 0077dc60f274b9a7e9aa705a34784fefb87e0eee Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:13 +0000 Subject: [PATCH 062/114] timekeeping: Provide ktime_get_with_offset() Provide a helper function which lets us implement ktime_t based interfaces for real, boot and tai clocks. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/timekeeping.h | 9 +++++++++ kernel/time/timekeeping.c | 27 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h index 3eb19e34cc20..a58e4b1879db 100644 --- a/include/linux/timekeeping.h +++ b/include/linux/timekeeping.h @@ -98,7 +98,16 @@ extern void getboottime(struct timespec *ts); /* * ktime_t based interfaces */ + +enum tk_offsets { + TK_OFFS_REAL, + TK_OFFS_BOOT, + TK_OFFS_TAI, + TK_OFFS_MAX, +}; + extern ktime_t ktime_get(void); +extern ktime_t ktime_get_with_offset(enum tk_offsets offs); extern ktime_t ktime_get_real(void); extern ktime_t ktime_get_boottime(void); extern ktime_t ktime_get_monotonic_offset(void); diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index d5be1425cc03..7c5f5e4a006c 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -423,6 +423,33 @@ ktime_t ktime_get(void) } EXPORT_SYMBOL_GPL(ktime_get); +static ktime_t *offsets[TK_OFFS_MAX] = { + [TK_OFFS_REAL] = &tk_core.timekeeper.offs_real, + [TK_OFFS_BOOT] = &tk_core.timekeeper.offs_boot, + [TK_OFFS_TAI] = &tk_core.timekeeper.offs_tai, +}; + +ktime_t ktime_get_with_offset(enum tk_offsets offs) +{ + struct timekeeper *tk = &tk_core.timekeeper; + unsigned int seq; + ktime_t base, *offset = offsets[offs]; + s64 nsecs; + + WARN_ON(timekeeping_suspended); + + do { + seq = read_seqcount_begin(&tk_core.seq); + base = ktime_add(tk->base_mono, *offset); + nsecs = timekeeping_get_ns(tk); + + } while (read_seqcount_retry(&tk_core.seq, seq)); + + return ktime_add_ns(base, nsecs); + +} +EXPORT_SYMBOL_GPL(ktime_get_with_offset); + /** * ktime_get_ts64 - get the monotonic clock in timespec64 format * @ts: pointer to timespec variable From f5264d5d5a0729306cc792d84432b97785d2662a Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:14 +0000 Subject: [PATCH 063/114] timekeeping: Use ktime_t based data for ktime_get_real() Speed up the readout. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/timekeeping.h | 9 ++++++++- kernel/time/timekeeping.c | 15 --------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h index a58e4b1879db..68e6678a743b 100644 --- a/include/linux/timekeeping.h +++ b/include/linux/timekeeping.h @@ -108,11 +108,18 @@ enum tk_offsets { extern ktime_t ktime_get(void); extern ktime_t ktime_get_with_offset(enum tk_offsets offs); -extern ktime_t ktime_get_real(void); extern ktime_t ktime_get_boottime(void); extern ktime_t ktime_get_monotonic_offset(void); extern ktime_t ktime_get_clocktai(void); +/** + * ktime_get_real - get the real (wall-) time in ktime_t format + */ +static inline ktime_t ktime_get_real(void) +{ + return ktime_get_with_offset(TK_OFFS_REAL); +} + /* * RTC specific */ diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 7c5f5e4a006c..56db2e16970a 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -773,21 +773,6 @@ int timekeeping_notify(struct clocksource *clock) return tk->clock == clock ? 0 : -1; } -/** - * ktime_get_real - get the real (wall-) time in ktime_t format - * - * returns the time in ktime_t format - */ -ktime_t ktime_get_real(void) -{ - struct timespec64 now; - - getnstimeofday64(&now); - - return timespec64_to_ktime(now); -} -EXPORT_SYMBOL_GPL(ktime_get_real); - /** * getrawmonotonic - Returns the raw monotonic time in a timespec * @ts: pointer to the timespec to be set From b82c817e2d16e818c472eb71019de521816000a3 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:16 +0000 Subject: [PATCH 064/114] timekeeping; Use ktime_t based data for ktime_get_boottime() Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/timekeeping.h | 12 +++++++++++- kernel/time/timekeeping.c | 17 ----------------- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h index 68e6678a743b..2fc606203c8c 100644 --- a/include/linux/timekeeping.h +++ b/include/linux/timekeeping.h @@ -108,7 +108,6 @@ enum tk_offsets { extern ktime_t ktime_get(void); extern ktime_t ktime_get_with_offset(enum tk_offsets offs); -extern ktime_t ktime_get_boottime(void); extern ktime_t ktime_get_monotonic_offset(void); extern ktime_t ktime_get_clocktai(void); @@ -120,6 +119,17 @@ static inline ktime_t ktime_get_real(void) return ktime_get_with_offset(TK_OFFS_REAL); } +/** + * ktime_get_boottime - Returns monotonic time since boot in ktime_t format + * + * This is similar to CLOCK_MONTONIC/ktime_get, but also includes the + * time spent in suspend. + */ +static inline ktime_t ktime_get_boottime(void) +{ + return ktime_get_with_offset(TK_OFFS_BOOT); +} + /* * RTC specific */ diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 56db2e16970a..5e60aa09af79 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -1565,23 +1565,6 @@ void get_monotonic_boottime(struct timespec *ts) } EXPORT_SYMBOL_GPL(get_monotonic_boottime); -/** - * ktime_get_boottime - Returns monotonic time since boot in a ktime - * - * Returns the monotonic time since boot in a ktime - * - * This is similar to CLOCK_MONTONIC/ktime_get, but also - * includes the time spent in suspend. - */ -ktime_t ktime_get_boottime(void) -{ - struct timespec ts; - - get_monotonic_boottime(&ts); - return timespec_to_ktime(ts); -} -EXPORT_SYMBOL_GPL(ktime_get_boottime); - /** * monotonic_to_bootbased - Convert the monotonic time to boot based. * @ts: pointer to the timespec to be converted From afab07c0e91ecf098abf34573ccfcd86d6be26f9 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:17 +0000 Subject: [PATCH 065/114] timekeeping: Use ktime_t based data for ktime_get_clocktai() Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/timekeeping.h | 9 ++++++++- kernel/time/timekeeping.c | 15 --------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h index 2fc606203c8c..3050a7d0a5a9 100644 --- a/include/linux/timekeeping.h +++ b/include/linux/timekeeping.h @@ -109,7 +109,6 @@ enum tk_offsets { extern ktime_t ktime_get(void); extern ktime_t ktime_get_with_offset(enum tk_offsets offs); extern ktime_t ktime_get_monotonic_offset(void); -extern ktime_t ktime_get_clocktai(void); /** * ktime_get_real - get the real (wall-) time in ktime_t format @@ -130,6 +129,14 @@ static inline ktime_t ktime_get_boottime(void) return ktime_get_with_offset(TK_OFFS_BOOT); } +/** + * ktime_get_clocktai - Returns the TAI time of day in ktime_t format + */ +static inline ktime_t ktime_get_clocktai(void) +{ + return ktime_get_with_offset(TK_OFFS_TAI); +} + /* * RTC specific */ diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 5e60aa09af79..c083ae2c34b5 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -512,21 +512,6 @@ void timekeeping_clocktai(struct timespec *ts) } EXPORT_SYMBOL(timekeeping_clocktai); - -/** - * ktime_get_clocktai - Returns the TAI time of day in a ktime - * - * Returns the time of day in a ktime. - */ -ktime_t ktime_get_clocktai(void) -{ - struct timespec ts; - - timekeeping_clocktai(&ts); - return timespec_to_ktime(ts); -} -EXPORT_SYMBOL(ktime_get_clocktai); - #ifdef CONFIG_NTP_PPS /** From a37c0aad6093575b52432b47b145304f1af18dff Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:19 +0000 Subject: [PATCH 066/114] timekeeping: Use ktime_t data for ktime_get_update_offsets_now() No need to juggle with timespecs. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- kernel/time/timekeeping.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index c083ae2c34b5..54d90529f99d 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -1668,14 +1668,14 @@ ktime_t ktime_get_update_offsets_now(ktime_t *offs_real, ktime_t *offs_boot, ktime_t *offs_tai) { struct timekeeper *tk = &tk_core.timekeeper; - ktime_t now; unsigned int seq; - u64 secs, nsecs; + ktime_t base; + u64 nsecs; do { seq = read_seqcount_begin(&tk_core.seq); - secs = tk->xtime_sec; + base = tk->base_mono; nsecs = timekeeping_get_ns(tk); *offs_real = tk->offs_real; @@ -1683,9 +1683,7 @@ ktime_t ktime_get_update_offsets_now(ktime_t *offs_real, ktime_t *offs_boot, *offs_tai = tk->offs_tai; } while (read_seqcount_retry(&tk_core.seq, seq)); - now = ktime_add_ns(ktime_set(secs, 0), nsecs); - now = ktime_sub(now, *offs_real); - return now; + return ktime_add_ns(base, nsecs); } #endif From 48064f5f67d58f95094305ac575d5372b58e265f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:20 +0000 Subject: [PATCH 067/114] timekeeping; Use ktime based data for ktime_get_update_offsets_tick() No need to juggle with timespecs. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- kernel/time/timekeeping.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 54d90529f99d..e99350319eec 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -1636,22 +1636,22 @@ ktime_t ktime_get_update_offsets_tick(ktime_t *offs_real, ktime_t *offs_boot, ktime_t *offs_tai) { struct timekeeper *tk = &tk_core.timekeeper; - struct timespec64 ts; - ktime_t now; unsigned int seq; + ktime_t base; + u64 nsecs; do { seq = read_seqcount_begin(&tk_core.seq); - ts = tk_xtime(tk); + base = tk->base_mono; + nsecs = tk->xtime_nsec >> tk->shift; + *offs_real = tk->offs_real; *offs_boot = tk->offs_boot; *offs_tai = tk->offs_tai; } while (read_seqcount_retry(&tk_core.seq, seq)); - now = ktime_set(ts.tv_sec, ts.tv_nsec); - now = ktime_sub(now, *offs_real); - return now; + return ktime_add_ns(base, nsecs); } #ifdef CONFIG_HIGH_RES_TIMERS From 9a6b51976ea3a326b6de534beec3fd87275f4ef6 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:22 +0000 Subject: [PATCH 068/114] timekeeping: Provide ktime_mono_to_any() ktime based conversion function to map a monotonic time stamp to a different CLOCK. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/timekeeping.h | 9 +++++++++ kernel/time/timekeeping.c | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h index 3050a7d0a5a9..910a98ef2154 100644 --- a/include/linux/timekeeping.h +++ b/include/linux/timekeeping.h @@ -109,6 +109,7 @@ enum tk_offsets { extern ktime_t ktime_get(void); extern ktime_t ktime_get_with_offset(enum tk_offsets offs); extern ktime_t ktime_get_monotonic_offset(void); +extern ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs); /** * ktime_get_real - get the real (wall-) time in ktime_t format @@ -137,6 +138,14 @@ static inline ktime_t ktime_get_clocktai(void) return ktime_get_with_offset(TK_OFFS_TAI); } +/** + * ktime_mono_to_real - Convert monotonic time to clock realtime + */ +static inline ktime_t ktime_mono_to_real(ktime_t mono) +{ + return ktime_mono_to_any(mono, TK_OFFS_REAL); +} + /* * RTC specific */ diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index e99350319eec..032e77a54a79 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -450,6 +450,26 @@ ktime_t ktime_get_with_offset(enum tk_offsets offs) } EXPORT_SYMBOL_GPL(ktime_get_with_offset); +/** + * ktime_mono_to_any() - convert mononotic time to any other time + * @tmono: time to convert. + * @offs: which offset to use + */ +ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs) +{ + ktime_t *offset = offsets[offs]; + unsigned long seq; + ktime_t tconv; + + do { + seq = read_seqcount_begin(&tk_core.seq); + tconv = ktime_add(tmono, *offset); + } while (read_seqcount_retry(&tk_core.seq, seq)); + + return tconv; +} +EXPORT_SYMBOL_GPL(ktime_mono_to_any); + /** * ktime_get_ts64 - get the monotonic clock in timespec64 format * @ts: pointer to timespec variable From 53cc7bad37fcb90e47ef729ef9818a2ed93ee862 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:23 +0000 Subject: [PATCH 069/114] timerfd: Use ktime_mono_to_real() We have a few other use cases of ktime_get_monotonic_offset() which can be optimized with ktime_mono_to_real(). The timerfd code uses the offset only for comparison, so we can use ktime_mono_to_real(0) for this as well. Funny enough text size shrinks with that on ARM and x8664 !? Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- fs/timerfd.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/timerfd.c b/fs/timerfd.c index 709603cac9e6..80c350216ea8 100644 --- a/fs/timerfd.c +++ b/fs/timerfd.c @@ -93,7 +93,7 @@ static enum alarmtimer_restart timerfd_alarmproc(struct alarm *alarm, */ void timerfd_clock_was_set(void) { - ktime_t moffs = ktime_get_monotonic_offset(); + ktime_t moffs = ktime_mono_to_real((ktime_t){ .tv64 = 0 }); struct timerfd_ctx *ctx; unsigned long flags; @@ -126,7 +126,7 @@ static bool timerfd_canceled(struct timerfd_ctx *ctx) { if (!ctx->might_cancel || ctx->moffs.tv64 != KTIME_MAX) return false; - ctx->moffs = ktime_get_monotonic_offset(); + ctx->moffs = ktime_mono_to_real((ktime_t){ .tv64 = 0 }); return true; } @@ -405,7 +405,7 @@ SYSCALL_DEFINE2(timerfd_create, int, clockid, int, flags) else hrtimer_init(&ctx->t.tmr, clockid, HRTIMER_MODE_ABS); - ctx->moffs = ktime_get_monotonic_offset(); + ctx->moffs = ktime_mono_to_real((ktime_t){ .tv64 = 0 }); ufd = anon_inode_getfd("[timerfd]", &timerfd_fops, ctx, O_RDWR | (flags & TFD_SHARED_FCNTL_FLAGS)); From 5cac2f4d1c5f3eb3d981fab85d926d19f7b6b889 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:25 +0000 Subject: [PATCH 070/114] input: evdev: Use ktime_mono_to_real() Convert the monotonic timestamp with ktime_mono_to_real() in evdev_events(). In evdev_queue_syn_dropped() we can call either ktime_get() or ktime_get_real() depending on the clkid. No point in having two calls for CLOCK_REALTIME. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- drivers/input/evdev.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/input/evdev.c b/drivers/input/evdev.c index fd325ec9f064..de055451d1af 100644 --- a/drivers/input/evdev.c +++ b/drivers/input/evdev.c @@ -108,9 +108,8 @@ static void evdev_queue_syn_dropped(struct evdev_client *client) struct input_event ev; ktime_t time; - time = ktime_get(); - if (client->clkid != CLOCK_MONOTONIC) - time = ktime_sub(time, ktime_get_monotonic_offset()); + time = (client->clkid == CLOCK_MONOTONIC) ? + ktime_get() : ktime_get_real(); ev.time = ktime_to_timeval(time); ev.type = EV_SYN; @@ -202,7 +201,7 @@ static void evdev_events(struct input_handle *handle, ktime_t time_mono, time_real; time_mono = ktime_get(); - time_real = ktime_sub(time_mono, ktime_get_monotonic_offset()); + time_real = ktime_mono_to_real(time_mono); rcu_read_lock(); From d2cb58c85ce4789caafa66fa8f1c641ab2504ead Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:26 +0000 Subject: [PATCH 071/114] drm: Use ktime_mono_to_real() Convert the monotonic timestamp with ktime_mono_to_real() in drm_calc_vbltimestamp_from_scanoutpos(). In get_drm_timestamp we can call either ktime_get() or ktime_get_real() depending on drm_timestamp_monotonic. No point in having two calls into the core for CLOCK_REALTIME. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- drivers/gpu/drm/drm_irq.c | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/drm_irq.c b/drivers/gpu/drm/drm_irq.c index 0de123afdb34..08ba1209228e 100644 --- a/drivers/gpu/drm/drm_irq.c +++ b/drivers/gpu/drm/drm_irq.c @@ -542,8 +542,8 @@ int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc, const struct drm_crtc *refcrtc, const struct drm_display_mode *mode) { - ktime_t stime, etime, mono_time_offset; struct timeval tv_etime; + ktime_t stime, etime; int vbl_status; int vpos, hpos, i; int framedur_ns, linedur_ns, pixeldur_ns, delta_ns, duration_ns; @@ -588,13 +588,6 @@ int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc, vbl_status = dev->driver->get_scanout_position(dev, crtc, flags, &vpos, &hpos, &stime, &etime); - /* - * Get correction for CLOCK_MONOTONIC -> CLOCK_REALTIME if - * CLOCK_REALTIME is requested. - */ - if (!drm_timestamp_monotonic) - mono_time_offset = ktime_get_monotonic_offset(); - /* Return as no-op if scanout query unsupported or failed. */ if (!(vbl_status & DRM_SCANOUTPOS_VALID)) { DRM_DEBUG("crtc %d : scanoutpos query failed [%d].\n", @@ -633,7 +626,7 @@ int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc, delta_ns = vpos * linedur_ns + hpos * pixeldur_ns; if (!drm_timestamp_monotonic) - etime = ktime_sub(etime, mono_time_offset); + etime = ktime_mono_to_real(etime); /* save this only for debugging purposes */ tv_etime = ktime_to_timeval(etime); @@ -664,10 +657,7 @@ static struct timeval get_drm_timestamp(void) { ktime_t now; - now = ktime_get(); - if (!drm_timestamp_monotonic) - now = ktime_sub(now, ktime_get_monotonic_offset()); - + now = drm_timestamp_monotonic ? ktime_get() : ktime_get_real(); return ktime_to_timeval(now); } From dcaab54e348c5b66cca4802815ceebd37059e70c Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:28 +0000 Subject: [PATCH 072/114] timekeeping: Remove ktime_get_monotonic_offset() No more users. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/timekeeping.h | 1 - kernel/time/timekeeping.c | 18 ------------------ 2 files changed, 19 deletions(-) diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h index 910a98ef2154..64c81f367866 100644 --- a/include/linux/timekeeping.h +++ b/include/linux/timekeeping.h @@ -108,7 +108,6 @@ enum tk_offsets { extern ktime_t ktime_get(void); extern ktime_t ktime_get_with_offset(enum tk_offsets offs); -extern ktime_t ktime_get_monotonic_offset(void); extern ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs); /** diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 032e77a54a79..f7378eaebe67 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -1707,24 +1707,6 @@ ktime_t ktime_get_update_offsets_now(ktime_t *offs_real, ktime_t *offs_boot, } #endif -/** - * ktime_get_monotonic_offset() - get wall_to_monotonic in ktime_t format - */ -ktime_t ktime_get_monotonic_offset(void) -{ - struct timekeeper *tk = &tk_core.timekeeper; - unsigned long seq; - struct timespec64 wtom; - - do { - seq = read_seqcount_begin(&tk_core.seq); - wtom = tk->wall_to_monotonic; - } while (read_seqcount_retry(&tk_core.seq, seq)); - - return timespec64_to_ktime(wtom); -} -EXPORT_SYMBOL_GPL(ktime_get_monotonic_offset); - /** * do_adjtimex() - Accessor function to NTP __do_adjtimex function */ From 897994e32b2b0a41ce4222c3b38a05bd2d1ee9fa Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:29 +0000 Subject: [PATCH 073/114] timekeeping: Provide ktime_get[*]_ns() helpers A lot of code converts either timespecs or ktime_t to nanoseconds. Provide helper functions. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/timekeeping.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h index 64c81f367866..903ecc10fcff 100644 --- a/include/linux/timekeeping.h +++ b/include/linux/timekeeping.h @@ -145,6 +145,21 @@ static inline ktime_t ktime_mono_to_real(ktime_t mono) return ktime_mono_to_any(mono, TK_OFFS_REAL); } +static inline u64 ktime_get_ns(void) +{ + return ktime_to_ns(ktime_get()); +} + +static inline u64 ktime_get_real_ns(void) +{ + return ktime_to_ns(ktime_get_real()); +} + +static inline u64 ktime_get_boot_ns(void) +{ + return ktime_to_ns(ktime_get_boottime()); +} + /* * RTC specific */ From d560fed6abe0f9975b509e4fb824e08ac19adc93 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:31 +0000 Subject: [PATCH 074/114] time: Export nsecs_to_jiffies() Required for moving drivers to the nanosecond based interfaces. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- kernel/time/time.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/time/time.c b/kernel/time/time.c index 278c63cc8054..f0294ba14634 100644 --- a/kernel/time/time.c +++ b/kernel/time/time.c @@ -757,6 +757,7 @@ unsigned long nsecs_to_jiffies(u64 n) { return (unsigned long)nsecs_to_jiffies64(n); } +EXPORT_SYMBOL_GPL(nsecs_to_jiffies); /* * Add two timespec values and do a safety check for overflow. From 57e0be041d9e21a7397eed3b67a7936ac4ac83c0 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:32 +0000 Subject: [PATCH 075/114] sched: Make task->real_start_time nanoseconds based Simplify the only user of this data by removing the timespec conversion. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- fs/proc/array.c | 7 +------ include/linux/sched.h | 2 +- kernel/fork.c | 3 +-- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/fs/proc/array.c b/fs/proc/array.c index 64db2bceac59..d7f9199217bb 100644 --- a/fs/proc/array.c +++ b/fs/proc/array.c @@ -473,13 +473,8 @@ static int do_task_stat(struct seq_file *m, struct pid_namespace *ns, priority = task_prio(task); nice = task_nice(task); - /* Temporary variable needed for gcc-2.96 */ - /* convert timespec -> nsec*/ - start_time = - (unsigned long long)task->real_start_time.tv_sec * NSEC_PER_SEC - + task->real_start_time.tv_nsec; /* convert nsec -> ticks */ - start_time = nsec_to_clock_t(start_time); + start_time = nsec_to_clock_t(task->real_start_time); seq_printf(m, "%d (%s) %c", pid_nr_ns(pid, ns), tcomm, state); seq_put_decimal_ll(m, ' ', ppid); diff --git a/include/linux/sched.h b/include/linux/sched.h index 306f4f0c987a..67678fa76f99 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1368,7 +1368,7 @@ struct task_struct { #endif unsigned long nvcsw, nivcsw; /* context switch counts */ struct timespec start_time; /* monotonic time */ - struct timespec real_start_time; /* boot based time */ + u64 real_start_time; /* boot based time in nsec */ /* mm fault and swap info: this can arguably be seen as either mm-specific or thread-specific */ unsigned long min_flt, maj_flt; diff --git a/kernel/fork.c b/kernel/fork.c index 8f541930ce26..a7ab82db2f60 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1263,8 +1263,7 @@ static struct task_struct *copy_process(unsigned long clone_flags, posix_cpu_timers_init(p); ktime_get_ts(&p->start_time); - p->real_start_time = p->start_time; - monotonic_to_bootbased(&p->real_start_time); + p->real_start_time = ktime_get_boot_ns(); p->io_context = NULL; p->audit_context = NULL; if (clone_flags & CLONE_THREAD) From ccbf62d8a284cf181ac28c8e8407dd077d90dd4b Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:34 +0000 Subject: [PATCH 076/114] sched: Make task->start_time nanoseconds based Simplify the timespec to nsec/usec conversions. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/sched.h | 2 +- kernel/acct.c | 10 +++------- kernel/fork.c | 2 +- kernel/tsacct.c | 19 +++++++++---------- 4 files changed, 14 insertions(+), 19 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index 67678fa76f99..10c6e829927f 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1367,7 +1367,7 @@ struct task_struct { } vtime_snap_whence; #endif unsigned long nvcsw, nivcsw; /* context switch counts */ - struct timespec start_time; /* monotonic time */ + u64 start_time; /* monotonic time in nsec */ u64 real_start_time; /* boot based time in nsec */ /* mm fault and swap info: this can arguably be seen as either mm-specific or thread-specific */ unsigned long min_flt, maj_flt; diff --git a/kernel/acct.c b/kernel/acct.c index 1be013c6053e..a1844f14c6d6 100644 --- a/kernel/acct.c +++ b/kernel/acct.c @@ -458,9 +458,7 @@ static void do_acct_process(struct bsd_acct_struct *acct, acct_t ac; mm_segment_t fs; unsigned long flim; - u64 elapsed; - u64 run_time; - struct timespec uptime; + u64 elapsed, run_time; struct tty_struct *tty; const struct cred *orig_cred; @@ -484,10 +482,8 @@ static void do_acct_process(struct bsd_acct_struct *acct, strlcpy(ac.ac_comm, current->comm, sizeof(ac.ac_comm)); /* calculate run_time in nsec*/ - ktime_get_ts(&uptime); - run_time = (u64)uptime.tv_sec*NSEC_PER_SEC + uptime.tv_nsec; - run_time -= (u64)current->group_leader->start_time.tv_sec * NSEC_PER_SEC - + current->group_leader->start_time.tv_nsec; + run_time = ktime_get_ns(); + run_time -= current->group_leader->start_time; /* convert nsec -> AHZ */ elapsed = nsec_to_AHZ(run_time); #if ACCT_VERSION==3 diff --git a/kernel/fork.c b/kernel/fork.c index a7ab82db2f60..627b7f80afb0 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1262,7 +1262,7 @@ static struct task_struct *copy_process(unsigned long clone_flags, posix_cpu_timers_init(p); - ktime_get_ts(&p->start_time); + p->start_time = ktime_get_ns(); p->real_start_time = ktime_get_boot_ns(); p->io_context = NULL; p->audit_context = NULL; diff --git a/kernel/tsacct.c b/kernel/tsacct.c index ea6d170452c4..975cb49e32bf 100644 --- a/kernel/tsacct.c +++ b/kernel/tsacct.c @@ -31,20 +31,19 @@ void bacct_add_tsk(struct user_namespace *user_ns, struct taskstats *stats, struct task_struct *tsk) { const struct cred *tcred; - struct timespec uptime, ts; cputime_t utime, stime, utimescaled, stimescaled; - u64 ac_etime; + u64 delta; BUILD_BUG_ON(TS_COMM_LEN < TASK_COMM_LEN); - /* calculate task elapsed time in timespec */ - ktime_get_ts(&uptime); - ts = timespec_sub(uptime, tsk->start_time); - /* rebase elapsed time to usec (should never be negative) */ - ac_etime = timespec_to_ns(&ts); - do_div(ac_etime, NSEC_PER_USEC); - stats->ac_etime = ac_etime; - stats->ac_btime = get_seconds() - ts.tv_sec; + /* calculate task elapsed time in nsec */ + delta = ktime_get_ns() - tsk->start_time; + /* Convert to micro seconds */ + do_div(delta, NSEC_PER_USEC); + stats->ac_etime = delta; + /* Convert to seconds for btime */ + do_div(delta, USEC_PER_SEC); + stats->ac_btime = get_seconds() - delta; if (thread_group_leader(tsk)) { stats->ac_exitcode = tsk->exit_code; if (tsk->flags & PF_FORKNOEXEC) From 9667a23db0dc0bd4892f0ada7e4e71528eaeed62 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:35 +0000 Subject: [PATCH 077/114] delayacct: Make accounting nanosecond based Kill the timespec juggling and calculate with plain nanoseconds. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/sched.h | 4 ++-- kernel/delayacct.c | 34 ++++++++++++---------------------- 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index 10c6e829927f..653744ae8d27 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -813,7 +813,7 @@ struct task_delay_info { * associated with the operation is added to XXX_delay. * XXX_delay contains the accumulated delay time in nanoseconds. */ - struct timespec blkio_start, blkio_end; /* Shared by blkio, swapin */ + u64 blkio_start; /* Shared by blkio, swapin */ u64 blkio_delay; /* wait for sync block io completion */ u64 swapin_delay; /* wait for swapin block io completion */ u32 blkio_count; /* total count of the number of sync block */ @@ -821,7 +821,7 @@ struct task_delay_info { u32 swapin_count; /* total count of the number of swapin block */ /* io operations performed */ - struct timespec freepages_start, freepages_end; + u64 freepages_start; u64 freepages_delay; /* wait for memory reclaim */ u32 freepages_count; /* total count of memory reclaim */ }; diff --git a/kernel/delayacct.c b/kernel/delayacct.c index de699f42f9bc..cf2e65dddb19 100644 --- a/kernel/delayacct.c +++ b/kernel/delayacct.c @@ -46,32 +46,25 @@ void __delayacct_tsk_init(struct task_struct *tsk) } /* - * Finish delay accounting for a statistic using - * its timestamps (@start, @end), accumalator (@total) and @count + * Finish delay accounting for a statistic using its timestamps (@start), + * accumalator (@total) and @count */ - -static void delayacct_end(struct timespec *start, struct timespec *end, - u64 *total, u32 *count) +static void delayacct_end(u64 *start, u64 *total, u32 *count) { - struct timespec ts; - s64 ns; + s64 ns = ktime_get_ns() - *start; unsigned long flags; - ktime_get_ts(end); - ts = timespec_sub(*end, *start); - ns = timespec_to_ns(&ts); - if (ns < 0) - return; - - spin_lock_irqsave(¤t->delays->lock, flags); - *total += ns; - (*count)++; - spin_unlock_irqrestore(¤t->delays->lock, flags); + if (ns > 0) { + spin_lock_irqsave(¤t->delays->lock, flags); + *total += ns; + (*count)++; + spin_unlock_irqrestore(¤t->delays->lock, flags); + } } void __delayacct_blkio_start(void) { - ktime_get_ts(¤t->delays->blkio_start); + current->delays->blkio_start = ktime_get_ns(); } void __delayacct_blkio_end(void) @@ -79,12 +72,10 @@ void __delayacct_blkio_end(void) if (current->delays->flags & DELAYACCT_PF_SWAPIN) /* Swapin block I/O */ delayacct_end(¤t->delays->blkio_start, - ¤t->delays->blkio_end, ¤t->delays->swapin_delay, ¤t->delays->swapin_count); else /* Other block I/O */ delayacct_end(¤t->delays->blkio_start, - ¤t->delays->blkio_end, ¤t->delays->blkio_delay, ¤t->delays->blkio_count); } @@ -159,13 +150,12 @@ __u64 __delayacct_blkio_ticks(struct task_struct *tsk) void __delayacct_freepages_start(void) { - ktime_get_ts(¤t->delays->freepages_start); + current->delays->freepages_start = ktime_get_ns(); } void __delayacct_freepages_end(void) { delayacct_end(¤t->delays->freepages_start, - ¤t->delays->freepages_end, ¤t->delays->freepages_delay, ¤t->delays->freepages_count); } From 68f6783d28316affcd2ce332d949e40e4c7416bd Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:37 +0000 Subject: [PATCH 078/114] delayacct: Remove braindamaged type conversions Converting cputime to timespec and timespec to nanoseconds makes no sense. Use cputime_to_ns() and be done with it. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- kernel/delayacct.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/kernel/delayacct.c b/kernel/delayacct.c index cf2e65dddb19..ef90b04d783f 100644 --- a/kernel/delayacct.c +++ b/kernel/delayacct.c @@ -82,23 +82,19 @@ void __delayacct_blkio_end(void) int __delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk) { - s64 tmp; - unsigned long t1; - unsigned long long t2, t3; - unsigned long flags; - struct timespec ts; cputime_t utime, stime, stimescaled, utimescaled; + unsigned long long t2, t3; + unsigned long flags, t1; + s64 tmp; - tmp = (s64)d->cpu_run_real_total; task_cputime(tsk, &utime, &stime); - cputime_to_timespec(utime + stime, &ts); - tmp += timespec_to_ns(&ts); + tmp = (s64)d->cpu_run_real_total; + tmp += cputime_to_nsecs(utime + stime); d->cpu_run_real_total = (tmp < (s64)d->cpu_run_real_total) ? 0 : tmp; - tmp = (s64)d->cpu_scaled_run_real_total; task_cputime_scaled(tsk, &utimescaled, &stimescaled); - cputime_to_timespec(utimescaled + stimescaled, &ts); - tmp += timespec_to_ns(&ts); + tmp = (s64)d->cpu_scaled_run_real_total; + tmp += cputime_to_nsecs(utimescaled + stimescaled); d->cpu_scaled_run_real_total = (tmp < (s64)d->cpu_scaled_run_real_total) ? 0 : tmp; From f2dec1eae8029bb056a3c1b2d3373681fa8e3e7a Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:38 +0000 Subject: [PATCH 079/114] powerpc: cell: Use ktime_get_ns() Replace the ever recurring: ts = ktime_get_ts(); ns = timespec_to_ns(&ts); with ns = ktime_get_ns(); Signed-off-by: Thomas Gleixner Acked-by: Arnd Bergmann Signed-off-by: John Stultz --- arch/powerpc/platforms/cell/spu_base.c | 11 +++-------- arch/powerpc/platforms/cell/spufs/context.c | 4 +--- arch/powerpc/platforms/cell/spufs/file.c | 4 +--- arch/powerpc/platforms/cell/spufs/sched.c | 4 +--- 4 files changed, 6 insertions(+), 17 deletions(-) diff --git a/arch/powerpc/platforms/cell/spu_base.c b/arch/powerpc/platforms/cell/spu_base.c index f85db3a69b4a..2930d1e81a05 100644 --- a/arch/powerpc/platforms/cell/spu_base.c +++ b/arch/powerpc/platforms/cell/spu_base.c @@ -611,7 +611,6 @@ static int __init create_spu(void *data) int ret; static int number; unsigned long flags; - struct timespec ts; ret = -ENOMEM; spu = kzalloc(sizeof (*spu), GFP_KERNEL); @@ -652,8 +651,7 @@ static int __init create_spu(void *data) mutex_unlock(&spu_full_list_mutex); spu->stats.util_state = SPU_UTIL_IDLE_LOADED; - ktime_get_ts(&ts); - spu->stats.tstamp = timespec_to_ns(&ts); + spu->stats.tstamp = ktime_get_ns(); INIT_LIST_HEAD(&spu->aff_list); @@ -676,7 +674,6 @@ static const char *spu_state_names[] = { static unsigned long long spu_acct_time(struct spu *spu, enum spu_utilization_state state) { - struct timespec ts; unsigned long long time = spu->stats.times[state]; /* @@ -684,10 +681,8 @@ static unsigned long long spu_acct_time(struct spu *spu, * statistics are not updated. Apply the time delta from the * last recorded state of the spu. */ - if (spu->stats.util_state == state) { - ktime_get_ts(&ts); - time += timespec_to_ns(&ts) - spu->stats.tstamp; - } + if (spu->stats.util_state == state) + time += ktime_get_ns() - spu->stats.tstamp; return time / NSEC_PER_MSEC; } diff --git a/arch/powerpc/platforms/cell/spufs/context.c b/arch/powerpc/platforms/cell/spufs/context.c index 9c6790d17eda..3b4152faeb1f 100644 --- a/arch/powerpc/platforms/cell/spufs/context.c +++ b/arch/powerpc/platforms/cell/spufs/context.c @@ -36,7 +36,6 @@ atomic_t nr_spu_contexts = ATOMIC_INIT(0); struct spu_context *alloc_spu_context(struct spu_gang *gang) { struct spu_context *ctx; - struct timespec ts; ctx = kzalloc(sizeof *ctx, GFP_KERNEL); if (!ctx) @@ -67,8 +66,7 @@ struct spu_context *alloc_spu_context(struct spu_gang *gang) __spu_update_sched_info(ctx); spu_set_timeslice(ctx); ctx->stats.util_state = SPU_UTIL_IDLE_LOADED; - ktime_get_ts(&ts); - ctx->stats.tstamp = timespec_to_ns(&ts); + ctx->stats.tstamp = ktime_get_ns(); atomic_inc(&nr_spu_contexts); goto out; diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c index 90986923a53a..d966bbe58b8f 100644 --- a/arch/powerpc/platforms/cell/spufs/file.c +++ b/arch/powerpc/platforms/cell/spufs/file.c @@ -2338,7 +2338,6 @@ static const char *ctx_state_names[] = { static unsigned long long spufs_acct_time(struct spu_context *ctx, enum spu_utilization_state state) { - struct timespec ts; unsigned long long time = ctx->stats.times[state]; /* @@ -2351,8 +2350,7 @@ static unsigned long long spufs_acct_time(struct spu_context *ctx, * of the spu context. */ if (ctx->spu && ctx->stats.util_state == state) { - ktime_get_ts(&ts); - time += timespec_to_ns(&ts) - ctx->stats.tstamp; + time += ktime_get_ns() - ctx->stats.tstamp; } return time / NSEC_PER_MSEC; diff --git a/arch/powerpc/platforms/cell/spufs/sched.c b/arch/powerpc/platforms/cell/spufs/sched.c index 4a0a64fe25df..998f632e7cce 100644 --- a/arch/powerpc/platforms/cell/spufs/sched.c +++ b/arch/powerpc/platforms/cell/spufs/sched.c @@ -1039,13 +1039,11 @@ void spuctx_switch_state(struct spu_context *ctx, { unsigned long long curtime; signed long long delta; - struct timespec ts; struct spu *spu; enum spu_utilization_state old_state; int node; - ktime_get_ts(&ts); - curtime = timespec_to_ns(&ts); + curtime = ktime_get_ns(); delta = curtime - ctx->stats.tstamp; WARN_ON(!mutex_is_locked(&ctx->state_mutex)); From 9e93f21b4309572c646bd8141a4853be5c031523 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:40 +0000 Subject: [PATCH 080/114] connector: Use ktime_get_ns() Replace the ever recurring: ts = ktime_get_ts(); ns = timespec_to_ns(&ts); with ns = ktime_get_ns(); Signed-off-by: Thomas Gleixner Cc: Evgeniy Polyakov Signed-off-by: John Stultz --- drivers/connector/cn_proc.c | 36 +++++++++--------------------------- 1 file changed, 9 insertions(+), 27 deletions(-) diff --git a/drivers/connector/cn_proc.c b/drivers/connector/cn_proc.c index ccdd4c7e748b..15d06fcf0b50 100644 --- a/drivers/connector/cn_proc.c +++ b/drivers/connector/cn_proc.c @@ -69,7 +69,6 @@ void proc_fork_connector(struct task_struct *task) struct cn_msg *msg; struct proc_event *ev; __u8 buffer[CN_PROC_MSG_SIZE] __aligned(8); - struct timespec ts; struct task_struct *parent; if (atomic_read(&proc_event_num_listeners) < 1) @@ -79,8 +78,7 @@ void proc_fork_connector(struct task_struct *task) ev = (struct proc_event *)msg->data; memset(&ev->event_data, 0, sizeof(ev->event_data)); get_seq(&msg->seq, &ev->cpu); - ktime_get_ts(&ts); /* get high res monotonic timestamp */ - ev->timestamp_ns = timespec_to_ns(&ts); + ev->timestamp_ns = ktime_get_ns(); ev->what = PROC_EVENT_FORK; rcu_read_lock(); parent = rcu_dereference(task->real_parent); @@ -102,7 +100,6 @@ void proc_exec_connector(struct task_struct *task) { struct cn_msg *msg; struct proc_event *ev; - struct timespec ts; __u8 buffer[CN_PROC_MSG_SIZE] __aligned(8); if (atomic_read(&proc_event_num_listeners) < 1) @@ -112,8 +109,7 @@ void proc_exec_connector(struct task_struct *task) ev = (struct proc_event *)msg->data; memset(&ev->event_data, 0, sizeof(ev->event_data)); get_seq(&msg->seq, &ev->cpu); - ktime_get_ts(&ts); /* get high res monotonic timestamp */ - ev->timestamp_ns = timespec_to_ns(&ts); + ev->timestamp_ns = ktime_get_ns(); ev->what = PROC_EVENT_EXEC; ev->event_data.exec.process_pid = task->pid; ev->event_data.exec.process_tgid = task->tgid; @@ -130,7 +126,6 @@ void proc_id_connector(struct task_struct *task, int which_id) struct cn_msg *msg; struct proc_event *ev; __u8 buffer[CN_PROC_MSG_SIZE] __aligned(8); - struct timespec ts; const struct cred *cred; if (atomic_read(&proc_event_num_listeners) < 1) @@ -156,8 +151,7 @@ void proc_id_connector(struct task_struct *task, int which_id) } rcu_read_unlock(); get_seq(&msg->seq, &ev->cpu); - ktime_get_ts(&ts); /* get high res monotonic timestamp */ - ev->timestamp_ns = timespec_to_ns(&ts); + ev->timestamp_ns = ktime_get_ns(); memcpy(&msg->id, &cn_proc_event_id, sizeof(msg->id)); msg->ack = 0; /* not used */ @@ -170,7 +164,6 @@ void proc_sid_connector(struct task_struct *task) { struct cn_msg *msg; struct proc_event *ev; - struct timespec ts; __u8 buffer[CN_PROC_MSG_SIZE] __aligned(8); if (atomic_read(&proc_event_num_listeners) < 1) @@ -180,8 +173,7 @@ void proc_sid_connector(struct task_struct *task) ev = (struct proc_event *)msg->data; memset(&ev->event_data, 0, sizeof(ev->event_data)); get_seq(&msg->seq, &ev->cpu); - ktime_get_ts(&ts); /* get high res monotonic timestamp */ - ev->timestamp_ns = timespec_to_ns(&ts); + ev->timestamp_ns = ktime_get_ns(); ev->what = PROC_EVENT_SID; ev->event_data.sid.process_pid = task->pid; ev->event_data.sid.process_tgid = task->tgid; @@ -197,7 +189,6 @@ void proc_ptrace_connector(struct task_struct *task, int ptrace_id) { struct cn_msg *msg; struct proc_event *ev; - struct timespec ts; __u8 buffer[CN_PROC_MSG_SIZE] __aligned(8); if (atomic_read(&proc_event_num_listeners) < 1) @@ -207,8 +198,7 @@ void proc_ptrace_connector(struct task_struct *task, int ptrace_id) ev = (struct proc_event *)msg->data; memset(&ev->event_data, 0, sizeof(ev->event_data)); get_seq(&msg->seq, &ev->cpu); - ktime_get_ts(&ts); /* get high res monotonic timestamp */ - ev->timestamp_ns = timespec_to_ns(&ts); + ev->timestamp_ns = ktime_get_ns(); ev->what = PROC_EVENT_PTRACE; ev->event_data.ptrace.process_pid = task->pid; ev->event_data.ptrace.process_tgid = task->tgid; @@ -232,7 +222,6 @@ void proc_comm_connector(struct task_struct *task) { struct cn_msg *msg; struct proc_event *ev; - struct timespec ts; __u8 buffer[CN_PROC_MSG_SIZE] __aligned(8); if (atomic_read(&proc_event_num_listeners) < 1) @@ -242,8 +231,7 @@ void proc_comm_connector(struct task_struct *task) ev = (struct proc_event *)msg->data; memset(&ev->event_data, 0, sizeof(ev->event_data)); get_seq(&msg->seq, &ev->cpu); - ktime_get_ts(&ts); /* get high res monotonic timestamp */ - ev->timestamp_ns = timespec_to_ns(&ts); + ev->timestamp_ns = ktime_get_ns(); ev->what = PROC_EVENT_COMM; ev->event_data.comm.process_pid = task->pid; ev->event_data.comm.process_tgid = task->tgid; @@ -261,7 +249,6 @@ void proc_coredump_connector(struct task_struct *task) struct cn_msg *msg; struct proc_event *ev; __u8 buffer[CN_PROC_MSG_SIZE] __aligned(8); - struct timespec ts; if (atomic_read(&proc_event_num_listeners) < 1) return; @@ -270,8 +257,7 @@ void proc_coredump_connector(struct task_struct *task) ev = (struct proc_event *)msg->data; memset(&ev->event_data, 0, sizeof(ev->event_data)); get_seq(&msg->seq, &ev->cpu); - ktime_get_ts(&ts); /* get high res monotonic timestamp */ - ev->timestamp_ns = timespec_to_ns(&ts); + ev->timestamp_ns = ktime_get_ns(); ev->what = PROC_EVENT_COREDUMP; ev->event_data.coredump.process_pid = task->pid; ev->event_data.coredump.process_tgid = task->tgid; @@ -288,7 +274,6 @@ void proc_exit_connector(struct task_struct *task) struct cn_msg *msg; struct proc_event *ev; __u8 buffer[CN_PROC_MSG_SIZE] __aligned(8); - struct timespec ts; if (atomic_read(&proc_event_num_listeners) < 1) return; @@ -297,8 +282,7 @@ void proc_exit_connector(struct task_struct *task) ev = (struct proc_event *)msg->data; memset(&ev->event_data, 0, sizeof(ev->event_data)); get_seq(&msg->seq, &ev->cpu); - ktime_get_ts(&ts); /* get high res monotonic timestamp */ - ev->timestamp_ns = timespec_to_ns(&ts); + ev->timestamp_ns = ktime_get_ns(); ev->what = PROC_EVENT_EXIT; ev->event_data.exit.process_pid = task->pid; ev->event_data.exit.process_tgid = task->tgid; @@ -325,7 +309,6 @@ static void cn_proc_ack(int err, int rcvd_seq, int rcvd_ack) struct cn_msg *msg; struct proc_event *ev; __u8 buffer[CN_PROC_MSG_SIZE] __aligned(8); - struct timespec ts; if (atomic_read(&proc_event_num_listeners) < 1) return; @@ -334,8 +317,7 @@ static void cn_proc_ack(int err, int rcvd_seq, int rcvd_ack) ev = (struct proc_event *)msg->data; memset(&ev->event_data, 0, sizeof(ev->event_data)); msg->seq = rcvd_seq; - ktime_get_ts(&ts); /* get high res monotonic timestamp */ - ev->timestamp_ns = timespec_to_ns(&ts); + ev->timestamp_ns = ktime_get_ns(); ev->cpu = -1; ev->what = PROC_EVENT_NONE; ev->event_data.ack.err = err; From 154adc14b1ce941b642a3775f2423c20d55c026b Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:41 +0000 Subject: [PATCH 081/114] mfd: cros_ec_spi: Use ktime_get_ns() Replace the ever recurring: ts = ktime_get_ts(); ns = timespec_to_ns(&ts); with ns = ktime_get_ns(); Signed-off-by: Thomas Gleixner Acked-by: Lee Jones Signed-off-by: John Stultz --- drivers/mfd/cros_ec_spi.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/drivers/mfd/cros_ec_spi.c b/drivers/mfd/cros_ec_spi.c index 0b8d32829166..8c1c7cc373f8 100644 --- a/drivers/mfd/cros_ec_spi.c +++ b/drivers/mfd/cros_ec_spi.c @@ -225,7 +225,6 @@ static int cros_ec_command_spi_xfer(struct cros_ec_device *ec_dev, u8 *ptr; int sum; int ret = 0, final_ret; - struct timespec ts; /* * We have the shared ec_dev buffer plus we do lots of separate spi_sync @@ -239,11 +238,9 @@ static int cros_ec_command_spi_xfer(struct cros_ec_device *ec_dev, /* If it's too soon to do another transaction, wait */ if (ec_spi->last_transfer_ns) { - struct timespec ts; unsigned long delay; /* The delay completed so far */ - ktime_get_ts(&ts); - delay = timespec_to_ns(&ts) - ec_spi->last_transfer_ns; + delay = ktime_get_ns() - ec_spi->last_transfer_ns; if (delay < EC_SPI_RECOVERY_TIME_NS) ndelay(EC_SPI_RECOVERY_TIME_NS - delay); } @@ -280,8 +277,7 @@ static int cros_ec_command_spi_xfer(struct cros_ec_device *ec_dev, } final_ret = spi_sync(ec_spi->spi, &msg); - ktime_get_ts(&ts); - ec_spi->last_transfer_ns = timespec_to_ns(&ts); + ec_spi->last_transfer_ns = ktime_get_ns(); if (!ret) ret = final_ret; if (ret < 0) { From 6d9b757c6ce850d0fb61d31f48e7bae56240643b Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:43 +0000 Subject: [PATCH 082/114] misc: ioc4: Use ktime_get_ns() Replace the ever recurring: ts = ktime_get_ts(); ns = timespec_to_ns(&ts); with ns = ktime_get_ns(); Signed-off-by: Thomas Gleixner Acked-by: Arnd Bergmann Acked-by: Greg Kroah-Hartman Signed-off-by: John Stultz --- drivers/misc/ioc4.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/drivers/misc/ioc4.c b/drivers/misc/ioc4.c index 06f6ad29ceff..3336ddca45ac 100644 --- a/drivers/misc/ioc4.c +++ b/drivers/misc/ioc4.c @@ -145,7 +145,6 @@ ioc4_clock_calibrate(struct ioc4_driver_data *idd) union ioc4_int_out int_out; union ioc4_gpcr gpcr; unsigned int state, last_state = 1; - struct timespec start_ts, end_ts; uint64_t start, end, period; unsigned int count = 0; @@ -174,10 +173,10 @@ ioc4_clock_calibrate(struct ioc4_driver_data *idd) if (!last_state && state) { count++; if (count == IOC4_CALIBRATE_END) { - ktime_get_ts(&end_ts); + end = ktime_get_ns(); break; } else if (count == IOC4_CALIBRATE_DISCARD) - ktime_get_ts(&start_ts); + start = ktime_get_ns(); } last_state = state; } while (1); @@ -192,8 +191,6 @@ ioc4_clock_calibrate(struct ioc4_driver_data *idd) * by which the IOC4 generates the square wave, to get the * period of an IOC4 INT_OUT count. */ - end = end_ts.tv_sec * NSEC_PER_SEC + end_ts.tv_nsec; - start = start_ts.tv_sec * NSEC_PER_SEC + start_ts.tv_nsec; period = (end - start) / (IOC4_CALIBRATE_CYCLES * 2 * (IOC4_CALIBRATE_COUNT + 1)); From 14a7004671246d1b799f545335995a9897de1268 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:44 +0000 Subject: [PATCH 083/114] net: mlx5: Use ktime_get_ns() This code is beyond silly: struct timespec ts = ktime_get_ts(); ktime_t ktime = timespec_to_ktime(ts); Further down the code builds the delta of two ktime_t values and converts the result to nanoseconds. Use ktime_get_ns() and replace all the nonsense. Signed-off-by: Thomas Gleixner Cc: Eli Cohen Signed-off-by: John Stultz --- drivers/net/ethernet/mellanox/mlx5/core/cmd.c | 16 ++++------------ include/linux/mlx5/driver.h | 4 ++-- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c index 87d1b018a9c3..67f8f5a1dc86 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c @@ -548,7 +548,7 @@ static void cmd_work_handler(struct work_struct *work) lay->status_own = CMD_OWNER_HW; set_signature(ent, !cmd->checksum_disabled); dump_command(dev, ent, 1); - ktime_get_ts(&ent->ts1); + ent->ts1 = ktime_get_ns(); /* ring doorbell after the descriptor is valid */ wmb(); @@ -637,7 +637,6 @@ static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in, { struct mlx5_cmd *cmd = &dev->cmd; struct mlx5_cmd_work_ent *ent; - ktime_t t1, t2, delta; struct mlx5_cmd_stats *stats; int err = 0; s64 ds; @@ -668,10 +667,7 @@ static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in, if (err == -ETIMEDOUT) goto out; - t1 = timespec_to_ktime(ent->ts1); - t2 = timespec_to_ktime(ent->ts2); - delta = ktime_sub(t2, t1); - ds = ktime_to_ns(delta); + ds = ent->ts2 - ent->ts1; op = be16_to_cpu(((struct mlx5_inbox_hdr *)in->first.data)->opcode); if (op < ARRAY_SIZE(cmd->stats)) { stats = &cmd->stats[op]; @@ -1135,7 +1131,6 @@ void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, unsigned long vector) void *context; int err; int i; - ktime_t t1, t2, delta; s64 ds; struct mlx5_cmd_stats *stats; unsigned long flags; @@ -1149,7 +1144,7 @@ void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, unsigned long vector) sem = &cmd->pages_sem; else sem = &cmd->sem; - ktime_get_ts(&ent->ts2); + ent->ts2 = ktime_get_ns(); memcpy(ent->out->first.data, ent->lay->out, sizeof(ent->lay->out)); dump_command(dev, ent, 0); if (!ent->ret) { @@ -1163,10 +1158,7 @@ void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, unsigned long vector) } free_ent(cmd, ent->idx); if (ent->callback) { - t1 = timespec_to_ktime(ent->ts1); - t2 = timespec_to_ktime(ent->ts2); - delta = ktime_sub(t2, t1); - ds = ktime_to_ns(delta); + ds = ent->ts2 - ent->ts1; if (ent->op < ARRAY_SIZE(cmd->stats)) { stats = &cmd->stats[ent->op]; spin_lock_irqsave(&stats->lock, flags); diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 2bce4aad2570..52d631ca32cf 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -604,8 +604,8 @@ struct mlx5_cmd_work_ent { int page_queue; u8 status; u8 token; - struct timespec ts1; - struct timespec ts2; + u64 ts1; + u64 ts2; u16 op; }; From 5eaaed4fe23247f0784edc2973f04134f8180251 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:46 +0000 Subject: [PATCH 084/114] fs: lockd: Use ktime_get_ns() Replace the ever recurring: ts = ktime_get_ts(); ns = timespec_to_ns(&ts); with ns = ktime_get_ns(); Signed-off-by: Thomas Gleixner Acked-by: Trond Myklebust Cc: "J. Bruce Fields" Signed-off-by: John Stultz --- fs/lockd/mon.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/lockd/mon.c b/fs/lockd/mon.c index 1812f026960c..daa8e7514eae 100644 --- a/fs/lockd/mon.c +++ b/fs/lockd/mon.c @@ -306,11 +306,9 @@ static struct nsm_handle *nsm_lookup_priv(const struct nsm_private *priv) static void nsm_init_private(struct nsm_handle *nsm) { u64 *p = (u64 *)&nsm->sm_priv.data; - struct timespec ts; s64 ns; - ktime_get_ts(&ts); - ns = timespec_to_ns(&ts); + ns = ktime_get_ns(); put_unaligned(ns, p); put_unaligned((unsigned long)nsm, p + 1); } From d659f9b135fe26b966a6a4ad7281d35d7dbb1366 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:47 +0000 Subject: [PATCH 085/114] hwmon: ibmaem: Use ktime_get_ns() Using the wall clock time for delta time calculations is wrong to begin with because wall clock time can be set from userspace and NTP. Such data wants to be based on clock monotonic. The calculations also are done on a nanosecond basis. Use the nanoseconds based interface right away. Signed-off-by: Thomas Gleixner Cc: Jean Delvare Acked-by: Jean Delvare Signed-off-by: John Stultz --- drivers/hwmon/ibmaem.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/hwmon/ibmaem.c b/drivers/hwmon/ibmaem.c index 632f1dc0fe1f..7a8a6fbf11ff 100644 --- a/drivers/hwmon/ibmaem.c +++ b/drivers/hwmon/ibmaem.c @@ -842,11 +842,10 @@ static ssize_t aem_show_power(struct device *dev, struct aem_data *data = dev_get_drvdata(dev); u64 before, after, delta, time; signed long leftover; - struct timespec b, a; mutex_lock(&data->lock); update_aem_energy_one(data, attr->index); - getnstimeofday(&b); + time = ktime_get_ns(); before = data->energy[attr->index]; leftover = schedule_timeout_interruptible( @@ -858,11 +857,10 @@ static ssize_t aem_show_power(struct device *dev, } update_aem_energy_one(data, attr->index); - getnstimeofday(&a); + time = ktime_get_ns() - time; after = data->energy[attr->index]; mutex_unlock(&data->lock); - time = timespec_to_ns(&a) - timespec_to_ns(&b); delta = (after - before) * UJ_PER_MJ; return sprintf(buf, "%llu\n", From fb31cc153dec0d4bdd9a5d7ce60d61acd04b4304 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:49 +0000 Subject: [PATCH 086/114] iio: Use ktime_get_real_ns() No idea why iio needs wall clock based time stamps, but we can avoid the timespec conversion dance by using the new interfaces. Signed-off-by: Thomas Gleixner Acked-by: Jonathan Cameron Signed-off-by: John Stultz --- include/linux/iio/iio.h | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/include/linux/iio/iio.h b/include/linux/iio/iio.h index ccde91725f98..15dc6bc2bdd2 100644 --- a/include/linux/iio/iio.h +++ b/include/linux/iio/iio.h @@ -277,14 +277,7 @@ static inline bool iio_channel_has_info(const struct iio_chan_spec *chan, **/ static inline s64 iio_get_time_ns(void) { - struct timespec ts; - /* - * calls getnstimeofday. - * If hrtimers then up to ns accurate, if not microsecond. - */ - ktime_get_real_ts(&ts); - - return timespec_to_ns(&ts); + return ktime_get_real_ns(); } /* Device operating modes */ From 41fa4215f8e8150bdc5d2a5f8704915f1b059fa8 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:50 +0000 Subject: [PATCH 087/114] arm: bL_switcher:k Use ktime_get_real_ns() Use the nanoseconds based interface instead of converting from a timespec. Signed-off-by: Thomas Gleixner Cc: Russell King Cc: linux-arm-kernel@lists.infradead.org Signed-off-by: John Stultz --- arch/arm/common/bL_switcher.c | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c index 490f3dced749..6eaddc47c43d 100644 --- a/arch/arm/common/bL_switcher.c +++ b/arch/arm/common/bL_switcher.c @@ -57,16 +57,6 @@ static int read_mpidr(void) return id & MPIDR_HWID_BITMASK; } -/* - * Get a global nanosecond time stamp for tracing. - */ -static s64 get_ns(void) -{ - struct timespec ts; - getnstimeofday(&ts); - return timespec_to_ns(&ts); -} - /* * bL switcher core code. */ @@ -224,7 +214,7 @@ static int bL_switch_to(unsigned int new_cluster_id) */ local_irq_disable(); local_fiq_disable(); - trace_cpu_migrate_begin(get_ns(), ob_mpidr); + trace_cpu_migrate_begin(ktime_get_real_ns(), ob_mpidr); /* redirect GIC's SGIs to our counterpart */ gic_migrate_target(bL_gic_id[ib_cpu][ib_cluster]); @@ -267,7 +257,7 @@ static int bL_switch_to(unsigned int new_cluster_id) tdev->evtdev->next_event, 1); } - trace_cpu_migrate_finish(get_ns(), ib_mpidr); + trace_cpu_migrate_finish(ktime_get_real_ns(), ib_mpidr); local_fiq_enable(); local_irq_enable(); @@ -558,7 +548,7 @@ int bL_switcher_get_logical_index(u32 mpidr) static void bL_switcher_trace_trigger_cpu(void *__always_unused info) { - trace_cpu_migrate_current(get_ns(), read_mpidr()); + trace_cpu_migrate_current(ktime_get_real_ns(), read_mpidr()); } int bL_switcher_trace_trigger(void) From bb0b58127c5add364cb597d58b1cf66eb279eae8 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:52 +0000 Subject: [PATCH 088/114] x86: kvm: Use ktime_get_boot_ns() Use the new nanoseconds based interface and get rid of the timespec conversion dance. Signed-off-by: Thomas Gleixner Cc: Gleb Natapov Cc: kvm@vger.kernel.org Acked-by: Paolo Bonzini Signed-off-by: John Stultz --- arch/x86/kvm/x86.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index f6449334ec45..65c430512132 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1109,11 +1109,7 @@ static void kvm_get_time_scale(uint32_t scaled_khz, uint32_t base_khz, static inline u64 get_kernel_ns(void) { - struct timespec ts; - - ktime_get_ts(&ts); - monotonic_to_bootbased(&ts); - return timespec_to_ns(&ts); + return ktime_get_boot_ns(); } #ifdef CONFIG_X86_64 From cbcf2dd3b3d4d990610259e8d878fc8dc1f17d80 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:54 +0000 Subject: [PATCH 089/114] x86: kvm: Make kvm_get_time_and_clockread() nanoseconds based Convert the relevant base data right away to nanoseconds instead of doing the conversion on every readout. Reduces text size by 160 bytes. Signed-off-by: Thomas Gleixner Cc: Gleb Natapov Cc: kvm@vger.kernel.org Acked-by: Paolo Bonzini Signed-off-by: John Stultz --- arch/x86/kvm/x86.c | 46 +++++++++++++++------------------------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 65c430512132..63832f5110b6 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -984,9 +984,8 @@ struct pvclock_gtod_data { u32 shift; } clock; - /* open coded 'struct timespec' */ - u64 monotonic_time_snsec; - time_t monotonic_time_sec; + u64 boot_ns; + u64 nsec_base; }; static struct pvclock_gtod_data pvclock_gtod_data; @@ -994,6 +993,9 @@ static struct pvclock_gtod_data pvclock_gtod_data; static void update_pvclock_gtod(struct timekeeper *tk) { struct pvclock_gtod_data *vdata = &pvclock_gtod_data; + u64 boot_ns; + + boot_ns = ktime_to_ns(ktime_add(tk->base_mono, tk->offs_boot)); write_seqcount_begin(&vdata->seq); @@ -1004,17 +1006,8 @@ static void update_pvclock_gtod(struct timekeeper *tk) vdata->clock.mult = tk->mult; vdata->clock.shift = tk->shift; - vdata->monotonic_time_sec = tk->xtime_sec - + tk->wall_to_monotonic.tv_sec; - vdata->monotonic_time_snsec = tk->xtime_nsec - + (tk->wall_to_monotonic.tv_nsec - << tk->shift); - while (vdata->monotonic_time_snsec >= - (((u64)NSEC_PER_SEC) << tk->shift)) { - vdata->monotonic_time_snsec -= - ((u64)NSEC_PER_SEC) << tk->shift; - vdata->monotonic_time_sec++; - } + vdata->boot_ns = boot_ns; + vdata->nsec_base = tk->xtime_nsec; write_seqcount_end(&vdata->seq); } @@ -1371,23 +1364,22 @@ static inline u64 vgettsc(cycle_t *cycle_now) return v * gtod->clock.mult; } -static int do_monotonic(struct timespec *ts, cycle_t *cycle_now) +static int do_monotonic_boot(s64 *t, cycle_t *cycle_now) { - unsigned long seq; - u64 ns; - int mode; struct pvclock_gtod_data *gtod = &pvclock_gtod_data; + unsigned long seq; + int mode; + u64 ns; - ts->tv_nsec = 0; do { seq = read_seqcount_begin(>od->seq); mode = gtod->clock.vclock_mode; - ts->tv_sec = gtod->monotonic_time_sec; - ns = gtod->monotonic_time_snsec; + ns = gtod->nsec_base; ns += vgettsc(cycle_now); ns >>= gtod->clock.shift; + ns += gtod->boot_ns; } while (unlikely(read_seqcount_retry(>od->seq, seq))); - timespec_add_ns(ts, ns); + *t = ns; return mode; } @@ -1395,19 +1387,11 @@ static int do_monotonic(struct timespec *ts, cycle_t *cycle_now) /* returns true if host is using tsc clocksource */ static bool kvm_get_time_and_clockread(s64 *kernel_ns, cycle_t *cycle_now) { - struct timespec ts; - /* checked again under seqlock below */ if (pvclock_gtod_data.clock.vclock_mode != VCLOCK_TSC) return false; - if (do_monotonic(&ts, cycle_now) != VCLOCK_TSC) - return false; - - monotonic_to_bootbased(&ts); - *kernel_ns = timespec_to_ns(&ts); - - return true; + return do_monotonic_boot(kernel_ns, cycle_now) == VCLOCK_TSC; } #endif From 250fade8af2ac5dda8d5106ea06738b6f9e768a7 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:55 +0000 Subject: [PATCH 090/114] timekeeping: Remove monotonic_to_bootbased No more users. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/timekeeping.h | 1 - kernel/time/timekeeping.c | 15 --------------- 2 files changed, 16 deletions(-) diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h index 903ecc10fcff..8ea3ca1b0ee5 100644 --- a/include/linux/timekeeping.h +++ b/include/linux/timekeeping.h @@ -27,7 +27,6 @@ struct timespec __current_kernel_time(void); */ struct timespec get_monotonic_coarse(void); extern void getrawmonotonic(struct timespec *ts); -extern void monotonic_to_bootbased(struct timespec *ts); extern void get_monotonic_boottime(struct timespec *ts); extern void ktime_get_ts64(struct timespec64 *ts); diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index f7378eaebe67..b35613508725 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -1570,21 +1570,6 @@ void get_monotonic_boottime(struct timespec *ts) } EXPORT_SYMBOL_GPL(get_monotonic_boottime); -/** - * monotonic_to_bootbased - Convert the monotonic time to boot based. - * @ts: pointer to the timespec to be converted - */ -void monotonic_to_bootbased(struct timespec *ts) -{ - struct timekeeper *tk = &tk_core.timekeeper; - struct timespec64 ts64; - - ts64 = timespec_to_timespec64(*ts); - ts64 = timespec64_add(ts64, tk->total_sleep_time); - *ts = timespec64_to_timespec(ts64); -} -EXPORT_SYMBOL_GPL(monotonic_to_bootbased); - unsigned long get_seconds(void) { struct timekeeper *tk = &tk_core.timekeeper; From 48f18fd6addc199f330d838d54fe7b0a0892adaa Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:57 +0000 Subject: [PATCH 091/114] timekeeping: Use ktime_get_boottime() for get_monotonic_boottime() get_monotonic_boottime() is not used in fast pathes, so the extra timespec conversion is not problematic. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/timekeeping.h | 9 ++++++++- kernel/time/timekeeping.c | 34 ---------------------------------- 2 files changed, 8 insertions(+), 35 deletions(-) diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h index 8ea3ca1b0ee5..7b8f20007871 100644 --- a/include/linux/timekeeping.h +++ b/include/linux/timekeeping.h @@ -27,7 +27,6 @@ struct timespec __current_kernel_time(void); */ struct timespec get_monotonic_coarse(void); extern void getrawmonotonic(struct timespec *ts); -extern void get_monotonic_boottime(struct timespec *ts); extern void ktime_get_ts64(struct timespec64 *ts); extern int __getnstimeofday64(struct timespec64 *tv); @@ -159,6 +158,14 @@ static inline u64 ktime_get_boot_ns(void) return ktime_to_ns(ktime_get_boottime()); } +/* + * Timespec interfaces utilizing the ktime based ones + */ +static inline void get_monotonic_boottime(struct timespec *ts) +{ + *ts = ktime_to_timespec(ktime_get_boottime()); +} + /* * RTC specific */ diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index b35613508725..f63476fb0daf 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -1536,40 +1536,6 @@ void getboottime(struct timespec *ts) } EXPORT_SYMBOL_GPL(getboottime); -/** - * get_monotonic_boottime - Returns monotonic time since boot - * @ts: pointer to the timespec to be set - * - * Returns the monotonic time since boot in a timespec. - * - * This is similar to CLOCK_MONTONIC/ktime_get_ts, but also - * includes the time spent in suspend. - */ -void get_monotonic_boottime(struct timespec *ts) -{ - struct timekeeper *tk = &tk_core.timekeeper; - struct timespec64 tomono, sleep, ret; - s64 nsec; - unsigned int seq; - - WARN_ON(timekeeping_suspended); - - do { - seq = read_seqcount_begin(&tk_core.seq); - ret.tv_sec = tk->xtime_sec; - nsec = timekeeping_get_ns(tk); - tomono = tk->wall_to_monotonic; - sleep = tk->total_sleep_time; - - } while (read_seqcount_retry(&tk_core.seq, seq)); - - ret.tv_sec += tomono.tv_sec + sleep.tv_sec; - ret.tv_nsec = 0; - timespec64_add_ns(&ret, nsec + tomono.tv_nsec + sleep.tv_nsec); - *ts = timespec64_to_timespec(ret); -} -EXPORT_SYMBOL_GPL(get_monotonic_boottime); - unsigned long get_seconds(void) { struct timekeeper *tk = &tk_core.timekeeper; From 02cba1598a2a3b689e79ad6dad2532521f638271 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:04:58 +0000 Subject: [PATCH 092/114] timekeeping: Simplify getboottime() Subtracting plain nsec values and converting to timespec is simpler than the whole timespec math. Not really fastpath code, so the division is not an issue. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- kernel/time/timekeeping.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index f63476fb0daf..3edc0c1d6fe8 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -1525,14 +1525,9 @@ out: void getboottime(struct timespec *ts) { struct timekeeper *tk = &tk_core.timekeeper; - struct timespec boottime = { - .tv_sec = tk->wall_to_monotonic.tv_sec + - tk->total_sleep_time.tv_sec, - .tv_nsec = tk->wall_to_monotonic.tv_nsec + - tk->total_sleep_time.tv_nsec - }; + ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot); - set_normalized_timespec(ts, -boottime.tv_sec, -boottime.tv_nsec); + *ts = ktime_to_timespec(t); } EXPORT_SYMBOL_GPL(getboottime); From 47da70d32535000ec29cc206cfc1d318fbd8761f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:05:00 +0000 Subject: [PATCH 093/114] timekeeping: Remove timekeeper.total_sleep_time No more users. Remove it Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/timekeeper_internal.h | 6 ++---- kernel/time/timekeeping.c | 14 +++----------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h index 87e0992564f2..8e5d77a01787 100644 --- a/include/linux/timekeeper_internal.h +++ b/include/linux/timekeeper_internal.h @@ -20,8 +20,8 @@ * the tv_nsec part positive so we can use the usual normalization. * * wall_to_monotonic is moved after resume from suspend for the - * monotonic time not to jump. We need to add total_sleep_time to - * wall_to_monotonic to get the real boot based time offset. + * monotonic time not to jump. To calculate the real boot time offset + * we need to do offs_real - offs_boot. * * - wall_to_monotonic is no longer the boot time, getboottime must be * used instead. @@ -51,8 +51,6 @@ struct timekeeper { /* Offset clock monotonic -> clock tai */ ktime_t offs_tai; - /* time spent in suspend */ - struct timespec64 total_sleep_time; /* The current UTC to TAI offset in seconds */ s32 tai_offset; diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 3edc0c1d6fe8..50d5de05b837 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -97,13 +97,9 @@ static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm) tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0)); } -static void tk_set_sleep_time(struct timekeeper *tk, struct timespec64 t) +static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta) { - /* Verify consistency before modifying */ - WARN_ON_ONCE(tk->offs_boot.tv64 != timespec64_to_ktime(tk->total_sleep_time).tv64); - - tk->total_sleep_time = t; - tk->offs_boot = timespec64_to_ktime(t); + tk->offs_boot = ktime_add(tk->offs_boot, delta); } /** @@ -919,10 +915,6 @@ void __init timekeeping_init(void) set_normalized_timespec64(&tmp, -boot.tv_sec, -boot.tv_nsec); tk_set_wall_to_mono(tk, tmp); - tmp.tv_sec = 0; - tmp.tv_nsec = 0; - tk_set_sleep_time(tk, tmp); - timekeeping_update(tk, TK_MIRROR); write_seqcount_end(&tk_core.seq); @@ -950,7 +942,7 @@ static void __timekeeping_inject_sleeptime(struct timekeeper *tk, } tk_xtime_add(tk, delta); tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta)); - tk_set_sleep_time(tk, timespec64_add(tk->total_sleep_time, *delta)); + tk_update_sleep_time(tk, timespec64_to_ktime(*delta)); tk_debug_account_sleep_time(delta); } From 61edec81d260bc96a73c878bbdb4c614460346da Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:05:01 +0000 Subject: [PATCH 094/114] timekeeping: Simplify timekeeping_clocktai() timekeeping_clocktai() is not used in fast pathes, so the extra timespec conversion is not problematic. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/timekeeping.h | 5 +++++ kernel/time/timekeeping.c | 31 ------------------------------- 2 files changed, 5 insertions(+), 31 deletions(-) diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h index 7b8f20007871..f0f12a84a31b 100644 --- a/include/linux/timekeeping.h +++ b/include/linux/timekeeping.h @@ -166,6 +166,11 @@ static inline void get_monotonic_boottime(struct timespec *ts) *ts = ktime_to_timespec(ktime_get_boottime()); } +static inline void timekeeping_clocktai(struct timespec *ts) +{ + *ts = ktime_to_timespec(ktime_get_clocktai()); +} + /* * RTC specific */ diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 50d5de05b837..118e91e3071c 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -497,37 +497,6 @@ void ktime_get_ts64(struct timespec64 *ts) } EXPORT_SYMBOL_GPL(ktime_get_ts64); - -/** - * timekeeping_clocktai - Returns the TAI time of day in a timespec - * @ts: pointer to the timespec to be set - * - * Returns the time of day in a timespec. - */ -void timekeeping_clocktai(struct timespec *ts) -{ - struct timekeeper *tk = &tk_core.timekeeper; - struct timespec64 ts64; - unsigned long seq; - u64 nsecs; - - WARN_ON(timekeeping_suspended); - - do { - seq = read_seqcount_begin(&tk_core.seq); - - ts64.tv_sec = tk->xtime_sec + tk->tai_offset; - nsecs = timekeeping_get_ns(tk); - - } while (read_seqcount_retry(&tk_core.seq, seq)); - - ts64.tv_nsec = 0; - timespec64_add_ns(&ts64, nsecs); - *ts = timespec64_to_timespec(ts64); - -} -EXPORT_SYMBOL(timekeeping_clocktai); - #ifdef CONFIG_NTP_PPS /** From 2044fdb03eb4e511d5028be0917899931f17461f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:05:03 +0000 Subject: [PATCH 095/114] hangcheck-timer: Use ktime_get_ns() There is no point in having a S390 private implementation and there is no point in using the raw monotonic time. The NTP freqeuency adjustment of CLOCK_MONOTONIC is really not doing any harm for the hang check timer. Use ktime_get_ns() for everything and get rid of the timespec conversions. V2: Drop the raw monotonic and the S390 special case Signed-off-by: Thomas Gleixner Cc: Arnd Bergmann Cc: Greg Kroah-Hartman Cc: Heiko Carstens Acked-by: Greg Kroah-Hartman Signed-off-by: John Stultz --- drivers/char/hangcheck-timer.c | 33 +++++---------------------------- 1 file changed, 5 insertions(+), 28 deletions(-) diff --git a/drivers/char/hangcheck-timer.c b/drivers/char/hangcheck-timer.c index f953c96efc86..ebc4c73d8ca4 100644 --- a/drivers/char/hangcheck-timer.c +++ b/drivers/char/hangcheck-timer.c @@ -49,7 +49,7 @@ #include #include #include -#include +#include #define VERSION_STR "0.9.1" @@ -117,24 +117,7 @@ __setup("hcheck_reboot", hangcheck_parse_reboot); __setup("hcheck_dump_tasks", hangcheck_parse_dump_tasks); #endif /* not MODULE */ -#if defined(CONFIG_S390) -# define HAVE_MONOTONIC -# define TIMER_FREQ 1000000000ULL -#else -# define TIMER_FREQ 1000000000ULL -#endif - -#ifdef HAVE_MONOTONIC -extern unsigned long long monotonic_clock(void); -#else -static inline unsigned long long monotonic_clock(void) -{ - struct timespec ts; - getrawmonotonic(&ts); - return timespec_to_ns(&ts); -} -#endif /* HAVE_MONOTONIC */ - +#define TIMER_FREQ 1000000000ULL /* Last time scheduled */ static unsigned long long hangcheck_tsc, hangcheck_tsc_margin; @@ -143,12 +126,11 @@ static void hangcheck_fire(unsigned long); static DEFINE_TIMER(hangcheck_ticktock, hangcheck_fire, 0, 0); - static void hangcheck_fire(unsigned long data) { unsigned long long cur_tsc, tsc_diff; - cur_tsc = monotonic_clock(); + cur_tsc = ktime_get_ns(); if (cur_tsc > hangcheck_tsc) tsc_diff = cur_tsc - hangcheck_tsc; @@ -177,7 +159,7 @@ static void hangcheck_fire(unsigned long data) tsc_diff, tsc_diff - hangcheck_tick*TIMER_FREQ); #endif mod_timer(&hangcheck_ticktock, jiffies + (hangcheck_tick*HZ)); - hangcheck_tsc = monotonic_clock(); + hangcheck_tsc = ktime_get_ns(); } @@ -185,16 +167,11 @@ static int __init hangcheck_init(void) { printk("Hangcheck: starting hangcheck timer %s (tick is %d seconds, margin is %d seconds).\n", VERSION_STR, hangcheck_tick, hangcheck_margin); -#if defined (HAVE_MONOTONIC) - printk("Hangcheck: Using monotonic_clock().\n"); -#else - printk("Hangcheck: Using getrawmonotonic().\n"); -#endif /* HAVE_MONOTONIC */ hangcheck_tsc_margin = (unsigned long long)(hangcheck_margin + hangcheck_tick); hangcheck_tsc_margin *= (unsigned long long)TIMER_FREQ; - hangcheck_tsc = monotonic_clock(); + hangcheck_tsc = ktime_get_ns(); mod_timer(&hangcheck_ticktock, jiffies + (hangcheck_tick*HZ)); return 0; From f519b1a2e08c913375324a927992bb328387f169 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:05:04 +0000 Subject: [PATCH 096/114] timekeeping: Provide ktime_get_raw() Provide a ktime_t based interface for raw monotonic time. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/timekeeper_internal.h | 3 +++ include/linux/timekeeping.h | 6 ++++++ kernel/time/timekeeping.c | 25 +++++++++++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h index 8e5d77a01787..2e20275a7083 100644 --- a/include/linux/timekeeper_internal.h +++ b/include/linux/timekeeper_internal.h @@ -54,6 +54,9 @@ struct timekeeper { /* The current UTC to TAI offset in seconds */ s32 tai_offset; + /* Monotonic raw base time */ + ktime_t base_raw; + /* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */ struct timespec64 raw_time; diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h index f0f12a84a31b..58ad7eff83ff 100644 --- a/include/linux/timekeeping.h +++ b/include/linux/timekeeping.h @@ -107,6 +107,7 @@ enum tk_offsets { extern ktime_t ktime_get(void); extern ktime_t ktime_get_with_offset(enum tk_offsets offs); extern ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs); +extern ktime_t ktime_get_raw(void); /** * ktime_get_real - get the real (wall-) time in ktime_t format @@ -158,6 +159,11 @@ static inline u64 ktime_get_boot_ns(void) return ktime_to_ns(ktime_get_boottime()); } +static inline u64 ktime_get_raw_ns(void) +{ + return ktime_to_ns(ktime_get_raw()); +} + /* * Timespec interfaces utilizing the ktime based ones */ diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 118e91e3071c..af8051f4420d 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -305,6 +305,9 @@ static inline void tk_update_ktime_data(struct timekeeper *tk) nsec *= NSEC_PER_SEC; nsec += tk->wall_to_monotonic.tv_nsec; tk->base_mono = ns_to_ktime(nsec); + + /* Update the monotonic raw base */ + tk->base_raw = timespec64_to_ktime(tk->raw_time); } /* must hold timekeeper_lock */ @@ -466,6 +469,27 @@ ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs) } EXPORT_SYMBOL_GPL(ktime_mono_to_any); +/** + * ktime_get_raw - Returns the raw monotonic time in ktime_t format + */ +ktime_t ktime_get_raw(void) +{ + struct timekeeper *tk = &tk_core.timekeeper; + unsigned int seq; + ktime_t base; + s64 nsecs; + + do { + seq = read_seqcount_begin(&tk_core.seq); + base = tk->base_raw; + nsecs = timekeeping_get_ns_raw(tk); + + } while (read_seqcount_retry(&tk_core.seq, seq)); + + return ktime_add_ns(base, nsecs); +} +EXPORT_SYMBOL_GPL(ktime_get_raw); + /** * ktime_get_ts64 - get the monotonic clock in timespec64 format * @ts: pointer to timespec variable @@ -878,6 +902,7 @@ void __init timekeeping_init(void) tk_set_xtime(tk, &now); tk->raw_time.tv_sec = 0; tk->raw_time.tv_nsec = 0; + tk->base_raw.tv64 = 0; if (boot.tv_sec == 0 && boot.tv_nsec == 0) boot = tk_xtime(tk); From 5ed0bdf21a85d78e04f89f15ccf227562177cbd9 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:05:06 +0000 Subject: [PATCH 097/114] drm: i915: Use nsec based interfaces Use ktime_get_raw_ns() and get rid of the back and forth timespec conversions. Signed-off-by: Thomas Gleixner Acked-by: Daniel Vetter Signed-off-by: John Stultz --- drivers/gpu/drm/i915/i915_drv.h | 2 +- drivers/gpu/drm/i915/i915_gem.c | 33 ++++++++++++--------------------- drivers/gpu/drm/i915/intel_pm.c | 12 +++++------- 3 files changed, 18 insertions(+), 29 deletions(-) diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 374f964323ad..1f7700897dfc 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -931,7 +931,7 @@ struct intel_ilk_power_mgmt { unsigned long last_time1; unsigned long chipset_power; u64 last_count2; - struct timespec last_time2; + u64 last_time2; unsigned long gfx_power; u8 corr; diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index f36126383d26..74531caa5191 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -1149,16 +1149,16 @@ static bool can_wait_boost(struct drm_i915_file_private *file_priv) static int __wait_seqno(struct intel_engine_cs *ring, u32 seqno, unsigned reset_counter, bool interruptible, - struct timespec *timeout, + s64 *timeout, struct drm_i915_file_private *file_priv) { struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; const bool irq_test_in_progress = ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring); - struct timespec before, now; DEFINE_WAIT(wait); unsigned long timeout_expire; + s64 before, now; int ret; WARN(dev_priv->pm.irqs_disabled, "IRQs disabled\n"); @@ -1166,7 +1166,7 @@ static int __wait_seqno(struct intel_engine_cs *ring, u32 seqno, if (i915_seqno_passed(ring->get_seqno(ring, true), seqno)) return 0; - timeout_expire = timeout ? jiffies + timespec_to_jiffies_timeout(timeout) : 0; + timeout_expire = timeout ? jiffies + nsecs_to_jiffies((u64)*timeout) : 0; if (INTEL_INFO(dev)->gen >= 6 && can_wait_boost(file_priv)) { gen6_rps_boost(dev_priv); @@ -1181,7 +1181,7 @@ static int __wait_seqno(struct intel_engine_cs *ring, u32 seqno, /* Record current time in case interrupted by signal, or wedged */ trace_i915_gem_request_wait_begin(ring, seqno); - getrawmonotonic(&before); + before = ktime_get_raw_ns(); for (;;) { struct timer_list timer; @@ -1230,7 +1230,7 @@ static int __wait_seqno(struct intel_engine_cs *ring, u32 seqno, destroy_timer_on_stack(&timer); } } - getrawmonotonic(&now); + now = ktime_get_raw_ns(); trace_i915_gem_request_wait_end(ring, seqno); if (!irq_test_in_progress) @@ -1239,10 +1239,9 @@ static int __wait_seqno(struct intel_engine_cs *ring, u32 seqno, finish_wait(&ring->irq_queue, &wait); if (timeout) { - struct timespec sleep_time = timespec_sub(now, before); - *timeout = timespec_sub(*timeout, sleep_time); - if (!timespec_valid(timeout)) /* i.e. negative time remains */ - set_normalized_timespec(timeout, 0, 0); + s64 tres = *timeout - (now - before); + + *timeout = tres < 0 ? 0 : tres; } return ret; @@ -2753,16 +2752,10 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) struct drm_i915_gem_wait *args = data; struct drm_i915_gem_object *obj; struct intel_engine_cs *ring = NULL; - struct timespec timeout_stack, *timeout = NULL; unsigned reset_counter; u32 seqno = 0; int ret = 0; - if (args->timeout_ns >= 0) { - timeout_stack = ns_to_timespec(args->timeout_ns); - timeout = &timeout_stack; - } - ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; @@ -2787,9 +2780,9 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) goto out; /* Do this after OLR check to make sure we make forward progress polling - * on this IOCTL with a 0 timeout (like busy ioctl) + * on this IOCTL with a timeout <=0 (like busy ioctl) */ - if (!args->timeout_ns) { + if (args->timeout_ns <= 0) { ret = -ETIME; goto out; } @@ -2798,10 +2791,8 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); mutex_unlock(&dev->struct_mutex); - ret = __wait_seqno(ring, seqno, reset_counter, true, timeout, file->driver_priv); - if (timeout) - args->timeout_ns = timespec_to_ns(timeout); - return ret; + return __wait_seqno(ring, seqno, reset_counter, true, &args->timeout_ns, + file->driver_priv); out: drm_gem_object_unreference(&obj->base); diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index ee72807069e4..f1233f544f3e 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -2993,7 +2993,7 @@ static void ironlake_enable_drps(struct drm_device *dev) I915_READ(0x112e0); dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies); dev_priv->ips.last_count2 = I915_READ(0x112f4); - getrawmonotonic(&dev_priv->ips.last_time2); + dev_priv->ips.last_time2 = ktime_get_raw_ns(); spin_unlock_irq(&mchdev_lock); } @@ -4314,18 +4314,16 @@ static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid) static void __i915_update_gfx_val(struct drm_i915_private *dev_priv) { - struct timespec now, diff1; - u64 diff; - unsigned long diffms; + u64 now, diff, diffms; u32 count; assert_spin_locked(&mchdev_lock); - getrawmonotonic(&now); - diff1 = timespec_sub(now, dev_priv->ips.last_time2); + now = ktime_get_raw_ns(); + diffms = now - dev_priv->ips.last_time2; + do_div(diffms, NSEC_PER_MSEC); /* Don't divide by 0 */ - diffms = diff1.tv_sec * 1000 + diff1.tv_nsec / 1000000; if (!diffms) return; From f166e6dcb7225c4193bcda68c9346583ed78b186 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:05:07 +0000 Subject: [PATCH 098/114] drm: vmwgfx: Use nsec based interfaces No point in converting timespecs back and forth. Signed-off-by: Thomas Gleixner Cc: Thomas Hellstrom Signed-off-by: John Stultz --- drivers/gpu/drm/vmwgfx/vmwgfx_drv.h | 4 +-- drivers/gpu/drm/vmwgfx/vmwgfx_marker.c | 44 ++++++++------------------ 2 files changed, 16 insertions(+), 32 deletions(-) diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h index 6b252a887ae2..c886c024c637 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h @@ -159,8 +159,8 @@ struct vmw_surface { struct vmw_marker_queue { struct list_head head; - struct timespec lag; - struct timespec lag_time; + u64 lag; + u64 lag_time; spinlock_t lock; }; diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_marker.c b/drivers/gpu/drm/vmwgfx/vmwgfx_marker.c index 8a8725c2716c..efd1ffd68185 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_marker.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_marker.c @@ -31,14 +31,14 @@ struct vmw_marker { struct list_head head; uint32_t seqno; - struct timespec submitted; + u64 submitted; }; void vmw_marker_queue_init(struct vmw_marker_queue *queue) { INIT_LIST_HEAD(&queue->head); - queue->lag = ns_to_timespec(0); - getrawmonotonic(&queue->lag_time); + queue->lag = 0; + queue->lag_time = ktime_get_raw_ns(); spin_lock_init(&queue->lock); } @@ -62,7 +62,7 @@ int vmw_marker_push(struct vmw_marker_queue *queue, return -ENOMEM; marker->seqno = seqno; - getrawmonotonic(&marker->submitted); + marker->submitted = ktime_get_raw_ns(); spin_lock(&queue->lock); list_add_tail(&marker->head, &queue->head); spin_unlock(&queue->lock); @@ -74,14 +74,14 @@ int vmw_marker_pull(struct vmw_marker_queue *queue, uint32_t signaled_seqno) { struct vmw_marker *marker, *next; - struct timespec now; bool updated = false; + u64 now; spin_lock(&queue->lock); - getrawmonotonic(&now); + now = ktime_get_raw_ns(); if (list_empty(&queue->head)) { - queue->lag = ns_to_timespec(0); + queue->lag = 0; queue->lag_time = now; updated = true; goto out_unlock; @@ -91,7 +91,7 @@ int vmw_marker_pull(struct vmw_marker_queue *queue, if (signaled_seqno - marker->seqno > (1 << 30)) continue; - queue->lag = timespec_sub(now, marker->submitted); + queue->lag = now - marker->submitted; queue->lag_time = now; updated = true; list_del(&marker->head); @@ -104,27 +104,13 @@ out_unlock: return (updated) ? 0 : -EBUSY; } -static struct timespec vmw_timespec_add(struct timespec t1, - struct timespec t2) +static u64 vmw_fifo_lag(struct vmw_marker_queue *queue) { - t1.tv_sec += t2.tv_sec; - t1.tv_nsec += t2.tv_nsec; - if (t1.tv_nsec >= 1000000000L) { - t1.tv_sec += 1; - t1.tv_nsec -= 1000000000L; - } - - return t1; -} - -static struct timespec vmw_fifo_lag(struct vmw_marker_queue *queue) -{ - struct timespec now; + u64 now; spin_lock(&queue->lock); - getrawmonotonic(&now); - queue->lag = vmw_timespec_add(queue->lag, - timespec_sub(now, queue->lag_time)); + now = ktime_get_raw_ns(); + queue->lag += now - queue->lag_time; queue->lag_time = now; spin_unlock(&queue->lock); return queue->lag; @@ -134,11 +120,9 @@ static struct timespec vmw_fifo_lag(struct vmw_marker_queue *queue) static bool vmw_lag_lt(struct vmw_marker_queue *queue, uint32_t us) { - struct timespec lag, cond; + u64 cond = (u64) us * NSEC_PER_USEC; - cond = ns_to_timespec((s64) us * 1000); - lag = vmw_fifo_lag(queue); - return (timespec_compare(&lag, &cond) < 1); + return vmw_fifo_lag(queue) <= cond; } int vmw_wait_lag(struct vmw_private *dev_priv, From 6438e0ddc870f282f7ad46c050c211063a574687 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:05:09 +0000 Subject: [PATCH 099/114] wireless: ath9k: Get rid of timespec conversions We have interfaces. Remove the open coded cruft. Reduces text size along with the code. Signed-off-by: Thomas Gleixner Cc: QCA ath9k Development Cc: John W. Linville Signed-off-by: John Stultz --- drivers/net/wireless/ath/ath9k/hw.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/drivers/net/wireless/ath/ath9k/hw.c b/drivers/net/wireless/ath/ath9k/hw.c index 2a8ed8375ec0..14b80b1b450c 100644 --- a/drivers/net/wireless/ath/ath9k/hw.c +++ b/drivers/net/wireless/ath/ath9k/hw.c @@ -1734,7 +1734,6 @@ int ath9k_hw_reset(struct ath_hw *ah, struct ath9k_channel *chan, struct ath9k_hw_cal_data *caldata, bool fastcc) { struct ath_common *common = ath9k_hw_common(ah); - struct timespec ts; u32 saveLedState; u32 saveDefAntenna; u32 macStaId1; @@ -1784,8 +1783,7 @@ int ath9k_hw_reset(struct ath_hw *ah, struct ath9k_channel *chan, /* Save TSF before chip reset, a cold reset clears it */ tsf = ath9k_hw_gettsf64(ah); - getrawmonotonic(&ts); - usec = ts.tv_sec * 1000000ULL + ts.tv_nsec / 1000; + usec = ktime_to_us(ktime_get_raw()); saveLedState = REG_READ(ah, AR_CFG_LED) & (AR_CFG_LED_ASSOC_CTL | AR_CFG_LED_MODE_SEL | @@ -1818,8 +1816,7 @@ int ath9k_hw_reset(struct ath_hw *ah, struct ath9k_channel *chan, } /* Restore TSF */ - getrawmonotonic(&ts); - usec = ts.tv_sec * 1000000ULL + ts.tv_nsec / 1000 - usec; + usec = ktime_to_us(ktime_get_raw()) - usec; ath9k_hw_settsf64(ah, tsf + usec); if (AR_SREV_9280_20_OR_LATER(ah)) From 3a97837784acbf9fed699fc04d1799b0eb742fdf Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:05:10 +0000 Subject: [PATCH 100/114] clocksource: Make delta calculation a function We want to move the TSC sanity check into core code to make NMI safe accessors to clock monotonic[_raw] possible. For this we need to sanity check the delta calculation. Create a helper function and convert all sites to use it. [ Build fix from jstultz ] Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- kernel/time/clocksource.c | 12 +++++++----- kernel/time/timekeeping.c | 26 ++++++++++++++------------ kernel/time/timekeeping_internal.h | 6 ++++++ 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index ba3e502c955a..2e949cc9c9f1 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -32,6 +32,7 @@ #include #include "tick-internal.h" +#include "timekeeping_internal.h" void timecounter_init(struct timecounter *tc, const struct cyclecounter *cc, @@ -249,7 +250,7 @@ void clocksource_mark_unstable(struct clocksource *cs) static void clocksource_watchdog(unsigned long data) { struct clocksource *cs; - cycle_t csnow, wdnow; + cycle_t csnow, wdnow, delta; int64_t wd_nsec, cs_nsec; int next_cpu, reset_pending; @@ -282,11 +283,12 @@ static void clocksource_watchdog(unsigned long data) continue; } - wd_nsec = clocksource_cyc2ns((wdnow - cs->wd_last) & watchdog->mask, - watchdog->mult, watchdog->shift); + delta = clocksource_delta(wdnow, cs->wd_last, watchdog->mask); + wd_nsec = clocksource_cyc2ns(delta, watchdog->mult, + watchdog->shift); - cs_nsec = clocksource_cyc2ns((csnow - cs->cs_last) & - cs->mask, cs->mult, cs->shift); + delta = clocksource_delta(csnow, cs->cs_last, cs->mask); + cs_nsec = clocksource_cyc2ns(delta, cs->mult, cs->shift); cs->cs_last = csnow; cs->wd_last = wdnow; diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index af8051f4420d..531805013786 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -173,7 +173,7 @@ static inline u32 arch_gettimeoffset(void) { return 0; } static inline s64 timekeeping_get_ns(struct timekeeper *tk) { - cycle_t cycle_now, cycle_delta; + cycle_t cycle_now, delta; struct clocksource *clock; s64 nsec; @@ -182,9 +182,9 @@ static inline s64 timekeeping_get_ns(struct timekeeper *tk) cycle_now = clock->read(clock); /* calculate the delta since the last update_wall_time: */ - cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; + delta = clocksource_delta(cycle_now, clock->cycle_last, clock->mask); - nsec = cycle_delta * tk->mult + tk->xtime_nsec; + nsec = delta * tk->mult + tk->xtime_nsec; nsec >>= tk->shift; /* If arch requires, add in get_arch_timeoffset() */ @@ -193,7 +193,7 @@ static inline s64 timekeeping_get_ns(struct timekeeper *tk) static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk) { - cycle_t cycle_now, cycle_delta; + cycle_t cycle_now, delta; struct clocksource *clock; s64 nsec; @@ -202,10 +202,10 @@ static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk) cycle_now = clock->read(clock); /* calculate the delta since the last update_wall_time: */ - cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; + delta = clocksource_delta(cycle_now, clock->cycle_last, clock->mask); /* convert delta to nanoseconds. */ - nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift); + nsec = clocksource_cyc2ns(delta, clock->mult, clock->shift); /* If arch requires, add in get_arch_timeoffset() */ return nsec + arch_gettimeoffset(); @@ -336,23 +336,23 @@ static void timekeeping_update(struct timekeeper *tk, unsigned int action) */ static void timekeeping_forward_now(struct timekeeper *tk) { - cycle_t cycle_now, cycle_delta; + cycle_t cycle_now, delta; struct clocksource *clock; s64 nsec; clock = tk->clock; cycle_now = clock->read(clock); - cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; + delta = clocksource_delta(cycle_now, clock->cycle_last, clock->mask); tk->cycle_last = clock->cycle_last = cycle_now; - tk->xtime_nsec += cycle_delta * tk->mult; + tk->xtime_nsec += delta * tk->mult; /* If arch requires, add in get_arch_timeoffset() */ tk->xtime_nsec += (u64)arch_gettimeoffset() << tk->shift; tk_normalize_xtime(tk); - nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift); + nsec = clocksource_cyc2ns(delta, clock->mult, clock->shift); timespec64_add_ns(&tk->raw_time, nsec); } @@ -1026,7 +1026,8 @@ static void timekeeping_resume(void) u32 shift = clock->shift; s64 nsec = 0; - cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; + cycle_delta = clocksource_delta(cycle_now, clock->cycle_last, + clock->mask); /* * "cycle_delta * mutl" may cause 64 bits overflow, if the @@ -1432,7 +1433,8 @@ void update_wall_time(void) #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET offset = real_tk->cycle_interval; #else - offset = (clock->read(clock) - clock->cycle_last) & clock->mask; + offset = clocksource_delta(clock->read(clock), clock->cycle_last, + clock->mask); #endif /* Check if there's really nothing to do */ diff --git a/kernel/time/timekeeping_internal.h b/kernel/time/timekeeping_internal.h index e3d28ad236f9..05dfa6b25dc4 100644 --- a/kernel/time/timekeeping_internal.h +++ b/kernel/time/timekeeping_internal.h @@ -3,6 +3,7 @@ /* * timekeeping debug functions */ +#include #include #ifdef CONFIG_DEBUG_FS @@ -11,4 +12,9 @@ extern void tk_debug_account_sleep_time(struct timespec64 *t); #define tk_debug_account_sleep_time(x) #endif +static inline cycle_t clocksource_delta(cycle_t now, cycle_t last, cycle_t mask) +{ + return (now - last) & mask; +} + #endif /* _TIMEKEEPING_INTERNAL_H */ From 09ec54429c6d10f87d1f084de53ae2c1c3a81108 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:05:12 +0000 Subject: [PATCH 101/114] clocksource: Move cycle_last validation to core code The only user of the cycle_last validation is the x86 TSC. In order to provide NMI safe accessor functions for clock monotonic and monotonic_raw we need to do that in the core. We can't do the TSC specific if (now < cycle_last) now = cycle_last; for the other wrapping around clocksources, but TSC has CLOCKSOURCE_MASK(64) which actually does not mask out anything so if now is less than cycle_last the subtraction will give a negative result. So we can check for that in clocksource_delta() and return 0 for that case. Implement and enable it for x86 Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- arch/x86/Kconfig | 1 + arch/x86/kernel/tsc.c | 21 +++++++++------------ kernel/time/Kconfig | 5 +++++ kernel/time/timekeeping_internal.h | 9 +++++++++ 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 7fa17b5ce668..d08e061c187a 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -109,6 +109,7 @@ config X86 select CLOCKSOURCE_WATCHDOG select GENERIC_CLOCKEVENTS select ARCH_CLOCKSOURCE_DATA + select CLOCKSOURCE_VALIDATE_LAST_CYCLE select GENERIC_CLOCKEVENTS_BROADCAST if X86_64 || (X86_32 && X86_LOCAL_APIC) select GENERIC_TIME_VSYSCALL select GENERIC_STRNCPY_FROM_USER diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 57e5ce126d5a..456c0e660c43 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -951,7 +951,7 @@ core_initcall(cpufreq_tsc); static struct clocksource clocksource_tsc; /* - * We compare the TSC to the cycle_last value in the clocksource + * We used to compare the TSC to the cycle_last value in the clocksource * structure to avoid a nasty time-warp. This can be observed in a * very small window right after one CPU updated cycle_last under * xtime/vsyscall_gtod lock and the other CPU reads a TSC value which @@ -961,26 +961,23 @@ static struct clocksource clocksource_tsc; * due to the unsigned delta calculation of the time keeping core * code, which is necessary to support wrapping clocksources like pm * timer. + * + * This sanity check is now done in the core timekeeping code. + * checking the result of read_tsc() - cycle_last for being negative. + * That works because CLOCKSOURCE_MASK(64) does not mask out any bit. */ static cycle_t read_tsc(struct clocksource *cs) { - cycle_t ret = (cycle_t)get_cycles(); - - return ret >= clocksource_tsc.cycle_last ? - ret : clocksource_tsc.cycle_last; -} - -static void resume_tsc(struct clocksource *cs) -{ - if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC_S3)) - clocksource_tsc.cycle_last = 0; + return (cycle_t)get_cycles(); } +/* + * .mask MUST be CLOCKSOURCE_MASK(64). See comment above read_tsc() + */ static struct clocksource clocksource_tsc = { .name = "tsc", .rating = 300, .read = read_tsc, - .resume = resume_tsc, .mask = CLOCKSOURCE_MASK(64), .flags = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_MUST_VERIFY, diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig index feccfd888732..d626dc98e8df 100644 --- a/kernel/time/Kconfig +++ b/kernel/time/Kconfig @@ -12,6 +12,11 @@ config CLOCKSOURCE_WATCHDOG config ARCH_CLOCKSOURCE_DATA bool +# Clocksources require validation of the clocksource against the last +# cycle update - x86/TSC misfeature +config CLOCKSOURCE_VALIDATE_LAST_CYCLE + bool + # Timekeeping vsyscall support config GENERIC_TIME_VSYSCALL bool diff --git a/kernel/time/timekeeping_internal.h b/kernel/time/timekeeping_internal.h index 05dfa6b25dc4..4ea005a7f9da 100644 --- a/kernel/time/timekeeping_internal.h +++ b/kernel/time/timekeeping_internal.h @@ -12,9 +12,18 @@ extern void tk_debug_account_sleep_time(struct timespec64 *t); #define tk_debug_account_sleep_time(x) #endif +#ifdef CONFIG_CLOCKSOURCE_VALIDATE_LAST_CYCLE +static inline cycle_t clocksource_delta(cycle_t now, cycle_t last, cycle_t mask) +{ + cycle_t ret = (now - last) & mask; + + return (s64) ret > 0 ? ret : 0; +} +#else static inline cycle_t clocksource_delta(cycle_t now, cycle_t last, cycle_t mask) { return (now - last) & mask; } +#endif #endif /* _TIMEKEEPING_INTERNAL_H */ From 4a0e637738f06673725792d74eed67f8779b62c7 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:05:13 +0000 Subject: [PATCH 102/114] clocksource: Get rid of cycle_last cycle_last was added to the clocksource to support the TSC validation. We moved that to the core code, so we can get rid of the extra copy. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- arch/arm64/kernel/vdso.c | 2 +- arch/ia64/kernel/time.c | 4 ++-- arch/powerpc/kernel/time.c | 4 ++-- arch/s390/kernel/time.c | 2 +- arch/tile/kernel/time.c | 2 +- arch/x86/kernel/vsyscall_gtod.c | 2 +- arch/x86/kvm/x86.c | 2 +- include/linux/clocksource.h | 2 -- include/linux/timekeeper_internal.h | 7 ++++--- kernel/time/timekeeping.c | 23 +++++++++++------------ 10 files changed, 24 insertions(+), 26 deletions(-) diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c index 50384fec56c4..574672f001f7 100644 --- a/arch/arm64/kernel/vdso.c +++ b/arch/arm64/kernel/vdso.c @@ -224,7 +224,7 @@ void update_vsyscall(struct timekeeper *tk) vdso_data->wtm_clock_nsec = tk->wall_to_monotonic.tv_nsec; if (!use_syscall) { - vdso_data->cs_cycle_last = tk->clock->cycle_last; + vdso_data->cs_cycle_last = tk->cycle_last; vdso_data->xtime_clock_sec = tk->xtime_sec; vdso_data->xtime_clock_nsec = tk->xtime_nsec; vdso_data->cs_mult = tk->mult; diff --git a/arch/ia64/kernel/time.c b/arch/ia64/kernel/time.c index 71c52bc7c28d..11dc42da7daf 100644 --- a/arch/ia64/kernel/time.c +++ b/arch/ia64/kernel/time.c @@ -441,7 +441,7 @@ void update_vsyscall_tz(void) } void update_vsyscall_old(struct timespec *wall, struct timespec *wtm, - struct clocksource *c, u32 mult) + struct clocksource *c, u32 mult, cycles_t cycle_last) { write_seqcount_begin(&fsyscall_gtod_data.seq); @@ -450,7 +450,7 @@ void update_vsyscall_old(struct timespec *wall, struct timespec *wtm, fsyscall_gtod_data.clk_mult = mult; fsyscall_gtod_data.clk_shift = c->shift; fsyscall_gtod_data.clk_fsys_mmio = c->archdata.fsys_mmio; - fsyscall_gtod_data.clk_cycle_last = c->cycle_last; + fsyscall_gtod_data.clk_cycle_last = cycle_last; /* copy kernel time structures */ fsyscall_gtod_data.wall_time.tv_sec = wall->tv_sec; diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index 9fff9cdcc519..368ab374d33c 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -741,7 +741,7 @@ static cycle_t timebase_read(struct clocksource *cs) } void update_vsyscall_old(struct timespec *wall_time, struct timespec *wtm, - struct clocksource *clock, u32 mult) + struct clocksource *clock, u32 mult, cycle_t cycle_last) { u64 new_tb_to_xs, new_stamp_xsec; u32 frac_sec; @@ -774,7 +774,7 @@ void update_vsyscall_old(struct timespec *wall_time, struct timespec *wtm, * We expect the caller to have done the first increment of * vdso_data->tb_update_count already. */ - vdso_data->tb_orig_stamp = clock->cycle_last; + vdso_data->tb_orig_stamp = cycle_last; vdso_data->stamp_xsec = new_stamp_xsec; vdso_data->tb_to_xs = new_tb_to_xs; vdso_data->wtom_clock_sec = wtm->tv_sec; diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index 0931b110c826..97950f392613 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -220,7 +220,7 @@ void update_vsyscall(struct timekeeper *tk) /* Make userspace gettimeofday spin until we're done. */ ++vdso_data->tb_update_count; smp_wmb(); - vdso_data->xtime_tod_stamp = tk->clock->cycle_last; + vdso_data->xtime_tod_stamp = tk->cycle_last; vdso_data->xtime_clock_sec = tk->xtime_sec; vdso_data->xtime_clock_nsec = tk->xtime_nsec; vdso_data->wtom_clock_sec = diff --git a/arch/tile/kernel/time.c b/arch/tile/kernel/time.c index ae70155c2f16..d22d5bfc1e4e 100644 --- a/arch/tile/kernel/time.c +++ b/arch/tile/kernel/time.c @@ -269,7 +269,7 @@ void update_vsyscall(struct timekeeper *tk) /* Userspace gettimeofday will spin while this value is odd. */ ++vdso_data->tb_update_count; smp_wmb(); - vdso_data->xtime_tod_stamp = clock->cycle_last; + vdso_data->xtime_tod_stamp = tk->cycle_last; vdso_data->xtime_clock_sec = tk->xtime_sec; vdso_data->xtime_clock_nsec = tk->xtime_nsec; vdso_data->wtom_clock_sec = wtm->tv_sec; diff --git a/arch/x86/kernel/vsyscall_gtod.c b/arch/x86/kernel/vsyscall_gtod.c index 9531fbb123ba..c3cb3c144591 100644 --- a/arch/x86/kernel/vsyscall_gtod.c +++ b/arch/x86/kernel/vsyscall_gtod.c @@ -32,7 +32,7 @@ void update_vsyscall(struct timekeeper *tk) /* copy vsyscall data */ vdata->vclock_mode = tk->clock->archdata.vclock_mode; - vdata->cycle_last = tk->clock->cycle_last; + vdata->cycle_last = tk->cycle_last; vdata->mask = tk->clock->mask; vdata->mult = tk->mult; vdata->shift = tk->shift; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 63832f5110b6..7b25125f3f42 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1001,7 +1001,7 @@ static void update_pvclock_gtod(struct timekeeper *tk) /* copy pvclock gtod data */ vdata->clock.vclock_mode = tk->clock->archdata.vclock_mode; - vdata->clock.cycle_last = tk->clock->cycle_last; + vdata->clock.cycle_last = tk->cycle_last; vdata->clock.mask = tk->clock->mask; vdata->clock.mult = tk->mult; vdata->clock.shift = tk->shift; diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h index a16b497d5159..653f0e2b6ca9 100644 --- a/include/linux/clocksource.h +++ b/include/linux/clocksource.h @@ -162,7 +162,6 @@ extern u64 timecounter_cyc2time(struct timecounter *tc, * @archdata: arch-specific data * @suspend: suspend function for the clocksource, if necessary * @resume: resume function for the clocksource, if necessary - * @cycle_last: most recent cycle counter value seen by ::read() * @owner: module reference, must be set by clocksource in modules */ struct clocksource { @@ -171,7 +170,6 @@ struct clocksource { * clocksource itself is cacheline aligned. */ cycle_t (*read)(struct clocksource *cs); - cycle_t cycle_last; cycle_t mask; u32 mult; u32 shift; diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h index 2e20275a7083..cb88096222c0 100644 --- a/include/linux/timekeeper_internal.h +++ b/include/linux/timekeeper_internal.h @@ -29,6 +29,8 @@ struct timekeeper { /* Current clocksource used for timekeeping. */ struct clocksource *clock; + /* Last cycle value */ + cycle_t cycle_last; /* NTP adjusted clock multiplier */ u32 mult; /* The shift value of the current clocksource. */ @@ -62,8 +64,6 @@ struct timekeeper { /* Number of clock cycles in one NTP interval. */ cycle_t cycle_interval; - /* Last cycle value (also stored in clock->cycle_last) */ - cycle_t cycle_last; /* Number of clock shifted nano seconds in one NTP interval. */ u64 xtime_interval; /* shifted nano seconds left over when rounding cycle_interval */ @@ -91,7 +91,8 @@ extern void update_vsyscall_tz(void); #elif defined(CONFIG_GENERIC_TIME_VSYSCALL_OLD) extern void update_vsyscall_old(struct timespec *ts, struct timespec *wtm, - struct clocksource *c, u32 mult); + struct clocksource *c, u32 mult, + cycles_t cycle_last); extern void update_vsyscall_tz(void); #else diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 531805013786..4e748c404749 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -121,7 +121,7 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) old_clock = tk->clock; tk->clock = clock; - tk->cycle_last = clock->cycle_last = clock->read(clock); + tk->cycle_last = clock->read(clock); /* Do the ns -> cycle conversion first, using original mult */ tmp = NTP_INTERVAL_LENGTH; @@ -182,7 +182,7 @@ static inline s64 timekeeping_get_ns(struct timekeeper *tk) cycle_now = clock->read(clock); /* calculate the delta since the last update_wall_time: */ - delta = clocksource_delta(cycle_now, clock->cycle_last, clock->mask); + delta = clocksource_delta(cycle_now, tk->cycle_last, clock->mask); nsec = delta * tk->mult + tk->xtime_nsec; nsec >>= tk->shift; @@ -202,7 +202,7 @@ static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk) cycle_now = clock->read(clock); /* calculate the delta since the last update_wall_time: */ - delta = clocksource_delta(cycle_now, clock->cycle_last, clock->mask); + delta = clocksource_delta(cycle_now, tk->cycle_last, clock->mask); /* convert delta to nanoseconds. */ nsec = clocksource_cyc2ns(delta, clock->mult, clock->shift); @@ -218,7 +218,8 @@ static inline void update_vsyscall(struct timekeeper *tk) struct timespec xt; xt = tk_xtime(tk); - update_vsyscall_old(&xt, &tk->wall_to_monotonic, tk->clock, tk->mult); + update_vsyscall_old(&xt, &tk->wall_to_monotonic, tk->clock, tk->mult, + tk->cycle_last); } static inline void old_vsyscall_fixup(struct timekeeper *tk) @@ -342,8 +343,8 @@ static void timekeeping_forward_now(struct timekeeper *tk) clock = tk->clock; cycle_now = clock->read(clock); - delta = clocksource_delta(cycle_now, clock->cycle_last, clock->mask); - tk->cycle_last = clock->cycle_last = cycle_now; + delta = clocksource_delta(cycle_now, tk->cycle_last, clock->mask); + tk->cycle_last = cycle_now; tk->xtime_nsec += delta * tk->mult; @@ -1020,13 +1021,13 @@ static void timekeeping_resume(void) */ cycle_now = clock->read(clock); if ((clock->flags & CLOCK_SOURCE_SUSPEND_NONSTOP) && - cycle_now > clock->cycle_last) { + cycle_now > tk->cycle_last) { u64 num, max = ULLONG_MAX; u32 mult = clock->mult; u32 shift = clock->shift; s64 nsec = 0; - cycle_delta = clocksource_delta(cycle_now, clock->cycle_last, + cycle_delta = clocksource_delta(cycle_now, tk->cycle_last, clock->mask); /* @@ -1053,7 +1054,7 @@ static void timekeeping_resume(void) __timekeeping_inject_sleeptime(tk, &ts_delta); /* Re-base the last cycle value */ - tk->cycle_last = clock->cycle_last = cycle_now; + tk->cycle_last = cycle_now; tk->ntp_error = 0; timekeeping_suspended = 0; timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET); @@ -1433,7 +1434,7 @@ void update_wall_time(void) #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET offset = real_tk->cycle_interval; #else - offset = clocksource_delta(clock->read(clock), clock->cycle_last, + offset = clocksource_delta(clock->read(clock), tk->cycle_last, clock->mask); #endif @@ -1477,8 +1478,6 @@ void update_wall_time(void) clock_set |= accumulate_nsecs_to_secs(tk); write_seqcount_begin(&tk_core.seq); - /* Update clock->cycle_last with the new value */ - clock->cycle_last = tk->cycle_last; /* * Update the real timekeeper. * From 6d3aadf3e180e09dbefab16478c6876b584ce16e Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:05:15 +0000 Subject: [PATCH 103/114] timekeeping: Restructure the timekeeper some more Access to time requires to touch two cachelines at minimum 1) The timekeeper data structure 2) The clocksource data structure The access to the clocksource data structure can be avoided as almost all clocksource implementations ignore the argument to the read callback, which is a pointer to the clocksource. But the core needs to touch it to access the members @read and @mask. So we are better off by copying the @read function pointer and the @mask from the clocksource to the core data structure itself. For the most used ktime_get() access all required data including the @read and @mask copies fits together with the sequence counter into a single 64 byte cacheline. For the other time access functions we touch in the current code three cache lines in the worst case. But with the clocksource data copies we can reduce that to two adjacent cachelines, which is more efficient than disjunct cache lines. Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/timekeeper_internal.h | 4 ++++ kernel/time/timekeeping.c | 35 +++++++++++++---------------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h index cb88096222c0..75bb8add78f5 100644 --- a/include/linux/timekeeper_internal.h +++ b/include/linux/timekeeper_internal.h @@ -29,6 +29,10 @@ struct timekeeper { /* Current clocksource used for timekeeping. */ struct clocksource *clock; + /* Read function of @clock */ + cycle_t (*read)(struct clocksource *cs); + /* Bitmask for two's complement subtraction of non 64bit counters */ + cycle_t mask; /* Last cycle value */ cycle_t cycle_last; /* NTP adjusted clock multiplier */ diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 4e748c404749..14b7367e6b94 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -121,7 +121,9 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) old_clock = tk->clock; tk->clock = clock; - tk->cycle_last = clock->read(clock); + tk->read = clock->read; + tk->mask = clock->mask; + tk->cycle_last = tk->read(clock); /* Do the ns -> cycle conversion first, using original mult */ tmp = NTP_INTERVAL_LENGTH; @@ -174,15 +176,13 @@ static inline u32 arch_gettimeoffset(void) { return 0; } static inline s64 timekeeping_get_ns(struct timekeeper *tk) { cycle_t cycle_now, delta; - struct clocksource *clock; s64 nsec; /* read clocksource: */ - clock = tk->clock; - cycle_now = clock->read(clock); + cycle_now = tk->read(tk->clock); /* calculate the delta since the last update_wall_time: */ - delta = clocksource_delta(cycle_now, tk->cycle_last, clock->mask); + delta = clocksource_delta(cycle_now, tk->cycle_last, tk->mask); nsec = delta * tk->mult + tk->xtime_nsec; nsec >>= tk->shift; @@ -193,16 +193,15 @@ static inline s64 timekeeping_get_ns(struct timekeeper *tk) static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk) { + struct clocksource *clock = tk->clock; cycle_t cycle_now, delta; - struct clocksource *clock; s64 nsec; /* read clocksource: */ - clock = tk->clock; - cycle_now = clock->read(clock); + cycle_now = tk->read(clock); /* calculate the delta since the last update_wall_time: */ - delta = clocksource_delta(cycle_now, tk->cycle_last, clock->mask); + delta = clocksource_delta(cycle_now, tk->cycle_last, tk->mask); /* convert delta to nanoseconds. */ nsec = clocksource_cyc2ns(delta, clock->mult, clock->shift); @@ -337,13 +336,12 @@ static void timekeeping_update(struct timekeeper *tk, unsigned int action) */ static void timekeeping_forward_now(struct timekeeper *tk) { + struct clocksource *clock = tk->clock; cycle_t cycle_now, delta; - struct clocksource *clock; s64 nsec; - clock = tk->clock; - cycle_now = clock->read(clock); - delta = clocksource_delta(cycle_now, tk->cycle_last, clock->mask); + cycle_now = tk->read(clock); + delta = clocksource_delta(cycle_now, tk->cycle_last, tk->mask); tk->cycle_last = cycle_now; tk->xtime_nsec += delta * tk->mult; @@ -1019,7 +1017,7 @@ static void timekeeping_resume(void) * The less preferred source will only be tried if there is no better * usable source. The rtc part is handled separately in rtc core code. */ - cycle_now = clock->read(clock); + cycle_now = tk->read(clock); if ((clock->flags & CLOCK_SOURCE_SUSPEND_NONSTOP) && cycle_now > tk->cycle_last) { u64 num, max = ULLONG_MAX; @@ -1028,7 +1026,7 @@ static void timekeeping_resume(void) s64 nsec = 0; cycle_delta = clocksource_delta(cycle_now, tk->cycle_last, - clock->mask); + tk->mask); /* * "cycle_delta * mutl" may cause 64 bits overflow, if the @@ -1415,7 +1413,6 @@ static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset, */ void update_wall_time(void) { - struct clocksource *clock; struct timekeeper *real_tk = &tk_core.timekeeper; struct timekeeper *tk = &shadow_timekeeper; cycle_t offset; @@ -1429,13 +1426,11 @@ void update_wall_time(void) if (unlikely(timekeeping_suspended)) goto out; - clock = real_tk->clock; - #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET offset = real_tk->cycle_interval; #else - offset = clocksource_delta(clock->read(clock), tk->cycle_last, - clock->mask); + offset = clocksource_delta(tk->read(tk->clock), tk->cycle_last, + tk->mask); #endif /* Check if there's really nothing to do */ From d28ede83791defee9a81e558540699dc46dbbe13 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:05:16 +0000 Subject: [PATCH 104/114] timekeeping: Create struct tk_read_base and use it in struct timekeeper The members of the new struct are the required ones for the new NMI safe accessor to clcok monotonic. In order to reuse the existing timekeeping code and to make the update of the fast NMI safe timekeepers a simple memcpy use the struct for the timekeeper as well and convert all users. Signed-off-by: Thomas Gleixner Cc: Peter Zijlstra Cc: Ingo Molnar Cc: Mathieu Desnoyers Signed-off-by: John Stultz --- arch/arm64/kernel/vdso.c | 10 +-- arch/s390/kernel/time.c | 16 ++-- arch/tile/kernel/time.c | 10 +-- arch/x86/kernel/vsyscall_gtod.c | 23 ++--- arch/x86/kvm/x86.c | 14 +-- include/linux/timekeeper_internal.h | 113 +++++++++++++----------- kernel/time/timekeeping.c | 132 ++++++++++++++-------------- 7 files changed, 163 insertions(+), 155 deletions(-) diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c index 574672f001f7..8296f7f5f0ba 100644 --- a/arch/arm64/kernel/vdso.c +++ b/arch/arm64/kernel/vdso.c @@ -211,7 +211,7 @@ struct vm_area_struct *get_gate_vma(struct mm_struct *mm) void update_vsyscall(struct timekeeper *tk) { struct timespec xtime_coarse; - u32 use_syscall = strcmp(tk->clock->name, "arch_sys_counter"); + u32 use_syscall = strcmp(tk->tkr.clock->name, "arch_sys_counter"); ++vdso_data->tb_seq_count; smp_wmb(); @@ -224,11 +224,11 @@ void update_vsyscall(struct timekeeper *tk) vdso_data->wtm_clock_nsec = tk->wall_to_monotonic.tv_nsec; if (!use_syscall) { - vdso_data->cs_cycle_last = tk->cycle_last; + vdso_data->cs_cycle_last = tk->tkr.cycle_last; vdso_data->xtime_clock_sec = tk->xtime_sec; - vdso_data->xtime_clock_nsec = tk->xtime_nsec; - vdso_data->cs_mult = tk->mult; - vdso_data->cs_shift = tk->shift; + vdso_data->xtime_clock_nsec = tk->tkr.xtime_nsec; + vdso_data->cs_mult = tk->tkr.mult; + vdso_data->cs_shift = tk->tkr.shift; } smp_wmb(); diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index 97950f392613..4cef607f3711 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -214,26 +214,26 @@ void update_vsyscall(struct timekeeper *tk) { u64 nsecps; - if (tk->clock != &clocksource_tod) + if (tk->tkr.clock != &clocksource_tod) return; /* Make userspace gettimeofday spin until we're done. */ ++vdso_data->tb_update_count; smp_wmb(); - vdso_data->xtime_tod_stamp = tk->cycle_last; + vdso_data->xtime_tod_stamp = tk->tkr.cycle_last; vdso_data->xtime_clock_sec = tk->xtime_sec; - vdso_data->xtime_clock_nsec = tk->xtime_nsec; + vdso_data->xtime_clock_nsec = tk->tkr.xtime_nsec; vdso_data->wtom_clock_sec = tk->xtime_sec + tk->wall_to_monotonic.tv_sec; - vdso_data->wtom_clock_nsec = tk->xtime_nsec + - + ((u64) tk->wall_to_monotonic.tv_nsec << tk->shift); - nsecps = (u64) NSEC_PER_SEC << tk->shift; + vdso_data->wtom_clock_nsec = tk->tkr.xtime_nsec + + + ((u64) tk->wall_to_monotonic.tv_nsec << tk->tkr.shift); + nsecps = (u64) NSEC_PER_SEC << tk->tkr.shift; while (vdso_data->wtom_clock_nsec >= nsecps) { vdso_data->wtom_clock_nsec -= nsecps; vdso_data->wtom_clock_sec++; } - vdso_data->tk_mult = tk->mult; - vdso_data->tk_shift = tk->shift; + vdso_data->tk_mult = tk->tkr.mult; + vdso_data->tk_shift = tk->tkr.shift; smp_wmb(); ++vdso_data->tb_update_count; } diff --git a/arch/tile/kernel/time.c b/arch/tile/kernel/time.c index d22d5bfc1e4e..d8fbc289e680 100644 --- a/arch/tile/kernel/time.c +++ b/arch/tile/kernel/time.c @@ -261,7 +261,7 @@ void update_vsyscall_tz(void) void update_vsyscall(struct timekeeper *tk) { struct timespec *wtm = &tk->wall_to_monotonic; - struct clocksource *clock = tk->clock; + struct clocksource *clock = tk->tkr.clock; if (clock != &cycle_counter_cs) return; @@ -269,13 +269,13 @@ void update_vsyscall(struct timekeeper *tk) /* Userspace gettimeofday will spin while this value is odd. */ ++vdso_data->tb_update_count; smp_wmb(); - vdso_data->xtime_tod_stamp = tk->cycle_last; + vdso_data->xtime_tod_stamp = tk->tkr.cycle_last; vdso_data->xtime_clock_sec = tk->xtime_sec; - vdso_data->xtime_clock_nsec = tk->xtime_nsec; + vdso_data->xtime_clock_nsec = tk->tkr.xtime_nsec; vdso_data->wtom_clock_sec = wtm->tv_sec; vdso_data->wtom_clock_nsec = wtm->tv_nsec; - vdso_data->mult = tk->mult; - vdso_data->shift = tk->shift; + vdso_data->mult = tk->tkr.mult; + vdso_data->shift = tk->tkr.shift; smp_wmb(); ++vdso_data->tb_update_count; } diff --git a/arch/x86/kernel/vsyscall_gtod.c b/arch/x86/kernel/vsyscall_gtod.c index c3cb3c144591..c7d791f32b98 100644 --- a/arch/x86/kernel/vsyscall_gtod.c +++ b/arch/x86/kernel/vsyscall_gtod.c @@ -31,29 +31,30 @@ void update_vsyscall(struct timekeeper *tk) gtod_write_begin(vdata); /* copy vsyscall data */ - vdata->vclock_mode = tk->clock->archdata.vclock_mode; - vdata->cycle_last = tk->cycle_last; - vdata->mask = tk->clock->mask; - vdata->mult = tk->mult; - vdata->shift = tk->shift; + vdata->vclock_mode = tk->tkr.clock->archdata.vclock_mode; + vdata->cycle_last = tk->tkr.cycle_last; + vdata->mask = tk->tkr.mask; + vdata->mult = tk->tkr.mult; + vdata->shift = tk->tkr.shift; vdata->wall_time_sec = tk->xtime_sec; - vdata->wall_time_snsec = tk->xtime_nsec; + vdata->wall_time_snsec = tk->tkr.xtime_nsec; vdata->monotonic_time_sec = tk->xtime_sec + tk->wall_to_monotonic.tv_sec; - vdata->monotonic_time_snsec = tk->xtime_nsec + vdata->monotonic_time_snsec = tk->tkr.xtime_nsec + ((u64)tk->wall_to_monotonic.tv_nsec - << tk->shift); + << tk->tkr.shift); while (vdata->monotonic_time_snsec >= - (((u64)NSEC_PER_SEC) << tk->shift)) { + (((u64)NSEC_PER_SEC) << tk->tkr.shift)) { vdata->monotonic_time_snsec -= - ((u64)NSEC_PER_SEC) << tk->shift; + ((u64)NSEC_PER_SEC) << tk->tkr.shift; vdata->monotonic_time_sec++; } vdata->wall_time_coarse_sec = tk->xtime_sec; - vdata->wall_time_coarse_nsec = (long)(tk->xtime_nsec >> tk->shift); + vdata->wall_time_coarse_nsec = (long)(tk->tkr.xtime_nsec >> + tk->tkr.shift); vdata->monotonic_time_coarse_sec = vdata->wall_time_coarse_sec + tk->wall_to_monotonic.tv_sec; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 7b25125f3f42..b7e57946d1c1 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -995,19 +995,19 @@ static void update_pvclock_gtod(struct timekeeper *tk) struct pvclock_gtod_data *vdata = &pvclock_gtod_data; u64 boot_ns; - boot_ns = ktime_to_ns(ktime_add(tk->base_mono, tk->offs_boot)); + boot_ns = ktime_to_ns(ktime_add(tk->tkr.base_mono, tk->offs_boot)); write_seqcount_begin(&vdata->seq); /* copy pvclock gtod data */ - vdata->clock.vclock_mode = tk->clock->archdata.vclock_mode; - vdata->clock.cycle_last = tk->cycle_last; - vdata->clock.mask = tk->clock->mask; - vdata->clock.mult = tk->mult; - vdata->clock.shift = tk->shift; + vdata->clock.vclock_mode = tk->tkr.clock->archdata.vclock_mode; + vdata->clock.cycle_last = tk->tkr.cycle_last; + vdata->clock.mask = tk->tkr.mask; + vdata->clock.mult = tk->tkr.mult; + vdata->clock.shift = tk->tkr.shift; vdata->boot_ns = boot_ns; - vdata->nsec_base = tk->xtime_nsec; + vdata->nsec_base = tk->tkr.xtime_nsec; write_seqcount_end(&vdata->seq); } diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h index 75bb8add78f5..97381997625b 100644 --- a/include/linux/timekeeper_internal.h +++ b/include/linux/timekeeper_internal.h @@ -10,80 +10,87 @@ #include #include -/* - * Structure holding internal timekeeping values. +/** + * struct tk_read_base - base structure for timekeeping readout + * @clock: Current clocksource used for timekeeping. + * @read: Read function of @clock + * @mask: Bitmask for two's complement subtraction of non 64bit clocks + * @cycle_last: @clock cycle value at last update + * @mult: NTP adjusted multiplier for scaled math conversion + * @shift: Shift value for scaled math conversion + * @xtime_nsec: Shifted (fractional) nano seconds offset for readout + * @base_mono: ktime_t (nanoseconds) base time for readout * - * Note: wall_to_monotonic is what we need to add to xtime (or xtime - * corrected for sub jiffie times) to get to monotonic time. - * Monotonic is pegged at zero at system boot time, so - * wall_to_monotonic will be negative, however, we will ALWAYS keep - * the tv_nsec part positive so we can use the usual normalization. + * This struct has size 56 byte on 64 bit. Together with a seqcount it + * occupies a single 64byte cache line. + * + * The struct is separate from struct timekeeper as it is also used + * for a fast NMI safe accessor to clock monotonic. + */ +struct tk_read_base { + struct clocksource *clock; + cycle_t (*read)(struct clocksource *cs); + cycle_t mask; + cycle_t cycle_last; + u32 mult; + u32 shift; + u64 xtime_nsec; + ktime_t base_mono; +}; + +/** + * struct timekeeper - Structure holding internal timekeeping values. + * @tkr: The readout base structure + * @xtime_sec: Current CLOCK_REALTIME time in seconds + * @wall_to_monotonic: CLOCK_REALTIME to CLOCK_MONOTONIC offset + * @offs_real: Offset clock monotonic -> clock realtime + * @offs_boot: Offset clock monotonic -> clock boottime + * @offs_tai: Offset clock monotonic -> clock tai + * @tai_offset: The current UTC to TAI offset in seconds + * @base_raw: Monotonic raw base time in ktime_t format + * @raw_time: Monotonic raw base time in timespec64 format + * @cycle_interval: Number of clock cycles in one NTP interval + * @xtime_interval: Number of clock shifted nano seconds in one NTP + * interval. + * @xtime_remainder: Shifted nano seconds left over when rounding + * @cycle_interval + * @raw_interval: Raw nano seconds accumulated per NTP interval. + * @ntp_error: Difference between accumulated time and NTP time in ntp + * shifted nano seconds. + * @ntp_error_shift: Shift conversion between clock shifted nano seconds and + * ntp shifted nano seconds. + * + * Note: For timespec(64) based interfaces wall_to_monotonic is what + * we need to add to xtime (or xtime corrected for sub jiffie times) + * to get to monotonic time. Monotonic is pegged at zero at system + * boot time, so wall_to_monotonic will be negative, however, we will + * ALWAYS keep the tv_nsec part positive so we can use the usual + * normalization. * * wall_to_monotonic is moved after resume from suspend for the - * monotonic time not to jump. To calculate the real boot time offset - * we need to do offs_real - offs_boot. + * monotonic time not to jump. We need to add total_sleep_time to + * wall_to_monotonic to get the real boot based time offset. * - * - wall_to_monotonic is no longer the boot time, getboottime must be + * wall_to_monotonic is no longer the boot time, getboottime must be * used instead. */ struct timekeeper { - /* Current clocksource used for timekeeping. */ - struct clocksource *clock; - /* Read function of @clock */ - cycle_t (*read)(struct clocksource *cs); - /* Bitmask for two's complement subtraction of non 64bit counters */ - cycle_t mask; - /* Last cycle value */ - cycle_t cycle_last; - /* NTP adjusted clock multiplier */ - u32 mult; - /* The shift value of the current clocksource. */ - u32 shift; - /* Clock shifted nano seconds */ - u64 xtime_nsec; - - /* Monotonic base time */ - ktime_t base_mono; - - /* Current CLOCK_REALTIME time in seconds */ + struct tk_read_base tkr; u64 xtime_sec; - /* CLOCK_REALTIME to CLOCK_MONOTONIC offset */ struct timespec64 wall_to_monotonic; - - /* Offset clock monotonic -> clock realtime */ ktime_t offs_real; - /* Offset clock monotonic -> clock boottime */ ktime_t offs_boot; - /* Offset clock monotonic -> clock tai */ ktime_t offs_tai; - - /* The current UTC to TAI offset in seconds */ s32 tai_offset; - - /* Monotonic raw base time */ ktime_t base_raw; - - /* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */ struct timespec64 raw_time; - /* Number of clock cycles in one NTP interval. */ + /* The following members are for timekeeping internal use */ cycle_t cycle_interval; - /* Number of clock shifted nano seconds in one NTP interval. */ u64 xtime_interval; - /* shifted nano seconds left over when rounding cycle_interval */ s64 xtime_remainder; - /* Raw nano seconds accumulated per NTP interval. */ u32 raw_interval; - - /* - * Difference between accumulated time and NTP time in ntp - * shifted nano seconds. - */ s64 ntp_error; - /* - * Shift conversion between clock shifted nano seconds and - * ntp shifted nano seconds. - */ u32 ntp_error_shift; }; diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 14b7367e6b94..ccb69980ef7e 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -52,8 +52,8 @@ bool __read_mostly persistent_clock_exist = false; static inline void tk_normalize_xtime(struct timekeeper *tk) { - while (tk->xtime_nsec >= ((u64)NSEC_PER_SEC << tk->shift)) { - tk->xtime_nsec -= (u64)NSEC_PER_SEC << tk->shift; + while (tk->tkr.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr.shift)) { + tk->tkr.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr.shift; tk->xtime_sec++; } } @@ -63,20 +63,20 @@ static inline struct timespec64 tk_xtime(struct timekeeper *tk) struct timespec64 ts; ts.tv_sec = tk->xtime_sec; - ts.tv_nsec = (long)(tk->xtime_nsec >> tk->shift); + ts.tv_nsec = (long)(tk->tkr.xtime_nsec >> tk->tkr.shift); return ts; } static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts) { tk->xtime_sec = ts->tv_sec; - tk->xtime_nsec = (u64)ts->tv_nsec << tk->shift; + tk->tkr.xtime_nsec = (u64)ts->tv_nsec << tk->tkr.shift; } static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts) { tk->xtime_sec += ts->tv_sec; - tk->xtime_nsec += (u64)ts->tv_nsec << tk->shift; + tk->tkr.xtime_nsec += (u64)ts->tv_nsec << tk->tkr.shift; tk_normalize_xtime(tk); } @@ -119,11 +119,11 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) u64 tmp, ntpinterval; struct clocksource *old_clock; - old_clock = tk->clock; - tk->clock = clock; - tk->read = clock->read; - tk->mask = clock->mask; - tk->cycle_last = tk->read(clock); + old_clock = tk->tkr.clock; + tk->tkr.clock = clock; + tk->tkr.read = clock->read; + tk->tkr.mask = clock->mask; + tk->tkr.cycle_last = tk->tkr.read(clock); /* Do the ns -> cycle conversion first, using original mult */ tmp = NTP_INTERVAL_LENGTH; @@ -147,11 +147,11 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) if (old_clock) { int shift_change = clock->shift - old_clock->shift; if (shift_change < 0) - tk->xtime_nsec >>= -shift_change; + tk->tkr.xtime_nsec >>= -shift_change; else - tk->xtime_nsec <<= shift_change; + tk->tkr.xtime_nsec <<= shift_change; } - tk->shift = clock->shift; + tk->tkr.shift = clock->shift; tk->ntp_error = 0; tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift; @@ -161,7 +161,7 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) * active clocksource. These value will be adjusted via NTP * to counteract clock drifting. */ - tk->mult = clock->mult; + tk->tkr.mult = clock->mult; } /* Timekeeper helper functions. */ @@ -179,13 +179,13 @@ static inline s64 timekeeping_get_ns(struct timekeeper *tk) s64 nsec; /* read clocksource: */ - cycle_now = tk->read(tk->clock); + cycle_now = tk->tkr.read(tk->tkr.clock); /* calculate the delta since the last update_wall_time: */ - delta = clocksource_delta(cycle_now, tk->cycle_last, tk->mask); + delta = clocksource_delta(cycle_now, tk->tkr.cycle_last, tk->tkr.mask); - nsec = delta * tk->mult + tk->xtime_nsec; - nsec >>= tk->shift; + nsec = delta * tk->tkr.mult + tk->tkr.xtime_nsec; + nsec >>= tk->tkr.shift; /* If arch requires, add in get_arch_timeoffset() */ return nsec + arch_gettimeoffset(); @@ -193,15 +193,15 @@ static inline s64 timekeeping_get_ns(struct timekeeper *tk) static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk) { - struct clocksource *clock = tk->clock; + struct clocksource *clock = tk->tkr.clock; cycle_t cycle_now, delta; s64 nsec; /* read clocksource: */ - cycle_now = tk->read(clock); + cycle_now = tk->tkr.read(clock); /* calculate the delta since the last update_wall_time: */ - delta = clocksource_delta(cycle_now, tk->cycle_last, tk->mask); + delta = clocksource_delta(cycle_now, tk->tkr.cycle_last, tk->tkr.mask); /* convert delta to nanoseconds. */ nsec = clocksource_cyc2ns(delta, clock->mult, clock->shift); @@ -217,8 +217,8 @@ static inline void update_vsyscall(struct timekeeper *tk) struct timespec xt; xt = tk_xtime(tk); - update_vsyscall_old(&xt, &tk->wall_to_monotonic, tk->clock, tk->mult, - tk->cycle_last); + update_vsyscall_old(&xt, &tk->wall_to_monotonic, tk->tkr.clock, tk->tkr.mult, + tk->tkr.cycle_last); } static inline void old_vsyscall_fixup(struct timekeeper *tk) @@ -235,11 +235,11 @@ static inline void old_vsyscall_fixup(struct timekeeper *tk) * (shifted nanoseconds), and CONFIG_GENERIC_TIME_VSYSCALL_OLD * users are removed, this can be killed. */ - remainder = tk->xtime_nsec & ((1ULL << tk->shift) - 1); - tk->xtime_nsec -= remainder; - tk->xtime_nsec += 1ULL << tk->shift; + remainder = tk->tkr.xtime_nsec & ((1ULL << tk->tkr.shift) - 1); + tk->tkr.xtime_nsec -= remainder; + tk->tkr.xtime_nsec += 1ULL << tk->tkr.shift; tk->ntp_error += remainder << tk->ntp_error_shift; - tk->ntp_error -= (1ULL << tk->shift) << tk->ntp_error_shift; + tk->ntp_error -= (1ULL << tk->tkr.shift) << tk->ntp_error_shift; } #else #define old_vsyscall_fixup(tk) @@ -304,7 +304,7 @@ static inline void tk_update_ktime_data(struct timekeeper *tk) nsec = (s64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec); nsec *= NSEC_PER_SEC; nsec += tk->wall_to_monotonic.tv_nsec; - tk->base_mono = ns_to_ktime(nsec); + tk->tkr.base_mono = ns_to_ktime(nsec); /* Update the monotonic raw base */ tk->base_raw = timespec64_to_ktime(tk->raw_time); @@ -336,18 +336,18 @@ static void timekeeping_update(struct timekeeper *tk, unsigned int action) */ static void timekeeping_forward_now(struct timekeeper *tk) { - struct clocksource *clock = tk->clock; + struct clocksource *clock = tk->tkr.clock; cycle_t cycle_now, delta; s64 nsec; - cycle_now = tk->read(clock); - delta = clocksource_delta(cycle_now, tk->cycle_last, tk->mask); - tk->cycle_last = cycle_now; + cycle_now = tk->tkr.read(clock); + delta = clocksource_delta(cycle_now, tk->tkr.cycle_last, tk->tkr.mask); + tk->tkr.cycle_last = cycle_now; - tk->xtime_nsec += delta * tk->mult; + tk->tkr.xtime_nsec += delta * tk->tkr.mult; /* If arch requires, add in get_arch_timeoffset() */ - tk->xtime_nsec += (u64)arch_gettimeoffset() << tk->shift; + tk->tkr.xtime_nsec += (u64)arch_gettimeoffset() << tk->tkr.shift; tk_normalize_xtime(tk); @@ -412,7 +412,7 @@ ktime_t ktime_get(void) do { seq = read_seqcount_begin(&tk_core.seq); - base = tk->base_mono; + base = tk->tkr.base_mono; nsecs = timekeeping_get_ns(tk); } while (read_seqcount_retry(&tk_core.seq, seq)); @@ -438,7 +438,7 @@ ktime_t ktime_get_with_offset(enum tk_offsets offs) do { seq = read_seqcount_begin(&tk_core.seq); - base = ktime_add(tk->base_mono, *offset); + base = ktime_add(tk->tkr.base_mono, *offset); nsecs = timekeeping_get_ns(tk); } while (read_seqcount_retry(&tk_core.seq, seq)); @@ -731,7 +731,7 @@ static int change_clocksource(void *data) */ if (try_module_get(new->owner)) { if (!new->enable || new->enable(new) == 0) { - old = tk->clock; + old = tk->tkr.clock; tk_setup_internals(tk, new); if (old->disable) old->disable(old); @@ -759,11 +759,11 @@ int timekeeping_notify(struct clocksource *clock) { struct timekeeper *tk = &tk_core.timekeeper; - if (tk->clock == clock) + if (tk->tkr.clock == clock) return 0; stop_machine(change_clocksource, clock, NULL); tick_clock_notify(); - return tk->clock == clock ? 0 : -1; + return tk->tkr.clock == clock ? 0 : -1; } /** @@ -803,7 +803,7 @@ int timekeeping_valid_for_hres(void) do { seq = read_seqcount_begin(&tk_core.seq); - ret = tk->clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; + ret = tk->tkr.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; } while (read_seqcount_retry(&tk_core.seq, seq)); @@ -822,7 +822,7 @@ u64 timekeeping_max_deferment(void) do { seq = read_seqcount_begin(&tk_core.seq); - ret = tk->clock->max_idle_ns; + ret = tk->tkr.clock->max_idle_ns; } while (read_seqcount_retry(&tk_core.seq, seq)); @@ -989,7 +989,7 @@ void timekeeping_inject_sleeptime(struct timespec *delta) static void timekeeping_resume(void) { struct timekeeper *tk = &tk_core.timekeeper; - struct clocksource *clock = tk->clock; + struct clocksource *clock = tk->tkr.clock; unsigned long flags; struct timespec64 ts_new, ts_delta; struct timespec tmp; @@ -1017,16 +1017,16 @@ static void timekeeping_resume(void) * The less preferred source will only be tried if there is no better * usable source. The rtc part is handled separately in rtc core code. */ - cycle_now = tk->read(clock); + cycle_now = tk->tkr.read(clock); if ((clock->flags & CLOCK_SOURCE_SUSPEND_NONSTOP) && - cycle_now > tk->cycle_last) { + cycle_now > tk->tkr.cycle_last) { u64 num, max = ULLONG_MAX; u32 mult = clock->mult; u32 shift = clock->shift; s64 nsec = 0; - cycle_delta = clocksource_delta(cycle_now, tk->cycle_last, - tk->mask); + cycle_delta = clocksource_delta(cycle_now, tk->tkr.cycle_last, + tk->tkr.mask); /* * "cycle_delta * mutl" may cause 64 bits overflow, if the @@ -1052,7 +1052,7 @@ static void timekeeping_resume(void) __timekeeping_inject_sleeptime(tk, &ts_delta); /* Re-base the last cycle value */ - tk->cycle_last = cycle_now; + tk->tkr.cycle_last = cycle_now; tk->ntp_error = 0; timekeeping_suspended = 0; timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET); @@ -1239,12 +1239,12 @@ static void timekeeping_adjust(struct timekeeper *tk, s64 offset) } } - if (unlikely(tk->clock->maxadj && - (tk->mult + adj > tk->clock->mult + tk->clock->maxadj))) { + if (unlikely(tk->tkr.clock->maxadj && + (tk->tkr.mult + adj > tk->tkr.clock->mult + tk->tkr.clock->maxadj))) { printk_deferred_once(KERN_WARNING "Adjusting %s more than 11%% (%ld vs %ld)\n", - tk->clock->name, (long)tk->mult + adj, - (long)tk->clock->mult + tk->clock->maxadj); + tk->tkr.clock->name, (long)tk->tkr.mult + adj, + (long)tk->tkr.clock->mult + tk->tkr.clock->maxadj); } /* * So the following can be confusing. @@ -1295,9 +1295,9 @@ static void timekeeping_adjust(struct timekeeper *tk, s64 offset) * * XXX - TODO: Doc ntp_error calculation. */ - tk->mult += adj; + tk->tkr.mult += adj; tk->xtime_interval += interval; - tk->xtime_nsec -= offset; + tk->tkr.xtime_nsec -= offset; tk->ntp_error -= (interval - offset) << tk->ntp_error_shift; out_adjust: @@ -1315,9 +1315,9 @@ out_adjust: * We'll correct this error next time through this function, when * xtime_nsec is not as small. */ - if (unlikely((s64)tk->xtime_nsec < 0)) { - s64 neg = -(s64)tk->xtime_nsec; - tk->xtime_nsec = 0; + if (unlikely((s64)tk->tkr.xtime_nsec < 0)) { + s64 neg = -(s64)tk->tkr.xtime_nsec; + tk->tkr.xtime_nsec = 0; tk->ntp_error += neg << tk->ntp_error_shift; } @@ -1333,13 +1333,13 @@ out_adjust: */ static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk) { - u64 nsecps = (u64)NSEC_PER_SEC << tk->shift; + u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr.shift; unsigned int clock_set = 0; - while (tk->xtime_nsec >= nsecps) { + while (tk->tkr.xtime_nsec >= nsecps) { int leap; - tk->xtime_nsec -= nsecps; + tk->tkr.xtime_nsec -= nsecps; tk->xtime_sec++; /* Figure out if its a leap sec and apply if needed */ @@ -1384,9 +1384,9 @@ static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset, /* Accumulate one shifted interval */ offset -= interval; - tk->cycle_last += interval; + tk->tkr.cycle_last += interval; - tk->xtime_nsec += tk->xtime_interval << shift; + tk->tkr.xtime_nsec += tk->xtime_interval << shift; *clock_set |= accumulate_nsecs_to_secs(tk); /* Accumulate raw time */ @@ -1429,8 +1429,8 @@ void update_wall_time(void) #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET offset = real_tk->cycle_interval; #else - offset = clocksource_delta(tk->read(tk->clock), tk->cycle_last, - tk->mask); + offset = clocksource_delta(tk->tkr.read(tk->tkr.clock), + tk->tkr.cycle_last, tk->tkr.mask); #endif /* Check if there's really nothing to do */ @@ -1591,8 +1591,8 @@ ktime_t ktime_get_update_offsets_tick(ktime_t *offs_real, ktime_t *offs_boot, do { seq = read_seqcount_begin(&tk_core.seq); - base = tk->base_mono; - nsecs = tk->xtime_nsec >> tk->shift; + base = tk->tkr.base_mono; + nsecs = tk->tkr.xtime_nsec >> tk->tkr.shift; *offs_real = tk->offs_real; *offs_boot = tk->offs_boot; @@ -1623,7 +1623,7 @@ ktime_t ktime_get_update_offsets_now(ktime_t *offs_real, ktime_t *offs_boot, do { seq = read_seqcount_begin(&tk_core.seq); - base = tk->base_mono; + base = tk->tkr.base_mono; nsecs = timekeeping_get_ns(tk); *offs_real = tk->offs_real; From 0e5ac3a8b100469ea154f87dd57b685fbdd356f6 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:05:18 +0000 Subject: [PATCH 105/114] timekeeping: Use tk_read_base as argument for timekeeping_get_ns() All the function needs is in the tk_read_base struct. No functional change for the current code, just a preparatory patch for the NMI safe accessor to clock monotonic which will use struct tk_read_base as well. Signed-off-by: Thomas Gleixner Cc: Steven Rostedt Cc: Peter Zijlstra Cc: Mathieu Desnoyers Signed-off-by: John Stultz --- kernel/time/timekeeping.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index ccb69980ef7e..dee23c9d6c21 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -173,19 +173,19 @@ u32 (*arch_gettimeoffset)(void) = default_arch_gettimeoffset; static inline u32 arch_gettimeoffset(void) { return 0; } #endif -static inline s64 timekeeping_get_ns(struct timekeeper *tk) +static inline s64 timekeeping_get_ns(struct tk_read_base *tkr) { cycle_t cycle_now, delta; s64 nsec; /* read clocksource: */ - cycle_now = tk->tkr.read(tk->tkr.clock); + cycle_now = tkr->read(tkr->clock); /* calculate the delta since the last update_wall_time: */ - delta = clocksource_delta(cycle_now, tk->tkr.cycle_last, tk->tkr.mask); + delta = clocksource_delta(cycle_now, tkr->cycle_last, tkr->mask); - nsec = delta * tk->tkr.mult + tk->tkr.xtime_nsec; - nsec >>= tk->tkr.shift; + nsec = delta * tkr->mult + tkr->xtime_nsec; + nsec >>= tkr->shift; /* If arch requires, add in get_arch_timeoffset() */ return nsec + arch_gettimeoffset(); @@ -372,7 +372,7 @@ int __getnstimeofday64(struct timespec64 *ts) seq = read_seqcount_begin(&tk_core.seq); ts->tv_sec = tk->xtime_sec; - nsecs = timekeeping_get_ns(tk); + nsecs = timekeeping_get_ns(&tk->tkr); } while (read_seqcount_retry(&tk_core.seq, seq)); @@ -413,7 +413,7 @@ ktime_t ktime_get(void) do { seq = read_seqcount_begin(&tk_core.seq); base = tk->tkr.base_mono; - nsecs = timekeeping_get_ns(tk); + nsecs = timekeeping_get_ns(&tk->tkr); } while (read_seqcount_retry(&tk_core.seq, seq)); @@ -439,7 +439,7 @@ ktime_t ktime_get_with_offset(enum tk_offsets offs) do { seq = read_seqcount_begin(&tk_core.seq); base = ktime_add(tk->tkr.base_mono, *offset); - nsecs = timekeeping_get_ns(tk); + nsecs = timekeeping_get_ns(&tk->tkr); } while (read_seqcount_retry(&tk_core.seq, seq)); @@ -509,7 +509,7 @@ void ktime_get_ts64(struct timespec64 *ts) do { seq = read_seqcount_begin(&tk_core.seq); ts->tv_sec = tk->xtime_sec; - nsec = timekeeping_get_ns(tk); + nsec = timekeeping_get_ns(&tk->tkr); tomono = tk->wall_to_monotonic; } while (read_seqcount_retry(&tk_core.seq, seq)); @@ -547,7 +547,7 @@ void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real) ts_real->tv_nsec = 0; nsecs_raw = timekeeping_get_ns_raw(tk); - nsecs_real = timekeeping_get_ns(tk); + nsecs_real = timekeeping_get_ns(&tk->tkr); } while (read_seqcount_retry(&tk_core.seq, seq)); @@ -1624,7 +1624,7 @@ ktime_t ktime_get_update_offsets_now(ktime_t *offs_real, ktime_t *offs_boot, seq = read_seqcount_begin(&tk_core.seq); base = tk->tkr.base_mono; - nsecs = timekeeping_get_ns(tk); + nsecs = timekeeping_get_ns(&tk->tkr); *offs_real = tk->offs_real; *offs_boot = tk->offs_boot; From 0ea5a520f73ca31abc4c10b6d5bc14a884a0641b Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:05:20 +0000 Subject: [PATCH 106/114] seqcount: Provide raw_read_seqcount() raw_read_seqcount opens a read critical section of the given seqcount without any lockdep checking and without checking or masking the LSB. Calling code is responsible for handling that. Preparatory patch to provide a NMI safe clock monotonic accessor function. Signed-off-by: Thomas Gleixner Cc: John Stultz Cc: Peter Zijlstra Cc: Ingo Molnar Cc: Mathieu Desnoyers Signed-off-by: John Stultz --- include/linux/seqlock.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index 535f158977b9..dcc64b9bfc41 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h @@ -116,6 +116,22 @@ repeat: return ret; } +/** + * raw_read_seqcount - Read the raw seqcount + * @s: pointer to seqcount_t + * Returns: count to be passed to read_seqcount_retry + * + * raw_read_seqcount opens a read critical section of the given + * seqcount without any lockdep checking and without checking or + * masking the LSB. Calling code is responsible for handling that. + */ +static inline unsigned raw_read_seqcount(const seqcount_t *s) +{ + unsigned ret = ACCESS_ONCE(s->sequence); + smp_rmb(); + return ret; +} + /** * raw_read_seqcount_begin - start seq-read critical section w/o lockdep * @s: pointer to seqcount_t From 9b0fd802e8c0545148324916055e7b40d97963fa Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Wed, 16 Jul 2014 21:05:21 +0000 Subject: [PATCH 107/114] seqcount: Add raw_write_seqcount_latch() For NMI safe access to clock monotonic we use the seqcount LSB as index of a timekeeper array. The update sequence looks like this: smp_wmb(); <- prior stores to a[1] seq++; smp_wmb(); <- seq increment before update of a[0] update(a[0]); smp_wmb(); <- update of a[0] seq++; smp_wmb(); <- seq increment before update of a[1] update(a[1]); To avoid open coded barriers, provide a helper function. [ tglx: Split out of a combo patch against the first implementation of the NMI safe accessor ] Signed-off-by: Mathieu Desnoyers Cc: John Stultz Cc: Peter Zijlstra Cc: Ingo Molnar Cc: Steven Rostedt Signed-off-by: Thomas Gleixner Signed-off-by: John Stultz --- include/linux/seqlock.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index dcc64b9bfc41..cce6e7453592 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h @@ -235,6 +235,17 @@ static inline void raw_write_seqcount_end(seqcount_t *s) s->sequence++; } +/* + * raw_write_seqcount_latch - redirect readers to even/odd copy + * @s: pointer to seqcount_t + */ +static inline void raw_write_seqcount_latch(seqcount_t *s) +{ + smp_wmb(); /* prior stores before incrementing "sequence" */ + s->sequence++; + smp_wmb(); /* increment "sequence" before following stores */ +} + /* * Sequence counter only version assumes that callers are using their * own mutexing. From 4396e058c52e167729729cf64ea3dfa229637086 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:05:23 +0000 Subject: [PATCH 108/114] timekeeping: Provide fast and NMI safe access to CLOCK_MONOTONIC Tracers want a correlated time between the kernel instrumentation and user space. We really do not want to export sched_clock() to user space, so we need to provide something sensible for this. Using separate data structures with an non blocking sequence count based update mechanism allows us to do that. The data structure required for the readout has a sequence counter and two copies of the timekeeping data. On the update side: smp_wmb(); tkf->seq++; smp_wmb(); update(tkf->base[0], tk); smp_wmb(); tkf->seq++; smp_wmb(); update(tkf->base[1], tk); On the reader side: do { seq = tkf->seq; smp_rmb(); idx = seq & 0x01; now = now(tkf->base[idx]); smp_rmb(); } while (seq != tkf->seq) So if a NMI hits the update of base[0] it will use base[1] which is still consistent, but this timestamp is not guaranteed to be monotonic across an update. The timestamp is calculated by: now = base_mono + clock_delta * slope So if the update lowers the slope, readers who are forced to the not yet updated second array are still using the old steeper slope. tmono ^ | o n | o n | u | o |o |12345678---> reader order o = old slope u = update n = new slope So reader 6 will observe time going backwards versus reader 5. While other CPUs are likely to be able observe that, the only way for a CPU local observation is when an NMI hits in the middle of the update. Timestamps taken from that NMI context might be ahead of the following timestamps. Callers need to be aware of that and deal with it. V2: Got rid of clock monotonic raw and reorganized the data structures. Folded in the barrier fix from Mathieu. Signed-off-by: Thomas Gleixner Cc: Peter Zijlstra Cc: Steven Rostedt Cc: Mathieu Desnoyers Signed-off-by: John Stultz --- include/linux/timekeeping.h | 2 + kernel/time/timekeeping.c | 124 ++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+) diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h index 58ad7eff83ff..1caa6b04fdc5 100644 --- a/include/linux/timekeeping.h +++ b/include/linux/timekeeping.h @@ -164,6 +164,8 @@ static inline u64 ktime_get_raw_ns(void) return ktime_to_ns(ktime_get_raw()); } +extern u64 ktime_get_mono_fast_ns(void); + /* * Timespec interfaces utilizing the ktime based ones */ diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index dee23c9d6c21..8980fb722fc5 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -44,6 +44,22 @@ static struct { static DEFINE_RAW_SPINLOCK(timekeeper_lock); static struct timekeeper shadow_timekeeper; +/** + * struct tk_fast - NMI safe timekeeper + * @seq: Sequence counter for protecting updates. The lowest bit + * is the index for the tk_read_base array + * @base: tk_read_base array. Access is indexed by the lowest bit of + * @seq. + * + * See @update_fast_timekeeper() below. + */ +struct tk_fast { + seqcount_t seq; + struct tk_read_base base[2]; +}; + +static struct tk_fast tk_fast_mono ____cacheline_aligned; + /* flag for if timekeeping is suspended */ int __read_mostly timekeeping_suspended; @@ -210,6 +226,112 @@ static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk) return nsec + arch_gettimeoffset(); } +/** + * update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper. + * @tk: The timekeeper from which we take the update + * @tkf: The fast timekeeper to update + * @tbase: The time base for the fast timekeeper (mono/raw) + * + * We want to use this from any context including NMI and tracing / + * instrumenting the timekeeping code itself. + * + * So we handle this differently than the other timekeeping accessor + * functions which retry when the sequence count has changed. The + * update side does: + * + * smp_wmb(); <- Ensure that the last base[1] update is visible + * tkf->seq++; + * smp_wmb(); <- Ensure that the seqcount update is visible + * update(tkf->base[0], tk); + * smp_wmb(); <- Ensure that the base[0] update is visible + * tkf->seq++; + * smp_wmb(); <- Ensure that the seqcount update is visible + * update(tkf->base[1], tk); + * + * The reader side does: + * + * do { + * seq = tkf->seq; + * smp_rmb(); + * idx = seq & 0x01; + * now = now(tkf->base[idx]); + * smp_rmb(); + * } while (seq != tkf->seq) + * + * As long as we update base[0] readers are forced off to + * base[1]. Once base[0] is updated readers are redirected to base[0] + * and the base[1] update takes place. + * + * So if a NMI hits the update of base[0] then it will use base[1] + * which is still consistent. In the worst case this can result is a + * slightly wrong timestamp (a few nanoseconds). See + * @ktime_get_mono_fast_ns. + */ +static void update_fast_timekeeper(struct timekeeper *tk) +{ + struct tk_read_base *base = tk_fast_mono.base; + + /* Force readers off to base[1] */ + raw_write_seqcount_latch(&tk_fast_mono.seq); + + /* Update base[0] */ + memcpy(base, &tk->tkr, sizeof(*base)); + + /* Force readers back to base[0] */ + raw_write_seqcount_latch(&tk_fast_mono.seq); + + /* Update base[1] */ + memcpy(base + 1, base, sizeof(*base)); +} + +/** + * ktime_get_mono_fast_ns - Fast NMI safe access to clock monotonic + * + * This timestamp is not guaranteed to be monotonic across an update. + * The timestamp is calculated by: + * + * now = base_mono + clock_delta * slope + * + * So if the update lowers the slope, readers who are forced to the + * not yet updated second array are still using the old steeper slope. + * + * tmono + * ^ + * | o n + * | o n + * | u + * | o + * |o + * |12345678---> reader order + * + * o = old slope + * u = update + * n = new slope + * + * So reader 6 will observe time going backwards versus reader 5. + * + * While other CPUs are likely to be able observe that, the only way + * for a CPU local observation is when an NMI hits in the middle of + * the update. Timestamps taken from that NMI context might be ahead + * of the following timestamps. Callers need to be aware of that and + * deal with it. + */ +u64 notrace ktime_get_mono_fast_ns(void) +{ + struct tk_read_base *tkr; + unsigned int seq; + u64 now; + + do { + seq = raw_read_seqcount(&tk_fast_mono.seq); + tkr = tk_fast_mono.base + (seq & 0x01); + now = ktime_to_ns(tkr->base_mono) + timekeeping_get_ns(tkr); + + } while (read_seqcount_retry(&tk_fast_mono.seq, seq)); + return now; +} +EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns); + #ifdef CONFIG_GENERIC_TIME_VSYSCALL_OLD static inline void update_vsyscall(struct timekeeper *tk) @@ -325,6 +447,8 @@ static void timekeeping_update(struct timekeeper *tk, unsigned int action) if (action & TK_MIRROR) memcpy(&shadow_timekeeper, &tk_core.timekeeper, sizeof(tk_core.timekeeper)); + + update_fast_timekeeper(tk); } /** From 1b3e5c0936046e7e023149ddc8946d21c2ea20eb Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Jul 2014 21:05:25 +0000 Subject: [PATCH 109/114] ftrace: Provide trace clocks monotonic Expose the new NMI safe accessor to clock monotonic to the tracer. Signed-off-by: Thomas Gleixner Cc: Steven Rostedt Cc: Peter Zijlstra Cc: Mathieu Desnoyers Signed-off-by: John Stultz --- kernel/trace/trace.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index f243444a3772..84e2b45c0934 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -806,11 +806,12 @@ static struct { const char *name; int in_ns; /* is this clock in nanoseconds? */ } trace_clocks[] = { - { trace_clock_local, "local", 1 }, - { trace_clock_global, "global", 1 }, - { trace_clock_counter, "counter", 0 }, - { trace_clock_jiffies, "uptime", 1 }, - { trace_clock, "perf", 1 }, + { trace_clock_local, "local", 1 }, + { trace_clock_global, "global", 1 }, + { trace_clock_counter, "counter", 0 }, + { trace_clock_jiffies, "uptime", 1 }, + { trace_clock, "perf", 1 }, + { ktime_get_mono_fast_ns, "mono", 1 }, ARCH_TRACE_CLOCKS }; From e2dff1ec0cc81fcf3e0696604bacc3e1c816538c Mon Sep 17 00:00:00 2001 From: John Stultz Date: Wed, 23 Jul 2014 14:35:39 -0700 Subject: [PATCH 110/114] timekeeping: Minor fixup for timespec64->timespec assignment In the GENERIC_TIME_VSYSCALL_OLD update_vsyscall implementation, we take the tk_xtime() value, which returns a timespec64, and store it in a timespec. This luckily is ok, since the only architectures that use GENERIC_TIME_VSYSCALL_OLD are ia64 and ppc64, which are both 64 bit systems where timespec64 is the same as a timespec. Even so, for cleanliness reasons, use the conversion function to assign the proper type. Signed-off-by: John Stultz --- kernel/time/timekeeping.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 8980fb722fc5..2b56b959615b 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -338,7 +338,7 @@ static inline void update_vsyscall(struct timekeeper *tk) { struct timespec xt; - xt = tk_xtime(tk); + xt = timespec64_to_timespec(tk_xtime(tk)); update_vsyscall_old(&xt, &tk->wall_to_monotonic, tk->tkr.clock, tk->tkr.mult, tk->tkr.cycle_last); } From dc491596f6394382fbc74ad331156207d619fa0a Mon Sep 17 00:00:00 2001 From: John Stultz Date: Fri, 6 Dec 2013 17:25:21 -0800 Subject: [PATCH 111/114] timekeeping: Rework frequency adjustments to work better w/ nohz The existing timekeeping_adjust logic has always been complicated to understand. Further, since it was developed prior to NOHZ becoming common, its not surprising it performs poorly when NOHZ is enabled. Since Miroslav pointed out the problematic nature of the existing code in the NOHZ case, I've tried to refactor the code to perform better. The problem with the previous approach was that it tried to adjust for the total cumulative error using a scaled dampening factor. This resulted in large errors to be corrected slowly, while small errors were corrected quickly. With NOHZ the timekeeping code doesn't know how far out the next tick will be, so this results in bad over-correction to small errors, and insufficient correction to large errors. Inspired by Miroslav's patch, I've refactored the code to try to address the correction in two steps. 1) Check the future freq error for the next tick, and if the frequency error is large, try to make sure we correct it so it doesn't cause much accumulated error. 2) Then make a small single unit adjustment to correct any cumulative error that has collected over time. This method performs fairly well in the simulator Miroslav created. Major credit to Miroslav for pointing out the issue, providing the original patch to resolve this, a simulator for testing, as well as helping debug and resolve issues in my implementation so that it performed closer to his original implementation. Cc: Miroslav Lichvar Cc: Richard Cochran Cc: Prarit Bhargava Reported-by: Miroslav Lichvar Signed-off-by: John Stultz --- include/linux/timekeeper_internal.h | 1 + kernel/time/timekeeping.c | 193 ++++++++++++---------------- 2 files changed, 84 insertions(+), 110 deletions(-) diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h index 97381997625b..f7ac48d2edf5 100644 --- a/include/linux/timekeeper_internal.h +++ b/include/linux/timekeeper_internal.h @@ -92,6 +92,7 @@ struct timekeeper { u32 raw_interval; s64 ntp_error; u32 ntp_error_shift; + u32 ntp_err_mult; }; #ifdef CONFIG_GENERIC_TIME_VSYSCALL diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 2b56b959615b..43c706a7a728 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -178,6 +178,7 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) * to counteract clock drifting. */ tk->tkr.mult = clock->mult; + tk->ntp_err_mult = 0; } /* Timekeeper helper functions. */ @@ -1257,125 +1258,34 @@ static int __init timekeeping_init_ops(void) register_syscore_ops(&timekeeping_syscore_ops); return 0; } - device_initcall(timekeeping_init_ops); /* - * If the error is already larger, we look ahead even further - * to compensate for late or lost adjustments. + * Apply a multiplier adjustment to the timekeeper */ -static __always_inline int timekeeping_bigadjust(struct timekeeper *tk, - s64 error, s64 *interval, - s64 *offset) +static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk, + s64 offset, + bool negative, + int adj_scale) { - s64 tick_error, i; - u32 look_ahead, adj; - s32 error2, mult; + s64 interval = tk->cycle_interval; + s32 mult_adj = 1; - /* - * Use the current error value to determine how much to look ahead. - * The larger the error the slower we adjust for it to avoid problems - * with losing too many ticks, otherwise we would overadjust and - * produce an even larger error. The smaller the adjustment the - * faster we try to adjust for it, as lost ticks can do less harm - * here. This is tuned so that an error of about 1 msec is adjusted - * within about 1 sec (or 2^20 nsec in 2^SHIFT_HZ ticks). - */ - error2 = tk->ntp_error >> (NTP_SCALE_SHIFT + 22 - 2 * SHIFT_HZ); - error2 = abs(error2); - for (look_ahead = 0; error2 > 0; look_ahead++) - error2 >>= 2; - - /* - * Now calculate the error in (1 << look_ahead) ticks, but first - * remove the single look ahead already included in the error. - */ - tick_error = ntp_tick_length() >> (tk->ntp_error_shift + 1); - tick_error -= tk->xtime_interval >> 1; - error = ((error - tick_error) >> look_ahead) + tick_error; - - /* Finally calculate the adjustment shift value. */ - i = *interval; - mult = 1; - if (error < 0) { - error = -error; - *interval = -*interval; - *offset = -*offset; - mult = -1; + if (negative) { + mult_adj = -mult_adj; + interval = -interval; + offset = -offset; } - for (adj = 0; error > i; adj++) - error >>= 1; + mult_adj <<= adj_scale; + interval <<= adj_scale; + offset <<= adj_scale; - *interval <<= adj; - *offset <<= adj; - return mult << adj; -} - -/* - * Adjust the multiplier to reduce the error value, - * this is optimized for the most common adjustments of -1,0,1, - * for other values we can do a bit more work. - */ -static void timekeeping_adjust(struct timekeeper *tk, s64 offset) -{ - s64 error, interval = tk->cycle_interval; - int adj; - - /* - * The point of this is to check if the error is greater than half - * an interval. - * - * First we shift it down from NTP_SHIFT to clocksource->shifted nsecs. - * - * Note we subtract one in the shift, so that error is really error*2. - * This "saves" dividing(shifting) interval twice, but keeps the - * (error > interval) comparison as still measuring if error is - * larger than half an interval. - * - * Note: It does not "save" on aggravation when reading the code. - */ - error = tk->ntp_error >> (tk->ntp_error_shift - 1); - if (error > interval) { - /* - * We now divide error by 4(via shift), which checks if - * the error is greater than twice the interval. - * If it is greater, we need a bigadjust, if its smaller, - * we can adjust by 1. - */ - error >>= 2; - if (likely(error <= interval)) - adj = 1; - else - adj = timekeeping_bigadjust(tk, error, &interval, &offset); - } else { - if (error < -interval) { - /* See comment above, this is just switched for the negative */ - error >>= 2; - if (likely(error >= -interval)) { - adj = -1; - interval = -interval; - offset = -offset; - } else { - adj = timekeeping_bigadjust(tk, error, &interval, &offset); - } - } else { - goto out_adjust; - } - } - - if (unlikely(tk->tkr.clock->maxadj && - (tk->tkr.mult + adj > tk->tkr.clock->mult + tk->tkr.clock->maxadj))) { - printk_deferred_once(KERN_WARNING - "Adjusting %s more than 11%% (%ld vs %ld)\n", - tk->tkr.clock->name, (long)tk->tkr.mult + adj, - (long)tk->tkr.clock->mult + tk->tkr.clock->maxadj); - } /* * So the following can be confusing. * - * To keep things simple, lets assume adj == 1 for now. + * To keep things simple, lets assume mult_adj == 1 for now. * - * When adj != 1, remember that the interval and offset values + * When mult_adj != 1, remember that the interval and offset values * have been appropriately scaled so the math is the same. * * The basic idea here is that we're increasing the multiplier @@ -1419,12 +1329,76 @@ static void timekeeping_adjust(struct timekeeper *tk, s64 offset) * * XXX - TODO: Doc ntp_error calculation. */ - tk->tkr.mult += adj; + tk->tkr.mult += mult_adj; tk->xtime_interval += interval; tk->tkr.xtime_nsec -= offset; tk->ntp_error -= (interval - offset) << tk->ntp_error_shift; +} + +/* + * Calculate the multiplier adjustment needed to match the frequency + * specified by NTP + */ +static __always_inline void timekeeping_freqadjust(struct timekeeper *tk, + s64 offset) +{ + s64 interval = tk->cycle_interval; + s64 xinterval = tk->xtime_interval; + s64 tick_error; + bool negative; + u32 adj; + + /* Remove any current error adj from freq calculation */ + if (tk->ntp_err_mult) + xinterval -= tk->cycle_interval; + + /* Calculate current error per tick */ + tick_error = ntp_tick_length() >> tk->ntp_error_shift; + tick_error -= (xinterval + tk->xtime_remainder); + + /* Don't worry about correcting it if its small */ + if (likely((tick_error >= 0) && (tick_error <= interval))) + return; + + /* preserve the direction of correction */ + negative = (tick_error < 0); + + /* Sort out the magnitude of the correction */ + tick_error = abs(tick_error); + for (adj = 0; tick_error > interval; adj++) + tick_error >>= 1; + + /* scale the corrections */ + timekeeping_apply_adjustment(tk, offset, negative, adj); +} + +/* + * Adjust the timekeeper's multiplier to the correct frequency + * and also to reduce the accumulated error value. + */ +static void timekeeping_adjust(struct timekeeper *tk, s64 offset) +{ + /* Correct for the current frequency error */ + timekeeping_freqadjust(tk, offset); + + /* Next make a small adjustment to fix any cumulative error */ + if (!tk->ntp_err_mult && (tk->ntp_error > 0)) { + tk->ntp_err_mult = 1; + timekeeping_apply_adjustment(tk, offset, 0, 0); + } else if (tk->ntp_err_mult && (tk->ntp_error <= 0)) { + /* Undo any existing error adjustment */ + timekeeping_apply_adjustment(tk, offset, 1, 0); + tk->ntp_err_mult = 0; + } + + if (unlikely(tk->tkr.clock->maxadj && + (tk->tkr.mult > tk->tkr.clock->mult + tk->tkr.clock->maxadj))) { + printk_once(KERN_WARNING + "Adjusting %s more than 11%% (%ld vs %ld)\n", + tk->tkr.clock->name, (long)tk->tkr.mult, + (long)tk->tkr.clock->mult + tk->tkr.clock->maxadj); + } -out_adjust: /* * It may be possible that when we entered this function, xtime_nsec * was very small. Further, if we're slightly speeding the clocksource @@ -1444,7 +1418,6 @@ out_adjust: tk->tkr.xtime_nsec = 0; tk->ntp_error += neg << tk->ntp_error_shift; } - } /** From 375f45b5b53a91dfa8f0c11328e0e044f82acbed Mon Sep 17 00:00:00 2001 From: John Stultz Date: Wed, 23 Apr 2014 20:53:29 -0700 Subject: [PATCH 112/114] timekeeping: Use cached ntp_tick_length when accumulating error By caching the ntp_tick_length() when we correct the frequency error, and then using that cached value to accumulate error, we avoid large initial errors when the tick length is changed. This makes convergence happen much faster in the simulator, since the initial error doesn't have to be slowly whittled away. This initially seems like an accounting error, but Miroslav pointed out that ntp_tick_length() can change mid-tick, so when we apply it in the error accumulation, we are applying any recent change to the entire tick. This approach chooses to apply changes in the ntp_tick_length() only to the next tick, which allows us to calculate the freq correction before using the new tick length, which avoids accummulating error. Credit to Miroslav for pointing this out and providing the original patch this functionality has been pulled out from, along with the rational. Cc: Miroslav Lichvar Cc: Richard Cochran Cc: Prarit Bhargava Reported-by: Miroslav Lichvar Signed-off-by: John Stultz --- include/linux/timekeeper_internal.h | 9 +++++++++ kernel/time/timekeeping.c | 5 ++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h index f7ac48d2edf5..e9660e52dc09 100644 --- a/include/linux/timekeeper_internal.h +++ b/include/linux/timekeeper_internal.h @@ -90,6 +90,15 @@ struct timekeeper { u64 xtime_interval; s64 xtime_remainder; u32 raw_interval; + /* The ntp_tick_length() value currently being used. + * This cached copy ensures we consistently apply the tick + * length for an entire tick, as ntp_tick_length may change + * mid-tick, and we don't want to apply that new value to + * the tick in progress. + */ + u64 ntp_tick; + /* Difference between accumulated time and NTP time in ntp + * shifted nano seconds. */ s64 ntp_error; u32 ntp_error_shift; u32 ntp_err_mult; diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 43c706a7a728..f36b02838a47 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -171,6 +171,7 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) tk->ntp_error = 0; tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift; + tk->ntp_tick = ntpinterval << tk->ntp_error_shift; /* * The timekeeper keeps its own mult values for the currently @@ -1352,6 +1353,8 @@ static __always_inline void timekeeping_freqadjust(struct timekeeper *tk, if (tk->ntp_err_mult) xinterval -= tk->cycle_interval; + tk->ntp_tick = ntp_tick_length(); + /* Calculate current error per tick */ tick_error = ntp_tick_length() >> tk->ntp_error_shift; tick_error -= (xinterval + tk->xtime_remainder); @@ -1497,7 +1500,7 @@ static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset, tk->raw_time.tv_nsec = raw_nsecs; /* Accumulate error between NTP and clock interval */ - tk->ntp_error += ntp_tick_length() << shift; + tk->ntp_error += tk->ntp_tick << shift; tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) << (tk->ntp_error_shift + shift); From 7806f60e1d205db46eca6ad24429b3f86eda2588 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Thu, 10 Jul 2014 09:52:27 +0200 Subject: [PATCH 113/114] clocksource: document some basic timekeeping concepts This adds some documentation about clock sources, clock events, the weak sched_clock() function and delay timers that answers questions that repeatedly arise on the mailing lists. Cc: Thomas Gleixner Cc: Nicolas Pitre Cc: Colin Cross Cc: John Stultz Cc: Peter Zijlstra Cc: Ingo Molnar Signed-off-by: Linus Walleij Acked-by: Nicolas Pitre Signed-off-by: John Stultz --- Documentation/timers/00-INDEX | 2 + Documentation/timers/timekeeping.txt | 179 +++++++++++++++++++++++++++ 2 files changed, 181 insertions(+) create mode 100644 Documentation/timers/timekeeping.txt diff --git a/Documentation/timers/00-INDEX b/Documentation/timers/00-INDEX index 6d042dc1cce0..ee212a27772f 100644 --- a/Documentation/timers/00-INDEX +++ b/Documentation/timers/00-INDEX @@ -12,6 +12,8 @@ Makefile - Build and link hpet_example NO_HZ.txt - Summary of the different methods for the scheduler clock-interrupts management. +timekeeping.txt + - Clock sources, clock events, sched_clock() and delay timer notes timers-howto.txt - how to insert delays in the kernel the right (tm) way. timer_stats.txt diff --git a/Documentation/timers/timekeeping.txt b/Documentation/timers/timekeeping.txt new file mode 100644 index 000000000000..f3a8cf28f802 --- /dev/null +++ b/Documentation/timers/timekeeping.txt @@ -0,0 +1,179 @@ +Clock sources, Clock events, sched_clock() and delay timers +----------------------------------------------------------- + +This document tries to briefly explain some basic kernel timekeeping +abstractions. It partly pertains to the drivers usually found in +drivers/clocksource in the kernel tree, but the code may be spread out +across the kernel. + +If you grep through the kernel source you will find a number of architecture- +specific implementations of clock sources, clockevents and several likewise +architecture-specific overrides of the sched_clock() function and some +delay timers. + +To provide timekeeping for your platform, the clock source provides +the basic timeline, whereas clock events shoot interrupts on certain points +on this timeline, providing facilities such as high-resolution timers. +sched_clock() is used for scheduling and timestamping, and delay timers +provide an accurate delay source using hardware counters. + + +Clock sources +------------- + +The purpose of the clock source is to provide a timeline for the system that +tells you where you are in time. For example issuing the command 'date' on +a Linux system will eventually read the clock source to determine exactly +what time it is. + +Typically the clock source is a monotonic, atomic counter which will provide +n bits which count from 0 to 2^(n-1) and then wraps around to 0 and start over. +It will ideally NEVER stop ticking as long as the system is running. It +may stop during system suspend. + +The clock source shall have as high resolution as possible, and the frequency +shall be as stable and correct as possible as compared to a real-world wall +clock. It should not move unpredictably back and forth in time or miss a few +cycles here and there. + +It must be immune to the kind of effects that occur in hardware where e.g. +the counter register is read in two phases on the bus lowest 16 bits first +and the higher 16 bits in a second bus cycle with the counter bits +potentially being updated in between leading to the risk of very strange +values from the counter. + +When the wall-clock accuracy of the clock source isn't satisfactory, there +are various quirks and layers in the timekeeping code for e.g. synchronizing +the user-visible time to RTC clocks in the system or against networked time +servers using NTP, but all they do basically is update an offset against +the clock source, which provides the fundamental timeline for the system. +These measures does not affect the clock source per se, they only adapt the +system to the shortcomings of it. + +The clock source struct shall provide means to translate the provided counter +into a nanosecond value as an unsigned long long (unsigned 64 bit) number. +Since this operation may be invoked very often, doing this in a strict +mathematical sense is not desirable: instead the number is taken as close as +possible to a nanosecond value using only the arithmetic operations +multiply and shift, so in clocksource_cyc2ns() you find: + + ns ~= (clocksource * mult) >> shift + +You will find a number of helper functions in the clock source code intended +to aid in providing these mult and shift values, such as +clocksource_khz2mult(), clocksource_hz2mult() that help determine the +mult factor from a fixed shift, and clocksource_register_hz() and +clocksource_register_khz() which will help out assigning both shift and mult +factors using the frequency of the clock source as the only input. + +For real simple clock sources accessed from a single I/O memory location +there is nowadays even clocksource_mmio_init() which will take a memory +location, bit width, a parameter telling whether the counter in the +register counts up or down, and the timer clock rate, and then conjure all +necessary parameters. + +Since a 32-bit counter at say 100 MHz will wrap around to zero after some 43 +seconds, the code handling the clock source will have to compensate for this. +That is the reason why the clock source struct also contains a 'mask' +member telling how many bits of the source are valid. This way the timekeeping +code knows when the counter will wrap around and can insert the necessary +compensation code on both sides of the wrap point so that the system timeline +remains monotonic. + + +Clock events +------------ + +Clock events are the conceptual reverse of clock sources: they take a +desired time specification value and calculate the values to poke into +hardware timer registers. + +Clock events are orthogonal to clock sources. The same hardware +and register range may be used for the clock event, but it is essentially +a different thing. The hardware driving clock events has to be able to +fire interrupts, so as to trigger events on the system timeline. On an SMP +system, it is ideal (and customary) to have one such event driving timer per +CPU core, so that each core can trigger events independently of any other +core. + +You will notice that the clock event device code is based on the same basic +idea about translating counters to nanoseconds using mult and shift +arithmetic, and you find the same family of helper functions again for +assigning these values. The clock event driver does not need a 'mask' +attribute however: the system will not try to plan events beyond the time +horizon of the clock event. + + +sched_clock() +------------- + +In addition to the clock sources and clock events there is a special weak +function in the kernel called sched_clock(). This function shall return the +number of nanoseconds since the system was started. An architecture may or +may not provide an implementation of sched_clock() on its own. If a local +implementation is not provided, the system jiffy counter will be used as +sched_clock(). + +As the name suggests, sched_clock() is used for scheduling the system, +determining the absolute timeslice for a certain process in the CFS scheduler +for example. It is also used for printk timestamps when you have selected to +include time information in printk for things like bootcharts. + +Compared to clock sources, sched_clock() has to be very fast: it is called +much more often, especially by the scheduler. If you have to do trade-offs +between accuracy compared to the clock source, you may sacrifice accuracy +for speed in sched_clock(). It however requires some of the same basic +characteristics as the clock source, i.e. it should be monotonic. + +The sched_clock() function may wrap only on unsigned long long boundaries, +i.e. after 64 bits. Since this is a nanosecond value this will mean it wraps +after circa 585 years. (For most practical systems this means "never".) + +If an architecture does not provide its own implementation of this function, +it will fall back to using jiffies, making its maximum resolution 1/HZ of the +jiffy frequency for the architecture. This will affect scheduling accuracy +and will likely show up in system benchmarks. + +The clock driving sched_clock() may stop or reset to zero during system +suspend/sleep. This does not matter to the function it serves of scheduling +events on the system. However it may result in interesting timestamps in +printk(). + +The sched_clock() function should be callable in any context, IRQ- and +NMI-safe and return a sane value in any context. + +Some architectures may have a limited set of time sources and lack a nice +counter to derive a 64-bit nanosecond value, so for example on the ARM +architecture, special helper functions have been created to provide a +sched_clock() nanosecond base from a 16- or 32-bit counter. Sometimes the +same counter that is also used as clock source is used for this purpose. + +On SMP systems, it is crucial for performance that sched_clock() can be called +independently on each CPU without any synchronization performance hits. +Some hardware (such as the x86 TSC) will cause the sched_clock() function to +drift between the CPUs on the system. The kernel can work around this by +enabling the CONFIG_HAVE_UNSTABLE_SCHED_CLOCK option. This is another aspect +that makes sched_clock() different from the ordinary clock source. + + +Delay timers (some architectures only) +-------------------------------------- + +On systems with variable CPU frequency, the various kernel delay() functions +will sometimes behave strangely. Basically these delays usually use a hard +loop to delay a certain number of jiffy fractions using a "lpj" (loops per +jiffy) value, calibrated on boot. + +Let's hope that your system is running on maximum frequency when this value +is calibrated: as an effect when the frequency is geared down to half the +full frequency, any delay() will be twice as long. Usually this does not +hurt, as you're commonly requesting that amount of delay *or more*. But +basically the semantics are quite unpredictable on such systems. + +Enter timer-based delays. Using these, a timer read may be used instead of +a hard-coded loop for providing the desired delay. + +This is done by declaring a struct delay_timer and assigning the appropriate +function pointers and rate settings for this delay timer. + +This is available on some architectures like OpenRISC or ARM. From 953dec21aed4038464fec02f96a2f1b8701a5bce Mon Sep 17 00:00:00 2001 From: John Stultz Date: Fri, 25 Jul 2014 21:37:19 -0700 Subject: [PATCH 114/114] timekeeping: Fixup typo in update_vsyscall_old definition In commit 4a0e637738f0 ("clocksource: Get rid of cycle_last"), currently in the -tip tree, there was a small typo where cycles_t was used intstead of cycle_t. This broke ppc64 builds. Fix this by using the proper cycle_t type for this usage, in both the definition and the ia64 implementation. Now, having both cycle_t and cycles_t types seems like a very bad idea just asking for these sorts of issues. But that will be a cleanup for another day. Reported-by: Stephen Rothwell Signed-off-by: John Stultz Cc: Ingo Molnar Cc: "H. Peter Anvin" Cc: Peter Zijlstra Link: http://lkml.kernel.org/r/1406349439-11785-1-git-send-email-john.stultz@linaro.org Signed-off-by: Thomas Gleixner --- arch/ia64/kernel/time.c | 2 +- include/linux/timekeeper_internal.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/ia64/kernel/time.c b/arch/ia64/kernel/time.c index 11dc42da7daf..3e71ef85e439 100644 --- a/arch/ia64/kernel/time.c +++ b/arch/ia64/kernel/time.c @@ -441,7 +441,7 @@ void update_vsyscall_tz(void) } void update_vsyscall_old(struct timespec *wall, struct timespec *wtm, - struct clocksource *c, u32 mult, cycles_t cycle_last) + struct clocksource *c, u32 mult, cycle_t cycle_last) { write_seqcount_begin(&fsyscall_gtod_data.seq); diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h index e9660e52dc09..95640dcd1899 100644 --- a/include/linux/timekeeper_internal.h +++ b/include/linux/timekeeper_internal.h @@ -113,7 +113,7 @@ extern void update_vsyscall_tz(void); extern void update_vsyscall_old(struct timespec *ts, struct timespec *wtm, struct clocksource *c, u32 mult, - cycles_t cycle_last); + cycle_t cycle_last); extern void update_vsyscall_tz(void); #else