iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
linux/drivers/memory/tegra/tegra210-emc-cc-r21021.c

1775 lines
51 KiB
C
Raw Normal View History

memory: tegra: Add EMC scaling sequence code for Tegra210 This patch includes the sequence for clock tuning and the dynamic training mechanism for the clock above 800MHz. And historically there have been different sequences to change the EMC clock. The sequence to be used is specified in the EMC table. However, for the currently supported upstreaming platform, only the most recent sequence is used. So only support that in this patch. Based on the work of Peter De Schrijver <pdeschrijver@nvidia.com>. Signed-off-by: Joseph Lo <josephl@nvidia.com> Signed-off-by: Thierry Reding <treding@nvidia.com>
2019-05-29 16:21:37 +08:00
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (c) 2014-2020, NVIDIA CORPORATION. All rights reserved.
*/
#include <linux/kernel.h>
#include <linux/io.h>
#include <linux/clk.h>
#include <linux/delay.h>
#include <linux/of.h>
#include <soc/tegra/mc.h>
#include "tegra210-emc.h"
#include "tegra210-mc.h"
/*
* Enable flags for specifying verbosity.
*/
#define INFO (1 << 0)
#define STEPS (1 << 1)
#define SUB_STEPS (1 << 2)
#define PRELOCK (1 << 3)
#define PRELOCK_STEPS (1 << 4)
#define ACTIVE_EN (1 << 5)
#define PRAMP_UP (1 << 6)
#define PRAMP_DN (1 << 7)
#define EMA_WRITES (1 << 10)
#define EMA_UPDATES (1 << 11)
#define PER_TRAIN (1 << 16)
#define CC_PRINT (1 << 17)
#define CCFIFO (1 << 29)
#define REGS (1 << 30)
#define REG_LISTS (1 << 31)
#define emc_dbg(emc, flags, ...) dev_dbg(emc->dev, __VA_ARGS__)
#define DVFS_CLOCK_CHANGE_VERSION 21021
#define EMC_PRELOCK_VERSION 2101
enum {
DVFS_SEQUENCE = 1,
WRITE_TRAINING_SEQUENCE = 2,
PERIODIC_TRAINING_SEQUENCE = 3,
DVFS_PT1 = 10,
DVFS_UPDATE = 11,
TRAINING_PT1 = 12,
TRAINING_UPDATE = 13,
PERIODIC_TRAINING_UPDATE = 14
};
/*
* PTFV defines - basically just indexes into the per table PTFV array.
*/
#define PTFV_DQSOSC_MOVAVG_C0D0U0_INDEX 0
#define PTFV_DQSOSC_MOVAVG_C0D0U1_INDEX 1
#define PTFV_DQSOSC_MOVAVG_C0D1U0_INDEX 2
#define PTFV_DQSOSC_MOVAVG_C0D1U1_INDEX 3
#define PTFV_DQSOSC_MOVAVG_C1D0U0_INDEX 4
#define PTFV_DQSOSC_MOVAVG_C1D0U1_INDEX 5
#define PTFV_DQSOSC_MOVAVG_C1D1U0_INDEX 6
#define PTFV_DQSOSC_MOVAVG_C1D1U1_INDEX 7
#define PTFV_DVFS_SAMPLES_INDEX 9
#define PTFV_MOVAVG_WEIGHT_INDEX 10
#define PTFV_CONFIG_CTRL_INDEX 11
#define PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA (1 << 0)
/*
* Do arithmetic in fixed point.
*/
#define MOVAVG_PRECISION_FACTOR 100
/*
* The division portion of the average operation.
*/
#define __AVERAGE_PTFV(dev) \
({ next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] = \
next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \
next->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; })
/*
* Convert val to fixed point and add it to the temporary average.
*/
#define __INCREMENT_PTFV(dev, val) \
({ next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] += \
((val) * MOVAVG_PRECISION_FACTOR); })
/*
* Convert a moving average back to integral form and return the value.
*/
#define __MOVAVG_AC(timing, dev) \
((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \
MOVAVG_PRECISION_FACTOR)
/* Weighted update. */
#define __WEIGHTED_UPDATE_PTFV(dev, nval) \
do { \
int w = PTFV_MOVAVG_WEIGHT_INDEX; \
int dqs = PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX; \
\
next->ptfv_list[dqs] = \
((nval * MOVAVG_PRECISION_FACTOR) + \
(next->ptfv_list[dqs] * \
next->ptfv_list[w])) / \
(next->ptfv_list[w] + 1); \
\
emc_dbg(emc, EMA_UPDATES, "%s: (s=%lu) EMA: %u\n", \
__stringify(dev), nval, next->ptfv_list[dqs]); \
} while (0)
/* Access a particular average. */
#define __MOVAVG(timing, dev) \
((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX])
static u32 update_clock_tree_delay(struct tegra210_emc *emc, int type)
{
bool periodic_training_update = type == PERIODIC_TRAINING_UPDATE;
struct tegra210_emc_timing *last = emc->last;
struct tegra210_emc_timing *next = emc->next;
u32 last_timing_rate_mhz = last->rate / 1000;
u32 next_timing_rate_mhz = next->rate / 1000;
bool dvfs_update = type == DVFS_UPDATE;
s32 tdel = 0, tmdel = 0, adel = 0;
bool dvfs_pt1 = type == DVFS_PT1;
unsigned long cval = 0;
u32 temp[2][2], value;
unsigned int i;
/*
* Dev0 MSB.
*/
if (dvfs_pt1 || periodic_training_update) {
value = tegra210_emc_mrr_read(emc, 2, 19);
for (i = 0; i < emc->num_channels; i++) {
temp[i][0] = (value & 0x00ff) << 8;
temp[i][1] = (value & 0xff00) << 0;
value >>= 16;
}
/*
* Dev0 LSB.
*/
value = tegra210_emc_mrr_read(emc, 2, 18);
for (i = 0; i < emc->num_channels; i++) {
temp[i][0] |= (value & 0x00ff) >> 0;
temp[i][1] |= (value & 0xff00) >> 8;
value >>= 16;
}
}
if (dvfs_pt1 || periodic_training_update) {
cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
cval *= 1000000;
cval /= last_timing_rate_mhz * 2 * temp[0][0];
}
if (dvfs_pt1)
__INCREMENT_PTFV(C0D0U0, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C0D0U0);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C0D0U0, cval);
if (dvfs_update || periodic_training_update) {
tdel = next->current_dram_clktree[C0D0U0] -
__MOVAVG_AC(next, C0D0U0);
tmdel = (tdel < 0) ? -1 * tdel : tdel;
adel = tmdel;
if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
next->tree_margin)
next->current_dram_clktree[C0D0U0] =
__MOVAVG_AC(next, C0D0U0);
}
if (dvfs_pt1 || periodic_training_update) {
cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
cval *= 1000000;
cval /= last_timing_rate_mhz * 2 * temp[0][1];
}
if (dvfs_pt1)
__INCREMENT_PTFV(C0D0U1, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C0D0U1);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C0D0U1, cval);
if (dvfs_update || periodic_training_update) {
tdel = next->current_dram_clktree[C0D0U1] -
__MOVAVG_AC(next, C0D0U1);
tmdel = (tdel < 0) ? -1 * tdel : tdel;
if (tmdel > adel)
adel = tmdel;
if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
next->tree_margin)
next->current_dram_clktree[C0D0U1] =
__MOVAVG_AC(next, C0D0U1);
}
if (emc->num_channels > 1) {
if (dvfs_pt1 || periodic_training_update) {
cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
cval *= 1000000;
cval /= last_timing_rate_mhz * 2 * temp[1][0];
}
if (dvfs_pt1)
__INCREMENT_PTFV(C1D0U0, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C1D0U0);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C1D0U0, cval);
if (dvfs_update || periodic_training_update) {
tdel = next->current_dram_clktree[C1D0U0] -
__MOVAVG_AC(next, C1D0U0);
tmdel = (tdel < 0) ? -1 * tdel : tdel;
if (tmdel > adel)
adel = tmdel;
if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
next->tree_margin)
next->current_dram_clktree[C1D0U0] =
__MOVAVG_AC(next, C1D0U0);
}
if (dvfs_pt1 || periodic_training_update) {
cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
cval *= 1000000;
cval /= last_timing_rate_mhz * 2 * temp[1][1];
}
if (dvfs_pt1)
__INCREMENT_PTFV(C1D0U1, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C1D0U1);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C1D0U1, cval);
if (dvfs_update || periodic_training_update) {
tdel = next->current_dram_clktree[C1D0U1] -
__MOVAVG_AC(next, C1D0U1);
tmdel = (tdel < 0) ? -1 * tdel : tdel;
if (tmdel > adel)
adel = tmdel;
if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
next->tree_margin)
next->current_dram_clktree[C1D0U1] =
__MOVAVG_AC(next, C1D0U1);
}
}
if (emc->num_devices < 2)
goto done;
/*
* Dev1 MSB.
*/
if (dvfs_pt1 || periodic_training_update) {
value = tegra210_emc_mrr_read(emc, 1, 19);
for (i = 0; i < emc->num_channels; i++) {
temp[i][0] = (value & 0x00ff) << 8;
temp[i][1] = (value & 0xff00) << 0;
value >>= 16;
}
/*
* Dev1 LSB.
*/
value = tegra210_emc_mrr_read(emc, 2, 18);
for (i = 0; i < emc->num_channels; i++) {
temp[i][0] |= (value & 0x00ff) >> 0;
temp[i][1] |= (value & 0xff00) >> 8;
value >>= 16;
}
}
if (dvfs_pt1 || periodic_training_update) {
cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
cval *= 1000000;
cval /= last_timing_rate_mhz * 2 * temp[0][0];
}
if (dvfs_pt1)
__INCREMENT_PTFV(C0D1U0, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C0D1U0);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C0D1U0, cval);
if (dvfs_update || periodic_training_update) {
tdel = next->current_dram_clktree[C0D1U0] -
__MOVAVG_AC(next, C0D1U0);
tmdel = (tdel < 0) ? -1 * tdel : tdel;
if (tmdel > adel)
adel = tmdel;
if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
next->tree_margin)
next->current_dram_clktree[C0D1U0] =
__MOVAVG_AC(next, C0D1U0);
}
if (dvfs_pt1 || periodic_training_update) {
cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
cval *= 1000000;
cval /= last_timing_rate_mhz * 2 * temp[0][1];
}
if (dvfs_pt1)
__INCREMENT_PTFV(C0D1U1, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C0D1U1);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C0D1U1, cval);
if (dvfs_update || periodic_training_update) {
tdel = next->current_dram_clktree[C0D1U1] -
__MOVAVG_AC(next, C0D1U1);
tmdel = (tdel < 0) ? -1 * tdel : tdel;
if (tmdel > adel)
adel = tmdel;
if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
next->tree_margin)
next->current_dram_clktree[C0D1U1] =
__MOVAVG_AC(next, C0D1U1);
}
if (emc->num_channels > 1) {
if (dvfs_pt1 || periodic_training_update) {
cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
cval *= 1000000;
cval /= last_timing_rate_mhz * 2 * temp[1][0];
}
if (dvfs_pt1)
__INCREMENT_PTFV(C1D1U0, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C1D1U0);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C1D1U0, cval);
if (dvfs_update || periodic_training_update) {
tdel = next->current_dram_clktree[C1D1U0] -
__MOVAVG_AC(next, C1D1U0);
tmdel = (tdel < 0) ? -1 * tdel : tdel;
if (tmdel > adel)
adel = tmdel;
if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
next->tree_margin)
next->current_dram_clktree[C1D1U0] =
__MOVAVG_AC(next, C1D1U0);
}
if (dvfs_pt1 || periodic_training_update) {
cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
cval *= 1000000;
cval /= last_timing_rate_mhz * 2 * temp[1][1];
}
if (dvfs_pt1)
__INCREMENT_PTFV(C1D1U1, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C1D1U1);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C1D1U1, cval);
if (dvfs_update || periodic_training_update) {
tdel = next->current_dram_clktree[C1D1U1] -
__MOVAVG_AC(next, C1D1U1);
tmdel = (tdel < 0) ? -1 * tdel : tdel;
if (tmdel > adel)
adel = tmdel;
if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
next->tree_margin)
next->current_dram_clktree[C1D1U1] =
__MOVAVG_AC(next, C1D1U1);
}
}
done:
return adel;
}
static u32 periodic_compensation_handler(struct tegra210_emc *emc, u32 type,
struct tegra210_emc_timing *last,
struct tegra210_emc_timing *next)
{
#define __COPY_EMA(nt, lt, dev) \
({ __MOVAVG(nt, dev) = __MOVAVG(lt, dev) * \
(nt)->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; })
u32 i, adel = 0, samples = next->ptfv_list[PTFV_DVFS_SAMPLES_INDEX];
u32 delay;
delay = tegra210_emc_actual_osc_clocks(last->run_clocks);
delay *= 1000;
delay = 2 + (delay / last->rate);
if (!next->periodic_training)
return 0;
if (type == DVFS_SEQUENCE) {
if (last->periodic_training &&
(next->ptfv_list[PTFV_CONFIG_CTRL_INDEX] &
PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA)) {
/*
* If the previous frequency was using periodic
* calibration then we can reuse the previous
* frequencies EMA data.
*/
__COPY_EMA(next, last, C0D0U0);
__COPY_EMA(next, last, C0D0U1);
__COPY_EMA(next, last, C1D0U0);
__COPY_EMA(next, last, C1D0U1);
__COPY_EMA(next, last, C0D1U0);
__COPY_EMA(next, last, C0D1U1);
__COPY_EMA(next, last, C1D1U0);
__COPY_EMA(next, last, C1D1U1);
} else {
/* Reset the EMA.*/
__MOVAVG(next, C0D0U0) = 0;
__MOVAVG(next, C0D0U1) = 0;
__MOVAVG(next, C1D0U0) = 0;
__MOVAVG(next, C1D0U1) = 0;
__MOVAVG(next, C0D1U0) = 0;
__MOVAVG(next, C0D1U1) = 0;
__MOVAVG(next, C1D1U0) = 0;
__MOVAVG(next, C1D1U1) = 0;
for (i = 0; i < samples; i++) {
tegra210_emc_start_periodic_compensation(emc);
udelay(delay);
/*
* Generate next sample of data.
*/
adel = update_clock_tree_delay(emc, DVFS_PT1);
}
}
/*
* Seems like it should be part of the
* 'if (last_timing->periodic_training)' conditional
* since is already done for the else clause.
*/
adel = update_clock_tree_delay(emc, DVFS_UPDATE);
}
if (type == PERIODIC_TRAINING_SEQUENCE) {
tegra210_emc_start_periodic_compensation(emc);
udelay(delay);
adel = update_clock_tree_delay(emc, PERIODIC_TRAINING_UPDATE);
}
return adel;
}
static u32 tegra210_emc_r21021_periodic_compensation(struct tegra210_emc *emc)
{
u32 emc_cfg, emc_cfg_o, emc_cfg_update, del, value;
u32 list[] = {
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0,
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1,
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2,
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3,
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0,
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1,
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2,
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3,
EMC_DATA_BRLSHFT_0,
EMC_DATA_BRLSHFT_1
};
struct tegra210_emc_timing *last = emc->last;
unsigned int items = ARRAY_SIZE(list), i;
unsigned long delay;
if (last->periodic_training) {
emc_dbg(emc, PER_TRAIN, "Periodic training starting\n");
value = emc_readl(emc, EMC_DBG);
emc_cfg_o = emc_readl(emc, EMC_CFG);
emc_cfg = emc_cfg_o & ~(EMC_CFG_DYN_SELF_REF |
EMC_CFG_DRAM_ACPD |
EMC_CFG_DRAM_CLKSTOP_PD);
/*
* 1. Power optimizations should be off.
*/
emc_writel(emc, emc_cfg, EMC_CFG);
/* Does emc_timing_update() for above changes. */
tegra210_emc_dll_disable(emc);
for (i = 0; i < emc->num_channels; i++)
tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK,
0);
for (i = 0; i < emc->num_channels; i++)
tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK,
0);
emc_cfg_update = value = emc_readl(emc, EMC_CFG_UPDATE);
value &= ~EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_MASK;
value |= (2 << EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_SHIFT);
emc_writel(emc, value, EMC_CFG_UPDATE);
/*
* 2. osc kick off - this assumes training and dvfs have set
* correct MR23.
*/
tegra210_emc_start_periodic_compensation(emc);
/*
* 3. Let dram capture its clock tree delays.
*/
delay = tegra210_emc_actual_osc_clocks(last->run_clocks);
delay *= 1000;
delay /= last->rate + 1;
udelay(delay);
/*
* 4. Check delta wrt previous values (save value if margin
* exceeds what is set in table).
*/
del = periodic_compensation_handler(emc,
PERIODIC_TRAINING_SEQUENCE,
last, last);
/*
* 5. Apply compensation w.r.t. trained values (if clock tree
* has drifted more than the set margin).
*/
if (last->tree_margin < ((del * 128 * (last->rate / 1000)) / 1000000)) {
for (i = 0; i < items; i++) {
value = tegra210_emc_compensate(last, list[i]);
emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n",
list[i], value);
emc_writel(emc, value, list[i]);
}
}
emc_writel(emc, emc_cfg_o, EMC_CFG);
/*
* 6. Timing update actally applies the new trimmers.
*/
tegra210_emc_timing_update(emc);
/* 6.1. Restore the UPDATE_DLL_IN_UPDATE field. */
emc_writel(emc, emc_cfg_update, EMC_CFG_UPDATE);
/* 6.2. Restore the DLL. */
tegra210_emc_dll_enable(emc);
}
return 0;
}
/*
* Do the clock change sequence.
*/
static void tegra210_emc_r21021_set_clock(struct tegra210_emc *emc, u32 clksrc)
{
/* state variables */
static bool fsp_for_next_freq;
/* constant configuration parameters */
const bool save_restore_clkstop_pd = true;
const u32 zqcal_before_cc_cutoff = 2400;
const bool cya_allow_ref_cc = false;
const bool cya_issue_pc_ref = false;
const bool opt_cc_short_zcal = true;
const bool ref_b4_sref_en = false;
const u32 tZQCAL_lpddr4 = 1000000;
const bool opt_short_zcal = true;
const bool opt_do_sw_qrst = true;
const u32 opt_dvfs_mode = MAN_SR;
/*
* This is the timing table for the source frequency. It does _not_
* necessarily correspond to the actual timing values in the EMC at the
* moment. If the boot BCT differs from the table then this can happen.
* However, we need it for accessing the dram_timings (which are not
* really registers) array for the current frequency.
*/
struct tegra210_emc_timing *fake, *last = emc->last, *next = emc->next;
u32 tRTM, RP_war, R2P_war, TRPab_war, deltaTWATM, W2P_war, tRPST;
u32 mr13_flip_fspwr, mr13_flip_fspop, ramp_up_wait, ramp_down_wait;
u32 zq_wait_long, zq_latch_dvfs_wait_time, tZQCAL_lpddr4_fc_adj;
u32 emc_auto_cal_config, auto_cal_en, emc_cfg, emc_sel_dpd_ctrl;
u32 tFC_lpddr4 = 1000 * next->dram_timings[T_FC_LPDDR4];
u32 bg_reg_mode_change, enable_bglp_reg, enable_bg_reg;
bool opt_zcal_en_cc = false, is_lpddr3 = false;
bool compensate_trimmer_applicable = false;
u32 emc_dbg, emc_cfg_pipe_clk, emc_pin;
u32 src_clk_period, dst_clk_period; /* in picoseconds */
bool shared_zq_resistor = false;
u32 value, dram_type;
u32 opt_dll_mode = 0;
unsigned long delay;
unsigned int i;
emc_dbg(emc, INFO, "Running clock change.\n");
/* XXX fake == last */
fake = tegra210_emc_find_timing(emc, last->rate * 1000UL);
fsp_for_next_freq = !fsp_for_next_freq;
value = emc_readl(emc, EMC_FBIO_CFG5) & EMC_FBIO_CFG5_DRAM_TYPE_MASK;
dram_type = value >> EMC_FBIO_CFG5_DRAM_TYPE_SHIFT;
if (last->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX] & BIT(31))
shared_zq_resistor = true;
if ((next->burst_regs[EMC_ZCAL_INTERVAL_INDEX] != 0 &&
last->burst_regs[EMC_ZCAL_INTERVAL_INDEX] == 0) ||
dram_type == DRAM_TYPE_LPDDR4)
opt_zcal_en_cc = true;
if (dram_type == DRAM_TYPE_DDR3)
opt_dll_mode = tegra210_emc_get_dll_state(next);
if ((next->burst_regs[EMC_FBIO_CFG5_INDEX] & BIT(25)) &&
(dram_type == DRAM_TYPE_LPDDR2))
is_lpddr3 = true;
emc_readl(emc, EMC_CFG);
emc_readl(emc, EMC_AUTO_CAL_CONFIG);
src_clk_period = 1000000000 / last->rate;
dst_clk_period = 1000000000 / next->rate;
if (dst_clk_period <= zqcal_before_cc_cutoff)
tZQCAL_lpddr4_fc_adj = tZQCAL_lpddr4 - tFC_lpddr4;
else
tZQCAL_lpddr4_fc_adj = tZQCAL_lpddr4;
tZQCAL_lpddr4_fc_adj /= dst_clk_period;
emc_dbg = emc_readl(emc, EMC_DBG);
emc_pin = emc_readl(emc, EMC_PIN);
emc_cfg_pipe_clk = emc_readl(emc, EMC_CFG_PIPE_CLK);
emc_cfg = next->burst_regs[EMC_CFG_INDEX];
emc_cfg &= ~(EMC_CFG_DYN_SELF_REF | EMC_CFG_DRAM_ACPD |
EMC_CFG_DRAM_CLKSTOP_SR | EMC_CFG_DRAM_CLKSTOP_PD);
emc_sel_dpd_ctrl = next->emc_sel_dpd_ctrl;
emc_sel_dpd_ctrl &= ~(EMC_SEL_DPD_CTRL_CLK_SEL_DPD_EN |
EMC_SEL_DPD_CTRL_CA_SEL_DPD_EN |
EMC_SEL_DPD_CTRL_RESET_SEL_DPD_EN |
EMC_SEL_DPD_CTRL_ODT_SEL_DPD_EN |
EMC_SEL_DPD_CTRL_DATA_SEL_DPD_EN);
emc_dbg(emc, INFO, "Clock change version: %d\n",
DVFS_CLOCK_CHANGE_VERSION);
emc_dbg(emc, INFO, "DRAM type = %d\n", dram_type);
emc_dbg(emc, INFO, "DRAM dev #: %u\n", emc->num_devices);
emc_dbg(emc, INFO, "Next EMC clksrc: 0x%08x\n", clksrc);
emc_dbg(emc, INFO, "DLL clksrc: 0x%08x\n", next->dll_clk_src);
emc_dbg(emc, INFO, "last rate: %u, next rate %u\n", last->rate,
next->rate);
emc_dbg(emc, INFO, "last period: %u, next period: %u\n",
src_clk_period, dst_clk_period);
emc_dbg(emc, INFO, " shared_zq_resistor: %d\n", !!shared_zq_resistor);
emc_dbg(emc, INFO, " num_channels: %u\n", emc->num_channels);
emc_dbg(emc, INFO, " opt_dll_mode: %d\n", opt_dll_mode);
/*
* Step 1:
* Pre DVFS SW sequence.
*/
emc_dbg(emc, STEPS, "Step 1\n");
emc_dbg(emc, STEPS, "Step 1.1: Disable DLL temporarily.\n");
value = emc_readl(emc, EMC_CFG_DIG_DLL);
value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
emc_writel(emc, value, EMC_CFG_DIG_DLL);
tegra210_emc_timing_update(emc);
for (i = 0; i < emc->num_channels; i++)
tegra210_emc_wait_for_update(emc, i, EMC_CFG_DIG_DLL,
EMC_CFG_DIG_DLL_CFG_DLL_EN, 0);
emc_dbg(emc, STEPS, "Step 1.2: Disable AUTOCAL temporarily.\n");
emc_auto_cal_config = next->emc_auto_cal_config;
auto_cal_en = emc_auto_cal_config & EMC_AUTO_CAL_CONFIG_AUTO_CAL_ENABLE;
emc_auto_cal_config &= ~EMC_AUTO_CAL_CONFIG_AUTO_CAL_START;
emc_auto_cal_config |= EMC_AUTO_CAL_CONFIG_AUTO_CAL_MEASURE_STALL;
emc_auto_cal_config |= EMC_AUTO_CAL_CONFIG_AUTO_CAL_UPDATE_STALL;
emc_auto_cal_config |= auto_cal_en;
emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG);
emc_readl(emc, EMC_AUTO_CAL_CONFIG); /* Flush write. */
emc_dbg(emc, STEPS, "Step 1.3: Disable other power features.\n");
tegra210_emc_set_shadow_bypass(emc, ACTIVE);
emc_writel(emc, emc_cfg, EMC_CFG);
emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
if (next->periodic_training) {
tegra210_emc_reset_dram_clktree_values(next);
for (i = 0; i < emc->num_channels; i++)
tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK,
0);
for (i = 0; i < emc->num_channels; i++)
tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK,
0);
tegra210_emc_start_periodic_compensation(emc);
delay = 1000 * tegra210_emc_actual_osc_clocks(last->run_clocks);
udelay((delay / last->rate) + 2);
value = periodic_compensation_handler(emc, DVFS_SEQUENCE, fake,
next);
value = (value * 128 * next->rate / 1000) / 1000000;
if (next->periodic_training && value > next->tree_margin)
compensate_trimmer_applicable = true;
}
emc_writel(emc, EMC_INTSTATUS_CLKCHANGE_COMPLETE, EMC_INTSTATUS);
tegra210_emc_set_shadow_bypass(emc, ACTIVE);
emc_writel(emc, emc_cfg, EMC_CFG);
emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
emc_writel(emc, emc_cfg_pipe_clk | EMC_CFG_PIPE_CLK_CLK_ALWAYS_ON,
EMC_CFG_PIPE_CLK);
emc_writel(emc, next->emc_fdpd_ctrl_cmd_no_ramp &
~EMC_FDPD_CTRL_CMD_NO_RAMP_CMD_DPD_NO_RAMP_ENABLE,
EMC_FDPD_CTRL_CMD_NO_RAMP);
bg_reg_mode_change =
((next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD) ^
(last->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD)) ||
((next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) ^
(last->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD));
enable_bglp_reg =
(next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD) == 0;
enable_bg_reg =
(next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) == 0;
if (bg_reg_mode_change) {
if (enable_bg_reg)
emc_writel(emc, last->burst_regs
[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD,
EMC_PMACRO_BG_BIAS_CTRL_0);
if (enable_bglp_reg)
emc_writel(emc, last->burst_regs
[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD,
EMC_PMACRO_BG_BIAS_CTRL_0);
}
/* Check if we need to turn on VREF generator. */
if ((((last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF) == 0) &&
((next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF) == 1)) ||
(((last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) == 0) &&
((next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) != 0))) {
u32 pad_tx_ctrl =
next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
u32 last_pad_tx_ctrl =
last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
u32 next_dq_e_ivref, next_dqs_e_ivref;
next_dqs_e_ivref = pad_tx_ctrl &
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF;
next_dq_e_ivref = pad_tx_ctrl &
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF;
value = (last_pad_tx_ctrl &
~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF &
~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) |
next_dq_e_ivref | next_dqs_e_ivref;
emc_writel(emc, value, EMC_PMACRO_DATA_PAD_TX_CTRL);
udelay(1);
} else if (bg_reg_mode_change) {
udelay(1);
}
tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
/*
* Step 2:
* Prelock the DLL.
*/
emc_dbg(emc, STEPS, "Step 2\n");
if (next->burst_regs[EMC_CFG_DIG_DLL_INDEX] &
EMC_CFG_DIG_DLL_CFG_DLL_EN) {
emc_dbg(emc, INFO, "Prelock enabled for target frequency.\n");
value = tegra210_emc_dll_prelock(emc, clksrc);
emc_dbg(emc, INFO, "DLL out: 0x%03x\n", value);
} else {
emc_dbg(emc, INFO, "Disabling DLL for target frequency.\n");
tegra210_emc_dll_disable(emc);
}
/*
* Step 3:
* Prepare autocal for the clock change.
*/
emc_dbg(emc, STEPS, "Step 3\n");
tegra210_emc_set_shadow_bypass(emc, ACTIVE);
emc_writel(emc, next->emc_auto_cal_config2, EMC_AUTO_CAL_CONFIG2);
emc_writel(emc, next->emc_auto_cal_config3, EMC_AUTO_CAL_CONFIG3);
emc_writel(emc, next->emc_auto_cal_config4, EMC_AUTO_CAL_CONFIG4);
emc_writel(emc, next->emc_auto_cal_config5, EMC_AUTO_CAL_CONFIG5);
emc_writel(emc, next->emc_auto_cal_config6, EMC_AUTO_CAL_CONFIG6);
emc_writel(emc, next->emc_auto_cal_config7, EMC_AUTO_CAL_CONFIG7);
emc_writel(emc, next->emc_auto_cal_config8, EMC_AUTO_CAL_CONFIG8);
tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
emc_auto_cal_config |= (EMC_AUTO_CAL_CONFIG_AUTO_CAL_COMPUTE_START |
auto_cal_en);
emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG);
/*
* Step 4:
* Update EMC_CFG. (??)
*/
emc_dbg(emc, STEPS, "Step 4\n");
if (src_clk_period > 50000 && dram_type == DRAM_TYPE_LPDDR4)
ccfifo_writel(emc, 1, EMC_SELF_REF, 0);
else
emc_writel(emc, next->emc_cfg_2, EMC_CFG_2);
/*
* Step 5:
* Prepare reference variables for ZQCAL regs.
*/
emc_dbg(emc, STEPS, "Step 5\n");
if (dram_type == DRAM_TYPE_LPDDR4)
zq_wait_long = max((u32)1, div_o3(1000000, dst_clk_period));
else if (dram_type == DRAM_TYPE_LPDDR2 || is_lpddr3)
zq_wait_long = max(next->min_mrs_wait,
div_o3(360000, dst_clk_period)) + 4;
else if (dram_type == DRAM_TYPE_DDR3)
zq_wait_long = max((u32)256,
div_o3(320000, dst_clk_period) + 2);
else
zq_wait_long = 0;
/*
* Step 6:
* Training code - removed.
*/
emc_dbg(emc, STEPS, "Step 6\n");
/*
* Step 7:
* Program FSP reference registers and send MRWs to new FSPWR.
*/
emc_dbg(emc, STEPS, "Step 7\n");
emc_dbg(emc, SUB_STEPS, "Step 7.1: Bug 200024907 - Patch RP R2P");
/* WAR 200024907 */
if (dram_type == DRAM_TYPE_LPDDR4) {
u32 nRTP = 16;
if (src_clk_period >= 1000000 / 1866) /* 535.91 ps */
nRTP = 14;
if (src_clk_period >= 1000000 / 1600) /* 625.00 ps */
nRTP = 12;
if (src_clk_period >= 1000000 / 1333) /* 750.19 ps */
nRTP = 10;
if (src_clk_period >= 1000000 / 1066) /* 938.09 ps */
nRTP = 8;
deltaTWATM = max_t(u32, div_o3(7500, src_clk_period), 8);
/*
* Originally there was a + .5 in the tRPST calculation.
* However since we can't do FP in the kernel and the tRTM
* computation was in a floating point ceiling function, adding
* one to tRTP should be ok. There is no other source of non
* integer values, so the result was always going to be
* something for the form: f_ceil(N + .5) = N + 1;
*/
tRPST = (last->emc_mrw & 0x80) >> 7;
tRTM = fake->dram_timings[RL] + div_o3(3600, src_clk_period) +
max_t(u32, div_o3(7500, src_clk_period), 8) + tRPST +
1 + nRTP;
emc_dbg(emc, INFO, "tRTM = %u, EMC_RP = %u\n", tRTM,
next->burst_regs[EMC_RP_INDEX]);
if (last->burst_regs[EMC_RP_INDEX] < tRTM) {
if (tRTM > (last->burst_regs[EMC_R2P_INDEX] +
last->burst_regs[EMC_RP_INDEX])) {
R2P_war = tRTM - last->burst_regs[EMC_RP_INDEX];
RP_war = last->burst_regs[EMC_RP_INDEX];
TRPab_war = last->burst_regs[EMC_TRPAB_INDEX];
if (R2P_war > 63) {
RP_war = R2P_war +
last->burst_regs[EMC_RP_INDEX] - 63;
if (TRPab_war < RP_war)
TRPab_war = RP_war;
R2P_war = 63;
}
} else {
R2P_war = last->burst_regs[EMC_R2P_INDEX];
RP_war = last->burst_regs[EMC_RP_INDEX];
TRPab_war = last->burst_regs[EMC_TRPAB_INDEX];
}
if (RP_war < deltaTWATM) {
W2P_war = last->burst_regs[EMC_W2P_INDEX]
+ deltaTWATM - RP_war;
if (W2P_war > 63) {
RP_war = RP_war + W2P_war - 63;
if (TRPab_war < RP_war)
TRPab_war = RP_war;
W2P_war = 63;
}
} else {
W2P_war = last->burst_regs[
EMC_W2P_INDEX];
}
if ((last->burst_regs[EMC_W2P_INDEX] ^ W2P_war) ||
(last->burst_regs[EMC_R2P_INDEX] ^ R2P_war) ||
(last->burst_regs[EMC_RP_INDEX] ^ RP_war) ||
(last->burst_regs[EMC_TRPAB_INDEX] ^ TRPab_war)) {
emc_writel(emc, RP_war, EMC_RP);
emc_writel(emc, R2P_war, EMC_R2P);
emc_writel(emc, W2P_war, EMC_W2P);
emc_writel(emc, TRPab_war, EMC_TRPAB);
}
tegra210_emc_timing_update(emc);
} else {
emc_dbg(emc, INFO, "Skipped WAR\n");
}
}
if (!fsp_for_next_freq) {
mr13_flip_fspwr = (next->emc_mrw3 & 0xffffff3f) | 0x80;
mr13_flip_fspop = (next->emc_mrw3 & 0xffffff3f) | 0x00;
} else {
mr13_flip_fspwr = (next->emc_mrw3 & 0xffffff3f) | 0x40;
mr13_flip_fspop = (next->emc_mrw3 & 0xffffff3f) | 0xc0;
}
if (dram_type == DRAM_TYPE_LPDDR4) {
emc_writel(emc, mr13_flip_fspwr, EMC_MRW3);
emc_writel(emc, next->emc_mrw, EMC_MRW);
emc_writel(emc, next->emc_mrw2, EMC_MRW2);
}
/*
* Step 8:
* Program the shadow registers.
*/
emc_dbg(emc, STEPS, "Step 8\n");
emc_dbg(emc, SUB_STEPS, "Writing burst_regs\n");
for (i = 0; i < next->num_burst; i++) {
const u16 *offsets = emc->offsets->burst;
u16 offset;
if (!offsets[i])
continue;
value = next->burst_regs[i];
offset = offsets[i];
if (dram_type != DRAM_TYPE_LPDDR4 &&
(offset == EMC_MRW6 || offset == EMC_MRW7 ||
offset == EMC_MRW8 || offset == EMC_MRW9 ||
offset == EMC_MRW10 || offset == EMC_MRW11 ||
offset == EMC_MRW12 || offset == EMC_MRW13 ||
offset == EMC_MRW14 || offset == EMC_MRW15 ||
offset == EMC_TRAINING_CTRL))
continue;
/* Pain... And suffering. */
if (offset == EMC_CFG) {
value &= ~EMC_CFG_DRAM_ACPD;
value &= ~EMC_CFG_DYN_SELF_REF;
if (dram_type == DRAM_TYPE_LPDDR4) {
value &= ~EMC_CFG_DRAM_CLKSTOP_SR;
value &= ~EMC_CFG_DRAM_CLKSTOP_PD;
}
} else if (offset == EMC_MRS_WAIT_CNT &&
dram_type == DRAM_TYPE_LPDDR2 &&
opt_zcal_en_cc && !opt_cc_short_zcal &&
opt_short_zcal) {
value = (value & ~(EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK <<
EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT)) |
((zq_wait_long & EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK) <<
EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT);
} else if (offset == EMC_ZCAL_WAIT_CNT &&
dram_type == DRAM_TYPE_DDR3 && opt_zcal_en_cc &&
!opt_cc_short_zcal && opt_short_zcal) {
value = (value & ~(EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK <<
EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_SHIFT)) |
memory: tegra: tegra210-emc: fix indentation Use tabs instead of spaces for indentation. Signed-off-by: Krzysztof Kozlowski <krzk@kernel.org> Link: https://lore.kernel.org/r/20200724074038.5597-25-krzk@kernel.org
2020-07-24 09:40:33 +02:00
((zq_wait_long & EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK) <<
memory: tegra: Add EMC scaling sequence code for Tegra210 This patch includes the sequence for clock tuning and the dynamic training mechanism for the clock above 800MHz. And historically there have been different sequences to change the EMC clock. The sequence to be used is specified in the EMC table. However, for the currently supported upstreaming platform, only the most recent sequence is used. So only support that in this patch. Based on the work of Peter De Schrijver <pdeschrijver@nvidia.com>. Signed-off-by: Joseph Lo <josephl@nvidia.com> Signed-off-by: Thierry Reding <treding@nvidia.com>
2019-05-29 16:21:37 +08:00
EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT);
} else if (offset == EMC_ZCAL_INTERVAL && opt_zcal_en_cc) {
value = 0; /* EMC_ZCAL_INTERVAL reset value. */
} else if (offset == EMC_PMACRO_AUTOCAL_CFG_COMMON) {
value |= EMC_PMACRO_AUTOCAL_CFG_COMMON_E_CAL_BYPASS_DVFS;
} else if (offset == EMC_PMACRO_DATA_PAD_TX_CTRL) {
value &= ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC |
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
} else if (offset == EMC_PMACRO_CMD_PAD_TX_CTRL) {
value |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
value &= ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC |
EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
} else if (offset == EMC_PMACRO_BRICK_CTRL_RFU1) {
value &= 0xf800f800;
} else if (offset == EMC_PMACRO_COMMON_PAD_TX_CTRL) {
value &= 0xfffffff0;
}
emc_writel(emc, value, offset);
}
memory: tegra: Support derated timings on Tegra210 Derated timings are used to ensure that the memory chips keep operating correctly at high temperatures. This adds code to support polling of the chip operating state when high temperatures are measured on the chip and change the refresh mode accordingly. Under very high temperatures, the driver will switch to the derated tables to ensure proper operation of the memory chips. Signed-off-by: Thierry Reding <treding@nvidia.com>
2020-04-03 20:03:15 +02:00
/* SW addition: do EMC refresh adjustment here. */
tegra210_emc_adjust_timing(emc, next);
memory: tegra: Add EMC scaling sequence code for Tegra210 This patch includes the sequence for clock tuning and the dynamic training mechanism for the clock above 800MHz. And historically there have been different sequences to change the EMC clock. The sequence to be used is specified in the EMC table. However, for the currently supported upstreaming platform, only the most recent sequence is used. So only support that in this patch. Based on the work of Peter De Schrijver <pdeschrijver@nvidia.com>. Signed-off-by: Joseph Lo <josephl@nvidia.com> Signed-off-by: Thierry Reding <treding@nvidia.com>
2019-05-29 16:21:37 +08:00
if (dram_type == DRAM_TYPE_LPDDR4) {
value = (23 << EMC_MRW_MRW_MA_SHIFT) |
(next->run_clocks & EMC_MRW_MRW_OP_MASK);
emc_writel(emc, value, EMC_MRW);
}
/* Per channel burst registers. */
emc_dbg(emc, SUB_STEPS, "Writing burst_regs_per_ch\n");
for (i = 0; i < next->num_burst_per_ch; i++) {
const struct tegra210_emc_per_channel_regs *burst =
emc->offsets->burst_per_channel;
if (!burst[i].offset)
continue;
if (dram_type != DRAM_TYPE_LPDDR4 &&
(burst[i].offset == EMC_MRW6 ||
burst[i].offset == EMC_MRW7 ||
burst[i].offset == EMC_MRW8 ||
burst[i].offset == EMC_MRW9 ||
burst[i].offset == EMC_MRW10 ||
burst[i].offset == EMC_MRW11 ||
burst[i].offset == EMC_MRW12 ||
burst[i].offset == EMC_MRW13 ||
burst[i].offset == EMC_MRW14 ||
burst[i].offset == EMC_MRW15))
continue;
/* Filter out second channel if not in DUAL_CHANNEL mode. */
if (emc->num_channels < 2 && burst[i].bank >= 1)
continue;
emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
next->burst_reg_per_ch[i], burst[i].offset);
emc_channel_writel(emc, burst[i].bank,
next->burst_reg_per_ch[i],
burst[i].offset);
}
/* Vref regs. */
emc_dbg(emc, SUB_STEPS, "Writing vref_regs\n");
for (i = 0; i < next->vref_num; i++) {
const struct tegra210_emc_per_channel_regs *vref =
emc->offsets->vref_per_channel;
if (!vref[i].offset)
continue;
if (emc->num_channels < 2 && vref[i].bank >= 1)
continue;
emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
next->vref_perch_regs[i], vref[i].offset);
emc_channel_writel(emc, vref[i].bank, next->vref_perch_regs[i],
vref[i].offset);
}
/* Trimmers. */
emc_dbg(emc, SUB_STEPS, "Writing trim_regs\n");
for (i = 0; i < next->num_trim; i++) {
const u16 *offsets = emc->offsets->trim;
if (!offsets[i])
continue;
if (compensate_trimmer_applicable &&
(offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 ||
offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 ||
offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 ||
offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 ||
offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 ||
offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 ||
offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 ||
offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 ||
offsets[i] == EMC_DATA_BRLSHFT_0 ||
offsets[i] == EMC_DATA_BRLSHFT_1)) {
value = tegra210_emc_compensate(next, offsets[i]);
emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
value, offsets[i]);
emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n",
(u32)(u64)offsets[i], value);
emc_writel(emc, value, offsets[i]);
} else {
emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
next->trim_regs[i], offsets[i]);
emc_writel(emc, next->trim_regs[i], offsets[i]);
}
}
/* Per channel trimmers. */
emc_dbg(emc, SUB_STEPS, "Writing trim_regs_per_ch\n");
for (i = 0; i < next->num_trim_per_ch; i++) {
const struct tegra210_emc_per_channel_regs *trim =
&emc->offsets->trim_per_channel[0];
unsigned int offset;
if (!trim[i].offset)
continue;
if (emc->num_channels < 2 && trim[i].bank >= 1)
continue;
offset = trim[i].offset;
if (compensate_trimmer_applicable &&
(offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 ||
offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 ||
offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 ||
offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 ||
offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 ||
offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 ||
offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 ||
offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 ||
offset == EMC_DATA_BRLSHFT_0 ||
offset == EMC_DATA_BRLSHFT_1)) {
value = tegra210_emc_compensate(next, offset);
emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
value, offset);
emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n", offset,
value);
emc_channel_writel(emc, trim[i].bank, value, offset);
} else {
emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
next->trim_perch_regs[i], offset);
emc_channel_writel(emc, trim[i].bank,
next->trim_perch_regs[i], offset);
}
}
emc_dbg(emc, SUB_STEPS, "Writing burst_mc_regs\n");
for (i = 0; i < next->num_mc_regs; i++) {
const u16 *offsets = emc->offsets->burst_mc;
u32 *values = next->burst_mc_regs;
emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
values[i], offsets[i]);
mc_writel(emc->mc, values[i], offsets[i]);
}
/* Registers to be programmed on the faster clock. */
if (next->rate < last->rate) {
const u16 *la = emc->offsets->la_scale;
emc_dbg(emc, SUB_STEPS, "Writing la_scale_regs\n");
for (i = 0; i < next->num_up_down; i++) {
emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
next->la_scale_regs[i], la[i]);
mc_writel(emc->mc, next->la_scale_regs[i], la[i]);
}
}
/* Flush all the burst register writes. */
mc_readl(emc->mc, MC_EMEM_ADR_CFG);
/*
* Step 9:
* LPDDR4 section A.
*/
emc_dbg(emc, STEPS, "Step 9\n");
value = next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX];
value &= ~EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK;
if (dram_type == DRAM_TYPE_LPDDR4) {
emc_writel(emc, 0, EMC_ZCAL_INTERVAL);
emc_writel(emc, value, EMC_ZCAL_WAIT_CNT);
value = emc_dbg | (EMC_DBG_WRITE_MUX_ACTIVE |
EMC_DBG_WRITE_ACTIVE_ONLY);
emc_writel(emc, value, EMC_DBG);
emc_writel(emc, 0, EMC_ZCAL_INTERVAL);
emc_writel(emc, emc_dbg, EMC_DBG);
}
/*
* Step 10:
* LPDDR4 and DDR3 common section.
*/
emc_dbg(emc, STEPS, "Step 10\n");
if (opt_dvfs_mode == MAN_SR || dram_type == DRAM_TYPE_LPDDR4) {
if (dram_type == DRAM_TYPE_LPDDR4)
ccfifo_writel(emc, 0x101, EMC_SELF_REF, 0);
else
ccfifo_writel(emc, 0x1, EMC_SELF_REF, 0);
if (dram_type == DRAM_TYPE_LPDDR4 &&
dst_clk_period <= zqcal_before_cc_cutoff) {
ccfifo_writel(emc, mr13_flip_fspwr ^ 0x40, EMC_MRW3, 0);
ccfifo_writel(emc, (next->burst_regs[EMC_MRW6_INDEX] &
0xFFFF3F3F) |
(last->burst_regs[EMC_MRW6_INDEX] &
0x0000C0C0), EMC_MRW6, 0);
ccfifo_writel(emc, (next->burst_regs[EMC_MRW14_INDEX] &
0xFFFF0707) |
(last->burst_regs[EMC_MRW14_INDEX] &
0x00003838), EMC_MRW14, 0);
if (emc->num_devices > 1) {
ccfifo_writel(emc,
(next->burst_regs[EMC_MRW7_INDEX] &
0xFFFF3F3F) |
(last->burst_regs[EMC_MRW7_INDEX] &
0x0000C0C0), EMC_MRW7, 0);
ccfifo_writel(emc,
(next->burst_regs[EMC_MRW15_INDEX] &
0xFFFF0707) |
(last->burst_regs[EMC_MRW15_INDEX] &
0x00003838), EMC_MRW15, 0);
}
if (opt_zcal_en_cc) {
if (emc->num_devices < 2)
ccfifo_writel(emc,
2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT
| EMC_ZQ_CAL_ZQ_CAL_CMD,
EMC_ZQ_CAL, 0);
else if (shared_zq_resistor)
ccfifo_writel(emc,
2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT
| EMC_ZQ_CAL_ZQ_CAL_CMD,
EMC_ZQ_CAL, 0);
else
ccfifo_writel(emc,
EMC_ZQ_CAL_ZQ_CAL_CMD,
EMC_ZQ_CAL, 0);
}
}
}
if (dram_type == DRAM_TYPE_LPDDR4) {
value = (1000 * fake->dram_timings[T_RP]) / src_clk_period;
ccfifo_writel(emc, mr13_flip_fspop | 0x8, EMC_MRW3, value);
ccfifo_writel(emc, 0, 0, tFC_lpddr4 / src_clk_period);
}
if (dram_type == DRAM_TYPE_LPDDR4 || opt_dvfs_mode != MAN_SR) {
delay = 30;
if (cya_allow_ref_cc) {
delay += (1000 * fake->dram_timings[T_RP]) /
src_clk_period;
delay += 4000 * fake->dram_timings[T_RFC];
}
ccfifo_writel(emc, emc_pin & ~(EMC_PIN_PIN_CKE_PER_DEV |
EMC_PIN_PIN_CKEB |
EMC_PIN_PIN_CKE),
EMC_PIN, delay);
}
/* calculate reference delay multiplier */
value = 1;
if (ref_b4_sref_en)
value++;
if (cya_allow_ref_cc)
value++;
if (cya_issue_pc_ref)
value++;
if (dram_type != DRAM_TYPE_LPDDR4) {
delay = ((1000 * fake->dram_timings[T_RP] / src_clk_period) +
(1000 * fake->dram_timings[T_RFC] / src_clk_period));
delay = value * delay + 20;
} else {
delay = 0;
}
/*
* Step 11:
* Ramp down.
*/
emc_dbg(emc, STEPS, "Step 11\n");
ccfifo_writel(emc, 0x0, EMC_CFG_SYNC, delay);
value = emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE | EMC_DBG_WRITE_ACTIVE_ONLY;
ccfifo_writel(emc, value, EMC_DBG, 0);
ramp_down_wait = tegra210_emc_dvfs_power_ramp_down(emc, src_clk_period,
0);
/*
* Step 12:
* And finally - trigger the clock change.
*/
emc_dbg(emc, STEPS, "Step 12\n");
ccfifo_writel(emc, 1, EMC_STALL_THEN_EXE_AFTER_CLKCHANGE, 0);
value &= ~EMC_DBG_WRITE_ACTIVE_ONLY;
ccfifo_writel(emc, value, EMC_DBG, 0);
/*
* Step 13:
* Ramp up.
*/
emc_dbg(emc, STEPS, "Step 13\n");
ramp_up_wait = tegra210_emc_dvfs_power_ramp_up(emc, dst_clk_period, 0);
ccfifo_writel(emc, emc_dbg, EMC_DBG, 0);
/*
* Step 14:
* Bringup CKE pins.
*/
emc_dbg(emc, STEPS, "Step 14\n");
if (dram_type == DRAM_TYPE_LPDDR4) {
value = emc_pin | EMC_PIN_PIN_CKE;
if (emc->num_devices <= 1)
value &= ~(EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE_PER_DEV);
else
value |= EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE_PER_DEV;
ccfifo_writel(emc, value, EMC_PIN, 0);
}
/*
* Step 15: (two step 15s ??)
* Calculate zqlatch wait time; has dependency on ramping times.
*/
emc_dbg(emc, STEPS, "Step 15\n");
if (dst_clk_period <= zqcal_before_cc_cutoff) {
s32 t = (s32)(ramp_up_wait + ramp_down_wait) /
(s32)dst_clk_period;
zq_latch_dvfs_wait_time = (s32)tZQCAL_lpddr4_fc_adj - t;
} else {
zq_latch_dvfs_wait_time = tZQCAL_lpddr4_fc_adj -
div_o3(1000 * next->dram_timings[T_PDEX],
dst_clk_period);
}
emc_dbg(emc, INFO, "tZQCAL_lpddr4_fc_adj = %u\n", tZQCAL_lpddr4_fc_adj);
emc_dbg(emc, INFO, "dst_clk_period = %u\n",
dst_clk_period);
emc_dbg(emc, INFO, "next->dram_timings[T_PDEX] = %u\n",
next->dram_timings[T_PDEX]);
emc_dbg(emc, INFO, "zq_latch_dvfs_wait_time = %d\n",
max_t(s32, 0, zq_latch_dvfs_wait_time));
if (dram_type == DRAM_TYPE_LPDDR4 && opt_zcal_en_cc) {
delay = div_o3(1000 * next->dram_timings[T_PDEX],
dst_clk_period);
if (emc->num_devices < 2) {
if (dst_clk_period > zqcal_before_cc_cutoff)
ccfifo_writel(emc,
2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL,
delay);
value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000;
ccfifo_writel(emc, value, EMC_MRW3, delay);
ccfifo_writel(emc, 0, EMC_SELF_REF, 0);
ccfifo_writel(emc, 0, EMC_REF, 0);
ccfifo_writel(emc, 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
EMC_ZQ_CAL_ZQ_LATCH_CMD,
EMC_ZQ_CAL,
max_t(s32, 0, zq_latch_dvfs_wait_time));
} else if (shared_zq_resistor) {
if (dst_clk_period > zqcal_before_cc_cutoff)
ccfifo_writel(emc,
2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL,
delay);
ccfifo_writel(emc, 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL,
max_t(s32, 0, zq_latch_dvfs_wait_time) +
delay);
ccfifo_writel(emc, 1UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
EMC_ZQ_CAL_ZQ_LATCH_CMD,
EMC_ZQ_CAL, 0);
value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000;
ccfifo_writel(emc, value, EMC_MRW3, 0);
ccfifo_writel(emc, 0, EMC_SELF_REF, 0);
ccfifo_writel(emc, 0, EMC_REF, 0);
ccfifo_writel(emc, 1UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL,
tZQCAL_lpddr4 / dst_clk_period);
} else {
if (dst_clk_period > zqcal_before_cc_cutoff)
ccfifo_writel(emc, EMC_ZQ_CAL_ZQ_CAL_CMD,
EMC_ZQ_CAL, delay);
value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000;
ccfifo_writel(emc, value, EMC_MRW3, delay);
ccfifo_writel(emc, 0, EMC_SELF_REF, 0);
ccfifo_writel(emc, 0, EMC_REF, 0);
ccfifo_writel(emc, EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL,
max_t(s32, 0, zq_latch_dvfs_wait_time));
}
}
/* WAR: delay for zqlatch */
ccfifo_writel(emc, 0, 0, 10);
/*
* Step 16:
* LPDDR4 Conditional Training Kickoff. Removed.
*/
/*
* Step 17:
* MANSR exit self refresh.
*/
emc_dbg(emc, STEPS, "Step 17\n");
if (opt_dvfs_mode == MAN_SR && dram_type != DRAM_TYPE_LPDDR4)
ccfifo_writel(emc, 0, EMC_SELF_REF, 0);
/*
* Step 18:
* Send MRWs to LPDDR3/DDR3.
*/
emc_dbg(emc, STEPS, "Step 18\n");
if (dram_type == DRAM_TYPE_LPDDR2) {
ccfifo_writel(emc, next->emc_mrw2, EMC_MRW2, 0);
ccfifo_writel(emc, next->emc_mrw, EMC_MRW, 0);
if (is_lpddr3)
ccfifo_writel(emc, next->emc_mrw4, EMC_MRW4, 0);
} else if (dram_type == DRAM_TYPE_DDR3) {
if (opt_dll_mode)
ccfifo_writel(emc, next->emc_emrs &
~EMC_EMRS_USE_EMRS_LONG_CNT, EMC_EMRS, 0);
ccfifo_writel(emc, next->emc_emrs2 &
~EMC_EMRS2_USE_EMRS2_LONG_CNT, EMC_EMRS2, 0);
ccfifo_writel(emc, next->emc_mrs |
EMC_EMRS_USE_EMRS_LONG_CNT, EMC_MRS, 0);
}
/*
* Step 19:
* ZQCAL for LPDDR3/DDR3
*/
emc_dbg(emc, STEPS, "Step 19\n");
if (opt_zcal_en_cc) {
if (dram_type == DRAM_TYPE_LPDDR2) {
value = opt_cc_short_zcal ? 90000 : 360000;
value = div_o3(value, dst_clk_period);
value = value <<
EMC_MRS_WAIT_CNT2_MRS_EXT2_WAIT_CNT_SHIFT |
value <<
EMC_MRS_WAIT_CNT2_MRS_EXT1_WAIT_CNT_SHIFT;
ccfifo_writel(emc, value, EMC_MRS_WAIT_CNT2, 0);
value = opt_cc_short_zcal ? 0x56 : 0xab;
ccfifo_writel(emc, 2 << EMC_MRW_MRW_DEV_SELECTN_SHIFT |
EMC_MRW_USE_MRW_EXT_CNT |
10 << EMC_MRW_MRW_MA_SHIFT |
value << EMC_MRW_MRW_OP_SHIFT,
EMC_MRW, 0);
if (emc->num_devices > 1) {
value = 1 << EMC_MRW_MRW_DEV_SELECTN_SHIFT |
EMC_MRW_USE_MRW_EXT_CNT |
10 << EMC_MRW_MRW_MA_SHIFT |
value << EMC_MRW_MRW_OP_SHIFT;
ccfifo_writel(emc, value, EMC_MRW, 0);
}
} else if (dram_type == DRAM_TYPE_DDR3) {
value = opt_cc_short_zcal ? 0 : EMC_ZQ_CAL_LONG;
ccfifo_writel(emc, value |
2 << EMC_ZQ_CAL_DEV_SEL_SHIFT |
EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL,
0);
if (emc->num_devices > 1) {
value = value | 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT |
EMC_ZQ_CAL_ZQ_CAL_CMD;
ccfifo_writel(emc, value, EMC_ZQ_CAL, 0);
}
}
}
if (bg_reg_mode_change) {
tegra210_emc_set_shadow_bypass(emc, ACTIVE);
if (ramp_up_wait <= 1250000)
delay = (1250000 - ramp_up_wait) / dst_clk_period;
else
delay = 0;
ccfifo_writel(emc,
next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX],
EMC_PMACRO_BG_BIAS_CTRL_0, delay);
tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
}
/*
* Step 20:
* Issue ref and optional QRST.
*/
emc_dbg(emc, STEPS, "Step 20\n");
if (dram_type != DRAM_TYPE_LPDDR4)
ccfifo_writel(emc, 0, EMC_REF, 0);
if (opt_do_sw_qrst) {
ccfifo_writel(emc, 1, EMC_ISSUE_QRST, 0);
ccfifo_writel(emc, 0, EMC_ISSUE_QRST, 2);
}
/*
* Step 21:
* Restore ZCAL and ZCAL interval.
*/
emc_dbg(emc, STEPS, "Step 21\n");
if (save_restore_clkstop_pd || opt_zcal_en_cc) {
ccfifo_writel(emc, emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE,
EMC_DBG, 0);
if (opt_zcal_en_cc && dram_type != DRAM_TYPE_LPDDR4)
ccfifo_writel(emc, next->burst_regs[EMC_ZCAL_INTERVAL_INDEX],
EMC_ZCAL_INTERVAL, 0);
if (save_restore_clkstop_pd)
ccfifo_writel(emc, next->burst_regs[EMC_CFG_INDEX] &
~EMC_CFG_DYN_SELF_REF,
EMC_CFG, 0);
ccfifo_writel(emc, emc_dbg, EMC_DBG, 0);
}
/*
* Step 22:
* Restore EMC_CFG_PIPE_CLK.
*/
emc_dbg(emc, STEPS, "Step 22\n");
ccfifo_writel(emc, emc_cfg_pipe_clk, EMC_CFG_PIPE_CLK, 0);
if (bg_reg_mode_change) {
if (enable_bg_reg)
emc_writel(emc,
next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD,
EMC_PMACRO_BG_BIAS_CTRL_0);
else
emc_writel(emc,
next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD,
EMC_PMACRO_BG_BIAS_CTRL_0);
}
/*
* Step 23:
*/
emc_dbg(emc, STEPS, "Step 23\n");
value = emc_readl(emc, EMC_CFG_DIG_DLL);
value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC;
value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK;
value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK;
value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
value = (value & ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK) |
(2 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT);
emc_writel(emc, value, EMC_CFG_DIG_DLL);
tegra210_emc_do_clock_change(emc, clksrc);
/*
* Step 24:
* Save training results. Removed.
*/
/*
* Step 25:
* Program MC updown registers.
*/
emc_dbg(emc, STEPS, "Step 25\n");
if (next->rate > last->rate) {
for (i = 0; i < next->num_up_down; i++)
mc_writel(emc->mc, next->la_scale_regs[i],
emc->offsets->la_scale[i]);
tegra210_emc_timing_update(emc);
}
/*
* Step 26:
* Restore ZCAL registers.
*/
emc_dbg(emc, STEPS, "Step 26\n");
if (dram_type == DRAM_TYPE_LPDDR4) {
tegra210_emc_set_shadow_bypass(emc, ACTIVE);
emc_writel(emc, next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX],
EMC_ZCAL_WAIT_CNT);
emc_writel(emc, next->burst_regs[EMC_ZCAL_INTERVAL_INDEX],
EMC_ZCAL_INTERVAL);
tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
}
if (dram_type != DRAM_TYPE_LPDDR4 && opt_zcal_en_cc &&
!opt_short_zcal && opt_cc_short_zcal) {
udelay(2);
tegra210_emc_set_shadow_bypass(emc, ACTIVE);
if (dram_type == DRAM_TYPE_LPDDR2)
emc_writel(emc, next->burst_regs[EMC_MRS_WAIT_CNT_INDEX],
EMC_MRS_WAIT_CNT);
else if (dram_type == DRAM_TYPE_DDR3)
emc_writel(emc, next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX],
EMC_ZCAL_WAIT_CNT);
tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
}
/*
* Step 27:
* Restore EMC_CFG, FDPD registers.
*/
emc_dbg(emc, STEPS, "Step 27\n");
tegra210_emc_set_shadow_bypass(emc, ACTIVE);
emc_writel(emc, next->burst_regs[EMC_CFG_INDEX], EMC_CFG);
tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
emc_writel(emc, next->emc_fdpd_ctrl_cmd_no_ramp,
EMC_FDPD_CTRL_CMD_NO_RAMP);
emc_writel(emc, next->emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
/*
* Step 28:
* Training recover. Removed.
*/
emc_dbg(emc, STEPS, "Step 28\n");
tegra210_emc_set_shadow_bypass(emc, ACTIVE);
emc_writel(emc,
next->burst_regs[EMC_PMACRO_AUTOCAL_CFG_COMMON_INDEX],
EMC_PMACRO_AUTOCAL_CFG_COMMON);
tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
/*
* Step 29:
* Power fix WAR.
*/
emc_dbg(emc, STEPS, "Step 29\n");
emc_writel(emc, EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE0 |
EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE1 |
EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE2 |
EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE3 |
EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE4 |
EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE5 |
EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE6 |
EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE7,
EMC_PMACRO_CFG_PM_GLOBAL_0);
emc_writel(emc, EMC_PMACRO_TRAINING_CTRL_0_CH0_TRAINING_E_WRPTR,
EMC_PMACRO_TRAINING_CTRL_0);
emc_writel(emc, EMC_PMACRO_TRAINING_CTRL_1_CH1_TRAINING_E_WRPTR,
EMC_PMACRO_TRAINING_CTRL_1);
emc_writel(emc, 0, EMC_PMACRO_CFG_PM_GLOBAL_0);
/*
* Step 30:
* Re-enable autocal.
*/
emc_dbg(emc, STEPS, "Step 30: Re-enable DLL and AUTOCAL\n");
if (next->burst_regs[EMC_CFG_DIG_DLL_INDEX] & EMC_CFG_DIG_DLL_CFG_DLL_EN) {
value = emc_readl(emc, EMC_CFG_DIG_DLL);
value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC;
value |= EMC_CFG_DIG_DLL_CFG_DLL_EN;
value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK;
value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK;
value = (value & ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK) |
(2 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT);
emc_writel(emc, value, EMC_CFG_DIG_DLL);
tegra210_emc_timing_update(emc);
}
emc_writel(emc, next->emc_auto_cal_config, EMC_AUTO_CAL_CONFIG);
/* Done! Yay. */
}
const struct tegra210_emc_sequence tegra210_emc_r21021 = {
.revision = 0x7,
.set_clock = tegra210_emc_r21021_set_clock,
.periodic_compensation = tegra210_emc_r21021_periodic_compensation,
};
Reference in New Issue Copy Permalink