git.sur5r.net Git - u-boot/blob - drivers/ddr/fsl/ctrl_regs.c

   1 /*
   2  * Copyright 2008-2016 Freescale Semiconductor, Inc.
   3  * Copyright 2017-2018 NXP Semiconductor
   4  *
   5  * SPDX-License-Identifier:     GPL-2.0+
   6  */
   7
   8 /*
   9  * Generic driver for Freescale DDR/DDR2/DDR3/DDR4 memory controller.
  10  * Based on code from spd_sdram.c
  11  * Author: James Yang [at freescale.com]
  12  */
  13
  14 #include <common.h>
  15 #include <fsl_ddr_sdram.h>
  16 #include <fsl_errata.h>
  17 #include <fsl_ddr.h>
  18 #include <fsl_immap.h>
  19 #include <asm/io.h>
  20 #if defined(CONFIG_FSL_LSCH2) || defined(CONFIG_FSL_LSCH3) || \
  21         defined(CONFIG_ARM)
  22 #include <asm/arch/clock.h>
  23 #endif
  24
  25 /*
  26  * Determine Rtt value.
  27  *
  28  * This should likely be either board or controller specific.
  29  *
  30  * Rtt(nominal) - DDR2:
  31  *      0 = Rtt disabled
  32  *      1 = 75 ohm
  33  *      2 = 150 ohm
  34  *      3 = 50 ohm
  35  * Rtt(nominal) - DDR3:
  36  *      0 = Rtt disabled
  37  *      1 = 60 ohm
  38  *      2 = 120 ohm
  39  *      3 = 40 ohm
  40  *      4 = 20 ohm
  41  *      5 = 30 ohm
  42  *
  43  * FIXME: Apparently 8641 needs a value of 2
  44  * FIXME: Old code seys if 667 MHz or higher, use 3 on 8572
  45  *
  46  * FIXME: There was some effort down this line earlier:
  47  *
  48  *      unsigned int i;
  49  *      for (i = 0; i < CONFIG_CHIP_SELECTS_PER_CTRL/2; i++) {
  50  *              if (popts->dimmslot[i].num_valid_cs
  51  *                  && (popts->cs_local_opts[2*i].odt_rd_cfg
  52  *                      || popts->cs_local_opts[2*i].odt_wr_cfg)) {
  53  *                      rtt = 2;
  54  *                      break;
  55  *              }
  56  *      }
  57  */
  58 static inline int fsl_ddr_get_rtt(void)
  59 {
  60         int rtt;
  61
  62 #if defined(CONFIG_SYS_FSL_DDR1)
  63         rtt = 0;
  64 #elif defined(CONFIG_SYS_FSL_DDR2)
  65         rtt = 3;
  66 #else
  67         rtt = 0;
  68 #endif
  69
  70         return rtt;
  71 }
  72
  73 #ifdef CONFIG_SYS_FSL_DDR4
  74 /*
  75  * compute CAS write latency according to DDR4 spec
  76  * CWL = 9 for <= 1600MT/s
  77  *       10 for <= 1866MT/s
  78  *       11 for <= 2133MT/s
  79  *       12 for <= 2400MT/s
  80  *       14 for <= 2667MT/s
  81  *       16 for <= 2933MT/s
  82  *       18 for higher
  83  */
  84 static inline unsigned int compute_cas_write_latency(
  85                                 const unsigned int ctrl_num)
  86 {
  87         unsigned int cwl;
  88         const unsigned int mclk_ps = get_memory_clk_period_ps(ctrl_num);
  89         if (mclk_ps >= 1250)
  90                 cwl = 9;
  91         else if (mclk_ps >= 1070)
  92                 cwl = 10;
  93         else if (mclk_ps >= 935)
  94                 cwl = 11;
  95         else if (mclk_ps >= 833)
  96                 cwl = 12;
  97         else if (mclk_ps >= 750)
  98                 cwl = 14;
  99         else if (mclk_ps >= 681)
 100                 cwl = 16;
 101         else
 102                 cwl = 18;
 103
 104         return cwl;
 105 }
 106 #else
 107 /*
 108  * compute the CAS write latency according to DDR3 spec
 109  * CWL = 5 if tCK >= 2.5ns
 110  *       6 if 2.5ns > tCK >= 1.875ns
 111  *       7 if 1.875ns > tCK >= 1.5ns
 112  *       8 if 1.5ns > tCK >= 1.25ns
 113  *       9 if 1.25ns > tCK >= 1.07ns
 114  *       10 if 1.07ns > tCK >= 0.935ns
 115  *       11 if 0.935ns > tCK >= 0.833ns
 116  *       12 if 0.833ns > tCK >= 0.75ns
 117  */
 118 static inline unsigned int compute_cas_write_latency(
 119                                 const unsigned int ctrl_num)
 120 {
 121         unsigned int cwl;
 122         const unsigned int mclk_ps = get_memory_clk_period_ps(ctrl_num);
 123
 124         if (mclk_ps >= 2500)
 125                 cwl = 5;
 126         else if (mclk_ps >= 1875)
 127                 cwl = 6;
 128         else if (mclk_ps >= 1500)
 129                 cwl = 7;
 130         else if (mclk_ps >= 1250)
 131                 cwl = 8;
 132         else if (mclk_ps >= 1070)
 133                 cwl = 9;
 134         else if (mclk_ps >= 935)
 135                 cwl = 10;
 136         else if (mclk_ps >= 833)
 137                 cwl = 11;
 138         else if (mclk_ps >= 750)
 139                 cwl = 12;
 140         else {
 141                 cwl = 12;
 142                 printf("Warning: CWL is out of range\n");
 143         }
 144         return cwl;
 145 }
 146 #endif
 147
 148 /* Chip Select Configuration (CSn_CONFIG) */
 149 static void set_csn_config(int dimm_number, int i, fsl_ddr_cfg_regs_t *ddr,
 150                                const memctl_options_t *popts,
 151                                const dimm_params_t *dimm_params)
 152 {
 153         unsigned int cs_n_en = 0; /* Chip Select enable */
 154         unsigned int intlv_en = 0; /* Memory controller interleave enable */
 155         unsigned int intlv_ctl = 0; /* Interleaving control */
 156         unsigned int ap_n_en = 0; /* Chip select n auto-precharge enable */
 157         unsigned int odt_rd_cfg = 0; /* ODT for reads configuration */
 158         unsigned int odt_wr_cfg = 0; /* ODT for writes configuration */
 159         unsigned int ba_bits_cs_n = 0; /* Num of bank bits for SDRAM on CSn */
 160         unsigned int row_bits_cs_n = 0; /* Num of row bits for SDRAM on CSn */
 161         unsigned int col_bits_cs_n = 0; /* Num of ocl bits for SDRAM on CSn */
 162         int go_config = 0;
 163 #ifdef CONFIG_SYS_FSL_DDR4
 164         unsigned int bg_bits_cs_n = 0; /* Num of bank group bits */
 165 #else
 166         unsigned int n_banks_per_sdram_device;
 167 #endif
 168
 169         /* Compute CS_CONFIG only for existing ranks of each DIMM.  */
 170         switch (i) {
 171         case 0:
 172                 if (dimm_params[dimm_number].n_ranks > 0) {
 173                         go_config = 1;
 174                         /* These fields only available in CS0_CONFIG */
 175                         if (!popts->memctl_interleaving)
 176                                 break;
 177                         switch (popts->memctl_interleaving_mode) {
 178                         case FSL_DDR_256B_INTERLEAVING:
 179                         case FSL_DDR_CACHE_LINE_INTERLEAVING:
 180                         case FSL_DDR_PAGE_INTERLEAVING:
 181                         case FSL_DDR_BANK_INTERLEAVING:
 182                         case FSL_DDR_SUPERBANK_INTERLEAVING:
 183                                 intlv_en = popts->memctl_interleaving;
 184                                 intlv_ctl = popts->memctl_interleaving_mode;
 185                                 break;
 186                         default:
 187                                 break;
 188                         }
 189                 }
 190                 break;
 191         case 1:
 192                 if ((dimm_number == 0 && dimm_params[0].n_ranks > 1) || \
 193                     (dimm_number == 1 && dimm_params[1].n_ranks > 0))
 194                         go_config = 1;
 195                 break;
 196         case 2:
 197                 if ((dimm_number == 0 && dimm_params[0].n_ranks > 2) || \
 198                    (dimm_number >= 1 && dimm_params[dimm_number].n_ranks > 0))
 199                         go_config = 1;
 200                 break;
 201         case 3:
 202                 if ((dimm_number == 0 && dimm_params[0].n_ranks > 3) || \
 203                     (dimm_number == 1 && dimm_params[1].n_ranks > 1) || \
 204                     (dimm_number == 3 && dimm_params[3].n_ranks > 0))
 205                         go_config = 1;
 206                 break;
 207         default:
 208                 break;
 209         }
 210         if (go_config) {
 211                 cs_n_en = 1;
 212                 ap_n_en = popts->cs_local_opts[i].auto_precharge;
 213                 odt_rd_cfg = popts->cs_local_opts[i].odt_rd_cfg;
 214                 odt_wr_cfg = popts->cs_local_opts[i].odt_wr_cfg;
 215 #ifdef CONFIG_SYS_FSL_DDR4
 216                 ba_bits_cs_n = dimm_params[dimm_number].bank_addr_bits;
 217                 bg_bits_cs_n = dimm_params[dimm_number].bank_group_bits;
 218 #else
 219                 n_banks_per_sdram_device
 220                         = dimm_params[dimm_number].n_banks_per_sdram_device;
 221                 ba_bits_cs_n = __ilog2(n_banks_per_sdram_device) - 2;
 222 #endif
 223                 row_bits_cs_n = dimm_params[dimm_number].n_row_addr - 12;
 224                 col_bits_cs_n = dimm_params[dimm_number].n_col_addr - 8;
 225         }
 226         ddr->cs[i].config = (0
 227                 | ((cs_n_en & 0x1) << 31)
 228                 | ((intlv_en & 0x3) << 29)
 229                 | ((intlv_ctl & 0xf) << 24)
 230                 | ((ap_n_en & 0x1) << 23)
 231
 232                 /* XXX: some implementation only have 1 bit starting at left */
 233                 | ((odt_rd_cfg & 0x7) << 20)
 234
 235                 /* XXX: Some implementation only have 1 bit starting at left */
 236                 | ((odt_wr_cfg & 0x7) << 16)
 237
 238                 | ((ba_bits_cs_n & 0x3) << 14)
 239                 | ((row_bits_cs_n & 0x7) << 8)
 240 #ifdef CONFIG_SYS_FSL_DDR4
 241                 | ((bg_bits_cs_n & 0x3) << 4)
 242 #endif
 243                 | ((col_bits_cs_n & 0x7) << 0)
 244                 );
 245         debug("FSLDDR: cs[%d]_config = 0x%08x\n", i,ddr->cs[i].config);
 246 }
 247
 248 /* Chip Select Configuration 2 (CSn_CONFIG_2) */
 249 /* FIXME: 8572 */
 250 static void set_csn_config_2(int i, fsl_ddr_cfg_regs_t *ddr)
 251 {
 252         unsigned int pasr_cfg = 0;      /* Partial array self refresh config */
 253
 254         ddr->cs[i].config_2 = ((pasr_cfg & 7) << 24);
 255         debug("FSLDDR: cs[%d]_config_2 = 0x%08x\n", i, ddr->cs[i].config_2);
 256 }
 257
 258 /* -3E = 667 CL5, -25 = CL6 800, -25E = CL5 800 */
 259
 260 #if !defined(CONFIG_SYS_FSL_DDR1)
 261 /*
 262  * Check DIMM configuration, return 2 if quad-rank or two dual-rank
 263  * Return 1 if other two slots configuration. Return 0 if single slot.
 264  */
 265 static inline int avoid_odt_overlap(const dimm_params_t *dimm_params)
 266 {
 267 #if CONFIG_DIMM_SLOTS_PER_CTLR == 1
 268         if (dimm_params[0].n_ranks == 4)
 269                 return 2;
 270 #endif
 271
 272 #if CONFIG_DIMM_SLOTS_PER_CTLR == 2
 273         if ((dimm_params[0].n_ranks == 2) &&
 274                 (dimm_params[1].n_ranks == 2))
 275                 return 2;
 276
 277 #ifdef CONFIG_FSL_DDR_FIRST_SLOT_QUAD_CAPABLE
 278         if (dimm_params[0].n_ranks == 4)
 279                 return 2;
 280 #endif
 281
 282         if ((dimm_params[0].n_ranks != 0) &&
 283             (dimm_params[2].n_ranks != 0))
 284                 return 1;
 285 #endif
 286         return 0;
 287 }
 288
 289 /*
 290  * DDR SDRAM Timing Configuration 0 (TIMING_CFG_0)
 291  *
 292  * Avoid writing for DDR I.  The new PQ38 DDR controller
 293  * dreams up non-zero default values to be backwards compatible.
 294  */
 295 static void set_timing_cfg_0(const unsigned int ctrl_num,
 296                                 fsl_ddr_cfg_regs_t *ddr,
 297                                 const memctl_options_t *popts,
 298                                 const dimm_params_t *dimm_params)
 299 {
 300         unsigned char trwt_mclk = 0;   /* Read-to-write turnaround */
 301         unsigned char twrt_mclk = 0;   /* Write-to-read turnaround */
 302         /* 7.5 ns on -3E; 0 means WL - CL + BL/2 + 1 */
 303         unsigned char trrt_mclk = 0;   /* Read-to-read turnaround */
 304         unsigned char twwt_mclk = 0;   /* Write-to-write turnaround */
 305
 306         /* Active powerdown exit timing (tXARD and tXARDS). */
 307         unsigned char act_pd_exit_mclk;
 308         /* Precharge powerdown exit timing (tXP). */
 309         unsigned char pre_pd_exit_mclk;
 310         /* ODT powerdown exit timing (tAXPD). */
 311         unsigned char taxpd_mclk = 0;
 312         /* Mode register set cycle time (tMRD). */
 313         unsigned char tmrd_mclk;
 314 #if defined(CONFIG_SYS_FSL_DDR4) || defined(CONFIG_SYS_FSL_DDR3)
 315         const unsigned int mclk_ps = get_memory_clk_period_ps(ctrl_num);
 316 #endif
 317
 318 #ifdef CONFIG_SYS_FSL_DDR4
 319         /* tXP=max(4nCK, 6ns) */
 320         int txp = max((int)mclk_ps * 4, 6000); /* unit=ps */
 321         unsigned int data_rate = get_ddr_freq(ctrl_num);
 322
 323         /* for faster clock, need more time for data setup */
 324         trwt_mclk = (data_rate/1000000 > 1900) ? 3 : 2;
 325
 326         /*
 327          * for single quad-rank DIMM and two-slot DIMMs
 328          * to avoid ODT overlap
 329          */
 330         switch (avoid_odt_overlap(dimm_params)) {
 331         case 2:
 332                 twrt_mclk = 2;
 333                 twwt_mclk = 2;
 334                 trrt_mclk = 2;
 335                 break;
 336         default:
 337                 twrt_mclk = 1;
 338                 twwt_mclk = 1;
 339                 trrt_mclk = 0;
 340                 break;
 341         }
 342
 343         act_pd_exit_mclk = picos_to_mclk(ctrl_num, txp);
 344         pre_pd_exit_mclk = act_pd_exit_mclk;
 345         /*
 346          * MRS_CYC = max(tMRD, tMOD)
 347          * tMRD = 8nCK, tMOD = max(24nCK, 15ns)
 348          */
 349         tmrd_mclk = max(24U, picos_to_mclk(ctrl_num, 15000));
 350 #elif defined(CONFIG_SYS_FSL_DDR3)
 351         unsigned int data_rate = get_ddr_freq(ctrl_num);
 352         int txp;
 353         unsigned int ip_rev;
 354         int odt_overlap;
 355         /*
 356          * (tXARD and tXARDS). Empirical?
 357          * The DDR3 spec has not tXARD,
 358          * we use the tXP instead of it.
 359          * tXP=max(3nCK, 7.5ns) for DDR3-800, 1066
 360          *     max(3nCK, 6ns) for DDR3-1333, 1600, 1866, 2133
 361          * spec has not the tAXPD, we use
 362          * tAXPD=1, need design to confirm.
 363          */
 364         txp = max((int)mclk_ps * 3, (mclk_ps > 1540 ? 7500 : 6000));
 365
 366         ip_rev = fsl_ddr_get_version(ctrl_num);
 367         if (ip_rev >= 0x40700) {
 368                 /*
 369                  * MRS_CYC = max(tMRD, tMOD)
 370                  * tMRD = 4nCK (8nCK for RDIMM)
 371                  * tMOD = max(12nCK, 15ns)
 372                  */
 373                 tmrd_mclk = max((unsigned int)12,
 374                                 picos_to_mclk(ctrl_num, 15000));
 375         } else {
 376                 /*
 377                  * MRS_CYC = tMRD
 378                  * tMRD = 4nCK (8nCK for RDIMM)
 379                  */
 380                 if (popts->registered_dimm_en)
 381                         tmrd_mclk = 8;
 382                 else
 383                         tmrd_mclk = 4;
 384         }
 385
 386         /* set the turnaround time */
 387
 388         /*
 389          * for single quad-rank DIMM and two-slot DIMMs
 390          * to avoid ODT overlap
 391          */
 392         odt_overlap = avoid_odt_overlap(dimm_params);
 393         switch (odt_overlap) {
 394         case 2:
 395                 twwt_mclk = 2;
 396                 trrt_mclk = 1;
 397                 break;
 398         case 1:
 399                 twwt_mclk = 1;
 400                 trrt_mclk = 0;
 401                 break;
 402         default:
 403                 break;
 404         }
 405
 406         /* for faster clock, need more time for data setup */
 407         trwt_mclk = (data_rate/1000000 > 1800) ? 2 : 1;
 408
 409         if ((data_rate/1000000 > 1150) || (popts->memctl_interleaving))
 410                 twrt_mclk = 1;
 411
 412         if (popts->dynamic_power == 0) {        /* powerdown is not used */
 413                 act_pd_exit_mclk = 1;
 414                 pre_pd_exit_mclk = 1;
 415                 taxpd_mclk = 1;
 416         } else {
 417                 /* act_pd_exit_mclk = tXARD, see above */
 418                 act_pd_exit_mclk = picos_to_mclk(ctrl_num, txp);
 419                 /* Mode register MR0[A12] is '1' - fast exit */
 420                 pre_pd_exit_mclk = act_pd_exit_mclk;
 421                 taxpd_mclk = 1;
 422         }
 423 #else /* CONFIG_SYS_FSL_DDR2 */
 424         /*
 425          * (tXARD and tXARDS). Empirical?
 426          * tXARD = 2 for DDR2
 427          * tXP=2
 428          * tAXPD=8
 429          */
 430         act_pd_exit_mclk = 2;
 431         pre_pd_exit_mclk = 2;
 432         taxpd_mclk = 8;
 433         tmrd_mclk = 2;
 434 #endif
 435
 436         if (popts->trwt_override)
 437                 trwt_mclk = popts->trwt;
 438
 439         ddr->timing_cfg_0 = (0
 440                 | ((trwt_mclk & 0x3) << 30)     /* RWT */
 441                 | ((twrt_mclk & 0x3) << 28)     /* WRT */
 442                 | ((trrt_mclk & 0x3) << 26)     /* RRT */
 443                 | ((twwt_mclk & 0x3) << 24)     /* WWT */
 444                 | ((act_pd_exit_mclk & 0xf) << 20)  /* ACT_PD_EXIT */
 445                 | ((pre_pd_exit_mclk & 0xF) << 16)  /* PRE_PD_EXIT */
 446                 | ((taxpd_mclk & 0xf) << 8)     /* ODT_PD_EXIT */
 447                 | ((tmrd_mclk & 0x1f) << 0)     /* MRS_CYC */
 448                 );
 449         debug("FSLDDR: timing_cfg_0 = 0x%08x\n", ddr->timing_cfg_0);
 450 }
 451 #endif  /* !defined(CONFIG_SYS_FSL_DDR1) */
 452
 453 /* DDR SDRAM Timing Configuration 3 (TIMING_CFG_3) */
 454 static void set_timing_cfg_3(const unsigned int ctrl_num,
 455                              fsl_ddr_cfg_regs_t *ddr,
 456                              const memctl_options_t *popts,
 457                              const common_timing_params_t *common_dimm,
 458                              unsigned int cas_latency,
 459                              unsigned int additive_latency)
 460 {
 461         /* Extended precharge to activate interval (tRP) */
 462         unsigned int ext_pretoact = 0;
 463         /* Extended Activate to precharge interval (tRAS) */
 464         unsigned int ext_acttopre = 0;
 465         /* Extended activate to read/write interval (tRCD) */
 466         unsigned int ext_acttorw = 0;
 467         /* Extended refresh recovery time (tRFC) */
 468         unsigned int ext_refrec;
 469         /* Extended MCAS latency from READ cmd */
 470         unsigned int ext_caslat = 0;
 471         /* Extended additive latency */
 472         unsigned int ext_add_lat = 0;
 473         /* Extended last data to precharge interval (tWR) */
 474         unsigned int ext_wrrec = 0;
 475         /* Control Adjust */
 476         unsigned int cntl_adj = 0;
 477
 478         ext_pretoact = picos_to_mclk(ctrl_num, common_dimm->trp_ps) >> 4;
 479         ext_acttopre = picos_to_mclk(ctrl_num, common_dimm->tras_ps) >> 4;
 480         ext_acttorw = picos_to_mclk(ctrl_num, common_dimm->trcd_ps) >> 4;
 481         ext_caslat = (2 * cas_latency - 1) >> 4;
 482         ext_add_lat = additive_latency >> 4;
 483 #ifdef CONFIG_SYS_FSL_DDR4
 484         ext_refrec = (picos_to_mclk(ctrl_num, common_dimm->trfc1_ps) - 8) >> 4;
 485 #else
 486         ext_refrec = (picos_to_mclk(ctrl_num, common_dimm->trfc_ps) - 8) >> 4;
 487         /* ext_wrrec only deals with 16 clock and above, or 14 with OTF */
 488 #endif
 489         ext_wrrec = (picos_to_mclk(ctrl_num, common_dimm->twr_ps) +
 490                 (popts->otf_burst_chop_en ? 2 : 0)) >> 4;
 491
 492         ddr->timing_cfg_3 = (0
 493                 | ((ext_pretoact & 0x1) << 28)
 494                 | ((ext_acttopre & 0x3) << 24)
 495                 | ((ext_acttorw & 0x1) << 22)
 496                 | ((ext_refrec & 0x3F) << 16)
 497                 | ((ext_caslat & 0x3) << 12)
 498                 | ((ext_add_lat & 0x1) << 10)
 499                 | ((ext_wrrec & 0x1) << 8)
 500                 | ((cntl_adj & 0x7) << 0)
 501                 );
 502         debug("FSLDDR: timing_cfg_3 = 0x%08x\n", ddr->timing_cfg_3);
 503 }
 504
 505 /* DDR SDRAM Timing Configuration 1 (TIMING_CFG_1) */
 506 static void set_timing_cfg_1(const unsigned int ctrl_num,
 507                              fsl_ddr_cfg_regs_t *ddr,
 508                              const memctl_options_t *popts,
 509                              const common_timing_params_t *common_dimm,
 510                              unsigned int cas_latency)
 511 {
 512         /* Precharge-to-activate interval (tRP) */
 513         unsigned char pretoact_mclk;
 514         /* Activate to precharge interval (tRAS) */
 515         unsigned char acttopre_mclk;
 516         /*  Activate to read/write interval (tRCD) */
 517         unsigned char acttorw_mclk;
 518         /* CASLAT */
 519         unsigned char caslat_ctrl;
 520         /*  Refresh recovery time (tRFC) ; trfc_low */
 521         unsigned char refrec_ctrl;
 522         /* Last data to precharge minimum interval (tWR) */
 523         unsigned char wrrec_mclk;
 524         /* Activate-to-activate interval (tRRD) */
 525         unsigned char acttoact_mclk;
 526         /* Last write data pair to read command issue interval (tWTR) */
 527         unsigned char wrtord_mclk;
 528 #ifdef CONFIG_SYS_FSL_DDR4
 529         /* DDR4 supports 10, 12, 14, 16, 18, 20, 24 */
 530         static const u8 wrrec_table[] = {
 531                 10, 10, 10, 10, 10,
 532                 10, 10, 10, 10, 10,
 533                 12, 12, 14, 14, 16,
 534                 16, 18, 18, 20, 20,
 535                 24, 24, 24, 24};
 536 #else
 537         /* DDR_SDRAM_MODE doesn't support 9,11,13,15 */
 538         static const u8 wrrec_table[] = {
 539                 1, 2, 3, 4, 5, 6, 7, 8, 10, 10, 12, 12, 14, 14, 0, 0};
 540 #endif
 541
 542         pretoact_mclk = picos_to_mclk(ctrl_num, common_dimm->trp_ps);
 543         acttopre_mclk = picos_to_mclk(ctrl_num, common_dimm->tras_ps);
 544         acttorw_mclk = picos_to_mclk(ctrl_num, common_dimm->trcd_ps);
 545
 546         /*
 547          * Translate CAS Latency to a DDR controller field value:
 548          *
 549          *      CAS Lat DDR I   DDR II  Ctrl
 550          *      Clocks  SPD Bit SPD Bit Value
 551          *      ------- ------- ------- -----
 552          *      1.0     0               0001
 553          *      1.5     1               0010
 554          *      2.0     2       2       0011
 555          *      2.5     3               0100
 556          *      3.0     4       3       0101
 557          *      3.5     5               0110
 558          *      4.0             4       0111
 559          *      4.5                     1000
 560          *      5.0             5       1001
 561          */
 562 #if defined(CONFIG_SYS_FSL_DDR1)
 563         caslat_ctrl = (cas_latency + 1) & 0x07;
 564 #elif defined(CONFIG_SYS_FSL_DDR2)
 565         caslat_ctrl = 2 * cas_latency - 1;
 566 #else
 567         /*
 568          * if the CAS latency more than 8 cycle,
 569          * we need set extend bit for it at
 570          * TIMING_CFG_3[EXT_CASLAT]
 571          */
 572         if (fsl_ddr_get_version(ctrl_num) <= 0x40400)
 573                 caslat_ctrl = 2 * cas_latency - 1;
 574         else
 575                 caslat_ctrl = (cas_latency - 1) << 1;
 576 #endif
 577
 578 #ifdef CONFIG_SYS_FSL_DDR4
 579         refrec_ctrl = picos_to_mclk(ctrl_num, common_dimm->trfc1_ps) - 8;
 580         wrrec_mclk = picos_to_mclk(ctrl_num, common_dimm->twr_ps);
 581         acttoact_mclk = max(picos_to_mclk(ctrl_num, common_dimm->trrds_ps), 4U);
 582         wrtord_mclk = max(2U, picos_to_mclk(ctrl_num, 2500));
 583         if ((wrrec_mclk < 1) || (wrrec_mclk > 24))
 584                 printf("Error: WRREC doesn't support %d clocks\n", wrrec_mclk);
 585         else
 586                 wrrec_mclk = wrrec_table[wrrec_mclk - 1];
 587 #else
 588         refrec_ctrl = picos_to_mclk(ctrl_num, common_dimm->trfc_ps) - 8;
 589         wrrec_mclk = picos_to_mclk(ctrl_num, common_dimm->twr_ps);
 590         acttoact_mclk = picos_to_mclk(ctrl_num, common_dimm->trrd_ps);
 591         wrtord_mclk = picos_to_mclk(ctrl_num, common_dimm->twtr_ps);
 592         if ((wrrec_mclk < 1) || (wrrec_mclk > 16))
 593                 printf("Error: WRREC doesn't support %d clocks\n", wrrec_mclk);
 594         else
 595                 wrrec_mclk = wrrec_table[wrrec_mclk - 1];
 596 #endif
 597         if (popts->otf_burst_chop_en)
 598                 wrrec_mclk += 2;
 599
 600         /*
 601          * JEDEC has min requirement for tRRD
 602          */
 603 #if defined(CONFIG_SYS_FSL_DDR3)
 604         if (acttoact_mclk < 4)
 605                 acttoact_mclk = 4;
 606 #endif
 607         /*
 608          * JEDEC has some min requirements for tWTR
 609          */
 610 #if defined(CONFIG_SYS_FSL_DDR2)
 611         if (wrtord_mclk < 2)
 612                 wrtord_mclk = 2;
 613 #elif defined(CONFIG_SYS_FSL_DDR3)
 614         if (wrtord_mclk < 4)
 615                 wrtord_mclk = 4;
 616 #endif
 617         if (popts->otf_burst_chop_en)
 618                 wrtord_mclk += 2;
 619
 620         ddr->timing_cfg_1 = (0
 621                 | ((pretoact_mclk & 0x0F) << 28)
 622                 | ((acttopre_mclk & 0x0F) << 24)
 623                 | ((acttorw_mclk & 0xF) << 20)
 624                 | ((caslat_ctrl & 0xF) << 16)
 625                 | ((refrec_ctrl & 0xF) << 12)
 626                 | ((wrrec_mclk & 0x0F) << 8)
 627                 | ((acttoact_mclk & 0x0F) << 4)
 628                 | ((wrtord_mclk & 0x0F) << 0)
 629                 );
 630         debug("FSLDDR: timing_cfg_1 = 0x%08x\n", ddr->timing_cfg_1);
 631 }
 632
 633 /* DDR SDRAM Timing Configuration 2 (TIMING_CFG_2) */
 634 static void set_timing_cfg_2(const unsigned int ctrl_num,
 635                              fsl_ddr_cfg_regs_t *ddr,
 636                              const memctl_options_t *popts,
 637                              const common_timing_params_t *common_dimm,
 638                              unsigned int cas_latency,
 639                              unsigned int additive_latency)
 640 {
 641         /* Additive latency */
 642         unsigned char add_lat_mclk;
 643         /* CAS-to-preamble override */
 644         unsigned short cpo;
 645         /* Write latency */
 646         unsigned char wr_lat;
 647         /*  Read to precharge (tRTP) */
 648         unsigned char rd_to_pre;
 649         /* Write command to write data strobe timing adjustment */
 650         unsigned char wr_data_delay;
 651         /* Minimum CKE pulse width (tCKE) */
 652         unsigned char cke_pls;
 653         /* Window for four activates (tFAW) */
 654         unsigned short four_act;
 655 #ifdef CONFIG_SYS_FSL_DDR3
 656         const unsigned int mclk_ps = get_memory_clk_period_ps(ctrl_num);
 657 #endif
 658
 659         /* FIXME add check that this must be less than acttorw_mclk */
 660         add_lat_mclk = additive_latency;
 661         cpo = popts->cpo_override;
 662
 663 #if defined(CONFIG_SYS_FSL_DDR1)
 664         /*
 665          * This is a lie.  It should really be 1, but if it is
 666          * set to 1, bits overlap into the old controller's
 667          * otherwise unused ACSM field.  If we leave it 0, then
 668          * the HW will magically treat it as 1 for DDR 1.  Oh Yea.
 669          */
 670         wr_lat = 0;
 671 #elif defined(CONFIG_SYS_FSL_DDR2)
 672         wr_lat = cas_latency - 1;
 673 #else
 674         wr_lat = compute_cas_write_latency(ctrl_num);
 675 #endif
 676
 677 #ifdef CONFIG_SYS_FSL_DDR4
 678         rd_to_pre = picos_to_mclk(ctrl_num, 7500);
 679 #else
 680         rd_to_pre = picos_to_mclk(ctrl_num, common_dimm->trtp_ps);
 681 #endif
 682         /*
 683          * JEDEC has some min requirements for tRTP
 684          */
 685 #if defined(CONFIG_SYS_FSL_DDR2)
 686         if (rd_to_pre  < 2)
 687                 rd_to_pre  = 2;
 688 #elif defined(CONFIG_SYS_FSL_DDR3) || defined(CONFIG_SYS_FSL_DDR4)
 689         if (rd_to_pre < 4)
 690                 rd_to_pre = 4;
 691 #endif
 692         if (popts->otf_burst_chop_en)
 693                 rd_to_pre += 2; /* according to UM */
 694
 695         wr_data_delay = popts->write_data_delay;
 696 #ifdef CONFIG_SYS_FSL_DDR4
 697         cpo = 0;
 698         cke_pls = max(3U, picos_to_mclk(ctrl_num, 5000));
 699 #elif defined(CONFIG_SYS_FSL_DDR3)
 700         /*
 701          * cke pulse = max(3nCK, 7.5ns) for DDR3-800
 702          *             max(3nCK, 5.625ns) for DDR3-1066, 1333
 703          *             max(3nCK, 5ns) for DDR3-1600, 1866, 2133
 704          */
 705         cke_pls = max(3U, picos_to_mclk(ctrl_num, mclk_ps > 1870 ? 7500 :
 706                                         (mclk_ps > 1245 ? 5625 : 5000)));
 707 #else
 708         cke_pls = FSL_DDR_MIN_TCKE_PULSE_WIDTH_DDR;
 709 #endif
 710         four_act = picos_to_mclk(ctrl_num,
 711                                  popts->tfaw_window_four_activates_ps);
 712
 713         ddr->timing_cfg_2 = (0
 714                 | ((add_lat_mclk & 0xf) << 28)
 715                 | ((cpo & 0x1f) << 23)
 716                 | ((wr_lat & 0xf) << 19)
 717                 | (((wr_lat & 0x10) >> 4) << 18)
 718                 | ((rd_to_pre & RD_TO_PRE_MASK) << RD_TO_PRE_SHIFT)
 719                 | ((wr_data_delay & WR_DATA_DELAY_MASK) << WR_DATA_DELAY_SHIFT)
 720                 | ((cke_pls & 0x7) << 6)
 721                 | ((four_act & 0x3f) << 0)
 722                 );
 723         debug("FSLDDR: timing_cfg_2 = 0x%08x\n", ddr->timing_cfg_2);
 724 }
 725
 726 /* DDR SDRAM Register Control Word */
 727 static void set_ddr_sdram_rcw(const unsigned int ctrl_num,
 728                               fsl_ddr_cfg_regs_t *ddr,
 729                               const memctl_options_t *popts,
 730                               const common_timing_params_t *common_dimm)
 731 {
 732         unsigned int ddr_freq = get_ddr_freq(ctrl_num) / 1000000;
 733         unsigned int rc0a, rc0f;
 734
 735         if (common_dimm->all_dimms_registered &&
 736             !common_dimm->all_dimms_unbuffered) {
 737                 if (popts->rcw_override) {
 738                         ddr->ddr_sdram_rcw_1 = popts->rcw_1;
 739                         ddr->ddr_sdram_rcw_2 = popts->rcw_2;
 740                         ddr->ddr_sdram_rcw_3 = popts->rcw_3;
 741                 } else {
 742                         rc0a = ddr_freq > 3200 ? 0x7 :
 743                                (ddr_freq > 2933 ? 0x6 :
 744                                 (ddr_freq > 2666 ? 0x5 :
 745                                  (ddr_freq > 2400 ? 0x4 :
 746                                   (ddr_freq > 2133 ? 0x3 :
 747                                    (ddr_freq > 1866 ? 0x2 :
 748                                     (ddr_freq > 1600 ? 1 : 0))))));
 749                         rc0f = ddr_freq > 3200 ? 0x3 :
 750                                (ddr_freq > 2400 ? 0x2 :
 751                                 (ddr_freq > 2133 ? 0x1 : 0));
 752                         ddr->ddr_sdram_rcw_1 =
 753                                 common_dimm->rcw[0] << 28 | \
 754                                 common_dimm->rcw[1] << 24 | \
 755                                 common_dimm->rcw[2] << 20 | \
 756                                 common_dimm->rcw[3] << 16 | \
 757                                 common_dimm->rcw[4] << 12 | \
 758                                 common_dimm->rcw[5] << 8 | \
 759                                 common_dimm->rcw[6] << 4 | \
 760                                 common_dimm->rcw[7];
 761                         ddr->ddr_sdram_rcw_2 =
 762                                 common_dimm->rcw[8] << 28 | \
 763                                 common_dimm->rcw[9] << 24 | \
 764                                 rc0a << 20 | \
 765                                 common_dimm->rcw[11] << 16 | \
 766                                 common_dimm->rcw[12] << 12 | \
 767                                 common_dimm->rcw[13] << 8 | \
 768                                 common_dimm->rcw[14] << 4 | \
 769                                 rc0f;
 770                         ddr->ddr_sdram_rcw_3 =
 771                                 ((ddr_freq - 1260 + 19) / 20) << 8;
 772                 }
 773                 debug("FSLDDR: ddr_sdram_rcw_1 = 0x%08x\n",
 774                       ddr->ddr_sdram_rcw_1);
 775                 debug("FSLDDR: ddr_sdram_rcw_2 = 0x%08x\n",
 776                       ddr->ddr_sdram_rcw_2);
 777                 debug("FSLDDR: ddr_sdram_rcw_3 = 0x%08x\n",
 778                       ddr->ddr_sdram_rcw_3);
 779         }
 780 }
 781
 782 /* DDR SDRAM control configuration (DDR_SDRAM_CFG) */
 783 static void set_ddr_sdram_cfg(fsl_ddr_cfg_regs_t *ddr,
 784                                const memctl_options_t *popts,
 785                                const common_timing_params_t *common_dimm)
 786 {
 787         unsigned int mem_en;            /* DDR SDRAM interface logic enable */
 788         unsigned int sren;              /* Self refresh enable (during sleep) */
 789         unsigned int ecc_en;            /* ECC enable. */
 790         unsigned int rd_en;             /* Registered DIMM enable */
 791         unsigned int sdram_type;        /* Type of SDRAM */
 792         unsigned int dyn_pwr;           /* Dynamic power management mode */
 793         unsigned int dbw;               /* DRAM dta bus width */
 794         unsigned int eight_be = 0;      /* 8-beat burst enable, DDR2 is zero */
 795         unsigned int ncap = 0;          /* Non-concurrent auto-precharge */
 796         unsigned int threet_en;         /* Enable 3T timing */
 797         unsigned int twot_en;           /* Enable 2T timing */
 798         unsigned int ba_intlv_ctl;      /* Bank (CS) interleaving control */
 799         unsigned int x32_en = 0;        /* x32 enable */
 800         unsigned int pchb8 = 0;         /* precharge bit 8 enable */
 801         unsigned int hse;               /* Global half strength override */
 802         unsigned int acc_ecc_en = 0;    /* Accumulated ECC enable */
 803         unsigned int mem_halt = 0;      /* memory controller halt */
 804         unsigned int bi = 0;            /* Bypass initialization */
 805
 806         mem_en = 1;
 807         sren = popts->self_refresh_in_sleep;
 808         if (common_dimm->all_dimms_ecc_capable) {
 809                 /* Allow setting of ECC only if all DIMMs are ECC. */
 810                 ecc_en = popts->ecc_mode;
 811         } else {
 812                 ecc_en = 0;
 813         }
 814
 815         if (common_dimm->all_dimms_registered &&
 816             !common_dimm->all_dimms_unbuffered) {
 817                 rd_en = 1;
 818                 twot_en = 0;
 819         } else {
 820                 rd_en = 0;
 821                 twot_en = popts->twot_en;
 822         }
 823
 824         sdram_type = CONFIG_FSL_SDRAM_TYPE;
 825
 826         dyn_pwr = popts->dynamic_power;
 827         dbw = popts->data_bus_width;
 828         /* 8-beat burst enable DDR-III case
 829          * we must clear it when use the on-the-fly mode,
 830          * must set it when use the 32-bits bus mode.
 831          */
 832         if ((sdram_type == SDRAM_TYPE_DDR3) ||
 833             (sdram_type == SDRAM_TYPE_DDR4)) {
 834                 if (popts->burst_length == DDR_BL8)
 835                         eight_be = 1;
 836                 if (popts->burst_length == DDR_OTF)
 837                         eight_be = 0;
 838                 if (dbw == 0x1)
 839                         eight_be = 1;
 840         }
 841
 842         threet_en = popts->threet_en;
 843         ba_intlv_ctl = popts->ba_intlv_ctl;
 844         hse = popts->half_strength_driver_enable;
 845
 846         /* set when ddr bus width < 64 */
 847         acc_ecc_en = (dbw != 0 && ecc_en == 1) ? 1 : 0;
 848
 849         ddr->ddr_sdram_cfg = (0
 850                         | ((mem_en & 0x1) << 31)
 851                         | ((sren & 0x1) << 30)
 852                         | ((ecc_en & 0x1) << 29)
 853                         | ((rd_en & 0x1) << 28)
 854                         | ((sdram_type & 0x7) << 24)
 855                         | ((dyn_pwr & 0x1) << 21)
 856                         | ((dbw & 0x3) << 19)
 857                         | ((eight_be & 0x1) << 18)
 858                         | ((ncap & 0x1) << 17)
 859                         | ((threet_en & 0x1) << 16)
 860                         | ((twot_en & 0x1) << 15)
 861                         | ((ba_intlv_ctl & 0x7F) << 8)
 862                         | ((x32_en & 0x1) << 5)
 863                         | ((pchb8 & 0x1) << 4)
 864                         | ((hse & 0x1) << 3)
 865                         | ((acc_ecc_en & 0x1) << 2)
 866                         | ((mem_halt & 0x1) << 1)
 867                         | ((bi & 0x1) << 0)
 868                         );
 869         debug("FSLDDR: ddr_sdram_cfg = 0x%08x\n", ddr->ddr_sdram_cfg);
 870 }
 871
 872 /* DDR SDRAM control configuration 2 (DDR_SDRAM_CFG_2) */
 873 static void set_ddr_sdram_cfg_2(const unsigned int ctrl_num,
 874                                fsl_ddr_cfg_regs_t *ddr,
 875                                const memctl_options_t *popts,
 876                                const unsigned int unq_mrs_en)
 877 {
 878         unsigned int frc_sr = 0;        /* Force self refresh */
 879         unsigned int sr_ie = 0;         /* Self-refresh interrupt enable */
 880         unsigned int odt_cfg = 0;       /* ODT configuration */
 881         unsigned int num_pr;            /* Number of posted refreshes */
 882         unsigned int slow = 0;          /* DDR will be run less than 1250 */
 883         unsigned int x4_en = 0;         /* x4 DRAM enable */
 884         unsigned int obc_cfg;           /* On-The-Fly Burst Chop Cfg */
 885         unsigned int ap_en;             /* Address Parity Enable */
 886         unsigned int d_init;            /* DRAM data initialization */
 887         unsigned int rcw_en = 0;        /* Register Control Word Enable */
 888         unsigned int md_en = 0;         /* Mirrored DIMM Enable */
 889         unsigned int qd_en = 0;         /* quad-rank DIMM Enable */
 890         int i;
 891 #ifndef CONFIG_SYS_FSL_DDR4
 892         unsigned int dll_rst_dis = 1;   /* DLL reset disable */
 893         unsigned int dqs_cfg;           /* DQS configuration */
 894
 895         dqs_cfg = popts->dqs_config;
 896 #endif
 897         for (i = 0; i < CONFIG_CHIP_SELECTS_PER_CTRL; i++) {
 898                 if (popts->cs_local_opts[i].odt_rd_cfg
 899                         || popts->cs_local_opts[i].odt_wr_cfg) {
 900                         odt_cfg = SDRAM_CFG2_ODT_ONLY_READ;
 901                         break;
 902                 }
 903         }
 904         sr_ie = popts->self_refresh_interrupt_en;
 905         num_pr = popts->package_3ds + 1;
 906
 907         /*
 908          * 8572 manual says
 909          *     {TIMING_CFG_1[PRETOACT]
 910          *      + [DDR_SDRAM_CFG_2[NUM_PR]
 911          *        * ({EXT_REFREC || REFREC} + 8 + 2)]}
 912          *      << DDR_SDRAM_INTERVAL[REFINT]
 913          */
 914 #if defined(CONFIG_SYS_FSL_DDR3) || defined(CONFIG_SYS_FSL_DDR4)
 915         obc_cfg = popts->otf_burst_chop_en;
 916 #else
 917         obc_cfg = 0;
 918 #endif
 919
 920 #if (CONFIG_SYS_FSL_DDR_VER >= FSL_DDR_VER_4_7)
 921         slow = get_ddr_freq(ctrl_num) < 1249000000;
 922 #endif
 923
 924         if (popts->registered_dimm_en)
 925                 rcw_en = 1;
 926
 927         /* DDR4 can have address parity for UDIMM and discrete */
 928         if ((CONFIG_FSL_SDRAM_TYPE != SDRAM_TYPE_DDR4) &&
 929             (!popts->registered_dimm_en)) {
 930                 ap_en = 0;
 931         } else {
 932                 ap_en = popts->ap_en;
 933         }
 934
 935         x4_en = popts->x4_en ? 1 : 0;
 936
 937 #if defined(CONFIG_ECC_INIT_VIA_DDRCONTROLLER)
 938         /* Use the DDR controller to auto initialize memory. */
 939         d_init = popts->ecc_init_using_memctl;
 940         ddr->ddr_data_init = CONFIG_MEM_INIT_VALUE;
 941         debug("DDR: ddr_data_init = 0x%08x\n", ddr->ddr_data_init);
 942 #else
 943         /* Memory will be initialized via DMA, or not at all. */
 944         d_init = 0;
 945 #endif
 946
 947 #if defined(CONFIG_SYS_FSL_DDR3) || defined(CONFIG_SYS_FSL_DDR4)
 948         md_en = popts->mirrored_dimm;
 949 #endif
 950         qd_en = popts->quad_rank_present ? 1 : 0;
 951         ddr->ddr_sdram_cfg_2 = (0
 952                 | ((frc_sr & 0x1) << 31)
 953                 | ((sr_ie & 0x1) << 30)
 954 #ifndef CONFIG_SYS_FSL_DDR4
 955                 | ((dll_rst_dis & 0x1) << 29)
 956                 | ((dqs_cfg & 0x3) << 26)
 957 #endif
 958                 | ((odt_cfg & 0x3) << 21)
 959                 | ((num_pr & 0xf) << 12)
 960                 | ((slow & 1) << 11)
 961                 | (x4_en << 10)
 962                 | (qd_en << 9)
 963                 | (unq_mrs_en << 8)
 964                 | ((obc_cfg & 0x1) << 6)
 965                 | ((ap_en & 0x1) << 5)
 966                 | ((d_init & 0x1) << 4)
 967                 | ((rcw_en & 0x1) << 2)
 968                 | ((md_en & 0x1) << 0)
 969                 );
 970         debug("FSLDDR: ddr_sdram_cfg_2 = 0x%08x\n", ddr->ddr_sdram_cfg_2);
 971 }
 972
 973 #ifdef CONFIG_SYS_FSL_DDR4
 974 /* DDR SDRAM Mode configuration 2 (DDR_SDRAM_MODE_2) */
 975 static void set_ddr_sdram_mode_2(const unsigned int ctrl_num,
 976                                 fsl_ddr_cfg_regs_t *ddr,
 977                                 const memctl_options_t *popts,
 978                                 const common_timing_params_t *common_dimm,
 979                                 const unsigned int unq_mrs_en)
 980 {
 981         unsigned short esdmode2 = 0;    /* Extended SDRAM mode 2 */
 982         unsigned short esdmode3 = 0;    /* Extended SDRAM mode 3 */
 983         int i;
 984         unsigned int wr_crc = 0;        /* Disable */
 985         unsigned int rtt_wr = 0;        /* Rtt_WR - dynamic ODT off */
 986         unsigned int srt = 0;   /* self-refresh temerature, normal range */
 987         unsigned int cwl = compute_cas_write_latency(ctrl_num) - 9;
 988         unsigned int mpr = 0;   /* serial */
 989         unsigned int wc_lat;
 990         const unsigned int mclk_ps = get_memory_clk_period_ps(ctrl_num);
 991
 992         if (popts->rtt_override)
 993                 rtt_wr = popts->rtt_wr_override_value;
 994         else
 995                 rtt_wr = popts->cs_local_opts[0].odt_rtt_wr;
 996
 997         if (common_dimm->extended_op_srt)
 998                 srt = common_dimm->extended_op_srt;
 999
1000         esdmode2 = (0
1001                 | ((wr_crc & 0x1) << 12)
1002                 | ((rtt_wr & 0x3) << 9)
1003                 | ((srt & 0x3) << 6)
1004                 | ((cwl & 0x7) << 3));
1005
1006         if (mclk_ps >= 1250)
1007                 wc_lat = 0;
1008         else if (mclk_ps >= 833)
1009                 wc_lat = 1;
1010         else
1011                 wc_lat = 2;
1012
1013         esdmode3 = (0
1014                 | ((mpr & 0x3) << 11)
1015                 | ((wc_lat & 0x3) << 9));
1016
1017         ddr->ddr_sdram_mode_2 = (0
1018                                  | ((esdmode2 & 0xFFFF) << 16)
1019                                  | ((esdmode3 & 0xFFFF) << 0)
1020                                  );
1021         debug("FSLDDR: ddr_sdram_mode_2 = 0x%08x\n", ddr->ddr_sdram_mode_2);
1022
1023         if (unq_mrs_en) {       /* unique mode registers are supported */
1024                 for (i = 1; i < CONFIG_CHIP_SELECTS_PER_CTRL; i++) {
1025                         if (popts->rtt_override)
1026                                 rtt_wr = popts->rtt_wr_override_value;
1027                         else
1028                                 rtt_wr = popts->cs_local_opts[i].odt_rtt_wr;
1029
1030                         esdmode2 &= 0xF9FF;     /* clear bit 10, 9 */
1031                         esdmode2 |= (rtt_wr & 0x3) << 9;
1032                         switch (i) {
1033                         case 1:
1034                                 ddr->ddr_sdram_mode_4 = (0
1035                                         | ((esdmode2 & 0xFFFF) << 16)
1036                                         | ((esdmode3 & 0xFFFF) << 0)
1037                                         );
1038                                 break;
1039                         case 2:
1040                                 ddr->ddr_sdram_mode_6 = (0
1041                                         | ((esdmode2 & 0xFFFF) << 16)
1042                                         | ((esdmode3 & 0xFFFF) << 0)
1043                                         );
1044                                 break;
1045                         case 3:
1046                                 ddr->ddr_sdram_mode_8 = (0
1047                                         | ((esdmode2 & 0xFFFF) << 16)
1048                                         | ((esdmode3 & 0xFFFF) << 0)
1049                                         );
1050                                 break;
1051                         }
1052                 }
1053                 debug("FSLDDR: ddr_sdram_mode_4 = 0x%08x\n",
1054                       ddr->ddr_sdram_mode_4);
1055                 debug("FSLDDR: ddr_sdram_mode_6 = 0x%08x\n",
1056                       ddr->ddr_sdram_mode_6);
1057                 debug("FSLDDR: ddr_sdram_mode_8 = 0x%08x\n",
1058                       ddr->ddr_sdram_mode_8);
1059         }
1060 }
1061 #elif defined(CONFIG_SYS_FSL_DDR3)
1062 /* DDR SDRAM Mode configuration 2 (DDR_SDRAM_MODE_2) */
1063 static void set_ddr_sdram_mode_2(const unsigned int ctrl_num,
1064                                 fsl_ddr_cfg_regs_t *ddr,
1065                                 const memctl_options_t *popts,
1066                                 const common_timing_params_t *common_dimm,
1067                                 const unsigned int unq_mrs_en)
1068 {
1069         unsigned short esdmode2 = 0;    /* Extended SDRAM mode 2 */
1070         unsigned short esdmode3 = 0;    /* Extended SDRAM mode 3 */
1071         int i;
1072         unsigned int rtt_wr = 0;        /* Rtt_WR - dynamic ODT off */
1073         unsigned int srt = 0;   /* self-refresh temerature, normal range */
1074         unsigned int asr = 0;   /* auto self-refresh disable */
1075         unsigned int cwl = compute_cas_write_latency(ctrl_num) - 5;
1076         unsigned int pasr = 0;  /* partial array self refresh disable */
1077
1078         if (popts->rtt_override)
1079                 rtt_wr = popts->rtt_wr_override_value;
1080         else
1081                 rtt_wr = popts->cs_local_opts[0].odt_rtt_wr;
1082
1083         if (common_dimm->extended_op_srt)
1084                 srt = common_dimm->extended_op_srt;
1085
1086         esdmode2 = (0
1087                 | ((rtt_wr & 0x3) << 9)
1088                 | ((srt & 0x1) << 7)
1089                 | ((asr & 0x1) << 6)
1090                 | ((cwl & 0x7) << 3)
1091                 | ((pasr & 0x7) << 0));
1092         ddr->ddr_sdram_mode_2 = (0
1093                                  | ((esdmode2 & 0xFFFF) << 16)
1094                                  | ((esdmode3 & 0xFFFF) << 0)
1095                                  );
1096         debug("FSLDDR: ddr_sdram_mode_2 = 0x%08x\n", ddr->ddr_sdram_mode_2);
1097
1098         if (unq_mrs_en) {       /* unique mode registers are supported */
1099                 for (i = 1; i < CONFIG_CHIP_SELECTS_PER_CTRL; i++) {
1100                         if (popts->rtt_override)
1101                                 rtt_wr = popts->rtt_wr_override_value;
1102                         else
1103                                 rtt_wr = popts->cs_local_opts[i].odt_rtt_wr;
1104
1105                         esdmode2 &= 0xF9FF;     /* clear bit 10, 9 */
1106                         esdmode2 |= (rtt_wr & 0x3) << 9;
1107                         switch (i) {
1108                         case 1:
1109                                 ddr->ddr_sdram_mode_4 = (0
1110                                         | ((esdmode2 & 0xFFFF) << 16)
1111                                         | ((esdmode3 & 0xFFFF) << 0)
1112                                         );
1113                                 break;
1114                         case 2:
1115                                 ddr->ddr_sdram_mode_6 = (0
1116                                         | ((esdmode2 & 0xFFFF) << 16)
1117                                         | ((esdmode3 & 0xFFFF) << 0)
1118                                         );
1119                                 break;
1120                         case 3:
1121                                 ddr->ddr_sdram_mode_8 = (0
1122                                         | ((esdmode2 & 0xFFFF) << 16)
1123                                         | ((esdmode3 & 0xFFFF) << 0)
1124                                         );
1125                                 break;
1126                         }
1127                 }
1128                 debug("FSLDDR: ddr_sdram_mode_4 = 0x%08x\n",
1129                         ddr->ddr_sdram_mode_4);
1130                 debug("FSLDDR: ddr_sdram_mode_6 = 0x%08x\n",
1131                         ddr->ddr_sdram_mode_6);
1132                 debug("FSLDDR: ddr_sdram_mode_8 = 0x%08x\n",
1133                         ddr->ddr_sdram_mode_8);
1134         }
1135 }
1136
1137 #else /* for DDR2 and DDR1 */
1138 /* DDR SDRAM Mode configuration 2 (DDR_SDRAM_MODE_2) */
1139 static void set_ddr_sdram_mode_2(const unsigned int ctrl_num,
1140                                 fsl_ddr_cfg_regs_t *ddr,
1141                                 const memctl_options_t *popts,
1142                                 const common_timing_params_t *common_dimm,
1143                                 const unsigned int unq_mrs_en)
1144 {
1145         unsigned short esdmode2 = 0;    /* Extended SDRAM mode 2 */
1146         unsigned short esdmode3 = 0;    /* Extended SDRAM mode 3 */
1147
1148         ddr->ddr_sdram_mode_2 = (0
1149                                  | ((esdmode2 & 0xFFFF) << 16)
1150                                  | ((esdmode3 & 0xFFFF) << 0)
1151                                  );
1152         debug("FSLDDR: ddr_sdram_mode_2 = 0x%08x\n", ddr->ddr_sdram_mode_2);
1153 }
1154 #endif
1155
1156 #ifdef CONFIG_SYS_FSL_DDR4
1157 /* DDR SDRAM Mode configuration 9 (DDR_SDRAM_MODE_9) */
1158 static void set_ddr_sdram_mode_9(fsl_ddr_cfg_regs_t *ddr,
1159                                 const memctl_options_t *popts,
1160                                 const common_timing_params_t *common_dimm,
1161                                 const unsigned int unq_mrs_en)
1162 {
1163         int i;
1164         unsigned short esdmode4 = 0;    /* Extended SDRAM mode 4 */
1165         unsigned short esdmode5;        /* Extended SDRAM mode 5 */
1166         int rtt_park = 0;
1167         bool four_cs = false;
1168         const unsigned int mclk_ps = get_memory_clk_period_ps(0);
1169
1170 #if CONFIG_CHIP_SELECTS_PER_CTRL == 4
1171         if ((ddr->cs[0].config & SDRAM_CS_CONFIG_EN) &&
1172             (ddr->cs[1].config & SDRAM_CS_CONFIG_EN) &&
1173             (ddr->cs[2].config & SDRAM_CS_CONFIG_EN) &&
1174             (ddr->cs[3].config & SDRAM_CS_CONFIG_EN))
1175                 four_cs = true;
1176 #endif
1177         if (ddr->cs[0].config & SDRAM_CS_CONFIG_EN) {
1178                 esdmode5 = 0x00000500;  /* Data mask enable, RTT_PARK CS0 */
1179                 rtt_park = four_cs ? 0 : 1;
1180         } else {
1181                 esdmode5 = 0x00000400;  /* Data mask enabled */
1182         }
1183
1184         /*
1185          * For DDR3, set C/A latency if address parity is enabled.
1186          * For DDR4, set C/A latency for UDIMM only. For RDIMM the delay is
1187          * handled by register chip and RCW settings.
1188          */
1189         if ((ddr->ddr_sdram_cfg_2 & SDRAM_CFG2_AP_EN) &&
1190             ((CONFIG_FSL_SDRAM_TYPE != SDRAM_TYPE_DDR4) ||
1191              !popts->registered_dimm_en)) {
1192                 if (mclk_ps >= 935) {
1193                         /* for DDR4-1600/1866/2133 */
1194                         esdmode5 |= DDR_MR5_CA_PARITY_LAT_4_CLK;
1195                 } else if (mclk_ps >= 833) {
1196                         /* for DDR4-2400 */
1197                         esdmode5 |= DDR_MR5_CA_PARITY_LAT_5_CLK;
1198                 } else {
1199                         printf("parity: mclk_ps = %d not supported\n", mclk_ps);
1200                 }
1201         }
1202
1203         ddr->ddr_sdram_mode_9 = (0
1204                                  | ((esdmode4 & 0xffff) << 16)
1205                                  | ((esdmode5 & 0xffff) << 0)
1206                                 );
1207
1208         /* Normally only the first enabled CS use 0x500, others use 0x400
1209          * But when four chip-selects are all enabled, all mode registers
1210          * need 0x500 to park.
1211          */
1212
1213         debug("FSLDDR: ddr_sdram_mode_9 = 0x%08x\n", ddr->ddr_sdram_mode_9);
1214         if (unq_mrs_en) {       /* unique mode registers are supported */
1215                 for (i = 1; i < CONFIG_CHIP_SELECTS_PER_CTRL; i++) {
1216                         if (!rtt_park &&
1217                             (ddr->cs[i].config & SDRAM_CS_CONFIG_EN)) {
1218                                 esdmode5 |= 0x00000500; /* RTT_PARK */
1219                                 rtt_park = four_cs ? 0 : 1;
1220                         } else {
1221                                 esdmode5 = 0x00000400;
1222                         }
1223
1224                         if ((ddr->ddr_sdram_cfg_2 & SDRAM_CFG2_AP_EN) &&
1225                             ((CONFIG_FSL_SDRAM_TYPE != SDRAM_TYPE_DDR4) ||
1226                              !popts->registered_dimm_en)) {
1227                                 if (mclk_ps >= 935) {
1228                                         /* for DDR4-1600/1866/2133 */
1229                                         esdmode5 |= DDR_MR5_CA_PARITY_LAT_4_CLK;
1230                                 } else if (mclk_ps >= 833) {
1231                                         /* for DDR4-2400 */
1232                                         esdmode5 |= DDR_MR5_CA_PARITY_LAT_5_CLK;
1233                                 } else {
1234                                         printf("parity: mclk_ps = %d not supported\n",
1235                                                mclk_ps);
1236                                 }
1237                         }
1238
1239                         switch (i) {
1240                         case 1:
1241                                 ddr->ddr_sdram_mode_11 = (0
1242                                         | ((esdmode4 & 0xFFFF) << 16)
1243                                         | ((esdmode5 & 0xFFFF) << 0)
1244                                         );
1245                                 break;
1246                         case 2:
1247                                 ddr->ddr_sdram_mode_13 = (0
1248                                         | ((esdmode4 & 0xFFFF) << 16)
1249                                         | ((esdmode5 & 0xFFFF) << 0)
1250                                         );
1251                                 break;
1252                         case 3:
1253                                 ddr->ddr_sdram_mode_15 = (0
1254                                         | ((esdmode4 & 0xFFFF) << 16)
1255                                         | ((esdmode5 & 0xFFFF) << 0)
1256                                         );
1257                                 break;
1258                         }
1259                 }
1260                 debug("FSLDDR: ddr_sdram_mode_11 = 0x%08x\n",
1261                       ddr->ddr_sdram_mode_11);
1262                 debug("FSLDDR: ddr_sdram_mode_13 = 0x%08x\n",
1263                       ddr->ddr_sdram_mode_13);
1264                 debug("FSLDDR: ddr_sdram_mode_15 = 0x%08x\n",
1265                       ddr->ddr_sdram_mode_15);
1266         }
1267 }
1268
1269 /* DDR SDRAM Mode configuration 10 (DDR_SDRAM_MODE_10) */
1270 static void set_ddr_sdram_mode_10(const unsigned int ctrl_num,
1271                                 fsl_ddr_cfg_regs_t *ddr,
1272                                 const memctl_options_t *popts,
1273                                 const common_timing_params_t *common_dimm,
1274                                 const unsigned int unq_mrs_en)
1275 {
1276         int i;
1277         unsigned short esdmode6 = 0;    /* Extended SDRAM mode 6 */
1278         unsigned short esdmode7 = 0;    /* Extended SDRAM mode 7 */
1279         unsigned int tccdl_min = picos_to_mclk(ctrl_num, common_dimm->tccdl_ps);
1280
1281         esdmode6 = ((tccdl_min - 4) & 0x7) << 10;
1282
1283         if (popts->ddr_cdr2 & DDR_CDR2_VREF_RANGE_2)
1284                 esdmode6 |= 1 << 6;     /* Range 2 */
1285
1286         ddr->ddr_sdram_mode_10 = (0
1287                                  | ((esdmode6 & 0xffff) << 16)
1288                                  | ((esdmode7 & 0xffff) << 0)
1289                                 );
1290         debug("FSLDDR: ddr_sdram_mode_10 = 0x%08x\n", ddr->ddr_sdram_mode_10);
1291         if (unq_mrs_en) {       /* unique mode registers are supported */
1292                 for (i = 1; i < CONFIG_CHIP_SELECTS_PER_CTRL; i++) {
1293                         switch (i) {
1294                         case 1:
1295                                 ddr->ddr_sdram_mode_12 = (0
1296                                         | ((esdmode6 & 0xFFFF) << 16)
1297                                         | ((esdmode7 & 0xFFFF) << 0)
1298                                         );
1299                                 break;
1300                         case 2:
1301                                 ddr->ddr_sdram_mode_14 = (0
1302                                         | ((esdmode6 & 0xFFFF) << 16)
1303                                         | ((esdmode7 & 0xFFFF) << 0)
1304                                         );
1305                                 break;
1306                         case 3:
1307                                 ddr->ddr_sdram_mode_16 = (0
1308                                         | ((esdmode6 & 0xFFFF) << 16)
1309                                         | ((esdmode7 & 0xFFFF) << 0)
1310                                         );
1311                                 break;
1312                         }
1313                 }
1314                 debug("FSLDDR: ddr_sdram_mode_12 = 0x%08x\n",
1315                       ddr->ddr_sdram_mode_12);
1316                 debug("FSLDDR: ddr_sdram_mode_14 = 0x%08x\n",
1317                       ddr->ddr_sdram_mode_14);
1318                 debug("FSLDDR: ddr_sdram_mode_16 = 0x%08x\n",
1319                       ddr->ddr_sdram_mode_16);
1320         }
1321 }
1322
1323 #endif
1324
1325 /* DDR SDRAM Interval Configuration (DDR_SDRAM_INTERVAL) */
1326 static void set_ddr_sdram_interval(const unsigned int ctrl_num,
1327                                 fsl_ddr_cfg_regs_t *ddr,
1328                                 const memctl_options_t *popts,
1329                                 const common_timing_params_t *common_dimm)
1330 {
1331         unsigned int refint;    /* Refresh interval */
1332         unsigned int bstopre;   /* Precharge interval */
1333
1334         refint = picos_to_mclk(ctrl_num, common_dimm->refresh_rate_ps);
1335
1336         bstopre = popts->bstopre;
1337
1338         /* refint field used 0x3FFF in earlier controllers */
1339         ddr->ddr_sdram_interval = (0
1340                                    | ((refint & 0xFFFF) << 16)
1341                                    | ((bstopre & 0x3FFF) << 0)
1342                                    );
1343         debug("FSLDDR: ddr_sdram_interval = 0x%08x\n", ddr->ddr_sdram_interval);
1344 }
1345
1346 #ifdef CONFIG_SYS_FSL_DDR4
1347 /* DDR SDRAM Mode configuration set (DDR_SDRAM_MODE) */
1348 static void set_ddr_sdram_mode(const unsigned int ctrl_num,
1349                                fsl_ddr_cfg_regs_t *ddr,
1350                                const memctl_options_t *popts,
1351                                const common_timing_params_t *common_dimm,
1352                                unsigned int cas_latency,
1353                                unsigned int additive_latency,
1354                                const unsigned int unq_mrs_en)
1355 {
1356         int i;
1357         unsigned short esdmode;         /* Extended SDRAM mode */
1358         unsigned short sdmode;          /* SDRAM mode */
1359
1360         /* Mode Register - MR1 */
1361         unsigned int qoff = 0;          /* Output buffer enable 0=yes, 1=no */
1362         unsigned int tdqs_en = 0;       /* TDQS Enable: 0=no, 1=yes */
1363         unsigned int rtt;
1364         unsigned int wrlvl_en = 0;      /* Write level enable: 0=no, 1=yes */
1365         unsigned int al = 0;            /* Posted CAS# additive latency (AL) */
1366         unsigned int dic = 0;           /* Output driver impedance, 40ohm */
1367         unsigned int dll_en = 1;        /* DLL Enable  1=Enable (Normal),
1368                                                        0=Disable (Test/Debug) */
1369
1370         /* Mode Register - MR0 */
1371         unsigned int wr = 0;    /* Write Recovery */
1372         unsigned int dll_rst;   /* DLL Reset */
1373         unsigned int mode;      /* Normal=0 or Test=1 */
1374         unsigned int caslat = 4;/* CAS# latency, default set as 6 cycles */
1375         /* BT: Burst Type (0=Nibble Sequential, 1=Interleaved) */
1376         unsigned int bt;
1377         unsigned int bl;        /* BL: Burst Length */
1378
1379         unsigned int wr_mclk;
1380         /* DDR4 support WR 10, 12, 14, 16, 18, 20, 24 */
1381         static const u8 wr_table[] = {
1382                 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 6, 6};
1383         /* DDR4 support CAS 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 22, 24 */
1384         static const u8 cas_latency_table[] = {
1385                 0, 1, 2, 3, 4, 5, 6, 7, 8, 8,
1386                 9, 9, 10, 10, 11, 11};
1387
1388         if (popts->rtt_override)
1389                 rtt = popts->rtt_override_value;
1390         else
1391                 rtt = popts->cs_local_opts[0].odt_rtt_norm;
1392
1393         if (additive_latency == (cas_latency - 1))
1394                 al = 1;
1395         if (additive_latency == (cas_latency - 2))
1396                 al = 2;
1397
1398         if (popts->quad_rank_present)
1399                 dic = 1;        /* output driver impedance 240/7 ohm */
1400
1401         /*
1402          * The esdmode value will also be used for writing
1403          * MR1 during write leveling for DDR3, although the
1404          * bits specifically related to the write leveling
1405          * scheme will be handled automatically by the DDR
1406          * controller. so we set the wrlvl_en = 0 here.
1407          */
1408         esdmode = (0
1409                 | ((qoff & 0x1) << 12)
1410                 | ((tdqs_en & 0x1) << 11)
1411                 | ((rtt & 0x7) << 8)
1412                 | ((wrlvl_en & 0x1) << 7)
1413                 | ((al & 0x3) << 3)
1414                 | ((dic & 0x3) << 1)   /* DIC field is split */
1415                 | ((dll_en & 0x1) << 0)
1416                 );
1417
1418         /*
1419          * DLL control for precharge PD
1420          * 0=slow exit DLL off (tXPDLL)
1421          * 1=fast exit DLL on (tXP)
1422          */
1423
1424         wr_mclk = picos_to_mclk(ctrl_num, common_dimm->twr_ps);
1425         if (wr_mclk <= 24) {
1426                 wr = wr_table[wr_mclk - 10];
1427         } else {
1428                 printf("Error: unsupported write recovery for mode register wr_mclk = %d\n",
1429                        wr_mclk);
1430         }
1431
1432         dll_rst = 0;    /* dll no reset */
1433         mode = 0;       /* normal mode */
1434
1435         /* look up table to get the cas latency bits */
1436         if (cas_latency >= 9 && cas_latency <= 24)
1437                 caslat = cas_latency_table[cas_latency - 9];
1438         else
1439                 printf("Error: unsupported cas latency for mode register\n");
1440
1441         bt = 0; /* Nibble sequential */
1442
1443         switch (popts->burst_length) {
1444         case DDR_BL8:
1445                 bl = 0;
1446                 break;
1447         case DDR_OTF:
1448                 bl = 1;
1449                 break;
1450         case DDR_BC4:
1451                 bl = 2;
1452                 break;
1453         default:
1454                 printf("Error: invalid burst length of %u specified. ",
1455                        popts->burst_length);
1456                 puts("Defaulting to on-the-fly BC4 or BL8 beats.\n");
1457                 bl = 1;
1458                 break;
1459         }
1460
1461         sdmode = (0
1462                   | ((wr & 0x7) << 9)
1463                   | ((dll_rst & 0x1) << 8)
1464                   | ((mode & 0x1) << 7)
1465                   | (((caslat >> 1) & 0x7) << 4)
1466                   | ((bt & 0x1) << 3)
1467                   | ((caslat & 1) << 2)
1468                   | ((bl & 0x3) << 0)
1469                   );
1470
1471         ddr->ddr_sdram_mode = (0
1472                                | ((esdmode & 0xFFFF) << 16)
1473                                | ((sdmode & 0xFFFF) << 0)
1474                                );
1475
1476         debug("FSLDDR: ddr_sdram_mode = 0x%08x\n", ddr->ddr_sdram_mode);
1477
1478         if (unq_mrs_en) {       /* unique mode registers are supported */
1479                 for (i = 1; i < CONFIG_CHIP_SELECTS_PER_CTRL; i++) {
1480                         if (popts->rtt_override)
1481                                 rtt = popts->rtt_override_value;
1482                         else
1483                                 rtt = popts->cs_local_opts[i].odt_rtt_norm;
1484
1485                         esdmode &= 0xF8FF;      /* clear bit 10,9,8 for rtt */
1486                         esdmode |= (rtt & 0x7) << 8;
1487                         switch (i) {
1488                         case 1:
1489                                 ddr->ddr_sdram_mode_3 = (0
1490                                        | ((esdmode & 0xFFFF) << 16)
1491                                        | ((sdmode & 0xFFFF) << 0)
1492                                        );
1493                                 break;
1494                         case 2:
1495                                 ddr->ddr_sdram_mode_5 = (0
1496                                        | ((esdmode & 0xFFFF) << 16)
1497                                        | ((sdmode & 0xFFFF) << 0)
1498                                        );
1499                                 break;
1500                         case 3:
1501                                 ddr->ddr_sdram_mode_7 = (0
1502                                        | ((esdmode & 0xFFFF) << 16)
1503                                        | ((sdmode & 0xFFFF) << 0)
1504                                        );
1505                                 break;
1506                         }
1507                 }
1508                 debug("FSLDDR: ddr_sdram_mode_3 = 0x%08x\n",
1509                       ddr->ddr_sdram_mode_3);
1510                 debug("FSLDDR: ddr_sdram_mode_5 = 0x%08x\n",
1511                       ddr->ddr_sdram_mode_5);
1512                 debug("FSLDDR: ddr_sdram_mode_5 = 0x%08x\n",
1513                       ddr->ddr_sdram_mode_5);
1514         }
1515 }
1516
1517 #elif defined(CONFIG_SYS_FSL_DDR3)
1518 /* DDR SDRAM Mode configuration set (DDR_SDRAM_MODE) */
1519 static void set_ddr_sdram_mode(const unsigned int ctrl_num,
1520                                fsl_ddr_cfg_regs_t *ddr,
1521                                const memctl_options_t *popts,
1522                                const common_timing_params_t *common_dimm,
1523                                unsigned int cas_latency,
1524                                unsigned int additive_latency,
1525                                const unsigned int unq_mrs_en)
1526 {
1527         int i;
1528         unsigned short esdmode;         /* Extended SDRAM mode */
1529         unsigned short sdmode;          /* SDRAM mode */
1530
1531         /* Mode Register - MR1 */
1532         unsigned int qoff = 0;          /* Output buffer enable 0=yes, 1=no */
1533         unsigned int tdqs_en = 0;       /* TDQS Enable: 0=no, 1=yes */
1534         unsigned int rtt;
1535         unsigned int wrlvl_en = 0;      /* Write level enable: 0=no, 1=yes */
1536         unsigned int al = 0;            /* Posted CAS# additive latency (AL) */
1537         unsigned int dic = 0;           /* Output driver impedance, 40ohm */
1538         unsigned int dll_en = 0;        /* DLL Enable  0=Enable (Normal),
1539                                                        1=Disable (Test/Debug) */
1540
1541         /* Mode Register - MR0 */
1542         unsigned int dll_on;    /* DLL control for precharge PD, 0=off, 1=on */
1543         unsigned int wr = 0;    /* Write Recovery */
1544         unsigned int dll_rst;   /* DLL Reset */
1545         unsigned int mode;      /* Normal=0 or Test=1 */
1546         unsigned int caslat = 4;/* CAS# latency, default set as 6 cycles */
1547         /* BT: Burst Type (0=Nibble Sequential, 1=Interleaved) */
1548         unsigned int bt;
1549         unsigned int bl;        /* BL: Burst Length */
1550
1551         unsigned int wr_mclk;
1552         /*
1553          * DDR_SDRAM_MODE doesn't support 9,11,13,15
1554          * Please refer JEDEC Standard No. 79-3E for Mode Register MR0
1555          * for this table
1556          */
1557         static const u8 wr_table[] = {1, 2, 3, 4, 5, 5, 6, 6, 7, 7, 0, 0};
1558
1559         if (popts->rtt_override)
1560                 rtt = popts->rtt_override_value;
1561         else
1562                 rtt = popts->cs_local_opts[0].odt_rtt_norm;
1563
1564         if (additive_latency == (cas_latency - 1))
1565                 al = 1;
1566         if (additive_latency == (cas_latency - 2))
1567                 al = 2;
1568
1569         if (popts->quad_rank_present)
1570                 dic = 1;        /* output driver impedance 240/7 ohm */
1571
1572         /*
1573          * The esdmode value will also be used for writing
1574          * MR1 during write leveling for DDR3, although the
1575          * bits specifically related to the write leveling
1576          * scheme will be handled automatically by the DDR
1577          * controller. so we set the wrlvl_en = 0 here.
1578          */
1579         esdmode = (0
1580                 | ((qoff & 0x1) << 12)
1581                 | ((tdqs_en & 0x1) << 11)
1582                 | ((rtt & 0x4) << 7)   /* rtt field is split */
1583                 | ((wrlvl_en & 0x1) << 7)
1584                 | ((rtt & 0x2) << 5)   /* rtt field is split */
1585                 | ((dic & 0x2) << 4)   /* DIC field is split */
1586                 | ((al & 0x3) << 3)
1587                 | ((rtt & 0x1) << 2)  /* rtt field is split */
1588                 | ((dic & 0x1) << 1)   /* DIC field is split */
1589                 | ((dll_en & 0x1) << 0)
1590                 );
1591
1592         /*
1593          * DLL control for precharge PD
1594          * 0=slow exit DLL off (tXPDLL)
1595          * 1=fast exit DLL on (tXP)
1596          */
1597         dll_on = 1;
1598
1599         wr_mclk = picos_to_mclk(ctrl_num, common_dimm->twr_ps);
1600         if (wr_mclk <= 16) {
1601                 wr = wr_table[wr_mclk - 5];
1602         } else {
1603                 printf("Error: unsupported write recovery for mode register "
1604                        "wr_mclk = %d\n", wr_mclk);
1605         }
1606
1607         dll_rst = 0;    /* dll no reset */
1608         mode = 0;       /* normal mode */
1609
1610         /* look up table to get the cas latency bits */
1611         if (cas_latency >= 5 && cas_latency <= 16) {
1612                 unsigned char cas_latency_table[] = {
1613                         0x2,    /* 5 clocks */
1614                         0x4,    /* 6 clocks */
1615                         0x6,    /* 7 clocks */
1616                         0x8,    /* 8 clocks */
1617                         0xa,    /* 9 clocks */
1618                         0xc,    /* 10 clocks */
1619                         0xe,    /* 11 clocks */
1620                         0x1,    /* 12 clocks */
1621                         0x3,    /* 13 clocks */
1622                         0x5,    /* 14 clocks */
1623                         0x7,    /* 15 clocks */
1624                         0x9,    /* 16 clocks */
1625                 };
1626                 caslat = cas_latency_table[cas_latency - 5];
1627         } else {
1628                 printf("Error: unsupported cas latency for mode register\n");
1629         }
1630
1631         bt = 0; /* Nibble sequential */
1632
1633         switch (popts->burst_length) {
1634         case DDR_BL8:
1635                 bl = 0;
1636                 break;
1637         case DDR_OTF:
1638                 bl = 1;
1639                 break;
1640         case DDR_BC4:
1641                 bl = 2;
1642                 break;
1643         default:
1644                 printf("Error: invalid burst length of %u specified. "
1645                         " Defaulting to on-the-fly BC4 or BL8 beats.\n",
1646                         popts->burst_length);
1647                 bl = 1;
1648                 break;
1649         }
1650
1651         sdmode = (0
1652                   | ((dll_on & 0x1) << 12)
1653                   | ((wr & 0x7) << 9)
1654                   | ((dll_rst & 0x1) << 8)
1655                   | ((mode & 0x1) << 7)
1656                   | (((caslat >> 1) & 0x7) << 4)
1657                   | ((bt & 0x1) << 3)
1658                   | ((caslat & 1) << 2)
1659                   | ((bl & 0x3) << 0)
1660                   );
1661
1662         ddr->ddr_sdram_mode = (0
1663                                | ((esdmode & 0xFFFF) << 16)
1664                                | ((sdmode & 0xFFFF) << 0)
1665                                );
1666
1667         debug("FSLDDR: ddr_sdram_mode = 0x%08x\n", ddr->ddr_sdram_mode);
1668
1669         if (unq_mrs_en) {       /* unique mode registers are supported */
1670                 for (i = 1; i < CONFIG_CHIP_SELECTS_PER_CTRL; i++) {
1671                         if (popts->rtt_override)
1672                                 rtt = popts->rtt_override_value;
1673                         else
1674                                 rtt = popts->cs_local_opts[i].odt_rtt_norm;
1675
1676                         esdmode &= 0xFDBB;      /* clear bit 9,6,2 */
1677                         esdmode |= (0
1678                                 | ((rtt & 0x4) << 7)   /* rtt field is split */
1679                                 | ((rtt & 0x2) << 5)   /* rtt field is split */
1680                                 | ((rtt & 0x1) << 2)  /* rtt field is split */
1681                                 );
1682                         switch (i) {
1683                         case 1:
1684                                 ddr->ddr_sdram_mode_3 = (0
1685                                        | ((esdmode & 0xFFFF) << 16)
1686                                        | ((sdmode & 0xFFFF) << 0)
1687                                        );
1688                                 break;
1689                         case 2:
1690                                 ddr->ddr_sdram_mode_5 = (0
1691                                        | ((esdmode & 0xFFFF) << 16)
1692                                        | ((sdmode & 0xFFFF) << 0)
1693                                        );
1694                                 break;
1695                         case 3:
1696                                 ddr->ddr_sdram_mode_7 = (0
1697                                        | ((esdmode & 0xFFFF) << 16)
1698                                        | ((sdmode & 0xFFFF) << 0)
1699                                        );
1700                                 break;
1701                         }
1702                 }
1703                 debug("FSLDDR: ddr_sdram_mode_3 = 0x%08x\n",
1704                         ddr->ddr_sdram_mode_3);
1705                 debug("FSLDDR: ddr_sdram_mode_5 = 0x%08x\n",
1706                         ddr->ddr_sdram_mode_5);
1707                 debug("FSLDDR: ddr_sdram_mode_5 = 0x%08x\n",
1708                         ddr->ddr_sdram_mode_5);
1709         }
1710 }
1711
1712 #else /* !CONFIG_SYS_FSL_DDR3 */
1713
1714 /* DDR SDRAM Mode configuration set (DDR_SDRAM_MODE) */
1715 static void set_ddr_sdram_mode(const unsigned int ctrl_num,
1716                                fsl_ddr_cfg_regs_t *ddr,
1717                                const memctl_options_t *popts,
1718                                const common_timing_params_t *common_dimm,
1719                                unsigned int cas_latency,
1720                                unsigned int additive_latency,
1721                                const unsigned int unq_mrs_en)
1722 {
1723         unsigned short esdmode;         /* Extended SDRAM mode */
1724         unsigned short sdmode;          /* SDRAM mode */
1725
1726         /*
1727          * FIXME: This ought to be pre-calculated in a
1728          * technology-specific routine,
1729          * e.g. compute_DDR2_mode_register(), and then the
1730          * sdmode and esdmode passed in as part of common_dimm.
1731          */
1732
1733         /* Extended Mode Register */
1734         unsigned int mrs = 0;           /* Mode Register Set */
1735         unsigned int outputs = 0;       /* 0=Enabled, 1=Disabled */
1736         unsigned int rdqs_en = 0;       /* RDQS Enable: 0=no, 1=yes */
1737         unsigned int dqs_en = 0;        /* DQS# Enable: 0=enable, 1=disable */
1738         unsigned int ocd = 0;           /* 0x0=OCD not supported,
1739                                            0x7=OCD default state */
1740         unsigned int rtt;
1741         unsigned int al;                /* Posted CAS# additive latency (AL) */
1742         unsigned int ods = 0;           /* Output Drive Strength:
1743                                                 0 = Full strength (18ohm)
1744                                                 1 = Reduced strength (4ohm) */
1745         unsigned int dll_en = 0;        /* DLL Enable  0=Enable (Normal),
1746                                                        1=Disable (Test/Debug) */
1747
1748         /* Mode Register (MR) */
1749         unsigned int mr;        /* Mode Register Definition */
1750         unsigned int pd;        /* Power-Down Mode */
1751         unsigned int wr;        /* Write Recovery */
1752         unsigned int dll_res;   /* DLL Reset */
1753         unsigned int mode;      /* Normal=0 or Test=1 */
1754         unsigned int caslat = 0;/* CAS# latency */
1755         /* BT: Burst Type (0=Sequential, 1=Interleaved) */
1756         unsigned int bt;
1757         unsigned int bl;        /* BL: Burst Length */
1758
1759         dqs_en = !popts->dqs_config;
1760         rtt = fsl_ddr_get_rtt();
1761
1762         al = additive_latency;
1763
1764         esdmode = (0
1765                 | ((mrs & 0x3) << 14)
1766                 | ((outputs & 0x1) << 12)
1767                 | ((rdqs_en & 0x1) << 11)
1768                 | ((dqs_en & 0x1) << 10)
1769                 | ((ocd & 0x7) << 7)
1770                 | ((rtt & 0x2) << 5)   /* rtt field is split */
1771                 | ((al & 0x7) << 3)
1772                 | ((rtt & 0x1) << 2)   /* rtt field is split */
1773                 | ((ods & 0x1) << 1)
1774                 | ((dll_en & 0x1) << 0)
1775                 );
1776
1777         mr = 0;          /* FIXME: CHECKME */
1778
1779         /*
1780          * 0 = Fast Exit (Normal)
1781          * 1 = Slow Exit (Low Power)
1782          */
1783         pd = 0;
1784
1785 #if defined(CONFIG_SYS_FSL_DDR1)
1786         wr = 0;       /* Historical */
1787 #elif defined(CONFIG_SYS_FSL_DDR2)
1788         wr = picos_to_mclk(ctrl_num, common_dimm->twr_ps);
1789 #endif
1790         dll_res = 0;
1791         mode = 0;
1792
1793 #if defined(CONFIG_SYS_FSL_DDR1)
1794         if (1 <= cas_latency && cas_latency <= 4) {
1795                 unsigned char mode_caslat_table[4] = {
1796                         0x5,    /* 1.5 clocks */
1797                         0x2,    /* 2.0 clocks */
1798                         0x6,    /* 2.5 clocks */
1799                         0x3     /* 3.0 clocks */
1800                 };
1801                 caslat = mode_caslat_table[cas_latency - 1];
1802         } else {
1803                 printf("Warning: unknown cas_latency %d\n", cas_latency);
1804         }
1805 #elif defined(CONFIG_SYS_FSL_DDR2)
1806         caslat = cas_latency;
1807 #endif
1808         bt = 0;
1809
1810         switch (popts->burst_length) {
1811         case DDR_BL4:
1812                 bl = 2;
1813                 break;
1814         case DDR_BL8:
1815                 bl = 3;
1816                 break;
1817         default:
1818                 printf("Error: invalid burst length of %u specified. "
1819                         " Defaulting to 4 beats.\n",
1820                         popts->burst_length);
1821                 bl = 2;
1822                 break;
1823         }
1824
1825         sdmode = (0
1826                   | ((mr & 0x3) << 14)
1827                   | ((pd & 0x1) << 12)
1828                   | ((wr & 0x7) << 9)
1829                   | ((dll_res & 0x1) << 8)
1830                   | ((mode & 0x1) << 7)
1831                   | ((caslat & 0x7) << 4)
1832                   | ((bt & 0x1) << 3)
1833                   | ((bl & 0x7) << 0)
1834                   );
1835
1836         ddr->ddr_sdram_mode = (0
1837                                | ((esdmode & 0xFFFF) << 16)
1838                                | ((sdmode & 0xFFFF) << 0)
1839                                );
1840         debug("FSLDDR: ddr_sdram_mode = 0x%08x\n", ddr->ddr_sdram_mode);
1841 }
1842 #endif
1843
1844 /* DDR SDRAM Data Initialization (DDR_DATA_INIT) */
1845 static void set_ddr_data_init(fsl_ddr_cfg_regs_t *ddr)
1846 {
1847         unsigned int init_value;        /* Initialization value */
1848
1849 #ifdef CONFIG_MEM_INIT_VALUE
1850         init_value = CONFIG_MEM_INIT_VALUE;
1851 #else
1852         init_value = 0xDEADBEEF;
1853 #endif
1854         ddr->ddr_data_init = init_value;
1855 }
1856
1857 /*
1858  * DDR SDRAM Clock Control (DDR_SDRAM_CLK_CNTL)
1859  * The old controller on the 8540/60 doesn't have this register.
1860  * Hope it's OK to set it (to 0) anyway.
1861  */
1862 static void set_ddr_sdram_clk_cntl(fsl_ddr_cfg_regs_t *ddr,
1863                                          const memctl_options_t *popts)
1864 {
1865         unsigned int clk_adjust;        /* Clock adjust */
1866         unsigned int ss_en = 0;         /* Source synchronous enable */
1867
1868 #if defined(CONFIG_ARCH_MPC8541) || defined(CONFIG_ARCH_MPC8555)
1869         /* Per FSL Application Note: AN2805 */
1870         ss_en = 1;
1871 #endif
1872         if (fsl_ddr_get_version(0) >= 0x40701) {
1873                 /* clk_adjust in 5-bits on T-series and LS-series */
1874                 clk_adjust = (popts->clk_adjust & 0x1F) << 22;
1875         } else {
1876                 /* clk_adjust in 4-bits on earlier MPC85xx and P-series */
1877                 clk_adjust = (popts->clk_adjust & 0xF) << 23;
1878         }
1879
1880         ddr->ddr_sdram_clk_cntl = (0
1881                                    | ((ss_en & 0x1) << 31)
1882                                    | clk_adjust
1883                                    );
1884         debug("FSLDDR: clk_cntl = 0x%08x\n", ddr->ddr_sdram_clk_cntl);
1885 }
1886
1887 /* DDR Initialization Address (DDR_INIT_ADDR) */
1888 static void set_ddr_init_addr(fsl_ddr_cfg_regs_t *ddr)
1889 {
1890         unsigned int init_addr = 0;     /* Initialization address */
1891
1892         ddr->ddr_init_addr = init_addr;
1893 }
1894
1895 /* DDR Initialization Address (DDR_INIT_EXT_ADDR) */
1896 static void set_ddr_init_ext_addr(fsl_ddr_cfg_regs_t *ddr)
1897 {
1898         unsigned int uia = 0;   /* Use initialization address */
1899         unsigned int init_ext_addr = 0; /* Initialization address */
1900
1901         ddr->ddr_init_ext_addr = (0
1902                                   | ((uia & 0x1) << 31)
1903                                   | (init_ext_addr & 0xF)
1904                                   );
1905 }
1906
1907 /* DDR SDRAM Timing Configuration 4 (TIMING_CFG_4) */
1908 static void set_timing_cfg_4(fsl_ddr_cfg_regs_t *ddr,
1909                                 const memctl_options_t *popts)
1910 {
1911         unsigned int rwt = 0; /* Read-to-write turnaround for same CS */
1912         unsigned int wrt = 0; /* Write-to-read turnaround for same CS */
1913         unsigned int rrt = 0; /* Read-to-read turnaround for same CS */
1914         unsigned int wwt = 0; /* Write-to-write turnaround for same CS */
1915         unsigned int trwt_mclk = 0;     /* ext_rwt */
1916         unsigned int dll_lock = 0; /* DDR SDRAM DLL Lock Time */
1917
1918 #if defined(CONFIG_SYS_FSL_DDR3) || defined(CONFIG_SYS_FSL_DDR4)
1919         if (popts->burst_length == DDR_BL8) {
1920                 /* We set BL/2 for fixed BL8 */
1921                 rrt = 0;        /* BL/2 clocks */
1922                 wwt = 0;        /* BL/2 clocks */
1923         } else {
1924                 /* We need to set BL/2 + 2 to BC4 and OTF */
1925                 rrt = 2;        /* BL/2 + 2 clocks */
1926                 wwt = 2;        /* BL/2 + 2 clocks */
1927         }
1928 #endif
1929 #ifdef CONFIG_SYS_FSL_DDR4
1930         dll_lock = 2;   /* tDLLK = 1024 clocks */
1931 #elif defined(CONFIG_SYS_FSL_DDR3)
1932         dll_lock = 1;   /* tDLLK = 512 clocks from spec */
1933 #endif
1934
1935         if (popts->trwt_override)
1936                 trwt_mclk = popts->trwt;
1937
1938         ddr->timing_cfg_4 = (0
1939                              | ((rwt & 0xf) << 28)
1940                              | ((wrt & 0xf) << 24)
1941                              | ((rrt & 0xf) << 20)
1942                              | ((wwt & 0xf) << 16)
1943                              | ((trwt_mclk & 0xc) << 12)
1944                              | (dll_lock & 0x3)
1945                              );
1946         debug("FSLDDR: timing_cfg_4 = 0x%08x\n", ddr->timing_cfg_4);
1947 }
1948
1949 /* DDR SDRAM Timing Configuration 5 (TIMING_CFG_5) */
1950 static void set_timing_cfg_5(fsl_ddr_cfg_regs_t *ddr, unsigned int cas_latency)
1951 {
1952         unsigned int rodt_on = 0;       /* Read to ODT on */
1953         unsigned int rodt_off = 0;      /* Read to ODT off */
1954         unsigned int wodt_on = 0;       /* Write to ODT on */
1955         unsigned int wodt_off = 0;      /* Write to ODT off */
1956
1957 #if defined(CONFIG_SYS_FSL_DDR3) || defined(CONFIG_SYS_FSL_DDR4)
1958         unsigned int wr_lat = ((ddr->timing_cfg_2 & 0x00780000) >> 19) +
1959                               ((ddr->timing_cfg_2 & 0x00040000) >> 14);
1960         /* rodt_on = timing_cfg_1[caslat] - timing_cfg_2[wrlat] + 1 */
1961         if (cas_latency >= wr_lat)
1962                 rodt_on = cas_latency - wr_lat + 1;
1963         rodt_off = 4;   /*  4 clocks */
1964         wodt_on = 1;    /*  1 clocks */
1965         wodt_off = 4;   /*  4 clocks */
1966 #endif
1967
1968         ddr->timing_cfg_5 = (0
1969                              | ((rodt_on & 0x1f) << 24)
1970                              | ((rodt_off & 0x7) << 20)
1971                              | ((wodt_on & 0x1f) << 12)
1972                              | ((wodt_off & 0x7) << 8)
1973                              );
1974         debug("FSLDDR: timing_cfg_5 = 0x%08x\n", ddr->timing_cfg_5);
1975 }
1976
1977 #ifdef CONFIG_SYS_FSL_DDR4
1978 static void set_timing_cfg_6(fsl_ddr_cfg_regs_t *ddr)
1979 {
1980         unsigned int hs_caslat = 0;
1981         unsigned int hs_wrlat = 0;
1982         unsigned int hs_wrrec = 0;
1983         unsigned int hs_clkadj = 0;
1984         unsigned int hs_wrlvl_start = 0;
1985
1986         ddr->timing_cfg_6 = (0
1987                              | ((hs_caslat & 0x1f) << 24)
1988                              | ((hs_wrlat & 0x1f) << 19)
1989                              | ((hs_wrrec & 0x1f) << 12)
1990                              | ((hs_clkadj & 0x1f) << 6)
1991                              | ((hs_wrlvl_start & 0x1f) << 0)
1992                             );
1993         debug("FSLDDR: timing_cfg_6 = 0x%08x\n", ddr->timing_cfg_6);
1994 }
1995
1996 static void set_timing_cfg_7(const unsigned int ctrl_num,
1997                              fsl_ddr_cfg_regs_t *ddr,
1998                              const memctl_options_t *popts,
1999                              const common_timing_params_t *common_dimm)
2000 {
2001         unsigned int txpr, tcksre, tcksrx;
2002         unsigned int cke_rst, cksre, cksrx, par_lat = 0, cs_to_cmd;
2003         const unsigned int mclk_ps = get_memory_clk_period_ps(ctrl_num);
2004
2005         txpr = max(5U, picos_to_mclk(ctrl_num, common_dimm->trfc1_ps + 10000));
2006         tcksre = max(5U, picos_to_mclk(ctrl_num, 10000));
2007         tcksrx = max(5U, picos_to_mclk(ctrl_num, 10000));
2008
2009         if (ddr->ddr_sdram_cfg_2 & SDRAM_CFG2_AP_EN &&
2010             CONFIG_FSL_SDRAM_TYPE == SDRAM_TYPE_DDR4) {
2011                 /* for DDR4 only */
2012                 par_lat = (ddr->ddr_sdram_rcw_2 & 0xf) + 1;
2013                 debug("PAR_LAT = %u for mclk_ps = %d\n", par_lat, mclk_ps);
2014         }
2015
2016         cs_to_cmd = 0;
2017
2018         if (txpr <= 200)
2019                 cke_rst = 0;
2020         else if (txpr <= 256)
2021                 cke_rst = 1;
2022         else if (txpr <= 512)
2023                 cke_rst = 2;
2024         else
2025                 cke_rst = 3;
2026
2027         if (tcksre <= 19)
2028                 cksre = tcksre - 5;
2029         else
2030                 cksre = 15;
2031
2032         if (tcksrx <= 19)
2033                 cksrx = tcksrx - 5;
2034         else
2035                 cksrx = 15;
2036
2037         ddr->timing_cfg_7 = (0
2038                              | ((cke_rst & 0x3) << 28)
2039                              | ((cksre & 0xf) << 24)
2040                              | ((cksrx & 0xf) << 20)
2041                              | ((par_lat & 0xf) << 16)
2042                              | ((cs_to_cmd & 0xf) << 4)
2043                             );
2044         debug("FSLDDR: timing_cfg_7 = 0x%08x\n", ddr->timing_cfg_7);
2045 }
2046
2047 static void set_timing_cfg_8(const unsigned int ctrl_num,
2048                              fsl_ddr_cfg_regs_t *ddr,
2049                              const memctl_options_t *popts,
2050                              const common_timing_params_t *common_dimm,
2051                              unsigned int cas_latency)
2052 {
2053         int rwt_bg, wrt_bg, rrt_bg, wwt_bg;
2054         unsigned int acttoact_bg, wrtord_bg, pre_all_rec;
2055         int tccdl = picos_to_mclk(ctrl_num, common_dimm->tccdl_ps);
2056         int wr_lat = ((ddr->timing_cfg_2 & 0x00780000) >> 19) +
2057                       ((ddr->timing_cfg_2 & 0x00040000) >> 14);
2058
2059         rwt_bg = cas_latency + 2 + 4 - wr_lat;
2060         if (rwt_bg < tccdl)
2061                 rwt_bg = tccdl - rwt_bg;
2062         else
2063                 rwt_bg = 0;
2064
2065         wrt_bg = wr_lat + 4 + 1 - cas_latency;
2066         if (wrt_bg < tccdl)
2067                 wrt_bg = tccdl - wrt_bg;
2068         else
2069                 wrt_bg = 0;
2070
2071         if (popts->burst_length == DDR_BL8) {
2072                 rrt_bg = tccdl - 4;
2073                 wwt_bg = tccdl - 4;
2074         } else {
2075                 rrt_bg = tccdl - 2;
2076                 wwt_bg = tccdl - 2;
2077         }
2078
2079         acttoact_bg = picos_to_mclk(ctrl_num, common_dimm->trrdl_ps);
2080         wrtord_bg = max(4U, picos_to_mclk(ctrl_num, 7500));
2081         if (popts->otf_burst_chop_en)
2082                 wrtord_bg += 2;
2083
2084         pre_all_rec = 0;
2085
2086         ddr->timing_cfg_8 = (0
2087                              | ((rwt_bg & 0xf) << 28)
2088                              | ((wrt_bg & 0xf) << 24)
2089                              | ((rrt_bg & 0xf) << 20)
2090                              | ((wwt_bg & 0xf) << 16)
2091                              | ((acttoact_bg & 0xf) << 12)
2092                              | ((wrtord_bg & 0xf) << 8)
2093                              | ((pre_all_rec & 0x1f) << 0)
2094                             );
2095
2096         debug("FSLDDR: timing_cfg_8 = 0x%08x\n", ddr->timing_cfg_8);
2097 }
2098
2099 static void set_timing_cfg_9(const unsigned int ctrl_num,
2100                              fsl_ddr_cfg_regs_t *ddr,
2101                              const memctl_options_t *popts,
2102                              const common_timing_params_t *common_dimm)
2103 {
2104         unsigned int refrec_cid_mclk = 0;
2105         unsigned int acttoact_cid_mclk = 0;
2106
2107         if (popts->package_3ds) {
2108                 refrec_cid_mclk =
2109                         picos_to_mclk(ctrl_num, common_dimm->trfc_slr_ps);
2110                 acttoact_cid_mclk = 4U; /* tRRDS_slr */
2111         }
2112
2113         ddr->timing_cfg_9 = (refrec_cid_mclk & 0x3ff) << 16     |
2114                             (acttoact_cid_mclk & 0xf) << 8;
2115
2116         debug("FSLDDR: timing_cfg_9 = 0x%08x\n", ddr->timing_cfg_9);
2117 }
2118
2119 /* This function needs to be called after set_ddr_sdram_cfg() is called */
2120 static void set_ddr_dq_mapping(fsl_ddr_cfg_regs_t *ddr,
2121                                const dimm_params_t *dimm_params)
2122 {
2123         unsigned int acc_ecc_en = (ddr->ddr_sdram_cfg >> 2) & 0x1;
2124         int i;
2125
2126         for (i = 0; i < CONFIG_DIMM_SLOTS_PER_CTLR; i++) {
2127                 if (dimm_params[i].n_ranks)
2128                         break;
2129         }
2130         if (i >= CONFIG_DIMM_SLOTS_PER_CTLR) {
2131                 puts("DDR error: no DIMM found!\n");
2132                 return;
2133         }
2134
2135         ddr->dq_map_0 = ((dimm_params[i].dq_mapping[0] & 0x3F) << 26) |
2136                         ((dimm_params[i].dq_mapping[1] & 0x3F) << 20) |
2137                         ((dimm_params[i].dq_mapping[2] & 0x3F) << 14) |
2138                         ((dimm_params[i].dq_mapping[3] & 0x3F) << 8) |
2139                         ((dimm_params[i].dq_mapping[4] & 0x3F) << 2);
2140
2141         ddr->dq_map_1 = ((dimm_params[i].dq_mapping[5] & 0x3F) << 26) |
2142                         ((dimm_params[i].dq_mapping[6] & 0x3F) << 20) |
2143                         ((dimm_params[i].dq_mapping[7] & 0x3F) << 14) |
2144                         ((dimm_params[i].dq_mapping[10] & 0x3F) << 8) |
2145                         ((dimm_params[i].dq_mapping[11] & 0x3F) << 2);
2146
2147         ddr->dq_map_2 = ((dimm_params[i].dq_mapping[12] & 0x3F) << 26) |
2148                         ((dimm_params[i].dq_mapping[13] & 0x3F) << 20) |
2149                         ((dimm_params[i].dq_mapping[14] & 0x3F) << 14) |
2150                         ((dimm_params[i].dq_mapping[15] & 0x3F) << 8) |
2151                         ((dimm_params[i].dq_mapping[16] & 0x3F) << 2);
2152
2153         /* dq_map for ECC[4:7] is set to 0 if accumulated ECC is enabled */
2154         ddr->dq_map_3 = ((dimm_params[i].dq_mapping[17] & 0x3F) << 26) |
2155                         ((dimm_params[i].dq_mapping[8] & 0x3F) << 20) |
2156                         (acc_ecc_en ? 0 :
2157                          (dimm_params[i].dq_mapping[9] & 0x3F) << 14) |
2158                         dimm_params[i].dq_mapping_ors;
2159
2160         debug("FSLDDR: dq_map_0 = 0x%08x\n", ddr->dq_map_0);
2161         debug("FSLDDR: dq_map_1 = 0x%08x\n", ddr->dq_map_1);
2162         debug("FSLDDR: dq_map_2 = 0x%08x\n", ddr->dq_map_2);
2163         debug("FSLDDR: dq_map_3 = 0x%08x\n", ddr->dq_map_3);
2164 }
2165 static void set_ddr_sdram_cfg_3(fsl_ddr_cfg_regs_t *ddr,
2166                                const memctl_options_t *popts)
2167 {
2168         int rd_pre;
2169
2170         rd_pre = popts->quad_rank_present ? 1 : 0;
2171
2172         ddr->ddr_sdram_cfg_3 = (rd_pre & 0x1) << 16;
2173         /* Disable MRS on parity error for RDIMMs */
2174         ddr->ddr_sdram_cfg_3 |= popts->registered_dimm_en ? 1 : 0;
2175
2176         if (popts->package_3ds) {       /* only 2,4,8 are supported */
2177                 if ((popts->package_3ds + 1) & 0x1) {
2178                         printf("Error: Unsupported 3DS DIMM with %d die\n",
2179                                popts->package_3ds + 1);
2180                 } else {
2181                         ddr->ddr_sdram_cfg_3 |= ((popts->package_3ds + 1) >> 1)
2182                                                 << 4;
2183                 }
2184         }
2185
2186         debug("FSLDDR: ddr_sdram_cfg_3 = 0x%08x\n", ddr->ddr_sdram_cfg_3);
2187 }
2188 #endif  /* CONFIG_SYS_FSL_DDR4 */
2189
2190 /* DDR ZQ Calibration Control (DDR_ZQ_CNTL) */
2191 static void set_ddr_zq_cntl(fsl_ddr_cfg_regs_t *ddr, unsigned int zq_en)
2192 {
2193         unsigned int zqinit = 0;/* POR ZQ Calibration Time (tZQinit) */
2194         /* Normal Operation Full Calibration Time (tZQoper) */
2195         unsigned int zqoper = 0;
2196         /* Normal Operation Short Calibration Time (tZQCS) */
2197         unsigned int zqcs = 0;
2198 #ifdef CONFIG_SYS_FSL_DDR4
2199         unsigned int zqcs_init;
2200 #endif
2201
2202         if (zq_en) {
2203 #ifdef CONFIG_SYS_FSL_DDR4
2204                 zqinit = 10;    /* 1024 clocks */
2205                 zqoper = 9;     /* 512 clocks */
2206                 zqcs = 7;       /* 128 clocks */
2207                 zqcs_init = 5;  /* 1024 refresh sequences */
2208 #else
2209                 zqinit = 9;     /* 512 clocks */
2210                 zqoper = 8;     /* 256 clocks */
2211                 zqcs = 6;       /* 64 clocks */
2212 #endif
2213         }
2214
2215         ddr->ddr_zq_cntl = (0
2216                             | ((zq_en & 0x1) << 31)
2217                             | ((zqinit & 0xF) << 24)
2218                             | ((zqoper & 0xF) << 16)
2219                             | ((zqcs & 0xF) << 8)
2220 #ifdef CONFIG_SYS_FSL_DDR4
2221                             | ((zqcs_init & 0xF) << 0)
2222 #endif
2223                             );
2224         debug("FSLDDR: zq_cntl = 0x%08x\n", ddr->ddr_zq_cntl);
2225 }
2226
2227 /* DDR Write Leveling Control (DDR_WRLVL_CNTL) */
2228 static void set_ddr_wrlvl_cntl(fsl_ddr_cfg_regs_t *ddr, unsigned int wrlvl_en,
2229                                 const memctl_options_t *popts)
2230 {
2231         /*
2232          * First DQS pulse rising edge after margining mode
2233          * is programmed (tWL_MRD)
2234          */
2235         unsigned int wrlvl_mrd = 0;
2236         /* ODT delay after margining mode is programmed (tWL_ODTEN) */
2237         unsigned int wrlvl_odten = 0;
2238         /* DQS/DQS_ delay after margining mode is programmed (tWL_DQSEN) */
2239         unsigned int wrlvl_dqsen = 0;
2240         /* WRLVL_SMPL: Write leveling sample time */
2241         unsigned int wrlvl_smpl = 0;
2242         /* WRLVL_WLR: Write leveling repeition time */
2243         unsigned int wrlvl_wlr = 0;
2244         /* WRLVL_START: Write leveling start time */
2245         unsigned int wrlvl_start = 0;
2246
2247         /* suggest enable write leveling for DDR3 due to fly-by topology */
2248         if (wrlvl_en) {
2249                 /* tWL_MRD min = 40 nCK, we set it 64 */
2250                 wrlvl_mrd = 0x6;
2251                 /* tWL_ODTEN 128 */
2252                 wrlvl_odten = 0x7;
2253                 /* tWL_DQSEN min = 25 nCK, we set it 32 */
2254                 wrlvl_dqsen = 0x5;
2255                 /*
2256                  * Write leveling sample time at least need 6 clocks
2257                  * higher than tWLO to allow enough time for progagation
2258                  * delay and sampling the prime data bits.
2259                  */
2260                 wrlvl_smpl = 0xf;
2261                 /*
2262                  * Write leveling repetition time
2263                  * at least tWLO + 6 clocks clocks
2264                  * we set it 64
2265                  */
2266                 wrlvl_wlr = 0x6;
2267                 /*
2268                  * Write leveling start time
2269                  * The value use for the DQS_ADJUST for the first sample
2270                  * when write leveling is enabled. It probably needs to be
2271                  * overridden per platform.
2272                  */
2273                 wrlvl_start = 0x8;
2274                 /*
2275                  * Override the write leveling sample and start time
2276                  * according to specific board
2277                  */
2278                 if (popts->wrlvl_override) {
2279                         wrlvl_smpl = popts->wrlvl_sample;
2280                         wrlvl_start = popts->wrlvl_start;
2281                 }
2282         }
2283
2284         ddr->ddr_wrlvl_cntl = (0
2285                                | ((wrlvl_en & 0x1) << 31)
2286                                | ((wrlvl_mrd & 0x7) << 24)
2287                                | ((wrlvl_odten & 0x7) << 20)
2288                                | ((wrlvl_dqsen & 0x7) << 16)
2289                                | ((wrlvl_smpl & 0xf) << 12)
2290                                | ((wrlvl_wlr & 0x7) << 8)
2291                                | ((wrlvl_start & 0x1F) << 0)
2292                                );
2293         debug("FSLDDR: wrlvl_cntl = 0x%08x\n", ddr->ddr_wrlvl_cntl);
2294         ddr->ddr_wrlvl_cntl_2 = popts->wrlvl_ctl_2;
2295         debug("FSLDDR: wrlvl_cntl_2 = 0x%08x\n", ddr->ddr_wrlvl_cntl_2);
2296         ddr->ddr_wrlvl_cntl_3 = popts->wrlvl_ctl_3;
2297         debug("FSLDDR: wrlvl_cntl_3 = 0x%08x\n", ddr->ddr_wrlvl_cntl_3);
2298
2299 }
2300
2301 /* DDR Self Refresh Counter (DDR_SR_CNTR) */
2302 static void set_ddr_sr_cntr(fsl_ddr_cfg_regs_t *ddr, unsigned int sr_it)
2303 {
2304         /* Self Refresh Idle Threshold */
2305         ddr->ddr_sr_cntr = (sr_it & 0xF) << 16;
2306 }
2307
2308 static void set_ddr_eor(fsl_ddr_cfg_regs_t *ddr, const memctl_options_t *popts)
2309 {
2310         if (popts->addr_hash) {
2311                 ddr->ddr_eor = 0x40000000;      /* address hash enable */
2312                 puts("Address hashing enabled.\n");
2313         }
2314 }
2315
2316 static void set_ddr_cdr1(fsl_ddr_cfg_regs_t *ddr, const memctl_options_t *popts)
2317 {
2318         ddr->ddr_cdr1 = popts->ddr_cdr1;
2319         debug("FSLDDR: ddr_cdr1 = 0x%08x\n", ddr->ddr_cdr1);
2320 }
2321
2322 static void set_ddr_cdr2(fsl_ddr_cfg_regs_t *ddr, const memctl_options_t *popts)
2323 {
2324         ddr->ddr_cdr2 = popts->ddr_cdr2;
2325         debug("FSLDDR: ddr_cdr2 = 0x%08x\n", ddr->ddr_cdr2);
2326 }
2327
2328 unsigned int
2329 check_fsl_memctl_config_regs(const fsl_ddr_cfg_regs_t *ddr)
2330 {
2331         unsigned int res = 0;
2332
2333         /*
2334          * Check that DDR_SDRAM_CFG[RD_EN] and DDR_SDRAM_CFG[2T_EN] are
2335          * not set at the same time.
2336          */
2337         if (ddr->ddr_sdram_cfg & 0x10000000
2338             && ddr->ddr_sdram_cfg & 0x00008000) {
2339                 printf("Error: DDR_SDRAM_CFG[RD_EN] and DDR_SDRAM_CFG[2T_EN] "
2340                                 " should not be set at the same time.\n");
2341                 res++;
2342         }
2343
2344         return res;
2345 }
2346
2347 unsigned int
2348 compute_fsl_memctl_config_regs(const unsigned int ctrl_num,
2349                                const memctl_options_t *popts,
2350                                fsl_ddr_cfg_regs_t *ddr,
2351                                const common_timing_params_t *common_dimm,
2352                                const dimm_params_t *dimm_params,
2353                                unsigned int dbw_cap_adj,
2354                                unsigned int size_only)
2355 {
2356         unsigned int i;
2357         unsigned int cas_latency;
2358         unsigned int additive_latency;
2359         unsigned int sr_it;
2360         unsigned int zq_en;
2361         unsigned int wrlvl_en;
2362         unsigned int ip_rev = 0;
2363         unsigned int unq_mrs_en = 0;
2364         int cs_en = 1;
2365 #ifdef CONFIG_SYS_FSL_ERRATUM_A009942
2366         unsigned int ddr_freq;
2367 #endif
2368 #if (defined(CONFIG_SYS_FSL_ERRATUM_A008378) && \
2369         defined(CONFIG_SYS_FSL_DDRC_GEN4)) || \
2370         defined(CONFIG_SYS_FSL_ERRATUM_A009942)
2371         struct ccsr_ddr __iomem *ddrc;
2372
2373         switch (ctrl_num) {
2374         case 0:
2375                 ddrc = (void *)CONFIG_SYS_FSL_DDR_ADDR;
2376                 break;
2377 #if defined(CONFIG_SYS_FSL_DDR2_ADDR) && (CONFIG_SYS_NUM_DDR_CTLRS > 1)
2378         case 1:
2379                 ddrc = (void *)CONFIG_SYS_FSL_DDR2_ADDR;
2380                 break;
2381 #endif
2382 #if defined(CONFIG_SYS_FSL_DDR3_ADDR) && (CONFIG_SYS_NUM_DDR_CTLRS > 2)
2383         case 2:
2384                 ddrc = (void *)CONFIG_SYS_FSL_DDR3_ADDR;
2385                 break;
2386 #endif
2387 #if defined(CONFIG_SYS_FSL_DDR4_ADDR) && (CONFIG_SYS_NUM_DDR_CTLRS > 3)
2388         case 3:
2389                 ddrc = (void *)CONFIG_SYS_FSL_DDR4_ADDR;
2390                 break;
2391 #endif
2392         default:
2393                 printf("%s unexpected ctrl_num = %u\n", __func__, ctrl_num);
2394                 return 1;
2395         }
2396 #endif
2397
2398         memset(ddr, 0, sizeof(fsl_ddr_cfg_regs_t));
2399
2400         if (common_dimm == NULL) {
2401                 printf("Error: subset DIMM params struct null pointer\n");
2402                 return 1;
2403         }
2404
2405         /*
2406          * Process overrides first.
2407          *
2408          * FIXME: somehow add dereated caslat to this
2409          */
2410         cas_latency = (popts->cas_latency_override)
2411                 ? popts->cas_latency_override_value
2412                 : common_dimm->lowest_common_spd_caslat;
2413
2414         additive_latency = (popts->additive_latency_override)
2415                 ? popts->additive_latency_override_value
2416                 : common_dimm->additive_latency;
2417
2418         sr_it = (popts->auto_self_refresh_en)
2419                 ? popts->sr_it
2420                 : 0;
2421         /* ZQ calibration */
2422         zq_en = (popts->zq_en) ? 1 : 0;
2423         /* write leveling */
2424         wrlvl_en = (popts->wrlvl_en) ? 1 : 0;
2425
2426         /* Chip Select Memory Bounds (CSn_BNDS) */
2427         for (i = 0; i < CONFIG_CHIP_SELECTS_PER_CTRL; i++) {
2428                 unsigned long long ea, sa;
2429                 unsigned int cs_per_dimm
2430                         = CONFIG_CHIP_SELECTS_PER_CTRL / CONFIG_DIMM_SLOTS_PER_CTLR;
2431                 unsigned int dimm_number
2432                         = i / cs_per_dimm;
2433                 unsigned long long rank_density
2434                         = dimm_params[dimm_number].rank_density >> dbw_cap_adj;
2435
2436                 if (dimm_params[dimm_number].n_ranks == 0) {
2437                         debug("Skipping setup of CS%u "
2438                                 "because n_ranks on DIMM %u is 0\n", i, dimm_number);
2439                         continue;
2440                 }
2441                 if (popts->memctl_interleaving) {
2442                         switch (popts->ba_intlv_ctl & FSL_DDR_CS0_CS1_CS2_CS3) {
2443                         case FSL_DDR_CS0_CS1_CS2_CS3:
2444                                 break;
2445                         case FSL_DDR_CS0_CS1:
2446                         case FSL_DDR_CS0_CS1_AND_CS2_CS3:
2447                                 if (i > 1)
2448                                         cs_en = 0;
2449                                 break;
2450                         case FSL_DDR_CS2_CS3:
2451                         default:
2452                                 if (i > 0)
2453                                         cs_en = 0;
2454                                 break;
2455                         }
2456                         sa = common_dimm->base_address;
2457                         ea = sa + common_dimm->total_mem - 1;
2458                 } else if (!popts->memctl_interleaving) {
2459                         /*
2460                          * If memory interleaving between controllers is NOT
2461                          * enabled, the starting address for each memory
2462                          * controller is distinct.  However, because rank
2463                          * interleaving is enabled, the starting and ending
2464                          * addresses of the total memory on that memory
2465                          * controller needs to be programmed into its
2466                          * respective CS0_BNDS.
2467                          */
2468                         switch (popts->ba_intlv_ctl & FSL_DDR_CS0_CS1_CS2_CS3) {
2469                         case FSL_DDR_CS0_CS1_CS2_CS3:
2470                                 sa = common_dimm->base_address;
2471                                 ea = sa + common_dimm->total_mem - 1;
2472                                 break;
2473                         case FSL_DDR_CS0_CS1_AND_CS2_CS3:
2474                                 if ((i >= 2) && (dimm_number == 0)) {
2475                                         sa = dimm_params[dimm_number].base_address +
2476                                               2 * rank_density;
2477                                         ea = sa + 2 * rank_density - 1;
2478                                 } else {
2479                                         sa = dimm_params[dimm_number].base_address;
2480                                         ea = sa + 2 * rank_density - 1;
2481                                 }
2482                                 break;
2483                         case FSL_DDR_CS0_CS1:
2484                                 if (dimm_params[dimm_number].n_ranks > (i % cs_per_dimm)) {
2485                                         sa = dimm_params[dimm_number].base_address;
2486                                         ea = sa + rank_density - 1;
2487                                         if (i != 1)
2488                                                 sa += (i % cs_per_dimm) * rank_density;
2489                                         ea += (i % cs_per_dimm) * rank_density;
2490                                 } else {
2491                                         sa = 0;
2492                                         ea = 0;
2493                                 }
2494                                 if (i == 0)
2495                                         ea += rank_density;
2496                                 break;
2497                         case FSL_DDR_CS2_CS3:
2498                                 if (dimm_params[dimm_number].n_ranks > (i % cs_per_dimm)) {
2499                                         sa = dimm_params[dimm_number].base_address;
2500                                         ea = sa + rank_density - 1;
2501                                         if (i != 3)
2502                                                 sa += (i % cs_per_dimm) * rank_density;
2503                                         ea += (i % cs_per_dimm) * rank_density;
2504                                 } else {
2505                                         sa = 0;
2506                                         ea = 0;
2507                                 }
2508                                 if (i == 2)
2509                                         ea += (rank_density >> dbw_cap_adj);
2510                                 break;
2511                         default:  /* No bank(chip-select) interleaving */
2512                                 sa = dimm_params[dimm_number].base_address;
2513                                 ea = sa + rank_density - 1;
2514                                 if (dimm_params[dimm_number].n_ranks > (i % cs_per_dimm)) {
2515                                         sa += (i % cs_per_dimm) * rank_density;
2516                                         ea += (i % cs_per_dimm) * rank_density;
2517                                 } else {
2518                                         sa = 0;
2519                                         ea = 0;
2520                                 }
2521                                 break;
2522                         }
2523                 }
2524
2525                 sa >>= 24;
2526                 ea >>= 24;
2527
2528                 if (cs_en) {
2529                         ddr->cs[i].bnds = (0
2530                                 | ((sa & 0xffff) << 16) /* starting address */
2531                                 | ((ea & 0xffff) << 0)  /* ending address */
2532                                 );
2533                 } else {
2534                         /* setting bnds to 0xffffffff for inactive CS */
2535                         ddr->cs[i].bnds = 0xffffffff;
2536                 }
2537
2538                 debug("FSLDDR: cs[%d]_bnds = 0x%08x\n", i, ddr->cs[i].bnds);
2539                 set_csn_config(dimm_number, i, ddr, popts, dimm_params);
2540                 set_csn_config_2(i, ddr);
2541         }
2542
2543         /*
2544          * In the case we only need to compute the ddr sdram size, we only need
2545          * to set csn registers, so return from here.
2546          */
2547         if (size_only)
2548                 return 0;
2549
2550         set_ddr_eor(ddr, popts);
2551
2552 #if !defined(CONFIG_SYS_FSL_DDR1)
2553         set_timing_cfg_0(ctrl_num, ddr, popts, dimm_params);
2554 #endif
2555
2556         set_timing_cfg_3(ctrl_num, ddr, popts, common_dimm, cas_latency,
2557                          additive_latency);
2558         set_timing_cfg_1(ctrl_num, ddr, popts, common_dimm, cas_latency);
2559         set_timing_cfg_2(ctrl_num, ddr, popts, common_dimm,
2560                          cas_latency, additive_latency);
2561
2562         set_ddr_cdr1(ddr, popts);
2563         set_ddr_cdr2(ddr, popts);
2564         set_ddr_sdram_cfg(ddr, popts, common_dimm);
2565         ip_rev = fsl_ddr_get_version(ctrl_num);
2566         if (ip_rev > 0x40400)
2567                 unq_mrs_en = 1;
2568
2569         if ((ip_rev > 0x40700) && (popts->cswl_override != 0))
2570                 ddr->debug[18] = popts->cswl_override;
2571
2572         set_ddr_sdram_cfg_2(ctrl_num, ddr, popts, unq_mrs_en);
2573         set_ddr_sdram_mode(ctrl_num, ddr, popts, common_dimm,
2574                            cas_latency, additive_latency, unq_mrs_en);
2575         set_ddr_sdram_mode_2(ctrl_num, ddr, popts, common_dimm, unq_mrs_en);
2576 #ifdef CONFIG_SYS_FSL_DDR4
2577         set_ddr_sdram_mode_9(ddr, popts, common_dimm, unq_mrs_en);
2578         set_ddr_sdram_mode_10(ctrl_num, ddr, popts, common_dimm, unq_mrs_en);
2579 #endif
2580         set_ddr_sdram_rcw(ctrl_num, ddr, popts, common_dimm);
2581
2582         set_ddr_sdram_interval(ctrl_num, ddr, popts, common_dimm);
2583         set_ddr_data_init(ddr);
2584         set_ddr_sdram_clk_cntl(ddr, popts);
2585         set_ddr_init_addr(ddr);
2586         set_ddr_init_ext_addr(ddr);
2587         set_timing_cfg_4(ddr, popts);
2588         set_timing_cfg_5(ddr, cas_latency);
2589 #ifdef CONFIG_SYS_FSL_DDR4
2590         set_ddr_sdram_cfg_3(ddr, popts);
2591         set_timing_cfg_6(ddr);
2592         set_timing_cfg_7(ctrl_num, ddr, popts, common_dimm);
2593         set_timing_cfg_8(ctrl_num, ddr, popts, common_dimm, cas_latency);
2594         set_timing_cfg_9(ctrl_num, ddr, popts, common_dimm);
2595         set_ddr_dq_mapping(ddr, dimm_params);
2596 #endif
2597
2598         set_ddr_zq_cntl(ddr, zq_en);
2599         set_ddr_wrlvl_cntl(ddr, wrlvl_en, popts);
2600
2601         set_ddr_sr_cntr(ddr, sr_it);
2602
2603 #ifdef CONFIG_SYS_FSL_DDR_EMU
2604         /* disble DDR training for emulator */
2605         ddr->debug[2] = 0x00000400;
2606         ddr->debug[4] = 0xff800800;
2607         ddr->debug[5] = 0x08000800;
2608         ddr->debug[6] = 0x08000800;
2609         ddr->debug[7] = 0x08000800;
2610         ddr->debug[8] = 0x08000800;
2611 #endif
2612 #ifdef CONFIG_SYS_FSL_ERRATUM_A004508
2613         if ((ip_rev >= 0x40000) && (ip_rev < 0x40400))
2614                 ddr->debug[2] |= 0x00000200;    /* set bit 22 */
2615 #endif
2616
2617 #if defined(CONFIG_SYS_FSL_ERRATUM_A008378) && defined(CONFIG_SYS_FSL_DDRC_GEN4)
2618         /* Erratum applies when accumulated ECC is used, or DBI is enabled */
2619 #define IS_ACC_ECC_EN(v) ((v) & 0x4)
2620 #define IS_DBI(v) ((((v) >> 12) & 0x3) == 0x2)
2621         if (has_erratum_a008378()) {
2622                 if (IS_ACC_ECC_EN(ddr->ddr_sdram_cfg) ||
2623                     IS_DBI(ddr->ddr_sdram_cfg_3)) {
2624                         ddr->debug[28] = ddr_in32(&ddrc->debug[28]);
2625                         ddr->debug[28] |= (0x9 << 20);
2626                 }
2627         }
2628 #endif
2629
2630 #ifdef CONFIG_SYS_FSL_ERRATUM_A009942
2631         ddr_freq = get_ddr_freq(ctrl_num) / 1000000;
2632         ddr->debug[28] |= ddr_in32(&ddrc->debug[28]);
2633         ddr->debug[28] &= 0xff0fff00;
2634         if (ddr_freq <= 1333)
2635                 ddr->debug[28] |= 0x0080006a;
2636         else if (ddr_freq <= 1600)
2637                 ddr->debug[28] |= 0x0070006f;
2638         else if (ddr_freq <= 1867)
2639                 ddr->debug[28] |= 0x00700076;
2640         else if (ddr_freq <= 2133)
2641                 ddr->debug[28] |= 0x0060007b;
2642         if (popts->cpo_sample)
2643                 ddr->debug[28] = (ddr->debug[28] & 0xffffff00) |
2644                                   popts->cpo_sample;
2645 #endif
2646
2647         return check_fsl_memctl_config_regs(ddr);
2648 }
2649
2650 #ifdef CONFIG_SYS_FSL_ERRATUM_A009942
2651 /*
2652  * This additional workaround of A009942 checks the condition to determine if
2653  * the CPO value set by the existing A009942 workaround needs to be updated.
2654  * If need, print a warning to prompt user reconfigure DDR debug_29[24:31] with
2655  * expected optimal value, the optimal value is highly board dependent.
2656  */
2657 void erratum_a009942_check_cpo(void)
2658 {
2659         struct ccsr_ddr __iomem *ddr =
2660                 (struct ccsr_ddr __iomem *)(CONFIG_SYS_FSL_DDR_ADDR);
2661         u32 cpo, cpo_e, cpo_o, cpo_target, cpo_optimal;
2662         u32 cpo_min = ddr_in32(&ddr->debug[9]) >> 24;
2663         u32 cpo_max = cpo_min;
2664         u32 sdram_cfg, i, tmp, lanes, ddr_type;
2665         bool update_cpo = false, has_ecc = false;
2666
2667         sdram_cfg = ddr_in32(&ddr->sdram_cfg);
2668         if (sdram_cfg & SDRAM_CFG_32_BE)
2669                 lanes = 4;
2670         else if (sdram_cfg & SDRAM_CFG_16_BE)
2671                 lanes = 2;
2672         else
2673                 lanes = 8;
2674
2675         if (sdram_cfg & SDRAM_CFG_ECC_EN)
2676                 has_ecc = true;
2677
2678         /* determine the maximum and minimum CPO values */
2679         for (i = 9; i < 9 + lanes / 2; i++) {
2680                 cpo = ddr_in32(&ddr->debug[i]);
2681                 cpo_e = cpo >> 24;
2682                 cpo_o = (cpo >> 8) & 0xff;
2683                 tmp = min(cpo_e, cpo_o);
2684                 if (tmp < cpo_min)
2685                         cpo_min = tmp;
2686                 tmp = max(cpo_e, cpo_o);
2687                 if (tmp > cpo_max)
2688                         cpo_max = tmp;
2689         }
2690
2691         if (has_ecc) {
2692                 cpo = ddr_in32(&ddr->debug[13]);
2693                 cpo = cpo >> 24;
2694                 if (cpo < cpo_min)
2695                         cpo_min = cpo;
2696                 if (cpo > cpo_max)
2697                         cpo_max = cpo;
2698         }
2699
2700         cpo_target = ddr_in32(&ddr->debug[28]) & 0xff;
2701         cpo_optimal = ((cpo_max + cpo_min) >> 1) + 0x27;
2702         debug("cpo_optimal = 0x%x, cpo_target = 0x%x\n", cpo_optimal,
2703               cpo_target);
2704         debug("cpo_max = 0x%x, cpo_min = 0x%x\n", cpo_max, cpo_min);
2705
2706         ddr_type = (sdram_cfg & SDRAM_CFG_SDRAM_TYPE_MASK) >>
2707                     SDRAM_CFG_SDRAM_TYPE_SHIFT;
2708         if (ddr_type == SDRAM_TYPE_DDR4)
2709                 update_cpo = (cpo_min + 0x3b) < cpo_target ? true : false;
2710         else if (ddr_type == SDRAM_TYPE_DDR3)
2711                 update_cpo = (cpo_min + 0x3f) < cpo_target ? true : false;
2712
2713         if (update_cpo) {
2714                 printf("WARN: pls set popts->cpo_sample = 0x%x ", cpo_optimal);
2715                 printf("in <board>/ddr.c to optimize cpo\n");
2716         }
2717 }
2718 #endif