git.sur5r.net Git - u-boot/blob - drivers/ddr/marvell/axp/ddr3_write_leveling.c

   1 /*
   2  * Copyright (C) Marvell International Ltd. and its affiliates
   3  *
   4  * SPDX-License-Identifier:     GPL-2.0
   5  */
   6
   7 #include <common.h>
   8 #include <i2c.h>
   9 #include <spl.h>
  10 #include <asm/io.h>
  11 #include <asm/arch/cpu.h>
  12 #include <asm/arch/soc.h>
  13
  14 #include "ddr3_hw_training.h"
  15
  16 /*
  17  * Debug
  18  */
  19 #define DEBUG_WL_C(s, d, l) \
  20         DEBUG_WL_S(s); DEBUG_WL_D(d, l); DEBUG_WL_S("\n")
  21 #define DEBUG_WL_FULL_C(s, d, l) \
  22         DEBUG_WL_FULL_S(s); DEBUG_WL_FULL_D(d, l); DEBUG_WL_FULL_S("\n")
  23
  24 #ifdef MV_DEBUG_WL
  25 #define DEBUG_RL_S(s) \
  26         debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%s", s)
  27 #define DEBUG_RL_D(d, l) \
  28         debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%x", d)
  29 #else
  30 #define DEBUG_WL_S(s)
  31 #define DEBUG_WL_D(d, l)
  32 #endif
  33
  34 #ifdef MV_DEBUG_WL_FULL
  35 #define DEBUG_WL_FULL_S(s)              puts(s)
  36 #define DEBUG_WL_FULL_D(d, l)           printf("%x", d)
  37 #else
  38 #define DEBUG_WL_FULL_S(s)
  39 #define DEBUG_WL_FULL_D(d, l)
  40 #endif
  41
  42 #define WL_SUP_EXPECTED_DATA            0x21
  43 #define WL_SUP_READ_DRAM_ENTRY          0x8
  44
  45 static int ddr3_write_leveling_single_cs(u32 cs, u32 freq, int ratio_2to1,
  46                                          u32 *result,
  47                                          MV_DRAM_INFO *dram_info);
  48 static void ddr3_write_ctrl_pup_reg(int bc_acc, u32 pup, u32 reg_addr,
  49                                     u32 data);
  50
  51 extern u16 odt_static[ODT_OPT][MAX_CS];
  52 extern u16 odt_dynamic[ODT_OPT][MAX_CS];
  53 extern u32 wl_sup_pattern[LEN_WL_SUP_PATTERN];
  54
  55 /*
  56  * Name:     ddr3_write_leveling_hw
  57  * Desc:     Execute Write leveling phase by HW
  58  * Args:     freq      - current sequence frequency
  59  *           dram_info   - main struct
  60  * Notes:
  61  * Returns:  MV_OK if success, MV_FAIL if fail.
  62  */
  63 int ddr3_write_leveling_hw(u32 freq, MV_DRAM_INFO *dram_info)
  64 {
  65         u32 reg, phase, delay, cs, pup;
  66 #ifdef MV88F67XX
  67         int dpde_flag = 0;
  68 #endif
  69         /* Debug message - Start Read leveling procedure */
  70         DEBUG_WL_S("DDR3 - Write Leveling - Starting HW WL procedure\n");
  71
  72 #ifdef MV88F67XX
  73         /* Dynamic pad issue (BTS669) during WL */
  74         reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
  75         if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
  76                 dpde_flag = 1;
  77                 reg_write(REG_DUNIT_CTRL_LOW_ADDR,
  78                           reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
  79         }
  80 #endif
  81
  82         reg = 1 << REG_DRAM_TRAINING_WL_OFFS;
  83         /* Config the retest number */
  84         reg |= (COUNT_HW_WL << REG_DRAM_TRAINING_RETEST_OFFS);
  85         reg |= (dram_info->cs_ena << (REG_DRAM_TRAINING_CS_OFFS));
  86         reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
  87
  88         reg =  reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) |
  89                 (1 << REG_DRAM_TRAINING_AUTO_OFFS);
  90         reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg);
  91
  92         /* Wait */
  93         do {
  94                 reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
  95                         (1 << REG_DRAM_TRAINING_AUTO_OFFS);
  96         } while (reg);          /* Wait for '0' */
  97
  98         reg = reg_read(REG_DRAM_TRAINING_ADDR);
  99         /* Check if Successful */
 100         if (reg & (1 << REG_DRAM_TRAINING_ERROR_OFFS)) {
 101                 /*
 102                  * Read results to arrays - Results are required for WL
 103                  * High freq Supplement and DQS Centralization
 104                  */
 105                 for (cs = 0; cs < MAX_CS; cs++) {
 106                         if (dram_info->cs_ena & (1 << cs)) {
 107                                 for (pup = 0;
 108                                      pup < dram_info->num_of_total_pups;
 109                                      pup++) {
 110                                         if (pup == dram_info->num_of_std_pups
 111                                             && dram_info->ecc_ena)
 112                                                 pup = ECC_PUP;
 113                                         reg =
 114                                             ddr3_read_pup_reg(PUP_WL_MODE, cs,
 115                                                               pup);
 116                                         phase =
 117                                             (reg >> REG_PHY_PHASE_OFFS) &
 118                                             PUP_PHASE_MASK;
 119                                         delay = reg & PUP_DELAY_MASK;
 120                                         dram_info->wl_val[cs][pup][P] = phase;
 121                                         dram_info->wl_val[cs][pup][D] = delay;
 122                                         dram_info->wl_val[cs][pup][S] =
 123                                             WL_HI_FREQ_STATE - 1;
 124                                         reg =
 125                                             ddr3_read_pup_reg(PUP_WL_MODE + 0x1,
 126                                                               cs, pup);
 127                                         dram_info->wl_val[cs][pup][DQS] =
 128                                             (reg & 0x3F);
 129                                 }
 130
 131 #ifdef MV_DEBUG_WL
 132                                 /* Debug message - Print res for cs[i]: cs,PUP,Phase,Delay */
 133                                 DEBUG_WL_S("DDR3 - Write Leveling - Write Leveling Cs - ");
 134                                 DEBUG_WL_D((u32) cs, 1);
 135                                 DEBUG_WL_S(" Results:\n");
 136                                 for (pup = 0;
 137                                      pup < dram_info->num_of_total_pups;
 138                                      pup++) {
 139                                         if (pup == dram_info->num_of_std_pups
 140                                             && dram_info->ecc_ena)
 141                                                 pup = ECC_PUP;
 142                                         DEBUG_WL_S("DDR3 - Write Leveling - PUP: ");
 143                                         DEBUG_WL_D((u32) pup, 1);
 144                                         DEBUG_WL_S(", Phase: ");
 145                                         DEBUG_WL_D((u32)
 146                                                    dram_info->wl_val[cs][pup]
 147                                                    [P], 1);
 148                                         DEBUG_WL_S(", Delay: ");
 149                                         DEBUG_WL_D((u32)
 150                                                    dram_info->wl_val[cs][pup]
 151                                                    [D], 2);
 152                                         DEBUG_WL_S("\n");
 153                                 }
 154 #endif
 155                         }
 156                 }
 157
 158                 /* Dynamic pad issue (BTS669) during WL */
 159 #ifdef MV88F67XX
 160                 if (dpde_flag) {
 161                         reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
 162                                 (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
 163                         reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
 164                 }
 165 #endif
 166
 167                 DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
 168
 169                 return MV_OK;
 170         } else {
 171                 DEBUG_WL_S("DDR3 - Write Leveling - HW WL Error\n");
 172                 return MV_FAIL;
 173         }
 174 }
 175
 176 /*
 177  * Name:     ddr3_wl_supplement
 178  * Desc:     Write Leveling Supplement
 179  * Args:     dram_info   - main struct
 180  * Notes:
 181  * Returns:  MV_OK if success, MV_FAIL if fail.
 182  */
 183 int ddr3_wl_supplement(MV_DRAM_INFO *dram_info)
 184 {
 185         u32 cs, cnt, pup_num, sum, phase, delay, max_pup_num, pup, sdram_offset;
 186         u32 tmp_count, ecc, reg;
 187         u32 ddr_width, tmp_pup, idx;
 188         u32 sdram_pup_val, uj;
 189         u32 one_clk_err = 0, align_err = 0, no_err = 0, err = 0, err_n = 0;
 190         u32 sdram_data[LEN_WL_SUP_PATTERN] __aligned(32) = { 0 };
 191
 192         ddr_width = dram_info->ddr_width;
 193         no_err = 0;
 194
 195         DEBUG_WL_S("DDR3 - Write Leveling Hi-Freq Supplement - Starting\n");
 196
 197         switch (ddr_width) {
 198                 /* Data error from pos-adge to pos-adge */
 199         case 16:
 200                 one_clk_err = 4;
 201                 align_err = 4;
 202                 break;
 203         case 32:
 204                 one_clk_err = 8;
 205                 align_err = 8;
 206                 break;
 207         case 64:
 208                 one_clk_err = 0x10;
 209                 align_err = 0x10;
 210                 break;
 211         default:
 212                 DEBUG_WL_S("Error - bus width!!!\n");
 213                 return MV_FAIL;
 214         }
 215
 216         /* Enable SW override */
 217         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 218                 (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 219
 220         /* [0] = 1 - Enable SW override  */
 221         /* 0x15B8 - Training SW 2 Register */
 222         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 223         DEBUG_WL_S("DDR3 - Write Leveling Hi-Freq Supplement - SW Override Enabled\n");
 224         reg = (1 << REG_DRAM_TRAINING_AUTO_OFFS);
 225         reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
 226         tmp_count = 0;
 227         for (cs = 0; cs < MAX_CS; cs++) {
 228                 if (dram_info->cs_ena & (1 << cs)) {
 229                         sum = 0;
 230                         /*
 231                          * 2 iterations loop: 1)actual WL results 2) fix WL
 232                          * if needed
 233                          */
 234                         for (cnt = 0; cnt < COUNT_WL_HI_FREQ; cnt++) {
 235                                 DEBUG_WL_C("COUNT = ", cnt, 1);
 236                                 for (ecc = 0; ecc < (dram_info->ecc_ena + 1);
 237                                      ecc++) {
 238                                         if (ecc) {
 239                                                 DEBUG_WL_S("ECC PUP:\n");
 240                                         } else {
 241                                                 DEBUG_WL_S("DATA PUP:\n");
 242                                         }
 243
 244                                         max_pup_num =
 245                                             dram_info->num_of_std_pups * (1 -
 246                                                                           ecc) +
 247                                             ecc;
 248                                         /* ECC Support - Switch ECC Mux on ecc=1 */
 249                                         reg =
 250                                             (reg_read(REG_DRAM_TRAINING_2_ADDR)
 251                                              & ~(1 <<
 252                                                  REG_DRAM_TRAINING_2_ECC_MUX_OFFS));
 253                                         reg |=
 254                                             (dram_info->ecc_ena *
 255                                              ecc <<
 256                                              REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 257                                         reg_write(REG_DRAM_TRAINING_2_ADDR,
 258                                                   reg);
 259                                         ddr3_reset_phy_read_fifo();
 260
 261                                         /* Write to memory */
 262                                         sdram_offset =
 263                                             tmp_count * (SDRAM_CS_SIZE + 1) +
 264                                             0x200;
 265                                         if (MV_OK != ddr3_dram_sram_burst((u32)
 266                                                                           wl_sup_pattern,
 267                                                                           sdram_offset,
 268                                                                           LEN_WL_SUP_PATTERN))
 269                                                 return MV_FAIL;
 270
 271                                         /* Read from memory */
 272                                         if (MV_OK !=
 273                                             ddr3_dram_sram_burst(sdram_offset,
 274                                                                  (u32)
 275                                                                  sdram_data,
 276                                                                  LEN_WL_SUP_PATTERN))
 277                                                 return MV_FAIL;
 278
 279                                         /* Print the buffer */
 280                                         for (uj = 0; uj < LEN_WL_SUP_PATTERN;
 281                                              uj++) {
 282                                                 if ((uj % 4 == 0) && (uj != 0)) {
 283                                                         DEBUG_WL_S("\n");
 284                                                 }
 285                                                 DEBUG_WL_D(sdram_data[uj],
 286                                                            8);
 287                                                 DEBUG_WL_S(" ");
 288                                         }
 289
 290                                         /* Check pup which DQS/DATA is error */
 291                                         for (pup = 0; pup < max_pup_num; pup++) {
 292                                                 /* ECC support - bit 8 */
 293                                                 pup_num = (ecc) ? ECC_PUP : pup;
 294                                                 if (pup < 4) {  /* lower 32 bit */
 295                                                         tmp_pup = pup;
 296                                                         idx =
 297                                                             WL_SUP_READ_DRAM_ENTRY;
 298                                                 } else {        /* higher 32 bit */
 299                                                         tmp_pup = pup - 4;
 300                                                         idx =
 301                                                             WL_SUP_READ_DRAM_ENTRY
 302                                                             + 1;
 303                                                 }
 304                                                 DEBUG_WL_S("\nCS: ");
 305                                                 DEBUG_WL_D((u32) cs, 1);
 306                                                 DEBUG_WL_S(" PUP: ");
 307                                                 DEBUG_WL_D((u32) pup_num, 1);
 308                                                 DEBUG_WL_S("\n");
 309                                                 sdram_pup_val =
 310                                                     ((sdram_data[idx] >>
 311                                                       ((tmp_pup) * 8)) & 0xFF);
 312                                                 DEBUG_WL_C("Actual Data = ",
 313                                                            sdram_pup_val, 2);
 314                                                 DEBUG_WL_C("Expected Data = ",
 315                                                            (WL_SUP_EXPECTED_DATA
 316                                                             + pup), 2);
 317                                                 /*
 318                                                  * ALINGHMENT: calculate
 319                                                  * expected data vs actual data
 320                                                  */
 321                                                 err =
 322                                                     (WL_SUP_EXPECTED_DATA +
 323                                                      pup) - sdram_pup_val;
 324                                                 /*
 325                                                  * CLOCK LONG: calculate
 326                                                  * expected data vs actual data
 327                                                  */
 328                                                 err_n =
 329                                                     sdram_pup_val -
 330                                                     (WL_SUP_EXPECTED_DATA +
 331                                                      pup);
 332                                                 DEBUG_WL_C("err = ", err, 2);
 333                                                 DEBUG_WL_C("err_n = ", err_n,
 334                                                            2);
 335                                                 if (err == no_err) {
 336                                                         /* PUP is correct - increment State */
 337                                                         dram_info->wl_val[cs]
 338                                                             [pup_num]
 339                                                             [S] = 1;
 340                                                 } else if (err_n == one_clk_err) {
 341                                                         /* clock is longer than DQS */
 342                                                         phase =
 343                                                             ((dram_info->wl_val
 344                                                               [cs]
 345                                                               [pup_num][P] +
 346                                                               WL_HI_FREQ_SHIFT)
 347                                                              % MAX_PHASE_2TO1);
 348                                                         dram_info->wl_val[cs]
 349                                                             [pup_num]
 350                                                             [P] = phase;
 351                                                         delay =
 352                                                             dram_info->wl_val
 353                                                             [cs][pup_num]
 354                                                             [D];
 355                                                         DEBUG_WL_S("#### Clock is longer than DQS more than one clk cycle ####\n");
 356                                                         ddr3_write_pup_reg
 357                                                             (PUP_WL_MODE, cs,
 358                                                              pup * (1 - ecc) +
 359                                                              ECC_PUP * ecc,
 360                                                              phase, delay);
 361                                                 } else if (err == align_err) {
 362                                                         /* clock is align to DQS */
 363                                                         phase =
 364                                                             dram_info->wl_val
 365                                                             [cs][pup_num]
 366                                                             [P];
 367                                                         delay =
 368                                                             dram_info->wl_val
 369                                                             [cs][pup_num]
 370                                                             [D];
 371                                                         DEBUG_WL_S("#### Alignment PUPS problem ####\n");
 372                                                         if ((phase == 0)
 373                                                             || ((phase == 1)
 374                                                                 && (delay <=
 375                                                                     0x10))) {
 376                                                                 DEBUG_WL_S("#### Warning - Possible Layout Violation (DQS is longer than CLK)####\n");
 377                                                         }
 378
 379                                                         phase = 0x0;
 380                                                         delay = 0x0;
 381                                                         dram_info->wl_val[cs]
 382                                                             [pup_num]
 383                                                             [P] = phase;
 384                                                         dram_info->wl_val[cs]
 385                                                             [pup_num]
 386                                                             [D] = delay;
 387                                                         ddr3_write_pup_reg
 388                                                             (PUP_WL_MODE, cs,
 389                                                              pup * (1 - ecc) +
 390                                                              ECC_PUP * ecc,
 391                                                              phase, delay);
 392                                                 }
 393                                                 /* Stop condition for ECC phase */
 394                                                 pup = (ecc) ? max_pup_num : pup;
 395                                         }
 396
 397                                         /* ECC Support - Disable ECC MUX */
 398                                         reg =
 399                                             (reg_read(REG_DRAM_TRAINING_2_ADDR)
 400                                              & ~(1 <<
 401                                                  REG_DRAM_TRAINING_2_ECC_MUX_OFFS));
 402                                         reg_write(REG_DRAM_TRAINING_2_ADDR,
 403                                                   reg);
 404                                 }
 405                         }
 406
 407                         for (pup = 0; pup < dram_info->num_of_std_pups; pup++)
 408                                 sum += dram_info->wl_val[cs][pup][S];
 409
 410                         if (dram_info->ecc_ena)
 411                                 sum += dram_info->wl_val[cs][ECC_PUP][S];
 412
 413                         /* Checks if any pup is not locked after the change */
 414                         if (sum < (WL_HI_FREQ_STATE * (dram_info->num_of_total_pups))) {
 415                                 DEBUG_WL_C("DDR3 - Write Leveling Hi-Freq Supplement - didn't work for Cs - ",
 416                                            (u32) cs, 1);
 417                                 return MV_FAIL;
 418                         }
 419                         tmp_count++;
 420                 }
 421         }
 422
 423         dram_info->wl_max_phase = 0;
 424         dram_info->wl_min_phase = 10;
 425
 426         /*
 427          * Read results to arrays - Results are required for DQS Centralization
 428          */
 429         for (cs = 0; cs < MAX_CS; cs++) {
 430                 if (dram_info->cs_ena & (1 << cs)) {
 431                         for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
 432                                 if (pup == dram_info->num_of_std_pups
 433                                     && dram_info->ecc_ena)
 434                                         pup = ECC_PUP;
 435                                 reg = ddr3_read_pup_reg(PUP_WL_MODE, cs, pup);
 436                                 phase =
 437                                     (reg >> REG_PHY_PHASE_OFFS) &
 438                                     PUP_PHASE_MASK;
 439                                 if (phase > dram_info->wl_max_phase)
 440                                         dram_info->wl_max_phase = phase;
 441                                 if (phase < dram_info->wl_min_phase)
 442                                         dram_info->wl_min_phase = phase;
 443                         }
 444                 }
 445         }
 446
 447         /* Disable SW override - Must be in a different stage */
 448         /* [0]=0 - Enable SW override  */
 449         reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 450         reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 451         /* 0x15B8 - Training SW 2 Register */
 452         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 453
 454         reg = reg_read(REG_DRAM_TRAINING_1_ADDR) |
 455                 (1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS);
 456         reg_write(REG_DRAM_TRAINING_1_ADDR, reg);
 457
 458         DEBUG_WL_S("DDR3 - Write Leveling Hi-Freq Supplement - Ended Successfully\n");
 459
 460         return MV_OK;
 461 }
 462
 463 /*
 464  * Name:     ddr3_write_leveling_hw_reg_dimm
 465  * Desc:     Execute Write leveling phase by HW
 466  * Args:     freq      - current sequence frequency
 467  *           dram_info   - main struct
 468  * Notes:
 469  * Returns:  MV_OK if success, MV_FAIL if fail.
 470  */
 471 int ddr3_write_leveling_hw_reg_dimm(u32 freq, MV_DRAM_INFO *dram_info)
 472 {
 473         u32 reg, phase, delay, cs, pup, pup_num;
 474         __maybe_unused int dpde_flag = 0;
 475
 476         /* Debug message - Start Read leveling procedure */
 477         DEBUG_WL_S("DDR3 - Write Leveling - Starting HW WL procedure\n");
 478
 479         if (dram_info->num_cs > 2) {
 480                 DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
 481                 return MV_NO_CHANGE;
 482         }
 483
 484         /* If target freq = 400 move clock start point */
 485         /* Write to control PUP to Control Deskew Regs */
 486         if (freq <= DDR_400) {
 487                 for (pup = 0; pup <= dram_info->num_of_total_pups; pup++) {
 488                         /* PUP_DELAY_MASK 0x1F */
 489                         /* reg = 0x0C10001F + (uj << 16); */
 490                         ddr3_write_ctrl_pup_reg(1, pup, CNTRL_PUP_DESKEW + pup,
 491                                                 0x1F);
 492                 }
 493         }
 494
 495 #ifdef MV88F67XX
 496         /* Dynamic pad issue (BTS669) during WL */
 497         reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
 498         if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
 499                 dpde_flag = 1;
 500                 reg_write(REG_DUNIT_CTRL_LOW_ADDR,
 501                           reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
 502         }
 503 #endif
 504
 505         reg = (1 << REG_DRAM_TRAINING_WL_OFFS);
 506         /* Config the retest number */
 507         reg |= (COUNT_HW_WL << REG_DRAM_TRAINING_RETEST_OFFS);
 508         reg |= (dram_info->cs_ena << (REG_DRAM_TRAINING_CS_OFFS));
 509         reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
 510
 511         reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) |
 512                 (1 << REG_DRAM_TRAINING_AUTO_OFFS);
 513         reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg);
 514
 515         /* Wait */
 516         do {
 517                 reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
 518                         (1 << REG_DRAM_TRAINING_AUTO_OFFS);
 519         } while (reg);          /* Wait for '0' */
 520
 521         reg = reg_read(REG_DRAM_TRAINING_ADDR);
 522         /* Check if Successful */
 523         if (reg & (1 << REG_DRAM_TRAINING_ERROR_OFFS)) {
 524                 /*
 525                  * Read results to arrays - Results are required for WL High
 526                  * freq Supplement and DQS Centralization
 527                  */
 528                 for (cs = 0; cs < MAX_CS; cs++) {
 529                         if (dram_info->cs_ena & (1 << cs)) {
 530                                 for (pup = 0;
 531                                      pup < dram_info->num_of_total_pups;
 532                                      pup++) {
 533                                         if (pup == dram_info->num_of_std_pups
 534                                             && dram_info->ecc_ena)
 535                                                 pup = ECC_BIT;
 536                                         reg =
 537                                             ddr3_read_pup_reg(PUP_WL_MODE, cs,
 538                                                               pup);
 539                                         phase =
 540                                             (reg >> REG_PHY_PHASE_OFFS) &
 541                                             PUP_PHASE_MASK;
 542                                         delay = reg & PUP_DELAY_MASK;
 543                                         dram_info->wl_val[cs][pup][P] = phase;
 544                                         dram_info->wl_val[cs][pup][D] = delay;
 545                                         if ((phase == 1) && (delay >= 0x1D)) {
 546                                                 /*
 547                                                  * Need to do it here for
 548                                                  * uncorrect WL values
 549                                                  */
 550                                                 ddr3_write_pup_reg(PUP_WL_MODE,
 551                                                                    cs, pup, 0,
 552                                                                    0);
 553                                                 dram_info->wl_val[cs][pup][P] =
 554                                                     0;
 555                                                 dram_info->wl_val[cs][pup][D] =
 556                                                     0;
 557                                         }
 558                                         dram_info->wl_val[cs][pup][S] =
 559                                             WL_HI_FREQ_STATE - 1;
 560                                         reg =
 561                                             ddr3_read_pup_reg(PUP_WL_MODE + 0x1,
 562                                                               cs, pup);
 563                                         dram_info->wl_val[cs][pup][DQS] =
 564                                             (reg & 0x3F);
 565                                 }
 566 #ifdef MV_DEBUG_WL
 567                                 /*
 568                                  * Debug message - Print res for cs[i]:
 569                                  * cs,PUP,Phase,Delay
 570                                  */
 571                                 DEBUG_WL_S("DDR3 - Write Leveling - Write Leveling Cs - ");
 572                                 DEBUG_WL_D((u32) cs, 1);
 573                                 DEBUG_WL_S(" Results:\n");
 574                                 for (pup = 0;
 575                                      pup < dram_info->num_of_total_pups;
 576                                      pup++) {
 577                                         DEBUG_WL_S
 578                                             ("DDR3 - Write Leveling - PUP: ");
 579                                         DEBUG_WL_D((u32) pup, 1);
 580                                         DEBUG_WL_S(", Phase: ");
 581                                         DEBUG_WL_D((u32)
 582                                                    dram_info->wl_val[cs][pup]
 583                                                    [P], 1);
 584                                         DEBUG_WL_S(", Delay: ");
 585                                         DEBUG_WL_D((u32)
 586                                                    dram_info->wl_val[cs][pup]
 587                                                    [D], 2);
 588                                         DEBUG_WL_S("\n");
 589                                 }
 590 #endif
 591                         }
 592                 }
 593
 594 #ifdef MV88F67XX
 595                 /* Dynamic pad issue (BTS669) during WL */
 596                 if (dpde_flag) {
 597                         reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
 598                                 (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
 599                         reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
 600                 }
 601 #endif
 602                 DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
 603
 604                 /* If target freq = 400 move clock back */
 605                 /* Write to control PUP to Control Deskew Regs */
 606                 if (freq <= DDR_400) {
 607                         for (pup = 0; pup <= dram_info->num_of_total_pups;
 608                              pup++) {
 609                                 ddr3_write_ctrl_pup_reg(1, pup,
 610                                                         CNTRL_PUP_DESKEW + pup, 0);
 611                         }
 612                 }
 613
 614                 return MV_OK;
 615         } else {
 616                 /* Configure Each PUP with locked leveling settings */
 617                 for (cs = 0; cs < MAX_CS; cs++) {
 618                         if (dram_info->cs_ena & (1 << cs)) {
 619                                 for (pup = 0;
 620                                      pup < dram_info->num_of_total_pups;
 621                                      pup++) {
 622                                         /* ECC support - bit 8 */
 623                                         pup_num = (pup == dram_info->num_of_std_pups) ?
 624                                                 ECC_BIT : pup;
 625                                         ddr3_write_pup_reg(PUP_WL_MODE, cs,
 626                                                            pup_num, 0, 0);
 627                                 }
 628                         }
 629                 }
 630
 631                 reg_write(REG_DRAM_TRAINING_ADDR, 0);
 632
 633                 /* If target freq = 400 move clock back */
 634                 /* Write to control PUP to Control Deskew Regs */
 635                 if (freq <= DDR_400) {
 636                         for (pup = 0; pup <= dram_info->num_of_total_pups;
 637                              pup++) {
 638                                 ddr3_write_ctrl_pup_reg(1, pup,
 639                                                         CNTRL_PUP_DESKEW + pup, 0);
 640                         }
 641                 }
 642
 643                 DEBUG_WL_S("DDR3 - Write Leveling - HW WL Ended Successfully\n");
 644                 return MV_NO_CHANGE;
 645         }
 646 }
 647
 648 /*
 649  * Name:     ddr3_write_leveling_sw
 650  * Desc:     Execute Write leveling phase by SW
 651  * Args:     freq      - current sequence frequency
 652  *           dram_info   - main struct
 653  * Notes:
 654  * Returns:  MV_OK if success, MV_FAIL if fail.
 655  */
 656 int ddr3_write_leveling_sw(u32 freq, int ratio_2to1, MV_DRAM_INFO *dram_info)
 657 {
 658         u32 reg, cs, cnt, pup, max_pup_num;
 659         u32 res[MAX_CS];
 660         max_pup_num = dram_info->num_of_total_pups;
 661         __maybe_unused int dpde_flag = 0;
 662
 663         /* Debug message - Start Write leveling procedure */
 664         DEBUG_WL_S("DDR3 - Write Leveling - Starting SW WL procedure\n");
 665
 666 #ifdef MV88F67XX
 667         /* Dynamic pad issue (BTS669) during WL */
 668         reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
 669         if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
 670                 dpde_flag = 1;
 671                 reg_write(REG_DUNIT_CTRL_LOW_ADDR,
 672                           reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
 673         }
 674 #endif
 675
 676         /* Set Output buffer-off to all CS and correct ODT values */
 677         for (cs = 0; cs < MAX_CS; cs++) {
 678                 if (dram_info->cs_ena & (1 << cs)) {
 679                         reg = reg_read(REG_DDR3_MR1_ADDR) &
 680                                 REG_DDR3_MR1_ODT_MASK;
 681                         reg |= odt_static[dram_info->cs_ena][cs];
 682                         reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
 683
 684                         /* 0x15D0 - DDR3 MR0 Register */
 685                         reg_write(REG_DDR3_MR1_ADDR, reg);
 686                         /* Issue MRS Command to current cs */
 687                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
 688                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 689                         /*
 690                          * [3-0] = 0x4 - MR1 Command, [11-8] -
 691                          * enable current cs
 692                          */
 693                         /* 0x1418 - SDRAM Operation Register */
 694                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
 695
 696                         udelay(MRS_DELAY);
 697                 }
 698         }
 699
 700         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Qoff and RTT Values are set for all Cs\n");
 701
 702         /* Enable SW override */
 703         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 704                 (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 705         /* [0] = 1 - Enable SW override  */
 706         /* 0x15B8 - Training SW 2 Register */
 707         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 708         DEBUG_WL_FULL_S("DDR3 - Write Leveling - SW Override Enabled\n");
 709
 710         /* Enable PHY write leveling mode */
 711         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 712                 ~(1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
 713         /* [2] = 0 - TrnWLMode - Enable */
 714         /* 0x15B8 - Training SW 2 Register */
 715         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 716         /* Reset WL results arry */
 717         memset(dram_info->wl_val, 0, sizeof(u32) * MAX_CS * MAX_PUP_NUM * 7);
 718
 719         /* Loop for each cs */
 720         for (cs = 0; cs < MAX_CS; cs++) {
 721                 if (dram_info->cs_ena & (1 << cs)) {
 722                         DEBUG_WL_FULL_C("DDR3 - Write Leveling - Starting working with Cs - ",
 723                                         (u32) cs, 1);
 724                         /* Refresh X9 current cs */
 725                         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Refresh X9\n");
 726                         for (cnt = 0; cnt < COUNT_WL_RFRS; cnt++) {
 727                                 reg =
 728                                     REG_SDRAM_OPERATION_CMD_RFRS & ~(1 <<
 729                                                                      (REG_SDRAM_OPERATION_CS_OFFS
 730                                                                       + cs));
 731                                 /* [3-0] = 0x2 - refresh, [11-8] - enable current cs */
 732                                 reg_write(REG_SDRAM_OPERATION_ADDR, reg);       /* 0x1418 - SDRAM Operation Register */
 733
 734                                 do {
 735                                         reg =
 736                                             ((reg_read
 737                                               (REG_SDRAM_OPERATION_ADDR)) &
 738                                              REG_SDRAM_OPERATION_CMD_RFRS_DONE);
 739                                 } while (reg);  /* Wait for '0' */
 740                         }
 741
 742                         /* Configure MR1 in Cs[CsNum] - write leveling on, output buffer on */
 743                         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Configure MR1 for current Cs: WL-on,OB-on\n");
 744                         reg = reg_read(REG_DDR3_MR1_ADDR) &
 745                                 REG_DDR3_MR1_OUTBUF_WL_MASK;
 746                         /* Set ODT Values */
 747                         reg &= REG_DDR3_MR1_ODT_MASK;
 748                         reg |= odt_static[dram_info->cs_ena][cs];
 749                         /* Enable WL MODE */
 750                         reg |= (1 << REG_DDR3_MR1_WL_ENA_OFFS);
 751                         /* [7]=1, [12]=0 - Output Buffer and write leveling enabled */
 752                         reg_write(REG_DDR3_MR1_ADDR, reg);      /* 0x15D4 - DDR3 MR1 Register */
 753                         /* Issue MRS Command to current cs */
 754                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
 755                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 756                         /*
 757                          * [3-0] = 0x4 - MR1 Command, [11-8] -
 758                          * enable current cs
 759                          */
 760                         /* 0x1418 - SDRAM Operation Register */
 761                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
 762
 763                         udelay(MRS_DELAY);
 764
 765                         /* Write leveling  cs[cs] */
 766                         if (MV_OK !=
 767                             ddr3_write_leveling_single_cs(cs, freq, ratio_2to1,
 768                                                           (u32 *)(res + cs),
 769                                                           dram_info)) {
 770                                 DEBUG_WL_FULL_C("DDR3 - Write Leveling single Cs - FAILED -  Cs - ",
 771                                                 (u32) cs, 1);
 772                                 for (pup = 0; pup < max_pup_num; pup++) {
 773                                         if (((res[cs] >> pup) & 0x1) == 0) {
 774                                                 DEBUG_WL_C("Failed Byte : ",
 775                                                            pup, 1);
 776                                         }
 777                                 }
 778                                 return MV_FAIL;
 779                         }
 780
 781                         /* Set TrnWLDeUpd - After each CS is done */
 782                         reg = reg_read(REG_TRAINING_WL_ADDR) |
 783                                 (1 << REG_TRAINING_WL_CS_DONE_OFFS);
 784                         /* 0x16AC - Training Write leveling register */
 785                         reg_write(REG_TRAINING_WL_ADDR, reg);
 786
 787                         /*
 788                          * Debug message - Finished Write leveling cs[cs] -
 789                          * each PUP Fail/Success
 790                          */
 791                         DEBUG_WL_FULL_C("DDR3 - Write Leveling - Finished Cs - ", (u32) cs,
 792                                         1);
 793                         DEBUG_WL_FULL_C("DDR3 - Write Leveling - The Results: 1-PUP locked, 0-PUP failed -",
 794                                         (u32) res[cs], 3);
 795
 796                         /*
 797                          * Configure MR1 in cs[cs] - write leveling off (0),
 798                          * output buffer off (1)
 799                          */
 800                         reg = reg_read(REG_DDR3_MR1_ADDR) &
 801                                 REG_DDR3_MR1_OUTBUF_WL_MASK;
 802                         reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
 803                         /* No need to sort ODT since it is same CS */
 804                         /* 0x15D4 - DDR3 MR1 Register */
 805                         reg_write(REG_DDR3_MR1_ADDR, reg);
 806                         /* Issue MRS Command to current cs */
 807                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
 808                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 809                         /*
 810                          * [3-0] = 0x4 - MR1 Command, [11-8] -
 811                          * enable current cs
 812                          */
 813                         /* 0x1418 - SDRAM Operation Register */
 814                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
 815
 816                         udelay(MRS_DELAY);
 817                 }
 818         }
 819
 820         /* Disable WL Mode */
 821         /* [2]=1 - TrnWLMode - Disable */
 822         reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 823         reg |= (1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
 824         /* 0x15B8 - Training SW 2 Register */
 825         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 826
 827         /* Disable SW override - Must be in a different stage */
 828         /* [0]=0 - Enable SW override  */
 829         reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
 830         reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 831         /* 0x15B8 - Training SW 2 Register */
 832         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 833
 834         /* Set Output buffer-on to all CS and correct ODT values */
 835         for (cs = 0; cs < MAX_CS; cs++) {
 836                 if (dram_info->cs_ena & (1 << cs)) {
 837                         reg = reg_read(REG_DDR3_MR1_ADDR) &
 838                                 REG_DDR3_MR1_ODT_MASK;
 839                         reg &= REG_DDR3_MR1_OUTBUF_WL_MASK;
 840                         reg |= odt_static[dram_info->cs_ena][cs];
 841
 842                         /* 0x15D0 - DDR3 MR1 Register */
 843                         reg_write(REG_DDR3_MR1_ADDR, reg);
 844                         /* Issue MRS Command to current cs */
 845                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
 846                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 847                         /*
 848                          * [3-0] = 0x4 - MR1 Command, [11-8] -
 849                          * enable current cs
 850                          */
 851                         /* 0x1418 - SDRAM Operation Register */
 852                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
 853
 854                         udelay(MRS_DELAY);
 855                 }
 856         }
 857
 858 #ifdef MV88F67XX
 859         /* Dynamic pad issue (BTS669) during WL */
 860         if (dpde_flag) {
 861                 reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
 862                         (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
 863                 reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
 864         }
 865 #endif
 866         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Finished WL procedure for all Cs\n");
 867
 868         return MV_OK;
 869 }
 870
 871 #if !defined(MV88F672X)
 872 /*
 873  * Name:     ddr3_write_leveling_sw
 874  * Desc:     Execute Write leveling phase by SW
 875  * Args:     freq        - current sequence frequency
 876  *           dram_info   - main struct
 877  * Notes:
 878  * Returns:  MV_OK if success, MV_FAIL if fail.
 879  */
 880 int ddr3_write_leveling_sw_reg_dimm(u32 freq, int ratio_2to1,
 881                                     MV_DRAM_INFO *dram_info)
 882 {
 883         u32 reg, cs, cnt, pup;
 884         u32 res[MAX_CS];
 885         __maybe_unused int dpde_flag = 0;
 886
 887         /* Debug message - Start Write leveling procedure */
 888         DEBUG_WL_S("DDR3 - Write Leveling - Starting SW WL procedure\n");
 889
 890 #ifdef MV88F67XX
 891         /* Dynamic pad issue (BTS669) during WL */
 892         reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR);
 893         if (reg & (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS)) {
 894                 dpde_flag = 1;
 895                 reg_write(REG_DUNIT_CTRL_LOW_ADDR,
 896                           reg & ~(1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS));
 897         }
 898 #endif
 899
 900         /* If target freq = 400 move clock start point */
 901         /* Write to control PUP to Control Deskew Regs */
 902         if (freq <= DDR_400) {
 903                 for (pup = 0; pup <= dram_info->num_of_total_pups; pup++) {
 904                         /* PUP_DELAY_MASK 0x1F */
 905                         /* reg = 0x0C10001F + (uj << 16); */
 906                         ddr3_write_ctrl_pup_reg(1, pup, CNTRL_PUP_DESKEW + pup,
 907                                                 0x1F);
 908                 }
 909         }
 910
 911         /* Set Output buffer-off to all CS and correct ODT values */
 912         for (cs = 0; cs < MAX_CS; cs++) {
 913                 if (dram_info->cs_ena & (1 << cs)) {
 914                         reg = reg_read(REG_DDR3_MR1_ADDR) &
 915                                 REG_DDR3_MR1_ODT_MASK;
 916                         reg |= odt_static[dram_info->cs_ena][cs];
 917                         reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
 918
 919                         /* 0x15D0 - DDR3 MR0 Register */
 920                         reg_write(REG_DDR3_MR1_ADDR, reg);
 921                         /* Issue MRS Command to current cs */
 922                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
 923                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 924                         /*
 925                          * [3-0] = 0x4 - MR1 Command, [11-8] -
 926                          * enable current cs
 927                          */
 928                         /* 0x1418 - SDRAM Operation Register */
 929                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
 930
 931                         udelay(MRS_DELAY);
 932                 }
 933         }
 934
 935         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Qoff and RTT Values are set for all Cs\n");
 936
 937         /* Enable SW override */
 938         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 939                 (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 940         /* [0] = 1 - Enable SW override  */
 941         /* 0x15B8 - Training SW 2 Register */
 942         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 943         DEBUG_WL_FULL_S("DDR3 - Write Leveling - SW Override Enabled\n");
 944
 945         /* Enable PHY write leveling mode */
 946         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 947                 ~(1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
 948         /* [2] = 0 - TrnWLMode - Enable */
 949         /* 0x15B8 - Training SW 2 Register */
 950         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 951
 952         /* Loop for each cs */
 953         for (cs = 0; cs < MAX_CS; cs++) {
 954                 if (dram_info->cs_ena & (1 << cs)) {
 955                         DEBUG_WL_FULL_C("DDR3 - Write Leveling - Starting working with Cs - ",
 956                                         (u32) cs, 1);
 957
 958                         /* Refresh X9 current cs */
 959                         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Refresh X9\n");
 960                         for (cnt = 0; cnt < COUNT_WL_RFRS; cnt++) {
 961                                 reg =
 962                                     REG_SDRAM_OPERATION_CMD_RFRS & ~(1 <<
 963                                                                      (REG_SDRAM_OPERATION_CS_OFFS
 964                                                                       + cs));
 965                                 /* [3-0] = 0x2 - refresh, [11-8] - enable current cs */
 966                                 reg_write(REG_SDRAM_OPERATION_ADDR, reg);       /* 0x1418 - SDRAM Operation Register */
 967
 968                                 do {
 969                                         reg =
 970                                             ((reg_read
 971                                               (REG_SDRAM_OPERATION_ADDR)) &
 972                                              REG_SDRAM_OPERATION_CMD_RFRS_DONE);
 973                                 } while (reg);  /* Wait for '0' */
 974                         }
 975
 976                         /*
 977                          * Configure MR1 in Cs[CsNum] - write leveling on,
 978                          * output buffer on
 979                          */
 980                         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Configure MR1 for current Cs: WL-on,OB-on\n");
 981                         reg = reg_read(REG_DDR3_MR1_ADDR) &
 982                                 REG_DDR3_MR1_OUTBUF_WL_MASK;
 983                         /* Set ODT Values */
 984                         reg &= REG_DDR3_MR1_ODT_MASK;
 985                         reg |= odt_static[dram_info->cs_ena][cs];
 986                         /* Enable WL MODE */
 987                         reg |= (1 << REG_DDR3_MR1_WL_ENA_OFFS);
 988                         /*
 989                          * [7]=1, [12]=0 - Output Buffer and write leveling
 990                          * enabled
 991                          */
 992                         /* 0x15D4 - DDR3 MR1 Register */
 993                         reg_write(REG_DDR3_MR1_ADDR, reg);
 994                         /* Issue MRS Command to current cs */
 995                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
 996                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
 997                         /*
 998                          * [3-0] = 0x4 - MR1 Command, [11-8] -
 999                          * enable current cs
1000                          */
1001                         /* 0x1418 - SDRAM Operation Register */
1002                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
1003
1004                         udelay(MRS_DELAY);
1005
1006                         /* Write leveling  cs[cs] */
1007                         if (MV_OK !=
1008                             ddr3_write_leveling_single_cs(cs, freq, ratio_2to1,
1009                                                           (u32 *)(res + cs),
1010                                                           dram_info)) {
1011                                 DEBUG_WL_FULL_C("DDR3 - Write Leveling single Cs - FAILED -  Cs - ",
1012                                                 (u32) cs, 1);
1013                                 return MV_FAIL;
1014                         }
1015
1016                         /* Set TrnWLDeUpd - After each CS is done */
1017                         reg = reg_read(REG_TRAINING_WL_ADDR) |
1018                                 (1 << REG_TRAINING_WL_CS_DONE_OFFS);
1019                         /* 0x16AC - Training Write leveling register */
1020                         reg_write(REG_TRAINING_WL_ADDR, reg);
1021
1022                         /*
1023                          * Debug message - Finished Write leveling cs[cs] -
1024                          * each PUP Fail/Success
1025                          */
1026                         DEBUG_WL_FULL_C("DDR3 - Write Leveling - Finished Cs - ", (u32) cs,
1027                                         1);
1028                         DEBUG_WL_FULL_C("DDR3 - Write Leveling - The Results: 1-PUP locked, 0-PUP failed -",
1029                                         (u32) res[cs], 3);
1030
1031                         /* Configure MR1 in cs[cs] - write leveling off (0), output buffer off (1) */
1032                         reg = reg_read(REG_DDR3_MR1_ADDR) &
1033                                 REG_DDR3_MR1_OUTBUF_WL_MASK;
1034                         reg |= (1 << REG_DDR3_MR1_OUTBUF_DIS_OFFS);
1035                         /* No need to sort ODT since it is same CS */
1036                         /* 0x15D4 - DDR3 MR1 Register */
1037                         reg_write(REG_DDR3_MR1_ADDR, reg);
1038                         /* Issue MRS Command to current cs */
1039                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
1040                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
1041                         /*
1042                          * [3-0] = 0x4 - MR1 Command, [11-8] -
1043                          * enable current cs
1044                          */
1045                         /* 0x1418 - SDRAM Operation Register */
1046                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
1047
1048                         udelay(MRS_DELAY);
1049                 }
1050         }
1051
1052         /* Disable WL Mode */
1053         /* [2]=1 - TrnWLMode - Disable */
1054         reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
1055         reg |= (1 << REG_DRAM_TRAINING_2_WL_MODE_OFFS);
1056         /* 0x15B8 - Training SW 2 Register */
1057         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
1058
1059         /* Disable SW override - Must be in a different stage */
1060         /* [0]=0 - Enable SW override  */
1061         reg = reg_read(REG_DRAM_TRAINING_2_ADDR);
1062         reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
1063         /* 0x15B8 - Training SW 2 Register */
1064         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
1065
1066         /* Set Output buffer-on to all CS and correct ODT values */
1067         for (cs = 0; cs < MAX_CS; cs++) {
1068                 if (dram_info->cs_ena & (1 << cs)) {
1069                         reg = reg_read(REG_DDR3_MR1_ADDR) &
1070                                 REG_DDR3_MR1_ODT_MASK;
1071                         reg &= REG_DDR3_MR1_OUTBUF_WL_MASK;
1072                         reg |= odt_static[dram_info->cs_ena][cs];
1073
1074                         /* 0x15D0 - DDR3 MR1 Register */
1075                         reg_write(REG_DDR3_MR1_ADDR, reg);
1076                         /* Issue MRS Command to current cs */
1077                         reg = REG_SDRAM_OPERATION_CMD_MR1 &
1078                                 ~(1 << (REG_SDRAM_OPERATION_CS_OFFS + cs));
1079                         /*
1080                          * [3-0] = 0x4 - MR1 Command, [11-8] -
1081                          * enable current cs
1082                          */
1083                         /* 0x1418 - SDRAM Operation Register */
1084                         reg_write(REG_SDRAM_OPERATION_ADDR, reg);
1085
1086                         udelay(MRS_DELAY);
1087                 }
1088         }
1089
1090 #ifdef MV88F67XX
1091         /* Dynamic pad issue (BTS669) during WL */
1092         if (dpde_flag) {
1093                 reg = reg_read(REG_DUNIT_CTRL_LOW_ADDR) |
1094                         (1 << REG_DUNIT_CTRL_LOW_DPDE_OFFS);
1095                 reg_write(REG_DUNIT_CTRL_LOW_ADDR, reg);
1096         }
1097 #endif
1098
1099         /* If target freq = 400 move clock back */
1100         /* Write to control PUP to Control Deskew Regs */
1101         if (freq <= DDR_400) {
1102                 for (pup = 0; pup <= dram_info->num_of_total_pups; pup++) {
1103                         ddr3_write_ctrl_pup_reg(1, pup, CNTRL_PUP_DESKEW + pup,
1104                                                 0);
1105                 }
1106         }
1107
1108         DEBUG_WL_FULL_S("DDR3 - Write Leveling - Finished WL procedure for all Cs\n");
1109         return MV_OK;
1110 }
1111 #endif
1112
1113 /*
1114  * Name:     ddr3_write_leveling_single_cs
1115  * Desc:     Execute Write leveling for single Chip select
1116  * Args:     cs          - current chip select
1117  *           freq        - current sequence frequency
1118  *           result      - res array
1119  *           dram_info   - main struct
1120  * Notes:
1121  * Returns:  MV_OK if success, MV_FAIL if fail.
1122  */
1123 static int ddr3_write_leveling_single_cs(u32 cs, u32 freq, int ratio_2to1,
1124                                          u32 *result, MV_DRAM_INFO *dram_info)
1125 {
1126         u32 reg, pup_num, delay, phase, phaseMax, max_pup_num, pup,
1127                 max_pup_mask;
1128
1129         max_pup_num = dram_info->num_of_total_pups;
1130         *result = 0;
1131         u32 flag[MAX_PUP_NUM] = { 0 };
1132
1133         DEBUG_WL_FULL_C("DDR3 - Write Leveling Single Cs - WL for Cs - ",
1134                         (u32) cs, 1);
1135
1136         switch (max_pup_num) {
1137         case 2:
1138                 max_pup_mask = 0x3;
1139                 break;
1140         case 4:
1141                 max_pup_mask = 0xf;
1142                 DEBUG_WL_C("max_pup_mask =  ", max_pup_mask, 3);
1143                 break;
1144         case 5:
1145                 max_pup_mask = 0x1f;
1146                 DEBUG_WL_C("max_pup_mask =  ", max_pup_mask, 3);
1147                 break;
1148         case 8:
1149                 max_pup_mask = 0xff;
1150                 DEBUG_WL_C("max_pup_mask =  ", max_pup_mask, 3);
1151                 break;
1152         case 9:
1153                 max_pup_mask = 0x1ff;
1154                 DEBUG_WL_C("max_pup_mask =  ", max_pup_mask, 3);
1155                 break;
1156         default:
1157                 DEBUG_WL_C("ddr3_write_leveling_single_cs wrong max_pup_num =  ",
1158                            max_pup_num, 3);
1159                 return MV_FAIL;
1160         }
1161
1162         /* CS ODT Override */
1163         reg = reg_read(REG_SDRAM_ODT_CTRL_HIGH_ADDR) &
1164                 REG_SDRAM_ODT_CTRL_HIGH_OVRD_MASK;
1165         reg |= (REG_SDRAM_ODT_CTRL_HIGH_OVRD_ENA << (2 * cs));
1166         /* Set 0x3 - Enable ODT on the curent cs and disable on other cs */
1167         /* 0x1498 - SDRAM ODT Control high */
1168         reg_write(REG_SDRAM_ODT_CTRL_HIGH_ADDR, reg);
1169
1170         DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - ODT Asserted for current Cs\n");
1171
1172         /* tWLMRD Delay */
1173         /* Delay of minimum 40 Dram clock cycles - 20 Tclk cycles */
1174         udelay(1);
1175
1176         /* [1:0] - current cs number */
1177         reg = (reg_read(REG_TRAINING_WL_ADDR) & REG_TRAINING_WL_CS_MASK) | cs;
1178         reg |= (1 << REG_TRAINING_WL_UPD_OFFS); /* [2] - trnWLCsUpd */
1179         /* 0x16AC - Training Write leveling register */
1180         reg_write(REG_TRAINING_WL_ADDR, reg);
1181
1182         /* Broadcast to all PUPs: Reset DQS phase, reset leveling delay */
1183         ddr3_write_pup_reg(PUP_WL_MODE, cs, PUP_BC, 0, 0);
1184
1185         /* Seek Edge */
1186         DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge - Current Cs\n");
1187
1188         /* Drive DQS high for one cycle - All data PUPs */
1189         DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge - Driving DQS high for one cycle\n");
1190         if (!ratio_2to1) {
1191                 reg = (reg_read(REG_TRAINING_WL_ADDR) &
1192                        REG_TRAINING_WL_RATIO_MASK) | REG_TRAINING_WL_1TO1;
1193         } else {
1194                 reg = (reg_read(REG_TRAINING_WL_ADDR) &
1195                        REG_TRAINING_WL_RATIO_MASK) | REG_TRAINING_WL_2TO1;
1196         }
1197         /* 0x16AC - Training Write leveling register */
1198         reg_write(REG_TRAINING_WL_ADDR, reg);
1199
1200         /* Wait tWLdelay */
1201         do {
1202                 /* [29] - trnWLDelayExp */
1203                 reg = (reg_read(REG_TRAINING_WL_ADDR)) &
1204                         REG_TRAINING_WL_DELAYEXP_MASK;
1205         } while (reg == 0x0);   /* Wait for '1' */
1206
1207         /* Read WL res */
1208         reg = (reg_read(REG_TRAINING_WL_ADDR) >> REG_TRAINING_WL_RESULTS_OFFS) &
1209                 REG_TRAINING_WL_RESULTS_MASK;
1210         /* [28:20] - TrnWLResult */
1211
1212         if (!ratio_2to1) /* Different phase options for 2:1 or 1:1 modes */
1213                 phaseMax = MAX_PHASE_1TO1;
1214         else
1215                 phaseMax = MAX_PHASE_2TO1;
1216
1217         DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge - Shift DQS + Octet Leveling\n");
1218
1219         /* Shift DQS + Octet leveling */
1220         for (phase = 0; phase < phaseMax; phase++) {
1221                 for (delay = 0; delay < MAX_DELAY; delay++) {
1222                         /*  Broadcast to all PUPs: DQS phase,leveling delay */
1223                         ddr3_write_pup_reg(PUP_WL_MODE, cs, PUP_BC, phase,
1224                                            delay);
1225
1226                         udelay(1);      /* Delay of  3 Tclk cycles */
1227
1228                         DEBUG_WL_FULL_S("DDR3 - Write Leveling Single Cs - Seek Edge: Phase = ");
1229                         DEBUG_WL_FULL_D((u32) phase, 1);
1230                         DEBUG_WL_FULL_S(", Delay = ");
1231                         DEBUG_WL_FULL_D((u32) delay, 1);
1232                         DEBUG_WL_FULL_S(", Counter = ");
1233                         DEBUG_WL_FULL_D((u32) i, 1);
1234                         DEBUG_WL_FULL_S("\n");
1235
1236                         /* Drive DQS high for one cycle - All data PUPs */
1237                         if (!ratio_2to1) {
1238                                 reg = (reg_read(REG_TRAINING_WL_ADDR) &
1239                                        REG_TRAINING_WL_RATIO_MASK) |
1240                                         REG_TRAINING_WL_1TO1;
1241                         } else {
1242                                 reg = (reg_read(REG_TRAINING_WL_ADDR) &
1243                                        REG_TRAINING_WL_RATIO_MASK) |
1244                                         REG_TRAINING_WL_2TO1;
1245                         }
1246                         reg_write(REG_TRAINING_WL_ADDR, reg);   /* 0x16AC  */
1247
1248                         /* Wait tWLdelay */
1249                         do {
1250                                 reg = (reg_read(REG_TRAINING_WL_ADDR)) &
1251                                         REG_TRAINING_WL_DELAYEXP_MASK;
1252                         } while (reg == 0x0);   /* [29] Wait for '1' */
1253
1254                         /* Read WL res */
1255                         reg = reg_read(REG_TRAINING_WL_ADDR);
1256                         reg = (reg >> REG_TRAINING_WL_RESULTS_OFFS) &
1257                                 REG_TRAINING_WL_RESULTS_MASK;   /* [28:20] */
1258
1259                         DEBUG_WL_FULL_C("DDR3 - Write Leveling Single Cs - Seek Edge: Results =  ",
1260                                         (u32) reg, 3);
1261
1262                         /* Update State machine */
1263                         for (pup = 0; pup < (max_pup_num); pup++) {
1264                                 /* ECC support - bit 8 */
1265                                 pup_num = (pup == dram_info->num_of_std_pups) ?
1266                                         ECC_BIT : pup;
1267                                 if (dram_info->wl_val[cs][pup][S] == 0) {
1268                                         /* Update phase to PUP */
1269                                         dram_info->wl_val[cs][pup][P] = phase;
1270                                         /* Update delay to PUP */
1271                                         dram_info->wl_val[cs][pup][D] = delay;
1272                                 }
1273
1274                                 if (((reg >> pup_num) & 0x1) == 0)
1275                                         flag[pup_num] = 1;
1276
1277                                 if (((reg >> pup_num) & 0x1)
1278                                     && (flag[pup_num] == 1)
1279                                     && (dram_info->wl_val[cs][pup][S] == 0)) {
1280                                         /*
1281                                          * If the PUP is locked now and in last
1282                                          * counter states
1283                                          */
1284                                         /* Go to next state */
1285                                         dram_info->wl_val[cs][pup][S] = 1;
1286                                         /* Set res */
1287                                         *result = *result | (1 << pup_num);
1288                                 }
1289                         }
1290
1291                         /* If all locked - Break the loops - Finished */
1292                         if (*result == max_pup_mask) {
1293                                 phase = phaseMax;
1294                                 delay = MAX_DELAY;
1295                                 DEBUG_WL_S("DDR3 - Write Leveling Single Cs - Seek Edge: All Locked\n");
1296                         }
1297                 }
1298         }
1299
1300         /* Debug message - Print res for cs[i]: cs,PUP,Phase,Delay */
1301         DEBUG_WL_C("DDR3 - Write Leveling - Results for CS - ", (u32) cs, 1);
1302         for (pup = 0; pup < (max_pup_num); pup++) {
1303                 DEBUG_WL_S("DDR3 - Write Leveling - PUP: ");
1304                 DEBUG_WL_D((u32) pup, 1);
1305                 DEBUG_WL_S(", Phase: ");
1306                 DEBUG_WL_D((u32) dram_info->wl_val[cs][pup][P], 1);
1307                 DEBUG_WL_S(", Delay: ");
1308                 DEBUG_WL_D((u32) dram_info->wl_val[cs][pup][D], 2);
1309                 DEBUG_WL_S("\n");
1310         }
1311
1312         /* Check if some not locked and return error */
1313         if (*result != max_pup_mask) {
1314                 DEBUG_WL_S("DDR3 - Write Leveling - ERROR - not all PUPS were locked\n");
1315                 return MV_FAIL;
1316         }
1317
1318         /* Configure Each PUP with locked leveling settings */
1319         for (pup = 0; pup < (max_pup_num); pup++) {
1320                 /* ECC support - bit 8 */
1321                 pup_num = (pup == dram_info->num_of_std_pups) ? ECC_BIT : pup;
1322                 phase = dram_info->wl_val[cs][pup][P];
1323                 delay = dram_info->wl_val[cs][pup][D];
1324                 ddr3_write_pup_reg(PUP_WL_MODE, cs, pup_num, phase, delay);
1325         }
1326
1327         /* CS ODT Override */
1328         reg =  reg_read(REG_SDRAM_ODT_CTRL_HIGH_ADDR) &
1329                 REG_SDRAM_ODT_CTRL_HIGH_OVRD_MASK;
1330         /* 0x1498 - SDRAM ODT Control high */
1331         reg_write(REG_SDRAM_ODT_CTRL_HIGH_ADDR, reg);
1332
1333         return MV_OK;
1334 }
1335
1336 /*
1337  * Perform DDR3 Control PUP Indirect Write
1338  */
1339 static void ddr3_write_ctrl_pup_reg(int bc_acc, u32 pup, u32 reg_addr, u32 data)
1340 {
1341         u32 reg = 0;
1342
1343         /* Store value for write */
1344         reg = (data & 0xFFFF);
1345
1346         /* Set bit 26 for control PHY access */
1347         reg |= (1 << REG_PHY_CNTRL_OFFS);
1348
1349         /* Configure BC or UC access to PHYs */
1350         if (bc_acc == 1)
1351                 reg |= (1 << REG_PHY_BC_OFFS);
1352         else
1353                 reg |= (pup << REG_PHY_PUP_OFFS);
1354
1355         /* Set PHY register address to write to */
1356         reg |= (reg_addr << REG_PHY_CS_OFFS);
1357
1358         reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg);      /* 0x16A0 */
1359         reg |= REG_PHY_REGISTRY_FILE_ACCESS_OP_WR;
1360         reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg);      /* 0x16A0 */
1361
1362         do {
1363                 reg = (reg_read(REG_PHY_REGISTRY_FILE_ACCESS_ADDR)) &
1364                         REG_PHY_REGISTRY_FILE_ACCESS_OP_DONE;
1365         } while (reg);          /* Wait for '0' to mark the end of the transaction */
1366 }