git.sur5r.net Git - u-boot/blob - drivers/ddr/marvell/axp/ddr3_read_leveling.c

   1 /*
   2  * Copyright (C) Marvell International Ltd. and its affiliates
   3  *
   4  * SPDX-License-Identifier:     GPL-2.0
   5  */
   6
   7 #include <common.h>
   8 #include <i2c.h>
   9 #include <spl.h>
  10 #include <asm/io.h>
  11 #include <asm/arch/cpu.h>
  12 #include <asm/arch/soc.h>
  13
  14 #include "ddr3_hw_training.h"
  15
  16 /*
  17  * Debug
  18  */
  19 #define DEBUG_RL_C(s, d, l) \
  20         DEBUG_RL_S(s); DEBUG_RL_D(d, l); DEBUG_RL_S("\n")
  21 #define DEBUG_RL_FULL_C(s, d, l) \
  22         DEBUG_RL_FULL_S(s); DEBUG_RL_FULL_D(d, l); DEBUG_RL_FULL_S("\n")
  23
  24 #ifdef MV_DEBUG_RL
  25 #define DEBUG_RL_S(s) \
  26         debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%s", s)
  27 #define DEBUG_RL_D(d, l) \
  28         debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%x", d)
  29 #else
  30 #define DEBUG_RL_S(s)
  31 #define DEBUG_RL_D(d, l)
  32 #endif
  33
  34 #ifdef MV_DEBUG_RL_FULL
  35 #define DEBUG_RL_FULL_S(s)              puts(s)
  36 #define DEBUG_RL_FULL_D(d, l)           printf("%x", d)
  37 #else
  38 #define DEBUG_RL_FULL_S(s)
  39 #define DEBUG_RL_FULL_D(d, l)
  40 #endif
  41
  42 extern u32 rl_pattern[LEN_STD_PATTERN];
  43
  44 #ifdef RL_MODE
  45 static int ddr3_read_leveling_single_cs_rl_mode(u32 cs, u32 freq,
  46                                                 int ratio_2to1, u32 ecc,
  47                                                 MV_DRAM_INFO *dram_info);
  48 #else
  49 static int ddr3_read_leveling_single_cs_window_mode(u32 cs, u32 freq,
  50                                                     int ratio_2to1, u32 ecc,
  51                                                     MV_DRAM_INFO *dram_info);
  52 #endif
  53
  54 /*
  55  * Name:     ddr3_read_leveling_hw
  56  * Desc:     Execute the Read leveling phase by HW
  57  * Args:     dram_info - main struct
  58  *           freq      - current sequence frequency
  59  * Notes:
  60  * Returns:  MV_OK if success, MV_FAIL if fail.
  61  */
  62 int ddr3_read_leveling_hw(u32 freq, MV_DRAM_INFO *dram_info)
  63 {
  64         u32 reg;
  65
  66         /* Debug message - Start Read leveling procedure */
  67         DEBUG_RL_S("DDR3 - Read Leveling - Starting HW RL procedure\n");
  68
  69         /* Start Auto Read Leveling procedure */
  70         reg = 1 << REG_DRAM_TRAINING_RL_OFFS;
  71         /* Config the retest number */
  72         reg |= (COUNT_HW_RL << REG_DRAM_TRAINING_RETEST_OFFS);
  73
  74         /* Enable CS in the automatic process */
  75         reg |= (dram_info->cs_ena << REG_DRAM_TRAINING_CS_OFFS);
  76
  77         reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
  78
  79         reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) |
  80                 (1 << REG_DRAM_TRAINING_AUTO_OFFS);
  81         reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg);
  82
  83         /* Wait */
  84         do {
  85                 reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
  86                         (1 << REG_DRAM_TRAINING_AUTO_OFFS);
  87         } while (reg);          /* Wait for '0' */
  88
  89         /* Check if Successful */
  90         if (reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
  91             (1 << REG_DRAM_TRAINING_ERROR_OFFS)) {
  92                 u32 delay, phase, pup, cs;
  93
  94                 dram_info->rl_max_phase = 0;
  95                 dram_info->rl_min_phase = 10;
  96
  97                 /* Read results to arrays */
  98                 for (cs = 0; cs < MAX_CS; cs++) {
  99                         if (dram_info->cs_ena & (1 << cs)) {
 100                                 for (pup = 0;
 101                                      pup < dram_info->num_of_total_pups;
 102                                      pup++) {
 103                                         if (pup == dram_info->num_of_std_pups
 104                                             && dram_info->ecc_ena)
 105                                                 pup = ECC_PUP;
 106                                         reg =
 107                                             ddr3_read_pup_reg(PUP_RL_MODE, cs,
 108                                                               pup);
 109                                         phase = (reg >> REG_PHY_PHASE_OFFS) &
 110                                                 PUP_PHASE_MASK;
 111                                         delay = reg & PUP_DELAY_MASK;
 112                                         dram_info->rl_val[cs][pup][P] = phase;
 113                                         if (phase > dram_info->rl_max_phase)
 114                                                 dram_info->rl_max_phase = phase;
 115                                         if (phase < dram_info->rl_min_phase)
 116                                                 dram_info->rl_min_phase = phase;
 117                                         dram_info->rl_val[cs][pup][D] = delay;
 118                                         dram_info->rl_val[cs][pup][S] =
 119                                             RL_FINAL_STATE;
 120                                         reg =
 121                                             ddr3_read_pup_reg(PUP_RL_MODE + 0x1,
 122                                                               cs, pup);
 123                                         dram_info->rl_val[cs][pup][DQS] =
 124                                             (reg & 0x3F);
 125                                 }
 126 #ifdef MV_DEBUG_RL
 127                                 /* Print results */
 128                                 DEBUG_RL_C("DDR3 - Read Leveling - Results for CS - ",
 129                                            (u32) cs, 1);
 130
 131                                 for (pup = 0;
 132                                      pup < (dram_info->num_of_total_pups);
 133                                      pup++) {
 134                                         if (pup == dram_info->num_of_std_pups
 135                                             && dram_info->ecc_ena)
 136                                                 pup = ECC_PUP;
 137                                         DEBUG_RL_S("DDR3 - Read Leveling - PUP: ");
 138                                         DEBUG_RL_D((u32) pup, 1);
 139                                         DEBUG_RL_S(", Phase: ");
 140                                         DEBUG_RL_D((u32) dram_info->
 141                                                    rl_val[cs][pup][P], 1);
 142                                         DEBUG_RL_S(", Delay: ");
 143                                         DEBUG_RL_D((u32) dram_info->
 144                                                    rl_val[cs][pup][D], 2);
 145                                         DEBUG_RL_S("\n");
 146                                 }
 147 #endif
 148                         }
 149                 }
 150
 151                 dram_info->rd_rdy_dly =
 152                         reg_read(REG_READ_DATA_READY_DELAYS_ADDR) &
 153                         REG_READ_DATA_SAMPLE_DELAYS_MASK;
 154                 dram_info->rd_smpl_dly =
 155                         reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR) &
 156                         REG_READ_DATA_READY_DELAYS_MASK;
 157 #ifdef MV_DEBUG_RL
 158                 DEBUG_RL_C("DDR3 - Read Leveling - Read Sample Delay: ",
 159                            dram_info->rd_smpl_dly, 2);
 160                 DEBUG_RL_C("DDR3 - Read Leveling - Read Ready Delay: ",
 161                            dram_info->rd_rdy_dly, 2);
 162                 DEBUG_RL_S("DDR3 - Read Leveling - HW RL Ended Successfully\n");
 163 #endif
 164                 return MV_OK;
 165
 166         } else {
 167                 DEBUG_RL_S("DDR3 - Read Leveling - HW RL Error\n");
 168                 return MV_FAIL;
 169         }
 170 }
 171
 172 /*
 173  * Name:     ddr3_read_leveling_sw
 174  * Desc:     Execute the Read leveling phase by SW
 175  * Args:     dram_info - main struct
 176  *           freq      - current sequence frequency
 177  * Notes:
 178  * Returns:  MV_OK if success, MV_FAIL if fail.
 179  */
 180 int ddr3_read_leveling_sw(u32 freq, int ratio_2to1, MV_DRAM_INFO *dram_info)
 181 {
 182         u32 reg, cs, ecc, pup_num, phase, delay, pup;
 183         int status;
 184
 185         /* Debug message - Start Read leveling procedure */
 186         DEBUG_RL_S("DDR3 - Read Leveling - Starting SW RL procedure\n");
 187
 188         /* Enable SW Read Leveling */
 189         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 190                 (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 191         reg &= ~(1 << REG_DRAM_TRAINING_2_RL_MODE_OFFS);
 192         /* [0]=1 - Enable SW override  */
 193         /* 0x15B8 - Training SW 2 Register */
 194         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 195
 196 #ifdef RL_MODE
 197         reg = (dram_info->cs_ena << REG_DRAM_TRAINING_CS_OFFS) |
 198                 (1 << REG_DRAM_TRAINING_AUTO_OFFS);
 199         reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */
 200 #endif
 201
 202         /* Loop for each CS */
 203         for (cs = 0; cs < dram_info->num_cs; cs++) {
 204                 DEBUG_RL_C("DDR3 - Read Leveling - CS - ", (u32) cs, 1);
 205
 206                 for (ecc = 0; ecc <= (dram_info->ecc_ena); ecc++) {
 207                         /* ECC Support - Switch ECC Mux on ecc=1 */
 208                         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 209                                 ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 210                         reg |= (dram_info->ecc_ena *
 211                                 ecc << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 212                         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 213
 214                         if (ecc)
 215                                 DEBUG_RL_S("DDR3 - Read Leveling - ECC Mux Enabled\n");
 216                         else
 217                                 DEBUG_RL_S("DDR3 - Read Leveling - ECC Mux Disabled\n");
 218
 219                         /* Set current sample delays */
 220                         reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
 221                         reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK <<
 222                                  (REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs));
 223                         reg |= (dram_info->cl <<
 224                                 (REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs));
 225                         reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR, reg);
 226
 227                         /* Set current Ready delay */
 228                         reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
 229                         reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
 230                                  (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 231                         if (!ratio_2to1) {
 232                                 /* 1:1 mode */
 233                                 reg |= ((dram_info->cl + 1) <<
 234                                         (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 235                         } else {
 236                                 /* 2:1 mode */
 237                                 reg |= ((dram_info->cl + 2) <<
 238                                         (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 239                         }
 240                         reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
 241
 242                         /* Read leveling Single CS[cs] */
 243 #ifdef RL_MODE
 244                         status =
 245                             ddr3_read_leveling_single_cs_rl_mode(cs, freq,
 246                                                                  ratio_2to1,
 247                                                                  ecc,
 248                                                                  dram_info);
 249                         if (MV_OK != status)
 250                                 return status;
 251 #else
 252                         status =
 253                             ddr3_read_leveling_single_cs_window_mode(cs, freq,
 254                                                                      ratio_2to1,
 255                                                                      ecc,
 256                                                                      dram_info)
 257                             if (MV_OK != status)
 258                                 return status;
 259 #endif
 260                 }
 261
 262                 /* Print results */
 263                 DEBUG_RL_C("DDR3 - Read Leveling - Results for CS - ", (u32) cs,
 264                            1);
 265
 266                 for (pup = 0;
 267                      pup < (dram_info->num_of_std_pups + dram_info->ecc_ena);
 268                      pup++) {
 269                         DEBUG_RL_S("DDR3 - Read Leveling - PUP: ");
 270                         DEBUG_RL_D((u32) pup, 1);
 271                         DEBUG_RL_S(", Phase: ");
 272                         DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][P], 1);
 273                         DEBUG_RL_S(", Delay: ");
 274                         DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][D], 2);
 275                         DEBUG_RL_S("\n");
 276                 }
 277
 278                 DEBUG_RL_C("DDR3 - Read Leveling - Read Sample Delay: ",
 279                            dram_info->rd_smpl_dly, 2);
 280                 DEBUG_RL_C("DDR3 - Read Leveling - Read Ready Delay: ",
 281                            dram_info->rd_rdy_dly, 2);
 282
 283                 /* Configure PHY with average of 3 locked leveling settings */
 284                 for (pup = 0;
 285                      pup < (dram_info->num_of_std_pups + dram_info->ecc_ena);
 286                      pup++) {
 287                         /* ECC support - bit 8 */
 288                         pup_num = (pup == dram_info->num_of_std_pups) ? ECC_BIT : pup;
 289
 290                         /* For now, set last cnt result */
 291                         phase = dram_info->rl_val[cs][pup][P];
 292                         delay = dram_info->rl_val[cs][pup][D];
 293                         ddr3_write_pup_reg(PUP_RL_MODE, cs, pup_num, phase,
 294                                            delay);
 295                 }
 296         }
 297
 298         /* Reset PHY read FIFO */
 299         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 300                 (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 301         /* 0x15B8 - Training SW 2 Register */
 302         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 303
 304         do {
 305                 reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
 306                         (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 307         } while (reg);          /* Wait for '0' */
 308
 309         /* ECC Support - Switch ECC Mux off ecc=0 */
 310         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 311                 ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
 312         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 313
 314 #ifdef RL_MODE
 315         reg_write(REG_DRAM_TRAINING_ADDR, 0);   /* 0x15B0 - Training Register */
 316 #endif
 317
 318         /* Disable SW Read Leveling */
 319         reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
 320                 ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
 321         /* [0] = 0 - Disable SW override  */
 322         reg = (reg | (0x1 << REG_DRAM_TRAINING_2_RL_MODE_OFFS));
 323         /* [3] = 1 - Disable RL MODE */
 324         /* 0x15B8 - Training SW 2 Register */
 325         reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 326
 327         DEBUG_RL_S("DDR3 - Read Leveling - Finished RL procedure for all CS\n");
 328         return MV_OK;
 329 }
 330
 331 #ifdef RL_MODE
 332 /*
 333  * overrun() extracted from ddr3_read_leveling_single_cs_rl_mode().
 334  * This just got too much indented making it hard to read / edit.
 335  */
 336 static void overrun(u32 cs, MV_DRAM_INFO *info, u32 pup, u32 locked_pups,
 337                     u32 *locked_sum, u32 ecc, int *first_octet_locked,
 338                     int *counter_in_progress, int final_delay, u32 delay,
 339                     u32 phase)
 340 {
 341         /* If no OverRun */
 342         if (((~locked_pups >> pup) & 0x1) && (final_delay == 0)) {
 343                 int idx;
 344
 345                 idx = pup + ecc * ECC_BIT;
 346
 347                 /* PUP passed, start examining */
 348                 if (info->rl_val[cs][idx][S] == RL_UNLOCK_STATE) {
 349                         /* Must be RL_UNLOCK_STATE */
 350                         /* Match expected value ? - Update State Machine */
 351                         if (info->rl_val[cs][idx][C] < RL_RETRY_COUNT) {
 352                                 DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have no overrun and a match on pup: ",
 353                                                 (u32)pup, 1);
 354                                 info->rl_val[cs][idx][C]++;
 355
 356                                 /* If pup got to last state - lock the delays */
 357                                 if (info->rl_val[cs][idx][C] == RL_RETRY_COUNT) {
 358                                         info->rl_val[cs][idx][C] = 0;
 359                                         info->rl_val[cs][idx][DS] = delay;
 360                                         info->rl_val[cs][idx][PS] = phase;
 361
 362                                         /* Go to Final State */
 363                                         info->rl_val[cs][idx][S] = RL_FINAL_STATE;
 364                                         *locked_sum = *locked_sum + 1;
 365                                         DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have locked pup: ",
 366                                                         (u32)pup, 1);
 367
 368                                         /*
 369                                          * If first lock - need to lock delays
 370                                          */
 371                                         if (*first_octet_locked == 0) {
 372                                                 DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got first lock on pup: ",
 373                                                                 (u32)pup, 1);
 374                                                 *first_octet_locked = 1;
 375                                         }
 376
 377                                         /*
 378                                          * If pup is in not in final state but
 379                                          * there was match - dont increment
 380                                          * counter
 381                                          */
 382                                 } else {
 383                                         *counter_in_progress = 1;
 384                                 }
 385                         }
 386                 }
 387         }
 388 }
 389
 390 /*
 391  * Name:     ddr3_read_leveling_single_cs_rl_mode
 392  * Desc:     Execute Read leveling for single Chip select
 393  * Args:     cs        - current chip select
 394  *           freq      - current sequence frequency
 395  *           ecc       - ecc iteration indication
 396  *           dram_info - main struct
 397  * Notes:
 398  * Returns:  MV_OK if success, MV_FAIL if fail.
 399  */
 400 static int ddr3_read_leveling_single_cs_rl_mode(u32 cs, u32 freq,
 401                                                 int ratio_2to1, u32 ecc,
 402                                                 MV_DRAM_INFO *dram_info)
 403 {
 404         u32 reg, delay, phase, pup, rd_sample_delay, add, locked_pups,
 405                 repeat_max_cnt, sdram_offset, locked_sum;
 406         u32 phase_min, ui_max_delay;
 407         int all_locked, first_octet_locked, counter_in_progress;
 408         int final_delay = 0;
 409
 410         DEBUG_RL_FULL_C("DDR3 - Read Leveling - Single CS - ", (u32) cs, 1);
 411
 412         /* Init values */
 413         phase = 0;
 414         delay = 0;
 415         rd_sample_delay = dram_info->cl;
 416         all_locked = 0;
 417         first_octet_locked = 0;
 418         repeat_max_cnt = 0;
 419         locked_sum = 0;
 420
 421         for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
 422              pup++)
 423                 dram_info->rl_val[cs][pup + ecc * ECC_BIT][S] = 0;
 424
 425         /* Main loop */
 426         while (!all_locked) {
 427                 counter_in_progress = 0;
 428
 429                 DEBUG_RL_FULL_S("DDR3 - Read Leveling - RdSmplDly = ");
 430                 DEBUG_RL_FULL_D(rd_sample_delay, 2);
 431                 DEBUG_RL_FULL_S(", RdRdyDly = ");
 432                 DEBUG_RL_FULL_D(dram_info->rd_rdy_dly, 2);
 433                 DEBUG_RL_FULL_S(", Phase = ");
 434                 DEBUG_RL_FULL_D(phase, 1);
 435                 DEBUG_RL_FULL_S(", Delay = ");
 436                 DEBUG_RL_FULL_D(delay, 2);
 437                 DEBUG_RL_FULL_S("\n");
 438
 439                 /*
 440                  * Broadcast to all PUPs current RL delays: DQS phase,
 441                  * leveling delay
 442                  */
 443                 ddr3_write_pup_reg(PUP_RL_MODE, cs, PUP_BC, phase, delay);
 444
 445                 /* Reset PHY read FIFO */
 446                 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 447                         (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 448                 /* 0x15B8 - Training SW 2 Register */
 449                 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 450
 451                 do {
 452                         reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
 453                                 (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 454                 } while (reg);  /* Wait for '0' */
 455
 456                 /* Read pattern from SDRAM */
 457                 sdram_offset = cs * (SDRAM_CS_SIZE + 1) + SDRAM_RL_OFFS;
 458                 locked_pups = 0;
 459                 if (MV_OK !=
 460                     ddr3_sdram_compare(dram_info, 0xFF, &locked_pups,
 461                                        rl_pattern, LEN_STD_PATTERN,
 462                                        sdram_offset, 0, 0, NULL, 0))
 463                         return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PATTERN;
 464
 465                 /* Octet evaluation */
 466                 /* pup_num = Q or 1 for ECC */
 467                 for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
 468                         /* Check Overrun */
 469                         if (!((reg_read(REG_DRAM_TRAINING_2_ADDR) >>
 470                                (REG_DRAM_TRAINING_2_OVERRUN_OFFS + pup)) & 0x1)) {
 471                                 overrun(cs, dram_info, pup, locked_pups,
 472                                         &locked_sum, ecc, &first_octet_locked,
 473                                         &counter_in_progress, final_delay,
 474                                         delay, phase);
 475                         } else {
 476                                 DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got overrun on pup: ",
 477                                                 (u32)pup, 1);
 478                         }
 479                 }
 480
 481                 if (locked_sum == (dram_info->num_of_std_pups *
 482                                    (1 - ecc) + ecc)) {
 483                         all_locked = 1;
 484                         DEBUG_RL_FULL_S("DDR3 - Read Leveling - Single Cs - All pups locked\n");
 485                 }
 486
 487                 /*
 488                  * This is a fix for unstable condition where pups are
 489                  * toggling between match and no match
 490                  */
 491                 /*
 492                  * If some of the pups is >1 <3, check if we did it too
 493                  * many times
 494                  */
 495                 if (counter_in_progress == 1) {
 496                         /* Notify at least one Counter is >=1 and < 3 */
 497                         if (repeat_max_cnt < RL_RETRY_COUNT) {
 498                                 repeat_max_cnt++;
 499                                 counter_in_progress = 1;
 500                                 DEBUG_RL_FULL_S("DDR3 - Read Leveling - Counter is >=1 and <3\n");
 501                                 DEBUG_RL_FULL_S("DDR3 - Read Leveling - So we will not increment the delay to see if locked again\n");
 502                         } else {
 503                                 DEBUG_RL_FULL_S("DDR3 - Read Leveling - repeat_max_cnt reached max so now we will increment the delay\n");
 504                                 counter_in_progress = 0;
 505                         }
 506                 }
 507
 508                 /*
 509                  * Check some of the pups are in the middle of state machine
 510                  * and don't increment the delays
 511                  */
 512                 if (!counter_in_progress && !all_locked) {
 513                         int idx;
 514
 515                         idx = pup + ecc * ECC_BIT;
 516
 517                         repeat_max_cnt = 0;
 518                         /* if 1:1 mode */
 519                         if ((!ratio_2to1) && ((phase == 0) || (phase == 4)))
 520                                 ui_max_delay = MAX_DELAY_INV;
 521                         else
 522                                 ui_max_delay = MAX_DELAY;
 523
 524                         /* Increment Delay */
 525                         if (delay < ui_max_delay) {
 526                                 delay++;
 527                                 /*
 528                                  * Mark the last delay/pahse place for
 529                                  * window final place
 530                                  */
 531                                 if (delay == ui_max_delay) {
 532                                         if ((!ratio_2to1 && phase ==
 533                                              MAX_PHASE_RL_L_1TO1)
 534                                             || (ratio_2to1 && phase ==
 535                                                 MAX_PHASE_RL_L_2TO1))
 536                                                 final_delay = 1;
 537                                 }
 538                         } else {
 539                                 /* Phase+CL Incrementation */
 540                                 delay = 0;
 541
 542                                 if (!ratio_2to1) {
 543                                         /* 1:1 mode */
 544                                         if (first_octet_locked) {
 545                                                 /* some Pup was Locked */
 546                                                 if (phase < MAX_PHASE_RL_L_1TO1) {
 547                                                         if (phase == 1) {
 548                                                                 phase = 4;
 549                                                         } else {
 550                                                                 phase++;
 551                                                                 delay = MIN_DELAY_PHASE_1_LIMIT;
 552                                                         }
 553                                                 } else {
 554                                                         DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 555                                                         DEBUG_RL_S("1)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked n");
 556                                                         return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PUP_UNLOCK;
 557                                                 }
 558                                         } else {
 559                                                 /* NO Pup was Locked */
 560                                                 if (phase < MAX_PHASE_RL_UL_1TO1) {
 561                                                         phase++;
 562                                                         delay =
 563                                                             MIN_DELAY_PHASE_1_LIMIT;
 564                                                 } else {
 565                                                         phase = 0;
 566                                                 }
 567                                         }
 568                                 } else {
 569                                         /* 2:1 mode */
 570                                         if (first_octet_locked) {
 571                                                 /* some Pup was Locked */
 572                                                 if (phase < MAX_PHASE_RL_L_2TO1) {
 573                                                         phase++;
 574                                                 } else {
 575                                                         DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 576                                                         DEBUG_RL_S("2)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 577                                                         for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
 578                                                                 /* pup_num = Q or 1 for ECC */
 579                                                                 if (dram_info->rl_val[cs][idx][S]
 580                                                                     == 0) {
 581                                                                         DEBUG_RL_C("Failed byte is = ",
 582                                                                                    pup, 1);
 583                                                                 }
 584                                                         }
 585                                                         return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PUP_UNLOCK;
 586                                                 }
 587                                         } else {
 588                                                 /* No Pup was Locked */
 589                                                 if (phase < MAX_PHASE_RL_UL_2TO1)
 590                                                         phase++;
 591                                                 else
 592                                                         phase = 0;
 593                                         }
 594                                 }
 595
 596                                 /*
 597                                  * If we finished a full Phases cycle (so now
 598                                  * phase = 0, need to increment rd_sample_dly
 599                                  */
 600                                 if (phase == 0 && first_octet_locked == 0) {
 601                                         rd_sample_delay++;
 602                                         if (rd_sample_delay == 0x10) {
 603                                                 DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 604                                                 DEBUG_RL_S("3)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 605                                                 for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
 606                                                         /* pup_num = Q or 1 for ECC */
 607                                                         if (dram_info->
 608                                                             rl_val[cs][idx][S] == 0) {
 609                                                                 DEBUG_RL_C("Failed byte is = ",
 610                                                                            pup, 1);
 611                                                         }
 612                                                 }
 613                                                 return MV_DDR3_TRAINING_ERR_RD_LVL_PUP_UNLOCK;
 614                                         }
 615
 616                                         /* Set current rd_sample_delay  */
 617                                         reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
 618                                         reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK
 619                                               << (REG_READ_DATA_SAMPLE_DELAYS_OFFS
 620                                                   * cs));
 621                                         reg |= (rd_sample_delay <<
 622                                                 (REG_READ_DATA_SAMPLE_DELAYS_OFFS *
 623                                                  cs));
 624                                         reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR,
 625                                                   reg);
 626                                 }
 627
 628                                 /*
 629                                  * Set current rdReadyDelay according to the
 630                                  * hash table (Need to do this in every phase
 631                                  * change)
 632                                  */
 633                                 if (!ratio_2to1) {
 634                                         /* 1:1 mode */
 635                                         add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
 636                                         switch (phase) {
 637                                         case 0:
 638                                                 add = (add >>
 639                                                        REG_TRAINING_DEBUG_2_OFFS);
 640                                                 break;
 641                                         case 1:
 642                                                 add = (add >>
 643                                                        (REG_TRAINING_DEBUG_2_OFFS
 644                                                         + 3));
 645                                                 break;
 646                                         case 4:
 647                                                 add = (add >>
 648                                                        (REG_TRAINING_DEBUG_2_OFFS
 649                                                         + 6));
 650                                                 break;
 651                                         case 5:
 652                                                 add = (add >>
 653                                                        (REG_TRAINING_DEBUG_2_OFFS
 654                                                         + 9));
 655                                                 break;
 656                                         }
 657                                         add &= REG_TRAINING_DEBUG_2_MASK;
 658                                 } else {
 659                                         /* 2:1 mode */
 660                                         add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
 661                                         add = (add >>
 662                                                (phase *
 663                                                 REG_TRAINING_DEBUG_3_OFFS));
 664                                         add &= REG_TRAINING_DEBUG_3_MASK;
 665                                 }
 666
 667                                 reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
 668                                 reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
 669                                          (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 670                                 reg |= ((rd_sample_delay + add) <<
 671                                         (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 672                                 reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
 673                                 dram_info->rd_smpl_dly = rd_sample_delay;
 674                                 dram_info->rd_rdy_dly = rd_sample_delay + add;
 675                         }
 676
 677                         /* Reset counters for pups with states<RD_STATE_COUNT */
 678                         for (pup = 0; pup <
 679                                      (dram_info->num_of_std_pups * (1 - ecc) + ecc);
 680                              pup++) {
 681                                 if (dram_info->rl_val[cs][idx][C] < RL_RETRY_COUNT)
 682                                         dram_info->rl_val[cs][idx][C] = 0;
 683                         }
 684                 }
 685         }
 686
 687         phase_min = 10;
 688
 689         for (pup = 0; pup < (dram_info->num_of_std_pups); pup++) {
 690                 if (dram_info->rl_val[cs][pup][PS] < phase_min)
 691                         phase_min = dram_info->rl_val[cs][pup][PS];
 692         }
 693
 694         /*
 695          * Set current rdReadyDelay according to the hash table (Need to
 696          * do this in every phase change)
 697          */
 698         if (!ratio_2to1) {
 699                 /* 1:1 mode */
 700                 add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
 701                 switch (phase_min) {
 702                 case 0:
 703                         add = (add >> REG_TRAINING_DEBUG_2_OFFS);
 704                         break;
 705                 case 1:
 706                         add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 3));
 707                         break;
 708                 case 4:
 709                         add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 6));
 710                         break;
 711                 case 5:
 712                         add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 9));
 713                         break;
 714                 }
 715                 add &= REG_TRAINING_DEBUG_2_MASK;
 716         } else {
 717                 /* 2:1 mode */
 718                 add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
 719                 add = (add >> (phase_min * REG_TRAINING_DEBUG_3_OFFS));
 720                 add &= REG_TRAINING_DEBUG_3_MASK;
 721         }
 722
 723         reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
 724         reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
 725                  (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 726         reg |= ((rd_sample_delay + add) << (REG_READ_DATA_READY_DELAYS_OFFS * cs));
 727         reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
 728         dram_info->rd_rdy_dly = rd_sample_delay + add;
 729
 730         for (cs = 0; cs < dram_info->num_cs; cs++) {
 731                 for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
 732                         reg = ddr3_read_pup_reg(PUP_RL_MODE + 0x1, cs, pup);
 733                         dram_info->rl_val[cs][pup][DQS] = (reg & 0x3F);
 734                 }
 735         }
 736
 737         return MV_OK;
 738 }
 739
 740 #else
 741
 742 /*
 743  * Name:     ddr3_read_leveling_single_cs_window_mode
 744  * Desc:     Execute Read leveling for single Chip select
 745  * Args:     cs        - current chip select
 746  *           freq      - current sequence frequency
 747  *           ecc       - ecc iteration indication
 748  *           dram_info - main struct
 749  * Notes:
 750  * Returns:  MV_OK if success, MV_FAIL if fail.
 751  */
 752 static int ddr3_read_leveling_single_cs_window_mode(u32 cs, u32 freq,
 753                                                     int ratio_2to1, u32 ecc,
 754                                                     MV_DRAM_INFO *dram_info)
 755 {
 756         u32 reg, delay, phase, sum, pup, rd_sample_delay, add, locked_pups,
 757             repeat_max_cnt, sdram_offset, final_sum, locked_sum;
 758         u32 delay_s, delay_e, tmp, phase_min, ui_max_delay;
 759         int all_locked, first_octet_locked, counter_in_progress;
 760         int final_delay = 0;
 761
 762         DEBUG_RL_FULL_C("DDR3 - Read Leveling - Single CS - ", (u32) cs, 1);
 763
 764         /* Init values */
 765         phase = 0;
 766         delay = 0;
 767         rd_sample_delay = dram_info->cl;
 768         all_locked = 0;
 769         first_octet_locked = 0;
 770         repeat_max_cnt = 0;
 771         sum = 0;
 772         final_sum = 0;
 773         locked_sum = 0;
 774
 775         for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
 776              pup++)
 777                 dram_info->rl_val[cs][pup + ecc * ECC_BIT][S] = 0;
 778
 779         /* Main loop */
 780         while (!all_locked) {
 781                 counter_in_progress = 0;
 782
 783                 DEBUG_RL_FULL_S("DDR3 - Read Leveling - RdSmplDly = ");
 784                 DEBUG_RL_FULL_D(rd_sample_delay, 2);
 785                 DEBUG_RL_FULL_S(", RdRdyDly = ");
 786                 DEBUG_RL_FULL_D(dram_info->rd_rdy_dly, 2);
 787                 DEBUG_RL_FULL_S(", Phase = ");
 788                 DEBUG_RL_FULL_D(phase, 1);
 789                 DEBUG_RL_FULL_S(", Delay = ");
 790                 DEBUG_RL_FULL_D(delay, 2);
 791                 DEBUG_RL_FULL_S("\n");
 792
 793                 /*
 794                  * Broadcast to all PUPs current RL delays: DQS phase,leveling
 795                  * delay
 796                  */
 797                 ddr3_write_pup_reg(PUP_RL_MODE, cs, PUP_BC, phase, delay);
 798
 799                 /* Reset PHY read FIFO */
 800                 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
 801                         (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 802                 /* 0x15B8 - Training SW 2 Register */
 803                 reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
 804
 805                 do {
 806                         reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
 807                                 (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
 808                 } while (reg);  /* Wait for '0' */
 809
 810                 /* Read pattern from SDRAM */
 811                 sdram_offset = cs * (SDRAM_CS_SIZE + 1) + SDRAM_RL_OFFS;
 812                 locked_pups = 0;
 813                 if (MV_OK !=
 814                     ddr3_sdram_compare(dram_info, 0xFF, &locked_pups,
 815                                        rl_pattern, LEN_STD_PATTERN,
 816                                        sdram_offset, 0, 0, NULL, 0))
 817                         return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PATTERN;
 818
 819                 /* Octet evaluation */
 820                 for (pup = 0; pup < (dram_info->num_of_std_pups *
 821                                      (1 - ecc) + ecc); pup++) {
 822                         /* pup_num = Q or 1 for ECC */
 823                         int idx;
 824
 825                         idx = pup + ecc * ECC_BIT;
 826
 827                         /* Check Overrun */
 828                         if (!((reg_read(REG_DRAM_TRAINING_2_ADDR) >>
 829                               (REG_DRAM_TRAINING_2_OVERRUN_OFFS +
 830                                pup)) & 0x1)) {
 831                                 /* If no OverRun */
 832
 833                                 /* Inside the window */
 834                                 if (dram_info->rl_val[cs][idx][S] == RL_WINDOW_STATE) {
 835                                         /*
 836                                          * Match expected value ? - Update
 837                                          * State Machine
 838                                          */
 839                                         if (((~locked_pups >> pup) & 0x1)
 840                                             && (final_delay == 0)) {
 841                                                 /* Match - Still inside the Window */
 842                                                 DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got another match inside the window  for pup: ",
 843                                                                 (u32)pup, 1);
 844
 845                                         } else {
 846                                                 /* We got fail -> this is the end of the window */
 847                                                 dram_info->rl_val[cs][idx][DE] = delay;
 848                                                 dram_info->rl_val[cs][idx][PE] = phase;
 849                                                 /* Go to Final State */
 850                                                 dram_info->rl_val[cs][idx][S]++;
 851                                                 final_sum++;
 852                                                 DEBUG_RL_FULL_C("DDR3 - Read Leveling - We finished the window for pup: ",
 853                                                                 (u32)pup, 1);
 854                                         }
 855
 856                                         /* Before the start of the window */
 857                                 } else if (dram_info->rl_val[cs][idx][S] ==
 858                                            RL_UNLOCK_STATE) {
 859                                         /* Must be RL_UNLOCK_STATE */
 860                                         /*
 861                                          * Match expected value ? - Update
 862                                          * State Machine
 863                                          */
 864                                         if (dram_info->rl_val[cs][idx][C] <
 865                                             RL_RETRY_COUNT) {
 866                                                 if (((~locked_pups >> pup) & 0x1)) {
 867                                                         /* Match */
 868                                                         DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have no overrun and a match on pup: ",
 869                                                                         (u32)pup, 1);
 870                                                         dram_info->rl_val[cs][idx][C]++;
 871
 872                                                         /* If pup got to last state - lock the delays */
 873                                                         if (dram_info->rl_val[cs][idx][C] ==
 874                                                             RL_RETRY_COUNT) {
 875                                                                 dram_info->rl_val[cs][idx][C] = 0;
 876                                                                 dram_info->rl_val[cs][idx][DS] =
 877                                                                         delay;
 878                                                                 dram_info->rl_val[cs][idx][PS] =
 879                                                                         phase;
 880                                                                 dram_info->rl_val[cs][idx][S]++;        /* Go to Window State */
 881                                                                 locked_sum++;
 882                                                                 /* Will count the pups that got locked */
 883
 884                                                                 /* IF First lock - need to lock delays */
 885                                                                 if (first_octet_locked == 0) {
 886                                                                         DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got first lock on pup: ",
 887                                                                                         (u32)pup, 1);
 888                                                                         first_octet_locked
 889                                                                             =
 890                                                                             1;
 891                                                                 }
 892                                                         }
 893
 894                                                         /* if pup is in not in final state but there was match - dont increment counter */
 895                                                         else {
 896                                                                 counter_in_progress
 897                                                                     = 1;
 898                                                         }
 899                                                 }
 900                                         }
 901                                 }
 902                         } else {
 903                                 DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got overrun on pup: ",
 904                                                 (u32)pup, 1);
 905                                 counter_in_progress = 1;
 906                         }
 907                 }
 908
 909                 if (final_sum == (dram_info->num_of_std_pups * (1 - ecc) + ecc)) {
 910                         all_locked = 1;
 911                         DEBUG_RL_FULL_S("DDR3 - Read Leveling - Single Cs - All pups locked\n");
 912                 }
 913
 914                 /*
 915                  * This is a fix for unstable condition where pups are
 916                  * toggling between match and no match
 917                  */
 918                 /*
 919                  * If some of the pups is >1 <3, check if we did it too many
 920                  * times
 921                  */
 922                 if (counter_in_progress == 1) {
 923                         if (repeat_max_cnt < RL_RETRY_COUNT) {
 924                                 /* Notify at least one Counter is >=1 and < 3 */
 925                                 repeat_max_cnt++;
 926                                 counter_in_progress = 1;
 927                                 DEBUG_RL_FULL_S("DDR3 - Read Leveling - Counter is >=1 and <3\n");
 928                                 DEBUG_RL_FULL_S("DDR3 - Read Leveling - So we will not increment the delay to see if locked again\n");
 929                         } else {
 930                                 DEBUG_RL_FULL_S("DDR3 - Read Leveling - repeat_max_cnt reached max so now we will increment the delay\n");
 931                                 counter_in_progress = 0;
 932                         }
 933                 }
 934
 935                 /*
 936                  * Check some of the pups are in the middle of state machine
 937                  * and don't increment the delays
 938                  */
 939                 if (!counter_in_progress && !all_locked) {
 940                         repeat_max_cnt = 0;
 941                         if (!ratio_2to1)
 942                                 ui_max_delay = MAX_DELAY_INV;
 943                         else
 944                                 ui_max_delay = MAX_DELAY;
 945
 946                         /* Increment Delay */
 947                         if (delay < ui_max_delay) {
 948                                 /* Delay Incrementation */
 949                                 delay++;
 950                                 if (delay == ui_max_delay) {
 951                                         /*
 952                                          * Mark the last delay/pahse place
 953                                          * for window final place
 954                                          */
 955                                         if ((!ratio_2to1
 956                                              && phase == MAX_PHASE_RL_L_1TO1)
 957                                             || (ratio_2to1
 958                                                 && phase ==
 959                                                 MAX_PHASE_RL_L_2TO1))
 960                                                 final_delay = 1;
 961                                 }
 962                         } else {
 963                                 /* Phase+CL Incrementation */
 964                                 delay = 0;
 965                                 if (!ratio_2to1) {
 966                                         /* 1:1 mode */
 967                                         if (first_octet_locked) {
 968                                                 /* some pupet was Locked */
 969                                                 if (phase < MAX_PHASE_RL_L_1TO1) {
 970 #ifdef RL_WINDOW_WA
 971                                                         if (phase == 0)
 972 #else
 973                                                         if (phase == 1)
 974 #endif
 975                                                                 phase = 4;
 976                                                         else
 977                                                                 phase++;
 978                                                 } else {
 979                                                         DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
 980                                                         return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PUP_UNLOCK;
 981                                                 }
 982                                         } else {
 983                                                 /* No Pup was Locked */
 984                                                 if (phase < MAX_PHASE_RL_UL_1TO1) {
 985 #ifdef RL_WINDOW_WA
 986                                                         if (phase == 0)
 987                                                                 phase = 4;
 988 #else
 989                                                         phase++;
 990 #endif
 991                                                 } else
 992                                                         phase = 0;
 993                                         }
 994                                 } else {
 995                                         /* 2:1 mode */
 996                                         if (first_octet_locked) {
 997                                                 /* Some Pup was Locked */
 998                                                 if (phase < MAX_PHASE_RL_L_2TO1) {
 999                                                         phase++;
1000                                                 } else {
1001                                                         DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
1002                                                         return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PUP_UNLOCK;
1003                                                 }
1004                                         } else {
1005                                                 /* No Pup was Locked */
1006                                                 if (phase < MAX_PHASE_RL_UL_2TO1)
1007                                                         phase++;
1008                                                 else
1009                                                         phase = 0;
1010                                         }
1011                                 }
1012
1013                                 /*
1014                                  * If we finished a full Phases cycle (so
1015                                  * now phase = 0, need to increment
1016                                  * rd_sample_dly
1017                                  */
1018                                 if (phase == 0 && first_octet_locked == 0) {
1019                                         rd_sample_delay++;
1020
1021                                         /* Set current rd_sample_delay  */
1022                                         reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
1023                                         reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK <<
1024                                                  (REG_READ_DATA_SAMPLE_DELAYS_OFFS
1025                                                   * cs));
1026                                         reg |= (rd_sample_delay <<
1027                                                 (REG_READ_DATA_SAMPLE_DELAYS_OFFS *
1028                                                  cs));
1029                                         reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR,
1030                                                   reg);
1031                                 }
1032
1033                                 /*
1034                                  * Set current rdReadyDelay according to the
1035                                  * hash table (Need to do this in every phase
1036                                  * change)
1037                                  */
1038                                 if (!ratio_2to1) {
1039                                         /* 1:1 mode */
1040                                         add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
1041                                         switch (phase) {
1042                                         case 0:
1043                                                 add = add >>
1044                                                         REG_TRAINING_DEBUG_2_OFFS;
1045                                                 break;
1046                                         case 1:
1047                                                 add = add >>
1048                                                         (REG_TRAINING_DEBUG_2_OFFS
1049                                                          + 3);
1050                                                 break;
1051                                         case 4:
1052                                                 add = add >>
1053                                                         (REG_TRAINING_DEBUG_2_OFFS
1054                                                          + 6);
1055                                                 break;
1056                                         case 5:
1057                                                 add = add >>
1058                                                         (REG_TRAINING_DEBUG_2_OFFS
1059                                                          + 9);
1060                                                 break;
1061                                         }
1062                                 } else {
1063                                         /* 2:1 mode */
1064                                         add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
1065                                         add = (add >> phase *
1066                                                REG_TRAINING_DEBUG_3_OFFS);
1067                                 }
1068                                 add &= REG_TRAINING_DEBUG_2_MASK;
1069                                 reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
1070                                 reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
1071                                          (REG_READ_DATA_READY_DELAYS_OFFS * cs));
1072                                 reg |= ((rd_sample_delay + add) <<
1073                                         (REG_READ_DATA_READY_DELAYS_OFFS * cs));
1074                                 reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
1075                                 dram_info->rd_smpl_dly = rd_sample_delay;
1076                                 dram_info->rd_rdy_dly = rd_sample_delay + add;
1077                         }
1078
1079                         /* Reset counters for pups with states<RD_STATE_COUNT */
1080                         for (pup = 0;
1081                              pup <
1082                              (dram_info->num_of_std_pups * (1 - ecc) + ecc);
1083                              pup++) {
1084                                 if (dram_info->rl_val[cs][idx][C] < RL_RETRY_COUNT)
1085                                         dram_info->rl_val[cs][idx][C] = 0;
1086                         }
1087                 }
1088         }
1089
1090         phase_min = 10;
1091
1092         for (pup = 0; pup < (dram_info->num_of_std_pups); pup++) {
1093                 DEBUG_RL_S("DDR3 - Read Leveling - Window info - PUP: ");
1094                 DEBUG_RL_D((u32) pup, 1);
1095                 DEBUG_RL_S(", PS: ");
1096                 DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][PS], 1);
1097                 DEBUG_RL_S(", DS: ");
1098                 DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][DS], 2);
1099                 DEBUG_RL_S(", PE: ");
1100                 DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][PE], 1);
1101                 DEBUG_RL_S(", DE: ");
1102                 DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][DE], 2);
1103                 DEBUG_RL_S("\n");
1104         }
1105
1106         /* Find center of the window procedure */
1107         for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
1108              pup++) {
1109 #ifdef RL_WINDOW_WA
1110                 if (!ratio_2to1) {      /* 1:1 mode */
1111                         if (dram_info->rl_val[cs][idx][PS] == 4)
1112                                 dram_info->rl_val[cs][idx][PS] = 1;
1113                         if (dram_info->rl_val[cs][idx][PE] == 4)
1114                                 dram_info->rl_val[cs][idx][PE] = 1;
1115
1116                         delay_s = dram_info->rl_val[cs][idx][PS] *
1117                                 MAX_DELAY_INV + dram_info->rl_val[cs][idx][DS];
1118                         delay_e = dram_info->rl_val[cs][idx][PE] *
1119                                 MAX_DELAY_INV + dram_info->rl_val[cs][idx][DE];
1120
1121                         tmp = (delay_e - delay_s) / 2 + delay_s;
1122                         phase = tmp / MAX_DELAY_INV;
1123                         if (phase == 1) /* 1:1 mode */
1124                                 phase = 4;
1125
1126                         if (phase < phase_min)  /* for the read ready delay */
1127                                 phase_min = phase;
1128
1129                         dram_info->rl_val[cs][idx][P] = phase;
1130                         dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY_INV;
1131
1132                 } else {
1133                         delay_s = dram_info->rl_val[cs][idx][PS] *
1134                                 MAX_DELAY + dram_info->rl_val[cs][idx][DS];
1135                         delay_e = dram_info->rl_val[cs][idx][PE] *
1136                                 MAX_DELAY + dram_info->rl_val[cs][idx][DE];
1137
1138                         tmp = (delay_e - delay_s) / 2 + delay_s;
1139                         phase = tmp / MAX_DELAY;
1140
1141                         if (phase < phase_min)  /* for the read ready delay */
1142                                 phase_min = phase;
1143
1144                         dram_info->rl_val[cs][idx][P] = phase;
1145                         dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY;
1146                 }
1147 #else
1148                 if (!ratio_2to1) {      /* 1:1 mode */
1149                         if (dram_info->rl_val[cs][idx][PS] > 1)
1150                                 dram_info->rl_val[cs][idx][PS] -= 2;
1151                         if (dram_info->rl_val[cs][idx][PE] > 1)
1152                                 dram_info->rl_val[cs][idx][PE] -= 2;
1153                 }
1154
1155                 delay_s = dram_info->rl_val[cs][idx][PS] * MAX_DELAY +
1156                         dram_info->rl_val[cs][idx][DS];
1157                 delay_e = dram_info->rl_val[cs][idx][PE] * MAX_DELAY +
1158                         dram_info->rl_val[cs][idx][DE];
1159
1160                 tmp = (delay_e - delay_s) / 2 + delay_s;
1161                 phase = tmp / MAX_DELAY;
1162                 if (!ratio_2to1 && phase > 1)   /* 1:1 mode */
1163                         phase += 2;
1164
1165                 if (phase < phase_min)  /* for the read ready delay */
1166                         phase_min = phase;
1167
1168                 dram_info->rl_val[cs][idx][P] = phase;
1169                 dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY;
1170 #endif
1171         }
1172
1173         /* Set current rdReadyDelay according to the hash table (Need to do this in every phase change) */
1174         if (!ratio_2to1) {      /* 1:1 mode */
1175                 add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
1176                 switch (phase_min) {
1177                 case 0:
1178                         add = (add >> REG_TRAINING_DEBUG_2_OFFS);
1179                         break;
1180                 case 1:
1181                         add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 3));
1182                         break;
1183                 case 4:
1184                         add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 6));
1185                         break;
1186                 case 5:
1187                         add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 9));
1188                         break;
1189                 }
1190         } else {                /* 2:1 mode */
1191                 add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
1192                 add = (add >> phase_min * REG_TRAINING_DEBUG_3_OFFS);
1193         }
1194
1195         add &= REG_TRAINING_DEBUG_2_MASK;
1196         reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
1197         reg &=
1198             ~(REG_READ_DATA_READY_DELAYS_MASK <<
1199               (REG_READ_DATA_READY_DELAYS_OFFS * cs));
1200         reg |=
1201             ((rd_sample_delay + add) << (REG_READ_DATA_READY_DELAYS_OFFS * cs));
1202         reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
1203         dram_info->rd_rdy_dly = rd_sample_delay + add;
1204
1205         for (cs = 0; cs < dram_info->num_cs; cs++) {
1206                 for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
1207                         reg = ddr3_read_pup_reg(PUP_RL_MODE + 0x1, cs, pup);
1208                         dram_info->rl_val[cs][pup][DQS] = (reg & 0x3F);
1209                 }
1210         }
1211
1212         return MV_OK;
1213 }
1214 #endif