git.sur5r.net Git - u-boot/blob - drivers/ddr/altera/sequencer.c

   1 /*
   2  * Copyright Altera Corporation (C) 2012-2015
   3  *
   4  * SPDX-License-Identifier:    BSD-3-Clause
   5  */
   6
   7 #include <common.h>
   8 #include <asm/io.h>
   9 #include <asm/arch/sdram.h>
  10 #include "sequencer.h"
  11 #include "sequencer_auto.h"
  12 #include "sequencer_auto_ac_init.h"
  13 #include "sequencer_auto_inst_init.h"
  14 #include "sequencer_defines.h"
  15
  16 static struct socfpga_sdr_rw_load_manager *sdr_rw_load_mgr_regs =
  17         (struct socfpga_sdr_rw_load_manager *)(SDR_PHYGRP_RWMGRGRP_ADDRESS | 0x800);
  18
  19 static struct socfpga_sdr_rw_load_jump_manager *sdr_rw_load_jump_mgr_regs =
  20         (struct socfpga_sdr_rw_load_jump_manager *)(SDR_PHYGRP_RWMGRGRP_ADDRESS | 0xC00);
  21
  22 static struct socfpga_sdr_reg_file *sdr_reg_file =
  23         (struct socfpga_sdr_reg_file *)SDR_PHYGRP_REGFILEGRP_ADDRESS;
  24
  25 static struct socfpga_sdr_scc_mgr *sdr_scc_mgr =
  26         (struct socfpga_sdr_scc_mgr *)(SDR_PHYGRP_SCCGRP_ADDRESS | 0xe00);
  27
  28 static struct socfpga_phy_mgr_cmd *phy_mgr_cmd =
  29         (struct socfpga_phy_mgr_cmd *)SDR_PHYGRP_PHYMGRGRP_ADDRESS;
  30
  31 static struct socfpga_phy_mgr_cfg *phy_mgr_cfg =
  32         (struct socfpga_phy_mgr_cfg *)(SDR_PHYGRP_PHYMGRGRP_ADDRESS | 0x40);
  33
  34 static struct socfpga_data_mgr *data_mgr =
  35         (struct socfpga_data_mgr *)SDR_PHYGRP_DATAMGRGRP_ADDRESS;
  36
  37 static struct socfpga_sdr_ctrl *sdr_ctrl =
  38         (struct socfpga_sdr_ctrl *)SDR_CTRLGRP_ADDRESS;
  39
  40 #define DELTA_D         1
  41
  42 /*
  43  * In order to reduce ROM size, most of the selectable calibration steps are
  44  * decided at compile time based on the user's calibration mode selection,
  45  * as captured by the STATIC_CALIB_STEPS selection below.
  46  *
  47  * However, to support simulation-time selection of fast simulation mode, where
  48  * we skip everything except the bare minimum, we need a few of the steps to
  49  * be dynamic.  In those cases, we either use the DYNAMIC_CALIB_STEPS for the
  50  * check, which is based on the rtl-supplied value, or we dynamically compute
  51  * the value to use based on the dynamically-chosen calibration mode
  52  */
  53
  54 #define DLEVEL 0
  55 #define STATIC_IN_RTL_SIM 0
  56 #define STATIC_SKIP_DELAY_LOOPS 0
  57
  58 #define STATIC_CALIB_STEPS (STATIC_IN_RTL_SIM | CALIB_SKIP_FULL_TEST | \
  59         STATIC_SKIP_DELAY_LOOPS)
  60
  61 /* calibration steps requested by the rtl */
  62 uint16_t dyn_calib_steps;
  63
  64 /*
  65  * To make CALIB_SKIP_DELAY_LOOPS a dynamic conditional option
  66  * instead of static, we use boolean logic to select between
  67  * non-skip and skip values
  68  *
  69  * The mask is set to include all bits when not-skipping, but is
  70  * zero when skipping
  71  */
  72
  73 uint16_t skip_delay_mask;       /* mask off bits when skipping/not-skipping */
  74
  75 #define SKIP_DELAY_LOOP_VALUE_OR_ZERO(non_skip_value) \
  76         ((non_skip_value) & skip_delay_mask)
  77
  78 struct gbl_type *gbl;
  79 struct param_type *param;
  80 uint32_t curr_shadow_reg;
  81
  82 static uint32_t rw_mgr_mem_calibrate_write_test(uint32_t rank_bgn,
  83         uint32_t write_group, uint32_t use_dm,
  84         uint32_t all_correct, uint32_t *bit_chk, uint32_t all_ranks);
  85
  86 static void set_failing_group_stage(uint32_t group, uint32_t stage,
  87         uint32_t substage)
  88 {
  89         /*
  90          * Only set the global stage if there was not been any other
  91          * failing group
  92          */
  93         if (gbl->error_stage == CAL_STAGE_NIL)  {
  94                 gbl->error_substage = substage;
  95                 gbl->error_stage = stage;
  96                 gbl->error_group = group;
  97         }
  98 }
  99
 100 static void reg_file_set_group(u16 set_group)
 101 {
 102         clrsetbits_le32(&sdr_reg_file->cur_stage, 0xffff0000, set_group << 16);
 103 }
 104
 105 static void reg_file_set_stage(u8 set_stage)
 106 {
 107         clrsetbits_le32(&sdr_reg_file->cur_stage, 0xffff, set_stage & 0xff);
 108 }
 109
 110 static void reg_file_set_sub_stage(u8 set_sub_stage)
 111 {
 112         set_sub_stage &= 0xff;
 113         clrsetbits_le32(&sdr_reg_file->cur_stage, 0xff00, set_sub_stage << 8);
 114 }
 115
 116 static void initialize(void)
 117 {
 118         debug("%s:%d\n", __func__, __LINE__);
 119         /* USER calibration has control over path to memory */
 120         /*
 121          * In Hard PHY this is a 2-bit control:
 122          * 0: AFI Mux Select
 123          * 1: DDIO Mux Select
 124          */
 125         writel(0x3, &phy_mgr_cfg->mux_sel);
 126
 127         /* USER memory clock is not stable we begin initialization  */
 128         writel(0, &phy_mgr_cfg->reset_mem_stbl);
 129
 130         /* USER calibration status all set to zero */
 131         writel(0, &phy_mgr_cfg->cal_status);
 132
 133         writel(0, &phy_mgr_cfg->cal_debug_info);
 134
 135         if ((dyn_calib_steps & CALIB_SKIP_ALL) != CALIB_SKIP_ALL) {
 136                 param->read_correct_mask_vg  = ((uint32_t)1 <<
 137                         (RW_MGR_MEM_DQ_PER_READ_DQS /
 138                         RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS)) - 1;
 139                 param->write_correct_mask_vg = ((uint32_t)1 <<
 140                         (RW_MGR_MEM_DQ_PER_READ_DQS /
 141                         RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS)) - 1;
 142                 param->read_correct_mask     = ((uint32_t)1 <<
 143                         RW_MGR_MEM_DQ_PER_READ_DQS) - 1;
 144                 param->write_correct_mask    = ((uint32_t)1 <<
 145                         RW_MGR_MEM_DQ_PER_WRITE_DQS) - 1;
 146                 param->dm_correct_mask       = ((uint32_t)1 <<
 147                         (RW_MGR_MEM_DATA_WIDTH / RW_MGR_MEM_DATA_MASK_WIDTH))
 148                         - 1;
 149         }
 150 }
 151
 152 static void set_rank_and_odt_mask(uint32_t rank, uint32_t odt_mode)
 153 {
 154         uint32_t odt_mask_0 = 0;
 155         uint32_t odt_mask_1 = 0;
 156         uint32_t cs_and_odt_mask;
 157
 158         if (odt_mode == RW_MGR_ODT_MODE_READ_WRITE) {
 159                 if (RW_MGR_MEM_NUMBER_OF_RANKS == 1) {
 160                         /*
 161                          * 1 Rank
 162                          * Read: ODT = 0
 163                          * Write: ODT = 1
 164                          */
 165                         odt_mask_0 = 0x0;
 166                         odt_mask_1 = 0x1;
 167                 } else if (RW_MGR_MEM_NUMBER_OF_RANKS == 2) {
 168                         /* 2 Ranks */
 169                         if (RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM == 1) {
 170                                 /* - Dual-Slot , Single-Rank
 171                                  * (1 chip-select per DIMM)
 172                                  * OR
 173                                  * - RDIMM, 4 total CS (2 CS per DIMM)
 174                                  * means 2 DIMM
 175                                  * Since MEM_NUMBER_OF_RANKS is 2 they are
 176                                  * both single rank
 177                                  * with 2 CS each (special for RDIMM)
 178                                  * Read: Turn on ODT on the opposite rank
 179                                  * Write: Turn on ODT on all ranks
 180                                  */
 181                                 odt_mask_0 = 0x3 & ~(1 << rank);
 182                                 odt_mask_1 = 0x3;
 183                         } else {
 184                                 /*
 185                                  * USER - Single-Slot , Dual-rank DIMMs
 186                                  * (2 chip-selects per DIMM)
 187                                  * USER Read: Turn on ODT off on all ranks
 188                                  * USER Write: Turn on ODT on active rank
 189                                  */
 190                                 odt_mask_0 = 0x0;
 191                                 odt_mask_1 = 0x3 & (1 << rank);
 192                         }
 193                 } else {
 194                         /* 4 Ranks
 195                          * Read:
 196                          * ----------+-----------------------+
 197                          *           |                       |
 198                          *           |         ODT           |
 199                          * Read From +-----------------------+
 200                          *   Rank    |  3  |  2  |  1  |  0  |
 201                          * ----------+-----+-----+-----+-----+
 202                          *     0     |  0  |  1  |  0  |  0  |
 203                          *     1     |  1  |  0  |  0  |  0  |
 204                          *     2     |  0  |  0  |  0  |  1  |
 205                          *     3     |  0  |  0  |  1  |  0  |
 206                          * ----------+-----+-----+-----+-----+
 207                          *
 208                          * Write:
 209                          * ----------+-----------------------+
 210                          *           |                       |
 211                          *           |         ODT           |
 212                          * Write To  +-----------------------+
 213                          *   Rank    |  3  |  2  |  1  |  0  |
 214                          * ----------+-----+-----+-----+-----+
 215                          *     0     |  0  |  1  |  0  |  1  |
 216                          *     1     |  1  |  0  |  1  |  0  |
 217                          *     2     |  0  |  1  |  0  |  1  |
 218                          *     3     |  1  |  0  |  1  |  0  |
 219                          * ----------+-----+-----+-----+-----+
 220                          */
 221                         switch (rank) {
 222                         case 0:
 223                                 odt_mask_0 = 0x4;
 224                                 odt_mask_1 = 0x5;
 225                                 break;
 226                         case 1:
 227                                 odt_mask_0 = 0x8;
 228                                 odt_mask_1 = 0xA;
 229                                 break;
 230                         case 2:
 231                                 odt_mask_0 = 0x1;
 232                                 odt_mask_1 = 0x5;
 233                                 break;
 234                         case 3:
 235                                 odt_mask_0 = 0x2;
 236                                 odt_mask_1 = 0xA;
 237                                 break;
 238                         }
 239                 }
 240         } else {
 241                 odt_mask_0 = 0x0;
 242                 odt_mask_1 = 0x0;
 243         }
 244
 245         cs_and_odt_mask =
 246                 (0xFF & ~(1 << rank)) |
 247                 ((0xFF & odt_mask_0) << 8) |
 248                 ((0xFF & odt_mask_1) << 16);
 249         writel(cs_and_odt_mask, SDR_PHYGRP_RWMGRGRP_ADDRESS |
 250                                 RW_MGR_SET_CS_AND_ODT_MASK_OFFSET);
 251 }
 252
 253 /**
 254  * scc_mgr_set() - Set SCC Manager register
 255  * @off:        Base offset in SCC Manager space
 256  * @grp:        Read/Write group
 257  * @val:        Value to be set
 258  *
 259  * This function sets the SCC Manager (Scan Chain Control Manager) register.
 260  */
 261 static void scc_mgr_set(u32 off, u32 grp, u32 val)
 262 {
 263         writel(val, SDR_PHYGRP_SCCGRP_ADDRESS | off | (grp << 2));
 264 }
 265
 266 /**
 267  * scc_mgr_initialize() - Initialize SCC Manager registers
 268  *
 269  * Initialize SCC Manager registers.
 270  */
 271 static void scc_mgr_initialize(void)
 272 {
 273         /*
 274          * Clear register file for HPS. 16 (2^4) is the size of the
 275          * full register file in the scc mgr:
 276          *      RFILE_DEPTH = 1 + log2(MEM_DQ_PER_DQS + 1 + MEM_DM_PER_DQS +
 277          *                             MEM_IF_READ_DQS_WIDTH - 1);
 278          */
 279         int i;
 280
 281         for (i = 0; i < 16; i++) {
 282                 debug_cond(DLEVEL == 1, "%s:%d: Clearing SCC RFILE index %u\n",
 283                            __func__, __LINE__, i);
 284                 scc_mgr_set(SCC_MGR_HHP_RFILE_OFFSET, 0, i);
 285         }
 286 }
 287
 288 static void scc_mgr_set_dqdqs_output_phase(uint32_t write_group, uint32_t phase)
 289 {
 290         scc_mgr_set(SCC_MGR_DQDQS_OUT_PHASE_OFFSET, write_group, phase);
 291 }
 292
 293 static void scc_mgr_set_dqs_bus_in_delay(uint32_t read_group, uint32_t delay)
 294 {
 295         scc_mgr_set(SCC_MGR_DQS_IN_DELAY_OFFSET, read_group, delay);
 296 }
 297
 298 static void scc_mgr_set_dqs_en_phase(uint32_t read_group, uint32_t phase)
 299 {
 300         scc_mgr_set(SCC_MGR_DQS_EN_PHASE_OFFSET, read_group, phase);
 301 }
 302
 303 static void scc_mgr_set_dqs_en_delay(uint32_t read_group, uint32_t delay)
 304 {
 305         scc_mgr_set(SCC_MGR_DQS_EN_DELAY_OFFSET, read_group, delay);
 306 }
 307
 308 static void scc_mgr_set_dqs_io_in_delay(uint32_t write_group, uint32_t delay)
 309 {
 310         scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET, RW_MGR_MEM_DQ_PER_WRITE_DQS,
 311                     delay);
 312 }
 313
 314 static void scc_mgr_set_dq_in_delay(uint32_t dq_in_group, uint32_t delay)
 315 {
 316         scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET, dq_in_group, delay);
 317 }
 318
 319 static void scc_mgr_set_dq_out1_delay(uint32_t dq_in_group, uint32_t delay)
 320 {
 321         scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET, dq_in_group, delay);
 322 }
 323
 324 static void scc_mgr_set_dqs_out1_delay(uint32_t write_group,
 325                                               uint32_t delay)
 326 {
 327         scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET, RW_MGR_MEM_DQ_PER_WRITE_DQS,
 328                     delay);
 329 }
 330
 331 static void scc_mgr_set_dm_out1_delay(uint32_t dm, uint32_t delay)
 332 {
 333         scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET,
 334                     RW_MGR_MEM_DQ_PER_WRITE_DQS + 1 + dm,
 335                     delay);
 336 }
 337
 338 /* load up dqs config settings */
 339 static void scc_mgr_load_dqs(uint32_t dqs)
 340 {
 341         writel(dqs, &sdr_scc_mgr->dqs_ena);
 342 }
 343
 344 /* load up dqs io config settings */
 345 static void scc_mgr_load_dqs_io(void)
 346 {
 347         writel(0, &sdr_scc_mgr->dqs_io_ena);
 348 }
 349
 350 /* load up dq config settings */
 351 static void scc_mgr_load_dq(uint32_t dq_in_group)
 352 {
 353         writel(dq_in_group, &sdr_scc_mgr->dq_ena);
 354 }
 355
 356 /* load up dm config settings */
 357 static void scc_mgr_load_dm(uint32_t dm)
 358 {
 359         writel(dm, &sdr_scc_mgr->dm_ena);
 360 }
 361
 362 /**
 363  * scc_mgr_set_all_ranks() - Set SCC Manager register for all ranks
 364  * @off:        Base offset in SCC Manager space
 365  * @grp:        Read/Write group
 366  * @val:        Value to be set
 367  * @update:     If non-zero, trigger SCC Manager update for all ranks
 368  *
 369  * This function sets the SCC Manager (Scan Chain Control Manager) register
 370  * and optionally triggers the SCC update for all ranks.
 371  */
 372 static void scc_mgr_set_all_ranks(const u32 off, const u32 grp, const u32 val,
 373                                   const int update)
 374 {
 375         u32 r;
 376
 377         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
 378              r += NUM_RANKS_PER_SHADOW_REG) {
 379                 scc_mgr_set(off, grp, val);
 380
 381                 if (update || (r == 0)) {
 382                         writel(grp, &sdr_scc_mgr->dqs_ena);
 383                         writel(0, &sdr_scc_mgr->update);
 384                 }
 385         }
 386 }
 387
 388 static void scc_mgr_set_dqs_en_phase_all_ranks(u32 read_group, u32 phase)
 389 {
 390         /*
 391          * USER although the h/w doesn't support different phases per
 392          * shadow register, for simplicity our scc manager modeling
 393          * keeps different phase settings per shadow reg, and it's
 394          * important for us to keep them in sync to match h/w.
 395          * for efficiency, the scan chain update should occur only
 396          * once to sr0.
 397          */
 398         scc_mgr_set_all_ranks(SCC_MGR_DQS_EN_PHASE_OFFSET,
 399                               read_group, phase, 0);
 400 }
 401
 402 static void scc_mgr_set_dqdqs_output_phase_all_ranks(uint32_t write_group,
 403                                                      uint32_t phase)
 404 {
 405         /*
 406          * USER although the h/w doesn't support different phases per
 407          * shadow register, for simplicity our scc manager modeling
 408          * keeps different phase settings per shadow reg, and it's
 409          * important for us to keep them in sync to match h/w.
 410          * for efficiency, the scan chain update should occur only
 411          * once to sr0.
 412          */
 413         scc_mgr_set_all_ranks(SCC_MGR_DQDQS_OUT_PHASE_OFFSET,
 414                               write_group, phase, 0);
 415 }
 416
 417 static void scc_mgr_set_dqs_en_delay_all_ranks(uint32_t read_group,
 418                                                uint32_t delay)
 419 {
 420         /*
 421          * In shadow register mode, the T11 settings are stored in
 422          * registers in the core, which are updated by the DQS_ENA
 423          * signals. Not issuing the SCC_MGR_UPD command allows us to
 424          * save lots of rank switching overhead, by calling
 425          * select_shadow_regs_for_update with update_scan_chains
 426          * set to 0.
 427          */
 428         scc_mgr_set_all_ranks(SCC_MGR_DQS_EN_DELAY_OFFSET,
 429                               read_group, delay, 1);
 430         writel(0, &sdr_scc_mgr->update);
 431 }
 432
 433 /**
 434  * scc_mgr_set_oct_out1_delay() - Set OCT output delay
 435  * @write_group:        Write group
 436  * @delay:              Delay value
 437  *
 438  * This function sets the OCT output delay in SCC manager.
 439  */
 440 static void scc_mgr_set_oct_out1_delay(const u32 write_group, const u32 delay)
 441 {
 442         const int ratio = RW_MGR_MEM_IF_READ_DQS_WIDTH /
 443                           RW_MGR_MEM_IF_WRITE_DQS_WIDTH;
 444         const int base = write_group * ratio;
 445         int i;
 446         /*
 447          * Load the setting in the SCC manager
 448          * Although OCT affects only write data, the OCT delay is controlled
 449          * by the DQS logic block which is instantiated once per read group.
 450          * For protocols where a write group consists of multiple read groups,
 451          * the setting must be set multiple times.
 452          */
 453         for (i = 0; i < ratio; i++)
 454                 scc_mgr_set(SCC_MGR_OCT_OUT1_DELAY_OFFSET, base + i, delay);
 455 }
 456
 457 static void scc_mgr_set_hhp_extras(void)
 458 {
 459         /*
 460          * Load the fixed setting in the SCC manager
 461          * bits: 0:0 = 1'b1   - dqs bypass
 462          * bits: 1:1 = 1'b1   - dq bypass
 463          * bits: 4:2 = 3'b001   - rfifo_mode
 464          * bits: 6:5 = 2'b01  - rfifo clock_select
 465          * bits: 7:7 = 1'b0  - separate gating from ungating setting
 466          * bits: 8:8 = 1'b0  - separate OE from Output delay setting
 467          */
 468         uint32_t value = (0<<8) | (0<<7) | (1<<5) | (1<<2) | (1<<1) | (1<<0);
 469         uint32_t addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_HHP_GLOBALS_OFFSET;
 470
 471         writel(value, addr + SCC_MGR_HHP_EXTRAS_OFFSET);
 472 }
 473
 474 /*
 475  * USER Zero all DQS config
 476  * TODO: maybe rename to scc_mgr_zero_dqs_config (or something)
 477  */
 478 static void scc_mgr_zero_all(void)
 479 {
 480         uint32_t i, r;
 481
 482         /*
 483          * USER Zero all DQS config settings, across all groups and all
 484          * shadow registers
 485          */
 486         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r +=
 487              NUM_RANKS_PER_SHADOW_REG) {
 488                 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
 489                         /*
 490                          * The phases actually don't exist on a per-rank basis,
 491                          * but there's no harm updating them several times, so
 492                          * let's keep the code simple.
 493                          */
 494                         scc_mgr_set_dqs_bus_in_delay(i, IO_DQS_IN_RESERVE);
 495                         scc_mgr_set_dqs_en_phase(i, 0);
 496                         scc_mgr_set_dqs_en_delay(i, 0);
 497                 }
 498
 499                 for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) {
 500                         scc_mgr_set_dqdqs_output_phase(i, 0);
 501                         /* av/cv don't have out2 */
 502                         scc_mgr_set_oct_out1_delay(i, IO_DQS_OUT_RESERVE);
 503                 }
 504         }
 505
 506         /* multicast to all DQS group enables */
 507         writel(0xff, &sdr_scc_mgr->dqs_ena);
 508         writel(0, &sdr_scc_mgr->update);
 509 }
 510
 511 /**
 512  * scc_set_bypass_mode() - Set bypass mode and trigger SCC update
 513  * @write_group:        Write group
 514  *
 515  * Set bypass mode and trigger SCC update.
 516  */
 517 static void scc_set_bypass_mode(const u32 write_group)
 518 {
 519         /* Only needed once to set all groups, pins, DQ, DQS, DM. */
 520         if (write_group == 0) {
 521                 debug_cond(DLEVEL == 1, "%s:%d Setting HHP Extras\n", __func__,
 522                            __LINE__);
 523                 scc_mgr_set_hhp_extras();
 524                 debug_cond(DLEVEL == 1, "%s:%d Done Setting HHP Extras\n",
 525                           __func__, __LINE__);
 526         }
 527
 528         /* Multicast to all DQ enables. */
 529         writel(0xff, &sdr_scc_mgr->dq_ena);
 530         writel(0xff, &sdr_scc_mgr->dm_ena);
 531
 532         /* Update current DQS IO enable. */
 533         writel(0, &sdr_scc_mgr->dqs_io_ena);
 534
 535         /* Update the DQS logic. */
 536         writel(write_group, &sdr_scc_mgr->dqs_ena);
 537
 538         /* Hit update. */
 539         writel(0, &sdr_scc_mgr->update);
 540 }
 541
 542 /**
 543  * scc_mgr_load_dqs_for_write_group() - Load DQS settings for Write Group
 544  * @write_group:        Write group
 545  *
 546  * Load DQS settings for Write Group, do not trigger SCC update.
 547  */
 548 static void scc_mgr_load_dqs_for_write_group(const u32 write_group)
 549 {
 550         const int ratio = RW_MGR_MEM_IF_READ_DQS_WIDTH /
 551                           RW_MGR_MEM_IF_WRITE_DQS_WIDTH;
 552         const int base = write_group * ratio;
 553         int i;
 554         /*
 555          * Load the setting in the SCC manager
 556          * Although OCT affects only write data, the OCT delay is controlled
 557          * by the DQS logic block which is instantiated once per read group.
 558          * For protocols where a write group consists of multiple read groups,
 559          * the setting must be set multiple times.
 560          */
 561         for (i = 0; i < ratio; i++)
 562                 writel(base + i, &sdr_scc_mgr->dqs_ena);
 563 }
 564
 565 static void scc_mgr_zero_group(uint32_t write_group, uint32_t test_begin,
 566                                int32_t out_only)
 567 {
 568         uint32_t i, r;
 569
 570         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r +=
 571                 NUM_RANKS_PER_SHADOW_REG) {
 572                 /* Zero all DQ config settings */
 573                 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
 574                         scc_mgr_set_dq_out1_delay(i, 0);
 575                         if (!out_only)
 576                                 scc_mgr_set_dq_in_delay(i, 0);
 577                 }
 578
 579                 /* multicast to all DQ enables */
 580                 writel(0xff, &sdr_scc_mgr->dq_ena);
 581
 582                 /* Zero all DM config settings */
 583                 for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
 584                         scc_mgr_set_dm_out1_delay(i, 0);
 585                 }
 586
 587                 /* multicast to all DM enables */
 588                 writel(0xff, &sdr_scc_mgr->dm_ena);
 589
 590                 /* zero all DQS io settings */
 591                 if (!out_only)
 592                         scc_mgr_set_dqs_io_in_delay(write_group, 0);
 593                 /* av/cv don't have out2 */
 594                 scc_mgr_set_dqs_out1_delay(write_group, IO_DQS_OUT_RESERVE);
 595                 scc_mgr_set_oct_out1_delay(write_group, IO_DQS_OUT_RESERVE);
 596                 scc_mgr_load_dqs_for_write_group(write_group);
 597
 598                 /* multicast to all DQS IO enables (only 1) */
 599                 writel(0, &sdr_scc_mgr->dqs_io_ena);
 600
 601                 /* hit update to zero everything */
 602                 writel(0, &sdr_scc_mgr->update);
 603         }
 604 }
 605
 606 /*
 607  * apply and load a particular input delay for the DQ pins in a group
 608  * group_bgn is the index of the first dq pin (in the write group)
 609  */
 610 static void scc_mgr_apply_group_dq_in_delay(uint32_t write_group,
 611                                             uint32_t group_bgn, uint32_t delay)
 612 {
 613         uint32_t i, p;
 614
 615         for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) {
 616                 scc_mgr_set_dq_in_delay(p, delay);
 617                 scc_mgr_load_dq(p);
 618         }
 619 }
 620
 621 /* apply and load a particular output delay for the DQ pins in a group */
 622 static void scc_mgr_apply_group_dq_out1_delay(uint32_t write_group,
 623                                               uint32_t group_bgn,
 624                                               uint32_t delay1)
 625 {
 626         uint32_t i, p;
 627
 628         for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) {
 629                 scc_mgr_set_dq_out1_delay(i, delay1);
 630                 scc_mgr_load_dq(i);
 631         }
 632 }
 633
 634 /* apply and load a particular output delay for the DM pins in a group */
 635 static void scc_mgr_apply_group_dm_out1_delay(uint32_t write_group,
 636                                               uint32_t delay1)
 637 {
 638         uint32_t i;
 639
 640         for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
 641                 scc_mgr_set_dm_out1_delay(i, delay1);
 642                 scc_mgr_load_dm(i);
 643         }
 644 }
 645
 646
 647 /* apply and load delay on both DQS and OCT out1 */
 648 static void scc_mgr_apply_group_dqs_io_and_oct_out1(uint32_t write_group,
 649                                                     uint32_t delay)
 650 {
 651         scc_mgr_set_dqs_out1_delay(write_group, delay);
 652         scc_mgr_load_dqs_io();
 653
 654         scc_mgr_set_oct_out1_delay(write_group, delay);
 655         scc_mgr_load_dqs_for_write_group(write_group);
 656 }
 657
 658 /* apply a delay to the entire output side: DQ, DM, DQS, OCT */
 659 static void scc_mgr_apply_group_all_out_delay_add(uint32_t write_group,
 660                                                   uint32_t group_bgn,
 661                                                   uint32_t delay)
 662 {
 663         uint32_t i, p, new_delay;
 664
 665         /* dq shift */
 666         for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) {
 667                 new_delay = READ_SCC_DQ_OUT2_DELAY;
 668                 new_delay += delay;
 669
 670                 if (new_delay > IO_IO_OUT2_DELAY_MAX) {
 671                         debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DQ[%u,%u]:\
 672                                    %u > %lu => %lu", __func__, __LINE__,
 673                                    write_group, group_bgn, delay, i, p, new_delay,
 674                                    (long unsigned int)IO_IO_OUT2_DELAY_MAX,
 675                                    (long unsigned int)IO_IO_OUT2_DELAY_MAX);
 676                         new_delay = IO_IO_OUT2_DELAY_MAX;
 677                 }
 678
 679                 scc_mgr_load_dq(i);
 680         }
 681
 682         /* dm shift */
 683         for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
 684                 new_delay = READ_SCC_DM_IO_OUT2_DELAY;
 685                 new_delay += delay;
 686
 687                 if (new_delay > IO_IO_OUT2_DELAY_MAX) {
 688                         debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DM[%u]:\
 689                                    %u > %lu => %lu\n",  __func__, __LINE__,
 690                                    write_group, group_bgn, delay, i, new_delay,
 691                                    (long unsigned int)IO_IO_OUT2_DELAY_MAX,
 692                                    (long unsigned int)IO_IO_OUT2_DELAY_MAX);
 693                         new_delay = IO_IO_OUT2_DELAY_MAX;
 694                 }
 695
 696                 scc_mgr_load_dm(i);
 697         }
 698
 699         /* dqs shift */
 700         new_delay = READ_SCC_DQS_IO_OUT2_DELAY;
 701         new_delay += delay;
 702
 703         if (new_delay > IO_IO_OUT2_DELAY_MAX) {
 704                 debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DQS: %u > %d => %d;"
 705                            " adding %u to OUT1\n", __func__, __LINE__,
 706                            write_group, group_bgn, delay, new_delay,
 707                            IO_IO_OUT2_DELAY_MAX, IO_IO_OUT2_DELAY_MAX,
 708                            new_delay - IO_IO_OUT2_DELAY_MAX);
 709                 scc_mgr_set_dqs_out1_delay(write_group, new_delay -
 710                                            IO_IO_OUT2_DELAY_MAX);
 711                 new_delay = IO_IO_OUT2_DELAY_MAX;
 712         }
 713
 714         scc_mgr_load_dqs_io();
 715
 716         /* oct shift */
 717         new_delay = READ_SCC_OCT_OUT2_DELAY;
 718         new_delay += delay;
 719
 720         if (new_delay > IO_IO_OUT2_DELAY_MAX) {
 721                 debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DQS: %u > %d => %d;"
 722                            " adding %u to OUT1\n", __func__, __LINE__,
 723                            write_group, group_bgn, delay, new_delay,
 724                            IO_IO_OUT2_DELAY_MAX, IO_IO_OUT2_DELAY_MAX,
 725                            new_delay - IO_IO_OUT2_DELAY_MAX);
 726                 scc_mgr_set_oct_out1_delay(write_group, new_delay -
 727                                            IO_IO_OUT2_DELAY_MAX);
 728                 new_delay = IO_IO_OUT2_DELAY_MAX;
 729         }
 730
 731         scc_mgr_load_dqs_for_write_group(write_group);
 732 }
 733
 734 /*
 735  * USER apply a delay to the entire output side (DQ, DM, DQS, OCT)
 736  * and to all ranks
 737  */
 738 static void scc_mgr_apply_group_all_out_delay_add_all_ranks(
 739         uint32_t write_group, uint32_t group_bgn, uint32_t delay)
 740 {
 741         uint32_t r;
 742
 743         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
 744                 r += NUM_RANKS_PER_SHADOW_REG) {
 745                 scc_mgr_apply_group_all_out_delay_add(write_group,
 746                                                       group_bgn, delay);
 747                 writel(0, &sdr_scc_mgr->update);
 748         }
 749 }
 750
 751 /* optimization used to recover some slots in ddr3 inst_rom */
 752 /* could be applied to other protocols if we wanted to */
 753 static void set_jump_as_return(void)
 754 {
 755         /*
 756          * to save space, we replace return with jump to special shared
 757          * RETURN instruction so we set the counter to large value so that
 758          * we always jump
 759          */
 760         writel(0xff, &sdr_rw_load_mgr_regs->load_cntr0);
 761         writel(RW_MGR_RETURN, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
 762 }
 763
 764 /*
 765  * should always use constants as argument to ensure all computations are
 766  * performed at compile time
 767  */
 768 static void delay_for_n_mem_clocks(const uint32_t clocks)
 769 {
 770         uint32_t afi_clocks;
 771         uint8_t inner = 0;
 772         uint8_t outer = 0;
 773         uint16_t c_loop = 0;
 774
 775         debug("%s:%d: clocks=%u ... start\n", __func__, __LINE__, clocks);
 776
 777
 778         afi_clocks = (clocks + AFI_RATE_RATIO-1) / AFI_RATE_RATIO;
 779         /* scale (rounding up) to get afi clocks */
 780
 781         /*
 782          * Note, we don't bother accounting for being off a little bit
 783          * because of a few extra instructions in outer loops
 784          * Note, the loops have a test at the end, and do the test before
 785          * the decrement, and so always perform the loop
 786          * 1 time more than the counter value
 787          */
 788         if (afi_clocks == 0) {
 789                 ;
 790         } else if (afi_clocks <= 0x100) {
 791                 inner = afi_clocks-1;
 792                 outer = 0;
 793                 c_loop = 0;
 794         } else if (afi_clocks <= 0x10000) {
 795                 inner = 0xff;
 796                 outer = (afi_clocks-1) >> 8;
 797                 c_loop = 0;
 798         } else {
 799                 inner = 0xff;
 800                 outer = 0xff;
 801                 c_loop = (afi_clocks-1) >> 16;
 802         }
 803
 804         /*
 805          * rom instructions are structured as follows:
 806          *
 807          *    IDLE_LOOP2: jnz cntr0, TARGET_A
 808          *    IDLE_LOOP1: jnz cntr1, TARGET_B
 809          *                return
 810          *
 811          * so, when doing nested loops, TARGET_A is set to IDLE_LOOP2, and
 812          * TARGET_B is set to IDLE_LOOP2 as well
 813          *
 814          * if we have no outer loop, though, then we can use IDLE_LOOP1 only,
 815          * and set TARGET_B to IDLE_LOOP1 and we skip IDLE_LOOP2 entirely
 816          *
 817          * a little confusing, but it helps save precious space in the inst_rom
 818          * and sequencer rom and keeps the delays more accurate and reduces
 819          * overhead
 820          */
 821         if (afi_clocks <= 0x100) {
 822                 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner),
 823                         &sdr_rw_load_mgr_regs->load_cntr1);
 824
 825                 writel(RW_MGR_IDLE_LOOP1,
 826                         &sdr_rw_load_jump_mgr_regs->load_jump_add1);
 827
 828                 writel(RW_MGR_IDLE_LOOP1, SDR_PHYGRP_RWMGRGRP_ADDRESS |
 829                                           RW_MGR_RUN_SINGLE_GROUP_OFFSET);
 830         } else {
 831                 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner),
 832                         &sdr_rw_load_mgr_regs->load_cntr0);
 833
 834                 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(outer),
 835                         &sdr_rw_load_mgr_regs->load_cntr1);
 836
 837                 writel(RW_MGR_IDLE_LOOP2,
 838                         &sdr_rw_load_jump_mgr_regs->load_jump_add0);
 839
 840                 writel(RW_MGR_IDLE_LOOP2,
 841                         &sdr_rw_load_jump_mgr_regs->load_jump_add1);
 842
 843                 /* hack to get around compiler not being smart enough */
 844                 if (afi_clocks <= 0x10000) {
 845                         /* only need to run once */
 846                         writel(RW_MGR_IDLE_LOOP2, SDR_PHYGRP_RWMGRGRP_ADDRESS |
 847                                                   RW_MGR_RUN_SINGLE_GROUP_OFFSET);
 848                 } else {
 849                         do {
 850                                 writel(RW_MGR_IDLE_LOOP2,
 851                                         SDR_PHYGRP_RWMGRGRP_ADDRESS |
 852                                         RW_MGR_RUN_SINGLE_GROUP_OFFSET);
 853                         } while (c_loop-- != 0);
 854                 }
 855         }
 856         debug("%s:%d clocks=%u ... end\n", __func__, __LINE__, clocks);
 857 }
 858
 859 static void rw_mgr_mem_initialize(void)
 860 {
 861         uint32_t r;
 862         uint32_t grpaddr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
 863                            RW_MGR_RUN_SINGLE_GROUP_OFFSET;
 864
 865         debug("%s:%d\n", __func__, __LINE__);
 866
 867         /* The reset / cke part of initialization is broadcasted to all ranks */
 868         writel(RW_MGR_RANK_ALL, SDR_PHYGRP_RWMGRGRP_ADDRESS |
 869                                 RW_MGR_SET_CS_AND_ODT_MASK_OFFSET);
 870
 871         /*
 872          * Here's how you load register for a loop
 873          * Counters are located @ 0x800
 874          * Jump address are located @ 0xC00
 875          * For both, registers 0 to 3 are selected using bits 3 and 2, like
 876          * in 0x800, 0x804, 0x808, 0x80C and 0xC00, 0xC04, 0xC08, 0xC0C
 877          * I know this ain't pretty, but Avalon bus throws away the 2 least
 878          * significant bits
 879          */
 880
 881         /* start with memory RESET activated */
 882
 883         /* tINIT = 200us */
 884
 885         /*
 886          * 200us @ 266MHz (3.75 ns) ~ 54000 clock cycles
 887          * If a and b are the number of iteration in 2 nested loops
 888          * it takes the following number of cycles to complete the operation:
 889          * number_of_cycles = ((2 + n) * a + 2) * b
 890          * where n is the number of instruction in the inner loop
 891          * One possible solution is n = 0 , a = 256 , b = 106 => a = FF,
 892          * b = 6A
 893          */
 894
 895         /* Load counters */
 896         writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR0_VAL),
 897                &sdr_rw_load_mgr_regs->load_cntr0);
 898         writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR1_VAL),
 899                &sdr_rw_load_mgr_regs->load_cntr1);
 900         writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR2_VAL),
 901                &sdr_rw_load_mgr_regs->load_cntr2);
 902
 903         /* Load jump address */
 904         writel(RW_MGR_INIT_RESET_0_CKE_0,
 905                 &sdr_rw_load_jump_mgr_regs->load_jump_add0);
 906         writel(RW_MGR_INIT_RESET_0_CKE_0,
 907                 &sdr_rw_load_jump_mgr_regs->load_jump_add1);
 908         writel(RW_MGR_INIT_RESET_0_CKE_0,
 909                 &sdr_rw_load_jump_mgr_regs->load_jump_add2);
 910
 911         /* Execute count instruction */
 912         writel(RW_MGR_INIT_RESET_0_CKE_0, grpaddr);
 913
 914         /* indicate that memory is stable */
 915         writel(1, &phy_mgr_cfg->reset_mem_stbl);
 916
 917         /*
 918          * transition the RESET to high
 919          * Wait for 500us
 920          */
 921
 922         /*
 923          * 500us @ 266MHz (3.75 ns) ~ 134000 clock cycles
 924          * If a and b are the number of iteration in 2 nested loops
 925          * it takes the following number of cycles to complete the operation
 926          * number_of_cycles = ((2 + n) * a + 2) * b
 927          * where n is the number of instruction in the inner loop
 928          * One possible solution is n = 2 , a = 131 , b = 256 => a = 83,
 929          * b = FF
 930          */
 931
 932         /* Load counters */
 933         writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TRESET_CNTR0_VAL),
 934                &sdr_rw_load_mgr_regs->load_cntr0);
 935         writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TRESET_CNTR1_VAL),
 936                &sdr_rw_load_mgr_regs->load_cntr1);
 937         writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TRESET_CNTR2_VAL),
 938                &sdr_rw_load_mgr_regs->load_cntr2);
 939
 940         /* Load jump address */
 941         writel(RW_MGR_INIT_RESET_1_CKE_0,
 942                 &sdr_rw_load_jump_mgr_regs->load_jump_add0);
 943         writel(RW_MGR_INIT_RESET_1_CKE_0,
 944                 &sdr_rw_load_jump_mgr_regs->load_jump_add1);
 945         writel(RW_MGR_INIT_RESET_1_CKE_0,
 946                 &sdr_rw_load_jump_mgr_regs->load_jump_add2);
 947
 948         writel(RW_MGR_INIT_RESET_1_CKE_0, grpaddr);
 949
 950         /* bring up clock enable */
 951
 952         /* tXRP < 250 ck cycles */
 953         delay_for_n_mem_clocks(250);
 954
 955         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
 956                 if (param->skip_ranks[r]) {
 957                         /* request to skip the rank */
 958                         continue;
 959                 }
 960
 961                 /* set rank */
 962                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);
 963
 964                 /*
 965                  * USER Use Mirror-ed commands for odd ranks if address
 966                  * mirrorring is on
 967                  */
 968                 if ((RW_MGR_MEM_ADDRESS_MIRRORING >> r) & 0x1) {
 969                         set_jump_as_return();
 970                         writel(RW_MGR_MRS2_MIRR, grpaddr);
 971                         delay_for_n_mem_clocks(4);
 972                         set_jump_as_return();
 973                         writel(RW_MGR_MRS3_MIRR, grpaddr);
 974                         delay_for_n_mem_clocks(4);
 975                         set_jump_as_return();
 976                         writel(RW_MGR_MRS1_MIRR, grpaddr);
 977                         delay_for_n_mem_clocks(4);
 978                         set_jump_as_return();
 979                         writel(RW_MGR_MRS0_DLL_RESET_MIRR, grpaddr);
 980                 } else {
 981                         set_jump_as_return();
 982                         writel(RW_MGR_MRS2, grpaddr);
 983                         delay_for_n_mem_clocks(4);
 984                         set_jump_as_return();
 985                         writel(RW_MGR_MRS3, grpaddr);
 986                         delay_for_n_mem_clocks(4);
 987                         set_jump_as_return();
 988                         writel(RW_MGR_MRS1, grpaddr);
 989                         set_jump_as_return();
 990                         writel(RW_MGR_MRS0_DLL_RESET, grpaddr);
 991                 }
 992                 set_jump_as_return();
 993                 writel(RW_MGR_ZQCL, grpaddr);
 994
 995                 /* tZQinit = tDLLK = 512 ck cycles */
 996                 delay_for_n_mem_clocks(512);
 997         }
 998 }
 999
1000 /*
1001  * At the end of calibration we have to program the user settings in, and
1002  * USER  hand off the memory to the user.
1003  */
1004 static void rw_mgr_mem_handoff(void)
1005 {
1006         uint32_t r;
1007         uint32_t grpaddr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
1008                            RW_MGR_RUN_SINGLE_GROUP_OFFSET;
1009
1010         debug("%s:%d\n", __func__, __LINE__);
1011         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
1012                 if (param->skip_ranks[r])
1013                         /* request to skip the rank */
1014                         continue;
1015                 /* set rank */
1016                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);
1017
1018                 /* precharge all banks ... */
1019                 writel(RW_MGR_PRECHARGE_ALL, grpaddr);
1020
1021                 /* load up MR settings specified by user */
1022
1023                 /*
1024                  * Use Mirror-ed commands for odd ranks if address
1025                  * mirrorring is on
1026                  */
1027                 if ((RW_MGR_MEM_ADDRESS_MIRRORING >> r) & 0x1) {
1028                         set_jump_as_return();
1029                         writel(RW_MGR_MRS2_MIRR, grpaddr);
1030                         delay_for_n_mem_clocks(4);
1031                         set_jump_as_return();
1032                         writel(RW_MGR_MRS3_MIRR, grpaddr);
1033                         delay_for_n_mem_clocks(4);
1034                         set_jump_as_return();
1035                         writel(RW_MGR_MRS1_MIRR, grpaddr);
1036                         delay_for_n_mem_clocks(4);
1037                         set_jump_as_return();
1038                         writel(RW_MGR_MRS0_USER_MIRR, grpaddr);
1039                 } else {
1040                         set_jump_as_return();
1041                         writel(RW_MGR_MRS2, grpaddr);
1042                         delay_for_n_mem_clocks(4);
1043                         set_jump_as_return();
1044                         writel(RW_MGR_MRS3, grpaddr);
1045                         delay_for_n_mem_clocks(4);
1046                         set_jump_as_return();
1047                         writel(RW_MGR_MRS1, grpaddr);
1048                         delay_for_n_mem_clocks(4);
1049                         set_jump_as_return();
1050                         writel(RW_MGR_MRS0_USER, grpaddr);
1051                 }
1052                 /*
1053                  * USER  need to wait tMOD (12CK or 15ns) time before issuing
1054                  * other commands, but we will have plenty of NIOS cycles before
1055                  * actual handoff so its okay.
1056                  */
1057         }
1058 }
1059
1060 /*
1061  * performs a guaranteed read on the patterns we are going to use during a
1062  * read test to ensure memory works
1063  */
1064 static uint32_t rw_mgr_mem_calibrate_read_test_patterns(uint32_t rank_bgn,
1065         uint32_t group, uint32_t num_tries, uint32_t *bit_chk,
1066         uint32_t all_ranks)
1067 {
1068         uint32_t r, vg;
1069         uint32_t correct_mask_vg;
1070         uint32_t tmp_bit_chk;
1071         uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
1072                 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
1073         uint32_t addr;
1074         uint32_t base_rw_mgr;
1075
1076         *bit_chk = param->read_correct_mask;
1077         correct_mask_vg = param->read_correct_mask_vg;
1078
1079         for (r = rank_bgn; r < rank_end; r++) {
1080                 if (param->skip_ranks[r])
1081                         /* request to skip the rank */
1082                         continue;
1083
1084                 /* set rank */
1085                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
1086
1087                 /* Load up a constant bursts of read commands */
1088                 writel(0x20, &sdr_rw_load_mgr_regs->load_cntr0);
1089                 writel(RW_MGR_GUARANTEED_READ,
1090                         &sdr_rw_load_jump_mgr_regs->load_jump_add0);
1091
1092                 writel(0x20, &sdr_rw_load_mgr_regs->load_cntr1);
1093                 writel(RW_MGR_GUARANTEED_READ_CONT,
1094                         &sdr_rw_load_jump_mgr_regs->load_jump_add1);
1095
1096                 tmp_bit_chk = 0;
1097                 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS-1; ; vg--) {
1098                         /* reset the fifos to get pointers to known state */
1099
1100                         writel(0, &phy_mgr_cmd->fifo_reset);
1101                         writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
1102                                   RW_MGR_RESET_READ_DATAPATH_OFFSET);
1103
1104                         tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_READ_DQS
1105                                 / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS);
1106
1107                         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
1108                         writel(RW_MGR_GUARANTEED_READ, addr +
1109                                ((group * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS +
1110                                 vg) << 2));
1111
1112                         base_rw_mgr = readl(SDR_PHYGRP_RWMGRGRP_ADDRESS);
1113                         tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & (~base_rw_mgr));
1114
1115                         if (vg == 0)
1116                                 break;
1117                 }
1118                 *bit_chk &= tmp_bit_chk;
1119         }
1120
1121         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
1122         writel(RW_MGR_CLEAR_DQS_ENABLE, addr + (group << 2));
1123
1124         set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1125         debug_cond(DLEVEL == 1, "%s:%d test_load_patterns(%u,ALL) => (%u == %u) =>\
1126                    %lu\n", __func__, __LINE__, group, *bit_chk, param->read_correct_mask,
1127                    (long unsigned int)(*bit_chk == param->read_correct_mask));
1128         return *bit_chk == param->read_correct_mask;
1129 }
1130
1131 static uint32_t rw_mgr_mem_calibrate_read_test_patterns_all_ranks
1132         (uint32_t group, uint32_t num_tries, uint32_t *bit_chk)
1133 {
1134         return rw_mgr_mem_calibrate_read_test_patterns(0, group,
1135                 num_tries, bit_chk, 1);
1136 }
1137
1138 /* load up the patterns we are going to use during a read test */
1139 static void rw_mgr_mem_calibrate_read_load_patterns(uint32_t rank_bgn,
1140         uint32_t all_ranks)
1141 {
1142         uint32_t r;
1143         uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
1144                 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
1145
1146         debug("%s:%d\n", __func__, __LINE__);
1147         for (r = rank_bgn; r < rank_end; r++) {
1148                 if (param->skip_ranks[r])
1149                         /* request to skip the rank */
1150                         continue;
1151
1152                 /* set rank */
1153                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
1154
1155                 /* Load up a constant bursts */
1156                 writel(0x20, &sdr_rw_load_mgr_regs->load_cntr0);
1157
1158                 writel(RW_MGR_GUARANTEED_WRITE_WAIT0,
1159                         &sdr_rw_load_jump_mgr_regs->load_jump_add0);
1160
1161                 writel(0x20, &sdr_rw_load_mgr_regs->load_cntr1);
1162
1163                 writel(RW_MGR_GUARANTEED_WRITE_WAIT1,
1164                         &sdr_rw_load_jump_mgr_regs->load_jump_add1);
1165
1166                 writel(0x04, &sdr_rw_load_mgr_regs->load_cntr2);
1167
1168                 writel(RW_MGR_GUARANTEED_WRITE_WAIT2,
1169                         &sdr_rw_load_jump_mgr_regs->load_jump_add2);
1170
1171                 writel(0x04, &sdr_rw_load_mgr_regs->load_cntr3);
1172
1173                 writel(RW_MGR_GUARANTEED_WRITE_WAIT3,
1174                         &sdr_rw_load_jump_mgr_regs->load_jump_add3);
1175
1176                 writel(RW_MGR_GUARANTEED_WRITE, SDR_PHYGRP_RWMGRGRP_ADDRESS |
1177                                                 RW_MGR_RUN_SINGLE_GROUP_OFFSET);
1178         }
1179
1180         set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1181 }
1182
1183 /*
1184  * try a read and see if it returns correct data back. has dummy reads
1185  * inserted into the mix used to align dqs enable. has more thorough checks
1186  * than the regular read test.
1187  */
1188 static uint32_t rw_mgr_mem_calibrate_read_test(uint32_t rank_bgn, uint32_t group,
1189         uint32_t num_tries, uint32_t all_correct, uint32_t *bit_chk,
1190         uint32_t all_groups, uint32_t all_ranks)
1191 {
1192         uint32_t r, vg;
1193         uint32_t correct_mask_vg;
1194         uint32_t tmp_bit_chk;
1195         uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
1196                 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
1197         uint32_t addr;
1198         uint32_t base_rw_mgr;
1199
1200         *bit_chk = param->read_correct_mask;
1201         correct_mask_vg = param->read_correct_mask_vg;
1202
1203         uint32_t quick_read_mode = (((STATIC_CALIB_STEPS) &
1204                 CALIB_SKIP_DELAY_SWEEPS) && ENABLE_SUPER_QUICK_CALIBRATION);
1205
1206         for (r = rank_bgn; r < rank_end; r++) {
1207                 if (param->skip_ranks[r])
1208                         /* request to skip the rank */
1209                         continue;
1210
1211                 /* set rank */
1212                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
1213
1214                 writel(0x10, &sdr_rw_load_mgr_regs->load_cntr1);
1215
1216                 writel(RW_MGR_READ_B2B_WAIT1,
1217                         &sdr_rw_load_jump_mgr_regs->load_jump_add1);
1218
1219                 writel(0x10, &sdr_rw_load_mgr_regs->load_cntr2);
1220                 writel(RW_MGR_READ_B2B_WAIT2,
1221                         &sdr_rw_load_jump_mgr_regs->load_jump_add2);
1222
1223                 if (quick_read_mode)
1224                         writel(0x1, &sdr_rw_load_mgr_regs->load_cntr0);
1225                         /* need at least two (1+1) reads to capture failures */
1226                 else if (all_groups)
1227                         writel(0x06, &sdr_rw_load_mgr_regs->load_cntr0);
1228                 else
1229                         writel(0x32, &sdr_rw_load_mgr_regs->load_cntr0);
1230
1231                 writel(RW_MGR_READ_B2B,
1232                         &sdr_rw_load_jump_mgr_regs->load_jump_add0);
1233                 if (all_groups)
1234                         writel(RW_MGR_MEM_IF_READ_DQS_WIDTH *
1235                                RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS - 1,
1236                                &sdr_rw_load_mgr_regs->load_cntr3);
1237                 else
1238                         writel(0x0, &sdr_rw_load_mgr_regs->load_cntr3);
1239
1240                 writel(RW_MGR_READ_B2B,
1241                         &sdr_rw_load_jump_mgr_regs->load_jump_add3);
1242
1243                 tmp_bit_chk = 0;
1244                 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS-1; ; vg--) {
1245                         /* reset the fifos to get pointers to known state */
1246                         writel(0, &phy_mgr_cmd->fifo_reset);
1247                         writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
1248                                   RW_MGR_RESET_READ_DATAPATH_OFFSET);
1249
1250                         tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_READ_DQS
1251                                 / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS);
1252
1253                         if (all_groups)
1254                                 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_ALL_GROUPS_OFFSET;
1255                         else
1256                                 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
1257
1258                         writel(RW_MGR_READ_B2B, addr +
1259                                ((group * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS +
1260                                vg) << 2));
1261
1262                         base_rw_mgr = readl(SDR_PHYGRP_RWMGRGRP_ADDRESS);
1263                         tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(base_rw_mgr));
1264
1265                         if (vg == 0)
1266                                 break;
1267                 }
1268                 *bit_chk &= tmp_bit_chk;
1269         }
1270
1271         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
1272         writel(RW_MGR_CLEAR_DQS_ENABLE, addr + (group << 2));
1273
1274         if (all_correct) {
1275                 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1276                 debug_cond(DLEVEL == 2, "%s:%d read_test(%u,ALL,%u) =>\
1277                            (%u == %u) => %lu", __func__, __LINE__, group,
1278                            all_groups, *bit_chk, param->read_correct_mask,
1279                            (long unsigned int)(*bit_chk ==
1280                            param->read_correct_mask));
1281                 return *bit_chk == param->read_correct_mask;
1282         } else  {
1283                 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1284                 debug_cond(DLEVEL == 2, "%s:%d read_test(%u,ONE,%u) =>\
1285                            (%u != %lu) => %lu\n", __func__, __LINE__,
1286                            group, all_groups, *bit_chk, (long unsigned int)0,
1287                            (long unsigned int)(*bit_chk != 0x00));
1288                 return *bit_chk != 0x00;
1289         }
1290 }
1291
1292 static uint32_t rw_mgr_mem_calibrate_read_test_all_ranks(uint32_t group,
1293         uint32_t num_tries, uint32_t all_correct, uint32_t *bit_chk,
1294         uint32_t all_groups)
1295 {
1296         return rw_mgr_mem_calibrate_read_test(0, group, num_tries, all_correct,
1297                                               bit_chk, all_groups, 1);
1298 }
1299
1300 static void rw_mgr_incr_vfifo(uint32_t grp, uint32_t *v)
1301 {
1302         writel(grp, &phy_mgr_cmd->inc_vfifo_hard_phy);
1303         (*v)++;
1304 }
1305
1306 static void rw_mgr_decr_vfifo(uint32_t grp, uint32_t *v)
1307 {
1308         uint32_t i;
1309
1310         for (i = 0; i < VFIFO_SIZE-1; i++)
1311                 rw_mgr_incr_vfifo(grp, v);
1312 }
1313
1314 static int find_vfifo_read(uint32_t grp, uint32_t *bit_chk)
1315 {
1316         uint32_t  v;
1317         uint32_t fail_cnt = 0;
1318         uint32_t test_status;
1319
1320         for (v = 0; v < VFIFO_SIZE; ) {
1321                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: vfifo %u\n",
1322                            __func__, __LINE__, v);
1323                 test_status = rw_mgr_mem_calibrate_read_test_all_ranks
1324                         (grp, 1, PASS_ONE_BIT, bit_chk, 0);
1325                 if (!test_status) {
1326                         fail_cnt++;
1327
1328                         if (fail_cnt == 2)
1329                                 break;
1330                 }
1331
1332                 /* fiddle with FIFO */
1333                 rw_mgr_incr_vfifo(grp, &v);
1334         }
1335
1336         if (v >= VFIFO_SIZE) {
1337                 /* no failing read found!! Something must have gone wrong */
1338                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: vfifo failed\n",
1339                            __func__, __LINE__);
1340                 return 0;
1341         } else {
1342                 return v;
1343         }
1344 }
1345
1346 static int find_working_phase(uint32_t *grp, uint32_t *bit_chk,
1347                               uint32_t dtaps_per_ptap, uint32_t *work_bgn,
1348                               uint32_t *v, uint32_t *d, uint32_t *p,
1349                               uint32_t *i, uint32_t *max_working_cnt)
1350 {
1351         uint32_t found_begin = 0;
1352         uint32_t tmp_delay = 0;
1353         uint32_t test_status;
1354
1355         for (*d = 0; *d <= dtaps_per_ptap; (*d)++, tmp_delay +=
1356                 IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
1357                 *work_bgn = tmp_delay;
1358                 scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d);
1359
1360                 for (*i = 0; *i < VFIFO_SIZE; (*i)++) {
1361                         for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX; (*p)++, *work_bgn +=
1362                                 IO_DELAY_PER_OPA_TAP) {
1363                                 scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p);
1364
1365                                 test_status =
1366                                 rw_mgr_mem_calibrate_read_test_all_ranks
1367                                 (*grp, 1, PASS_ONE_BIT, bit_chk, 0);
1368
1369                                 if (test_status) {
1370                                         *max_working_cnt = 1;
1371                                         found_begin = 1;
1372                                         break;
1373                                 }
1374                         }
1375
1376                         if (found_begin)
1377                                 break;
1378
1379                         if (*p > IO_DQS_EN_PHASE_MAX)
1380                                 /* fiddle with FIFO */
1381                                 rw_mgr_incr_vfifo(*grp, v);
1382                 }
1383
1384                 if (found_begin)
1385                         break;
1386         }
1387
1388         if (*i >= VFIFO_SIZE) {
1389                 /* cannot find working solution */
1390                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: no vfifo/\
1391                            ptap/dtap\n", __func__, __LINE__);
1392                 return 0;
1393         } else {
1394                 return 1;
1395         }
1396 }
1397
1398 static void sdr_backup_phase(uint32_t *grp, uint32_t *bit_chk,
1399                              uint32_t *work_bgn, uint32_t *v, uint32_t *d,
1400                              uint32_t *p, uint32_t *max_working_cnt)
1401 {
1402         uint32_t found_begin = 0;
1403         uint32_t tmp_delay;
1404
1405         /* Special case code for backing up a phase */
1406         if (*p == 0) {
1407                 *p = IO_DQS_EN_PHASE_MAX;
1408                 rw_mgr_decr_vfifo(*grp, v);
1409         } else {
1410                 (*p)--;
1411         }
1412         tmp_delay = *work_bgn - IO_DELAY_PER_OPA_TAP;
1413         scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p);
1414
1415         for (*d = 0; *d <= IO_DQS_EN_DELAY_MAX && tmp_delay < *work_bgn;
1416                 (*d)++, tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
1417                 scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d);
1418
1419                 if (rw_mgr_mem_calibrate_read_test_all_ranks(*grp, 1,
1420                                                              PASS_ONE_BIT,
1421                                                              bit_chk, 0)) {
1422                         found_begin = 1;
1423                         *work_bgn = tmp_delay;
1424                         break;
1425                 }
1426         }
1427
1428         /* We have found a working dtap before the ptap found above */
1429         if (found_begin == 1)
1430                 (*max_working_cnt)++;
1431
1432         /*
1433          * Restore VFIFO to old state before we decremented it
1434          * (if needed).
1435          */
1436         (*p)++;
1437         if (*p > IO_DQS_EN_PHASE_MAX) {
1438                 *p = 0;
1439                 rw_mgr_incr_vfifo(*grp, v);
1440         }
1441
1442         scc_mgr_set_dqs_en_delay_all_ranks(*grp, 0);
1443 }
1444
1445 static int sdr_nonworking_phase(uint32_t *grp, uint32_t *bit_chk,
1446                              uint32_t *work_bgn, uint32_t *v, uint32_t *d,
1447                              uint32_t *p, uint32_t *i, uint32_t *max_working_cnt,
1448                              uint32_t *work_end)
1449 {
1450         uint32_t found_end = 0;
1451
1452         (*p)++;
1453         *work_end += IO_DELAY_PER_OPA_TAP;
1454         if (*p > IO_DQS_EN_PHASE_MAX) {
1455                 /* fiddle with FIFO */
1456                 *p = 0;
1457                 rw_mgr_incr_vfifo(*grp, v);
1458         }
1459
1460         for (; *i < VFIFO_SIZE + 1; (*i)++) {
1461                 for (; *p <= IO_DQS_EN_PHASE_MAX; (*p)++, *work_end
1462                         += IO_DELAY_PER_OPA_TAP) {
1463                         scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p);
1464
1465                         if (!rw_mgr_mem_calibrate_read_test_all_ranks
1466                                 (*grp, 1, PASS_ONE_BIT, bit_chk, 0)) {
1467                                 found_end = 1;
1468                                 break;
1469                         } else {
1470                                 (*max_working_cnt)++;
1471                         }
1472                 }
1473
1474                 if (found_end)
1475                         break;
1476
1477                 if (*p > IO_DQS_EN_PHASE_MAX) {
1478                         /* fiddle with FIFO */
1479                         rw_mgr_incr_vfifo(*grp, v);
1480                         *p = 0;
1481                 }
1482         }
1483
1484         if (*i >= VFIFO_SIZE + 1) {
1485                 /* cannot see edge of failing read */
1486                 debug_cond(DLEVEL == 2, "%s:%d sdr_nonworking_phase: end:\
1487                            failed\n", __func__, __LINE__);
1488                 return 0;
1489         } else {
1490                 return 1;
1491         }
1492 }
1493
1494 static int sdr_find_window_centre(uint32_t *grp, uint32_t *bit_chk,
1495                                   uint32_t *work_bgn, uint32_t *v, uint32_t *d,
1496                                   uint32_t *p, uint32_t *work_mid,
1497                                   uint32_t *work_end)
1498 {
1499         int i;
1500         int tmp_delay = 0;
1501
1502         *work_mid = (*work_bgn + *work_end) / 2;
1503
1504         debug_cond(DLEVEL == 2, "work_bgn=%d work_end=%d work_mid=%d\n",
1505                    *work_bgn, *work_end, *work_mid);
1506         /* Get the middle delay to be less than a VFIFO delay */
1507         for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX;
1508                 (*p)++, tmp_delay += IO_DELAY_PER_OPA_TAP)
1509                 ;
1510         debug_cond(DLEVEL == 2, "vfifo ptap delay %d\n", tmp_delay);
1511         while (*work_mid > tmp_delay)
1512                 *work_mid -= tmp_delay;
1513         debug_cond(DLEVEL == 2, "new work_mid %d\n", *work_mid);
1514
1515         tmp_delay = 0;
1516         for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX && tmp_delay < *work_mid;
1517                 (*p)++, tmp_delay += IO_DELAY_PER_OPA_TAP)
1518                 ;
1519         tmp_delay -= IO_DELAY_PER_OPA_TAP;
1520         debug_cond(DLEVEL == 2, "new p %d, tmp_delay=%d\n", (*p) - 1, tmp_delay);
1521         for (*d = 0; *d <= IO_DQS_EN_DELAY_MAX && tmp_delay < *work_mid; (*d)++,
1522                 tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP)
1523                 ;
1524         debug_cond(DLEVEL == 2, "new d %d, tmp_delay=%d\n", *d, tmp_delay);
1525
1526         scc_mgr_set_dqs_en_phase_all_ranks(*grp, (*p) - 1);
1527         scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d);
1528
1529         /*
1530          * push vfifo until we can successfully calibrate. We can do this
1531          * because the largest possible margin in 1 VFIFO cycle.
1532          */
1533         for (i = 0; i < VFIFO_SIZE; i++) {
1534                 debug_cond(DLEVEL == 2, "find_dqs_en_phase: center: vfifo=%u\n",
1535                            *v);
1536                 if (rw_mgr_mem_calibrate_read_test_all_ranks(*grp, 1,
1537                                                              PASS_ONE_BIT,
1538                                                              bit_chk, 0)) {
1539                         break;
1540                 }
1541
1542                 /* fiddle with FIFO */
1543                 rw_mgr_incr_vfifo(*grp, v);
1544         }
1545
1546         if (i >= VFIFO_SIZE) {
1547                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: center: \
1548                            failed\n", __func__, __LINE__);
1549                 return 0;
1550         } else {
1551                 return 1;
1552         }
1553 }
1554
1555 /* find a good dqs enable to use */
1556 static uint32_t rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(uint32_t grp)
1557 {
1558         uint32_t v, d, p, i;
1559         uint32_t max_working_cnt;
1560         uint32_t bit_chk;
1561         uint32_t dtaps_per_ptap;
1562         uint32_t work_bgn, work_mid, work_end;
1563         uint32_t found_passing_read, found_failing_read, initial_failing_dtap;
1564
1565         debug("%s:%d %u\n", __func__, __LINE__, grp);
1566
1567         reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
1568
1569         scc_mgr_set_dqs_en_delay_all_ranks(grp, 0);
1570         scc_mgr_set_dqs_en_phase_all_ranks(grp, 0);
1571
1572         /* ************************************************************** */
1573         /* * Step 0 : Determine number of delay taps for each phase tap * */
1574         dtaps_per_ptap = IO_DELAY_PER_OPA_TAP/IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
1575
1576         /* ********************************************************* */
1577         /* * Step 1 : First push vfifo until we get a failing read * */
1578         v = find_vfifo_read(grp, &bit_chk);
1579
1580         max_working_cnt = 0;
1581
1582         /* ******************************************************** */
1583         /* * step 2: find first working phase, increment in ptaps * */
1584         work_bgn = 0;
1585         if (find_working_phase(&grp, &bit_chk, dtaps_per_ptap, &work_bgn, &v, &d,
1586                                 &p, &i, &max_working_cnt) == 0)
1587                 return 0;
1588
1589         work_end = work_bgn;
1590
1591         /*
1592          * If d is 0 then the working window covers a phase tap and
1593          * we can follow the old procedure otherwise, we've found the beginning,
1594          * and we need to increment the dtaps until we find the end.
1595          */
1596         if (d == 0) {
1597                 /* ********************************************************* */
1598                 /* * step 3a: if we have room, back off by one and
1599                 increment in dtaps * */
1600
1601                 sdr_backup_phase(&grp, &bit_chk, &work_bgn, &v, &d, &p,
1602                                  &max_working_cnt);
1603
1604                 /* ********************************************************* */
1605                 /* * step 4a: go forward from working phase to non working
1606                 phase, increment in ptaps * */
1607                 if (sdr_nonworking_phase(&grp, &bit_chk, &work_bgn, &v, &d, &p,
1608                                          &i, &max_working_cnt, &work_end) == 0)
1609                         return 0;
1610
1611                 /* ********************************************************* */
1612                 /* * step 5a:  back off one from last, increment in dtaps  * */
1613
1614                 /* Special case code for backing up a phase */
1615                 if (p == 0) {
1616                         p = IO_DQS_EN_PHASE_MAX;
1617                         rw_mgr_decr_vfifo(grp, &v);
1618                 } else {
1619                         p = p - 1;
1620                 }
1621
1622                 work_end -= IO_DELAY_PER_OPA_TAP;
1623                 scc_mgr_set_dqs_en_phase_all_ranks(grp, p);
1624
1625                 /* * The actual increment of dtaps is done outside of
1626                 the if/else loop to share code */
1627                 d = 0;
1628
1629                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: v/p: \
1630                            vfifo=%u ptap=%u\n", __func__, __LINE__,
1631                            v, p);
1632         } else {
1633                 /* ******************************************************* */
1634                 /* * step 3-5b:  Find the right edge of the window using
1635                 delay taps   * */
1636                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase:vfifo=%u \
1637                            ptap=%u dtap=%u bgn=%u\n", __func__, __LINE__,
1638                            v, p, d, work_bgn);
1639
1640                 work_end = work_bgn;
1641
1642                 /* * The actual increment of dtaps is done outside of the
1643                 if/else loop to share code */
1644
1645                 /* Only here to counterbalance a subtract later on which is
1646                 not needed if this branch of the algorithm is taken */
1647                 max_working_cnt++;
1648         }
1649
1650         /* The dtap increment to find the failing edge is done here */
1651         for (; d <= IO_DQS_EN_DELAY_MAX; d++, work_end +=
1652                 IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
1653                         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: \
1654                                    end-2: dtap=%u\n", __func__, __LINE__, d);
1655                         scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
1656
1657                         if (!rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
1658                                                                       PASS_ONE_BIT,
1659                                                                       &bit_chk, 0)) {
1660                                 break;
1661                         }
1662         }
1663
1664         /* Go back to working dtap */
1665         if (d != 0)
1666                 work_end -= IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
1667
1668         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: v/p/d: vfifo=%u \
1669                    ptap=%u dtap=%u end=%u\n", __func__, __LINE__,
1670                    v, p, d-1, work_end);
1671
1672         if (work_end < work_bgn) {
1673                 /* nil range */
1674                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: end-2: \
1675                            failed\n", __func__, __LINE__);
1676                 return 0;
1677         }
1678
1679         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: found range [%u,%u]\n",
1680                    __func__, __LINE__, work_bgn, work_end);
1681
1682         /* *************************************************************** */
1683         /*
1684          * * We need to calculate the number of dtaps that equal a ptap
1685          * * To do that we'll back up a ptap and re-find the edge of the
1686          * * window using dtaps
1687          */
1688
1689         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: calculate dtaps_per_ptap \
1690                    for tracking\n", __func__, __LINE__);
1691
1692         /* Special case code for backing up a phase */
1693         if (p == 0) {
1694                 p = IO_DQS_EN_PHASE_MAX;
1695                 rw_mgr_decr_vfifo(grp, &v);
1696                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: backedup \
1697                            cycle/phase: v=%u p=%u\n", __func__, __LINE__,
1698                            v, p);
1699         } else {
1700                 p = p - 1;
1701                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: backedup \
1702                            phase only: v=%u p=%u", __func__, __LINE__,
1703                            v, p);
1704         }
1705
1706         scc_mgr_set_dqs_en_phase_all_ranks(grp, p);
1707
1708         /*
1709          * Increase dtap until we first see a passing read (in case the
1710          * window is smaller than a ptap),
1711          * and then a failing read to mark the edge of the window again
1712          */
1713
1714         /* Find a passing read */
1715         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: find passing read\n",
1716                    __func__, __LINE__);
1717         found_passing_read = 0;
1718         found_failing_read = 0;
1719         initial_failing_dtap = d;
1720         for (; d <= IO_DQS_EN_DELAY_MAX; d++) {
1721                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: testing \
1722                            read d=%u\n", __func__, __LINE__, d);
1723                 scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
1724
1725                 if (rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
1726                                                              PASS_ONE_BIT,
1727                                                              &bit_chk, 0)) {
1728                         found_passing_read = 1;
1729                         break;
1730                 }
1731         }
1732
1733         if (found_passing_read) {
1734                 /* Find a failing read */
1735                 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: find failing \
1736                            read\n", __func__, __LINE__);
1737                 for (d = d + 1; d <= IO_DQS_EN_DELAY_MAX; d++) {
1738                         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: \
1739                                    testing read d=%u\n", __func__, __LINE__, d);
1740                         scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
1741
1742                         if (!rw_mgr_mem_calibrate_read_test_all_ranks
1743                                 (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) {
1744                                 found_failing_read = 1;
1745                                 break;
1746                         }
1747                 }
1748         } else {
1749                 debug_cond(DLEVEL == 1, "%s:%d find_dqs_en_phase: failed to \
1750                            calculate dtaps", __func__, __LINE__);
1751                 debug_cond(DLEVEL == 1, "per ptap. Fall back on static value\n");
1752         }
1753
1754         /*
1755          * The dynamically calculated dtaps_per_ptap is only valid if we
1756          * found a passing/failing read. If we didn't, it means d hit the max
1757          * (IO_DQS_EN_DELAY_MAX). Otherwise, dtaps_per_ptap retains its
1758          * statically calculated value.
1759          */
1760         if (found_passing_read && found_failing_read)
1761                 dtaps_per_ptap = d - initial_failing_dtap;
1762
1763         writel(dtaps_per_ptap, &sdr_reg_file->dtaps_per_ptap);
1764         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: dtaps_per_ptap=%u \
1765                    - %u = %u",  __func__, __LINE__, d,
1766                    initial_failing_dtap, dtaps_per_ptap);
1767
1768         /* ******************************************** */
1769         /* * step 6:  Find the centre of the window   * */
1770         if (sdr_find_window_centre(&grp, &bit_chk, &work_bgn, &v, &d, &p,
1771                                    &work_mid, &work_end) == 0)
1772                 return 0;
1773
1774         debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: center found: \
1775                    vfifo=%u ptap=%u dtap=%u\n", __func__, __LINE__,
1776                    v, p-1, d);
1777         return 1;
1778 }
1779
1780 /*
1781  * Try rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase across different
1782  * dq_in_delay values
1783  */
1784 static uint32_t
1785 rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay
1786 (uint32_t write_group, uint32_t read_group, uint32_t test_bgn)
1787 {
1788         uint32_t found;
1789         uint32_t i;
1790         uint32_t p;
1791         uint32_t d;
1792         uint32_t r;
1793
1794         const uint32_t delay_step = IO_IO_IN_DELAY_MAX /
1795                 (RW_MGR_MEM_DQ_PER_READ_DQS-1);
1796                 /* we start at zero, so have one less dq to devide among */
1797
1798         debug("%s:%d (%u,%u,%u)", __func__, __LINE__, write_group, read_group,
1799               test_bgn);
1800
1801         /* try different dq_in_delays since the dq path is shorter than dqs */
1802
1803         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
1804              r += NUM_RANKS_PER_SHADOW_REG) {
1805                 for (i = 0, p = test_bgn, d = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS;
1806                         i++, p++, d += delay_step) {
1807                         debug_cond(DLEVEL == 1, "%s:%d rw_mgr_mem_calibrate_\
1808                                    vfifo_find_dqs_", __func__, __LINE__);
1809                         debug_cond(DLEVEL == 1, "en_phase_sweep_dq_in_delay: g=%u/%u ",
1810                                write_group, read_group);
1811                         debug_cond(DLEVEL == 1, "r=%u, i=%u p=%u d=%u\n", r, i , p, d);
1812                         scc_mgr_set_dq_in_delay(p, d);
1813                         scc_mgr_load_dq(p);
1814                 }
1815                 writel(0, &sdr_scc_mgr->update);
1816         }
1817
1818         found = rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(read_group);
1819
1820         debug_cond(DLEVEL == 1, "%s:%d rw_mgr_mem_calibrate_vfifo_find_dqs_\
1821                    en_phase_sweep_dq", __func__, __LINE__);
1822         debug_cond(DLEVEL == 1, "_in_delay: g=%u/%u found=%u; Reseting delay \
1823                    chain to zero\n", write_group, read_group, found);
1824
1825         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
1826              r += NUM_RANKS_PER_SHADOW_REG) {
1827                 for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS;
1828                         i++, p++) {
1829                         scc_mgr_set_dq_in_delay(p, 0);
1830                         scc_mgr_load_dq(p);
1831                 }
1832                 writel(0, &sdr_scc_mgr->update);
1833         }
1834
1835         return found;
1836 }
1837
1838 /* per-bit deskew DQ and center */
1839 static uint32_t rw_mgr_mem_calibrate_vfifo_center(uint32_t rank_bgn,
1840         uint32_t write_group, uint32_t read_group, uint32_t test_bgn,
1841         uint32_t use_read_test, uint32_t update_fom)
1842 {
1843         uint32_t i, p, d, min_index;
1844         /*
1845          * Store these as signed since there are comparisons with
1846          * signed numbers.
1847          */
1848         uint32_t bit_chk;
1849         uint32_t sticky_bit_chk;
1850         int32_t left_edge[RW_MGR_MEM_DQ_PER_READ_DQS];
1851         int32_t right_edge[RW_MGR_MEM_DQ_PER_READ_DQS];
1852         int32_t final_dq[RW_MGR_MEM_DQ_PER_READ_DQS];
1853         int32_t mid;
1854         int32_t orig_mid_min, mid_min;
1855         int32_t new_dqs, start_dqs, start_dqs_en, shift_dq, final_dqs,
1856                 final_dqs_en;
1857         int32_t dq_margin, dqs_margin;
1858         uint32_t stop;
1859         uint32_t temp_dq_in_delay1, temp_dq_in_delay2;
1860         uint32_t addr;
1861
1862         debug("%s:%d: %u %u", __func__, __LINE__, read_group, test_bgn);
1863
1864         addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_DQS_IN_DELAY_OFFSET;
1865         start_dqs = readl(addr + (read_group << 2));
1866         if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS)
1867                 start_dqs_en = readl(addr + ((read_group << 2)
1868                                      - IO_DQS_EN_DELAY_OFFSET));
1869
1870         /* set the left and right edge of each bit to an illegal value */
1871         /* use (IO_IO_IN_DELAY_MAX + 1) as an illegal value */
1872         sticky_bit_chk = 0;
1873         for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
1874                 left_edge[i]  = IO_IO_IN_DELAY_MAX + 1;
1875                 right_edge[i] = IO_IO_IN_DELAY_MAX + 1;
1876         }
1877
1878         /* Search for the left edge of the window for each bit */
1879         for (d = 0; d <= IO_IO_IN_DELAY_MAX; d++) {
1880                 scc_mgr_apply_group_dq_in_delay(write_group, test_bgn, d);
1881
1882                 writel(0, &sdr_scc_mgr->update);
1883
1884                 /*
1885                  * Stop searching when the read test doesn't pass AND when
1886                  * we've seen a passing read on every bit.
1887                  */
1888                 if (use_read_test) {
1889                         stop = !rw_mgr_mem_calibrate_read_test(rank_bgn,
1890                                 read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT,
1891                                 &bit_chk, 0, 0);
1892                 } else {
1893                         rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
1894                                                         0, PASS_ONE_BIT,
1895                                                         &bit_chk, 0);
1896                         bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS *
1897                                 (read_group - (write_group *
1898                                         RW_MGR_MEM_IF_READ_DQS_WIDTH /
1899                                         RW_MGR_MEM_IF_WRITE_DQS_WIDTH)));
1900                         stop = (bit_chk == 0);
1901                 }
1902                 sticky_bit_chk = sticky_bit_chk | bit_chk;
1903                 stop = stop && (sticky_bit_chk == param->read_correct_mask);
1904                 debug_cond(DLEVEL == 2, "%s:%d vfifo_center(left): dtap=%u => %u == %u \
1905                            && %u", __func__, __LINE__, d,
1906                            sticky_bit_chk,
1907                         param->read_correct_mask, stop);
1908
1909                 if (stop == 1) {
1910                         break;
1911                 } else {
1912                         for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
1913                                 if (bit_chk & 1) {
1914                                         /* Remember a passing test as the
1915                                         left_edge */
1916                                         left_edge[i] = d;
1917                                 } else {
1918                                         /* If a left edge has not been seen yet,
1919                                         then a future passing test will mark
1920                                         this edge as the right edge */
1921                                         if (left_edge[i] ==
1922                                                 IO_IO_IN_DELAY_MAX + 1) {
1923                                                 right_edge[i] = -(d + 1);
1924                                         }
1925                                 }
1926                                 bit_chk = bit_chk >> 1;
1927                         }
1928                 }
1929         }
1930
1931         /* Reset DQ delay chains to 0 */
1932         scc_mgr_apply_group_dq_in_delay(write_group, test_bgn, 0);
1933         sticky_bit_chk = 0;
1934         for (i = RW_MGR_MEM_DQ_PER_READ_DQS - 1;; i--) {
1935                 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: left_edge[%u]: \
1936                            %d right_edge[%u]: %d\n", __func__, __LINE__,
1937                            i, left_edge[i], i, right_edge[i]);
1938
1939                 /*
1940                  * Check for cases where we haven't found the left edge,
1941                  * which makes our assignment of the the right edge invalid.
1942                  * Reset it to the illegal value.
1943                  */
1944                 if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) && (
1945                         right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) {
1946                         right_edge[i] = IO_IO_IN_DELAY_MAX + 1;
1947                         debug_cond(DLEVEL == 2, "%s:%d vfifo_center: reset \
1948                                    right_edge[%u]: %d\n", __func__, __LINE__,
1949                                    i, right_edge[i]);
1950                 }
1951
1952                 /*
1953                  * Reset sticky bit (except for bits where we have seen
1954                  * both the left and right edge).
1955                  */
1956                 sticky_bit_chk = sticky_bit_chk << 1;
1957                 if ((left_edge[i] != IO_IO_IN_DELAY_MAX + 1) &&
1958                     (right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) {
1959                         sticky_bit_chk = sticky_bit_chk | 1;
1960                 }
1961
1962                 if (i == 0)
1963                         break;
1964         }
1965
1966         /* Search for the right edge of the window for each bit */
1967         for (d = 0; d <= IO_DQS_IN_DELAY_MAX - start_dqs; d++) {
1968                 scc_mgr_set_dqs_bus_in_delay(read_group, d + start_dqs);
1969                 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
1970                         uint32_t delay = d + start_dqs_en;
1971                         if (delay > IO_DQS_EN_DELAY_MAX)
1972                                 delay = IO_DQS_EN_DELAY_MAX;
1973                         scc_mgr_set_dqs_en_delay(read_group, delay);
1974                 }
1975                 scc_mgr_load_dqs(read_group);
1976
1977                 writel(0, &sdr_scc_mgr->update);
1978
1979                 /*
1980                  * Stop searching when the read test doesn't pass AND when
1981                  * we've seen a passing read on every bit.
1982                  */
1983                 if (use_read_test) {
1984                         stop = !rw_mgr_mem_calibrate_read_test(rank_bgn,
1985                                 read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT,
1986                                 &bit_chk, 0, 0);
1987                 } else {
1988                         rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
1989                                                         0, PASS_ONE_BIT,
1990                                                         &bit_chk, 0);
1991                         bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS *
1992                                 (read_group - (write_group *
1993                                         RW_MGR_MEM_IF_READ_DQS_WIDTH /
1994                                         RW_MGR_MEM_IF_WRITE_DQS_WIDTH)));
1995                         stop = (bit_chk == 0);
1996                 }
1997                 sticky_bit_chk = sticky_bit_chk | bit_chk;
1998                 stop = stop && (sticky_bit_chk == param->read_correct_mask);
1999
2000                 debug_cond(DLEVEL == 2, "%s:%d vfifo_center(right): dtap=%u => %u == \
2001                            %u && %u", __func__, __LINE__, d,
2002                            sticky_bit_chk, param->read_correct_mask, stop);
2003
2004                 if (stop == 1) {
2005                         break;
2006                 } else {
2007                         for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
2008                                 if (bit_chk & 1) {
2009                                         /* Remember a passing test as
2010                                         the right_edge */
2011                                         right_edge[i] = d;
2012                                 } else {
2013                                         if (d != 0) {
2014                                                 /* If a right edge has not been
2015                                                 seen yet, then a future passing
2016                                                 test will mark this edge as the
2017                                                 left edge */
2018                                                 if (right_edge[i] ==
2019                                                 IO_IO_IN_DELAY_MAX + 1) {
2020                                                         left_edge[i] = -(d + 1);
2021                                                 }
2022                                         } else {
2023                                                 /* d = 0 failed, but it passed
2024                                                 when testing the left edge,
2025                                                 so it must be marginal,
2026                                                 set it to -1 */
2027                                                 if (right_edge[i] ==
2028                                                         IO_IO_IN_DELAY_MAX + 1 &&
2029                                                         left_edge[i] !=
2030                                                         IO_IO_IN_DELAY_MAX
2031                                                         + 1) {
2032                                                         right_edge[i] = -1;
2033                                                 }
2034                                                 /* If a right edge has not been
2035                                                 seen yet, then a future passing
2036                                                 test will mark this edge as the
2037                                                 left edge */
2038                                                 else if (right_edge[i] ==
2039                                                         IO_IO_IN_DELAY_MAX +
2040                                                         1) {
2041                                                         left_edge[i] = -(d + 1);
2042                                                 }
2043                                         }
2044                                 }
2045
2046                                 debug_cond(DLEVEL == 2, "%s:%d vfifo_center[r,\
2047                                            d=%u]: ", __func__, __LINE__, d);
2048                                 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d ",
2049                                            (int)(bit_chk & 1), i, left_edge[i]);
2050                                 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
2051                                            right_edge[i]);
2052                                 bit_chk = bit_chk >> 1;
2053                         }
2054                 }
2055         }
2056
2057         /* Check that all bits have a window */
2058         for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
2059                 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: left_edge[%u]: \
2060                            %d right_edge[%u]: %d", __func__, __LINE__,
2061                            i, left_edge[i], i, right_edge[i]);
2062                 if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) || (right_edge[i]
2063                         == IO_IO_IN_DELAY_MAX + 1)) {
2064                         /*
2065                          * Restore delay chain settings before letting the loop
2066                          * in rw_mgr_mem_calibrate_vfifo to retry different
2067                          * dqs/ck relationships.
2068                          */
2069                         scc_mgr_set_dqs_bus_in_delay(read_group, start_dqs);
2070                         if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
2071                                 scc_mgr_set_dqs_en_delay(read_group,
2072                                                          start_dqs_en);
2073                         }
2074                         scc_mgr_load_dqs(read_group);
2075                         writel(0, &sdr_scc_mgr->update);
2076
2077                         debug_cond(DLEVEL == 1, "%s:%d vfifo_center: failed to \
2078                                    find edge [%u]: %d %d", __func__, __LINE__,
2079                                    i, left_edge[i], right_edge[i]);
2080                         if (use_read_test) {
2081                                 set_failing_group_stage(read_group *
2082                                         RW_MGR_MEM_DQ_PER_READ_DQS + i,
2083                                         CAL_STAGE_VFIFO,
2084                                         CAL_SUBSTAGE_VFIFO_CENTER);
2085                         } else {
2086                                 set_failing_group_stage(read_group *
2087                                         RW_MGR_MEM_DQ_PER_READ_DQS + i,
2088                                         CAL_STAGE_VFIFO_AFTER_WRITES,
2089                                         CAL_SUBSTAGE_VFIFO_CENTER);
2090                         }
2091                         return 0;
2092                 }
2093         }
2094
2095         /* Find middle of window for each DQ bit */
2096         mid_min = left_edge[0] - right_edge[0];
2097         min_index = 0;
2098         for (i = 1; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
2099                 mid = left_edge[i] - right_edge[i];
2100                 if (mid < mid_min) {
2101                         mid_min = mid;
2102                         min_index = i;
2103                 }
2104         }
2105
2106         /*
2107          * -mid_min/2 represents the amount that we need to move DQS.
2108          * If mid_min is odd and positive we'll need to add one to
2109          * make sure the rounding in further calculations is correct
2110          * (always bias to the right), so just add 1 for all positive values.
2111          */
2112         if (mid_min > 0)
2113                 mid_min++;
2114
2115         mid_min = mid_min / 2;
2116
2117         debug_cond(DLEVEL == 1, "%s:%d vfifo_center: mid_min=%d (index=%u)\n",
2118                    __func__, __LINE__, mid_min, min_index);
2119
2120         /* Determine the amount we can change DQS (which is -mid_min) */
2121         orig_mid_min = mid_min;
2122         new_dqs = start_dqs - mid_min;
2123         if (new_dqs > IO_DQS_IN_DELAY_MAX)
2124                 new_dqs = IO_DQS_IN_DELAY_MAX;
2125         else if (new_dqs < 0)
2126                 new_dqs = 0;
2127
2128         mid_min = start_dqs - new_dqs;
2129         debug_cond(DLEVEL == 1, "vfifo_center: new mid_min=%d new_dqs=%d\n",
2130                    mid_min, new_dqs);
2131
2132         if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
2133                 if (start_dqs_en - mid_min > IO_DQS_EN_DELAY_MAX)
2134                         mid_min += start_dqs_en - mid_min - IO_DQS_EN_DELAY_MAX;
2135                 else if (start_dqs_en - mid_min < 0)
2136                         mid_min += start_dqs_en - mid_min;
2137         }
2138         new_dqs = start_dqs - mid_min;
2139
2140         debug_cond(DLEVEL == 1, "vfifo_center: start_dqs=%d start_dqs_en=%d \
2141                    new_dqs=%d mid_min=%d\n", start_dqs,
2142                    IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS ? start_dqs_en : -1,
2143                    new_dqs, mid_min);
2144
2145         /* Initialize data for export structures */
2146         dqs_margin = IO_IO_IN_DELAY_MAX + 1;
2147         dq_margin  = IO_IO_IN_DELAY_MAX + 1;
2148
2149         /* add delay to bring centre of all DQ windows to the same "level" */
2150         for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) {
2151                 /* Use values before divide by 2 to reduce round off error */
2152                 shift_dq = (left_edge[i] - right_edge[i] -
2153                         (left_edge[min_index] - right_edge[min_index]))/2  +
2154                         (orig_mid_min - mid_min);
2155
2156                 debug_cond(DLEVEL == 2, "vfifo_center: before: \
2157                            shift_dq[%u]=%d\n", i, shift_dq);
2158
2159                 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_IN_DELAY_OFFSET;
2160                 temp_dq_in_delay1 = readl(addr + (p << 2));
2161                 temp_dq_in_delay2 = readl(addr + (i << 2));
2162
2163                 if (shift_dq + (int32_t)temp_dq_in_delay1 >
2164                         (int32_t)IO_IO_IN_DELAY_MAX) {
2165                         shift_dq = (int32_t)IO_IO_IN_DELAY_MAX - temp_dq_in_delay2;
2166                 } else if (shift_dq + (int32_t)temp_dq_in_delay1 < 0) {
2167                         shift_dq = -(int32_t)temp_dq_in_delay1;
2168                 }
2169                 debug_cond(DLEVEL == 2, "vfifo_center: after: \
2170                            shift_dq[%u]=%d\n", i, shift_dq);
2171                 final_dq[i] = temp_dq_in_delay1 + shift_dq;
2172                 scc_mgr_set_dq_in_delay(p, final_dq[i]);
2173                 scc_mgr_load_dq(p);
2174
2175                 debug_cond(DLEVEL == 2, "vfifo_center: margin[%u]=[%d,%d]\n", i,
2176                            left_edge[i] - shift_dq + (-mid_min),
2177                            right_edge[i] + shift_dq - (-mid_min));
2178                 /* To determine values for export structures */
2179                 if (left_edge[i] - shift_dq + (-mid_min) < dq_margin)
2180                         dq_margin = left_edge[i] - shift_dq + (-mid_min);
2181
2182                 if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin)
2183                         dqs_margin = right_edge[i] + shift_dq - (-mid_min);
2184         }
2185
2186         final_dqs = new_dqs;
2187         if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS)
2188                 final_dqs_en = start_dqs_en - mid_min;
2189
2190         /* Move DQS-en */
2191         if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
2192                 scc_mgr_set_dqs_en_delay(read_group, final_dqs_en);
2193                 scc_mgr_load_dqs(read_group);
2194         }
2195
2196         /* Move DQS */
2197         scc_mgr_set_dqs_bus_in_delay(read_group, final_dqs);
2198         scc_mgr_load_dqs(read_group);
2199         debug_cond(DLEVEL == 2, "%s:%d vfifo_center: dq_margin=%d \
2200                    dqs_margin=%d", __func__, __LINE__,
2201                    dq_margin, dqs_margin);
2202
2203         /*
2204          * Do not remove this line as it makes sure all of our decisions
2205          * have been applied. Apply the update bit.
2206          */
2207         writel(0, &sdr_scc_mgr->update);
2208
2209         return (dq_margin >= 0) && (dqs_margin >= 0);
2210 }
2211
2212 /*
2213  * calibrate the read valid prediction FIFO.
2214  *
2215  *  - read valid prediction will consist of finding a good DQS enable phase,
2216  * DQS enable delay, DQS input phase, and DQS input delay.
2217  *  - we also do a per-bit deskew on the DQ lines.
2218  */
2219 static uint32_t rw_mgr_mem_calibrate_vfifo(uint32_t read_group,
2220                                            uint32_t test_bgn)
2221 {
2222         uint32_t p, d, rank_bgn, sr;
2223         uint32_t dtaps_per_ptap;
2224         uint32_t tmp_delay;
2225         uint32_t bit_chk;
2226         uint32_t grp_calibrated;
2227         uint32_t write_group, write_test_bgn;
2228         uint32_t failed_substage;
2229
2230         debug("%s:%d: %u %u\n", __func__, __LINE__, read_group, test_bgn);
2231
2232         /* update info for sims */
2233         reg_file_set_stage(CAL_STAGE_VFIFO);
2234
2235         write_group = read_group;
2236         write_test_bgn = test_bgn;
2237
2238         /* USER Determine number of delay taps for each phase tap */
2239         dtaps_per_ptap = 0;
2240         tmp_delay = 0;
2241         while (tmp_delay < IO_DELAY_PER_OPA_TAP) {
2242                 dtaps_per_ptap++;
2243                 tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
2244         }
2245         dtaps_per_ptap--;
2246         tmp_delay = 0;
2247
2248         /* update info for sims */
2249         reg_file_set_group(read_group);
2250
2251         grp_calibrated = 0;
2252
2253         reg_file_set_sub_stage(CAL_SUBSTAGE_GUARANTEED_READ);
2254         failed_substage = CAL_SUBSTAGE_GUARANTEED_READ;
2255
2256         for (d = 0; d <= dtaps_per_ptap && grp_calibrated == 0; d += 2) {
2257                 /*
2258                  * In RLDRAMX we may be messing the delay of pins in
2259                  * the same write group but outside of the current read
2260                  * the group, but that's ok because we haven't
2261                  * calibrated output side yet.
2262                  */
2263                 if (d > 0) {
2264                         scc_mgr_apply_group_all_out_delay_add_all_ranks
2265                         (write_group, write_test_bgn, d);
2266                 }
2267
2268                 for (p = 0; p <= IO_DQDQS_OUT_PHASE_MAX && grp_calibrated == 0;
2269                         p++) {
2270                         /* set a particular dqdqs phase */
2271                         scc_mgr_set_dqdqs_output_phase_all_ranks(read_group, p);
2272
2273                         debug_cond(DLEVEL == 1, "%s:%d calibrate_vfifo: g=%u \
2274                                    p=%u d=%u\n", __func__, __LINE__,
2275                                    read_group, p, d);
2276
2277                         /*
2278                          * Load up the patterns used by read calibration
2279                          * using current DQDQS phase.
2280                          */
2281                         rw_mgr_mem_calibrate_read_load_patterns(0, 1);
2282                         if (!(gbl->phy_debug_mode_flags &
2283                                 PHY_DEBUG_DISABLE_GUARANTEED_READ)) {
2284                                 if (!rw_mgr_mem_calibrate_read_test_patterns_all_ranks
2285                                     (read_group, 1, &bit_chk)) {
2286                                         debug_cond(DLEVEL == 1, "%s:%d Guaranteed read test failed:",
2287                                                    __func__, __LINE__);
2288                                         debug_cond(DLEVEL == 1, " g=%u p=%u d=%u\n",
2289                                                    read_group, p, d);
2290                                         break;
2291                                 }
2292                         }
2293
2294 /* case:56390 */
2295                         grp_calibrated = 1;
2296                 if (rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay
2297                     (write_group, read_group, test_bgn)) {
2298                                 /*
2299                                  * USER Read per-bit deskew can be done on a
2300                                  * per shadow register basis.
2301                                  */
2302                                 for (rank_bgn = 0, sr = 0;
2303                                         rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS;
2304                                         rank_bgn += NUM_RANKS_PER_SHADOW_REG,
2305                                         ++sr) {
2306                                         /*
2307                                          * Determine if this set of ranks
2308                                          * should be skipped entirely.
2309                                          */
2310                                         if (!param->skip_shadow_regs[sr]) {
2311                                                 /*
2312                                                  * If doing read after write
2313                                                  * calibration, do not update
2314                                                  * FOM, now - do it then.
2315                                                  */
2316                                         if (!rw_mgr_mem_calibrate_vfifo_center
2317                                                 (rank_bgn, write_group,
2318                                                 read_group, test_bgn, 1, 0)) {
2319                                                         grp_calibrated = 0;
2320                                                         failed_substage =
2321                                                 CAL_SUBSTAGE_VFIFO_CENTER;
2322                                                 }
2323                                         }
2324                                 }
2325                         } else {
2326                                 grp_calibrated = 0;
2327                                 failed_substage = CAL_SUBSTAGE_DQS_EN_PHASE;
2328                         }
2329                 }
2330         }
2331
2332         if (grp_calibrated == 0) {
2333                 set_failing_group_stage(write_group, CAL_STAGE_VFIFO,
2334                                         failed_substage);
2335                 return 0;
2336         }
2337
2338         /*
2339          * Reset the delay chains back to zero if they have moved > 1
2340          * (check for > 1 because loop will increase d even when pass in
2341          * first case).
2342          */
2343         if (d > 2)
2344                 scc_mgr_zero_group(write_group, write_test_bgn, 1);
2345
2346         return 1;
2347 }
2348
2349 /* VFIFO Calibration -- Read Deskew Calibration after write deskew */
2350 static uint32_t rw_mgr_mem_calibrate_vfifo_end(uint32_t read_group,
2351                                                uint32_t test_bgn)
2352 {
2353         uint32_t rank_bgn, sr;
2354         uint32_t grp_calibrated;
2355         uint32_t write_group;
2356
2357         debug("%s:%d %u %u", __func__, __LINE__, read_group, test_bgn);
2358
2359         /* update info for sims */
2360
2361         reg_file_set_stage(CAL_STAGE_VFIFO_AFTER_WRITES);
2362         reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
2363
2364         write_group = read_group;
2365
2366         /* update info for sims */
2367         reg_file_set_group(read_group);
2368
2369         grp_calibrated = 1;
2370         /* Read per-bit deskew can be done on a per shadow register basis */
2371         for (rank_bgn = 0, sr = 0; rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS;
2372                 rank_bgn += NUM_RANKS_PER_SHADOW_REG, ++sr) {
2373                 /* Determine if this set of ranks should be skipped entirely */
2374                 if (!param->skip_shadow_regs[sr]) {
2375                 /* This is the last calibration round, update FOM here */
2376                         if (!rw_mgr_mem_calibrate_vfifo_center(rank_bgn,
2377                                                                 write_group,
2378                                                                 read_group,
2379                                                                 test_bgn, 0,
2380                                                                 1)) {
2381                                 grp_calibrated = 0;
2382                         }
2383                 }
2384         }
2385
2386
2387         if (grp_calibrated == 0) {
2388                 set_failing_group_stage(write_group,
2389                                         CAL_STAGE_VFIFO_AFTER_WRITES,
2390                                         CAL_SUBSTAGE_VFIFO_CENTER);
2391                 return 0;
2392         }
2393
2394         return 1;
2395 }
2396
2397 /* Calibrate LFIFO to find smallest read latency */
2398 static uint32_t rw_mgr_mem_calibrate_lfifo(void)
2399 {
2400         uint32_t found_one;
2401         uint32_t bit_chk;
2402
2403         debug("%s:%d\n", __func__, __LINE__);
2404
2405         /* update info for sims */
2406         reg_file_set_stage(CAL_STAGE_LFIFO);
2407         reg_file_set_sub_stage(CAL_SUBSTAGE_READ_LATENCY);
2408
2409         /* Load up the patterns used by read calibration for all ranks */
2410         rw_mgr_mem_calibrate_read_load_patterns(0, 1);
2411         found_one = 0;
2412
2413         do {
2414                 writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
2415                 debug_cond(DLEVEL == 2, "%s:%d lfifo: read_lat=%u",
2416                            __func__, __LINE__, gbl->curr_read_lat);
2417
2418                 if (!rw_mgr_mem_calibrate_read_test_all_ranks(0,
2419                                                               NUM_READ_TESTS,
2420                                                               PASS_ALL_BITS,
2421                                                               &bit_chk, 1)) {
2422                         break;
2423                 }
2424
2425                 found_one = 1;
2426                 /* reduce read latency and see if things are working */
2427                 /* correctly */
2428                 gbl->curr_read_lat--;
2429         } while (gbl->curr_read_lat > 0);
2430
2431         /* reset the fifos to get pointers to known state */
2432
2433         writel(0, &phy_mgr_cmd->fifo_reset);
2434
2435         if (found_one) {
2436                 /* add a fudge factor to the read latency that was determined */
2437                 gbl->curr_read_lat += 2;
2438                 writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
2439                 debug_cond(DLEVEL == 2, "%s:%d lfifo: success: using \
2440                            read_lat=%u\n", __func__, __LINE__,
2441                            gbl->curr_read_lat);
2442                 return 1;
2443         } else {
2444                 set_failing_group_stage(0xff, CAL_STAGE_LFIFO,
2445                                         CAL_SUBSTAGE_READ_LATENCY);
2446
2447                 debug_cond(DLEVEL == 2, "%s:%d lfifo: failed at initial \
2448                            read_lat=%u\n", __func__, __LINE__,
2449                            gbl->curr_read_lat);
2450                 return 0;
2451         }
2452 }
2453
2454 /*
2455  * issue write test command.
2456  * two variants are provided. one that just tests a write pattern and
2457  * another that tests datamask functionality.
2458  */
2459 static void rw_mgr_mem_calibrate_write_test_issue(uint32_t group,
2460                                                   uint32_t test_dm)
2461 {
2462         uint32_t mcc_instruction;
2463         uint32_t quick_write_mode = (((STATIC_CALIB_STEPS) & CALIB_SKIP_WRITES) &&
2464                 ENABLE_SUPER_QUICK_CALIBRATION);
2465         uint32_t rw_wl_nop_cycles;
2466         uint32_t addr;
2467
2468         /*
2469          * Set counter and jump addresses for the right
2470          * number of NOP cycles.
2471          * The number of supported NOP cycles can range from -1 to infinity
2472          * Three different cases are handled:
2473          *
2474          * 1. For a number of NOP cycles greater than 0, the RW Mgr looping
2475          *    mechanism will be used to insert the right number of NOPs
2476          *
2477          * 2. For a number of NOP cycles equals to 0, the micro-instruction
2478          *    issuing the write command will jump straight to the
2479          *    micro-instruction that turns on DQS (for DDRx), or outputs write
2480          *    data (for RLD), skipping
2481          *    the NOP micro-instruction all together
2482          *
2483          * 3. A number of NOP cycles equal to -1 indicates that DQS must be
2484          *    turned on in the same micro-instruction that issues the write
2485          *    command. Then we need
2486          *    to directly jump to the micro-instruction that sends out the data
2487          *
2488          * NOTE: Implementing this mechanism uses 2 RW Mgr jump-counters
2489          *       (2 and 3). One jump-counter (0) is used to perform multiple
2490          *       write-read operations.
2491          *       one counter left to issue this command in "multiple-group" mode
2492          */
2493
2494         rw_wl_nop_cycles = gbl->rw_wl_nop_cycles;
2495
2496         if (rw_wl_nop_cycles == -1) {
2497                 /*
2498                  * CNTR 2 - We want to execute the special write operation that
2499                  * turns on DQS right away and then skip directly to the
2500                  * instruction that sends out the data. We set the counter to a
2501                  * large number so that the jump is always taken.
2502                  */
2503                 writel(0xFF, &sdr_rw_load_mgr_regs->load_cntr2);
2504
2505                 /* CNTR 3 - Not used */
2506                 if (test_dm) {
2507                         mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0_WL_1;
2508                         writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_DATA,
2509                                &sdr_rw_load_jump_mgr_regs->load_jump_add2);
2510                         writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP,
2511                                &sdr_rw_load_jump_mgr_regs->load_jump_add3);
2512                 } else {
2513                         mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0_WL_1;
2514                         writel(RW_MGR_LFSR_WR_RD_BANK_0_DATA,
2515                                 &sdr_rw_load_jump_mgr_regs->load_jump_add2);
2516                         writel(RW_MGR_LFSR_WR_RD_BANK_0_NOP,
2517                                 &sdr_rw_load_jump_mgr_regs->load_jump_add3);
2518                 }
2519         } else if (rw_wl_nop_cycles == 0) {
2520                 /*
2521                  * CNTR 2 - We want to skip the NOP operation and go straight
2522                  * to the DQS enable instruction. We set the counter to a large
2523                  * number so that the jump is always taken.
2524                  */
2525                 writel(0xFF, &sdr_rw_load_mgr_regs->load_cntr2);
2526
2527                 /* CNTR 3 - Not used */
2528                 if (test_dm) {
2529                         mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0;
2530                         writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_DQS,
2531                                &sdr_rw_load_jump_mgr_regs->load_jump_add2);
2532                 } else {
2533                         mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0;
2534                         writel(RW_MGR_LFSR_WR_RD_BANK_0_DQS,
2535                                 &sdr_rw_load_jump_mgr_regs->load_jump_add2);
2536                 }
2537         } else {
2538                 /*
2539                  * CNTR 2 - In this case we want to execute the next instruction
2540                  * and NOT take the jump. So we set the counter to 0. The jump
2541                  * address doesn't count.
2542                  */
2543                 writel(0x0, &sdr_rw_load_mgr_regs->load_cntr2);
2544                 writel(0x0, &sdr_rw_load_jump_mgr_regs->load_jump_add2);
2545
2546                 /*
2547                  * CNTR 3 - Set the nop counter to the number of cycles we
2548                  * need to loop for, minus 1.
2549                  */
2550                 writel(rw_wl_nop_cycles - 1, &sdr_rw_load_mgr_regs->load_cntr3);
2551                 if (test_dm) {
2552                         mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0;
2553                         writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP,
2554                                 &sdr_rw_load_jump_mgr_regs->load_jump_add3);
2555                 } else {
2556                         mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0;
2557                         writel(RW_MGR_LFSR_WR_RD_BANK_0_NOP,
2558                                 &sdr_rw_load_jump_mgr_regs->load_jump_add3);
2559                 }
2560         }
2561
2562         writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
2563                   RW_MGR_RESET_READ_DATAPATH_OFFSET);
2564
2565         if (quick_write_mode)
2566                 writel(0x08, &sdr_rw_load_mgr_regs->load_cntr0);
2567         else
2568                 writel(0x40, &sdr_rw_load_mgr_regs->load_cntr0);
2569
2570         writel(mcc_instruction, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
2571
2572         /*
2573          * CNTR 1 - This is used to ensure enough time elapses
2574          * for read data to come back.
2575          */
2576         writel(0x30, &sdr_rw_load_mgr_regs->load_cntr1);
2577
2578         if (test_dm) {
2579                 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_WAIT,
2580                         &sdr_rw_load_jump_mgr_regs->load_jump_add1);
2581         } else {
2582                 writel(RW_MGR_LFSR_WR_RD_BANK_0_WAIT,
2583                         &sdr_rw_load_jump_mgr_regs->load_jump_add1);
2584         }
2585
2586         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
2587         writel(mcc_instruction, addr + (group << 2));
2588 }
2589
2590 /* Test writes, can check for a single bit pass or multiple bit pass */
2591 static uint32_t rw_mgr_mem_calibrate_write_test(uint32_t rank_bgn,
2592         uint32_t write_group, uint32_t use_dm, uint32_t all_correct,
2593         uint32_t *bit_chk, uint32_t all_ranks)
2594 {
2595         uint32_t r;
2596         uint32_t correct_mask_vg;
2597         uint32_t tmp_bit_chk;
2598         uint32_t vg;
2599         uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
2600                 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
2601         uint32_t addr_rw_mgr;
2602         uint32_t base_rw_mgr;
2603
2604         *bit_chk = param->write_correct_mask;
2605         correct_mask_vg = param->write_correct_mask_vg;
2606
2607         for (r = rank_bgn; r < rank_end; r++) {
2608                 if (param->skip_ranks[r]) {
2609                         /* request to skip the rank */
2610                         continue;
2611                 }
2612
2613                 /* set rank */
2614                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
2615
2616                 tmp_bit_chk = 0;
2617                 addr_rw_mgr = SDR_PHYGRP_RWMGRGRP_ADDRESS;
2618                 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS-1; ; vg--) {
2619                         /* reset the fifos to get pointers to known state */
2620                         writel(0, &phy_mgr_cmd->fifo_reset);
2621
2622                         tmp_bit_chk = tmp_bit_chk <<
2623                                 (RW_MGR_MEM_DQ_PER_WRITE_DQS /
2624                                 RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS);
2625                         rw_mgr_mem_calibrate_write_test_issue(write_group *
2626                                 RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS+vg,
2627                                 use_dm);
2628
2629                         base_rw_mgr = readl(addr_rw_mgr);
2630                         tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(base_rw_mgr));
2631                         if (vg == 0)
2632                                 break;
2633                 }
2634                 *bit_chk &= tmp_bit_chk;
2635         }
2636
2637         if (all_correct) {
2638                 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
2639                 debug_cond(DLEVEL == 2, "write_test(%u,%u,ALL) : %u == \
2640                            %u => %lu", write_group, use_dm,
2641                            *bit_chk, param->write_correct_mask,
2642                            (long unsigned int)(*bit_chk ==
2643                            param->write_correct_mask));
2644                 return *bit_chk == param->write_correct_mask;
2645         } else {
2646                 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
2647                 debug_cond(DLEVEL == 2, "write_test(%u,%u,ONE) : %u != ",
2648                        write_group, use_dm, *bit_chk);
2649                 debug_cond(DLEVEL == 2, "%lu" " => %lu", (long unsigned int)0,
2650                         (long unsigned int)(*bit_chk != 0));
2651                 return *bit_chk != 0x00;
2652         }
2653 }
2654
2655 /*
2656  * center all windows. do per-bit-deskew to possibly increase size of
2657  * certain windows.
2658  */
2659 static uint32_t rw_mgr_mem_calibrate_writes_center(uint32_t rank_bgn,
2660         uint32_t write_group, uint32_t test_bgn)
2661 {
2662         uint32_t i, p, min_index;
2663         int32_t d;
2664         /*
2665          * Store these as signed since there are comparisons with
2666          * signed numbers.
2667          */
2668         uint32_t bit_chk;
2669         uint32_t sticky_bit_chk;
2670         int32_t left_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS];
2671         int32_t right_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS];
2672         int32_t mid;
2673         int32_t mid_min, orig_mid_min;
2674         int32_t new_dqs, start_dqs, shift_dq;
2675         int32_t dq_margin, dqs_margin, dm_margin;
2676         uint32_t stop;
2677         uint32_t temp_dq_out1_delay;
2678         uint32_t addr;
2679
2680         debug("%s:%d %u %u", __func__, __LINE__, write_group, test_bgn);
2681
2682         dm_margin = 0;
2683
2684         addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_OUT1_DELAY_OFFSET;
2685         start_dqs = readl(addr +
2686                           (RW_MGR_MEM_DQ_PER_WRITE_DQS << 2));
2687
2688         /* per-bit deskew */
2689
2690         /*
2691          * set the left and right edge of each bit to an illegal value
2692          * use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value.
2693          */
2694         sticky_bit_chk = 0;
2695         for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2696                 left_edge[i]  = IO_IO_OUT1_DELAY_MAX + 1;
2697                 right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
2698         }
2699
2700         /* Search for the left edge of the window for each bit */
2701         for (d = 0; d <= IO_IO_OUT1_DELAY_MAX; d++) {
2702                 scc_mgr_apply_group_dq_out1_delay(write_group, test_bgn, d);
2703
2704                 writel(0, &sdr_scc_mgr->update);
2705
2706                 /*
2707                  * Stop searching when the read test doesn't pass AND when
2708                  * we've seen a passing read on every bit.
2709                  */
2710                 stop = !rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
2711                         0, PASS_ONE_BIT, &bit_chk, 0);
2712                 sticky_bit_chk = sticky_bit_chk | bit_chk;
2713                 stop = stop && (sticky_bit_chk == param->write_correct_mask);
2714                 debug_cond(DLEVEL == 2, "write_center(left): dtap=%d => %u \
2715                            == %u && %u [bit_chk= %u ]\n",
2716                         d, sticky_bit_chk, param->write_correct_mask,
2717                         stop, bit_chk);
2718
2719                 if (stop == 1) {
2720                         break;
2721                 } else {
2722                         for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2723                                 if (bit_chk & 1) {
2724                                         /*
2725                                          * Remember a passing test as the
2726                                          * left_edge.
2727                                          */
2728                                         left_edge[i] = d;
2729                                 } else {
2730                                         /*
2731                                          * If a left edge has not been seen
2732                                          * yet, then a future passing test will
2733                                          * mark this edge as the right edge.
2734                                          */
2735                                         if (left_edge[i] ==
2736                                                 IO_IO_OUT1_DELAY_MAX + 1) {
2737                                                 right_edge[i] = -(d + 1);
2738                                         }
2739                                 }
2740                                 debug_cond(DLEVEL == 2, "write_center[l,d=%d):", d);
2741                                 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d",
2742                                            (int)(bit_chk & 1), i, left_edge[i]);
2743                                 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
2744                                        right_edge[i]);
2745                                 bit_chk = bit_chk >> 1;
2746                         }
2747                 }
2748         }
2749
2750         /* Reset DQ delay chains to 0 */
2751         scc_mgr_apply_group_dq_out1_delay(write_group, test_bgn, 0);
2752         sticky_bit_chk = 0;
2753         for (i = RW_MGR_MEM_DQ_PER_WRITE_DQS - 1;; i--) {
2754                 debug_cond(DLEVEL == 2, "%s:%d write_center: left_edge[%u]: \
2755                            %d right_edge[%u]: %d\n", __func__, __LINE__,
2756                            i, left_edge[i], i, right_edge[i]);
2757
2758                 /*
2759                  * Check for cases where we haven't found the left edge,
2760                  * which makes our assignment of the the right edge invalid.
2761                  * Reset it to the illegal value.
2762                  */
2763                 if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) &&
2764                     (right_edge[i] != IO_IO_OUT1_DELAY_MAX + 1)) {
2765                         right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
2766                         debug_cond(DLEVEL == 2, "%s:%d write_center: reset \
2767                                    right_edge[%u]: %d\n", __func__, __LINE__,
2768                                    i, right_edge[i]);
2769                 }
2770
2771                 /*
2772                  * Reset sticky bit (except for bits where we have
2773                  * seen the left edge).
2774                  */
2775                 sticky_bit_chk = sticky_bit_chk << 1;
2776                 if ((left_edge[i] != IO_IO_OUT1_DELAY_MAX + 1))
2777                         sticky_bit_chk = sticky_bit_chk | 1;
2778
2779                 if (i == 0)
2780                         break;
2781         }
2782
2783         /* Search for the right edge of the window for each bit */
2784         for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - start_dqs; d++) {
2785                 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group,
2786                                                         d + start_dqs);
2787
2788                 writel(0, &sdr_scc_mgr->update);
2789
2790                 /*
2791                  * Stop searching when the read test doesn't pass AND when
2792                  * we've seen a passing read on every bit.
2793                  */
2794                 stop = !rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
2795                         0, PASS_ONE_BIT, &bit_chk, 0);
2796
2797                 sticky_bit_chk = sticky_bit_chk | bit_chk;
2798                 stop = stop && (sticky_bit_chk == param->write_correct_mask);
2799
2800                 debug_cond(DLEVEL == 2, "write_center (right): dtap=%u => %u == \
2801                            %u && %u\n", d, sticky_bit_chk,
2802                            param->write_correct_mask, stop);
2803
2804                 if (stop == 1) {
2805                         if (d == 0) {
2806                                 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS;
2807                                         i++) {
2808                                         /* d = 0 failed, but it passed when
2809                                         testing the left edge, so it must be
2810                                         marginal, set it to -1 */
2811                                         if (right_edge[i] ==
2812                                                 IO_IO_OUT1_DELAY_MAX + 1 &&
2813                                                 left_edge[i] !=
2814                                                 IO_IO_OUT1_DELAY_MAX + 1) {
2815                                                 right_edge[i] = -1;
2816                                         }
2817                                 }
2818                         }
2819                         break;
2820                 } else {
2821                         for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2822                                 if (bit_chk & 1) {
2823                                         /*
2824                                          * Remember a passing test as
2825                                          * the right_edge.
2826                                          */
2827                                         right_edge[i] = d;
2828                                 } else {
2829                                         if (d != 0) {
2830                                                 /*
2831                                                  * If a right edge has not
2832                                                  * been seen yet, then a future
2833                                                  * passing test will mark this
2834                                                  * edge as the left edge.
2835                                                  */
2836                                                 if (right_edge[i] ==
2837                                                     IO_IO_OUT1_DELAY_MAX + 1)
2838                                                         left_edge[i] = -(d + 1);
2839                                         } else {
2840                                                 /*
2841                                                  * d = 0 failed, but it passed
2842                                                  * when testing the left edge,
2843                                                  * so it must be marginal, set
2844                                                  * it to -1.
2845                                                  */
2846                                                 if (right_edge[i] ==
2847                                                     IO_IO_OUT1_DELAY_MAX + 1 &&
2848                                                     left_edge[i] !=
2849                                                     IO_IO_OUT1_DELAY_MAX + 1)
2850                                                         right_edge[i] = -1;
2851                                                 /*
2852                                                  * If a right edge has not been
2853                                                  * seen yet, then a future
2854                                                  * passing test will mark this
2855                                                  * edge as the left edge.
2856                                                  */
2857                                                 else if (right_edge[i] ==
2858                                                         IO_IO_OUT1_DELAY_MAX +
2859                                                         1)
2860                                                         left_edge[i] = -(d + 1);
2861                                         }
2862                                 }
2863                                 debug_cond(DLEVEL == 2, "write_center[r,d=%d):", d);
2864                                 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d",
2865                                            (int)(bit_chk & 1), i, left_edge[i]);
2866                                 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
2867                                            right_edge[i]);
2868                                 bit_chk = bit_chk >> 1;
2869                         }
2870                 }
2871         }
2872
2873         /* Check that all bits have a window */
2874         for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2875                 debug_cond(DLEVEL == 2, "%s:%d write_center: left_edge[%u]: \
2876                            %d right_edge[%u]: %d", __func__, __LINE__,
2877                            i, left_edge[i], i, right_edge[i]);
2878                 if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) ||
2879                     (right_edge[i] == IO_IO_OUT1_DELAY_MAX + 1)) {
2880                         set_failing_group_stage(test_bgn + i,
2881                                                 CAL_STAGE_WRITES,
2882                                                 CAL_SUBSTAGE_WRITES_CENTER);
2883                         return 0;
2884                 }
2885         }
2886
2887         /* Find middle of window for each DQ bit */
2888         mid_min = left_edge[0] - right_edge[0];
2889         min_index = 0;
2890         for (i = 1; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2891                 mid = left_edge[i] - right_edge[i];
2892                 if (mid < mid_min) {
2893                         mid_min = mid;
2894                         min_index = i;
2895                 }
2896         }
2897
2898         /*
2899          * -mid_min/2 represents the amount that we need to move DQS.
2900          * If mid_min is odd and positive we'll need to add one to
2901          * make sure the rounding in further calculations is correct
2902          * (always bias to the right), so just add 1 for all positive values.
2903          */
2904         if (mid_min > 0)
2905                 mid_min++;
2906         mid_min = mid_min / 2;
2907         debug_cond(DLEVEL == 1, "%s:%d write_center: mid_min=%d\n", __func__,
2908                    __LINE__, mid_min);
2909
2910         /* Determine the amount we can change DQS (which is -mid_min) */
2911         orig_mid_min = mid_min;
2912         new_dqs = start_dqs;
2913         mid_min = 0;
2914         debug_cond(DLEVEL == 1, "%s:%d write_center: start_dqs=%d new_dqs=%d \
2915                    mid_min=%d\n", __func__, __LINE__, start_dqs, new_dqs, mid_min);
2916         /* Initialize data for export structures */
2917         dqs_margin = IO_IO_OUT1_DELAY_MAX + 1;
2918         dq_margin  = IO_IO_OUT1_DELAY_MAX + 1;
2919
2920         /* add delay to bring centre of all DQ windows to the same "level" */
2921         for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) {
2922                 /* Use values before divide by 2 to reduce round off error */
2923                 shift_dq = (left_edge[i] - right_edge[i] -
2924                         (left_edge[min_index] - right_edge[min_index]))/2  +
2925                 (orig_mid_min - mid_min);
2926
2927                 debug_cond(DLEVEL == 2, "%s:%d write_center: before: shift_dq \
2928                            [%u]=%d\n", __func__, __LINE__, i, shift_dq);
2929
2930                 addr = SDR_PHYGRP_SCCGRP_ADDRESS | SCC_MGR_IO_OUT1_DELAY_OFFSET;
2931                 temp_dq_out1_delay = readl(addr + (i << 2));
2932                 if (shift_dq + (int32_t)temp_dq_out1_delay >
2933                         (int32_t)IO_IO_OUT1_DELAY_MAX) {
2934                         shift_dq = (int32_t)IO_IO_OUT1_DELAY_MAX - temp_dq_out1_delay;
2935                 } else if (shift_dq + (int32_t)temp_dq_out1_delay < 0) {
2936                         shift_dq = -(int32_t)temp_dq_out1_delay;
2937                 }
2938                 debug_cond(DLEVEL == 2, "write_center: after: shift_dq[%u]=%d\n",
2939                            i, shift_dq);
2940                 scc_mgr_set_dq_out1_delay(i, temp_dq_out1_delay + shift_dq);
2941                 scc_mgr_load_dq(i);
2942
2943                 debug_cond(DLEVEL == 2, "write_center: margin[%u]=[%d,%d]\n", i,
2944                            left_edge[i] - shift_dq + (-mid_min),
2945                            right_edge[i] + shift_dq - (-mid_min));
2946                 /* To determine values for export structures */
2947                 if (left_edge[i] - shift_dq + (-mid_min) < dq_margin)
2948                         dq_margin = left_edge[i] - shift_dq + (-mid_min);
2949
2950                 if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin)
2951                         dqs_margin = right_edge[i] + shift_dq - (-mid_min);
2952         }
2953
2954         /* Move DQS */
2955         scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs);
2956         writel(0, &sdr_scc_mgr->update);
2957
2958         /* Centre DM */
2959         debug_cond(DLEVEL == 2, "%s:%d write_center: DM\n", __func__, __LINE__);
2960
2961         /*
2962          * set the left and right edge of each bit to an illegal value,
2963          * use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value,
2964          */
2965         left_edge[0]  = IO_IO_OUT1_DELAY_MAX + 1;
2966         right_edge[0] = IO_IO_OUT1_DELAY_MAX + 1;
2967         int32_t bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
2968         int32_t end_curr = IO_IO_OUT1_DELAY_MAX + 1;
2969         int32_t bgn_best = IO_IO_OUT1_DELAY_MAX + 1;
2970         int32_t end_best = IO_IO_OUT1_DELAY_MAX + 1;
2971         int32_t win_best = 0;
2972
2973         /* Search for the/part of the window with DM shift */
2974         for (d = IO_IO_OUT1_DELAY_MAX; d >= 0; d -= DELTA_D) {
2975                 scc_mgr_apply_group_dm_out1_delay(write_group, d);
2976                 writel(0, &sdr_scc_mgr->update);
2977
2978                 if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1,
2979                                                     PASS_ALL_BITS, &bit_chk,
2980                                                     0)) {
2981                         /* USE Set current end of the window */
2982                         end_curr = -d;
2983                         /*
2984                          * If a starting edge of our window has not been seen
2985                          * this is our current start of the DM window.
2986                          */
2987                         if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1)
2988                                 bgn_curr = -d;
2989
2990                         /*
2991                          * If current window is bigger than best seen.
2992                          * Set best seen to be current window.
2993                          */
2994                         if ((end_curr-bgn_curr+1) > win_best) {
2995                                 win_best = end_curr-bgn_curr+1;
2996                                 bgn_best = bgn_curr;
2997                                 end_best = end_curr;
2998                         }
2999                 } else {
3000                         /* We just saw a failing test. Reset temp edge */
3001                         bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
3002                         end_curr = IO_IO_OUT1_DELAY_MAX + 1;
3003                         }
3004                 }
3005
3006
3007         /* Reset DM delay chains to 0 */
3008         scc_mgr_apply_group_dm_out1_delay(write_group, 0);
3009
3010         /*
3011          * Check to see if the current window nudges up aganist 0 delay.
3012          * If so we need to continue the search by shifting DQS otherwise DQS
3013          * search begins as a new search. */
3014         if (end_curr != 0) {
3015                 bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
3016                 end_curr = IO_IO_OUT1_DELAY_MAX + 1;
3017         }
3018
3019         /* Search for the/part of the window with DQS shifts */
3020         for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - new_dqs; d += DELTA_D) {
3021                 /*
3022                  * Note: This only shifts DQS, so are we limiting ourselve to
3023                  * width of DQ unnecessarily.
3024                  */
3025                 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group,
3026                                                         d + new_dqs);
3027
3028                 writel(0, &sdr_scc_mgr->update);
3029                 if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1,
3030                                                     PASS_ALL_BITS, &bit_chk,
3031                                                     0)) {
3032                         /* USE Set current end of the window */
3033                         end_curr = d;
3034                         /*
3035                          * If a beginning edge of our window has not been seen
3036                          * this is our current begin of the DM window.
3037                          */
3038                         if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1)
3039                                 bgn_curr = d;
3040
3041                         /*
3042                          * If current window is bigger than best seen. Set best
3043                          * seen to be current window.
3044                          */
3045                         if ((end_curr-bgn_curr+1) > win_best) {
3046                                 win_best = end_curr-bgn_curr+1;
3047                                 bgn_best = bgn_curr;
3048                                 end_best = end_curr;
3049                         }
3050                 } else {
3051                         /* We just saw a failing test. Reset temp edge */
3052                         bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
3053                         end_curr = IO_IO_OUT1_DELAY_MAX + 1;
3054
3055                         /* Early exit optimization: if ther remaining delay
3056                         chain space is less than already seen largest window
3057                         we can exit */
3058                         if ((win_best-1) >
3059                                 (IO_IO_OUT1_DELAY_MAX - new_dqs - d)) {
3060                                         break;
3061                                 }
3062                         }
3063                 }
3064
3065         /* assign left and right edge for cal and reporting; */
3066         left_edge[0] = -1*bgn_best;
3067         right_edge[0] = end_best;
3068
3069         debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d\n", __func__,
3070                    __LINE__, left_edge[0], right_edge[0]);
3071
3072         /* Move DQS (back to orig) */
3073         scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs);
3074
3075         /* Move DM */
3076
3077         /* Find middle of window for the DM bit */
3078         mid = (left_edge[0] - right_edge[0]) / 2;
3079
3080         /* only move right, since we are not moving DQS/DQ */
3081         if (mid < 0)
3082                 mid = 0;
3083
3084         /* dm_marign should fail if we never find a window */
3085         if (win_best == 0)
3086                 dm_margin = -1;
3087         else
3088                 dm_margin = left_edge[0] - mid;
3089
3090         scc_mgr_apply_group_dm_out1_delay(write_group, mid);
3091         writel(0, &sdr_scc_mgr->update);
3092
3093         debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d mid=%d \
3094                    dm_margin=%d\n", __func__, __LINE__, left_edge[0],
3095                    right_edge[0], mid, dm_margin);
3096         /* Export values */
3097         gbl->fom_out += dq_margin + dqs_margin;
3098
3099         debug_cond(DLEVEL == 2, "%s:%d write_center: dq_margin=%d \
3100                    dqs_margin=%d dm_margin=%d\n", __func__, __LINE__,
3101                    dq_margin, dqs_margin, dm_margin);
3102
3103         /*
3104          * Do not remove this line as it makes sure all of our
3105          * decisions have been applied.
3106          */
3107         writel(0, &sdr_scc_mgr->update);
3108         return (dq_margin >= 0) && (dqs_margin >= 0) && (dm_margin >= 0);
3109 }
3110
3111 /* calibrate the write operations */
3112 static uint32_t rw_mgr_mem_calibrate_writes(uint32_t rank_bgn, uint32_t g,
3113         uint32_t test_bgn)
3114 {
3115         /* update info for sims */
3116         debug("%s:%d %u %u\n", __func__, __LINE__, g, test_bgn);
3117
3118         reg_file_set_stage(CAL_STAGE_WRITES);
3119         reg_file_set_sub_stage(CAL_SUBSTAGE_WRITES_CENTER);
3120
3121         reg_file_set_group(g);
3122
3123         if (!rw_mgr_mem_calibrate_writes_center(rank_bgn, g, test_bgn)) {
3124                 set_failing_group_stage(g, CAL_STAGE_WRITES,
3125                                         CAL_SUBSTAGE_WRITES_CENTER);
3126                 return 0;
3127         }
3128
3129         return 1;
3130 }
3131
3132 /* precharge all banks and activate row 0 in bank "000..." and bank "111..." */
3133 static void mem_precharge_and_activate(void)
3134 {
3135         uint32_t r;
3136
3137         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
3138                 if (param->skip_ranks[r]) {
3139                         /* request to skip the rank */
3140                         continue;
3141                 }
3142
3143                 /* set rank */
3144                 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);
3145
3146                 /* precharge all banks ... */
3147                 writel(RW_MGR_PRECHARGE_ALL, SDR_PHYGRP_RWMGRGRP_ADDRESS |
3148                                              RW_MGR_RUN_SINGLE_GROUP_OFFSET);
3149
3150                 writel(0x0F, &sdr_rw_load_mgr_regs->load_cntr0);
3151                 writel(RW_MGR_ACTIVATE_0_AND_1_WAIT1,
3152                         &sdr_rw_load_jump_mgr_regs->load_jump_add0);
3153
3154                 writel(0x0F, &sdr_rw_load_mgr_regs->load_cntr1);
3155                 writel(RW_MGR_ACTIVATE_0_AND_1_WAIT2,
3156                         &sdr_rw_load_jump_mgr_regs->load_jump_add1);
3157
3158                 /* activate rows */
3159                 writel(RW_MGR_ACTIVATE_0_AND_1, SDR_PHYGRP_RWMGRGRP_ADDRESS |
3160                                                 RW_MGR_RUN_SINGLE_GROUP_OFFSET);
3161         }
3162 }
3163
3164 /* Configure various memory related parameters. */
3165 static void mem_config(void)
3166 {
3167         uint32_t rlat, wlat;
3168         uint32_t rw_wl_nop_cycles;
3169         uint32_t max_latency;
3170
3171         debug("%s:%d\n", __func__, __LINE__);
3172         /* read in write and read latency */
3173         wlat = readl(&data_mgr->t_wl_add);
3174         wlat += readl(&data_mgr->mem_t_add);
3175
3176         /* WL for hard phy does not include additive latency */
3177
3178         /*
3179          * add addtional write latency to offset the address/command extra
3180          * clock cycle. We change the AC mux setting causing AC to be delayed
3181          * by one mem clock cycle. Only do this for DDR3
3182          */
3183         wlat = wlat + 1;
3184
3185         rlat = readl(&data_mgr->t_rl_add);
3186
3187         rw_wl_nop_cycles = wlat - 2;
3188         gbl->rw_wl_nop_cycles = rw_wl_nop_cycles;
3189
3190         /*
3191          * For AV/CV, lfifo is hardened and always runs at full rate so
3192          * max latency in AFI clocks, used here, is correspondingly smaller.
3193          */
3194         max_latency = (1<<MAX_LATENCY_COUNT_WIDTH)/1 - 1;
3195         /* configure for a burst length of 8 */
3196
3197         /* write latency */
3198         /* Adjust Write Latency for Hard PHY */
3199         wlat = wlat + 1;
3200
3201         /* set a pretty high read latency initially */
3202         gbl->curr_read_lat = rlat + 16;
3203
3204         if (gbl->curr_read_lat > max_latency)
3205                 gbl->curr_read_lat = max_latency;
3206
3207         writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
3208
3209         /* advertise write latency */
3210         gbl->curr_write_lat = wlat;
3211         writel(wlat - 2, &phy_mgr_cfg->afi_wlat);
3212
3213         /* initialize bit slips */
3214         mem_precharge_and_activate();
3215 }
3216
3217 /* Set VFIFO and LFIFO to instant-on settings in skip calibration mode */
3218 static void mem_skip_calibrate(void)
3219 {
3220         uint32_t vfifo_offset;
3221         uint32_t i, j, r;
3222
3223         debug("%s:%d\n", __func__, __LINE__);
3224         /* Need to update every shadow register set used by the interface */
3225         for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
3226                 r += NUM_RANKS_PER_SHADOW_REG) {
3227                 /*
3228                  * Set output phase alignment settings appropriate for
3229                  * skip calibration.
3230                  */
3231                 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
3232                         scc_mgr_set_dqs_en_phase(i, 0);
3233 #if IO_DLL_CHAIN_LENGTH == 6
3234                         scc_mgr_set_dqdqs_output_phase(i, 6);
3235 #else
3236                         scc_mgr_set_dqdqs_output_phase(i, 7);
3237 #endif
3238                         /*
3239                          * Case:33398
3240                          *
3241                          * Write data arrives to the I/O two cycles before write
3242                          * latency is reached (720 deg).
3243                          *   -> due to bit-slip in a/c bus
3244                          *   -> to allow board skew where dqs is longer than ck
3245                          *      -> how often can this happen!?
3246                          *      -> can claim back some ptaps for high freq
3247                          *       support if we can relax this, but i digress...
3248                          *
3249                          * The write_clk leads mem_ck by 90 deg
3250                          * The minimum ptap of the OPA is 180 deg
3251                          * Each ptap has (360 / IO_DLL_CHAIN_LENGH) deg of delay
3252                          * The write_clk is always delayed by 2 ptaps
3253                          *
3254                          * Hence, to make DQS aligned to CK, we need to delay
3255                          * DQS by:
3256                          *    (720 - 90 - 180 - 2 * (360 / IO_DLL_CHAIN_LENGTH))
3257                          *
3258                          * Dividing the above by (360 / IO_DLL_CHAIN_LENGTH)
3259                          * gives us the number of ptaps, which simplies to:
3260                          *
3261                          *    (1.25 * IO_DLL_CHAIN_LENGTH - 2)
3262                          */
3263                         scc_mgr_set_dqdqs_output_phase(i, (1.25 *
3264                                 IO_DLL_CHAIN_LENGTH - 2));
3265                 }
3266                 writel(0xff, &sdr_scc_mgr->dqs_ena);
3267                 writel(0xff, &sdr_scc_mgr->dqs_io_ena);
3268
3269                 for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) {
3270                         writel(i, SDR_PHYGRP_SCCGRP_ADDRESS |
3271                                   SCC_MGR_GROUP_COUNTER_OFFSET);
3272                 }
3273                 writel(0xff, &sdr_scc_mgr->dq_ena);
3274                 writel(0xff, &sdr_scc_mgr->dm_ena);
3275                 writel(0, &sdr_scc_mgr->update);
3276         }
3277
3278         /* Compensate for simulation model behaviour */
3279         for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
3280                 scc_mgr_set_dqs_bus_in_delay(i, 10);
3281                 scc_mgr_load_dqs(i);
3282         }
3283         writel(0, &sdr_scc_mgr->update);
3284
3285         /*
3286          * ArriaV has hard FIFOs that can only be initialized by incrementing
3287          * in sequencer.
3288          */
3289         vfifo_offset = CALIB_VFIFO_OFFSET;
3290         for (j = 0; j < vfifo_offset; j++) {
3291                 writel(0xff, &phy_mgr_cmd->inc_vfifo_hard_phy);
3292         }
3293         writel(0, &phy_mgr_cmd->fifo_reset);
3294
3295         /*
3296          * For ACV with hard lfifo, we get the skip-cal setting from
3297          * generation-time constant.
3298          */
3299         gbl->curr_read_lat = CALIB_LFIFO_OFFSET;
3300         writel(gbl->curr_read_lat, &phy_mgr_cfg->phy_rlat);
3301 }
3302
3303 /* Memory calibration entry point */
3304 static uint32_t mem_calibrate(void)
3305 {
3306         uint32_t i;
3307         uint32_t rank_bgn, sr;
3308         uint32_t write_group, write_test_bgn;
3309         uint32_t read_group, read_test_bgn;
3310         uint32_t run_groups, current_run;
3311         uint32_t failing_groups = 0;
3312         uint32_t group_failed = 0;
3313         uint32_t sr_failed = 0;
3314
3315         debug("%s:%d\n", __func__, __LINE__);
3316         /* Initialize the data settings */
3317
3318         gbl->error_substage = CAL_SUBSTAGE_NIL;
3319         gbl->error_stage = CAL_STAGE_NIL;
3320         gbl->error_group = 0xff;
3321         gbl->fom_in = 0;
3322         gbl->fom_out = 0;
3323
3324         mem_config();
3325
3326         for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
3327                 writel(i, SDR_PHYGRP_SCCGRP_ADDRESS |
3328                           SCC_MGR_GROUP_COUNTER_OFFSET);
3329                 scc_set_bypass_mode(i);
3330         }
3331
3332         if ((dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL) {
3333                 /*
3334                  * Set VFIFO and LFIFO to instant-on settings in skip
3335                  * calibration mode.
3336                  */
3337                 mem_skip_calibrate();
3338         } else {
3339                 for (i = 0; i < NUM_CALIB_REPEAT; i++) {
3340                         /*
3341                          * Zero all delay chain/phase settings for all
3342                          * groups and all shadow register sets.
3343                          */
3344                         scc_mgr_zero_all();
3345
3346                         run_groups = ~param->skip_groups;
3347
3348                         for (write_group = 0, write_test_bgn = 0; write_group
3349                                 < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; write_group++,
3350                                 write_test_bgn += RW_MGR_MEM_DQ_PER_WRITE_DQS) {
3351                                 /* Initialized the group failure */
3352                                 group_failed = 0;
3353
3354                                 current_run = run_groups & ((1 <<
3355                                         RW_MGR_NUM_DQS_PER_WRITE_GROUP) - 1);
3356                                 run_groups = run_groups >>
3357                                         RW_MGR_NUM_DQS_PER_WRITE_GROUP;
3358
3359                                 if (current_run == 0)
3360                                         continue;
3361
3362                                 writel(write_group, SDR_PHYGRP_SCCGRP_ADDRESS |
3363                                                     SCC_MGR_GROUP_COUNTER_OFFSET);
3364                                 scc_mgr_zero_group(write_group, write_test_bgn,
3365                                                    0);
3366
3367                                 for (read_group = write_group *
3368                                         RW_MGR_MEM_IF_READ_DQS_WIDTH /
3369                                         RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
3370                                         read_test_bgn = 0;
3371                                         read_group < (write_group + 1) *
3372                                         RW_MGR_MEM_IF_READ_DQS_WIDTH /
3373                                         RW_MGR_MEM_IF_WRITE_DQS_WIDTH &&
3374                                         group_failed == 0;
3375                                         read_group++, read_test_bgn +=
3376                                         RW_MGR_MEM_DQ_PER_READ_DQS) {
3377                                         /* Calibrate the VFIFO */
3378                                         if (!((STATIC_CALIB_STEPS) &
3379                                                 CALIB_SKIP_VFIFO)) {
3380                                                 if (!rw_mgr_mem_calibrate_vfifo
3381                                                         (read_group,
3382                                                         read_test_bgn)) {
3383                                                         group_failed = 1;
3384
3385                                                         if (!(gbl->
3386                                                         phy_debug_mode_flags &
3387                                                 PHY_DEBUG_SWEEP_ALL_GROUPS)) {
3388                                                                 return 0;
3389                                                         }
3390                                                 }
3391                                         }
3392                                 }
3393
3394                                 /* Calibrate the output side */
3395                                 if (group_failed == 0)  {
3396                                         for (rank_bgn = 0, sr = 0; rank_bgn
3397                                                 < RW_MGR_MEM_NUMBER_OF_RANKS;
3398                                                 rank_bgn +=
3399                                                 NUM_RANKS_PER_SHADOW_REG,
3400                                                 ++sr) {
3401                                                 sr_failed = 0;
3402                                                 if (!((STATIC_CALIB_STEPS) &
3403                                                 CALIB_SKIP_WRITES)) {
3404                                                         if ((STATIC_CALIB_STEPS)
3405                                                 & CALIB_SKIP_DELAY_SWEEPS) {
3406                                                 /* not needed in quick mode! */
3407                                                         } else {
3408                                                 /*
3409                                                  * Determine if this set of
3410                                                  * ranks should be skipped
3411                                                  * entirely.
3412                                                  */
3413                                         if (!param->skip_shadow_regs[sr]) {
3414                                                 if (!rw_mgr_mem_calibrate_writes
3415                                                 (rank_bgn, write_group,
3416                                                 write_test_bgn)) {
3417                                                         sr_failed = 1;
3418                                                         if (!(gbl->
3419                                                         phy_debug_mode_flags &
3420                                                 PHY_DEBUG_SWEEP_ALL_GROUPS)) {
3421                                                                 return 0;
3422                                                                         }
3423                                                                         }
3424                                                                 }
3425                                                         }
3426                                                 }
3427                                                 if (sr_failed != 0)
3428                                                         group_failed = 1;
3429                                         }
3430                                 }
3431
3432                                 if (group_failed == 0) {
3433                                         for (read_group = write_group *
3434                                         RW_MGR_MEM_IF_READ_DQS_WIDTH /
3435                                         RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
3436                                         read_test_bgn = 0;
3437                                                 read_group < (write_group + 1)
3438                                                 * RW_MGR_MEM_IF_READ_DQS_WIDTH
3439                                                 / RW_MGR_MEM_IF_WRITE_DQS_WIDTH &&
3440                                                 group_failed == 0;
3441                                                 read_group++, read_test_bgn +=
3442                                                 RW_MGR_MEM_DQ_PER_READ_DQS) {
3443                                                 if (!((STATIC_CALIB_STEPS) &
3444                                                         CALIB_SKIP_WRITES)) {
3445                                         if (!rw_mgr_mem_calibrate_vfifo_end
3446                                                 (read_group, read_test_bgn)) {
3447                                                         group_failed = 1;
3448
3449                                                 if (!(gbl->phy_debug_mode_flags
3450                                                 & PHY_DEBUG_SWEEP_ALL_GROUPS)) {
3451                                                                 return 0;
3452                                                                 }
3453                                                         }
3454                                                 }
3455                                         }
3456                                 }
3457
3458                                 if (group_failed != 0)
3459                                         failing_groups++;
3460                         }
3461
3462                         /*
3463                          * USER If there are any failing groups then report
3464                          * the failure.
3465                          */
3466                         if (failing_groups != 0)
3467                                 return 0;
3468
3469                         /* Calibrate the LFIFO */
3470                         if (!((STATIC_CALIB_STEPS) & CALIB_SKIP_LFIFO)) {
3471                                 /*
3472                                  * If we're skipping groups as part of debug,
3473                                  * don't calibrate LFIFO.
3474                                  */
3475                                 if (param->skip_groups == 0) {
3476                                         if (!rw_mgr_mem_calibrate_lfifo())
3477                                                 return 0;
3478                                 }
3479                         }
3480                 }
3481         }
3482
3483         /*
3484          * Do not remove this line as it makes sure all of our decisions
3485          * have been applied.
3486          */
3487         writel(0, &sdr_scc_mgr->update);
3488         return 1;
3489 }
3490
3491 static uint32_t run_mem_calibrate(void)
3492 {
3493         uint32_t pass;
3494         uint32_t debug_info;
3495
3496         debug("%s:%d\n", __func__, __LINE__);
3497
3498         /* Reset pass/fail status shown on afi_cal_success/fail */
3499         writel(PHY_MGR_CAL_RESET, &phy_mgr_cfg->cal_status);
3500
3501         /* stop tracking manger */
3502         uint32_t ctrlcfg = readl(&sdr_ctrl->ctrl_cfg);
3503
3504         writel(ctrlcfg & 0xFFBFFFFF, &sdr_ctrl->ctrl_cfg);
3505
3506         initialize();
3507         rw_mgr_mem_initialize();
3508
3509         pass = mem_calibrate();
3510
3511         mem_precharge_and_activate();
3512         writel(0, &phy_mgr_cmd->fifo_reset);
3513
3514         /*
3515          * Handoff:
3516          * Don't return control of the PHY back to AFI when in debug mode.
3517          */
3518         if ((gbl->phy_debug_mode_flags & PHY_DEBUG_IN_DEBUG_MODE) == 0) {
3519                 rw_mgr_mem_handoff();
3520                 /*
3521                  * In Hard PHY this is a 2-bit control:
3522                  * 0: AFI Mux Select
3523                  * 1: DDIO Mux Select
3524                  */
3525                 writel(0x2, &phy_mgr_cfg->mux_sel);
3526         }
3527
3528         writel(ctrlcfg, &sdr_ctrl->ctrl_cfg);
3529
3530         if (pass) {
3531                 printf("%s: CALIBRATION PASSED\n", __FILE__);
3532
3533                 gbl->fom_in /= 2;
3534                 gbl->fom_out /= 2;
3535
3536                 if (gbl->fom_in > 0xff)
3537                         gbl->fom_in = 0xff;
3538
3539                 if (gbl->fom_out > 0xff)
3540                         gbl->fom_out = 0xff;
3541
3542                 /* Update the FOM in the register file */
3543                 debug_info = gbl->fom_in;
3544                 debug_info |= gbl->fom_out << 8;
3545                 writel(debug_info, &sdr_reg_file->fom);
3546
3547                 writel(debug_info, &phy_mgr_cfg->cal_debug_info);
3548                 writel(PHY_MGR_CAL_SUCCESS, &phy_mgr_cfg->cal_status);
3549         } else {
3550                 printf("%s: CALIBRATION FAILED\n", __FILE__);
3551
3552                 debug_info = gbl->error_stage;
3553                 debug_info |= gbl->error_substage << 8;
3554                 debug_info |= gbl->error_group << 16;
3555
3556                 writel(debug_info, &sdr_reg_file->failing_stage);
3557                 writel(debug_info, &phy_mgr_cfg->cal_debug_info);
3558                 writel(PHY_MGR_CAL_FAIL, &phy_mgr_cfg->cal_status);
3559
3560                 /* Update the failing group/stage in the register file */
3561                 debug_info = gbl->error_stage;
3562                 debug_info |= gbl->error_substage << 8;
3563                 debug_info |= gbl->error_group << 16;
3564                 writel(debug_info, &sdr_reg_file->failing_stage);
3565         }
3566
3567         return pass;
3568 }
3569
3570 /**
3571  * hc_initialize_rom_data() - Initialize ROM data
3572  *
3573  * Initialize ROM data.
3574  */
3575 static void hc_initialize_rom_data(void)
3576 {
3577         u32 i, addr;
3578
3579         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_INST_ROM_WRITE_OFFSET;
3580         for (i = 0; i < ARRAY_SIZE(inst_rom_init); i++)
3581                 writel(inst_rom_init[i], addr + (i << 2));
3582
3583         addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_AC_ROM_WRITE_OFFSET;
3584         for (i = 0; i < ARRAY_SIZE(ac_rom_init); i++)
3585                 writel(ac_rom_init[i], addr + (i << 2));
3586 }
3587
3588 /**
3589  * initialize_reg_file() - Initialize SDR register file
3590  *
3591  * Initialize SDR register file.
3592  */
3593 static void initialize_reg_file(void)
3594 {
3595         /* Initialize the register file with the correct data */
3596         writel(REG_FILE_INIT_SEQ_SIGNATURE, &sdr_reg_file->signature);
3597         writel(0, &sdr_reg_file->debug_data_addr);
3598         writel(0, &sdr_reg_file->cur_stage);
3599         writel(0, &sdr_reg_file->fom);
3600         writel(0, &sdr_reg_file->failing_stage);
3601         writel(0, &sdr_reg_file->debug1);
3602         writel(0, &sdr_reg_file->debug2);
3603 }
3604
3605 /**
3606  * initialize_hps_phy() - Initialize HPS PHY
3607  *
3608  * Initialize HPS PHY.
3609  */
3610 static void initialize_hps_phy(void)
3611 {
3612         uint32_t reg;
3613         /*
3614          * Tracking also gets configured here because it's in the
3615          * same register.
3616          */
3617         uint32_t trk_sample_count = 7500;
3618         uint32_t trk_long_idle_sample_count = (10 << 16) | 100;
3619         /*
3620          * Format is number of outer loops in the 16 MSB, sample
3621          * count in 16 LSB.
3622          */
3623
3624         reg = 0;
3625         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ACDELAYEN_SET(2);
3626         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQDELAYEN_SET(1);
3627         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSDELAYEN_SET(1);
3628         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSLOGICDELAYEN_SET(1);
3629         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_RESETDELAYEN_SET(0);
3630         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_LPDDRDIS_SET(1);
3631         /*
3632          * This field selects the intrinsic latency to RDATA_EN/FULL path.
3633          * 00-bypass, 01- add 5 cycles, 10- add 10 cycles, 11- add 15 cycles.
3634          */
3635         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ADDLATSEL_SET(0);
3636         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_SET(
3637                 trk_sample_count);
3638         writel(reg, &sdr_ctrl->phy_ctrl0);
3639
3640         reg = 0;
3641         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_SAMPLECOUNT_31_20_SET(
3642                 trk_sample_count >>
3643                 SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_WIDTH);
3644         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_SET(
3645                 trk_long_idle_sample_count);
3646         writel(reg, &sdr_ctrl->phy_ctrl1);
3647
3648         reg = 0;
3649         reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_2_LONGIDLESAMPLECOUNT_31_20_SET(
3650                 trk_long_idle_sample_count >>
3651                 SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_WIDTH);
3652         writel(reg, &sdr_ctrl->phy_ctrl2);
3653 }
3654
3655 static void initialize_tracking(void)
3656 {
3657         uint32_t concatenated_longidle = 0x0;
3658         uint32_t concatenated_delays = 0x0;
3659         uint32_t concatenated_rw_addr = 0x0;
3660         uint32_t concatenated_refresh = 0x0;
3661         uint32_t trk_sample_count = 7500;
3662         uint32_t dtaps_per_ptap;
3663         uint32_t tmp_delay;
3664
3665         /*
3666          * compute usable version of value in case we skip full
3667          * computation later
3668          */
3669         dtaps_per_ptap = 0;
3670         tmp_delay = 0;
3671         while (tmp_delay < IO_DELAY_PER_OPA_TAP) {
3672                 dtaps_per_ptap++;
3673                 tmp_delay += IO_DELAY_PER_DCHAIN_TAP;
3674         }
3675         dtaps_per_ptap--;
3676
3677         concatenated_longidle = concatenated_longidle ^ 10;
3678                 /*longidle outer loop */
3679         concatenated_longidle = concatenated_longidle << 16;
3680         concatenated_longidle = concatenated_longidle ^ 100;
3681                 /*longidle sample count */
3682         concatenated_delays = concatenated_delays ^ 243;
3683                 /* trfc, worst case of 933Mhz 4Gb */
3684         concatenated_delays = concatenated_delays << 8;
3685         concatenated_delays = concatenated_delays ^ 14;
3686                 /* trcd, worst case */
3687         concatenated_delays = concatenated_delays << 8;
3688         concatenated_delays = concatenated_delays ^ 10;
3689                 /* vfifo wait */
3690         concatenated_delays = concatenated_delays << 8;
3691         concatenated_delays = concatenated_delays ^ 4;
3692                 /* mux delay */
3693
3694         concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_IDLE;
3695         concatenated_rw_addr = concatenated_rw_addr << 8;
3696         concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_ACTIVATE_1;
3697         concatenated_rw_addr = concatenated_rw_addr << 8;
3698         concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_SGLE_READ;
3699         concatenated_rw_addr = concatenated_rw_addr << 8;
3700         concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_PRECHARGE_ALL;
3701
3702         concatenated_refresh = concatenated_refresh ^ RW_MGR_REFRESH_ALL;
3703         concatenated_refresh = concatenated_refresh << 24;
3704         concatenated_refresh = concatenated_refresh ^ 1000; /* trefi */
3705
3706         /* Initialize the register file with the correct data */
3707         writel(dtaps_per_ptap, &sdr_reg_file->dtaps_per_ptap);
3708         writel(trk_sample_count, &sdr_reg_file->trk_sample_count);
3709         writel(concatenated_longidle, &sdr_reg_file->trk_longidle);
3710         writel(concatenated_delays, &sdr_reg_file->delays);
3711         writel(concatenated_rw_addr, &sdr_reg_file->trk_rw_mgr_addr);
3712         writel(RW_MGR_MEM_IF_READ_DQS_WIDTH, &sdr_reg_file->trk_read_dqs_width);
3713         writel(concatenated_refresh, &sdr_reg_file->trk_rfsh);
3714 }
3715
3716 int sdram_calibration_full(void)
3717 {
3718         struct param_type my_param;
3719         struct gbl_type my_gbl;
3720         uint32_t pass;
3721         uint32_t i;
3722
3723         param = &my_param;
3724         gbl = &my_gbl;
3725
3726         /* Initialize the debug mode flags */
3727         gbl->phy_debug_mode_flags = 0;
3728         /* Set the calibration enabled by default */
3729         gbl->phy_debug_mode_flags |= PHY_DEBUG_ENABLE_CAL_RPT;
3730         /*
3731          * Only sweep all groups (regardless of fail state) by default
3732          * Set enabled read test by default.
3733          */
3734 #if DISABLE_GUARANTEED_READ
3735         gbl->phy_debug_mode_flags |= PHY_DEBUG_DISABLE_GUARANTEED_READ;
3736 #endif
3737         /* Initialize the register file */
3738         initialize_reg_file();
3739
3740         /* Initialize any PHY CSR */
3741         initialize_hps_phy();
3742
3743         scc_mgr_initialize();
3744
3745         initialize_tracking();
3746
3747         /* USER Enable all ranks, groups */
3748         for (i = 0; i < RW_MGR_MEM_NUMBER_OF_RANKS; i++)
3749                 param->skip_ranks[i] = 0;
3750         for (i = 0; i < NUM_SHADOW_REGS; ++i)
3751                 param->skip_shadow_regs[i] = 0;
3752         param->skip_groups = 0;
3753
3754         printf("%s: Preparing to start memory calibration\n", __FILE__);
3755
3756         debug("%s:%d\n", __func__, __LINE__);
3757         debug_cond(DLEVEL == 1,
3758                    "DDR3 FULL_RATE ranks=%u cs/dimm=%u dq/dqs=%u,%u vg/dqs=%u,%u ",
3759                    RW_MGR_MEM_NUMBER_OF_RANKS, RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM,
3760                    RW_MGR_MEM_DQ_PER_READ_DQS, RW_MGR_MEM_DQ_PER_WRITE_DQS,
3761                    RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS,
3762                    RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS);
3763         debug_cond(DLEVEL == 1,
3764                    "dqs=%u,%u dq=%u dm=%u ptap_delay=%u dtap_delay=%u ",
3765                    RW_MGR_MEM_IF_READ_DQS_WIDTH, RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
3766                    RW_MGR_MEM_DATA_WIDTH, RW_MGR_MEM_DATA_MASK_WIDTH,
3767                    IO_DELAY_PER_OPA_TAP, IO_DELAY_PER_DCHAIN_TAP);
3768         debug_cond(DLEVEL == 1, "dtap_dqsen_delay=%u, dll=%u",
3769                    IO_DELAY_PER_DQS_EN_DCHAIN_TAP, IO_DLL_CHAIN_LENGTH);
3770         debug_cond(DLEVEL == 1, "max values: en_p=%u dqdqs_p=%u en_d=%u dqs_in_d=%u ",
3771                    IO_DQS_EN_PHASE_MAX, IO_DQDQS_OUT_PHASE_MAX,
3772                    IO_DQS_EN_DELAY_MAX, IO_DQS_IN_DELAY_MAX);
3773         debug_cond(DLEVEL == 1, "io_in_d=%u io_out1_d=%u io_out2_d=%u ",
3774                    IO_IO_IN_DELAY_MAX, IO_IO_OUT1_DELAY_MAX,
3775                    IO_IO_OUT2_DELAY_MAX);
3776         debug_cond(DLEVEL == 1, "dqs_in_reserve=%u dqs_out_reserve=%u\n",
3777                    IO_DQS_IN_RESERVE, IO_DQS_OUT_RESERVE);
3778
3779         hc_initialize_rom_data();
3780
3781         /* update info for sims */
3782         reg_file_set_stage(CAL_STAGE_NIL);
3783         reg_file_set_group(0);
3784
3785         /*
3786          * Load global needed for those actions that require
3787          * some dynamic calibration support.
3788          */
3789         dyn_calib_steps = STATIC_CALIB_STEPS;
3790         /*
3791          * Load global to allow dynamic selection of delay loop settings
3792          * based on calibration mode.
3793          */
3794         if (!(dyn_calib_steps & CALIB_SKIP_DELAY_LOOPS))
3795                 skip_delay_mask = 0xff;
3796         else
3797                 skip_delay_mask = 0x0;
3798
3799         pass = run_mem_calibrate();
3800
3801         printf("%s: Calibration complete\n", __FILE__);
3802         return pass;
3803 }