2 * Copyright Altera Corporation (C) 2012-2015
4 * SPDX-License-Identifier: BSD-3-Clause
9 #include <asm/arch/sdram.h>
10 #include "sequencer.h"
11 #include "sequencer_auto.h"
12 #include "sequencer_auto_ac_init.h"
13 #include "sequencer_auto_inst_init.h"
14 #include "sequencer_defines.h"
16 static void scc_mgr_load_dqs_for_write_group(uint32_t write_group);
18 static struct socfpga_sdr_rw_load_manager *sdr_rw_load_mgr_regs =
19 (struct socfpga_sdr_rw_load_manager *)(SDR_PHYGRP_RWMGRGRP_ADDRESS | 0x800);
21 static struct socfpga_sdr_rw_load_jump_manager *sdr_rw_load_jump_mgr_regs =
22 (struct socfpga_sdr_rw_load_jump_manager *)(SDR_PHYGRP_RWMGRGRP_ADDRESS | 0xC00);
24 static struct socfpga_sdr_reg_file *sdr_reg_file =
25 (struct socfpga_sdr_reg_file *)SDR_PHYGRP_REGFILEGRP_ADDRESS;
27 static struct socfpga_sdr_scc_mgr *sdr_scc_mgr =
28 (struct socfpga_sdr_scc_mgr *)(SDR_PHYGRP_SCCGRP_ADDRESS | 0xe00);
30 static struct socfpga_phy_mgr_cmd *phy_mgr_cmd =
31 (struct socfpga_phy_mgr_cmd *)SDR_PHYGRP_PHYMGRGRP_ADDRESS;
33 static struct socfpga_phy_mgr_cfg *phy_mgr_cfg =
34 (struct socfpga_phy_mgr_cfg *)(SDR_PHYGRP_PHYMGRGRP_ADDRESS | 0x40);
36 static struct socfpga_data_mgr *data_mgr =
37 (struct socfpga_data_mgr *)(BASE_DATA_MGR);
40 #define MGR_SELECT_MASK 0xf8000
43 * In order to reduce ROM size, most of the selectable calibration steps are
44 * decided at compile time based on the user's calibration mode selection,
45 * as captured by the STATIC_CALIB_STEPS selection below.
47 * However, to support simulation-time selection of fast simulation mode, where
48 * we skip everything except the bare minimum, we need a few of the steps to
49 * be dynamic. In those cases, we either use the DYNAMIC_CALIB_STEPS for the
50 * check, which is based on the rtl-supplied value, or we dynamically compute
51 * the value to use based on the dynamically-chosen calibration mode
55 #define STATIC_IN_RTL_SIM 0
56 #define STATIC_SKIP_DELAY_LOOPS 0
58 #define STATIC_CALIB_STEPS (STATIC_IN_RTL_SIM | CALIB_SKIP_FULL_TEST | \
59 STATIC_SKIP_DELAY_LOOPS)
61 /* calibration steps requested by the rtl */
62 uint16_t dyn_calib_steps;
65 * To make CALIB_SKIP_DELAY_LOOPS a dynamic conditional option
66 * instead of static, we use boolean logic to select between
67 * non-skip and skip values
69 * The mask is set to include all bits when not-skipping, but is
73 uint16_t skip_delay_mask; /* mask off bits when skipping/not-skipping */
75 #define SKIP_DELAY_LOOP_VALUE_OR_ZERO(non_skip_value) \
76 ((non_skip_value) & skip_delay_mask)
79 struct param_type *param;
80 uint32_t curr_shadow_reg;
82 static uint32_t rw_mgr_mem_calibrate_write_test(uint32_t rank_bgn,
83 uint32_t write_group, uint32_t use_dm,
84 uint32_t all_correct, uint32_t *bit_chk, uint32_t all_ranks);
86 static u32 sdr_get_addr(u32 *base)
88 u32 addr = (u32)base & MGR_SELECT_MASK;
92 addr = (((u32)base >> 8) & (1 << 6)) | ((u32)base & 0x3f) |
93 SDR_PHYGRP_PHYMGRGRP_ADDRESS;
96 addr = ((u32)base & 0x1fff) | SDR_PHYGRP_RWMGRGRP_ADDRESS;
99 addr = ((u32)base & 0x7ff) | SDR_PHYGRP_DATAMGRGRP_ADDRESS;
102 addr = ((u32)base & 0xfff) | SDR_PHYGRP_SCCGRP_ADDRESS;
105 addr = ((u32)base & 0x7ff) | SDR_PHYGRP_REGFILEGRP_ADDRESS;
108 addr = ((u32)base & 0xfff) | SDR_CTRLGRP_ADDRESS;
117 static void set_failing_group_stage(uint32_t group, uint32_t stage,
121 * Only set the global stage if there was not been any other
124 if (gbl->error_stage == CAL_STAGE_NIL) {
125 gbl->error_substage = substage;
126 gbl->error_stage = stage;
127 gbl->error_group = group;
131 static void reg_file_set_group(uint32_t set_group)
133 u32 addr = (u32)&sdr_reg_file->cur_stage;
135 /* Read the current group and stage */
136 uint32_t cur_stage_group = readl(SOCFPGA_SDR_ADDRESS + addr);
138 /* Clear the group */
139 cur_stage_group &= 0x0000FFFF;
142 cur_stage_group |= (set_group << 16);
144 /* Write the data back */
145 writel(cur_stage_group, SOCFPGA_SDR_ADDRESS + addr);
148 static void reg_file_set_stage(uint32_t set_stage)
150 u32 addr = (u32)&sdr_reg_file->cur_stage;
152 /* Read the current group and stage */
153 uint32_t cur_stage_group = readl(SOCFPGA_SDR_ADDRESS + addr);
155 /* Clear the stage and substage */
156 cur_stage_group &= 0xFFFF0000;
159 cur_stage_group |= (set_stage & 0x000000FF);
161 /* Write the data back */
162 writel(cur_stage_group, SOCFPGA_SDR_ADDRESS + addr);
165 static void reg_file_set_sub_stage(uint32_t set_sub_stage)
167 u32 addr = (u32)&sdr_reg_file->cur_stage;
169 /* Read the current group and stage */
170 uint32_t cur_stage_group = readl(SOCFPGA_SDR_ADDRESS + addr);
172 /* Clear the substage */
173 cur_stage_group &= 0xFFFF00FF;
175 /* Set the sub stage */
176 cur_stage_group |= ((set_sub_stage << 8) & 0x0000FF00);
178 /* Write the data back */
179 writel(cur_stage_group, SOCFPGA_SDR_ADDRESS + addr);
182 static void initialize(void)
184 u32 addr = (u32)&phy_mgr_cfg->mux_sel;
186 debug("%s:%d\n", __func__, __LINE__);
187 /* USER calibration has control over path to memory */
189 * In Hard PHY this is a 2-bit control:
193 writel(0x3, SOCFPGA_SDR_ADDRESS + addr);
195 /* USER memory clock is not stable we begin initialization */
196 addr = (u32)&phy_mgr_cfg->reset_mem_stbl;
197 writel(0, SOCFPGA_SDR_ADDRESS + addr);
199 /* USER calibration status all set to zero */
200 addr = (u32)&phy_mgr_cfg->cal_status;
201 writel(0, SOCFPGA_SDR_ADDRESS + addr);
203 addr = (u32)&phy_mgr_cfg->cal_debug_info;
204 writel(0, SOCFPGA_SDR_ADDRESS + addr);
206 if ((dyn_calib_steps & CALIB_SKIP_ALL) != CALIB_SKIP_ALL) {
207 param->read_correct_mask_vg = ((uint32_t)1 <<
208 (RW_MGR_MEM_DQ_PER_READ_DQS /
209 RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS)) - 1;
210 param->write_correct_mask_vg = ((uint32_t)1 <<
211 (RW_MGR_MEM_DQ_PER_READ_DQS /
212 RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS)) - 1;
213 param->read_correct_mask = ((uint32_t)1 <<
214 RW_MGR_MEM_DQ_PER_READ_DQS) - 1;
215 param->write_correct_mask = ((uint32_t)1 <<
216 RW_MGR_MEM_DQ_PER_WRITE_DQS) - 1;
217 param->dm_correct_mask = ((uint32_t)1 <<
218 (RW_MGR_MEM_DATA_WIDTH / RW_MGR_MEM_DATA_MASK_WIDTH))
223 static void set_rank_and_odt_mask(uint32_t rank, uint32_t odt_mode)
225 uint32_t odt_mask_0 = 0;
226 uint32_t odt_mask_1 = 0;
227 uint32_t cs_and_odt_mask;
230 if (odt_mode == RW_MGR_ODT_MODE_READ_WRITE) {
231 if (RW_MGR_MEM_NUMBER_OF_RANKS == 1) {
239 } else if (RW_MGR_MEM_NUMBER_OF_RANKS == 2) {
241 if (RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM == 1) {
242 /* - Dual-Slot , Single-Rank
243 * (1 chip-select per DIMM)
245 * - RDIMM, 4 total CS (2 CS per DIMM)
247 * Since MEM_NUMBER_OF_RANKS is 2 they are
249 * with 2 CS each (special for RDIMM)
250 * Read: Turn on ODT on the opposite rank
251 * Write: Turn on ODT on all ranks
253 odt_mask_0 = 0x3 & ~(1 << rank);
257 * USER - Single-Slot , Dual-rank DIMMs
258 * (2 chip-selects per DIMM)
259 * USER Read: Turn on ODT off on all ranks
260 * USER Write: Turn on ODT on active rank
263 odt_mask_1 = 0x3 & (1 << rank);
268 * ----------+-----------------------+
271 * Read From +-----------------------+
272 * Rank | 3 | 2 | 1 | 0 |
273 * ----------+-----+-----+-----+-----+
274 * 0 | 0 | 1 | 0 | 0 |
275 * 1 | 1 | 0 | 0 | 0 |
276 * 2 | 0 | 0 | 0 | 1 |
277 * 3 | 0 | 0 | 1 | 0 |
278 * ----------+-----+-----+-----+-----+
281 * ----------+-----------------------+
284 * Write To +-----------------------+
285 * Rank | 3 | 2 | 1 | 0 |
286 * ----------+-----+-----+-----+-----+
287 * 0 | 0 | 1 | 0 | 1 |
288 * 1 | 1 | 0 | 1 | 0 |
289 * 2 | 0 | 1 | 0 | 1 |
290 * 3 | 1 | 0 | 1 | 0 |
291 * ----------+-----+-----+-----+-----+
318 (0xFF & ~(1 << rank)) |
319 ((0xFF & odt_mask_0) << 8) |
320 ((0xFF & odt_mask_1) << 16);
321 addr = sdr_get_addr((u32 *)RW_MGR_SET_CS_AND_ODT_MASK);
322 writel(cs_and_odt_mask, SOCFPGA_SDR_ADDRESS + addr);
325 static void scc_mgr_initialize(void)
327 u32 addr = sdr_get_addr((u32 *)SCC_MGR_HHP_RFILE);
330 * Clear register file for HPS
331 * 16 (2^4) is the size of the full register file in the scc mgr:
332 * RFILE_DEPTH = log2(MEM_DQ_PER_DQS + 1 + MEM_DM_PER_DQS +
333 * MEM_IF_READ_DQS_WIDTH - 1) + 1;
336 for (i = 0; i < 16; i++) {
337 debug_cond(DLEVEL == 1, "%s:%d: Clearing SCC RFILE index %u\n",
338 __func__, __LINE__, i);
339 writel(0, SOCFPGA_SDR_ADDRESS + addr + (i << 2));
343 static void scc_mgr_set_dqs_bus_in_delay(uint32_t read_group,
346 u32 addr = sdr_get_addr((u32 *)SCC_MGR_DQS_IN_DELAY);
348 /* Load the setting in the SCC manager */
349 writel(delay, SOCFPGA_SDR_ADDRESS + addr + (read_group << 2));
352 static void scc_mgr_set_dqs_io_in_delay(uint32_t write_group,
355 u32 addr = sdr_get_addr((u32 *)SCC_MGR_IO_IN_DELAY);
357 writel(delay, SOCFPGA_SDR_ADDRESS + addr + (RW_MGR_MEM_DQ_PER_WRITE_DQS << 2));
360 static void scc_mgr_set_dqs_en_phase(uint32_t read_group, uint32_t phase)
362 u32 addr = sdr_get_addr((u32 *)SCC_MGR_DQS_EN_PHASE);
364 /* Load the setting in the SCC manager */
365 writel(phase, SOCFPGA_SDR_ADDRESS + addr + (read_group << 2));
368 static void scc_mgr_set_dqs_en_phase_all_ranks(uint32_t read_group,
372 uint32_t update_scan_chains;
375 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
376 r += NUM_RANKS_PER_SHADOW_REG) {
378 * USER although the h/w doesn't support different phases per
379 * shadow register, for simplicity our scc manager modeling
380 * keeps different phase settings per shadow reg, and it's
381 * important for us to keep them in sync to match h/w.
382 * for efficiency, the scan chain update should occur only
385 update_scan_chains = (r == 0) ? 1 : 0;
387 scc_mgr_set_dqs_en_phase(read_group, phase);
389 if (update_scan_chains) {
390 addr = (u32)&sdr_scc_mgr->dqs_ena;
391 writel(read_group, SOCFPGA_SDR_ADDRESS + addr);
393 addr = (u32)&sdr_scc_mgr->update;
394 writel(0, SOCFPGA_SDR_ADDRESS + addr);
399 static void scc_mgr_set_dqdqs_output_phase(uint32_t write_group,
402 u32 addr = sdr_get_addr((u32 *)SCC_MGR_DQDQS_OUT_PHASE);
404 /* Load the setting in the SCC manager */
405 writel(phase, SOCFPGA_SDR_ADDRESS + addr + (write_group << 2));
408 static void scc_mgr_set_dqdqs_output_phase_all_ranks(uint32_t write_group,
412 uint32_t update_scan_chains;
415 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
416 r += NUM_RANKS_PER_SHADOW_REG) {
418 * USER although the h/w doesn't support different phases per
419 * shadow register, for simplicity our scc manager modeling
420 * keeps different phase settings per shadow reg, and it's
421 * important for us to keep them in sync to match h/w.
422 * for efficiency, the scan chain update should occur only
425 update_scan_chains = (r == 0) ? 1 : 0;
427 scc_mgr_set_dqdqs_output_phase(write_group, phase);
429 if (update_scan_chains) {
430 addr = (u32)&sdr_scc_mgr->dqs_ena;
431 writel(write_group, SOCFPGA_SDR_ADDRESS + addr);
433 addr = (u32)&sdr_scc_mgr->update;
434 writel(0, SOCFPGA_SDR_ADDRESS + addr);
439 static void scc_mgr_set_dqs_en_delay(uint32_t read_group, uint32_t delay)
441 uint32_t addr = sdr_get_addr((u32 *)SCC_MGR_DQS_EN_DELAY);
443 /* Load the setting in the SCC manager */
444 writel(delay + IO_DQS_EN_DELAY_OFFSET, SOCFPGA_SDR_ADDRESS + addr +
448 static void scc_mgr_set_dqs_en_delay_all_ranks(uint32_t read_group,
454 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
455 r += NUM_RANKS_PER_SHADOW_REG) {
456 scc_mgr_set_dqs_en_delay(read_group, delay);
458 addr = (u32)&sdr_scc_mgr->dqs_ena;
459 writel(read_group, SOCFPGA_SDR_ADDRESS + addr);
461 * In shadow register mode, the T11 settings are stored in
462 * registers in the core, which are updated by the DQS_ENA
463 * signals. Not issuing the SCC_MGR_UPD command allows us to
464 * save lots of rank switching overhead, by calling
465 * select_shadow_regs_for_update with update_scan_chains
468 addr = (u32)&sdr_scc_mgr->update;
469 writel(0, SOCFPGA_SDR_ADDRESS + addr);
472 * In shadow register mode, the T11 settings are stored in
473 * registers in the core, which are updated by the DQS_ENA
474 * signals. Not issuing the SCC_MGR_UPD command allows us to
475 * save lots of rank switching overhead, by calling
476 * select_shadow_regs_for_update with update_scan_chains
479 addr = (u32)&sdr_scc_mgr->update;
480 writel(0, SOCFPGA_SDR_ADDRESS + addr);
483 static void scc_mgr_set_oct_out1_delay(uint32_t write_group, uint32_t delay)
486 uint32_t addr = sdr_get_addr((u32 *)SCC_MGR_OCT_OUT1_DELAY);
489 * Load the setting in the SCC manager
490 * Although OCT affects only write data, the OCT delay is controlled
491 * by the DQS logic block which is instantiated once per read group.
492 * For protocols where a write group consists of multiple read groups,
493 * the setting must be set multiple times.
495 for (read_group = write_group * RW_MGR_MEM_IF_READ_DQS_WIDTH /
496 RW_MGR_MEM_IF_WRITE_DQS_WIDTH;
497 read_group < (write_group + 1) * RW_MGR_MEM_IF_READ_DQS_WIDTH /
498 RW_MGR_MEM_IF_WRITE_DQS_WIDTH; ++read_group)
499 writel(delay, SOCFPGA_SDR_ADDRESS + addr + (read_group << 2));
502 static void scc_mgr_set_dq_out1_delay(uint32_t write_group,
503 uint32_t dq_in_group, uint32_t delay)
505 uint32_t addr = sdr_get_addr((u32 *)SCC_MGR_IO_OUT1_DELAY);
507 /* Load the setting in the SCC manager */
508 writel(delay, SOCFPGA_SDR_ADDRESS + addr + (dq_in_group << 2));
511 static void scc_mgr_set_dq_in_delay(uint32_t write_group,
512 uint32_t dq_in_group, uint32_t delay)
514 uint32_t addr = sdr_get_addr((u32 *)SCC_MGR_IO_IN_DELAY);
516 /* Load the setting in the SCC manager */
517 writel(delay, SOCFPGA_SDR_ADDRESS + addr + (dq_in_group << 2));
520 static void scc_mgr_set_hhp_extras(void)
523 * Load the fixed setting in the SCC manager
524 * bits: 0:0 = 1'b1 - dqs bypass
525 * bits: 1:1 = 1'b1 - dq bypass
526 * bits: 4:2 = 3'b001 - rfifo_mode
527 * bits: 6:5 = 2'b01 - rfifo clock_select
528 * bits: 7:7 = 1'b0 - separate gating from ungating setting
529 * bits: 8:8 = 1'b0 - separate OE from Output delay setting
531 uint32_t value = (0<<8) | (0<<7) | (1<<5) | (1<<2) | (1<<1) | (1<<0);
532 uint32_t addr = sdr_get_addr((u32 *)SCC_MGR_HHP_GLOBALS);
534 writel(value, SOCFPGA_SDR_ADDRESS + addr + SCC_MGR_HHP_EXTRAS_OFFSET);
537 static void scc_mgr_set_dqs_out1_delay(uint32_t write_group,
540 uint32_t addr = sdr_get_addr((u32 *)SCC_MGR_IO_OUT1_DELAY);
542 /* Load the setting in the SCC manager */
543 writel(delay, SOCFPGA_SDR_ADDRESS + addr + (RW_MGR_MEM_DQ_PER_WRITE_DQS << 2));
546 static void scc_mgr_set_dm_out1_delay(uint32_t write_group,
547 uint32_t dm, uint32_t delay)
549 uint32_t addr = sdr_get_addr((u32 *)SCC_MGR_IO_OUT1_DELAY);
551 /* Load the setting in the SCC manager */
552 writel(delay, SOCFPGA_SDR_ADDRESS + addr +
553 ((RW_MGR_MEM_DQ_PER_WRITE_DQS + 1 + dm) << 2));
557 * USER Zero all DQS config
558 * TODO: maybe rename to scc_mgr_zero_dqs_config (or something)
560 static void scc_mgr_zero_all(void)
566 * USER Zero all DQS config settings, across all groups and all
569 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r +=
570 NUM_RANKS_PER_SHADOW_REG) {
571 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
573 * The phases actually don't exist on a per-rank basis,
574 * but there's no harm updating them several times, so
575 * let's keep the code simple.
577 scc_mgr_set_dqs_bus_in_delay(i, IO_DQS_IN_RESERVE);
578 scc_mgr_set_dqs_en_phase(i, 0);
579 scc_mgr_set_dqs_en_delay(i, 0);
582 for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) {
583 scc_mgr_set_dqdqs_output_phase(i, 0);
584 /* av/cv don't have out2 */
585 scc_mgr_set_oct_out1_delay(i, IO_DQS_OUT_RESERVE);
589 /* multicast to all DQS group enables */
590 addr = (u32)&sdr_scc_mgr->dqs_ena;
591 writel(0xff, SOCFPGA_SDR_ADDRESS + addr);
593 addr = (u32)&sdr_scc_mgr->update;
594 writel(0, SOCFPGA_SDR_ADDRESS + addr);
597 static void scc_set_bypass_mode(uint32_t write_group, uint32_t mode)
600 /* mode = 0 : Do NOT bypass - Half Rate Mode */
601 /* mode = 1 : Bypass - Full Rate Mode */
603 /* only need to set once for all groups, pins, dq, dqs, dm */
604 if (write_group == 0) {
605 debug_cond(DLEVEL == 1, "%s:%d Setting HHP Extras\n", __func__,
607 scc_mgr_set_hhp_extras();
608 debug_cond(DLEVEL == 1, "%s:%d Done Setting HHP Extras\n",
611 /* multicast to all DQ enables */
612 addr = (u32)&sdr_scc_mgr->dq_ena;
613 writel(0xff, SOCFPGA_SDR_ADDRESS + addr);
615 addr = (u32)&sdr_scc_mgr->dm_ena;
616 writel(0xff, SOCFPGA_SDR_ADDRESS + addr);
618 /* update current DQS IO enable */
619 addr = (u32)&sdr_scc_mgr->dqs_io_ena;
620 writel(0, SOCFPGA_SDR_ADDRESS + addr);
622 /* update the DQS logic */
623 addr = (u32)&sdr_scc_mgr->dqs_ena;
624 writel(write_group, SOCFPGA_SDR_ADDRESS + addr);
627 addr = (u32)&sdr_scc_mgr->update;
628 writel(0, SOCFPGA_SDR_ADDRESS + addr);
631 static void scc_mgr_zero_group(uint32_t write_group, uint32_t test_begin,
637 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r +=
638 NUM_RANKS_PER_SHADOW_REG) {
639 /* Zero all DQ config settings */
640 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
641 scc_mgr_set_dq_out1_delay(write_group, i, 0);
643 scc_mgr_set_dq_in_delay(write_group, i, 0);
646 /* multicast to all DQ enables */
647 addr = (u32)&sdr_scc_mgr->dq_ena;
648 writel(0xff, SOCFPGA_SDR_ADDRESS + addr);
650 /* Zero all DM config settings */
651 for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
652 scc_mgr_set_dm_out1_delay(write_group, i, 0);
655 /* multicast to all DM enables */
656 addr = (u32)&sdr_scc_mgr->dm_ena;
657 writel(0xff, SOCFPGA_SDR_ADDRESS + addr);
659 /* zero all DQS io settings */
661 scc_mgr_set_dqs_io_in_delay(write_group, 0);
662 /* av/cv don't have out2 */
663 scc_mgr_set_dqs_out1_delay(write_group, IO_DQS_OUT_RESERVE);
664 scc_mgr_set_oct_out1_delay(write_group, IO_DQS_OUT_RESERVE);
665 scc_mgr_load_dqs_for_write_group(write_group);
667 /* multicast to all DQS IO enables (only 1) */
668 addr = (u32)&sdr_scc_mgr->dqs_io_ena;
669 writel(0, SOCFPGA_SDR_ADDRESS + addr);
671 /* hit update to zero everything */
672 addr = (u32)&sdr_scc_mgr->update;
673 writel(0, SOCFPGA_SDR_ADDRESS + addr);
677 /* load up dqs config settings */
678 static void scc_mgr_load_dqs(uint32_t dqs)
680 uint32_t addr = (u32)&sdr_scc_mgr->dqs_ena;
682 writel(dqs, SOCFPGA_SDR_ADDRESS + addr);
685 static void scc_mgr_load_dqs_for_write_group(uint32_t write_group)
688 uint32_t addr = (u32)&sdr_scc_mgr->dqs_ena;
690 * Although OCT affects only write data, the OCT delay is controlled
691 * by the DQS logic block which is instantiated once per read group.
692 * For protocols where a write group consists of multiple read groups,
693 * the setting must be scanned multiple times.
695 for (read_group = write_group * RW_MGR_MEM_IF_READ_DQS_WIDTH /
696 RW_MGR_MEM_IF_WRITE_DQS_WIDTH;
697 read_group < (write_group + 1) * RW_MGR_MEM_IF_READ_DQS_WIDTH /
698 RW_MGR_MEM_IF_WRITE_DQS_WIDTH; ++read_group)
699 writel(read_group, SOCFPGA_SDR_ADDRESS + addr);
702 /* load up dqs io config settings */
703 static void scc_mgr_load_dqs_io(void)
705 uint32_t addr = (u32)&sdr_scc_mgr->dqs_io_ena;
707 writel(0, SOCFPGA_SDR_ADDRESS + addr);
710 /* load up dq config settings */
711 static void scc_mgr_load_dq(uint32_t dq_in_group)
713 uint32_t addr = (u32)&sdr_scc_mgr->dq_ena;
715 writel(dq_in_group, SOCFPGA_SDR_ADDRESS + addr);
718 /* load up dm config settings */
719 static void scc_mgr_load_dm(uint32_t dm)
721 uint32_t addr = (u32)&sdr_scc_mgr->dm_ena;
723 writel(dm, SOCFPGA_SDR_ADDRESS + addr);
727 * apply and load a particular input delay for the DQ pins in a group
728 * group_bgn is the index of the first dq pin (in the write group)
730 static void scc_mgr_apply_group_dq_in_delay(uint32_t write_group,
731 uint32_t group_bgn, uint32_t delay)
735 for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) {
736 scc_mgr_set_dq_in_delay(write_group, p, delay);
741 /* apply and load a particular output delay for the DQ pins in a group */
742 static void scc_mgr_apply_group_dq_out1_delay(uint32_t write_group,
748 for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) {
749 scc_mgr_set_dq_out1_delay(write_group, i, delay1);
754 /* apply and load a particular output delay for the DM pins in a group */
755 static void scc_mgr_apply_group_dm_out1_delay(uint32_t write_group,
760 for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
761 scc_mgr_set_dm_out1_delay(write_group, i, delay1);
767 /* apply and load delay on both DQS and OCT out1 */
768 static void scc_mgr_apply_group_dqs_io_and_oct_out1(uint32_t write_group,
771 scc_mgr_set_dqs_out1_delay(write_group, delay);
772 scc_mgr_load_dqs_io();
774 scc_mgr_set_oct_out1_delay(write_group, delay);
775 scc_mgr_load_dqs_for_write_group(write_group);
778 /* apply a delay to the entire output side: DQ, DM, DQS, OCT */
779 static void scc_mgr_apply_group_all_out_delay_add(uint32_t write_group,
783 uint32_t i, p, new_delay;
786 for (i = 0, p = group_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) {
787 new_delay = READ_SCC_DQ_OUT2_DELAY;
790 if (new_delay > IO_IO_OUT2_DELAY_MAX) {
791 debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DQ[%u,%u]:\
792 %u > %lu => %lu", __func__, __LINE__,
793 write_group, group_bgn, delay, i, p, new_delay,
794 (long unsigned int)IO_IO_OUT2_DELAY_MAX,
795 (long unsigned int)IO_IO_OUT2_DELAY_MAX);
796 new_delay = IO_IO_OUT2_DELAY_MAX;
803 for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
804 new_delay = READ_SCC_DM_IO_OUT2_DELAY;
807 if (new_delay > IO_IO_OUT2_DELAY_MAX) {
808 debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DM[%u]:\
809 %u > %lu => %lu\n", __func__, __LINE__,
810 write_group, group_bgn, delay, i, new_delay,
811 (long unsigned int)IO_IO_OUT2_DELAY_MAX,
812 (long unsigned int)IO_IO_OUT2_DELAY_MAX);
813 new_delay = IO_IO_OUT2_DELAY_MAX;
820 new_delay = READ_SCC_DQS_IO_OUT2_DELAY;
823 if (new_delay > IO_IO_OUT2_DELAY_MAX) {
824 debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DQS: %u > %d => %d;"
825 " adding %u to OUT1\n", __func__, __LINE__,
826 write_group, group_bgn, delay, new_delay,
827 IO_IO_OUT2_DELAY_MAX, IO_IO_OUT2_DELAY_MAX,
828 new_delay - IO_IO_OUT2_DELAY_MAX);
829 scc_mgr_set_dqs_out1_delay(write_group, new_delay -
830 IO_IO_OUT2_DELAY_MAX);
831 new_delay = IO_IO_OUT2_DELAY_MAX;
834 scc_mgr_load_dqs_io();
837 new_delay = READ_SCC_OCT_OUT2_DELAY;
840 if (new_delay > IO_IO_OUT2_DELAY_MAX) {
841 debug_cond(DLEVEL == 1, "%s:%d (%u, %u, %u) DQS: %u > %d => %d;"
842 " adding %u to OUT1\n", __func__, __LINE__,
843 write_group, group_bgn, delay, new_delay,
844 IO_IO_OUT2_DELAY_MAX, IO_IO_OUT2_DELAY_MAX,
845 new_delay - IO_IO_OUT2_DELAY_MAX);
846 scc_mgr_set_oct_out1_delay(write_group, new_delay -
847 IO_IO_OUT2_DELAY_MAX);
848 new_delay = IO_IO_OUT2_DELAY_MAX;
851 scc_mgr_load_dqs_for_write_group(write_group);
855 * USER apply a delay to the entire output side (DQ, DM, DQS, OCT)
858 static void scc_mgr_apply_group_all_out_delay_add_all_ranks(
859 uint32_t write_group, uint32_t group_bgn, uint32_t delay)
862 uint32_t addr = (u32)&sdr_scc_mgr->update;
864 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
865 r += NUM_RANKS_PER_SHADOW_REG) {
866 scc_mgr_apply_group_all_out_delay_add(write_group,
868 writel(0, SOCFPGA_SDR_ADDRESS + addr);
872 /* optimization used to recover some slots in ddr3 inst_rom */
873 /* could be applied to other protocols if we wanted to */
874 static void set_jump_as_return(void)
876 uint32_t addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0;
879 * to save space, we replace return with jump to special shared
880 * RETURN instruction so we set the counter to large value so that
883 writel(0xff, SOCFPGA_SDR_ADDRESS + addr);
884 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0;
885 writel(RW_MGR_RETURN, SOCFPGA_SDR_ADDRESS + addr);
889 * should always use constants as argument to ensure all computations are
890 * performed at compile time
892 static void delay_for_n_mem_clocks(const uint32_t clocks)
900 debug("%s:%d: clocks=%u ... start\n", __func__, __LINE__, clocks);
903 afi_clocks = (clocks + AFI_RATE_RATIO-1) / AFI_RATE_RATIO;
904 /* scale (rounding up) to get afi clocks */
907 * Note, we don't bother accounting for being off a little bit
908 * because of a few extra instructions in outer loops
909 * Note, the loops have a test at the end, and do the test before
910 * the decrement, and so always perform the loop
911 * 1 time more than the counter value
913 if (afi_clocks == 0) {
915 } else if (afi_clocks <= 0x100) {
916 inner = afi_clocks-1;
919 } else if (afi_clocks <= 0x10000) {
921 outer = (afi_clocks-1) >> 8;
926 c_loop = (afi_clocks-1) >> 16;
930 * rom instructions are structured as follows:
932 * IDLE_LOOP2: jnz cntr0, TARGET_A
933 * IDLE_LOOP1: jnz cntr1, TARGET_B
936 * so, when doing nested loops, TARGET_A is set to IDLE_LOOP2, and
937 * TARGET_B is set to IDLE_LOOP2 as well
939 * if we have no outer loop, though, then we can use IDLE_LOOP1 only,
940 * and set TARGET_B to IDLE_LOOP1 and we skip IDLE_LOOP2 entirely
942 * a little confusing, but it helps save precious space in the inst_rom
943 * and sequencer rom and keeps the delays more accurate and reduces
946 if (afi_clocks <= 0x100) {
947 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1;
948 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner), SOCFPGA_SDR_ADDRESS + addr);
950 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1;
951 writel(RW_MGR_IDLE_LOOP1, SOCFPGA_SDR_ADDRESS + addr);
953 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
954 writel(RW_MGR_IDLE_LOOP1, SOCFPGA_SDR_ADDRESS + addr);
956 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0;
957 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner), SOCFPGA_SDR_ADDRESS + addr);
959 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1;
960 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(outer), SOCFPGA_SDR_ADDRESS + addr);
962 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0;
963 writel(RW_MGR_IDLE_LOOP2, SOCFPGA_SDR_ADDRESS + addr);
965 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1;
966 writel(RW_MGR_IDLE_LOOP2, SOCFPGA_SDR_ADDRESS + addr);
968 /* hack to get around compiler not being smart enough */
969 if (afi_clocks <= 0x10000) {
970 /* only need to run once */
971 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
972 writel(RW_MGR_IDLE_LOOP2, SOCFPGA_SDR_ADDRESS + addr);
975 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
976 writel(RW_MGR_IDLE_LOOP2, SOCFPGA_SDR_ADDRESS + addr);
977 } while (c_loop-- != 0);
980 debug("%s:%d clocks=%u ... end\n", __func__, __LINE__, clocks);
983 static void rw_mgr_mem_initialize(void)
988 debug("%s:%d\n", __func__, __LINE__);
990 /* The reset / cke part of initialization is broadcasted to all ranks */
991 addr = sdr_get_addr((u32 *)RW_MGR_SET_CS_AND_ODT_MASK);
992 writel(RW_MGR_RANK_ALL, SOCFPGA_SDR_ADDRESS + addr);
995 * Here's how you load register for a loop
996 * Counters are located @ 0x800
997 * Jump address are located @ 0xC00
998 * For both, registers 0 to 3 are selected using bits 3 and 2, like
999 * in 0x800, 0x804, 0x808, 0x80C and 0xC00, 0xC04, 0xC08, 0xC0C
1000 * I know this ain't pretty, but Avalon bus throws away the 2 least
1004 /* start with memory RESET activated */
1009 * 200us @ 266MHz (3.75 ns) ~ 54000 clock cycles
1010 * If a and b are the number of iteration in 2 nested loops
1011 * it takes the following number of cycles to complete the operation:
1012 * number_of_cycles = ((2 + n) * a + 2) * b
1013 * where n is the number of instruction in the inner loop
1014 * One possible solution is n = 0 , a = 256 , b = 106 => a = FF,
1019 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0;
1020 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR0_VAL),
1021 SOCFPGA_SDR_ADDRESS + addr);
1022 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1;
1023 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR1_VAL),
1024 SOCFPGA_SDR_ADDRESS + addr);
1025 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2;
1026 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TINIT_CNTR2_VAL),
1027 SOCFPGA_SDR_ADDRESS + addr);
1029 /* Load jump address */
1030 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0;
1031 writel(RW_MGR_INIT_RESET_0_CKE_0, SOCFPGA_SDR_ADDRESS + addr);
1033 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1;
1034 writel(RW_MGR_INIT_RESET_0_CKE_0, SOCFPGA_SDR_ADDRESS + addr);
1036 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2;
1037 writel(RW_MGR_INIT_RESET_0_CKE_0, SOCFPGA_SDR_ADDRESS + addr);
1039 /* Execute count instruction */
1040 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
1041 writel(RW_MGR_INIT_RESET_0_CKE_0, SOCFPGA_SDR_ADDRESS + addr);
1043 /* indicate that memory is stable */
1044 addr = (u32)&phy_mgr_cfg->reset_mem_stbl;
1045 writel(1, SOCFPGA_SDR_ADDRESS + addr);
1048 * transition the RESET to high
1053 * 500us @ 266MHz (3.75 ns) ~ 134000 clock cycles
1054 * If a and b are the number of iteration in 2 nested loops
1055 * it takes the following number of cycles to complete the operation
1056 * number_of_cycles = ((2 + n) * a + 2) * b
1057 * where n is the number of instruction in the inner loop
1058 * One possible solution is n = 2 , a = 131 , b = 256 => a = 83,
1063 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0;
1064 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TRESET_CNTR0_VAL),
1065 SOCFPGA_SDR_ADDRESS + addr);
1066 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1;
1067 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TRESET_CNTR1_VAL),
1068 SOCFPGA_SDR_ADDRESS + addr);
1069 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2;
1070 writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(SEQ_TRESET_CNTR2_VAL),
1071 SOCFPGA_SDR_ADDRESS + addr);
1073 /* Load jump address */
1074 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0;
1075 writel(RW_MGR_INIT_RESET_1_CKE_0, SOCFPGA_SDR_ADDRESS + addr);
1076 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1;
1077 writel(RW_MGR_INIT_RESET_1_CKE_0, SOCFPGA_SDR_ADDRESS + addr);
1078 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2;
1079 writel(RW_MGR_INIT_RESET_1_CKE_0, SOCFPGA_SDR_ADDRESS + addr);
1081 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
1082 writel(RW_MGR_INIT_RESET_1_CKE_0, SOCFPGA_SDR_ADDRESS + addr);
1084 /* bring up clock enable */
1086 /* tXRP < 250 ck cycles */
1087 delay_for_n_mem_clocks(250);
1089 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
1090 if (param->skip_ranks[r]) {
1091 /* request to skip the rank */
1096 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);
1099 * USER Use Mirror-ed commands for odd ranks if address
1102 if ((RW_MGR_MEM_ADDRESS_MIRRORING >> r) & 0x1) {
1103 set_jump_as_return();
1104 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
1105 writel(RW_MGR_MRS2_MIRR, SOCFPGA_SDR_ADDRESS + addr);
1106 delay_for_n_mem_clocks(4);
1107 set_jump_as_return();
1108 writel(RW_MGR_MRS3_MIRR, SOCFPGA_SDR_ADDRESS + addr);
1109 delay_for_n_mem_clocks(4);
1110 set_jump_as_return();
1111 writel(RW_MGR_MRS1_MIRR, SOCFPGA_SDR_ADDRESS + addr);
1112 delay_for_n_mem_clocks(4);
1113 set_jump_as_return();
1114 writel(RW_MGR_MRS0_DLL_RESET_MIRR, SOCFPGA_SDR_ADDRESS + addr);
1116 set_jump_as_return();
1117 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
1118 writel(RW_MGR_MRS2, SOCFPGA_SDR_ADDRESS + addr);
1119 delay_for_n_mem_clocks(4);
1120 set_jump_as_return();
1121 writel(RW_MGR_MRS3, SOCFPGA_SDR_ADDRESS + addr);
1122 delay_for_n_mem_clocks(4);
1123 set_jump_as_return();
1124 writel(RW_MGR_MRS1, SOCFPGA_SDR_ADDRESS + addr);
1125 set_jump_as_return();
1126 writel(RW_MGR_MRS0_DLL_RESET, SOCFPGA_SDR_ADDRESS + addr);
1128 set_jump_as_return();
1129 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
1130 writel(RW_MGR_ZQCL, SOCFPGA_SDR_ADDRESS + addr);
1132 /* tZQinit = tDLLK = 512 ck cycles */
1133 delay_for_n_mem_clocks(512);
1138 * At the end of calibration we have to program the user settings in, and
1139 * USER hand off the memory to the user.
1141 static void rw_mgr_mem_handoff(void)
1146 debug("%s:%d\n", __func__, __LINE__);
1147 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
1148 if (param->skip_ranks[r])
1149 /* request to skip the rank */
1152 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);
1154 /* precharge all banks ... */
1155 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
1156 writel(RW_MGR_PRECHARGE_ALL, SOCFPGA_SDR_ADDRESS + addr);
1158 /* load up MR settings specified by user */
1161 * Use Mirror-ed commands for odd ranks if address
1164 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
1165 if ((RW_MGR_MEM_ADDRESS_MIRRORING >> r) & 0x1) {
1166 set_jump_as_return();
1167 writel(RW_MGR_MRS2_MIRR, SOCFPGA_SDR_ADDRESS + addr);
1168 delay_for_n_mem_clocks(4);
1169 set_jump_as_return();
1170 writel(RW_MGR_MRS3_MIRR, SOCFPGA_SDR_ADDRESS + addr);
1171 delay_for_n_mem_clocks(4);
1172 set_jump_as_return();
1173 writel(RW_MGR_MRS1_MIRR, SOCFPGA_SDR_ADDRESS + addr);
1174 delay_for_n_mem_clocks(4);
1175 set_jump_as_return();
1176 writel(RW_MGR_MRS0_USER_MIRR, SOCFPGA_SDR_ADDRESS + addr);
1178 set_jump_as_return();
1179 writel(RW_MGR_MRS2, SOCFPGA_SDR_ADDRESS + addr);
1180 delay_for_n_mem_clocks(4);
1181 set_jump_as_return();
1182 writel(RW_MGR_MRS3, SOCFPGA_SDR_ADDRESS + addr);
1183 delay_for_n_mem_clocks(4);
1184 set_jump_as_return();
1185 writel(RW_MGR_MRS1, SOCFPGA_SDR_ADDRESS + addr);
1186 delay_for_n_mem_clocks(4);
1187 set_jump_as_return();
1188 writel(RW_MGR_MRS0_USER, SOCFPGA_SDR_ADDRESS + addr);
1191 * USER need to wait tMOD (12CK or 15ns) time before issuing
1192 * other commands, but we will have plenty of NIOS cycles before
1193 * actual handoff so its okay.
1199 * performs a guaranteed read on the patterns we are going to use during a
1200 * read test to ensure memory works
1202 static uint32_t rw_mgr_mem_calibrate_read_test_patterns(uint32_t rank_bgn,
1203 uint32_t group, uint32_t num_tries, uint32_t *bit_chk,
1207 uint32_t correct_mask_vg;
1208 uint32_t tmp_bit_chk;
1209 uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
1210 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
1212 uint32_t base_rw_mgr;
1214 *bit_chk = param->read_correct_mask;
1215 correct_mask_vg = param->read_correct_mask_vg;
1217 for (r = rank_bgn; r < rank_end; r++) {
1218 if (param->skip_ranks[r])
1219 /* request to skip the rank */
1223 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
1225 /* Load up a constant bursts of read commands */
1226 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0;
1227 writel(0x20, SOCFPGA_SDR_ADDRESS + addr);
1228 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0;
1229 writel(RW_MGR_GUARANTEED_READ, SOCFPGA_SDR_ADDRESS + addr);
1231 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1;
1232 writel(0x20, SOCFPGA_SDR_ADDRESS + addr);
1233 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1;
1234 writel(RW_MGR_GUARANTEED_READ_CONT, SOCFPGA_SDR_ADDRESS + addr);
1237 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS-1; ; vg--) {
1238 /* reset the fifos to get pointers to known state */
1240 addr = (u32)&phy_mgr_cmd->fifo_reset;
1241 writel(0, SOCFPGA_SDR_ADDRESS + addr);
1242 addr = sdr_get_addr((u32 *)RW_MGR_RESET_READ_DATAPATH);
1243 writel(0, SOCFPGA_SDR_ADDRESS + addr);
1245 tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_READ_DQS
1246 / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS);
1248 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
1249 writel(RW_MGR_GUARANTEED_READ, SOCFPGA_SDR_ADDRESS + addr +
1250 ((group * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS +
1253 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS;
1254 base_rw_mgr = readl(SOCFPGA_SDR_ADDRESS + addr);
1255 tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & (~base_rw_mgr));
1260 *bit_chk &= tmp_bit_chk;
1263 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
1264 writel(RW_MGR_CLEAR_DQS_ENABLE, SOCFPGA_SDR_ADDRESS + addr + (group << 2));
1266 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1267 debug_cond(DLEVEL == 1, "%s:%d test_load_patterns(%u,ALL) => (%u == %u) =>\
1268 %lu\n", __func__, __LINE__, group, *bit_chk, param->read_correct_mask,
1269 (long unsigned int)(*bit_chk == param->read_correct_mask));
1270 return *bit_chk == param->read_correct_mask;
1273 static uint32_t rw_mgr_mem_calibrate_read_test_patterns_all_ranks
1274 (uint32_t group, uint32_t num_tries, uint32_t *bit_chk)
1276 return rw_mgr_mem_calibrate_read_test_patterns(0, group,
1277 num_tries, bit_chk, 1);
1280 /* load up the patterns we are going to use during a read test */
1281 static void rw_mgr_mem_calibrate_read_load_patterns(uint32_t rank_bgn,
1286 uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
1287 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
1289 debug("%s:%d\n", __func__, __LINE__);
1290 for (r = rank_bgn; r < rank_end; r++) {
1291 if (param->skip_ranks[r])
1292 /* request to skip the rank */
1296 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
1298 /* Load up a constant bursts */
1299 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0;
1300 writel(0x20, SOCFPGA_SDR_ADDRESS + addr);
1302 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0;
1303 writel(RW_MGR_GUARANTEED_WRITE_WAIT0, SOCFPGA_SDR_ADDRESS + addr);
1305 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1;
1306 writel(0x20, SOCFPGA_SDR_ADDRESS + addr);
1308 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1;
1309 writel(RW_MGR_GUARANTEED_WRITE_WAIT1, SOCFPGA_SDR_ADDRESS + addr);
1311 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2;
1312 writel(0x04, SOCFPGA_SDR_ADDRESS + addr);
1314 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2;
1315 writel(RW_MGR_GUARANTEED_WRITE_WAIT2, SOCFPGA_SDR_ADDRESS + addr);
1317 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr3;
1318 writel(0x04, SOCFPGA_SDR_ADDRESS + addr);
1320 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3;
1321 writel(RW_MGR_GUARANTEED_WRITE_WAIT3, SOCFPGA_SDR_ADDRESS + addr);
1323 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
1324 writel(RW_MGR_GUARANTEED_WRITE, SOCFPGA_SDR_ADDRESS + addr);
1327 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1331 * try a read and see if it returns correct data back. has dummy reads
1332 * inserted into the mix used to align dqs enable. has more thorough checks
1333 * than the regular read test.
1335 static uint32_t rw_mgr_mem_calibrate_read_test(uint32_t rank_bgn, uint32_t group,
1336 uint32_t num_tries, uint32_t all_correct, uint32_t *bit_chk,
1337 uint32_t all_groups, uint32_t all_ranks)
1340 uint32_t correct_mask_vg;
1341 uint32_t tmp_bit_chk;
1342 uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
1343 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
1345 uint32_t base_rw_mgr;
1347 *bit_chk = param->read_correct_mask;
1348 correct_mask_vg = param->read_correct_mask_vg;
1350 uint32_t quick_read_mode = (((STATIC_CALIB_STEPS) &
1351 CALIB_SKIP_DELAY_SWEEPS) && ENABLE_SUPER_QUICK_CALIBRATION);
1353 for (r = rank_bgn; r < rank_end; r++) {
1354 if (param->skip_ranks[r])
1355 /* request to skip the rank */
1359 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
1361 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1;
1362 writel(0x10, SOCFPGA_SDR_ADDRESS + addr);
1364 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1;
1365 writel(RW_MGR_READ_B2B_WAIT1, SOCFPGA_SDR_ADDRESS + addr);
1367 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2;
1368 writel(0x10, SOCFPGA_SDR_ADDRESS + addr);
1369 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2;
1370 writel(RW_MGR_READ_B2B_WAIT2, SOCFPGA_SDR_ADDRESS + addr);
1372 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0;
1373 if (quick_read_mode)
1374 writel(0x1, SOCFPGA_SDR_ADDRESS + addr);
1375 /* need at least two (1+1) reads to capture failures */
1376 else if (all_groups)
1377 writel(0x06, SOCFPGA_SDR_ADDRESS + addr);
1379 writel(0x32, SOCFPGA_SDR_ADDRESS + addr);
1381 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0;
1382 writel(RW_MGR_READ_B2B, SOCFPGA_SDR_ADDRESS + addr);
1383 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr3;
1385 writel(RW_MGR_MEM_IF_READ_DQS_WIDTH *
1386 RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS - 1,
1387 SOCFPGA_SDR_ADDRESS + addr);
1389 writel(0x0, SOCFPGA_SDR_ADDRESS + addr);
1391 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3;
1392 writel(RW_MGR_READ_B2B, SOCFPGA_SDR_ADDRESS + addr);
1395 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS-1; ; vg--) {
1396 /* reset the fifos to get pointers to known state */
1397 addr = (u32)&phy_mgr_cmd->fifo_reset;
1398 writel(0, SOCFPGA_SDR_ADDRESS + addr);
1399 addr = sdr_get_addr((u32 *)RW_MGR_RESET_READ_DATAPATH);
1400 writel(0, SOCFPGA_SDR_ADDRESS + addr);
1402 tmp_bit_chk = tmp_bit_chk << (RW_MGR_MEM_DQ_PER_READ_DQS
1403 / RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS);
1405 addr = sdr_get_addr((u32 *)(all_groups ? RW_MGR_RUN_ALL_GROUPS :
1406 RW_MGR_RUN_SINGLE_GROUP));
1407 writel(RW_MGR_READ_B2B, SOCFPGA_SDR_ADDRESS + addr +
1408 ((group * RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS +
1411 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS;
1412 base_rw_mgr = readl(SOCFPGA_SDR_ADDRESS + addr);
1413 tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(base_rw_mgr));
1418 *bit_chk &= tmp_bit_chk;
1421 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
1422 writel(RW_MGR_CLEAR_DQS_ENABLE, SOCFPGA_SDR_ADDRESS + addr + (group << 2));
1425 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1426 debug_cond(DLEVEL == 2, "%s:%d read_test(%u,ALL,%u) =>\
1427 (%u == %u) => %lu", __func__, __LINE__, group,
1428 all_groups, *bit_chk, param->read_correct_mask,
1429 (long unsigned int)(*bit_chk ==
1430 param->read_correct_mask));
1431 return *bit_chk == param->read_correct_mask;
1433 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
1434 debug_cond(DLEVEL == 2, "%s:%d read_test(%u,ONE,%u) =>\
1435 (%u != %lu) => %lu\n", __func__, __LINE__,
1436 group, all_groups, *bit_chk, (long unsigned int)0,
1437 (long unsigned int)(*bit_chk != 0x00));
1438 return *bit_chk != 0x00;
1442 static uint32_t rw_mgr_mem_calibrate_read_test_all_ranks(uint32_t group,
1443 uint32_t num_tries, uint32_t all_correct, uint32_t *bit_chk,
1444 uint32_t all_groups)
1446 return rw_mgr_mem_calibrate_read_test(0, group, num_tries, all_correct,
1447 bit_chk, all_groups, 1);
1450 static void rw_mgr_incr_vfifo(uint32_t grp, uint32_t *v)
1452 uint32_t addr = (u32)&phy_mgr_cmd->inc_vfifo_hard_phy;
1454 writel(grp, SOCFPGA_SDR_ADDRESS + addr);
1458 static void rw_mgr_decr_vfifo(uint32_t grp, uint32_t *v)
1462 for (i = 0; i < VFIFO_SIZE-1; i++)
1463 rw_mgr_incr_vfifo(grp, v);
1466 static int find_vfifo_read(uint32_t grp, uint32_t *bit_chk)
1469 uint32_t fail_cnt = 0;
1470 uint32_t test_status;
1472 for (v = 0; v < VFIFO_SIZE; ) {
1473 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: vfifo %u\n",
1474 __func__, __LINE__, v);
1475 test_status = rw_mgr_mem_calibrate_read_test_all_ranks
1476 (grp, 1, PASS_ONE_BIT, bit_chk, 0);
1484 /* fiddle with FIFO */
1485 rw_mgr_incr_vfifo(grp, &v);
1488 if (v >= VFIFO_SIZE) {
1489 /* no failing read found!! Something must have gone wrong */
1490 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: vfifo failed\n",
1491 __func__, __LINE__);
1498 static int find_working_phase(uint32_t *grp, uint32_t *bit_chk,
1499 uint32_t dtaps_per_ptap, uint32_t *work_bgn,
1500 uint32_t *v, uint32_t *d, uint32_t *p,
1501 uint32_t *i, uint32_t *max_working_cnt)
1503 uint32_t found_begin = 0;
1504 uint32_t tmp_delay = 0;
1505 uint32_t test_status;
1507 for (*d = 0; *d <= dtaps_per_ptap; (*d)++, tmp_delay +=
1508 IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
1509 *work_bgn = tmp_delay;
1510 scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d);
1512 for (*i = 0; *i < VFIFO_SIZE; (*i)++) {
1513 for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX; (*p)++, *work_bgn +=
1514 IO_DELAY_PER_OPA_TAP) {
1515 scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p);
1518 rw_mgr_mem_calibrate_read_test_all_ranks
1519 (*grp, 1, PASS_ONE_BIT, bit_chk, 0);
1522 *max_working_cnt = 1;
1531 if (*p > IO_DQS_EN_PHASE_MAX)
1532 /* fiddle with FIFO */
1533 rw_mgr_incr_vfifo(*grp, v);
1540 if (*i >= VFIFO_SIZE) {
1541 /* cannot find working solution */
1542 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: no vfifo/\
1543 ptap/dtap\n", __func__, __LINE__);
1550 static void sdr_backup_phase(uint32_t *grp, uint32_t *bit_chk,
1551 uint32_t *work_bgn, uint32_t *v, uint32_t *d,
1552 uint32_t *p, uint32_t *max_working_cnt)
1554 uint32_t found_begin = 0;
1557 /* Special case code for backing up a phase */
1559 *p = IO_DQS_EN_PHASE_MAX;
1560 rw_mgr_decr_vfifo(*grp, v);
1564 tmp_delay = *work_bgn - IO_DELAY_PER_OPA_TAP;
1565 scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p);
1567 for (*d = 0; *d <= IO_DQS_EN_DELAY_MAX && tmp_delay < *work_bgn;
1568 (*d)++, tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
1569 scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d);
1571 if (rw_mgr_mem_calibrate_read_test_all_ranks(*grp, 1,
1575 *work_bgn = tmp_delay;
1580 /* We have found a working dtap before the ptap found above */
1581 if (found_begin == 1)
1582 (*max_working_cnt)++;
1585 * Restore VFIFO to old state before we decremented it
1589 if (*p > IO_DQS_EN_PHASE_MAX) {
1591 rw_mgr_incr_vfifo(*grp, v);
1594 scc_mgr_set_dqs_en_delay_all_ranks(*grp, 0);
1597 static int sdr_nonworking_phase(uint32_t *grp, uint32_t *bit_chk,
1598 uint32_t *work_bgn, uint32_t *v, uint32_t *d,
1599 uint32_t *p, uint32_t *i, uint32_t *max_working_cnt,
1602 uint32_t found_end = 0;
1605 *work_end += IO_DELAY_PER_OPA_TAP;
1606 if (*p > IO_DQS_EN_PHASE_MAX) {
1607 /* fiddle with FIFO */
1609 rw_mgr_incr_vfifo(*grp, v);
1612 for (; *i < VFIFO_SIZE + 1; (*i)++) {
1613 for (; *p <= IO_DQS_EN_PHASE_MAX; (*p)++, *work_end
1614 += IO_DELAY_PER_OPA_TAP) {
1615 scc_mgr_set_dqs_en_phase_all_ranks(*grp, *p);
1617 if (!rw_mgr_mem_calibrate_read_test_all_ranks
1618 (*grp, 1, PASS_ONE_BIT, bit_chk, 0)) {
1622 (*max_working_cnt)++;
1629 if (*p > IO_DQS_EN_PHASE_MAX) {
1630 /* fiddle with FIFO */
1631 rw_mgr_incr_vfifo(*grp, v);
1636 if (*i >= VFIFO_SIZE + 1) {
1637 /* cannot see edge of failing read */
1638 debug_cond(DLEVEL == 2, "%s:%d sdr_nonworking_phase: end:\
1639 failed\n", __func__, __LINE__);
1646 static int sdr_find_window_centre(uint32_t *grp, uint32_t *bit_chk,
1647 uint32_t *work_bgn, uint32_t *v, uint32_t *d,
1648 uint32_t *p, uint32_t *work_mid,
1654 *work_mid = (*work_bgn + *work_end) / 2;
1656 debug_cond(DLEVEL == 2, "work_bgn=%d work_end=%d work_mid=%d\n",
1657 *work_bgn, *work_end, *work_mid);
1658 /* Get the middle delay to be less than a VFIFO delay */
1659 for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX;
1660 (*p)++, tmp_delay += IO_DELAY_PER_OPA_TAP)
1662 debug_cond(DLEVEL == 2, "vfifo ptap delay %d\n", tmp_delay);
1663 while (*work_mid > tmp_delay)
1664 *work_mid -= tmp_delay;
1665 debug_cond(DLEVEL == 2, "new work_mid %d\n", *work_mid);
1668 for (*p = 0; *p <= IO_DQS_EN_PHASE_MAX && tmp_delay < *work_mid;
1669 (*p)++, tmp_delay += IO_DELAY_PER_OPA_TAP)
1671 tmp_delay -= IO_DELAY_PER_OPA_TAP;
1672 debug_cond(DLEVEL == 2, "new p %d, tmp_delay=%d\n", (*p) - 1, tmp_delay);
1673 for (*d = 0; *d <= IO_DQS_EN_DELAY_MAX && tmp_delay < *work_mid; (*d)++,
1674 tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP)
1676 debug_cond(DLEVEL == 2, "new d %d, tmp_delay=%d\n", *d, tmp_delay);
1678 scc_mgr_set_dqs_en_phase_all_ranks(*grp, (*p) - 1);
1679 scc_mgr_set_dqs_en_delay_all_ranks(*grp, *d);
1682 * push vfifo until we can successfully calibrate. We can do this
1683 * because the largest possible margin in 1 VFIFO cycle.
1685 for (i = 0; i < VFIFO_SIZE; i++) {
1686 debug_cond(DLEVEL == 2, "find_dqs_en_phase: center: vfifo=%u\n",
1688 if (rw_mgr_mem_calibrate_read_test_all_ranks(*grp, 1,
1694 /* fiddle with FIFO */
1695 rw_mgr_incr_vfifo(*grp, v);
1698 if (i >= VFIFO_SIZE) {
1699 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: center: \
1700 failed\n", __func__, __LINE__);
1707 /* find a good dqs enable to use */
1708 static uint32_t rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(uint32_t grp)
1710 uint32_t v, d, p, i;
1711 uint32_t max_working_cnt;
1713 uint32_t dtaps_per_ptap;
1714 uint32_t work_bgn, work_mid, work_end;
1715 uint32_t found_passing_read, found_failing_read, initial_failing_dtap;
1718 debug("%s:%d %u\n", __func__, __LINE__, grp);
1720 reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
1722 scc_mgr_set_dqs_en_delay_all_ranks(grp, 0);
1723 scc_mgr_set_dqs_en_phase_all_ranks(grp, 0);
1725 /* ************************************************************** */
1726 /* * Step 0 : Determine number of delay taps for each phase tap * */
1727 dtaps_per_ptap = IO_DELAY_PER_OPA_TAP/IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
1729 /* ********************************************************* */
1730 /* * Step 1 : First push vfifo until we get a failing read * */
1731 v = find_vfifo_read(grp, &bit_chk);
1733 max_working_cnt = 0;
1735 /* ******************************************************** */
1736 /* * step 2: find first working phase, increment in ptaps * */
1738 if (find_working_phase(&grp, &bit_chk, dtaps_per_ptap, &work_bgn, &v, &d,
1739 &p, &i, &max_working_cnt) == 0)
1742 work_end = work_bgn;
1745 * If d is 0 then the working window covers a phase tap and
1746 * we can follow the old procedure otherwise, we've found the beginning,
1747 * and we need to increment the dtaps until we find the end.
1750 /* ********************************************************* */
1751 /* * step 3a: if we have room, back off by one and
1752 increment in dtaps * */
1754 sdr_backup_phase(&grp, &bit_chk, &work_bgn, &v, &d, &p,
1757 /* ********************************************************* */
1758 /* * step 4a: go forward from working phase to non working
1759 phase, increment in ptaps * */
1760 if (sdr_nonworking_phase(&grp, &bit_chk, &work_bgn, &v, &d, &p,
1761 &i, &max_working_cnt, &work_end) == 0)
1764 /* ********************************************************* */
1765 /* * step 5a: back off one from last, increment in dtaps * */
1767 /* Special case code for backing up a phase */
1769 p = IO_DQS_EN_PHASE_MAX;
1770 rw_mgr_decr_vfifo(grp, &v);
1775 work_end -= IO_DELAY_PER_OPA_TAP;
1776 scc_mgr_set_dqs_en_phase_all_ranks(grp, p);
1778 /* * The actual increment of dtaps is done outside of
1779 the if/else loop to share code */
1782 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: v/p: \
1783 vfifo=%u ptap=%u\n", __func__, __LINE__,
1786 /* ******************************************************* */
1787 /* * step 3-5b: Find the right edge of the window using
1789 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase:vfifo=%u \
1790 ptap=%u dtap=%u bgn=%u\n", __func__, __LINE__,
1793 work_end = work_bgn;
1795 /* * The actual increment of dtaps is done outside of the
1796 if/else loop to share code */
1798 /* Only here to counterbalance a subtract later on which is
1799 not needed if this branch of the algorithm is taken */
1803 /* The dtap increment to find the failing edge is done here */
1804 for (; d <= IO_DQS_EN_DELAY_MAX; d++, work_end +=
1805 IO_DELAY_PER_DQS_EN_DCHAIN_TAP) {
1806 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: \
1807 end-2: dtap=%u\n", __func__, __LINE__, d);
1808 scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
1810 if (!rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
1817 /* Go back to working dtap */
1819 work_end -= IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
1821 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: v/p/d: vfifo=%u \
1822 ptap=%u dtap=%u end=%u\n", __func__, __LINE__,
1823 v, p, d-1, work_end);
1825 if (work_end < work_bgn) {
1827 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: end-2: \
1828 failed\n", __func__, __LINE__);
1832 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: found range [%u,%u]\n",
1833 __func__, __LINE__, work_bgn, work_end);
1835 /* *************************************************************** */
1837 * * We need to calculate the number of dtaps that equal a ptap
1838 * * To do that we'll back up a ptap and re-find the edge of the
1839 * * window using dtaps
1842 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: calculate dtaps_per_ptap \
1843 for tracking\n", __func__, __LINE__);
1845 /* Special case code for backing up a phase */
1847 p = IO_DQS_EN_PHASE_MAX;
1848 rw_mgr_decr_vfifo(grp, &v);
1849 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: backedup \
1850 cycle/phase: v=%u p=%u\n", __func__, __LINE__,
1854 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: backedup \
1855 phase only: v=%u p=%u", __func__, __LINE__,
1859 scc_mgr_set_dqs_en_phase_all_ranks(grp, p);
1862 * Increase dtap until we first see a passing read (in case the
1863 * window is smaller than a ptap),
1864 * and then a failing read to mark the edge of the window again
1867 /* Find a passing read */
1868 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: find passing read\n",
1869 __func__, __LINE__);
1870 found_passing_read = 0;
1871 found_failing_read = 0;
1872 initial_failing_dtap = d;
1873 for (; d <= IO_DQS_EN_DELAY_MAX; d++) {
1874 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: testing \
1875 read d=%u\n", __func__, __LINE__, d);
1876 scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
1878 if (rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
1881 found_passing_read = 1;
1886 if (found_passing_read) {
1887 /* Find a failing read */
1888 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: find failing \
1889 read\n", __func__, __LINE__);
1890 for (d = d + 1; d <= IO_DQS_EN_DELAY_MAX; d++) {
1891 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: \
1892 testing read d=%u\n", __func__, __LINE__, d);
1893 scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
1895 if (!rw_mgr_mem_calibrate_read_test_all_ranks
1896 (grp, 1, PASS_ONE_BIT, &bit_chk, 0)) {
1897 found_failing_read = 1;
1902 debug_cond(DLEVEL == 1, "%s:%d find_dqs_en_phase: failed to \
1903 calculate dtaps", __func__, __LINE__);
1904 debug_cond(DLEVEL == 1, "per ptap. Fall back on static value\n");
1908 * The dynamically calculated dtaps_per_ptap is only valid if we
1909 * found a passing/failing read. If we didn't, it means d hit the max
1910 * (IO_DQS_EN_DELAY_MAX). Otherwise, dtaps_per_ptap retains its
1911 * statically calculated value.
1913 if (found_passing_read && found_failing_read)
1914 dtaps_per_ptap = d - initial_failing_dtap;
1916 addr = (u32)&sdr_reg_file->dtaps_per_ptap;
1917 writel(dtaps_per_ptap, SOCFPGA_SDR_ADDRESS + addr);
1918 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: dtaps_per_ptap=%u \
1919 - %u = %u", __func__, __LINE__, d,
1920 initial_failing_dtap, dtaps_per_ptap);
1922 /* ******************************************** */
1923 /* * step 6: Find the centre of the window * */
1924 if (sdr_find_window_centre(&grp, &bit_chk, &work_bgn, &v, &d, &p,
1925 &work_mid, &work_end) == 0)
1928 debug_cond(DLEVEL == 2, "%s:%d find_dqs_en_phase: center found: \
1929 vfifo=%u ptap=%u dtap=%u\n", __func__, __LINE__,
1935 * Try rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase across different
1936 * dq_in_delay values
1939 rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay
1940 (uint32_t write_group, uint32_t read_group, uint32_t test_bgn)
1949 const uint32_t delay_step = IO_IO_IN_DELAY_MAX /
1950 (RW_MGR_MEM_DQ_PER_READ_DQS-1);
1951 /* we start at zero, so have one less dq to devide among */
1953 debug("%s:%d (%u,%u,%u)", __func__, __LINE__, write_group, read_group,
1956 /* try different dq_in_delays since the dq path is shorter than dqs */
1958 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
1959 r += NUM_RANKS_PER_SHADOW_REG) {
1960 for (i = 0, p = test_bgn, d = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS;
1961 i++, p++, d += delay_step) {
1962 debug_cond(DLEVEL == 1, "%s:%d rw_mgr_mem_calibrate_\
1963 vfifo_find_dqs_", __func__, __LINE__);
1964 debug_cond(DLEVEL == 1, "en_phase_sweep_dq_in_delay: g=%u/%u ",
1965 write_group, read_group);
1966 debug_cond(DLEVEL == 1, "r=%u, i=%u p=%u d=%u\n", r, i , p, d);
1967 scc_mgr_set_dq_in_delay(write_group, p, d);
1970 addr = (u32)&sdr_scc_mgr->update;
1971 writel(0, SOCFPGA_SDR_ADDRESS + addr);
1974 found = rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(read_group);
1976 debug_cond(DLEVEL == 1, "%s:%d rw_mgr_mem_calibrate_vfifo_find_dqs_\
1977 en_phase_sweep_dq", __func__, __LINE__);
1978 debug_cond(DLEVEL == 1, "_in_delay: g=%u/%u found=%u; Reseting delay \
1979 chain to zero\n", write_group, read_group, found);
1981 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
1982 r += NUM_RANKS_PER_SHADOW_REG) {
1983 for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS;
1985 scc_mgr_set_dq_in_delay(write_group, p, 0);
1988 addr = (u32)&sdr_scc_mgr->update;
1989 writel(0, SOCFPGA_SDR_ADDRESS + addr);
1995 /* per-bit deskew DQ and center */
1996 static uint32_t rw_mgr_mem_calibrate_vfifo_center(uint32_t rank_bgn,
1997 uint32_t write_group, uint32_t read_group, uint32_t test_bgn,
1998 uint32_t use_read_test, uint32_t update_fom)
2000 uint32_t i, p, d, min_index;
2002 * Store these as signed since there are comparisons with
2006 uint32_t sticky_bit_chk;
2007 int32_t left_edge[RW_MGR_MEM_DQ_PER_READ_DQS];
2008 int32_t right_edge[RW_MGR_MEM_DQ_PER_READ_DQS];
2009 int32_t final_dq[RW_MGR_MEM_DQ_PER_READ_DQS];
2011 int32_t orig_mid_min, mid_min;
2012 int32_t new_dqs, start_dqs, start_dqs_en, shift_dq, final_dqs,
2014 int32_t dq_margin, dqs_margin;
2016 uint32_t temp_dq_in_delay1, temp_dq_in_delay2;
2019 debug("%s:%d: %u %u", __func__, __LINE__, read_group, test_bgn);
2021 addr = sdr_get_addr((u32 *)SCC_MGR_DQS_IN_DELAY);
2022 start_dqs = readl(SOCFPGA_SDR_ADDRESS + addr + (read_group << 2));
2023 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS)
2024 start_dqs_en = readl(SOCFPGA_SDR_ADDRESS + addr + ((read_group << 2)
2025 - IO_DQS_EN_DELAY_OFFSET));
2027 /* set the left and right edge of each bit to an illegal value */
2028 /* use (IO_IO_IN_DELAY_MAX + 1) as an illegal value */
2030 for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
2031 left_edge[i] = IO_IO_IN_DELAY_MAX + 1;
2032 right_edge[i] = IO_IO_IN_DELAY_MAX + 1;
2035 addr = (u32)&sdr_scc_mgr->update;
2036 /* Search for the left edge of the window for each bit */
2037 for (d = 0; d <= IO_IO_IN_DELAY_MAX; d++) {
2038 scc_mgr_apply_group_dq_in_delay(write_group, test_bgn, d);
2040 writel(0, SOCFPGA_SDR_ADDRESS + addr);
2043 * Stop searching when the read test doesn't pass AND when
2044 * we've seen a passing read on every bit.
2046 if (use_read_test) {
2047 stop = !rw_mgr_mem_calibrate_read_test(rank_bgn,
2048 read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT,
2051 rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
2054 bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS *
2055 (read_group - (write_group *
2056 RW_MGR_MEM_IF_READ_DQS_WIDTH /
2057 RW_MGR_MEM_IF_WRITE_DQS_WIDTH)));
2058 stop = (bit_chk == 0);
2060 sticky_bit_chk = sticky_bit_chk | bit_chk;
2061 stop = stop && (sticky_bit_chk == param->read_correct_mask);
2062 debug_cond(DLEVEL == 2, "%s:%d vfifo_center(left): dtap=%u => %u == %u \
2063 && %u", __func__, __LINE__, d,
2065 param->read_correct_mask, stop);
2070 for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
2072 /* Remember a passing test as the
2076 /* If a left edge has not been seen yet,
2077 then a future passing test will mark
2078 this edge as the right edge */
2080 IO_IO_IN_DELAY_MAX + 1) {
2081 right_edge[i] = -(d + 1);
2084 bit_chk = bit_chk >> 1;
2089 /* Reset DQ delay chains to 0 */
2090 scc_mgr_apply_group_dq_in_delay(write_group, test_bgn, 0);
2092 for (i = RW_MGR_MEM_DQ_PER_READ_DQS - 1;; i--) {
2093 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: left_edge[%u]: \
2094 %d right_edge[%u]: %d\n", __func__, __LINE__,
2095 i, left_edge[i], i, right_edge[i]);
2098 * Check for cases where we haven't found the left edge,
2099 * which makes our assignment of the the right edge invalid.
2100 * Reset it to the illegal value.
2102 if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) && (
2103 right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) {
2104 right_edge[i] = IO_IO_IN_DELAY_MAX + 1;
2105 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: reset \
2106 right_edge[%u]: %d\n", __func__, __LINE__,
2111 * Reset sticky bit (except for bits where we have seen
2112 * both the left and right edge).
2114 sticky_bit_chk = sticky_bit_chk << 1;
2115 if ((left_edge[i] != IO_IO_IN_DELAY_MAX + 1) &&
2116 (right_edge[i] != IO_IO_IN_DELAY_MAX + 1)) {
2117 sticky_bit_chk = sticky_bit_chk | 1;
2124 addr = (u32)&sdr_scc_mgr->update;
2125 /* Search for the right edge of the window for each bit */
2126 for (d = 0; d <= IO_DQS_IN_DELAY_MAX - start_dqs; d++) {
2127 scc_mgr_set_dqs_bus_in_delay(read_group, d + start_dqs);
2128 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
2129 uint32_t delay = d + start_dqs_en;
2130 if (delay > IO_DQS_EN_DELAY_MAX)
2131 delay = IO_DQS_EN_DELAY_MAX;
2132 scc_mgr_set_dqs_en_delay(read_group, delay);
2134 scc_mgr_load_dqs(read_group);
2136 writel(0, SOCFPGA_SDR_ADDRESS + addr);
2139 * Stop searching when the read test doesn't pass AND when
2140 * we've seen a passing read on every bit.
2142 if (use_read_test) {
2143 stop = !rw_mgr_mem_calibrate_read_test(rank_bgn,
2144 read_group, NUM_READ_PB_TESTS, PASS_ONE_BIT,
2147 rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
2150 bit_chk = bit_chk >> (RW_MGR_MEM_DQ_PER_READ_DQS *
2151 (read_group - (write_group *
2152 RW_MGR_MEM_IF_READ_DQS_WIDTH /
2153 RW_MGR_MEM_IF_WRITE_DQS_WIDTH)));
2154 stop = (bit_chk == 0);
2156 sticky_bit_chk = sticky_bit_chk | bit_chk;
2157 stop = stop && (sticky_bit_chk == param->read_correct_mask);
2159 debug_cond(DLEVEL == 2, "%s:%d vfifo_center(right): dtap=%u => %u == \
2160 %u && %u", __func__, __LINE__, d,
2161 sticky_bit_chk, param->read_correct_mask, stop);
2166 for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
2168 /* Remember a passing test as
2173 /* If a right edge has not been
2174 seen yet, then a future passing
2175 test will mark this edge as the
2177 if (right_edge[i] ==
2178 IO_IO_IN_DELAY_MAX + 1) {
2179 left_edge[i] = -(d + 1);
2182 /* d = 0 failed, but it passed
2183 when testing the left edge,
2184 so it must be marginal,
2186 if (right_edge[i] ==
2187 IO_IO_IN_DELAY_MAX + 1 &&
2193 /* If a right edge has not been
2194 seen yet, then a future passing
2195 test will mark this edge as the
2197 else if (right_edge[i] ==
2198 IO_IO_IN_DELAY_MAX +
2200 left_edge[i] = -(d + 1);
2205 debug_cond(DLEVEL == 2, "%s:%d vfifo_center[r,\
2206 d=%u]: ", __func__, __LINE__, d);
2207 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d ",
2208 (int)(bit_chk & 1), i, left_edge[i]);
2209 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
2211 bit_chk = bit_chk >> 1;
2216 /* Check that all bits have a window */
2217 addr = (u32)&sdr_scc_mgr->update;
2218 for (i = 0; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
2219 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: left_edge[%u]: \
2220 %d right_edge[%u]: %d", __func__, __LINE__,
2221 i, left_edge[i], i, right_edge[i]);
2222 if ((left_edge[i] == IO_IO_IN_DELAY_MAX + 1) || (right_edge[i]
2223 == IO_IO_IN_DELAY_MAX + 1)) {
2225 * Restore delay chain settings before letting the loop
2226 * in rw_mgr_mem_calibrate_vfifo to retry different
2227 * dqs/ck relationships.
2229 scc_mgr_set_dqs_bus_in_delay(read_group, start_dqs);
2230 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
2231 scc_mgr_set_dqs_en_delay(read_group,
2234 scc_mgr_load_dqs(read_group);
2235 writel(0, SOCFPGA_SDR_ADDRESS + addr);
2237 debug_cond(DLEVEL == 1, "%s:%d vfifo_center: failed to \
2238 find edge [%u]: %d %d", __func__, __LINE__,
2239 i, left_edge[i], right_edge[i]);
2240 if (use_read_test) {
2241 set_failing_group_stage(read_group *
2242 RW_MGR_MEM_DQ_PER_READ_DQS + i,
2244 CAL_SUBSTAGE_VFIFO_CENTER);
2246 set_failing_group_stage(read_group *
2247 RW_MGR_MEM_DQ_PER_READ_DQS + i,
2248 CAL_STAGE_VFIFO_AFTER_WRITES,
2249 CAL_SUBSTAGE_VFIFO_CENTER);
2255 /* Find middle of window for each DQ bit */
2256 mid_min = left_edge[0] - right_edge[0];
2258 for (i = 1; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++) {
2259 mid = left_edge[i] - right_edge[i];
2260 if (mid < mid_min) {
2267 * -mid_min/2 represents the amount that we need to move DQS.
2268 * If mid_min is odd and positive we'll need to add one to
2269 * make sure the rounding in further calculations is correct
2270 * (always bias to the right), so just add 1 for all positive values.
2275 mid_min = mid_min / 2;
2277 debug_cond(DLEVEL == 1, "%s:%d vfifo_center: mid_min=%d (index=%u)\n",
2278 __func__, __LINE__, mid_min, min_index);
2280 /* Determine the amount we can change DQS (which is -mid_min) */
2281 orig_mid_min = mid_min;
2282 new_dqs = start_dqs - mid_min;
2283 if (new_dqs > IO_DQS_IN_DELAY_MAX)
2284 new_dqs = IO_DQS_IN_DELAY_MAX;
2285 else if (new_dqs < 0)
2288 mid_min = start_dqs - new_dqs;
2289 debug_cond(DLEVEL == 1, "vfifo_center: new mid_min=%d new_dqs=%d\n",
2292 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
2293 if (start_dqs_en - mid_min > IO_DQS_EN_DELAY_MAX)
2294 mid_min += start_dqs_en - mid_min - IO_DQS_EN_DELAY_MAX;
2295 else if (start_dqs_en - mid_min < 0)
2296 mid_min += start_dqs_en - mid_min;
2298 new_dqs = start_dqs - mid_min;
2300 debug_cond(DLEVEL == 1, "vfifo_center: start_dqs=%d start_dqs_en=%d \
2301 new_dqs=%d mid_min=%d\n", start_dqs,
2302 IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS ? start_dqs_en : -1,
2305 /* Initialize data for export structures */
2306 dqs_margin = IO_IO_IN_DELAY_MAX + 1;
2307 dq_margin = IO_IO_IN_DELAY_MAX + 1;
2309 addr = sdr_get_addr((u32 *)SCC_MGR_IO_IN_DELAY);
2310 /* add delay to bring centre of all DQ windows to the same "level" */
2311 for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_READ_DQS; i++, p++) {
2312 /* Use values before divide by 2 to reduce round off error */
2313 shift_dq = (left_edge[i] - right_edge[i] -
2314 (left_edge[min_index] - right_edge[min_index]))/2 +
2315 (orig_mid_min - mid_min);
2317 debug_cond(DLEVEL == 2, "vfifo_center: before: \
2318 shift_dq[%u]=%d\n", i, shift_dq);
2320 temp_dq_in_delay1 = readl(SOCFPGA_SDR_ADDRESS + addr + (p << 2));
2321 temp_dq_in_delay2 = readl(SOCFPGA_SDR_ADDRESS + addr + (i << 2));
2323 if (shift_dq + (int32_t)temp_dq_in_delay1 >
2324 (int32_t)IO_IO_IN_DELAY_MAX) {
2325 shift_dq = (int32_t)IO_IO_IN_DELAY_MAX - temp_dq_in_delay2;
2326 } else if (shift_dq + (int32_t)temp_dq_in_delay1 < 0) {
2327 shift_dq = -(int32_t)temp_dq_in_delay1;
2329 debug_cond(DLEVEL == 2, "vfifo_center: after: \
2330 shift_dq[%u]=%d\n", i, shift_dq);
2331 final_dq[i] = temp_dq_in_delay1 + shift_dq;
2332 scc_mgr_set_dq_in_delay(write_group, p, final_dq[i]);
2335 debug_cond(DLEVEL == 2, "vfifo_center: margin[%u]=[%d,%d]\n", i,
2336 left_edge[i] - shift_dq + (-mid_min),
2337 right_edge[i] + shift_dq - (-mid_min));
2338 /* To determine values for export structures */
2339 if (left_edge[i] - shift_dq + (-mid_min) < dq_margin)
2340 dq_margin = left_edge[i] - shift_dq + (-mid_min);
2342 if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin)
2343 dqs_margin = right_edge[i] + shift_dq - (-mid_min);
2346 final_dqs = new_dqs;
2347 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS)
2348 final_dqs_en = start_dqs_en - mid_min;
2351 if (IO_SHIFT_DQS_EN_WHEN_SHIFT_DQS) {
2352 scc_mgr_set_dqs_en_delay(read_group, final_dqs_en);
2353 scc_mgr_load_dqs(read_group);
2357 scc_mgr_set_dqs_bus_in_delay(read_group, final_dqs);
2358 scc_mgr_load_dqs(read_group);
2359 debug_cond(DLEVEL == 2, "%s:%d vfifo_center: dq_margin=%d \
2360 dqs_margin=%d", __func__, __LINE__,
2361 dq_margin, dqs_margin);
2364 * Do not remove this line as it makes sure all of our decisions
2365 * have been applied. Apply the update bit.
2367 addr = (u32)&sdr_scc_mgr->update;
2368 writel(0, SOCFPGA_SDR_ADDRESS + addr);
2370 return (dq_margin >= 0) && (dqs_margin >= 0);
2374 * calibrate the read valid prediction FIFO.
2376 * - read valid prediction will consist of finding a good DQS enable phase,
2377 * DQS enable delay, DQS input phase, and DQS input delay.
2378 * - we also do a per-bit deskew on the DQ lines.
2380 static uint32_t rw_mgr_mem_calibrate_vfifo(uint32_t read_group,
2383 uint32_t p, d, rank_bgn, sr;
2384 uint32_t dtaps_per_ptap;
2387 uint32_t grp_calibrated;
2388 uint32_t write_group, write_test_bgn;
2389 uint32_t failed_substage;
2391 debug("%s:%d: %u %u\n", __func__, __LINE__, read_group, test_bgn);
2393 /* update info for sims */
2394 reg_file_set_stage(CAL_STAGE_VFIFO);
2396 write_group = read_group;
2397 write_test_bgn = test_bgn;
2399 /* USER Determine number of delay taps for each phase tap */
2402 while (tmp_delay < IO_DELAY_PER_OPA_TAP) {
2404 tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
2409 /* update info for sims */
2410 reg_file_set_group(read_group);
2414 reg_file_set_sub_stage(CAL_SUBSTAGE_GUARANTEED_READ);
2415 failed_substage = CAL_SUBSTAGE_GUARANTEED_READ;
2417 for (d = 0; d <= dtaps_per_ptap && grp_calibrated == 0; d += 2) {
2419 * In RLDRAMX we may be messing the delay of pins in
2420 * the same write group but outside of the current read
2421 * the group, but that's ok because we haven't
2422 * calibrated output side yet.
2425 scc_mgr_apply_group_all_out_delay_add_all_ranks
2426 (write_group, write_test_bgn, d);
2429 for (p = 0; p <= IO_DQDQS_OUT_PHASE_MAX && grp_calibrated == 0;
2431 /* set a particular dqdqs phase */
2432 scc_mgr_set_dqdqs_output_phase_all_ranks(read_group, p);
2434 debug_cond(DLEVEL == 1, "%s:%d calibrate_vfifo: g=%u \
2435 p=%u d=%u\n", __func__, __LINE__,
2439 * Load up the patterns used by read calibration
2440 * using current DQDQS phase.
2442 rw_mgr_mem_calibrate_read_load_patterns(0, 1);
2443 if (!(gbl->phy_debug_mode_flags &
2444 PHY_DEBUG_DISABLE_GUARANTEED_READ)) {
2445 if (!rw_mgr_mem_calibrate_read_test_patterns_all_ranks
2446 (read_group, 1, &bit_chk)) {
2447 debug_cond(DLEVEL == 1, "%s:%d Guaranteed read test failed:",
2448 __func__, __LINE__);
2449 debug_cond(DLEVEL == 1, " g=%u p=%u d=%u\n",
2457 if (rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase_sweep_dq_in_delay
2458 (write_group, read_group, test_bgn)) {
2460 * USER Read per-bit deskew can be done on a
2461 * per shadow register basis.
2463 for (rank_bgn = 0, sr = 0;
2464 rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS;
2465 rank_bgn += NUM_RANKS_PER_SHADOW_REG,
2468 * Determine if this set of ranks
2469 * should be skipped entirely.
2471 if (!param->skip_shadow_regs[sr]) {
2473 * If doing read after write
2474 * calibration, do not update
2475 * FOM, now - do it then.
2477 if (!rw_mgr_mem_calibrate_vfifo_center
2478 (rank_bgn, write_group,
2479 read_group, test_bgn, 1, 0)) {
2482 CAL_SUBSTAGE_VFIFO_CENTER;
2488 failed_substage = CAL_SUBSTAGE_DQS_EN_PHASE;
2493 if (grp_calibrated == 0) {
2494 set_failing_group_stage(write_group, CAL_STAGE_VFIFO,
2500 * Reset the delay chains back to zero if they have moved > 1
2501 * (check for > 1 because loop will increase d even when pass in
2505 scc_mgr_zero_group(write_group, write_test_bgn, 1);
2510 /* VFIFO Calibration -- Read Deskew Calibration after write deskew */
2511 static uint32_t rw_mgr_mem_calibrate_vfifo_end(uint32_t read_group,
2514 uint32_t rank_bgn, sr;
2515 uint32_t grp_calibrated;
2516 uint32_t write_group;
2518 debug("%s:%d %u %u", __func__, __LINE__, read_group, test_bgn);
2520 /* update info for sims */
2522 reg_file_set_stage(CAL_STAGE_VFIFO_AFTER_WRITES);
2523 reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
2525 write_group = read_group;
2527 /* update info for sims */
2528 reg_file_set_group(read_group);
2531 /* Read per-bit deskew can be done on a per shadow register basis */
2532 for (rank_bgn = 0, sr = 0; rank_bgn < RW_MGR_MEM_NUMBER_OF_RANKS;
2533 rank_bgn += NUM_RANKS_PER_SHADOW_REG, ++sr) {
2534 /* Determine if this set of ranks should be skipped entirely */
2535 if (!param->skip_shadow_regs[sr]) {
2536 /* This is the last calibration round, update FOM here */
2537 if (!rw_mgr_mem_calibrate_vfifo_center(rank_bgn,
2548 if (grp_calibrated == 0) {
2549 set_failing_group_stage(write_group,
2550 CAL_STAGE_VFIFO_AFTER_WRITES,
2551 CAL_SUBSTAGE_VFIFO_CENTER);
2558 /* Calibrate LFIFO to find smallest read latency */
2559 static uint32_t rw_mgr_mem_calibrate_lfifo(void)
2565 debug("%s:%d\n", __func__, __LINE__);
2567 /* update info for sims */
2568 reg_file_set_stage(CAL_STAGE_LFIFO);
2569 reg_file_set_sub_stage(CAL_SUBSTAGE_READ_LATENCY);
2571 /* Load up the patterns used by read calibration for all ranks */
2572 rw_mgr_mem_calibrate_read_load_patterns(0, 1);
2575 addr = (u32)&phy_mgr_cfg->phy_rlat;
2577 writel(gbl->curr_read_lat, SOCFPGA_SDR_ADDRESS + addr);
2578 debug_cond(DLEVEL == 2, "%s:%d lfifo: read_lat=%u",
2579 __func__, __LINE__, gbl->curr_read_lat);
2581 if (!rw_mgr_mem_calibrate_read_test_all_ranks(0,
2589 /* reduce read latency and see if things are working */
2591 gbl->curr_read_lat--;
2592 } while (gbl->curr_read_lat > 0);
2594 /* reset the fifos to get pointers to known state */
2596 addr = (u32)&phy_mgr_cmd->fifo_reset;
2597 writel(0, SOCFPGA_SDR_ADDRESS + addr);
2600 /* add a fudge factor to the read latency that was determined */
2601 gbl->curr_read_lat += 2;
2602 addr = (u32)&phy_mgr_cfg->phy_rlat;
2603 writel(gbl->curr_read_lat, SOCFPGA_SDR_ADDRESS + addr);
2604 debug_cond(DLEVEL == 2, "%s:%d lfifo: success: using \
2605 read_lat=%u\n", __func__, __LINE__,
2606 gbl->curr_read_lat);
2609 set_failing_group_stage(0xff, CAL_STAGE_LFIFO,
2610 CAL_SUBSTAGE_READ_LATENCY);
2612 debug_cond(DLEVEL == 2, "%s:%d lfifo: failed at initial \
2613 read_lat=%u\n", __func__, __LINE__,
2614 gbl->curr_read_lat);
2620 * issue write test command.
2621 * two variants are provided. one that just tests a write pattern and
2622 * another that tests datamask functionality.
2624 static void rw_mgr_mem_calibrate_write_test_issue(uint32_t group,
2627 uint32_t mcc_instruction;
2628 uint32_t quick_write_mode = (((STATIC_CALIB_STEPS) & CALIB_SKIP_WRITES) &&
2629 ENABLE_SUPER_QUICK_CALIBRATION);
2630 uint32_t rw_wl_nop_cycles;
2634 * Set counter and jump addresses for the right
2635 * number of NOP cycles.
2636 * The number of supported NOP cycles can range from -1 to infinity
2637 * Three different cases are handled:
2639 * 1. For a number of NOP cycles greater than 0, the RW Mgr looping
2640 * mechanism will be used to insert the right number of NOPs
2642 * 2. For a number of NOP cycles equals to 0, the micro-instruction
2643 * issuing the write command will jump straight to the
2644 * micro-instruction that turns on DQS (for DDRx), or outputs write
2645 * data (for RLD), skipping
2646 * the NOP micro-instruction all together
2648 * 3. A number of NOP cycles equal to -1 indicates that DQS must be
2649 * turned on in the same micro-instruction that issues the write
2650 * command. Then we need
2651 * to directly jump to the micro-instruction that sends out the data
2653 * NOTE: Implementing this mechanism uses 2 RW Mgr jump-counters
2654 * (2 and 3). One jump-counter (0) is used to perform multiple
2655 * write-read operations.
2656 * one counter left to issue this command in "multiple-group" mode
2659 rw_wl_nop_cycles = gbl->rw_wl_nop_cycles;
2661 if (rw_wl_nop_cycles == -1) {
2663 * CNTR 2 - We want to execute the special write operation that
2664 * turns on DQS right away and then skip directly to the
2665 * instruction that sends out the data. We set the counter to a
2666 * large number so that the jump is always taken.
2668 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2;
2669 writel(0xFF, SOCFPGA_SDR_ADDRESS + addr);
2671 /* CNTR 3 - Not used */
2673 mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0_WL_1;
2674 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2;
2675 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_DATA,
2676 SOCFPGA_SDR_ADDRESS + addr);
2677 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3;
2678 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP,
2679 SOCFPGA_SDR_ADDRESS + addr);
2681 mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0_WL_1;
2682 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2;
2683 writel(RW_MGR_LFSR_WR_RD_BANK_0_DATA, SOCFPGA_SDR_ADDRESS + addr);
2684 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3;
2685 writel(RW_MGR_LFSR_WR_RD_BANK_0_NOP, SOCFPGA_SDR_ADDRESS + addr);
2687 } else if (rw_wl_nop_cycles == 0) {
2689 * CNTR 2 - We want to skip the NOP operation and go straight
2690 * to the DQS enable instruction. We set the counter to a large
2691 * number so that the jump is always taken.
2693 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2;
2694 writel(0xFF, SOCFPGA_SDR_ADDRESS + addr);
2696 /* CNTR 3 - Not used */
2698 mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0;
2699 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2;
2700 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_DQS,
2701 SOCFPGA_SDR_ADDRESS + addr);
2703 mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0;
2704 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2;
2705 writel(RW_MGR_LFSR_WR_RD_BANK_0_DQS, SOCFPGA_SDR_ADDRESS + addr);
2709 * CNTR 2 - In this case we want to execute the next instruction
2710 * and NOT take the jump. So we set the counter to 0. The jump
2711 * address doesn't count.
2713 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr2;
2714 writel(0x0, SOCFPGA_SDR_ADDRESS + addr);
2715 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add2;
2716 writel(0x0, SOCFPGA_SDR_ADDRESS + addr);
2719 * CNTR 3 - Set the nop counter to the number of cycles we
2720 * need to loop for, minus 1.
2722 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr3;
2723 writel(rw_wl_nop_cycles - 1, SOCFPGA_SDR_ADDRESS + addr);
2725 mcc_instruction = RW_MGR_LFSR_WR_RD_DM_BANK_0;
2726 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3;
2727 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_NOP, SOCFPGA_SDR_ADDRESS + addr);
2729 mcc_instruction = RW_MGR_LFSR_WR_RD_BANK_0;
2730 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add3;
2731 writel(RW_MGR_LFSR_WR_RD_BANK_0_NOP, SOCFPGA_SDR_ADDRESS + addr);
2735 addr = sdr_get_addr((u32 *)RW_MGR_RESET_READ_DATAPATH);
2736 writel(0, SOCFPGA_SDR_ADDRESS + addr);
2738 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0;
2739 if (quick_write_mode)
2740 writel(0x08, SOCFPGA_SDR_ADDRESS + addr);
2742 writel(0x40, SOCFPGA_SDR_ADDRESS + addr);
2744 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0;
2745 writel(mcc_instruction, SOCFPGA_SDR_ADDRESS + addr);
2748 * CNTR 1 - This is used to ensure enough time elapses
2749 * for read data to come back.
2751 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1;
2752 writel(0x30, SOCFPGA_SDR_ADDRESS + addr);
2754 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1;
2756 writel(RW_MGR_LFSR_WR_RD_DM_BANK_0_WAIT, SOCFPGA_SDR_ADDRESS + addr);
2758 writel(RW_MGR_LFSR_WR_RD_BANK_0_WAIT, SOCFPGA_SDR_ADDRESS + addr);
2761 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
2762 writel(mcc_instruction, SOCFPGA_SDR_ADDRESS + addr + (group << 2));
2765 /* Test writes, can check for a single bit pass or multiple bit pass */
2766 static uint32_t rw_mgr_mem_calibrate_write_test(uint32_t rank_bgn,
2767 uint32_t write_group, uint32_t use_dm, uint32_t all_correct,
2768 uint32_t *bit_chk, uint32_t all_ranks)
2772 uint32_t correct_mask_vg;
2773 uint32_t tmp_bit_chk;
2775 uint32_t rank_end = all_ranks ? RW_MGR_MEM_NUMBER_OF_RANKS :
2776 (rank_bgn + NUM_RANKS_PER_SHADOW_REG);
2777 uint32_t addr_rw_mgr;
2778 uint32_t base_rw_mgr;
2780 *bit_chk = param->write_correct_mask;
2781 correct_mask_vg = param->write_correct_mask_vg;
2783 for (r = rank_bgn; r < rank_end; r++) {
2784 if (param->skip_ranks[r]) {
2785 /* request to skip the rank */
2790 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_READ_WRITE);
2793 addr = (u32)&phy_mgr_cmd->fifo_reset;
2794 addr_rw_mgr = SDR_PHYGRP_RWMGRGRP_ADDRESS;
2795 for (vg = RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS-1; ; vg--) {
2796 /* reset the fifos to get pointers to known state */
2797 writel(0, SOCFPGA_SDR_ADDRESS + addr);
2799 tmp_bit_chk = tmp_bit_chk <<
2800 (RW_MGR_MEM_DQ_PER_WRITE_DQS /
2801 RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS);
2802 rw_mgr_mem_calibrate_write_test_issue(write_group *
2803 RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS+vg,
2806 base_rw_mgr = readl(SOCFPGA_SDR_ADDRESS + addr_rw_mgr);
2807 tmp_bit_chk = tmp_bit_chk | (correct_mask_vg & ~(base_rw_mgr));
2811 *bit_chk &= tmp_bit_chk;
2815 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
2816 debug_cond(DLEVEL == 2, "write_test(%u,%u,ALL) : %u == \
2817 %u => %lu", write_group, use_dm,
2818 *bit_chk, param->write_correct_mask,
2819 (long unsigned int)(*bit_chk ==
2820 param->write_correct_mask));
2821 return *bit_chk == param->write_correct_mask;
2823 set_rank_and_odt_mask(0, RW_MGR_ODT_MODE_OFF);
2824 debug_cond(DLEVEL == 2, "write_test(%u,%u,ONE) : %u != ",
2825 write_group, use_dm, *bit_chk);
2826 debug_cond(DLEVEL == 2, "%lu" " => %lu", (long unsigned int)0,
2827 (long unsigned int)(*bit_chk != 0));
2828 return *bit_chk != 0x00;
2833 * center all windows. do per-bit-deskew to possibly increase size of
2836 static uint32_t rw_mgr_mem_calibrate_writes_center(uint32_t rank_bgn,
2837 uint32_t write_group, uint32_t test_bgn)
2839 uint32_t i, p, min_index;
2842 * Store these as signed since there are comparisons with
2846 uint32_t sticky_bit_chk;
2847 int32_t left_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS];
2848 int32_t right_edge[RW_MGR_MEM_DQ_PER_WRITE_DQS];
2850 int32_t mid_min, orig_mid_min;
2851 int32_t new_dqs, start_dqs, shift_dq;
2852 int32_t dq_margin, dqs_margin, dm_margin;
2854 uint32_t temp_dq_out1_delay;
2857 debug("%s:%d %u %u", __func__, __LINE__, write_group, test_bgn);
2861 addr = sdr_get_addr((u32 *)SCC_MGR_IO_OUT1_DELAY);
2862 start_dqs = readl(SOCFPGA_SDR_ADDRESS + addr +
2863 (RW_MGR_MEM_DQ_PER_WRITE_DQS << 2));
2865 /* per-bit deskew */
2868 * set the left and right edge of each bit to an illegal value
2869 * use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value.
2872 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2873 left_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
2874 right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
2877 /* Search for the left edge of the window for each bit */
2878 addr = (u32)&sdr_scc_mgr->update;
2879 for (d = 0; d <= IO_IO_OUT1_DELAY_MAX; d++) {
2880 scc_mgr_apply_group_dq_out1_delay(write_group, test_bgn, d);
2882 writel(0, SOCFPGA_SDR_ADDRESS + addr);
2885 * Stop searching when the read test doesn't pass AND when
2886 * we've seen a passing read on every bit.
2888 stop = !rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
2889 0, PASS_ONE_BIT, &bit_chk, 0);
2890 sticky_bit_chk = sticky_bit_chk | bit_chk;
2891 stop = stop && (sticky_bit_chk == param->write_correct_mask);
2892 debug_cond(DLEVEL == 2, "write_center(left): dtap=%d => %u \
2893 == %u && %u [bit_chk= %u ]\n",
2894 d, sticky_bit_chk, param->write_correct_mask,
2900 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
2903 * Remember a passing test as the
2909 * If a left edge has not been seen
2910 * yet, then a future passing test will
2911 * mark this edge as the right edge.
2914 IO_IO_OUT1_DELAY_MAX + 1) {
2915 right_edge[i] = -(d + 1);
2918 debug_cond(DLEVEL == 2, "write_center[l,d=%d):", d);
2919 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d",
2920 (int)(bit_chk & 1), i, left_edge[i]);
2921 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
2923 bit_chk = bit_chk >> 1;
2928 /* Reset DQ delay chains to 0 */
2929 scc_mgr_apply_group_dq_out1_delay(write_group, test_bgn, 0);
2931 for (i = RW_MGR_MEM_DQ_PER_WRITE_DQS - 1;; i--) {
2932 debug_cond(DLEVEL == 2, "%s:%d write_center: left_edge[%u]: \
2933 %d right_edge[%u]: %d\n", __func__, __LINE__,
2934 i, left_edge[i], i, right_edge[i]);
2937 * Check for cases where we haven't found the left edge,
2938 * which makes our assignment of the the right edge invalid.
2939 * Reset it to the illegal value.
2941 if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) &&
2942 (right_edge[i] != IO_IO_OUT1_DELAY_MAX + 1)) {
2943 right_edge[i] = IO_IO_OUT1_DELAY_MAX + 1;
2944 debug_cond(DLEVEL == 2, "%s:%d write_center: reset \
2945 right_edge[%u]: %d\n", __func__, __LINE__,
2950 * Reset sticky bit (except for bits where we have
2951 * seen the left edge).
2953 sticky_bit_chk = sticky_bit_chk << 1;
2954 if ((left_edge[i] != IO_IO_OUT1_DELAY_MAX + 1))
2955 sticky_bit_chk = sticky_bit_chk | 1;
2961 /* Search for the right edge of the window for each bit */
2962 addr = (u32)&sdr_scc_mgr->update;
2963 for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - start_dqs; d++) {
2964 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group,
2967 writel(0, SOCFPGA_SDR_ADDRESS + addr);
2970 * Stop searching when the read test doesn't pass AND when
2971 * we've seen a passing read on every bit.
2973 stop = !rw_mgr_mem_calibrate_write_test(rank_bgn, write_group,
2974 0, PASS_ONE_BIT, &bit_chk, 0);
2976 sticky_bit_chk = sticky_bit_chk | bit_chk;
2977 stop = stop && (sticky_bit_chk == param->write_correct_mask);
2979 debug_cond(DLEVEL == 2, "write_center (right): dtap=%u => %u == \
2980 %u && %u\n", d, sticky_bit_chk,
2981 param->write_correct_mask, stop);
2985 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS;
2987 /* d = 0 failed, but it passed when
2988 testing the left edge, so it must be
2989 marginal, set it to -1 */
2990 if (right_edge[i] ==
2991 IO_IO_OUT1_DELAY_MAX + 1 &&
2993 IO_IO_OUT1_DELAY_MAX + 1) {
3000 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
3003 * Remember a passing test as
3010 * If a right edge has not
3011 * been seen yet, then a future
3012 * passing test will mark this
3013 * edge as the left edge.
3015 if (right_edge[i] ==
3016 IO_IO_OUT1_DELAY_MAX + 1)
3017 left_edge[i] = -(d + 1);
3020 * d = 0 failed, but it passed
3021 * when testing the left edge,
3022 * so it must be marginal, set
3025 if (right_edge[i] ==
3026 IO_IO_OUT1_DELAY_MAX + 1 &&
3028 IO_IO_OUT1_DELAY_MAX + 1)
3031 * If a right edge has not been
3032 * seen yet, then a future
3033 * passing test will mark this
3034 * edge as the left edge.
3036 else if (right_edge[i] ==
3037 IO_IO_OUT1_DELAY_MAX +
3039 left_edge[i] = -(d + 1);
3042 debug_cond(DLEVEL == 2, "write_center[r,d=%d):", d);
3043 debug_cond(DLEVEL == 2, "bit_chk_test=%d left_edge[%u]: %d",
3044 (int)(bit_chk & 1), i, left_edge[i]);
3045 debug_cond(DLEVEL == 2, "right_edge[%u]: %d\n", i,
3047 bit_chk = bit_chk >> 1;
3052 /* Check that all bits have a window */
3053 for (i = 0; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
3054 debug_cond(DLEVEL == 2, "%s:%d write_center: left_edge[%u]: \
3055 %d right_edge[%u]: %d", __func__, __LINE__,
3056 i, left_edge[i], i, right_edge[i]);
3057 if ((left_edge[i] == IO_IO_OUT1_DELAY_MAX + 1) ||
3058 (right_edge[i] == IO_IO_OUT1_DELAY_MAX + 1)) {
3059 set_failing_group_stage(test_bgn + i,
3061 CAL_SUBSTAGE_WRITES_CENTER);
3066 /* Find middle of window for each DQ bit */
3067 mid_min = left_edge[0] - right_edge[0];
3069 for (i = 1; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++) {
3070 mid = left_edge[i] - right_edge[i];
3071 if (mid < mid_min) {
3078 * -mid_min/2 represents the amount that we need to move DQS.
3079 * If mid_min is odd and positive we'll need to add one to
3080 * make sure the rounding in further calculations is correct
3081 * (always bias to the right), so just add 1 for all positive values.
3085 mid_min = mid_min / 2;
3086 debug_cond(DLEVEL == 1, "%s:%d write_center: mid_min=%d\n", __func__,
3089 /* Determine the amount we can change DQS (which is -mid_min) */
3090 orig_mid_min = mid_min;
3091 new_dqs = start_dqs;
3093 debug_cond(DLEVEL == 1, "%s:%d write_center: start_dqs=%d new_dqs=%d \
3094 mid_min=%d\n", __func__, __LINE__, start_dqs, new_dqs, mid_min);
3095 /* Initialize data for export structures */
3096 dqs_margin = IO_IO_OUT1_DELAY_MAX + 1;
3097 dq_margin = IO_IO_OUT1_DELAY_MAX + 1;
3099 /* add delay to bring centre of all DQ windows to the same "level" */
3100 addr = sdr_get_addr((u32 *)SCC_MGR_IO_OUT1_DELAY);
3101 for (i = 0, p = test_bgn; i < RW_MGR_MEM_DQ_PER_WRITE_DQS; i++, p++) {
3102 /* Use values before divide by 2 to reduce round off error */
3103 shift_dq = (left_edge[i] - right_edge[i] -
3104 (left_edge[min_index] - right_edge[min_index]))/2 +
3105 (orig_mid_min - mid_min);
3107 debug_cond(DLEVEL == 2, "%s:%d write_center: before: shift_dq \
3108 [%u]=%d\n", __func__, __LINE__, i, shift_dq);
3110 temp_dq_out1_delay = readl(SOCFPGA_SDR_ADDRESS + addr + (i << 2));
3111 if (shift_dq + (int32_t)temp_dq_out1_delay >
3112 (int32_t)IO_IO_OUT1_DELAY_MAX) {
3113 shift_dq = (int32_t)IO_IO_OUT1_DELAY_MAX - temp_dq_out1_delay;
3114 } else if (shift_dq + (int32_t)temp_dq_out1_delay < 0) {
3115 shift_dq = -(int32_t)temp_dq_out1_delay;
3117 debug_cond(DLEVEL == 2, "write_center: after: shift_dq[%u]=%d\n",
3119 scc_mgr_set_dq_out1_delay(write_group, i, temp_dq_out1_delay +
3123 debug_cond(DLEVEL == 2, "write_center: margin[%u]=[%d,%d]\n", i,
3124 left_edge[i] - shift_dq + (-mid_min),
3125 right_edge[i] + shift_dq - (-mid_min));
3126 /* To determine values for export structures */
3127 if (left_edge[i] - shift_dq + (-mid_min) < dq_margin)
3128 dq_margin = left_edge[i] - shift_dq + (-mid_min);
3130 if (right_edge[i] + shift_dq - (-mid_min) < dqs_margin)
3131 dqs_margin = right_edge[i] + shift_dq - (-mid_min);
3135 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs);
3136 addr = (u32)&sdr_scc_mgr->update;
3137 writel(0, SOCFPGA_SDR_ADDRESS + addr);
3140 debug_cond(DLEVEL == 2, "%s:%d write_center: DM\n", __func__, __LINE__);
3143 * set the left and right edge of each bit to an illegal value,
3144 * use (IO_IO_OUT1_DELAY_MAX + 1) as an illegal value,
3146 left_edge[0] = IO_IO_OUT1_DELAY_MAX + 1;
3147 right_edge[0] = IO_IO_OUT1_DELAY_MAX + 1;
3148 int32_t bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
3149 int32_t end_curr = IO_IO_OUT1_DELAY_MAX + 1;
3150 int32_t bgn_best = IO_IO_OUT1_DELAY_MAX + 1;
3151 int32_t end_best = IO_IO_OUT1_DELAY_MAX + 1;
3152 int32_t win_best = 0;
3154 /* Search for the/part of the window with DM shift */
3155 addr = (u32)&sdr_scc_mgr->update;
3156 for (d = IO_IO_OUT1_DELAY_MAX; d >= 0; d -= DELTA_D) {
3157 scc_mgr_apply_group_dm_out1_delay(write_group, d);
3158 writel(0, SOCFPGA_SDR_ADDRESS + addr);
3160 if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1,
3161 PASS_ALL_BITS, &bit_chk,
3163 /* USE Set current end of the window */
3166 * If a starting edge of our window has not been seen
3167 * this is our current start of the DM window.
3169 if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1)
3173 * If current window is bigger than best seen.
3174 * Set best seen to be current window.
3176 if ((end_curr-bgn_curr+1) > win_best) {
3177 win_best = end_curr-bgn_curr+1;
3178 bgn_best = bgn_curr;
3179 end_best = end_curr;
3182 /* We just saw a failing test. Reset temp edge */
3183 bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
3184 end_curr = IO_IO_OUT1_DELAY_MAX + 1;
3189 /* Reset DM delay chains to 0 */
3190 scc_mgr_apply_group_dm_out1_delay(write_group, 0);
3193 * Check to see if the current window nudges up aganist 0 delay.
3194 * If so we need to continue the search by shifting DQS otherwise DQS
3195 * search begins as a new search. */
3196 if (end_curr != 0) {
3197 bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
3198 end_curr = IO_IO_OUT1_DELAY_MAX + 1;
3201 /* Search for the/part of the window with DQS shifts */
3202 addr = (u32)&sdr_scc_mgr->update;
3203 for (d = 0; d <= IO_IO_OUT1_DELAY_MAX - new_dqs; d += DELTA_D) {
3205 * Note: This only shifts DQS, so are we limiting ourselve to
3206 * width of DQ unnecessarily.
3208 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group,
3211 writel(0, SOCFPGA_SDR_ADDRESS + addr);
3212 if (rw_mgr_mem_calibrate_write_test(rank_bgn, write_group, 1,
3213 PASS_ALL_BITS, &bit_chk,
3215 /* USE Set current end of the window */
3218 * If a beginning edge of our window has not been seen
3219 * this is our current begin of the DM window.
3221 if (bgn_curr == IO_IO_OUT1_DELAY_MAX + 1)
3225 * If current window is bigger than best seen. Set best
3226 * seen to be current window.
3228 if ((end_curr-bgn_curr+1) > win_best) {
3229 win_best = end_curr-bgn_curr+1;
3230 bgn_best = bgn_curr;
3231 end_best = end_curr;
3234 /* We just saw a failing test. Reset temp edge */
3235 bgn_curr = IO_IO_OUT1_DELAY_MAX + 1;
3236 end_curr = IO_IO_OUT1_DELAY_MAX + 1;
3238 /* Early exit optimization: if ther remaining delay
3239 chain space is less than already seen largest window
3242 (IO_IO_OUT1_DELAY_MAX - new_dqs - d)) {
3248 /* assign left and right edge for cal and reporting; */
3249 left_edge[0] = -1*bgn_best;
3250 right_edge[0] = end_best;
3252 debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d\n", __func__,
3253 __LINE__, left_edge[0], right_edge[0]);
3255 /* Move DQS (back to orig) */
3256 scc_mgr_apply_group_dqs_io_and_oct_out1(write_group, new_dqs);
3260 /* Find middle of window for the DM bit */
3261 mid = (left_edge[0] - right_edge[0]) / 2;
3263 /* only move right, since we are not moving DQS/DQ */
3267 /* dm_marign should fail if we never find a window */
3271 dm_margin = left_edge[0] - mid;
3273 scc_mgr_apply_group_dm_out1_delay(write_group, mid);
3274 addr = (u32)&sdr_scc_mgr->update;
3275 writel(0, SOCFPGA_SDR_ADDRESS + addr);
3277 debug_cond(DLEVEL == 2, "%s:%d dm_calib: left=%d right=%d mid=%d \
3278 dm_margin=%d\n", __func__, __LINE__, left_edge[0],
3279 right_edge[0], mid, dm_margin);
3281 gbl->fom_out += dq_margin + dqs_margin;
3283 debug_cond(DLEVEL == 2, "%s:%d write_center: dq_margin=%d \
3284 dqs_margin=%d dm_margin=%d\n", __func__, __LINE__,
3285 dq_margin, dqs_margin, dm_margin);
3288 * Do not remove this line as it makes sure all of our
3289 * decisions have been applied.
3291 addr = (u32)&sdr_scc_mgr->update;
3292 writel(0, SOCFPGA_SDR_ADDRESS + addr);
3293 return (dq_margin >= 0) && (dqs_margin >= 0) && (dm_margin >= 0);
3296 /* calibrate the write operations */
3297 static uint32_t rw_mgr_mem_calibrate_writes(uint32_t rank_bgn, uint32_t g,
3300 /* update info for sims */
3301 debug("%s:%d %u %u\n", __func__, __LINE__, g, test_bgn);
3303 reg_file_set_stage(CAL_STAGE_WRITES);
3304 reg_file_set_sub_stage(CAL_SUBSTAGE_WRITES_CENTER);
3306 reg_file_set_group(g);
3308 if (!rw_mgr_mem_calibrate_writes_center(rank_bgn, g, test_bgn)) {
3309 set_failing_group_stage(g, CAL_STAGE_WRITES,
3310 CAL_SUBSTAGE_WRITES_CENTER);
3317 /* precharge all banks and activate row 0 in bank "000..." and bank "111..." */
3318 static void mem_precharge_and_activate(void)
3323 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS; r++) {
3324 if (param->skip_ranks[r]) {
3325 /* request to skip the rank */
3330 set_rank_and_odt_mask(r, RW_MGR_ODT_MODE_OFF);
3332 /* precharge all banks ... */
3333 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
3334 writel(RW_MGR_PRECHARGE_ALL, SOCFPGA_SDR_ADDRESS + addr);
3336 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr0;
3337 writel(0x0F, SOCFPGA_SDR_ADDRESS + addr);
3338 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add0;
3339 writel(RW_MGR_ACTIVATE_0_AND_1_WAIT1, SOCFPGA_SDR_ADDRESS + addr);
3341 addr = (u32)&sdr_rw_load_mgr_regs->load_cntr1;
3342 writel(0x0F, SOCFPGA_SDR_ADDRESS + addr);
3343 addr = (u32)&sdr_rw_load_jump_mgr_regs->load_jump_add1;
3344 writel(RW_MGR_ACTIVATE_0_AND_1_WAIT2, SOCFPGA_SDR_ADDRESS + addr);
3347 addr = sdr_get_addr((u32 *)RW_MGR_RUN_SINGLE_GROUP);
3348 writel(RW_MGR_ACTIVATE_0_AND_1, SOCFPGA_SDR_ADDRESS + addr);
3352 /* Configure various memory related parameters. */
3353 static void mem_config(void)
3355 uint32_t rlat, wlat;
3356 uint32_t rw_wl_nop_cycles;
3357 uint32_t max_latency;
3360 debug("%s:%d\n", __func__, __LINE__);
3361 /* read in write and read latency */
3362 addr = sdr_get_addr(&data_mgr->t_wl_add);
3363 wlat = readl(SOCFPGA_SDR_ADDRESS + addr);
3365 addr = sdr_get_addr(&data_mgr->mem_t_add);
3366 wlat += readl(SOCFPGA_SDR_ADDRESS + addr);
3367 /* WL for hard phy does not include additive latency */
3370 * add addtional write latency to offset the address/command extra
3371 * clock cycle. We change the AC mux setting causing AC to be delayed
3372 * by one mem clock cycle. Only do this for DDR3
3376 addr = sdr_get_addr(&data_mgr->t_rl_add);
3377 rlat = readl(SOCFPGA_SDR_ADDRESS + addr);
3379 rw_wl_nop_cycles = wlat - 2;
3380 gbl->rw_wl_nop_cycles = rw_wl_nop_cycles;
3383 * For AV/CV, lfifo is hardened and always runs at full rate so
3384 * max latency in AFI clocks, used here, is correspondingly smaller.
3386 max_latency = (1<<MAX_LATENCY_COUNT_WIDTH)/1 - 1;
3387 /* configure for a burst length of 8 */
3390 /* Adjust Write Latency for Hard PHY */
3393 /* set a pretty high read latency initially */
3394 gbl->curr_read_lat = rlat + 16;
3396 if (gbl->curr_read_lat > max_latency)
3397 gbl->curr_read_lat = max_latency;
3399 addr = (u32)&phy_mgr_cfg->phy_rlat;
3400 writel(gbl->curr_read_lat, SOCFPGA_SDR_ADDRESS + addr);
3402 /* advertise write latency */
3403 gbl->curr_write_lat = wlat;
3404 addr = (u32)&phy_mgr_cfg->afi_wlat;
3405 writel(wlat - 2, SOCFPGA_SDR_ADDRESS + addr);
3407 /* initialize bit slips */
3408 mem_precharge_and_activate();
3411 /* Set VFIFO and LFIFO to instant-on settings in skip calibration mode */
3412 static void mem_skip_calibrate(void)
3414 uint32_t vfifo_offset;
3418 debug("%s:%d\n", __func__, __LINE__);
3419 /* Need to update every shadow register set used by the interface */
3420 for (r = 0; r < RW_MGR_MEM_NUMBER_OF_RANKS;
3421 r += NUM_RANKS_PER_SHADOW_REG) {
3423 * Set output phase alignment settings appropriate for
3426 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
3427 scc_mgr_set_dqs_en_phase(i, 0);
3428 #if IO_DLL_CHAIN_LENGTH == 6
3429 scc_mgr_set_dqdqs_output_phase(i, 6);
3431 scc_mgr_set_dqdqs_output_phase(i, 7);
3436 * Write data arrives to the I/O two cycles before write
3437 * latency is reached (720 deg).
3438 * -> due to bit-slip in a/c bus
3439 * -> to allow board skew where dqs is longer than ck
3440 * -> how often can this happen!?
3441 * -> can claim back some ptaps for high freq
3442 * support if we can relax this, but i digress...
3444 * The write_clk leads mem_ck by 90 deg
3445 * The minimum ptap of the OPA is 180 deg
3446 * Each ptap has (360 / IO_DLL_CHAIN_LENGH) deg of delay
3447 * The write_clk is always delayed by 2 ptaps
3449 * Hence, to make DQS aligned to CK, we need to delay
3451 * (720 - 90 - 180 - 2 * (360 / IO_DLL_CHAIN_LENGTH))
3453 * Dividing the above by (360 / IO_DLL_CHAIN_LENGTH)
3454 * gives us the number of ptaps, which simplies to:
3456 * (1.25 * IO_DLL_CHAIN_LENGTH - 2)
3458 scc_mgr_set_dqdqs_output_phase(i, (1.25 *
3459 IO_DLL_CHAIN_LENGTH - 2));
3461 addr = (u32)&sdr_scc_mgr->dqs_ena;
3462 writel(0xff, SOCFPGA_SDR_ADDRESS + addr);
3463 addr = (u32)&sdr_scc_mgr->dqs_io_ena;
3464 writel(0xff, SOCFPGA_SDR_ADDRESS + addr);
3466 addr = sdr_get_addr((u32 *)SCC_MGR_GROUP_COUNTER);
3467 for (i = 0; i < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; i++) {
3468 writel(i, SOCFPGA_SDR_ADDRESS + addr);
3470 addr = (u32)&sdr_scc_mgr->dq_ena;
3471 writel(0xff, SOCFPGA_SDR_ADDRESS + addr);
3472 addr = (u32)&sdr_scc_mgr->dm_ena;
3473 writel(0xff, SOCFPGA_SDR_ADDRESS + addr);
3474 addr = (u32)&sdr_scc_mgr->update;
3475 writel(0, SOCFPGA_SDR_ADDRESS + addr);
3478 /* Compensate for simulation model behaviour */
3479 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
3480 scc_mgr_set_dqs_bus_in_delay(i, 10);
3481 scc_mgr_load_dqs(i);
3483 addr = (u32)&sdr_scc_mgr->update;
3484 writel(0, SOCFPGA_SDR_ADDRESS + addr);
3487 * ArriaV has hard FIFOs that can only be initialized by incrementing
3490 vfifo_offset = CALIB_VFIFO_OFFSET;
3491 addr = (u32)&phy_mgr_cmd->inc_vfifo_hard_phy;
3492 for (j = 0; j < vfifo_offset; j++) {
3493 writel(0xff, SOCFPGA_SDR_ADDRESS + addr);
3495 addr = (u32)&phy_mgr_cmd->fifo_reset;
3496 writel(0, SOCFPGA_SDR_ADDRESS + addr);
3499 * For ACV with hard lfifo, we get the skip-cal setting from
3500 * generation-time constant.
3502 gbl->curr_read_lat = CALIB_LFIFO_OFFSET;
3503 addr = (u32)&phy_mgr_cfg->phy_rlat;
3504 writel(gbl->curr_read_lat, SOCFPGA_SDR_ADDRESS + addr);
3507 /* Memory calibration entry point */
3508 static uint32_t mem_calibrate(void)
3511 uint32_t rank_bgn, sr;
3512 uint32_t write_group, write_test_bgn;
3513 uint32_t read_group, read_test_bgn;
3514 uint32_t run_groups, current_run;
3515 uint32_t failing_groups = 0;
3516 uint32_t group_failed = 0;
3517 uint32_t sr_failed = 0;
3520 debug("%s:%d\n", __func__, __LINE__);
3521 /* Initialize the data settings */
3523 gbl->error_substage = CAL_SUBSTAGE_NIL;
3524 gbl->error_stage = CAL_STAGE_NIL;
3525 gbl->error_group = 0xff;
3531 uint32_t bypass_mode = 0x1;
3532 addr = sdr_get_addr((u32 *)SCC_MGR_GROUP_COUNTER);
3533 for (i = 0; i < RW_MGR_MEM_IF_READ_DQS_WIDTH; i++) {
3534 writel(i, SOCFPGA_SDR_ADDRESS + addr);
3535 scc_set_bypass_mode(i, bypass_mode);
3538 if ((dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL) {
3540 * Set VFIFO and LFIFO to instant-on settings in skip
3543 mem_skip_calibrate();
3545 for (i = 0; i < NUM_CALIB_REPEAT; i++) {
3547 * Zero all delay chain/phase settings for all
3548 * groups and all shadow register sets.
3552 run_groups = ~param->skip_groups;
3554 for (write_group = 0, write_test_bgn = 0; write_group
3555 < RW_MGR_MEM_IF_WRITE_DQS_WIDTH; write_group++,
3556 write_test_bgn += RW_MGR_MEM_DQ_PER_WRITE_DQS) {
3557 /* Initialized the group failure */
3560 current_run = run_groups & ((1 <<
3561 RW_MGR_NUM_DQS_PER_WRITE_GROUP) - 1);
3562 run_groups = run_groups >>
3563 RW_MGR_NUM_DQS_PER_WRITE_GROUP;
3565 if (current_run == 0)
3568 addr = sdr_get_addr((u32 *)SCC_MGR_GROUP_COUNTER);
3569 writel(write_group, SOCFPGA_SDR_ADDRESS + addr);
3570 scc_mgr_zero_group(write_group, write_test_bgn,
3573 for (read_group = write_group *
3574 RW_MGR_MEM_IF_READ_DQS_WIDTH /
3575 RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
3577 read_group < (write_group + 1) *
3578 RW_MGR_MEM_IF_READ_DQS_WIDTH /
3579 RW_MGR_MEM_IF_WRITE_DQS_WIDTH &&
3581 read_group++, read_test_bgn +=
3582 RW_MGR_MEM_DQ_PER_READ_DQS) {
3583 /* Calibrate the VFIFO */
3584 if (!((STATIC_CALIB_STEPS) &
3585 CALIB_SKIP_VFIFO)) {
3586 if (!rw_mgr_mem_calibrate_vfifo
3592 phy_debug_mode_flags &
3593 PHY_DEBUG_SWEEP_ALL_GROUPS)) {
3600 /* Calibrate the output side */
3601 if (group_failed == 0) {
3602 for (rank_bgn = 0, sr = 0; rank_bgn
3603 < RW_MGR_MEM_NUMBER_OF_RANKS;
3605 NUM_RANKS_PER_SHADOW_REG,
3608 if (!((STATIC_CALIB_STEPS) &
3609 CALIB_SKIP_WRITES)) {
3610 if ((STATIC_CALIB_STEPS)
3611 & CALIB_SKIP_DELAY_SWEEPS) {
3612 /* not needed in quick mode! */
3615 * Determine if this set of
3616 * ranks should be skipped
3619 if (!param->skip_shadow_regs[sr]) {
3620 if (!rw_mgr_mem_calibrate_writes
3621 (rank_bgn, write_group,
3625 phy_debug_mode_flags &
3626 PHY_DEBUG_SWEEP_ALL_GROUPS)) {
3638 if (group_failed == 0) {
3639 for (read_group = write_group *
3640 RW_MGR_MEM_IF_READ_DQS_WIDTH /
3641 RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
3643 read_group < (write_group + 1)
3644 * RW_MGR_MEM_IF_READ_DQS_WIDTH
3645 / RW_MGR_MEM_IF_WRITE_DQS_WIDTH &&
3647 read_group++, read_test_bgn +=
3648 RW_MGR_MEM_DQ_PER_READ_DQS) {
3649 if (!((STATIC_CALIB_STEPS) &
3650 CALIB_SKIP_WRITES)) {
3651 if (!rw_mgr_mem_calibrate_vfifo_end
3652 (read_group, read_test_bgn)) {
3655 if (!(gbl->phy_debug_mode_flags
3656 & PHY_DEBUG_SWEEP_ALL_GROUPS)) {
3664 if (group_failed != 0)
3669 * USER If there are any failing groups then report
3672 if (failing_groups != 0)
3675 /* Calibrate the LFIFO */
3676 if (!((STATIC_CALIB_STEPS) & CALIB_SKIP_LFIFO)) {
3678 * If we're skipping groups as part of debug,
3679 * don't calibrate LFIFO.
3681 if (param->skip_groups == 0) {
3682 if (!rw_mgr_mem_calibrate_lfifo())
3690 * Do not remove this line as it makes sure all of our decisions
3691 * have been applied.
3693 addr = (u32)&sdr_scc_mgr->update;
3694 writel(0, SOCFPGA_SDR_ADDRESS + addr);
3698 static uint32_t run_mem_calibrate(void)
3701 uint32_t debug_info;
3704 debug("%s:%d\n", __func__, __LINE__);
3706 /* Reset pass/fail status shown on afi_cal_success/fail */
3707 addr = (u32)&phy_mgr_cfg->cal_status;
3708 writel(PHY_MGR_CAL_RESET, SOCFPGA_SDR_ADDRESS + addr);
3710 addr = sdr_get_addr((u32 *)BASE_MMR);
3711 /* stop tracking manger */
3712 uint32_t ctrlcfg = readl(SOCFPGA_SDR_ADDRESS + addr);
3714 addr = sdr_get_addr((u32 *)BASE_MMR);
3715 writel(ctrlcfg & 0xFFBFFFFF, SOCFPGA_SDR_ADDRESS + addr);
3718 rw_mgr_mem_initialize();
3720 pass = mem_calibrate();
3722 mem_precharge_and_activate();
3723 addr = (u32)&phy_mgr_cmd->fifo_reset;
3724 writel(0, SOCFPGA_SDR_ADDRESS + addr);
3728 * Don't return control of the PHY back to AFI when in debug mode.
3730 if ((gbl->phy_debug_mode_flags & PHY_DEBUG_IN_DEBUG_MODE) == 0) {
3731 rw_mgr_mem_handoff();
3733 * In Hard PHY this is a 2-bit control:
3735 * 1: DDIO Mux Select
3737 addr = (u32)&phy_mgr_cfg->mux_sel;
3738 writel(0x2, SOCFPGA_SDR_ADDRESS + addr);
3741 addr = sdr_get_addr((u32 *)BASE_MMR);
3742 writel(ctrlcfg, SOCFPGA_SDR_ADDRESS + addr);
3745 printf("%s: CALIBRATION PASSED\n", __FILE__);
3750 if (gbl->fom_in > 0xff)
3753 if (gbl->fom_out > 0xff)
3754 gbl->fom_out = 0xff;
3756 /* Update the FOM in the register file */
3757 debug_info = gbl->fom_in;
3758 debug_info |= gbl->fom_out << 8;
3759 addr = (u32)&sdr_reg_file->fom;
3760 writel(debug_info, SOCFPGA_SDR_ADDRESS + addr);
3762 addr = (u32)&phy_mgr_cfg->cal_debug_info;
3763 writel(debug_info, SOCFPGA_SDR_ADDRESS + addr);
3764 addr = (u32)&phy_mgr_cfg->cal_status;
3765 writel(PHY_MGR_CAL_SUCCESS, SOCFPGA_SDR_ADDRESS + addr);
3767 printf("%s: CALIBRATION FAILED\n", __FILE__);
3769 debug_info = gbl->error_stage;
3770 debug_info |= gbl->error_substage << 8;
3771 debug_info |= gbl->error_group << 16;
3773 addr = (u32)&sdr_reg_file->failing_stage;
3774 writel(debug_info, SOCFPGA_SDR_ADDRESS + addr);
3775 addr = (u32)&phy_mgr_cfg->cal_debug_info;
3776 writel(debug_info, SOCFPGA_SDR_ADDRESS + addr);
3777 addr = (u32)&phy_mgr_cfg->cal_status;
3778 writel(PHY_MGR_CAL_FAIL, SOCFPGA_SDR_ADDRESS + addr);
3780 /* Update the failing group/stage in the register file */
3781 debug_info = gbl->error_stage;
3782 debug_info |= gbl->error_substage << 8;
3783 debug_info |= gbl->error_group << 16;
3784 addr = (u32)&sdr_reg_file->failing_stage;
3785 writel(debug_info, SOCFPGA_SDR_ADDRESS + addr);
3791 static void hc_initialize_rom_data(void)
3796 addr = sdr_get_addr((u32 *)(RW_MGR_INST_ROM_WRITE));
3797 for (i = 0; i < ARRAY_SIZE(inst_rom_init); i++) {
3798 uint32_t data = inst_rom_init[i];
3799 writel(data, SOCFPGA_SDR_ADDRESS + addr + (i << 2));
3802 addr = sdr_get_addr((u32 *)(RW_MGR_AC_ROM_WRITE));
3803 for (i = 0; i < ARRAY_SIZE(ac_rom_init); i++) {
3804 uint32_t data = ac_rom_init[i];
3805 writel(data, SOCFPGA_SDR_ADDRESS + addr + (i << 2));
3809 static void initialize_reg_file(void)
3813 /* Initialize the register file with the correct data */
3814 addr = (u32)&sdr_reg_file->signature;
3815 writel(REG_FILE_INIT_SEQ_SIGNATURE, SOCFPGA_SDR_ADDRESS + addr);
3817 addr = (u32)&sdr_reg_file->debug_data_addr;
3818 writel(0, SOCFPGA_SDR_ADDRESS + addr);
3820 addr = (u32)&sdr_reg_file->cur_stage;
3821 writel(0, SOCFPGA_SDR_ADDRESS + addr);
3823 addr = (u32)&sdr_reg_file->fom;
3824 writel(0, SOCFPGA_SDR_ADDRESS + addr);
3826 addr = (u32)&sdr_reg_file->failing_stage;
3827 writel(0, SOCFPGA_SDR_ADDRESS + addr);
3829 addr = (u32)&sdr_reg_file->debug1;
3830 writel(0, SOCFPGA_SDR_ADDRESS + addr);
3832 addr = (u32)&sdr_reg_file->debug2;
3833 writel(0, SOCFPGA_SDR_ADDRESS + addr);
3836 static void initialize_hps_phy(void)
3841 * Tracking also gets configured here because it's in the
3844 uint32_t trk_sample_count = 7500;
3845 uint32_t trk_long_idle_sample_count = (10 << 16) | 100;
3847 * Format is number of outer loops in the 16 MSB, sample
3852 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ACDELAYEN_SET(2);
3853 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQDELAYEN_SET(1);
3854 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSDELAYEN_SET(1);
3855 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSLOGICDELAYEN_SET(1);
3856 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_RESETDELAYEN_SET(0);
3857 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_LPDDRDIS_SET(1);
3859 * This field selects the intrinsic latency to RDATA_EN/FULL path.
3860 * 00-bypass, 01- add 5 cycles, 10- add 10 cycles, 11- add 15 cycles.
3862 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ADDLATSEL_SET(0);
3863 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_SET(
3865 addr = sdr_get_addr((u32 *)BASE_MMR);
3866 writel(reg, SOCFPGA_SDR_ADDRESS + addr + SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_OFFSET);
3869 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_SAMPLECOUNT_31_20_SET(
3871 SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_WIDTH);
3872 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_SET(
3873 trk_long_idle_sample_count);
3874 writel(reg, SOCFPGA_SDR_ADDRESS + addr + SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_OFFSET);
3877 reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_2_LONGIDLESAMPLECOUNT_31_20_SET(
3878 trk_long_idle_sample_count >>
3879 SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_WIDTH);
3880 writel(reg, SOCFPGA_SDR_ADDRESS + addr + SDR_CTRLGRP_PHYCTRL_PHYCTRL_2_OFFSET);
3883 static void initialize_tracking(void)
3885 uint32_t concatenated_longidle = 0x0;
3886 uint32_t concatenated_delays = 0x0;
3887 uint32_t concatenated_rw_addr = 0x0;
3888 uint32_t concatenated_refresh = 0x0;
3889 uint32_t trk_sample_count = 7500;
3890 uint32_t dtaps_per_ptap;
3895 * compute usable version of value in case we skip full
3900 while (tmp_delay < IO_DELAY_PER_OPA_TAP) {
3902 tmp_delay += IO_DELAY_PER_DCHAIN_TAP;
3906 concatenated_longidle = concatenated_longidle ^ 10;
3907 /*longidle outer loop */
3908 concatenated_longidle = concatenated_longidle << 16;
3909 concatenated_longidle = concatenated_longidle ^ 100;
3910 /*longidle sample count */
3911 concatenated_delays = concatenated_delays ^ 243;
3912 /* trfc, worst case of 933Mhz 4Gb */
3913 concatenated_delays = concatenated_delays << 8;
3914 concatenated_delays = concatenated_delays ^ 14;
3915 /* trcd, worst case */
3916 concatenated_delays = concatenated_delays << 8;
3917 concatenated_delays = concatenated_delays ^ 10;
3919 concatenated_delays = concatenated_delays << 8;
3920 concatenated_delays = concatenated_delays ^ 4;
3923 concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_IDLE;
3924 concatenated_rw_addr = concatenated_rw_addr << 8;
3925 concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_ACTIVATE_1;
3926 concatenated_rw_addr = concatenated_rw_addr << 8;
3927 concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_SGLE_READ;
3928 concatenated_rw_addr = concatenated_rw_addr << 8;
3929 concatenated_rw_addr = concatenated_rw_addr ^ RW_MGR_PRECHARGE_ALL;
3931 concatenated_refresh = concatenated_refresh ^ RW_MGR_REFRESH_ALL;
3932 concatenated_refresh = concatenated_refresh << 24;
3933 concatenated_refresh = concatenated_refresh ^ 1000; /* trefi */
3935 /* Initialize the register file with the correct data */
3936 addr = (u32)&sdr_reg_file->dtaps_per_ptap;
3937 writel(dtaps_per_ptap, SOCFPGA_SDR_ADDRESS + addr);
3939 addr = (u32)&sdr_reg_file->trk_sample_count;
3940 writel(trk_sample_count, SOCFPGA_SDR_ADDRESS + addr);
3942 addr = (u32)&sdr_reg_file->trk_longidle;
3943 writel(concatenated_longidle, SOCFPGA_SDR_ADDRESS + addr);
3945 addr = (u32)&sdr_reg_file->delays;
3946 writel(concatenated_delays, SOCFPGA_SDR_ADDRESS + addr);
3948 addr = (u32)&sdr_reg_file->trk_rw_mgr_addr;
3949 writel(concatenated_rw_addr, SOCFPGA_SDR_ADDRESS + addr);
3951 addr = (u32)&sdr_reg_file->trk_read_dqs_width;
3952 writel(RW_MGR_MEM_IF_READ_DQS_WIDTH, SOCFPGA_SDR_ADDRESS + addr);
3954 addr = (u32)&sdr_reg_file->trk_rfsh;
3955 writel(concatenated_refresh, SOCFPGA_SDR_ADDRESS + addr);
3958 int sdram_calibration_full(void)
3960 struct param_type my_param;
3961 struct gbl_type my_gbl;
3968 /* Initialize the debug mode flags */
3969 gbl->phy_debug_mode_flags = 0;
3970 /* Set the calibration enabled by default */
3971 gbl->phy_debug_mode_flags |= PHY_DEBUG_ENABLE_CAL_RPT;
3973 * Only sweep all groups (regardless of fail state) by default
3974 * Set enabled read test by default.
3976 #if DISABLE_GUARANTEED_READ
3977 gbl->phy_debug_mode_flags |= PHY_DEBUG_DISABLE_GUARANTEED_READ;
3979 /* Initialize the register file */
3980 initialize_reg_file();
3982 /* Initialize any PHY CSR */
3983 initialize_hps_phy();
3985 scc_mgr_initialize();
3987 initialize_tracking();
3989 /* USER Enable all ranks, groups */
3990 for (i = 0; i < RW_MGR_MEM_NUMBER_OF_RANKS; i++)
3991 param->skip_ranks[i] = 0;
3992 for (i = 0; i < NUM_SHADOW_REGS; ++i)
3993 param->skip_shadow_regs[i] = 0;
3994 param->skip_groups = 0;
3996 printf("%s: Preparing to start memory calibration\n", __FILE__);
3998 debug("%s:%d\n", __func__, __LINE__);
3999 debug_cond(DLEVEL == 1,
4000 "DDR3 FULL_RATE ranks=%u cs/dimm=%u dq/dqs=%u,%u vg/dqs=%u,%u ",
4001 RW_MGR_MEM_NUMBER_OF_RANKS, RW_MGR_MEM_NUMBER_OF_CS_PER_DIMM,
4002 RW_MGR_MEM_DQ_PER_READ_DQS, RW_MGR_MEM_DQ_PER_WRITE_DQS,
4003 RW_MGR_MEM_VIRTUAL_GROUPS_PER_READ_DQS,
4004 RW_MGR_MEM_VIRTUAL_GROUPS_PER_WRITE_DQS);
4005 debug_cond(DLEVEL == 1,
4006 "dqs=%u,%u dq=%u dm=%u ptap_delay=%u dtap_delay=%u ",
4007 RW_MGR_MEM_IF_READ_DQS_WIDTH, RW_MGR_MEM_IF_WRITE_DQS_WIDTH,
4008 RW_MGR_MEM_DATA_WIDTH, RW_MGR_MEM_DATA_MASK_WIDTH,
4009 IO_DELAY_PER_OPA_TAP, IO_DELAY_PER_DCHAIN_TAP);
4010 debug_cond(DLEVEL == 1, "dtap_dqsen_delay=%u, dll=%u",
4011 IO_DELAY_PER_DQS_EN_DCHAIN_TAP, IO_DLL_CHAIN_LENGTH);
4012 debug_cond(DLEVEL == 1, "max values: en_p=%u dqdqs_p=%u en_d=%u dqs_in_d=%u ",
4013 IO_DQS_EN_PHASE_MAX, IO_DQDQS_OUT_PHASE_MAX,
4014 IO_DQS_EN_DELAY_MAX, IO_DQS_IN_DELAY_MAX);
4015 debug_cond(DLEVEL == 1, "io_in_d=%u io_out1_d=%u io_out2_d=%u ",
4016 IO_IO_IN_DELAY_MAX, IO_IO_OUT1_DELAY_MAX,
4017 IO_IO_OUT2_DELAY_MAX);
4018 debug_cond(DLEVEL == 1, "dqs_in_reserve=%u dqs_out_reserve=%u\n",
4019 IO_DQS_IN_RESERVE, IO_DQS_OUT_RESERVE);
4021 hc_initialize_rom_data();
4023 /* update info for sims */
4024 reg_file_set_stage(CAL_STAGE_NIL);
4025 reg_file_set_group(0);
4028 * Load global needed for those actions that require
4029 * some dynamic calibration support.
4031 dyn_calib_steps = STATIC_CALIB_STEPS;
4033 * Load global to allow dynamic selection of delay loop settings
4034 * based on calibration mode.
4036 if (!(dyn_calib_steps & CALIB_SKIP_DELAY_LOOPS))
4037 skip_delay_mask = 0xff;
4039 skip_delay_mask = 0x0;
4041 pass = run_mem_calibrate();
4043 printf("%s: Calibration complete\n", __FILE__);