2 * Copyright (C) Marvell International Ltd. and its affiliates
4 * SPDX-License-Identifier: GPL-2.0
10 #include <asm/arch/cpu.h>
11 #include <asm/arch/soc.h>
13 #include "ddr3_init.h"
15 #define VALIDATE_WIN_LENGTH(e1, e2, maxsize) \
16 (((e2) + 1 > (e1) + (u8)MIN_WINDOW_SIZE) && \
17 ((e2) + 1 < (e1) + (u8)maxsize))
18 #define IS_WINDOW_OUT_BOUNDARY(e1, e2, maxsize) \
19 (((e1) == 0 && (e2) != 0) || \
20 ((e1) != (maxsize - 1) && (e2) == (maxsize - 1)))
23 #define NUM_OF_CENTRAL_TYPES 2
25 u32 start_pattern = PATTERN_KILLER_DQ0, end_pattern = PATTERN_KILLER_DQ7;
26 u32 start_if = 0, end_if = (MAX_INTERFACE_NUM - 1);
27 u8 bus_end_window[NUM_OF_CENTRAL_TYPES][MAX_INTERFACE_NUM][MAX_BUS_NUM];
28 u8 bus_start_window[NUM_OF_CENTRAL_TYPES][MAX_INTERFACE_NUM][MAX_BUS_NUM];
29 u8 centralization_state[MAX_INTERFACE_NUM][MAX_BUS_NUM];
30 static u8 ddr3_tip_special_rx_run_once_flag;
32 static int ddr3_tip_centralization(u32 dev_num, u32 mode);
35 * Centralization RX Flow
37 int ddr3_tip_centralization_rx(u32 dev_num)
39 CHECK_STATUS(ddr3_tip_special_rx(dev_num));
40 CHECK_STATUS(ddr3_tip_centralization(dev_num, CENTRAL_RX));
46 * Centralization TX Flow
48 int ddr3_tip_centralization_tx(u32 dev_num)
50 CHECK_STATUS(ddr3_tip_centralization(dev_num, CENTRAL_TX));
58 static int ddr3_tip_centralization(u32 dev_num, u32 mode)
60 enum hws_training_ip_stat training_result[MAX_INTERFACE_NUM];
61 u32 if_id, pattern_id, bit_id;
63 u8 cur_start_win[BUS_WIDTH_IN_BITS];
64 u8 centralization_result[MAX_INTERFACE_NUM][BUS_WIDTH_IN_BITS];
65 u8 cur_end_win[BUS_WIDTH_IN_BITS];
66 u8 current_window[BUS_WIDTH_IN_BITS];
67 u8 opt_window, waste_window, start_window_skew, end_window_skew;
68 u8 final_pup_window[MAX_INTERFACE_NUM][BUS_WIDTH_IN_BITS];
69 struct hws_topology_map *tm = ddr3_get_topology_map();
70 enum hws_training_result result_type = RESULT_PER_BIT;
71 enum hws_dir direction;
72 u32 *result[HWS_SEARCH_DIR_LIMIT];
76 u8 cur_end_win_min, cur_start_win_max;
77 u32 cs_enable_reg_val[MAX_INTERFACE_NUM];
79 enum hws_result *flow_result = ddr3_tip_get_result_ptr(training_stage);
80 u32 pup_win_length = 0;
81 enum hws_search_dir search_dir_id;
82 u8 cons_tap = (mode == CENTRAL_TX) ? (64) : (0);
84 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
85 VALIDATE_ACTIVE(tm->if_act_mask, if_id);
86 /* save current cs enable reg val */
87 CHECK_STATUS(ddr3_tip_if_read
88 (dev_num, ACCESS_TYPE_UNICAST, if_id,
89 CS_ENABLE_REG, cs_enable_reg_val, MASK_ALL_BITS));
90 /* enable single cs */
91 CHECK_STATUS(ddr3_tip_if_write
92 (dev_num, ACCESS_TYPE_UNICAST, if_id,
93 CS_ENABLE_REG, (1 << 3), (1 << 3)));
96 if (mode == CENTRAL_TX) {
97 max_win_size = MAX_WINDOW_SIZE_TX;
98 reg_phy_off = WRITE_CENTRALIZATION_PHY_REG + (effective_cs * 4);
99 direction = OPER_WRITE;
101 max_win_size = MAX_WINDOW_SIZE_RX;
102 reg_phy_off = READ_CENTRALIZATION_PHY_REG + (effective_cs * 4);
103 direction = OPER_READ;
106 /* DB initialization */
107 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
108 VALIDATE_ACTIVE(tm->if_act_mask, if_id);
110 bus_id < tm->num_of_bus_per_interface; bus_id++) {
111 VALIDATE_ACTIVE(tm->bus_act_mask, bus_id);
112 centralization_state[if_id][bus_id] = 0;
113 bus_end_window[mode][if_id][bus_id] =
114 (max_win_size - 1) + cons_tap;
115 bus_start_window[mode][if_id][bus_id] = 0;
116 centralization_result[if_id][bus_id] = 0;
121 for (pattern_id = start_pattern; pattern_id <= end_pattern;
123 ddr3_tip_ip_training_wrapper(dev_num, ACCESS_TYPE_MULTICAST,
125 ACCESS_TYPE_MULTICAST,
126 PARAM_NOT_CARE, result_type,
127 HWS_CONTROL_ELEMENT_ADLL,
128 PARAM_NOT_CARE, direction,
133 pattern_id, EDGE_FPF, CS_SINGLE,
134 PARAM_NOT_CARE, training_result);
136 for (if_id = start_if; if_id <= end_if; if_id++) {
137 VALIDATE_ACTIVE(tm->if_act_mask, if_id);
139 bus_id <= tm->num_of_bus_per_interface - 1;
141 VALIDATE_ACTIVE(tm->bus_act_mask, bus_id);
143 for (search_dir_id = HWS_LOW2HIGH;
144 search_dir_id <= HWS_HIGH2LOW;
147 (ddr3_tip_read_training_result
149 ACCESS_TYPE_UNICAST, bus_id,
152 direction, result_type,
153 TRAINING_LOAD_OPERATION_UNLOAD,
155 &result[search_dir_id],
157 DEBUG_CENTRALIZATION_ENGINE
159 ("%s pat %d IF %d pup %d Regs: 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
161 CENTRAL_TX) ? "TX" : "RX"),
162 pattern_id, if_id, bus_id,
163 result[search_dir_id][0],
164 result[search_dir_id][1],
165 result[search_dir_id][2],
166 result[search_dir_id][3],
167 result[search_dir_id][4],
168 result[search_dir_id][5],
169 result[search_dir_id][6],
170 result[search_dir_id][7]));
173 for (bit_id = 0; bit_id < BUS_WIDTH_IN_BITS;
175 /* check if this code is valid for 2 edge, probably not :( */
176 cur_start_win[bit_id] =
177 GET_TAP_RESULT(result
181 cur_end_win[bit_id] =
182 GET_TAP_RESULT(result
187 current_window[bit_id] =
188 cur_end_win[bit_id] -
189 cur_start_win[bit_id] + 1;
190 DEBUG_CENTRALIZATION_ENGINE
192 ("cs %x patern %d IF %d pup %d cur_start_win %d cur_end_win %d current_window %d\n",
193 effective_cs, pattern_id,
195 cur_start_win[bit_id],
197 current_window[bit_id]));
200 if ((ddr3_tip_is_pup_lock
201 (result[HWS_LOW2HIGH], result_type)) &&
202 (ddr3_tip_is_pup_lock
203 (result[HWS_HIGH2LOW], result_type))) {
204 /* read result success */
205 DEBUG_CENTRALIZATION_ENGINE
207 ("Pup locked, pat %d IF %d pup %d\n",
208 pattern_id, if_id, bus_id));
210 /* read result failure */
211 DEBUG_CENTRALIZATION_ENGINE
213 ("fail Lock, pat %d IF %d pup %d\n",
214 pattern_id, if_id, bus_id));
215 if (centralization_state[if_id][bus_id]
217 /* continue with next pup */
218 DEBUG_CENTRALIZATION_ENGINE
220 ("continue to next pup %d %d\n",
226 bit_id < BUS_WIDTH_IN_BITS;
229 * the next check is relevant
230 * only when using search
233 if (cur_start_win[bit_id] > 0 &&
234 cur_end_win[bit_id] == 0) {
238 DEBUG_CENTRALIZATION_ENGINE
240 ("fail, IF %d pup %d bit %d fail #1\n",
248 DEBUG_CENTRALIZATION_ENGINE
250 ("fail, IF %d pup %d bit %d fail #2\n",
256 if (centralization_state[if_id][bus_id]
258 /* going to next pup */
264 ddr3_tip_get_buf_min(current_window);
265 /* final pup window length */
266 final_pup_window[if_id][bus_id] =
267 ddr3_tip_get_buf_min(cur_end_win) -
268 ddr3_tip_get_buf_max(cur_start_win) +
272 final_pup_window[if_id][bus_id];
274 ddr3_tip_get_buf_max(cur_start_win) -
275 ddr3_tip_get_buf_min(
278 ddr3_tip_get_buf_max(
280 ddr3_tip_get_buf_min(
282 /* min/max updated with pattern change */
284 ddr3_tip_get_buf_min(
287 ddr3_tip_get_buf_max(
289 bus_end_window[mode][if_id][bus_id] =
290 GET_MIN(bus_end_window[mode][if_id]
293 bus_start_window[mode][if_id][bus_id] =
294 GET_MAX(bus_start_window[mode][if_id]
297 DEBUG_CENTRALIZATION_ENGINE(
299 ("pat %d IF %d pup %d opt_win %d final_win %d waste_win %d st_win_skew %d end_win_skew %d cur_st_win_max %d cur_end_win_min %d bus_st_win %d bus_end_win %d\n",
300 pattern_id, if_id, bus_id, opt_window,
301 final_pup_window[if_id][bus_id],
302 waste_window, start_window_skew,
306 bus_start_window[mode][if_id][bus_id],
307 bus_end_window[mode][if_id][bus_id]));
309 /* check if window is valid */
310 if (ddr3_tip_centr_skip_min_win_check == 0) {
311 if ((VALIDATE_WIN_LENGTH
312 (bus_start_window[mode][if_id]
314 bus_end_window[mode][if_id]
316 max_win_size) == 1) ||
317 (IS_WINDOW_OUT_BOUNDARY
318 (bus_start_window[mode][if_id]
320 bus_end_window[mode][if_id]
322 max_win_size) == 1)) {
323 DEBUG_CENTRALIZATION_ENGINE
325 ("win valid, pat %d IF %d pup %d\n",
328 /* window is valid */
330 DEBUG_CENTRALIZATION_ENGINE
332 ("fail win, pat %d IF %d pup %d bus_st_win %d bus_end_win %d\n",
333 pattern_id, if_id, bus_id,
334 bus_start_window[mode]
338 centralization_state[if_id]
343 } /* ddr3_tip_centr_skip_min_win_check */
348 for (if_id = start_if; if_id <= end_if; if_id++) {
349 if (IS_ACTIVE(tm->if_act_mask, if_id) == 0)
353 flow_result[if_id] = TEST_SUCCESS;
356 bus_id <= (tm->num_of_bus_per_interface - 1); bus_id++) {
357 VALIDATE_ACTIVE(tm->bus_act_mask, bus_id);
359 /* continue only if lock */
360 if (centralization_state[if_id][bus_id] != 1) {
361 if (ddr3_tip_centr_skip_min_win_check == 0) {
363 [mode][if_id][bus_id] ==
364 (max_win_size - 1)) &&
366 [mode][if_id][bus_id] -
367 bus_start_window[mode][if_id]
368 [bus_id]) < MIN_WINDOW_SIZE) &&
369 ((bus_end_window[mode][if_id]
370 [bus_id] - bus_start_window
371 [mode][if_id][bus_id]) > 2)) {
372 /* prevent false lock */
373 /* TBD change to enum */
378 if ((bus_end_window[mode][if_id][bus_id]
380 ((bus_end_window[mode][if_id]
382 bus_start_window[mode][if_id]
383 [bus_id]) < MIN_WINDOW_SIZE) &&
384 ((bus_end_window[mode][if_id]
386 bus_start_window[mode][if_id]
388 /*prevent false lock */
389 centralization_state[if_id]
393 if ((bus_end_window[mode][if_id][bus_id] >
394 (max_win_size - 1)) && direction ==
396 DEBUG_CENTRALIZATION_ENGINE
398 ("Tx special pattern\n"));
404 if (centralization_state[if_id][bus_id] == 3) {
405 DEBUG_CENTRALIZATION_ENGINE(
407 ("SSW - TBD IF %d pup %d\n",
410 } else if (centralization_state[if_id][bus_id] == 2) {
411 DEBUG_CENTRALIZATION_ENGINE(
413 ("SEW - TBD IF %d pup %d\n",
416 } else if (centralization_state[if_id][bus_id] == 0) {
419 DEBUG_CENTRALIZATION_ENGINE(
421 ("fail, IF %d pup %d\n",
426 if (lock_success == 1) {
427 centralization_result[if_id][bus_id] =
428 (bus_end_window[mode][if_id][bus_id] +
429 bus_start_window[mode][if_id][bus_id])
431 DEBUG_CENTRALIZATION_ENGINE(
433 (" bus_id %d Res= %d\n", bus_id,
434 centralization_result[if_id][bus_id]));
435 /* copy results to registers */
437 bus_end_window[mode][if_id][bus_id] -
438 bus_start_window[mode][if_id][bus_id] +
441 ddr3_tip_bus_read(dev_num, if_id,
442 ACCESS_TYPE_UNICAST, bus_id,
444 RESULT_DB_PHY_REG_ADDR +
446 reg = (reg & (~0x1f <<
447 ((mode == CENTRAL_TX) ?
448 (RESULT_DB_PHY_REG_TX_OFFSET) :
449 (RESULT_DB_PHY_REG_RX_OFFSET))))
451 ((mode == CENTRAL_TX) ?
452 (RESULT_DB_PHY_REG_TX_OFFSET) :
453 (RESULT_DB_PHY_REG_RX_OFFSET));
454 CHECK_STATUS(ddr3_tip_bus_write
455 (dev_num, ACCESS_TYPE_UNICAST,
456 if_id, ACCESS_TYPE_UNICAST,
457 bus_id, DDR_PHY_DATA,
458 RESULT_DB_PHY_REG_ADDR +
461 /* offset per CS is calculated earlier */
463 ddr3_tip_bus_write(dev_num,
470 centralization_result
479 flow_result[if_id] = TEST_FAILED;
482 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
483 /* restore cs enable value */
484 VALIDATE_ACTIVE(tm->if_act_mask, if_id);
485 CHECK_STATUS(ddr3_tip_if_write(dev_num, ACCESS_TYPE_UNICAST,
486 if_id, CS_ENABLE_REG,
487 cs_enable_reg_val[if_id],
495 * Centralization Flow
497 int ddr3_tip_special_rx(u32 dev_num)
499 enum hws_training_ip_stat training_result[MAX_INTERFACE_NUM];
500 u32 if_id, pup_id, pattern_id, bit_id;
501 u8 cur_start_win[BUS_WIDTH_IN_BITS];
502 u8 cur_end_win[BUS_WIDTH_IN_BITS];
503 enum hws_training_result result_type = RESULT_PER_BIT;
504 enum hws_dir direction;
505 enum hws_search_dir search_dir_id;
506 u32 *result[HWS_SEARCH_DIR_LIMIT];
508 u8 cur_end_win_min, cur_start_win_max;
509 u32 cs_enable_reg_val[MAX_INTERFACE_NUM];
512 struct hws_topology_map *tm = ddr3_get_topology_map();
514 if (ddr3_tip_special_rx_run_once_flag != 0)
517 ddr3_tip_special_rx_run_once_flag = 1;
519 for (if_id = 0; if_id < MAX_INTERFACE_NUM; if_id++) {
520 VALIDATE_ACTIVE(tm->if_act_mask, if_id);
521 /* save current cs enable reg val */
522 CHECK_STATUS(ddr3_tip_if_read(dev_num, ACCESS_TYPE_UNICAST,
523 if_id, CS_ENABLE_REG,
526 /* enable single cs */
527 CHECK_STATUS(ddr3_tip_if_write(dev_num, ACCESS_TYPE_UNICAST,
528 if_id, CS_ENABLE_REG,
529 (1 << 3), (1 << 3)));
532 max_win_size = MAX_WINDOW_SIZE_RX;
533 direction = OPER_READ;
534 pattern_id = PATTERN_VREF;
537 ddr3_tip_ip_training_wrapper(dev_num, ACCESS_TYPE_MULTICAST,
538 PARAM_NOT_CARE, ACCESS_TYPE_MULTICAST,
539 PARAM_NOT_CARE, result_type,
540 HWS_CONTROL_ELEMENT_ADLL,
541 PARAM_NOT_CARE, direction,
542 tm->if_act_mask, 0x0,
543 max_win_size - 1, max_win_size - 1,
544 pattern_id, EDGE_FPF, CS_SINGLE,
545 PARAM_NOT_CARE, training_result);
547 for (if_id = start_if; if_id <= end_if; if_id++) {
548 VALIDATE_ACTIVE(tm->if_act_mask, if_id);
550 pup_id <= tm->num_of_bus_per_interface; pup_id++) {
551 VALIDATE_ACTIVE(tm->bus_act_mask, pup_id);
553 for (search_dir_id = HWS_LOW2HIGH;
554 search_dir_id <= HWS_HIGH2LOW;
556 CHECK_STATUS(ddr3_tip_read_training_result
558 ACCESS_TYPE_UNICAST, pup_id,
559 ALL_BITS_PER_PUP, search_dir_id,
560 direction, result_type,
561 TRAINING_LOAD_OPERATION_UNLOAD,
562 CS_SINGLE, &result[search_dir_id],
564 DEBUG_CENTRALIZATION_ENGINE(DEBUG_LEVEL_INFO,
565 ("Special: pat %d IF %d pup %d Regs: 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
587 for (bit_id = 0; bit_id < BUS_WIDTH_IN_BITS; bit_id++) {
589 * check if this code is valid for 2 edge,
592 cur_start_win[bit_id] =
593 GET_TAP_RESULT(result[HWS_LOW2HIGH]
595 cur_end_win[bit_id] =
596 GET_TAP_RESULT(result[HWS_HIGH2LOW]
599 if (!((ddr3_tip_is_pup_lock
600 (result[HWS_LOW2HIGH], result_type)) &&
601 (ddr3_tip_is_pup_lock
602 (result[HWS_HIGH2LOW], result_type)))) {
603 DEBUG_CENTRALIZATION_ENGINE(
605 ("Special: Pup lock fail, pat %d IF %d pup %d\n",
606 pattern_id, if_id, pup_id));
611 ddr3_tip_get_buf_min(cur_end_win);
613 ddr3_tip_get_buf_max(cur_start_win);
615 if (cur_start_win_max <= 1) { /* Align left */
616 for (bit_id = 0; bit_id < BUS_WIDTH_IN_BITS;
619 dq_map_table[bit_id +
625 num_of_bus_per_interface];
626 CHECK_STATUS(ddr3_tip_bus_read
629 pup_id, DDR_PHY_DATA,
630 PBS_RX_PHY_REG + pad_num,
632 temp = (temp + 0xa > 31) ?
634 CHECK_STATUS(ddr3_tip_bus_write
639 pup_id, DDR_PHY_DATA,
640 PBS_RX_PHY_REG + pad_num,
643 DEBUG_CENTRALIZATION_ENGINE(
645 ("Special: PBS:: I/F# %d , Bus# %d fix align to the Left\n",
649 if (cur_end_win_min > 30) { /* Align right */
650 CHECK_STATUS(ddr3_tip_bus_read
652 ACCESS_TYPE_UNICAST, pup_id,
653 DDR_PHY_DATA, PBS_RX_PHY_REG + 4,
656 CHECK_STATUS(ddr3_tip_bus_write
657 (dev_num, ACCESS_TYPE_UNICAST,
658 if_id, ACCESS_TYPE_UNICAST,
659 pup_id, DDR_PHY_DATA,
660 PBS_RX_PHY_REG + 4, temp));
661 CHECK_STATUS(ddr3_tip_bus_read
663 ACCESS_TYPE_UNICAST, pup_id,
664 DDR_PHY_DATA, PBS_RX_PHY_REG + 5,
667 CHECK_STATUS(ddr3_tip_bus_write
668 (dev_num, ACCESS_TYPE_UNICAST,
669 if_id, ACCESS_TYPE_UNICAST,
670 pup_id, DDR_PHY_DATA,
671 PBS_RX_PHY_REG + 5, temp));
672 DEBUG_CENTRALIZATION_ENGINE(
674 ("Special: PBS:: I/F# %d , Bus# %d fix align to the right\n",
678 vref_window_size[if_id][pup_id] =
680 cur_start_win_max + 1;
681 DEBUG_CENTRALIZATION_ENGINE(
683 ("Special: Winsize I/F# %d , Bus# %d is %d\n",
684 if_id, pup_id, vref_window_size
687 } /* end of interface */
693 * Print Centralization Result
695 int ddr3_tip_print_centralization_result(u32 dev_num)
697 u32 if_id = 0, bus_id = 0;
698 struct hws_topology_map *tm = ddr3_get_topology_map();
700 printf("Centralization Results\n");
701 printf("I/F0 Result[0 - success 1-fail 2 - state_2 3 - state_3] ...\n");
702 for (if_id = 0; if_id <= MAX_INTERFACE_NUM - 1; if_id++) {
703 VALIDATE_ACTIVE(tm->if_act_mask, if_id);
704 for (bus_id = 0; bus_id < tm->num_of_bus_per_interface;
706 VALIDATE_ACTIVE(tm->bus_act_mask, bus_id);
707 printf("%d ,\n", centralization_state[if_id][bus_id]);