+}
+
+/**
+ * rw_mgr_mem_calibrate_vfifo_center() - Per-bit deskew DQ and centering
+ * @rank_bgn: Rank number
+ * @rw_group: Read/Write Group
+ * @test_bgn: Rank at which the test begins
+ * @use_read_test: Perform a read test
+ * @update_fom: Update FOM
+ *
+ * Per-bit deskew DQ and centering.
+ */
+static int rw_mgr_mem_calibrate_vfifo_center(const u32 rank_bgn,
+ const u32 rw_group, const u32 test_bgn,
+ const int use_read_test, const int update_fom)
+{
+ const u32 addr =
+ SDR_PHYGRP_SCCGRP_ADDRESS + SCC_MGR_DQS_IN_DELAY_OFFSET +
+ (rw_group << 2);
+ /*
+ * Store these as signed since there are comparisons with
+ * signed numbers.
+ */
+ uint32_t sticky_bit_chk;
+ int32_t left_edge[rwcfg->mem_dq_per_read_dqs];
+ int32_t right_edge[rwcfg->mem_dq_per_read_dqs];
+ int32_t orig_mid_min, mid_min;
+ int32_t new_dqs, start_dqs, start_dqs_en = 0, final_dqs_en;
+ int32_t dq_margin, dqs_margin;
+ int i, min_index;
+ int ret;
+
+ debug("%s:%d: %u %u", __func__, __LINE__, rw_group, test_bgn);
+
+ start_dqs = readl(addr);
+ if (iocfg->shift_dqs_en_when_shift_dqs)
+ start_dqs_en = readl(addr - iocfg->dqs_en_delay_offset);
+
+ /* set the left and right edge of each bit to an illegal value */
+ /* use (iocfg->io_in_delay_max + 1) as an illegal value */
+ sticky_bit_chk = 0;
+ for (i = 0; i < rwcfg->mem_dq_per_read_dqs; i++) {
+ left_edge[i] = iocfg->io_in_delay_max + 1;
+ right_edge[i] = iocfg->io_in_delay_max + 1;
+ }
+
+ /* Search for the left edge of the window for each bit */
+ search_left_edge(0, rank_bgn, rw_group, rw_group, test_bgn,
+ &sticky_bit_chk,
+ left_edge, right_edge, use_read_test);
+
+
+ /* Search for the right edge of the window for each bit */
+ ret = search_right_edge(0, rank_bgn, rw_group, rw_group,
+ start_dqs, start_dqs_en,
+ &sticky_bit_chk,
+ left_edge, right_edge, use_read_test);
+ if (ret) {
+ /*
+ * Restore delay chain settings before letting the loop
+ * in rw_mgr_mem_calibrate_vfifo to retry different
+ * dqs/ck relationships.
+ */
+ scc_mgr_set_dqs_bus_in_delay(rw_group, start_dqs);
+ if (iocfg->shift_dqs_en_when_shift_dqs)
+ scc_mgr_set_dqs_en_delay(rw_group, start_dqs_en);
+
+ scc_mgr_load_dqs(rw_group);
+ writel(0, &sdr_scc_mgr->update);
+
+ debug_cond(DLEVEL == 1,
+ "%s:%d vfifo_center: failed to find edge [%u]: %d %d",
+ __func__, __LINE__, i, left_edge[i], right_edge[i]);
+ if (use_read_test) {
+ set_failing_group_stage(rw_group *
+ rwcfg->mem_dq_per_read_dqs + i,
+ CAL_STAGE_VFIFO,
+ CAL_SUBSTAGE_VFIFO_CENTER);
+ } else {
+ set_failing_group_stage(rw_group *
+ rwcfg->mem_dq_per_read_dqs + i,
+ CAL_STAGE_VFIFO_AFTER_WRITES,
+ CAL_SUBSTAGE_VFIFO_CENTER);
+ }
+ return -EIO;
+ }
+
+ min_index = get_window_mid_index(0, left_edge, right_edge, &mid_min);
+
+ /* Determine the amount we can change DQS (which is -mid_min) */
+ orig_mid_min = mid_min;
+ new_dqs = start_dqs - mid_min;
+ if (new_dqs > iocfg->dqs_in_delay_max)
+ new_dqs = iocfg->dqs_in_delay_max;
+ else if (new_dqs < 0)
+ new_dqs = 0;
+
+ mid_min = start_dqs - new_dqs;
+ debug_cond(DLEVEL == 1, "vfifo_center: new mid_min=%d new_dqs=%d\n",
+ mid_min, new_dqs);
+
+ if (iocfg->shift_dqs_en_when_shift_dqs) {
+ if (start_dqs_en - mid_min > iocfg->dqs_en_delay_max)
+ mid_min += start_dqs_en - mid_min - iocfg->dqs_en_delay_max;
+ else if (start_dqs_en - mid_min < 0)
+ mid_min += start_dqs_en - mid_min;
+ }
+ new_dqs = start_dqs - mid_min;
+
+ debug_cond(DLEVEL == 1,
+ "vfifo_center: start_dqs=%d start_dqs_en=%d new_dqs=%d mid_min=%d\n",
+ start_dqs,
+ iocfg->shift_dqs_en_when_shift_dqs ? start_dqs_en : -1,
+ new_dqs, mid_min);
+
+ /* Add delay to bring centre of all DQ windows to the same "level". */
+ center_dq_windows(0, left_edge, right_edge, mid_min, orig_mid_min,
+ min_index, test_bgn, &dq_margin, &dqs_margin);