+/**
+ * sdr_backup_phase() - Find DQS enable backup phase
+ * @grp: Read/Write group
+ * @work_bgn: Working window start position
+ * @p: DQS Phase Iterator
+ *
+ * Find DQS enable backup phase setting.
+ */
+static void sdr_backup_phase(const u32 grp, u32 *work_bgn, u32 *p)
+{
+ u32 tmp_delay, d;
+ int ret;
+
+ /* Special case code for backing up a phase */
+ if (*p == 0) {
+ *p = IO_DQS_EN_PHASE_MAX;
+ rw_mgr_decr_vfifo(grp);
+ } else {
+ (*p)--;
+ }
+ tmp_delay = *work_bgn - IO_DELAY_PER_OPA_TAP;
+ scc_mgr_set_dqs_en_phase_all_ranks(grp, *p);
+
+ for (d = 0; d <= IO_DQS_EN_DELAY_MAX && tmp_delay < *work_bgn; d++) {
+ scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
+
+ ret = rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
+ PASS_ONE_BIT, 0);
+ if (ret) {
+ *work_bgn = tmp_delay;
+ break;
+ }
+
+ tmp_delay += IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
+ }
+
+ /* Restore VFIFO to old state before we decremented it (if needed). */
+ (*p)++;
+ if (*p > IO_DQS_EN_PHASE_MAX) {
+ *p = 0;
+ rw_mgr_incr_vfifo(grp);
+ }
+
+ scc_mgr_set_dqs_en_delay_all_ranks(grp, 0);
+}
+
+/**
+ * sdr_nonworking_phase() - Find non-working DQS enable phase
+ * @grp: Read/Write group
+ * @work_end: Working window end position
+ * @p: DQS Phase Iterator
+ * @i: Iterator
+ *
+ * Find non-working DQS enable phase setting.
+ */
+static int sdr_nonworking_phase(const u32 grp, u32 *work_end, u32 *p, u32 *i)
+{
+ int ret;
+
+ (*p)++;
+ *work_end += IO_DELAY_PER_OPA_TAP;
+ if (*p > IO_DQS_EN_PHASE_MAX) {
+ /* Fiddle with FIFO. */
+ *p = 0;
+ rw_mgr_incr_vfifo(grp);
+ }
+
+ ret = sdr_find_phase(0, grp, work_end, i, p);
+ if (ret) {
+ /* Cannot see edge of failing read. */
+ debug_cond(DLEVEL == 2, "%s:%d: end: failed\n",
+ __func__, __LINE__);
+ }
+
+ return ret;
+}
+
+/**
+ * sdr_find_window_center() - Find center of the working DQS window.
+ * @grp: Read/Write group
+ * @work_bgn: First working settings
+ * @work_end: Last working settings
+ *
+ * Find center of the working DQS enable window.
+ */
+static int sdr_find_window_center(const u32 grp, const u32 work_bgn,
+ const u32 work_end)
+{
+ u32 work_mid;
+ int tmp_delay = 0;
+ int i, p, d;
+
+ work_mid = (work_bgn + work_end) / 2;
+
+ debug_cond(DLEVEL == 2, "work_bgn=%d work_end=%d work_mid=%d\n",
+ work_bgn, work_end, work_mid);
+ /* Get the middle delay to be less than a VFIFO delay */
+ tmp_delay = (IO_DQS_EN_PHASE_MAX + 1) * IO_DELAY_PER_OPA_TAP;
+
+ debug_cond(DLEVEL == 2, "vfifo ptap delay %d\n", tmp_delay);
+ work_mid %= tmp_delay;
+ debug_cond(DLEVEL == 2, "new work_mid %d\n", work_mid);
+
+ tmp_delay = rounddown(work_mid, IO_DELAY_PER_OPA_TAP);
+ if (tmp_delay > IO_DQS_EN_PHASE_MAX * IO_DELAY_PER_OPA_TAP)
+ tmp_delay = IO_DQS_EN_PHASE_MAX * IO_DELAY_PER_OPA_TAP;
+ p = tmp_delay / IO_DELAY_PER_OPA_TAP;
+
+ debug_cond(DLEVEL == 2, "new p %d, tmp_delay=%d\n", p, tmp_delay);
+
+ d = DIV_ROUND_UP(work_mid - tmp_delay, IO_DELAY_PER_DQS_EN_DCHAIN_TAP);
+ if (d > IO_DQS_EN_DELAY_MAX)
+ d = IO_DQS_EN_DELAY_MAX;
+ tmp_delay += d * IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
+
+ debug_cond(DLEVEL == 2, "new d %d, tmp_delay=%d\n", d, tmp_delay);
+
+ scc_mgr_set_dqs_en_phase_all_ranks(grp, p);
+ scc_mgr_set_dqs_en_delay_all_ranks(grp, d);
+
+ /*
+ * push vfifo until we can successfully calibrate. We can do this
+ * because the largest possible margin in 1 VFIFO cycle.
+ */
+ for (i = 0; i < VFIFO_SIZE; i++) {
+ debug_cond(DLEVEL == 2, "find_dqs_en_phase: center\n");
+ if (rw_mgr_mem_calibrate_read_test_all_ranks(grp, 1,
+ PASS_ONE_BIT,
+ 0)) {
+ debug_cond(DLEVEL == 2,
+ "%s:%d center: found: ptap=%u dtap=%u\n",
+ __func__, __LINE__, p, d);
+ return 0;
+ }
+
+ /* Fiddle with FIFO. */
+ rw_mgr_incr_vfifo(grp);
+ }
+
+ debug_cond(DLEVEL == 2, "%s:%d center: failed.\n",
+ __func__, __LINE__);
+ return -EINVAL;
+}
+
+/**
+ * rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase() - Find a good DQS enable to use
+ * @grp: Read/Write Group
+ *
+ * Find a good DQS enable to use.
+ */
+static int rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(const u32 grp)
+{
+ u32 d, p, i;
+ u32 dtaps_per_ptap;
+ u32 work_bgn, work_end;
+ u32 found_passing_read, found_failing_read, initial_failing_dtap;
+ int ret;
+
+ debug("%s:%d %u\n", __func__, __LINE__, grp);
+
+ reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
+
+ scc_mgr_set_dqs_en_delay_all_ranks(grp, 0);
+ scc_mgr_set_dqs_en_phase_all_ranks(grp, 0);
+
+ /* Step 0: Determine number of delay taps for each phase tap. */
+ dtaps_per_ptap = IO_DELAY_PER_OPA_TAP / IO_DELAY_PER_DQS_EN_DCHAIN_TAP;
+
+ /* Step 1: First push vfifo until we get a failing read. */
+ find_vfifo_failing_read(grp);
+
+ /* Step 2: Find first working phase, increment in ptaps. */
+ work_bgn = 0;
+ ret = sdr_working_phase(grp, &work_bgn, &d, &p, &i);
+ if (ret)
+ return ret;
+
+ work_end = work_bgn;
+
+ /*
+ * If d is 0 then the working window covers a phase tap and we can
+ * follow the old procedure. Otherwise, we've found the beginning
+ * and we need to increment the dtaps until we find the end.
+ */
+ if (d == 0) {
+ /*
+ * Step 3a: If we have room, back off by one and
+ * increment in dtaps.
+ */
+ sdr_backup_phase(grp, &work_bgn, &p);
+
+ /*
+ * Step 4a: go forward from working phase to non working
+ * phase, increment in ptaps.
+ */
+ ret = sdr_nonworking_phase(grp, &work_end, &p, &i);
+ if (ret)
+ return ret;
+
+ /* Step 5a: Back off one from last, increment in dtaps. */
+
+ /* Special case code for backing up a phase */
+ if (p == 0) {
+ p = IO_DQS_EN_PHASE_MAX;
+ rw_mgr_decr_vfifo(grp);
+ } else {
+ p = p - 1;
+ }
+
+ work_end -= IO_DELAY_PER_OPA_TAP;
+ scc_mgr_set_dqs_en_phase_all_ranks(grp, p);
+
+ d = 0;
+
+ debug_cond(DLEVEL == 2, "%s:%d p: ptap=%u\n",
+ __func__, __LINE__, p);
+ }
+
+ /* The dtap increment to find the failing edge is done here. */
+ sdr_find_phase_delay(0, 1, grp, &work_end,
+ IO_DELAY_PER_DQS_EN_DCHAIN_TAP, &d);