diff --git a/sound/soc/sh/rcar/ssi.c b/sound/soc/sh/rcar/ssi.c
index 411bda2387ad..135c5669f796 100644
--- a/sound/soc/sh/rcar/ssi.c
+++ b/sound/soc/sh/rcar/ssi.c
@@ -227,6 +227,15 @@ static int rsnd_ssi_master_clk_start(struct rsnd_mod *mod,
 	 */
 	for (j = 0; j < ARRAY_SIZE(ssi_clk_mul_table); j++) {
 
+		/*
+		 * It will set SSIWSR.CONT here, but SSICR.CKDV = 000
+		 * with it is not allowed. (SSIWSR.WS_MODE with
+		 * SSICR.CKDV = 000 is not allowed either).
+		 * Skip it. See SSICR.CKDV
+		 */
+		if (j == 0)
+			continue;
+
 		/*
 		 * this driver is assuming that
 		 * system word is 32bit x chan