diff --git a/sound/soc/sh/rcar/ssi.c b/sound/soc/sh/rcar/ssi.c
index 38aae96267c9..a001331a53c1 100644
--- a/sound/soc/sh/rcar/ssi.c
+++ b/sound/soc/sh/rcar/ssi.c
@@ -142,6 +142,15 @@ static int rsnd_ssi_master_clk_start(struct rsnd_ssi *ssi,
 	 */
 	for (j = 0; j < ARRAY_SIZE(ssi_clk_mul_table); j++) {
 
+		/*
+		 * It will set SSIWSR.CONT here, but SSICR.CKDV = 000
+		 * with it is not allowed. (SSIWSR.WS_MODE with
+		 * SSICR.CKDV = 000 is not allowed either).
+		 * Skip it. See SSICR.CKDV
+		 */
+		if (j == 0)
+			continue;
+
 		/*
 		 * this driver is assuming that
 		 * system word is 64fs (= 2 x 32bit)