From 9994bb3f36e3d181d9f0a078609038080cfd7a3d Mon Sep 17 00:00:00 2001 From: Miquel Raynal Date: Wed, 30 Sep 2020 01:01:13 +0200 Subject: [PATCH] mtd: nand: ecc-bch: Create the software BCH engine Let's continue introducing the generic ECC engine abstraction in the NAND subsystem by instantiating a first ECC engine: the software BCH one. While at it, make a very tidy ecc_sw_bch_init() function and move all the sanity checks and user input management in nand_ecc_sw_bch_init_ctx(). This second helper will be called from the raw RAND core. Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/linux-mtd/20200929230124.31491-10-miquel.raynal@bootlin.com --- drivers/mtd/nand/ecc-sw-bch.c | 335 ++++++++++++++++++++++------ drivers/mtd/nand/raw/nand_base.c | 62 +---- include/linux/mtd/nand-ecc-sw-bch.h | 16 +- include/linux/mtd/nand.h | 9 + 4 files changed, 297 insertions(+), 125 deletions(-) diff --git a/drivers/mtd/nand/ecc-sw-bch.c b/drivers/mtd/nand/ecc-sw-bch.c index 16a54bd2ca31..0a0ac11d5725 100644 --- a/drivers/mtd/nand/ecc-sw-bch.c +++ b/drivers/mtd/nand/ecc-sw-bch.c @@ -75,6 +75,19 @@ int nand_ecc_sw_bch_correct(struct nand_device *nand, unsigned char *buf, } EXPORT_SYMBOL(nand_ecc_sw_bch_correct); +/** + * nand_ecc_sw_bch_cleanup - Cleanup software BCH ECC resources + * @nand: NAND device + */ +static void nand_ecc_sw_bch_cleanup(struct nand_device *nand) +{ + struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv; + + bch_free(engine_conf->bch); + kfree(engine_conf->errloc); + kfree(engine_conf->eccmask); +} + /** * nand_ecc_sw_bch_init - Initialize software BCH ECC engine * @nand: NAND device @@ -92,71 +105,36 @@ EXPORT_SYMBOL(nand_ecc_sw_bch_correct); * step_size = 512 (thus, m = 13 is the smallest integer such that 2^m - 1 > 512 * 8) * bytes = 7 (7 bytes are required to store m * t = 13 * 4 = 52 bits) */ -int nand_ecc_sw_bch_init(struct nand_device *nand) +static int nand_ecc_sw_bch_init(struct nand_device *nand) { - struct mtd_info *mtd = nanddev_to_mtd(nand); - unsigned int m, t, eccsteps, i; struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv; - unsigned char *erased_page; unsigned int eccsize = nand->ecc.ctx.conf.step_size; unsigned int eccbytes = engine_conf->code_size; - unsigned int eccstrength = nand->ecc.ctx.conf.strength; + unsigned int m, t, i; + unsigned char *erased_page; + int ret; - if (!eccbytes && eccstrength) { - eccbytes = DIV_ROUND_UP(eccstrength * fls(8 * eccsize), 8); - engine_conf->code_size = eccbytes; - } - - if (!eccsize || !eccbytes) { - pr_warn("ecc parameters not supplied\n"); - return -EINVAL; - } - - m = fls(1+8*eccsize); - t = (eccbytes*8)/m; + m = fls(1 + (8 * eccsize)); + t = (eccbytes * 8) / m; engine_conf->bch = bch_init(m, t, 0, false); if (!engine_conf->bch) return -EINVAL; - /* verify that eccbytes has the expected value */ - if (engine_conf->bch->ecc_bytes != eccbytes) { - pr_warn("invalid eccbytes %u, should be %u\n", - eccbytes, engine_conf->bch->ecc_bytes); - goto fail; - } - - eccsteps = mtd->writesize/eccsize; - - /* Check that we have an oob layout description. */ - if (!mtd->ooblayout) { - pr_warn("missing oob scheme"); - goto fail; - } - - /* sanity checks */ - if (8*(eccsize+eccbytes) >= (1 << m)) { - pr_warn("eccsize %u is too large\n", eccsize); - goto fail; - } - - if (mtd_ooblayout_count_eccbytes(mtd) != (eccsteps*eccbytes)) { - pr_warn("invalid ecc layout\n"); - goto fail; - } - engine_conf->eccmask = kzalloc(eccbytes, GFP_KERNEL); engine_conf->errloc = kmalloc_array(t, sizeof(*engine_conf->errloc), GFP_KERNEL); - if (!engine_conf->eccmask || !engine_conf->errloc) - goto fail; + if (!engine_conf->eccmask || !engine_conf->errloc) { + ret = -ENOMEM; + goto cleanup; + } - /* - * compute and store the inverted ecc of an erased ecc block - */ + /* Compute and store the inverted ECC of an erased step */ erased_page = kmalloc(eccsize, GFP_KERNEL); - if (!erased_page) - goto fail; + if (!erased_page) { + ret = -ENOMEM; + goto cleanup; + } memset(erased_page, 0xff, eccsize); bch_encode(engine_conf->bch, erased_page, eccsize, @@ -166,33 +144,262 @@ int nand_ecc_sw_bch_init(struct nand_device *nand) for (i = 0; i < eccbytes; i++) engine_conf->eccmask[i] ^= 0xff; - if (!eccstrength) - nand->ecc.ctx.conf.strength = (eccbytes * 8) / fls(8 * eccsize); + /* Verify that the number of code bytes has the expected value */ + if (engine_conf->bch->ecc_bytes != eccbytes) { + pr_err("Invalid number of ECC bytes: %u, expected: %u\n", + eccbytes, engine_conf->bch->ecc_bytes); + ret = -EINVAL; + goto cleanup; + } + + /* Sanity checks */ + if (8 * (eccsize + eccbytes) >= (1 << m)) { + pr_err("ECC step size is too large (%u)\n", eccsize); + ret = -EINVAL; + goto cleanup; + } return 0; -fail: +cleanup: nand_ecc_sw_bch_cleanup(nand); - return -EINVAL; + return ret; } -EXPORT_SYMBOL(nand_ecc_sw_bch_init); -/** - * nand_ecc_sw_bch_cleanup - Cleanup software BCH ECC resources - * @nand: NAND device - */ -void nand_ecc_sw_bch_cleanup(struct nand_device *nand) +int nand_ecc_sw_bch_init_ctx(struct nand_device *nand) +{ + struct nand_ecc_props *conf = &nand->ecc.ctx.conf; + struct mtd_info *mtd = nanddev_to_mtd(nand); + struct nand_ecc_sw_bch_conf *engine_conf; + unsigned int code_size = 0, nsteps; + int ret; + + /* Only large page NAND chips may use BCH */ + if (mtd->oobsize < 64) { + pr_err("BCH cannot be used with small page NAND chips\n"); + return -EINVAL; + } + + if (!mtd->ooblayout) + mtd_set_ooblayout(mtd, nand_get_large_page_ooblayout()); + + conf->engine_type = NAND_ECC_ENGINE_TYPE_SOFT; + conf->algo = NAND_ECC_ALGO_BCH; + conf->step_size = nand->ecc.user_conf.step_size; + conf->strength = nand->ecc.user_conf.strength; + + /* + * Board driver should supply ECC size and ECC strength + * values to select how many bits are correctable. + * Otherwise, default to 512 bytes for large page devices and 256 for + * small page devices. + */ + if (!conf->step_size) { + if (mtd->oobsize >= 64) + conf->step_size = 512; + else + conf->step_size = 256; + + conf->strength = 4; + } + + nsteps = mtd->writesize / conf->step_size; + + /* Maximize */ + if (nand->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH) { + conf->step_size = 1024; + nsteps = mtd->writesize / conf->step_size; + /* Reserve 2 bytes for the BBM */ + code_size = (mtd->oobsize - 2) / nsteps; + conf->strength = code_size * 8 / fls(8 * conf->step_size); + } + + if (!code_size) + code_size = DIV_ROUND_UP(conf->strength * + fls(8 * conf->step_size), 8); + + if (!conf->strength) + conf->strength = (code_size * 8) / fls(8 * conf->step_size); + + if (!code_size && !conf->strength) { + pr_err("Missing ECC parameters\n"); + return -EINVAL; + } + + engine_conf = kzalloc(sizeof(*engine_conf), GFP_KERNEL); + if (!engine_conf) + return -ENOMEM; + + ret = nand_ecc_init_req_tweaking(&engine_conf->req_ctx, nand); + if (ret) + goto free_engine_conf; + + engine_conf->code_size = code_size; + engine_conf->nsteps = nsteps; + engine_conf->calc_buf = kzalloc(mtd->oobsize, GFP_KERNEL); + engine_conf->code_buf = kzalloc(mtd->oobsize, GFP_KERNEL); + if (!engine_conf->calc_buf || !engine_conf->code_buf) { + ret = -ENOMEM; + goto free_bufs; + } + + nand->ecc.ctx.priv = engine_conf; + nand->ecc.ctx.total = nsteps * code_size; + + ret = nand_ecc_sw_bch_init(nand); + if (ret) + goto free_bufs; + + /* Verify the layout validity */ + if (mtd_ooblayout_count_eccbytes(mtd) != + engine_conf->nsteps * engine_conf->code_size) { + pr_err("Invalid ECC layout\n"); + ret = -EINVAL; + goto cleanup_bch_ctx; + } + + return 0; + +cleanup_bch_ctx: + nand_ecc_sw_bch_cleanup(nand); +free_bufs: + nand_ecc_cleanup_req_tweaking(&engine_conf->req_ctx); + kfree(engine_conf->calc_buf); + kfree(engine_conf->code_buf); +free_engine_conf: + kfree(engine_conf); + + return ret; +} +EXPORT_SYMBOL(nand_ecc_sw_bch_init_ctx); + +void nand_ecc_sw_bch_cleanup_ctx(struct nand_device *nand) { struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv; if (engine_conf) { - bch_free(engine_conf->bch); - kfree(engine_conf->errloc); - kfree(engine_conf->eccmask); + nand_ecc_sw_bch_cleanup(nand); + nand_ecc_cleanup_req_tweaking(&engine_conf->req_ctx); + kfree(engine_conf->calc_buf); + kfree(engine_conf->code_buf); + kfree(engine_conf); } } -EXPORT_SYMBOL(nand_ecc_sw_bch_cleanup); +EXPORT_SYMBOL(nand_ecc_sw_bch_cleanup_ctx); + +static int nand_ecc_sw_bch_prepare_io_req(struct nand_device *nand, + struct nand_page_io_req *req) +{ + struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv; + struct mtd_info *mtd = nanddev_to_mtd(nand); + int eccsize = nand->ecc.ctx.conf.step_size; + int eccbytes = engine_conf->code_size; + int eccsteps = engine_conf->nsteps; + int total = nand->ecc.ctx.total; + u8 *ecccalc = engine_conf->calc_buf; + const u8 *data; + int i; + + /* Nothing to do for a raw operation */ + if (req->mode == MTD_OPS_RAW) + return 0; + + /* This engine does not provide BBM/free OOB bytes protection */ + if (!req->datalen) + return 0; + + nand_ecc_tweak_req(&engine_conf->req_ctx, req); + + /* No more preparation for page read */ + if (req->type == NAND_PAGE_READ) + return 0; + + /* Preparation for page write: derive the ECC bytes and place them */ + for (i = 0, data = req->databuf.out; + eccsteps; + eccsteps--, i += eccbytes, data += eccsize) + nand_ecc_sw_bch_calculate(nand, data, &ecccalc[i]); + + return mtd_ooblayout_set_eccbytes(mtd, ecccalc, (void *)req->oobbuf.out, + 0, total); +} + +static int nand_ecc_sw_bch_finish_io_req(struct nand_device *nand, + struct nand_page_io_req *req) +{ + struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv; + struct mtd_info *mtd = nanddev_to_mtd(nand); + int eccsize = nand->ecc.ctx.conf.step_size; + int total = nand->ecc.ctx.total; + int eccbytes = engine_conf->code_size; + int eccsteps = engine_conf->nsteps; + u8 *ecccalc = engine_conf->calc_buf; + u8 *ecccode = engine_conf->code_buf; + unsigned int max_bitflips = 0; + u8 *data = req->databuf.in; + int i, ret; + + /* Nothing to do for a raw operation */ + if (req->mode == MTD_OPS_RAW) + return 0; + + /* This engine does not provide BBM/free OOB bytes protection */ + if (!req->datalen) + return 0; + + /* No more preparation for page write */ + if (req->type == NAND_PAGE_WRITE) { + nand_ecc_restore_req(&engine_conf->req_ctx, req); + return 0; + } + + /* Finish a page read: retrieve the (raw) ECC bytes*/ + ret = mtd_ooblayout_get_eccbytes(mtd, ecccode, req->oobbuf.in, 0, + total); + if (ret) + return ret; + + /* Calculate the ECC bytes */ + for (i = 0; eccsteps; eccsteps--, i += eccbytes, data += eccsize) + nand_ecc_sw_bch_calculate(nand, data, &ecccalc[i]); + + /* Finish a page read: compare and correct */ + for (eccsteps = engine_conf->nsteps, i = 0, data = req->databuf.in; + eccsteps; + eccsteps--, i += eccbytes, data += eccsize) { + int stat = nand_ecc_sw_bch_correct(nand, data, + &ecccode[i], + &ecccalc[i]); + if (stat < 0) { + mtd->ecc_stats.failed++; + } else { + mtd->ecc_stats.corrected += stat; + max_bitflips = max_t(unsigned int, max_bitflips, stat); + } + } + + nand_ecc_restore_req(&engine_conf->req_ctx, req); + + return max_bitflips; +} + +static struct nand_ecc_engine_ops nand_ecc_sw_bch_engine_ops = { + .init_ctx = nand_ecc_sw_bch_init_ctx, + .cleanup_ctx = nand_ecc_sw_bch_cleanup_ctx, + .prepare_io_req = nand_ecc_sw_bch_prepare_io_req, + .finish_io_req = nand_ecc_sw_bch_finish_io_req, +}; + +static struct nand_ecc_engine nand_ecc_sw_bch_engine = { + .ops = &nand_ecc_sw_bch_engine_ops, +}; + +struct nand_ecc_engine *nand_ecc_sw_bch_get_engine(void) +{ + return &nand_ecc_sw_bch_engine; +} +EXPORT_SYMBOL(nand_ecc_sw_bch_get_engine); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Ivan Djelic "); diff --git a/drivers/mtd/nand/raw/nand_base.c b/drivers/mtd/nand/raw/nand_base.c index 03106bf629dd..ebaf3bbfc1b2 100644 --- a/drivers/mtd/nand/raw/nand_base.c +++ b/drivers/mtd/nand/raw/nand_base.c @@ -5150,17 +5150,11 @@ int rawnand_sw_bch_init(struct nand_chip *chip) base->ecc.user_conf.step_size = chip->ecc.size; base->ecc.user_conf.strength = chip->ecc.strength; - engine_conf = kzalloc(sizeof(*engine_conf), GFP_KERNEL); - if (!engine_conf) - return -ENOMEM; - - engine_conf->code_size = chip->ecc.bytes; - - base->ecc.ctx.priv = engine_conf; - - ret = nand_ecc_sw_bch_init(base); + ret = nand_ecc_sw_bch_init_ctx(base); if (ret) - kfree(base->ecc.ctx.priv); + return ret; + + engine_conf = base->ecc.ctx.priv; chip->ecc.size = base->ecc.ctx.conf.step_size; chip->ecc.strength = base->ecc.ctx.conf.strength; @@ -5168,7 +5162,7 @@ int rawnand_sw_bch_init(struct nand_chip *chip) chip->ecc.steps = engine_conf->nsteps; chip->ecc.bytes = engine_conf->code_size; - return ret; + return 0; } EXPORT_SYMBOL(rawnand_sw_bch_init); @@ -5194,9 +5188,7 @@ void rawnand_sw_bch_cleanup(struct nand_chip *chip) { struct nand_device *base = &chip->base; - nand_ecc_sw_bch_cleanup(base); - - kfree(base->ecc.ctx.priv); + nand_ecc_sw_bch_cleanup_ctx(base); } EXPORT_SYMBOL(rawnand_sw_bch_cleanup); @@ -5308,51 +5300,15 @@ static int nand_set_ecc_soft_ops(struct nand_chip *chip) ecc->read_oob = nand_read_oob_std; ecc->write_oob = nand_write_oob_std; - /* - * Board driver should supply ecc.size and ecc.strength - * values to select how many bits are correctable. - * Otherwise, default to 4 bits for large page devices. - */ - if (!ecc->size && (mtd->oobsize >= 64)) { - ecc->size = 512; - ecc->strength = 4; - } - - /* - * if no ecc placement scheme was provided pickup the default - * large page one. - */ - if (!mtd->ooblayout) { - /* handle large page devices only */ - if (mtd->oobsize < 64) { - WARN(1, "OOB layout is required when using software BCH on small pages\n"); - return -EINVAL; - } - - mtd_set_ooblayout(mtd, nand_get_large_page_ooblayout()); - - } - /* * We can only maximize ECC config when the default layout is * used, otherwise we don't know how many bytes can really be * used. */ - if (mtd->ooblayout == nand_get_large_page_ooblayout() && - nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH) { - int steps, bytes; + if (nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH && + mtd->ooblayout != nand_get_large_page_ooblayout()) + nanddev->ecc.user_conf.flags &= ~NAND_ECC_MAXIMIZE_STRENGTH; - /* Always prefer 1k blocks over 512bytes ones */ - ecc->size = 1024; - steps = mtd->writesize / ecc->size; - - /* Reserve 2 bytes for the BBM */ - bytes = (mtd->oobsize - 2) / steps; - ecc->strength = bytes * 8 / fls(8 * ecc->size); - } - - /* See the software BCH ECC initialization for details */ - ecc->bytes = 0; ret = rawnand_sw_bch_init(chip); if (ret) { WARN(1, "BCH ECC initialization failed!\n"); diff --git a/include/linux/mtd/nand-ecc-sw-bch.h b/include/linux/mtd/nand-ecc-sw-bch.h index ce005528e55f..22c92073b3dd 100644 --- a/include/linux/mtd/nand-ecc-sw-bch.h +++ b/include/linux/mtd/nand-ecc-sw-bch.h @@ -13,8 +13,8 @@ /** * struct nand_ecc_sw_bch_conf - private software BCH ECC engine structure - * @reqooblen: Save the actual user OOB length requested before overwriting it - * @spare_oobbuf: Spare OOB buffer if none is provided + * @req_ctx: Save request context and tweak the original request to fit the + * engine needs * @code_size: Number of bytes needed to store a code (one code per step) * @nsteps: Number of steps * @calc_buf: Buffer to use when calculating ECC bytes @@ -24,8 +24,7 @@ * @eccmask: XOR ecc mask, allows erased pages to be decoded as valid */ struct nand_ecc_sw_bch_conf { - unsigned int reqooblen; - void *spare_oobbuf; + struct nand_ecc_req_tweak_ctx req_ctx; unsigned int code_size; unsigned int nsteps; u8 *calc_buf; @@ -41,8 +40,9 @@ int nand_ecc_sw_bch_calculate(struct nand_device *nand, const unsigned char *buf, unsigned char *code); int nand_ecc_sw_bch_correct(struct nand_device *nand, unsigned char *buf, unsigned char *read_ecc, unsigned char *calc_ecc); -int nand_ecc_sw_bch_init(struct nand_device *nand); -void nand_ecc_sw_bch_cleanup(struct nand_device *nand); +int nand_ecc_sw_bch_init_ctx(struct nand_device *nand); +void nand_ecc_sw_bch_cleanup_ctx(struct nand_device *nand); +struct nand_ecc_engine *nand_ecc_sw_bch_get_engine(void); #else /* !CONFIG_MTD_NAND_ECC_SW_BCH */ @@ -61,12 +61,12 @@ static inline int nand_ecc_sw_bch_correct(struct nand_device *nand, return -ENOTSUPP; } -static inline int nand_ecc_sw_bch_init(struct nand_device *nand) +static inline int nand_ecc_sw_bch_init_ctx(struct nand_device *nand) { return -ENOTSUPP; } -static inline void nand_ecc_sw_bch_cleanup(struct nand_device *nand) {} +static inline void nand_ecc_sw_bch_cleanup_ctx(struct nand_device *nand) {} #endif /* CONFIG_MTD_NAND_ECC_SW_BCH */ diff --git a/include/linux/mtd/nand.h b/include/linux/mtd/nand.h index 36e4fe08d0ea..df8548187713 100644 --- a/include/linux/mtd/nand.h +++ b/include/linux/mtd/nand.h @@ -278,6 +278,15 @@ int nand_ecc_finish_io_req(struct nand_device *nand, struct nand_page_io_req *req); bool nand_ecc_is_strong_enough(struct nand_device *nand); +#if IS_ENABLED(CONFIG_MTD_NAND_ECC_SW_BCH) +struct nand_ecc_engine *nand_ecc_sw_bch_get_engine(void); +#else +static inline struct nand_ecc_engine *nand_ecc_sw_bch_get_engine(void) +{ + return NULL; +} +#endif /* CONFIG_MTD_NAND_ECC_SW_BCH */ + /** * struct nand_ecc_req_tweak_ctx - Help for automatically tweaking requests * @orig_req: Pointer to the original IO request