gx_image_enum_begin(gx_device * dev, const gs_gstate * pgs,
                    const gs_matrix *pmat, const gs_image_common_t * pic,
                const gx_drawing_color * pdcolor, const gx_clip_path * pcpath,
                gs_memory_t * mem, gx_image_enum *penum)
{
    const gs_pixel_image_t *pim = (const gs_pixel_image_t *)pic;
    gs_image_format_t format = pim->format;
    const int width = pim->Width;
    const int height = pim->Height;
    const int bps = pim->BitsPerComponent;
    bool masked = penum->masked;
    const float *decode = pim->Decode;
    gs_matrix_double mat;
    int index_bps;
    const gs_color_space *pcs = pim->ColorSpace;
    gs_logical_operation_t lop = (pgs ? pgs->log_op : lop_default);
    int code;
    int log2_xbytes = (bps <= 8 ? 0 : arch_log2_sizeof_frac);
    int spp, nplanes, spread;
    uint bsize;
    byte *buffer;
    fixed mtx, mty;
    gs_fixed_point row_extent, col_extent, x_extent, y_extent;
    bool device_color = true;
    gs_fixed_rect obox, cbox;
    bool gridfitimages = 0;
    bool in_pattern_accumulator = 0;
    int orthogonal;
    int force_interpolation = 0;

    penum->clues = NULL;
    penum->icc_setup.has_transfer = false;
    penum->icc_setup.is_lab = false;
    penum->icc_setup.must_halftone = false;
    penum->icc_setup.need_decode = false;
    penum->Width = width;
    penum->Height = height;

    if ((code = gx_image_compute_mat(pgs, pmat, &(pim->ImageMatrix), &mat)) < 0) {
        return code;
    }
    /* Grid fit: A common construction in postscript/PDF files is for images
     * to be constructed as a series of 'stacked' 1 pixel high images.
     * Furthermore, many of these are implemented as an imagemask plotted on
     * top of thin rectangles. The different fill rules for images and line
     * art produces problems; line art fills a pixel if any part of it is
     * touched - images only fill a pixel if the centre of the pixel is
     * covered. Bug 692666 is such a problem.
     *
     * As a workaround for this problem, the code below was introduced. The
     * concept is that orthogonal images can be 'grid fitted' (or 'stretch')
     * to entirely cover pixels that they touch. Initially I had this working
     * for all images regardless of type, but as testing has proceeded, this
     * showed more and more regressions, so I've cut the cases back in which
     * this code is used until it now only triggers on imagemasks that are
     * either 1 pixel high, or wide, and then not if we are rendering a
     * glyph (such as from a type3 font).
     */

    /* Ask the device if we are in a pattern accumulator */
    in_pattern_accumulator = (dev_proc(dev, dev_spec_op)(dev, gxdso_in_pattern_accumulator, NULL, 0));
    if (in_pattern_accumulator < 0)
        in_pattern_accumulator = 0;

    /* Figure out if we are orthogonal */
    if (mat.xy == 0 && mat.yx == 0)
        orthogonal = 1;
    else if (mat.xx == 0 && mat.yy == 0)
        orthogonal = 2;
    else
        orthogonal = 0;

    /* If we are in a pattern accumulator, we choose to always grid fit
     * orthogonal images. We do this by asking the device whether we
     * should grid fit. This allows us to avoid nasty blank lines around
     * the edges of cells.
     */
    gridfitimages = in_pattern_accumulator && orthogonal;

    if (pgs != NULL && pgs->show_gstate != NULL) {
        /* If we're a graphics state, and we're in a text object, then we
         * must be in a type3 font. Don't fiddle with it. */
    } else if (!gridfitimages &&
               (!penum->masked || penum->image_parent_type != 0)) {
        /* Other than for images we are specifically looking to grid fit (such as
         * ones in a pattern device), we only grid fit imagemasks */
    } else if (gridfitimages && (penum->masked && penum->image_parent_type == 0)) {
        /* We don't gridfit imagemasks in a pattern accumulator */
    } else if (pgs != NULL && pgs->fill_adjust.x == 0 && pgs->fill_adjust.y == 0) {
        /* If fill adjust is disabled, so is grid fitting */
    } else if (orthogonal == 1) {
        if (width == 1 || gridfitimages) {
            if (mat.xx > 0) {
                fixed ix0 = int2fixed(fixed2int(float2fixed(mat.tx)));
                double x1 = mat.tx + mat.xx * width;
                fixed ix1 = int2fixed(fixed2int_ceiling(float2fixed(x1)));
                mat.tx = (double)fixed2float(ix0);
                mat.xx = (double)(fixed2float(ix1 - ix0)/width);
            } else if (mat.xx < 0) {
                fixed ix0 = int2fixed(fixed2int_ceiling(float2fixed(mat.tx)));
                double x1 = mat.tx + mat.xx * width;
                fixed ix1 = int2fixed(fixed2int(float2fixed(x1)));
                mat.tx = (double)fixed2float(ix0);
                mat.xx = (double)(fixed2float(ix1 - ix0)/width);
            }
        }
        if (height == 1 || gridfitimages) {
            if (mat.yy > 0) {
                fixed iy0 = int2fixed(fixed2int(float2fixed(mat.ty)));
                double y1 = mat.ty + mat.yy * height;
                fixed iy1 = int2fixed(fixed2int_ceiling(float2fixed(y1)));
                mat.ty = (double)fixed2float(iy0);
                mat.yy = (double)(fixed2float(iy1 - iy0)/height);
            } else if (mat.yy < 0) {
                fixed iy0 = int2fixed(fixed2int_ceiling(float2fixed(mat.ty)));
                double y1 = mat.ty + mat.yy * height;
                fixed iy1 = int2fixed(fixed2int(float2fixed(y1)));
                mat.ty = (double)fixed2float(iy0);
                mat.yy = ((double)fixed2float(iy1 - iy0)/height);
            }
        }
    } else if (orthogonal == 2) {
        if (height == 1 || gridfitimages) {
            if (mat.yx > 0) {
                fixed ix0 = int2fixed(fixed2int(float2fixed(mat.tx)));
                double x1 = mat.tx + mat.yx * height;
                fixed ix1 = int2fixed(fixed2int_ceiling(float2fixed(x1)));
                mat.tx = (double)fixed2float(ix0);
                mat.yx = (double)(fixed2float(ix1 - ix0)/height);
            } else if (mat.yx < 0) {
                fixed ix0 = int2fixed(fixed2int_ceiling(float2fixed(mat.tx)));
                double x1 = mat.tx + mat.yx * height;
                fixed ix1 = int2fixed(fixed2int(float2fixed(x1)));
                mat.tx = (double)fixed2float(ix0);
                mat.yx = (double)(fixed2float(ix1 - ix0)/height);
            }
        }
        if (width == 1 || gridfitimages) {
            if (mat.xy > 0) {
                fixed iy0 = int2fixed(fixed2int(float2fixed(mat.ty)));
                double y1 = mat.ty + mat.xy * width;
                fixed iy1 = int2fixed(fixed2int_ceiling(float2fixed(y1)));
                mat.ty = (double)fixed2float(iy0);
                mat.xy = (double)(fixed2float(iy1 - iy0)/width);
            } else if (mat.xy < 0) {
                fixed iy0 = int2fixed(fixed2int_ceiling(float2fixed(mat.ty)));
                double y1 = mat.ty + mat.xy * width;
                fixed iy1 = int2fixed(fixed2int(float2fixed(y1)));
                mat.ty = (double)fixed2float(iy0);
                mat.xy = ((double)fixed2float(iy1 - iy0)/width);
            }
        }
    }

    /* When rendering to a pattern accumulator, if we are downscaling
     * then enable interpolation, as otherwise dropouts can cause
     * serious problems. */
    if (in_pattern_accumulator) {
        double ome = ((double)(fixed_1 - fixed_epsilon)) / (double)fixed_1; /* One Minus Epsilon */

        if (orthogonal == 1) {
            if ((mat.xx > -ome && mat.xx < ome) || (mat.yy > -ome && mat.yy < ome)) {
                force_interpolation = true;
            }
        } else if (orthogonal == 2) {
            if ((mat.xy > -ome && mat.xy < ome) || (mat.yx > -ome && mat.yx < ome)) {
                force_interpolation = true;
            }
        }
    }

    /* Can we restrict the amount of image we need? */
    while (pcpath) /* So we can break out of it */
    {
        gs_rect rect, rect_out;
        gs_matrix mi;
        const gs_matrix *m = pgs != NULL ? &ctm_only(pgs) : NULL;
        gs_fixed_rect obox;
        gs_int_rect irect;
        if (m == NULL || (code = gs_matrix_invert(m, &mi)) < 0 ||
            (code = gs_matrix_multiply(&mi, &pic->ImageMatrix, &mi)) < 0) {
            /* Give up trying to shrink the render box, but continue processing */
            break;
        }
        gx_cpath_outer_box(pcpath, &obox);
        rect.p.x = fixed2float(obox.p.x);
        rect.p.y = fixed2float(obox.p.y);
        rect.q.x = fixed2float(obox.q.x);
        rect.q.y = fixed2float(obox.q.y);
        code = gs_bbox_transform(&rect, &mi, &rect_out);
        if (code < 0) {
            /* Give up trying to shrink the render box, but continue processing */
            break;
        }
        irect.p.x = (int)(rect_out.p.x-1.0);
        irect.p.y = (int)(rect_out.p.y-1.0);
        irect.q.x = (int)(rect_out.q.x+1.0);
        irect.q.y = (int)(rect_out.q.y+1.0);
        /* Need to expand the region to allow for the fact that the mitchell
         * scaler reads multiple pixels in. This calculation can probably be
         * improved. */
        /* If mi.{xx,yy} > 1 then we are downscaling. During downscaling,
         * the support increases to ensure that we don't lose pixels contributions
         * entirely. */
        /* I do not understand the need for the +/- 1 fudge factors,
         * but they seem to be required. Increasing the render rectangle can
         * never be bad at least... RJW */
        {
            float support = any_abs(mi.xx);
            int isupport;
            if (any_abs(mi.yy) > support)
                support = any_abs(mi.yy);
            if (any_abs(mi.xy) > support)
                support = any_abs(mi.xy);
            if (any_abs(mi.yx) > support)
                support = any_abs(mi.yx);
            isupport = (int)(MAX_ISCALE_SUPPORT * (support+1)) + 1;
            irect.p.x -= isupport;
            irect.p.y -= isupport;
            irect.q.x += isupport;
            irect.q.y += isupport;
        }
        if (penum->rrect.x < irect.p.x) {
            penum->rrect.w -= irect.p.x - penum->rrect.x;
            if (penum->rrect.w < 0)
               penum->rrect.w = 0;
            penum->rrect.x = irect.p.x;
        }
        if (penum->rrect.x + penum->rrect.w > irect.q.x) {
            penum->rrect.w = irect.q.x - penum->rrect.x;
            if (penum->rrect.w < 0)
                penum->rrect.w = 0;
        }
        if (penum->rrect.y < irect.p.y) {
            penum->rrect.h -= irect.p.y - penum->rrect.y;
            if (penum->rrect.h < 0)
                penum->rrect.h = 0;
            penum->rrect.y = irect.p.y;
        }
        if (penum->rrect.y + penum->rrect.h > irect.q.y) {
            penum->rrect.h = irect.q.y - penum->rrect.y;
            if (penum->rrect.h < 0)
                penum->rrect.h = 0;
        }
        break; /* Out of the while */
    }
    /* Check for the intersection being null */
    if (penum->rrect.x + penum->rrect.w <= penum->rect.x  ||
        penum->rect.x  + penum->rect.w  <= penum->rrect.x ||
        penum->rrect.y + penum->rrect.h <= penum->rect.y  ||
        penum->rect.y  + penum->rect.h  <= penum->rrect.y)
    {
          /* Something may have gone wrong with the floating point above.
           * set the region to something sane. */
        penum->rrect.x = penum->rect.x;
        penum->rrect.y = penum->rect.y;
        penum->rrect.w = 0;
        penum->rrect.h = 0;
    }

    /*penum->matrix = mat;*/
    penum->matrix.xx = mat.xx;
    penum->matrix.xy = mat.xy;
    penum->matrix.yx = mat.yx;
    penum->matrix.yy = mat.yy;
    penum->matrix.tx = mat.tx;
    penum->matrix.ty = mat.ty;
    if_debug6m('b', mem, " [%g %g %g %g %g %g]\n",
              mat.xx, mat.xy, mat.yx, mat.yy, mat.tx, mat.ty);
    /* following works for 1, 2, 4, 8, 12, 16 */
    index_bps = (bps < 8 ? bps >> 1 : (bps >> 2) + 1);
    /*
     * Compute extents with distance transformation.
     */
    if (mat.tx > 0)
        mtx = float2fixed(mat.tx);
    else { /* Use positive values to ensure round down. */
        int f = (int)-mat.tx + 1;

        mtx = float2fixed(mat.tx + f) - int2fixed(f);
    }
    if (mat.ty > 0)
        mty = float2fixed(mat.ty);
    else {  /* Use positive values to ensure round down. */
        int f = (int)-mat.ty + 1;

        mty = float2fixed(mat.ty + f) - int2fixed(f);
    }

    row_extent.x = float2fixed_rounded_boxed(width * mat.xx);
    row_extent.y =
        (is_fzero(mat.xy) ? fixed_0 :
         float2fixed_rounded_boxed(width * mat.xy));
    col_extent.x =
        (is_fzero(mat.yx) ? fixed_0 :
         float2fixed_rounded_boxed(height * mat.yx));
    col_extent.y = float2fixed_rounded_boxed(height * mat.yy);
    gx_image_enum_common_init((gx_image_enum_common_t *)penum,
                              (const gs_data_image_t *)pim,
                              &image1_enum_procs, dev,
                              (masked ? 1 : (penum->alpha ? cs_num_components(pcs)+1 : cs_num_components(pcs))),
                              format);
    if (penum->rect.w == width && penum->rect.h == height) {
        x_extent = row_extent;
        y_extent = col_extent;
    } else {
        int rw = penum->rect.w, rh = penum->rect.h;

        x_extent.x = float2fixed_rounded_boxed(rw * mat.xx);
        x_extent.y =
            (is_fzero(mat.xy) ? fixed_0 :
             float2fixed_rounded_boxed(rw * mat.xy));
        y_extent.x =
            (is_fzero(mat.yx) ? fixed_0 :
             float2fixed_rounded_boxed(rh * mat.yx));
        y_extent.y = float2fixed_rounded_boxed(rh * mat.yy);
    }
    /* Set icolor0 and icolor1 to point to image clues locations if we have
       1spp or an imagemask, otherwise image clues is not used and
       we have these values point to other member variables */
    if (masked || cs_num_components(pcs) == 1) {
        /* Go ahead and allocate now if not already done.  For a mask
           we really should only do 2 values. For now, the goal is to
           eliminate the 256 bytes for the >8bpp image enumerator */
        penum->clues = (gx_image_clue*) gs_alloc_bytes(mem, sizeof(gx_image_clue)*256,
                             "gx_image_enum_begin");
        if (penum->clues == NULL)
            return_error(gs_error_VMerror);
        penum->icolor0 = &(penum->clues[0].dev_color);
        penum->icolor1 = &(penum->clues[255].dev_color);
    } else {
        penum->icolor0 = &(penum->icolor0_val);
        penum->icolor1 = &(penum->icolor1_val);
    }
    if (masked) {       /* This is imagemask. */
        if (bps != 1 || pcs != NULL || penum->alpha || decode[0] == decode[1]) {
            return_error(gs_error_rangecheck);
        }
        /* Initialize color entries 0 and 255. */
        set_nonclient_dev_color(penum->icolor0, gx_no_color_index);
        set_nonclient_dev_color(penum->icolor1, gx_no_color_index);
        *(penum->icolor1) = *pdcolor;
        memcpy(&penum->map[0].table.lookup4x1to32[0],
               (decode[0] < decode[1] ? lookup4x1to32_inverted :
                lookup4x1to32_identity),
               16 * 4);
        penum->map[0].decoding = sd_none;
        spp = 1;
        lop = rop3_know_S_0(lop);
    } else {                    /* This is image, not imagemask. */
        const gs_color_space_type *pcst = pcs->type;
        int b_w_color;

        spp = cs_num_components(pcs);
        if (spp < 0) {          /* Pattern not allowed */
            return_error(gs_error_rangecheck);
        }
        if (penum->alpha)
            ++spp;
        /* Use a less expensive format if possible. */
        switch (format) {
        case gs_image_format_bit_planar:
            if (bps > 1)
                break;
            format = gs_image_format_component_planar;
        case gs_image_format_component_planar:
            if (spp == 1)
                format = gs_image_format_chunky;
        default:                /* chunky */
            break;
        }

        if (pcs->cmm_icc_profile_data != NULL) {
            device_color = false;
        } else {
            device_color = (*pcst->concrete_space) (pcs, pgs) == pcs;
        }

        code = image_init_colors(penum, bps, spp, format, decode, pgs, dev,
                          pcs, &device_color);
        if (code < 0)
            return gs_throw(code, "Image colors initialization failed");
        /* If we have a CIE based color space and the icc equivalent profile
           is not yet set, go ahead and handle that now.  It may already
           be done due to the above init_colors which may go through remap. */
        if (gs_color_space_is_PSCIE(pcs) && pcs->icc_equivalent == NULL) {
            code = gs_colorspace_set_icc_equivalent((gs_color_space *)pcs, &(penum->icc_setup.is_lab),
                                                pgs->memory);
            if (code < 0)
                return code;
            if (penum->icc_setup.is_lab) {
                /* Free what ever profile was created and use the icc manager's
                   cielab profile */
                gs_color_space *curr_pcs = (gs_color_space *)pcs;
                rc_decrement(curr_pcs->icc_equivalent,"gx_image_enum_begin");
                rc_decrement(curr_pcs->cmm_icc_profile_data,"gx_image_enum_begin");
                curr_pcs->cmm_icc_profile_data = pgs->icc_manager->lab_profile;
                rc_increment(curr_pcs->cmm_icc_profile_data);
            }
        }
        /* Try to transform non-default RasterOps to something */
        /* that we implement less expensively. */
        if (!pim->CombineWithColor)
            lop = rop3_know_T_0(lop) & ~lop_T_transparent;
        else if ((rop3_uses_T(lop) && color_draws_b_w(dev, pdcolor) == 0))
            lop = rop3_know_T_0(lop);

        if (lop != rop3_S &&    /* if best case, no more work needed */
            !rop3_uses_T(lop) && bps == 1 && spp == 1 &&
            (b_w_color =
             color_draws_b_w(dev, penum->icolor0)) >= 0 &&
            color_draws_b_w(dev, penum->icolor1) == (b_w_color ^ 1)
            ) {
            if (b_w_color) {    /* Swap the colors and invert the RasterOp source. */
                gx_device_color dcolor;

                dcolor = *(penum->icolor0);
                *(penum->icolor0) = *(penum->icolor1);
                *(penum->icolor1) = dcolor;
                lop = rop3_invert_S(lop);
            }
            /*
             * At this point, we know that the source pixels
             * correspond directly to the S input for the raster op,
             * i.e., icolor0 is black and icolor1 is white.
             */
            switch (lop) {
                case rop3_D & rop3_S:
                    /* Implement this as an inverted mask writing 0s. */
                    *(penum->icolor1) = *(penum->icolor0);
                    /* (falls through) */
                case rop3_D | rop3_not(rop3_S):
                    /* Implement this as an inverted mask writing 1s. */
                    memcpy(&penum->map[0].table.lookup4x1to32[0],
                           lookup4x1to32_inverted, 16 * 4);
                  rmask:        /* Fill in the remaining parameters for a mask. */
                    penum->masked = masked = true;
                    set_nonclient_dev_color(penum->icolor0, gx_no_color_index);
                    penum->map[0].decoding = sd_none;
                    lop = rop3_T;
                    break;
                case rop3_D & rop3_not(rop3_S):
                    /* Implement this as a mask writing 0s. */
                    *(penum->icolor1) = *(penum->icolor0);
                    /* (falls through) */
                case rop3_D | rop3_S:
                    /* Implement this as a mask writing 1s. */
                    memcpy(&penum->map[0].table.lookup4x1to32[0],
                           lookup4x1to32_identity, 16 * 4);
                    goto rmask;
                default:
                    ;
            }
        }
    }
    penum->device_color = device_color;
    /*
     * Adjust width upward for unpacking up to 7 trailing bits in
     * the row, plus 1 byte for end-of-run, plus up to 7 leading
     * bits for data_x offset within a packed byte.
     */
    bsize = ((bps > 8 ? width * 2 : width) + 15) * spp;
    buffer = gs_alloc_bytes(mem, bsize, "image buffer");
    if (buffer == 0) {
        return_error(gs_error_VMerror);
    }
    penum->bps = bps;
    penum->unpack_bps = bps;
    penum->log2_xbytes = log2_xbytes;
    penum->spp = spp;
    switch (format) {
    case gs_image_format_chunky:
        nplanes = 1;
        spread = 1 << log2_xbytes;
        break;
    case gs_image_format_component_planar:
        nplanes = spp;
        spread = spp << log2_xbytes;
        break;
    case gs_image_format_bit_planar:
        nplanes = spp * bps;
        spread = spp << log2_xbytes;
        break;
    default:
        /* No other cases are possible (checked by gx_image_enum_alloc). */
        return_error(gs_error_Fatal);
    }
    penum->num_planes = nplanes;
    penum->spread = spread;
    /*
     * If we're asked to interpolate in a partial image, we have to
     * assume that the client either really only is interested in
     * the given sub-image, or else is constructing output out of
     * overlapping pieces.
     */
    penum->interpolate = force_interpolation ? interp_force : pim->Interpolate ? interp_on : interp_off;
    penum->x_extent = x_extent;
    penum->y_extent = y_extent;
    penum->posture =
        ((x_extent.y | y_extent.x) == 0 ? image_portrait :
         (x_extent.x | y_extent.y) == 0 ? image_landscape :
         image_skewed);
    penum->pgs = pgs;
    penum->pcs = pcs;
    penum->memory = mem;
    penum->buffer = buffer;
    penum->buffer_size = bsize;
    penum->line = NULL;
    penum->icc_link = NULL;
    penum->color_cache = NULL;
    penum->ht_buffer = NULL;
    penum->thresh_buffer = NULL;
    penum->use_cie_range = false;
    penum->line_size = 0;
    penum->use_rop = lop != (masked ? rop3_T : rop3_S);
#ifdef DEBUG
    if (gs_debug_c('*')) {
        if (penum->use_rop)
            dmprintf1(mem, "[%03x]", lop);
        dmprintf5(mem, "%c%d%c%dx%d ",
                 (masked ? (color_is_pure(pdcolor) ? 'm' : 'h') : 'i'),
                 bps,
                 (penum->posture == image_portrait ? ' ' :
                  penum->posture == image_landscape ? 'L' : 'T'),
                 width, height);
    }
#endif
    penum->slow_loop = 0;
    if (pcpath == 0) {
        (*dev_proc(dev, get_clipping_box)) (dev, &obox);
        cbox = obox;
        penum->clip_image = 0;
    } else
        penum->clip_image =
            (gx_cpath_outer_box(pcpath, &obox) |        /* not || */
             gx_cpath_inner_box(pcpath, &cbox) ?
             0 : image_clip_region);
    penum->clip_outer = obox;
    penum->clip_inner = cbox;
    penum->log_op = rop3_T;     /* rop device takes care of this */
    penum->clip_dev = 0;        /* in case we bail out */
    penum->rop_dev = 0;         /* ditto */
    penum->scaler = 0;          /* ditto */
    /*
     * If all four extrema of the image fall within the clipping
     * rectangle, clipping is never required.  When making this check,
     * we must carefully take into account the fact that we only care
     * about pixel centers.
     */
    {
        fixed
            epx = min(row_extent.x, 0) + min(col_extent.x, 0),
            eqx = max(row_extent.x, 0) + max(col_extent.x, 0),
            epy = min(row_extent.y, 0) + min(col_extent.y, 0),
            eqy = max(row_extent.y, 0) + max(col_extent.y, 0);

        {
            int hwx, hwy;

            switch (penum->posture) {
                case image_portrait:
                    hwx = width, hwy = height;
                    break;
                case image_landscape:
                    hwx = height, hwy = width;
                    break;
                default:
                    hwx = hwy = 0;
            }
            /*
             * If the image is only 1 sample wide or high,
             * and is less than 1 device pixel wide or high,
             * move it slightly so that it covers pixel centers.
             * This is a hack to work around a bug in some old
             * versions of TeX/dvips, which use 1-bit-high images
             * to draw horizontal and vertical lines without
             * positioning them properly.
             */
            if (hwx == 1 && eqx - epx < fixed_1) {
                fixed diff =
                arith_rshift_1(row_extent.x + col_extent.x);

                mtx = (((mtx + diff) | fixed_half) & -fixed_half) - diff;
            }
            if (hwy == 1 && eqy - epy < fixed_1) {
                fixed diff =
                arith_rshift_1(row_extent.y + col_extent.y);

                mty = (((mty + diff) | fixed_half) & -fixed_half) - diff;
            }
        }
        if_debug5m('b', mem, "[b]Image: %sspp=%d, bps=%d, mt=(%g,%g)\n",
                   (masked? "masked, " : ""), spp, bps,
                   fixed2float(mtx), fixed2float(mty));
        if_debug9m('b', mem,
                   "[b]   cbox=(%g,%g),(%g,%g), obox=(%g,%g),(%g,%g), clip_image=0x%x\n",
                   fixed2float(cbox.p.x), fixed2float(cbox.p.y),
                   fixed2float(cbox.q.x), fixed2float(cbox.q.y),
                   fixed2float(obox.p.x), fixed2float(obox.p.y),
                   fixed2float(obox.q.x), fixed2float(obox.q.y),
                   penum->clip_image);
        /* These DDAs enumerate the starting position of each source pixel
         * row in device space. */
        dda_init(penum->dda.row.x, mtx, col_extent.x, height);
        dda_init(penum->dda.row.y, mty, col_extent.y, height);
        if (penum->posture == image_portrait) {
            penum->dst_width = row_extent.x;
            penum->dst_height = col_extent.y;
        } else {
            penum->dst_width = col_extent.x;
            penum->dst_height = row_extent.y;
        }
        /* For gs_image_class_0_interpolate. */
        penum->yi0 = fixed2int_pixround_perfect(dda_current(penum->dda.row.y)); /* For gs_image_class_0_interpolate. */
        if (penum->rect.y) {
            int y = penum->rect.y;

            while (y--) {
                dda_next(penum->dda.row.x);
                dda_next(penum->dda.row.y);
            }
        }
        penum->cur.x = penum->prev.x = dda_current(penum->dda.row.x);
        penum->cur.y = penum->prev.y = dda_current(penum->dda.row.y);
        /* These DDAs enumerate the starting positions of each row of our
         * source pixel data, in the subrectangle ('strip') that we are
         * actually rendering. */
        dda_init(penum->dda.strip.x, penum->cur.x, row_extent.x, width);
        dda_init(penum->dda.strip.y, penum->cur.y, row_extent.y, width);
        if (penum->rect.x) {
            dda_advance(penum->dda.strip.x, penum->rect.x);
            dda_advance(penum->dda.strip.y, penum->rect.x);
        }
        {
            fixed ox = dda_current(penum->dda.strip.x);
            fixed oy = dda_current(penum->dda.strip.y);

            if (!penum->clip_image)     /* i.e., not clip region */
                penum->clip_image =
                    (fixed_pixround(ox + epx) < fixed_pixround(cbox.p.x) ?
                     image_clip_xmin : 0) +
                    (fixed_pixround(ox + eqx) >= fixed_pixround(cbox.q.x) ?
                     image_clip_xmax : 0) +
                    (fixed_pixround(oy + epy) < fixed_pixround(cbox.p.y) ?
                     image_clip_ymin : 0) +
                    (fixed_pixround(oy + eqy) >= fixed_pixround(cbox.q.y) ?
                     image_clip_ymax : 0);
        }
    }
    penum->y = 0;
    penum->used.x = 0;
    penum->used.y = 0;
    {
        static sample_unpack_proc_t procs[2][6] = {
        {   sample_unpack_1, sample_unpack_2,
            sample_unpack_4, sample_unpack_8,
            sample_unpack_12, sample_unpack_16
        },
        {   sample_unpack_1_interleaved, sample_unpack_2_interleaved,
            sample_unpack_4_interleaved, sample_unpack_8_interleaved,
            sample_unpack_12, sample_unpack_16
        }};
        int num_planes = penum->num_planes;
        bool interleaved = (num_planes == 1 && penum->plane_depths[0] != penum->bps);
        int i;

        if (interleaved) {
            int num_components = penum->plane_depths[0] / penum->bps;

            for (i = 1; i < num_components; i++) {
                if (decode[0] != decode[i * 2 + 0] ||
                    decode[1] != decode[i * 2 + 1])
                    break;
            }
            if (i == num_components)
                interleaved = false; /* Use single table. */
        }
        penum->unpack = procs[interleaved][index_bps];

        if_debug1m('b', mem, "[b]unpack=%d\n", bps);
        /* Set up pixel0 for image class procedures. */
        penum->dda.pixel0 = penum->dda.strip;
        for (i = 0; i < gx_image_class_table_count; ++i)
            if ((penum->render = gx_image_class_table[i](penum)) != 0)
                break;
        if (i == gx_image_class_table_count) {
            /* No available class can handle this image. */
            gx_default_end_image(dev, (gx_image_enum_common_t *) penum,
                                 false);
            return_error(gs_error_rangecheck);
        }
    }
    if (penum->clip_image && pcpath) {  /* Set up the clipping device. */
        gx_device_clip *cdev =
            gs_alloc_struct(mem, gx_device_clip,
                            &st_device_clip, "image clipper");

        if (cdev == 0) {
            gx_default_end_image(dev,
                                 (gx_image_enum_common_t *) penum,
                                 false);
            return_error(gs_error_VMerror);
        }
        gx_make_clip_device_in_heap(cdev, pcpath, dev, mem);
        penum->clip_dev = cdev;
    }
    if (penum->use_rop) {       /* Set up the RasterOp source device. */
        gx_device_rop_texture *rtdev;

        code = gx_alloc_rop_texture_device(&rtdev, mem,
                                           "image RasterOp");
        if (code < 0) {
            gx_default_end_image(dev, (gx_image_enum_common_t *) penum,
                                 false);
            return code;
        }
        /* The 'target' must not be NULL for gx_make_rop_texture_device */
        if (!penum->clip_dev && !dev)
            return_error(gs_error_undefined);

        gx_make_rop_texture_device(rtdev,
                                   (penum->clip_dev != 0 ?
                                    (gx_device *) penum->clip_dev :
                                    dev), lop, pdcolor);
        gx_device_retain((gx_device *)rtdev, true);
        penum->rop_dev = rtdev;
    }
    return 0;
}


threshold_16_bit(byte *contone_ptr, byte *thresh_ptr, byte *ht_data)
{
    int j;

    for (j = 2; j > 0; j--) {
        byte h = 0;
        byte bit_init = 0x80;
        do {
            if (*contone_ptr++ < *thresh_ptr++) {
                h |=  bit_init;
            }
            bit_init >>= 1;
        } while (bit_init != 0);
        *ht_data++ = h;
    }
}

Fix bug 697459 Buffer overflow in fill_threshold_buffer

There was an overflow check for ht_buffer size, but none for the larger
threshold_buffer. Note that this file didn't fail on Windows because the
combination of the ht_buffer and the size of the (miscalculated due to
overflow) threshold_buffer would have exceeded the 2Gb limit.


jetp3852_print_page(gx_device_printer *pdev, gp_file *prn_stream)
{
#define DATA_SIZE (LINE_SIZE * 8)

    unsigned int cnt_2prn;
    unsigned int count,tempcnt;
    unsigned char vtp,cntc1,cntc2;
    int line_size_color_plane;

    byte data[DATA_SIZE];
    byte plane_data[LINE_SIZE * 3];

    /* Initialise data to zeros, otherwise later on, uninitialised bytes in
    dp[] can be greater than 7, which breaks spr8[dp[]]. */
    memset(data, 0x00, DATA_SIZE);

    /* Set initial condition for printer */
    gp_fputs("\033@",prn_stream);

    /* Send each scan line in turn */
    {
        int lnum;
        int line_size = gdev_mem_bytes_per_scan_line((gx_device *)pdev);
        int num_blank_lines = 0;

        if (line_size > DATA_SIZE) {
            emprintf2(pdev->memory, "invalid resolution and/or width gives line_size = %d, max. is %d\n",
                      line_size, DATA_SIZE);
            return_error(gs_error_rangecheck);
        }

        for ( lnum = 0; lnum < pdev->height; lnum++ ) {
            byte *end_data = data + line_size;
            gdev_prn_copy_scan_lines(pdev, lnum,
                                     (byte *)data, line_size);
            /* Remove trailing 0s. */
            while ( end_data > data && end_data[-1] == 0 )
                end_data--;
            if ( end_data == data ) {
                /* Blank line */
                num_blank_lines++;
            } else {
                int i;
                byte *odp;
                byte *row;

                /* Transpose the data to get pixel planes. */
                for ( i = 0, odp = plane_data; i < DATA_SIZE;
                      i += 8, odp++
                    ) { /* The following is for 16-bit machines */
#define spread3(c)\
 { 0, c, c*0x100, c*0x101, c*0x10000L, c*0x10001L, c*0x10100L, c*0x10101L }
                    static ulong spr40[8] = spread3(0x40);
                    static ulong spr8[8] = spread3(8);
                    static ulong spr2[8] = spread3(2);
                    register byte *dp = data + i;
                    register ulong pword =
                                     (spr40[dp[0]] << 1) +
                                     (spr40[dp[1]]) +
                                     (spr40[dp[2]] >> 1) +
                                     (spr8[dp[3]] << 1) +
                                     (spr8[dp[4]]) +
                                     (spr8[dp[5]] >> 1) +
                                     (spr2[dp[6]]) +
                                     (spr2[dp[7]] >> 1);
                    odp[0] = (byte)(pword >> 16);
                    odp[LINE_SIZE] = (byte)(pword >> 8);
                    odp[LINE_SIZE*2] = (byte)(pword);
                }
                /* Skip blank lines if any */
                if ( num_blank_lines > 0 ) {
                    /* Do "dot skips" */
                    while(num_blank_lines > 255) {
                        gp_fputs("\033e\377",prn_stream);
                        num_blank_lines -= 255;
                    }
                    vtp = num_blank_lines;
                    gp_fprintf(prn_stream,"\033e%c",vtp);
                    num_blank_lines = 0;
                }

                /* Transfer raster graphics in the order R, G, B. */
                /* Apparently it is stored in B, G, R */
                /* Calculate the amount of data to send by what */
                /* Ghostscript tells us the scan line_size in (bytes) */

                count = line_size / 3;
                line_size_color_plane = count / 3;
                cnt_2prn = line_size_color_plane * 3 + 5;
                tempcnt = cnt_2prn;
                cntc1 = (tempcnt & 0xFF00) >> 8;
                cntc2 = (tempcnt & 0x00FF);
                gp_fprintf(prn_stream, "\033[O%c%c\200\037",cntc2,cntc1);
                gp_fputc('\000',prn_stream);
                gp_fputs("\124\124",prn_stream);

                for ( row = plane_data + LINE_SIZE * 2, i = 0;
                      i < 3; row -= LINE_SIZE, i++ ) {
                    int jj;
                    byte ctemp;
                    odp = row;
                    /* Complement bytes */
                    for (jj=0; jj< line_size_color_plane; jj++) {
                        ctemp = *odp;
                        *odp++ = ~ctemp;
                    }
                    gp_fwrite(row, sizeof(byte),
                              line_size_color_plane, prn_stream);
                }
            }
        }
    }

    /* eject page */
    gp_fputs("\014", prn_stream);

    return 0;
}

Bug 701815: avoid uninitialised bytes being >7, which broke indexing.

Fixes:
    ./sanbin/gs -dBATCH -dNOPAUSE -sOutputFile=tmp -sDEVICE=jetp3852 ../bug-701815.pdf


static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
{
	__s32 __user *fds = arg;
	int fd;

	if (ctx->cq_ev_fd)
		return -EBUSY;

	if (copy_from_user(&fd, fds, sizeof(*fds)))
		return -EFAULT;

	ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
	if (IS_ERR(ctx->cq_ev_fd)) {
		int ret = PTR_ERR(ctx->cq_ev_fd);

		ctx->cq_ev_fd = NULL;
		return ret;
	}

	return 0;

io_uring: reexpand under-reexpanded iters

[   74.211232] BUG: KASAN: stack-out-of-bounds in iov_iter_revert+0x809/0x900
[   74.212778] Read of size 8 at addr ffff888025dc78b8 by task
syz-executor.0/828
[   74.214756] CPU: 0 PID: 828 Comm: syz-executor.0 Not tainted
5.14.0-rc3-next-20210730 #1
[   74.216525] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996),
BIOS rel-1.14.0-0-g155821a1990b-prebuilt.qemu.org 04/01/2014
[   74.219033] Call Trace:
[   74.219683]  dump_stack_lvl+0x8b/0xb3
[   74.220706]  print_address_description.constprop.0+0x1f/0x140
[   74.224226]  kasan_report.cold+0x7f/0x11b
[   74.226085]  iov_iter_revert+0x809/0x900
[   74.227960]  io_write+0x57d/0xe40
[   74.232647]  io_issue_sqe+0x4da/0x6a80
[   74.242578]  __io_queue_sqe+0x1ac/0xe60
[   74.245358]  io_submit_sqes+0x3f6e/0x76a0
[   74.248207]  __do_sys_io_uring_enter+0x90c/0x1a20
[   74.257167]  do_syscall_64+0x3b/0x90
[   74.257984]  entry_SYSCALL_64_after_hwframe+0x44/0xae

old_size = iov_iter_count();
...
iov_iter_revert(old_size - iov_iter_count());

If iov_iter_revert() is done base on the initial size as above, and the
iter is truncated and not reexpanded in the middle, it miscalculates
borders causing problems. This trace is due to no one reexpanding after
generic_write_checks().

Now iters store how many bytes has been truncated, so reexpand them to
the initial state right before reverting.

Cc: stable@vger.kernel.org
Reported-by: Palash Oswal <oswalpalash@gmail.com>
Reported-by: Sudip Mukherjee <sudipm.mukherjee@gmail.com>
Reported-and-tested-by: syzbot+9671693590ef5aad8953@syzkaller.appspotmail.com
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>


static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_provide_buf *p = &req->pbuf;
	struct io_ring_ctx *ctx = req->ctx;
	struct io_buffer *head;
	int ret = 0;
	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;

	io_ring_submit_lock(ctx, !force_nonblock);

	lockdep_assert_held(&ctx->uring_lock);

	ret = -ENOENT;
	head = xa_load(&ctx->io_buffers, p->bgid);
	if (head)
		ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs);
	if (ret < 0)
		req_set_fail(req);

	/* complete before unlock, IOPOLL may need the lock */
	__io_req_complete(req, issue_flags, ret, 0);
	io_ring_submit_unlock(ctx, !force_nonblock);
	return 0;
}


static void io_req_task_cancel(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;

	/* ctx is guaranteed to stay alive while we hold uring_lock */
	mutex_lock(&ctx->uring_lock);
	io_req_complete_failed(req, req->result);
	mutex_unlock(&ctx->uring_lock);
}


static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
{
	struct io_rsrc_data *rsrc_data = ref_node->rsrc_data;
	struct io_ring_ctx *ctx = rsrc_data->ctx;
	struct io_rsrc_put *prsrc, *tmp;

	list_for_each_entry_safe(prsrc, tmp, &ref_node->rsrc_list, list) {
		list_del(&prsrc->list);

		if (prsrc->tag) {
			bool lock_ring = ctx->flags & IORING_SETUP_IOPOLL;

			io_ring_submit_lock(ctx, lock_ring);
			spin_lock_irq(&ctx->completion_lock);
			io_cqring_fill_event(ctx, prsrc->tag, 0, 0);
			ctx->cq_extra++;
			io_commit_cqring(ctx);
			spin_unlock_irq(&ctx->completion_lock);
			io_cqring_ev_posted(ctx);
			io_ring_submit_unlock(ctx, lock_ring);
		}

		rsrc_data->do_put(ctx, prsrc);
		kfree(prsrc);
	}

	io_rsrc_node_destroy(ref_node);
	if (atomic_dec_and_test(&rsrc_data->refs))
		complete(&rsrc_data->done);


static void io_iopoll_req_issued(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;
	const bool in_async = io_wq_current_is_worker();

	/* workqueue context doesn't hold uring_lock, grab it now */
	if (unlikely(in_async))
		mutex_lock(&ctx->uring_lock);

	/*
	 * Track whether we have multiple files in our lists. This will impact
	 * how we do polling eventually, not spinning if we're on potentially
	 * different devices.
	 */
	if (list_empty(&ctx->iopoll_list)) {
		ctx->poll_multi_queue = false;
	} else if (!ctx->poll_multi_queue) {
		struct io_kiocb *list_req;
		unsigned int queue_num0, queue_num1;

		list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb,
						inflight_entry);

		if (list_req->file != req->file) {
			ctx->poll_multi_queue = true;
		} else {
			queue_num0 = blk_qc_t_to_queue_num(list_req->rw.kiocb.ki_cookie);
			queue_num1 = blk_qc_t_to_queue_num(req->rw.kiocb.ki_cookie);
			if (queue_num0 != queue_num1)
				ctx->poll_multi_queue = true;
		}
	}

	/*
	 * For fast devices, IO may have already completed. If it has, add
	 * it to the front so we find it first.
	 */
	if (READ_ONCE(req->iopoll_completed))
		list_add(&req->inflight_entry, &ctx->iopoll_list);
	else
		list_add_tail(&req->inflight_entry, &ctx->iopoll_list);

	if (unlikely(in_async)) {
		/*
		 * If IORING_SETUP_SQPOLL is enabled, sqes are either handle
		 * in sq thread task context or in io worker task context. If
		 * current task context is sq thread, we don't need to check
		 * whether should wake up sq thread.
		 */
		if ((ctx->flags & IORING_SETUP_SQPOLL) &&
		    wq_has_sleeper(&ctx->sq_data->wait))
			wake_up(&ctx->sq_data->wait);

		mutex_unlock(&ctx->uring_lock);
	}
}


static void io_poll_task_func(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;
	struct io_kiocb *nxt;

	if (io_poll_rewait(req, &req->poll)) {
		spin_unlock_irq(&ctx->completion_lock);
	} else {
		bool done;

		done = io_poll_complete(req, req->result);
		if (done) {
			hash_del(&req->hash_node);
		} else {
			req->result = 0;
			add_wait_queue(req->poll.head, &req->poll.wait);
		}
		spin_unlock_irq(&ctx->completion_lock);
		io_cqring_ev_posted(ctx);

		if (done) {
			nxt = io_put_req_find_next(req);
			if (nxt)
				io_req_task_submit(nxt);
		}
	}


static inline void req_set_fail(struct io_kiocb *req)
{
	req->flags |= REQ_F_FAIL;
}


static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_async_msghdr iomsg, *kmsg;
	struct socket *sock;
	unsigned flags;
	int min_ret = 0;
	int ret;

	sock = sock_from_file(req->file);
	if (unlikely(!sock))
		return -ENOTSOCK;

	kmsg = req->async_data;
	if (!kmsg) {
		ret = io_sendmsg_copy_hdr(req, &iomsg);
		if (ret)
			return ret;
		kmsg = &iomsg;
	}

	flags = req->sr_msg.msg_flags;
	if (issue_flags & IO_URING_F_NONBLOCK)
		flags |= MSG_DONTWAIT;
	if (flags & MSG_WAITALL)
		min_ret = iov_iter_count(&kmsg->msg.msg_iter);

	ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
	if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN)
		return io_setup_async_msg(req, kmsg);
	if (ret == -ERESTARTSYS)
		ret = -EINTR;

	/* fast path, check for non-NULL to avoid function call */
	if (kmsg->free_iov)
		kfree(kmsg->free_iov);
	req->flags &= ~REQ_F_NEED_CLEANUP;
	if (ret < min_ret)
		req_set_fail(req);
	__io_req_complete(req, issue_flags, ret, 0);
	return 0;
}


static inline int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)
{
	if (req->file->f_op->read_iter)
		return call_read_iter(req->file, &req->rw.kiocb, iter);
	else if (req->file->f_op->read)
		return loop_rw_iter(READ, req, iter);
	else
		return -EINVAL;
}


static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries)
{
	unsigned int to_submit;
	int ret = 0;

	to_submit = io_sqring_entries(ctx);
	/* if we're handling multiple rings, cap submit size for fairness */
	if (cap_entries && to_submit > IORING_SQPOLL_CAP_ENTRIES_VALUE)
		to_submit = IORING_SQPOLL_CAP_ENTRIES_VALUE;

	if (!list_empty(&ctx->iopoll_list) || to_submit) {
		unsigned nr_events = 0;
		const struct cred *creds = NULL;

		if (ctx->sq_creds != current_cred())
			creds = override_creds(ctx->sq_creds);

		mutex_lock(&ctx->uring_lock);
		if (!list_empty(&ctx->iopoll_list))
			io_do_iopoll(ctx, &nr_events, 0);

		/*
		 * Don't submit if refs are dying, good for io_uring_register(),
		 * but also it is relied upon by io_ring_exit_work()
		 */
		if (to_submit && likely(!percpu_ref_is_dying(&ctx->refs)) &&
		    !(ctx->flags & IORING_SETUP_R_DISABLED))
			ret = io_submit_sqes(ctx, to_submit);
		mutex_unlock(&ctx->uring_lock);

		if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait))
			wake_up(&ctx->sqo_sq_wait);
		if (creds)
			revert_creds(creds);
	}

	return ret;


static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
{
	unsigned long flags;
	bool all_flushed, posted;

	if (!force && __io_cqring_events(ctx) == ctx->cq_entries)
		return false;

	posted = false;
	spin_lock_irqsave(&ctx->completion_lock, flags);
	while (!list_empty(&ctx->cq_overflow_list)) {
		struct io_uring_cqe *cqe = io_get_cqe(ctx);
		struct io_overflow_cqe *ocqe;

		if (!cqe && !force)
			break;
		ocqe = list_first_entry(&ctx->cq_overflow_list,
					struct io_overflow_cqe, list);
		if (cqe)
			memcpy(cqe, &ocqe->cqe, sizeof(*cqe));
		else
			io_account_cq_overflow(ctx);

		posted = true;
		list_del(&ocqe->list);
		kfree(ocqe);
	}

	all_flushed = list_empty(&ctx->cq_overflow_list);
	if (all_flushed) {
		clear_bit(0, &ctx->check_cq_overflow);
		ctx->rings->sq_flags &= ~IORING_SQ_CQ_OVERFLOW;
	}

	if (posted)
		io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);
	if (posted)
		io_cqring_ev_posted(ctx);
	return all_flushed;
}


static int io_rsrc_update_prep(struct io_kiocb *req,
				const struct io_uring_sqe *sqe)
{
	if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
		return -EINVAL;
	if (sqe->ioprio || sqe->rw_flags)
		return -EINVAL;

	req->rsrc_update.offset = READ_ONCE(sqe->off);
	req->rsrc_update.nr_args = READ_ONCE(sqe->len);
	if (!req->rsrc_update.nr_args)
		return -EINVAL;
	req->rsrc_update.arg = READ_ONCE(sqe->addr);
	return 0;


static int io_renameat_prep(struct io_kiocb *req,
			    const struct io_uring_sqe *sqe)
{
	struct io_rename *ren = &req->rename;
	const char __user *oldf, *newf;

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->ioprio || sqe->buf_index)
		return -EINVAL;
	if (unlikely(req->flags & REQ_F_FIXED_FILE))
		return -EBADF;

	ren->old_dfd = READ_ONCE(sqe->fd);
	oldf = u64_to_user_ptr(READ_ONCE(sqe->addr));
	newf = u64_to_user_ptr(READ_ONCE(sqe->addr2));
	ren->new_dfd = READ_ONCE(sqe->len);
	ren->flags = READ_ONCE(sqe->rename_flags);

	ren->oldpath = getname(oldf);
	if (IS_ERR(ren->oldpath))
		return PTR_ERR(ren->oldpath);

	ren->newpath = getname(newf);
	if (IS_ERR(ren->newpath)) {
		putname(ren->oldpath);
		return PTR_ERR(ren->newpath);
	}

	req->flags |= REQ_F_NEED_CLEANUP;
	return 0;
}


static void io_free_file_tables(struct io_file_table *table, unsigned nr_files)
{
	size_t size = nr_files * sizeof(struct io_fixed_file);

	io_free_page_table((void **)table->files, size);
	table->files = NULL;


static bool io_register_op_must_quiesce(int op)
{
	switch (op) {
	case IORING_REGISTER_BUFFERS:
	case IORING_UNREGISTER_BUFFERS:
	case IORING_REGISTER_FILES:
	case IORING_UNREGISTER_FILES:
	case IORING_REGISTER_FILES_UPDATE:
	case IORING_REGISTER_PROBE:
	case IORING_REGISTER_PERSONALITY:
	case IORING_UNREGISTER_PERSONALITY:
	case IORING_REGISTER_FILES2:
	case IORING_REGISTER_FILES_UPDATE2:
	case IORING_REGISTER_BUFFERS2:
	case IORING_REGISTER_BUFFERS_UPDATE:
	case IORING_REGISTER_IOWQ_AFF:
	case IORING_UNREGISTER_IOWQ_AFF:
		return false;
	default:
		return true;
	}


static void __io_queue_sqe(struct io_kiocb *req)
{
	struct io_kiocb *linked_timeout = io_prep_linked_timeout(req);
	int ret;

issue_sqe:
	ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER);

	/*
	 * We async punt it if the file wasn't marked NOWAIT, or if the file
	 * doesn't support non-blocking read/write attempts
	 */
	if (likely(!ret)) {
		/* drop submission reference */
		if (req->flags & REQ_F_COMPLETE_INLINE) {
			struct io_ring_ctx *ctx = req->ctx;
			struct io_comp_state *cs = &ctx->submit_state.comp;

			cs->reqs[cs->nr++] = req;
			if (cs->nr == ARRAY_SIZE(cs->reqs))
				io_submit_flush_completions(ctx);
		} else {
			io_put_req(req);
		}
	} else if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
		switch (io_arm_poll_handler(req)) {
		case IO_APOLL_READY:
			goto issue_sqe;
		case IO_APOLL_ABORTED:
			/*
			 * Queued up for async execution, worker will release
			 * submit reference when the iocb is actually submitted.
			 */
			io_queue_async_work(req);
			break;
		}
	} else {
		io_req_complete_failed(req, ret);
	}
	if (linked_timeout)
		io_queue_linked_timeout(linked_timeout);


static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt,
			    struct wait_queue_head *head,
			    struct io_poll_iocb **poll_ptr)
{
	struct io_kiocb *req = pt->req;

	/*
	 * If poll->head is already set, it's because the file being polled
	 * uses multiple waitqueues for poll handling (eg one for read, one
	 * for write). Setup a separate io_poll_iocb if this happens.
	 */
	if (unlikely(poll->head)) {
		struct io_poll_iocb *poll_one = poll;

		/* already have a 2nd entry, fail a third attempt */
		if (*poll_ptr) {
			pt->error = -EINVAL;
			return;
		}
		/*
		 * Can't handle multishot for double wait for now, turn it
		 * into one-shot mode.
		 */
		if (!(poll_one->events & EPOLLONESHOT))
			poll_one->events |= EPOLLONESHOT;
		/* double add on the same waitqueue head, ignore */
		if (poll_one->head == head)
			return;
		poll = kmalloc(sizeof(*poll), GFP_ATOMIC);
		if (!poll) {
			pt->error = -ENOMEM;
			return;
		}
		io_init_poll_iocb(poll, poll_one->events, io_poll_double_wake);
		req_ref_get(req);
		poll->wait.private = req;
		*poll_ptr = poll;
	}

	pt->error = 0;
	poll->head = head;

	if (poll->events & EPOLLEXCLUSIVE)
		add_wait_queue_exclusive(head, &poll->wait);
	else
		add_wait_queue(head, &poll->wait);


static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
{
	struct io_timeout_data *data = container_of(timer,
						struct io_timeout_data, timer);
	struct io_kiocb *req = data->req;
	struct io_ring_ctx *ctx = req->ctx;
	unsigned long flags;

	spin_lock_irqsave(&ctx->completion_lock, flags);
	list_del_init(&req->timeout.list);
	atomic_set(&req->ctx->cq_timeouts,
		atomic_read(&req->ctx->cq_timeouts) + 1);

	io_cqring_fill_event(ctx, req->user_data, -ETIME, 0);
	io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);

	io_cqring_ev_posted(ctx);
	req_set_fail(req);
	io_put_req(req);
	return HRTIMER_NORESTART;


static void io_tctx_exit_cb(struct callback_head *cb)
{
	struct io_uring_task *tctx = current->io_uring;
	struct io_tctx_exit *work;

	work = container_of(cb, struct io_tctx_exit, task_work);
	/*
	 * When @in_idle, we're in cancellation and it's racy to remove the
	 * node. It'll be removed by the end of cancellation, just ignore it.
	 */
	if (!atomic_read(&tctx->in_idle))
		io_uring_del_tctx_node((unsigned long)work->ctx);
	complete(&work->completion);


static void io_dismantle_req(struct io_kiocb *req)
{
	unsigned int flags = req->flags;

	if (io_req_needs_clean(req))
		io_clean_op(req);
	if (!(flags & REQ_F_FIXED_FILE))
		io_put_file(req->file);
	if (req->fixed_rsrc_refs)
		percpu_ref_put(req->fixed_rsrc_refs);
	if (req->async_data) {
		kfree(req->async_data);
		req->async_data = NULL;
	}
}


static void io_clean_op(struct io_kiocb *req)
{
	if (req->flags & REQ_F_BUFFER_SELECTED) {
		switch (req->opcode) {
		case IORING_OP_READV:
		case IORING_OP_READ_FIXED:
		case IORING_OP_READ:
			kfree((void *)(unsigned long)req->rw.addr);
			break;
		case IORING_OP_RECVMSG:
		case IORING_OP_RECV:
			kfree(req->sr_msg.kbuf);
			break;
		}
	}

	if (req->flags & REQ_F_NEED_CLEANUP) {
		switch (req->opcode) {
		case IORING_OP_READV:
		case IORING_OP_READ_FIXED:
		case IORING_OP_READ:
		case IORING_OP_WRITEV:
		case IORING_OP_WRITE_FIXED:
		case IORING_OP_WRITE: {
			struct io_async_rw *io = req->async_data;

			kfree(io->free_iovec);
			break;
			}
		case IORING_OP_RECVMSG:
		case IORING_OP_SENDMSG: {
			struct io_async_msghdr *io = req->async_data;

			kfree(io->free_iov);
			break;
			}
		case IORING_OP_SPLICE:
		case IORING_OP_TEE:
			if (!(req->splice.flags & SPLICE_F_FD_IN_FIXED))
				io_put_file(req->splice.file_in);
			break;
		case IORING_OP_OPENAT:
		case IORING_OP_OPENAT2:
			if (req->open.filename)
				putname(req->open.filename);
			break;
		case IORING_OP_RENAMEAT:
			putname(req->rename.oldpath);
			putname(req->rename.newpath);
			break;
		case IORING_OP_UNLINKAT:
			putname(req->unlink.filename);
			break;
		}
	}
	if ((req->flags & REQ_F_POLLED) && req->apoll) {
		kfree(req->apoll->double_poll);
		kfree(req->apoll);
		req->apoll = NULL;
	}
	if (req->flags & REQ_F_INFLIGHT) {
		struct io_uring_task *tctx = req->task->io_uring;

		atomic_dec(&tctx->inflight_tracked);
	}
	if (req->flags & REQ_F_CREDS)
		put_cred(req->creds);

	req->flags &= ~IO_REQ_CLEAN_FLAGS;


static int io_uring_create(unsigned entries, struct io_uring_params *p,
			   struct io_uring_params __user *params)
{
	struct io_ring_ctx *ctx;
	struct file *file;
	int ret;

	if (!entries)
		return -EINVAL;
	if (entries > IORING_MAX_ENTRIES) {
		if (!(p->flags & IORING_SETUP_CLAMP))
			return -EINVAL;
		entries = IORING_MAX_ENTRIES;
	}

	/*
	 * Use twice as many entries for the CQ ring. It's possible for the
	 * application to drive a higher depth than the size of the SQ ring,
	 * since the sqes are only used at submission time. This allows for
	 * some flexibility in overcommitting a bit. If the application has
	 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
	 * of CQ ring entries manually.
	 */
	p->sq_entries = roundup_pow_of_two(entries);
	if (p->flags & IORING_SETUP_CQSIZE) {
		/*
		 * If IORING_SETUP_CQSIZE is set, we do the same roundup
		 * to a power-of-two, if it isn't already. We do NOT impose
		 * any cq vs sq ring sizing.
		 */
		if (!p->cq_entries)
			return -EINVAL;
		if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
			if (!(p->flags & IORING_SETUP_CLAMP))
				return -EINVAL;
			p->cq_entries = IORING_MAX_CQ_ENTRIES;
		}
		p->cq_entries = roundup_pow_of_two(p->cq_entries);
		if (p->cq_entries < p->sq_entries)
			return -EINVAL;
	} else {
		p->cq_entries = 2 * p->sq_entries;
	}

	ctx = io_ring_ctx_alloc(p);
	if (!ctx)
		return -ENOMEM;
	ctx->compat = in_compat_syscall();
	if (!capable(CAP_IPC_LOCK))
		ctx->user = get_uid(current_user());

	/*
	 * This is just grabbed for accounting purposes. When a process exits,
	 * the mm is exited and dropped before the files, hence we need to hang
	 * on to this mm purely for the purposes of being able to unaccount
	 * memory (locked/pinned vm). It's not used for anything else.
	 */
	mmgrab(current->mm);
	ctx->mm_account = current->mm;

	ret = io_allocate_scq_urings(ctx, p);
	if (ret)
		goto err;

	ret = io_sq_offload_create(ctx, p);
	if (ret)
		goto err;
	/* always set a rsrc node */
	ret = io_rsrc_node_switch_start(ctx);
	if (ret)
		goto err;
	io_rsrc_node_switch(ctx, NULL);

	memset(&p->sq_off, 0, sizeof(p->sq_off));
	p->sq_off.head = offsetof(struct io_rings, sq.head);
	p->sq_off.tail = offsetof(struct io_rings, sq.tail);
	p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
	p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
	p->sq_off.flags = offsetof(struct io_rings, sq_flags);
	p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
	p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;

	memset(&p->cq_off, 0, sizeof(p->cq_off));
	p->cq_off.head = offsetof(struct io_rings, cq.head);
	p->cq_off.tail = offsetof(struct io_rings, cq.tail);
	p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
	p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
	p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
	p->cq_off.cqes = offsetof(struct io_rings, cqes);
	p->cq_off.flags = offsetof(struct io_rings, cq_flags);

	p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
			IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
			IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL |
			IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED |
			IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS |
			IORING_FEAT_RSRC_TAGS;

	if (copy_to_user(params, p, sizeof(*p))) {
		ret = -EFAULT;
		goto err;
	}

	file = io_uring_get_file(ctx);
	if (IS_ERR(file)) {
		ret = PTR_ERR(file);
		goto err;
	}

	/*
	 * Install ring fd as the very last thing, so we don't risk someone
	 * having closed it before we finish setup
	 */
	ret = io_uring_install_fd(ctx, file);
	if (ret < 0) {
		/* fput will clean it up */
		fput(file);
		return ret;
	}

	trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
	return ret;
err:
	io_ring_ctx_wait_and_kill(ctx);
	return ret;


static struct io_kiocb *__io_req_find_next(struct io_kiocb *req)
{
	struct io_kiocb *nxt;

	/*
	 * If LINK is set, we have dependent requests in this chain. If we
	 * didn't fail this request, queue the first one up, moving any other
	 * dependencies to the next request. In case of failure, fail the rest
	 * of the chain.
	 */
	if (req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_FAIL)) {
		struct io_ring_ctx *ctx = req->ctx;
		unsigned long flags;
		bool posted;

		spin_lock_irqsave(&ctx->completion_lock, flags);
		posted = io_disarm_next(req);
		if (posted)
			io_commit_cqring(req->ctx);
		spin_unlock_irqrestore(&ctx->completion_lock, flags);
		if (posted)
			io_cqring_ev_posted(ctx);
	}
	nxt = req->link;
	req->link = NULL;
	return nxt;
}


static int __io_uring_add_tctx_node(struct io_ring_ctx *ctx)
{
	struct io_uring_task *tctx = current->io_uring;
	struct io_tctx_node *node;
	int ret;

	if (unlikely(!tctx)) {
		ret = io_uring_alloc_task_context(current, ctx);
		if (unlikely(ret))
			return ret;
		tctx = current->io_uring;
	}
	if (!xa_load(&tctx->xa, (unsigned long)ctx)) {
		node = kmalloc(sizeof(*node), GFP_KERNEL);
		if (!node)
			return -ENOMEM;
		node->ctx = ctx;
		node->task = current;

		ret = xa_err(xa_store(&tctx->xa, (unsigned long)ctx,
					node, GFP_KERNEL));
		if (ret) {
			kfree(node);
			return ret;
		}

		mutex_lock(&ctx->uring_lock);
		list_add(&node->ctx_node, &ctx->tctx_list);
		mutex_unlock(&ctx->uring_lock);
	}
	tctx->last = ctx;
	return 0;


static void io_uring_clean_tctx(struct io_uring_task *tctx)
{
	struct io_wq *wq = tctx->io_wq;
	struct io_tctx_node *node;
	unsigned long index;

	xa_for_each(&tctx->xa, index, node)
		io_uring_del_tctx_node(index);
	if (wq) {
		/*
		 * Must be after io_uring_del_task_file() (removes nodes under
		 * uring_lock) to avoid race with io_uring_try_cancel_iowq().
		 */
		tctx->io_wq = NULL;
		io_wq_put_and_exit(wq);
	}


static int io_write(struct io_kiocb *req, unsigned int issue_flags)
{
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
	struct kiocb *kiocb = &req->rw.kiocb;
	struct iov_iter __iter, *iter = &__iter;
	struct io_async_rw *rw = req->async_data;
	ssize_t ret, ret2, io_size;
	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;

	if (rw) {
		iter = &rw->iter;
		iovec = NULL;
	} else {
		ret = io_import_iovec(WRITE, req, &iovec, iter, !force_nonblock);
		if (ret < 0)
			return ret;
	}
	io_size = iov_iter_count(iter);
	req->result = io_size;

	/* Ensure we clear previously set non-block flag */
	if (!force_nonblock)
		kiocb->ki_flags &= ~IOCB_NOWAIT;
	else
		kiocb->ki_flags |= IOCB_NOWAIT;

	/* If the file doesn't support async, just async punt */
	if (force_nonblock && !io_file_supports_async(req, WRITE))
		goto copy_iov;

	/* file path doesn't support NOWAIT for non-direct_IO */
	if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
	    (req->flags & REQ_F_ISREG))
		goto copy_iov;

	ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), io_size);
	if (unlikely(ret))
		goto out_free;

	/*
	 * Open-code file_start_write here to grab freeze protection,
	 * which will be released by another thread in
	 * io_complete_rw().  Fool lockdep by telling it the lock got
	 * released so that it doesn't complain about the held lock when
	 * we return to userspace.
	 */
	if (req->flags & REQ_F_ISREG) {
		sb_start_write(file_inode(req->file)->i_sb);
		__sb_writers_release(file_inode(req->file)->i_sb,
					SB_FREEZE_WRITE);
	}
	kiocb->ki_flags |= IOCB_WRITE;

	if (req->file->f_op->write_iter)
		ret2 = call_write_iter(req->file, kiocb, iter);
	else if (req->file->f_op->write)
		ret2 = loop_rw_iter(WRITE, req, iter);
	else
		ret2 = -EINVAL;

	if (req->flags & REQ_F_REISSUE) {
		req->flags &= ~REQ_F_REISSUE;
		ret2 = -EAGAIN;
	}

	/*
	 * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just
	 * retry them without IOCB_NOWAIT.
	 */
	if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
		ret2 = -EAGAIN;
	/* no retry on NONBLOCK nor RWF_NOWAIT */
	if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT))
		goto done;
	if (!force_nonblock || ret2 != -EAGAIN) {
		/* IOPOLL retry should happen for io-wq threads */
		if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN)
			goto copy_iov;
done:
		kiocb_done(kiocb, ret2, issue_flags);
	} else {
copy_iov:
		/* some cases will consume bytes even on error returns */
		iov_iter_reexpand(iter, iter->count + iter->truncated);
		iov_iter_revert(iter, io_size - iov_iter_count(iter));
		ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false);
		return ret ?: -EAGAIN;
	}
out_free:
	/* it's reportedly faster than delegating the null check to kfree() */
	if (iovec)
		kfree(iovec);
	return ret;
}


static bool io_match_task(struct io_kiocb *head, struct task_struct *task,
			  bool cancel_all)
{
	struct io_kiocb *req;

	if (task && head->task != task)
		return false;
	if (cancel_all)
		return true;

	io_for_each_link(req, head) {
		if (req->flags & REQ_F_INFLIGHT)
			return true;
	}
	return false;
}


static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
{
	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
	struct io_task_cancel *cancel = data;
	bool ret;

	if (!cancel->all && (req->flags & REQ_F_LINK_TIMEOUT)) {
		unsigned long flags;
		struct io_ring_ctx *ctx = req->ctx;

		/* protect against races with linked timeouts */
		spin_lock_irqsave(&ctx->completion_lock, flags);
		ret = io_match_task(req, cancel->task, cancel->all);
		spin_unlock_irqrestore(&ctx->completion_lock, flags);
	} else {
		ret = io_match_task(req, cancel->task, cancel->all);
	}
	return ret;


static void io_async_find_and_cancel(struct io_ring_ctx *ctx,
				     struct io_kiocb *req, __u64 sqe_addr,
				     int success_ret)
{
	unsigned long flags;
	int ret;

	ret = io_async_cancel_one(req->task->io_uring, sqe_addr, ctx);
	spin_lock_irqsave(&ctx->completion_lock, flags);
	if (ret != -ENOENT)
		goto done;
	ret = io_timeout_cancel(ctx, sqe_addr);
	if (ret != -ENOENT)
		goto done;
	ret = io_poll_cancel(ctx, sqe_addr, false);
done:
	if (!ret)
		ret = success_ret;
	io_cqring_fill_event(ctx, req->user_data, ret, 0);
	io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);
	io_cqring_ev_posted(ctx);

	if (ret < 0)
		req_set_fail(req);


void __io_uring_free(struct task_struct *tsk)
{
	struct io_uring_task *tctx = tsk->io_uring;

	WARN_ON_ONCE(!xa_empty(&tctx->xa));
	WARN_ON_ONCE(tctx->io_wq);
	WARN_ON_ONCE(tctx->cached_refs);

	percpu_counter_destroy(&tctx->inflight);
	kfree(tctx);
	tsk->io_uring = NULL;


static void **io_alloc_page_table(size_t size)
{
	unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE);
	size_t init_size = size;
	void **table;

	table = kcalloc(nr_tables, sizeof(*table), GFP_KERNEL);
	if (!table)
		return NULL;

	for (i = 0; i < nr_tables; i++) {
		unsigned int this_size = min_t(size_t, size, PAGE_SIZE);

		table[i] = kzalloc(this_size, GFP_KERNEL);
		if (!table[i]) {
			io_free_page_table(table, init_size);
			return NULL;
		}
		size -= this_size;
	}
	return table;


static void io_req_task_queue_fail(struct io_kiocb *req, int ret)
{
	req->result = ret;
	req->io_task_work.func = io_req_task_cancel;
	io_req_task_work_add(req);
}


static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->ioprio || sqe->buf_index)
		return -EINVAL;
	if (req->flags & REQ_F_FIXED_FILE)
		return -EBADF;

	req->statx.dfd = READ_ONCE(sqe->fd);
	req->statx.mask = READ_ONCE(sqe->len);
	req->statx.filename = u64_to_user_ptr(READ_ONCE(sqe->addr));
	req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2));
	req->statx.flags = READ_ONCE(sqe->statx_flags);

	return 0;
}


static int io_read(struct io_kiocb *req, unsigned int issue_flags)
{
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
	struct kiocb *kiocb = &req->rw.kiocb;
	struct iov_iter __iter, *iter = &__iter;
	struct io_async_rw *rw = req->async_data;
	ssize_t io_size, ret, ret2;
	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;

	if (rw) {
		iter = &rw->iter;
		iovec = NULL;
	} else {
		ret = io_import_iovec(READ, req, &iovec, iter, !force_nonblock);
		if (ret < 0)
			return ret;
	}
	io_size = iov_iter_count(iter);
	req->result = io_size;

	/* Ensure we clear previously set non-block flag */
	if (!force_nonblock)
		kiocb->ki_flags &= ~IOCB_NOWAIT;
	else
		kiocb->ki_flags |= IOCB_NOWAIT;

	/* If the file doesn't support async, just async punt */
	if (force_nonblock && !io_file_supports_async(req, READ)) {
		ret = io_setup_async_rw(req, iovec, inline_vecs, iter, true);
		return ret ?: -EAGAIN;
	}

	ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), io_size);
	if (unlikely(ret)) {
		kfree(iovec);
		return ret;
	}

	ret = io_iter_do_read(req, iter);

	if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) {
		req->flags &= ~REQ_F_REISSUE;
		/* IOPOLL retry should happen for io-wq threads */
		if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL))
			goto done;
		/* no retry on NONBLOCK nor RWF_NOWAIT */
		if (req->flags & REQ_F_NOWAIT)
			goto done;
		/* some cases will consume bytes even on error returns */
		iov_iter_reexpand(iter, iter->count + iter->truncated);
		iov_iter_revert(iter, io_size - iov_iter_count(iter));
		ret = 0;
	} else if (ret == -EIOCBQUEUED) {
		goto out_free;
	} else if (ret <= 0 || ret == io_size || !force_nonblock ||
		   (req->flags & REQ_F_NOWAIT) || !(req->flags & REQ_F_ISREG)) {
		/* read all, failed, already did sync or don't want to retry */
		goto done;
	}

	ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true);
	if (ret2)
		return ret2;

	iovec = NULL;
	rw = req->async_data;
	/* now use our persistent iterator, if we aren't already */
	iter = &rw->iter;

	do {
		io_size -= ret;
		rw->bytes_done += ret;
		/* if we can retry, do so with the callbacks armed */
		if (!io_rw_should_retry(req)) {
			kiocb->ki_flags &= ~IOCB_WAITQ;
			return -EAGAIN;
		}

		/*
		 * Now retry read with the IOCB_WAITQ parts set in the iocb. If
		 * we get -EIOCBQUEUED, then we'll get a notification when the
		 * desired page gets unlocked. We can also get a partial read
		 * here, and if we do, then just retry at the new offset.
		 */
		ret = io_iter_do_read(req, iter);
		if (ret == -EIOCBQUEUED)
			return 0;
		/* we got some bytes, but not all. retry. */
		kiocb->ki_flags &= ~IOCB_WAITQ;
	} while (ret > 0 && ret < io_size);
done:
	kiocb_done(kiocb, ret, issue_flags);
out_free:
	/* it's faster to check here then delegate to kfree */
	if (iovec)
		kfree(iovec);
	return 0;
}


 */
static void io_submit_state_end(struct io_submit_state *state,
				struct io_ring_ctx *ctx)
{
	if (state->link.head)
		io_queue_sqe(state->link.head);
	if (state->comp.nr)
		io_submit_flush_completions(ctx);
	if (state->plug_started)
		blk_finish_plug(&state->plug);
	io_state_file_put(state);


static void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
{
	if (ctx->off_timeout_used)
		io_flush_timeouts(ctx);
	if (ctx->drain_active)
		io_queue_deferred(ctx);
}


static int io_register_iowq_aff(struct io_ring_ctx *ctx, void __user *arg,
				unsigned len)
{
	struct io_uring_task *tctx = current->io_uring;
	cpumask_var_t new_mask;
	int ret;

	if (!tctx || !tctx->io_wq)
		return -EINVAL;

	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
		return -ENOMEM;

	cpumask_clear(new_mask);
	if (len > cpumask_size())
		len = cpumask_size();

	if (copy_from_user(new_mask, arg, len)) {
		free_cpumask_var(new_mask);
		return -EFAULT;
	}

	ret = io_wq_cpu_affinity(tctx->io_wq, new_mask);
	free_cpumask_var(new_mask);
	return ret;


SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
		u32, min_complete, u32, flags, const void __user *, argp,
		size_t, argsz)
{
	struct io_ring_ctx *ctx;
	int submitted = 0;
	struct fd f;
	long ret;

	io_run_task_work();

	if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP |
			       IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG)))
		return -EINVAL;

	f = fdget(fd);
	if (unlikely(!f.file))
		return -EBADF;

	ret = -EOPNOTSUPP;
	if (unlikely(f.file->f_op != &io_uring_fops))
		goto out_fput;

	ret = -ENXIO;
	ctx = f.file->private_data;
	if (unlikely(!percpu_ref_tryget(&ctx->refs)))
		goto out_fput;

	ret = -EBADFD;
	if (unlikely(ctx->flags & IORING_SETUP_R_DISABLED))
		goto out;

	/*
	 * For SQ polling, the thread will do all submissions and completions.
	 * Just return the requested submit count, and wake the thread if
	 * we were asked to.
	 */
	ret = 0;
	if (ctx->flags & IORING_SETUP_SQPOLL) {
		io_cqring_overflow_flush(ctx, false);

		ret = -EOWNERDEAD;
		if (unlikely(ctx->sq_data->thread == NULL))
			goto out;
		if (flags & IORING_ENTER_SQ_WAKEUP)
			wake_up(&ctx->sq_data->wait);
		if (flags & IORING_ENTER_SQ_WAIT) {
			ret = io_sqpoll_wait_sq(ctx);
			if (ret)
				goto out;
		}
		submitted = to_submit;
	} else if (to_submit) {
		ret = io_uring_add_tctx_node(ctx);
		if (unlikely(ret))
			goto out;
		mutex_lock(&ctx->uring_lock);
		submitted = io_submit_sqes(ctx, to_submit);
		mutex_unlock(&ctx->uring_lock);

		if (submitted != to_submit)
			goto out;
	}
	if (flags & IORING_ENTER_GETEVENTS) {
		const sigset_t __user *sig;
		struct __kernel_timespec __user *ts;

		ret = io_get_ext_arg(flags, argp, &argsz, &ts, &sig);
		if (unlikely(ret))
			goto out;

		min_complete = min(min_complete, ctx->cq_entries);

		/*
		 * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
		 * space applications don't need to do io completion events
		 * polling again, they can rely on io_sq_thread to do polling
		 * work, which can reduce cpu usage and uring_lock contention.
		 */
		if (ctx->flags & IORING_SETUP_IOPOLL &&
		    !(ctx->flags & IORING_SETUP_SQPOLL)) {
			ret = io_iopoll_check(ctx, min_complete);
		} else {
			ret = io_cqring_wait(ctx, min_complete, sig, argsz, ts);
		}
	}

out:
	percpu_ref_put(&ctx->refs);
out_fput:
	fdput(f);
	return submitted ? submitted : ret;


static bool io_poll_remove_one(struct io_kiocb *req)
	__must_hold(&req->ctx->completion_lock)
{
	bool do_complete;

	do_complete = io_poll_remove_waitqs(req);
	if (do_complete) {
		io_cqring_fill_event(req->ctx, req->user_data, -ECANCELED, 0);
		io_commit_cqring(req->ctx);
		req_set_fail(req);
		io_put_req_deferred(req, 1);
	}

	return do_complete;


static void ctx_flush_and_put(struct io_ring_ctx *ctx)
{
	if (!ctx)
		return;
	if (ctx->submit_state.comp.nr) {
		mutex_lock(&ctx->uring_lock);
		io_submit_flush_completions(ctx);
		mutex_unlock(&ctx->uring_lock);
	}
	percpu_ref_put(&ctx->refs);
}


static int io_splice(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_splice *sp = &req->splice;
	struct file *in = sp->file_in;
	struct file *out = sp->file_out;
	unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
	loff_t *poff_in, *poff_out;
	long ret = 0;

	if (issue_flags & IO_URING_F_NONBLOCK)
		return -EAGAIN;

	poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
	poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;

	if (sp->len)
		ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);

	if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
		io_put_file(in);
	req->flags &= ~REQ_F_NEED_CLEANUP;

	if (ret != sp->len)
		req_set_fail(req);
	io_req_complete(req, ret);
	return 0;
}


static bool io_cancel_defer_files(struct io_ring_ctx *ctx,
				  struct task_struct *task, bool cancel_all)
{
	struct io_defer_entry *de;
	LIST_HEAD(list);

	spin_lock_irq(&ctx->completion_lock);
	list_for_each_entry_reverse(de, &ctx->defer_list, list) {
		if (io_match_task(de->req, task, cancel_all)) {
			list_cut_position(&list, &ctx->defer_list, &de->list);
			break;
		}
	}
	spin_unlock_irq(&ctx->completion_lock);
	if (list_empty(&list))
		return false;

	while (!list_empty(&list)) {
		de = list_first_entry(&list, struct io_defer_entry, list);
		list_del_init(&de->list);
		io_req_complete_failed(de->req, -ECANCELED);
		kfree(de);
	}
	return true;


static bool __io_file_supports_async(struct file *file, int rw)
{
	umode_t mode = file_inode(file)->i_mode;

	if (S_ISBLK(mode)) {
		if (IS_ENABLED(CONFIG_BLOCK) &&
		    io_bdev_nowait(I_BDEV(file->f_mapping->host)))
			return true;
		return false;
	}
	if (S_ISSOCK(mode))
		return true;
	if (S_ISREG(mode)) {
		if (IS_ENABLED(CONFIG_BLOCK) &&
		    io_bdev_nowait(file->f_inode->i_sb->s_bdev) &&
		    file->f_op != &io_uring_fops)
			return true;
		return false;
	}

	/* any ->read/write should understand O_NONBLOCK */
	if (file->f_flags & O_NONBLOCK)
		return true;

	if (!(file->f_mode & FMODE_NOWAIT))
		return false;

	if (rw == READ)
		return file->f_op->read_iter != NULL;

	return file->f_op->write_iter != NULL;
}


static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
				 unsigned nr_args, u64 __user *tags)
{
	__s32 __user *fds = (__s32 __user *) arg;
	struct file *file;
	int fd, ret;
	unsigned i;

	if (ctx->file_data)
		return -EBUSY;
	if (!nr_args)
		return -EINVAL;
	if (nr_args > IORING_MAX_FIXED_FILES)
		return -EMFILE;
	ret = io_rsrc_node_switch_start(ctx);
	if (ret)
		return ret;
	ret = io_rsrc_data_alloc(ctx, io_rsrc_file_put, tags, nr_args,
				 &ctx->file_data);
	if (ret)
		return ret;

	ret = -ENOMEM;
	if (!io_alloc_file_tables(&ctx->file_table, nr_args))
		goto out_free;

	for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
		if (copy_from_user(&fd, &fds[i], sizeof(fd))) {
			ret = -EFAULT;
			goto out_fput;
		}
		/* allow sparse sets */
		if (fd == -1) {
			ret = -EINVAL;
			if (unlikely(*io_get_tag_slot(ctx->file_data, i)))
				goto out_fput;
			continue;
		}

		file = fget(fd);
		ret = -EBADF;
		if (unlikely(!file))
			goto out_fput;

		/*
		 * Don't allow io_uring instances to be registered. If UNIX
		 * isn't enabled, then this causes a reference cycle and this
		 * instance can never get freed. If UNIX is enabled we'll
		 * handle it just fine, but there's still no point in allowing
		 * a ring fd as it doesn't support regular read/write anyway.
		 */
		if (file->f_op == &io_uring_fops) {
			fput(file);
			goto out_fput;
		}
		io_fixed_file_set(io_fixed_file_slot(&ctx->file_table, i), file);
	}

	ret = io_sqe_files_scm(ctx);
	if (ret) {
		__io_sqe_files_unregister(ctx);
		return ret;
	}

	io_rsrc_node_switch(ctx, NULL);
	return ret;
out_fput:
	for (i = 0; i < ctx->nr_user_files; i++) {
		file = io_file_from_index(ctx, i);
		if (file)
			fput(file);
	}
	io_free_file_tables(&ctx->file_table, nr_args);
	ctx->nr_user_files = 0;
out_free:
	io_rsrc_data_free(ctx->file_data);
	ctx->file_data = NULL;
	return ret;


static int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
				 struct io_rsrc_node *node, void *rsrc)
{
	struct io_rsrc_put *prsrc;

	prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL);
	if (!prsrc)
		return -ENOMEM;

	prsrc->tag = *io_get_tag_slot(data, idx);
	prsrc->rsrc = rsrc;
	list_add(&prsrc->list, &node->rsrc_list);
	return 0;


static int __init io_uring_init(void)
{
#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \
	BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
	BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \
} while (0)

#define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
	__BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename)
	BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
	BUILD_BUG_SQE_ELEM(0,  __u8,   opcode);
	BUILD_BUG_SQE_ELEM(1,  __u8,   flags);
	BUILD_BUG_SQE_ELEM(2,  __u16,  ioprio);
	BUILD_BUG_SQE_ELEM(4,  __s32,  fd);
	BUILD_BUG_SQE_ELEM(8,  __u64,  off);
	BUILD_BUG_SQE_ELEM(8,  __u64,  addr2);
	BUILD_BUG_SQE_ELEM(16, __u64,  addr);
	BUILD_BUG_SQE_ELEM(16, __u64,  splice_off_in);
	BUILD_BUG_SQE_ELEM(24, __u32,  len);
	BUILD_BUG_SQE_ELEM(28,     __kernel_rwf_t, rw_flags);
	BUILD_BUG_SQE_ELEM(28, /* compat */   int, rw_flags);
	BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  fsync_flags);
	BUILD_BUG_SQE_ELEM(28, /* compat */ __u16,  poll_events);
	BUILD_BUG_SQE_ELEM(28, __u32,  poll32_events);
	BUILD_BUG_SQE_ELEM(28, __u32,  sync_range_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  msg_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  timeout_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  accept_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  cancel_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  open_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  statx_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  fadvise_advice);
	BUILD_BUG_SQE_ELEM(28, __u32,  splice_flags);
	BUILD_BUG_SQE_ELEM(32, __u64,  user_data);
	BUILD_BUG_SQE_ELEM(40, __u16,  buf_index);
	BUILD_BUG_SQE_ELEM(40, __u16,  buf_group);
	BUILD_BUG_SQE_ELEM(42, __u16,  personality);
	BUILD_BUG_SQE_ELEM(44, __s32,  splice_fd_in);

	BUILD_BUG_ON(sizeof(struct io_uring_files_update) !=
		     sizeof(struct io_uring_rsrc_update));
	BUILD_BUG_ON(sizeof(struct io_uring_rsrc_update) >
		     sizeof(struct io_uring_rsrc_update2));
	/* should fit into one byte */
	BUILD_BUG_ON(SQE_VALID_FLAGS >= (1 << 8));

	BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
	BUILD_BUG_ON(__REQ_F_LAST_BIT >= 8 * sizeof(int));

	req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC |
				SLAB_ACCOUNT);
	return 0;


static int io_unregister_iowq_aff(struct io_ring_ctx *ctx)
{
	struct io_uring_task *tctx = current->io_uring;

	if (!tctx || !tctx->io_wq)
		return -EINVAL;

	return io_wq_cpu_affinity(tctx->io_wq, NULL);


static int io_close(struct io_kiocb *req, unsigned int issue_flags)
{
	struct files_struct *files = current->files;
	struct io_close *close = &req->close;
	struct fdtable *fdt;
	struct file *file = NULL;
	int ret = -EBADF;

	spin_lock(&files->file_lock);
	fdt = files_fdtable(files);
	if (close->fd >= fdt->max_fds) {
		spin_unlock(&files->file_lock);
		goto err;
	}
	file = fdt->fd[close->fd];
	if (!file || file->f_op == &io_uring_fops) {
		spin_unlock(&files->file_lock);
		file = NULL;
		goto err;
	}

	/* if the file has a flush method, be safe and punt to async */
	if (file->f_op->flush && (issue_flags & IO_URING_F_NONBLOCK)) {
		spin_unlock(&files->file_lock);
		return -EAGAIN;
	}

	ret = __close_fd_get_file(close->fd, &file);
	spin_unlock(&files->file_lock);
	if (ret < 0) {
		if (ret == -ENOENT)
			ret = -EBADF;
		goto err;
	}

	/* No ->flush() or already async, safely close from here */
	ret = filp_close(file, current->files);
err:
	if (ret < 0)
		req_set_fail(req);
	if (file)
		fput(file);
	__io_req_complete(req, issue_flags, ret, 0);
	return 0;
}


static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_async_msghdr iomsg, *kmsg;
	struct socket *sock;
	struct io_buffer *kbuf;
	unsigned flags;
	int min_ret = 0;
	int ret, cflags = 0;
	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;

	sock = sock_from_file(req->file);
	if (unlikely(!sock))
		return -ENOTSOCK;

	kmsg = req->async_data;
	if (!kmsg) {
		ret = io_recvmsg_copy_hdr(req, &iomsg);
		if (ret)
			return ret;
		kmsg = &iomsg;
	}

	if (req->flags & REQ_F_BUFFER_SELECT) {
		kbuf = io_recv_buffer_select(req, !force_nonblock);
		if (IS_ERR(kbuf))
			return PTR_ERR(kbuf);
		kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
		kmsg->fast_iov[0].iov_len = req->sr_msg.len;
		iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov,
				1, req->sr_msg.len);
	}

	flags = req->sr_msg.msg_flags;
	if (force_nonblock)
		flags |= MSG_DONTWAIT;
	if (flags & MSG_WAITALL)
		min_ret = iov_iter_count(&kmsg->msg.msg_iter);

	ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.umsg,
					kmsg->uaddr, flags);
	if (force_nonblock && ret == -EAGAIN)
		return io_setup_async_msg(req, kmsg);
	if (ret == -ERESTARTSYS)
		ret = -EINTR;

	if (req->flags & REQ_F_BUFFER_SELECTED)
		cflags = io_put_recv_kbuf(req);
	/* fast path, check for non-NULL to avoid function call */
	if (kmsg->free_iov)
		kfree(kmsg->free_iov);
	req->flags &= ~REQ_F_NEED_CLEANUP;
	if (ret < min_ret || ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))))
		req_set_fail(req);
	__io_req_complete(req, issue_flags, ret, cflags);
	return 0;
}


static inline void io_commit_cqring(struct io_ring_ctx *ctx)
{
	if (unlikely(ctx->off_timeout_used || ctx->drain_active))
		__io_commit_cqring_flush(ctx);
	/* order cqe stores with ring update */
	smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail);
}


static void io_req_complete_post(struct io_kiocb *req, long res,
				 unsigned int cflags)
{
	struct io_ring_ctx *ctx = req->ctx;
	unsigned long flags;

	spin_lock_irqsave(&ctx->completion_lock, flags);
	__io_cqring_fill_event(ctx, req->user_data, res, cflags);
	/*
	 * If we're the last reference to this request, add to our locked
	 * free_list cache.
	 */
	if (req_ref_put_and_test(req)) {
		if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
			if (req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_FAIL))
				io_disarm_next(req);
			if (req->link) {
				io_req_task_queue(req->link);
				req->link = NULL;
			}
		}
		io_dismantle_req(req);
		io_put_task(req->task, 1);
		list_add(&req->compl.list, &ctx->locked_free_list);
		ctx->locked_free_nr++;
	} else {
		if (!percpu_ref_tryget(&ctx->refs))
			req = NULL;
	}
	io_commit_cqring(ctx);
	spin_unlock_irqrestore(&ctx->completion_lock, flags);

	if (req) {
		io_cqring_ev_posted(ctx);
		percpu_ref_put(&ctx->refs);
	}
}


static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
		       const struct io_uring_sqe *sqe)
{
	struct io_submit_state *state;
	unsigned int sqe_flags;
	int personality, ret = 0;

	req->opcode = READ_ONCE(sqe->opcode);
	/* same numerical values with corresponding REQ_F_*, safe to copy */
	req->flags = sqe_flags = READ_ONCE(sqe->flags);
	req->user_data = READ_ONCE(sqe->user_data);
	req->file = NULL;
	req->fixed_rsrc_refs = NULL;
	/* one is dropped after submission, the other at completion */
	atomic_set(&req->refs, 2);
	req->task = current;

	/* enforce forwards compatibility on users */
	if (unlikely(sqe_flags & ~SQE_VALID_FLAGS))
		return -EINVAL;
	if (unlikely(req->opcode >= IORING_OP_LAST))
		return -EINVAL;
	if (!io_check_restriction(ctx, req, sqe_flags))
		return -EACCES;

	if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
	    !io_op_defs[req->opcode].buffer_select)
		return -EOPNOTSUPP;
	if (unlikely(sqe_flags & IOSQE_IO_DRAIN))
		ctx->drain_active = true;

	personality = READ_ONCE(sqe->personality);
	if (personality) {
		req->creds = xa_load(&ctx->personalities, personality);
		if (!req->creds)
			return -EINVAL;
		get_cred(req->creds);
		req->flags |= REQ_F_CREDS;
	}
	state = &ctx->submit_state;

	/*
	 * Plug now if we have more than 1 IO left after this, and the target
	 * is potentially a read/write to block based storage.
	 */
	if (!state->plug_started && state->ios_left > 1 &&
	    io_op_defs[req->opcode].plug) {
		blk_start_plug(&state->plug);
		state->plug_started = true;
	}

	if (io_op_defs[req->opcode].needs_file) {
		bool fixed = req->flags & REQ_F_FIXED_FILE;

		req->file = io_file_get(state, req, READ_ONCE(sqe->fd), fixed);
		if (unlikely(!req->file))
			ret = -EBADF;
	}

	state->ios_left--;
	return ret;


static int io_recv(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_buffer *kbuf;
	struct io_sr_msg *sr = &req->sr_msg;
	struct msghdr msg;
	void __user *buf = sr->buf;
	struct socket *sock;
	struct iovec iov;
	unsigned flags;
	int min_ret = 0;
	int ret, cflags = 0;
	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;

	sock = sock_from_file(req->file);
	if (unlikely(!sock))
		return -ENOTSOCK;

	if (req->flags & REQ_F_BUFFER_SELECT) {
		kbuf = io_recv_buffer_select(req, !force_nonblock);
		if (IS_ERR(kbuf))
			return PTR_ERR(kbuf);
		buf = u64_to_user_ptr(kbuf->addr);
	}

	ret = import_single_range(READ, buf, sr->len, &iov, &msg.msg_iter);
	if (unlikely(ret))
		goto out_free;

	msg.msg_name = NULL;
	msg.msg_control = NULL;
	msg.msg_controllen = 0;
	msg.msg_namelen = 0;
	msg.msg_iocb = NULL;
	msg.msg_flags = 0;

	flags = req->sr_msg.msg_flags;
	if (force_nonblock)
		flags |= MSG_DONTWAIT;
	if (flags & MSG_WAITALL)
		min_ret = iov_iter_count(&msg.msg_iter);

	ret = sock_recvmsg(sock, &msg, flags);
	if (force_nonblock && ret == -EAGAIN)
		return -EAGAIN;
	if (ret == -ERESTARTSYS)
		ret = -EINTR;
out_free:
	if (req->flags & REQ_F_BUFFER_SELECTED)
		cflags = io_put_recv_kbuf(req);
	if (ret < min_ret || ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))))
		req_set_fail(req);
	__io_req_complete(req, issue_flags, ret, cflags);
	return 0;
}


static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
					 struct task_struct *task,
					 bool cancel_all)
{
	struct io_task_cancel cancel = { .task = task, .all = cancel_all, };
	struct io_uring_task *tctx = task ? task->io_uring : NULL;

	while (1) {
		enum io_wq_cancel cret;
		bool ret = false;

		if (!task) {
			ret |= io_uring_try_cancel_iowq(ctx);
		} else if (tctx && tctx->io_wq) {
			/*
			 * Cancels requests of all rings, not only @ctx, but
			 * it's fine as the task is in exit/exec.
			 */
			cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_task_cb,
					       &cancel, true);
			ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
		}

		/* SQPOLL thread does its own polling */
		if ((!(ctx->flags & IORING_SETUP_SQPOLL) && cancel_all) ||
		    (ctx->sq_data && ctx->sq_data->thread == current)) {
			while (!list_empty_careful(&ctx->iopoll_list)) {
				io_iopoll_try_reap_events(ctx);
				ret = true;
			}
		}

		ret |= io_cancel_defer_files(ctx, task, cancel_all);
		ret |= io_poll_remove_all(ctx, task, cancel_all);
		ret |= io_kill_timeouts(ctx, task, cancel_all);
		if (task)
			ret |= io_run_task_work();
		if (!ret)
			break;
		cond_resched();
	}


	__releases(ctx->uring_lock)
	__acquires(ctx->uring_lock)
{
	int ret;

	/*
	 * We're inside the ring mutex, if the ref is already dying, then
	 * someone else killed the ctx or is already going through
	 * io_uring_register().
	 */
	if (percpu_ref_is_dying(&ctx->refs))
		return -ENXIO;

	if (ctx->restricted) {
		if (opcode >= IORING_REGISTER_LAST)
			return -EINVAL;
		opcode = array_index_nospec(opcode, IORING_REGISTER_LAST);
		if (!test_bit(opcode, ctx->restrictions.register_op))
			return -EACCES;
	}

	if (io_register_op_must_quiesce(opcode)) {
		percpu_ref_kill(&ctx->refs);

		/*
		 * Drop uring mutex before waiting for references to exit. If
		 * another thread is currently inside io_uring_enter() it might
		 * need to grab the uring_lock to make progress. If we hold it
		 * here across the drain wait, then we can deadlock. It's safe
		 * to drop the mutex here, since no new references will come in
		 * after we've killed the percpu ref.
		 */
		mutex_unlock(&ctx->uring_lock);
		do {
			ret = wait_for_completion_interruptible(&ctx->ref_comp);
			if (!ret)
				break;
			ret = io_run_task_work_sig();
			if (ret < 0)
				break;
		} while (1);
		mutex_lock(&ctx->uring_lock);

		if (ret) {
			io_refs_resurrect(&ctx->refs, &ctx->ref_comp);
			return ret;
		}
	}

	switch (opcode) {
	case IORING_REGISTER_BUFFERS:
		ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL);
		break;
	case IORING_UNREGISTER_BUFFERS:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_sqe_buffers_unregister(ctx);
		break;
	case IORING_REGISTER_FILES:
		ret = io_sqe_files_register(ctx, arg, nr_args, NULL);
		break;
	case IORING_UNREGISTER_FILES:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_sqe_files_unregister(ctx);
		break;
	case IORING_REGISTER_FILES_UPDATE:
		ret = io_register_files_update(ctx, arg, nr_args);
		break;
	case IORING_REGISTER_EVENTFD:
	case IORING_REGISTER_EVENTFD_ASYNC:
		ret = -EINVAL;
		if (nr_args != 1)
			break;
		ret = io_eventfd_register(ctx, arg);
		if (ret)
			break;
		if (opcode == IORING_REGISTER_EVENTFD_ASYNC)
			ctx->eventfd_async = 1;
		else
			ctx->eventfd_async = 0;
		break;
	case IORING_UNREGISTER_EVENTFD:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_eventfd_unregister(ctx);
		break;
	case IORING_REGISTER_PROBE:
		ret = -EINVAL;
		if (!arg || nr_args > 256)
			break;
		ret = io_probe(ctx, arg, nr_args);
		break;
	case IORING_REGISTER_PERSONALITY:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_register_personality(ctx);
		break;
	case IORING_UNREGISTER_PERSONALITY:
		ret = -EINVAL;
		if (arg)
			break;
		ret = io_unregister_personality(ctx, nr_args);
		break;
	case IORING_REGISTER_ENABLE_RINGS:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_register_enable_rings(ctx);
		break;
	case IORING_REGISTER_RESTRICTIONS:
		ret = io_register_restrictions(ctx, arg, nr_args);
		break;
	case IORING_REGISTER_FILES2:
		ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_FILE);
		break;
	case IORING_REGISTER_FILES_UPDATE2:
		ret = io_register_rsrc_update(ctx, arg, nr_args,
					      IORING_RSRC_FILE);
		break;
	case IORING_REGISTER_BUFFERS2:
		ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_BUFFER);
		break;
	case IORING_REGISTER_BUFFERS_UPDATE:
		ret = io_register_rsrc_update(ctx, arg, nr_args,
					      IORING_RSRC_BUFFER);
		break;
	case IORING_REGISTER_IOWQ_AFF:
		ret = -EINVAL;
		if (!arg || !nr_args)
			break;
		ret = io_register_iowq_aff(ctx, arg, nr_args);
		break;
	case IORING_UNREGISTER_IOWQ_AFF:
		ret = -EINVAL;
		if (arg || nr_args)
			break;
		ret = io_unregister_iowq_aff(ctx);
		break;
	default:
		ret = -EINVAL;
		break;
	}

	if (io_register_op_must_quiesce(opcode)) {
		/* bring the ctx back to life */
		percpu_ref_reinit(&ctx->refs);
		reinit_completion(&ctx->ref_comp);
	}
	return ret;


static inline void io_queue_sqe(struct io_kiocb *req)
{
	if (unlikely(req->ctx->drain_active) && io_drain_req(req))
		return;

	if (likely(!(req->flags & REQ_F_FORCE_ASYNC))) {
		__io_queue_sqe(req);
	} else {
		int ret = io_req_prep_async(req);

		if (unlikely(ret))
			io_req_complete_failed(req, ret);
		else
			io_queue_async_work(req);
	}


static void io_free_req_deferred(struct io_kiocb *req)
{
	req->io_task_work.func = io_free_req;
	io_req_task_work_add(req);
}


 */
static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,
			       bool cancel_all)
{
	struct hlist_node *tmp;
	struct io_kiocb *req;
	int posted = 0, i;

	spin_lock_irq(&ctx->completion_lock);
	for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
		struct hlist_head *list;

		list = &ctx->cancel_hash[i];
		hlist_for_each_entry_safe(req, tmp, list, hash_node) {
			if (io_match_task(req, tsk, cancel_all))
				posted += io_poll_remove_one(req);
		}
	}
	spin_unlock_irq(&ctx->completion_lock);

	if (posted)
		io_cqring_ev_posted(ctx);

	return posted != 0;


static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
			 const struct io_uring_sqe *sqe)
{
	struct io_submit_link *link = &ctx->submit_state.link;
	int ret;

	ret = io_init_req(ctx, req, sqe);
	if (unlikely(ret)) {
fail_req:
		if (link->head) {
			/* fail even hard links since we don't submit */
			req_set_fail(link->head);
			io_req_complete_failed(link->head, -ECANCELED);
			link->head = NULL;
		}
		io_req_complete_failed(req, ret);
		return ret;
	}

	ret = io_req_prep(req, sqe);
	if (unlikely(ret))
		goto fail_req;

	/* don't need @sqe from now on */
	trace_io_uring_submit_sqe(ctx, req, req->opcode, req->user_data,
				  req->flags, true,
				  ctx->flags & IORING_SETUP_SQPOLL);

	/*
	 * If we already have a head request, queue this one for async
	 * submittal once the head completes. If we don't have a head but
	 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
	 * submitted sync once the chain is complete. If none of those
	 * conditions are true (normal request), then just queue it.
	 */
	if (link->head) {
		struct io_kiocb *head = link->head;

		ret = io_req_prep_async(req);
		if (unlikely(ret))
			goto fail_req;
		trace_io_uring_link(ctx, req, head);
		link->last->link = req;
		link->last = req;

		/* last request of a link, enqueue the link */
		if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) {
			link->head = NULL;
			io_queue_sqe(head);
		}
	} else {
		if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
			link->head = req;
			link->last = req;
		} else {
			io_queue_sqe(req);
		}
	}

	return 0;


/* Returns true if we found and killed one or more timeouts */
static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,
			     bool cancel_all)
{
	struct io_kiocb *req, *tmp;
	int canceled = 0;

	spin_lock_irq(&ctx->completion_lock);
	list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) {
		if (io_match_task(req, tsk, cancel_all)) {
			io_kill_timeout(req, -ECANCELED);
			canceled++;
		}
	}
	if (canceled != 0)
		io_commit_cqring(ctx);
	spin_unlock_irq(&ctx->completion_lock);
	if (canceled != 0)
		io_cqring_ev_posted(ctx);
	return canceled != 0;


static int io_tee(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_splice *sp = &req->splice;
	struct file *in = sp->file_in;
	struct file *out = sp->file_out;
	unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
	long ret = 0;

	if (issue_flags & IO_URING_F_NONBLOCK)
		return -EAGAIN;
	if (sp->len)
		ret = do_tee(in, out, sp->len, flags);

	if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
		io_put_file(in);
	req->flags &= ~REQ_F_NEED_CLEANUP;

	if (ret != sp->len)
		req_set_fail(req);
	io_req_complete(req, ret);
	return 0;
}


static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
{
	struct io_timeout_data *data = container_of(timer,
						struct io_timeout_data, timer);
	struct io_kiocb *prev, *req = data->req;
	struct io_ring_ctx *ctx = req->ctx;
	unsigned long flags;

	spin_lock_irqsave(&ctx->completion_lock, flags);
	prev = req->timeout.head;
	req->timeout.head = NULL;

	/*
	 * We don't expect the list to be empty, that will only happen if we
	 * race with the completion of the linked work.
	 */
	if (prev) {
		io_remove_next_linked(prev);
		if (!req_ref_inc_not_zero(prev))
			prev = NULL;
	}
	spin_unlock_irqrestore(&ctx->completion_lock, flags);

	if (prev) {
		io_async_find_and_cancel(ctx, req, prev->user_data, -ETIME);
		io_put_req_deferred(prev, 1);
		io_put_req_deferred(req, 1);
	} else {
		io_req_complete_post(req, -ETIME, 0);
	}
	return HRTIMER_NORESTART;


static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
{
	unsigned int nr_events = 0;
	int ret = 0;

	/*
	 * We disallow the app entering submit/complete with polling, but we
	 * still need to lock the ring to prevent racing with polled issue
	 * that got punted to a workqueue.
	 */
	mutex_lock(&ctx->uring_lock);
	/*
	 * Don't enter poll loop if we already have events pending.
	 * If we do, we can potentially be spinning for commands that
	 * already triggered a CQE (eg in error).
	 */
	if (test_bit(0, &ctx->check_cq_overflow))
		__io_cqring_overflow_flush(ctx, false);
	if (io_cqring_events(ctx))
		goto out;
	do {
		/*
		 * If a submit got punted to a workqueue, we can have the
		 * application entering polling for a command before it gets
		 * issued. That app will hold the uring_lock for the duration
		 * of the poll right here, so we need to take a breather every
		 * now and then to ensure that the issue has a chance to add
		 * the poll to the issued list. Otherwise we can spin here
		 * forever, while the workqueue is stuck trying to acquire the
		 * very same mutex.
		 */
		if (list_empty(&ctx->iopoll_list)) {
			u32 tail = ctx->cached_cq_tail;

			mutex_unlock(&ctx->uring_lock);
			io_run_task_work();
			mutex_lock(&ctx->uring_lock);

			/* some requests don't go through iopoll_list */
			if (tail != ctx->cached_cq_tail ||
			    list_empty(&ctx->iopoll_list))
				break;
		}
		ret = io_do_iopoll(ctx, &nr_events, min);
	} while (!ret && nr_events < min && !need_resched());
out:
	mutex_unlock(&ctx->uring_lock);
	return ret;
}


static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
				     long res, unsigned int cflags)
{
	struct io_overflow_cqe *ocqe;

	ocqe = kmalloc(sizeof(*ocqe), GFP_ATOMIC | __GFP_ACCOUNT);
	if (!ocqe) {
		/*
		 * If we're in ring overflow flush mode, or in task cancel mode,
		 * or cannot allocate an overflow entry, then we need to drop it
		 * on the floor.
		 */
		io_account_cq_overflow(ctx);
		return false;
	}
	if (list_empty(&ctx->cq_overflow_list)) {
		set_bit(0, &ctx->check_cq_overflow);
		ctx->rings->sq_flags |= IORING_SQ_CQ_OVERFLOW;
	}
	ocqe->cqe.user_data = user_data;
	ocqe->cqe.res = res;
	ocqe->cqe.flags = cflags;
	list_add_tail(&ocqe->list, &ctx->cq_overflow_list);
	return true;
}


static bool io_drain_req(struct io_kiocb *req)
{
	struct io_kiocb *pos;
	struct io_ring_ctx *ctx = req->ctx;
	struct io_defer_entry *de;
	int ret;
	u32 seq;

	/*
	 * If we need to drain a request in the middle of a link, drain the
	 * head request and the next request/link after the current link.
	 * Considering sequential execution of links, IOSQE_IO_DRAIN will be
	 * maintained for every request of our link.
	 */
	if (ctx->drain_next) {
		req->flags |= REQ_F_IO_DRAIN;
		ctx->drain_next = false;
	}
	/* not interested in head, start from the first linked */
	io_for_each_link(pos, req->link) {
		if (pos->flags & REQ_F_IO_DRAIN) {
			ctx->drain_next = true;
			req->flags |= REQ_F_IO_DRAIN;
			break;
		}
	}

	/* Still need defer if there is pending req in defer list. */
	if (likely(list_empty_careful(&ctx->defer_list) &&
		!(req->flags & REQ_F_IO_DRAIN))) {
		ctx->drain_active = false;
		return false;
	}

	seq = io_get_sequence(req);
	/* Still a chance to pass the sequence check */
	if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list))
		return false;

	ret = io_req_prep_async(req);
	if (ret)
		return ret;
	io_prep_async_link(req);
	de = kmalloc(sizeof(*de), GFP_KERNEL);
	if (!de) {
		io_req_complete_failed(req, -ENOMEM);
		return true;
	}

	spin_lock_irq(&ctx->completion_lock);
	if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) {
		spin_unlock_irq(&ctx->completion_lock);
		kfree(de);
		io_queue_async_work(req);
		return true;
	}

	trace_io_uring_defer(ctx, req, req->user_data);
	de->req = req;
	de->seq = seq;
	list_add_tail(&de->list, &ctx->defer_list);
	spin_unlock_irq(&ctx->completion_lock);
	return true;