njs_typed_array_prototype_join(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t unused)
{
    u_char             *p;
    size_t             size, length, arr_length;
    njs_int_t          ret;
    njs_chb_t          chain;
    njs_value_t        *this, *separator;
    njs_typed_array_t  *array;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_typed_array(this))) {
        njs_type_error(vm, "this is not a typed array");
        return NJS_ERROR;
    }

    array = njs_typed_array(this);
    if (njs_slow_path(njs_is_detached_buffer(array->buffer))) {
        njs_type_error(vm, "detached buffer");
        return NJS_ERROR;
    }

    arr_length = njs_typed_array_length(array);

    separator = njs_arg(args, nargs, 1);

    if (njs_slow_path(!njs_is_string(separator))) {
        if (njs_is_undefined(separator)) {
            separator = njs_value_arg(&njs_string_comma);

        } else {
            ret = njs_value_to_string(vm, separator, separator);
            if (njs_slow_path(ret != NJS_OK)) {
                return ret;
            }
        }
    }

    if (arr_length == 0) {
        vm->retval = njs_string_empty;
        return NJS_OK;
    }

    if (njs_slow_path(njs_is_detached_buffer(array->buffer))) {
        njs_type_error(vm, "detached buffer");
        return NJS_ERROR;
    }

    njs_chb_init(&chain, vm->mem_pool);

    length = njs_typed_array_to_chain(vm, &chain, array, separator);
    size = njs_chb_size(&chain);

    p = njs_string_alloc(vm, &vm->retval, size, length);
    if (njs_slow_path(p == NULL)) {
        return NJS_ERROR;
    }

    njs_chb_join_to(&chain, p);
    njs_chb_destroy(&chain);

    return NJS_OK;
}


njs_typed_array_from(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
    njs_index_t unused)
{
    double             num;
    int64_t            length, i;
    njs_int_t          ret;
    njs_value_t        *this, *source, *mapfn;
    njs_value_t        arguments[3], retval;
    njs_function_t     *function;
    njs_typed_array_t  *array;

    this = njs_argument(args, 0);

    if (njs_slow_path(!njs_is_constructor(this))) {
        njs_type_error(vm, "%s is not a constructor",
                       njs_type_string(this->type));
        return NJS_ERROR;
    }

    mapfn = njs_arg(args, nargs, 2);

    if (njs_slow_path(!njs_is_function_or_undefined(mapfn))) {
        njs_type_error(vm, "\"mapfn\" argument is not callable");
        return NJS_ERROR;
    }

    function = NULL;
    if (njs_is_function(mapfn)) {
        function = njs_function(mapfn);
    }

    source = njs_arg(args, nargs, 1);

    ret = njs_value_to_object(vm, source);
    if (njs_slow_path(ret != NJS_OK)) {
        return ret;
    }

    ret = njs_object_length(vm, source, &length);
    if (njs_slow_path(ret == NJS_ERROR)) {
        return ret;
    }

    njs_set_number(&arguments[0], length);
    ret = njs_typed_array_create(vm, this, arguments, 1, &vm->retval);
    if (njs_slow_path(ret != NJS_OK)) {
        return NJS_ERROR;
    }

    array = njs_typed_array(&vm->retval);
    arguments[0] = *njs_arg(args, nargs, 3);

    for (i = 0; i < length; i++) {
        ret = njs_value_property_i64(vm, source, i, &retval);
        if (njs_slow_path(ret == NJS_ERROR)) {
            return NJS_ERROR;
        }

        if (function != NULL) {
            arguments[1] = retval;
            njs_set_number(&arguments[2], i);
            ret = njs_function_apply(vm, function, arguments, 3, &retval);
            if (njs_slow_path(ret != NJS_OK)) {
                return NJS_ERROR;
            }
        }

        ret = njs_value_to_number(vm, &retval, &num);
        if (njs_slow_path(ret != NJS_OK)) {
            return NJS_ERROR;
        }

        njs_typed_array_prop_set(vm, array, i, num);
    }

    njs_set_typed_array(&vm->retval, array);

    return NJS_OK;
}

Fixed typed-array ctor when source array is changed while iterating.

Previously, the function used optimization for ordinary arrays with no
gaps (so called fast arrays).  For a fast array code took elements
directly from internal flat C array. The direct pointer may become
invalid as side-effect of custom valueOf() method for an element.

The fix is to eliminate the micro-optimization which uses direct
pointers.

The problem is similar to the 9578cc729205 (0.7.2) commit.

This closes #523 issue on Github.


njs_typed_array_generic_compare(const void *a, const void *b, void *c)
{
    double                      num;
    njs_int_t                   ret;
    njs_value_t                 arguments[3], retval;
    njs_typed_array_sort_ctx_t  *ctx;

    ctx = c;

    if (njs_slow_path(ctx->exception)) {
        return 0;
    }

    njs_set_undefined(&arguments[0]);
    njs_set_number(&arguments[1], ctx->get(a));
    njs_set_number(&arguments[2], ctx->get(b));

    ret = njs_function_apply(ctx->vm, ctx->function, arguments, 3, &retval);
    if (njs_slow_path(ret != NJS_OK)) {
        goto exception;
    }

    ret = njs_value_to_number(ctx->vm, &retval, &num);
    if (njs_slow_path(ret != NJS_OK)) {
        goto exception;
    }

    if (njs_slow_path(njs_is_detached_buffer(ctx->buffer))) {
        njs_type_error(ctx->vm, "detached buffer");
        goto exception;
    }

    if (njs_slow_path(isnan(num))) {
        return 0;
    }

    if (num != 0) {
        return (num > 0) - (num < 0);
    }

    return 0;

exception:

    ctx->exception = 1;

    return 0;
}


njs_typed_array_writable(njs_vm_t *vm, njs_typed_array_t *array)
{
    njs_int_t           ret;
    njs_array_buffer_t  *buffer;

    buffer = array->buffer;
    if (njs_slow_path(njs_is_detached_buffer(buffer))) {
        njs_type_error(vm, "detached buffer");
        return NULL;
    }

    ret = njs_array_buffer_writable(vm, buffer);
    if (njs_slow_path(ret != NJS_OK)) {
        return NULL;
    }

    return buffer;
}


njs_data_view_prototype_set(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t type)
{
    double              v;
    uint8_t             *u8;
    uint32_t            u32;
    uint64_t            index;
    njs_int_t           ret;
    njs_bool_t          swap;
    njs_value_t         *this;
    njs_conv_f32_t      conv_f32;
    njs_conv_f64_t      conv_f64;
    njs_data_view_t     *view;
    njs_array_buffer_t  *buffer;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_data_view(this))) {
        njs_type_error(vm, "this is not a DataView");
        return NJS_ERROR;
    }

    ret = njs_value_to_index(vm, njs_arg(args, nargs, 1), &index);
    if (njs_slow_path(ret != NJS_OK)) {
        return NJS_ERROR;
    }

    ret = njs_value_to_number(vm, njs_arg(args, nargs, 2), &v);
    if (njs_slow_path(ret != NJS_OK)) {
        return NJS_ERROR;
    }

    swap = njs_bool(njs_arg(args, nargs, 3));

#if NJS_HAVE_LITTLE_ENDIAN
    swap = !swap;
#endif

    view = njs_data_view(this);
    if (njs_slow_path(njs_is_detached_buffer(view->buffer))) {
        njs_type_error(vm, "detached buffer");
        return NJS_ERROR;
    }

    if (njs_typed_array_element_size(type) + index > view->byte_length) {
        njs_range_error(vm, "index %uL is outside the bound of the buffer",
                        index);
        return NJS_ERROR;
    }

    buffer = view->buffer;
    u8 = &buffer->u.u8[index + view->offset];

    switch (type) {
    case NJS_OBJ_TYPE_UINT8_ARRAY:
    case NJS_OBJ_TYPE_INT8_ARRAY:
        *u8 = njs_number_to_int32(v);
        break;

    case NJS_OBJ_TYPE_UINT16_ARRAY:
    case NJS_OBJ_TYPE_INT16_ARRAY:
        u32 = (uint16_t) njs_number_to_int32(v);

        if (swap) {
            u32 = njs_bswap_u16(u32);
        }

        njs_set_u16(u8, u32);
        break;

    case NJS_OBJ_TYPE_UINT32_ARRAY:
    case NJS_OBJ_TYPE_INT32_ARRAY:
        u32 = njs_number_to_int32(v);

        if (swap) {
            u32 = njs_bswap_u32(u32);
        }

        njs_set_u32(u8, u32);
        break;

    case NJS_OBJ_TYPE_FLOAT32_ARRAY:
        conv_f32.f = (float) v;

        if (swap) {
            conv_f32.u = njs_bswap_u32(conv_f32.u);
        }

        njs_set_u32(u8, conv_f32.u);
        break;

    default:
        /* NJS_OBJ_TYPE_FLOAT64_ARRAY. */

        conv_f64.f = v;

        if (swap) {
            conv_f64.u = njs_bswap_u64(conv_f64.u);
        }

        njs_set_u64(u8, conv_f64.u);
    }

    njs_set_undefined(&vm->retval);

    return NJS_OK;
}


njs_typed_array_prototype_copy_within(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t unused)
{
    int64_t             length, to, from, final, count;
    uint32_t            element_size;
    njs_int_t           ret;
    njs_value_t         *this, *value;
    njs_typed_array_t   *array;
    njs_array_buffer_t  *buffer;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_typed_array(this))) {
        njs_type_error(vm, "this is not a typed array");
        return NJS_ERROR;
    }

    array = njs_typed_array(this);
    if (njs_slow_path(njs_is_detached_buffer(array->buffer))) {
        njs_type_error(vm, "detached buffer");
        return NJS_ERROR;
    }

    length = njs_typed_array_length(array);

    value = njs_arg(args, nargs, 1);

    ret = njs_value_to_integer(vm, value, &to);
    if (njs_slow_path(ret != NJS_OK)) {
        return NJS_ERROR;
    }

    to = to < 0 ? njs_max(to + length, 0) : njs_min(to, length);

    value = njs_arg(args, nargs, 2);

    ret = njs_value_to_integer(vm, value, &from);
    if (njs_slow_path(ret != NJS_OK)) {
        return ret;
    }

    from = from < 0 ? njs_max(from + length, 0) : njs_min(from, length);

    value = njs_arg(args, nargs, 3);

    final = length;

    if (njs_is_defined(value)) {
        ret = njs_value_to_integer(vm, value, &final);
        if (njs_slow_path(ret != NJS_OK)) {
            return NJS_ERROR;
        }
    }

    final = (final < 0) ? njs_max(final + length, 0) : njs_min(final, length);

    njs_set_typed_array(&vm->retval, array);

    count = njs_min(final - from, length - to);

    if (count <= 0) {
        return NJS_OK;
    }

    buffer = njs_typed_array_writable(vm, array);
    if (njs_slow_path(buffer == NULL)) {
        return NJS_ERROR;
    }

    element_size = njs_typed_array_element_size(array->type);

    to = (to + array->offset) * element_size;
    from = (from + array->offset) * element_size;
    count = count * element_size;

    memmove(&buffer->u.u8[to], &buffer->u.u8[from], count);

    return NJS_OK;
}


njs_typed_array_prototype_reverse(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t unused)
{
    double              *f64;
    uint8_t             *u8;
    int64_t             i, length;
    uint16_t            *u16;
    uint32_t            *u32;
    njs_value_t         *this;
    njs_typed_array_t   *array;
    njs_array_buffer_t  *buffer;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_typed_array(this))) {
        njs_type_error(vm, "this is not a typed array");
        return NJS_ERROR;
    }

    array = njs_typed_array(this);
    length = njs_typed_array_length(array);

    buffer = njs_typed_array_writable(vm, array);
    if (njs_slow_path(buffer == NULL)) {
        return NJS_ERROR;
    }

    switch (array->type) {
    case NJS_OBJ_TYPE_UINT8_ARRAY:
    case NJS_OBJ_TYPE_UINT8_CLAMPED_ARRAY:
    case NJS_OBJ_TYPE_INT8_ARRAY:
        u8 = &buffer->u.u8[array->offset];

        for (i = 0; i < length / 2; i++) {
            njs_swap_u8(&u8[i], &u8[length - i - 1], 0);
        }

        break;

    case NJS_OBJ_TYPE_INT16_ARRAY:
    case NJS_OBJ_TYPE_UINT16_ARRAY:
        u16 = &buffer->u.u16[array->offset];

        for (i = 0; i < length / 2; i++) {
            njs_swap_u16(&u16[i], &u16[length - i - 1], 0);
        }

        break;

    case NJS_OBJ_TYPE_INT32_ARRAY:
    case NJS_OBJ_TYPE_UINT32_ARRAY:
    case NJS_OBJ_TYPE_FLOAT32_ARRAY:
        u32 = &buffer->u.u32[array->offset];

        for (i = 0; i < length / 2; i++) {
            njs_swap_u32(&u32[i], &u32[length - i - 1], 0);
        }

        break;

    default:

        /* NJS_OBJ_TYPE_FLOAT64_ARRAY. */

        f64 = &buffer->u.f64[array->offset];

        for (i = 0; i < length / 2; i++) {
            njs_swap_u64(&f64[i], &f64[length - i - 1], 0);
        }
    }

    njs_set_typed_array(&vm->retval, array);

    return NJS_OK;
}


njs_typed_array_get_i32(const void *a)
{
    return *(const int32_t *) a;
}


njs_typed_array_get_u32(const void *a)
{
    return *(const uint32_t *) a;
}


njs_typed_array_prototype_buffer(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t unused)
{
    njs_value_t        *this;
    njs_typed_array_t  *array;

    this = njs_argument(args, 0);
    if (!njs_is_typed_array(this) && !njs_is_data_view(this)) {
        njs_type_error(vm, "Method TypedArray.prototype.buffer called "
                       "on incompatible receiver");
        return NJS_ERROR;
    }

    array = njs_typed_array(this);

    njs_set_array_buffer(&vm->retval, njs_typed_array_buffer(array));

    return NJS_OK;
}


njs_typed_array_set_value(njs_vm_t *vm, njs_typed_array_t *array,
    uint32_t index, njs_value_t *setval)
{
    double              num;
    njs_int_t           ret;
    njs_array_buffer_t  *buffer;

    ret = njs_value_to_number(vm, setval, &num);
    if (njs_slow_path(ret != NJS_OK)) {
        return ret;
    }

    buffer = njs_typed_array_writable(vm, array);
    if (njs_slow_path(buffer == NULL)) {
        return NJS_ERROR;
    }

    njs_typed_array_prop_set(vm, array, index, num);

    return NJS_OK;
}


njs_typed_array_get_this(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t unused)
{
    vm->retval = args[0];

    return NJS_OK;
}


njs_typed_array_prototype_length(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t unused)
{
    uint32_t           length;
    njs_value_t        *this;
    njs_typed_array_t  *array;

    this = njs_argument(args, 0);
    if (!njs_is_typed_array(this)) {
        njs_type_error(vm, "Method TypedArray.prototype.length called "
                       "on incompatible receiver");
        return NJS_ERROR;
    }

    array = njs_typed_array(this);
    length = njs_typed_array_length(array);

    if (njs_slow_path(njs_is_detached_buffer(array->buffer))) {
        length = 0;
    }

    njs_set_number(&vm->retval, length);

    return NJS_OK;
}


njs_typed_array_prototype_set(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t unused)
{
    double              num;
    int64_t             i, length, src_length, offset;
    njs_int_t           ret;
    njs_value_t         *this, *src, *value, prop;
    njs_array_t         *array;
    njs_typed_array_t   *self, *src_tarray;
    njs_array_buffer_t  *buffer;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_typed_array(this))) {
        njs_type_error(vm, "this is not a typed array");
        return NJS_ERROR;
    }

    self = njs_typed_array(this);
    src = njs_arg(args, nargs, 1);
    value = njs_arg(args, nargs, 2);

    ret = njs_value_to_integer(vm, value, &offset);
    if (njs_slow_path(ret != NJS_OK)) {
        return NJS_ERROR;
    }

    if (njs_slow_path(offset < 0)) {
        njs_range_error(vm, "offset is out of bounds");
        return NJS_ERROR;
    }

    buffer = njs_typed_array_writable(vm, self);
    if (njs_slow_path(buffer == NULL)) {
        return NJS_ERROR;
    }

    length = njs_typed_array_length(self);

    if (njs_is_typed_array(src)) {
        src_tarray = njs_typed_array(src);
        if (njs_slow_path(njs_is_detached_buffer(src_tarray->buffer))) {
            njs_type_error(vm, "detached buffer");
            return NJS_ERROR;
        }

        src_length = njs_typed_array_length(src_tarray);

        if (njs_slow_path((src_length > length)
                          || (offset > length - src_length)))
        {
            njs_range_error(vm, "source is too large");
            return NJS_ERROR;
        }

        length = njs_min(njs_typed_array_length(src_tarray), length - offset);

        for (i = 0; i < length; i++) {
            njs_typed_array_prop_set(vm, self, offset + i,
                                     njs_typed_array_prop(src_tarray, i));
        }

    } else {
        if (njs_is_fast_array(src)) {
            array = njs_array(src);
            src_length = array->length;

            if (njs_slow_path((src_length > length)
                              || (offset > length - src_length)))
            {
                njs_range_error(vm, "source is too large");
                return NJS_ERROR;
            }

            length = njs_min(array->length, length - offset);

            for (i = 0; i < length; i++) {
                ret = njs_value_to_number(vm, &array->start[i], &num);
                if (ret == NJS_OK) {
                    njs_typed_array_prop_set(vm, self, offset + i, num);
                }
            }

            goto done;
        }

        ret = njs_value_to_object(vm, src);
        if (njs_slow_path(ret != NJS_OK)) {
            return ret;
        }

        ret = njs_object_length(vm, src, &src_length);
        if (njs_slow_path(ret == NJS_ERROR)) {
            return ret;
        }

        if (njs_slow_path((src_length > length)
                          || (offset > length - src_length)))
        {
            njs_range_error(vm, "source is too large");
            return NJS_ERROR;
        }

        length = njs_min(src_length, length - offset);

        for (i = 0; i < length; i++) {
            ret = njs_value_property_i64(vm, src, i, &prop);
            if (njs_slow_path(ret == NJS_ERROR)) {
                return NJS_ERROR;
            }

            num = NAN;

            if (ret == NJS_OK) {
                ret = njs_value_to_number(vm, &prop, &num);
                if (njs_slow_path(ret == NJS_ERROR)) {
                    return NJS_ERROR;
                }
            }

            if (njs_slow_path(njs_is_detached_buffer(buffer))) {
                njs_type_error(vm, "detached buffer");
                return NJS_ERROR;
            }

            njs_typed_array_prop_set(vm, self, offset + i, num);
        }
    }

done:

    njs_set_undefined(&vm->retval);

    return NJS_OK;
}


njs_data_view_prototype_get(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t type)
{
    double              v;
    uint32_t            u32;
    uint64_t            index;
    njs_int_t           ret;
    njs_bool_t          swap;
    njs_value_t         *this;
    const uint8_t       *u8;
    njs_conv_f32_t      conv_f32;
    njs_conv_f64_t      conv_f64;
    njs_data_view_t     *view;
    njs_array_buffer_t  *buffer;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_data_view(this))) {
        njs_type_error(vm, "this is not a DataView");
        return NJS_ERROR;
    }

    ret = njs_value_to_index(vm, njs_arg(args, nargs, 1), &index);
    if (njs_slow_path(ret != NJS_OK)) {
        return NJS_ERROR;
    }

    swap = njs_bool(njs_arg(args, nargs, 2));

#if NJS_HAVE_LITTLE_ENDIAN
    swap = !swap;
#endif

    view = njs_data_view(this);
    if (njs_slow_path(njs_is_detached_buffer(view->buffer))) {
        njs_type_error(vm, "detached buffer");
        return NJS_ERROR;
    }

    if (njs_typed_array_element_size(type) + index > view->byte_length) {
        njs_range_error(vm, "index %uL is outside the bound of the buffer",
                        index);
        return NJS_ERROR;
    }

    buffer = view->buffer;
    u8 = &buffer->u.u8[index + view->offset];

    switch (type) {
    case NJS_OBJ_TYPE_UINT8_ARRAY:
        v = *u8;
        break;

    case NJS_OBJ_TYPE_INT8_ARRAY:
        v = (int8_t) *u8;
        break;

    case NJS_OBJ_TYPE_UINT16_ARRAY:
        u32 = njs_get_u16(u8);

        if (swap) {
            u32 = njs_bswap_u16(u32);
        }

        v = u32;
        break;

    case NJS_OBJ_TYPE_INT16_ARRAY:
        u32 = njs_get_u16(u8);

        if (swap) {
            u32 = njs_bswap_u16(u32);
        }

        v = (int16_t) u32;
        break;

    case NJS_OBJ_TYPE_UINT32_ARRAY:
    case NJS_OBJ_TYPE_INT32_ARRAY:
    case NJS_OBJ_TYPE_FLOAT32_ARRAY:
        u32 = njs_get_u32(u8);

        if (swap) {
            u32 = njs_bswap_u32(u32);
        }

        switch (type) {
        case NJS_OBJ_TYPE_UINT32_ARRAY:
            v = u32;
            break;

        case NJS_OBJ_TYPE_INT32_ARRAY:
            v = (int32_t) u32;
            break;

        default:
            conv_f32.u = u32;
            v = conv_f32.f;
        }

        break;

    default:
        /* NJS_OBJ_TYPE_FLOAT64_ARRAY. */

        conv_f64.u = njs_get_u64(u8);

        if (swap) {
            conv_f64.u = njs_bswap_u64(conv_f64.u);
        }

        v = conv_f64.f;
    }

    njs_set_number(&vm->retval, v);

    return NJS_OK;
}


njs_typed_array_to_chain(njs_vm_t *vm, njs_chb_t *chain,
    njs_typed_array_t *array, njs_value_t *sep)
{
    size_t             size, length, arr_length;
    uint32_t           i;
    njs_string_prop_t  separator;

    if (sep == NULL) {
        sep = njs_value_arg(&njs_string_comma);
    }

    (void) njs_string_prop(&separator, sep);

    arr_length = njs_typed_array_length(array);

    if (arr_length == 0) {
        return 0;
    }

    for (i = 0; i < arr_length; i++) {
        njs_number_to_chain(vm, chain, njs_typed_array_prop(array, i));
        njs_chb_append(chain, separator.start, separator.size);
    }

    njs_chb_drop(chain, separator.size);

    size = njs_chb_size(chain);

    if (njs_utf8_length(separator.start, separator.size) >= 0) {
        length = size - (separator.size - separator.length) * (arr_length - 1);

    } else {
        length = 0;
    }

    return length;
}


njs_typed_array_get_u8(const void *a)
{
    return *(const uint8_t *) a;
}


njs_typed_array_get_f32(const void *a)
{
    return *(const float *) a;
}


njs_data_view_constructor(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
    njs_index_t unused)
{
    uint64_t            size, offset;
    njs_int_t           ret;
    njs_data_view_t     *view;
    njs_array_buffer_t  *buffer;

    if (!vm->top_frame->ctor) {
        njs_type_error(vm, "Constructor of DataView requires 'new'");
        return NJS_ERROR;
    }

    if (!njs_is_array_buffer(njs_arg(args, nargs, 1))) {
        njs_type_error(vm, "buffer is not an ArrayBuffer");
        return NJS_ERROR;
    }

    size = 0;
    offset = 0;

    ret = njs_value_to_index(vm, njs_arg(args, nargs, 2), &offset);
    if (njs_slow_path(ret != NJS_OK)) {
        return NJS_ERROR;
    }

    buffer = njs_array_buffer(njs_argument(args, 1));
    if (njs_slow_path(njs_is_detached_buffer(buffer))) {
        njs_type_error(vm, "detached buffer");
        return NJS_ERROR;
    }

    if (!njs_is_undefined(njs_arg(args, nargs, 3))) {
        ret = njs_value_to_index(vm, njs_argument(args, 3), &size);
        if (njs_slow_path(ret != NJS_OK)) {
            return NJS_ERROR;
        }

        if (njs_slow_path((offset + size) > buffer->size)) {
            njs_range_error(vm, "Invalid DataView length: %uL", size);
            return NJS_ERROR;
        }

    } else {
        if (offset > buffer->size) {
            njs_range_error(vm, "byteOffset %uL is outside the bound of "
                            "the buffer", offset);
            return NJS_ERROR;
        }

        size = buffer->size - offset;
    }

    view = njs_mp_zalloc(vm->mem_pool, sizeof(njs_data_view_t));
    if (njs_slow_path(view == NULL)) {
        goto memory_error;
    }

    view->buffer = buffer;
    view->offset = offset;
    view->byte_length = size;
    view->type = NJS_OBJ_TYPE_DATA_VIEW;

    njs_lvlhsh_init(&view->object.hash);
    njs_lvlhsh_init(&view->object.shared_hash);
    view->object.__proto__ = &vm->prototypes[view->type].object;
    view->object.type = NJS_DATA_VIEW;
    view->object.extensible = 1;

    njs_set_data_view(&vm->retval, view);

    return NJS_OK;

memory_error:

    njs_memory_error(vm);

    return NJS_ERROR;
}


njs_typed_array_prototype_index_of(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t type)
{
    double              v;
    int64_t             i, i64, from, to, index, increment, offset, length;
    njs_int_t           ret, integer;
    njs_value_t         *this;
    const float         *f32;
    const double        *f64;
    const uint8_t       *u8;
    const uint16_t      *u16;
    const uint32_t      *u32;
    njs_typed_array_t   *array;
    njs_array_buffer_t  *buffer;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_typed_array(this))) {
        njs_type_error(vm, "this is not a typed array");
        return NJS_ERROR;
    }

    index = -1;
    array = njs_typed_array(this);
    length = njs_typed_array_length(array);

    if (!njs_is_number(njs_arg(args, nargs, 1)) || length == 0) {
        goto done;
    }

    if (type & 2) {
        /* lastIndexOf(). */

        if (nargs > 2) {
            ret = njs_value_to_integer(vm, njs_arg(args, nargs, 2), &from);
            if (njs_slow_path(ret != NJS_OK)) {
                return ret;
            }

        } else {
            from = length - 1;
        }

        if (from >= 0) {
            from = njs_min(from, length - 1);

        } else if (from < 0) {
            from += length;
        }

        to = -1;
        increment = -1;

        if (from <= to) {
            goto done;
        }

    } else {
        /* indexOf(), includes(). */

        ret = njs_value_to_integer(vm, njs_arg(args, nargs, 2), &from);
        if (njs_slow_path(ret != NJS_OK)) {
            return ret;
        }

        if (from < 0) {
            from += length;

            if (from < 0) {
                from = 0;
            }
        }

        to = length;
        increment = 1;

        if (from >= to) {
            goto done;
        }
    }

    if (njs_slow_path(njs_is_detached_buffer(array->buffer))) {
        njs_type_error(vm, "detached buffer");
        return NJS_ERROR;
    }

    v = njs_number(njs_argument(args, 1));

    i64 = v;
    integer = (v == i64);

    buffer = array->buffer;
    offset = array->offset;

    switch (array->type) {
    case NJS_OBJ_TYPE_INT8_ARRAY:
        if (integer && ((int8_t) i64 == i64)) {
            goto search8;
        }

        break;

    case NJS_OBJ_TYPE_UINT8_CLAMPED_ARRAY:
    case NJS_OBJ_TYPE_UINT8_ARRAY:
        if (integer && ((uint8_t) i64 == i64)) {
search8:
            u8 = &buffer->u.u8[0];
            for (i = from; i != to; i += increment) {
                if (u8[offset + i] == (uint8_t) i64) {
                    index = i;
                    break;
                }
            }
        }

        break;

    case NJS_OBJ_TYPE_INT16_ARRAY:
        if (integer && ((int16_t) i64 == i64)) {
            goto search16;
        }

        break;

    case NJS_OBJ_TYPE_UINT16_ARRAY:
        if (integer && ((uint16_t) i64 == i64)) {
search16:
            u16 = &buffer->u.u16[0];
            for (i = from; i != to; i += increment) {
                if (u16[offset + i] == (uint16_t) i64) {
                    index = i;
                    break;
                }
            }
        }

        break;

    case NJS_OBJ_TYPE_INT32_ARRAY:
        if (integer && ((int32_t) i64 == i64)) {
            goto search32;
        }

        break;

    case NJS_OBJ_TYPE_UINT32_ARRAY:
        if (integer && ((uint32_t) i64 == i64)) {
search32:
            u32 = &buffer->u.u32[0];
            for (i = from; i != to; i += increment) {
                if (u32[offset + i] == (uint32_t) i64) {
                    index = i;
                    break;
                }
            }
        }

        break;

    case NJS_OBJ_TYPE_FLOAT32_ARRAY:
        f32 = &buffer->u.f32[0];

        if (((float) v == v)) {
            for (i = from; i != to; i += increment) {
                if (f32[offset + i] == (float) v) {
                    index = i;
                    break;
                }
            }

        } else if ((type & 1) && isnan(v)) {
            /* includes() handles NaN. */

            for (i = from; i != to; i += increment) {
                if (isnan(f32[offset + i])) {
                    index = i;
                    break;
                }
            }
        }

        break;

    default:

        /* NJS_OBJ_TYPE_FLOAT64_ARRAY. */

        f64 = &buffer->u.f64[0];

        if ((type & 1) && isnan(v)) {
            /* includes() handles NaN. */

            for (i = from; i != to; i += increment) {
                if (isnan(f64[offset + i])) {
                    index = i;
                    break;
                }
            }

        } else {
            for (i = from; i != to; i += increment) {
                if (f64[offset + i] == v) {
                    index = i;
                    break;
                }
            }
        }
    }

done:

    /* Default values. */

    if (type & 1) {
        njs_set_boolean(&vm->retval, index != -1);

    } else {
        njs_set_number(&vm->retval, index);
    }

    return NJS_OK;
}


njs_typed_array_constructor(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
    njs_index_t magic)
{
    njs_typed_array_t  *array;

    if (!vm->top_frame->ctor) {
        njs_type_error(vm, "Constructor of TypedArray requires 'new'");
        return NJS_ERROR;
    }

    array = njs_typed_array_alloc(vm, &args[1], nargs - 1, 1, magic);
    if (njs_slow_path(array == NULL)) {
        return NJS_ERROR;
    }

    njs_set_typed_array(&vm->retval, array);

    return NJS_OK;
}


njs_typed_array_prototype_reduce(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t right)
{
    int64_t             i, from, to, increment, length;
    njs_int_t           ret;
    njs_value_t         *this, accumulator;
    njs_value_t         arguments[5];
    njs_function_t      *function;
    njs_typed_array_t   *array;
    njs_array_buffer_t  *buffer;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_typed_array(this))) {
        njs_type_error(vm, "this is not a typed array");
        return NJS_ERROR;
    }

    array = njs_typed_array(this);
    length = njs_typed_array_length(array);

    if (njs_slow_path(!njs_is_function(njs_arg(args, nargs, 1)))) {
        njs_type_error(vm, "callback argument is not callable");
        return NJS_ERROR;
    }

    function = njs_function(njs_argument(args, 1));

    if (length == 0 && nargs <= 2) {
        njs_type_error(vm, "Reduce of empty object with no initial value");
        return NJS_ERROR;
    }

    if (right) {
        from = length - 1;
        to = -1;
        increment = -1;

    } else {
        from = 0;
        to = length;
        increment = 1;
    }

    buffer = array->buffer;

    if (nargs > 2) {
        accumulator = *njs_argument(args, 2);

    } else {
        if (njs_slow_path(njs_is_detached_buffer(buffer))) {
            njs_type_error(vm, "detached buffer");
            return NJS_ERROR;
        }

        njs_set_number(&accumulator, njs_typed_array_prop(array, from));
        from += increment;
    }

    for (i = from; i != to; i += increment) {
        if (njs_slow_path(njs_is_detached_buffer(buffer))) {
            njs_type_error(vm, "detached buffer");
            return NJS_ERROR;
        }

        njs_set_undefined(&arguments[0]);
        arguments[1] = accumulator;
        njs_set_number(&arguments[2], njs_typed_array_prop(array, i));
        njs_set_number(&arguments[3], i);
        njs_set_typed_array(&arguments[4], array);

        ret =  njs_function_apply(vm, function, arguments, 5, &accumulator);
        if (njs_slow_path(ret != NJS_OK)) {
            return ret;
        }
    }

    vm->retval = accumulator;

    return NJS_OK;
}


njs_typed_array_get_i16(const void *a)
{
    return *(const int16_t *) a;
}


njs_typed_array_alloc(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
    njs_bool_t zeroing, njs_object_type_t type)
{
    double              num;
    int64_t             i, length;
    uint32_t            element_size;
    uint64_t            size, offset;
    njs_int_t           ret;
    njs_value_t         *value, prop;
    njs_typed_array_t   *array, *src_tarray;
    njs_array_buffer_t  *buffer;

    size = 0;
    length = 0;
    offset = 0;

    buffer = NULL;
    src_tarray = NULL;

    element_size = njs_typed_array_element_size(type);

    value = njs_arg(args, nargs, 0);

    if (njs_is_array_buffer(value)) {
        buffer = njs_array_buffer(value);

        ret = njs_value_to_index(vm, njs_arg(args, nargs, 1), &offset);
        if (njs_slow_path(ret != NJS_OK)) {
            return NULL;
        }

        if (njs_slow_path((offset % element_size) != 0)) {
            njs_range_error(vm, "start offset must be multiple of %uD",
                            element_size);
            return NULL;
        }

        if (njs_is_defined(njs_arg(args, nargs, 2))) {
            ret = njs_value_to_index(vm, njs_argument(args, 2), &size);
            if (njs_slow_path(ret != NJS_OK)) {
                return NULL;
            }
        }

        if (njs_slow_path(njs_is_detached_buffer(buffer))) {
            njs_type_error(vm, "detached buffer");
            return NULL;
        }

        if (njs_is_defined(njs_arg(args, nargs, 2))) {
            ret = njs_value_to_index(vm, njs_argument(args, 2), &size);
            if (njs_slow_path(ret != NJS_OK)) {
                return NULL;
            }

            size *= element_size;

            if (njs_slow_path((offset + size) > buffer->size)) {
                njs_range_error(vm, "Invalid typed array length: %uL", size);
                return NULL;
            }

        } else {
            if (njs_slow_path((buffer->size % element_size) != 0)) {
                njs_range_error(vm, "byteLength of buffer must be "
                                "multiple of %uD", element_size);
                return NULL;
            }

            if (offset > buffer->size) {
                njs_range_error(vm, "byteOffset %uL is outside the bound of "
                                "the buffer", offset);
                return NULL;
            }

            size = buffer->size - offset;
        }

    } else if (njs_is_typed_array(value)) {
        src_tarray = njs_typed_array(value);
        if (njs_slow_path(njs_is_detached_buffer(src_tarray->buffer))) {
            njs_type_error(vm, "detached buffer");
            return NULL;
        }

        size = (uint64_t) njs_typed_array_length(src_tarray) * element_size;

    } else if (njs_is_object(value)) {
        ret = njs_object_length(vm, value, &length);
        if (njs_slow_path(ret == NJS_ERROR)) {
            return NULL;
        }

        size = length * element_size;

    } else {
        ret = njs_value_to_index(vm, value, &size);
        if (njs_slow_path(ret != NJS_OK)) {
            return NULL;
        }

        size *= element_size;
    }

    if (buffer == NULL) {
        buffer = njs_array_buffer_alloc(vm, size, zeroing);
        if (njs_slow_path(buffer == NULL)) {
            return NULL;
        }
    }

    array = njs_mp_zalloc(vm->mem_pool, sizeof(njs_typed_array_t));
    if (njs_slow_path(array == NULL)) {
        goto memory_error;
    }

    array->buffer = buffer;
    array->offset = offset / element_size;
    array->byte_length = size;
    array->type = type;

    if (src_tarray != NULL) {
        if (type != src_tarray->type) {
            length = njs_typed_array_length(src_tarray);
            for (i = 0; i < length; i++) {
                njs_typed_array_prop_set(vm, array, i,
                                         njs_typed_array_prop(src_tarray, i));
            }

        } else {
            memcpy(&buffer->u.u8[0], &src_tarray->buffer->u.u8[0], size);
        }

    } else if (!njs_is_array_buffer(value) && njs_is_object(value)) {
        for (i = 0; i < length; i++) {
            ret = njs_value_property_i64(vm, value, i, &prop);
            if (njs_slow_path(ret == NJS_ERROR)) {
                return NULL;
            }

            num = NAN;

            if (ret == NJS_OK) {
                ret = njs_value_to_number(vm, &prop, &num);
                if (njs_slow_path(ret == NJS_ERROR)) {
                    return NULL;
                }
            }

            njs_typed_array_prop_set(vm, array, i, num);
        }
    }

    njs_lvlhsh_init(&array->object.hash);
    njs_lvlhsh_init(&array->object.shared_hash);
    array->object.__proto__ = &vm->prototypes[type].object;
    array->object.type = NJS_TYPED_ARRAY;
    array->object.extensible = 1;
    array->object.fast_array = 1;

    return array;

memory_error:

    njs_memory_error(vm);

    return NULL;
}


njs_typed_array_compare_u16(const void *a, const void *b, void *c)
{
    return *((const uint16_t *) a) - *((const uint16_t *) b);
}


njs_typed_array_get_u16(const void *a)
{
    return *(const uint16_t *) a;
}


njs_typed_array_compare_u8(const void *a, const void *b, void *c)
{
    return *((const uint8_t *) a) - *((const uint8_t *) b);
}


njs_typed_array_create(njs_vm_t *vm, njs_value_t *constructor,
    njs_value_t *args, njs_uint_t nargs, njs_value_t *retval)
{
    njs_int_t          ret;
    njs_value_t        this;
    njs_object_t       *object;
    njs_typed_array_t  *array;

    object = njs_function_new_object(vm, constructor);
    if (njs_slow_path(object == NULL)) {
        return NJS_ERROR;
    }

    njs_set_object(&this, object);

    ret = njs_function_call2(vm, njs_function(constructor), &this, args,
                             nargs, retval, 1);
    if (njs_slow_path(ret != NJS_OK)) {
        return NJS_ERROR;
    }

    if (njs_slow_path(!njs_is_typed_array(retval))) {
        njs_type_error(vm, "Derived TypedArray constructor "
                       "returned not a typed array");
        return NJS_ERROR;
    }

    array = njs_typed_array(retval);
    if (njs_slow_path(njs_is_detached_buffer(array->buffer))) {
        njs_type_error(vm, "detached buffer");
        return NJS_ERROR;
    }

    if (njs_slow_path(nargs == 1 && njs_is_number(&args[0])
                      && njs_typed_array_length(array)
                         < njs_number(&args[0])))
    {
        njs_type_error(vm, "Derived TypedArray constructor "
                       "returned too short array");
        return NJS_ERROR;
    }

    return NJS_OK;
}


njs_typed_array_get_string_tag(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t unused)
{
    njs_value_t  *this;

    static const njs_value_t  *tags[NJS_OBJ_TYPE_TYPED_ARRAY_SIZE] = {
        &njs_typed_array_uint8_tag,
        &njs_typed_array_uint8_clamped_tag,
        &njs_typed_array_int8_tag,
        &njs_typed_array_uint16_tag,
        &njs_typed_array_int16_tag,
        &njs_typed_array_uint32_tag,
        &njs_typed_array_int32_tag,
        &njs_typed_array_float32_tag,
        &njs_typed_array_float64_tag,
    };

    this = njs_argument(args, 0);

    if (!njs_is_typed_array(this)) {
        njs_set_undefined(&vm->retval);
        return NJS_OK;
    }

    vm->retval = *tags[njs_typed_array_index(njs_typed_array(this)->type)];

    return NJS_OK;
}


njs_typed_array_prototype_slice(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t copy)
{
    int64_t             start, end, count, offset;
    uint32_t            i, element_size, length;
    njs_int_t           ret;
    njs_value_t         arguments[3], *this, *value;
    njs_typed_array_t   *array, *new_array;
    njs_array_buffer_t  *buffer, *new_buffer;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_typed_array(this))) {
        njs_type_error(vm, "this is not a typed array");
        return NJS_ERROR;
    }

    array = njs_typed_array(this);
    length = njs_typed_array_length(array);
    buffer = njs_typed_array_buffer(array);
    if (njs_slow_path(copy && njs_is_detached_buffer(buffer))) {
        njs_type_error(vm, "detached buffer");
        return NJS_ERROR;
    }

    ret = njs_value_to_integer(vm, njs_arg(args, nargs, 1), &start);
    if (njs_slow_path(ret != NJS_OK)) {
        njs_range_error(vm, "invalid start");
        return NJS_ERROR;
    }

    start = (start < 0) ? njs_max(length + start, 0) : njs_min(start, length);

    value = njs_arg(args, nargs, 2);

    if (njs_is_undefined(value)) {
        end = length;

    } else {
        ret = njs_value_to_integer(vm, value, &end);
        if (njs_slow_path(ret != NJS_OK)) {
            njs_range_error(vm, "invalid end");
            return NJS_ERROR;
        }
    }

    end = (end < 0) ? njs_max(length + end, 0) : njs_min(end, length);

    element_size = njs_typed_array_element_size(array->type);

    if (copy) {
        count = njs_max(end - start, 0);
        njs_set_number(&arguments[0], count);

        ret = njs_typed_array_species_create(vm, this,
                                             njs_value_arg(arguments), 1,
                                             &vm->retval);
        if (njs_slow_path(ret != NJS_OK)) {
            return ret;
        }

        if (count == 0) {
            return NJS_OK;
        }

        if (njs_slow_path(njs_is_detached_buffer(buffer))) {
            njs_type_error(vm, "detached buffer");
            return NJS_ERROR;
        }

        new_array = njs_typed_array(&vm->retval);
        new_buffer = njs_typed_array_buffer(new_array);
        element_size = njs_typed_array_element_size(array->type);

        if (njs_fast_path(array->type == new_array->type)) {
            start = start * element_size;
            count = count * element_size;

            memcpy(&new_buffer->u.u8[0], &buffer->u.u8[start], count);

        } else {
            for (i = 0; i < count; i++) {
                njs_typed_array_prop_set(vm, new_array, i,
                                        njs_typed_array_prop(array, i + start));
            }
        }

        return NJS_OK;
    }

    offset = array->offset * element_size;
    offset += start * element_size;

    njs_set_array_buffer(&arguments[0], buffer);
    njs_set_number(&arguments[1], offset);
    njs_set_number(&arguments[2], njs_max(end - start, 0));

    return njs_typed_array_species_create(vm, this, njs_value_arg(arguments), 3,
                                          &vm->retval);
}


njs_typed_array_compare_u32(const void *a, const void *b, void *c)
{
    uint32_t  au, bu;

    au = *(const uint32_t *) a;
    bu = *(const uint32_t *) b;

    return (au > bu) - (au < bu);
}


njs_typed_array_prototype_sort(njs_vm_t *vm, njs_value_t *args,
    njs_uint_t nargs, njs_index_t unused)
{
    u_char                      *base, *orig;
    int64_t                     length;
    uint32_t                    element_size;
    njs_value_t                 *this, *comparefn;
    njs_typed_array_t           *array;
    njs_array_buffer_t          *buffer;
    njs_typed_array_cmp_t       cmp;
    njs_typed_array_sort_ctx_t  ctx;

    this = njs_argument(args, 0);
    if (njs_slow_path(!njs_is_typed_array(this))) {
        njs_type_error(vm, "this is not a typed array");
        return NJS_ERROR;
    }

    array = njs_typed_array(this);
    if (njs_slow_path(njs_is_detached_buffer(array->buffer))) {
        njs_type_error(vm, "detached buffer");
        return NJS_ERROR;
    }

    ctx.vm = vm;
    ctx.buffer = array->buffer;
    ctx.exception = 0;

    comparefn = njs_arg(args, nargs, 1);

    if (njs_is_defined(comparefn)) {
        if (njs_slow_path(!njs_is_function(comparefn))) {
            njs_type_error(vm, "comparefn must be callable or undefined");
            return NJS_ERROR;
        }

        ctx.function = njs_function(comparefn);

    } else {
        ctx.function = NULL;
    }

    switch (array->type) {
    case NJS_OBJ_TYPE_UINT8_ARRAY:
    case NJS_OBJ_TYPE_UINT8_CLAMPED_ARRAY:
        cmp = njs_typed_array_compare_u8;
        ctx.get = njs_typed_array_get_u8;
        break;

    case NJS_OBJ_TYPE_INT8_ARRAY:
        cmp = njs_typed_array_compare_i8;
        ctx.get = njs_typed_array_get_i8;
        break;

    case NJS_OBJ_TYPE_UINT16_ARRAY:
        cmp = njs_typed_array_compare_u16;
        ctx.get = njs_typed_array_get_u16;
        break;

    case NJS_OBJ_TYPE_INT16_ARRAY:
        cmp = njs_typed_array_compare_i16;
        ctx.get = njs_typed_array_get_i16;
        break;

    case NJS_OBJ_TYPE_UINT32_ARRAY:
        cmp = njs_typed_array_compare_u32;
        ctx.get = njs_typed_array_get_u32;
        break;

    case NJS_OBJ_TYPE_INT32_ARRAY:
        cmp = njs_typed_array_compare_i32;
        ctx.get = njs_typed_array_get_i32;
        break;

    case NJS_OBJ_TYPE_FLOAT32_ARRAY:
        cmp = njs_typed_array_compare_f32;
        ctx.get = njs_typed_array_get_f32;
        break;

    default:

        /* NJS_OBJ_TYPE_FLOAT64_ARRAY. */

        cmp = njs_typed_array_compare_f64;
        ctx.get = njs_typed_array_get_f64;
        break;
    }

    buffer = njs_typed_array_writable(vm, array);
    if (njs_slow_path(buffer == NULL)) {
        return NJS_ERROR;
    }

    length = njs_typed_array_length(array);
    element_size = njs_typed_array_element_size(array->type);
    base = &buffer->u.u8[array->offset * element_size];
    orig = base;

    if (ctx.function != NULL) {
        cmp = njs_typed_array_generic_compare;
        base = njs_mp_alloc(vm->mem_pool, length * element_size);
        if (njs_slow_path(base == NULL)) {
            njs_memory_error(vm);
            return NJS_ERROR;
        }

        memcpy(base, &buffer->u.u8[array->offset * element_size],
               length * element_size);
    }

    njs_qsort(base, length, element_size, cmp, &ctx);
    if (ctx.function != NULL) {
        if (&buffer->u.u8[array->offset * element_size] == orig) {
            memcpy(orig, base, length * element_size);
        }

        njs_mp_free(vm->mem_pool, base);
    }

    if (njs_slow_path(ctx.exception)) {
        return NJS_ERROR;
    }

    njs_set_typed_array(&vm->retval, array);

    return NJS_OK;
}


njs_typed_array_get_f64(const void *a)
{
    return *(const double *) a;
}


const char* DocumentSourceGroup::getSourceName() const {
    return kStageName.rawData();
}

SERVER-60218-44: SERVER-60218 add initialize helper function for document_source_group (cherry picked from commit 867f52afbb79bc00e35c70f8e0681b7d602f97b2)


intrusive_ptr<DocumentSourceGroup> DocumentSourceGroup::create(
    const intrusive_ptr<ExpressionContext>& pExpCtx,
    const boost::intrusive_ptr<Expression>& groupByExpression,
    std::vector<AccumulationStatement> accumulationStatements,
    boost::optional<size_t> maxMemoryUsageBytes) {
    size_t memoryBytes = maxMemoryUsageBytes ? *maxMemoryUsageBytes
                                             : internalDocumentSourceGroupMaxMemoryBytes.load();
    intrusive_ptr<DocumentSourceGroup> groupStage(new DocumentSourceGroup(pExpCtx, memoryBytes));
    groupStage->setIdExpression(groupByExpression);
    for (auto&& statement : accumulationStatements) {
        groupStage->addAccumulator(statement);
    }

    return groupStage;
}


    bool operator()(const GroupsMap::value_type* lhs, const GroupsMap::value_type* rhs) const {
        return _valueComparator.evaluate(lhs->first < rhs->first);
    }


int DocumentSourceGroup::freeMemory() {
    invariant(_groups);
    int totalMemorySaved = 0;
    for (auto&& group : *_groups) {
        for (auto&& groupObj : group.second) {
            auto prevMemUsage = groupObj->memUsageForSorter();
            groupObj->reduceMemoryConsumptionIfAble();

            // Update the memory usage for this group.
            totalMemorySaved += (prevMemUsage - groupObj->memUsageForSorter());
        }
    }
    return totalMemorySaved;
}


DocumentSource::GetNextResult DocumentSourceGroup::doGetNext() {
    if (!_initialized) {
        const auto initializationResult = initialize();
        if (initializationResult.isPaused()) {
            return initializationResult;
        }
        invariant(initializationResult.isEOF());
    }

    for (auto&& accum : _currentAccumulators) {
        accum->reset();  // Prep accumulators for a new group.
    }

    if (_spilled) {
        return getNextSpilled();
    } else {
        return getNextStandard();
    }
}


boost::optional<DocumentSource::DistributedPlanLogic> DocumentSourceGroup::distributedPlanLogic() {
    intrusive_ptr<DocumentSourceGroup> mergingGroup(new DocumentSourceGroup(pExpCtx));
    mergingGroup->setDoingMerge(true);

    VariablesParseState vps = pExpCtx->variablesParseState;
    /* the merger will use the same grouping key */
    mergingGroup->setIdExpression(ExpressionFieldPath::parse(pExpCtx, "$$ROOT._id", vps));

    for (auto&& accumulatedField : _accumulatedFields) {
        // The merger's output field names will be the same, as will the accumulator factories.
        // However, for some accumulators, the expression to be accumulated will be different. The
        // original accumulator may be collecting an expression based on a field expression or
        // constant.  Here, we accumulate the output of the same name from the prior group.
        auto copiedAccumulatedField = accumulatedField;
        copiedAccumulatedField.expr.argument =
            ExpressionFieldPath::parse(pExpCtx, "$$ROOT." + copiedAccumulatedField.fieldName, vps);
        mergingGroup->addAccumulator(copiedAccumulatedField);
    }

    // {shardsStage, mergingStage, sortPattern}
    return DistributedPlanLogic{this, mergingGroup, boost::none};
}


Document GroupFromFirstDocumentTransformation::serializeTransformation(
    boost::optional<ExplainOptions::Verbosity> explain) const {

    MutableDocument newRoot(_accumulatorExprs.size());
    for (auto&& expr : _accumulatorExprs) {
        newRoot.addField(expr.first, expr.second->serialize(static_cast<bool>(explain)));
    }

    return {{"newRoot", newRoot.freezeToValue()}};
}


Value DocumentSourceGroup::serialize(boost::optional<ExplainOptions::Verbosity> explain) const {
    MutableDocument insides;

    // Add the _id.
    if (_idFieldNames.empty()) {
        invariant(_idExpressions.size() == 1);
        insides["_id"] = _idExpressions[0]->serialize(static_cast<bool>(explain));
    } else {
        // Decomposed document case.
        invariant(_idExpressions.size() == _idFieldNames.size());
        MutableDocument md;
        for (size_t i = 0; i < _idExpressions.size(); i++) {
            md[_idFieldNames[i]] = _idExpressions[i]->serialize(static_cast<bool>(explain));
        }
        insides["_id"] = md.freezeToValue();
    }

    // Add the remaining fields.
    for (auto&& accumulatedField : _accumulatedFields) {
        intrusive_ptr<AccumulatorState> accum = accumulatedField.makeAccumulator();
        insides[accumulatedField.fieldName] =
            Value(accum->serialize(accumulatedField.expr.initializer,
                                   accumulatedField.expr.argument,
                                   static_cast<bool>(explain)));
    }

    if (_doingMerge) {
        // This makes the output unparsable (with error) on pre 2.6 shards, but it will never
        // be sent to old shards when this flag is true since they can't do a merge anyway.
        insides["$doingMerge"] = Value(true);
    }

    return Value(DOC(getSourceName() << insides.freeze()));
}


bool DocumentSourceGroup::canRunInParallelBeforeWriteStage(
    const std::set<std::string>& nameOfShardKeyFieldsUponEntryToStage) const {
    if (_doingMerge) {
        return true;  // This is fine.
    }

    // Certain $group stages are allowed to execute on each exchange consumer. In order to
    // guarantee each consumer will only group together data from its own shard, the $group must
    // group on a superset of the shard key.
    for (auto&& currentPathOfShardKey : nameOfShardKeyFieldsUponEntryToStage) {
        if (!pathIncludedInGroupKeys(currentPathOfShardKey)) {
            // This requires an exact path match, but as a future optimization certain path
            // prefixes should be okay. For example, if the shard key path is "a.b", and we're
            // grouping by "a", then each group of "a" is strictly more specific than "a.b", so
            // we can deduce that grouping by "a" will not need to group together documents
            // across different values of the shard key field "a.b", and thus as long as any
            // other shard key fields are similarly preserved will not need to consume a merged
            // stream to perform the group.
            return false;
        }
    }
    return true;
}


Value DocumentSourceGroup::expandId(const Value& val) {
    // _id doesn't get wrapped in a document
    if (_idFieldNames.empty())
        return val;

    // _id is a single-field document containing val
    if (_idFieldNames.size() == 1)
        return Value(DOC(_idFieldNames[0] << val));

    // _id is a multi-field document containing the elements of val
    const vector<Value>& vals = val.getArray();
    invariant(_idFieldNames.size() == vals.size());
    MutableDocument md(vals.size());
    for (size_t i = 0; i < vals.size(); i++) {
        md[_idFieldNames[i]] = vals[i];
    }
    return md.freezeToValue();
}


MONGO_COMPILER_NOINLINE DocumentSource::GetNextResult DocumentSourceGroup::initializeSelf(
    GetNextResult input) {
    const size_t numAccumulators = _accumulatedFields.size();
    // Barring any pausing, this loop exhausts 'pSource' and populates '_groups'.
    for (; input.isAdvanced(); input = pSource->getNext()) {
        if (_memoryTracker.shouldSpillWithAttemptToSaveMemory([this]() { return freeMemory(); })) {
            _sortedFiles.push_back(spill());
        }

        // We release the result document here so that it does not outlive the end of this loop
        // iteration. Not releasing could lead to an array copy when this group follows an unwind.
        auto rootDocument = input.releaseDocument();
        Value id = computeId(rootDocument);

        // Look for the _id value in the map. If it's not there, add a new entry with a blank
        // accumulator. This is done in a somewhat odd way in order to avoid hashing 'id' and
        // looking it up in '_groups' multiple times.
        const size_t oldSize = _groups->size();
        vector<intrusive_ptr<AccumulatorState>>& group = (*_groups)[id];
        const bool inserted = _groups->size() != oldSize;

        if (inserted) {
            _memoryTracker.memoryUsageBytes += id.getApproximateSize();

            // Initialize and add the accumulators
            Value expandedId = expandId(id);
            Document idDoc =
                expandedId.getType() == BSONType::Object ? expandedId.getDocument() : Document();
            group.reserve(numAccumulators);
            for (auto&& accumulatedField : _accumulatedFields) {
                auto accum = accumulatedField.makeAccumulator();
                Value initializerValue =
                    accumulatedField.expr.initializer->evaluate(idDoc, &pExpCtx->variables);
                accum->startNewGroup(initializerValue);
                group.push_back(accum);
            }
        } else {
            for (auto&& groupObj : group) {
                // subtract old mem usage. New usage added back after processing.
                _memoryTracker.memoryUsageBytes -= groupObj->memUsageForSorter();
            }
        }

        /* tickle all the accumulators for the group we found */
        dassert(numAccumulators == group.size());

        for (size_t i = 0; i < numAccumulators; i++) {
            group[i]->process(
                _accumulatedFields[i].expr.argument->evaluate(rootDocument, &pExpCtx->variables),
                _doingMerge);

            _memoryTracker.memoryUsageBytes += group[i]->memUsageForSorter();
        }

        if (kDebugBuild && !storageGlobalParams.readOnly) {
            // In debug mode, spill every time we have a duplicate id to stress merge logic.
            if (!inserted &&                     // is a dup
                !pExpCtx->inMongos &&            // can't spill to disk in mongos
                !_memoryTracker.allowDiskUse &&  // don't change behavior when testing external sort
                _sortedFiles.size() < 20) {      // don't open too many FDs

                _sortedFiles.push_back(spill());
            }
        }
    }

    switch (input.getStatus()) {
        case DocumentSource::GetNextResult::ReturnStatus::kAdvanced: {
            MONGO_UNREACHABLE;  // We consumed all advances above.
        }
        case DocumentSource::GetNextResult::ReturnStatus::kPauseExecution: {
            return input;  // Propagate pause.
        }
        case DocumentSource::GetNextResult::ReturnStatus::kEOF: {
            // Do any final steps necessary to prepare to output results.
            if (!_sortedFiles.empty()) {
                _spilled = true;
                if (!_groups->empty()) {
                    _sortedFiles.push_back(spill());
                }

                // We won't be using groups again so free its memory.
                _groups = pExpCtx->getValueComparator().makeUnorderedValueMap<Accumulators>();

                _sorterIterator.reset(Sorter<Value, Value>::Iterator::merge(
                    _sortedFiles, SortOptions(), SorterComparator(pExpCtx->getValueComparator())));

                // prepare current to accumulate data
                _currentAccumulators.reserve(numAccumulators);
                for (auto&& accumulatedField : _accumulatedFields) {
                    _currentAccumulators.push_back(accumulatedField.makeAccumulator());
                }

                verify(_sorterIterator->more());  // we put data in, we should get something out.
                _firstPartOfNextGroup = _sorterIterator->next();
            } else {
                // start the group iterator
                groupsIterator = _groups->begin();
            }

            // This must happen last so that, unless control gets here, we will re-enter
            // initialization after getting a GetNextResult::ResultState::kPauseExecution.
            _initialized = true;
            return input;
        }
    }
    MONGO_UNREACHABLE;
}


const std::vector<AccumulationStatement>& DocumentSourceGroup::getAccumulatedFields() const {
    return _accumulatedFields;
}


bool DocumentSourceGroup::pathIncludedInGroupKeys(const std::string& dottedPath) const {
    return std::any_of(
        _idExpressions.begin(), _idExpressions.end(), [&dottedPath](const auto& exp) {
            if (auto fieldExp = dynamic_cast<ExpressionFieldPath*>(exp.get())) {
                if (fieldExp->representsPath(dottedPath)) {
                    return true;
                }
            }
            return false;
        });
}


void DocumentSourceGroup::addAccumulator(AccumulationStatement accumulationStatement) {
    _accumulatedFields.push_back(accumulationStatement);
}


bool DocumentSourceGroup::MemoryUsageTracker::shouldSpillWithAttemptToSaveMemory(
    std::function<int()> saveMemory) {
    if (!allowDiskUse && (memoryUsageBytes > maxMemoryUsageBytes)) {
        memoryUsageBytes -= saveMemory();
    }

    if (memoryUsageBytes > maxMemoryUsageBytes) {
        uassert(ErrorCodes::QueryExceededMemoryLimitNoDiskUseAllowed,
                "Exceeded memory limit for $group, but didn't allow external sort."
                " Pass allowDiskUse:true to opt in.",
                allowDiskUse);
        memoryUsageBytes = 0;
        return true;
    }
    return false;
}


SpillSTLComparator(ValueComparator valueComparator) : _valueComparator(valueComparator) {}


Document DocumentSourceGroup::makeDocument(const Value& id,
                                           const Accumulators& accums,
                                           bool mergeableOutput) {
    const size_t n = _accumulatedFields.size();
    MutableDocument out(1 + n);

    /* add the _id field */
    out.addField("_id", expandId(id));

    /* add the rest of the fields */
    for (size_t i = 0; i < n; ++i) {
        Value val = accums[i]->getValue(mergeableOutput);
        if (val.missing()) {
            // we return null in this case so return objects are predictable
            out.addField(_accumulatedFields[i].fieldName, Value(BSONNULL));
        } else {
            out.addField(_accumulatedFields[i].fieldName, val);
        }
    }

    return out.freeze();
}


StringMap<boost::intrusive_ptr<Expression>> DocumentSourceGroup::getIdFields() const {
    if (_idFieldNames.empty()) {
        invariant(_idExpressions.size() == 1);
        return {{"_id", _idExpressions[0]}};
    } else {
        invariant(_idFieldNames.size() == _idExpressions.size());
        StringMap<boost::intrusive_ptr<Expression>> result;
        for (std::size_t i = 0; i < _idFieldNames.size(); ++i) {
            result["_id." + _idFieldNames[i]] = _idExpressions[i];
        }
        return result;
    }
}


DepsTracker::State DocumentSourceGroup::getDependencies(DepsTracker* deps) const {
    // add the _id
    for (size_t i = 0; i < _idExpressions.size(); i++) {
        _idExpressions[i]->addDependencies(deps);
    }

    // add the rest
    for (auto&& accumulatedField : _accumulatedFields) {
        accumulatedField.expr.argument->addDependencies(deps);
        // Don't add initializer, because it doesn't refer to docs from the input stream.
    }

    return DepsTracker::State::EXHAUSTIVE_ALL;
}


DocumentSource::GetModPathsReturn DocumentSourceGroup::getModifiedPaths() const {
    // We preserve none of the fields, but any fields referenced as part of the group key are
    // logically just renamed.
    StringMap<std::string> renames;
    for (std::size_t i = 0; i < _idExpressions.size(); ++i) {
        auto idExp = _idExpressions[i];
        auto pathToPutResultOfExpression =
            _idFieldNames.empty() ? "_id" : "_id." + _idFieldNames[i];
        auto computedPaths = idExp->getComputedPaths(pathToPutResultOfExpression);
        for (auto&& rename : computedPaths.renames) {
            renames[rename.first] = rename.second;
        }
    }

    return {DocumentSource::GetModPathsReturn::Type::kAllExcept,
            std::set<std::string>{},  // No fields are preserved.
            std::move(renames)};
}


void DocumentSourceGroup::doDispose() {
    // Free our resources.
    _groups = pExpCtx->getValueComparator().makeUnorderedValueMap<Accumulators>();
    _sorterIterator.reset();

    // Make us look done.
    groupsIterator = _groups->end();
}


DepsTracker::State GroupFromFirstDocumentTransformation::addDependencies(DepsTracker* deps) const {
    for (auto&& expr : _accumulatorExprs) {
        expr.second->addDependencies(deps);
    }

    // This stage will replace the entire document with a new document, so any existing fields
    // will be replaced and cannot be required as dependencies. We use EXHAUSTIVE_ALL here
    // instead of EXHAUSTIVE_FIELDS, as in ReplaceRootTransformation, because the stages that
    // follow a $group stage should not depend on document metadata.
    return DepsTracker::State::EXHAUSTIVE_ALL;
}


std::string nextFileName() {
    static AtomicWord<unsigned> documentSourceGroupFileCounter;
    return "extsort-doc-group." + std::to_string(documentSourceGroupFileCounter.fetchAndAdd(1));
}


intrusive_ptr<DocumentSource> DocumentSourceGroup::createFromBson(
    BSONElement elem, const intrusive_ptr<ExpressionContext>& pExpCtx) {
    uassert(15947, "a group's fields must be specified in an object", elem.type() == Object);

    intrusive_ptr<DocumentSourceGroup> pGroup(new DocumentSourceGroup(pExpCtx));

    BSONObj groupObj(elem.Obj());
    BSONObjIterator groupIterator(groupObj);
    VariablesParseState vps = pExpCtx->variablesParseState;
    while (groupIterator.more()) {
        BSONElement groupField(groupIterator.next());
        StringData pFieldName = groupField.fieldNameStringData();
        if (pFieldName == "_id") {
            uassert(
                15948, "a group's _id may only be specified once", pGroup->_idExpressions.empty());
            pGroup->setIdExpression(parseIdExpression(pExpCtx, groupField, vps));
            invariant(!pGroup->_idExpressions.empty());
        } else if (pFieldName == "$doingMerge") {
            massert(17030, "$doingMerge should be true if present", groupField.Bool());

            pGroup->setDoingMerge(true);
        } else {
            // Any other field will be treated as an accumulator specification.
            pGroup->addAccumulator(
                AccumulationStatement::parseAccumulationStatement(pExpCtx, groupField, vps));
        }
    }

    uassert(15955, "a group specification must include an _id", !pGroup->_idExpressions.empty());
    return pGroup;
}


DocumentSource::GetNextResult DocumentSourceGroup::getNextStandard() {
    // Not spilled, and not streaming.
    if (_groups->empty())
        return GetNextResult::makeEOF();

    Document out = makeDocument(groupsIterator->first, groupsIterator->second, pExpCtx->needsMerge);

    if (++groupsIterator == _groups->end())
        dispose();

    return std::move(out);
}


    int operator()(const Data& lhs, const Data& rhs) const {
        return _valueComparator.compare(lhs.first, rhs.first);
    }


bool DocumentSourceGroup::usedDisk() {
    return _usedDisk;
}


intrusive_ptr<Expression> parseIdExpression(const intrusive_ptr<ExpressionContext>& expCtx,
                                            BSONElement groupField,
                                            const VariablesParseState& vps) {
    if (groupField.type() == Object) {
        // {_id: {}} is treated as grouping on a constant, not an expression
        if (groupField.Obj().isEmpty()) {
            return ExpressionConstant::create(expCtx, Value(groupField));
        }

        const BSONObj idKeyObj = groupField.Obj();
        if (idKeyObj.firstElementFieldName()[0] == '$') {
            // grouping on a $op expression
            return Expression::parseObject(expCtx, idKeyObj, vps);
        } else {
            for (auto&& field : idKeyObj) {
                uassert(17390,
                        "$group does not support inclusion-style expressions",
                        !field.isNumber() && field.type() != Bool);
            }
            return ExpressionObject::parse(expCtx, idKeyObj, vps);
        }
    } else {
        return Expression::parseOperand(expCtx, groupField, vps);
    }
}


shared_ptr<Sorter<Value, Value>::Iterator> DocumentSourceGroup::spill() {
    _usedDisk = true;
    vector<const GroupsMap::value_type*> ptrs;  // using pointers to speed sorting
    ptrs.reserve(_groups->size());
    for (GroupsMap::const_iterator it = _groups->begin(), end = _groups->end(); it != end; ++it) {
        ptrs.push_back(&*it);
    }

    stable_sort(ptrs.begin(), ptrs.end(), SpillSTLComparator(pExpCtx->getValueComparator()));

    SortedFileWriter<Value, Value> writer(SortOptions().TempDir(pExpCtx->tempDir), _file);
    switch (_accumulatedFields.size()) {  // same as ptrs[i]->second.size() for all i.
        case 0:                           // no values, essentially a distinct
            for (size_t i = 0; i < ptrs.size(); i++) {
                writer.addAlreadySorted(ptrs[i]->first, Value());
            }
            break;

        case 1:  // just one value, use optimized serialization as single Value
            for (size_t i = 0; i < ptrs.size(); i++) {
                writer.addAlreadySorted(ptrs[i]->first,
                                        ptrs[i]->second[0]->getValue(/*toBeMerged=*/true));
            }
            break;

        default:  // multiple values, serialize as array-typed Value
            for (size_t i = 0; i < ptrs.size(); i++) {
                vector<Value> accums;
                for (size_t j = 0; j < ptrs[i]->second.size(); j++) {
                    accums.push_back(ptrs[i]->second[j]->getValue(/*toBeMerged=*/true));
                }
                writer.addAlreadySorted(ptrs[i]->first, Value(std::move(accums)));
            }
            break;
    }

    _groups->clear();

    Sorter<Value, Value>::Iterator* iteratorPtr = writer.done();
    return shared_ptr<Sorter<Value, Value>::Iterator>(iteratorPtr);
}


Value DocumentSourceGroup::computeId(const Document& root) {
    // If only one expression, return result directly
    if (_idExpressions.size() == 1) {
        Value retValue = _idExpressions[0]->evaluate(root, &pExpCtx->variables);
        return retValue.missing() ? Value(BSONNULL) : std::move(retValue);
    }

    // Multiple expressions get results wrapped in a vector
    vector<Value> vals;
    vals.reserve(_idExpressions.size());
    for (size_t i = 0; i < _idExpressions.size(); i++) {
        vals.push_back(_idExpressions[i]->evaluate(root, &pExpCtx->variables));
    }
    return Value(std::move(vals));
}


DocumentSourceGroup::DocumentSourceGroup(const intrusive_ptr<ExpressionContext>& pExpCtx,
                                         boost::optional<size_t> maxMemoryUsageBytes)
    : DocumentSource(kStageName, pExpCtx),
      _usedDisk(false),
      _doingMerge(false),
      _memoryTracker{pExpCtx->allowDiskUse && !pExpCtx->inMongos,
                     maxMemoryUsageBytes ? *maxMemoryUsageBytes
                                         : internalDocumentSourceGroupMaxMemoryBytes.load()},
      // We spill to disk in debug mode, regardless of allowDiskUse, to stress the system.
      _file(!pExpCtx->inMongos && (pExpCtx->allowDiskUse || kDebugBuild)
                ? std::make_shared<Sorter<Value, Value>::File>(pExpCtx->tempDir + "/" +
                                                               nextFileName())
                : nullptr),
      _initialized(false),
      _groups(pExpCtx->getValueComparator().makeUnorderedValueMap<Accumulators>()),
      _spilled(false) {}


DocumentSource::GetModPathsReturn GroupFromFirstDocumentTransformation::getModifiedPaths() const {
    // Replaces the entire root, so all paths are modified.
    return {DocumentSource::GetModPathsReturn::Type::kAllPaths, std::set<std::string>{}, {}};
}


DocumentSource::GetNextResult DocumentSourceGroup::initialize() {
    GetNextResult input = pSource->getNext();
    return initializeSelf(input);
}


DocumentSourceGroup::rewriteGroupAsTransformOnFirstDocument() const {
    if (_idExpressions.size() != 1) {
        // This transformation is only intended for $group stages that group on a single field.
        return nullptr;
    }

    auto fieldPathExpr = dynamic_cast<ExpressionFieldPath*>(_idExpressions.front().get());
    if (!fieldPathExpr || !fieldPathExpr->isRootFieldPath()) {
        return nullptr;
    }

    const auto fieldPath = fieldPathExpr->getFieldPath();
    if (fieldPath.getPathLength() == 1) {
        // The path is $$CURRENT or $$ROOT. This isn't really a sensible value to group by (since
        // each document has a unique _id, it will just return the entire collection). We only
        // apply the rewrite when grouping by a single field, so we cannot apply it in this case,
        // where we are grouping by the entire document.
        invariant(fieldPath.getFieldName(0) == "CURRENT" || fieldPath.getFieldName(0) == "ROOT");
        return nullptr;
    }

    const auto groupId = fieldPath.tail().fullPath();

    // We can't do this transformation if there are any non-$first accumulators.
    for (auto&& accumulator : _accumulatedFields) {
        if (AccumulatorDocumentsNeeded::kFirstDocument !=
            accumulator.makeAccumulator()->documentsNeeded()) {
            return nullptr;
        }
    }

    std::vector<std::pair<std::string, boost::intrusive_ptr<Expression>>> fields;

    boost::intrusive_ptr<Expression> idField;
    // The _id field can be specified either as a fieldpath (ex. _id: "$a") or as a singleton
    // object (ex. _id: {v: "$a"}).
    if (_idFieldNames.empty()) {
        idField = ExpressionFieldPath::create(pExpCtx.get(), groupId);
    } else {
        invariant(_idFieldNames.size() == 1);
        idField = ExpressionObject::create(pExpCtx.get(),
                                           {{_idFieldNames.front(), _idExpressions.front()}});
    }
    fields.push_back(std::make_pair("_id", idField));

    for (auto&& accumulator : _accumulatedFields) {
        fields.push_back(std::make_pair(accumulator.fieldName, accumulator.expr.argument));

        // Since we don't attempt this transformation for non-$first accumulators,
        // the initializer should always be trivial.
    }

    return GroupFromFirstDocumentTransformation::create(pExpCtx, groupId, std::move(fields));
}


void GroupFromFirstDocumentTransformation::optimize() {
    for (auto&& expr : _accumulatorExprs) {
        expr.second = expr.second->optimize();
    }
}


std::unique_ptr<GroupFromFirstDocumentTransformation> GroupFromFirstDocumentTransformation::create(
    const intrusive_ptr<ExpressionContext>& expCtx,
    const std::string& groupId,
    vector<pair<std::string, intrusive_ptr<Expression>>> accumulatorExprs) {
    return std::make_unique<GroupFromFirstDocumentTransformation>(groupId,
                                                                  std::move(accumulatorExprs));
}


intrusive_ptr<DocumentSource> DocumentSourceGroup::optimize() {
    // TODO: If all _idExpressions are ExpressionConstants after optimization, then we know there
    // will be only one group. We should take advantage of that to avoid going through the hash
    // table.
    for (size_t i = 0; i < _idExpressions.size(); i++) {
        _idExpressions[i] = _idExpressions[i]->optimize();
    }

    for (auto&& accumulatedField : _accumulatedFields) {
        accumulatedField.expr.initializer = accumulatedField.expr.initializer->optimize();
        accumulatedField.expr.argument = accumulatedField.expr.argument->optimize();
    }

    return this;
}


Document GroupFromFirstDocumentTransformation::applyTransformation(const Document& input) {
    MutableDocument output(_accumulatorExprs.size());

    for (auto&& expr : _accumulatorExprs) {
        auto value = expr.second->evaluate(input, &expr.second->getExpressionContext()->variables);
        output.addField(expr.first, value.missing() ? Value(BSONNULL) : value);
    }

    return output.freeze();
}


SorterComparator(ValueComparator valueComparator) : _valueComparator(valueComparator) {}


DocumentSource::GetNextResult DocumentSourceGroup::getNextSpilled() {
    // We aren't streaming, and we have spilled to disk.
    if (!_sorterIterator)
        return GetNextResult::makeEOF();

    _currentId = _firstPartOfNextGroup.first;
    const size_t numAccumulators = _accumulatedFields.size();

    // Call startNewGroup on every accumulator.
    Value expandedId = expandId(_currentId);
    Document idDoc =
        expandedId.getType() == BSONType::Object ? expandedId.getDocument() : Document();
    for (size_t i = 0; i < numAccumulators; ++i) {
        Value initializerValue =
            _accumulatedFields[i].expr.initializer->evaluate(idDoc, &pExpCtx->variables);
        _currentAccumulators[i]->startNewGroup(initializerValue);
    }

    while (pExpCtx->getValueComparator().evaluate(_currentId == _firstPartOfNextGroup.first)) {
        // Inside of this loop, _firstPartOfNextGroup is the current data being processed.
        // At loop exit, it is the first value to be processed in the next group.
        switch (numAccumulators) {  // mirrors switch in spill()
            case 1:                 // Single accumulators serialize as a single Value.
                _currentAccumulators[0]->process(_firstPartOfNextGroup.second, true);
            case 0:  // No accumulators so no Values.
                break;
            default: {  // Multiple accumulators serialize as an array of Values.
                const vector<Value>& accumulatorStates = _firstPartOfNextGroup.second.getArray();
                for (size_t i = 0; i < numAccumulators; i++) {
                    _currentAccumulators[i]->process(accumulatorStates[i], true);
                }
            }
        }

        if (!_sorterIterator->more()) {
            dispose();
            break;
        }

        _firstPartOfNextGroup = _sorterIterator->next();
    }

    return makeDocument(_currentId, _currentAccumulators, pExpCtx->needsMerge);
}


void DocumentSourceGroup::setIdExpression(const boost::intrusive_ptr<Expression> idExpression) {

    if (auto object = dynamic_cast<ExpressionObject*>(idExpression.get())) {
        auto& childExpressions = object->getChildExpressions();
        invariant(!childExpressions.empty());  // We expect to have converted an empty object into a
                                               // constant expression.

        // grouping on an "artificial" object. Rather than create the object for each input
        // in initialize(), instead group on the output of the raw expressions. The artificial
        // object will be created at the end in makeDocument() while outputting results.
        for (auto&& childExpPair : childExpressions) {
            _idFieldNames.push_back(childExpPair.first);
            _idExpressions.push_back(childExpPair.second);
        }
    } else {
        _idExpressions.push_back(idExpression);
    }
}


static int adts_write_header(AVFormatContext *s)
{
    ADTSContext *adts = s->priv_data;

    if (adts->id3v2tag)
        ff_id3v2_write_simple(s, 4, ID3v2_DEFAULT_MAGIC);

    return 0;
}

avformat/adtsenc: return value check for init_get_bits in adts_decode_extradata

As the second argument for init_get_bits (buf) can be crafted, a return value check for this function call is necessary.
'buf' is  part of  'AVPacket pkt'.
replace init_get_bits with init_get_bits8.

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>


static int adts_write_frame_header(ADTSContext *ctx,
                                   uint8_t *buf, int size, int pce_size)
{
    PutBitContext pb;

    unsigned full_frame_size = (unsigned)ADTS_HEADER_SIZE + size + pce_size;
    if (full_frame_size > ADTS_MAX_FRAME_BYTES) {
        av_log(NULL, AV_LOG_ERROR, "ADTS frame size too large: %u (max %d)\n",
               full_frame_size, ADTS_MAX_FRAME_BYTES);
        return AVERROR_INVALIDDATA;
    }

    init_put_bits(&pb, buf, ADTS_HEADER_SIZE);

    /* adts_fixed_header */
    put_bits(&pb, 12, 0xfff);   /* syncword */
    put_bits(&pb, 1, ctx->mpeg_id); /* ID */
    put_bits(&pb, 2, 0);        /* layer */
    put_bits(&pb, 1, 1);        /* protection_absent */
    put_bits(&pb, 2, ctx->objecttype); /* profile_objecttype */
    put_bits(&pb, 4, ctx->sample_rate_index);
    put_bits(&pb, 1, 0);        /* private_bit */
    put_bits(&pb, 3, ctx->channel_conf); /* channel_configuration */
    put_bits(&pb, 1, 0);        /* original_copy */
    put_bits(&pb, 1, 0);        /* home */

    /* adts_variable_header */
    put_bits(&pb, 1, 0);        /* copyright_identification_bit */
    put_bits(&pb, 1, 0);        /* copyright_identification_start */
    put_bits(&pb, 13, full_frame_size); /* aac_frame_length */
    put_bits(&pb, 11, 0x7ff);   /* adts_buffer_fullness */
    put_bits(&pb, 2, 0);        /* number_of_raw_data_blocks_in_frame */

    flush_put_bits(&pb);

    return 0;
}


static int adts_decode_extradata(AVFormatContext *s, ADTSContext *adts, const uint8_t *buf, int size)
{
    GetBitContext gb;
    PutBitContext pb;
    MPEG4AudioConfig m4ac;
    int off, ret;

    ret = init_get_bits8(&gb, buf, size);
    if (ret < 0)
        return ret;
    off = avpriv_mpeg4audio_get_config2(&m4ac, buf, size, 1, s);
    if (off < 0)
        return off;
    skip_bits_long(&gb, off);
    adts->objecttype        = m4ac.object_type - 1;
    adts->sample_rate_index = m4ac.sampling_index;
    adts->channel_conf      = m4ac.chan_config;

    if (adts->objecttype > 3U) {
        av_log(s, AV_LOG_ERROR, "MPEG-4 AOT %d is not allowed in ADTS\n", adts->objecttype+1);
        return AVERROR_INVALIDDATA;
    }
    if (adts->sample_rate_index == 15) {
        av_log(s, AV_LOG_ERROR, "Escape sample rate index illegal in ADTS\n");
        return AVERROR_INVALIDDATA;
    }
    if (get_bits(&gb, 1)) {
        av_log(s, AV_LOG_ERROR, "960/120 MDCT window is not allowed in ADTS\n");
        return AVERROR_INVALIDDATA;
    }
    if (get_bits(&gb, 1)) {
        av_log(s, AV_LOG_ERROR, "Scalable configurations are not allowed in ADTS\n");
        return AVERROR_INVALIDDATA;
    }
    if (get_bits(&gb, 1)) {
        av_log(s, AV_LOG_ERROR, "Extension flag is not allowed in ADTS\n");
        return AVERROR_INVALIDDATA;
    }
    if (!adts->channel_conf) {
        init_put_bits(&pb, adts->pce_data, MAX_PCE_SIZE);

        put_bits(&pb, 3, 5); //ID_PCE
        adts->pce_size = (ff_copy_pce_data(&pb, &gb) + 3) / 8;
        flush_put_bits(&pb);
    }

    adts->write_adts = 1;

    return 0;
}


static int adts_write_trailer(AVFormatContext *s)
{
    ADTSContext *adts = s->priv_data;

    if (adts->apetag)
        ff_ape_write_tag(s);

    return 0;
}


static int adts_init(AVFormatContext *s)
{
    ADTSContext *adts = s->priv_data;
    AVCodecParameters *par = s->streams[0]->codecpar;

    if (par->codec_id != AV_CODEC_ID_AAC) {
        av_log(s, AV_LOG_ERROR, "Only AAC streams can be muxed by the ADTS muxer\n");
        return AVERROR(EINVAL);
    }
    if (par->extradata_size > 0)
        return adts_decode_extradata(s, adts, par->extradata,
                                     par->extradata_size);

    return 0;
}


static int adts_write_packet(AVFormatContext *s, AVPacket *pkt)
{
    ADTSContext *adts = s->priv_data;
    AVCodecParameters *par = s->streams[0]->codecpar;
    AVIOContext *pb = s->pb;
    uint8_t buf[ADTS_HEADER_SIZE];

    if (!pkt->size)
        return 0;
    if (!par->extradata_size) {
        uint8_t *side_data;
        size_t side_data_size;
        int ret;

        side_data = av_packet_get_side_data(pkt, AV_PKT_DATA_NEW_EXTRADATA,
                                            &side_data_size);
        if (side_data_size) {
            ret = adts_decode_extradata(s, adts, side_data, side_data_size);
            if (ret < 0)
                return ret;
            ret = ff_alloc_extradata(par, side_data_size);
            if (ret < 0)
                return ret;
            memcpy(par->extradata, side_data, side_data_size);
        }
    }
    if (adts->write_adts) {
        int err = adts_write_frame_header(adts, buf, pkt->size,
                                             adts->pce_size);
        if (err < 0)
            return err;
        avio_write(pb, buf, ADTS_HEADER_SIZE);
        if (adts->pce_size) {
            avio_write(pb, adts->pce_data, adts->pce_size);
            adts->pce_size = 0;
        }
    }
    avio_write(pb, pkt->data, pkt->size);

    return 0;
}


void ompl::geometric::VFRRT::setup()
{
    RRT::setup();
    vfdim_ = si_->getStateSpace()->getValueLocations().size();
}

fix memory leak in VFRRT. closes #839


Eigen::VectorXd ompl::geometric::VFRRT::computeAlphaBeta(double omega, const Eigen::VectorXd &vrand,
                                                         const Eigen::VectorXd &vfield)
{
    double w2 = omega * omega;
    double c = vfield.dot(vrand);
    double cc_1 = c * c - 1.;
    double root = std::sqrt(cc_1 * w2 * (w2 - 4.));
    double beta = -root / (2. * cc_1);
    double sign = (beta < 0.) ? -1. : 1.;
    beta *= sign;
    double alpha = (sign * c * root + cc_1 * (2. - w2)) / (2. * cc_1);
    return alpha * vfield + beta * vrand;
}

fix memory leak in VFRRT. closes #839


void ompl::geometric::VFRRT::clear()
{
    RRT::clear();
    efficientCount_ = 0;
    inefficientCount_ = 0;
    explorationInefficiency_ = 0.;
    step_ = 0;
}

fix memory leak in VFRRT. closes #839


double ompl::geometric::VFRRT::determineMeanNorm()
{
    ompl::base::State *rstate = si_->allocState();
    double sum = 0.;
    for (unsigned int i = 0; i < magic::VFRRT_MEAN_NORM_SAMPLES; i++)
    {
        sampler_->sampleUniform(rstate);
        sum += vf_(rstate).norm();
    }
    si_->freeState(rstate);
    return sum / magic::VFRRT_MEAN_NORM_SAMPLES;
}

fix memory leak in VFRRT. closes #839


void ompl::geometric::VFRRT::updateExplorationEfficiency(Motion *m)
{
    Motion *near = nn_->nearest(m);
    if (distanceFunction(m, near) < si_->getStateValidityCheckingResolution())
        inefficientCount_++;
    else
        efficientCount_++;
    explorationInefficiency_ = inefficientCount_ / (double)(efficientCount_ + inefficientCount_);
}

fix memory leak in VFRRT. closes #839


double ompl::geometric::VFRRT::biasedSampling(const Eigen::VectorXd &vrand, const Eigen::VectorXd &vfield,
                                              double lambdaScale)
{
    double sigma = .25 * (vrand - vfield).squaredNorm();
    updateGain();
    double scaledLambda = lambda_ * lambdaScale / meanNorm_;
    double phi = scaledLambda / (1. - std::exp(-2. * scaledLambda));
    double z = -std::log(1. - sigma * scaledLambda / phi) / scaledLambda;
    return std::sqrt(2. * z);
}

fix memory leak in VFRRT. closes #839


ompl::geometric::VFRRT::VFRRT(const base::SpaceInformationPtr &si, VectorField vf, double exploration,
                              double initial_lambda, unsigned int update_freq)
  : RRT(si)
  , vf_(std::move(vf))
  , explorationSetting_(exploration)
  , lambda_(initial_lambda)
  , nth_step_(update_freq)
{
    setName("VFRRT");
    maxDistance_ = si->getStateValidityCheckingResolution();
}

fix memory leak in VFRRT. closes #839


Eigen::VectorXd ompl::geometric::VFRRT::getNewDirection(const base::State *qnear, const base::State *qrand)
{
    // Set vrand to be the normalized vector from qnear to qrand
    Eigen::VectorXd vrand(vfdim_);
    for (unsigned int i = 0; i < vfdim_; i++)
        vrand[i] = *si_->getStateSpace()->getValueAddressAtIndex(qrand, i) -
                   *si_->getStateSpace()->getValueAddressAtIndex(qnear, i);
    vrand /= si_->distance(qnear, qrand);

    // Get the vector at qnear, and normalize
    Eigen::VectorXd vfield = vf_(qnear);
    const double lambdaScale = vfield.norm();
    // In the case where there is no vector field present, vfield.norm() == 0,
    // return the direction of the random state.
    if (lambdaScale < std::numeric_limits<float>::epsilon())
        return vrand;
    vfield /= lambdaScale;
    // Sample a weight from the distribution
    const double omega = biasedSampling(vrand, vfield, lambdaScale);
    // Determine updated direction
    return computeAlphaBeta(omega, vrand, vfield);
}

fix memory leak in VFRRT. closes #839


ompl::geometric::VFRRT::Motion *ompl::geometric::VFRRT::extendTree(Motion *m, base::State *rstate,
                                                                   const Eigen::VectorXd &v)
{
    base::State *newState = si_->allocState();
    si_->copyState(newState, m->state);

    double d = si_->distance(m->state, rstate);
    if (d > maxDistance_)
        d = maxDistance_;

    const base::StateSpacePtr &space = si_->getStateSpace();
    for (unsigned int i = 0; i < vfdim_; i++)
        *space->getValueAddressAtIndex(newState, i) += d * v[i];
    if (!v.hasNaN() && si_->checkMotion(m->state, newState))
    {
        auto *motion = new Motion();
        motion->state = newState;
        motion->parent = m;
        updateExplorationEfficiency(motion);
        nn_->add(motion);
        return motion;
    }
    else
    {
        si_->freeState(newState);
        inefficientCount_++;
        return nullptr;
    }
}

fix memory leak in VFRRT. closes #839


ompl::base::PlannerStatus ompl::geometric::VFRRT::solve(const base::PlannerTerminationCondition &ptc)
{
    checkValidity();
    base::Goal *goal = pdef_->getGoal().get();
    auto *goal_s = dynamic_cast<base::GoalSampleableRegion *>(goal);

    if (!sampler_)
        sampler_ = si_->allocStateSampler();

    meanNorm_ = determineMeanNorm();

    while (const base::State *st = pis_.nextStart())
    {
        auto *motion = new Motion(si_);
        si_->copyState(motion->state, st);
        nn_->add(motion);
    }

    if (nn_->size() == 0)
    {
        OMPL_ERROR("%s: There are no valid initial states!", getName().c_str());
        return base::PlannerStatus::INVALID_START;
    }

    OMPL_INFORM("%s: Starting planning with %u states already in datastructure", getName().c_str(), nn_->size());

    Motion *solution = nullptr;
    Motion *approxsol = nullptr;
    double approxdif = std::numeric_limits<double>::infinity();
    auto *rmotion = new Motion(si_);
    base::State *rstate = rmotion->state;
    base::State *xstate = si_->allocState();

    while (ptc == false)
    {
        // Sample random state (with goal biasing)
        if (goal_s && rng_.uniform01() < goalBias_ && goal_s->canSample())
            goal_s->sampleGoal(rstate);
        else
            sampler_->sampleUniform(rstate);

        // Find closest state in the tree
        Motion *nmotion = nn_->nearest(rmotion);

        // Modify direction based on vector field before extending
        Motion *motion = extendTree(nmotion, rstate, getNewDirection(nmotion->state, rstate));
        if (!motion)
            continue;

        // Check if we can connect to the goal
        double dist = 0;
        bool sat = goal->isSatisfied(motion->state, &dist);
        if (sat)
        {
            approxdif = dist;
            solution = motion;
            break;
        }
        if (dist < approxdif)
        {
            approxdif = dist;
            approxsol = motion;
        }
    }

    bool solved = false;
    bool approximate = false;
    if (solution == nullptr)
    {
        solution = approxsol;
        approximate = true;
    }

    if (solution != nullptr)
    {
        lastGoalMotion_ = solution;

        // Construct the solution path
        std::vector<Motion *> mpath;
        while (solution != nullptr)
        {
            mpath.push_back(solution);
            solution = solution->parent;
        }

        // Set the solution path
        auto path(std::make_shared<PathGeometric>(si_));
        for (int i = mpath.size() - 1; i >= 0; --i)
            path->append(mpath[i]->state);
        pdef_->addSolutionPath(path, approximate, approxdif, name_);
        solved = true;
    }

    si_->freeState(xstate);
    if (rmotion->state)
        si_->freeState(rmotion->state);
    delete rmotion;

    OMPL_INFORM("%s: Created %u states", getName().c_str(), nn_->size());

    return {solved, approximate};
}

fix memory leak in VFRRT. closes #839


void ompl::geometric::VFRRT::updateGain()
{
    if (step_ == nth_step_)
    {
        lambda_ = lambda_ * (1 - explorationInefficiency_ + explorationSetting_);
        efficientCount_ = inefficientCount_ = 0;
        explorationInefficiency_ = 0;
        step_ = 0;
    }
    else
        step_++;
}

fix memory leak in VFRRT. closes #839


int unit_name_to_instance(const char *n, char **ret) {
        const char *p, *d;

        assert(n);

        if (!unit_name_is_valid(n, UNIT_NAME_ANY))
                return -EINVAL;

        /* Everything past the first @ and before the last . is the instance */
        p = strchr(n, '@');
        if (!p) {
                if (ret)
                        *ret = NULL;
                return UNIT_NAME_PLAIN;
        }

        p++;

        d = strrchr(p, '.');
        if (!d)
                return -EINVAL;

        if (ret) {
                char *i = strndup(p, d-p);
                if (!i)
                        return -ENOMEM;

                *ret = i;
        }
        return d > p ? UNIT_NAME_INSTANCE : UNIT_NAME_TEMPLATE;
}

basic/unit-name: do not use strdupa() on a path

The path may have unbounded length, for example through a fuse mount.

CVE-2021-33910: attacked controlled alloca() leads to crash in systemd and
ultimately a kernel panic. Systemd parses the content of /proc/self/mountinfo
and each mountpoint is passed to mount_setup_unit(), which calls
unit_name_path_escape() underneath. A local attacker who is able to mount a
filesystem with a very long path can crash systemd and the whole system.

https://bugzilla.redhat.com/show_bug.cgi?id=1970887

The resulting string length is bounded by UNIT_NAME_MAX, which is 256. But we
can't easily check the length after simplification before doing the
simplification, which in turns uses a copy of the string we can write to.
So we can't reject paths that are too long before doing the duplication.
Hence the most obvious solution is to switch back to strdup(), as before
7410616cd9dbbec97cf98d75324da5cda2b2f7a2.

(cherry picked from commit 441e0115646d54f080e5c3bb0ba477c892861ab9)
(cherry picked from commit 764b74113e36ac5219a4b82a05f311b5a92136ce)
(cherry picked from commit 4a1c5f34bd3e1daed4490e9d97918e504d19733b)


int unit_name_build(const char *prefix, const char *instance, const char *suffix, char **ret) {
        UnitType type;

        assert(prefix);
        assert(suffix);
        assert(ret);

        if (suffix[0] != '.')
                return -EINVAL;

        type = unit_type_from_string(suffix + 1);
        if (type < 0)
                return -EINVAL;

        return unit_name_build_from_type(prefix, instance, type, ret);
}


int unit_name_path_escape(const char *f, char **ret) {
        _cleanup_free_ char *p = NULL;
        char *s;

        assert(f);
        assert(ret);

        p = strdup(f);
        if (!p)
                return -ENOMEM;

        path_simplify(p, false);

        if (empty_or_root(p))
                s = strdup("-");
        else {
                if (!path_is_normalized(p))
                        return -EINVAL;

                /* Truncate trailing slashes and skip leading slashes */
                delete_trailing_chars(p, "/");
                s = unit_name_escape(skip_leading_chars(p, "/"));
        }
        if (!s)
                return -ENOMEM;

        *ret = s;
        return 0;
}


njs_module_require(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs,
    njs_index_t unused)
{
    njs_int_t    ret;
    njs_str_t    name;
    njs_mod_t    *module;
    njs_value_t  *path;

    if (nargs < 2) {
        njs_type_error(vm, "missing path");
        return NJS_ERROR;
    }

    path = njs_argument(args, 1);
    ret = njs_value_to_string(vm, path, path);
    if (njs_slow_path(ret != NJS_OK)) {
        return ret;
    }

    njs_string_get(path, &name);

    module = njs_module_find(vm, &name, 0);
    if (njs_slow_path(module == NULL)) {
        njs_error(vm, "Cannot find module \"%V\"", &name);

        return NJS_ERROR;
    }

    njs_value_assign(&vm->retval, &module->value);

    return NJS_OK;
}

Fixed typo while calculating module path length.

The issue was introduced in 77c398f26d7e (not released yet).


njs_module_find(njs_vm_t *vm, njs_str_t *name, njs_bool_t shared)
{
    njs_int_t           ret;
    njs_mod_t           *shrd, *module;
    njs_object_t        *object;
    njs_lvlhsh_query_t  lhq;

    lhq.key = *name;
    lhq.key_hash = njs_djb_hash(name->start, name->length);
    lhq.proto = &njs_modules_hash_proto;

    if (njs_lvlhsh_find(&vm->modules_hash, &lhq) == NJS_OK) {
        return lhq.value;
    }

    if (njs_lvlhsh_find(&vm->shared->modules_hash, &lhq) == NJS_OK) {
        shrd = lhq.value;

        if (shared) {
            return shrd;
        }

        module = njs_mp_alloc(vm->mem_pool, sizeof(njs_mod_t));
        if (njs_slow_path(module == NULL)) {
            njs_memory_error(vm);
            return NULL;
        }

        memcpy(module, shrd, sizeof(njs_mod_t));

        object = njs_object_value_copy(vm, &module->value);
        if (njs_slow_path(object == NULL)) {
            return NULL;
        }

        lhq.replace = 0;
        lhq.value = module;
        lhq.pool = vm->mem_pool;

        ret = njs_lvlhsh_insert(&vm->modules_hash, &lhq);
        if (njs_fast_path(ret == NJS_OK)) {
            return module;
        }
    }

    return NULL;
}


njs_module_add(njs_vm_t *vm, njs_str_t *name)
{
    njs_int_t           ret;
    njs_mod_t           *module;
    njs_lvlhsh_query_t  lhq;

    module = njs_mp_zalloc(vm->mem_pool, sizeof(njs_mod_t));
    if (njs_slow_path(module == NULL)) {
        njs_memory_error(vm);
        return NULL;
    }

    ret = njs_name_copy(vm, &module->name, name);
    if (njs_slow_path(ret != NJS_OK)) {
        njs_memory_error(vm);
        return NULL;
    }

    lhq.replace = 0;
    lhq.key = *name;
    lhq.key_hash = njs_djb_hash(name->start, name->length);
    lhq.value = module;
    lhq.pool = vm->mem_pool;
    lhq.proto = &njs_modules_hash_proto;

    ret = njs_lvlhsh_insert(&vm->shared->modules_hash, &lhq);
    if (njs_fast_path(ret == NJS_OK)) {
        return module;
    }

    njs_mp_free(vm->mem_pool, module->name.start);
    njs_mp_free(vm->mem_pool, module);

    njs_internal_error(vm, "lvlhsh insert failed");

    return NULL;
}


njs_module_path(njs_vm_t *vm, const njs_str_t *dir, njs_module_info_t *info)
{
    char        *p;
    size_t      length;
    njs_bool_t  trail;
    char        src[NJS_MAX_PATH + 1];

    trail = 0;
    length = info->name.length;

    if (dir != NULL) {
        length += dir->length;

        if (length == 0) {
            return NJS_DECLINED;
        }

        trail = (dir->start[dir->length - 1] != '/');

        if (trail) {
            length++;
        }
    }

    if (njs_slow_path(length > NJS_MAX_PATH)) {
        return NJS_ERROR;
    }

    p = &src[0];

    if (dir != NULL) {
        p = (char *) njs_cpymem(p, dir->start, dir->length);

        if (trail) {
            *p++ = '/';
        }
    }

    p = (char *) njs_cpymem(p, info->name.start, info->name.length);
    *p = '\0';

    p = realpath(&src[0], &info->path[0]);
    if (p == NULL) {
        return NJS_DECLINED;
    }

    info->fd = open(&info->path[0], O_RDONLY);
    if (info->fd < 0) {
        return NJS_DECLINED;
    }

    info->file.start = (u_char *) &info->path[0];
    info->file.length = njs_strlen(info->file.start);

    return NJS_OK;
}


njs_module_read(njs_vm_t *vm, int fd, njs_str_t *text)
{
    ssize_t      n;
    struct stat  sb;

    text->start = NULL;

    if (fstat(fd, &sb) == -1) {
        goto fail;
    }

    if (!S_ISREG(sb.st_mode)) {
        goto fail;
    }

    text->length = sb.st_size;

    text->start = njs_mp_alloc(vm->mem_pool, text->length);
    if (text->start == NULL) {
        goto fail;
    }

    n = read(fd, text->start, sb.st_size);

    if (n < 0 || n != sb.st_size) {
        goto fail;
    }

    return NJS_OK;

fail:

    if (text->start != NULL) {
        njs_mp_free(vm->mem_pool, text->start);
    }

    return NJS_ERROR;
}


njs_parser_module(njs_parser_t *parser, njs_str_t *name)
{
    njs_mod_t            *module;
    njs_vm_t             *vm;
    njs_external_ptr_t   external;
    njs_module_loader_t  loader;

    vm = parser->vm;

    if (name->length == 0) {
        njs_parser_syntax_error(parser, "Cannot find module \"%V\"", name);
        return NULL;
    }

    module = njs_module_find(vm, name, 1);
    if (module != NULL) {
        goto done;
    }

    external = parser;
    loader = njs_default_module_loader;

    if (vm->options.ops != NULL && vm->options.ops->module_loader != NULL) {
        loader = vm->options.ops->module_loader;
        external = vm->external;
    }

    module = loader(vm, external, name);
    if (module == NULL) {
        njs_parser_syntax_error(parser, "Cannot find module \"%V\"", name);
        return NULL;
    }

done:

    if (module->index == 0) {
        module->index = vm->shared->module_items++;
    }

    return module;
}