str_new(VALUE klass, const char *ptr, long len)
{
    VALUE str;

    if (len < 0) {
	rb_raise(rb_eArgError, "negative string size (or size too big)");
    }

    str = str_alloc(klass);
    if (len > RSTRING_EMBED_LEN_MAX) {
	RSTRING(str)->as.heap.aux.capa = len;
	RSTRING(str)->as.heap.ptr = ALLOC_N(char,len+1);
	STR_SET_NOEMBED(str);
    }
    else if (len == 0) {
	ENC_CODERANGE_SET(str, ENC_CODERANGE_7BIT);
    }
    if (ptr) {
	memcpy(RSTRING_PTR(str), ptr, len);
    }
    STR_SET_LEN(str, len);
    RSTRING_PTR(str)[len] = '\0';
    return str;
}


rb_str_resize(VALUE str, long len)
{
    long slen;

    if (len < 0) {
	rb_raise(rb_eArgError, "negative string size (or size too big)");
    }

    rb_str_modify(str);
    slen = RSTRING_LEN(str);
    if (len != slen) {
	if (STR_EMBED_P(str)) {
	    char *ptr;
	    if (len <= RSTRING_EMBED_LEN_MAX) {
		STR_SET_EMBED_LEN(str, len);
		RSTRING(str)->as.ary[len] = '\0';
		return str;
	    }
	    ptr = ALLOC_N(char,len+1);
	    MEMCPY(ptr, RSTRING(str)->as.ary, char, slen);
	    RSTRING(str)->as.heap.ptr = ptr;
	    STR_SET_NOEMBED(str);
	}
	else if (len <= RSTRING_EMBED_LEN_MAX) {
	    char *ptr = RSTRING(str)->as.heap.ptr;
	    STR_SET_EMBED(str);
	    if (slen > 0) MEMCPY(RSTRING(str)->as.ary, ptr, char, len);
	    RSTRING(str)->as.ary[len] = '\0';
	    STR_SET_EMBED_LEN(str, len);
	    xfree(ptr);
	    return str;
	}
	else if (slen < len || slen - len > 1024) {
	    REALLOC_N(RSTRING(str)->as.heap.ptr, char, len+1);
	}
	if (!STR_NOCAPA_P(str)) {
	    RSTRING(str)->as.heap.aux.capa = len;
	}
	RSTRING(str)->as.heap.len = len;
	RSTRING(str)->as.heap.ptr[len] = '\0';	/* sentinel */
    }
    return str;
}

* string.c (rb_str_justify): CVE-2009-4124.
  Fixes a bug reported by
  Emmanouel Kellinis <Emmanouel.Kellinis AT kpmg.co.uk>, KPMG London;
  Patch by nobu.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@26038 b2dd03c8-39d4-4d8f-98ff-823fe69b080e


rb_str_chomp(int argc, VALUE *argv, VALUE str)
{
    str = rb_str_dup(str);
    rb_str_chomp_bang(argc, argv, str);
    return str;
}


rb_str_casecmp(VALUE str1, VALUE str2)
{
    long len;
    rb_encoding *enc;
    char *p1, *p1end, *p2, *p2end;

    StringValue(str2);
    enc = rb_enc_compatible(str1, str2);
    if (!enc) {
	return Qnil;
    }

    p1 = RSTRING_PTR(str1); p1end = RSTRING_END(str1);
    p2 = RSTRING_PTR(str2); p2end = RSTRING_END(str2);
    if (single_byte_optimizable(str1) && single_byte_optimizable(str2)) {
	while (p1 < p1end && p2 < p2end) {
	    if (*p1 != *p2) {
		unsigned int c1 = TOUPPER(*p1 & 0xff);
		unsigned int c2 = TOUPPER(*p2 & 0xff);
                if (c1 != c2)
                    return INT2FIX(c1 < c2 ? -1 : 1);
	    }
	    p1++;
	    p2++;
	}
    }
    else {
	while (p1 < p1end && p2 < p2end) {
            int l1, c1 = rb_enc_ascget(p1, p1end, &l1, enc);
            int l2, c2 = rb_enc_ascget(p2, p2end, &l2, enc);

            if (0 <= c1 && 0 <= c2) {
                c1 = TOUPPER(c1);
                c2 = TOUPPER(c2);
                if (c1 != c2)
                    return INT2FIX(c1 < c2 ? -1 : 1);
            }
            else {
                int r;
                l1 = rb_enc_mbclen(p1, p1end, enc);
                l2 = rb_enc_mbclen(p2, p2end, enc);
                len = l1 < l2 ? l1 : l2;
                r = memcmp(p1, p2, len);
                if (r != 0)
                    return INT2FIX(r < 0 ? -1 : 1);
                if (l1 != l2)
                    return INT2FIX(l1 < l2 ? -1 : 1);
            }
	    p1 += l1;
	    p2 += l2;
	}
    }
    if (RSTRING_LEN(str1) == RSTRING_LEN(str2)) return INT2FIX(0);
    if (RSTRING_LEN(str1) > RSTRING_LEN(str2)) return INT2FIX(1);
    return INT2FIX(-1);
}


rb_str_set_len(VALUE str, long len)
{
    STR_SET_LEN(str, len);
    RSTRING_PTR(str)[len] = '\0';
}


rb_str_reverse_bang(VALUE str)
{
    if (RSTRING_LEN(str) > 1) {
	if (single_byte_optimizable(str)) {
	    char *s, *e, c;

	    str_modify_keep_cr(str);
	    s = RSTRING_PTR(str);
	    e = RSTRING_END(str) - 1;
	    while (s < e) {
		c = *s;
		*s++ = *e;
		*e-- = c;
	    }
	}
	else {
	    rb_str_shared_replace(str, rb_str_reverse(str));
	}
    }
    else {
	str_modify_keep_cr(str);
    }
    return str;
}


str_make_independent(VALUE str)
{
    char *ptr;
    long len = RSTRING_LEN(str);

    ptr = ALLOC_N(char, len+1);
    if (RSTRING_PTR(str)) {
	memcpy(ptr, RSTRING_PTR(str), len);
    }
    STR_SET_NOEMBED(str);
    ptr[len] = 0;
    RSTRING(str)->as.heap.ptr = ptr;
    RSTRING(str)->as.heap.len = len;
    RSTRING(str)->as.heap.aux.capa = len;
    STR_UNSET_NOCAPA(str);
}


tr_find(unsigned int c, char table[256], VALUE del, VALUE nodel)
{
    if (c < 256) {
	return table[c] != 0;
    }
    else {
	VALUE v = UINT2NUM(c);

	if (del && !NIL_P(rb_hash_lookup(del, v))) {
	    if (!nodel || NIL_P(rb_hash_lookup(nodel, v))) {
		return TRUE;
	    }
	}
	return FALSE;
    }
}


rb_str_tmp_new(long len)
{
    return str_new(0, 0, len);
}


rb_str_chomp_bang(int argc, VALUE *argv, VALUE str)
{
    rb_encoding *enc;
    VALUE rs;
    int newline;
    char *p, *pp, *e;
    long len, rslen;

    str_modify_keep_cr(str);
    len = RSTRING_LEN(str);
    if (len == 0) return Qnil;
    p = RSTRING_PTR(str);
    e = p + len;
    if (argc == 0) {
	rs = rb_rs;
	if (rs == rb_default_rs) {
	  smart_chomp:
	    enc = rb_enc_get(str);
	    if (rb_enc_mbminlen(enc) > 1) {
		pp = rb_enc_left_char_head(p, e-rb_enc_mbminlen(enc), e, enc);
		if (rb_enc_is_newline(pp, e, enc)) {
		    e = pp;
		}
		pp = e - rb_enc_mbminlen(enc);
		if (pp >= p) {
		    pp = rb_enc_left_char_head(p, pp, e, enc);
		    if (rb_enc_ascget(pp, e, 0, enc) == '\r') {
			e = pp;
		    }
		}
		if (e == RSTRING_END(str)) {
		    return Qnil;
		}
		len = e - RSTRING_PTR(str);
		STR_SET_LEN(str, len);
	    }
	    else {
		if (RSTRING_PTR(str)[len-1] == '\n') {
		    STR_DEC_LEN(str);
		    if (RSTRING_LEN(str) > 0 &&
			RSTRING_PTR(str)[RSTRING_LEN(str)-1] == '\r') {
			STR_DEC_LEN(str);
		    }
		}
		else if (RSTRING_PTR(str)[len-1] == '\r') {
		    STR_DEC_LEN(str);
		}
		else {
		    return Qnil;
		}
	    }
	    RSTRING_PTR(str)[RSTRING_LEN(str)] = '\0';
	    return str;
	}
    }
    else {
	rb_scan_args(argc, argv, "01", &rs);
    }
    if (NIL_P(rs)) return Qnil;
    StringValue(rs);
    rslen = RSTRING_LEN(rs);
    if (rslen == 0) {
	while (len>0 && p[len-1] == '\n') {
	    len--;
	    if (len>0 && p[len-1] == '\r')
		len--;
	}
	if (len < RSTRING_LEN(str)) {
	    STR_SET_LEN(str, len);
	    RSTRING_PTR(str)[len] = '\0';
	    return str;
	}
	return Qnil;
    }
    if (rslen > len) return Qnil;
    newline = RSTRING_PTR(rs)[rslen-1];
    if (rslen == 1 && newline == '\n')
	goto smart_chomp;

    enc = rb_enc_check(str, rs);
    if (is_broken_string(rs)) {
	return Qnil;
    }
    pp = e - rslen;
    if (p[len-1] == newline &&
	(rslen <= 1 ||
	 memcmp(RSTRING_PTR(rs), pp, rslen) == 0)) {
	if (rb_enc_left_char_head(p, pp, e, enc) != pp)
	    return Qnil;
	if (ENC_CODERANGE(str) != ENC_CODERANGE_7BIT) {
	    ENC_CODERANGE_CLEAR(str);
	}
	STR_SET_LEN(str, RSTRING_LEN(str) - rslen);
	RSTRING_PTR(str)[RSTRING_LEN(str)] = '\0';
	return str;
    }
    return Qnil;
}


rb_str_rstrip_bang(VALUE str)
{
    rb_encoding *enc;
    char *s, *t, *e;

    str_modify_keep_cr(str);
    enc = STR_ENC_GET(str);
    rb_str_check_dummy_enc(enc);
    s = RSTRING_PTR(str);
    if (!s || RSTRING_LEN(str) == 0) return Qnil;
    t = e = RSTRING_END(str);

    /* remove trailing spaces or '\0's */
    if (single_byte_optimizable(str)) {
	unsigned char c;
	while (s < t && ((c = *(t-1)) == '\0' || ascii_isspace(c))) t--;
    }
    else {
	char *tp;

        while ((tp = rb_enc_prev_char(s, t, e, enc)) != NULL) {
	    unsigned int c = rb_enc_codepoint(tp, e, enc);
	    if (c && !rb_isspace(c)) break;
	    t = tp;
	}
    }
    if (t < e) {
	long len = t-RSTRING_PTR(str);

	STR_SET_LEN(str, len);
	RSTRING_PTR(str)[len] = '\0';
	return str;
    }
    return Qnil;
}


rb_str_ord(VALUE s)
{
    unsigned int c;

    c = rb_enc_codepoint(RSTRING_PTR(s), RSTRING_END(s), STR_ENC_GET(s));
    return UINT2NUM(c);
}


rb_str_tr_bang(VALUE str, VALUE src, VALUE repl)
{
    return tr_trans(str, src, repl, 0);
}


str_new4(VALUE klass, VALUE str)
{
    VALUE str2;

    str2 = str_alloc(klass);
    STR_SET_NOEMBED(str2);
    RSTRING(str2)->as.heap.len = RSTRING_LEN(str);
    RSTRING(str2)->as.heap.ptr = RSTRING_PTR(str);
    if (STR_SHARED_P(str)) {
	VALUE shared = RSTRING(str)->as.heap.aux.shared;
	assert(OBJ_FROZEN(shared));
	FL_SET(str2, ELTS_SHARED);
	RSTRING(str2)->as.heap.aux.shared = shared;
    }
    else {
	FL_SET(str, ELTS_SHARED);
	RSTRING(str)->as.heap.aux.shared = str2;
    }
    rb_enc_cr_str_exact_copy(str2, str);
    OBJ_INFECT(str2, str);
    return str2;
}


rb_str_partition(VALUE str, VALUE sep)
{
    long pos;
    int regex = FALSE;

    if (TYPE(sep) == T_REGEXP) {
	pos = rb_reg_search(sep, str, 0, 0);
	regex = TRUE;
    }
    else {
	VALUE tmp;

	tmp = rb_check_string_type(sep);
	if (NIL_P(tmp)) {
	    rb_raise(rb_eTypeError, "type mismatch: %s given",
		     rb_obj_classname(sep));
	}
	sep = tmp;
	pos = rb_str_index(str, sep, 0);
    }
    if (pos < 0) {
      failed:
	return rb_ary_new3(3, str, rb_str_new(0,0),rb_str_new(0,0));
    }
    if (regex) {
	sep = rb_str_subpat(str, sep, INT2FIX(0));
	if (pos == 0 && RSTRING_LEN(sep) == 0) goto failed;
    }
    return rb_ary_new3(3, rb_str_subseq(str, 0, pos),
		          sep,
		          rb_str_subseq(str, pos+RSTRING_LEN(sep),
					     RSTRING_LEN(str)-pos-RSTRING_LEN(sep)));
}


str_cat_char(VALUE str, unsigned int c, rb_encoding *enc)
{
    char s[RUBY_MAX_CHAR_LEN];
    int n = rb_enc_codelen(c, enc);

    rb_enc_mbcput(c, s, enc);
    rb_enc_str_buf_cat(str, s, n, enc);
}


rb_str_subseq(VALUE str, long beg, long len)
{
    VALUE str2;

    if (RSTRING_LEN(str) == beg + len &&
        RSTRING_EMBED_LEN_MAX < len) {
        str2 = rb_str_new_shared(rb_str_new_frozen(str));
        rb_str_drop_bytes(str2, beg);
    }
    else {
        str2 = rb_str_new5(str, RSTRING_PTR(str)+beg, len);
    }

    rb_enc_cr_str_copy_for_substr(str2, str);
    OBJ_INFECT(str2, str);

    return str2;
}


rb_str_equal(VALUE str1, VALUE str2)
{
    if (str1 == str2) return Qtrue;
    if (TYPE(str2) != T_STRING) {
	if (!rb_respond_to(str2, rb_intern("to_str"))) {
	    return Qfalse;
	}
	return rb_equal(str2, str1);
    }
    return str_eql(str1, str2);
}


chopped_length(VALUE str)
{
    rb_encoding *enc = STR_ENC_GET(str);
    const char *p, *p2, *beg, *end;

    beg = RSTRING_PTR(str);
    end = beg + RSTRING_LEN(str);
    if (beg > end) return 0;
    p = rb_enc_prev_char(beg, end, end, enc);
    if (!p) return 0;
    if (p > beg && rb_enc_ascget(p, end, 0, enc) == '\n') {
	p2 = rb_enc_prev_char(beg, p, end, enc);
	if (p2 && rb_enc_ascget(p2, end, 0, enc) == '\r') p = p2;
    }
    return p - beg;
}


rb_str_new_shared(VALUE str)
{
    VALUE str2 = str_new3(rb_obj_class(str), str);

    OBJ_INFECT(str2, str);
    return str2;
}


rb_str_succ(VALUE orig)
{
    rb_encoding *enc;
    VALUE str;
    char *sbeg, *s, *e, *last_alnum = 0;
    int c = -1;
    long l;
    char carry[ONIGENC_CODE_TO_MBC_MAXLEN] = "\1";
    long carry_pos = 0, carry_len = 1;
    enum neighbor_char neighbor = NEIGHBOR_FOUND;

    str = rb_str_new5(orig, RSTRING_PTR(orig), RSTRING_LEN(orig));
    rb_enc_cr_str_copy_for_substr(str, orig);
    OBJ_INFECT(str, orig);
    if (RSTRING_LEN(str) == 0) return str;

    enc = STR_ENC_GET(orig);
    sbeg = RSTRING_PTR(str);
    s = e = sbeg + RSTRING_LEN(str);

    while ((s = rb_enc_prev_char(sbeg, s, e, enc)) != 0) {
	if (neighbor == NEIGHBOR_NOT_CHAR && last_alnum) {
	    if (ISALPHA(*last_alnum) ? ISDIGIT(*s) :
		ISDIGIT(*last_alnum) ? ISALPHA(*s) : 0) {
		s = last_alnum;
		break;
	    }
	}
	if ((l = rb_enc_precise_mbclen(s, e, enc)) <= 0) continue;
        neighbor = enc_succ_alnum_char(s, l, enc, carry);
        switch (neighbor) {
	  case NEIGHBOR_NOT_CHAR:
	    continue;
	  case NEIGHBOR_FOUND:
	    return str;
	  case NEIGHBOR_WRAPPED:
	    last_alnum = s;
	    break;
	}
        c = 1;
        carry_pos = s - sbeg;
        carry_len = l;
    }
    if (c == -1) {		/* str contains no alnum */
	s = e;
	while ((s = rb_enc_prev_char(sbeg, s, e, enc)) != 0) {
            enum neighbor_char neighbor;
            if ((l = rb_enc_precise_mbclen(s, e, enc)) <= 0) continue;
            neighbor = enc_succ_char(s, l, enc);
            if (neighbor == NEIGHBOR_FOUND)
                return str;
            if (rb_enc_precise_mbclen(s, s+l, enc) != l) {
                /* wrapped to \0...\0.  search next valid char. */
                enc_succ_char(s, l, enc);
            }
            if (!rb_enc_asciicompat(enc)) {
                MEMCPY(carry, s, char, l);
                carry_len = l;
            }
            carry_pos = s - sbeg;
	}
    }
    RESIZE_CAPA(str, RSTRING_LEN(str) + carry_len);
    s = RSTRING_PTR(str) + carry_pos;
    memmove(s + carry_len, s, RSTRING_LEN(str) - carry_pos);
    memmove(s, carry, carry_len);
    STR_SET_LEN(str, RSTRING_LEN(str) + carry_len);
    RSTRING_PTR(str)[RSTRING_LEN(str)] = '\0';
    rb_enc_str_coderange(str);
    return str;
}


rb_str_locktmp(VALUE str)
{
    if (FL_TEST(str, STR_TMPLOCK)) {
	rb_raise(rb_eRuntimeError, "temporal locking already locked string");
    }
    FL_SET(str, STR_TMPLOCK);
    return str;
}


rb_str_strip(VALUE str)
{
    str = rb_str_dup(str);
    rb_str_strip_bang(str);
    return str;
}


rb_str_offset(VALUE str, long pos)
{
    return str_offset(RSTRING_PTR(str), RSTRING_END(str), pos,
		      STR_ENC_GET(str), single_byte_optimizable(str));
}


str_eql(const VALUE str1, const VALUE str2)
{
    const long len = RSTRING_LEN(str1);

    if (len != RSTRING_LEN(str2)) return Qfalse;
    if (!rb_str_comparable(str1, str2)) return Qfalse;
    if (memcmp(RSTRING_PTR(str1), RSTRING_PTR(str2), len) == 0)
	return Qtrue;
    return Qfalse;
}


rb_str_gsub_bang(int argc, VALUE *argv, VALUE str)
{
    str_modify_keep_cr(str);
    return str_gsub(argc, argv, str, 1);
}


rb_locale_str_new_cstr(const char *ptr)
{
    return rb_external_str_new_with_enc(ptr, strlen(ptr), rb_locale_encoding());
}


rb_str_free(VALUE str)
{
    if (!STR_EMBED_P(str) && !STR_SHARED_P(str)) {
	xfree(RSTRING(str)->as.heap.ptr);
    }
}


rb_str_include(VALUE str, VALUE arg)
{
    long i;

    StringValue(arg);
    i = rb_str_index(str, arg, 0);

    if (i == -1) return Qfalse;
    return Qtrue;
}


str_strlen(VALUE str, rb_encoding *enc)
{
    const char *p, *e;
    long n;
    int cr;

    if (single_byte_optimizable(str)) return RSTRING_LEN(str);
    if (!enc) enc = STR_ENC_GET(str);
    p = RSTRING_PTR(str);
    e = RSTRING_END(str);
    cr = ENC_CODERANGE(str);
#ifdef NONASCII_MASK
    if (ENC_CODERANGE(str) == ENC_CODERANGE_VALID &&
        enc == rb_utf8_encoding()) {

	VALUE len = 0;
	if ((int)sizeof(VALUE) * 2 < e - p) {
	    const VALUE *s, *t;
	    const VALUE lowbits = sizeof(VALUE) - 1;
	    s = (const VALUE*)(~lowbits & ((VALUE)p + lowbits));
	    t = (const VALUE*)(~lowbits & (VALUE)e);
	    while (p < (const char *)s) {
		if (is_utf8_lead_byte(*p)) len++;
		p++;
	    }
	    while (s < t) {
		len += count_utf8_lead_bytes_with_word(s);
		s++;
	    }
	    p = (const char *)s;
	}
	while (p < e) {
	    if (is_utf8_lead_byte(*p)) len++;
	    p++;
	}
	return (long)len;
    }
#endif
    n = rb_enc_strlen_cr(p, e, enc, &cr);
    if (cr) {
        ENC_CODERANGE_SET(str, cr);
    }
    return n;
}


rb_string_value_ptr(volatile VALUE *ptr)
{
    VALUE str = rb_string_value(ptr);
    return RSTRING_PTR(str);
}


rb_str_dup(VALUE str)
{
    return str_duplicate(rb_obj_class(str), str);
}


rb_enc_cr_str_copy_for_substr(VALUE dest, VALUE src)
{
    /* this function is designed for copying encoding and coderange
     * from src to new string "dest" which is made from the part of src.
     */
    str_enc_copy(dest, src);
    switch (ENC_CODERANGE(src)) {
      case ENC_CODERANGE_7BIT:
	ENC_CODERANGE_SET(dest, ENC_CODERANGE_7BIT);
	break;
      case ENC_CODERANGE_VALID:
	if (!rb_enc_asciicompat(STR_ENC_GET(src)) ||
	    search_nonascii(RSTRING_PTR(dest), RSTRING_END(dest)))
	    ENC_CODERANGE_SET(dest, ENC_CODERANGE_VALID);
	else
	    ENC_CODERANGE_SET(dest, ENC_CODERANGE_7BIT);
	break;
      default:
	if (RSTRING_LEN(dest) == 0) {
	    if (!rb_enc_asciicompat(STR_ENC_GET(src)))
		ENC_CODERANGE_SET(dest, ENC_CODERANGE_VALID);
	    else
		ENC_CODERANGE_SET(dest, ENC_CODERANGE_7BIT);
	}
	break;
    }
}


rb_str_cat2(VALUE str, const char *ptr)
{
    return rb_str_cat(str, ptr, strlen(ptr));
}


rb_tainted_str_new(const char *ptr, long len)
{
    VALUE str = rb_str_new(ptr, len);

    OBJ_TAINT(str);
    return str;
}


rb_str_swapcase(VALUE str)
{
    str = rb_str_dup(str);
    rb_str_swapcase_bang(str);
    return str;
}


rb_str_delete(int argc, VALUE *argv, VALUE str)
{
    str = rb_str_dup(str);
    rb_str_delete_bang(argc, argv, str);
    return str;
}


sym_equal(VALUE sym1, VALUE sym2)
{
    if (sym1 == sym2) return Qtrue;
    return Qfalse;
}


rb_external_str_new_cstr(const char *ptr)
{
    return rb_external_str_new_with_enc(ptr, strlen(ptr), rb_default_external_encoding());
}


rb_external_str_new(const char *ptr, long len)
{
    return rb_external_str_new_with_enc(ptr, len, rb_default_external_encoding());
}


rb_str_is_ascii_only_p(VALUE str)
{
    int cr = rb_enc_str_coderange(str);

    return cr == ENC_CODERANGE_7BIT ? Qtrue : Qfalse;
}


rb_str_split_m(int argc, VALUE *argv, VALUE str)
{
    rb_encoding *enc;
    VALUE spat;
    VALUE limit;
    enum {awk, string, regexp} split_type;
    long beg, end, i = 0;
    int lim = 0;
    VALUE result, tmp;

    if (rb_scan_args(argc, argv, "02", &spat, &limit) == 2) {
	lim = NUM2INT(limit);
	if (lim <= 0) limit = Qnil;
	else if (lim == 1) {
	    if (RSTRING_LEN(str) == 0)
		return rb_ary_new2(0);
	    return rb_ary_new3(1, str);
	}
	i = 1;
    }

    enc = STR_ENC_GET(str);
    if (NIL_P(spat)) {
	if (!NIL_P(rb_fs)) {
	    spat = rb_fs;
	    goto fs_set;
	}
	split_type = awk;
    }
    else {
      fs_set:
	if (TYPE(spat) == T_STRING) {
	    rb_encoding *enc2 = STR_ENC_GET(spat);

	    split_type = string;
	    if (RSTRING_LEN(spat) == 0) {
		/* Special case - split into chars */
		spat = rb_reg_regcomp(spat);
		split_type = regexp;
	    }
	    else if (rb_enc_asciicompat(enc2) == 1) {
		if (RSTRING_LEN(spat) == 1 && RSTRING_PTR(spat)[0] == ' '){
		    split_type = awk;
		}
	    }
	    else {
		int l;
		if (rb_enc_ascget(RSTRING_PTR(spat), RSTRING_END(spat), &l, enc2) == ' ' &&
		    RSTRING_LEN(spat) == l) {
		    split_type = awk;
		}
	    }
	}
	else {
	    spat = get_pat(spat, 1);
	    split_type = regexp;
	}
    }

    result = rb_ary_new();
    beg = 0;
    if (split_type == awk) {
	char *ptr = RSTRING_PTR(str);
	char *eptr = RSTRING_END(str);
	char *bptr = ptr;
	int skip = 1;
	unsigned int c;

	end = beg;
	if (is_ascii_string(str)) {
	    while (ptr < eptr) {
		c = (unsigned char)*ptr++;
		if (skip) {
		    if (ascii_isspace(c)) {
			beg = ptr - bptr;
		    }
		    else {
			end = ptr - bptr;
			skip = 0;
			if (!NIL_P(limit) && lim <= i) break;
		    }
		}
		else if (ascii_isspace(c)) {
		    rb_ary_push(result, rb_str_subseq(str, beg, end-beg));
		    skip = 1;
		    beg = ptr - bptr;
		    if (!NIL_P(limit)) ++i;
		}
		else {
		    end = ptr - bptr;
		}
	    }
	}
	else {
	    while (ptr < eptr) {
		int n;

		c = rb_enc_codepoint_len(ptr, eptr, &n, enc);
		ptr += n;
		if (skip) {
		    if (rb_isspace(c)) {
			beg = ptr - bptr;
		    }
		    else {
			end = ptr - bptr;
			skip = 0;
			if (!NIL_P(limit) && lim <= i) break;
		    }
		}
		else if (rb_isspace(c)) {
		    rb_ary_push(result, rb_str_subseq(str, beg, end-beg));
		    skip = 1;
		    beg = ptr - bptr;
		    if (!NIL_P(limit)) ++i;
		}
		else {
		    end = ptr - bptr;
		}
	    }
	}
    }
    else if (split_type == string) {
	char *ptr = RSTRING_PTR(str);
	char *temp = ptr;
	char *eptr = RSTRING_END(str);
	char *sptr = RSTRING_PTR(spat);
	long slen = RSTRING_LEN(spat);

	if (is_broken_string(str)) {
	    rb_raise(rb_eArgError, "invalid byte sequence in %s", rb_enc_name(STR_ENC_GET(str)));
	}
	if (is_broken_string(spat)) {
	    rb_raise(rb_eArgError, "invalid byte sequence in %s", rb_enc_name(STR_ENC_GET(spat)));
	}
	enc = rb_enc_check(str, spat);
	while (ptr < eptr &&
	       (end = rb_memsearch(sptr, slen, ptr, eptr - ptr, enc)) >= 0) {
	    /* Check we are at the start of a char */
	    char *t = rb_enc_right_char_head(ptr, ptr + end, eptr, enc);
	    if (t != ptr + end) {
		ptr = t;
		continue;
	    }
	    rb_ary_push(result, rb_str_subseq(str, ptr - temp, end));
	    ptr += end + slen;
	    if (!NIL_P(limit) && lim <= ++i) break;
	}
	beg = ptr - temp;
    }
    else {
	char *ptr = RSTRING_PTR(str);
	long len = RSTRING_LEN(str);
	long start = beg;
	long idx;
	int last_null = 0;
	struct re_registers *regs;

	while ((end = rb_reg_search(spat, str, start, 0)) >= 0) {
	    regs = RMATCH_REGS(rb_backref_get());
	    if (start == end && BEG(0) == END(0)) {
		if (!ptr) {
		    rb_ary_push(result, rb_str_new("", 0));
		    break;
		}
		else if (last_null == 1) {
		    rb_ary_push(result, rb_str_subseq(str, beg,
						      rb_enc_fast_mbclen(ptr+beg,
									 ptr+len,
									 enc)));
		    beg = start;
		}
		else {
                    if (ptr+start == ptr+len)
                        start++;
                    else
                        start += rb_enc_fast_mbclen(ptr+start,ptr+len,enc);
		    last_null = 1;
		    continue;
		}
	    }
	    else {
		rb_ary_push(result, rb_str_subseq(str, beg, end-beg));
		beg = start = END(0);
	    }
	    last_null = 0;

	    for (idx=1; idx < regs->num_regs; idx++) {
		if (BEG(idx) == -1) continue;
		if (BEG(idx) == END(idx))
		    tmp = rb_str_new5(str, 0, 0);
		else
		    tmp = rb_str_subseq(str, BEG(idx), END(idx)-BEG(idx));
		rb_ary_push(result, tmp);
	    }
	    if (!NIL_P(limit) && lim <= ++i) break;
	}
    }
    if (RSTRING_LEN(str) > 0 && (!NIL_P(limit) || RSTRING_LEN(str) > beg || lim < 0)) {
	if (RSTRING_LEN(str) == beg)
	    tmp = rb_str_new5(str, 0, 0);
	else
	    tmp = rb_str_subseq(str, beg, RSTRING_LEN(str)-beg);
	rb_ary_push(result, tmp);
    }
    if (NIL_P(limit) && lim == 0) {
	long len;
	while ((len = RARRAY_LEN(result)) > 0 &&
	       (tmp = RARRAY_PTR(result)[len-1], RSTRING_LEN(tmp) == 0))
	    rb_ary_pop(result);
    }

    return result;
}


str_frozen_check(VALUE s)
{
    if (OBJ_FROZEN(s)) {
	rb_raise(rb_eRuntimeError, "string frozen");
    }
}


rb_str_gsub(int argc, VALUE *argv, VALUE str)
{
    return str_gsub(argc, argv, str, 0);
}


str_gsub(int argc, VALUE *argv, VALUE str, int bang)
{
    VALUE pat, val, repl, match, dest, hash = Qnil;
    struct re_registers *regs;
    long beg, n;
    long beg0, end0;
    long offset, blen, slen, len, last;
    int iter = 0;
    char *sp, *cp;
    int tainted = 0;
    rb_encoding *str_enc;

    switch (argc) {
      case 1:
	RETURN_ENUMERATOR(str, argc, argv);
	iter = 1;
	break;
      case 2:
	repl = argv[1];
	hash = rb_check_convert_type(argv[1], T_HASH, "Hash", "to_hash");
	if (NIL_P(hash)) {
	    StringValue(repl);
	}
	if (OBJ_TAINTED(repl)) tainted = 1;
	break;
      default:
	rb_raise(rb_eArgError, "wrong number of arguments (%d for 2)", argc);
    }

    pat = get_pat(argv[0], 1);
    beg = rb_reg_search(pat, str, 0, 0);
    if (beg < 0) {
	if (bang) return Qnil;	/* no match, no substitution */
	return rb_str_dup(str);
    }

    offset = 0;
    n = 0;
    blen = RSTRING_LEN(str) + 30; /* len + margin */
    dest = rb_str_buf_new(blen);
    sp = RSTRING_PTR(str);
    slen = RSTRING_LEN(str);
    cp = sp;
    str_enc = STR_ENC_GET(str);

    do {
	n++;
	match = rb_backref_get();
	regs = RMATCH_REGS(match);
	beg0 = BEG(0);
	end0 = END(0);
	if (iter || !NIL_P(hash)) {
            if (iter) {
                val = rb_obj_as_string(rb_yield(rb_reg_nth_match(0, match)));
            }
            else {
                val = rb_hash_aref(hash, rb_str_subseq(str, BEG(0), END(0) - BEG(0)));
                val = rb_obj_as_string(val);
            }
	    str_mod_check(str, sp, slen);
	    if (val == dest) { 	/* paranoid check [ruby-dev:24827] */
		rb_raise(rb_eRuntimeError, "block should not cheat");
	    }
	}
	else {
	    val = rb_reg_regsub(repl, str, regs, pat);
	}

	if (OBJ_TAINTED(val)) tainted = 1;

	len = beg - offset;	/* copy pre-match substr */
        if (len) {
            rb_enc_str_buf_cat(dest, cp, len, str_enc);
        }

        rb_str_buf_append(dest, val);

	last = offset;
	offset = end0;
	if (beg0 == end0) {
	    /*
	     * Always consume at least one character of the input string
	     * in order to prevent infinite loops.
	     */
	    if (RSTRING_LEN(str) <= end0) break;
	    len = rb_enc_fast_mbclen(RSTRING_PTR(str)+end0, RSTRING_END(str), str_enc);
            rb_enc_str_buf_cat(dest, RSTRING_PTR(str)+end0, len, str_enc);
	    offset = end0 + len;
	}
	cp = RSTRING_PTR(str) + offset;
	if (offset > RSTRING_LEN(str)) break;
	beg = rb_reg_search(pat, str, offset, 0);
    } while (beg >= 0);
    if (RSTRING_LEN(str) > offset) {
        rb_enc_str_buf_cat(dest, cp, RSTRING_LEN(str) - offset, str_enc);
    }
    rb_reg_search(pat, str, last, 0);
    if (bang) {
        rb_str_shared_replace(str, dest);
    }
    else {
	RBASIC(dest)->klass = rb_obj_class(str);
	OBJ_INFECT(dest, str);
	str = dest;
    }

    if (tainted) OBJ_TAINT(str);
    return str;
}


sym_length(VALUE sym)
{
    return rb_str_length(rb_id2str(SYM2ID(sym)));
}


rb_str_center(int argc, VALUE *argv, VALUE str)
{
    return rb_str_justify(argc, argv, str, 'c');
}


scan_once(VALUE str, VALUE pat, long *start)
{
    VALUE result, match;
    struct re_registers *regs;
    int i;

    if (rb_reg_search(pat, str, *start, 0) >= 0) {
	match = rb_backref_get();
	regs = RMATCH_REGS(match);
	if (BEG(0) == END(0)) {
	    rb_encoding *enc = STR_ENC_GET(str);
	    /*
	     * Always consume at least one character of the input string
	     */
	    if (RSTRING_LEN(str) > END(0))
		*start = END(0)+rb_enc_fast_mbclen(RSTRING_PTR(str)+END(0),
						   RSTRING_END(str), enc);
	    else
		*start = END(0)+1;
	}
	else {
	    *start = END(0);
	}
	if (regs->num_regs == 1) {
	    return rb_reg_nth_match(0, match);
	}
	result = rb_ary_new2(regs->num_regs);
	for (i=1; i < regs->num_regs; i++) {
	    rb_ary_push(result, rb_reg_nth_match(i, match));
	}

	return result;
    }
    return Qnil;
}


coderange_scan(const char *p, long len, rb_encoding *enc)
{
    const char *e = p + len;

    if (rb_enc_to_index(enc) == 0) {
        /* enc is ASCII-8BIT.  ASCII-8BIT string never be broken. */
        p = search_nonascii(p, e);
        return p ? ENC_CODERANGE_VALID : ENC_CODERANGE_7BIT;
    }

    if (rb_enc_asciicompat(enc)) {
        p = search_nonascii(p, e);
        if (!p) {
            return ENC_CODERANGE_7BIT;
        }
        while (p < e) {
            int ret = rb_enc_precise_mbclen(p, e, enc);
            if (!MBCLEN_CHARFOUND_P(ret)) {
                return ENC_CODERANGE_BROKEN;
            }
            p += MBCLEN_CHARFOUND_LEN(ret);
            if (p < e) {
                p = search_nonascii(p, e);
                if (!p) {
                    return ENC_CODERANGE_VALID;
                }
            }
        }
        if (e < p) {
            return ENC_CODERANGE_BROKEN;
        }
        return ENC_CODERANGE_VALID;
    }

    while (p < e) {
        int ret = rb_enc_precise_mbclen(p, e, enc);

        if (!MBCLEN_CHARFOUND_P(ret)) {
            return ENC_CODERANGE_BROKEN;
        }
        p += MBCLEN_CHARFOUND_LEN(ret);
    }
    if (e < p) {
        return ENC_CODERANGE_BROKEN;
    }
    return ENC_CODERANGE_VALID;
}


rb_str_splice_0(VALUE str, long beg, long len, VALUE val)
{
    if (beg == 0 && RSTRING_LEN(val) == 0) {
	rb_str_drop_bytes(str, len);
	OBJ_INFECT(str, val);
	return;
    }

    rb_str_modify(str);
    if (len < RSTRING_LEN(val)) {
	/* expand string */
	RESIZE_CAPA(str, RSTRING_LEN(str) + RSTRING_LEN(val) - len + 1);
    }

    if (RSTRING_LEN(val) != len) {
	memmove(RSTRING_PTR(str) + beg + RSTRING_LEN(val),
		RSTRING_PTR(str) + beg + len,
		RSTRING_LEN(str) - (beg + len));
    }
    if (RSTRING_LEN(val) < beg && len < 0) {
	MEMZERO(RSTRING_PTR(str) + RSTRING_LEN(str), char, -len);
    }
    if (RSTRING_LEN(val) > 0) {
	memmove(RSTRING_PTR(str)+beg, RSTRING_PTR(val), RSTRING_LEN(val));
    }
    STR_SET_LEN(str, RSTRING_LEN(str) + RSTRING_LEN(val) - len);
    if (RSTRING_PTR(str)) {
	RSTRING_PTR(str)[RSTRING_LEN(str)] = '\0';
    }
    OBJ_INFECT(str, val);
}


rb_str_subpat(VALUE str, VALUE re, VALUE backref)
{
    if (rb_reg_search(re, str, 0, 0) >= 0) {
        VALUE match = rb_backref_get();
        int nth = rb_reg_backref_number(match, backref);
	return rb_reg_nth_match(nth, match);
    }
    return Qnil;
}


rb_str_getbyte(VALUE str, VALUE index)
{
    long pos = NUM2LONG(index);

    if (pos < 0)
        pos += RSTRING_LEN(str);
    if (pos < 0 ||  RSTRING_LEN(str) <= pos)
        return Qnil;

    return INT2FIX((unsigned char)RSTRING_PTR(str)[pos]);
}


str_modifiable(VALUE str)
{
    if (FL_TEST(str, STR_TMPLOCK)) {
	rb_raise(rb_eRuntimeError, "can't modify string; temporarily locked");
    }
    if (OBJ_FROZEN(str)) rb_error_frozen("string");
    if (!OBJ_UNTRUSTED(str) && rb_safe_level() >= 4)
	rb_raise(rb_eSecurityError, "Insecure: can't modify string");
}


rb_str_lstrip(VALUE str)
{
    str = rb_str_dup(str);
    rb_str_lstrip_bang(str);
    return str;
}


rb_str_buf_cat2(VALUE str, const char *ptr)
{
    return rb_str_buf_cat(str, ptr, strlen(ptr));
}


rb_str_export_to_enc(VALUE str, rb_encoding *enc)
{
    return rb_str_conv_enc(str, STR_ENC_GET(str), enc);
}


rb_str_export_locale(VALUE str)
{
    return rb_str_conv_enc(str, STR_ENC_GET(str), rb_locale_encoding());
}


rb_str_to_s(VALUE str)
{
    if (rb_obj_class(str) != rb_cString) {
	return str_duplicate(rb_cString, str);
    }
    return str;
}


rb_enc_strlen(const char *p, const char *e, rb_encoding *enc)
{
    return enc_strlen(p, e, enc, ENC_CODERANGE_UNKNOWN);
}


rb_str_times(VALUE str, VALUE times)
{
    VALUE str2;
    long n, len;
    char *ptr2;

    len = NUM2LONG(times);
    if (len < 0) {
	rb_raise(rb_eArgError, "negative argument");
    }
    if (len && LONG_MAX/len <  RSTRING_LEN(str)) {
	rb_raise(rb_eArgError, "argument too big");
    }

    str2 = rb_str_new5(str, 0, len *= RSTRING_LEN(str));
    ptr2 = RSTRING_PTR(str2);
    if (len) {
        n = RSTRING_LEN(str);
        memcpy(ptr2, RSTRING_PTR(str), n);
        while (n <= len/2) {
            memcpy(ptr2 + n, ptr2, n);
            n *= 2;
        }
        memcpy(ptr2 + n, ptr2, len-n);
    }
    ptr2[RSTRING_LEN(str2)] = '\0';
    OBJ_INFECT(str2, str);
    rb_enc_cr_str_copy_for_substr(str2, str);

    return str2;
}


rb_str_succ_bang(VALUE str)
{
    rb_str_shared_replace(str, rb_str_succ(str));

    return str;
}


rb_str_aref(VALUE str, VALUE indx)
{
    long idx;

    switch (TYPE(indx)) {
      case T_FIXNUM:
	idx = FIX2LONG(indx);

      num_index:
	str = rb_str_substr(str, idx, 1);
	if (!NIL_P(str) && RSTRING_LEN(str) == 0) return Qnil;
	return str;

      case T_REGEXP:
	return rb_str_subpat(str, indx, INT2FIX(0));

      case T_STRING:
	if (rb_str_index(str, indx, 0) != -1)
	    return rb_str_dup(indx);
	return Qnil;

      default:
	/* check if indx is Range */
	{
	    long beg, len;
	    VALUE tmp;

	    len = str_strlen(str, STR_ENC_GET(str));
	    switch (rb_range_beg_len(indx, &beg, &len, len, 0)) {
	      case Qfalse:
		break;
	      case Qnil:
		return Qnil;
	      default:
		tmp = rb_str_substr(str, beg, len);
		return tmp;
	    }
	}
	idx = NUM2LONG(indx);
	goto num_index;
    }
    return Qnil;		/* not reached */
}


str_nth(const char *p, const char *e, long nth, rb_encoding *enc, int singlebyte)
{
    if (singlebyte)
	p += nth;
    else {
	p = rb_enc_nth(p, e, nth, enc);
    }
    if (!p) return 0;
    if (p > e) p = e;
    return (char *)p;
}


rb_str_chop(VALUE str)
{
    VALUE str2 = rb_str_new5(str, RSTRING_PTR(str), chopped_length(str));
    rb_enc_cr_str_copy_for_substr(str2, str);
    OBJ_INFECT(str2, str);
    return str2;
}


rb_str_chr(VALUE str)
{
    return rb_str_substr(str, 0, 1);
}


rb_str_buf_new_cstr(const char *ptr)
{
    VALUE str;
    long len = strlen(ptr);

    str = rb_str_buf_new(len);
    rb_str_buf_cat(str, ptr, len);

    return str;
}


sym_to_proc(VALUE sym)
{
    static VALUE sym_proc_cache = Qfalse;
    enum {SYM_PROC_CACHE_SIZE = 67};
    VALUE proc;
    long id, index;
    VALUE *aryp;

    if (!sym_proc_cache) {
	sym_proc_cache = rb_ary_tmp_new(SYM_PROC_CACHE_SIZE * 2);
	rb_gc_register_mark_object(sym_proc_cache);
	rb_ary_store(sym_proc_cache, SYM_PROC_CACHE_SIZE*2 - 1, Qnil);
    }

    id = SYM2ID(sym);
    index = (id % SYM_PROC_CACHE_SIZE) << 1;

    aryp = RARRAY_PTR(sym_proc_cache);
    if (aryp[index] == sym) {
	return aryp[index + 1];
    }
    else {
	proc = rb_proc_new(sym_call, (VALUE)id);
	aryp[index] = sym;
	aryp[index + 1] = proc;
	return proc;
    }
}


rb_str_sub_bang(int argc, VALUE *argv, VALUE str)
{
    VALUE pat, repl, hash = Qnil;
    int iter = 0;
    int tainted = 0;
    int untrusted = 0;
    long plen;

    if (argc == 1 && rb_block_given_p()) {
	iter = 1;
    }
    else if (argc == 2) {
	repl = argv[1];
	hash = rb_check_convert_type(argv[1], T_HASH, "Hash", "to_hash");
	if (NIL_P(hash)) {
	    StringValue(repl);
	}
	if (OBJ_TAINTED(repl)) tainted = 1;
	if (OBJ_UNTRUSTED(repl)) untrusted = 1;
    }
    else {
	rb_raise(rb_eArgError, "wrong number of arguments (%d for 2)", argc);
    }

    pat = get_pat(argv[0], 1);
    if (rb_reg_search(pat, str, 0, 0) >= 0) {
	rb_encoding *enc;
	int cr = ENC_CODERANGE(str);
	VALUE match = rb_backref_get();
	struct re_registers *regs = RMATCH_REGS(match);
	long beg0 = BEG(0);
	long end0 = END(0);
	char *p, *rp;
	long len, rlen;

	if (iter || !NIL_P(hash)) {
	    p = RSTRING_PTR(str); len = RSTRING_LEN(str);

            if (iter) {
                repl = rb_obj_as_string(rb_yield(rb_reg_nth_match(0, match)));
            }
            else {
                repl = rb_hash_aref(hash, rb_str_subseq(str, beg0, end0 - beg0));
                repl = rb_obj_as_string(repl);
            }
	    str_mod_check(str, p, len);
	    str_frozen_check(str);
	}
	else {
	    repl = rb_reg_regsub(repl, str, regs, pat);
	}
        enc = rb_enc_compatible(str, repl);
        if (!enc) {
            rb_encoding *str_enc = STR_ENC_GET(str);
	    p = RSTRING_PTR(str); len = RSTRING_LEN(str);
	    if (coderange_scan(p, beg0, str_enc) != ENC_CODERANGE_7BIT ||
		coderange_scan(p+end0, len-end0, str_enc) != ENC_CODERANGE_7BIT) {
                rb_raise(rb_eEncCompatError, "incompatible character encodings: %s and %s",
			 rb_enc_name(str_enc),
			 rb_enc_name(STR_ENC_GET(repl)));
            }
            enc = STR_ENC_GET(repl);
        }
	rb_str_modify(str);
	rb_enc_associate(str, enc);
	if (OBJ_TAINTED(repl)) tainted = 1;
	if (OBJ_UNTRUSTED(repl)) untrusted = 1;
	if (ENC_CODERANGE_UNKNOWN < cr && cr < ENC_CODERANGE_BROKEN) {
	    int cr2 = ENC_CODERANGE(repl);
            if (cr2 == ENC_CODERANGE_BROKEN ||
                (cr == ENC_CODERANGE_VALID && cr2 == ENC_CODERANGE_7BIT))
                cr = ENC_CODERANGE_UNKNOWN;
            else
                cr = cr2;
	}
	plen = end0 - beg0;
	rp = RSTRING_PTR(repl); rlen = RSTRING_LEN(repl);
	len = RSTRING_LEN(str);
	if (rlen > plen) {
	    RESIZE_CAPA(str, len + rlen - plen);
	}
	p = RSTRING_PTR(str);
	if (rlen != plen) {
	    memmove(p + beg0 + rlen, p + beg0 + plen, len - beg0 - plen);
	}
	memcpy(p + beg0, rp, rlen);
	len += rlen - plen;
	STR_SET_LEN(str, len);
	RSTRING_PTR(str)[len] = '\0';
	ENC_CODERANGE_SET(str, cr);
	if (tainted) OBJ_TAINT(str);
	if (untrusted) OBJ_UNTRUST(str);

	return str;
    }
    return Qnil;
}


rb_str_new_frozen(VALUE orig)
{
    VALUE klass, str;

    if (OBJ_FROZEN(orig)) return orig;
    klass = rb_obj_class(orig);
    if (STR_SHARED_P(orig) && (str = RSTRING(orig)->as.heap.aux.shared)) {
	long ofs;
	assert(OBJ_FROZEN(str));
	ofs = RSTRING_LEN(str) - RSTRING_LEN(orig);
	if ((ofs > 0) || (klass != RBASIC(str)->klass) ||
	    (!OBJ_TAINTED(str) && OBJ_TAINTED(orig)) ||
	    ENCODING_GET(str) != ENCODING_GET(orig)) {
	    str = str_new3(klass, str);
	    RSTRING(str)->as.heap.ptr += ofs;
	    RSTRING(str)->as.heap.len -= ofs;
	    rb_enc_cr_str_exact_copy(str, orig);
	    OBJ_INFECT(str, orig);
	}
    }
    else if (STR_EMBED_P(orig)) {
	str = str_new(klass, RSTRING_PTR(orig), RSTRING_LEN(orig));
	rb_enc_cr_str_exact_copy(str, orig);
	OBJ_INFECT(str, orig);
    }
    else if (STR_ASSOC_P(orig)) {
	VALUE assoc = RSTRING(orig)->as.heap.aux.shared;
	FL_UNSET(orig, STR_ASSOC);
	str = str_new4(klass, orig);
	FL_SET(str, STR_ASSOC);
	RSTRING(str)->as.heap.aux.shared = assoc;
    }
    else {
	str = str_new4(klass, orig);
    }
    OBJ_FREEZE(str);
    return str;
}


rb_str_split(VALUE str, const char *sep0)
{
    VALUE sep;

    StringValue(str);
    sep = rb_str_new2(sep0);
    return rb_str_split_m(1, &sep, str);
}


rb_str_concat(VALUE str1, VALUE str2)
{
    if (FIXNUM_P(str2) || TYPE(str2) == T_BIGNUM) {
	rb_encoding *enc = STR_ENC_GET(str1);
	unsigned int c = NUM2UINT(str2);
	long pos = RSTRING_LEN(str1);
	int len = rb_enc_codelen(c, enc);
	int cr = ENC_CODERANGE(str1);

	rb_str_resize(str1, pos+len);
	rb_enc_mbcput(c, RSTRING_PTR(str1)+pos, enc);
	ENC_CODERANGE_SET(str1, cr);
	return str1;
    }
    return rb_str_append(str1, str2);
}


rb_str_start_with(int argc, VALUE *argv, VALUE str)
{
    int i;

    for (i=0; i<argc; i++) {
	VALUE tmp = rb_check_string_type(argv[i]);
	if (NIL_P(tmp)) continue;
	rb_enc_check(str, tmp);
	if (RSTRING_LEN(str) < RSTRING_LEN(tmp)) continue;
	if (memcmp(RSTRING_PTR(str), RSTRING_PTR(tmp), RSTRING_LEN(tmp)) == 0)
	    return Qtrue;
    }
    return Qfalse;
}


rb_enc_str_asciionly_p(VALUE str)
{
    rb_encoding *enc = STR_ENC_GET(str);

    if (!rb_enc_asciicompat(enc))
        return FALSE;
    else if (rb_enc_str_coderange(str) == ENC_CODERANGE_7BIT)
        return TRUE;
    return FALSE;
}


rb_str_export(VALUE str)
{
    return rb_str_conv_enc(str, STR_ENC_GET(str), rb_default_external_encoding());
}


rb_obj_as_string(VALUE obj)
{
    VALUE str;

    if (TYPE(obj) == T_STRING) {
	return obj;
    }
    str = rb_funcall(obj, id_to_s, 0);
    if (TYPE(str) != T_STRING)
	return rb_any_to_s(obj);
    if (OBJ_TAINTED(obj)) OBJ_TAINT(str);
    return str;
}


rb_str_justify(int argc, VALUE *argv, VALUE str, char jflag)
{
    rb_encoding *enc;
    VALUE w;
    long width, len, flen = 1, fclen = 1;
    VALUE res;
    char *p;
    const char *f = " ";
    long n, size, llen, rlen, llen2 = 0, rlen2 = 0;
    volatile VALUE pad;
    int singlebyte = 1, cr;

    rb_scan_args(argc, argv, "11", &w, &pad);
    enc = STR_ENC_GET(str);
    width = NUM2LONG(w);
    if (argc == 2) {
	StringValue(pad);
	enc = rb_enc_check(str, pad);
	f = RSTRING_PTR(pad);
	flen = RSTRING_LEN(pad);
	fclen = str_strlen(pad, enc);
	singlebyte = single_byte_optimizable(pad);
	if (flen == 0 || fclen == 0) {
	    rb_raise(rb_eArgError, "zero width padding");
	}
    }
    len = str_strlen(str, enc);
    if (width < 0 || len >= width) return rb_str_dup(str);
    n = width - len;
    llen = (jflag == 'l') ? 0 : ((jflag == 'r') ? n : n/2);
    rlen = n - llen;
    cr = ENC_CODERANGE(str);
    if (flen > 1) {
       llen2 = str_offset(f, f + flen, llen % fclen, enc, singlebyte);
       rlen2 = str_offset(f, f + flen, rlen % fclen, enc, singlebyte);
    }
    size = RSTRING_LEN(str);
    if ((len = llen / fclen + rlen / fclen) >= LONG_MAX / flen ||
       (len *= flen) >= LONG_MAX - llen2 - rlen2 ||
       (len += llen2 + rlen2) >= LONG_MAX - size) {
       rb_raise(rb_eArgError, "argument too big");
    }
    len += size;
    res = rb_str_new5(str, 0, len);
    p = RSTRING_PTR(res);
    if (flen <= 1) {
       memset(p, *f, llen);
       p += llen;
    }
    else {
       while (llen > fclen) {
	    memcpy(p,f,flen);
	    p += flen;
	    llen -= fclen;
	}
       if (llen > 0) {
           memcpy(p, f, llen2);
           p += llen2;
	}
    }
    memcpy(p, RSTRING_PTR(str), size);
    p += size;
    if (flen <= 1) {
       memset(p, *f, rlen);
       p += rlen;
    }
    else {
       while (rlen > fclen) {
	    memcpy(p,f,flen);
	    p += flen;
	    rlen -= fclen;
	}
       if (rlen > 0) {
           memcpy(p, f, rlen2);
           p += rlen2;
	}
    }
    *p = '\0';
    STR_SET_LEN(res, p-RSTRING_PTR(res));
    OBJ_INFECT(res, str);
    if (!NIL_P(pad)) OBJ_INFECT(res, pad);
    rb_enc_associate(res, enc);
    if (argc == 2)
	cr = ENC_CODERANGE_AND(cr, ENC_CODERANGE(pad));
    if (cr != ENC_CODERANGE_BROKEN)
	ENC_CODERANGE_SET(res, cr);
    return res;
}


rb_str_each_byte(VALUE str)
{
    long i;

    RETURN_ENUMERATOR(str, 0, 0);
    for (i=0; i<RSTRING_LEN(str); i++) {
	rb_yield(INT2FIX(RSTRING_PTR(str)[i] & 0xff));
    }
    return str;
}


rb_str_capacity(VALUE str)
{
    if (STR_EMBED_P(str)) {
	return RSTRING_EMBED_LEN_MAX;
    }
    else if (STR_NOCAPA_P(str)) {
	return RSTRING(str)->as.heap.len;
    }
    else {
	return RSTRING(str)->as.heap.aux.capa;
    }
}


enc_pred_char(char *p, long len, rb_encoding *enc)
{
    long i;
    int l;
    while (1) {
        for (i = len-1; 0 <= i && (unsigned char)p[i] == 0; i--)
            p[i] = '\xff';
        if (i < 0)
            return NEIGHBOR_WRAPPED;
        --((unsigned char*)p)[i];
        l = rb_enc_precise_mbclen(p, p+len, enc);
        if (MBCLEN_CHARFOUND_P(l)) {
            l = MBCLEN_CHARFOUND_LEN(l);
            if (l == len) {
                return NEIGHBOR_FOUND;
            }
            else {
                memset(p+l, 0, len-l);
            }
        }
        if (MBCLEN_INVALID_P(l) && i < len-1) {
            long len2;
            int l2;
            for (len2 = len-1; 0 < len2; len2--) {
                l2 = rb_enc_precise_mbclen(p, p+len2, enc);
                if (!MBCLEN_INVALID_P(l2))
                    break;
            }
            memset(p+len2+1, 0, len-(len2+1));
        }
    }
}


sym_casecmp(VALUE sym, VALUE other)
{
    if (!SYMBOL_P(other)) {
	return Qnil;
    }
    return rb_str_casecmp(rb_sym_to_s(sym), rb_sym_to_s(other));
}


rb_str_associated(VALUE str)
{
    if (STR_SHARED_P(str)) str = RSTRING(str)->as.heap.aux.shared;
    if (STR_ASSOC_P(str)) {
	return RSTRING(str)->as.heap.aux.shared;
    }
    return Qfalse;
}


rb_str_buf_append(VALUE str, VALUE str2)
{
    int str2_cr;

    str2_cr = ENC_CODERANGE(str2);

    rb_enc_cr_str_buf_cat(str, RSTRING_PTR(str2), RSTRING_LEN(str2),
        ENCODING_GET(str2), str2_cr, &str2_cr);

    OBJ_INFECT(str, str2);
    ENC_CODERANGE_SET(str2, str2_cr);

    return str;
}


rb_str_cmp(VALUE str1, VALUE str2)
{
    long len;
    int retval;

    len = lesser(RSTRING_LEN(str1), RSTRING_LEN(str2));
    retval = memcmp(RSTRING_PTR(str1), RSTRING_PTR(str2), len);
    if (retval == 0) {
	if (RSTRING_LEN(str1) == RSTRING_LEN(str2)) {
	    if (!rb_str_comparable(str1, str2)) {
		if (ENCODING_GET(str1) > ENCODING_GET(str2))
		    return 1;
		return -1;
	    }
	    return 0;
	}
	if (RSTRING_LEN(str1) > RSTRING_LEN(str2)) return 1;
	return -1;
    }
    if (retval > 0) return 1;
    return -1;
}


sym_call(VALUE args, VALUE sym, int argc, VALUE *argv)
{
    VALUE obj;

    if (argc < 1) {
	rb_raise(rb_eArgError, "no receiver given");
    }
    obj = argv[0];
    return rb_funcall3(obj, (ID)sym, argc - 1, argv + 1);
}


rb_enc_str_coderange(VALUE str)
{
    int cr = ENC_CODERANGE(str);

    if (cr == ENC_CODERANGE_UNKNOWN) {
	rb_encoding *enc = STR_ENC_GET(str);
        cr = coderange_scan(RSTRING_PTR(str), RSTRING_LEN(str), enc);
        ENC_CODERANGE_SET(str, cr);
    }
    return cr;
}


str_independent(VALUE str)
{
    str_modifiable(str);
    if (!STR_SHARED_P(str)) return 1;
    if (STR_EMBED_P(str)) return 1;
    return 0;
}


str_buf_cat(VALUE str, const char *ptr, long len)
{
    long capa, total, off = -1;

    if (ptr >= RSTRING_PTR(str) && ptr <= RSTRING_END(str)) {
        off = ptr - RSTRING_PTR(str);
    }
    rb_str_modify(str);
    if (len == 0) return 0;
    if (STR_ASSOC_P(str)) {
	FL_UNSET(str, STR_ASSOC);
	capa = RSTRING(str)->as.heap.aux.capa = RSTRING_LEN(str);
    }
    else if (STR_EMBED_P(str)) {
	capa = RSTRING_EMBED_LEN_MAX;
    }
    else {
	capa = RSTRING(str)->as.heap.aux.capa;
    }
    if (RSTRING_LEN(str) >= LONG_MAX - len) {
	rb_raise(rb_eArgError, "string sizes too big");
    }
    total = RSTRING_LEN(str)+len;
    if (capa <= total) {
	while (total > capa) {
	    if (capa + 1 >= LONG_MAX / 2) {
		capa = (total + 4095) / 4096;
		break;
	    }
	    capa = (capa + 1) * 2;
	}
	RESIZE_CAPA(str, capa);
    }
    if (off != -1) {
        ptr = RSTRING_PTR(str) + off;
    }
    memcpy(RSTRING_PTR(str) + RSTRING_LEN(str), ptr, len);
    STR_SET_LEN(str, total);
    RSTRING_PTR(str)[total] = '\0'; /* sentinel */

    return str;
}


rb_str_downcase(VALUE str)
{
    str = rb_str_dup(str);
    rb_str_downcase_bang(str);
    return str;
}


rb_str_cat(VALUE str, const char *ptr, long len)
{
    if (len < 0) {
	rb_raise(rb_eArgError, "negative string size (or size too big)");
    }
    if (STR_ASSOC_P(str)) {
	rb_str_modify(str);
	if (STR_EMBED_P(str)) str_make_independent(str);
	REALLOC_N(RSTRING(str)->as.heap.ptr, char, RSTRING(str)->as.heap.len+len+1);
	memcpy(RSTRING(str)->as.heap.ptr + RSTRING(str)->as.heap.len, ptr, len);
	RSTRING(str)->as.heap.len += len;
	RSTRING(str)->as.heap.ptr[RSTRING(str)->as.heap.len] = '\0'; /* sentinel */
	return str;
    }

    return rb_str_buf_cat(str, ptr, len);
}


rb_str_rjust(int argc, VALUE *argv, VALUE str)
{
    return rb_str_justify(argc, argv, str, 'r');
}


rb_string_value(volatile VALUE *ptr)
{
    VALUE s = *ptr;
    if (TYPE(s) != T_STRING) {
	if (SYMBOL_P(s)) {
	    s = rb_sym_to_s(s);
	}
	else {
	    s = rb_str_to_str(s);
	}
	*ptr = s;
    }
    return s;
}


sym_printable(const char *s, const char *send, rb_encoding *enc)
{
    while (s < send) {
	int n;
	int c = rb_enc_codepoint_len(s, send, &n, enc);

	if (!rb_enc_isprint(c, enc)) return FALSE;
	s += n;
    }
    return TRUE;
}


rb_sym_to_s(VALUE sym)
{
    ID id = SYM2ID(sym);

    return str_new3(rb_cString, rb_id2str(id));
}


rb_usascii_str_new_cstr(const char *ptr)
{
    VALUE str = rb_str_new2(ptr);
    ENCODING_CODERANGE_SET(str, rb_usascii_encindex(), ENC_CODERANGE_7BIT);
    return str;
}


static int cmd_queue (IMAP_DATA* idata, const char* cmdstr, int flags)
{
  IMAP_COMMAND* cmd;
  int rc;

  if (cmd_queue_full (idata))
  {
    dprint (3, (debugfile, "Draining IMAP command pipeline\n"));

    rc = imap_exec (idata, NULL, IMAP_CMD_FAIL_OK | (flags & IMAP_CMD_POLL));

    if (rc < 0 && rc != -2)
      return rc;
  }

  if (!(cmd = cmd_new (idata)))
    return IMAP_CMD_BAD;

  if (mutt_buffer_printf (idata->cmdbuf, "%s %s\r\n", cmd->seq, cmdstr) < 0)
    return IMAP_CMD_BAD;

  return 0;
}

Handle NO response without message properly


int imap_exec (IMAP_DATA* idata, const char* cmdstr, int flags)
{
  int rc;

  if ((rc = cmd_start (idata, cmdstr, flags)) < 0)
  {
    cmd_handle_fatal (idata);
    return -1;
  }

  if (flags & IMAP_CMD_QUEUE)
    return 0;

  if ((flags & IMAP_CMD_POLL) &&
      (ImapPollTimeout > 0) &&
      (mutt_socket_poll (idata->conn, ImapPollTimeout)) == 0)
  {
    mutt_error (_("Connection to %s timed out"), idata->conn->account.host);
    mutt_sleep (2);
    cmd_handle_fatal (idata);
    return -1;
  }

  do
    rc = imap_cmd_step (idata);
  while (rc == IMAP_CMD_CONTINUE);

  if (rc == IMAP_CMD_NO && (flags & IMAP_CMD_FAIL_OK))
    return -2;

  if (rc != IMAP_CMD_OK)
  {
    if ((flags & IMAP_CMD_FAIL_OK) && idata->status != IMAP_FATAL)
      return -2;

    dprint (1, (debugfile, "imap_exec: command failed: %s\n", idata->buf));
    return -1;
  }

  return 0;
}


static void cmd_handle_fatal (IMAP_DATA* idata)
{
  idata->status = IMAP_FATAL;

  if ((idata->state >= IMAP_SELECTED) &&
      (idata->reopen & IMAP_REOPEN_ALLOW))
  {
    mx_fastclose_mailbox (idata->ctx);
    mutt_socket_close (idata->conn);
    mutt_error (_("Mailbox %s@%s closed"),
	idata->conn->account.login, idata->conn->account.host);
    mutt_sleep (1);
    idata->state = IMAP_DISCONNECTED;
  }

  if (idata->state < IMAP_SELECTED)
    imap_close_connection (idata);
}


int imap_cmd_idle (IMAP_DATA* idata)
{
  int rc;

  if (cmd_start (idata, "IDLE", IMAP_CMD_POLL) < 0)
  {
    cmd_handle_fatal (idata);
    return -1;
  }

  if ((ImapPollTimeout > 0) &&
      (mutt_socket_poll (idata->conn, ImapPollTimeout)) == 0)
  {
    mutt_error (_("Connection to %s timed out"), idata->conn->account.host);
    mutt_sleep (2);
    cmd_handle_fatal (idata);
    return -1;
  }

  do
    rc = imap_cmd_step (idata);
  while (rc == IMAP_CMD_CONTINUE);

  if (rc == IMAP_CMD_RESPOND)
  {
    /* successfully entered IDLE state */
    idata->state = IMAP_IDLE;
    /* queue automatic exit when next command is issued */
    mutt_buffer_printf (idata->cmdbuf, "DONE\r\n");
    rc = IMAP_CMD_OK;
  }
  if (rc != IMAP_CMD_OK)
  {
    dprint (1, (debugfile, "imap_cmd_idle: error starting IDLE\n"));
    return -1;
  }

  return 0;
}


int imap_cmd_start (IMAP_DATA* idata, const char* cmdstr)
{
  return cmd_start (idata, cmdstr, 0);
}


static int cmd_handle_untagged (IMAP_DATA* idata)
{
  char* s;
  char* pn;
  unsigned int count;

  s = imap_next_word (idata->buf);
  pn = imap_next_word (s);

  if ((idata->state >= IMAP_SELECTED) && isdigit ((unsigned char) *s))
  {
    pn = s;
    s = imap_next_word (s);

    /* EXISTS and EXPUNGE are always related to the SELECTED mailbox for the
     * connection, so update that one.
     */
    if (ascii_strncasecmp ("EXISTS", s, 6) == 0)
    {
      dprint (2, (debugfile, "Handling EXISTS\n"));

      /* new mail arrived */
      mutt_atoui (pn, &count);

      if ( !(idata->reopen & IMAP_EXPUNGE_PENDING) &&
	   count < idata->max_msn)
      {
        /* Notes 6.0.3 has a tendency to report fewer messages exist than
         * it should. */
	dprint (1, (debugfile, "Message count is out of sync"));
	return 0;
      }
      /* at least the InterChange server sends EXISTS messages freely,
       * even when there is no new mail */
      else if (count == idata->max_msn)
	dprint (3, (debugfile,
          "cmd_handle_untagged: superfluous EXISTS message.\n"));
      else
      {
	if (!(idata->reopen & IMAP_EXPUNGE_PENDING))
        {
          dprint (2, (debugfile,
            "cmd_handle_untagged: New mail in %s - %d messages total.\n",
            idata->mailbox, count));
	  idata->reopen |= IMAP_NEWMAIL_PENDING;
        }
	idata->newMailCount = count;
      }
    }
    /* pn vs. s: need initial seqno */
    else if (ascii_strncasecmp ("EXPUNGE", s, 7) == 0)
      cmd_parse_expunge (idata, pn);
    else if (ascii_strncasecmp ("FETCH", s, 5) == 0)
      cmd_parse_fetch (idata, pn);
  }
  else if (ascii_strncasecmp ("CAPABILITY", s, 10) == 0)
    cmd_parse_capability (idata, s);
  else if (!ascii_strncasecmp ("OK [CAPABILITY", s, 14))
    cmd_parse_capability (idata, pn);
  else if (!ascii_strncasecmp ("OK [CAPABILITY", pn, 14))
    cmd_parse_capability (idata, imap_next_word (pn));
  else if (ascii_strncasecmp ("LIST", s, 4) == 0)
    cmd_parse_list (idata, s);
  else if (ascii_strncasecmp ("LSUB", s, 4) == 0)
    cmd_parse_lsub (idata, s);
  else if (ascii_strncasecmp ("MYRIGHTS", s, 8) == 0)
    cmd_parse_myrights (idata, s);
  else if (ascii_strncasecmp ("SEARCH", s, 6) == 0)
    cmd_parse_search (idata, s);
  else if (ascii_strncasecmp ("STATUS", s, 6) == 0)
    cmd_parse_status (idata, s);
  else if (ascii_strncasecmp ("ENABLED", s, 7) == 0)
    cmd_parse_enabled (idata, s);
  else if (ascii_strncasecmp ("BYE", s, 3) == 0)
  {
    dprint (2, (debugfile, "Handling BYE\n"));

    /* check if we're logging out */
    if (idata->status == IMAP_BYE)
      return 0;

    /* server shut down our connection */
    s += 3;
    SKIPWS (s);
    mutt_error ("%s", s);
    mutt_sleep (2);
    cmd_handle_fatal (idata);

    return -1;
  }
  else if (option (OPTIMAPSERVERNOISE) && (ascii_strncasecmp ("NO", s, 2) == 0))
  {
    dprint (2, (debugfile, "Handling untagged NO\n"));

    /* Display the warning message from the server */
    mutt_error ("%s", s+2);
    mutt_sleep (2);
  }

  return 0;
}


static int cmd_start (IMAP_DATA* idata, const char* cmdstr, int flags)
{
  int rc;

  if (idata->status == IMAP_FATAL)
  {
    cmd_handle_fatal (idata);
    return -1;
  }

  if (cmdstr && ((rc = cmd_queue (idata, cmdstr, flags)) < 0))
    return rc;

  if (flags & IMAP_CMD_QUEUE)
    return 0;

  if (idata->cmdbuf->dptr == idata->cmdbuf->data)
    return IMAP_CMD_BAD;

  rc = mutt_socket_write_d (idata->conn, idata->cmdbuf->data, -1,
                            flags & IMAP_CMD_PASS ? IMAP_LOG_PASS : IMAP_LOG_CMD);
  idata->cmdbuf->dptr = idata->cmdbuf->data;

  /* unidle when command queue is flushed */
  if (idata->state == IMAP_IDLE)
    idata->state = IMAP_SELECTED;

  return (rc < 0) ? IMAP_CMD_BAD : 0;
}


static void cmd_parse_expunge (IMAP_DATA* idata, const char* s)
{
  unsigned int exp_msn, cur;
  HEADER* h;

  dprint (2, (debugfile, "Handling EXPUNGE\n"));

  if (mutt_atoui (s, &exp_msn) < 0 ||
      exp_msn < 1 || exp_msn > idata->max_msn)
    return;

  h = idata->msn_index[exp_msn - 1];
  if (h)
  {
    /* imap_expunge_mailbox() will rewrite h->index.
     * It needs to resort using SORT_ORDER anyway, so setting to INT_MAX
     * makes the code simpler and possibly more efficient. */
    h->index = INT_MAX;
    HEADER_DATA(h)->msn = 0;
  }

  /* decrement seqno of those above. */
  for (cur = exp_msn; cur < idata->max_msn; cur++)
  {
    h = idata->msn_index[cur];
    if (h)
      HEADER_DATA(h)->msn--;
    idata->msn_index[cur - 1] = h;
  }

  idata->msn_index[idata->max_msn - 1] = NULL;
  idata->max_msn--;

  idata->reopen |= IMAP_EXPUNGE_PENDING;
}


static void cmd_parse_search (IMAP_DATA* idata, const char* s)
{
  unsigned int uid;
  HEADER *h;

  dprint (2, (debugfile, "Handling SEARCH\n"));

  while ((s = imap_next_word ((char*)s)) && *s != '\0')
  {
    if (mutt_atoui (s, &uid) < 0)
      continue;
    h = (HEADER *)int_hash_find (idata->uid_hash, uid);
    if (h)
      h->matched = 1;
  }
}


static IMAP_COMMAND* cmd_new (IMAP_DATA* idata)
{
  IMAP_COMMAND* cmd;

  if (cmd_queue_full (idata))
  {
    dprint (3, (debugfile, "cmd_new: IMAP command queue full\n"));
    return NULL;
  }

  cmd = idata->cmds + idata->nextcmd;
  idata->nextcmd = (idata->nextcmd + 1) % idata->cmdslots;

  snprintf (cmd->seq, sizeof (cmd->seq), "a%04u", idata->seqno++);
  if (idata->seqno > 9999)
    idata->seqno = 0;

  cmd->state = IMAP_CMD_NEW;

  return cmd;
}


int imap_cmd_step (IMAP_DATA* idata)
{
  size_t len = 0;
  int c;
  int rc;
  int stillrunning = 0;
  IMAP_COMMAND* cmd;

  if (idata->status == IMAP_FATAL)
  {
    cmd_handle_fatal (idata);
    return IMAP_CMD_BAD;
  }

  /* read into buffer, expanding buffer as necessary until we have a full
   * line */
  do
  {
    if (len == idata->blen)
    {
      safe_realloc (&idata->buf, idata->blen + IMAP_CMD_BUFSIZE);
      idata->blen = idata->blen + IMAP_CMD_BUFSIZE;
      dprint (3, (debugfile, "imap_cmd_step: grew buffer to %u bytes\n",
		  idata->blen));
    }

    /* back up over '\0' */
    if (len)
      len--;
    c = mutt_socket_readln (idata->buf + len, idata->blen - len, idata->conn);
    if (c <= 0)
    {
      dprint (1, (debugfile, "imap_cmd_step: Error reading server response.\n"));
      cmd_handle_fatal (idata);
      return IMAP_CMD_BAD;
    }

    len += c;
  }
  /* if we've read all the way to the end of the buffer, we haven't read a
   * full line (mutt_socket_readln strips the \r, so we always have at least
   * one character free when we've read a full line) */
  while (len == idata->blen);

  /* don't let one large string make cmd->buf hog memory forever */
  if ((idata->blen > IMAP_CMD_BUFSIZE) && (len <= IMAP_CMD_BUFSIZE))
  {
    safe_realloc (&idata->buf, IMAP_CMD_BUFSIZE);
    idata->blen = IMAP_CMD_BUFSIZE;
    dprint (3, (debugfile, "imap_cmd_step: shrank buffer to %u bytes\n", idata->blen));
  }

  idata->lastread = time (NULL);

  /* handle untagged messages. The caller still gets its shot afterwards. */
  if ((!ascii_strncmp (idata->buf, "* ", 2)
       || !ascii_strncmp (imap_next_word (idata->buf), "OK [", 4))
      && cmd_handle_untagged (idata))
    return IMAP_CMD_BAD;

  /* server demands a continuation response from us */
  if (idata->buf[0] == '+')
    return IMAP_CMD_RESPOND;

  /* Look for tagged command completions.
   *
   * Some response handlers can end up recursively calling
   * imap_cmd_step() and end up handling all tagged command
   * completions.
   * (e.g. FETCH->set_flag->set_header_color->~h pattern match.)
   *
   * Other callers don't even create an idata->cmds entry.
   *
   * For both these cases, we default to returning OK */
  rc = IMAP_CMD_OK;
  c = idata->lastcmd;
  do
  {
    cmd = &idata->cmds[c];
    if (cmd->state == IMAP_CMD_NEW)
    {
      if (!ascii_strncmp (idata->buf, cmd->seq, SEQLEN)) {
	if (!stillrunning)
	{
	  /* first command in queue has finished - move queue pointer up */
	  idata->lastcmd = (idata->lastcmd + 1) % idata->cmdslots;
	}
	cmd->state = cmd_status (idata->buf);
	/* bogus - we don't know which command result to return here. Caller
	 * should provide a tag. */
	rc = cmd->state;
      }
      else
	stillrunning++;
    }

    c = (c + 1) % idata->cmdslots;
  }
  while (c != idata->nextcmd);

  if (stillrunning)
    rc = IMAP_CMD_CONTINUE;
  else
  {
    dprint (3, (debugfile, "IMAP queue drained\n"));
    imap_cmd_finish (idata);
  }

  return rc;
}


int imap_code (const char *s)
{
  return cmd_status (s) == IMAP_CMD_OK;
}


void imap_cmd_finish (IMAP_DATA* idata)
{
  if (idata->status == IMAP_FATAL)
  {
    cmd_handle_fatal (idata);
    return;
  }

  if (!(idata->state >= IMAP_SELECTED) || idata->ctx->closing)
    return;

  if (idata->reopen & IMAP_REOPEN_ALLOW)
  {
    unsigned int count = idata->newMailCount;

    if (!(idata->reopen & IMAP_EXPUNGE_PENDING) &&
	(idata->reopen & IMAP_NEWMAIL_PENDING)
	&& count > idata->max_msn)
    {
      /* read new mail messages */
      dprint (2, (debugfile, "imap_cmd_finish: Fetching new mail\n"));
      /* check_status: curs_main uses imap_check_mailbox to detect
       *   whether the index needs updating */
      idata->check_status = IMAP_NEWMAIL_PENDING;
      imap_read_headers (idata, idata->max_msn+1, count);
    }
    else if (idata->reopen & IMAP_EXPUNGE_PENDING)
    {
      dprint (2, (debugfile, "imap_cmd_finish: Expunging mailbox\n"));
      imap_expunge_mailbox (idata);
      /* Detect whether we've gotten unexpected EXPUNGE messages */
      if ((idata->reopen & IMAP_EXPUNGE_PENDING) &&
	  !(idata->reopen & IMAP_EXPUNGE_EXPECTED))
	idata->check_status = IMAP_EXPUNGE_PENDING;
      idata->reopen &= ~(IMAP_EXPUNGE_PENDING | IMAP_NEWMAIL_PENDING |
			 IMAP_EXPUNGE_EXPECTED);
    }
  }

  idata->status = 0;
}


const char* imap_cmd_trailer (IMAP_DATA* idata)
{
  static const char* notrailer = "";
  const char* s = idata->buf;

  if (!s)
  {
    dprint (2, (debugfile, "imap_cmd_trailer: not a tagged response"));
    return notrailer;
  }

  s = imap_next_word ((char *)s);
  if (!s || (ascii_strncasecmp (s, "OK", 2) &&
	     ascii_strncasecmp (s, "NO", 2) &&
	     ascii_strncasecmp (s, "BAD", 3)))
  {
    dprint (2, (debugfile, "imap_cmd_trailer: not a command completion: %s",
		idata->buf));
    return notrailer;
  }

  s = imap_next_word ((char *)s);
  if (!s)
    return notrailer;

  return s;
}


static void cmd_parse_capability (IMAP_DATA* idata, char* s)
{
  int x;
  char* bracket;

  dprint (3, (debugfile, "Handling CAPABILITY\n"));

  s = imap_next_word (s);
  if ((bracket = strchr (s, ']')))
    *bracket = '\0';
  FREE(&idata->capstr);
  idata->capstr = safe_strdup (s);

  memset (idata->capabilities, 0, sizeof (idata->capabilities));

  while (*s)
  {
    for (x = 0; x < CAPMAX; x++)
      if (imap_wordcasecmp(Capabilities[x], s) == 0)
      {
	mutt_bit_set (idata->capabilities, x);
	break;
      }
    s = imap_next_word (s);
  }
}


static void cmd_parse_enabled (IMAP_DATA* idata, const char* s)
{
  dprint (2, (debugfile, "Handling ENABLED\n"));

  while ((s = imap_next_word ((char*)s)) && *s != '\0')
  {
    if (ascii_strncasecmp(s, "UTF8=ACCEPT", 11) == 0 ||
        ascii_strncasecmp(s, "UTF8=ONLY", 9) == 0)
      idata->unicode = 1;
  }
}


static int cmd_queue_full (IMAP_DATA* idata)
{
  if ((idata->nextcmd + 1) % idata->cmdslots == idata->lastcmd)
    return 1;

  return 0;
}


static void cmd_parse_fetch (IMAP_DATA* idata, char* s)
{
  unsigned int msn, uid;
  HEADER *h;
  int server_changes = 0;

  dprint (3, (debugfile, "Handling FETCH\n"));

  if (mutt_atoui (s, &msn) < 0 ||
      msn < 1 || msn > idata->max_msn)
  {
    dprint (3, (debugfile, "FETCH response ignored for this message\n"));
    return;
  }

  h = idata->msn_index[msn - 1];
  if (!h || !h->active)
  {
    dprint (3, (debugfile, "FETCH response ignored for this message\n"));
    return;
  }

  dprint (2, (debugfile, "Message UID %u updated\n", HEADER_DATA(h)->uid));
  /* skip FETCH */
  s = imap_next_word (s);
  s = imap_next_word (s);

  if (*s != '(')
  {
    dprint (1, (debugfile, "Malformed FETCH response"));
    return;
  }
  s++;

  while (*s)
  {
    SKIPWS (s);

    if (ascii_strncasecmp ("FLAGS", s, 5) == 0)
    {
      imap_set_flags (idata, h, s, &server_changes);
      if (server_changes)
      {
        /* If server flags could conflict with mutt's flags, reopen the mailbox. */
        if (h->changed)
          idata->reopen |= IMAP_EXPUNGE_PENDING;
        else
          idata->check_status = IMAP_FLAGS_PENDING;
      }
      return;
    }
    else if (ascii_strncasecmp ("UID", s, 3) == 0)
    {
      s += 3;
      SKIPWS (s);
      if (mutt_atoui (s, &uid) < 0)
      {
        dprint (2, (debugfile, "Illegal UID.  Skipping update.\n"));
        return;
      }
      if (uid != HEADER_DATA(h)->uid)
      {
        dprint (2, (debugfile, "FETCH UID vs MSN mismatch.  Skipping update.\n"));
        return;
      }
      s = imap_next_word (s);
    }
    else if (*s == ')')
      s++; /* end of request */
    else if (*s)
    {
      dprint (2, (debugfile, "Only handle FLAGS updates\n"));
      return;
    }
  }
}


static int cmd_status (const char *s)
{
  s = imap_next_word((char*)s);

  if (!ascii_strncasecmp("OK", s, 2))
    return IMAP_CMD_OK;
  if (!ascii_strncasecmp("NO", s, 2))
    return IMAP_CMD_NO;

  return IMAP_CMD_BAD;
}


static void cmd_parse_myrights (IMAP_DATA* idata, const char* s)
{
  dprint (2, (debugfile, "Handling MYRIGHTS\n"));

  s = imap_next_word ((char*)s);
  s = imap_next_word ((char*)s);

  /* zero out current rights set */
  memset (idata->ctx->rights, 0, sizeof (idata->ctx->rights));

  while (*s && !isspace((unsigned char) *s))
  {
    switch (*s)
    {
      case 'l':
	mutt_bit_set (idata->ctx->rights, MUTT_ACL_LOOKUP);
	break;
      case 'r':
	mutt_bit_set (idata->ctx->rights, MUTT_ACL_READ);
	break;
      case 's':
	mutt_bit_set (idata->ctx->rights, MUTT_ACL_SEEN);
	break;
      case 'w':
	mutt_bit_set (idata->ctx->rights, MUTT_ACL_WRITE);
	break;
      case 'i':
	mutt_bit_set (idata->ctx->rights, MUTT_ACL_INSERT);
	break;
      case 'p':
	mutt_bit_set (idata->ctx->rights, MUTT_ACL_POST);
	break;
      case 'a':
	mutt_bit_set (idata->ctx->rights, MUTT_ACL_ADMIN);
	break;
      case 'k':
	mutt_bit_set (idata->ctx->rights, MUTT_ACL_CREATE);
        break;
      case 'x':
        mutt_bit_set (idata->ctx->rights, MUTT_ACL_DELMX);
        break;
      case 't':
	mutt_bit_set (idata->ctx->rights, MUTT_ACL_DELETE);
        break;
      case 'e':
        mutt_bit_set (idata->ctx->rights, MUTT_ACL_EXPUNGE);
        break;

        /* obsolete rights */
      case 'c':
	mutt_bit_set (idata->ctx->rights, MUTT_ACL_CREATE);
        mutt_bit_set (idata->ctx->rights, MUTT_ACL_DELMX);
	break;
      case 'd':
	mutt_bit_set (idata->ctx->rights, MUTT_ACL_DELETE);
        mutt_bit_set (idata->ctx->rights, MUTT_ACL_EXPUNGE);
	break;
      default:
        dprint(1, (debugfile, "Unknown right: %c\n", *s));
    }
    s++;
  }
}


static void cmd_parse_lsub (IMAP_DATA* idata, char* s)
{
  char buf[STRING];
  char errstr[STRING];
  BUFFER err, token;
  ciss_url_t url;
  IMAP_LIST list;

  if (idata->cmddata && idata->cmdtype == IMAP_CT_LIST)
  {
    /* caller will handle response itself */
    cmd_parse_list (idata, s);
    return;
  }

  if (!option (OPTIMAPCHECKSUBSCRIBED))
    return;

  idata->cmdtype = IMAP_CT_LIST;
  idata->cmddata = &list;
  cmd_parse_list (idata, s);
  idata->cmddata = NULL;
  /* noselect is for a gmail quirk (#3445) */
  if (!list.name || list.noselect)
    return;

  dprint (3, (debugfile, "Subscribing to %s\n", list.name));

  strfcpy (buf, "mailboxes \"", sizeof (buf));
  mutt_account_tourl (&idata->conn->account, &url);
  /* escape \ and ". Also escape ` because the resulting
   * string will be passed to mutt_parse_rc_line. */
  imap_quote_string_and_backquotes (errstr, sizeof (errstr), list.name);
  url.path = errstr + 1;
  url.path[strlen(url.path) - 1] = '\0';
  if (!mutt_strcmp (url.user, ImapUser))
    url.user = NULL;
  url_ciss_tostring (&url, buf + 11, sizeof (buf) - 11, 0);
  safe_strcat (buf, sizeof (buf), "\"");
  mutt_buffer_init (&token);
  mutt_buffer_init (&err);
  err.data = errstr;
  err.dsize = sizeof (errstr);
  if (mutt_parse_rc_line (buf, &token, &err))
    dprint (1, (debugfile, "Error adding subscribed mailbox: %s\n", errstr));
  FREE (&token.data);
}


static void cmd_parse_list (IMAP_DATA* idata, char* s)
{
  IMAP_LIST* list;
  IMAP_LIST lb;
  char delimbuf[5]; /* worst case: "\\"\0 */
  unsigned int litlen;

  if (idata->cmddata && idata->cmdtype == IMAP_CT_LIST)
    list = (IMAP_LIST*)idata->cmddata;
  else
    list = &lb;

  memset (list, 0, sizeof (IMAP_LIST));

  /* flags */
  s = imap_next_word (s);
  if (*s != '(')
  {
    dprint (1, (debugfile, "Bad LIST response\n"));
    return;
  }
  s++;
  while (*s)
  {
    if (!ascii_strncasecmp (s, "\\NoSelect", 9))
      list->noselect = 1;
    else if (!ascii_strncasecmp (s, "\\NoInferiors", 12))
      list->noinferiors = 1;
    /* See draft-gahrns-imap-child-mailbox-?? */
    else if (!ascii_strncasecmp (s, "\\HasNoChildren", 14))
      list->noinferiors = 1;

    s = imap_next_word (s);
    if (*(s - 2) == ')')
      break;
  }

  /* Delimiter */
  if (ascii_strncasecmp (s, "NIL", 3))
  {
    delimbuf[0] = '\0';
    safe_strcat (delimbuf, 5, s);
    imap_unquote_string (delimbuf);
    list->delim = delimbuf[0];
  }

  /* Name */
  s = imap_next_word (s);
  /* Notes often responds with literals here. We need a real tokenizer. */
  if (!imap_get_literal_count (s, &litlen))
  {
    if (imap_cmd_step (idata) != IMAP_CMD_CONTINUE)
    {
      idata->status = IMAP_FATAL;
      return;
    }
    list->name = idata->buf;
  }
  else
  {
    imap_unmunge_mbox_name (idata, s);
    list->name = s;
  }

  if (list->name[0] == '\0')
  {
    idata->delim = list->delim;
    dprint (3, (debugfile, "Root delimiter: %c\n", idata->delim));
  }
}


MagickExport Image *DisposeImages(const Image *images,ExceptionInfo *exception)
{
  Image
    *dispose_image,
    *dispose_images;

  RectangleInfo
    bounds;

  register Image
    *image,
    *next;

  /*
    Run the image through the animation sequence
  */
  assert(images != (Image *) NULL);
  assert(images->signature == MagickCoreSignature);
  if (images->debug != MagickFalse)
    (void) LogMagickEvent(TraceEvent,GetMagickModule(),"%s",images->filename);
  assert(exception != (ExceptionInfo *) NULL);
  assert(exception->signature == MagickCoreSignature);
  image=GetFirstImageInList(images);
  dispose_image=CloneImage(image,image->page.width,image->page.height,
    MagickTrue,exception);
  if (dispose_image == (Image *) NULL)
    return((Image *) NULL);
  dispose_image->page=image->page;
  dispose_image->page.x=0;
  dispose_image->page.y=0;
  dispose_image->dispose=NoneDispose;
  dispose_image->background_color.alpha_trait=BlendPixelTrait;
  dispose_image->background_color.alpha=(MagickRealType) TransparentAlpha;
  (void) SetImageBackgroundColor(dispose_image,exception);
  dispose_images=NewImageList();
  for (next=image; image != (Image *) NULL; image=GetNextImageInList(image))
  {
    Image
      *current_image;

    /*
      Overlay this frame's image over the previous disposal image.
    */
    current_image=CloneImage(dispose_image,0,0,MagickTrue,exception);
    if (current_image == (Image *) NULL)
      {
        dispose_images=DestroyImageList(dispose_images);
        dispose_image=DestroyImage(dispose_image);
        return((Image *) NULL);
      }
    current_image->background_color.alpha_trait=BlendPixelTrait;
    (void) CompositeImage(current_image,next,
      next->alpha_trait != UndefinedPixelTrait ? OverCompositeOp : CopyCompositeOp,
      MagickTrue,next->page.x,next->page.y,exception);
    /*
      Handle Background dispose: image is displayed for the delay period.
    */
    if (next->dispose == BackgroundDispose)
      {
        bounds=next->page;
        bounds.width=next->columns;
        bounds.height=next->rows;
        if (bounds.x < 0)
          {
            bounds.width+=bounds.x;
            bounds.x=0;
          }
        if ((ssize_t) (bounds.x+bounds.width) > (ssize_t) current_image->columns)
          bounds.width=current_image->columns-bounds.x;
        if (bounds.y < 0)
          {
            bounds.height+=bounds.y;
            bounds.y=0;
          }
        if ((ssize_t) (bounds.y+bounds.height) > (ssize_t) current_image->rows)
          bounds.height=current_image->rows-bounds.y;
        ClearBounds(current_image,&bounds,exception);
      }
    /*
      Select the appropriate previous/disposed image.
    */
    if (next->dispose == PreviousDispose)
      current_image=DestroyImage(current_image);
    else
      {
        dispose_image=DestroyImage(dispose_image);
        dispose_image=current_image;
        current_image=(Image *) NULL;
      }
    /*
      Save the dispose image just calculated for return.
    */
    {
      Image
        *dispose;

      dispose=CloneImage(dispose_image,0,0,MagickTrue,exception);
      if (dispose == (Image *) NULL)
        {
          dispose_images=DestroyImageList(dispose_images);
          dispose_image=DestroyImage(dispose_image);
          return((Image *) NULL);
        }
      dispose_image->background_color.alpha_trait=BlendPixelTrait;
      (void) CloneImageProfiles(dispose,next);
      (void) CloneImageProperties(dispose,next);
      (void) CloneImageArtifacts(dispose,next);
      dispose->page.x=0;
      dispose->page.y=0;
      dispose->dispose=next->dispose;
      AppendImageToList(&dispose_images,dispose);
    }
  }
  dispose_image=DestroyImage(dispose_image);
  return(GetFirstImageInList(dispose_images));
}

There is a Division by Zero in function OptimizeLayerFrames (#2743)

in file MagickCore/layer.c. cur->ticks_per_seconds can be zero
with a crafted input argument *image. This is similar to
CVE-2019-13454.


MagickExport Image *CompareImagesLayers(const Image *image,
  const LayerMethod method,ExceptionInfo *exception)
{
  Image
    *image_a,
    *image_b,
    *layers;

  RectangleInfo
    *bounds;

  register const Image
    *next;

  register ssize_t
    i;

  assert(image != (const Image *) NULL);
  assert(image->signature == MagickCoreSignature);
  if (image->debug != MagickFalse)
    (void) LogMagickEvent(TraceEvent,GetMagickModule(),"%s",image->filename);
  assert(exception != (ExceptionInfo *) NULL);
  assert(exception->signature == MagickCoreSignature);
  assert((method == CompareAnyLayer) ||
         (method == CompareClearLayer) ||
         (method == CompareOverlayLayer));
  /*
    Allocate bounds memory.
  */
  next=GetFirstImageInList(image);
  bounds=(RectangleInfo *) AcquireQuantumMemory((size_t)
    GetImageListLength(next),sizeof(*bounds));
  if (bounds == (RectangleInfo *) NULL)
    ThrowImageException(ResourceLimitError,"MemoryAllocationFailed");
  /*
    Set up first comparision images.
  */
  image_a=CloneImage(next,next->page.width,next->page.height,
    MagickTrue,exception);
  if (image_a == (Image *) NULL)
    {
      bounds=(RectangleInfo *) RelinquishMagickMemory(bounds);
      return((Image *) NULL);
    }
  image_a->background_color.alpha_trait=BlendPixelTrait;
  image_a->background_color.alpha=(MagickRealType) TransparentAlpha;
  (void) SetImageBackgroundColor(image_a,exception);
  image_a->page=next->page;
  image_a->page.x=0;
  image_a->page.y=0;
  (void) CompositeImage(image_a,next,CopyCompositeOp,MagickTrue,next->page.x,
    next->page.y,exception);
  /*
    Compute the bounding box of changes for the later images
  */
  i=0;
  next=GetNextImageInList(next);
  for ( ; next != (const Image *) NULL; next=GetNextImageInList(next))
  {
    image_b=CloneImage(image_a,0,0,MagickTrue,exception);
    if (image_b == (Image *) NULL)
      {
        image_a=DestroyImage(image_a);
        bounds=(RectangleInfo *) RelinquishMagickMemory(bounds);
        return((Image *) NULL);
      }
    image_b->background_color.alpha_trait=BlendPixelTrait;
    (void) CompositeImage(image_a,next,CopyCompositeOp,MagickTrue,next->page.x,
      next->page.y,exception);
    bounds[i]=CompareImagesBounds(image_b,image_a,method,exception);
    image_b=DestroyImage(image_b);
    i++;
  }
  image_a=DestroyImage(image_a);
  /*
    Clone first image in sequence.
  */
  next=GetFirstImageInList(image);
  layers=CloneImage(next,0,0,MagickTrue,exception);
  if (layers == (Image *) NULL)
    {
      bounds=(RectangleInfo *) RelinquishMagickMemory(bounds);
      return((Image *) NULL);
    }
  layers->background_color.alpha_trait=BlendPixelTrait;
  /*
    Deconstruct the image sequence.
  */
  i=0;
  next=GetNextImageInList(next);
  for ( ; next != (const Image *) NULL; next=GetNextImageInList(next))
  {
    if ((bounds[i].x == -1) && (bounds[i].y == -1) &&
        (bounds[i].width == 1) && (bounds[i].height == 1))
      {
        /*
          An empty frame is returned from CompareImageBounds(), which means the
          current frame is identical to the previous frame.
        */
        i++;
        continue;
      }
    image_a=CloneImage(next,0,0,MagickTrue,exception);
    if (image_a == (Image *) NULL)
      break;
    image_a->background_color.alpha_trait=BlendPixelTrait;
    image_b=CropImage(image_a,&bounds[i],exception);
    image_a=DestroyImage(image_a);
    if (image_b == (Image *) NULL)
      break;
    AppendImageToList(&layers,image_b);
    i++;
  }
  bounds=(RectangleInfo *) RelinquishMagickMemory(bounds);
  if (next != (Image *) NULL)
    {
      layers=DestroyImageList(layers);
      return((Image *) NULL);
    }
  return(GetFirstImageInList(layers));
}


MagickExport Image *MergeImageLayers(Image *image,const LayerMethod method,
  ExceptionInfo *exception)
{
#define MergeLayersTag  "Merge/Layers"

  Image
    *canvas;

  MagickBooleanType
    proceed;

  RectangleInfo
    page;

  register const Image
    *next;

  size_t
    number_images,
    height,
    width;

  ssize_t
    scene;

  assert(image != (Image *) NULL);
  assert(image->signature == MagickCoreSignature);
  if (image->debug != MagickFalse)
    (void) LogMagickEvent(TraceEvent,GetMagickModule(),"%s",image->filename);
  assert(exception != (ExceptionInfo *) NULL);
  assert(exception->signature == MagickCoreSignature);
  /*
    Determine canvas image size, and its virtual canvas size and offset
  */
  page=image->page;
  width=image->columns;
  height=image->rows;
  switch (method)
  {
    case TrimBoundsLayer:
    case MergeLayer:
    default:
    {
      next=GetNextImageInList(image);
      for ( ; next != (Image *) NULL;  next=GetNextImageInList(next))
      {
        if (page.x > next->page.x)
          {
            width+=page.x-next->page.x;
            page.x=next->page.x;
          }
        if (page.y > next->page.y)
          {
            height+=page.y-next->page.y;
            page.y=next->page.y;
          }
        if ((ssize_t) width < (next->page.x+(ssize_t) next->columns-page.x))
          width=(size_t) next->page.x+(ssize_t) next->columns-page.x;
        if ((ssize_t) height < (next->page.y+(ssize_t) next->rows-page.y))
          height=(size_t) next->page.y+(ssize_t) next->rows-page.y;
      }
      break;
    }
    case FlattenLayer:
    {
      if (page.width > 0)
        width=page.width;
      if (page.height > 0)
        height=page.height;
      page.x=0;
      page.y=0;
      break;
    }
    case MosaicLayer:
    {
      if (page.width > 0)
        width=page.width;
      if (page.height > 0)
        height=page.height;
      for (next=image; next != (Image *) NULL; next=GetNextImageInList(next))
      {
        if (method == MosaicLayer)
          {
            page.x=next->page.x;
            page.y=next->page.y;
            if ((ssize_t) width < (next->page.x+(ssize_t) next->columns))
              width=(size_t) next->page.x+next->columns;
            if ((ssize_t) height < (next->page.y+(ssize_t) next->rows))
              height=(size_t) next->page.y+next->rows;
          }
      }
      page.width=width;
      page.height=height;
      page.x=0;
      page.y=0;
    }
    break;
  }
  /*
    Set virtual canvas size if not defined.
  */
  if (page.width == 0)
    page.width=page.x < 0 ? width : width+page.x;
  if (page.height == 0)
    page.height=page.y < 0 ? height : height+page.y;
  /*
    Handle "TrimBoundsLayer" method separately to normal 'layer merge'.
  */
  if (method == TrimBoundsLayer)
    {
      number_images=GetImageListLength(image);
      for (scene=0; scene < (ssize_t) number_images; scene++)
      {
        image->page.x-=page.x;
        image->page.y-=page.y;
        image->page.width=width;
        image->page.height=height;
        proceed=SetImageProgress(image,MergeLayersTag,(MagickOffsetType) scene,
          number_images);
        if (proceed == MagickFalse)
          break;
        image=GetNextImageInList(image);
        if (image == (Image *) NULL)
          break;
      }
      return((Image *) NULL);
    }
  /*
    Create canvas size of width and height, and background color.
  */
  canvas=CloneImage(image,width,height,MagickTrue,exception);
  if (canvas == (Image *) NULL)
    return((Image *) NULL);
  canvas->background_color.alpha_trait=BlendPixelTrait;
  (void) SetImageBackgroundColor(canvas,exception);
  canvas->page=page;
  canvas->dispose=UndefinedDispose;
  /*
    Compose images onto canvas, with progress monitor
  */
  number_images=GetImageListLength(image);
  for (scene=0; scene < (ssize_t) number_images; scene++)
  {
    (void) CompositeImage(canvas,image,image->compose,MagickTrue,image->page.x-
      canvas->page.x,image->page.y-canvas->page.y,exception);
    proceed=SetImageProgress(image,MergeLayersTag,(MagickOffsetType) scene,
      number_images);
    if (proceed == MagickFalse)
      break;
    image=GetNextImageInList(image);
    if (image == (Image *) NULL)
      break;
  }
  return(canvas);
}


MagickExport Image *OptimizeImageLayers(const Image *image,
  ExceptionInfo *exception)
{
  return(OptimizeLayerFrames(image,OptimizeImageLayer,exception));
}


static Image *OptimizeLayerFrames(const Image *image,const LayerMethod method,
  ExceptionInfo *exception)
{
  ExceptionInfo
    *sans_exception;

  Image
    *prev_image,
    *dup_image,
    *bgnd_image,
    *optimized_image;

  RectangleInfo
    try_bounds,
    bgnd_bounds,
    dup_bounds,
    *bounds;

  MagickBooleanType
    add_frames,
    try_cleared,
    cleared;

  DisposeType
    *disposals;

  register const Image
    *curr;

  register ssize_t
    i;

  assert(image != (const Image *) NULL);
  assert(image->signature == MagickCoreSignature);
  if (image->debug != MagickFalse)
    (void) LogMagickEvent(TraceEvent,GetMagickModule(),"%s",image->filename);
  assert(exception != (ExceptionInfo *) NULL);
  assert(exception->signature == MagickCoreSignature);
  assert(method == OptimizeLayer ||
         method == OptimizeImageLayer ||
         method == OptimizePlusLayer);
  /*
    Are we allowed to add/remove frames from animation?
  */
  add_frames=method == OptimizePlusLayer ? MagickTrue : MagickFalse;
  /*
    Ensure  all the images are the same size.
  */
  curr=GetFirstImageInList(image);
  for (; curr != (Image *) NULL; curr=GetNextImageInList(curr))
  {
    if ((curr->columns != image->columns) || (curr->rows != image->rows))
      ThrowImageException(OptionError,"ImagesAreNotTheSameSize");

    if ((curr->page.x != 0) || (curr->page.y != 0) ||
        (curr->page.width != image->page.width) ||
        (curr->page.height != image->page.height))
      ThrowImageException(OptionError,"ImagePagesAreNotCoalesced");
  }
  /*
    Allocate memory (times 2 if we allow the use of frame duplications)
  */
  curr=GetFirstImageInList(image);
  bounds=(RectangleInfo *) AcquireQuantumMemory((size_t)
    GetImageListLength(curr),(add_frames != MagickFalse ? 2UL : 1UL)*
    sizeof(*bounds));
  if (bounds == (RectangleInfo *) NULL)
    ThrowImageException(ResourceLimitError,"MemoryAllocationFailed");
  disposals=(DisposeType *) AcquireQuantumMemory((size_t)
    GetImageListLength(image),(add_frames != MagickFalse ? 2UL : 1UL)*
    sizeof(*disposals));
  if (disposals == (DisposeType *) NULL)
    {
      bounds=(RectangleInfo *) RelinquishMagickMemory(bounds);
      ThrowImageException(ResourceLimitError,"MemoryAllocationFailed");
    }
  /*
    Initialise Previous Image as fully transparent
  */
  prev_image=CloneImage(curr,curr->columns,curr->rows,MagickTrue,exception);
  if (prev_image == (Image *) NULL)
    {
      bounds=(RectangleInfo *) RelinquishMagickMemory(bounds);
      disposals=(DisposeType *) RelinquishMagickMemory(disposals);
      return((Image *) NULL);
    }
  prev_image->page=curr->page;  /* ERROR: <-- should not be need, but is! */
  prev_image->page.x=0;
  prev_image->page.y=0;
  prev_image->dispose=NoneDispose;
  prev_image->background_color.alpha_trait=BlendPixelTrait;
  prev_image->background_color.alpha=(MagickRealType) TransparentAlpha;
  (void) SetImageBackgroundColor(prev_image,exception);
  /*
    Figure out the area of overlay of the first frame
    No pixel could be cleared as all pixels are already cleared.
  */
#if DEBUG_OPT_FRAME
  i=0;
  (void) FormatLocaleFile(stderr,"frame %.20g :-\n",(double) i);
#endif
  disposals[0]=NoneDispose;
  bounds[0]=CompareImagesBounds(prev_image,curr,CompareAnyLayer,exception);
#if DEBUG_OPT_FRAME
  (void) FormatLocaleFile(stderr, "overlay: %.20gx%.20g%+.20g%+.20g\n\n",
    (double) bounds[i].width,(double) bounds[i].height,
    (double) bounds[i].x,(double) bounds[i].y );
#endif
  /*
    Compute the bounding box of changes for each pair of images.
  */
  i=1;
  bgnd_image=(Image *) NULL;
  dup_image=(Image *) NULL;
  dup_bounds.width=0;
  dup_bounds.height=0;
  dup_bounds.x=0;
  dup_bounds.y=0;
  curr=GetNextImageInList(curr);
  for ( ; curr != (const Image *) NULL; curr=GetNextImageInList(curr))
  {
#if DEBUG_OPT_FRAME
    (void) FormatLocaleFile(stderr,"frame %.20g :-\n",(double) i);
#endif
    /*
      Assume none disposal is the best
    */
    bounds[i]=CompareImagesBounds(curr->previous,curr,CompareAnyLayer,exception);
    cleared=IsBoundsCleared(curr->previous,curr,&bounds[i],exception);
    disposals[i-1]=NoneDispose;
#if DEBUG_OPT_FRAME
    (void) FormatLocaleFile(stderr, "overlay: %.20gx%.20g%+.20g%+.20g%s%s\n",
         (double) bounds[i].width,(double) bounds[i].height,
         (double) bounds[i].x,(double) bounds[i].y,
         bounds[i].x < 0?"  (unchanged)":"",
         cleared?"  (pixels cleared)":"");
#endif
    if ( bounds[i].x < 0 ) {
      /*
        Image frame is exactly the same as the previous frame!
        If not adding frames leave it to be cropped down to a null image.
        Otherwise mark previous image for deleted, transfering its crop bounds
        to the current image.
      */
      if ( add_frames && i>=2 ) {
        disposals[i-1]=DelDispose;
        disposals[i]=NoneDispose;
        bounds[i]=bounds[i-1];
        i++;
        continue;
      }
    }
    else
      {
        /*
          Compare a none disposal against a previous disposal
        */
        try_bounds=CompareImagesBounds(prev_image,curr,CompareAnyLayer,exception);
        try_cleared=IsBoundsCleared(prev_image,curr,&try_bounds,exception);
#if DEBUG_OPT_FRAME
    (void) FormatLocaleFile(stderr, "test_prev: %.20gx%.20g%+.20g%+.20g%s\n",
         (double) try_bounds.width,(double) try_bounds.height,
         (double) try_bounds.x,(double) try_bounds.y,
         try_cleared?"  (pixels were cleared)":"");
#endif
        if ( (!try_cleared && cleared ) ||
                try_bounds.width * try_bounds.height
                    <  bounds[i].width * bounds[i].height )
          {
            cleared=try_cleared;
            bounds[i]=try_bounds;
            disposals[i-1]=PreviousDispose;
#if DEBUG_OPT_FRAME
            (void) FormatLocaleFile(stderr,"previous: accepted\n");
          } else {
            (void) FormatLocaleFile(stderr,"previous: rejected\n");
#endif
          }

        /*
          If we are allowed lets try a complex frame duplication.
          It is useless if the previous image already clears pixels correctly.
          This method will always clear all the pixels that need to be cleared.
        */
        dup_bounds.width=dup_bounds.height=0; /* no dup, no pixel added */
        if ( add_frames )
          {
            dup_image=CloneImage(curr->previous,0,0,MagickTrue,exception);
            if (dup_image == (Image *) NULL)
              {
                bounds=(RectangleInfo *) RelinquishMagickMemory(bounds);
                disposals=(DisposeType *) RelinquishMagickMemory(disposals);
                prev_image=DestroyImage(prev_image);
                return((Image *) NULL);
              }
            dup_image->background_color.alpha_trait=BlendPixelTrait;
            dup_bounds=CompareImagesBounds(dup_image,curr,CompareClearLayer,exception);
            ClearBounds(dup_image,&dup_bounds,exception);
            try_bounds=CompareImagesBounds(dup_image,curr,CompareAnyLayer,exception);
            if ( cleared ||
                   dup_bounds.width*dup_bounds.height
                      +try_bounds.width*try_bounds.height
                   < bounds[i].width * bounds[i].height )
              {
                cleared=MagickFalse;
                bounds[i]=try_bounds;
                disposals[i-1]=DupDispose;
                /* to be finalised later, if found to be optimial */
              }
            else
              dup_bounds.width=dup_bounds.height=0;
          }
        /*
          Now compare against a simple background disposal
        */
        bgnd_image=CloneImage(curr->previous,0,0,MagickTrue,exception);
        if (bgnd_image == (Image *) NULL)
          {
            bounds=(RectangleInfo *) RelinquishMagickMemory(bounds);
            disposals=(DisposeType *) RelinquishMagickMemory(disposals);
            prev_image=DestroyImage(prev_image);
            if ( dup_image != (Image *) NULL)
              dup_image=DestroyImage(dup_image);
            return((Image *) NULL);
          }
        bgnd_image->background_color.alpha_trait=BlendPixelTrait;
        bgnd_bounds=bounds[i-1]; /* interum bounds of the previous image */
        ClearBounds(bgnd_image,&bgnd_bounds,exception);
        try_bounds=CompareImagesBounds(bgnd_image,curr,CompareAnyLayer,exception);
        try_cleared=IsBoundsCleared(bgnd_image,curr,&try_bounds,exception);
#if DEBUG_OPT_FRAME
    (void) FormatLocaleFile(stderr, "background: %s\n",
         try_cleared?"(pixels cleared)":"");
#endif
        if ( try_cleared )
          {
            /*
              Straight background disposal failed to clear pixels needed!
              Lets try expanding the disposal area of the previous frame, to
              include the pixels that are cleared.  This guaranteed
              to work, though may not be the most optimized solution.
            */
            try_bounds=CompareImagesBounds(curr->previous,curr,CompareClearLayer,exception);
#if DEBUG_OPT_FRAME
            (void) FormatLocaleFile(stderr, "expand_clear: %.20gx%.20g%+.20g%+.20g%s\n",
                (double) try_bounds.width,(double) try_bounds.height,
                (double) try_bounds.x,(double) try_bounds.y,
                try_bounds.x<0?"  (no expand nessary)":"");
#endif
            if ( bgnd_bounds.x < 0 )
              bgnd_bounds = try_bounds;
            else
              {
#if DEBUG_OPT_FRAME
                (void) FormatLocaleFile(stderr, "expand_bgnd: %.20gx%.20g%+.20g%+.20g\n",
                    (double) bgnd_bounds.width,(double) bgnd_bounds.height,
                    (double) bgnd_bounds.x,(double) bgnd_bounds.y );
#endif
                if ( try_bounds.x < bgnd_bounds.x )
                  {
                     bgnd_bounds.width+= bgnd_bounds.x-try_bounds.x;
                     if ( bgnd_bounds.width < try_bounds.width )
                       bgnd_bounds.width = try_bounds.width;
                     bgnd_bounds.x = try_bounds.x;
                  }
                else
                  {
                     try_bounds.width += try_bounds.x - bgnd_bounds.x;
                     if ( bgnd_bounds.width < try_bounds.width )
                       bgnd_bounds.width = try_bounds.width;
                  }
                if ( try_bounds.y < bgnd_bounds.y )
                  {
                     bgnd_bounds.height += bgnd_bounds.y - try_bounds.y;
                     if ( bgnd_bounds.height < try_bounds.height )
                       bgnd_bounds.height = try_bounds.height;
                     bgnd_bounds.y = try_bounds.y;
                  }
                else
                  {
                    try_bounds.height += try_bounds.y - bgnd_bounds.y;
                     if ( bgnd_bounds.height < try_bounds.height )
                       bgnd_bounds.height = try_bounds.height;
                  }
#if DEBUG_OPT_FRAME
                (void) FormatLocaleFile(stderr, "        to : %.20gx%.20g%+.20g%+.20g\n",
                    (double) bgnd_bounds.width,(double) bgnd_bounds.height,
                    (double) bgnd_bounds.x,(double) bgnd_bounds.y );
#endif
              }
            ClearBounds(bgnd_image,&bgnd_bounds,exception);
#if DEBUG_OPT_FRAME
/* Something strange is happening with a specific animation
 * CompareAnyLayers (normal method) and CompareClearLayers returns the whole
 * image, which is not posibly correct!  As verified by previous tests.
 * Something changed beyond the bgnd_bounds clearing.  But without being able
 * to see, or writet he image at this point it is hard to tell what is wrong!
 * Only CompareOverlay seemed to return something sensible.
 */
            try_bounds=CompareImagesBounds(bgnd_image,curr,CompareClearLayer,exception);
            (void) FormatLocaleFile(stderr, "expand_ctst: %.20gx%.20g%+.20g%+.20g\n",
                (double) try_bounds.width,(double) try_bounds.height,
                (double) try_bounds.x,(double) try_bounds.y );
            try_bounds=CompareImagesBounds(bgnd_image,curr,CompareAnyLayer,exception);
            try_cleared=IsBoundsCleared(bgnd_image,curr,&try_bounds,exception);
            (void) FormatLocaleFile(stderr, "expand_any : %.20gx%.20g%+.20g%+.20g%s\n",
                (double) try_bounds.width,(double) try_bounds.height,
                (double) try_bounds.x,(double) try_bounds.y,
                try_cleared?"   (pixels cleared)":"");
#endif
            try_bounds=CompareImagesBounds(bgnd_image,curr,CompareOverlayLayer,exception);
#if DEBUG_OPT_FRAME
            try_cleared=IsBoundsCleared(bgnd_image,curr,&try_bounds,exception);
            (void) FormatLocaleFile(stderr, "expand_test: %.20gx%.20g%+.20g%+.20g%s\n",
                (double) try_bounds.width,(double) try_bounds.height,
                (double) try_bounds.x,(double) try_bounds.y,
                try_cleared?"   (pixels cleared)":"");
#endif
          }
        /*
          Test if this background dispose is smaller than any of the
          other methods we tryed before this (including duplicated frame)
        */
        if ( cleared ||
              bgnd_bounds.width*bgnd_bounds.height
                +try_bounds.width*try_bounds.height
              < bounds[i-1].width*bounds[i-1].height
                  +dup_bounds.width*dup_bounds.height
                  +bounds[i].width*bounds[i].height )
          {
            cleared=MagickFalse;
            bounds[i-1]=bgnd_bounds;
            bounds[i]=try_bounds;
            if ( disposals[i-1] == DupDispose )
              dup_image=DestroyImage(dup_image);
            disposals[i-1]=BackgroundDispose;
#if DEBUG_OPT_FRAME
    (void) FormatLocaleFile(stderr,"expand_bgnd: accepted\n");
          } else {
    (void) FormatLocaleFile(stderr,"expand_bgnd: reject\n");
#endif
          }
      }
    /*
       Finalise choice of dispose, set new prev_image,
       and junk any extra images as appropriate,
    */
    if ( disposals[i-1] == DupDispose )
      {
         if (bgnd_image != (Image *) NULL)
           bgnd_image=DestroyImage(bgnd_image);
         prev_image=DestroyImage(prev_image);
         prev_image=dup_image, dup_image=(Image *) NULL;
         bounds[i+1]=bounds[i];
         bounds[i]=dup_bounds;
         disposals[i-1]=DupDispose;
         disposals[i]=BackgroundDispose;
         i++;
      }
    else
      {
        if ( dup_image != (Image *) NULL)
          dup_image=DestroyImage(dup_image);
        if ( disposals[i-1] != PreviousDispose )
          prev_image=DestroyImage(prev_image);
        if ( disposals[i-1] == BackgroundDispose )
          prev_image=bgnd_image, bgnd_image=(Image *) NULL;
        if (bgnd_image != (Image *) NULL)
          bgnd_image=DestroyImage(bgnd_image);
        if ( disposals[i-1] == NoneDispose )
          {
            prev_image=ReferenceImage(curr->previous);
            if (prev_image == (Image *) NULL)
              {
                bounds=(RectangleInfo *) RelinquishMagickMemory(bounds);
                disposals=(DisposeType *) RelinquishMagickMemory(disposals);
                return((Image *) NULL);
              }
          }

      }
    assert(prev_image != (Image *) NULL);
    disposals[i]=disposals[i-1];
#if DEBUG_OPT_FRAME
    (void) FormatLocaleFile(stderr, "final   %.20g : %s  %.20gx%.20g%+.20g%+.20g\n",
         (double) i-1,
         CommandOptionToMnemonic(MagickDisposeOptions,disposals[i-1]),
         (double) bounds[i-1].width,(double) bounds[i-1].height,
         (double) bounds[i-1].x,(double) bounds[i-1].y );
#endif
#if DEBUG_OPT_FRAME
    (void) FormatLocaleFile(stderr, "interum %.20g : %s  %.20gx%.20g%+.20g%+.20g\n",
         (double) i,
         CommandOptionToMnemonic(MagickDisposeOptions,disposals[i]),
         (double) bounds[i].width,(double) bounds[i].height,
         (double) bounds[i].x,(double) bounds[i].y );
    (void) FormatLocaleFile(stderr,"\n");
#endif
    i++;
  }
  prev_image=DestroyImage(prev_image);
  /*
    Optimize all images in sequence.
  */
  sans_exception=AcquireExceptionInfo();
  i=0;
  curr=GetFirstImageInList(image);
  optimized_image=NewImageList();
  while ( curr != (const Image *) NULL )
  {
    prev_image=CloneImage(curr,0,0,MagickTrue,exception);
    if (prev_image == (Image *) NULL)
      break;
    prev_image->background_color.alpha_trait=BlendPixelTrait;
    if ( disposals[i] == DelDispose ) {
      size_t time = 0;
      while ( disposals[i] == DelDispose ) {
        time +=(size_t) (curr->delay*1000*
          PerceptibleReciprocal((double) curr->ticks_per_second));
        curr=GetNextImageInList(curr);
        i++;
      }
      time += (size_t)(curr->delay*1000*
        PerceptibleReciprocal((double) curr->ticks_per_second));
      prev_image->ticks_per_second = 100L;
      prev_image->delay = time*prev_image->ticks_per_second/1000;
    }
    bgnd_image=CropImage(prev_image,&bounds[i],sans_exception);
    prev_image=DestroyImage(prev_image);
    if (bgnd_image == (Image *) NULL)
      break;
    bgnd_image->dispose=disposals[i];
    if ( disposals[i] == DupDispose ) {
      bgnd_image->delay=0;
      bgnd_image->dispose=NoneDispose;
    }
    else
      curr=GetNextImageInList(curr);
    AppendImageToList(&optimized_image,bgnd_image);
    i++;
  }
  sans_exception=DestroyExceptionInfo(sans_exception);
  bounds=(RectangleInfo *) RelinquishMagickMemory(bounds);
  disposals=(DisposeType *) RelinquishMagickMemory(disposals);
  if (curr != (Image *) NULL)
    {
      optimized_image=DestroyImageList(optimized_image);
      return((Image *) NULL);
    }
  return(GetFirstImageInList(optimized_image));
}


static MagickBooleanType IsBoundsCleared(const Image *image1,
  const Image *image2,RectangleInfo *bounds,ExceptionInfo *exception)
{
  register const Quantum
    *p,
    *q;

  register ssize_t
    x;

  ssize_t
    y;

  if (bounds->x < 0)
    return(MagickFalse);
  for (y=0; y < (ssize_t) bounds->height; y++)
  {
    p=GetVirtualPixels(image1,bounds->x,bounds->y+y,bounds->width,1,exception);
    q=GetVirtualPixels(image2,bounds->x,bounds->y+y,bounds->width,1,exception);
    if ((p == (const Quantum *) NULL) || (q == (Quantum *) NULL))
      break;
    for (x=0; x < (ssize_t) bounds->width; x++)
    {
      if ((GetPixelAlpha(image1,p) >= (Quantum) (QuantumRange/2)) &&
          (GetPixelAlpha(image2,q) < (Quantum) (QuantumRange/2)))
        break;
      p+=GetPixelChannels(image1);
      q+=GetPixelChannels(image2);
    }
    if (x < (ssize_t) bounds->width)
      break;
  }
  return(y < (ssize_t) bounds->height ? MagickTrue : MagickFalse);
}


static MagickBooleanType ComparePixels(const LayerMethod method,
  const PixelInfo *p,const PixelInfo *q)
{
  double
    o1,
    o2;

  /*
    Any change in pixel values
  */
  if (method == CompareAnyLayer)
    return(IsFuzzyEquivalencePixelInfo(p,q) == MagickFalse ? MagickTrue : MagickFalse);
  o1 = (p->alpha_trait != UndefinedPixelTrait) ? p->alpha : OpaqueAlpha;
  o2 = (q->alpha_trait != UndefinedPixelTrait) ? q->alpha : OpaqueAlpha;
  /*
    Pixel goes from opaque to transprency.
  */
  if (method == CompareClearLayer)
    return((MagickBooleanType) ( (o1 >= ((double) QuantumRange/2.0)) &&
      (o2 < ((double) QuantumRange/2.0)) ) );
  /*
    Overlay would change first pixel by second.
  */
  if (method == CompareOverlayLayer)
    {
      if (o2 < ((double) QuantumRange/2.0))
        return MagickFalse;
      return(IsFuzzyEquivalencePixelInfo(p,q) == MagickFalse ? MagickTrue :
        MagickFalse);
    }
  return(MagickFalse);
}