sn_coap_hdr_s *sn_coap_parser_alloc_message_with_options(struct coap_s *handle)
{
    // check the handle just as in any other place
    if (handle == NULL) {
        return NULL;
    }

    sn_coap_hdr_s *coap_msg_ptr = sn_coap_parser_alloc_message(handle);

    sn_coap_options_list_s *options_list_ptr = sn_coap_parser_alloc_options(handle, coap_msg_ptr);

    if ((coap_msg_ptr == NULL) || (options_list_ptr == NULL)) {

        // oops, out of memory free if got already any
        handle->sn_coap_protocol_free(coap_msg_ptr);
        handle->sn_coap_protocol_free(options_list_ptr);

        coap_msg_ptr = NULL;
    }

    return coap_msg_ptr;
}


sn_coap_hdr_s *sn_coap_parser_alloc_message(struct coap_s *handle)
{
    sn_coap_hdr_s *returned_coap_msg_ptr;

    /* * * * Check given pointer * * * */
    if (handle == NULL) {
        return NULL;
    }

    /* * * * Allocate memory for returned CoAP message and initialize allocated memory with with default values  * * * */
    returned_coap_msg_ptr = handle->sn_coap_protocol_malloc(sizeof(sn_coap_hdr_s));

    return sn_coap_parser_init_message(returned_coap_msg_ptr);
}


static int8_t sn_coap_parser_payload_parse(uint16_t packet_data_len, uint8_t *packet_data_start_ptr, uint8_t **packet_data_pptr, sn_coap_hdr_s *dst_coap_msg_ptr)
{
    /* If there is payload */
    if ((*packet_data_pptr - packet_data_start_ptr) < packet_data_len) {
        if (**packet_data_pptr == 0xff) {
            (*packet_data_pptr)++;
            /* Parse Payload length */
            dst_coap_msg_ptr->payload_len = packet_data_len - (*packet_data_pptr - packet_data_start_ptr);

            /* The presence of a marker followed by a zero-length payload MUST be processed as a message format error */
            if (dst_coap_msg_ptr->payload_len == 0) {
                return -1;
            }

            /* Parse Payload by setting CoAP message's payload_ptr to point Payload in Packet data */
            dst_coap_msg_ptr->payload_ptr = *packet_data_pptr;
        }
        /* No payload marker.. */
        else {
            tr_error("sn_coap_parser_payload_parse - payload marker not found!");
            return -1;
        }
    }
    return 0;
}


static uint32_t sn_coap_parser_options_parse_uint(uint8_t **packet_data_pptr, uint8_t option_len)
{
    uint32_t value = 0;
    while (option_len--) {
        value <<= 8;
        value |= *(*packet_data_pptr)++;
    }
    return value;
}


static void sn_coap_parser_header_parse(uint8_t **packet_data_pptr, sn_coap_hdr_s *dst_coap_msg_ptr, coap_version_e *coap_version_ptr)
{
    /* Parse CoAP Version and message type*/
    *coap_version_ptr = (coap_version_e)(**packet_data_pptr & COAP_HEADER_VERSION_MASK);
    dst_coap_msg_ptr->msg_type = (sn_coap_msg_type_e)(**packet_data_pptr & COAP_HEADER_MSG_TYPE_MASK);
    (*packet_data_pptr) += 1;

    /* Parse Message code */
    dst_coap_msg_ptr->msg_code = (sn_coap_msg_code_e) **packet_data_pptr;
    (*packet_data_pptr) += 1;

    /* Parse Message ID */
    dst_coap_msg_ptr->msg_id = *(*packet_data_pptr + 1);
    dst_coap_msg_ptr->msg_id += **packet_data_pptr << COAP_HEADER_MSG_ID_MSB_SHIFT;
    (*packet_data_pptr) += 2;

}


static int8_t sn_coap_parser_options_parse(struct coap_s *handle, uint8_t **packet_data_pptr, sn_coap_hdr_s *dst_coap_msg_ptr, uint8_t *packet_data_start_ptr, uint16_t packet_len)
{
    uint8_t previous_option_number = 0;
    int8_t  ret_status             = 0;
    uint16_t message_left          = sn_coap_parser_move_packet_ptr(packet_data_pptr,
                                                                    packet_data_start_ptr,
                                                                    packet_len,
                                                                    0);

    /*  Parse token, if exists  */
    dst_coap_msg_ptr->token_len = *packet_data_start_ptr & COAP_HEADER_TOKEN_LENGTH_MASK;

    if (dst_coap_msg_ptr->token_len) {
        int8_t ptr_check_result;
        if ((dst_coap_msg_ptr->token_len > 8) || dst_coap_msg_ptr->token_ptr) {
            tr_error("sn_coap_parser_options_parse - token not valid!");
            return -1;
        }

        ptr_check_result = sn_coap_parser_check_packet_ptr(*packet_data_pptr, packet_data_start_ptr, packet_len, dst_coap_msg_ptr->token_len);
        if (0 != ptr_check_result) {
            tr_error("sn_coap_parser_options_parse - **packet_data_pptr overflow !");
            return -1;
        }

        dst_coap_msg_ptr->token_ptr = sn_coap_protocol_malloc_copy(handle, *packet_data_pptr, dst_coap_msg_ptr->token_len);

        if (dst_coap_msg_ptr->token_ptr == NULL) {
            tr_error("sn_coap_parser_options_parse - failed to allocate token!");
            return -1;
        }

        message_left = sn_coap_parser_move_packet_ptr(packet_data_pptr,
                                                      packet_data_start_ptr,
                                                      packet_len,
                                                      dst_coap_msg_ptr->token_len);
    }

    /* Loop all Options */
    while (message_left && (**packet_data_pptr != 0xff)) {
        /* Get option length WITHOUT extensions */
        uint16_t option_len = (**packet_data_pptr & 0x0F);
        /* Get option number WITHOUT extensions */
        uint16_t  option_number = (**packet_data_pptr >> COAP_OPTIONS_OPTION_NUMBER_SHIFT);

        message_left = sn_coap_parser_move_packet_ptr(packet_data_pptr, packet_data_start_ptr, packet_len, 1);

        int8_t    option_parse_result;
        /* Add possible option delta extension */
        option_parse_result = parse_ext_option(&option_number,
                                                packet_data_pptr,
                                                packet_data_start_ptr,
                                                packet_len,
                                                &message_left);
        if (option_parse_result != 0) {
            return -1;
        }
        /* Add previous option to option delta and get option number */
        if(sn_coap_parser_add_u16_limit(option_number, previous_option_number, &option_number) != 0)
        {
            return -1;
        }

        /* Add possible option length extension to resolve full length of the option */
        option_parse_result = parse_ext_option(&option_len,
                                                packet_data_pptr,
                                                packet_data_start_ptr,
                                                packet_len,
                                                &message_left);
        if (option_parse_result != 0) {
            return -1;
        }

        /* * * Parse option itself * * */
        /* Some options are handled independently in own functions */
        previous_option_number = option_number;
        /* Allocate options_list_ptr if needed */
        switch (option_number) {
            case COAP_OPTION_MAX_AGE:
            case COAP_OPTION_PROXY_URI:
            case COAP_OPTION_ETAG:
            case COAP_OPTION_URI_HOST:
            case COAP_OPTION_LOCATION_PATH:
            case COAP_OPTION_URI_PORT:
            case COAP_OPTION_LOCATION_QUERY:
            case COAP_OPTION_OBSERVE:
            case COAP_OPTION_URI_QUERY:
            case COAP_OPTION_BLOCK2:
            case COAP_OPTION_BLOCK1:
            case COAP_OPTION_ACCEPT:
            case COAP_OPTION_SIZE1:
            case COAP_OPTION_SIZE2:
                if (sn_coap_parser_alloc_options(handle, dst_coap_msg_ptr) == NULL) {
                    tr_error("sn_coap_parser_options_parse - failed to allocate options!");
                    return -1;
                }
                break;
        }

        if (message_left < option_len){
            /* packet_data_pptr would overflow! */
            tr_error("sn_coap_parser_options_parse - **packet_data_pptr would overflow when parsing options!");
            return -1;
        }

        /* Parse option */
        switch (option_number) {
            case COAP_OPTION_CONTENT_FORMAT:
                if ((option_len > 2) || (dst_coap_msg_ptr->content_format != COAP_CT_NONE)) {
                    tr_error("sn_coap_parser_options_parse - COAP_OPTION_CONTENT_FORMAT not valid!");
                    return -1;
                }
                dst_coap_msg_ptr->content_format = (sn_coap_content_format_e) sn_coap_parser_options_parse_uint(packet_data_pptr, option_len);
                break;

            case COAP_OPTION_MAX_AGE:
                if (option_len > 4) {
                    tr_error("sn_coap_parser_options_parse - COAP_OPTION_MAX_AGE not valid!");
                    return -1;
                }
                dst_coap_msg_ptr->options_list_ptr->max_age = sn_coap_parser_options_parse_uint(packet_data_pptr, option_len);
                break;

            case COAP_OPTION_PROXY_URI:
                if ((option_len > 1034) || (option_len < 1) || dst_coap_msg_ptr->options_list_ptr->proxy_uri_ptr) {
                    tr_error("sn_coap_parser_options_parse - COAP_OPTION_PROXY_URI not valid!");
                    return -1;
                }
                dst_coap_msg_ptr->options_list_ptr->proxy_uri_len = option_len;
                dst_coap_msg_ptr->options_list_ptr->proxy_uri_ptr = sn_coap_protocol_malloc_copy(handle, *packet_data_pptr, option_len);

                if (dst_coap_msg_ptr->options_list_ptr->proxy_uri_ptr == NULL) {
                    tr_error("sn_coap_parser_options_parse - COAP_OPTION_PROXY_URI allocation failed!");
                    return -1;
                }
                message_left = sn_coap_parser_move_packet_ptr(packet_data_pptr, packet_data_start_ptr, packet_len, option_len);
                break;

            case COAP_OPTION_ETAG:
                if (dst_coap_msg_ptr->options_list_ptr->etag_ptr)
                {
                    tr_error("sn_coap_parser_options_parse - COAP_OPTION_ETAG exists!");
                    return -1;
                }
                /* This is managed independently because User gives this option in one character table */
                ret_status = sn_coap_parser_options_parse_multiple_options(handle, packet_data_pptr,
                             message_left,
                             &dst_coap_msg_ptr->options_list_ptr->etag_ptr,
                             (uint16_t *)&dst_coap_msg_ptr->options_list_ptr->etag_len,
                             COAP_OPTION_ETAG, option_len);
                if (ret_status < 0) {
                    tr_error("sn_coap_parser_options_parse - COAP_OPTION_ETAG not valid!");
                    return -1;
                }
                break;

            case COAP_OPTION_URI_HOST:
                if ((option_len > 255) || (option_len < 1) || dst_coap_msg_ptr->options_list_ptr->uri_host_ptr) {
                    tr_error("sn_coap_parser_options_parse - COAP_OPTION_URI_HOST not valid!");
                    return -1;
                }
                dst_coap_msg_ptr->options_list_ptr->uri_host_len = option_len;
                dst_coap_msg_ptr->options_list_ptr->uri_host_ptr = sn_coap_protocol_malloc_copy(handle, *packet_data_pptr, option_len);

                if (dst_coap_msg_ptr->options_list_ptr->uri_host_ptr == NULL) {
                    tr_error("sn_coap_parser_options_parse - COAP_OPTION_URI_HOST allocation failed!");
                    return -1;
                }
                message_left = sn_coap_parser_move_packet_ptr(packet_data_pptr, packet_data_start_ptr, packet_len, option_len);
                break;

            case COAP_OPTION_LOCATION_PATH:
                if (dst_coap_msg_ptr->options_list_ptr->location_path_ptr) {
                    tr_error("sn_coap_parser_options_parse - COAP_OPTION_LOCATION_PATH exists!");
                    return -1;
                }
                /* This is managed independently because User gives this option in one character table */
                ret_status = sn_coap_parser_options_parse_multiple_options(handle, packet_data_pptr, message_left,
                             &dst_coap_msg_ptr->options_list_ptr->location_path_ptr, &dst_coap_msg_ptr->options_list_ptr->location_path_len,
                             COAP_OPTION_LOCATION_PATH, option_len);
                if (ret_status <0) {
                    tr_error("sn_coap_parser_options_parse - COAP_OPTION_LOCATION_PATH not valid!");
                    return -1;
                }
                break;

            case COAP_OPTION_URI_PORT:
                if ((option_len > 2) || dst_coap_msg_ptr->options_list_ptr->uri_port != COAP_OPTION_URI_PORT_NONE) {
                    tr_error("sn_coap_parser_options_parse - COAP_OPTION_URI_PORT not valid!");
                    return -1;
                }
                dst_coap_msg_ptr->options_list_ptr->uri_port = sn_coap_parser_options_parse_uint(packet_data_pptr, option_len);
                break;

            case COAP_OPTION_LOCATION_QUERY:
                if (dst_coap_msg_ptr->options_list_ptr->location_query_ptr)
                {
                    tr_error("sn_coap_parser_options_parse - COAP_OPTION_LOCATION_QUERY exists!");
                    return -1;
                }
                ret_status = sn_coap_parser_options_parse_multiple_options(handle, packet_data_pptr, message_left,
                             &dst_coap_msg_ptr->options_list_ptr->location_query_ptr, &dst_coap_msg_ptr->options_list_ptr->location_query_len,
                             COAP_OPTION_LOCATION_QUERY, option_len);
                if (ret_status < 0) {
                    tr_error("sn_coap_parser_options_parse - COAP_OPTION_LOCATION_QUERY not valid!");
                    return -1;
                }

                break;

            case COAP_OPTION_URI_PATH:
                if (dst_coap_msg_ptr->uri_path_ptr)
                {
                    tr_error("sn_coap_parser_options_parse - COAP_OPTION_URI_PATH exists!");
                    return -1;
                }
                ret_status = sn_coap_parser_options_parse_multiple_options(handle, packet_data_pptr, message_left,
                             &dst_coap_msg_ptr->uri_path_ptr, &dst_coap_msg_ptr->uri_path_len,
                             COAP_OPTION_URI_PATH, option_len);
                if (ret_status < 0) {
                    tr_error("sn_coap_parser_options_parse - COAP_OPTION_URI_PATH not valid!");
                    return -1;
                }
                break;

            case COAP_OPTION_OBSERVE:
                if ((option_len > 2) || dst_coap_msg_ptr->options_list_ptr->observe != COAP_OBSERVE_NONE) {
                    tr_error("sn_coap_parser_options_parse - COAP_OPTION_OBSERVE not valid!");
                    return -1;
                }
                dst_coap_msg_ptr->options_list_ptr->observe = sn_coap_parser_options_parse_uint(packet_data_pptr, option_len);
                break;

            case COAP_OPTION_URI_QUERY:
                ret_status = sn_coap_parser_options_parse_multiple_options(handle, packet_data_pptr, message_left,
                             &dst_coap_msg_ptr->options_list_ptr->uri_query_ptr, &dst_coap_msg_ptr->options_list_ptr->uri_query_len,
                             COAP_OPTION_URI_QUERY, option_len);
                if (ret_status < 0) {
                    tr_error("sn_coap_parser_options_parse - COAP_OPTION_URI_QUERY not valid!");
                    return -1;
                }
                break;

            case COAP_OPTION_BLOCK2:
                if ((option_len > 3) || dst_coap_msg_ptr->options_list_ptr->block2 != COAP_OPTION_BLOCK_NONE) {
                    tr_error("sn_coap_parser_options_parse - COAP_OPTION_BLOCK2 not valid!");
                    return -1;
                }
                dst_coap_msg_ptr->options_list_ptr->block2 = sn_coap_parser_options_parse_uint(packet_data_pptr, option_len);
                break;

            case COAP_OPTION_BLOCK1:
                if ((option_len > 3) || dst_coap_msg_ptr->options_list_ptr->block1 != COAP_OPTION_BLOCK_NONE) {
                    tr_error("sn_coap_parser_options_parse - COAP_OPTION_BLOCK1 not valid!");
                    return -1;
                }
                dst_coap_msg_ptr->options_list_ptr->block1 = sn_coap_parser_options_parse_uint(packet_data_pptr, option_len);
                break;

            case COAP_OPTION_ACCEPT:
                if ((option_len > 2) || (dst_coap_msg_ptr->options_list_ptr->accept != COAP_CT_NONE)) {
                    tr_error("sn_coap_parser_options_parse - COAP_OPTION_ACCEPT not valid!");
                    return -1;
                }
                dst_coap_msg_ptr->options_list_ptr->accept = (sn_coap_content_format_e) sn_coap_parser_options_parse_uint(packet_data_pptr, option_len);
                break;

            case COAP_OPTION_SIZE1:
                if ((option_len > 4) || dst_coap_msg_ptr->options_list_ptr->use_size1) {
                    tr_error("sn_coap_parser_options_parse - COAP_OPTION_SIZE1 not valid!");
                    return -1;
                }
                dst_coap_msg_ptr->options_list_ptr->use_size1 = true;
                dst_coap_msg_ptr->options_list_ptr->size1 = sn_coap_parser_options_parse_uint(packet_data_pptr, option_len);
                break;

            case COAP_OPTION_SIZE2:
                if ((option_len > 4) || dst_coap_msg_ptr->options_list_ptr->use_size2) {
                    tr_error("sn_coap_parser_options_parse - COAP_OPTION_SIZE2 not valid!");
                    return -1;
                }
                dst_coap_msg_ptr->options_list_ptr->use_size2 = true;
                dst_coap_msg_ptr->options_list_ptr->size2 = sn_coap_parser_options_parse_uint(packet_data_pptr, option_len);
                break;

            default:
                tr_error("sn_coap_parser_options_parse - unknown option!");
                return -1;
        }

        /* Check for overflow */
        if ((*packet_data_pptr - packet_data_start_ptr) > packet_len) {
            return -1;
        }
        message_left = sn_coap_parser_move_packet_ptr(packet_data_pptr,
                                                      packet_data_start_ptr,
                                                      packet_len,
                                                      0);
    }
    return 0;
}


static int8_t sn_coap_parser_read_packet_u8(uint8_t *dst, uint8_t *packet_data_ptr, uint8_t *packet_data_start_ptr, uint16_t packet_len)
{
    int8_t ptr_check_result;

    ptr_check_result = sn_coap_parser_check_packet_ptr(packet_data_ptr, packet_data_start_ptr, packet_len, 1);

    if (ptr_check_result != 0) {
        return ptr_check_result;
    }

    *dst = *packet_data_ptr;

    return 0;
}


sn_coap_options_list_s *sn_coap_parser_alloc_options(struct coap_s *handle, sn_coap_hdr_s *coap_msg_ptr)
{
    sn_coap_options_list_s *options_list_ptr;

    /* * * * Check given pointers * * * */
    if (handle == NULL || coap_msg_ptr == NULL) {
        return NULL;
    }

    /* * * * If the message already has options, return them * * * */
    if (coap_msg_ptr->options_list_ptr) {
        return coap_msg_ptr->options_list_ptr;
    }

    /* * * * Allocate memory for options and initialize allocated memory with with default values  * * * */
    /* XXX not technically legal to memset pointers to 0 */
    options_list_ptr = sn_coap_protocol_calloc(handle, sizeof(sn_coap_options_list_s));

    if (options_list_ptr == NULL) {
        tr_error("sn_coap_parser_alloc_options - failed to allocate options list!");
        return NULL;
    }

    coap_msg_ptr->options_list_ptr = options_list_ptr;

    options_list_ptr->uri_port = COAP_OPTION_URI_PORT_NONE;
    options_list_ptr->observe = COAP_OBSERVE_NONE;
    options_list_ptr->accept = COAP_CT_NONE;
    options_list_ptr->block2 = COAP_OPTION_BLOCK_NONE;
    options_list_ptr->block1 = COAP_OPTION_BLOCK_NONE;

    return options_list_ptr;
}


static int8_t sn_coap_parser_read_packet_u16(uint16_t *dst, uint8_t *packet_data_ptr, uint8_t *packet_data_start_ptr, uint16_t packet_len)
{
    int8_t ptr_check_result;
    uint16_t value;

    ptr_check_result = sn_coap_parser_check_packet_ptr(packet_data_ptr, packet_data_start_ptr, packet_len, 2);

    if (ptr_check_result != 0) {
        return ptr_check_result;
    }

    value = *(packet_data_ptr) << 8;
    value |= *(packet_data_ptr + 1);
    *dst = value;

    return 0;
}


static int8_t sn_coap_parser_add_u16_limit(uint16_t a, uint16_t b, uint16_t *result)
{
    uint16_t c;

    c = a + b;
    if (c < a || c < b)
    {
        return -1;
    }

    *result = c;

    return 0;
}


static int8_t parse_ext_option(uint16_t *dst, uint8_t **packet_data_pptr, uint8_t *packet_data_start_ptr, uint16_t packet_len, uint16_t *message_left)
{
    uint16_t option_number = *dst;

    if (option_number == 13) {
        uint8_t option_ext;
        int8_t read_result = sn_coap_parser_read_packet_u8(&option_ext,
                                                            *packet_data_pptr,
                                                            packet_data_start_ptr,
                                                            packet_len);
        if (read_result != 0) {
            /* packet_data_pptr would overflow! */
            tr_error("sn_coap_parser_options_parse - **packet_data_pptr overflow !");
            return -1;
        }
        else {
            if(sn_coap_parser_add_u16_limit(option_number, option_ext, &option_number) != 0)
            {
                return -1;
            }

            *message_left = sn_coap_parser_move_packet_ptr(packet_data_pptr,
                                                            packet_data_start_ptr,
                                                            packet_len,
                                                            1);
        }
    } else if (option_number == 14) {
            int8_t read_result = sn_coap_parser_read_packet_u16(&option_number,
                                                                *packet_data_pptr,
                                                                packet_data_start_ptr,
                                                                packet_len);
            if (read_result != 0) {
                /* packet_data_pptr would overflow! */
                tr_error("sn_coap_parser_options_parse - **packet_data_pptr overflow !");
                return -1;
            }
            else {
                if(sn_coap_parser_add_u16_limit(option_number, 269, &option_number) != 0)
                {
                    return -1;
                }

                *message_left = sn_coap_parser_move_packet_ptr(packet_data_pptr,
                                                            packet_data_start_ptr,
                                                            packet_len,
                                                            2);
            }
    }
    /* Option number 15 reserved for payload marker. This is handled as a error! */
    else if (option_number == 15) {
        tr_error("sn_coap_parser_options_parse - invalid option number(15)!");
        return -1;
    }

    *dst = option_number;
    return 0;
}


sn_coap_hdr_s *sn_coap_parser(struct coap_s *handle, uint16_t packet_data_len, uint8_t *packet_data_ptr, coap_version_e *coap_version_ptr)
{
    uint8_t       *data_temp_ptr                    = packet_data_ptr;
    sn_coap_hdr_s *parsed_and_returned_coap_msg_ptr = NULL;

    /* * * * Check given pointer * * * */
    if (packet_data_ptr == NULL || packet_data_len < 4 || handle == NULL) {
        return NULL;
    }

    /* * * * Allocate and initialize CoAP message  * * * */
    parsed_and_returned_coap_msg_ptr = sn_coap_parser_alloc_message(handle);

    if (parsed_and_returned_coap_msg_ptr == NULL) {
        tr_error("sn_coap_parser - failed to allocate message!");
        return NULL;
    }

    /* * * * Header parsing, move pointer over the header...  * * * */
    sn_coap_parser_header_parse(&data_temp_ptr, parsed_and_returned_coap_msg_ptr, coap_version_ptr);
    /* * * * Options parsing, move pointer over the options... * * * */
    if (sn_coap_parser_options_parse(handle, &data_temp_ptr, parsed_and_returned_coap_msg_ptr, packet_data_ptr, packet_data_len) != 0) {
        parsed_and_returned_coap_msg_ptr->coap_status = COAP_STATUS_PARSER_ERROR_IN_HEADER;
        return parsed_and_returned_coap_msg_ptr;
    }

    /* * * * Payload parsing * * * */
    if (sn_coap_parser_payload_parse(packet_data_len, packet_data_ptr, &data_temp_ptr, parsed_and_returned_coap_msg_ptr) == -1) {
        parsed_and_returned_coap_msg_ptr->coap_status = COAP_STATUS_PARSER_ERROR_IN_HEADER;
        return parsed_and_returned_coap_msg_ptr;
    }

    parsed_and_returned_coap_msg_ptr->coap_status = COAP_STATUS_OK;

    /* * * * Return parsed CoAP message  * * * * */
    return parsed_and_returned_coap_msg_ptr;
}


static uint16_t sn_coap_parser_move_packet_ptr(uint8_t **packet_data_pptr, uint8_t *packet_data_start_ptr, uint16_t packet_len, uint16_t delta)
{
    uint8_t *packet_end = packet_data_start_ptr + packet_len;
    uint8_t *new_data_ptr = *packet_data_pptr + delta;

    if (new_data_ptr < packet_data_start_ptr){
        return 0;
    } else if (new_data_ptr >= packet_end) {
        *packet_data_pptr = packet_end;
        return 0;
    }

    *packet_data_pptr = new_data_ptr;

    return (uint16_t)(packet_end - new_data_ptr);
}


int PasswdMgr::updatePasswdSpecialFile(const std::string& userName,
                                       const std::string& newUserName)
{
    phosphor::user::shadow::Lock lock();

    size_t bytesWritten = 0;
    size_t inBytesLen = 0;
    size_t isUsrFound = false;
    const EVP_CIPHER* cipher = EVP_aes_128_cbc();
    std::vector<uint8_t> dataBuf;

    // Read the encrypted file and get the file data
    // Check user existance and return if not exist.
    if (readPasswdFileData(dataBuf) != 0)
    {
        log<level::DEBUG>("Error in reading the encrypted pass file");
        return -EIO;
    }

    if (dataBuf.size() != 0)
    {
        inBytesLen =
            dataBuf.size() + newUserName.size() + EVP_CIPHER_block_size(cipher);
    }

    std::vector<uint8_t> inBytes(inBytesLen);
    if (inBytesLen != 0)
    {
        char* outPtr = reinterpret_cast<char*>(dataBuf.data());
        char* nToken = NULL;
        char* linePtr = strtok_r(outPtr, "\n", &nToken);
        while (linePtr != NULL)
        {
            size_t userEPos = 0;

            std::string lineStr(linePtr);
            if ((userEPos = lineStr.find(":")) != std::string::npos)
            {
                if (userName.compare(lineStr.substr(0, userEPos)) == 0)
                {
                    isUsrFound = true;
                    if (!newUserName.empty())
                    {
                        bytesWritten += std::snprintf(
                            reinterpret_cast<char*>(&inBytes[0]) + bytesWritten,
                            (inBytesLen - bytesWritten), "%s%s\n",
                            newUserName.c_str(),
                            lineStr.substr(userEPos, lineStr.size()).data());
                    }
                }
                else
                {
                    bytesWritten += std::snprintf(
                        reinterpret_cast<char*>(&inBytes[0]) + bytesWritten,
                        (inBytesLen - bytesWritten), "%s\n", lineStr.data());
                }
            }
            linePtr = strtok_r(NULL, "\n", &nToken);
        }
        inBytesLen = bytesWritten;
    }
    if (!isUsrFound)
    {
        log<level::DEBUG>("User doesn't exist");
        return 0;
    }

    // Read the key buff from key file
    std::array<uint8_t, maxKeySize> keyBuff;
    std::ifstream keyFile(encryptKeyFileName, std::ios::in | std::ios::binary);
    if (!keyFile.good())
    {
        log<level::DEBUG>("Error in opening encryption key file");
        return -EIO;
    }
    keyFile.read(reinterpret_cast<char*>(keyBuff.data()), keyBuff.size());
    if (keyFile.fail())
    {
        log<level::DEBUG>("Error in reading encryption key file");
        return -EIO;
    }
    keyFile.close();

    // Read the original passwd file mode
    struct stat st = {};
    if (stat(passwdFileName, &st) != 0)
    {
        log<level::DEBUG>("Error in getting password file fstat()");
        return -EIO;
    }

    // Create temporary file for write
    std::string pwdFile(passwdFileName);
    std::vector<char> tempFileName(pwdFile.begin(), pwdFile.end());
    std::vector<char> fileTemplate = {'_', '_', 'X', 'X', 'X',
                                      'X', 'X', 'X', '\0'};
    tempFileName.insert(tempFileName.end(), fileTemplate.begin(),
                        fileTemplate.end());
    int fd = mkstemp((char*)tempFileName.data());
    if (fd == -1)
    {
        log<level::DEBUG>("Error creating temp file");
        return -EIO;
    }

    std::string strTempFileName(tempFileName.data());
    // Open the temp file for writing from provided fd
    // By "true", remove it at exit if still there.
    // This is needed to cleanup the temp file at exception
    phosphor::user::File temp(fd, strTempFileName, "w", true);
    if ((temp)() == NULL)
    {
        close(fd);
        log<level::DEBUG>("Error creating temp file");
        return -EIO;
    }

    // Set the file mode as read-write for owner only
    if (fchmod(fileno((temp)()), S_IRUSR | S_IWUSR) < 0)
    {
        log<level::DEBUG>("Error setting fchmod for temp file");
        return -EIO;
    }

    const EVP_MD* digest = EVP_sha256();
    size_t hashLen = EVP_MD_block_size(digest);
    std::vector<uint8_t> hash(hashLen);
    size_t ivLen = EVP_CIPHER_iv_length(cipher);
    std::vector<uint8_t> iv(ivLen);
    std::array<uint8_t, EVP_MAX_KEY_LENGTH> key;
    size_t keyLen = key.size();
    std::array<uint8_t, EVP_MAX_MD_SIZE> mac;
    size_t macLen = mac.size();

    // Create random hash and generate hash key which will be used for
    // encryption.
    if (RAND_bytes(hash.data(), hashLen) != 1)
    {
        log<level::DEBUG>("Hash genertion failed, bailing out");
        return -EIO;
    }
    if (NULL == HMAC(digest, keyBuff.data(), keyBuff.size(), hash.data(),
                     hashLen, key.data(),
                     reinterpret_cast<unsigned int*>(&keyLen)))
    {
        log<level::DEBUG>("Failed to create MAC for authentication");
        return -EIO;
    }

    // Generate IV values
    if (RAND_bytes(iv.data(), ivLen) != 1)
    {
        log<level::DEBUG>("UV genertion failed, bailing out");
        return -EIO;
    }

    // Encrypt the input data
    std::vector<uint8_t> outBytes(inBytesLen + EVP_MAX_BLOCK_LENGTH);
    size_t outBytesLen = 0;
    if (inBytesLen != 0)
    {
        if (encryptDecryptData(true, EVP_aes_128_cbc(), key.data(), keyLen,
                               iv.data(), ivLen, inBytes.data(), inBytesLen,
                               mac.data(), &macLen, outBytes.data(),
                               &outBytesLen) != 0)
        {
            log<level::DEBUG>("Error while encrypting the data");
            return -EIO;
        }
        outBytes[outBytesLen] = 0;
    }
    OPENSSL_cleanse(key.data(), keyLen);

    // Update the meta password structure.
    MetaPassStruct metaData = {META_PASSWD_SIG, {0, 0}, 0, 0, 0, 0, 0};
    metaData.hashSize = hashLen;
    metaData.ivSize = ivLen;
    metaData.dataSize = bytesWritten;
    metaData.padSize = outBytesLen - bytesWritten;
    metaData.macSize = macLen;

    if (fwrite(&metaData, 1, sizeof(metaData), (temp)()) != sizeof(metaData))
    {
        log<level::DEBUG>("Error in writing meta data");
        return -EIO;
    }

    if (fwrite(&hash[0], 1, hashLen, (temp)()) != hashLen)
    {
        log<level::DEBUG>("Error in writing hash data");
        return -EIO;
    }

    if (fwrite(&iv[0], 1, ivLen, (temp)()) != ivLen)
    {
        log<level::DEBUG>("Error in writing IV data");
        return -EIO;
    }

    if (fwrite(&outBytes[0], 1, outBytesLen, (temp)()) != outBytesLen)
    {
        log<level::DEBUG>("Error in writing encrypted data");
        return -EIO;
    }

    if (fwrite(&mac[0], 1, macLen, (temp)()) != macLen)
    {
        log<level::DEBUG>("Error in writing MAC data");
        return -EIO;
    }

    if (fflush((temp)()))
    {
        log<level::DEBUG>(
            "File fflush error while writing entries to special file");
        return -EIO;
    }

    OPENSSL_cleanse(iv.data(), ivLen);

    // Rename the tmp  file to actual file
    if (std::rename(strTempFileName.data(), passwdFileName) != 0)
    {
        log<level::DEBUG>("Failed to rename tmp file to ipmi-pass");
        return -EIO;
    }

    return 0;
}


int PasswdMgr::encryptDecryptData(bool doEncrypt, const EVP_CIPHER* cipher,
                                  uint8_t* key, size_t keyLen, uint8_t* iv,
                                  size_t ivLen, uint8_t* inBytes,
                                  size_t inBytesLen, uint8_t* mac,
                                  size_t* macLen, unsigned char* outBytes,
                                  size_t* outBytesLen)
{
    if (cipher == NULL || key == NULL || iv == NULL || inBytes == NULL ||
        outBytes == NULL || mac == NULL || inBytesLen == 0 ||
        (size_t)EVP_CIPHER_key_length(cipher) > keyLen ||
        (size_t)EVP_CIPHER_iv_length(cipher) > ivLen)
    {
        log<level::DEBUG>("Error Invalid Inputs");
        return -EINVAL;
    }

    if (!doEncrypt)
    {
        // verify MAC before decrypting the data.
        std::array<uint8_t, EVP_MAX_MD_SIZE> calMac;
        size_t calMacLen = calMac.size();
        // calculate MAC for the encrypted message.
        if (NULL == HMAC(EVP_sha256(), key, keyLen, inBytes, inBytesLen,
                         calMac.data(),
                         reinterpret_cast<unsigned int*>(&calMacLen)))
        {
            log<level::DEBUG>("Error: Failed to calculate MAC");
            return -EIO;
        }
        if (!((calMacLen == *macLen) &&
              (std::memcmp(calMac.data(), mac, calMacLen) == 0)))
        {
            log<level::DEBUG>("Authenticated message doesn't match");
            return -EBADMSG;
        }
    }

    std::unique_ptr<EVP_CIPHER_CTX, decltype(&::EVP_CIPHER_CTX_free)> ctx(
        EVP_CIPHER_CTX_new(), ::EVP_CIPHER_CTX_free);
    EVP_CIPHER_CTX_set_padding(ctx.get(), 1);

    // Set key & IV
    int retval = EVP_CipherInit_ex(ctx.get(), cipher, NULL, key, iv,
                                   static_cast<int>(doEncrypt));
    if (!retval)
    {
        log<level::DEBUG>("EVP_CipherInit_ex failed",
                          entry("RET_VAL=%d", retval));
        return -EIO;
    }

    int outLen = 0, outEVPLen = 0;
    if ((retval = EVP_CipherUpdate(ctx.get(), outBytes + outLen, &outEVPLen,
                                   inBytes, inBytesLen)))
    {
        outLen += outEVPLen;
        if ((retval =
                 EVP_CipherFinal(ctx.get(), outBytes + outLen, &outEVPLen)))
        {
            outLen += outEVPLen;
            *outBytesLen = outLen;
        }
        else
        {
            log<level::DEBUG>("EVP_CipherFinal fails",
                              entry("RET_VAL=%d", retval));
            return -EIO;
        }
    }
    else
    {
        log<level::DEBUG>("EVP_CipherUpdate fails",
                          entry("RET_VAL=%d", retval));
        return -EIO;
    }

    if (doEncrypt)
    {
        // Create MAC for the encrypted message
        if (NULL == HMAC(EVP_sha256(), key, keyLen, outBytes, *outBytesLen, mac,
                         reinterpret_cast<unsigned int*>(macLen)))
        {
            log<level::DEBUG>("Failed to create authentication");
            return -EIO;
        }
    }
    return 0;
}


std::time_t PasswdMgr::getUpdatedFileTime()
{
    struct stat fileStat = {};
    if (stat(passwdFileName, &fileStat) != 0)
    {
        log<level::DEBUG>("Error - Getting passwd file time stamp");
        return -EIO;
    }
    return fileStat.st_mtime;
}


int PasswdMgr::readPasswdFileData(std::vector<uint8_t>& outBytes)
{
    std::array<uint8_t, maxKeySize> keyBuff;
    std::ifstream keyFile(encryptKeyFileName, std::ios::in | std::ios::binary);
    if (!keyFile.is_open())
    {
        log<level::DEBUG>("Error in opening encryption key file");
        return -EIO;
    }
    keyFile.read(reinterpret_cast<char*>(keyBuff.data()), keyBuff.size());
    if (keyFile.fail())
    {
        log<level::DEBUG>("Error in reading encryption key file");
        return -EIO;
    }

    std::ifstream passwdFile(passwdFileName, std::ios::in | std::ios::binary);
    if (!passwdFile.is_open())
    {
        log<level::DEBUG>("Error in opening ipmi password file");
        return -EIO;
    }

    // calculate file size and read the data
    passwdFile.seekg(0, std::ios::end);
    ssize_t fileSize = passwdFile.tellg();
    passwdFile.seekg(0, std::ios::beg);
    std::vector<uint8_t> input(fileSize);
    passwdFile.read(reinterpret_cast<char*>(input.data()), fileSize);
    if (passwdFile.fail())
    {
        log<level::DEBUG>("Error in reading encryption key file");
        return -EIO;
    }

    // verify the signature first
    MetaPassStruct* metaData = reinterpret_cast<MetaPassStruct*>(input.data());
    if (std::strncmp(metaData->signature, META_PASSWD_SIG,
                     sizeof(metaData->signature)))
    {
        log<level::DEBUG>("Error signature mismatch in password file");
        return -EBADMSG;
    }

    size_t inBytesLen = metaData->dataSize + metaData->padSize;
    // If data is empty i.e no password map then return success
    if (inBytesLen == 0)
    {
        log<level::DEBUG>("Empty password file");
        return 0;
    }

    // compute the key needed to decrypt
    std::array<uint8_t, EVP_MAX_KEY_LENGTH> key;
    auto keyLen = key.size();
    if (NULL == HMAC(EVP_sha256(), keyBuff.data(), keyBuff.size(),
                     input.data() + sizeof(*metaData), metaData->hashSize,
                     key.data(), reinterpret_cast<unsigned int*>(&keyLen)))
    {
        log<level::DEBUG>("Failed to create MAC for authentication");
        return -EIO;
    }

    // decrypt the data
    uint8_t* iv = input.data() + sizeof(*metaData) + metaData->hashSize;
    size_t ivLen = metaData->ivSize;
    uint8_t* inBytes = iv + ivLen;
    uint8_t* mac = inBytes + inBytesLen;
    size_t macLen = metaData->macSize;

    size_t outBytesLen = 0;
    // Resize to actual data size
    outBytes.resize(inBytesLen + EVP_MAX_BLOCK_LENGTH);
    if (encryptDecryptData(false, EVP_aes_128_cbc(), key.data(), keyLen, iv,
                           ivLen, inBytes, inBytesLen, mac, &macLen,
                           outBytes.data(), &outBytesLen) != 0)
    {
        log<level::DEBUG>("Error in decryption");
        return -EIO;
    }
    // Resize the vector to outBytesLen
    outBytes.resize(outBytesLen);

    OPENSSL_cleanse(key.data(), keyLen);
    OPENSSL_cleanse(iv, ivLen);

    return 0;
}


int PasswdMgr::updateUserEntry(const std::string& userName,
                               const std::string& newUserName)
{
    std::time_t updatedTime = getUpdatedFileTime();
    // Check file time stamp to know passwdMapList is up-to-date.
    // If not up-to-date, then updatePasswdSpecialFile will read and
    // check the user entry existance.
    if (fileLastUpdatedTime == updatedTime && updatedTime != -EIO)
    {
        if (passwdMapList.find(userName) == passwdMapList.end())
        {
            log<level::DEBUG>("User not found");
            return 0;
        }
    }

    // Write passwdMap to Encryted file
    if (updatePasswdSpecialFile(userName, newUserName) != 0)
    {
        log<level::DEBUG>("Passwd file update failed");
        return -EIO;
    }

    log<level::DEBUG>("Passwd file updated successfully");
    return 0;
}


std::string PasswdMgr::getPasswdByUserName(const std::string& userName)
{
    checkAndReload();
    auto iter = passwdMapList.find(userName);
    if (iter == passwdMapList.end())
    {
        return std::string();
    }
    return iter->second;
}


void PasswdMgr::checkAndReload(void)
{
    std::time_t updatedTime = getUpdatedFileTime();
    if (fileLastUpdatedTime != updatedTime && updatedTime != -1)
    {
        log<level::DEBUG>("Reloading password map list");
        passwdMapList.clear();
        initPasswordMap();
    }
}


PasswdMgr::PasswdMgr()
{
    initPasswordMap();
}


void PasswdMgr::initPasswordMap(void)
{
    phosphor::user::shadow::Lock lock();
    std::vector<uint8_t> dataBuf;

    if (readPasswdFileData(dataBuf) != 0)
    {
        log<level::DEBUG>("Error in reading the encrypted pass file");
        return;
    }

    if (dataBuf.size() != 0)
    {
        // populate the user list with password
        char* outPtr = reinterpret_cast<char*>(dataBuf.data());
        char* nToken = NULL;
        char* linePtr = strtok_r(outPtr, "\n", &nToken);
        size_t lineSize = 0;
        while (linePtr != NULL)
        {
            size_t userEPos = 0;
            std::string lineStr(linePtr);
            if ((userEPos = lineStr.find(":")) != std::string::npos)
            {
                lineSize = lineStr.size();
                passwdMapList.emplace(
                    lineStr.substr(0, userEPos),
                    lineStr.substr(userEPos + 1, lineSize - (userEPos + 1)));
            }
            linePtr = strtok_r(NULL, "\n", &nToken);
        }
    }

    // Update the timestamp
    fileLastUpdatedTime = getUpdatedFileTime();
    return;
}


static PyObject *encode_int64(PyObject *self, PyObject *value)
{
    long long x;
    char bs[8], sign = 0x00;
    int i;

    if (!PyArg_Parse(value, "L", &x))
        return NULL;

    if (x < 0) {
        x = -x;
        sign = 0x80;
    }
    for (i = 0; i < 8; i++) {
        bs[i] = x & 0xff;
        x >>= 8;  /* x /= 256 */
    }
    bs[7] |= sign;
    return PyBytes_FromStringAndSize(bs, 8);
}


PyInit_core(void)
{
    PyObject *m;

    m = PyModule_Create(&moduledef);
    if (m == NULL)
        return NULL;
    return m;
}


static off_t matchlen(unsigned char *old, off_t oldsize,
                      unsigned char *new, off_t newsize)
{
    off_t i;

    for (i = 0; (i < oldsize) && (i < newsize); i++)
        if (old[i] != new[i])
            break;
    return i;
}


initcore(void)
{
    Py_InitModule("core", module_functions);
}


static void qsufsort(off_t *I, off_t *V, unsigned char *old, off_t oldsize)
{
    off_t buckets[256], i, h, len;

    for (i = 0; i < 256; i++)
        buckets[i] = 0;
    for (i = 0; i < oldsize; i++)
        buckets[old[i]]++;
    for (i = 1; i < 256; i++)
        buckets[i] += buckets[i - 1];
    for (i = 255; i > 0; i--)
        buckets[i] = buckets[i - 1];
    buckets[0] = 0;

    for (i = 0; i < oldsize; i++)
        I[++buckets[old[i]]] = i;
    I[0] = oldsize;
    for (i = 0; i < oldsize; i++)
        V[i] = buckets[old[i]];
    V[oldsize] = 0;
    for (i = 1; i < 256; i++)
        if (buckets[i] == buckets[i - 1] + 1)
            I[buckets[i]] = -1;
    I[0] = -1;

    for (h = 1; I[0] != -(oldsize + 1); h += h) {
        len = 0;
        for (i = 0; i < oldsize + 1;) {
            if (I[i] < 0) {
                len -= I[i];
                i -= I[i];
            } else {
                if (len)
                    I[i - len] = -len;
                len = V[I[i]] + 1 - i;
                split(I, V, i, len, h);
                i += len;
                len=0;
            }
        }
        if (len)
            I[i - len] = -len;
    }

    for (i = 0; i < oldsize + 1; i++)
        I[V[i]] = i;
}


static PyObject* patch(PyObject* self, PyObject* args)
{
    char *origData, *newData, *diffBlock, *extraBlock, *diffPtr, *extraPtr;
    Py_ssize_t origDataLength, newDataLength, diffBlockLength, extraBlockLength;
    PyObject *controlTuples, *tuple, *results;
    off_t oldpos, newpos, x, y, z;
    int i, j, numTuples;

    if (!PyArg_ParseTuple(args, "s#nO!s#s#",
                          &origData, &origDataLength, &newDataLength,
                          &PyList_Type, &controlTuples,
                          &diffBlock, &diffBlockLength,
                          &extraBlock, &extraBlockLength))
        return NULL;

    /* allocate the memory for the new data */
    newData = PyMem_Malloc(newDataLength + 1);
    if (!newData)
        return PyErr_NoMemory();

    oldpos = 0;
    newpos = 0;
    diffPtr = diffBlock;
    extraPtr = extraBlock;
    numTuples = PyList_GET_SIZE(controlTuples);
    for (i = 0; i < numTuples; i++) {
        tuple = PyList_GET_ITEM(controlTuples, i);
        if (!PyTuple_Check(tuple)) {
            PyMem_Free(newData);
            PyErr_SetString(PyExc_TypeError, "expecting tuple");
            return NULL;
        }
        if (PyTuple_GET_SIZE(tuple) != 3) {
            PyMem_Free(newData);
            PyErr_SetString(PyExc_TypeError, "expecting tuple of size 3");
            return NULL;
        }
        x = PyLong_AsLong(PyTuple_GET_ITEM(tuple, 0));
        y = PyLong_AsLong(PyTuple_GET_ITEM(tuple, 1));
        z = PyLong_AsLong(PyTuple_GET_ITEM(tuple, 2));
        if (newpos + x > newDataLength ||
                diffPtr + x > diffBlock + diffBlockLength) {
            PyMem_Free(newData);
            PyErr_SetString(PyExc_ValueError, "corrupt patch (overflow)");
            return NULL;
        }
        memcpy(newData + newpos, diffPtr, x);
        diffPtr += x;
        for (j = 0; j < x; j++)
            if ((oldpos + j >= 0) && (oldpos + j < origDataLength))
                newData[newpos + j] += origData[oldpos + j];
        newpos += x;
        oldpos += x;
        if (newpos + y > newDataLength ||
                extraPtr + y > extraBlock + extraBlockLength) {
            PyMem_Free(newData);
            PyErr_SetString(PyExc_ValueError, "corrupt patch (overflow)");
            return NULL;
        }
        memcpy(newData + newpos, extraPtr, y);
        extraPtr += y;
        newpos += y;
        oldpos += z;
    }

    /* confirm that a valid patch was applied */
    if (newpos != newDataLength ||
            diffPtr != diffBlock + diffBlockLength ||
            extraPtr != extraBlock + extraBlockLength) {
        PyMem_Free(newData);
        PyErr_SetString(PyExc_ValueError, "corrupt patch (underflow)");
        return NULL;
    }

    results = PyBytes_FromStringAndSize(newData, newDataLength);
    PyMem_Free(newData);
    return results;
}


static off_t search(off_t *I,
                    unsigned char *old, off_t oldsize,
                    unsigned char *new, off_t newsize,
                    off_t st, off_t en, off_t *pos)
{
    off_t x, y;

    if (en - st < 2) {
        x = matchlen(old + I[st], oldsize - I[st], new, newsize);
        y = matchlen(old + I[en], oldsize - I[en], new, newsize);

        if (x > y) {
            *pos = I[st];
            return x;
        } else {
            *pos = I[en];
            return y;
        }
    }

    x = st + (en - st) / 2;
    if (memcmp(old + I[x], new, MIN(oldsize - I[x], newsize)) < 0) {
        return search(I, old, oldsize, new, newsize, x, en, pos);
    } else {
        return search(I, old, oldsize, new, newsize, st, x, pos);
    }
}


static PyObject *decode_int64(PyObject *self, PyObject *string)
{
    long long x;
    char *bs;
    int i;

    if (!PyBytes_Check(string)) {
        PyErr_SetString(PyExc_TypeError, "bytes expected");
        return NULL;
    }
    if (PyBytes_Size(string) != 8) {
        PyErr_SetString(PyExc_ValueError, "8 bytes expected");
        return NULL;
    }
    bs = PyBytes_AsString(string);

    x = bs[7] & 0x7F;
    for (i = 6; i >= 0; i--) {
        x <<= 8;  /* x = x * 256 + (unsigned char) bs[i]; */
        x |= (unsigned char) bs[i];
    }
    if (bs[7] & 0x80)
        x = -x;
    return PyLong_FromLongLong(x);
}


static void split(off_t *I, off_t *V, off_t start, off_t len, off_t h)
{
    off_t i, j, k, x, tmp, jj, kk;

    if (len < 16) {
        for (k = start; k < start + len; k += j) {
            j = 1;
            x = V[I[k] + h];
            for (i = 1; k + i < start + len; i++) {
                if (V[I[k + i] + h] < x) {
                    x = V[I[k + i] + h];
                    j = 0;
                }
                if (V[I[k + i] + h] == x) {
                    tmp = I[k + j];
                    I[k + j] = I[k + i];
                    I[k + i] = tmp;
                    j++;
                }
            }
            for (i = 0; i < j; i++)
                V[I[k + i]] = k + j - 1;
            if (j == 1)
                I[k] = -1;
        }

    } else {

        jj = 0;
        kk = 0;
        x = V[I[start + len / 2] + h];
        for (i = start; i < start + len; i++) {
            if (V[I[i] + h] < x)
                jj++;
            if (V[I[i] + h] == x)
                kk++;
        }
        jj += start;
        kk += jj;

        j = 0;
        k = 0;
        i = start;
        while (i < jj) {
            if (V[I[i] + h] < x) {
                i++;
            } else if (V[I[i] + h] == x) {
                tmp = I[i];
                I[i] = I[jj + j];
                I[jj + j] = tmp;
                j++;
            } else {
                tmp = I[i];
                I[i] = I[kk + k];
                I[kk + k] = tmp;
                k++;
            }
        }

        while (jj + j < kk) {
            if (V[I[jj + j] + h] == x) {
                j++;
            } else {
                tmp = I[jj + j];
                I[jj + j] = I[kk + k];
                I[kk + k] = tmp;
                k++;
            }
        }

        if (jj > start)
            split(I, V, start, jj - start, h);

        for (i = 0; i < kk - jj; i++)
            V[I[jj + i]] = kk - 1;
        if (jj == kk - 1)
            I[jj] = -1;
        if (start + len > kk)
            split(I, V, kk, start + len - kk, h);
    }
}


static PyObject* diff(PyObject* self, PyObject* args)
{
    off_t lastscan, lastpos, lastoffset, oldscore, scsc, overlap, Ss, lens;
    off_t *I, *V, dblen, eblen, scan, pos, len, s, Sf, lenf, Sb, lenb, i;
    PyObject *controlTuples, *tuple, *results, *temp;
    Py_ssize_t origDataLength, newDataLength;
    char *origData, *newData;
    unsigned char *db, *eb;

    if (!PyArg_ParseTuple(args, "s#s#",
                          &origData, &origDataLength,
                          &newData, &newDataLength))
        return NULL;

    /* create the control tuple */
    controlTuples = PyList_New(0);
    if (!controlTuples)
        return NULL;

    /* perform sort on original data */
    I = PyMem_Malloc((origDataLength + 1) * sizeof(off_t));
    if (!I) {
        Py_DECREF(controlTuples);
        return PyErr_NoMemory();
    }
    V = PyMem_Malloc((origDataLength + 1) * sizeof(off_t));
    if (!V) {
        Py_DECREF(controlTuples);
        PyMem_Free(I);
        return PyErr_NoMemory();
    }
    Py_BEGIN_ALLOW_THREADS  /* release GIL */
    qsufsort(I, V, (unsigned char *) origData, origDataLength);
    Py_END_ALLOW_THREADS
    PyMem_Free(V);

    /* allocate memory for the diff and extra blocks */
    db = PyMem_Malloc(newDataLength + 1);
    if (!db) {
        Py_DECREF(controlTuples);
        PyMem_Free(I);
        return PyErr_NoMemory();
    }
    eb = PyMem_Malloc(newDataLength + 1);
    if (!eb) {
        Py_DECREF(controlTuples);
        PyMem_Free(I);
        PyMem_Free(db);
        return PyErr_NoMemory();
    }
    dblen = 0;
    eblen = 0;

    /* perform the diff */
    len = 0;
    scan = 0;
    lastscan = 0;
    lastpos = 0;
    lastoffset = 0;
    pos = 0;
    while (scan < newDataLength) {
        oldscore = 0;

        Py_BEGIN_ALLOW_THREADS  /* release GIL */
        for (scsc = scan += len; scan < newDataLength; scan++) {
            len = search(I, (unsigned char *) origData, origDataLength,
                         (unsigned char *) newData + scan,
                         newDataLength - scan, 0, origDataLength, &pos);
            for (; scsc < scan + len; scsc++)
                if ((scsc + lastoffset < origDataLength) &&
                          (origData[scsc + lastoffset] == newData[scsc]))
                    oldscore++;
            if (((len == oldscore) && (len != 0)) || (len > oldscore + 8))
                break;
            if ((scan + lastoffset < origDataLength) &&
                      (origData[scan + lastoffset] == newData[scan]))
                oldscore--;
        }
        Py_END_ALLOW_THREADS

        if ((len != oldscore) || (scan == newDataLength)) {
            s = 0;
            Sf = 0;
            lenf = 0;
            for (i = 0; (lastscan + i < scan) &&
                     (lastpos + i < origDataLength);) {
                if (origData[lastpos + i] == newData[lastscan + i])
                    s++;
                i++;
                if (s * 2 - i > Sf * 2 - lenf) {
                    Sf = s;
                    lenf = i;
                }
            }

            lenb = 0;
            if (scan < newDataLength) {
                s = 0;
                Sb = 0;
                for (i = 1; (scan >= lastscan + i) && (pos >= i); i++) {
                    if (origData[pos - i] == newData[scan - i])
                        s++;
                    if (s * 2 - i > Sb * 2 - lenb) {
                        Sb = s;
                        lenb = i;
                    }
                }
            }

            if (lastscan + lenf > scan - lenb) {
                overlap = (lastscan + lenf) - (scan - lenb);
                s = 0;
                Ss = 0;
                lens = 0;
                for (i = 0; i < overlap; i++) {
                    if (newData[lastscan + lenf - overlap + i] ==
                            origData[lastpos + lenf - overlap + i])
                        s++;
                    if (newData[scan - lenb + i]== origData[pos - lenb + i])
                        s--;
                    if (s > Ss) {
                        Ss = s;
                        lens = i + 1;
                    }
                }

                lenf += lens - overlap;
                lenb -= lens;
            }

            for (i = 0; i < lenf; i++)
                db[dblen + i] = newData[lastscan + i] - origData[lastpos + i];
            for (i = 0; i < (scan - lenb) - (lastscan + lenf); i++)
                eb[eblen + i] = newData[lastscan + lenf + i];

            dblen += lenf;
            eblen += (scan - lenb) - (lastscan + lenf);

            tuple = PyTuple_New(3);
            if (!tuple) {
                Py_DECREF(controlTuples);
                PyMem_Free(I);
                PyMem_Free(db);
                PyMem_Free(eb);
                return NULL;
            }
            PyTuple_SET_ITEM(tuple, 0, PyLong_FromLong(lenf));
            PyTuple_SET_ITEM(tuple, 1,
                    PyLong_FromLong((scan - lenb) - (lastscan + lenf)));
            PyTuple_SET_ITEM(tuple, 2,
                    PyLong_FromLong((pos - lenb) - (lastpos + lenf)));
            if (PyList_Append(controlTuples, tuple) < 0) {
                Py_DECREF(controlTuples);
                Py_DECREF(tuple);
                PyMem_Free(I);
                PyMem_Free(db);
                PyMem_Free(eb);
                return NULL;
            }
            Py_DECREF(tuple);

            lastscan = scan - lenb;
            lastpos = pos - lenb;
            lastoffset = pos - scan;
        }
    }

    PyMem_Free(I);
    results = PyTuple_New(3);
    if (!results) {
        PyMem_Free(db);
        PyMem_Free(eb);
        return NULL;
    }
    PyTuple_SET_ITEM(results, 0, controlTuples);
    temp = PyBytes_FromStringAndSize((char *) db, dblen);
    PyMem_Free(db);
    if (!temp) {
        PyMem_Free(eb);
        Py_DECREF(results);
        return NULL;
    }
    PyTuple_SET_ITEM(results, 1, temp);
    temp = PyBytes_FromStringAndSize((char *) eb, eblen);
    PyMem_Free(eb);
    if (!temp) {
        Py_DECREF(results);
        return NULL;
    }
    PyTuple_SET_ITEM(results, 2, temp);

    return results;
}


static inline int p_buf_refill(struct parser_buf *buf, const int input_fd)
{
    int read_size;

    read_size = read(input_fd, buf->f_buf, SRC_PARSER_F_BUF_SIZE);
    buf->f_indx = 0;
    buf->f_read_size = read_size;
    return read_size;
}


static void print_file_full(int fd)
{
    char f_buf[SRC_PARSER_F_BUF_SIZE];
    int read_size;

    if (lseek(fd, 0, SEEK_SET)) {
        fprintf(stderr, "**Error: Could not set offset.\n");
        return;
    }

    while ((read_size = read(fd, f_buf, SRC_PARSER_F_BUF_SIZE)) > 0) {
        int read_indx = 0;

        while (read_indx < read_size)
            putchar(f_buf[read_indx++]);
    }
}


static inline int p_buf_push_tmp_char(struct parser_buf *buf, const char c)
{
    buf->tmp_buf[buf->tmp_indx++] = c;
    buf->f_indx++;
    return buf->tmp_indx;
}


static inline int p_buf_write_f_char(struct parser_buf *buf, const int output_fd)
{
    return write(output_fd, &buf->f_buf[buf->f_indx++], 1);
}


static int src_parser_trans_stage_1_2_3(const int tmp_fd, const char *src, const struct trans_config cfg)
{
    struct parser_buf pbuf = {
        .f_indx = 0,
        .tmp_indx = 0,
        .f_read_size = 0
    };

    int write_count = 0;
    int src_fd;
    int p_state = P_STATE_CODE;

    src_fd = open(src, O_RDONLY);
    if (src_fd == -1) {
        fprintf(stderr, "**Error: Could not open source file: %s.\n", src);
        return -1;
    }

    while (p_buf_refill(&pbuf, src_fd) > 0) {

        while (PBUF_F_REMD(pbuf)) {

            switch (p_state) {
            case P_STATE_COMMENT_C:

                switch (PBUF_F_CHAR(pbuf)) {
                case '*':
                    p_buf_push_tmp_char(&pbuf, '*');
                    continue;

                case '/':
                    if (pbuf.tmp_indx && (PBUF_TMP_PREV_CHAR(pbuf) == '*')) {
                        pbuf.tmp_indx--;
                        p_state = P_STATE_CODE;
                    }
                    break;

                default:
                    if (pbuf.tmp_indx && (PBUF_TMP_PREV_CHAR(pbuf) == '*'))
                        pbuf.tmp_indx--;
                    break;
                }

                pbuf.f_indx++;

            case P_STATE_CODE:
            default:

                /* TODO: add trigraph support */

                switch (PBUF_F_CHAR(pbuf)) {
                case ' ':
                case '\t':
                    if (pbuf.tmp_indx &&
                            (PBUF_TMP_PREV_CHAR(pbuf) == ' ' || PBUF_TMP_PREV_CHAR(pbuf) == '\t' ||
                             PBUF_TMP_PREV_CHAR(pbuf) == '\n'))
                        pbuf.f_indx++;
                    else
                        p_buf_push_tmp_char(&pbuf, ' ');

                    continue;

                case '\r':
                case '\n':
                    if (pbuf.tmp_indx &&
                            (PBUF_TMP_PREV_CHAR(pbuf) == ' ' || PBUF_TMP_PREV_CHAR(pbuf) == '\t' ||
                             PBUF_TMP_PREV_CHAR(pbuf) == '\n')) {
                        pbuf.f_indx++;
                    } else if (pbuf.tmp_indx &&
                            (PBUF_TMP_PREV_CHAR(pbuf) == '\\')) {
                        pbuf.tmp_indx--;
                        pbuf.f_indx++;
                    } else {
                        p_buf_push_tmp_char(&pbuf, '\n');
                    }

                    continue;

                case '\\':
                    p_buf_write_tmp(&pbuf, tmp_fd);
                    p_buf_push_tmp_char(&pbuf, '\\');
                    continue;

                case '/':
                    p_buf_write_tmp(&pbuf, tmp_fd);
                    p_buf_push_tmp_char(&pbuf, '/');
                    continue;

                case '*':
                    if (pbuf.tmp_indx &&
                            (PBUF_TMP_PREV_CHAR(pbuf) == '/')) {
                        pbuf.tmp_indx--;
                        pbuf.f_indx++;
                        p_state = P_STATE_COMMENT_C;
                        continue;
                    }

                default:
                    break;
                }

                /* TODO: check return values */
                p_buf_write_tmp(&pbuf, tmp_fd);
                p_buf_write_f_char(&pbuf, tmp_fd);
            }
        }
    }

    p_buf_write_tmp(&pbuf, tmp_fd);
    return 0;
}


static inline int p_buf_write_tmp(struct parser_buf *buf, const int output_fd)
{
    int write_size;

    if (!buf->tmp_indx)
        return 0;

    write_size = write(output_fd, buf->tmp_buf, buf->tmp_indx);
    buf->tmp_indx = 0;

    return write_size;
}


int src_parser_cpp(const char *src, const struct trans_config cfg)
{
    int tmp_fd;
    char fname[TMP_FILE_NAME_SIZE];

    strncpy(fname, TMP_FILE_NAME, TMP_FILE_NAME_SIZE);
    tmp_fd = mkstemp(fname);
    if (tmp_fd == -1) {
        fprintf(stderr, "**Error: could not create a working file.\n");
        return -1;
    }

    src_parser_trans_stage_1_2_3(tmp_fd, src, cfg);

    print_file_full(tmp_fd);

    unlink(fname);
}


XMLNode::property(const string& name)
{
	XMLPropertyIterator iter = _proplist.begin();

	while (iter != _proplist.end()) {
		if ((*iter)->name() == name) {
			return *iter;
		}
		++iter;
	}

	return 0;
}


XMLTree::read_buffer(const string& buffer, bool to_tree_doc)
{
	xmlDocPtr doc;

	_filename = "";

	delete _root;
	_root = 0;

	doc = xmlParseMemory(const_cast<char*>(buffer.c_str()), buffer.length());
	if (!doc) {
		return false;
	}

	_root = readnode(xmlDocGetRootElement(doc));
	if (to_tree_doc) {
		if (_doc) {
			xmlFreeDoc (_doc);
		}
		_doc = doc;
	} else {
		xmlFreeDoc (doc);
	}

	return true;
}


static XMLSharedNodeList* find_impl(xmlXPathContext* ctxt, const string& xpath)
{
	xmlXPathObject* result = xmlXPathEval((const xmlChar*)xpath.c_str(), ctxt);

	if (!result) {
		xmlFreeDoc(ctxt->doc);
		xmlXPathFreeContext(ctxt);

		throw XMLException("Invalid XPath: " + xpath);
	}

	if (result->type != XPATH_NODESET) {
		xmlXPathFreeObject(result);
		xmlFreeDoc(ctxt->doc);
		xmlXPathFreeContext(ctxt);

		throw XMLException("Only nodeset result types are supported.");
	}

	xmlNodeSet* nodeset = result->nodesetval;
	XMLSharedNodeList* nodes = new XMLSharedNodeList();
	if (nodeset) {
		for (int i = 0; i < nodeset->nodeNr; ++i) {
			XMLNode* node = readnode(nodeset->nodeTab[i]);
			nodes->push_back(boost::shared_ptr<XMLNode>(node));
		}
	} else {
		// return empty set
	}

	xmlXPathFreeObject(result);

	return nodes;
}


XMLNode::remove_nodes_and_delete(const string& propname, const string& val)
{
	XMLNodeIterator i = _children.begin();
	XMLProperty const * prop;

	while (i != _children.end()) {
		prop = (*i)->property(propname);
		if (prop && prop->value() == val) {
			delete *i;
			i = _children.erase(i);
		} else {
			++i;
		}
	}
}


XMLNode::child (const char* name) const
{
	/* returns first child matching name */

	XMLNodeConstIterator cur;

	if (name == 0) {
		return 0;
	}

	for (cur = _children.begin(); cur != _children.end(); ++cur) {
		if ((*cur)->name() == name) {
			return *cur;
		}
	}

	return 0;
}


XMLNode::XMLNode(const string& n, const string& c)
	: _name(n)
	, _is_content(true)
	, _content(c)
{
	_proplist.reserve (PROPERTY_RESERVE_COUNT);
}


XMLNode::~XMLNode()
{
	clear_lists ();
}


XMLNode::operator== (const XMLNode& other) const
{
	if (is_content () != other.is_content ()) {
		return false;
	}

	if (is_content ()) {
		if (content () != other.content ()) {
			return false;
		}
	} else {
		if (name () != other.name ()) {
			return false;
		}
	}

	XMLPropertyList const& other_properties = other.properties ();

	if (_proplist.size () != other_properties.size ()) {
		return false;
	}

	XMLPropertyConstIterator our_prop_iter = _proplist.begin();
	XMLPropertyConstIterator other_prop_iter = other_properties.begin();

	while (our_prop_iter != _proplist.end ()) {
		XMLProperty const* our_prop = *our_prop_iter;
		XMLProperty const* other_prop = *other_prop_iter;
		if (our_prop->name () != other_prop->name () || our_prop->value () != other_prop->value ()) {
			return false;
		}
		++our_prop_iter;
		++other_prop_iter;
	}

	XMLNodeList const& other_children = other.children();

	if (_children.size() != other_children.size()) {
		return false;
	}

	XMLNodeConstIterator our_child_iter = _children.begin ();
	XMLNodeConstIterator other_child_iter = other_children.begin ();

	while (our_child_iter != _children.end()) {
		XMLNode const* our_child = *our_child_iter;
		XMLNode const* other_child = *other_child_iter;

		if (*our_child != *other_child) {
			return false;
		}
		++our_child_iter;
		++other_child_iter;
	}
	return true;
}


readnode(xmlNodePtr node)
{
	string name, content;
	xmlNodePtr child;
	XMLNode* tmp;
	xmlAttrPtr attr;

	if (node->name) {
		name = (const char*)node->name;
	}

	tmp = new XMLNode(name);

	for (attr = node->properties; attr; attr = attr->next) {
		content = "";
		if (attr->children) {
			content = (char*)attr->children->content;
		}
		tmp->set_property((const char*)attr->name, content);
	}

	if (node->content) {
		tmp->set_content((char*)node->content);
	} else {
		tmp->set_content(string());
	}

	for (child = node->children; child; child = child->next) {
		tmp->add_child_nocopy (*readnode(child));
	}

	return tmp;
}


XMLNode::dump (ostream& s, string p) const
{
	if (_is_content) {
		s << p << "  " << content() << "\n";
	} else {
		s << p << "<" << _name;
		for (XMLPropertyList::const_iterator i = _proplist.begin(); i != _proplist.end(); ++i) {
			s << " " << (*i)->name() << "=\"" << (*i)->value() << "\"";
		}
		s << ">\n";

		for (XMLNodeList::const_iterator i = _children.begin(); i != _children.end(); ++i) {
			(*i)->dump (s, p + "  ");
		}

		s << p << "</" << _name << ">\n";
	}
}


XMLNode::has_property_with_value (const string& name, const string& value) const
{
	XMLPropertyConstIterator iter = _proplist.begin();

	while (iter != _proplist.end()) {
		if ((*iter)->name() == name && (*iter)->value() == value) {
			return true;
		}
		++iter;
	}
	return false;
}


XMLTree::debug(FILE* out) const
{
#ifdef LIBXML_DEBUG_ENABLED
	xmlDocPtr doc;
	XMLNodeList children;

	xmlKeepBlanksDefault(0);
	doc = xmlNewDoc(xml_version);
	xmlSetDocCompressMode(doc, _compression);
	writenode(doc, _root, doc->children, 1);
	xmlDebugDumpDocument (out, doc);
	xmlFreeDoc(doc);
#endif
}


XMLTree::XMLTree(const XMLTree* from)
	: _filename(from->filename())
	, _root(new XMLNode(*from->root()))
	, _doc (xmlCopyDoc (from->_doc, 1))
	, _compression(from->compression())
{

}


XMLNode::XMLNode(const string& n)
	: _name(n)
	, _is_content(false)
{
	_proplist.reserve (PROPERTY_RESERVE_COUNT);
}


XMLNode::clear_lists ()
{
	XMLNodeIterator curchild;
	XMLPropertyIterator curprop;

	_selected_children.clear ();

	for (curchild = _children.begin(); curchild != _children.end(); ++curchild) {
		delete *curchild;
	}

	_children.clear ();

	for (curprop = _proplist.begin(); curprop != _proplist.end(); ++curprop) {
		delete *curprop;
	}

	_proplist.clear ();
}


XMLNode::add_child_nocopy(XMLNode& n)
{
	_children.insert(_children.end(), &n);
}


XMLTree::XMLTree()
	: _filename()
	, _root(0)
	, _doc (0)
	, _compression(0)
{
}


XMLTree::write_buffer() const
{
	static string retval;
	char* ptr;
	int len;
	xmlDocPtr doc;
	XMLNodeList children;

	xmlKeepBlanksDefault(0);
	doc = xmlNewDoc(xml_version);
	xmlSetDocCompressMode(doc, _compression);
	writenode(doc, _root, doc->children, 1);
	xmlDocDumpMemory(doc, (xmlChar **) & ptr, &len);
	xmlFreeDoc(doc);

	retval = ptr;

	free(ptr);

	return retval;
}


XMLTree::read_internal(bool validate)
{
	//shouldnt be used anywhere ATM, remove if so!
	assert(!validate);

	delete _root;
	_root = 0;

	if (_doc) {
		xmlFreeDoc (_doc);
		_doc = 0;
	}

	/* Calling this prevents libxml2 from treating whitespace as active
	   nodes. It needs to be called before we create a parser context.
	*/
	xmlKeepBlanksDefault(0);

	/* create a parser context */
	xmlParserCtxtPtr ctxt = xmlNewParserCtxt();
	if (ctxt == NULL) {
		return false;
	}

	/* parse the file, activating the DTD validation option */
	if (validate) {
		_doc = xmlCtxtReadFile(ctxt, _filename.c_str(), NULL, XML_PARSE_DTDVALID);
	} else {
		_doc = xmlCtxtReadFile(ctxt, _filename.c_str(), NULL, XML_PARSE_HUGE);
	}

	/* check if parsing suceeded */
	if (_doc == NULL) {
		xmlFreeParserCtxt(ctxt);
		return false;
	} else {
		/* check if validation suceeded */
		if (validate && ctxt->valid == 0) {
			xmlFreeParserCtxt(ctxt);
			throw XMLException("Failed to validate document " + _filename);
		}
	}

	_root = readnode(xmlDocGetRootElement(_doc));

	/* free up the parser context */
	xmlFreeParserCtxt(ctxt);

	return true;
}


XMLTree::~XMLTree()
{
	delete _root;

	if (_doc) {
		xmlFreeDoc (_doc);
	}
}


XMLNode::set_content(const string& c)
{
	if (c.empty()) {
		_is_content = false;
	} else {
		_is_content = true;
	}

	_content = c;

	return _content;
}


XMLTree::find(const string xpath, XMLNode* node) const
{
	xmlXPathContext* ctxt;
	xmlDocPtr doc = 0;

	if (node) {
		doc = xmlNewDoc(xml_version);
		writenode(doc, node, doc->children, 1);
		ctxt = xmlXPathNewContext(doc);
	} else {
		ctxt = xmlXPathNewContext(_doc);
	}

	boost::shared_ptr<XMLSharedNodeList> result =
		boost::shared_ptr<XMLSharedNodeList>(find_impl(ctxt, xpath));

	xmlXPathFreeContext(ctxt);
	if (doc) {
		xmlFreeDoc (doc);
	}

	return result;
}


XMLNode::operator= (const XMLNode& from)
{
	if (&from == this) {
		return *this;
	}

	clear_lists ();

	_name = from.name ();
	set_content (from.content ());

	const XMLPropertyList& props = from.properties ();

	for (XMLPropertyConstIterator prop_iter = props.begin (); prop_iter != props.end (); ++prop_iter) {
		set_property ((*prop_iter)->name ().c_str (), (*prop_iter)->value ());
	}

	const XMLNodeList& nodes = from.children ();
	for (XMLNodeConstIterator child_iter = nodes.begin (); child_iter != nodes.end (); ++child_iter) {
		add_child_copy (**child_iter);
	}

	return *this;
}


XMLNode::XMLNode(const XMLNode& from)
{
	_proplist.reserve (PROPERTY_RESERVE_COUNT);
	*this = from;
}


XMLNode::add_child_copy(const XMLNode& n)
{
	XMLNode *copy = new XMLNode(n);
	_children.insert(_children.end(), copy);
	return copy;
}


XMLNode::property(const char* name) const
{
	XMLPropertyConstIterator iter = _proplist.begin();

	while (iter != _proplist.end()) {
		if ((*iter)->name() == name) {
			return *iter;
		}
		++iter;
	}

	return 0;
}


XMLNode::attribute_value()
{
	XMLNodeList children = this->children();
	if (_is_content)
		throw XMLException("XMLNode: attribute_value failed (is_content) for requested node: " + name());

	if (children.size() != 1)
		throw XMLException("XMLNode: attribute_value failed (children.size != 1) for requested node: " + name());

	XMLNode* child = *(children.begin());
	if (!child->is_content())
		throw XMLException("XMLNode: attribute_value failed (!child->is_content()) for requested node: " + name());

	return child->content();
}


XMLNode::operator!= (const XMLNode& other) const
{
	return !(*this == other);
}


XMLNode::property(const string& name) const
{
	XMLPropertyConstIterator iter = _proplist.begin();

	while (iter != _proplist.end()) {
		if ((*iter)->name() == name) {
			return *iter;
		}
		++iter;
	}
	return 0;
}


XMLNode::add_content(const string& c)
{
	if (c.empty ()) {
		/* this would add a "</>" child, leading to invalid XML.
		 * Also in XML, empty string content is equivalent to no content.
		 */
		return NULL;
	}
	return add_child_copy(XMLNode (string(), c));
}


XMLNode::remove_node_and_delete(const string& n, const string& propname, const string& val)
{
	for (XMLNodeIterator i = _children.begin(); i != _children.end(); ++i) {
		if ((*i)->name() == n) {
			XMLProperty const * prop = (*i)->property (propname);
			if (prop && prop->value() == val) {
				delete *i;
				_children.erase (i);
				break;
			}
		}
	}
}


XMLNode::remove_property(const string& name)
{
	XMLPropertyIterator iter = _proplist.begin();

	while (iter != _proplist.end()) {
		if ((*iter)->name() == name) {
			XMLProperty* property = *iter;
			_proplist.erase (iter);
			delete property;
			break;
		}
		++iter;
	}
}


XMLNode::remove_nodes(const string& n)
{
	XMLNodeIterator i = _children.begin();
	while (i != _children.end()) {
		if ((*i)->name() == n) {
			i = _children.erase (i);
		} else {
			++i;
		}
	}
}


XMLNode::get_property(const char* name, std::string& value) const
{
	XMLProperty const* const prop = property (name);
	if (!prop)
		return false;

	value = prop->value ();

	return true;
}


XMLNode::children(const string& n) const
{
	/* returns all children matching name */

	XMLNodeConstIterator cur;

	if (n.empty()) {
		return _children;
	}

	_selected_children.clear();

	for (cur = _children.begin(); cur != _children.end(); ++cur) {
		if ((*cur)->name() == n) {
			_selected_children.insert(_selected_children.end(), *cur);
		}
	}

	return _selected_children;
}


XMLProperty::XMLProperty(const string& n, const string& v)
	: _name(n)
	, _value(v)
{
}


writenode(xmlDocPtr doc, XMLNode* n, xmlNodePtr p, int root = 0)
{
	xmlNodePtr node;

	if (root) {
		node = doc->children = xmlNewDocNode(doc, 0, (const xmlChar*) n->name().c_str(), 0);
	} else {
		node = xmlNewChild(p, 0, (const xmlChar*) n->name().c_str(), 0);
	}

	if (n->is_content()) {
		node->type = XML_TEXT_NODE;
		xmlNodeSetContentLen(node, (const xmlChar*)n->content().c_str(), n->content().length());
	}

	const XMLPropertyList& props = n->properties();

	for (XMLPropertyConstIterator prop_iter = props.begin (); prop_iter != props.end ();
	     ++prop_iter) {
		xmlSetProp (node, (const xmlChar*)(*prop_iter)->name ().c_str (),
		            (const xmlChar*)(*prop_iter)->value ().c_str ());
	}

	const XMLNodeList& children = n->children ();
	for (XMLNodeConstIterator child_iter = children.begin (); child_iter != children.end ();
	     ++child_iter) {
		writenode (doc, *child_iter, node);
	}
}


XMLNode::remove_property_recursively(const string& n)
{
	remove_property (n);
	for (XMLNodeIterator i = _children.begin(); i != _children.end(); ++i) {
		(*i)->remove_property_recursively (n);
	}
}


XMLNode::set_property(const char* name, const string& value)
{
	XMLPropertyIterator iter = _proplist.begin();

	while (iter != _proplist.end()) {
		if ((*iter)->name() == name) {
			(*iter)->set_value (value);
			return *iter;
		}
		++iter;
	}

	XMLProperty* new_property = new XMLProperty(name, value);

	if (!new_property) {
		return 0;
	}

	_proplist.insert(_proplist.end(), new_property);

	return new_property;
}


XMLTree::set_compression(int c)
{
	if (c > 9) {
		c = 9;
	} else if (c < 0) {
		c = 0;
	}

	_compression = c;

	return _compression;
}


XMLNode::remove_nodes_and_delete(const string& n)
{
	XMLNodeIterator i = _children.begin();

	while (i != _children.end()) {
		if ((*i)->name() == n) {
			delete *i;
			i = _children.erase (i);
		} else {
			++i;
		}
	}
}


XMLNode::add_child(const char* n)
{
	return add_child_copy(XMLNode (n));
}


XMLTree::XMLTree(const string& fn, bool validate)
	: _filename(fn)
	, _root(0)
	, _doc (0)
	, _compression(0)
{
	read_internal(validate);
}


XMLTree::write() const
{
	xmlDocPtr doc;
	XMLNodeList children;
	int result;

	xmlKeepBlanksDefault(0);
	doc = xmlNewDoc(xml_version);
	xmlSetDocCompressMode(doc, _compression);
	writenode(doc, _root, doc->children, 1);
	result = xmlSaveFormatFileEnc(_filename.c_str(), doc, "UTF-8", 1);
#ifndef NDEBUG
	if (result == -1) {
		xmlErrorPtr xerr = xmlGetLastError ();
		if (!xerr) {
			std::cerr << "unknown XML error during xmlSaveFormatFileEnc()." << std::endl;
		} else {
			std::cerr << "xmlSaveFormatFileEnc: error"
				<< " domain: " << xerr->domain
				<< " code: " << xerr->code
				<< " msg: " << xerr->message
				<< std::endl;
		}
	}
#endif
	xmlFreeDoc(doc);

	if (result == -1) {
		return false;
	}

	return true;
}


XMLNode::property(const char* name)
{
	XMLPropertyIterator iter = _proplist.begin();

	while (iter != _proplist.end()) {
		if ((*iter)->name() == name) {
			return *iter;
		}
		++iter;
	}
	return 0;
}


static Jsi_RC CDataEnumFindCmd(Jsi_Interp *interp, Jsi_Value *args, Jsi_Value *_this,
                                Jsi_Value **ret, Jsi_Func *funcPtr)
{

    Jsi_Value *arg2 = Jsi_ValueArrayIndex(interp, args, 1);
    char *arg1 = Jsi_ValueArrayIndexToStr(interp, args, 0, NULL);
    Jsi_EnumSpec *ei, *el;
    if (!(el = jsi_csEnumGet(interp, arg1)))
        return Jsi_LogError("Unknown enum: %s", arg1);
    Jsi_Wide wval;
    if (Jsi_GetWideFromValue(interp, arg2, &wval) != JSI_OK)
        return JSI_ERROR;

    ei = (typeof(ei))el->extData;
    uint i;
    for (i=0; i<el->size; i++) {
        if (wval == (Jsi_Wide)ei[i].value) {
            Jsi_ValueMakeStringKey(interp, ret, ei[i].name);
            return JSI_OK;
        }
    }
    return JSI_OK;
}


static Jsi_RC jsi_csBitGetSet(Jsi_Interp *interp, void *vrec,  Jsi_Wide* vPtr, Jsi_OptionSpec *spec, int idx, bool isSet) {
    bool us = (spec->tname && spec->tname[0] == 'u');
    if (!us)
        return jsi_csSBitGetSet(interp, vrec, vPtr, spec, idx, isSet);

    if (*vPtr<0)
        return JSI_ERROR;

    Jsi_UWide *valPtr = (typeof(valPtr))vPtr;
    int bits = spec->bits;
    int boffs = spec->boffset;
    if (bits<1 || bits>=64) return JSI_ERROR;
    int ofs = (boffs/8);
    int bo = (boffs%8); // 0 if byte-aligned
    int Bsz = ((bits+bo+7)/8);
    uchar *rec = (uchar*)vrec;
#ifdef __SIZEOF_INT128__
    typedef unsigned __int128 utvalType;
#else
    typedef Jsi_UWide utvalType;
#endif
    utvalType tbuf[2] = {};
    uchar sbuf[20], *bptr = (uchar*)tbuf;
    memcpy(tbuf, rec+ofs, Bsz);
    Jsi_UWide mval;
    Jsi_UWide amask = ((1LL<<(bits-1))-1LL);
    utvalType tval = 0, kval = 0, lmask;
    if (bo) { // If not byte aligned, get tval and shift
        bptr = sbuf;
        kval = tval = *(typeof(tval)*)tbuf;
        tval >>= bo;
        if (!isSet) {
            mval = (Jsi_UWide)tval;
            *(Jsi_UWide*)bptr = mval;
        }
    } else
         mval = *valPtr;

    if (!isSet) { // Get value.
        if (!jsi_csBitSetGet(0, bptr, bits, &mval))
            return JSI_ERROR;
        *valPtr = mval;
        return JSI_OK;
    }

    if (!jsi_csBitSetGet(1, bptr, bits, &mval))
        return JSI_ERROR;
    if (bo) {
        tval = (typeof(tval))mval;
        lmask=(amask<<bo);
        kval &= ~lmask;
        tval <<= bo;
        tval = (kval | tval);
        *(typeof(tval)*)tbuf = tval;
    }
    memcpy(rec+ofs, tbuf, Bsz);

    return JSI_OK;
}


Jsi_TreeEntry *Jsi_TreeObjSetValue(Jsi_Obj *obj, const char *key, Jsi_Value *val, int isstrkey) {
    Jsi_Tree *treePtr = obj->tree;
    bool isNew;
    Jsi_TreeEntry *hPtr;
    Jsi_Interp *interp = treePtr->opts.interp;
    if (!isstrkey) {
        const char *okey = key;
        Jsi_MapEntry *hePtr = Jsi_MapEntryNew(interp->strKeyTbl, key, &isNew);
        key = (const char*)Jsi_MapKeyGet(hePtr, 0);
        if (!key) {
            Jsi_MapEntry *hePtr = Jsi_MapEntryNew(interp->strKeyTbl, okey, &isNew);
            key = (const char*)Jsi_MapKeyGet(hePtr, 0);
        }
    }
    //return Jsi_TreeSet(treePtr, key, val);
    hPtr = Jsi_TreeEntryNew(treePtr, key, &isNew);
    if (!hPtr)
        return NULL;
    if (val)
        SIGASSERT(val,VALUE);
    if (!isNew)
        Jsi_ValueReplace(interp, (Jsi_Value**)&(hPtr->value), val);
    else {
        hPtr->value = val;
        if (val)
            Jsi_IncrRefCount(interp, val);
    }
//    Jsi_Value *oldVal;  /* FYI: This let kitty.breed() work in tests/proto2.js */
//    Assert(val->refCnt>0);
//    if (!isNew) {
//        oldVal = Jsi_TreeValueGet(hPtr);
//        if (oldVal) {
//            Jsi_ValueReset(interp, &oldVal);
//            Jsi_ValueCopy(interp, oldVal, val);
//        }
//    }
//    else
//        hPtr->value = val;

    return hPtr;
}


static Jsi_RC CDataUnsetCmd(Jsi_Interp *interp, Jsi_Value *args, Jsi_Value *_this,
                              Jsi_Value **ret, Jsi_Func *funcPtr)
{
    UdcGet(cd, _this, funcPtr);
    Jsi_Value *arg2 = Jsi_ValueArrayIndex(interp, args, 0);
    char kbuf[BUFSIZ];
    void *key = kbuf;
    if (!cd->mapPtr && cd->arrSize<=0)
        return Jsi_LogError("can not call unset on non map/array");
    if (JSI_OK != jsi_csGetKey(interp, cd, arg2, &key, sizeof(kbuf), 2)) {
        return JSI_ERROR;
    }
    uchar *dptr = NULL;
    Jsi_MapEntry *mPtr = Jsi_MapEntryFind(*cd->mapPtr, key);
    if (mPtr)
        dptr = (uchar*)Jsi_MapValueGet(mPtr);
    if (!dptr) {
        if (cd->keyType != JSI_KEYS_ONEWORD)
            return Jsi_LogError("no data in map: %s", (char*)key);
        else
            return Jsi_LogError("no data in map: %p", key);
        return JSI_ERROR;
    }
    Jsi_Free(dptr);
    Jsi_MapEntryDelete(mPtr);
    return JSI_OK;

}


void jsi_csInitType(Jsi_Interp *interp)
{
    if (interp->CTypeHash->numEntries) return;
    bool isNew;
    Jsi_HashEntry *entry;
    const Jsi_OptionTypedef *tl;
    if (!interp->typeInit) {
        int i;
        for (i = JSI_OPTION_BOOL; i!=JSI_OPTION_END; i++) {
            tl = Jsi_OptionTypeInfo((Jsi_OptionId)i);
            entry = Jsi_HashEntryNew(interp->TYPEHash, tl->idName, &isNew);
            if (!isNew)
                Jsi_LogBug("duplicate type: %s", tl->idName);
            Jsi_HashValueSet(entry, (void*)tl);
            if (tl->cName && tl->cName[0])
                Jsi_HashSet(interp->CTypeHash, tl->cName, (void*)tl);
        }
    }
    interp->typeInit = 1;
}


static Jsi_RC CDataSetCmd(Jsi_Interp *interp, Jsi_Value *args, Jsi_Value *_this,
                             Jsi_Value **ret, Jsi_Func *funcPtr) {
    return CDataSetCmd_(interp, args, _this, ret, funcPtr, 1);
}


Jsi_Obj *Jsi_ObjNewType(Jsi_Interp *interp, Jsi_otype otype)
{
    Jsi_Obj *obj = Jsi_ObjNew(interp);
    obj->ot = (otype==JSI_OT_ARRAY?JSI_OT_OBJECT:otype);
    switch (otype) {
        case JSI_OT_BOOL:   obj->__proto__ = interp->Boolean_prototype; break;
        case JSI_OT_NUMBER: obj->__proto__ = interp->Number_prototype; break;
        case JSI_OT_STRING: obj->__proto__ = interp->String_prototype; break;
        case JSI_OT_FUNCTION:obj->__proto__ = interp->Function_prototype; break;
        case JSI_OT_REGEXP: obj->__proto__ = interp->RegExp_prototype; break;
        case JSI_OT_OBJECT: obj->__proto__ = interp->Object_prototype; break;
        case JSI_OT_ARRAY:  obj->__proto__ = interp->Array_prototype;
            obj->isarrlist = 1;
            break;
        default: assert(0); break;
    }
    if (interp->protoInit)
        assert(obj->__proto__);
    return obj;
}


static bool jsi_csBitSetGet(int isSet, uchar *tbuf, int bits, Jsi_UWide *valPtr) {
    union bitfield *bms = (union bitfield *)tbuf;
    Jsi_UWide val = *valPtr;
    union bitfield {
        Jsi_UWide b1:1; Jsi_UWide b2:2; Jsi_UWide b3:3; Jsi_UWide b4:4; Jsi_UWide b5:5; Jsi_UWide b6:6;
        Jsi_UWide b7:7; Jsi_UWide b8:8; Jsi_UWide b9:9; Jsi_UWide b10:10; Jsi_UWide b11:11; Jsi_UWide b12:12;
        Jsi_UWide b13:13; Jsi_UWide b14:14; Jsi_UWide b15:15; Jsi_UWide b16:16; Jsi_UWide b17:17;
        Jsi_UWide b18:18; Jsi_UWide b19:19; Jsi_UWide b20:20; Jsi_UWide b21:21; Jsi_UWide b22:22;
        Jsi_UWide b23:23; Jsi_UWide b24:24; Jsi_UWide b25:25; Jsi_UWide b26:26; Jsi_UWide b27:27;
        Jsi_UWide b28:28; Jsi_UWide b29:29; Jsi_UWide b30:30; Jsi_UWide b31:31; Jsi_UWide b32:32;
        Jsi_UWide b33:33; Jsi_UWide b34:34; Jsi_UWide b35:35; Jsi_UWide b36:36; Jsi_UWide b37:37;
        Jsi_UWide b38:38; Jsi_UWide b39:39; Jsi_UWide b40:40; Jsi_UWide b41:41; Jsi_UWide b42:42;
        Jsi_UWide b43:43; Jsi_UWide b44:44; Jsi_UWide b45:45; Jsi_UWide b46:46; Jsi_UWide b47:47;
        Jsi_UWide b48:48; Jsi_UWide b49:49; Jsi_UWide b50:50; Jsi_UWide b51:51; Jsi_UWide b52:52;
        Jsi_UWide b53:53; Jsi_UWide b54:54; Jsi_UWide b55:55; Jsi_UWide b56:56; Jsi_UWide b57:57;
        Jsi_UWide b58:58; Jsi_UWide b59:59; Jsi_UWide b60:60; Jsi_UWide b61:61; Jsi_UWide b62:62;
        Jsi_UWide b63:63; Jsi_UWide b64:64;
    };
    if (isSet) {
        switch (bits) {
    #define CBSN(n) \
            case n: bms->b##n = val; return (bms->b##n == val)
           CBSN(1); CBSN(2); CBSN(3); CBSN(4); CBSN(5); CBSN(6); CBSN(7); CBSN(8);
           CBSN(9); CBSN(10); CBSN(11); CBSN(12); CBSN(13); CBSN(14); CBSN(15); CBSN(16);
           CBSN(17); CBSN(18); CBSN(19); CBSN(20); CBSN(21); CBSN(22); CBSN(23); CBSN(24);
           CBSN(25); CBSN(26); CBSN(27); CBSN(28); CBSN(29); CBSN(30); CBSN(31); CBSN(32);
           CBSN(33); CBSN(34); CBSN(35); CBSN(36); CBSN(37); CBSN(38); CBSN(39); CBSN(40);
           CBSN(41); CBSN(42); CBSN(43); CBSN(44); CBSN(45); CBSN(46); CBSN(47); CBSN(48);
           CBSN(49); CBSN(50); CBSN(51); CBSN(52); CBSN(53); CBSN(54); CBSN(55); CBSN(56);
           CBSN(57); CBSN(58); CBSN(59); CBSN(60); CBSN(61); CBSN(62); CBSN(63); CBSN(64);
        }
        assert(0);
    }
    switch (bits) {
#define CBGN(n) \
        case n: val = bms->b##n; break
       CBGN(1); CBGN(2); CBGN(3); CBGN(4); CBGN(5); CBGN(6); CBGN(7); CBGN(8);
       CBGN(9); CBGN(10); CBGN(11); CBGN(12); CBGN(13); CBGN(14); CBGN(15); CBGN(16);
       CBGN(17); CBGN(18); CBGN(19); CBGN(20); CBGN(21); CBGN(22); CBGN(23); CBGN(24);
       CBGN(25); CBGN(26); CBGN(27); CBGN(28); CBGN(29); CBGN(30); CBGN(31); CBGN(32);
       CBGN(33); CBGN(34); CBGN(35); CBGN(36); CBGN(37); CBGN(38); CBGN(39); CBGN(40);
       CBGN(41); CBGN(42); CBGN(43); CBGN(44); CBGN(45); CBGN(46); CBGN(47); CBGN(48);
       CBGN(49); CBGN(50); CBGN(51); CBGN(52); CBGN(53); CBGN(54); CBGN(55); CBGN(56);
       CBGN(57); CBGN(58); CBGN(59); CBGN(60); CBGN(61); CBGN(62); CBGN(63); CBGN(64);
       default: assert(0);
    }
    *valPtr = val;
    return 1;
}


static Jsi_RC CDataEnumGetCmd(Jsi_Interp *interp, Jsi_Value *args, Jsi_Value *_this,
    Jsi_Value **ret, Jsi_Func *funcPtr)
{

    char *name = Jsi_ValueArrayIndexToStr(interp, args, 0, NULL);
    Jsi_EnumSpec *sl = jsi_csEnumGet(interp, name);
    if (!sl)
        return JSI_OK;
    JSI_DSTRING_VAR(dsPtr, 400);
    Jsi_RC rc = CDataEnumGetDfn(interp, sl, dsPtr);
    if (JSI_OK == rc)
        rc = Jsi_JSONParse(interp, Jsi_DSValue(dsPtr), ret, 0);
    Jsi_DSFree(dsPtr);
    return rc;
}