GF_Err gf_isom_box_parse_ex(GF_Box **outBox, GF_BitStream *bs, u32 parent_type, Bool is_root_box)
{
	u32 type, uuid_type, hdr_size;
	u64 size, start, payload_start, end;
	char uuid[16];
	GF_Err e;
	GF_Box *newBox;
	Bool skip_logs = gf_bs_get_cookie(bs) ? GF_TRUE : GF_FALSE;
	Bool is_special = GF_TRUE;

	if ((bs == NULL) || (outBox == NULL) ) return GF_BAD_PARAM;
	*outBox = NULL;
	if (gf_bs_available(bs) < 8) {
		return GF_ISOM_INCOMPLETE_FILE;
	}

	start = gf_bs_get_position(bs);

	uuid_type = 0;
	size = (u64) gf_bs_read_u32(bs);
	hdr_size = 4;
	/*fix for some boxes found in some old hinted files*/
	if ((size >= 2) && (size <= 4)) {
		size = 4;
		type = GF_ISOM_BOX_TYPE_VOID;
	} else {
		type = gf_bs_read_u32(bs);
		hdr_size += 4;
		/*no size means till end of file - EXCEPT FOR some old QuickTime boxes...*/
		if (type == GF_ISOM_BOX_TYPE_TOTL)
			size = 12;
		if (!size) {
			if (is_root_box) {
				if (!skip_logs) {
					GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[iso file] Warning Read Box type %s (0x%08X) size 0 reading till the end of file\n", gf_4cc_to_str(type), type));
				}
				size = gf_bs_available(bs) + 8;
			} else {
				if (!skip_logs) {
					GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[iso file] Read Box type %s (0x%08X) at position "LLU" has size 0 but is not at root/file level, skipping\n", gf_4cc_to_str(type), type, start));
				}
				return GF_OK;
//				return GF_ISOM_INVALID_FILE;
			}
		}
	}
	/*handle uuid*/
	memset(uuid, 0, 16);
	if (type == GF_ISOM_BOX_TYPE_UUID ) {
		if (gf_bs_available(bs) < 16) {
			return GF_ISOM_INCOMPLETE_FILE;
		}
		gf_bs_read_data(bs, uuid, 16);
		hdr_size += 16;
		uuid_type = gf_isom_solve_uuid_box(uuid);
	}

	//handle large box
	if (size == 1) {
		if (gf_bs_available(bs) < 8) {
			return GF_ISOM_INCOMPLETE_FILE;
		}
		size = gf_bs_read_u64(bs);
		hdr_size += 8;
	}
	GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[iso file] Read Box type %s size "LLD" start "LLD"\n", gf_4cc_to_str(type), LLD_CAST size, LLD_CAST start));

	if ( size < hdr_size ) {
		GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[iso file] Box size "LLD" less than box header size %d\n", LLD_CAST size, hdr_size));
		return GF_ISOM_INVALID_FILE;
	}

	//some special boxes (references and track groups) are handled by a single generic box with an associated ref/group type
	if (parent_type && (parent_type == GF_ISOM_BOX_TYPE_TREF)) {
		newBox = gf_isom_box_new(GF_ISOM_BOX_TYPE_REFT);
		if (!newBox) return GF_OUT_OF_MEM;
		((GF_TrackReferenceTypeBox*)newBox)->reference_type = type;
	} else if (parent_type && (parent_type == GF_ISOM_BOX_TYPE_IREF)) {
		newBox = gf_isom_box_new(GF_ISOM_BOX_TYPE_REFI);
		if (!newBox) return GF_OUT_OF_MEM;
		((GF_ItemReferenceTypeBox*)newBox)->reference_type = type;
	} else if (parent_type && (parent_type == GF_ISOM_BOX_TYPE_TRGR)) {
		newBox = gf_isom_box_new(GF_ISOM_BOX_TYPE_TRGT);
		if (!newBox) return GF_OUT_OF_MEM;
		((GF_TrackGroupTypeBox*)newBox)->group_type = type;
	} else if (parent_type && (parent_type == GF_ISOM_BOX_TYPE_GRPL)) {
		newBox = gf_isom_box_new(GF_ISOM_BOX_TYPE_GRPT);
		if (!newBox) return GF_OUT_OF_MEM;
		((GF_EntityToGroupTypeBox*)newBox)->grouping_type = type;
	} else {
		//OK, create the box based on the type
		is_special = GF_FALSE;
		newBox = gf_isom_box_new_ex(uuid_type ? uuid_type : type, parent_type, skip_logs, is_root_box);
		if (!newBox) return GF_OUT_OF_MEM;
	}

	//OK, init and read this box
	if (type==GF_ISOM_BOX_TYPE_UUID && !is_special) {
		memcpy(((GF_UUIDBox *)newBox)->uuid, uuid, 16);
		((GF_UUIDBox *)newBox)->internal_4cc = uuid_type;
	}

	if (!newBox->type) newBox->type = type;
	payload_start = gf_bs_get_position(bs);

retry_unknown_box:

	end = gf_bs_available(bs);
	if (size - hdr_size > end ) {
		newBox->size = size - hdr_size - end;
		*outBox = newBox;
		return GF_ISOM_INCOMPLETE_FILE;
	}

	newBox->size = size - hdr_size;

	if (newBox->size) {
		e = gf_isom_full_box_read(newBox, bs);
		if (!e) e = gf_isom_box_read(newBox, bs);
		newBox->size = size;
		end = gf_bs_get_position(bs);
	} else {
		newBox->size = size;
		//empty box
		e = GF_OK;
		end = gf_bs_get_position(bs);
	}

	if (e && (e != GF_ISOM_INCOMPLETE_FILE)) {
		gf_isom_box_del(newBox);
		*outBox = NULL;

		if (parent_type==GF_ISOM_BOX_TYPE_STSD) {
			newBox = gf_isom_box_new(GF_ISOM_BOX_TYPE_UNKNOWN);
			((GF_UnknownBox *)newBox)->original_4cc = type;
			newBox->size = size;
			gf_bs_seek(bs, payload_start);
			goto retry_unknown_box;
		}
		if (!skip_logs) {
			GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[iso file] Read Box \"%s\" (start "LLU") failed (%s) - skipping\n", gf_4cc_to_str(type), start, gf_error_to_string(e)));
		}
		//we don't try to reparse known boxes that have been failing (too dangerous)
		return e;
	}

	if (end-start > size) {
		if (!skip_logs) {
			GF_LOG(GF_LOG_WARNING, GF_LOG_CONTAINER, ("[iso file] Box \"%s\" size "LLU" (start "LLU") invalid (read "LLU")\n", gf_4cc_to_str(type), LLU_CAST size, start, LLU_CAST (end-start) ));
		}
		/*let's still try to load the file since no error was notified*/
		gf_bs_seek(bs, start+size);
	} else if (end-start < size) {
		u32 to_skip = (u32) (size-(end-start));
		if (!skip_logs) {
			if ((to_skip!=4) || gf_bs_peek_bits(bs, 32, 0)) {
				GF_LOG(GF_LOG_WARNING, GF_LOG_CONTAINER, ("[iso file] Box \"%s\" (start "LLU") has %u extra bytes\n", gf_4cc_to_str(type), start, to_skip));
			}
		}
		gf_bs_skip_bytes(bs, to_skip);
	}
	*outBox = newBox;

	return e;
}

fixed potential crash - cf #1406


GF_Err gf_isom_box_parse_ex(GF_Box **outBox, GF_BitStream *bs, u32 parent_type, Bool is_root_box)
{
	u32 type, uuid_type, hdr_size;
	u64 size, start, payload_start, end;
	char uuid[16];
	GF_Err e;
	GF_Box *newBox;
	Bool skip_logs = gf_bs_get_cookie(bs) ? GF_TRUE : GF_FALSE;
	Bool is_special = GF_TRUE;

	if ((bs == NULL) || (outBox == NULL) ) return GF_BAD_PARAM;
	*outBox = NULL;
	if (gf_bs_available(bs) < 8) {
		return GF_ISOM_INCOMPLETE_FILE;
	}

	start = gf_bs_get_position(bs);

	uuid_type = 0;
	size = (u64) gf_bs_read_u32(bs);
	hdr_size = 4;
	/*fix for some boxes found in some old hinted files*/
	if ((size >= 2) && (size <= 4)) {
		size = 4;
		type = GF_ISOM_BOX_TYPE_VOID;
	} else {
		type = gf_bs_read_u32(bs);
		hdr_size += 4;
		/*no size means till end of file - EXCEPT FOR some old QuickTime boxes...*/
		if (type == GF_ISOM_BOX_TYPE_TOTL)
			size = 12;
		if (!size) {
			if (is_root_box) {
				if (!skip_logs) {
					GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[iso file] Warning Read Box type %s (0x%08X) size 0 reading till the end of file\n", gf_4cc_to_str(type), type));
				}
				size = gf_bs_available(bs) + 8;
			} else {
				if (!skip_logs) {
					GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[iso file] Read Box type %s (0x%08X) at position "LLU" has size 0 but is not at root/file level, skipping\n", gf_4cc_to_str(type), type, start));
				}
				return GF_OK;
//				return GF_ISOM_INVALID_FILE;
			}
		}
	}
	/*handle uuid*/
	memset(uuid, 0, 16);
	if (type == GF_ISOM_BOX_TYPE_UUID ) {
		if (gf_bs_available(bs) < 16) {
			return GF_ISOM_INCOMPLETE_FILE;
		}
		gf_bs_read_data(bs, uuid, 16);
		hdr_size += 16;
		uuid_type = gf_isom_solve_uuid_box(uuid);
	}

	//handle large box
	if (size == 1) {
		if (gf_bs_available(bs) < 8) {
			return GF_ISOM_INCOMPLETE_FILE;
		}
		size = gf_bs_read_u64(bs);
		hdr_size += 8;
	}
	GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[iso file] Read Box type %s size "LLD" start "LLD"\n", gf_4cc_to_str(type), LLD_CAST size, LLD_CAST start));

	if ( size < hdr_size ) {
		GF_LOG(GF_LOG_DEBUG, GF_LOG_CONTAINER, ("[iso file] Box size "LLD" less than box header size %d\n", LLD_CAST size, hdr_size));
		return GF_ISOM_INVALID_FILE;
	}

	//some special boxes (references and track groups) are handled by a single generic box with an associated ref/group type
	if (parent_type && (parent_type == GF_ISOM_BOX_TYPE_TREF)) {
		newBox = gf_isom_box_new(GF_ISOM_BOX_TYPE_REFT);
		if (!newBox) return GF_OUT_OF_MEM;
		((GF_TrackReferenceTypeBox*)newBox)->reference_type = type;
	} else if (parent_type && (parent_type == GF_ISOM_BOX_TYPE_IREF)) {
		newBox = gf_isom_box_new(GF_ISOM_BOX_TYPE_REFI);
		if (!newBox) return GF_OUT_OF_MEM;
		((GF_ItemReferenceTypeBox*)newBox)->reference_type = type;
	} else if (parent_type && (parent_type == GF_ISOM_BOX_TYPE_TRGR)) {
		newBox = gf_isom_box_new(GF_ISOM_BOX_TYPE_TRGT);
		if (!newBox) return GF_OUT_OF_MEM;
		((GF_TrackGroupTypeBox*)newBox)->group_type = type;
	} else if (parent_type && (parent_type == GF_ISOM_BOX_TYPE_GRPL)) {
		newBox = gf_isom_box_new(GF_ISOM_BOX_TYPE_GRPT);
		if (!newBox) return GF_OUT_OF_MEM;
		((GF_EntityToGroupTypeBox*)newBox)->grouping_type = type;
	} else {
		//OK, create the box based on the type
		is_special = GF_FALSE;
		newBox = gf_isom_box_new_ex(uuid_type ? uuid_type : type, parent_type, skip_logs, is_root_box);
		if (!newBox) return GF_OUT_OF_MEM;
	}

	//OK, init and read this box
	if (type==GF_ISOM_BOX_TYPE_UUID && !is_special) {
		memcpy(((GF_UUIDBox *)newBox)->uuid, uuid, 16);
		((GF_UUIDBox *)newBox)->internal_4cc = uuid_type;
	}

	if (!newBox->type) newBox->type = type;
	payload_start = gf_bs_get_position(bs);

retry_unknown_box:

	end = gf_bs_available(bs);
	if (size - hdr_size > end ) {
		newBox->size = size - hdr_size - end;
		*outBox = newBox;
		return GF_ISOM_INCOMPLETE_FILE;
	}

	newBox->size = size - hdr_size;

	//parse even if size is 0 - this makes sure that we perform box parsing (usually in box->read)
	e = gf_isom_full_box_read(newBox, bs);
	if (!e) e = gf_isom_box_read(newBox, bs);
	newBox->size = size;
	end = gf_bs_get_position(bs);

	if (e && (e != GF_ISOM_INCOMPLETE_FILE)) {
		gf_isom_box_del(newBox);
		*outBox = NULL;

		if (parent_type==GF_ISOM_BOX_TYPE_STSD) {
			newBox = gf_isom_box_new(GF_ISOM_BOX_TYPE_UNKNOWN);
			((GF_UnknownBox *)newBox)->original_4cc = type;
			newBox->size = size;
			gf_bs_seek(bs, payload_start);
			goto retry_unknown_box;
		}
		if (!skip_logs) {
			GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[iso file] Read Box \"%s\" (start "LLU") failed (%s) - skipping\n", gf_4cc_to_str(type), start, gf_error_to_string(e)));
		}
		//we don't try to reparse known boxes that have been failing (too dangerous)
		return e;
	}

	if (end-start > size) {
		if (!skip_logs) {
			GF_LOG(GF_LOG_WARNING, GF_LOG_CONTAINER, ("[iso file] Box \"%s\" size "LLU" (start "LLU") invalid (read "LLU")\n", gf_4cc_to_str(type), LLU_CAST size, start, LLU_CAST (end-start) ));
		}
		/*let's still try to load the file since no error was notified*/
		gf_bs_seek(bs, start+size);
	} else if (end-start < size) {
		u32 to_skip = (u32) (size-(end-start));
		if (!skip_logs) {
			if ((to_skip!=4) || gf_bs_peek_bits(bs, 32, 0)) {
				GF_LOG(GF_LOG_WARNING, GF_LOG_CONTAINER, ("[iso file] Box \"%s\" (start "LLU") has %u extra bytes\n", gf_4cc_to_str(type), start, to_skip));
			}
		}
		gf_bs_skip_bytes(bs, to_skip);
	}
	*outBox = newBox;

	return e;
}

fixed potential crash - cf #1406


snmp_ber_decode_string_len_buffer(unsigned char *buf, uint32_t *buff_len, const char **str, uint32_t *length)
{
  uint8_t type, i, length_bytes;

  buf = snmp_ber_decode_type(buf, buff_len, &type);

  if(buf == NULL || type != BER_DATA_TYPE_OCTET_STRING) {
    /*
     * Sanity check
     * Invalid type in buffer
     */
    return NULL;
  }

  if((*buf & 0x80) == 0) {
    *length = (uint32_t)*buf++;
    (*buff_len)--;
  } else {

    length_bytes = (uint8_t)(*buf++ & 0x7F);
    (*buff_len)--;
    if(length_bytes > 4) {
      /*
       * Sanity check
       * It will not fit in the uint32_t
       */
      return NULL;
    }

    *length = (uint32_t)*buf++;
    (*buff_len)--;
    for(i = 1; i < length_bytes; ++i) {
      *length <<= 8;
      *length |= *buf++;
      (*buff_len)--;
    }
  }

  *str = (const char *)buf;
  *buff_len -= *length;

  return buf + *length;
}

Refactored SNMP engine after vulnerabilities


snmp_ber_decode_string_len_buffer(snmp_packet_t *snmp_packet, const char **str, uint32_t *length)
{
  uint8_t type, i, length_bytes;

  if(!snmp_ber_decode_type(snmp_packet, &type)) {
    return 0;
  }

  if(type != BER_DATA_TYPE_OCTET_STRING) {
    /*
     * Sanity check
     * Invalid type in buffer
     */
    return 0;
  }

  if((*snmp_packet->in & 0x80) == 0) {

    if(snmp_packet->used == 0) {
      return 0;
    }

    *length = (uint32_t)*snmp_packet->in++;
    snmp_packet->used--;
  } else {

    if(snmp_packet->used == 0) {
      return 0;
    }

    length_bytes = (uint8_t)(*snmp_packet->in++ & 0x7F);
    snmp_packet->used--;

    if(length_bytes > 4) {
      /*
       * Sanity check
       * It will not fit in the uint32_t
       */
      return 0;
    }

    if(snmp_packet->used == 0) {
      return 0;
    }

    *length = (uint32_t)*snmp_packet->in++;
    snmp_packet->used--;

    for(i = 1; i < length_bytes; ++i) {
      *length <<= 8;

      if(snmp_packet->used == 0) {
        return 0;
      }

      *length |= *snmp_packet->in++;
      snmp_packet->used--;
    }
  }

  *str = (const char *)snmp_packet->in;

  if(snmp_packet->used == 0 || snmp_packet->used - *length <= 0) {
    return 0;
  }

  snmp_packet->used -= *length;
  snmp_packet->in += *length;

  return 1;
}


  bool handleBackslash(signed char& out) {
    char ch = *p++;
    switch (ch) {
      case 0: return false;
      case '"': out = ch; return true;
      case '\\': out = ch; return true;
      case '/': out = ch; return true;
      case 'b': out = '\b'; return true;
      case 'f': out = '\f'; return true;
      case 'n': out = '\n'; return true;
      case 'r': out = '\r'; return true;
      case 't': out = '\t'; return true;
      case 'u': {
        if (UNLIKELY(is_tsimplejson)) {
          auto const ch1 = *p++;
          auto const ch2 = *p++;
          auto const dch3 = dehexchar(*p++);
          auto const dch4 = dehexchar(*p++);
          if (UNLIKELY(ch1 != '0' || ch2 != '0' || dch3 < 0 || dch4 < 0)) {
            return false;
          }
          out = (dch3 << 4) | dch4;
          return true;
        } else {
          uint16_t u16cp = 0;
          for (int i = 0; i < 4; i++) {
            auto const hexv = dehexchar(*p++);
            if (hexv < 0) return false; // includes check for end of string
            u16cp <<= 4;
            u16cp |= hexv;
          }
          if (u16cp > 0x7f) {
            return false;
          } else {
            out = u16cp;
            return true;
          }
        }
      }
      default: return false;
    }
  }

Fix buffer overrun in SimpleParser::handleBackslash

Summary:
It read 4 chars, then checked for validity, but any of them could have
been the end of the string, so check after each one instead.

Reviewed By: oulgen

Differential Revision: D19611163

fbshipit-source-id: 3da0a39555cb85a93f4fd98048368f17cf37e2e4


  bool handleBackslash(signed char& out) {
    char ch = *p++;
    switch (ch) {
      case 0: return false;
      case '"': out = ch; return true;
      case '\\': out = ch; return true;
      case '/': out = ch; return true;
      case 'b': out = '\b'; return true;
      case 'f': out = '\f'; return true;
      case 'n': out = '\n'; return true;
      case 'r': out = '\r'; return true;
      case 't': out = '\t'; return true;
      case 'u': {
        if (UNLIKELY(is_tsimplejson)) {
          auto const ch1 = *p++;
          if (UNLIKELY(ch1 != '0')) return false;
          auto const ch2 = *p++;
          if (UNLIKELY(ch2 != '0')) return false;
          auto const dch3 = dehexchar(*p++);
          if (UNLIKELY(dch3 < 0)) return false;
          auto const dch4 = dehexchar(*p++);
          if (UNLIKELY(dch4 < 0)) return false;
          out = (dch3 << 4) | dch4;
          return true;
        } else {
          uint16_t u16cp = 0;
          for (int i = 0; i < 4; i++) {
            auto const hexv = dehexchar(*p++);
            if (hexv < 0) return false; // includes check for end of string
            u16cp <<= 4;
            u16cp |= hexv;
          }
          if (u16cp > 0x7f) {
            return false;
          } else {
            out = u16cp;
            return true;
          }
        }
      }
      default: return false;
    }
  }


void LanLinkProvider::addLink(const QString& deviceId, QSslSocket* socket, NetworkPacket* receivedPacket, LanDeviceLink::ConnectionStarted connectionOrigin)
{
    // Socket disconnection will now be handled by LanDeviceLink
    disconnect(socket, &QAbstractSocket::disconnected, socket, &QObject::deleteLater);

    LanDeviceLink* deviceLink;
    //Do we have a link for this device already?
    QMap< QString, LanDeviceLink* >::iterator linkIterator = m_links.find(deviceId);
    if (linkIterator != m_links.end()) {
        //qCDebug(KDECONNECT_CORE) << "Reusing link to" << deviceId;
        deviceLink = linkIterator.value();
        deviceLink->reset(socket, connectionOrigin);
    } else {
        deviceLink = new LanDeviceLink(deviceId, this, socket, connectionOrigin);
        connect(deviceLink, &QObject::destroyed, this, &LanLinkProvider::deviceLinkDestroyed);
        m_links[deviceId] = deviceLink;
        if (m_pairingHandlers.contains(deviceId)) {
            //We shouldn't have a pairinghandler if we didn't have a link.
            //Crash if debug, recover if release (by setting the new devicelink to the old pairinghandler)
            Q_ASSERT(m_pairingHandlers.contains(deviceId));
            m_pairingHandlers[deviceId]->setDeviceLink(deviceLink);
        }
    }
    Q_EMIT onConnectionReceived(*receivedPacket, deviceLink);
}

Limit number of connected sockets from unpaired devices

Thanks Matthias Gerstner <mgerstner@suse.de> for reporting this.


void LanLinkProvider::addLink(const QString& deviceId, QSslSocket* socket, NetworkPacket* receivedPacket, LanDeviceLink::ConnectionStarted connectionOrigin)
{
    // Socket disconnection will now be handled by LanDeviceLink
    disconnect(socket, &QAbstractSocket::disconnected, socket, &QObject::deleteLater);

    LanDeviceLink* deviceLink;
    //Do we have a link for this device already?
    QMap< QString, LanDeviceLink* >::iterator linkIterator = m_links.find(deviceId);
    if (linkIterator != m_links.end()) {
        //qCDebug(KDECONNECT_CORE) << "Reusing link to" << deviceId;
        deviceLink = linkIterator.value();
        deviceLink->reset(socket, connectionOrigin);
    } else {
        deviceLink = new LanDeviceLink(deviceId, this, socket, connectionOrigin);
        // Socket disconnection will now be handled by LanDeviceLink
        disconnect(socket, &QAbstractSocket::disconnected, socket, &QObject::deleteLater);
        bool isDeviceTrusted = KdeConnectConfig::instance().trustedDevices().contains(deviceId);
        if (!isDeviceTrusted && m_links.size() > MAX_UNPAIRED_CONNECTIONS) {
            qCWarning(KDECONNECT_CORE) << "Too many unpaired devices to remember them all. Ignoring " << deviceId;
            socket->disconnectFromHost();
            socket->deleteLater();
            return;
        }
        connect(deviceLink, &QObject::destroyed, this, &LanLinkProvider::deviceLinkDestroyed);
        m_links[deviceId] = deviceLink;
        if (m_pairingHandlers.contains(deviceId)) {
            //We shouldn't have a pairinghandler if we didn't have a link.
            //Crash if debug, recover if release (by setting the new devicelink to the old pairinghandler)
            Q_ASSERT(m_pairingHandlers.contains(deviceId));
            m_pairingHandlers[deviceId]->setDeviceLink(deviceLink);
        }
    }
    Q_EMIT onConnectionReceived(*receivedPacket, deviceLink);
}


_libssh2_packet_add(LIBSSH2_SESSION * session, unsigned char *data,
                    size_t datalen, int macstate)
{
    int rc = 0;
    char *message = NULL;
    char *language = NULL;
    size_t message_len = 0;
    size_t language_len = 0;
    LIBSSH2_CHANNEL *channelp = NULL;
    size_t data_head = 0;
    unsigned char msg = data[0];

    switch(session->packAdd_state) {
    case libssh2_NB_state_idle:
        _libssh2_debug(session, LIBSSH2_TRACE_TRANS,
                       "Packet type %d received, length=%d",
                       (int) msg, (int) datalen);

        if((macstate == LIBSSH2_MAC_INVALID) &&
            (!session->macerror ||
             LIBSSH2_MACERROR(session, (char *) data, datalen))) {
            /* Bad MAC input, but no callback set or non-zero return from the
               callback */

            LIBSSH2_FREE(session, data);
            return _libssh2_error(session, LIBSSH2_ERROR_INVALID_MAC,
                                  "Invalid MAC received");
        }
        session->packAdd_state = libssh2_NB_state_allocated;
        break;
    case libssh2_NB_state_jump1:
        goto libssh2_packet_add_jump_point1;
    case libssh2_NB_state_jump2:
        goto libssh2_packet_add_jump_point2;
    case libssh2_NB_state_jump3:
        goto libssh2_packet_add_jump_point3;
    case libssh2_NB_state_jump4:
        goto libssh2_packet_add_jump_point4;
    case libssh2_NB_state_jump5:
        goto libssh2_packet_add_jump_point5;
    default: /* nothing to do */
        break;
    }

    if(session->packAdd_state == libssh2_NB_state_allocated) {
        /* A couple exceptions to the packet adding rule: */
        switch(msg) {

            /*
              byte      SSH_MSG_DISCONNECT
              uint32    reason code
              string    description in ISO-10646 UTF-8 encoding [RFC3629]
              string    language tag [RFC3066]
            */

        case SSH_MSG_DISCONNECT:
            if(datalen >= 5) {
                size_t reason = _libssh2_ntohu32(data + 1);

                if(datalen >= 9) {
                    message_len = _libssh2_ntohu32(data + 5);

                    if(message_len < datalen-13) {
                        /* 9 = packet_type(1) + reason(4) + message_len(4) */
                        message = (char *) data + 9;

                        language_len =
                            _libssh2_ntohu32(data + 9 + message_len);
                        language = (char *) data + 9 + message_len + 4;

                        if(language_len > (datalen-13-message_len)) {
                            /* bad input, clear info */
                            language = message = NULL;
                            language_len = message_len = 0;
                        }
                    }
                    else
                        /* bad size, clear it */
                        message_len = 0;
                }
                if(session->ssh_msg_disconnect) {
                    LIBSSH2_DISCONNECT(session, reason, message,
                                       message_len, language, language_len);
                }
                _libssh2_debug(session, LIBSSH2_TRACE_TRANS,
                               "Disconnect(%d): %s(%s)", reason,
                               message, language);
            }

            LIBSSH2_FREE(session, data);
            session->socket_state = LIBSSH2_SOCKET_DISCONNECTED;
            session->packAdd_state = libssh2_NB_state_idle;
            return _libssh2_error(session, LIBSSH2_ERROR_SOCKET_DISCONNECT,
                                  "socket disconnect");
            /*
              byte      SSH_MSG_IGNORE
              string    data
            */

        case SSH_MSG_IGNORE:
            if(datalen >= 2) {
                if(session->ssh_msg_ignore) {
                    LIBSSH2_IGNORE(session, (char *) data + 1, datalen - 1);
                }
            }
            else if(session->ssh_msg_ignore) {
                LIBSSH2_IGNORE(session, "", 0);
            }
            LIBSSH2_FREE(session, data);
            session->packAdd_state = libssh2_NB_state_idle;
            return 0;

            /*
              byte      SSH_MSG_DEBUG
              boolean   always_display
              string    message in ISO-10646 UTF-8 encoding [RFC3629]
              string    language tag [RFC3066]
            */

        case SSH_MSG_DEBUG:
            if(datalen >= 2) {
                int always_display = data[1];

                if(datalen >= 6) {
                    message_len = _libssh2_ntohu32(data + 2);

                    if(message_len <= (datalen - 10)) {
                        /* 6 = packet_type(1) + display(1) + message_len(4) */
                        message = (char *) data + 6;
                        language_len = _libssh2_ntohu32(data + 6 +
                                                        message_len);

                        if(language_len <= (datalen - 10 - message_len))
                            language = (char *) data + 10 + message_len;
                    }
                }

                if(session->ssh_msg_debug) {
                    LIBSSH2_DEBUG(session, always_display, message,
                                  message_len, language, language_len);
                }
            }
            /*
             * _libssh2_debug will actually truncate this for us so
             * that it's not an inordinate about of data
             */
            _libssh2_debug(session, LIBSSH2_TRACE_TRANS,
                           "Debug Packet: %s", message);
            LIBSSH2_FREE(session, data);
            session->packAdd_state = libssh2_NB_state_idle;
            return 0;

            /*
              byte      SSH_MSG_GLOBAL_REQUEST
              string    request name in US-ASCII only
              boolean   want reply
              ....      request-specific data follows
            */

        case SSH_MSG_GLOBAL_REQUEST:
            if(datalen >= 5) {
                uint32_t len = 0;
                unsigned char want_reply = 0;
                len = _libssh2_ntohu32(data + 1);
                if(datalen >= (6 + len)) {
                    want_reply = data[5 + len];
                    _libssh2_debug(session,
                                   LIBSSH2_TRACE_CONN,
                                   "Received global request type %.*s (wr %X)",
                                   len, data + 5, want_reply);
                }

                if(want_reply) {
                    static const unsigned char packet =
                        SSH_MSG_REQUEST_FAILURE;
                  libssh2_packet_add_jump_point5:
                    session->packAdd_state = libssh2_NB_state_jump5;
                    rc = _libssh2_transport_send(session, &packet, 1, NULL, 0);
                    if(rc == LIBSSH2_ERROR_EAGAIN)
                        return rc;
                }
            }
            LIBSSH2_FREE(session, data);
            session->packAdd_state = libssh2_NB_state_idle;
            return 0;

            /*
              byte      SSH_MSG_CHANNEL_EXTENDED_DATA
              uint32    recipient channel
              uint32    data_type_code
              string    data
            */

        case SSH_MSG_CHANNEL_EXTENDED_DATA:
            /* streamid(4) */
            data_head += 4;

            /* fall-through */

            /*
              byte      SSH_MSG_CHANNEL_DATA
              uint32    recipient channel
              string    data
            */

        case SSH_MSG_CHANNEL_DATA:
            /* packet_type(1) + channelno(4) + datalen(4) */
            data_head += 9;

            if(datalen >= data_head)
                channelp =
                    _libssh2_channel_locate(session,
                                            _libssh2_ntohu32(data + 1));

            if(!channelp) {
                _libssh2_error(session, LIBSSH2_ERROR_CHANNEL_UNKNOWN,
                               "Packet received for unknown channel");
                LIBSSH2_FREE(session, data);
                session->packAdd_state = libssh2_NB_state_idle;
                return 0;
            }
#ifdef LIBSSH2DEBUG
            {
                uint32_t stream_id = 0;
                if(msg == SSH_MSG_CHANNEL_EXTENDED_DATA)
                    stream_id = _libssh2_ntohu32(data + 5);

                _libssh2_debug(session, LIBSSH2_TRACE_CONN,
                               "%d bytes packet_add() for %lu/%lu/%lu",
                               (int) (datalen - data_head),
                               channelp->local.id,
                               channelp->remote.id,
                               stream_id);
            }
#endif
            if((channelp->remote.extended_data_ignore_mode ==
                 LIBSSH2_CHANNEL_EXTENDED_DATA_IGNORE) &&
                (msg == SSH_MSG_CHANNEL_EXTENDED_DATA)) {
                /* Pretend we didn't receive this */
                LIBSSH2_FREE(session, data);

                _libssh2_debug(session, LIBSSH2_TRACE_CONN,
                               "Ignoring extended data and refunding %d bytes",
                               (int) (datalen - 13));
                if(channelp->read_avail + datalen - data_head >=
                    channelp->remote.window_size)
                    datalen = channelp->remote.window_size -
                        channelp->read_avail + data_head;

                channelp->remote.window_size -= datalen - data_head;
                _libssh2_debug(session, LIBSSH2_TRACE_CONN,
                               "shrinking window size by %lu bytes to %lu, "
                               "read_avail %lu",
                               datalen - data_head,
                               channelp->remote.window_size,
                               channelp->read_avail);

                session->packAdd_channelp = channelp;

                /* Adjust the window based on the block we just freed */
              libssh2_packet_add_jump_point1:
                session->packAdd_state = libssh2_NB_state_jump1;
                rc = _libssh2_channel_receive_window_adjust(session->
                                                            packAdd_channelp,
                                                            datalen - 13,
                                                            1, NULL);
                if(rc == LIBSSH2_ERROR_EAGAIN)
                    return rc;

                session->packAdd_state = libssh2_NB_state_idle;
                return 0;
            }

            /*
             * REMEMBER! remote means remote as source of data,
             * NOT remote window!
             */
            if(channelp->remote.packet_size < (datalen - data_head)) {
                /*
                 * Spec says we MAY ignore bytes sent beyond
                 * packet_size
                 */
                _libssh2_error(session,
                               LIBSSH2_ERROR_CHANNEL_PACKET_EXCEEDED,
                               "Packet contains more data than we offered"
                               " to receive, truncating");
                datalen = channelp->remote.packet_size + data_head;
            }
            if(channelp->remote.window_size <= channelp->read_avail) {
                /*
                 * Spec says we MAY ignore bytes sent beyond
                 * window_size
                 */
                _libssh2_error(session,
                               LIBSSH2_ERROR_CHANNEL_WINDOW_EXCEEDED,
                               "The current receive window is full,"
                               " data ignored");
                LIBSSH2_FREE(session, data);
                session->packAdd_state = libssh2_NB_state_idle;
                return 0;
            }
            /* Reset EOF status */
            channelp->remote.eof = 0;

            if(channelp->read_avail + datalen - data_head >
                channelp->remote.window_size) {
                _libssh2_error(session,
                               LIBSSH2_ERROR_CHANNEL_WINDOW_EXCEEDED,
                               "Remote sent more data than current "
                               "window allows, truncating");
                datalen = channelp->remote.window_size -
                    channelp->read_avail + data_head;
            }

            /* Update the read_avail counter. The window size will be
             * updated once the data is actually read from the queue
             * from an upper layer */
            channelp->read_avail += datalen - data_head;

            _libssh2_debug(session, LIBSSH2_TRACE_CONN,
                           "increasing read_avail by %lu bytes to %lu/%lu",
                           (long)(datalen - data_head),
                           (long)channelp->read_avail,
                           (long)channelp->remote.window_size);

            break;

            /*
              byte      SSH_MSG_CHANNEL_EOF
              uint32    recipient channel
            */

        case SSH_MSG_CHANNEL_EOF:
            if(datalen >= 5)
                channelp =
                    _libssh2_channel_locate(session,
                                            _libssh2_ntohu32(data + 1));
            if(!channelp)
                /* We may have freed already, just quietly ignore this... */
                ;
            else {
                _libssh2_debug(session,
                               LIBSSH2_TRACE_CONN,
                               "EOF received for channel %lu/%lu",
                               channelp->local.id,
                               channelp->remote.id);
                channelp->remote.eof = 1;
            }
            LIBSSH2_FREE(session, data);
            session->packAdd_state = libssh2_NB_state_idle;
            return 0;

            /*
              byte      SSH_MSG_CHANNEL_REQUEST
              uint32    recipient channel
              string    request type in US-ASCII characters only
              boolean   want reply
              ....      type-specific data follows
            */

        case SSH_MSG_CHANNEL_REQUEST:
            if(datalen >= 9) {
                uint32_t channel = _libssh2_ntohu32(data + 1);
                uint32_t len = _libssh2_ntohu32(data + 5);
                unsigned char want_reply = 1;

                if((len + 9) < datalen)
                    want_reply = data[len + 9];

                _libssh2_debug(session,
                               LIBSSH2_TRACE_CONN,
                               "Channel %d received request type %.*s (wr %X)",
                               channel, len, data + 9, want_reply);

                if(len == sizeof("exit-status") - 1
                    && (sizeof("exit-status") - 1 + 9) <= datalen
                    && !memcmp("exit-status", data + 9,
                               sizeof("exit-status") - 1)) {

                    /* we've got "exit-status" packet. Set the session value */
                    if(datalen >= 20)
                        channelp =
                            _libssh2_channel_locate(session, channel);

                    if(channelp && (sizeof("exit-status") + 13) <= datalen) {
                        channelp->exit_status =
                            _libssh2_ntohu32(data + 9 + sizeof("exit-status"));
                        _libssh2_debug(session, LIBSSH2_TRACE_CONN,
                                       "Exit status %lu received for "
                                       "channel %lu/%lu",
                                       channelp->exit_status,
                                       channelp->local.id,
                                       channelp->remote.id);
                    }

                }
                else if(len == sizeof("exit-signal") - 1
                         && (sizeof("exit-signal") - 1 + 9) <= datalen
                         && !memcmp("exit-signal", data + 9,
                                    sizeof("exit-signal") - 1)) {
                    /* command terminated due to signal */
                    if(datalen >= 20)
                        channelp = _libssh2_channel_locate(session, channel);

                    if(channelp && (sizeof("exit-signal") + 13) <= datalen) {
                        /* set signal name (without SIG prefix) */
                        uint32_t namelen =
                            _libssh2_ntohu32(data + 9 + sizeof("exit-signal"));

                        if(namelen <= UINT_MAX - 1) {
                            channelp->exit_signal =
                                LIBSSH2_ALLOC(session, namelen + 1);
                        }
                        else {
                            channelp->exit_signal = NULL;
                        }

                        if(!channelp->exit_signal)
                            rc = _libssh2_error(session, LIBSSH2_ERROR_ALLOC,
                                                "memory for signal name");
                        else if((sizeof("exit-signal") + 13 + namelen <=
                                 datalen)) {
                            memcpy(channelp->exit_signal,
                                   data + 13 + sizeof("exit-signal"), namelen);
                            channelp->exit_signal[namelen] = '\0';
                            /* TODO: save error message and language tag */
                            _libssh2_debug(session, LIBSSH2_TRACE_CONN,
                                           "Exit signal %s received for "
                                           "channel %lu/%lu",
                                           channelp->exit_signal,
                                           channelp->local.id,
                                           channelp->remote.id);
                        }
                    }
                }

                if(want_reply) {
                    unsigned char packet[5];
                  libssh2_packet_add_jump_point4:
                    session->packAdd_state = libssh2_NB_state_jump4;
                    packet[0] = SSH_MSG_CHANNEL_FAILURE;
                    memcpy(&packet[1], data + 1, 4);
                    rc = _libssh2_transport_send(session, packet, 5, NULL, 0);
                    if(rc == LIBSSH2_ERROR_EAGAIN)
                        return rc;
                }
            }
            LIBSSH2_FREE(session, data);
            session->packAdd_state = libssh2_NB_state_idle;
            return rc;

            /*
              byte      SSH_MSG_CHANNEL_CLOSE
              uint32    recipient channel
            */

        case SSH_MSG_CHANNEL_CLOSE:
            if(datalen >= 5)
                channelp =
                    _libssh2_channel_locate(session,
                                            _libssh2_ntohu32(data + 1));
            if(!channelp) {
                /* We may have freed already, just quietly ignore this... */
                LIBSSH2_FREE(session, data);
                session->packAdd_state = libssh2_NB_state_idle;
                return 0;
            }
            _libssh2_debug(session, LIBSSH2_TRACE_CONN,
                           "Close received for channel %lu/%lu",
                           channelp->local.id,
                           channelp->remote.id);

            channelp->remote.close = 1;
            channelp->remote.eof = 1;

            LIBSSH2_FREE(session, data);
            session->packAdd_state = libssh2_NB_state_idle;
            return 0;

            /*
              byte      SSH_MSG_CHANNEL_OPEN
              string    "session"
              uint32    sender channel
              uint32    initial window size
              uint32    maximum packet size
            */

        case SSH_MSG_CHANNEL_OPEN:
            if(datalen < 17)
                ;
            else if((datalen >= (sizeof("forwarded-tcpip") + 4)) &&
                     ((sizeof("forwarded-tcpip") - 1) ==
                      _libssh2_ntohu32(data + 1))
                     &&
                     (memcmp(data + 5, "forwarded-tcpip",
                             sizeof("forwarded-tcpip") - 1) == 0)) {

                /* init the state struct */
                memset(&session->packAdd_Qlstn_state, 0,
                       sizeof(session->packAdd_Qlstn_state));

              libssh2_packet_add_jump_point2:
                session->packAdd_state = libssh2_NB_state_jump2;
                rc = packet_queue_listener(session, data, datalen,
                                           &session->packAdd_Qlstn_state);
            }
            else if((datalen >= (sizeof("x11") + 4)) &&
                     ((sizeof("x11") - 1) == _libssh2_ntohu32(data + 1)) &&
                     (memcmp(data + 5, "x11", sizeof("x11") - 1) == 0)) {

                /* init the state struct */
                memset(&session->packAdd_x11open_state, 0,
                       sizeof(session->packAdd_x11open_state));

              libssh2_packet_add_jump_point3:
                session->packAdd_state = libssh2_NB_state_jump3;
                rc = packet_x11_open(session, data, datalen,
                                     &session->packAdd_x11open_state);
            }
            if(rc == LIBSSH2_ERROR_EAGAIN)
                return rc;

            LIBSSH2_FREE(session, data);
            session->packAdd_state = libssh2_NB_state_idle;
            return rc;

            /*
              byte      SSH_MSG_CHANNEL_WINDOW_ADJUST
              uint32    recipient channel
              uint32    bytes to add
            */
        case SSH_MSG_CHANNEL_WINDOW_ADJUST:
            if(datalen < 9)
                ;
            else {
                uint32_t bytestoadd = _libssh2_ntohu32(data + 5);
                channelp =
                    _libssh2_channel_locate(session,
                                            _libssh2_ntohu32(data + 1));
                if(channelp) {
                    channelp->local.window_size += bytestoadd;

                    _libssh2_debug(session, LIBSSH2_TRACE_CONN,
                                   "Window adjust for channel %lu/%lu, "
                                   "adding %lu bytes, new window_size=%lu",
                                   channelp->local.id,
                                   channelp->remote.id,
                                   bytestoadd,
                                   channelp->local.window_size);
                }
            }
            LIBSSH2_FREE(session, data);
            session->packAdd_state = libssh2_NB_state_idle;
            return 0;
        default:
            break;
        }

        session->packAdd_state = libssh2_NB_state_sent;
    }

    if(session->packAdd_state == libssh2_NB_state_sent) {
        LIBSSH2_PACKET *packetp =
            LIBSSH2_ALLOC(session, sizeof(LIBSSH2_PACKET));
        if(!packetp) {
            _libssh2_debug(session, LIBSSH2_ERROR_ALLOC,
                           "memory for packet");
            LIBSSH2_FREE(session, data);
            session->packAdd_state = libssh2_NB_state_idle;
            return LIBSSH2_ERROR_ALLOC;
        }
        packetp->data = data;
        packetp->data_len = datalen;
        packetp->data_head = data_head;

        _libssh2_list_add(&session->packets, &packetp->node);

        session->packAdd_state = libssh2_NB_state_sent1;
    }

    if((msg == SSH_MSG_KEXINIT &&
         !(session->state & LIBSSH2_STATE_EXCHANGING_KEYS)) ||
        (session->packAdd_state == libssh2_NB_state_sent2)) {
        if(session->packAdd_state == libssh2_NB_state_sent1) {
            /*
             * Remote wants new keys
             * Well, it's already in the brigade,
             * let's just call back into ourselves
             */
            _libssh2_debug(session, LIBSSH2_TRACE_TRANS, "Renegotiating Keys");

            session->packAdd_state = libssh2_NB_state_sent2;
        }

        /*
         * The KEXINIT message has been added to the queue.  The packAdd and
         * readPack states need to be reset because _libssh2_kex_exchange
         * (eventually) calls upon _libssh2_transport_read to read the rest of
         * the key exchange conversation.
         */
        session->readPack_state = libssh2_NB_state_idle;
        session->packet.total_num = 0;
        session->packAdd_state = libssh2_NB_state_idle;
        session->fullpacket_state = libssh2_NB_state_idle;

        memset(&session->startup_key_state, 0, sizeof(key_exchange_state_t));

        /*
         * If there was a key reexchange failure, let's just hope we didn't
         * send NEWKEYS yet, otherwise remote will drop us like a rock
         */
        rc = _libssh2_kex_exchange(session, 1, &session->startup_key_state);
        if(rc == LIBSSH2_ERROR_EAGAIN)
            return rc;
    }

    session->packAdd_state = libssh2_NB_state_idle;
    return 0;
}

packet.c: improve message parsing (#402)

* packet.c: improve parsing of packets

file: packet.c

notes:
Use _libssh2_get_string API in SSH_MSG_DEBUG/SSH_MSG_DISCONNECT. Additional uint32 bounds check in SSH_MSG_GLOBAL_REQUEST.


_libssh2_packet_add(LIBSSH2_SESSION * session, unsigned char *data,
                    size_t datalen, int macstate)
{
    int rc = 0;
    unsigned char *message = NULL;
    unsigned char *language = NULL;
    size_t message_len = 0;
    size_t language_len = 0;
    LIBSSH2_CHANNEL *channelp = NULL;
    size_t data_head = 0;
    unsigned char msg = data[0];

    switch(session->packAdd_state) {
    case libssh2_NB_state_idle:
        _libssh2_debug(session, LIBSSH2_TRACE_TRANS,
                       "Packet type %d received, length=%d",
                       (int) msg, (int) datalen);

        if((macstate == LIBSSH2_MAC_INVALID) &&
            (!session->macerror ||
             LIBSSH2_MACERROR(session, (char *) data, datalen))) {
            /* Bad MAC input, but no callback set or non-zero return from the
               callback */

            LIBSSH2_FREE(session, data);
            return _libssh2_error(session, LIBSSH2_ERROR_INVALID_MAC,
                                  "Invalid MAC received");
        }
        session->packAdd_state = libssh2_NB_state_allocated;
        break;
    case libssh2_NB_state_jump1:
        goto libssh2_packet_add_jump_point1;
    case libssh2_NB_state_jump2:
        goto libssh2_packet_add_jump_point2;
    case libssh2_NB_state_jump3:
        goto libssh2_packet_add_jump_point3;
    case libssh2_NB_state_jump4:
        goto libssh2_packet_add_jump_point4;
    case libssh2_NB_state_jump5:
        goto libssh2_packet_add_jump_point5;
    default: /* nothing to do */
        break;
    }

    if(session->packAdd_state == libssh2_NB_state_allocated) {
        /* A couple exceptions to the packet adding rule: */
        switch(msg) {

            /*
              byte      SSH_MSG_DISCONNECT
              uint32    reason code
              string    description in ISO-10646 UTF-8 encoding [RFC3629]
              string    language tag [RFC3066]
            */

        case SSH_MSG_DISCONNECT:
            if(datalen >= 5) {
                uint32_t reason = 0;
                struct string_buf buf;
                buf.data = (unsigned char *)data;
                buf.dataptr = buf.data;
                buf.len = datalen;
                buf.dataptr++; /* advance past type */

                _libssh2_get_u32(&buf, &reason);
                _libssh2_get_string(&buf, &message, &message_len);
                _libssh2_get_string(&buf, &language, &language_len);

                if(session->ssh_msg_disconnect) {
                    LIBSSH2_DISCONNECT(session, reason, (const char *)message,
                                       message_len, (const char *)language,
                                       language_len);
                }

                _libssh2_debug(session, LIBSSH2_TRACE_TRANS,
                               "Disconnect(%d): %s(%s)", reason,
                               message, language);
            }

            LIBSSH2_FREE(session, data);
            session->socket_state = LIBSSH2_SOCKET_DISCONNECTED;
            session->packAdd_state = libssh2_NB_state_idle;
            return _libssh2_error(session, LIBSSH2_ERROR_SOCKET_DISCONNECT,
                                  "socket disconnect");
            /*
              byte      SSH_MSG_IGNORE
              string    data
            */

        case SSH_MSG_IGNORE:
            if(datalen >= 2) {
                if(session->ssh_msg_ignore) {
                    LIBSSH2_IGNORE(session, (char *) data + 1, datalen - 1);
                }
            }
            else if(session->ssh_msg_ignore) {
                LIBSSH2_IGNORE(session, "", 0);
            }
            LIBSSH2_FREE(session, data);
            session->packAdd_state = libssh2_NB_state_idle;
            return 0;

            /*
              byte      SSH_MSG_DEBUG
              boolean   always_display
              string    message in ISO-10646 UTF-8 encoding [RFC3629]
              string    language tag [RFC3066]
            */

        case SSH_MSG_DEBUG:
            if(datalen >= 2) {
                int always_display = data[1];

                if(datalen >= 6) {
                    struct string_buf buf;
                    buf.data = (unsigned char *)data;
                    buf.dataptr = buf.data;
                    buf.len = datalen;
                    buf.dataptr += 2; /* advance past type & always display */

                    _libssh2_get_string(&buf, &message, &message_len);
                    _libssh2_get_string(&buf, &language, &language_len);
                }

                if(session->ssh_msg_debug) {
                    LIBSSH2_DEBUG(session, always_display,
                                  (const char *)message,
                                  message_len, (const char *)language,
                                  language_len);
                }
            }

            /*
             * _libssh2_debug will actually truncate this for us so
             * that it's not an inordinate about of data
             */
            _libssh2_debug(session, LIBSSH2_TRACE_TRANS,
                           "Debug Packet: %s", message);
            LIBSSH2_FREE(session, data);
            session->packAdd_state = libssh2_NB_state_idle;
            return 0;

            /*
              byte      SSH_MSG_GLOBAL_REQUEST
              string    request name in US-ASCII only
              boolean   want reply
              ....      request-specific data follows
            */

        case SSH_MSG_GLOBAL_REQUEST:
            if(datalen >= 5) {
                uint32_t len = 0;
                unsigned char want_reply = 0;
                len = _libssh2_ntohu32(data + 1);
                if((len <= (UINT_MAX - 6)) && (datalen >= (6 + len))) {
                    want_reply = data[5 + len];
                    _libssh2_debug(session,
                                   LIBSSH2_TRACE_CONN,
                                   "Received global request type %.*s (wr %X)",
                                   len, data + 5, want_reply);
                }

                if(want_reply) {
                    static const unsigned char packet =
                        SSH_MSG_REQUEST_FAILURE;
                  libssh2_packet_add_jump_point5:
                    session->packAdd_state = libssh2_NB_state_jump5;
                    rc = _libssh2_transport_send(session, &packet, 1, NULL, 0);
                    if(rc == LIBSSH2_ERROR_EAGAIN)
                        return rc;
                }
            }
            LIBSSH2_FREE(session, data);
            session->packAdd_state = libssh2_NB_state_idle;
            return 0;

            /*
              byte      SSH_MSG_CHANNEL_EXTENDED_DATA
              uint32    recipient channel
              uint32    data_type_code
              string    data
            */

        case SSH_MSG_CHANNEL_EXTENDED_DATA:
            /* streamid(4) */
            data_head += 4;

            /* fall-through */

            /*
              byte      SSH_MSG_CHANNEL_DATA
              uint32    recipient channel
              string    data
            */

        case SSH_MSG_CHANNEL_DATA:
            /* packet_type(1) + channelno(4) + datalen(4) */
            data_head += 9;

            if(datalen >= data_head)
                channelp =
                    _libssh2_channel_locate(session,
                                            _libssh2_ntohu32(data + 1));

            if(!channelp) {
                _libssh2_error(session, LIBSSH2_ERROR_CHANNEL_UNKNOWN,
                               "Packet received for unknown channel");
                LIBSSH2_FREE(session, data);
                session->packAdd_state = libssh2_NB_state_idle;
                return 0;
            }
#ifdef LIBSSH2DEBUG
            {
                uint32_t stream_id = 0;
                if(msg == SSH_MSG_CHANNEL_EXTENDED_DATA)
                    stream_id = _libssh2_ntohu32(data + 5);

                _libssh2_debug(session, LIBSSH2_TRACE_CONN,
                               "%d bytes packet_add() for %lu/%lu/%lu",
                               (int) (datalen - data_head),
                               channelp->local.id,
                               channelp->remote.id,
                               stream_id);
            }
#endif
            if((channelp->remote.extended_data_ignore_mode ==
                 LIBSSH2_CHANNEL_EXTENDED_DATA_IGNORE) &&
                (msg == SSH_MSG_CHANNEL_EXTENDED_DATA)) {
                /* Pretend we didn't receive this */
                LIBSSH2_FREE(session, data);

                _libssh2_debug(session, LIBSSH2_TRACE_CONN,
                               "Ignoring extended data and refunding %d bytes",
                               (int) (datalen - 13));
                if(channelp->read_avail + datalen - data_head >=
                    channelp->remote.window_size)
                    datalen = channelp->remote.window_size -
                        channelp->read_avail + data_head;

                channelp->remote.window_size -= datalen - data_head;
                _libssh2_debug(session, LIBSSH2_TRACE_CONN,
                               "shrinking window size by %lu bytes to %lu, "
                               "read_avail %lu",
                               datalen - data_head,
                               channelp->remote.window_size,
                               channelp->read_avail);

                session->packAdd_channelp = channelp;

                /* Adjust the window based on the block we just freed */
              libssh2_packet_add_jump_point1:
                session->packAdd_state = libssh2_NB_state_jump1;
                rc = _libssh2_channel_receive_window_adjust(session->
                                                            packAdd_channelp,
                                                            datalen - 13,
                                                            1, NULL);
                if(rc == LIBSSH2_ERROR_EAGAIN)
                    return rc;

                session->packAdd_state = libssh2_NB_state_idle;
                return 0;
            }

            /*
             * REMEMBER! remote means remote as source of data,
             * NOT remote window!
             */
            if(channelp->remote.packet_size < (datalen - data_head)) {
                /*
                 * Spec says we MAY ignore bytes sent beyond
                 * packet_size
                 */
                _libssh2_error(session,
                               LIBSSH2_ERROR_CHANNEL_PACKET_EXCEEDED,
                               "Packet contains more data than we offered"
                               " to receive, truncating");
                datalen = channelp->remote.packet_size + data_head;
            }
            if(channelp->remote.window_size <= channelp->read_avail) {
                /*
                 * Spec says we MAY ignore bytes sent beyond
                 * window_size
                 */
                _libssh2_error(session,
                               LIBSSH2_ERROR_CHANNEL_WINDOW_EXCEEDED,
                               "The current receive window is full,"
                               " data ignored");
                LIBSSH2_FREE(session, data);
                session->packAdd_state = libssh2_NB_state_idle;
                return 0;
            }
            /* Reset EOF status */
            channelp->remote.eof = 0;

            if(channelp->read_avail + datalen - data_head >
                channelp->remote.window_size) {
                _libssh2_error(session,
                               LIBSSH2_ERROR_CHANNEL_WINDOW_EXCEEDED,
                               "Remote sent more data than current "
                               "window allows, truncating");
                datalen = channelp->remote.window_size -
                    channelp->read_avail + data_head;
            }

            /* Update the read_avail counter. The window size will be
             * updated once the data is actually read from the queue
             * from an upper layer */
            channelp->read_avail += datalen - data_head;

            _libssh2_debug(session, LIBSSH2_TRACE_CONN,
                           "increasing read_avail by %lu bytes to %lu/%lu",
                           (long)(datalen - data_head),
                           (long)channelp->read_avail,
                           (long)channelp->remote.window_size);

            break;

            /*
              byte      SSH_MSG_CHANNEL_EOF
              uint32    recipient channel
            */

        case SSH_MSG_CHANNEL_EOF:
            if(datalen >= 5)
                channelp =
                    _libssh2_channel_locate(session,
                                            _libssh2_ntohu32(data + 1));
            if(!channelp)
                /* We may have freed already, just quietly ignore this... */
                ;
            else {
                _libssh2_debug(session,
                               LIBSSH2_TRACE_CONN,
                               "EOF received for channel %lu/%lu",
                               channelp->local.id,
                               channelp->remote.id);
                channelp->remote.eof = 1;
            }
            LIBSSH2_FREE(session, data);
            session->packAdd_state = libssh2_NB_state_idle;
            return 0;

            /*
              byte      SSH_MSG_CHANNEL_REQUEST
              uint32    recipient channel
              string    request type in US-ASCII characters only
              boolean   want reply
              ....      type-specific data follows
            */

        case SSH_MSG_CHANNEL_REQUEST:
            if(datalen >= 9) {
                uint32_t channel = _libssh2_ntohu32(data + 1);
                uint32_t len = _libssh2_ntohu32(data + 5);
                unsigned char want_reply = 1;

                if((len + 9) < datalen)
                    want_reply = data[len + 9];

                _libssh2_debug(session,
                               LIBSSH2_TRACE_CONN,
                               "Channel %d received request type %.*s (wr %X)",
                               channel, len, data + 9, want_reply);

                if(len == sizeof("exit-status") - 1
                    && (sizeof("exit-status") - 1 + 9) <= datalen
                    && !memcmp("exit-status", data + 9,
                               sizeof("exit-status") - 1)) {

                    /* we've got "exit-status" packet. Set the session value */
                    if(datalen >= 20)
                        channelp =
                            _libssh2_channel_locate(session, channel);

                    if(channelp && (sizeof("exit-status") + 13) <= datalen) {
                        channelp->exit_status =
                            _libssh2_ntohu32(data + 9 + sizeof("exit-status"));
                        _libssh2_debug(session, LIBSSH2_TRACE_CONN,
                                       "Exit status %lu received for "
                                       "channel %lu/%lu",
                                       channelp->exit_status,
                                       channelp->local.id,
                                       channelp->remote.id);
                    }

                }
                else if(len == sizeof("exit-signal") - 1
                         && (sizeof("exit-signal") - 1 + 9) <= datalen
                         && !memcmp("exit-signal", data + 9,
                                    sizeof("exit-signal") - 1)) {
                    /* command terminated due to signal */
                    if(datalen >= 20)
                        channelp = _libssh2_channel_locate(session, channel);

                    if(channelp && (sizeof("exit-signal") + 13) <= datalen) {
                        /* set signal name (without SIG prefix) */
                        uint32_t namelen =
                            _libssh2_ntohu32(data + 9 + sizeof("exit-signal"));

                        if(namelen <= UINT_MAX - 1) {
                            channelp->exit_signal =
                                LIBSSH2_ALLOC(session, namelen + 1);
                        }
                        else {
                            channelp->exit_signal = NULL;
                        }

                        if(!channelp->exit_signal)
                            rc = _libssh2_error(session, LIBSSH2_ERROR_ALLOC,
                                                "memory for signal name");
                        else if((sizeof("exit-signal") + 13 + namelen <=
                                 datalen)) {
                            memcpy(channelp->exit_signal,
                                   data + 13 + sizeof("exit-signal"), namelen);
                            channelp->exit_signal[namelen] = '\0';
                            /* TODO: save error message and language tag */
                            _libssh2_debug(session, LIBSSH2_TRACE_CONN,
                                           "Exit signal %s received for "
                                           "channel %lu/%lu",
                                           channelp->exit_signal,
                                           channelp->local.id,
                                           channelp->remote.id);
                        }
                    }
                }

                if(want_reply) {
                    unsigned char packet[5];
                  libssh2_packet_add_jump_point4:
                    session->packAdd_state = libssh2_NB_state_jump4;
                    packet[0] = SSH_MSG_CHANNEL_FAILURE;
                    memcpy(&packet[1], data + 1, 4);
                    rc = _libssh2_transport_send(session, packet, 5, NULL, 0);
                    if(rc == LIBSSH2_ERROR_EAGAIN)
                        return rc;
                }
            }
            LIBSSH2_FREE(session, data);
            session->packAdd_state = libssh2_NB_state_idle;
            return rc;

            /*
              byte      SSH_MSG_CHANNEL_CLOSE
              uint32    recipient channel
            */

        case SSH_MSG_CHANNEL_CLOSE:
            if(datalen >= 5)
                channelp =
                    _libssh2_channel_locate(session,
                                            _libssh2_ntohu32(data + 1));
            if(!channelp) {
                /* We may have freed already, just quietly ignore this... */
                LIBSSH2_FREE(session, data);
                session->packAdd_state = libssh2_NB_state_idle;
                return 0;
            }
            _libssh2_debug(session, LIBSSH2_TRACE_CONN,
                           "Close received for channel %lu/%lu",
                           channelp->local.id,
                           channelp->remote.id);

            channelp->remote.close = 1;
            channelp->remote.eof = 1;

            LIBSSH2_FREE(session, data);
            session->packAdd_state = libssh2_NB_state_idle;
            return 0;

            /*
              byte      SSH_MSG_CHANNEL_OPEN
              string    "session"
              uint32    sender channel
              uint32    initial window size
              uint32    maximum packet size
            */

        case SSH_MSG_CHANNEL_OPEN:
            if(datalen < 17)
                ;
            else if((datalen >= (sizeof("forwarded-tcpip") + 4)) &&
                     ((sizeof("forwarded-tcpip") - 1) ==
                      _libssh2_ntohu32(data + 1))
                     &&
                     (memcmp(data + 5, "forwarded-tcpip",
                             sizeof("forwarded-tcpip") - 1) == 0)) {

                /* init the state struct */
                memset(&session->packAdd_Qlstn_state, 0,
                       sizeof(session->packAdd_Qlstn_state));

              libssh2_packet_add_jump_point2:
                session->packAdd_state = libssh2_NB_state_jump2;
                rc = packet_queue_listener(session, data, datalen,
                                           &session->packAdd_Qlstn_state);
            }
            else if((datalen >= (sizeof("x11") + 4)) &&
                     ((sizeof("x11") - 1) == _libssh2_ntohu32(data + 1)) &&
                     (memcmp(data + 5, "x11", sizeof("x11") - 1) == 0)) {

                /* init the state struct */
                memset(&session->packAdd_x11open_state, 0,
                       sizeof(session->packAdd_x11open_state));

              libssh2_packet_add_jump_point3:
                session->packAdd_state = libssh2_NB_state_jump3;
                rc = packet_x11_open(session, data, datalen,
                                     &session->packAdd_x11open_state);
            }
            if(rc == LIBSSH2_ERROR_EAGAIN)
                return rc;

            LIBSSH2_FREE(session, data);
            session->packAdd_state = libssh2_NB_state_idle;
            return rc;

            /*
              byte      SSH_MSG_CHANNEL_WINDOW_ADJUST
              uint32    recipient channel
              uint32    bytes to add
            */
        case SSH_MSG_CHANNEL_WINDOW_ADJUST:
            if(datalen < 9)
                ;
            else {
                uint32_t bytestoadd = _libssh2_ntohu32(data + 5);
                channelp =
                    _libssh2_channel_locate(session,
                                            _libssh2_ntohu32(data + 1));
                if(channelp) {
                    channelp->local.window_size += bytestoadd;

                    _libssh2_debug(session, LIBSSH2_TRACE_CONN,
                                   "Window adjust for channel %lu/%lu, "
                                   "adding %lu bytes, new window_size=%lu",
                                   channelp->local.id,
                                   channelp->remote.id,
                                   bytestoadd,
                                   channelp->local.window_size);
                }
            }
            LIBSSH2_FREE(session, data);
            session->packAdd_state = libssh2_NB_state_idle;
            return 0;
        default:
            break;
        }

        session->packAdd_state = libssh2_NB_state_sent;
    }

    if(session->packAdd_state == libssh2_NB_state_sent) {
        LIBSSH2_PACKET *packetp =
            LIBSSH2_ALLOC(session, sizeof(LIBSSH2_PACKET));
        if(!packetp) {
            _libssh2_debug(session, LIBSSH2_ERROR_ALLOC,
                           "memory for packet");
            LIBSSH2_FREE(session, data);
            session->packAdd_state = libssh2_NB_state_idle;
            return LIBSSH2_ERROR_ALLOC;
        }
        packetp->data = data;
        packetp->data_len = datalen;
        packetp->data_head = data_head;

        _libssh2_list_add(&session->packets, &packetp->node);

        session->packAdd_state = libssh2_NB_state_sent1;
    }

    if((msg == SSH_MSG_KEXINIT &&
         !(session->state & LIBSSH2_STATE_EXCHANGING_KEYS)) ||
        (session->packAdd_state == libssh2_NB_state_sent2)) {
        if(session->packAdd_state == libssh2_NB_state_sent1) {
            /*
             * Remote wants new keys
             * Well, it's already in the brigade,
             * let's just call back into ourselves
             */
            _libssh2_debug(session, LIBSSH2_TRACE_TRANS, "Renegotiating Keys");

            session->packAdd_state = libssh2_NB_state_sent2;
        }

        /*
         * The KEXINIT message has been added to the queue.  The packAdd and
         * readPack states need to be reset because _libssh2_kex_exchange
         * (eventually) calls upon _libssh2_transport_read to read the rest of
         * the key exchange conversation.
         */
        session->readPack_state = libssh2_NB_state_idle;
        session->packet.total_num = 0;
        session->packAdd_state = libssh2_NB_state_idle;
        session->fullpacket_state = libssh2_NB_state_idle;

        memset(&session->startup_key_state, 0, sizeof(key_exchange_state_t));

        /*
         * If there was a key reexchange failure, let's just hope we didn't
         * send NEWKEYS yet, otherwise remote will drop us like a rock
         */
        rc = _libssh2_kex_exchange(session, 1, &session->startup_key_state);
        if(rc == LIBSSH2_ERROR_EAGAIN)
            return rc;
    }

    session->packAdd_state = libssh2_NB_state_idle;
    return 0;
}


  void Compute(OpKernelContext* context) override {
    const Tensor& logits_in = context->input(0);
    const Tensor& labels_in = context->input(1);

    TensorShape shape_in = logits_in.shape();

    BCast bcast(BCast::FromShape(logits_in.shape()),
                BCast::FromShape(labels_in.shape()));
    if (!logits_in.IsSameSize(labels_in)) {
      OP_REQUIRES(context, bcast.IsValid(),
                  errors::InvalidArgument(
                      "logits and labels must be broadcastable: logits_size=",
                      logits_in.shape().DebugString(),
                      " labels_size=", labels_in.shape().DebugString()));
      shape_in = BCast::ToShape(bcast.output_shape());
    }
    OP_REQUIRES(context, TensorShapeUtils::IsMatrix(shape_in),
                errors::InvalidArgument("logits and labels must be either "
                                        "2-dimensional, or broadcasted to be "
                                        "2-dimensional"));

    if (std::is_same<Device, GPUDevice>::value) {
      OP_REQUIRES(context, !OpDeterminismRequired(),
                  errors::Unimplemented(
                      "The GPU implementation of SoftmaxCrossEntropyWithLogits"
                      " that would have been executed is not deterministic."
                      " Note that the Python API uses an alternative,"
                      " deterministic, GPU-accelerated path when determinism is"
                      " enabled."));
    }

    // loss is 1-D (one per example), and size is batch_size.

    Tensor scratch;
    OP_REQUIRES_OK(
        context, context->allocate_temp(DataTypeToEnum<T>::value,
                                        TensorShape({shape_in.dim_size(0), 1}),
                                        &scratch));

    Tensor* loss_out = nullptr;
    OP_REQUIRES_OK(context,
                   context->allocate_output(
                       0, TensorShape({shape_in.dim_size(0)}), &loss_out));
    Tensor* back_out = nullptr;
    // Try to reuse the logits_in buffer for the backprop output.
    OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
                                {0}, 1, shape_in, &back_out));
    if (shape_in.dim_size(0) > 0) {
      functor::XentFunctor<Device, T> functor;
      if (logits_in.IsSameSize(labels_in)) {
        functor(context->eigen_device<Device>(), shape_in.AsEigenDSizes<2>(),
                Eigen::array<Eigen::DenseIndex, 2>{1, 1},
                Eigen::array<Eigen::DenseIndex, 2>{1, 1}, logits_in.matrix<T>(),
                labels_in.matrix<T>(), scratch.matrix<T>(), loss_out->vec<T>(),
                back_out->matrix<T>());
      } else {
        functor(context->eigen_device<Device>(), shape_in.AsEigenDSizes<2>(),
                BCast::ToIndexArray<2>(bcast.x_bcast()),
                BCast::ToIndexArray<2>(bcast.y_bcast()),
                logits_in.template shaped<T, 2>(bcast.x_reshape()),
                labels_in.template shaped<T, 2>(bcast.y_reshape()),
                scratch.matrix<T>(), loss_out->vec<T>(), back_out->matrix<T>());
      }
    }
  }

Fix crash in softmax-xent when some input dimensions are 1.

Before, tf.nn.softmax_cross_entropy_with_logits would fail a CHECK if one input tensor had shape (1, 1) and the other did not.

In particular, the call to ToIndexArray<2> here https://github.com/tensorflow/tensorflow/blob/1f3da84a89702d3b4f234ee83762d738caffe098/tensorflow/core/kernels/xent_op.cc#L99 would fail, since the call assumed the array had two dimensions. If both dimensions were 1, BCast would merge the two dimensions into a single dimension. Passing fewer_dims_optimization=false stops this optimization

PiperOrigin-RevId: 384844496
Change-Id: Ifb02dc74964132c3ed3f3bc98b0858dbe4e258b7


  void Compute(OpKernelContext* context) override {
    const Tensor& logits_in = context->input(0);
    const Tensor& labels_in = context->input(1);

    TensorShape shape_in = logits_in.shape();

    BCast bcast(BCast::FromShape(logits_in.shape()),
                BCast::FromShape(labels_in.shape()),
                /*fewer_dims_optimization=*/false);
    if (!logits_in.IsSameSize(labels_in)) {
      OP_REQUIRES(context, bcast.IsValid(),
                  errors::InvalidArgument(
                      "logits and labels must be broadcastable: logits_size=",
                      logits_in.shape().DebugString(),
                      " labels_size=", labels_in.shape().DebugString()));
      shape_in = BCast::ToShape(bcast.output_shape());
    }
    OP_REQUIRES(context, TensorShapeUtils::IsMatrix(shape_in),
                errors::InvalidArgument("logits and labels must be either "
                                        "2-dimensional, or broadcasted to be "
                                        "2-dimensional"));

    if (std::is_same<Device, GPUDevice>::value) {
      OP_REQUIRES(context, !OpDeterminismRequired(),
                  errors::Unimplemented(
                      "The GPU implementation of SoftmaxCrossEntropyWithLogits"
                      " that would have been executed is not deterministic."
                      " Note that the Python API uses an alternative,"
                      " deterministic, GPU-accelerated path when determinism is"
                      " enabled."));
    }

    // loss is 1-D (one per example), and size is batch_size.

    Tensor scratch;
    OP_REQUIRES_OK(
        context, context->allocate_temp(DataTypeToEnum<T>::value,
                                        TensorShape({shape_in.dim_size(0), 1}),
                                        &scratch));

    Tensor* loss_out = nullptr;
    OP_REQUIRES_OK(context,
                   context->allocate_output(
                       0, TensorShape({shape_in.dim_size(0)}), &loss_out));
    Tensor* back_out = nullptr;
    // Try to reuse the logits_in buffer for the backprop output.
    OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
                                {0}, 1, shape_in, &back_out));
    if (shape_in.dim_size(0) > 0) {
      functor::XentFunctor<Device, T> functor;
      functor(context->eigen_device<Device>(), shape_in.AsEigenDSizes<2>(),
              BCast::ToIndexArray<2>(bcast.x_bcast()),
              BCast::ToIndexArray<2>(bcast.y_bcast()),
              logits_in.template shaped<T, 2>(bcast.x_reshape()),
              labels_in.template shaped<T, 2>(bcast.y_reshape()),
              scratch.matrix<T>(), loss_out->vec<T>(), back_out->matrix<T>());
    }
  }


  Status BuildFeatureReaders(const OpInputList& ragged_values_list,
                             const OpInputList& ragged_splits_list,
                             const OpInputList& sparse_indices_list,
                             const OpInputList& sparse_values_list,
                             const OpInputList& dense_list, int64 batch_size,
                             FeatureReaders* features) {
    features->reserve(input_order_.size());

    int next_ragged = 0;
    int next_sparse = 0;
    int next_dense = 0;
    for (char c : input_order_) {
      if (c == 'R') {
        TF_RETURN_IF_ERROR(BuildRaggedFeatureReader(
            ragged_values_list[next_ragged], ragged_splits_list[next_ragged],
            features));
        next_ragged++;
      } else if (c == 'S') {
        TF_RETURN_IF_ERROR(BuildSparseFeatureReader(
            sparse_indices_list[next_sparse], sparse_values_list[next_sparse],
            batch_size, features));
        next_sparse++;
      } else if (c == 'D') {
        TF_RETURN_IF_ERROR(
            BuildDenseFeatureReader(dense_list[next_dense++], features));
      } else {
        return errors::InvalidArgument("Unexpected input_order value.");
      }
    }

    return Status::OK();
  }

Fix out of bounds read in `ragged_cross_op.cc`.

PiperOrigin-RevId: 369757702
Change-Id: Ie6e5d2c21513a8d56bf41fcf35960caf76e890f9


  Status BuildFeatureReaders(const OpInputList& ragged_values_list,
                             const OpInputList& ragged_splits_list,
                             const OpInputList& sparse_indices_list,
                             const OpInputList& sparse_values_list,
                             const OpInputList& dense_list, int64 batch_size,
                             FeatureReaders* features) {
    features->reserve(input_order_.size());

    int next_ragged = 0;
    int next_sparse = 0;
    int next_dense = 0;
    for (char c : input_order_) {
      if (c == 'R') {
        if (next_ragged >= ragged_values_list.size())
          return errors::InvalidArgument(
              "input_order \"", input_order_,
              "\" specifies reading a ragged tensor value at index ",
              next_ragged, " from a list of ", ragged_values_list.size(),
              " values.");
        if (next_ragged >= ragged_splits_list.size())
          return errors::InvalidArgument(
              "input_order \"", input_order_,
              "\" specifies reading a ragged tensor split at index ",
              next_ragged, " from a list of ", ragged_splits_list.size(),
              " splits.");
        TF_RETURN_IF_ERROR(BuildRaggedFeatureReader(
            ragged_values_list[next_ragged], ragged_splits_list[next_ragged],
            features));
        next_ragged++;
      } else if (c == 'S') {
        if (next_sparse >= sparse_values_list.size())
          return errors::InvalidArgument(
              "input_order \"", input_order_,
              "\" specifies reading a sparse tensor value at index ",
              next_sparse, " from a list of ", sparse_values_list.size(),
              " values.");
        if (next_sparse >= sparse_indices_list.size())
          return errors::InvalidArgument(
              "input_order \"", input_order_,
              "\" specifies reading a sparse tensor index at index ",
              next_sparse, " from a list of ", sparse_indices_list.size(),
              " indices.");
        TF_RETURN_IF_ERROR(BuildSparseFeatureReader(
            sparse_indices_list[next_sparse], sparse_values_list[next_sparse],
            batch_size, features));
        next_sparse++;
      } else if (c == 'D') {
        if (next_dense >= dense_list.size())
          return errors::InvalidArgument(
              "input_order \"", input_order_,
              "\" specifies reading a dense tensor at index ", next_dense,
              " from a list of ", dense_list.size(), " tensors.");
        TF_RETURN_IF_ERROR(
            BuildDenseFeatureReader(dense_list[next_dense++], features));
      } else {
        return errors::InvalidArgument("Unexpected input_order value.");
      }
    }

    return Status::OK();
  }


sc_oberthur_read_file(struct sc_pkcs15_card *p15card, const char *in_path,
		unsigned char **out, size_t *out_len,
		int verify_pin)
{
	struct sc_context *ctx = p15card->card->ctx;
	struct sc_card *card = p15card->card;
	struct sc_file *file = NULL;
	struct sc_path path;
	size_t sz;
	int rv;

	LOG_FUNC_CALLED(ctx);
	if (!in_path || !out || !out_len)
		LOG_TEST_RET(ctx, SC_ERROR_INVALID_ARGUMENTS, "Cannot read oberthur file");

	sc_log(ctx, "read file '%s'; verify_pin:%i", in_path, verify_pin);

	*out = NULL;
	*out_len = 0;

	sc_format_path(in_path, &path);
	rv = sc_select_file(card, &path, &file);
	if (rv != SC_SUCCESS) {
		sc_file_free(file);
		LOG_TEST_RET(ctx, rv, "Cannot select oberthur file to read");
	}

	if (file->ef_structure == SC_FILE_EF_TRANSPARENT)
		sz = file->size;
	else
		sz = (file->record_length + 2) * file->record_count;

	*out = calloc(sz, 1);
	if (*out == NULL) {
		sc_file_free(file);
		LOG_TEST_RET(ctx, SC_ERROR_OUT_OF_MEMORY, "Cannot read oberthur file");
	}

	if (file->ef_structure == SC_FILE_EF_TRANSPARENT)   {
		rv = sc_read_binary(card, 0, *out, sz, 0);
	}
	else	{
		size_t rec;
		size_t offs = 0;
		size_t rec_len = file->record_length;

		for (rec = 1; ; rec++)   {
			if (rec > file->record_count) {
				rv = 0;
				break;
			}
			rv = sc_read_record(card, rec, *out + offs + 2, rec_len, SC_RECORD_BY_REC_NR);
			if (rv == SC_ERROR_RECORD_NOT_FOUND)   {
				rv = 0;
				break;
			}
			else if (rv < 0)   {
				break;
			}

			rec_len = rv;

			*(*out + offs) = 'R';
			*(*out + offs + 1) = rv;

			offs += rv + 2;
		}

		sz = offs;
	}

	sc_log(ctx, "read oberthur file result %i", rv);
	if (verify_pin && rv == SC_ERROR_SECURITY_STATUS_NOT_SATISFIED)   {
		struct sc_pkcs15_object *objs[0x10], *pin_obj = NULL;
		const struct sc_acl_entry *acl = sc_file_get_acl_entry(file, SC_AC_OP_READ);
		int ii;

		if (acl == NULL) {
			sc_file_free(file);
			free(*out);
			*out = NULL;
			LOG_FUNC_RETURN(ctx, SC_ERROR_INVALID_DATA);
		}

		rv = sc_pkcs15_get_objects(p15card, SC_PKCS15_TYPE_AUTH_PIN, objs, 0x10);
		if (rv != SC_SUCCESS) {
			sc_file_free(file);
			free(*out);
			*out = NULL;
			LOG_TEST_RET(ctx, rv, "Cannot read oberthur file: get AUTH objects error");
		}

		for (ii=0; ii<rv; ii++)   {
			struct sc_pkcs15_auth_info *auth_info = (struct sc_pkcs15_auth_info *) objs[ii]->data;
			sc_log(ctx, "compare PIN/ACL refs:%i/%i, method:%i/%i",
					auth_info->attrs.pin.reference, acl->key_ref, auth_info->auth_method, acl->method);
			if (auth_info->attrs.pin.reference == (int)acl->key_ref && auth_info->auth_method == (unsigned)acl->method)   {
				pin_obj = objs[ii];
				break;
			}
		}

		if (!pin_obj || !pin_obj->content.value)    {
			rv = SC_ERROR_SECURITY_STATUS_NOT_SATISFIED;
		}
		else    {
			rv = sc_pkcs15_verify_pin(p15card, pin_obj, pin_obj->content.value, pin_obj->content.len);
			if (!rv)
				rv = sc_oberthur_read_file(p15card, in_path, out, out_len, 0);
		}
	}

	sc_file_free(file);

	if (rv < 0)   {
		free(*out);
		*out = NULL;
		*out_len = 0;
	}

	*out_len = sz;

	LOG_FUNC_RETURN(ctx, rv);
}

oberthur: Correctly check for return values

Thanks oss-fuzz

https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=28843


sc_oberthur_read_file(struct sc_pkcs15_card *p15card, const char *in_path,
		unsigned char **out, size_t *out_len,
		int verify_pin)
{
	struct sc_context *ctx = p15card->card->ctx;
	struct sc_card *card = p15card->card;
	struct sc_file *file = NULL;
	struct sc_path path;
	size_t sz;
	int rv;

	LOG_FUNC_CALLED(ctx);
	if (!in_path || !out || !out_len)
		LOG_TEST_RET(ctx, SC_ERROR_INVALID_ARGUMENTS, "Cannot read oberthur file");

	sc_log(ctx, "read file '%s'; verify_pin:%i", in_path, verify_pin);

	*out = NULL;
	*out_len = 0;

	sc_format_path(in_path, &path);
	rv = sc_select_file(card, &path, &file);
	if (rv != SC_SUCCESS) {
		sc_file_free(file);
		LOG_TEST_RET(ctx, rv, "Cannot select oberthur file to read");
	}

	if (file->ef_structure == SC_FILE_EF_TRANSPARENT)
		sz = file->size;
	else
		sz = (file->record_length + 2) * file->record_count;

	*out = calloc(sz, 1);
	if (*out == NULL) {
		sc_file_free(file);
		LOG_TEST_RET(ctx, SC_ERROR_OUT_OF_MEMORY, "Cannot read oberthur file");
	}

	if (file->ef_structure == SC_FILE_EF_TRANSPARENT)   {
		rv = sc_read_binary(card, 0, *out, sz, 0);
	}
	else	{
		size_t rec;
		size_t offs = 0;
		size_t rec_len = file->record_length;

		for (rec = 1; ; rec++)   {
			if (rec > file->record_count) {
				rv = 0;
				break;
			}
			rv = sc_read_record(card, rec, *out + offs + 2, rec_len, SC_RECORD_BY_REC_NR);
			if (rv == SC_ERROR_RECORD_NOT_FOUND)   {
				rv = 0;
				break;
			}
			else if (rv < 0)   {
				break;
			}

			rec_len = rv;

			*(*out + offs) = 'R';
			*(*out + offs + 1) = rv;

			offs += rv + 2;
		}

		sz = offs;
	}

	sc_log(ctx, "read oberthur file result %i", rv);
	if (verify_pin && rv == SC_ERROR_SECURITY_STATUS_NOT_SATISFIED)   {
		struct sc_pkcs15_object *objs[0x10], *pin_obj = NULL;
		const struct sc_acl_entry *acl = sc_file_get_acl_entry(file, SC_AC_OP_READ);
		int ii, nobjs;

		if (acl == NULL) {
			sc_file_free(file);
			free(*out);
			*out = NULL;
			LOG_FUNC_RETURN(ctx, SC_ERROR_INVALID_DATA);
		}

		nobjs = sc_pkcs15_get_objects(p15card, SC_PKCS15_TYPE_AUTH_PIN, objs, 0x10);
		if (nobjs < 1) {
			sc_file_free(file);
			free(*out);
			*out = NULL;
			LOG_TEST_RET(ctx, SC_ERROR_DATA_OBJECT_NOT_FOUND,
				"Cannot read oberthur file: get AUTH objects error");
		}

		for (ii = 0; ii < nobjs; ii++) {
			struct sc_pkcs15_auth_info *auth_info = (struct sc_pkcs15_auth_info *) objs[ii]->data;
			sc_log(ctx, "compare PIN/ACL refs:%i/%i, method:%i/%i",
				auth_info->attrs.pin.reference, acl->key_ref, auth_info->auth_method, acl->method);
			if (auth_info->attrs.pin.reference == (int)acl->key_ref && auth_info->auth_method == (unsigned)acl->method)   {
				pin_obj = objs[ii];
				break;
			}
		}

		if (!pin_obj || !pin_obj->content.value)    {
			rv = SC_ERROR_SECURITY_STATUS_NOT_SATISFIED;
		}
		else    {
			rv = sc_pkcs15_verify_pin(p15card, pin_obj, pin_obj->content.value, pin_obj->content.len);
			if (!rv)
				rv = sc_oberthur_read_file(p15card, in_path, out, out_len, 0);
		}
	}

	sc_file_free(file);

	if (rv < 0)   {
		free(*out);
		*out = NULL;
		*out_len = 0;
	}

	*out_len = sz;

	LOG_FUNC_RETURN(ctx, rv);
}


  void Compute(OpKernelContext* ctx) override {
    const Tensor& input = ctx->input(0);
    const Tensor& input_min_tensor = ctx->input(1);
    const Tensor& input_max_tensor = ctx->input(2);

    int num_slices = 1;
    if (axis_ > -1) {
      num_slices = input.dim_size(axis_);
    }

    Tensor* output = nullptr;
    OP_REQUIRES_OK(ctx, ctx->allocate_output(0, input.shape(), &output));
    Tensor float_output =
        need_cast_ ? tensorflow::Tensor(DT_FLOAT, input.shape()) : *output;
    if (num_slices == 1) {
      const float min_range = input_min_tensor.flat<float>()(0);
      const float max_range = input_max_tensor.flat<float>()(0);
      DequantizeTensor(ctx, input, min_range, max_range, &float_output);
    } else {
      OP_REQUIRES(ctx, mode_ != QUANTIZE_MODE_MIN_FIRST,
                  errors::Unimplemented("MIN_FIRST mode is not implemented for "
                                        "Dequantize with axis != -1."));

      int64 pre_dim = 1, post_dim = 1;
      for (int i = 0; i < axis_; ++i) {
        pre_dim *= float_output.dim_size(i);
      }
      for (int i = axis_ + 1; i < float_output.dims(); ++i) {
        post_dim *= float_output.dim_size(i);
      }
      auto input_tensor = input.template bit_casted_shaped<T, 3>(
          {pre_dim, num_slices, post_dim});
      auto output_tensor =
          float_output.flat_inner_outer_dims<float, 3>(axis_ - 1);
      auto min_ranges = input_min_tensor.vec<float>();
      auto max_ranges = input_max_tensor.vec<float>();
      for (int i = 0; i < num_slices; ++i) {
        DequantizeSlice(ctx->eigen_device<Device>(), ctx,
                        input_tensor.template chip<1>(i), min_ranges(i),
                        max_ranges(i), output_tensor.template chip<1>(i));
      }
    }
    if (need_cast_) {
      S* out_ptr = output->flat<S>().data();
      float* in_ptr = float_output.flat<float>().data();
      for (int64 i = 0; i < float_output.NumElements(); ++i) {
        out_ptr[i] = static_cast<S>(in_ptr[i]);
      }
    }
  }

Fix heap OOB read in dequantize op.

Also fixes SEGV in same op

PiperOrigin-RevId: 372437896
Change-Id: I135e94d360c2a1ce374c10f7e0fed1af603dbc02


  void Compute(OpKernelContext* ctx) override {
    const Tensor& input = ctx->input(0);
    const Tensor& input_min_tensor = ctx->input(1);
    const Tensor& input_max_tensor = ctx->input(2);

    int num_slices = 1;
    if (axis_ > -1) {
      num_slices = input.dim_size(axis_);
    }
    OP_REQUIRES(ctx, input_min_tensor.NumElements() == num_slices,
                errors::InvalidArgument(
                    "input_min_tensor must have as many elements as input on "
                    "the dequantization axis (",
                    axis_, "), got ", input_min_tensor.NumElements(),
                    ", expected ", num_slices));
    OP_REQUIRES(ctx, input_max_tensor.NumElements() == num_slices,
                errors::InvalidArgument(
                    "input_max_tensor must have as many elements as input on "
                    "the dequantization axis (",
                    axis_, "), got ", input_max_tensor.NumElements(),
                    ", expected ", num_slices));

    Tensor* output = nullptr;
    OP_REQUIRES_OK(ctx, ctx->allocate_output(0, input.shape(), &output));
    Tensor float_output =
        need_cast_ ? tensorflow::Tensor(DT_FLOAT, input.shape()) : *output;
    if (num_slices == 1) {
      const float min_range = input_min_tensor.flat<float>()(0);
      const float max_range = input_max_tensor.flat<float>()(0);
      DequantizeTensor(ctx, input, min_range, max_range, &float_output);
    } else {
      OP_REQUIRES(ctx, mode_ != QUANTIZE_MODE_MIN_FIRST,
                  errors::Unimplemented("MIN_FIRST mode is not implemented for "
                                        "Dequantize with axis != -1."));

      int64 pre_dim = 1, post_dim = 1;
      for (int i = 0; i < axis_; ++i) {
        pre_dim *= float_output.dim_size(i);
      }
      for (int i = axis_ + 1; i < float_output.dims(); ++i) {
        post_dim *= float_output.dim_size(i);
      }
      auto input_tensor = input.template bit_casted_shaped<T, 3>(
          {pre_dim, num_slices, post_dim});
      auto output_tensor =
          float_output.flat_inner_outer_dims<float, 3>(axis_ - 1);
      auto min_ranges = input_min_tensor.vec<float>();
      auto max_ranges = input_max_tensor.vec<float>();
      for (int i = 0; i < num_slices; ++i) {
        DequantizeSlice(ctx->eigen_device<Device>(), ctx,
                        input_tensor.template chip<1>(i), min_ranges(i),
                        max_ranges(i), output_tensor.template chip<1>(i));
      }
    }
    if (need_cast_) {
      S* out_ptr = output->flat<S>().data();
      float* in_ptr = float_output.flat<float>().data();
      for (int64 i = 0; i < float_output.NumElements(); ++i) {
        out_ptr[i] = static_cast<S>(in_ptr[i]);
      }
    }
  }


CallResult<PseudoHandle<>> JSObject::getComputedWithReceiver_RJS(
    Handle<JSObject> selfHandle,
    Runtime *runtime,
    Handle<> nameValHandle,
    Handle<> receiver) {
  // Try the fast-path first: no "index-like" properties and the "name" already
  // is a valid integer index.
  if (selfHandle->flags_.fastIndexProperties) {
    if (auto arrayIndex = toArrayIndexFastPath(*nameValHandle)) {
      // Do we have this value present in our array storage? If so, return it.
      PseudoHandle<> ourValue = createPseudoHandle(
          getOwnIndexed(selfHandle.get(), runtime, *arrayIndex));
      if (LLVM_LIKELY(!ourValue->isEmpty()))
        return ourValue;
    }
  }

  // If nameValHandle is an object, we should convert it to string now,
  // because toString may have side-effect, and we want to do this only
  // once.
  auto converted = toPropertyKeyIfObject(runtime, nameValHandle);
  if (LLVM_UNLIKELY(converted == ExecutionStatus::EXCEPTION)) {
    return ExecutionStatus::EXCEPTION;
  }
  auto nameValPrimitiveHandle = *converted;

  ComputedPropertyDescriptor desc;

  // Locate the descriptor. propObj contains the object which may be anywhere
  // along the prototype chain.
  MutableHandle<JSObject> propObj{runtime};
  if (LLVM_UNLIKELY(
          getComputedPrimitiveDescriptor(
              selfHandle, runtime, nameValPrimitiveHandle, propObj, desc) ==
          ExecutionStatus::EXCEPTION)) {
    return ExecutionStatus::EXCEPTION;
  }

  if (!propObj)
    return createPseudoHandle(HermesValue::encodeUndefinedValue());

  if (LLVM_LIKELY(
          !desc.flags.accessor && !desc.flags.hostObject &&
          !desc.flags.proxyObject))
    return createPseudoHandle(
        getComputedSlotValue(propObj.get(), runtime, desc));

  if (desc.flags.accessor) {
    auto *accessor = vmcast<PropertyAccessor>(
        getComputedSlotValue(propObj.get(), runtime, desc));
    if (!accessor->getter)
      return createPseudoHandle(HermesValue::encodeUndefinedValue());

    // Execute the accessor on this object.
    return accessor->getter.get(runtime)->executeCall0(
        runtime->makeHandle(accessor->getter), runtime, receiver);
  } else if (desc.flags.hostObject) {
    SymbolID id{};
    LAZY_TO_IDENTIFIER(runtime, nameValPrimitiveHandle, id);
    auto propRes = vmcast<HostObject>(selfHandle.get())->get(id);
    if (propRes == ExecutionStatus::EXCEPTION)
      return ExecutionStatus::EXCEPTION;
    return createPseudoHandle(*propRes);
  } else {
    assert(desc.flags.proxyObject && "descriptor flags are impossible");
    CallResult<Handle<>> key = toPropertyKey(runtime, nameValPrimitiveHandle);
    if (key == ExecutionStatus::EXCEPTION)
      return ExecutionStatus::EXCEPTION;
    return JSProxy::getComputed(propObj, runtime, *key, receiver);
  }
}

[CVE-2020-1911] Look up HostObject computed properties on the right object in the prototype chain.

Summary:
The change in the hermes repository fixes the security vulnerability
CVE-2020-1911.  This vulnerability only affects applications which
allow evaluation of uncontrolled, untrusted JavaScript code not
shipped with the app, so React Native apps will generally not be affected.

This revision includes a test for the bug.  The test is generic JSI
code, so it is included in the hermes and react-native repositories.

Changelog: [Internal]

Reviewed By: tmikov

Differential Revision: D23322992

fbshipit-source-id: 4e88c974afe1ad33a263f9cac03e9dc98d33649a


CallResult<PseudoHandle<>> JSObject::getComputedWithReceiver_RJS(
    Handle<JSObject> selfHandle,
    Runtime *runtime,
    Handle<> nameValHandle,
    Handle<> receiver) {
  // Try the fast-path first: no "index-like" properties and the "name" already
  // is a valid integer index.
  if (selfHandle->flags_.fastIndexProperties) {
    if (auto arrayIndex = toArrayIndexFastPath(*nameValHandle)) {
      // Do we have this value present in our array storage? If so, return it.
      PseudoHandle<> ourValue = createPseudoHandle(
          getOwnIndexed(selfHandle.get(), runtime, *arrayIndex));
      if (LLVM_LIKELY(!ourValue->isEmpty()))
        return ourValue;
    }
  }

  // If nameValHandle is an object, we should convert it to string now,
  // because toString may have side-effect, and we want to do this only
  // once.
  auto converted = toPropertyKeyIfObject(runtime, nameValHandle);
  if (LLVM_UNLIKELY(converted == ExecutionStatus::EXCEPTION)) {
    return ExecutionStatus::EXCEPTION;
  }
  auto nameValPrimitiveHandle = *converted;

  ComputedPropertyDescriptor desc;

  // Locate the descriptor. propObj contains the object which may be anywhere
  // along the prototype chain.
  MutableHandle<JSObject> propObj{runtime};
  if (LLVM_UNLIKELY(
          getComputedPrimitiveDescriptor(
              selfHandle, runtime, nameValPrimitiveHandle, propObj, desc) ==
          ExecutionStatus::EXCEPTION)) {
    return ExecutionStatus::EXCEPTION;
  }

  if (!propObj)
    return createPseudoHandle(HermesValue::encodeUndefinedValue());

  if (LLVM_LIKELY(
          !desc.flags.accessor && !desc.flags.hostObject &&
          !desc.flags.proxyObject))
    return createPseudoHandle(
        getComputedSlotValue(propObj.get(), runtime, desc));

  if (desc.flags.accessor) {
    auto *accessor = vmcast<PropertyAccessor>(
        getComputedSlotValue(propObj.get(), runtime, desc));
    if (!accessor->getter)
      return createPseudoHandle(HermesValue::encodeUndefinedValue());

    // Execute the accessor on this object.
    return accessor->getter.get(runtime)->executeCall0(
        runtime->makeHandle(accessor->getter), runtime, receiver);
  } else if (desc.flags.hostObject) {
    SymbolID id{};
    LAZY_TO_IDENTIFIER(runtime, nameValPrimitiveHandle, id);
    auto propRes = vmcast<HostObject>(propObj.get())->get(id);
    if (propRes == ExecutionStatus::EXCEPTION)
      return ExecutionStatus::EXCEPTION;
    return createPseudoHandle(*propRes);
  } else {
    assert(desc.flags.proxyObject && "descriptor flags are impossible");
    CallResult<Handle<>> key = toPropertyKey(runtime, nameValPrimitiveHandle);
    if (key == ExecutionStatus::EXCEPTION)
      return ExecutionStatus::EXCEPTION;
    return JSProxy::getComputed(propObj, runtime, *key, receiver);
  }
}