static int xennet_connect(struct net_device *dev)
{
	struct netfront_info *np = netdev_priv(dev);
	unsigned int num_queues = 0;
	int err;
	unsigned int j = 0;
	struct netfront_queue *queue = NULL;

	if (!xenbus_read_unsigned(np->xbdev->otherend, "feature-rx-copy", 0)) {
		dev_info(&dev->dev,
			 "backend does not support copying receive path\n");
		return -ENODEV;
	}

	err = talk_to_netback(np->xbdev, np);
	if (err)
		return err;
	if (np->netback_has_xdp_headroom)
		pr_info("backend supports XDP headroom\n");
	if (np->bounce)
		dev_info(&np->xbdev->dev,
			 "bouncing transmitted data to zeroed pages\n");

	/* talk_to_netback() sets the correct number of queues */
	num_queues = dev->real_num_tx_queues;

	if (dev->reg_state == NETREG_UNINITIALIZED) {
		err = register_netdev(dev);
		if (err) {
			pr_warn("%s: register_netdev err=%d\n", __func__, err);
			device_unregister(&np->xbdev->dev);
			return err;
		}
	}

	rtnl_lock();
	netdev_update_features(dev);
	rtnl_unlock();

	/*
	 * All public and private state should now be sane.  Get
	 * ready to start sending and receiving packets and give the driver
	 * domain a kick because we've probably just requeued some
	 * packets.
	 */
	netif_tx_lock_bh(np->netdev);
	netif_device_attach(np->netdev);
	netif_tx_unlock_bh(np->netdev);

	netif_carrier_on(np->netdev);
	for (j = 0; j < num_queues; ++j) {
		queue = &np->queues[j];

		notify_remote_via_irq(queue->tx_irq);
		if (queue->tx_irq != queue->rx_irq)
			notify_remote_via_irq(queue->rx_irq);

		spin_lock_irq(&queue->tx_lock);
		xennet_tx_buf_gc(queue);
		spin_unlock_irq(&queue->tx_lock);

		spin_lock_bh(&queue->rx_lock);
		xennet_alloc_rx_buffers(queue);
		spin_unlock_bh(&queue->rx_lock);
	}

	return 0;
}


static int setup_netfront_split(struct netfront_queue *queue)
{
	int err;

	err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn);
	if (err < 0)
		goto fail;
	err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->rx_evtchn);
	if (err < 0)
		goto alloc_rx_evtchn_fail;

	snprintf(queue->tx_irq_name, sizeof(queue->tx_irq_name),
		 "%s-tx", queue->name);
	err = bind_evtchn_to_irqhandler_lateeoi(queue->tx_evtchn,
						xennet_tx_interrupt, 0,
						queue->tx_irq_name, queue);
	if (err < 0)
		goto bind_tx_fail;
	queue->tx_irq = err;

	snprintf(queue->rx_irq_name, sizeof(queue->rx_irq_name),
		 "%s-rx", queue->name);
	err = bind_evtchn_to_irqhandler_lateeoi(queue->rx_evtchn,
						xennet_rx_interrupt, 0,
						queue->rx_irq_name, queue);
	if (err < 0)
		goto bind_rx_fail;
	queue->rx_irq = err;

	return 0;

bind_rx_fail:
	unbind_from_irqhandler(queue->tx_irq, queue);
	queue->tx_irq = 0;
bind_tx_fail:
	xenbus_free_evtchn(queue->info->xbdev, queue->rx_evtchn);
	queue->rx_evtchn = 0;
alloc_rx_evtchn_fail:
	xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn);
	queue->tx_evtchn = 0;
fail:
	return err;
}

xen-netfront: restore __skb_queue_tail() positioning in xennet_get_responses()

The commit referenced below moved the invocation past the "next" label,
without any explanation. In fact this allows misbehaving backends undue
control over the domain the frontend runs in, as earlier detected errors
require the skb to not be freed (it may be retained for later processing
via xennet_move_rx_slot(), or it may simply be unsafe to have it freed).

This is CVE-2022-33743 / XSA-405.

Fixes: 6c5aa6fc4def ("xen networking: add basic XDP support for xen-netfront")
Signed-off-by: Jan Beulich <jbeulich@suse.com>
Reviewed-by: Juergen Gross <jgross@suse.com>
Signed-off-by: Juergen Gross <jgross@suse.com>


static void xennet_set_rx_rsp_cons(struct netfront_queue *queue, RING_IDX val)
{
	unsigned long flags;

	spin_lock_irqsave(&queue->rx_cons_lock, flags);
	queue->rx.rsp_cons = val;
	queue->rx_rsp_unconsumed = XEN_RING_NR_UNCONSUMED_RESPONSES(&queue->rx);
	spin_unlock_irqrestore(&queue->rx_cons_lock, flags);
}


static netdev_features_t xennet_fix_features(struct net_device *dev,
	netdev_features_t features)
{
	struct netfront_info *np = netdev_priv(dev);

	if (features & NETIF_F_SG &&
	    !xenbus_read_unsigned(np->xbdev->otherend, "feature-sg", 0))
		features &= ~NETIF_F_SG;

	if (features & NETIF_F_IPV6_CSUM &&
	    !xenbus_read_unsigned(np->xbdev->otherend,
				  "feature-ipv6-csum-offload", 0))
		features &= ~NETIF_F_IPV6_CSUM;

	if (features & NETIF_F_TSO &&
	    !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv4", 0))
		features &= ~NETIF_F_TSO;

	if (features & NETIF_F_TSO6 &&
	    !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv6", 0))
		features &= ~NETIF_F_TSO6;

	return features;
}


static struct sk_buff *xennet_get_rx_skb(struct netfront_queue *queue,
					 RING_IDX ri)
{
	int i = xennet_rxidx(ri);
	struct sk_buff *skb = queue->rx_skbs[i];
	queue->rx_skbs[i] = NULL;
	return skb;
}


static ssize_t store_rxbuf(struct device *dev,
			   struct device_attribute *attr,
			   const char *buf, size_t len)
{
	char *endp;

	if (!capable(CAP_NET_ADMIN))
		return -EPERM;

	simple_strtoul(buf, &endp, 0);
	if (endp == buf)
		return -EBADMSG;

	/* rxbuf_min and rxbuf_max are no longer configurable. */

	return len;
}


static int xennet_xdp_xmit_one(struct net_device *dev,
			       struct netfront_queue *queue,
			       struct xdp_frame *xdpf)
{
	struct netfront_info *np = netdev_priv(dev);
	struct netfront_stats *tx_stats = this_cpu_ptr(np->tx_stats);
	struct xennet_gnttab_make_txreq info = {
		.queue = queue,
		.skb = NULL,
		.page = virt_to_page(xdpf->data),
	};
	int notify;

	xennet_make_first_txreq(&info,
				offset_in_page(xdpf->data),
				xdpf->len);

	xennet_mark_tx_pending(queue);

	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify);
	if (notify)
		notify_remote_via_irq(queue->tx_irq);

	u64_stats_update_begin(&tx_stats->syncp);
	tx_stats->bytes += xdpf->len;
	tx_stats->packets++;
	u64_stats_update_end(&tx_stats->syncp);

	xennet_tx_buf_gc(queue);

	return 0;
}


static int xennet_get_sset_count(struct net_device *dev, int string_set)
{
	switch (string_set) {
	case ETH_SS_STATS:
		return ARRAY_SIZE(xennet_stats);
	default:
		return -EINVAL;
	}
}


static int talk_to_netback_xdp(struct netfront_info *np, int xdp)
{
	int err;
	unsigned short headroom;

	headroom = xdp ? XDP_PACKET_HEADROOM : 0;
	err = xenbus_printf(XBT_NIL, np->xbdev->nodename,
			    "xdp-headroom", "%hu",
			    headroom);
	if (err)
		pr_warn("Error writing xdp-headroom\n");

	return err;
}


static irqreturn_t xennet_rx_interrupt(int irq, void *dev_id)
{
	unsigned int eoiflag = XEN_EOI_FLAG_SPURIOUS;

	if (likely(xennet_handle_rx(dev_id, &eoiflag)))
		xen_irq_lateeoi(irq, eoiflag);

	return IRQ_HANDLED;
}


static int setup_netfront(struct xenbus_device *dev,
			struct netfront_queue *queue, unsigned int feature_split_evtchn)
{
	struct xen_netif_tx_sring *txs;
	struct xen_netif_rx_sring *rxs;
	int err;

	queue->tx_ring_ref = INVALID_GRANT_REF;
	queue->rx_ring_ref = INVALID_GRANT_REF;
	queue->rx.sring = NULL;
	queue->tx.sring = NULL;

	err = xenbus_setup_ring(dev, GFP_NOIO | __GFP_HIGH, (void **)&txs,
				1, &queue->tx_ring_ref);
	if (err)
		goto fail;

	XEN_FRONT_RING_INIT(&queue->tx, txs, XEN_PAGE_SIZE);

	err = xenbus_setup_ring(dev, GFP_NOIO | __GFP_HIGH, (void **)&rxs,
				1, &queue->rx_ring_ref);
	if (err)
		goto fail;

	XEN_FRONT_RING_INIT(&queue->rx, rxs, XEN_PAGE_SIZE);

	if (feature_split_evtchn)
		err = setup_netfront_split(queue);
	/* setup single event channel if
	 *  a) feature-split-event-channels == 0
	 *  b) feature-split-event-channels == 1 but failed to setup
	 */
	if (!feature_split_evtchn || err)
		err = setup_netfront_single(queue);

	if (err)
		goto fail;

	return 0;

 fail:
	xenbus_teardown_ring((void **)&queue->rx.sring, 1, &queue->rx_ring_ref);
	xenbus_teardown_ring((void **)&queue->tx.sring, 1, &queue->tx_ring_ref);

	return err;
}


static struct sk_buff *xennet_alloc_one_rx_buffer(struct netfront_queue *queue)
{
	struct sk_buff *skb;
	struct page *page;

	skb = __netdev_alloc_skb(queue->info->netdev,
				 RX_COPY_THRESHOLD + NET_IP_ALIGN,
				 GFP_ATOMIC | __GFP_NOWARN);
	if (unlikely(!skb))
		return NULL;

	page = page_pool_alloc_pages(queue->page_pool,
				     GFP_ATOMIC | __GFP_NOWARN | __GFP_ZERO);
	if (unlikely(!page)) {
		kfree_skb(skb);
		return NULL;
	}
	skb_add_rx_frag(skb, 0, page, 0, 0, PAGE_SIZE);

	/* Align ip header to a 16 bytes boundary */
	skb_reserve(skb, NET_IP_ALIGN);
	skb->dev = queue->info->netdev;

	return skb;
}


static void xennet_maybe_wake_tx(struct netfront_queue *queue)
{
	struct net_device *dev = queue->info->netdev;
	struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, queue->id);

	if (unlikely(netif_tx_queue_stopped(dev_queue)) &&
	    netfront_tx_slot_available(queue) &&
	    likely(netif_running(dev)))
		netif_tx_wake_queue(netdev_get_tx_queue(dev, queue->id));
}


static int setup_netfront_single(struct netfront_queue *queue)
{
	int err;

	err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn);
	if (err < 0)
		goto fail;

	err = bind_evtchn_to_irqhandler_lateeoi(queue->tx_evtchn,
						xennet_interrupt, 0,
						queue->info->netdev->name,
						queue);
	if (err < 0)
		goto bind_fail;
	queue->rx_evtchn = queue->tx_evtchn;
	queue->rx_irq = queue->tx_irq = err;

	return 0;

bind_fail:
	xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn);
	queue->tx_evtchn = 0;
fail:
	return err;
}


static void xennet_uninit(struct net_device *dev)
{
	struct netfront_info *np = netdev_priv(dev);
	xennet_destroy_queues(np);
}


static int xennet_set_skb_gso(struct sk_buff *skb,
			      struct xen_netif_extra_info *gso)
{
	if (!gso->u.gso.size) {
		if (net_ratelimit())
			pr_warn("GSO size must not be zero\n");
		return -EINVAL;
	}

	if (gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV4 &&
	    gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV6) {
		if (net_ratelimit())
			pr_warn("Bad GSO type %d\n", gso->u.gso.type);
		return -EINVAL;
	}

	skb_shinfo(skb)->gso_size = gso->u.gso.size;
	skb_shinfo(skb)->gso_type =
		(gso->u.gso.type == XEN_NETIF_GSO_TYPE_TCPV4) ?
		SKB_GSO_TCPV4 :
		SKB_GSO_TCPV6;

	/* Header must be checked, and gso_segs computed. */
	skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
	skb_shinfo(skb)->gso_segs = 0;

	return 0;
}


static int xennet_open(struct net_device *dev)
{
	struct netfront_info *np = netdev_priv(dev);
	unsigned int num_queues = dev->real_num_tx_queues;
	unsigned int i = 0;
	struct netfront_queue *queue = NULL;

	if (!np->queues || np->broken)
		return -ENODEV;

	for (i = 0; i < num_queues; ++i) {
		queue = &np->queues[i];
		napi_enable(&queue->napi);

		spin_lock_bh(&queue->rx_lock);
		if (netif_carrier_ok(dev)) {
			xennet_alloc_rx_buffers(queue);
			queue->rx.sring->rsp_event = queue->rx.rsp_cons + 1;
			if (RING_HAS_UNCONSUMED_RESPONSES(&queue->rx))
				napi_schedule(&queue->napi);
		}
		spin_unlock_bh(&queue->rx_lock);
	}

	netif_tx_start_all_queues(dev);

	return 0;
}


static void xennet_release_rx_bufs(struct netfront_queue *queue)
{
	int id, ref;

	spin_lock_bh(&queue->rx_lock);

	for (id = 0; id < NET_RX_RING_SIZE; id++) {
		struct sk_buff *skb;
		struct page *page;

		skb = queue->rx_skbs[id];
		if (!skb)
			continue;

		ref = queue->grant_rx_ref[id];
		if (ref == INVALID_GRANT_REF)
			continue;

		page = skb_frag_page(&skb_shinfo(skb)->frags[0]);

		/* gnttab_end_foreign_access() needs a page ref until
		 * foreign access is ended (which may be deferred).
		 */
		get_page(page);
		gnttab_end_foreign_access(ref, page);
		queue->grant_rx_ref[id] = INVALID_GRANT_REF;

		kfree_skb(skb);
	}

	spin_unlock_bh(&queue->rx_lock);
}


static int xennet_xdp(struct net_device *dev, struct netdev_bpf *xdp)
{
	struct netfront_info *np = netdev_priv(dev);

	if (np->broken)
		return -ENODEV;

	switch (xdp->command) {
	case XDP_SETUP_PROG:
		return xennet_xdp_set(dev, xdp->prog, xdp->extack);
	default:
		return -EINVAL;
	}
}


static int xennet_create_page_pool(struct netfront_queue *queue)
{
	int err;
	struct page_pool_params pp_params = {
		.order = 0,
		.flags = 0,
		.pool_size = NET_RX_RING_SIZE,
		.nid = NUMA_NO_NODE,
		.dev = &queue->info->netdev->dev,
		.offset = XDP_PACKET_HEADROOM,
		.max_len = XEN_PAGE_SIZE - XDP_PACKET_HEADROOM,
	};

	queue->page_pool = page_pool_create(&pp_params);
	if (IS_ERR(queue->page_pool)) {
		err = PTR_ERR(queue->page_pool);
		queue->page_pool = NULL;
		return err;
	}

	err = xdp_rxq_info_reg(&queue->xdp_rxq, queue->info->netdev,
			       queue->id, 0);
	if (err) {
		netdev_err(queue->info->netdev, "xdp_rxq_info_reg failed\n");
		goto err_free_pp;
	}

	err = xdp_rxq_info_reg_mem_model(&queue->xdp_rxq,
					 MEM_TYPE_PAGE_POOL, queue->page_pool);
	if (err) {
		netdev_err(queue->info->netdev, "xdp_rxq_info_reg_mem_model failed\n");
		goto err_unregister_rxq;
	}
	return 0;

err_unregister_rxq:
	xdp_rxq_info_unreg(&queue->xdp_rxq);
err_free_pp:
	page_pool_destroy(queue->page_pool);
	queue->page_pool = NULL;
	return err;
}


static void xennet_alloc_rx_buffers(struct netfront_queue *queue)
{
	RING_IDX req_prod = queue->rx.req_prod_pvt;
	int notify;
	int err = 0;

	if (unlikely(!netif_carrier_ok(queue->info->netdev)))
		return;

	for (req_prod = queue->rx.req_prod_pvt;
	     req_prod - queue->rx.rsp_cons < NET_RX_RING_SIZE;
	     req_prod++) {
		struct sk_buff *skb;
		unsigned short id;
		grant_ref_t ref;
		struct page *page;
		struct xen_netif_rx_request *req;

		skb = xennet_alloc_one_rx_buffer(queue);
		if (!skb) {
			err = -ENOMEM;
			break;
		}

		id = xennet_rxidx(req_prod);

		BUG_ON(queue->rx_skbs[id]);
		queue->rx_skbs[id] = skb;

		ref = gnttab_claim_grant_reference(&queue->gref_rx_head);
		WARN_ON_ONCE(IS_ERR_VALUE((unsigned long)(int)ref));
		queue->grant_rx_ref[id] = ref;

		page = skb_frag_page(&skb_shinfo(skb)->frags[0]);

		req = RING_GET_REQUEST(&queue->rx, req_prod);
		gnttab_page_grant_foreign_access_ref_one(ref,
							 queue->info->xbdev->otherend_id,
							 page,
							 0);
		req->id = id;
		req->gref = ref;
	}

	queue->rx.req_prod_pvt = req_prod;

	/* Try again later if there are not enough requests or skb allocation
	 * failed.
	 * Enough requests is quantified as the sum of newly created slots and
	 * the unconsumed slots at the backend.
	 */
	if (req_prod - queue->rx.rsp_cons < NET_RX_SLOTS_MIN ||
	    unlikely(err)) {
		mod_timer(&queue->rx_refill_timer, jiffies + (HZ/10));
		return;
	}

	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->rx, notify);
	if (notify)
		notify_remote_via_irq(queue->rx_irq);
}


static int xennet_poll(struct napi_struct *napi, int budget)
{
	struct netfront_queue *queue = container_of(napi, struct netfront_queue, napi);
	struct net_device *dev = queue->info->netdev;
	struct sk_buff *skb;
	struct netfront_rx_info rinfo;
	struct xen_netif_rx_response *rx = &rinfo.rx;
	struct xen_netif_extra_info *extras = rinfo.extras;
	RING_IDX i, rp;
	int work_done;
	struct sk_buff_head rxq;
	struct sk_buff_head errq;
	struct sk_buff_head tmpq;
	int err;
	bool need_xdp_flush = false;

	spin_lock(&queue->rx_lock);

	skb_queue_head_init(&rxq);
	skb_queue_head_init(&errq);
	skb_queue_head_init(&tmpq);

	rp = queue->rx.sring->rsp_prod;
	if (RING_RESPONSE_PROD_OVERFLOW(&queue->rx, rp)) {
		dev_alert(&dev->dev, "Illegal number of responses %u\n",
			  rp - queue->rx.rsp_cons);
		queue->info->broken = true;
		spin_unlock(&queue->rx_lock);
		return 0;
	}
	rmb(); /* Ensure we see queued responses up to 'rp'. */

	i = queue->rx.rsp_cons;
	work_done = 0;
	while ((i != rp) && (work_done < budget)) {
		RING_COPY_RESPONSE(&queue->rx, i, rx);
		memset(extras, 0, sizeof(rinfo.extras));

		err = xennet_get_responses(queue, &rinfo, rp, &tmpq,
					   &need_xdp_flush);

		if (unlikely(err)) {
			if (queue->info->broken) {
				spin_unlock(&queue->rx_lock);
				return 0;
			}
err:
			while ((skb = __skb_dequeue(&tmpq)))
				__skb_queue_tail(&errq, skb);
			dev->stats.rx_errors++;
			i = queue->rx.rsp_cons;
			continue;
		}

		skb = __skb_dequeue(&tmpq);

		if (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type) {
			struct xen_netif_extra_info *gso;
			gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1];

			if (unlikely(xennet_set_skb_gso(skb, gso))) {
				__skb_queue_head(&tmpq, skb);
				xennet_set_rx_rsp_cons(queue,
						       queue->rx.rsp_cons +
						       skb_queue_len(&tmpq));
				goto err;
			}
		}

		NETFRONT_SKB_CB(skb)->pull_to = rx->status;
		if (NETFRONT_SKB_CB(skb)->pull_to > RX_COPY_THRESHOLD)
			NETFRONT_SKB_CB(skb)->pull_to = RX_COPY_THRESHOLD;

		skb_frag_off_set(&skb_shinfo(skb)->frags[0], rx->offset);
		skb_frag_size_set(&skb_shinfo(skb)->frags[0], rx->status);
		skb->data_len = rx->status;
		skb->len += rx->status;

		if (unlikely(xennet_fill_frags(queue, skb, &tmpq)))
			goto err;

		if (rx->flags & XEN_NETRXF_csum_blank)
			skb->ip_summed = CHECKSUM_PARTIAL;
		else if (rx->flags & XEN_NETRXF_data_validated)
			skb->ip_summed = CHECKSUM_UNNECESSARY;

		__skb_queue_tail(&rxq, skb);

		i = queue->rx.rsp_cons + 1;
		xennet_set_rx_rsp_cons(queue, i);
		work_done++;
	}
	if (need_xdp_flush)
		xdp_do_flush();

	__skb_queue_purge(&errq);

	work_done -= handle_incoming_queue(queue, &rxq);

	xennet_alloc_rx_buffers(queue);

	if (work_done < budget) {
		int more_to_do = 0;

		napi_complete_done(napi, work_done);

		RING_FINAL_CHECK_FOR_RESPONSES(&queue->rx, more_to_do);
		if (more_to_do)
			napi_schedule(napi);
	}

	spin_unlock(&queue->rx_lock);

	return work_done;
}


static int checksum_setup(struct net_device *dev, struct sk_buff *skb)
{
	bool recalculate_partial_csum = false;

	/*
	 * A GSO SKB must be CHECKSUM_PARTIAL. However some buggy
	 * peers can fail to set NETRXF_csum_blank when sending a GSO
	 * frame. In this case force the SKB to CHECKSUM_PARTIAL and
	 * recalculate the partial checksum.
	 */
	if (skb->ip_summed != CHECKSUM_PARTIAL && skb_is_gso(skb)) {
		struct netfront_info *np = netdev_priv(dev);
		atomic_inc(&np->rx_gso_checksum_fixup);
		skb->ip_summed = CHECKSUM_PARTIAL;
		recalculate_partial_csum = true;
	}

	/* A non-CHECKSUM_PARTIAL SKB does not require setup. */
	if (skb->ip_summed != CHECKSUM_PARTIAL)
		return 0;

	return skb_checksum_setup(skb, recalculate_partial_csum);
}


static int netfront_probe(struct xenbus_device *dev,
			  const struct xenbus_device_id *id)
{
	int err;
	struct net_device *netdev;
	struct netfront_info *info;

	netdev = xennet_create_dev(dev);
	if (IS_ERR(netdev)) {
		err = PTR_ERR(netdev);
		xenbus_dev_fatal(dev, err, "creating netdev");
		return err;
	}

	info = netdev_priv(netdev);
	dev_set_drvdata(&dev->dev, info);
#ifdef CONFIG_SYSFS
	info->netdev->sysfs_groups[0] = &xennet_dev_group;
#endif

	return 0;
}


static void add_id_to_list(unsigned *head, unsigned short *list,
			   unsigned short id)
{
	list[id] = *head;
	*head = id;
}


static netdev_tx_t xennet_start_xmit(struct sk_buff *skb, struct net_device *dev)
{
	struct netfront_info *np = netdev_priv(dev);
	struct netfront_stats *tx_stats = this_cpu_ptr(np->tx_stats);
	struct xen_netif_tx_request *first_tx;
	unsigned int i;
	int notify;
	int slots;
	struct page *page;
	unsigned int offset;
	unsigned int len;
	unsigned long flags;
	struct netfront_queue *queue = NULL;
	struct xennet_gnttab_make_txreq info = { };
	unsigned int num_queues = dev->real_num_tx_queues;
	u16 queue_index;
	struct sk_buff *nskb;

	/* Drop the packet if no queues are set up */
	if (num_queues < 1)
		goto drop;
	if (unlikely(np->broken))
		goto drop;
	/* Determine which queue to transmit this SKB on */
	queue_index = skb_get_queue_mapping(skb);
	queue = &np->queues[queue_index];

	/* If skb->len is too big for wire format, drop skb and alert
	 * user about misconfiguration.
	 */
	if (unlikely(skb->len > XEN_NETIF_MAX_TX_SIZE)) {
		net_alert_ratelimited(
			"xennet: skb->len = %u, too big for wire format\n",
			skb->len);
		goto drop;
	}

	slots = xennet_count_skb_slots(skb);
	if (unlikely(slots > MAX_XEN_SKB_FRAGS + 1)) {
		net_dbg_ratelimited("xennet: skb rides the rocket: %d slots, %d bytes\n",
				    slots, skb->len);
		if (skb_linearize(skb))
			goto drop;
	}

	page = virt_to_page(skb->data);
	offset = offset_in_page(skb->data);

	/* The first req should be at least ETH_HLEN size or the packet will be
	 * dropped by netback.
	 *
	 * If the backend is not trusted bounce all data to zeroed pages to
	 * avoid exposing contiguous data on the granted page not belonging to
	 * the skb.
	 */
	if (np->bounce || unlikely(PAGE_SIZE - offset < ETH_HLEN)) {
		nskb = bounce_skb(skb);
		if (!nskb)
			goto drop;
		dev_consume_skb_any(skb);
		skb = nskb;
		page = virt_to_page(skb->data);
		offset = offset_in_page(skb->data);
	}

	len = skb_headlen(skb);

	spin_lock_irqsave(&queue->tx_lock, flags);

	if (unlikely(!netif_carrier_ok(dev) ||
		     (slots > 1 && !xennet_can_sg(dev)) ||
		     netif_needs_gso(skb, netif_skb_features(skb)))) {
		spin_unlock_irqrestore(&queue->tx_lock, flags);
		goto drop;
	}

	/* First request for the linear area. */
	info.queue = queue;
	info.skb = skb;
	info.page = page;
	first_tx = xennet_make_first_txreq(&info, offset, len);
	offset += info.tx_local.size;
	if (offset == PAGE_SIZE) {
		page++;
		offset = 0;
	}
	len -= info.tx_local.size;

	if (skb->ip_summed == CHECKSUM_PARTIAL)
		/* local packet? */
		first_tx->flags |= XEN_NETTXF_csum_blank |
				   XEN_NETTXF_data_validated;
	else if (skb->ip_summed == CHECKSUM_UNNECESSARY)
		/* remote but checksummed. */
		first_tx->flags |= XEN_NETTXF_data_validated;

	/* Optional extra info after the first request. */
	if (skb_shinfo(skb)->gso_size) {
		struct xen_netif_extra_info *gso;

		gso = (struct xen_netif_extra_info *)
			RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++);

		first_tx->flags |= XEN_NETTXF_extra_info;

		gso->u.gso.size = skb_shinfo(skb)->gso_size;
		gso->u.gso.type = (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) ?
			XEN_NETIF_GSO_TYPE_TCPV6 :
			XEN_NETIF_GSO_TYPE_TCPV4;
		gso->u.gso.pad = 0;
		gso->u.gso.features = 0;

		gso->type = XEN_NETIF_EXTRA_TYPE_GSO;
		gso->flags = 0;
	}

	/* Requests for the rest of the linear area. */
	xennet_make_txreqs(&info, page, offset, len);

	/* Requests for all the frags. */
	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
		xennet_make_txreqs(&info, skb_frag_page(frag),
					skb_frag_off(frag),
					skb_frag_size(frag));
	}

	/* First request has the packet length. */
	first_tx->size = skb->len;

	/* timestamp packet in software */
	skb_tx_timestamp(skb);

	xennet_mark_tx_pending(queue);

	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify);
	if (notify)
		notify_remote_via_irq(queue->tx_irq);

	u64_stats_update_begin(&tx_stats->syncp);
	tx_stats->bytes += skb->len;
	tx_stats->packets++;
	u64_stats_update_end(&tx_stats->syncp);

	/* Note: It is not safe to access skb after xennet_tx_buf_gc()! */
	xennet_tx_buf_gc(queue);

	if (!netfront_tx_slot_available(queue))
		netif_tx_stop_queue(netdev_get_tx_queue(dev, queue->id));

	spin_unlock_irqrestore(&queue->tx_lock, flags);

	return NETDEV_TX_OK;

 drop:
	dev->stats.tx_dropped++;
	dev_kfree_skb_any(skb);
	return NETDEV_TX_OK;
}


static void __exit netif_exit(void)
{
	xenbus_unregister_driver(&netfront_driver);
}


static int netfront_tx_slot_available(struct netfront_queue *queue)
{
	return (queue->tx.req_prod_pvt - queue->tx.rsp_cons) <
		(NET_TX_RING_SIZE - XEN_NETIF_NR_SLOTS_MIN - 1);
}


struct sk_buff *bounce_skb(const struct sk_buff *skb)
{
	unsigned int headerlen = skb_headroom(skb);
	/* Align size to allocate full pages and avoid contiguous data leaks */
	unsigned int size = ALIGN(skb_end_offset(skb) + skb->data_len,
				  XEN_PAGE_SIZE);
	struct sk_buff *n = alloc_skb(size, GFP_ATOMIC | __GFP_ZERO);

	if (!n)
		return NULL;

	if (!IS_ALIGNED((uintptr_t)n->head, XEN_PAGE_SIZE)) {
		WARN_ONCE(1, "misaligned skb allocated\n");
		kfree_skb(n);
		return NULL;
	}

	/* Set the data pointer */
	skb_reserve(n, headerlen);
	/* Set the tail pointer and length */
	skb_put(n, skb->len);

	BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len));

	skb_copy_header(n, skb);
	return n;
}


static irqreturn_t xennet_interrupt(int irq, void *dev_id)
{
	unsigned int eoiflag = XEN_EOI_FLAG_SPURIOUS;

	if (xennet_handle_tx(dev_id, &eoiflag) &&
	    xennet_handle_rx(dev_id, &eoiflag))
		xen_irq_lateeoi(irq, eoiflag);

	return IRQ_HANDLED;
}


static int xennet_fill_frags(struct netfront_queue *queue,
			     struct sk_buff *skb,
			     struct sk_buff_head *list)
{
	RING_IDX cons = queue->rx.rsp_cons;
	struct sk_buff *nskb;

	while ((nskb = __skb_dequeue(list))) {
		struct xen_netif_rx_response rx;
		skb_frag_t *nfrag = &skb_shinfo(nskb)->frags[0];

		RING_COPY_RESPONSE(&queue->rx, ++cons, &rx);

		if (skb_shinfo(skb)->nr_frags == MAX_SKB_FRAGS) {
			unsigned int pull_to = NETFRONT_SKB_CB(skb)->pull_to;

			BUG_ON(pull_to < skb_headlen(skb));
			__pskb_pull_tail(skb, pull_to - skb_headlen(skb));
		}
		if (unlikely(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS)) {
			xennet_set_rx_rsp_cons(queue,
					       ++cons + skb_queue_len(list));
			kfree_skb(nskb);
			return -ENOENT;
		}

		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
				skb_frag_page(nfrag),
				rx.offset, rx.status, PAGE_SIZE);

		skb_shinfo(nskb)->nr_frags = 0;
		kfree_skb(nskb);
	}

	xennet_set_rx_rsp_cons(queue, cons);

	return 0;
}


static int xennet_set_features(struct net_device *dev,
	netdev_features_t features)
{
	if (!(features & NETIF_F_SG) && dev->mtu > ETH_DATA_LEN) {
		netdev_info(dev, "Reducing MTU because no SG offload");
		dev->mtu = ETH_DATA_LEN;
	}

	return 0;
}


static bool xennet_can_sg(struct net_device *dev)
{
	return dev->features & NETIF_F_SG;
}


static int talk_to_netback(struct xenbus_device *dev,
			   struct netfront_info *info)
{
	const char *message;
	struct xenbus_transaction xbt;
	int err;
	unsigned int feature_split_evtchn;
	unsigned int i = 0;
	unsigned int max_queues = 0;
	struct netfront_queue *queue = NULL;
	unsigned int num_queues = 1;
	u8 addr[ETH_ALEN];

	info->netdev->irq = 0;

	/* Check if backend is trusted. */
	info->bounce = !xennet_trusted ||
		       !xenbus_read_unsigned(dev->nodename, "trusted", 1);

	/* Check if backend supports multiple queues */
	max_queues = xenbus_read_unsigned(info->xbdev->otherend,
					  "multi-queue-max-queues", 1);
	num_queues = min(max_queues, xennet_max_queues);

	/* Check feature-split-event-channels */
	feature_split_evtchn = xenbus_read_unsigned(info->xbdev->otherend,
					"feature-split-event-channels", 0);

	/* Read mac addr. */
	err = xen_net_read_mac(dev, addr);
	if (err) {
		xenbus_dev_fatal(dev, err, "parsing %s/mac", dev->nodename);
		goto out_unlocked;
	}
	eth_hw_addr_set(info->netdev, addr);

	info->netback_has_xdp_headroom = xenbus_read_unsigned(info->xbdev->otherend,
							      "feature-xdp-headroom", 0);
	if (info->netback_has_xdp_headroom) {
		/* set the current xen-netfront xdp state */
		err = talk_to_netback_xdp(info, info->netfront_xdp_enabled ?
					  NETBACK_XDP_HEADROOM_ENABLE :
					  NETBACK_XDP_HEADROOM_DISABLE);
		if (err)
			goto out_unlocked;
	}

	rtnl_lock();
	if (info->queues)
		xennet_destroy_queues(info);

	/* For the case of a reconnect reset the "broken" indicator. */
	info->broken = false;

	err = xennet_create_queues(info, &num_queues);
	if (err < 0) {
		xenbus_dev_fatal(dev, err, "creating queues");
		kfree(info->queues);
		info->queues = NULL;
		goto out;
	}
	rtnl_unlock();

	/* Create shared ring, alloc event channel -- for each queue */
	for (i = 0; i < num_queues; ++i) {
		queue = &info->queues[i];
		err = setup_netfront(dev, queue, feature_split_evtchn);
		if (err)
			goto destroy_ring;
	}

again:
	err = xenbus_transaction_start(&xbt);
	if (err) {
		xenbus_dev_fatal(dev, err, "starting transaction");
		goto destroy_ring;
	}

	if (xenbus_exists(XBT_NIL,
			  info->xbdev->otherend, "multi-queue-max-queues")) {
		/* Write the number of queues */
		err = xenbus_printf(xbt, dev->nodename,
				    "multi-queue-num-queues", "%u", num_queues);
		if (err) {
			message = "writing multi-queue-num-queues";
			goto abort_transaction_no_dev_fatal;
		}
	}

	if (num_queues == 1) {
		err = write_queue_xenstore_keys(&info->queues[0], &xbt, 0); /* flat */
		if (err)
			goto abort_transaction_no_dev_fatal;
	} else {
		/* Write the keys for each queue */
		for (i = 0; i < num_queues; ++i) {
			queue = &info->queues[i];
			err = write_queue_xenstore_keys(queue, &xbt, 1); /* hierarchical */
			if (err)
				goto abort_transaction_no_dev_fatal;
		}
	}

	/* The remaining keys are not queue-specific */
	err = xenbus_printf(xbt, dev->nodename, "request-rx-copy", "%u",
			    1);
	if (err) {
		message = "writing request-rx-copy";
		goto abort_transaction;
	}

	err = xenbus_printf(xbt, dev->nodename, "feature-rx-notify", "%d", 1);
	if (err) {
		message = "writing feature-rx-notify";
		goto abort_transaction;
	}

	err = xenbus_printf(xbt, dev->nodename, "feature-sg", "%d", 1);
	if (err) {
		message = "writing feature-sg";
		goto abort_transaction;
	}

	err = xenbus_printf(xbt, dev->nodename, "feature-gso-tcpv4", "%d", 1);
	if (err) {
		message = "writing feature-gso-tcpv4";
		goto abort_transaction;
	}

	err = xenbus_write(xbt, dev->nodename, "feature-gso-tcpv6", "1");
	if (err) {
		message = "writing feature-gso-tcpv6";
		goto abort_transaction;
	}

	err = xenbus_write(xbt, dev->nodename, "feature-ipv6-csum-offload",
			   "1");
	if (err) {
		message = "writing feature-ipv6-csum-offload";
		goto abort_transaction;
	}

	err = xenbus_transaction_end(xbt, 0);
	if (err) {
		if (err == -EAGAIN)
			goto again;
		xenbus_dev_fatal(dev, err, "completing transaction");
		goto destroy_ring;
	}

	return 0;

 abort_transaction:
	xenbus_dev_fatal(dev, err, "%s", message);
abort_transaction_no_dev_fatal:
	xenbus_transaction_end(xbt, 1);
 destroy_ring:
	xennet_disconnect_backend(info);
	rtnl_lock();
	xennet_destroy_queues(info);
 out:
	rtnl_unlock();
out_unlocked:
	device_unregister(&dev->dev);
	return err;
}


static void xennet_tx_setup_grant(unsigned long gfn, unsigned int offset,
				  unsigned int len, void *data)
{
	struct xennet_gnttab_make_txreq *info = data;
	unsigned int id;
	struct xen_netif_tx_request *tx;
	grant_ref_t ref;
	/* convenient aliases */
	struct page *page = info->page;
	struct netfront_queue *queue = info->queue;
	struct sk_buff *skb = info->skb;

	id = get_id_from_list(&queue->tx_skb_freelist, queue->tx_link);
	tx = RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++);
	ref = gnttab_claim_grant_reference(&queue->gref_tx_head);
	WARN_ON_ONCE(IS_ERR_VALUE((unsigned long)(int)ref));

	gnttab_grant_foreign_access_ref(ref, queue->info->xbdev->otherend_id,
					gfn, GNTMAP_readonly);

	queue->tx_skbs[id] = skb;
	queue->grant_tx_page[id] = page;
	queue->grant_tx_ref[id] = ref;

	info->tx_local.id = id;
	info->tx_local.gref = ref;
	info->tx_local.offset = offset;
	info->tx_local.size = len;
	info->tx_local.flags = 0;

	*tx = info->tx_local;

	/*
	 * Put the request in the pending queue, it will be set to be pending
	 * when the producer index is about to be raised.
	 */
	add_id_to_list(&queue->tx_pend_queue, queue->tx_link, id);

	info->tx = tx;
	info->size += info->tx_local.size;
}


static int netfront_resume(struct xenbus_device *dev)
{
	struct netfront_info *info = dev_get_drvdata(&dev->dev);

	dev_dbg(&dev->dev, "%s\n", dev->nodename);

	netif_tx_lock_bh(info->netdev);
	netif_device_detach(info->netdev);
	netif_tx_unlock_bh(info->netdev);

	xennet_disconnect_backend(info);
	return 0;
}


static void xennet_end_access(int ref, void *page)
{
	/* This frees the page as a side-effect */
	if (ref != INVALID_GRANT_REF)
		gnttab_end_foreign_access(ref, virt_to_page(page));
}


static u16 xennet_select_queue(struct net_device *dev, struct sk_buff *skb,
			       struct net_device *sb_dev)
{
	unsigned int num_queues = dev->real_num_tx_queues;
	u32 hash;
	u16 queue_idx;

	/* First, check if there is only one queue */
	if (num_queues == 1) {
		queue_idx = 0;
	} else {
		hash = skb_get_hash(skb);
		queue_idx = hash % num_queues;
	}

	return queue_idx;
}


static int xennet_remove(struct xenbus_device *dev)
{
	struct netfront_info *info = dev_get_drvdata(&dev->dev);

	xennet_bus_close(dev);
	xennet_disconnect_backend(info);

	if (info->netdev->reg_state == NETREG_REGISTERED)
		unregister_netdev(info->netdev);

	if (info->queues) {
		rtnl_lock();
		xennet_destroy_queues(info);
		rtnl_unlock();
	}
	xennet_free_netdev(info->netdev);

	return 0;
}


static int xen_net_read_mac(struct xenbus_device *dev, u8 mac[])
{
	char *s, *e, *macstr;
	int i;

	macstr = s = xenbus_read(XBT_NIL, dev->nodename, "mac", NULL);
	if (IS_ERR(macstr))
		return PTR_ERR(macstr);

	for (i = 0; i < ETH_ALEN; i++) {
		mac[i] = simple_strtoul(s, &e, 16);
		if ((s == e) || (*e != ((i == ETH_ALEN-1) ? '\0' : ':'))) {
			kfree(macstr);
			return -ENOENT;
		}
		s = e+1;
	}

	kfree(macstr);
	return 0;
}


static void xennet_disconnect_backend(struct netfront_info *info)
{
	unsigned int i = 0;
	unsigned int num_queues = info->netdev->real_num_tx_queues;

	netif_carrier_off(info->netdev);

	for (i = 0; i < num_queues && info->queues; ++i) {
		struct netfront_queue *queue = &info->queues[i];

		del_timer_sync(&queue->rx_refill_timer);

		if (queue->tx_irq && (queue->tx_irq == queue->rx_irq))
			unbind_from_irqhandler(queue->tx_irq, queue);
		if (queue->tx_irq && (queue->tx_irq != queue->rx_irq)) {
			unbind_from_irqhandler(queue->tx_irq, queue);
			unbind_from_irqhandler(queue->rx_irq, queue);
		}
		queue->tx_evtchn = queue->rx_evtchn = 0;
		queue->tx_irq = queue->rx_irq = 0;

		if (netif_running(info->netdev))
			napi_synchronize(&queue->napi);

		xennet_release_tx_bufs(queue);
		xennet_release_rx_bufs(queue);
		gnttab_free_grant_references(queue->gref_tx_head);
		gnttab_free_grant_references(queue->gref_rx_head);

		/* End access and free the pages */
		xennet_end_access(queue->tx_ring_ref, queue->tx.sring);
		xennet_end_access(queue->rx_ring_ref, queue->rx.sring);

		queue->tx_ring_ref = INVALID_GRANT_REF;
		queue->rx_ring_ref = INVALID_GRANT_REF;
		queue->tx.sring = NULL;
		queue->rx.sring = NULL;

		page_pool_destroy(queue->page_pool);
	}
}


static struct xen_netif_tx_request *xennet_make_first_txreq(
	struct xennet_gnttab_make_txreq *info,
	unsigned int offset, unsigned int len)
{
	info->size = 0;

	gnttab_for_one_grant(info->page, offset, len, xennet_tx_setup_grant, info);

	return info->tx;
}


static void xennet_bus_close(struct xenbus_device *dev)
{
	int ret;

	if (xenbus_read_driver_state(dev->otherend) == XenbusStateClosed)
		return;
	do {
		xenbus_switch_state(dev, XenbusStateClosing);
		ret = wait_event_timeout(module_wq,
				   xenbus_read_driver_state(dev->otherend) ==
				   XenbusStateClosing ||
				   xenbus_read_driver_state(dev->otherend) ==
				   XenbusStateClosed ||
				   xenbus_read_driver_state(dev->otherend) ==
				   XenbusStateUnknown,
				   XENNET_TIMEOUT);
	} while (!ret);

	if (xenbus_read_driver_state(dev->otherend) == XenbusStateClosed)
		return;

	do {
		xenbus_switch_state(dev, XenbusStateClosed);
		ret = wait_event_timeout(module_wq,
				   xenbus_read_driver_state(dev->otherend) ==
				   XenbusStateClosed ||
				   xenbus_read_driver_state(dev->otherend) ==
				   XenbusStateUnknown,
				   XENNET_TIMEOUT);
	} while (!ret);
}


static void xennet_make_one_txreq(unsigned long gfn, unsigned int offset,
				  unsigned int len, void *data)
{
	struct xennet_gnttab_make_txreq *info = data;

	info->tx->flags |= XEN_NETTXF_more_data;
	skb_get(info->skb);
	xennet_tx_setup_grant(gfn, offset, len, data);
}


static void xennet_mark_tx_pending(struct netfront_queue *queue)
{
	unsigned int i;

	while ((i = get_id_from_list(&queue->tx_pend_queue, queue->tx_link)) !=
	       TX_LINK_NONE)
		queue->tx_link[i] = TX_PENDING;
}


static struct net_device *xennet_create_dev(struct xenbus_device *dev)
{
	int err;
	struct net_device *netdev;
	struct netfront_info *np;

	netdev = alloc_etherdev_mq(sizeof(struct netfront_info), xennet_max_queues);
	if (!netdev)
		return ERR_PTR(-ENOMEM);

	np                   = netdev_priv(netdev);
	np->xbdev            = dev;

	np->queues = NULL;

	err = -ENOMEM;
	np->rx_stats = netdev_alloc_pcpu_stats(struct netfront_stats);
	if (np->rx_stats == NULL)
		goto exit;
	np->tx_stats = netdev_alloc_pcpu_stats(struct netfront_stats);
	if (np->tx_stats == NULL)
		goto exit;

	netdev->netdev_ops	= &xennet_netdev_ops;

	netdev->features        = NETIF_F_IP_CSUM | NETIF_F_RXCSUM |
				  NETIF_F_GSO_ROBUST;
	netdev->hw_features	= NETIF_F_SG |
				  NETIF_F_IPV6_CSUM |
				  NETIF_F_TSO | NETIF_F_TSO6;

	/*
         * Assume that all hw features are available for now. This set
         * will be adjusted by the call to netdev_update_features() in
         * xennet_connect() which is the earliest point where we can
         * negotiate with the backend regarding supported features.
         */
	netdev->features |= netdev->hw_features;

	netdev->ethtool_ops = &xennet_ethtool_ops;
	netdev->min_mtu = ETH_MIN_MTU;
	netdev->max_mtu = XEN_NETIF_MAX_TX_SIZE;
	SET_NETDEV_DEV(netdev, &dev->dev);

	np->netdev = netdev;
	np->netfront_xdp_enabled = false;

	netif_carrier_off(netdev);

	do {
		xenbus_switch_state(dev, XenbusStateInitialising);
		err = wait_event_timeout(module_wq,
				 xenbus_read_driver_state(dev->otherend) !=
				 XenbusStateClosed &&
				 xenbus_read_driver_state(dev->otherend) !=
				 XenbusStateUnknown, XENNET_TIMEOUT);
	} while (!err);

	return netdev;

 exit:
	xennet_free_netdev(netdev);
	return ERR_PTR(err);
}


static int unzzip_cat (int argc, char ** argv, int extract)
{
    int argn;
    ZZIP_DIR* disk;
    zzip_error_t error;

    if (argc == 1)
    {
        printf (__FILE__" version "ZZIP_PACKAGE" "ZZIP_VERSION"\n");
        return -1; /* better provide an archive argument */
    }

    disk = zzip_dir_open (argv[1], &error);
    if (! disk) {
	fprintf(stderr, "%s: %s\n", argv[1], zzip_strerror(error));
	return -1;
    }

    if (argc == 2)
    {  /* list all */
	ZZIP_DIRENT entry;
	while(zzip_dir_read(disk, &entry))
	{
	    char* name = entry.d_name;
	    FILE* out = stdout;
	    if (extract) out = create_fopen(name, "w", 1);
	    if (out) {
	        unzzip_cat_file (disk, name, out);
	        if (extract) fclose(out);
	    }
	}
    }
    else
    {   /* list only the matching entries - in order of zip directory */
	ZZIP_DIRENT entry;
	while(zzip_dir_read(disk, &entry))
	{
	    char* name = entry.d_name;
	    for (argn=1; argn < argc; argn++)
	    {
		if (! _zzip_fnmatch (argv[argn], name,
		    FNM_NOESCAPE|FNM_PATHNAME|FNM_PERIOD))
	        {
	            FILE* out = stdout;
	            if (extract) out = create_fopen(name, "w", 1);
	            if (out) {
		        unzzip_cat_file (disk, name, out);
		        if (extract) fclose(out);
		    }
		    break; /* match loop */
	        }
	    }
	}
    }
    zzip_dir_close(disk);
    return 0;
}

Reject the ZIP file and report it as corrupt if the size of the
central directory and/or the offset of start of central directory
point beyond the end of the ZIP file. [CVE-2018-6484]


__zzip_aligned4(char *p)
{
#define aligned4   __zzip_aligned4
    p += ((long) p) & 1;        /* warnings about truncation of a "pointer" */
    p += ((long) p) & 2;        /* to a "long int" may be safely ignored :) */
    return p;
}


int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
{
	const struct x86_emulate_ops *ops = ctxt->ops;
	int rc = X86EMUL_CONTINUE;
	int saved_dst_type = ctxt->dst.type;

	ctxt->mem_read.pos = 0;

	/* LOCK prefix is allowed only with some instructions */
	if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
		rc = emulate_ud(ctxt);
		goto done;
	}

	if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
		rc = emulate_ud(ctxt);
		goto done;
	}

	if (unlikely(ctxt->d &
		     (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
		if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
				(ctxt->d & Undefined)) {
			rc = emulate_ud(ctxt);
			goto done;
		}

		if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
		    || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
			rc = emulate_ud(ctxt);
			goto done;
		}

		if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
			rc = emulate_nm(ctxt);
			goto done;
		}

		if (ctxt->d & Mmx) {
			rc = flush_pending_x87_faults(ctxt);
			if (rc != X86EMUL_CONTINUE)
				goto done;
			/*
			 * Now that we know the fpu is exception safe, we can fetch
			 * operands from it.
			 */
			fetch_possible_mmx_operand(ctxt, &ctxt->src);
			fetch_possible_mmx_operand(ctxt, &ctxt->src2);
			if (!(ctxt->d & Mov))
				fetch_possible_mmx_operand(ctxt, &ctxt->dst);
		}

		if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
			rc = emulator_check_intercept(ctxt, ctxt->intercept,
						      X86_ICPT_PRE_EXCEPT);
			if (rc != X86EMUL_CONTINUE)
				goto done;
		}

		/* Privileged instruction can be executed only in CPL=0 */
		if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
			if (ctxt->d & PrivUD)
				rc = emulate_ud(ctxt);
			else
				rc = emulate_gp(ctxt, 0);
			goto done;
		}

		/* Instruction can only be executed in protected mode */
		if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
			rc = emulate_ud(ctxt);
			goto done;
		}

		/* Do instruction specific permission checks */
		if (ctxt->d & CheckPerm) {
			rc = ctxt->check_perm(ctxt);
			if (rc != X86EMUL_CONTINUE)
				goto done;
		}

		if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
			rc = emulator_check_intercept(ctxt, ctxt->intercept,
						      X86_ICPT_POST_EXCEPT);
			if (rc != X86EMUL_CONTINUE)
				goto done;
		}

		if (ctxt->rep_prefix && (ctxt->d & String)) {
			/* All REP prefixes have the same first termination condition */
			if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
				ctxt->eip = ctxt->_eip;
				ctxt->eflags &= ~EFLG_RF;
				goto done;
			}
		}
	}

	if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
		rc = segmented_read(ctxt, ctxt->src.addr.mem,
				    ctxt->src.valptr, ctxt->src.bytes);
		if (rc != X86EMUL_CONTINUE)
			goto done;
		ctxt->src.orig_val64 = ctxt->src.val64;
	}

	if (ctxt->src2.type == OP_MEM) {
		rc = segmented_read(ctxt, ctxt->src2.addr.mem,
				    &ctxt->src2.val, ctxt->src2.bytes);
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

	if ((ctxt->d & DstMask) == ImplicitOps)
		goto special_insn;

	if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
		/* optimisation - avoid slow emulated read if Mov */
		rc = segmented_read(ctxt, ctxt->dst.addr.mem,
				   &ctxt->dst.val, ctxt->dst.bytes);
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}
	ctxt->dst.orig_val = ctxt->dst.val;

special_insn:

	if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
		rc = emulator_check_intercept(ctxt, ctxt->intercept,
					      X86_ICPT_POST_MEMACCESS);
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

	if (ctxt->rep_prefix && (ctxt->d & String))
		ctxt->eflags |= EFLG_RF;
	else
		ctxt->eflags &= ~EFLG_RF;

	if (ctxt->execute) {
		if (ctxt->d & Fastop) {
			void (*fop)(struct fastop *) = (void *)ctxt->execute;
			rc = fastop(ctxt, fop);
			if (rc != X86EMUL_CONTINUE)
				goto done;
			goto writeback;
		}
		rc = ctxt->execute(ctxt);
		if (rc != X86EMUL_CONTINUE)
			goto done;
		goto writeback;
	}

	if (ctxt->opcode_len == 2)
		goto twobyte_insn;
	else if (ctxt->opcode_len == 3)
		goto threebyte_insn;

	switch (ctxt->b) {
	case 0x63:		/* movsxd */
		if (ctxt->mode != X86EMUL_MODE_PROT64)
			goto cannot_emulate;
		ctxt->dst.val = (s32) ctxt->src.val;
		break;
	case 0x70 ... 0x7f: /* jcc (short) */
		if (test_cc(ctxt->b, ctxt->eflags))
			rc = jmp_rel(ctxt, ctxt->src.val);
		break;
	case 0x8d: /* lea r16/r32, m */
		ctxt->dst.val = ctxt->src.addr.mem.ea;
		break;
	case 0x90 ... 0x97: /* nop / xchg reg, rax */
		if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
			ctxt->dst.type = OP_NONE;
		else
			rc = em_xchg(ctxt);
		break;
	case 0x98: /* cbw/cwde/cdqe */
		switch (ctxt->op_bytes) {
		case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
		case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
		case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
		}
		break;
	case 0xcc:		/* int3 */
		rc = emulate_int(ctxt, 3);
		break;
	case 0xcd:		/* int n */
		rc = emulate_int(ctxt, ctxt->src.val);
		break;
	case 0xce:		/* into */
		if (ctxt->eflags & EFLG_OF)
			rc = emulate_int(ctxt, 4);
		break;
	case 0xe9: /* jmp rel */
	case 0xeb: /* jmp rel short */
		rc = jmp_rel(ctxt, ctxt->src.val);
		ctxt->dst.type = OP_NONE; /* Disable writeback. */
		break;
	case 0xf4:              /* hlt */
		ctxt->ops->halt(ctxt);
		break;
	case 0xf5:	/* cmc */
		/* complement carry flag from eflags reg */
		ctxt->eflags ^= EFLG_CF;
		break;
	case 0xf8: /* clc */
		ctxt->eflags &= ~EFLG_CF;
		break;
	case 0xf9: /* stc */
		ctxt->eflags |= EFLG_CF;
		break;
	case 0xfc: /* cld */
		ctxt->eflags &= ~EFLG_DF;
		break;
	case 0xfd: /* std */
		ctxt->eflags |= EFLG_DF;
		break;
	default:
		goto cannot_emulate;
	}

	if (rc != X86EMUL_CONTINUE)
		goto done;

writeback:
	if (ctxt->d & SrcWrite) {
		BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
		rc = writeback(ctxt, &ctxt->src);
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}
	if (!(ctxt->d & NoWrite)) {
		rc = writeback(ctxt, &ctxt->dst);
		if (rc != X86EMUL_CONTINUE)
			goto done;
	}

	/*
	 * restore dst type in case the decoding will be reused
	 * (happens for string instruction )
	 */
	ctxt->dst.type = saved_dst_type;

	if ((ctxt->d & SrcMask) == SrcSI)
		string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);

	if ((ctxt->d & DstMask) == DstDI)
		string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);

	if (ctxt->rep_prefix && (ctxt->d & String)) {
		unsigned int count;
		struct read_cache *r = &ctxt->io_read;
		if ((ctxt->d & SrcMask) == SrcSI)
			count = ctxt->src.count;
		else
			count = ctxt->dst.count;
		register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
				-count);

		if (!string_insn_completed(ctxt)) {
			/*
			 * Re-enter guest when pio read ahead buffer is empty
			 * or, if it is not used, after each 1024 iteration.
			 */
			if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
			    (r->end == 0 || r->end != r->pos)) {
				/*
				 * Reset read cache. Usually happens before
				 * decode, but since instruction is restarted
				 * we have to do it here.
				 */
				ctxt->mem_read.end = 0;
				writeback_registers(ctxt);
				return EMULATION_RESTART;
			}
			goto done; /* skip rip writeback */
		}
		ctxt->eflags &= ~EFLG_RF;
	}

	ctxt->eip = ctxt->_eip;

done:
	if (rc == X86EMUL_PROPAGATE_FAULT) {
		WARN_ON(ctxt->exception.vector > 0x1f);
		ctxt->have_exception = true;
	}
	if (rc == X86EMUL_INTERCEPTED)
		return EMULATION_INTERCEPTED;

	if (rc == X86EMUL_CONTINUE)
		writeback_registers(ctxt);

	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;

twobyte_insn:
	switch (ctxt->b) {
	case 0x09:		/* wbinvd */
		(ctxt->ops->wbinvd)(ctxt);
		break;
	case 0x08:		/* invd */
	case 0x0d:		/* GrpP (prefetch) */
	case 0x18:		/* Grp16 (prefetch/nop) */
	case 0x1f:		/* nop */
		break;
	case 0x20: /* mov cr, reg */
		ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
		break;
	case 0x21: /* mov from dr to reg */
		ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
		break;
	case 0x40 ... 0x4f:	/* cmov */
		if (test_cc(ctxt->b, ctxt->eflags))
			ctxt->dst.val = ctxt->src.val;
		else if (ctxt->mode != X86EMUL_MODE_PROT64 ||
			 ctxt->op_bytes != 4)
			ctxt->dst.type = OP_NONE; /* no writeback */
		break;
	case 0x80 ... 0x8f: /* jnz rel, etc*/
		if (test_cc(ctxt->b, ctxt->eflags))
			rc = jmp_rel(ctxt, ctxt->src.val);
		break;
	case 0x90 ... 0x9f:     /* setcc r/m8 */
		ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
		break;
	case 0xb6 ... 0xb7:	/* movzx */
		ctxt->dst.bytes = ctxt->op_bytes;
		ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
						       : (u16) ctxt->src.val;
		break;
	case 0xbe ... 0xbf:	/* movsx */
		ctxt->dst.bytes = ctxt->op_bytes;
		ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
							(s16) ctxt->src.val;
		break;
	case 0xc3:		/* movnti */
		ctxt->dst.bytes = ctxt->op_bytes;
		ctxt->dst.val = (ctxt->op_bytes == 8) ? (u64) ctxt->src.val :
							(u32) ctxt->src.val;
		break;
	default:
		goto cannot_emulate;
	}

threebyte_insn:

	if (rc != X86EMUL_CONTINUE)
		goto done;

	goto writeback;

cannot_emulate:
	return EMULATION_FAILED;
}

KVM: x86: Emulator does not decode clflush well

Currently, all group15 instructions are decoded as clflush (e.g., mfence,
xsave).  In addition, the clflush instruction requires no prefix (66/f2/f3)
would exist. If prefix exists it may encode a different instruction (e.g.,
clflushopt).

Creating a group for clflush, and different group for each prefix.

This has been the case forever, but the next patch needs the cflush group
in order to fix a bug introduced in 3.17.

Fixes: 41061cdb98a0bec464278b4db8e894a3121671f5
Cc: stable@vger.kernel.org
Signed-off-by: Nadav Amit <namit@cs.technion.ac.il>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>


static int em_clflush(struct x86_emulate_ctxt *ctxt)
{
	/* emulating clflush regardless of cpuid */
	return X86EMUL_CONTINUE;
}


static inline unsigned tx_desc_get_length(uint32_t *desc)
{
    return desc[1] & DESC_1_LENGTH;
}

cadence_gem: switch to use qemu_receive_packet() for loopback

This patch switches to use qemu_receive_packet() which can detect
reentrancy and return early.

This is intended to address CVE-2021-3416.

Cc: Prasad J Pandit <ppandit@redhat.com>
Cc: qemu-stable@nongnu.org
Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>
Signed-off-by: Alexander Bulekov <alxndr@bu.edu>
Signed-off-by: Jason Wang <jasowang@redhat.com>


static inline uint32_t gem_get_rx_queue_base_addr(CadenceGEMState *s, int q)
{
    return gem_get_queue_base_addr(s, false, q);
}


static hwaddr gem_get_rx_desc_addr(CadenceGEMState *s, int q)
{
    return gem_get_desc_addr(s, false, q);
}


static void phy_update_link(CadenceGEMState *s)
{
    DB_PRINT("down %d\n", qemu_get_queue(s->nic)->link_down);

    /* Autonegotiation status mirrors link status.  */
    if (qemu_get_queue(s->nic)->link_down) {
        s->phy_regs[PHY_REG_STATUS] &= ~(PHY_REG_STATUS_ANEGCMPL |
                                         PHY_REG_STATUS_LINK);
        s->phy_regs[PHY_REG_INT_ST] |= PHY_REG_INT_ST_LINKC;
    } else {
        s->phy_regs[PHY_REG_STATUS] |= (PHY_REG_STATUS_ANEGCMPL |
                                         PHY_REG_STATUS_LINK);
        s->phy_regs[PHY_REG_INT_ST] |= (PHY_REG_INT_ST_LINKC |
                                        PHY_REG_INT_ST_ANEGCMPL |
                                        PHY_REG_INT_ST_ENERGY);
    }
}


static inline int gem_get_desc_len(CadenceGEMState *s, bool rx_n_tx)
{
    int ret = 2;

    if (s->regs[GEM_DMACFG] & GEM_DMACFG_ADDR_64B) {
        ret += 2;
    }
    if (s->regs[GEM_DMACFG] & (rx_n_tx ? GEM_DMACFG_RX_BD_EXT
                                       : GEM_DMACFG_TX_BD_EXT)) {
        ret += 2;
    }

    assert(ret <= DESC_MAX_NUM_WORDS);
    return ret;
}


static void gem_set_link(NetClientState *nc)
{
    CadenceGEMState *s = qemu_get_nic_opaque(nc);

    DB_PRINT("\n");
    phy_update_link(s);
    gem_update_int_status(s);
}


static inline uint64_t rx_desc_get_buffer(CadenceGEMState *s, uint32_t *desc)
{
    uint64_t ret = desc[0] & ~0x3UL;

    if (s->regs[GEM_DMACFG] & GEM_DMACFG_ADDR_64B) {
        ret |= (uint64_t)desc[2] << 32;
    }
    return ret;
}


static hwaddr gem_get_tx_desc_addr(CadenceGEMState *s, int q)
{
    return gem_get_desc_addr(s, true, q);
}


static void gem_phy_reset(CadenceGEMState *s)
{
    memset(&s->phy_regs[0], 0, sizeof(s->phy_regs));
    s->phy_regs[PHY_REG_CONTROL] = 0x1140;
    s->phy_regs[PHY_REG_STATUS] = 0x7969;
    s->phy_regs[PHY_REG_PHYID1] = 0x0141;
    s->phy_regs[PHY_REG_PHYID2] = 0x0CC2;
    s->phy_regs[PHY_REG_ANEGADV] = 0x01E1;
    s->phy_regs[PHY_REG_LINKPABIL] = 0xCDE1;
    s->phy_regs[PHY_REG_ANEGEXP] = 0x000F;
    s->phy_regs[PHY_REG_NEXTP] = 0x2001;
    s->phy_regs[PHY_REG_LINKPNEXTP] = 0x40E6;
    s->phy_regs[PHY_REG_100BTCTRL] = 0x0300;
    s->phy_regs[PHY_REG_1000BTSTAT] = 0x7C00;
    s->phy_regs[PHY_REG_EXTSTAT] = 0x3000;
    s->phy_regs[PHY_REG_PHYSPCFC_CTL] = 0x0078;
    s->phy_regs[PHY_REG_PHYSPCFC_ST] = 0x7C00;
    s->phy_regs[PHY_REG_EXT_PHYSPCFC_CTL] = 0x0C60;
    s->phy_regs[PHY_REG_LED] = 0x4100;
    s->phy_regs[PHY_REG_EXT_PHYSPCFC_CTL2] = 0x000A;
    s->phy_regs[PHY_REG_EXT_PHYSPCFC_ST] = 0x848B;

    phy_update_link(s);
}


static inline void tx_desc_set_used(uint32_t *desc)
{
    desc[1] |= DESC_1_USED;
}


static void gem_update_int_status(CadenceGEMState *s)
{
    int i;

    qemu_set_irq(s->irq[0], !!s->regs[GEM_ISR]);

    for (i = 1; i < s->num_priority_queues; ++i) {
        qemu_set_irq(s->irq[i], !!s->regs[GEM_INT_Q1_STATUS + i - 1]);
    }
}


static inline unsigned tx_desc_get_wrap(uint32_t *desc)
{
    return (desc[1] & DESC_1_TX_WRAP) ? 1 : 0;
}


static void gem_transmit(CadenceGEMState *s)
{
    uint32_t desc[DESC_MAX_NUM_WORDS];
    hwaddr packet_desc_addr;
    uint8_t     *p;
    unsigned    total_bytes;
    int q = 0;

    /* Do nothing if transmit is not enabled. */
    if (!(s->regs[GEM_NWCTRL] & GEM_NWCTRL_TXENA)) {
        return;
    }

    DB_PRINT("\n");

    /* The packet we will hand off to QEMU.
     * Packets scattered across multiple descriptors are gathered to this
     * one contiguous buffer first.
     */
    p = s->tx_packet;
    total_bytes = 0;

    for (q = s->num_priority_queues - 1; q >= 0; q--) {
        /* read current descriptor */
        packet_desc_addr = gem_get_tx_desc_addr(s, q);

        DB_PRINT("read descriptor 0x%" HWADDR_PRIx "\n", packet_desc_addr);
        address_space_read(&s->dma_as, packet_desc_addr,
                           MEMTXATTRS_UNSPECIFIED, desc,
                           sizeof(uint32_t) * gem_get_desc_len(s, false));
        /* Handle all descriptors owned by hardware */
        while (tx_desc_get_used(desc) == 0) {

            /* Do nothing if transmit is not enabled. */
            if (!(s->regs[GEM_NWCTRL] & GEM_NWCTRL_TXENA)) {
                return;
            }
            print_gem_tx_desc(desc, q);

            /* The real hardware would eat this (and possibly crash).
             * For QEMU let's lend a helping hand.
             */
            if ((tx_desc_get_buffer(s, desc) == 0) ||
                (tx_desc_get_length(desc) == 0)) {
                DB_PRINT("Invalid TX descriptor @ 0x%" HWADDR_PRIx "\n",
                         packet_desc_addr);
                break;
            }

            if (tx_desc_get_length(desc) > gem_get_max_buf_len(s, true) -
                                               (p - s->tx_packet)) {
                qemu_log_mask(LOG_GUEST_ERROR, "TX descriptor @ 0x%" \
                         HWADDR_PRIx " too large: size 0x%x space 0x%zx\n",
                         packet_desc_addr, tx_desc_get_length(desc),
                         gem_get_max_buf_len(s, true) - (p - s->tx_packet));
                gem_set_isr(s, q, GEM_INT_AMBA_ERR);
                break;
            }

            /* Gather this fragment of the packet from "dma memory" to our
             * contig buffer.
             */
            address_space_read(&s->dma_as, tx_desc_get_buffer(s, desc),
                               MEMTXATTRS_UNSPECIFIED,
                               p, tx_desc_get_length(desc));
            p += tx_desc_get_length(desc);
            total_bytes += tx_desc_get_length(desc);

            /* Last descriptor for this packet; hand the whole thing off */
            if (tx_desc_get_last(desc)) {
                uint32_t desc_first[DESC_MAX_NUM_WORDS];
                hwaddr desc_addr = gem_get_tx_desc_addr(s, q);

                /* Modify the 1st descriptor of this packet to be owned by
                 * the processor.
                 */
                address_space_read(&s->dma_as, desc_addr,
                                   MEMTXATTRS_UNSPECIFIED, desc_first,
                                   sizeof(desc_first));
                tx_desc_set_used(desc_first);
                address_space_write(&s->dma_as, desc_addr,
                                    MEMTXATTRS_UNSPECIFIED, desc_first,
                                    sizeof(desc_first));
                /* Advance the hardware current descriptor past this packet */
                if (tx_desc_get_wrap(desc)) {
                    s->tx_desc_addr[q] = gem_get_tx_queue_base_addr(s, q);
                } else {
                    s->tx_desc_addr[q] = packet_desc_addr +
                                         4 * gem_get_desc_len(s, false);
                }
                DB_PRINT("TX descriptor next: 0x%08x\n", s->tx_desc_addr[q]);

                s->regs[GEM_TXSTATUS] |= GEM_TXSTATUS_TXCMPL;
                gem_set_isr(s, q, GEM_INT_TXCMPL);

                /* Handle interrupt consequences */
                gem_update_int_status(s);

                /* Is checksum offload enabled? */
                if (s->regs[GEM_DMACFG] & GEM_DMACFG_TXCSUM_OFFL) {
                    net_checksum_calculate(s->tx_packet, total_bytes, CSUM_ALL);
                }

                /* Update MAC statistics */
                gem_transmit_updatestats(s, s->tx_packet, total_bytes);

                /* Send the packet somewhere */
                if (s->phy_loop || (s->regs[GEM_NWCTRL] &
                                    GEM_NWCTRL_LOCALLOOP)) {
                    qemu_receive_packet(qemu_get_queue(s->nic), s->tx_packet,
                                        total_bytes);
                } else {
                    qemu_send_packet(qemu_get_queue(s->nic), s->tx_packet,
                                     total_bytes);
                }

                /* Prepare for next packet */
                p = s->tx_packet;
                total_bytes = 0;
            }

            /* read next descriptor */
            if (tx_desc_get_wrap(desc)) {

                if (s->regs[GEM_DMACFG] & GEM_DMACFG_ADDR_64B) {
                    packet_desc_addr = s->regs[GEM_TBQPH];
                    packet_desc_addr <<= 32;
                } else {
                    packet_desc_addr = 0;
                }
                packet_desc_addr |= gem_get_tx_queue_base_addr(s, q);
            } else {
                packet_desc_addr += 4 * gem_get_desc_len(s, false);
            }
            DB_PRINT("read descriptor 0x%" HWADDR_PRIx "\n", packet_desc_addr);
            address_space_read(&s->dma_as, packet_desc_addr,
                               MEMTXATTRS_UNSPECIFIED, desc,
                               sizeof(uint32_t) * gem_get_desc_len(s, false));
        }

        if (tx_desc_get_used(desc)) {
            s->regs[GEM_TXSTATUS] |= GEM_TXSTATUS_USED;
            /* IRQ TXUSED is defined only for queue 0 */
            if (q == 0) {
                gem_set_isr(s, 0, GEM_INT_TXUSED);
            }
            gem_update_int_status(s);
        }
    }
}


static inline void rx_desc_set_sof(uint32_t *desc)
{
    desc[1] |= DESC_1_RX_SOF;
}


static void gem_get_rx_desc(CadenceGEMState *s, int q)
{
    hwaddr desc_addr = gem_get_rx_desc_addr(s, q);

    DB_PRINT("read descriptor 0x%" HWADDR_PRIx "\n", desc_addr);

    /* read current descriptor */
    address_space_read(&s->dma_as, desc_addr, MEMTXATTRS_UNSPECIFIED,
                       s->rx_desc[q],
                       sizeof(uint32_t) * gem_get_desc_len(s, true));

    /* Descriptor owned by software ? */
    if (rx_desc_get_ownership(s->rx_desc[q]) == 1) {
        DB_PRINT("descriptor 0x%" HWADDR_PRIx " owned by sw.\n", desc_addr);
        s->regs[GEM_RXSTATUS] |= GEM_RXSTATUS_NOBUF;
        gem_set_isr(s, q, GEM_INT_RXUSED);
        /* Handle interrupt consequences */
        gem_update_int_status(s);
    }
}


static void gem_init(Object *obj)
{
    CadenceGEMState *s = CADENCE_GEM(obj);
    DeviceState *dev = DEVICE(obj);

    DB_PRINT("\n");

    gem_init_register_masks(s);
    memory_region_init_io(&s->iomem, OBJECT(s), &gem_ops, s,
                          "enet", sizeof(s->regs));

    sysbus_init_mmio(SYS_BUS_DEVICE(dev), &s->iomem);

    object_property_add_link(obj, "dma", TYPE_MEMORY_REGION,
                             (Object **)&s->dma_mr,
                             qdev_prop_allow_set_link_before_realize,
                             OBJ_PROP_LINK_STRONG);
}


static inline void rx_desc_set_multicast_hash(uint32_t *desc)
{
    desc[1] |= R_DESC_1_RX_MULTICAST_HASH;
}


static inline void rx_desc_set_unicast_hash(uint32_t *desc)
{
    desc[1] |= R_DESC_1_RX_UNICAST_HASH;
}


static inline unsigned tx_desc_get_used(uint32_t *desc)
{
    return (desc[1] & DESC_1_USED) ? 1 : 0;
}


static int get_queue_from_screen(CadenceGEMState *s, uint8_t *rxbuf_ptr,
                                 unsigned rxbufsize)
{
    uint32_t reg;
    bool matched, mismatched;
    int i, j;

    for (i = 0; i < s->num_type1_screeners; i++) {
        reg = s->regs[GEM_SCREENING_TYPE1_REGISTER_0 + i];
        matched = false;
        mismatched = false;

        /* Screening is based on UDP Port */
        if (reg & GEM_ST1R_UDP_PORT_MATCH_ENABLE) {
            uint16_t udp_port = rxbuf_ptr[14 + 22] << 8 | rxbuf_ptr[14 + 23];
            if (udp_port == extract32(reg, GEM_ST1R_UDP_PORT_MATCH_SHIFT,
                                           GEM_ST1R_UDP_PORT_MATCH_WIDTH)) {
                matched = true;
            } else {
                mismatched = true;
            }
        }

        /* Screening is based on DS/TC */
        if (reg & GEM_ST1R_DSTC_ENABLE) {
            uint8_t dscp = rxbuf_ptr[14 + 1];
            if (dscp == extract32(reg, GEM_ST1R_DSTC_MATCH_SHIFT,
                                       GEM_ST1R_DSTC_MATCH_WIDTH)) {
                matched = true;
            } else {
                mismatched = true;
            }
        }

        if (matched && !mismatched) {
            return extract32(reg, GEM_ST1R_QUEUE_SHIFT, GEM_ST1R_QUEUE_WIDTH);
        }
    }

    for (i = 0; i < s->num_type2_screeners; i++) {
        reg = s->regs[GEM_SCREENING_TYPE2_REGISTER_0 + i];
        matched = false;
        mismatched = false;

        if (reg & GEM_ST2R_ETHERTYPE_ENABLE) {
            uint16_t type = rxbuf_ptr[12] << 8 | rxbuf_ptr[13];
            int et_idx = extract32(reg, GEM_ST2R_ETHERTYPE_INDEX_SHIFT,
                                        GEM_ST2R_ETHERTYPE_INDEX_WIDTH);

            if (et_idx > s->num_type2_screeners) {
                qemu_log_mask(LOG_GUEST_ERROR, "Out of range ethertype "
                              "register index: %d\n", et_idx);
            }
            if (type == s->regs[GEM_SCREENING_TYPE2_ETHERTYPE_REG_0 +
                                et_idx]) {
                matched = true;
            } else {
                mismatched = true;
            }
        }

        /* Compare A, B, C */
        for (j = 0; j < 3; j++) {
            uint32_t cr0, cr1, mask;
            uint16_t rx_cmp;
            int offset;
            int cr_idx = extract32(reg, GEM_ST2R_COMPARE_A_SHIFT + j * 6,
                                        GEM_ST2R_COMPARE_WIDTH);

            if (!(reg & (GEM_ST2R_COMPARE_A_ENABLE << (j * 6)))) {
                continue;
            }
            if (cr_idx > s->num_type2_screeners) {
                qemu_log_mask(LOG_GUEST_ERROR, "Out of range compare "
                              "register index: %d\n", cr_idx);
            }

            cr0 = s->regs[GEM_TYPE2_COMPARE_0_WORD_0 + cr_idx * 2];
            cr1 = s->regs[GEM_TYPE2_COMPARE_0_WORD_0 + cr_idx * 2 + 1];
            offset = extract32(cr1, GEM_T2CW1_OFFSET_VALUE_SHIFT,
                                    GEM_T2CW1_OFFSET_VALUE_WIDTH);

            switch (extract32(cr1, GEM_T2CW1_COMPARE_OFFSET_SHIFT,
                                   GEM_T2CW1_COMPARE_OFFSET_WIDTH)) {
            case 3: /* Skip UDP header */
                qemu_log_mask(LOG_UNIMP, "TCP compare offsets"
                              "unimplemented - assuming UDP\n");
                offset += 8;
                /* Fallthrough */
            case 2: /* skip the IP header */
                offset += 20;
                /* Fallthrough */
            case 1: /* Count from after the ethertype */
                offset += 14;
                break;
            case 0:
                /* Offset from start of frame */
                break;
            }

            rx_cmp = rxbuf_ptr[offset] << 8 | rxbuf_ptr[offset];
            mask = extract32(cr0, 0, 16);

            if ((rx_cmp & mask) == (extract32(cr0, 16, 16) & mask)) {
                matched = true;
            } else {
                mismatched = true;
            }
        }

        if (matched && !mismatched) {
            return extract32(reg, GEM_ST2R_QUEUE_SHIFT, GEM_ST2R_QUEUE_WIDTH);
        }
    }

    /* We made it here, assume it's queue 0 */
    return 0;
}


static void gem_write(void *opaque, hwaddr offset, uint64_t val,
        unsigned size)
{
    CadenceGEMState *s = (CadenceGEMState *)opaque;
    uint32_t readonly;
    int i;

    DB_PRINT("offset: 0x%04x write: 0x%08x ", (unsigned)offset, (unsigned)val);
    offset >>= 2;

    /* Squash bits which are read only in write value */
    val &= ~(s->regs_ro[offset]);
    /* Preserve (only) bits which are read only and wtc in register */
    readonly = s->regs[offset] & (s->regs_ro[offset] | s->regs_w1c[offset]);

    /* Copy register write to backing store */
    s->regs[offset] = (val & ~s->regs_w1c[offset]) | readonly;

    /* do w1c */
    s->regs[offset] &= ~(s->regs_w1c[offset] & val);

    /* Handle register write side effects */
    switch (offset) {
    case GEM_NWCTRL:
        if (val & GEM_NWCTRL_RXENA) {
            for (i = 0; i < s->num_priority_queues; ++i) {
                gem_get_rx_desc(s, i);
            }
        }
        if (val & GEM_NWCTRL_TXSTART) {
            gem_transmit(s);
        }
        if (!(val & GEM_NWCTRL_TXENA)) {
            /* Reset to start of Q when transmit disabled. */
            for (i = 0; i < s->num_priority_queues; i++) {
                s->tx_desc_addr[i] = gem_get_tx_queue_base_addr(s, i);
            }
        }
        if (gem_can_receive(qemu_get_queue(s->nic))) {
            qemu_flush_queued_packets(qemu_get_queue(s->nic));
        }
        break;

    case GEM_TXSTATUS:
        gem_update_int_status(s);
        break;
    case GEM_RXQBASE:
        s->rx_desc_addr[0] = val;
        break;
    case GEM_RECEIVE_Q1_PTR ... GEM_RECEIVE_Q7_PTR:
        s->rx_desc_addr[offset - GEM_RECEIVE_Q1_PTR + 1] = val;
        break;
    case GEM_TXQBASE:
        s->tx_desc_addr[0] = val;
        break;
    case GEM_TRANSMIT_Q1_PTR ... GEM_TRANSMIT_Q7_PTR:
        s->tx_desc_addr[offset - GEM_TRANSMIT_Q1_PTR + 1] = val;
        break;
    case GEM_RXSTATUS:
        gem_update_int_status(s);
        break;
    case GEM_IER:
        s->regs[GEM_IMR] &= ~val;
        gem_update_int_status(s);
        break;
    case GEM_JUMBO_MAX_LEN:
        s->regs[GEM_JUMBO_MAX_LEN] = val & MAX_JUMBO_FRAME_SIZE_MASK;
        break;
    case GEM_INT_Q1_ENABLE ... GEM_INT_Q7_ENABLE:
        s->regs[GEM_INT_Q1_MASK + offset - GEM_INT_Q1_ENABLE] &= ~val;
        gem_update_int_status(s);
        break;
    case GEM_IDR:
        s->regs[GEM_IMR] |= val;
        gem_update_int_status(s);
        break;
    case GEM_INT_Q1_DISABLE ... GEM_INT_Q7_DISABLE:
        s->regs[GEM_INT_Q1_MASK + offset - GEM_INT_Q1_DISABLE] |= val;
        gem_update_int_status(s);
        break;
    case GEM_SPADDR1LO:
    case GEM_SPADDR2LO:
    case GEM_SPADDR3LO:
    case GEM_SPADDR4LO:
        s->sar_active[(offset - GEM_SPADDR1LO) / 2] = false;
        break;
    case GEM_SPADDR1HI:
    case GEM_SPADDR2HI:
    case GEM_SPADDR3HI:
    case GEM_SPADDR4HI:
        s->sar_active[(offset - GEM_SPADDR1HI) / 2] = true;
        break;
    case GEM_PHYMNTNC:
        if (val & GEM_PHYMNTNC_OP_W) {
            uint32_t phy_addr, reg_num;

            phy_addr = (val & GEM_PHYMNTNC_ADDR) >> GEM_PHYMNTNC_ADDR_SHFT;
            if (phy_addr == s->phy_addr) {
                reg_num = (val & GEM_PHYMNTNC_REG) >> GEM_PHYMNTNC_REG_SHIFT;
                gem_phy_write(s, reg_num, val);
            }
        }
        break;
    }

    DB_PRINT("newval: 0x%08x\n", s->regs[offset]);
}


static inline void rx_desc_set_length(uint32_t *desc, unsigned len)
{
    desc[1] &= ~DESC_1_LENGTH;
    desc[1] |= len;
}


static inline uint64_t tx_desc_get_buffer(CadenceGEMState *s, uint32_t *desc)
{
    uint64_t ret = desc[0];

    if (s->regs[GEM_DMACFG] & GEM_DMACFG_ADDR_64B) {
        ret |= (uint64_t)desc[2] << 32;
    }
    return ret;
}


static void gem_init_register_masks(CadenceGEMState *s)
{
    unsigned int i;
    /* Mask of register bits which are read only */
    memset(&s->regs_ro[0], 0, sizeof(s->regs_ro));
    s->regs_ro[GEM_NWCTRL]   = 0xFFF80000;
    s->regs_ro[GEM_NWSTATUS] = 0xFFFFFFFF;
    s->regs_ro[GEM_DMACFG]   = 0x8E00F000;
    s->regs_ro[GEM_TXSTATUS] = 0xFFFFFE08;
    s->regs_ro[GEM_RXQBASE]  = 0x00000003;
    s->regs_ro[GEM_TXQBASE]  = 0x00000003;
    s->regs_ro[GEM_RXSTATUS] = 0xFFFFFFF0;
    s->regs_ro[GEM_ISR]      = 0xFFFFFFFF;
    s->regs_ro[GEM_IMR]      = 0xFFFFFFFF;
    s->regs_ro[GEM_MODID]    = 0xFFFFFFFF;
    for (i = 0; i < s->num_priority_queues; i++) {
        s->regs_ro[GEM_INT_Q1_STATUS + i] = 0xFFFFFFFF;
        s->regs_ro[GEM_INT_Q1_ENABLE + i] = 0xFFFFF319;
        s->regs_ro[GEM_INT_Q1_DISABLE + i] = 0xFFFFF319;
        s->regs_ro[GEM_INT_Q1_MASK + i] = 0xFFFFFFFF;
    }

    /* Mask of register bits which are clear on read */
    memset(&s->regs_rtc[0], 0, sizeof(s->regs_rtc));
    s->regs_rtc[GEM_ISR]      = 0xFFFFFFFF;
    for (i = 0; i < s->num_priority_queues; i++) {
        s->regs_rtc[GEM_INT_Q1_STATUS + i] = 0x00000CE6;
    }

    /* Mask of register bits which are write 1 to clear */
    memset(&s->regs_w1c[0], 0, sizeof(s->regs_w1c));
    s->regs_w1c[GEM_TXSTATUS] = 0x000001F7;
    s->regs_w1c[GEM_RXSTATUS] = 0x0000000F;

    /* Mask of register bits which are write only */
    memset(&s->regs_wo[0], 0, sizeof(s->regs_wo));
    s->regs_wo[GEM_NWCTRL]   = 0x00073E60;
    s->regs_wo[GEM_IER]      = 0x07FFFFFF;
    s->regs_wo[GEM_IDR]      = 0x07FFFFFF;
    for (i = 0; i < s->num_priority_queues; i++) {
        s->regs_wo[GEM_INT_Q1_ENABLE + i] = 0x00000CE6;
        s->regs_wo[GEM_INT_Q1_DISABLE + i] = 0x00000CE6;
    }
}


static void gem_transmit_updatestats(CadenceGEMState *s, const uint8_t *packet,
                                     unsigned bytes)
{
    uint64_t octets;

    /* Total octets (bytes) transmitted */
    octets = ((uint64_t)(s->regs[GEM_OCTTXLO]) << 32) |
             s->regs[GEM_OCTTXHI];
    octets += bytes;
    s->regs[GEM_OCTTXLO] = octets >> 32;
    s->regs[GEM_OCTTXHI] = octets;

    /* Error-free Frames transmitted */
    s->regs[GEM_TXCNT]++;

    /* Error-free Broadcast Frames counter */
    if (!memcmp(packet, broadcast_addr, 6)) {
        s->regs[GEM_TXBCNT]++;
    }

    /* Error-free Multicast Frames counter */
    if (packet[0] == 0x01) {
        s->regs[GEM_TXMCNT]++;
    }

    if (bytes <= 64) {
        s->regs[GEM_TX64CNT]++;
    } else if (bytes <= 127) {
        s->regs[GEM_TX65CNT]++;
    } else if (bytes <= 255) {
        s->regs[GEM_TX128CNT]++;
    } else if (bytes <= 511) {
        s->regs[GEM_TX256CNT]++;
    } else if (bytes <= 1023) {
        s->regs[GEM_TX512CNT]++;
    } else if (bytes <= 1518) {
        s->regs[GEM_TX1024CNT]++;
    } else {
        s->regs[GEM_TX1519CNT]++;
    }
}


static void gem_set_isr(CadenceGEMState *s, int q, uint32_t flag)
{
    if (q == 0) {
        s->regs[GEM_ISR] |= flag & ~(s->regs[GEM_IMR]);
    } else {
        s->regs[GEM_INT_Q1_STATUS + q - 1] |= flag &
                                      ~(s->regs[GEM_INT_Q1_MASK + q - 1]);
    }
}


static inline unsigned tx_desc_get_last(uint32_t *desc)
{
    return (desc[1] & DESC_1_TX_LAST) ? 1 : 0;
}


static inline void rx_desc_clear_control(uint32_t *desc)
{
    desc[1]  = 0;
}


static inline uint32_t gem_get_tx_queue_base_addr(CadenceGEMState *s, int q)
{
    return gem_get_queue_base_addr(s, true, q);
}


static void gem_receive_updatestats(CadenceGEMState *s, const uint8_t *packet,
                                    unsigned bytes)
{
    uint64_t octets;

    /* Total octets (bytes) received */
    octets = ((uint64_t)(s->regs[GEM_OCTRXLO]) << 32) |
             s->regs[GEM_OCTRXHI];
    octets += bytes;
    s->regs[GEM_OCTRXLO] = octets >> 32;
    s->regs[GEM_OCTRXHI] = octets;

    /* Error-free Frames received */
    s->regs[GEM_RXCNT]++;

    /* Error-free Broadcast Frames counter */
    if (!memcmp(packet, broadcast_addr, 6)) {
        s->regs[GEM_RXBROADCNT]++;
    }

    /* Error-free Multicast Frames counter */
    if (packet[0] == 0x01) {
        s->regs[GEM_RXMULTICNT]++;
    }

    if (bytes <= 64) {
        s->regs[GEM_RX64CNT]++;
    } else if (bytes <= 127) {
        s->regs[GEM_RX65CNT]++;
    } else if (bytes <= 255) {
        s->regs[GEM_RX128CNT]++;
    } else if (bytes <= 511) {
        s->regs[GEM_RX256CNT]++;
    } else if (bytes <= 1023) {
        s->regs[GEM_RX512CNT]++;
    } else if (bytes <= 1518) {
        s->regs[GEM_RX1024CNT]++;
    } else {
        s->regs[GEM_RX1519CNT]++;
    }
}


static inline void rx_desc_set_sar(uint32_t *desc, int sar_idx)
{
    desc[1] = deposit32(desc[1], R_DESC_1_RX_SAR_SHIFT, R_DESC_1_RX_SAR_LENGTH,
                        sar_idx);
    desc[1] |= R_DESC_1_RX_SAR_MATCH;
}


static uint32_t gem_get_queue_base_addr(CadenceGEMState *s, bool tx, int q)
{
    uint32_t base_addr = 0;

    switch (q) {
    case 0:
        base_addr = s->regs[tx ? GEM_TXQBASE : GEM_RXQBASE];
        break;
    case 1 ... (MAX_PRIORITY_QUEUES - 1):
        base_addr = s->regs[(tx ? GEM_TRANSMIT_Q1_PTR :
                                 GEM_RECEIVE_Q1_PTR) + q - 1];
        break;
    default:
        g_assert_not_reached();
    };

    return base_addr;
}


static void gem_realize(DeviceState *dev, Error **errp)
{
    CadenceGEMState *s = CADENCE_GEM(dev);
    int i;

    address_space_init(&s->dma_as,
                       s->dma_mr ? s->dma_mr : get_system_memory(), "dma");

    if (s->num_priority_queues == 0 ||
        s->num_priority_queues > MAX_PRIORITY_QUEUES) {
        error_setg(errp, "Invalid num-priority-queues value: %" PRIx8,
                   s->num_priority_queues);
        return;
    } else if (s->num_type1_screeners > MAX_TYPE1_SCREENERS) {
        error_setg(errp, "Invalid num-type1-screeners value: %" PRIx8,
                   s->num_type1_screeners);
        return;
    } else if (s->num_type2_screeners > MAX_TYPE2_SCREENERS) {
        error_setg(errp, "Invalid num-type2-screeners value: %" PRIx8,
                   s->num_type2_screeners);
        return;
    }

    for (i = 0; i < s->num_priority_queues; ++i) {
        sysbus_init_irq(SYS_BUS_DEVICE(dev), &s->irq[i]);
    }

    qemu_macaddr_default_if_unset(&s->conf.macaddr);

    s->nic = qemu_new_nic(&net_gem_info, &s->conf,
                          object_get_typename(OBJECT(dev)), dev->id, s);

    if (s->jumbo_max_len > MAX_FRAME_SIZE) {
        error_setg(errp, "jumbo-max-len is greater than %d",
                  MAX_FRAME_SIZE);
        return;
    }
}


static uint32_t gem_get_max_buf_len(CadenceGEMState *s, bool tx)
{
    uint32_t size;
    if (s->regs[GEM_NWCFG] & GEM_NWCFG_JUMBO_FRAME) {
        size = s->regs[GEM_JUMBO_MAX_LEN];
        if (size > s->jumbo_max_len) {
            size = s->jumbo_max_len;
            qemu_log_mask(LOG_GUEST_ERROR, "GEM_JUMBO_MAX_LEN reg cannot be"
                " greater than 0x%" PRIx32 "\n", s->jumbo_max_len);
        }
    } else if (tx) {
        size = 1518;
    } else {
        size = s->regs[GEM_NWCFG] & GEM_NWCFG_RCV_1538 ? 1538 : 1518;
    }
    return size;
}


static inline void rx_desc_set_eof(uint32_t *desc)
{
    desc[1] |= DESC_1_RX_EOF;
}


static bool gem_can_receive(NetClientState *nc)
{
    CadenceGEMState *s;
    int i;

    s = qemu_get_nic_opaque(nc);

    /* Do nothing if receive is not enabled. */
    if (!(s->regs[GEM_NWCTRL] & GEM_NWCTRL_RXENA)) {
        if (s->can_rx_state != 1) {
            s->can_rx_state = 1;
            DB_PRINT("can't receive - no enable\n");
        }
        return false;
    }

    for (i = 0; i < s->num_priority_queues; i++) {
        if (rx_desc_get_ownership(s->rx_desc[i]) != 1) {
            break;
        }
    };

    if (i == s->num_priority_queues) {
        if (s->can_rx_state != 2) {
            s->can_rx_state = 2;
            DB_PRINT("can't receive - all the buffer descriptors are busy\n");
        }
        return false;
    }

    if (s->can_rx_state != 0) {
        s->can_rx_state = 0;
        DB_PRINT("can receive\n");
    }
    return true;
}


static inline unsigned rx_desc_get_ownership(uint32_t *desc)
{
    return desc[0] & DESC_0_RX_OWNERSHIP ? 1 : 0;
}


static inline void rx_desc_set_broadcast(uint32_t *desc)
{
    desc[1] |= R_DESC_1_RX_BROADCAST;
}


static ssize_t gem_receive(NetClientState *nc, const uint8_t *buf, size_t size)
{
    CadenceGEMState *s = qemu_get_nic_opaque(nc);
    unsigned   rxbufsize, bytes_to_copy;
    unsigned   rxbuf_offset;
    uint8_t   *rxbuf_ptr;
    bool first_desc = true;
    int maf;
    int q = 0;

    /* Is this destination MAC address "for us" ? */
    maf = gem_mac_address_filter(s, buf);
    if (maf == GEM_RX_REJECT) {
        return size;  /* no, drop siliently b/c it's not an error */
    }

    /* Discard packets with receive length error enabled ? */
    if (s->regs[GEM_NWCFG] & GEM_NWCFG_LERR_DISC) {
        unsigned type_len;

        /* Fish the ethertype / length field out of the RX packet */
        type_len = buf[12] << 8 | buf[13];
        /* It is a length field, not an ethertype */
        if (type_len < 0x600) {
            if (size < type_len) {
                /* discard */
                return -1;
            }
        }
    }

    /*
     * Determine configured receive buffer offset (probably 0)
     */
    rxbuf_offset = (s->regs[GEM_NWCFG] & GEM_NWCFG_BUFF_OFST_M) >>
                   GEM_NWCFG_BUFF_OFST_S;

    /* The configure size of each receive buffer.  Determines how many
     * buffers needed to hold this packet.
     */
    rxbufsize = ((s->regs[GEM_DMACFG] & GEM_DMACFG_RBUFSZ_M) >>
                 GEM_DMACFG_RBUFSZ_S) * GEM_DMACFG_RBUFSZ_MUL;
    bytes_to_copy = size;

    /* Hardware allows a zero value here but warns against it. To avoid QEMU
     * indefinite loops we enforce a minimum value here
     */
    if (rxbufsize < GEM_DMACFG_RBUFSZ_MUL) {
        rxbufsize = GEM_DMACFG_RBUFSZ_MUL;
    }

    /* Pad to minimum length. Assume FCS field is stripped, logic
     * below will increment it to the real minimum of 64 when
     * not FCS stripping
     */
    if (size < 60) {
        size = 60;
    }

    /* Strip of FCS field ? (usually yes) */
    if (s->regs[GEM_NWCFG] & GEM_NWCFG_STRIP_FCS) {
        rxbuf_ptr = (void *)buf;
    } else {
        unsigned crc_val;

        if (size > MAX_FRAME_SIZE - sizeof(crc_val)) {
            size = MAX_FRAME_SIZE - sizeof(crc_val);
        }
        bytes_to_copy = size;
        /* The application wants the FCS field, which QEMU does not provide.
         * We must try and calculate one.
         */

        memcpy(s->rx_packet, buf, size);
        memset(s->rx_packet + size, 0, MAX_FRAME_SIZE - size);
        rxbuf_ptr = s->rx_packet;
        crc_val = cpu_to_le32(crc32(0, s->rx_packet, MAX(size, 60)));
        memcpy(s->rx_packet + size, &crc_val, sizeof(crc_val));

        bytes_to_copy += 4;
        size += 4;
    }

    DB_PRINT("config bufsize: %u packet size: %zd\n", rxbufsize, size);

    /* Find which queue we are targeting */
    q = get_queue_from_screen(s, rxbuf_ptr, rxbufsize);

    if (size > gem_get_max_buf_len(s, false)) {
        qemu_log_mask(LOG_GUEST_ERROR, "rx frame too long\n");
        gem_set_isr(s, q, GEM_INT_AMBA_ERR);
        return -1;
    }

    while (bytes_to_copy) {
        hwaddr desc_addr;

        /* Do nothing if receive is not enabled. */
        if (!gem_can_receive(nc)) {
            return -1;
        }

        DB_PRINT("copy %" PRIu32 " bytes to 0x%" PRIx64 "\n",
                MIN(bytes_to_copy, rxbufsize),
                rx_desc_get_buffer(s, s->rx_desc[q]));

        /* Copy packet data to emulated DMA buffer */
        address_space_write(&s->dma_as, rx_desc_get_buffer(s, s->rx_desc[q]) +
                                                                  rxbuf_offset,
                            MEMTXATTRS_UNSPECIFIED, rxbuf_ptr,
                            MIN(bytes_to_copy, rxbufsize));
        rxbuf_ptr += MIN(bytes_to_copy, rxbufsize);
        bytes_to_copy -= MIN(bytes_to_copy, rxbufsize);

        rx_desc_clear_control(s->rx_desc[q]);

        /* Update the descriptor.  */
        if (first_desc) {
            rx_desc_set_sof(s->rx_desc[q]);
            first_desc = false;
        }
        if (bytes_to_copy == 0) {
            rx_desc_set_eof(s->rx_desc[q]);
            rx_desc_set_length(s->rx_desc[q], size);
        }
        rx_desc_set_ownership(s->rx_desc[q]);

        switch (maf) {
        case GEM_RX_PROMISCUOUS_ACCEPT:
            break;
        case GEM_RX_BROADCAST_ACCEPT:
            rx_desc_set_broadcast(s->rx_desc[q]);
            break;
        case GEM_RX_UNICAST_HASH_ACCEPT:
            rx_desc_set_unicast_hash(s->rx_desc[q]);
            break;
        case GEM_RX_MULTICAST_HASH_ACCEPT:
            rx_desc_set_multicast_hash(s->rx_desc[q]);
            break;
        case GEM_RX_REJECT:
            abort();
        default: /* SAR */
            rx_desc_set_sar(s->rx_desc[q], maf);
        }

        /* Descriptor write-back.  */
        desc_addr = gem_get_rx_desc_addr(s, q);
        address_space_write(&s->dma_as, desc_addr, MEMTXATTRS_UNSPECIFIED,
                            s->rx_desc[q],
                            sizeof(uint32_t) * gem_get_desc_len(s, true));

        /* Next descriptor */
        if (rx_desc_get_wrap(s->rx_desc[q])) {
            DB_PRINT("wrapping RX descriptor list\n");
            s->rx_desc_addr[q] = gem_get_rx_queue_base_addr(s, q);
        } else {
            DB_PRINT("incrementing RX descriptor list\n");
            s->rx_desc_addr[q] += 4 * gem_get_desc_len(s, true);
        }

        gem_get_rx_desc(s, q);
    }

    /* Count it */
    gem_receive_updatestats(s, buf, size);

    s->regs[GEM_RXSTATUS] |= GEM_RXSTATUS_FRMRCVD;
    gem_set_isr(s, q, GEM_INT_RXCMPL);

    /* Handle interrupt consequences */
    gem_update_int_status(s);

    return size;
}


static unsigned get_bit(const uint8_t *mac, unsigned bit)
{
    unsigned byte;

    byte = mac[bit / 8];
    byte >>= (bit & 0x7);
    byte &= 1;

    return byte;
}


static void gem_phy_write(CadenceGEMState *s, unsigned reg_num, uint16_t val)
{
    DB_PRINT("reg: %d value: 0x%04x\n", reg_num, val);

    switch (reg_num) {
    case PHY_REG_CONTROL:
        if (val & PHY_REG_CONTROL_RST) {
            /* Phy reset */
            gem_phy_reset(s);
            val &= ~(PHY_REG_CONTROL_RST | PHY_REG_CONTROL_LOOP);
            s->phy_loop = 0;
        }
        if (val & PHY_REG_CONTROL_ANEG) {
            /* Complete autonegotiation immediately */
            val &= ~(PHY_REG_CONTROL_ANEG | PHY_REG_CONTROL_ANRESTART);
            s->phy_regs[PHY_REG_STATUS] |= PHY_REG_STATUS_ANEGCMPL;
        }
        if (val & PHY_REG_CONTROL_LOOP) {
            DB_PRINT("PHY placed in loopback\n");
            s->phy_loop = 1;
        } else {
            s->phy_loop = 0;
        }
        break;
    }
    s->phy_regs[reg_num] = val;
}


static inline void rx_desc_set_ownership(uint32_t *desc)
{
    desc[0] |= DESC_0_RX_OWNERSHIP;
}


static unsigned calc_mac_hash(const uint8_t *mac)
{
    int index_bit, mac_bit;
    unsigned hash_index;

    hash_index = 0;
    mac_bit = 5;
    for (index_bit = 5; index_bit >= 0; index_bit--) {
        hash_index |= (get_bit(mac,  mac_bit) ^
                               get_bit(mac, mac_bit + 6) ^
                               get_bit(mac, mac_bit + 12) ^
                               get_bit(mac, mac_bit + 18) ^
                               get_bit(mac, mac_bit + 24) ^
                               get_bit(mac, mac_bit + 30) ^
                               get_bit(mac, mac_bit + 36) ^
                               get_bit(mac, mac_bit + 42)) << index_bit;
        mac_bit--;
    }

    return hash_index;
}


static uint16_t gem_phy_read(CadenceGEMState *s, unsigned reg_num)
{
    DB_PRINT("reg: %d value: 0x%04x\n", reg_num, s->phy_regs[reg_num]);
    return s->phy_regs[reg_num];
}


static inline void print_gem_tx_desc(uint32_t *desc, uint8_t queue)
{
    DB_PRINT("TXDESC (queue %" PRId8 "):\n", queue);
    DB_PRINT("bufaddr: 0x%08x\n", *desc);
    DB_PRINT("used_hw: %d\n", tx_desc_get_used(desc));
    DB_PRINT("wrap:    %d\n", tx_desc_get_wrap(desc));
    DB_PRINT("last:    %d\n", tx_desc_get_last(desc));
    DB_PRINT("length:  %d\n", tx_desc_get_length(desc));
}


static hwaddr gem_get_desc_addr(CadenceGEMState *s, bool tx, int q)
{
    hwaddr desc_addr = 0;

    if (s->regs[GEM_DMACFG] & GEM_DMACFG_ADDR_64B) {
        desc_addr = s->regs[tx ? GEM_TBQPH : GEM_RBQPH];
    }
    desc_addr <<= 32;
    desc_addr |= tx ? s->tx_desc_addr[q] : s->rx_desc_addr[q];
    return desc_addr;
}


static uint64_t gem_read(void *opaque, hwaddr offset, unsigned size)
{
    CadenceGEMState *s;
    uint32_t retval;
    s = (CadenceGEMState *)opaque;

    offset >>= 2;
    retval = s->regs[offset];

    DB_PRINT("offset: 0x%04x read: 0x%08x\n", (unsigned)offset*4, retval);

    switch (offset) {
    case GEM_ISR:
        DB_PRINT("lowering irqs on ISR read\n");
        /* The interrupts get updated at the end of the function. */
        break;
    case GEM_PHYMNTNC:
        if (retval & GEM_PHYMNTNC_OP_R) {
            uint32_t phy_addr, reg_num;

            phy_addr = (retval & GEM_PHYMNTNC_ADDR) >> GEM_PHYMNTNC_ADDR_SHFT;
            if (phy_addr == s->phy_addr) {
                reg_num = (retval & GEM_PHYMNTNC_REG) >> GEM_PHYMNTNC_REG_SHIFT;
                retval &= 0xFFFF0000;
                retval |= gem_phy_read(s, reg_num);
            } else {
                retval |= 0xFFFF; /* No device at this address */
            }
        }
        break;
    }

    /* Squash read to clear bits */
    s->regs[offset] &= ~(s->regs_rtc[offset]);

    /* Do not provide write only bits */
    retval &= ~(s->regs_wo[offset]);

    DB_PRINT("0x%08x\n", retval);
    gem_update_int_status(s);
    return retval;
}


static inline unsigned rx_desc_get_wrap(uint32_t *desc)
{
    return desc[0] & DESC_0_RX_WRAP ? 1 : 0;
}


static void gem_reset(DeviceState *d)
{
    int i;
    CadenceGEMState *s = CADENCE_GEM(d);
    const uint8_t *a;
    uint32_t queues_mask = 0;

    DB_PRINT("\n");

    /* Set post reset register values */
    memset(&s->regs[0], 0, sizeof(s->regs));
    s->regs[GEM_NWCFG] = 0x00080000;
    s->regs[GEM_NWSTATUS] = 0x00000006;
    s->regs[GEM_DMACFG] = 0x00020784;
    s->regs[GEM_IMR] = 0x07ffffff;
    s->regs[GEM_TXPAUSE] = 0x0000ffff;
    s->regs[GEM_TXPARTIALSF] = 0x000003ff;
    s->regs[GEM_RXPARTIALSF] = 0x000003ff;
    s->regs[GEM_MODID] = s->revision;
    s->regs[GEM_DESCONF] = 0x02D00111;
    s->regs[GEM_DESCONF2] = 0x2ab10000 | s->jumbo_max_len;
    s->regs[GEM_DESCONF5] = 0x002f2045;
    s->regs[GEM_DESCONF6] = GEM_DESCONF6_64B_MASK;
    s->regs[GEM_INT_Q1_MASK] = 0x00000CE6;
    s->regs[GEM_JUMBO_MAX_LEN] = s->jumbo_max_len;

    if (s->num_priority_queues > 1) {
        queues_mask = MAKE_64BIT_MASK(1, s->num_priority_queues - 1);
        s->regs[GEM_DESCONF6] |= queues_mask;
    }

    /* Set MAC address */
    a = &s->conf.macaddr.a[0];
    s->regs[GEM_SPADDR1LO] = a[0] | (a[1] << 8) | (a[2] << 16) | (a[3] << 24);
    s->regs[GEM_SPADDR1HI] = a[4] | (a[5] << 8);

    for (i = 0; i < 4; i++) {
        s->sar_active[i] = false;
    }

    gem_phy_reset(s);

    gem_update_int_status(s);
}


static void gem_register_types(void)
{
    type_register_static(&gem_info);
}


static int gem_mac_address_filter(CadenceGEMState *s, const uint8_t *packet)
{
    uint8_t *gem_spaddr;
    int i, is_mc;

    /* Promiscuous mode? */
    if (s->regs[GEM_NWCFG] & GEM_NWCFG_PROMISC) {
        return GEM_RX_PROMISCUOUS_ACCEPT;
    }

    if (!memcmp(packet, broadcast_addr, 6)) {
        /* Reject broadcast packets? */
        if (s->regs[GEM_NWCFG] & GEM_NWCFG_BCAST_REJ) {
            return GEM_RX_REJECT;
        }
        return GEM_RX_BROADCAST_ACCEPT;
    }

    /* Accept packets -w- hash match? */
    is_mc = is_multicast_ether_addr(packet);
    if ((is_mc && (s->regs[GEM_NWCFG] & GEM_NWCFG_MCAST_HASH)) ||
        (!is_mc && (s->regs[GEM_NWCFG] & GEM_NWCFG_UCAST_HASH))) {
        uint64_t buckets;
        unsigned hash_index;

        hash_index = calc_mac_hash(packet);
        buckets = ((uint64_t)s->regs[GEM_HASHHI] << 32) | s->regs[GEM_HASHLO];
        if ((buckets >> hash_index) & 1) {
            return is_mc ? GEM_RX_MULTICAST_HASH_ACCEPT
                         : GEM_RX_UNICAST_HASH_ACCEPT;
        }
    }

    /* Check all 4 specific addresses */
    gem_spaddr = (uint8_t *)&(s->regs[GEM_SPADDR1LO]);
    for (i = 3; i >= 0; i--) {
        if (s->sar_active[i] && !memcmp(packet, gem_spaddr + 8 * i, 6)) {
            return GEM_RX_SAR_ACCEPT + i;
        }
    }

    /* No address match; reject the packet */
    return GEM_RX_REJECT;
}


static void gem_class_init(ObjectClass *klass, void *data)
{
    DeviceClass *dc = DEVICE_CLASS(klass);

    dc->realize = gem_realize;
    device_class_set_props(dc, gem_properties);
    dc->vmsd = &vmstate_cadence_gem;
    dc->reset = gem_reset;
}