smb2_set_ea(const unsigned int xid, struct cifs_tcon *tcon,
	    const char *path, const char *ea_name, const void *ea_value,
	    const __u16 ea_value_len, const struct nls_table *nls_codepage,
	    struct cifs_sb_info *cifs_sb)
{
	struct cifs_ses *ses = tcon->ses;
	struct TCP_Server_Info *server = cifs_pick_channel(ses);
	__le16 *utf16_path = NULL;
	int ea_name_len = strlen(ea_name);
	int flags = CIFS_CP_CREATE_CLOSE_OP;
	int len;
	struct smb_rqst rqst[3];
	int resp_buftype[3];
	struct kvec rsp_iov[3];
	struct kvec open_iov[SMB2_CREATE_IOV_SIZE];
	struct cifs_open_parms oparms;
	__u8 oplock = SMB2_OPLOCK_LEVEL_NONE;
	struct cifs_fid fid;
	struct kvec si_iov[SMB2_SET_INFO_IOV_SIZE];
	unsigned int size[1];
	void *data[1];
	struct smb2_file_full_ea_info *ea = NULL;
	struct kvec close_iov[1];
	struct smb2_query_info_rsp *rsp;
	int rc, used_len = 0;

	if (smb3_encryption_required(tcon))
		flags |= CIFS_TRANSFORM_REQ;

	if (ea_name_len > 255)
		return -EINVAL;

	utf16_path = cifs_convert_path_to_utf16(path, cifs_sb);
	if (!utf16_path)
		return -ENOMEM;

	memset(rqst, 0, sizeof(rqst));
	resp_buftype[0] = resp_buftype[1] = resp_buftype[2] = CIFS_NO_BUFFER;
	memset(rsp_iov, 0, sizeof(rsp_iov));

	if (ses->server->ops->query_all_EAs) {
		if (!ea_value) {
			rc = ses->server->ops->query_all_EAs(xid, tcon, path,
							     ea_name, NULL, 0,
							     cifs_sb);
			if (rc == -ENODATA)
				goto sea_exit;
		} else {
			/* If we are adding a attribute we should first check
			 * if there will be enough space available to store
			 * the new EA. If not we should not add it since we
			 * would not be able to even read the EAs back.
			 */
			rc = smb2_query_info_compound(xid, tcon, path,
				      FILE_READ_EA,
				      FILE_FULL_EA_INFORMATION,
				      SMB2_O_INFO_FILE,
				      CIFSMaxBufSize -
				      MAX_SMB2_CREATE_RESPONSE_SIZE -
				      MAX_SMB2_CLOSE_RESPONSE_SIZE,
				      &rsp_iov[1], &resp_buftype[1], cifs_sb);
			if (rc == 0) {
				rsp = (struct smb2_query_info_rsp *)rsp_iov[1].iov_base;
				used_len = le32_to_cpu(rsp->OutputBufferLength);
			}
			free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base);
			resp_buftype[1] = CIFS_NO_BUFFER;
			memset(&rsp_iov[1], 0, sizeof(rsp_iov[1]));
			rc = 0;

			/* Use a fudge factor of 256 bytes in case we collide
			 * with a different set_EAs command.
			 */
			if(CIFSMaxBufSize - MAX_SMB2_CREATE_RESPONSE_SIZE -
			   MAX_SMB2_CLOSE_RESPONSE_SIZE - 256 <
			   used_len + ea_name_len + ea_value_len + 1) {
				rc = -ENOSPC;
				goto sea_exit;
			}
		}
	}

	/* Open */
	memset(&open_iov, 0, sizeof(open_iov));
	rqst[0].rq_iov = open_iov;
	rqst[0].rq_nvec = SMB2_CREATE_IOV_SIZE;

	memset(&oparms, 0, sizeof(oparms));
	oparms.tcon = tcon;
	oparms.desired_access = FILE_WRITE_EA;
	oparms.disposition = FILE_OPEN;
	oparms.create_options = cifs_create_options(cifs_sb, 0);
	oparms.fid = &fid;
	oparms.reconnect = false;

	rc = SMB2_open_init(tcon, server,
			    &rqst[0], &oplock, &oparms, utf16_path);
	if (rc)
		goto sea_exit;
	smb2_set_next_command(tcon, &rqst[0]);

	/* Set Info */
	memset(&si_iov, 0, sizeof(si_iov));
	rqst[1].rq_iov = si_iov;
	rqst[1].rq_nvec = 1;

	len = sizeof(*ea) + ea_name_len + ea_value_len + 1;
	ea = kzalloc(len, GFP_KERNEL);
	if (ea == NULL) {
		rc = -ENOMEM;
		goto sea_exit;
	}

	ea->ea_name_length = ea_name_len;
	ea->ea_value_length = cpu_to_le16(ea_value_len);
	memcpy(ea->ea_data, ea_name, ea_name_len + 1);
	memcpy(ea->ea_data + ea_name_len + 1, ea_value, ea_value_len);

	size[0] = len;
	data[0] = ea;

	rc = SMB2_set_info_init(tcon, server,
				&rqst[1], COMPOUND_FID,
				COMPOUND_FID, current->tgid,
				FILE_FULL_EA_INFORMATION,
				SMB2_O_INFO_FILE, 0, data, size);
	smb2_set_next_command(tcon, &rqst[1]);
	smb2_set_related(&rqst[1]);

	/* Close */
	memset(&close_iov, 0, sizeof(close_iov));
	rqst[2].rq_iov = close_iov;
	rqst[2].rq_nvec = 1;
	rc = SMB2_close_init(tcon, server,
			     &rqst[2], COMPOUND_FID, COMPOUND_FID, false);
	smb2_set_related(&rqst[2]);

	rc = compound_send_recv(xid, ses, server,
				flags, 3, rqst,
				resp_buftype, rsp_iov);
	/* no need to bump num_remote_opens because handle immediately closed */

 sea_exit:
	kfree(ea);
	kfree(utf16_path);
	SMB2_open_free(&rqst[0]);
	SMB2_set_info_free(&rqst[1]);
	SMB2_close_free(&rqst[2]);
	free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base);
	free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base);
	free_rsp_buf(resp_buftype[2], rsp_iov[2].iov_base);
	return rc;
}


smb2_negotiate_wsize(struct cifs_tcon *tcon, struct smb3_fs_context *ctx)
{
	struct TCP_Server_Info *server = tcon->ses->server;
	unsigned int wsize;

	/* start with specified wsize, or default */
	wsize = ctx->wsize ? ctx->wsize : CIFS_DEFAULT_IOSIZE;
	wsize = min_t(unsigned int, wsize, server->max_write);
	if (!(server->capabilities & SMB2_GLOBAL_CAP_LARGE_MTU))
		wsize = min_t(unsigned int, wsize, SMB2_MAX_BUFFER_SIZE);

	return wsize;
}

cifs: fix NULL ptr dereference in smb2_ioctl_query_info()

When calling smb2_ioctl_query_info() with invalid
smb_query_info::flags, a NULL ptr dereference is triggered when trying
to kfree() uninitialised rqst[n].rq_iov array.

This also fixes leaked paths that are created in SMB2_open_init()
which required SMB2_open_free() to properly free them.

Here is a small C reproducer that triggers it

	#include <stdio.h>
	#include <stdlib.h>
	#include <stdint.h>
	#include <unistd.h>
	#include <fcntl.h>
	#include <sys/ioctl.h>

	#define die(s) perror(s), exit(1)
	#define QUERY_INFO 0xc018cf07

	int main(int argc, char *argv[])
	{
		int fd;

		if (argc < 2)
			exit(1);
		fd = open(argv[1], O_RDONLY);
		if (fd == -1)
			die("open");
		if (ioctl(fd, QUERY_INFO, (uint32_t[]) { 0, 0, 0, 4, 0, 0}) == -1)
			die("ioctl");
		close(fd);
		return 0;
	}

	mount.cifs //srv/share /mnt -o ...
	gcc repro.c && ./a.out /mnt/f0

	[ 1832.124468] CIFS: VFS: \\w22-dc.zelda.test\test Invalid passthru query flags: 0x4
	[ 1832.125043] general protection fault, probably for non-canonical address 0xdffffc0000000000: 0000 [#1] PREEMPT SMP KASAN NOPTI
	[ 1832.125764] KASAN: null-ptr-deref in range [0x0000000000000000-0x0000000000000007]
	[ 1832.126241] CPU: 3 PID: 1133 Comm: a.out Not tainted 5.17.0-rc8 #2
	[ 1832.126630] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.15.0-0-g2dd4b9b-rebuilt.opensuse.org 04/01/2014
	[ 1832.127322] RIP: 0010:smb2_ioctl_query_info+0x7a3/0xe30 [cifs]
	[ 1832.127749] Code: 00 00 00 fc ff df 48 c1 ea 03 80 3c 02 00 0f 85 6c 05 00 00 48 b8 00 00 00 00 00 fc ff df 4d 8b 74 24 28 4c 89 f2 48 c1 ea 03 <80> 3c 02 00 0f 85 cb 04 00 00 49 8b 3e e8 bb fc fa ff 48 89 da 48
	[ 1832.128911] RSP: 0018:ffffc90000957b08 EFLAGS: 00010256
	[ 1832.129243] RAX: dffffc0000000000 RBX: ffff888117e9b850 RCX: ffffffffa020580d
	[ 1832.129691] RDX: 0000000000000000 RSI: 0000000000000004 RDI: ffffffffa043a2c0
	[ 1832.130137] RBP: ffff888117e9b878 R08: 0000000000000001 R09: 0000000000000003
	[ 1832.130585] R10: fffffbfff4087458 R11: 0000000000000001 R12: ffff888117e9b800
	[ 1832.131037] R13: 00000000ffffffea R14: 0000000000000000 R15: ffff888117e9b8a8
	[ 1832.131485] FS:  00007fcee9900740(0000) GS:ffff888151a00000(0000) knlGS:0000000000000000
	[ 1832.131993] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
	[ 1832.132354] CR2: 00007fcee9a1ef5e CR3: 0000000114cd2000 CR4: 0000000000350ee0
	[ 1832.132801] Call Trace:
	[ 1832.132962]  <TASK>
	[ 1832.133104]  ? smb2_query_reparse_tag+0x890/0x890 [cifs]
	[ 1832.133489]  ? cifs_mapchar+0x460/0x460 [cifs]
	[ 1832.133822]  ? rcu_read_lock_sched_held+0x3f/0x70
	[ 1832.134125]  ? cifs_strndup_to_utf16+0x15b/0x250 [cifs]
	[ 1832.134502]  ? lock_downgrade+0x6f0/0x6f0
	[ 1832.134760]  ? cifs_convert_path_to_utf16+0x198/0x220 [cifs]
	[ 1832.135170]  ? smb2_check_message+0x1080/0x1080 [cifs]
	[ 1832.135545]  cifs_ioctl+0x1577/0x3320 [cifs]
	[ 1832.135864]  ? lock_downgrade+0x6f0/0x6f0
	[ 1832.136125]  ? cifs_readdir+0x2e60/0x2e60 [cifs]
	[ 1832.136468]  ? rcu_read_lock_sched_held+0x3f/0x70
	[ 1832.136769]  ? __rseq_handle_notify_resume+0x80b/0xbe0
	[ 1832.137096]  ? __up_read+0x192/0x710
	[ 1832.137327]  ? __ia32_sys_rseq+0xf0/0xf0
	[ 1832.137578]  ? __x64_sys_openat+0x11f/0x1d0
	[ 1832.137850]  __x64_sys_ioctl+0x127/0x190
	[ 1832.138103]  do_syscall_64+0x3b/0x90
	[ 1832.138378]  entry_SYSCALL_64_after_hwframe+0x44/0xae
	[ 1832.138702] RIP: 0033:0x7fcee9a253df
	[ 1832.138937] Code: 00 48 89 44 24 18 31 c0 48 8d 44 24 60 c7 04 24 10 00 00 00 48 89 44 24 08 48 8d 44 24 20 48 89 44 24 10 b8 10 00 00 00 0f 05 <41> 89 c0 3d 00 f0 ff ff 77 1f 48 8b 44 24 18 64 48 2b 04 25 28 00
	[ 1832.140107] RSP: 002b:00007ffeba94a8a0 EFLAGS: 00000246 ORIG_RAX: 0000000000000010
	[ 1832.140606] RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007fcee9a253df
	[ 1832.141058] RDX: 00007ffeba94a910 RSI: 00000000c018cf07 RDI: 0000000000000003
	[ 1832.141503] RBP: 00007ffeba94a930 R08: 00007fcee9b24db0 R09: 00007fcee9b45c4e
	[ 1832.141948] R10: 00007fcee9918d40 R11: 0000000000000246 R12: 00007ffeba94aa48
	[ 1832.142396] R13: 0000000000401176 R14: 0000000000403df8 R15: 00007fcee9b78000
	[ 1832.142851]  </TASK>
	[ 1832.142994] Modules linked in: cifs cifs_arc4 cifs_md4 bpf_preload [last unloaded: cifs]

Cc: stable@vger.kernel.org
Signed-off-by: Paulo Alcantara (SUSE) <pc@cjr.nz>
Signed-off-by: Steve French <stfrench@microsoft.com>


smb2_get_credits(struct mid_q_entry *mid)
{
	return mid->credits_received;
}


smb2_revert_current_mid(struct TCP_Server_Info *server, const unsigned int val)
{
	spin_lock(&GlobalMid_Lock);
	if (server->CurrentMid >= val)
		server->CurrentMid -= val;
	spin_unlock(&GlobalMid_Lock);
}


smb2_query_info_compound(const unsigned int xid, struct cifs_tcon *tcon,
			 const char *path, u32 desired_access,
			 u32 class, u32 type, u32 output_len,
			 struct kvec *rsp, int *buftype,
			 struct cifs_sb_info *cifs_sb)
{
	struct cifs_ses *ses = tcon->ses;
	struct TCP_Server_Info *server = cifs_pick_channel(ses);
	int flags = CIFS_CP_CREATE_CLOSE_OP;
	struct smb_rqst rqst[3];
	int resp_buftype[3];
	struct kvec rsp_iov[3];
	struct kvec open_iov[SMB2_CREATE_IOV_SIZE];
	struct kvec qi_iov[1];
	struct kvec close_iov[1];
	u8 oplock = SMB2_OPLOCK_LEVEL_NONE;
	struct cifs_open_parms oparms;
	struct cifs_fid fid;
	int rc;
	__le16 *utf16_path;
	struct cached_fid *cfid = NULL;

	if (!path)
		path = "";
	utf16_path = cifs_convert_path_to_utf16(path, cifs_sb);
	if (!utf16_path)
		return -ENOMEM;

	if (smb3_encryption_required(tcon))
		flags |= CIFS_TRANSFORM_REQ;

	memset(rqst, 0, sizeof(rqst));
	resp_buftype[0] = resp_buftype[1] = resp_buftype[2] = CIFS_NO_BUFFER;
	memset(rsp_iov, 0, sizeof(rsp_iov));

	rc = open_cached_dir(xid, tcon, path, cifs_sb, &cfid);

	memset(&open_iov, 0, sizeof(open_iov));
	rqst[0].rq_iov = open_iov;
	rqst[0].rq_nvec = SMB2_CREATE_IOV_SIZE;

	oparms.tcon = tcon;
	oparms.desired_access = desired_access;
	oparms.disposition = FILE_OPEN;
	oparms.create_options = cifs_create_options(cifs_sb, 0);
	oparms.fid = &fid;
	oparms.reconnect = false;

	rc = SMB2_open_init(tcon, server,
			    &rqst[0], &oplock, &oparms, utf16_path);
	if (rc)
		goto qic_exit;
	smb2_set_next_command(tcon, &rqst[0]);

	memset(&qi_iov, 0, sizeof(qi_iov));
	rqst[1].rq_iov = qi_iov;
	rqst[1].rq_nvec = 1;

	if (cfid) {
		rc = SMB2_query_info_init(tcon, server,
					  &rqst[1],
					  cfid->fid->persistent_fid,
					  cfid->fid->volatile_fid,
					  class, type, 0,
					  output_len, 0,
					  NULL);
	} else {
		rc = SMB2_query_info_init(tcon, server,
					  &rqst[1],
					  COMPOUND_FID,
					  COMPOUND_FID,
					  class, type, 0,
					  output_len, 0,
					  NULL);
	}
	if (rc)
		goto qic_exit;
	if (!cfid) {
		smb2_set_next_command(tcon, &rqst[1]);
		smb2_set_related(&rqst[1]);
	}

	memset(&close_iov, 0, sizeof(close_iov));
	rqst[2].rq_iov = close_iov;
	rqst[2].rq_nvec = 1;

	rc = SMB2_close_init(tcon, server,
			     &rqst[2], COMPOUND_FID, COMPOUND_FID, false);
	if (rc)
		goto qic_exit;
	smb2_set_related(&rqst[2]);

	if (cfid) {
		rc = compound_send_recv(xid, ses, server,
					flags, 1, &rqst[1],
					&resp_buftype[1], &rsp_iov[1]);
	} else {
		rc = compound_send_recv(xid, ses, server,
					flags, 3, rqst,
					resp_buftype, rsp_iov);
	}
	if (rc) {
		free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base);
		if (rc == -EREMCHG) {
			tcon->need_reconnect = true;
			pr_warn_once("server share %s deleted\n",
				     tcon->treeName);
		}
		goto qic_exit;
	}
	*rsp = rsp_iov[1];
	*buftype = resp_buftype[1];

 qic_exit:
	kfree(utf16_path);
	SMB2_open_free(&rqst[0]);
	SMB2_query_info_free(&rqst[1]);
	SMB2_close_free(&rqst[2]);
	free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base);
	free_rsp_buf(resp_buftype[2], rsp_iov[2].iov_base);
	if (cfid)
		close_cached_dir(cfid);
	return rc;
}


static long smb3_insert_range(struct file *file, struct cifs_tcon *tcon,
			      loff_t off, loff_t len)
{
	int rc;
	unsigned int xid;
	struct cifsFileInfo *cfile = file->private_data;
	__le64 eof;
	__u64  count;

	xid = get_xid();

	if (off >= i_size_read(file->f_inode)) {
		rc = -EINVAL;
		goto out;
	}

	count = i_size_read(file->f_inode) - off;
	eof = cpu_to_le64(i_size_read(file->f_inode) + len);

	rc = SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid,
			  cfile->fid.volatile_fid, cfile->pid, &eof);
	if (rc < 0)
		goto out;

	rc = smb2_copychunk_range(xid, cfile, cfile, off, count, off + len);
	if (rc < 0)
		goto out;

	rc = smb3_zero_range(file, tcon, off, len, 1);
	if (rc < 0)
		goto out;

	rc = 0;
 out:
	free_xid(xid);
	return rc;
}


smb2_dump_share_caps(struct seq_file *m, struct cifs_tcon *tcon)
{
	seq_puts(m, "\n\tShare Capabilities:");
	if (tcon->capabilities & SMB2_SHARE_CAP_DFS)
		seq_puts(m, " DFS,");
	if (tcon->capabilities & SMB2_SHARE_CAP_CONTINUOUS_AVAILABILITY)
		seq_puts(m, " CONTINUOUS AVAILABILITY,");
	if (tcon->capabilities & SMB2_SHARE_CAP_SCALEOUT)
		seq_puts(m, " SCALEOUT,");
	if (tcon->capabilities & SMB2_SHARE_CAP_CLUSTER)
		seq_puts(m, " CLUSTER,");
	if (tcon->capabilities & SMB2_SHARE_CAP_ASYMMETRIC)
		seq_puts(m, " ASYMMETRIC,");
	if (tcon->capabilities == 0)
		seq_puts(m, " None");
	if (tcon->ss_flags & SSINFO_FLAGS_ALIGNED_DEVICE)
		seq_puts(m, " Aligned,");
	if (tcon->ss_flags & SSINFO_FLAGS_PARTITION_ALIGNED_ON_DEVICE)
		seq_puts(m, " Partition Aligned,");
	if (tcon->ss_flags & SSINFO_FLAGS_NO_SEEK_PENALTY)
		seq_puts(m, " SSD,");
	if (tcon->ss_flags & SSINFO_FLAGS_TRIM_ENABLED)
		seq_puts(m, " TRIM-support,");

	seq_printf(m, "\tShare Flags: 0x%x", tcon->share_flags);
	seq_printf(m, "\n\ttid: 0x%x", tcon->tid);
	if (tcon->perf_sector_size)
		seq_printf(m, "\tOptimal sector size: 0x%x",
			   tcon->perf_sector_size);
	seq_printf(m, "\tMaximal Access: 0x%x", tcon->maximal_access);
}


smb3_downgrade_oplock(struct TCP_Server_Info *server,
		       struct cifsInodeInfo *cinode, __u32 oplock,
		       unsigned int epoch, bool *purge_cache)
{
	unsigned int old_state = cinode->oplock;
	unsigned int old_epoch = cinode->epoch;
	unsigned int new_state;

	if (epoch > old_epoch) {
		smb21_set_oplock_level(cinode, oplock, 0, NULL);
		cinode->epoch = epoch;
	}

	new_state = cinode->oplock;
	*purge_cache = false;

	if ((old_state & CIFS_CACHE_READ_FLG) != 0 &&
	    (new_state & CIFS_CACHE_READ_FLG) == 0)
		*purge_cache = true;
	else if (old_state == new_state && (epoch - old_epoch > 1))
		*purge_cache = true;
}


smb2_query_dir_first(const unsigned int xid, struct cifs_tcon *tcon,
		     const char *path, struct cifs_sb_info *cifs_sb,
		     struct cifs_fid *fid, __u16 search_flags,
		     struct cifs_search_info *srch_inf)
{
	__le16 *utf16_path;
	struct smb_rqst rqst[2];
	struct kvec rsp_iov[2];
	int resp_buftype[2];
	struct kvec open_iov[SMB2_CREATE_IOV_SIZE];
	struct kvec qd_iov[SMB2_QUERY_DIRECTORY_IOV_SIZE];
	int rc, flags = 0;
	u8 oplock = SMB2_OPLOCK_LEVEL_NONE;
	struct cifs_open_parms oparms;
	struct smb2_query_directory_rsp *qd_rsp = NULL;
	struct smb2_create_rsp *op_rsp = NULL;
	struct TCP_Server_Info *server = cifs_pick_channel(tcon->ses);
	int retry_count = 0;

	utf16_path = cifs_convert_path_to_utf16(path, cifs_sb);
	if (!utf16_path)
		return -ENOMEM;

	if (smb3_encryption_required(tcon))
		flags |= CIFS_TRANSFORM_REQ;

	memset(rqst, 0, sizeof(rqst));
	resp_buftype[0] = resp_buftype[1] = CIFS_NO_BUFFER;
	memset(rsp_iov, 0, sizeof(rsp_iov));

	/* Open */
	memset(&open_iov, 0, sizeof(open_iov));
	rqst[0].rq_iov = open_iov;
	rqst[0].rq_nvec = SMB2_CREATE_IOV_SIZE;

	oparms.tcon = tcon;
	oparms.desired_access = FILE_READ_ATTRIBUTES | FILE_READ_DATA;
	oparms.disposition = FILE_OPEN;
	oparms.create_options = cifs_create_options(cifs_sb, 0);
	oparms.fid = fid;
	oparms.reconnect = false;

	rc = SMB2_open_init(tcon, server,
			    &rqst[0], &oplock, &oparms, utf16_path);
	if (rc)
		goto qdf_free;
	smb2_set_next_command(tcon, &rqst[0]);

	/* Query directory */
	srch_inf->entries_in_buffer = 0;
	srch_inf->index_of_last_entry = 2;

	memset(&qd_iov, 0, sizeof(qd_iov));
	rqst[1].rq_iov = qd_iov;
	rqst[1].rq_nvec = SMB2_QUERY_DIRECTORY_IOV_SIZE;

	rc = SMB2_query_directory_init(xid, tcon, server,
				       &rqst[1],
				       COMPOUND_FID, COMPOUND_FID,
				       0, srch_inf->info_level);
	if (rc)
		goto qdf_free;

	smb2_set_related(&rqst[1]);

again:
	rc = compound_send_recv(xid, tcon->ses, server,
				flags, 2, rqst,
				resp_buftype, rsp_iov);

	if (rc == -EAGAIN && retry_count++ < 10)
		goto again;

	/* If the open failed there is nothing to do */
	op_rsp = (struct smb2_create_rsp *)rsp_iov[0].iov_base;
	if (op_rsp == NULL || op_rsp->hdr.Status != STATUS_SUCCESS) {
		cifs_dbg(FYI, "query_dir_first: open failed rc=%d\n", rc);
		goto qdf_free;
	}
	fid->persistent_fid = op_rsp->PersistentFileId;
	fid->volatile_fid = op_rsp->VolatileFileId;

	/* Anything else than ENODATA means a genuine error */
	if (rc && rc != -ENODATA) {
		SMB2_close(xid, tcon, fid->persistent_fid, fid->volatile_fid);
		cifs_dbg(FYI, "query_dir_first: query directory failed rc=%d\n", rc);
		trace_smb3_query_dir_err(xid, fid->persistent_fid,
					 tcon->tid, tcon->ses->Suid, 0, 0, rc);
		goto qdf_free;
	}

	atomic_inc(&tcon->num_remote_opens);

	qd_rsp = (struct smb2_query_directory_rsp *)rsp_iov[1].iov_base;
	if (qd_rsp->hdr.Status == STATUS_NO_MORE_FILES) {
		trace_smb3_query_dir_done(xid, fid->persistent_fid,
					  tcon->tid, tcon->ses->Suid, 0, 0);
		srch_inf->endOfSearch = true;
		rc = 0;
		goto qdf_free;
	}

	rc = smb2_parse_query_directory(tcon, &rsp_iov[1], resp_buftype[1],
					srch_inf);
	if (rc) {
		trace_smb3_query_dir_err(xid, fid->persistent_fid, tcon->tid,
			tcon->ses->Suid, 0, 0, rc);
		goto qdf_free;
	}
	resp_buftype[1] = CIFS_NO_BUFFER;

	trace_smb3_query_dir_done(xid, fid->persistent_fid, tcon->tid,
			tcon->ses->Suid, 0, srch_inf->entries_in_buffer);

 qdf_free:
	kfree(utf16_path);
	SMB2_open_free(&rqst[0]);
	SMB2_query_directory_free(&rqst[1]);
	free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base);
	free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base);
	return rc;
}


smb2_need_neg(struct TCP_Server_Info *server)
{
	return server->max_read == 0;
}


smb2_query_file_info(const unsigned int xid, struct cifs_tcon *tcon,
		     struct cifs_fid *fid, FILE_ALL_INFO *data)
{
	int rc;
	struct smb2_file_all_info *smb2_data;

	smb2_data = kzalloc(sizeof(struct smb2_file_all_info) + PATH_MAX * 2,
			    GFP_KERNEL);
	if (smb2_data == NULL)
		return -ENOMEM;

	rc = SMB2_query_info(xid, tcon, fid->persistent_fid, fid->volatile_fid,
			     smb2_data);
	if (!rc)
		move_smb2_info_to_cifs(data, smb2_data);
	kfree(smb2_data);
	return rc;
}


fill_transform_hdr(struct smb2_transform_hdr *tr_hdr, unsigned int orig_len,
		   struct smb_rqst *old_rq, __le16 cipher_type)
{
	struct smb2_hdr *shdr =
			(struct smb2_hdr *)old_rq->rq_iov[0].iov_base;

	memset(tr_hdr, 0, sizeof(struct smb2_transform_hdr));
	tr_hdr->ProtocolId = SMB2_TRANSFORM_PROTO_NUM;
	tr_hdr->OriginalMessageSize = cpu_to_le32(orig_len);
	tr_hdr->Flags = cpu_to_le16(0x01);
	if ((cipher_type == SMB2_ENCRYPTION_AES128_GCM) ||
	    (cipher_type == SMB2_ENCRYPTION_AES256_GCM))
		get_random_bytes(&tr_hdr->Nonce, SMB3_AES_GCM_NONCE);
	else
		get_random_bytes(&tr_hdr->Nonce, SMB3_AES_CCM_NONCE);
	memcpy(&tr_hdr->SessionId, &shdr->SessionId, 8);
}


smb3_is_transform_hdr(void *buf)
{
	struct smb2_transform_hdr *trhdr = buf;

	return trhdr->ProtocolId == SMB2_TRANSFORM_PROTO_NUM;
}


smb2_negotiate(const unsigned int xid,
	       struct cifs_ses *ses,
	       struct TCP_Server_Info *server)
{
	int rc;

	spin_lock(&GlobalMid_Lock);
	server->CurrentMid = 0;
	spin_unlock(&GlobalMid_Lock);
	rc = SMB2_negotiate(xid, ses, server);
	/* BB we probably don't need to retry with modern servers */
	if (rc == -EAGAIN)
		rc = -EHOSTDOWN;
	return rc;
}


smb2_can_echo(struct TCP_Server_Info *server)
{
	return server->echoes;
}


static int compare_iface(const void *ia, const void *ib)
{
	const struct cifs_server_iface *a = (struct cifs_server_iface *)ia;
	const struct cifs_server_iface *b = (struct cifs_server_iface *)ib;

	return a->speed == b->speed ? 0 : (a->speed > b->speed ? -1 : 1);
}


get_smb2_acl_by_fid(struct cifs_sb_info *cifs_sb,
		    const struct cifs_fid *cifsfid, u32 *pacllen, u32 info)
{
	struct cifs_ntsd *pntsd = NULL;
	unsigned int xid;
	int rc = -EOPNOTSUPP;
	struct tcon_link *tlink = cifs_sb_tlink(cifs_sb);

	if (IS_ERR(tlink))
		return ERR_CAST(tlink);

	xid = get_xid();
	cifs_dbg(FYI, "trying to get acl\n");

	rc = SMB2_query_acl(xid, tlink_tcon(tlink), cifsfid->persistent_fid,
			    cifsfid->volatile_fid, (void **)&pntsd, pacllen,
			    info);
	free_xid(xid);

	cifs_put_tlink(tlink);

	cifs_dbg(FYI, "%s: rc = %d ACL len %d\n", __func__, rc, *pacllen);
	if (rc)
		return ERR_PTR(rc);
	return pntsd;

}


smb3_init_transform_rq(struct TCP_Server_Info *server, int num_rqst,
		       struct smb_rqst *new_rq, struct smb_rqst *old_rq)
{
	struct page **pages;
	struct smb2_transform_hdr *tr_hdr = new_rq[0].rq_iov[0].iov_base;
	unsigned int npages;
	unsigned int orig_len = 0;
	int i, j;
	int rc = -ENOMEM;

	for (i = 1; i < num_rqst; i++) {
		npages = old_rq[i - 1].rq_npages;
		pages = kmalloc_array(npages, sizeof(struct page *),
				      GFP_KERNEL);
		if (!pages)
			goto err_free;

		new_rq[i].rq_pages = pages;
		new_rq[i].rq_npages = npages;
		new_rq[i].rq_offset = old_rq[i - 1].rq_offset;
		new_rq[i].rq_pagesz = old_rq[i - 1].rq_pagesz;
		new_rq[i].rq_tailsz = old_rq[i - 1].rq_tailsz;
		new_rq[i].rq_iov = old_rq[i - 1].rq_iov;
		new_rq[i].rq_nvec = old_rq[i - 1].rq_nvec;

		orig_len += smb_rqst_len(server, &old_rq[i - 1]);

		for (j = 0; j < npages; j++) {
			pages[j] = alloc_page(GFP_KERNEL|__GFP_HIGHMEM);
			if (!pages[j])
				goto err_free;
		}

		/* copy pages form the old */
		for (j = 0; j < npages; j++) {
			char *dst, *src;
			unsigned int offset, len;

			rqst_page_get_length(&new_rq[i], j, &len, &offset);

			dst = (char *) kmap(new_rq[i].rq_pages[j]) + offset;
			src = (char *) kmap(old_rq[i - 1].rq_pages[j]) + offset;

			memcpy(dst, src, len);
			kunmap(new_rq[i].rq_pages[j]);
			kunmap(old_rq[i - 1].rq_pages[j]);
		}
	}

	/* fill the 1st iov with a transform header */
	fill_transform_hdr(tr_hdr, orig_len, old_rq, server->cipher_type);

	rc = crypt_message(server, num_rqst, new_rq, 1);
	cifs_dbg(FYI, "Encrypt message returned %d\n", rc);
	if (rc)
		goto err_free;

	return rc;

err_free:
	smb3_free_compound_rqst(num_rqst - 1, &new_rq[1]);
	return rc;
}


smb3_negotiate_rsize(struct cifs_tcon *tcon, struct smb3_fs_context *ctx)
{
	struct TCP_Server_Info *server = tcon->ses->server;
	unsigned int rsize;

	/* start with specified rsize, or default */
	rsize = ctx->rsize ? ctx->rsize : SMB3_DEFAULT_IOSIZE;
	rsize = min_t(unsigned int, rsize, server->max_read);
#ifdef CONFIG_CIFS_SMB_DIRECT
	if (server->rdma) {
		if (server->sign)
			/*
			 * Account for SMB2 data transfer packet header and
			 * possible encryption header
			 */
			rsize = min_t(unsigned int,
				rsize,
				server->smbd_conn->max_fragmented_recv_size -
					SMB2_READWRITE_PDU_HEADER_SIZE -
					sizeof(struct smb2_transform_hdr));
		else
			rsize = min_t(unsigned int,
				rsize, server->smbd_conn->max_readwrite_size);
	}
#endif

	if (!(server->capabilities & SMB2_GLOBAL_CAP_LARGE_MTU))
		rsize = min_t(unsigned int, rsize, SMB2_MAX_BUFFER_SIZE);

	return rsize;
}


smb2_set_file_size(const unsigned int xid, struct cifs_tcon *tcon,
		   struct cifsFileInfo *cfile, __u64 size, bool set_alloc)
{
	__le64 eof = cpu_to_le64(size);
	struct inode *inode;

	/*
	 * If extending file more than one page make sparse. Many Linux fs
	 * make files sparse by default when extending via ftruncate
	 */
	inode = d_inode(cfile->dentry);

	if (!set_alloc && (size > inode->i_size + 8192)) {
		__u8 set_sparse = 1;

		/* whether set sparse succeeds or not, extend the file */
		smb2_set_sparse(xid, tcon, cfile, inode, set_sparse);
	}

	return SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid,
			    cfile->fid.volatile_fid, cfile->pid, &eof);
}


smb2_copychunk_range(const unsigned int xid,
			struct cifsFileInfo *srcfile,
			struct cifsFileInfo *trgtfile, u64 src_off,
			u64 len, u64 dest_off)
{
	int rc;
	unsigned int ret_data_len;
	struct copychunk_ioctl *pcchunk;
	struct copychunk_ioctl_rsp *retbuf = NULL;
	struct cifs_tcon *tcon;
	int chunks_copied = 0;
	bool chunk_sizes_updated = false;
	ssize_t bytes_written, total_bytes_written = 0;

	pcchunk = kmalloc(sizeof(struct copychunk_ioctl), GFP_KERNEL);

	if (pcchunk == NULL)
		return -ENOMEM;

	cifs_dbg(FYI, "%s: about to call request res key\n", __func__);
	/* Request a key from the server to identify the source of the copy */
	rc = SMB2_request_res_key(xid, tlink_tcon(srcfile->tlink),
				srcfile->fid.persistent_fid,
				srcfile->fid.volatile_fid, pcchunk);

	/* Note: request_res_key sets res_key null only if rc !=0 */
	if (rc)
		goto cchunk_out;

	/* For now array only one chunk long, will make more flexible later */
	pcchunk->ChunkCount = cpu_to_le32(1);
	pcchunk->Reserved = 0;
	pcchunk->Reserved2 = 0;

	tcon = tlink_tcon(trgtfile->tlink);

	while (len > 0) {
		pcchunk->SourceOffset = cpu_to_le64(src_off);
		pcchunk->TargetOffset = cpu_to_le64(dest_off);
		pcchunk->Length =
			cpu_to_le32(min_t(u32, len, tcon->max_bytes_chunk));

		/* Request server copy to target from src identified by key */
		kfree(retbuf);
		retbuf = NULL;
		rc = SMB2_ioctl(xid, tcon, trgtfile->fid.persistent_fid,
			trgtfile->fid.volatile_fid, FSCTL_SRV_COPYCHUNK_WRITE,
			true /* is_fsctl */, (char *)pcchunk,
			sizeof(struct copychunk_ioctl),	CIFSMaxBufSize,
			(char **)&retbuf, &ret_data_len);
		if (rc == 0) {
			if (ret_data_len !=
					sizeof(struct copychunk_ioctl_rsp)) {
				cifs_tcon_dbg(VFS, "Invalid cchunk response size\n");
				rc = -EIO;
				goto cchunk_out;
			}
			if (retbuf->TotalBytesWritten == 0) {
				cifs_dbg(FYI, "no bytes copied\n");
				rc = -EIO;
				goto cchunk_out;
			}
			/*
			 * Check if server claimed to write more than we asked
			 */
			if (le32_to_cpu(retbuf->TotalBytesWritten) >
			    le32_to_cpu(pcchunk->Length)) {
				cifs_tcon_dbg(VFS, "Invalid copy chunk response\n");
				rc = -EIO;
				goto cchunk_out;
			}
			if (le32_to_cpu(retbuf->ChunksWritten) != 1) {
				cifs_tcon_dbg(VFS, "Invalid num chunks written\n");
				rc = -EIO;
				goto cchunk_out;
			}
			chunks_copied++;

			bytes_written = le32_to_cpu(retbuf->TotalBytesWritten);
			src_off += bytes_written;
			dest_off += bytes_written;
			len -= bytes_written;
			total_bytes_written += bytes_written;

			cifs_dbg(FYI, "Chunks %d PartialChunk %d Total %zu\n",
				le32_to_cpu(retbuf->ChunksWritten),
				le32_to_cpu(retbuf->ChunkBytesWritten),
				bytes_written);
		} else if (rc == -EINVAL) {
			if (ret_data_len != sizeof(struct copychunk_ioctl_rsp))
				goto cchunk_out;

			cifs_dbg(FYI, "MaxChunks %d BytesChunk %d MaxCopy %d\n",
				le32_to_cpu(retbuf->ChunksWritten),
				le32_to_cpu(retbuf->ChunkBytesWritten),
				le32_to_cpu(retbuf->TotalBytesWritten));

			/*
			 * Check if this is the first request using these sizes,
			 * (ie check if copy succeed once with original sizes
			 * and check if the server gave us different sizes after
			 * we already updated max sizes on previous request).
			 * if not then why is the server returning an error now
			 */
			if ((chunks_copied != 0) || chunk_sizes_updated)
				goto cchunk_out;

			/* Check that server is not asking us to grow size */
			if (le32_to_cpu(retbuf->ChunkBytesWritten) <
					tcon->max_bytes_chunk)
				tcon->max_bytes_chunk =
					le32_to_cpu(retbuf->ChunkBytesWritten);
			else
				goto cchunk_out; /* server gave us bogus size */

			/* No need to change MaxChunks since already set to 1 */
			chunk_sizes_updated = true;
		} else
			goto cchunk_out;
	}

cchunk_out:
	kfree(pcchunk);
	kfree(retbuf);
	if (rc)
		return rc;
	else
		return total_bytes_written;
}


static inline void smb2_sg_set_buf(struct scatterlist *sg, const void *buf,
				   unsigned int buflen)
{
	void *addr;
	/*
	 * VMAP_STACK (at least) puts stack into the vmalloc address space
	 */
	if (is_vmalloc_addr(buf))
		addr = vmalloc_to_page(buf);
	else
		addr = virt_to_page(buf);
	sg_set_page(sg, addr, buflen, offset_in_page(buf));
}


smb3_negotiate_wsize(struct cifs_tcon *tcon, struct smb3_fs_context *ctx)
{
	struct TCP_Server_Info *server = tcon->ses->server;
	unsigned int wsize;

	/* start with specified wsize, or default */
	wsize = ctx->wsize ? ctx->wsize : SMB3_DEFAULT_IOSIZE;
	wsize = min_t(unsigned int, wsize, server->max_write);
#ifdef CONFIG_CIFS_SMB_DIRECT
	if (server->rdma) {
		if (server->sign)
			/*
			 * Account for SMB2 data transfer packet header and
			 * possible encryption header
			 */
			wsize = min_t(unsigned int,
				wsize,
				server->smbd_conn->max_fragmented_send_size -
					SMB2_READWRITE_PDU_HEADER_SIZE -
					sizeof(struct smb2_transform_hdr));
		else
			wsize = min_t(unsigned int,
				wsize, server->smbd_conn->max_readwrite_size);
	}
#endif
	if (!(server->capabilities & SMB2_GLOBAL_CAP_LARGE_MTU))
		wsize = min_t(unsigned int, wsize, SMB2_MAX_BUFFER_SIZE);

	return wsize;
}


smb3_create_lease_buf(u8 *lease_key, u8 oplock)
{
	struct create_lease_v2 *buf;

	buf = kzalloc(sizeof(struct create_lease_v2), GFP_KERNEL);
	if (!buf)
		return NULL;

	memcpy(&buf->lcontext.LeaseKey, lease_key, SMB2_LEASE_KEY_SIZE);
	buf->lcontext.LeaseState = map_oplock_to_lease(oplock);

	buf->ccontext.DataOffset = cpu_to_le16(offsetof
					(struct create_lease_v2, lcontext));
	buf->ccontext.DataLength = cpu_to_le32(sizeof(struct lease_context_v2));
	buf->ccontext.NameOffset = cpu_to_le16(offsetof
				(struct create_lease_v2, Name));
	buf->ccontext.NameLength = cpu_to_le16(4);
	/* SMB2_CREATE_REQUEST_LEASE is "RqLs" */
	buf->Name[0] = 'R';
	buf->Name[1] = 'q';
	buf->Name[2] = 'L';
	buf->Name[3] = 's';
	return (char *)buf;
}


smb2_downgrade_oplock(struct TCP_Server_Info *server,
		      struct cifsInodeInfo *cinode, __u32 oplock,
		      unsigned int epoch, bool *purge_cache)
{
	server->ops->set_oplock_level(cinode, oplock, 0, NULL);
}


smb2_is_network_name_deleted(char *buf, struct TCP_Server_Info *server)
{
	struct smb2_hdr *shdr = (struct smb2_hdr *)buf;
	struct list_head *tmp, *tmp1;
	struct cifs_ses *ses;
	struct cifs_tcon *tcon;

	if (shdr->Status != STATUS_NETWORK_NAME_DELETED)
		return;

	spin_lock(&cifs_tcp_ses_lock);
	list_for_each(tmp, &server->smb_ses_list) {
		ses = list_entry(tmp, struct cifs_ses, smb_ses_list);
		list_for_each(tmp1, &ses->tcon_list) {
			tcon = list_entry(tmp1, struct cifs_tcon, tcon_list);
			if (tcon->tid == le32_to_cpu(shdr->Id.SyncId.TreeId)) {
				tcon->need_reconnect = true;
				spin_unlock(&cifs_tcp_ses_lock);
				pr_warn_once("Server share %s deleted.\n",
					     tcon->treeName);
				return;
			}
		}
	}
	spin_unlock(&cifs_tcp_ses_lock);
}


set_smb2_acl(struct cifs_ntsd *pnntsd, __u32 acllen,
		struct inode *inode, const char *path, int aclflag)
{
	u8 oplock = SMB2_OPLOCK_LEVEL_NONE;
	unsigned int xid;
	int rc, access_flags = 0;
	struct cifs_tcon *tcon;
	struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb);
	struct tcon_link *tlink = cifs_sb_tlink(cifs_sb);
	struct cifs_fid fid;
	struct cifs_open_parms oparms;
	__le16 *utf16_path;

	cifs_dbg(FYI, "set smb3 acl for path %s\n", path);
	if (IS_ERR(tlink))
		return PTR_ERR(tlink);

	tcon = tlink_tcon(tlink);
	xid = get_xid();

	if (aclflag & CIFS_ACL_OWNER || aclflag & CIFS_ACL_GROUP)
		access_flags |= WRITE_OWNER;
	if (aclflag & CIFS_ACL_SACL)
		access_flags |= SYSTEM_SECURITY;
	if (aclflag & CIFS_ACL_DACL)
		access_flags |= WRITE_DAC;

	utf16_path = cifs_convert_path_to_utf16(path, cifs_sb);
	if (!utf16_path) {
		rc = -ENOMEM;
		free_xid(xid);
		return rc;
	}

	oparms.tcon = tcon;
	oparms.desired_access = access_flags;
	oparms.create_options = cifs_create_options(cifs_sb, 0);
	oparms.disposition = FILE_OPEN;
	oparms.path = path;
	oparms.fid = &fid;
	oparms.reconnect = false;

	rc = SMB2_open(xid, &oparms, utf16_path, &oplock, NULL, NULL,
		       NULL, NULL);
	kfree(utf16_path);
	if (!rc) {
		rc = SMB2_set_acl(xid, tlink_tcon(tlink), fid.persistent_fid,
			    fid.volatile_fid, pnntsd, acllen, aclflag);
		SMB2_close(xid, tcon, fid.persistent_fid, fid.volatile_fid);
	}

	cifs_put_tlink(tlink);
	free_xid(xid);
	return rc;
}


smb2_is_path_accessible(const unsigned int xid, struct cifs_tcon *tcon,
			struct cifs_sb_info *cifs_sb, const char *full_path)
{
	int rc;
	__le16 *utf16_path;
	__u8 oplock = SMB2_OPLOCK_LEVEL_NONE;
	struct cifs_open_parms oparms;
	struct cifs_fid fid;

	if ((*full_path == 0) && tcon->crfid.is_valid)
		return 0;

	utf16_path = cifs_convert_path_to_utf16(full_path, cifs_sb);
	if (!utf16_path)
		return -ENOMEM;

	oparms.tcon = tcon;
	oparms.desired_access = FILE_READ_ATTRIBUTES;
	oparms.disposition = FILE_OPEN;
	oparms.create_options = cifs_create_options(cifs_sb, 0);
	oparms.fid = &fid;
	oparms.reconnect = false;

	rc = SMB2_open(xid, &oparms, utf16_path, &oplock, NULL, NULL, NULL,
		       NULL);
	if (rc) {
		kfree(utf16_path);
		return rc;
	}

	rc = SMB2_close(xid, tcon, fid.persistent_fid, fid.volatile_fid);
	kfree(utf16_path);
	return rc;
}


parse_reparse_point(struct reparse_data_buffer *buf,
		    u32 plen, char **target_path,
		    struct cifs_sb_info *cifs_sb)
{
	if (plen < sizeof(struct reparse_data_buffer)) {
		cifs_dbg(VFS, "reparse buffer is too small. Must be at least 8 bytes but was %d\n",
			 plen);
		return -EIO;
	}

	if (plen < le16_to_cpu(buf->ReparseDataLength) +
	    sizeof(struct reparse_data_buffer)) {
		cifs_dbg(VFS, "srv returned invalid reparse buf length: %d\n",
			 plen);
		return -EIO;
	}

	/* See MS-FSCC 2.1.2 */
	switch (le32_to_cpu(buf->ReparseTag)) {
	case IO_REPARSE_TAG_NFS:
		return parse_reparse_posix(
			(struct reparse_posix_data *)buf,
			plen, target_path, cifs_sb);
	case IO_REPARSE_TAG_SYMLINK:
		return parse_reparse_symlink(
			(struct reparse_symlink_data_buffer *)buf,
			plen, target_path, cifs_sb);
	default:
		cifs_dbg(VFS, "srv returned unknown symlink buffer tag:0x%08x\n",
			 le32_to_cpu(buf->ReparseTag));
		return -EOPNOTSUPP;
	}
}


SMB3_request_interfaces(const unsigned int xid, struct cifs_tcon *tcon)
{
	int rc;
	unsigned int ret_data_len = 0;
	struct network_interface_info_ioctl_rsp *out_buf = NULL;
	struct cifs_server_iface *iface_list;
	size_t iface_count;
	struct cifs_ses *ses = tcon->ses;

	rc = SMB2_ioctl(xid, tcon, NO_FILE_ID, NO_FILE_ID,
			FSCTL_QUERY_NETWORK_INTERFACE_INFO, true /* is_fsctl */,
			NULL /* no data input */, 0 /* no data input */,
			CIFSMaxBufSize, (char **)&out_buf, &ret_data_len);
	if (rc == -EOPNOTSUPP) {
		cifs_dbg(FYI,
			 "server does not support query network interfaces\n");
		goto out;
	} else if (rc != 0) {
		cifs_tcon_dbg(VFS, "error %d on ioctl to get interface list\n", rc);
		goto out;
	}

	rc = parse_server_interfaces(out_buf, ret_data_len,
				     &iface_list, &iface_count);
	if (rc)
		goto out;

	/* sort interfaces from fastest to slowest */
	sort(iface_list, iface_count, sizeof(*iface_list), compare_iface, NULL);

	spin_lock(&ses->iface_lock);
	kfree(ses->iface_list);
	ses->iface_list = iface_list;
	ses->iface_count = iface_count;
	ses->iface_last_update = jiffies;
	spin_unlock(&ses->iface_lock);

out:
	kfree(out_buf);
	return rc;
}


smb2_is_status_pending(char *buf, struct TCP_Server_Info *server)
{
	struct smb2_hdr *shdr = (struct smb2_hdr *)buf;
	int scredits, in_flight;

	if (shdr->Status != STATUS_PENDING)
		return false;

	if (shdr->CreditRequest) {
		spin_lock(&server->req_lock);
		server->credits += le16_to_cpu(shdr->CreditRequest);
		scredits = server->credits;
		in_flight = server->in_flight;
		spin_unlock(&server->req_lock);
		wake_up(&server->request_q);

		trace_smb3_add_credits(server->CurrentMid,
				server->conn_id, server->hostname, scredits,
				le16_to_cpu(shdr->CreditRequest), in_flight);
		cifs_dbg(FYI, "%s: status pending add %u credits total=%d\n",
				__func__, le16_to_cpu(shdr->CreditRequest), scredits);
	}

	return true;
}


smb2_dump_detail(void *buf, struct TCP_Server_Info *server)
{
#ifdef CONFIG_CIFS_DEBUG2
	struct smb2_hdr *shdr = (struct smb2_hdr *)buf;

	cifs_server_dbg(VFS, "Cmd: %d Err: 0x%x Flags: 0x%x Mid: %llu Pid: %d\n",
		 shdr->Command, shdr->Status, shdr->Flags, shdr->MessageId,
		 shdr->Id.SyncId.ProcessId);
	cifs_server_dbg(VFS, "smb buf %p len %u\n", buf,
		 server->ops->calc_smb_size(buf, server));
#endif
}


smb2_duplicate_extents(const unsigned int xid,
			struct cifsFileInfo *srcfile,
			struct cifsFileInfo *trgtfile, u64 src_off,
			u64 len, u64 dest_off)
{
	int rc;
	unsigned int ret_data_len;
	struct inode *inode;
	struct duplicate_extents_to_file dup_ext_buf;
	struct cifs_tcon *tcon = tlink_tcon(trgtfile->tlink);

	/* server fileays advertise duplicate extent support with this flag */
	if ((le32_to_cpu(tcon->fsAttrInfo.Attributes) &
	     FILE_SUPPORTS_BLOCK_REFCOUNTING) == 0)
		return -EOPNOTSUPP;

	dup_ext_buf.VolatileFileHandle = srcfile->fid.volatile_fid;
	dup_ext_buf.PersistentFileHandle = srcfile->fid.persistent_fid;
	dup_ext_buf.SourceFileOffset = cpu_to_le64(src_off);
	dup_ext_buf.TargetFileOffset = cpu_to_le64(dest_off);
	dup_ext_buf.ByteCount = cpu_to_le64(len);
	cifs_dbg(FYI, "Duplicate extents: src off %lld dst off %lld len %lld\n",
		src_off, dest_off, len);

	inode = d_inode(trgtfile->dentry);
	if (inode->i_size < dest_off + len) {
		rc = smb2_set_file_size(xid, tcon, trgtfile, dest_off + len, false);
		if (rc)
			goto duplicate_extents_out;

		/*
		 * Although also could set plausible allocation size (i_blocks)
		 * here in addition to setting the file size, in reflink
		 * it is likely that the target file is sparse. Its allocation
		 * size will be queried on next revalidate, but it is important
		 * to make sure that file's cached size is updated immediately
		 */
		cifs_setsize(inode, dest_off + len);
	}
	rc = SMB2_ioctl(xid, tcon, trgtfile->fid.persistent_fid,
			trgtfile->fid.volatile_fid,
			FSCTL_DUPLICATE_EXTENTS_TO_FILE,
			true /* is_fsctl */,
			(char *)&dup_ext_buf,
			sizeof(struct duplicate_extents_to_file),
			CIFSMaxBufSize, NULL,
			&ret_data_len);

	if (ret_data_len > 0)
		cifs_dbg(FYI, "Non-zero response length in duplicate extents\n");

duplicate_extents_out:
	return rc;
}


smb2_wp_retry_size(struct inode *inode)
{
	return min_t(unsigned int, CIFS_SB(inode->i_sb)->ctx->wsize,
		     SMB2_MAX_BUFFER_SIZE);
}


smb3_receive_transform(struct TCP_Server_Info *server,
		       struct mid_q_entry **mids, char **bufs, int *num_mids)
{
	char *buf = server->smallbuf;
	unsigned int pdu_length = server->pdu_size;
	struct smb2_transform_hdr *tr_hdr = (struct smb2_transform_hdr *)buf;
	unsigned int orig_len = le32_to_cpu(tr_hdr->OriginalMessageSize);

	if (pdu_length < sizeof(struct smb2_transform_hdr) +
						sizeof(struct smb2_hdr)) {
		cifs_server_dbg(VFS, "Transform message is too small (%u)\n",
			 pdu_length);
		cifs_reconnect(server, true);
		return -ECONNABORTED;
	}

	if (pdu_length < orig_len + sizeof(struct smb2_transform_hdr)) {
		cifs_server_dbg(VFS, "Transform message is broken\n");
		cifs_reconnect(server, true);
		return -ECONNABORTED;
	}

	/* TODO: add support for compounds containing READ. */
	if (pdu_length > CIFSMaxBufSize + MAX_HEADER_SIZE(server)) {
		return receive_encrypted_read(server, &mids[0], num_mids);
	}

	return receive_encrypted_standard(server, mids, bufs, num_mids);
}


init_sg(int num_rqst, struct smb_rqst *rqst, u8 *sign)
{
	unsigned int sg_len;
	struct scatterlist *sg;
	unsigned int i;
	unsigned int j;
	unsigned int idx = 0;
	int skip;

	sg_len = 1;
	for (i = 0; i < num_rqst; i++)
		sg_len += rqst[i].rq_nvec + rqst[i].rq_npages;

	sg = kmalloc_array(sg_len, sizeof(struct scatterlist), GFP_KERNEL);
	if (!sg)
		return NULL;

	sg_init_table(sg, sg_len);
	for (i = 0; i < num_rqst; i++) {
		for (j = 0; j < rqst[i].rq_nvec; j++) {
			/*
			 * The first rqst has a transform header where the
			 * first 20 bytes are not part of the encrypted blob
			 */
			skip = (i == 0) && (j == 0) ? 20 : 0;
			smb2_sg_set_buf(&sg[idx++],
					rqst[i].rq_iov[j].iov_base + skip,
					rqst[i].rq_iov[j].iov_len - skip);
			}

		for (j = 0; j < rqst[i].rq_npages; j++) {
			unsigned int len, offset;

			rqst_page_get_length(&rqst[i], j, &len, &offset);
			sg_set_page(&sg[idx++], rqst[i].rq_pages[j], len, offset);
		}
	}
	smb2_sg_set_buf(&sg[idx], sign, SMB2_SIGNATURE_SIZE);
	return sg;
}


smb2_close_getattr(const unsigned int xid, struct cifs_tcon *tcon,
		   struct cifsFileInfo *cfile)
{
	struct smb2_file_network_open_info file_inf;
	struct inode *inode;
	int rc;

	rc = __SMB2_close(xid, tcon, cfile->fid.persistent_fid,
		   cfile->fid.volatile_fid, &file_inf);
	if (rc)
		return;

	inode = d_inode(cfile->dentry);

	spin_lock(&inode->i_lock);
	CIFS_I(inode)->time = jiffies;

	/* Creation time should not need to be updated on close */
	if (file_inf.LastWriteTime)
		inode->i_mtime = cifs_NTtimeToUnix(file_inf.LastWriteTime);
	if (file_inf.ChangeTime)
		inode->i_ctime = cifs_NTtimeToUnix(file_inf.ChangeTime);
	if (file_inf.LastAccessTime)
		inode->i_atime = cifs_NTtimeToUnix(file_inf.LastAccessTime);

	/*
	 * i_blocks is not related to (i_size / i_blksize),
	 * but instead 512 byte (2**9) size is required for
	 * calculating num blocks.
	 */
	if (le64_to_cpu(file_inf.AllocationSize) > 4096)
		inode->i_blocks =
			(512 - 1 + le64_to_cpu(file_inf.AllocationSize)) >> 9;

	/* End of file and Attributes should not have to be updated on close */
	spin_unlock(&inode->i_lock);
}


get_smb2_acl_by_path(struct cifs_sb_info *cifs_sb,
		     const char *path, u32 *pacllen, u32 info)
{
	struct cifs_ntsd *pntsd = NULL;
	u8 oplock = SMB2_OPLOCK_LEVEL_NONE;
	unsigned int xid;
	int rc;
	struct cifs_tcon *tcon;
	struct tcon_link *tlink = cifs_sb_tlink(cifs_sb);
	struct cifs_fid fid;
	struct cifs_open_parms oparms;
	__le16 *utf16_path;

	cifs_dbg(FYI, "get smb3 acl for path %s\n", path);
	if (IS_ERR(tlink))
		return ERR_CAST(tlink);

	tcon = tlink_tcon(tlink);
	xid = get_xid();

	utf16_path = cifs_convert_path_to_utf16(path, cifs_sb);
	if (!utf16_path) {
		rc = -ENOMEM;
		free_xid(xid);
		return ERR_PTR(rc);
	}

	oparms.tcon = tcon;
	oparms.desired_access = READ_CONTROL;
	oparms.disposition = FILE_OPEN;
	/*
	 * When querying an ACL, even if the file is a symlink we want to open
	 * the source not the target, and so the protocol requires that the
	 * client specify this flag when opening a reparse point
	 */
	oparms.create_options = cifs_create_options(cifs_sb, 0) | OPEN_REPARSE_POINT;
	oparms.fid = &fid;
	oparms.reconnect = false;

	if (info & SACL_SECINFO)
		oparms.desired_access |= SYSTEM_SECURITY;

	rc = SMB2_open(xid, &oparms, utf16_path, &oplock, NULL, NULL, NULL,
		       NULL);
	kfree(utf16_path);
	if (!rc) {
		rc = SMB2_query_acl(xid, tlink_tcon(tlink), fid.persistent_fid,
				    fid.volatile_fid, (void **)&pntsd, pacllen,
				    info);
		SMB2_close(xid, tcon, fid.persistent_fid, fid.volatile_fid);
	}

	cifs_put_tlink(tlink);
	free_xid(xid);

	cifs_dbg(FYI, "%s: rc = %d ACL len %d\n", __func__, rc, *pacllen);
	if (rc)
		return ERR_PTR(rc);
	return pntsd;
}


static void smb2_decrypt_offload(struct work_struct *work)
{
	struct smb2_decrypt_work *dw = container_of(work,
				struct smb2_decrypt_work, decrypt);
	int i, rc;
	struct mid_q_entry *mid;

	rc = decrypt_raw_data(dw->server, dw->buf, dw->server->vals->read_rsp_size,
			      dw->ppages, dw->npages, dw->len, true);
	if (rc) {
		cifs_dbg(VFS, "error decrypting rc=%d\n", rc);
		goto free_pages;
	}

	dw->server->lstrp = jiffies;
	mid = smb2_find_dequeue_mid(dw->server, dw->buf);
	if (mid == NULL)
		cifs_dbg(FYI, "mid not found\n");
	else {
		mid->decrypted = true;
		rc = handle_read_data(dw->server, mid, dw->buf,
				      dw->server->vals->read_rsp_size,
				      dw->ppages, dw->npages, dw->len,
				      true);
		if (rc >= 0) {
#ifdef CONFIG_CIFS_STATS2
			mid->when_received = jiffies;
#endif
			if (dw->server->ops->is_network_name_deleted)
				dw->server->ops->is_network_name_deleted(dw->buf,
									 dw->server);

			mid->callback(mid);
		} else {
			spin_lock(&cifs_tcp_ses_lock);
			spin_lock(&GlobalMid_Lock);
			if (dw->server->tcpStatus == CifsNeedReconnect) {
				mid->mid_state = MID_RETRY_NEEDED;
				spin_unlock(&GlobalMid_Lock);
				spin_unlock(&cifs_tcp_ses_lock);
				mid->callback(mid);
			} else {
				mid->mid_state = MID_REQUEST_SUBMITTED;
				mid->mid_flags &= ~(MID_DELETED);
				list_add_tail(&mid->qhead,
					&dw->server->pending_mid_q);
				spin_unlock(&GlobalMid_Lock);
				spin_unlock(&cifs_tcp_ses_lock);
			}
		}
		cifs_mid_q_entry_release(mid);
	}

free_pages:
	for (i = dw->npages-1; i >= 0; i--)
		put_page(dw->ppages[i]);

	kfree(dw->ppages);
	cifs_small_buf_release(dw->buf);
	kfree(dw);
}


smb2_cached_lease_break(struct work_struct *work)
{
	struct cached_fid *cfid = container_of(work,
				struct cached_fid, lease_break);

	close_cached_dir_lease(cfid);
}


smb2_close_file(const unsigned int xid, struct cifs_tcon *tcon,
		struct cifs_fid *fid)
{
	SMB2_close(xid, tcon, fid->persistent_fid, fid->volatile_fid);
}


init_read_bvec(struct page **pages, unsigned int npages, unsigned int data_size,
	       unsigned int cur_off, struct bio_vec **page_vec)
{
	struct bio_vec *bvec;
	int i;

	bvec = kcalloc(npages, sizeof(struct bio_vec), GFP_KERNEL);
	if (!bvec)
		return -ENOMEM;

	for (i = 0; i < npages; i++) {
		bvec[i].bv_page = pages[i];
		bvec[i].bv_offset = (i == 0) ? cur_off : 0;
		bvec[i].bv_len = min_t(unsigned int, PAGE_SIZE, data_size);
		data_size -= bvec[i].bv_len;
	}

	if (data_size != 0) {
		cifs_dbg(VFS, "%s: something went wrong\n", __func__);
		kfree(bvec);
		return -EIO;
	}

	*page_vec = bvec;
	return 0;
}


move_smb2_ea_to_cifs(char *dst, size_t dst_size,
		     struct smb2_file_full_ea_info *src, size_t src_size,
		     const unsigned char *ea_name)
{
	int rc = 0;
	unsigned int ea_name_len = ea_name ? strlen(ea_name) : 0;
	char *name, *value;
	size_t buf_size = dst_size;
	size_t name_len, value_len, user_name_len;

	while (src_size > 0) {
		name = &src->ea_data[0];
		name_len = (size_t)src->ea_name_length;
		value = &src->ea_data[src->ea_name_length + 1];
		value_len = (size_t)le16_to_cpu(src->ea_value_length);

		if (name_len == 0)
			break;

		if (src_size < 8 + name_len + 1 + value_len) {
			cifs_dbg(FYI, "EA entry goes beyond length of list\n");
			rc = -EIO;
			goto out;
		}

		if (ea_name) {
			if (ea_name_len == name_len &&
			    memcmp(ea_name, name, name_len) == 0) {
				rc = value_len;
				if (dst_size == 0)
					goto out;
				if (dst_size < value_len) {
					rc = -ERANGE;
					goto out;
				}
				memcpy(dst, value, value_len);
				goto out;
			}
		} else {
			/* 'user.' plus a terminating null */
			user_name_len = 5 + 1 + name_len;

			if (buf_size == 0) {
				/* skip copy - calc size only */
				rc += user_name_len;
			} else if (dst_size >= user_name_len) {
				dst_size -= user_name_len;
				memcpy(dst, "user.", 5);
				dst += 5;
				memcpy(dst, src->ea_data, name_len);
				dst += name_len;
				*dst = 0;
				++dst;
				rc += user_name_len;
			} else {
				/* stop before overrun buffer */
				rc = -ERANGE;
				break;
			}
		}

		if (!src->next_entry_offset)
			break;

		if (src_size < le32_to_cpu(src->next_entry_offset)) {
			/* stop before overrun buffer */
			rc = -ERANGE;
			break;
		}
		src_size -= le32_to_cpu(src->next_entry_offset);
		src = (void *)((char *)src +
			       le32_to_cpu(src->next_entry_offset));
	}

	/* didn't find the named attribute */
	if (ea_name)
		rc = -ENODATA;

out:
	return (ssize_t)rc;
}


smb2_new_lease_key(struct cifs_fid *fid)
{
	generate_random_uuid(fid->lease_key);
}


smb3_qfs_tcon(const unsigned int xid, struct cifs_tcon *tcon,
	      struct cifs_sb_info *cifs_sb)
{
	int rc;
	__le16 srch_path = 0; /* Null - open root of share */
	u8 oplock = SMB2_OPLOCK_LEVEL_NONE;
	struct cifs_open_parms oparms;
	struct cifs_fid fid;
	struct cached_fid *cfid = NULL;

	oparms.tcon = tcon;
	oparms.desired_access = FILE_READ_ATTRIBUTES;
	oparms.disposition = FILE_OPEN;
	oparms.create_options = cifs_create_options(cifs_sb, 0);
	oparms.fid = &fid;
	oparms.reconnect = false;

	rc = open_cached_dir(xid, tcon, "", cifs_sb, &cfid);
	if (rc == 0)
		memcpy(&fid, cfid->fid, sizeof(struct cifs_fid));
	else
		rc = SMB2_open(xid, &oparms, &srch_path, &oplock, NULL, NULL,
			       NULL, NULL);
	if (rc)
		return;

	SMB3_request_interfaces(xid, tcon);

	SMB2_QFS_attr(xid, tcon, fid.persistent_fid, fid.volatile_fid,
			FS_ATTRIBUTE_INFORMATION);
	SMB2_QFS_attr(xid, tcon, fid.persistent_fid, fid.volatile_fid,
			FS_DEVICE_INFORMATION);
	SMB2_QFS_attr(xid, tcon, fid.persistent_fid, fid.volatile_fid,
			FS_VOLUME_INFORMATION);
	SMB2_QFS_attr(xid, tcon, fid.persistent_fid, fid.volatile_fid,
			FS_SECTOR_SIZE_INFORMATION); /* SMB3 specific */
	if (cfid == NULL)
		SMB2_close(xid, tcon, fid.persistent_fid, fid.volatile_fid);
	else
		close_cached_dir(cfid);
}


smb2_ioctl_query_info(const unsigned int xid,
		      struct cifs_tcon *tcon,
		      struct cifs_sb_info *cifs_sb,
		      __le16 *path, int is_dir,
		      unsigned long p)
{
	struct iqi_vars *vars;
	struct smb_rqst *rqst;
	struct kvec *rsp_iov;
	struct cifs_ses *ses = tcon->ses;
	struct TCP_Server_Info *server = cifs_pick_channel(ses);
	char __user *arg = (char __user *)p;
	struct smb_query_info qi;
	struct smb_query_info __user *pqi;
	int rc = 0;
	int flags = CIFS_CP_CREATE_CLOSE_OP;
	struct smb2_query_info_rsp *qi_rsp = NULL;
	struct smb2_ioctl_rsp *io_rsp = NULL;
	void *buffer = NULL;
	int resp_buftype[3];
	struct cifs_open_parms oparms;
	u8 oplock = SMB2_OPLOCK_LEVEL_NONE;
	struct cifs_fid fid;
	unsigned int size[2];
	void *data[2];
	int create_options = is_dir ? CREATE_NOT_FILE : CREATE_NOT_DIR;
	void (*free_req1_func)(struct smb_rqst *r);

	vars = kzalloc(sizeof(*vars), GFP_ATOMIC);
	if (vars == NULL)
		return -ENOMEM;
	rqst = &vars->rqst[0];
	rsp_iov = &vars->rsp_iov[0];

	resp_buftype[0] = resp_buftype[1] = resp_buftype[2] = CIFS_NO_BUFFER;

	if (copy_from_user(&qi, arg, sizeof(struct smb_query_info))) {
		rc = -EFAULT;
		goto free_vars;
	}
	if (qi.output_buffer_length > 1024) {
		rc = -EINVAL;
		goto free_vars;
	}

	if (!ses || !server) {
		rc = -EIO;
		goto free_vars;
	}

	if (smb3_encryption_required(tcon))
		flags |= CIFS_TRANSFORM_REQ;

	if (qi.output_buffer_length) {
		buffer = memdup_user(arg + sizeof(struct smb_query_info), qi.output_buffer_length);
		if (IS_ERR(buffer)) {
			rc = PTR_ERR(buffer);
			goto free_vars;
		}
	}

	/* Open */
	rqst[0].rq_iov = &vars->open_iov[0];
	rqst[0].rq_nvec = SMB2_CREATE_IOV_SIZE;

	memset(&oparms, 0, sizeof(oparms));
	oparms.tcon = tcon;
	oparms.disposition = FILE_OPEN;
	oparms.create_options = cifs_create_options(cifs_sb, create_options);
	oparms.fid = &fid;
	oparms.reconnect = false;

	if (qi.flags & PASSTHRU_FSCTL) {
		switch (qi.info_type & FSCTL_DEVICE_ACCESS_MASK) {
		case FSCTL_DEVICE_ACCESS_FILE_READ_WRITE_ACCESS:
			oparms.desired_access = FILE_READ_DATA | FILE_WRITE_DATA | FILE_READ_ATTRIBUTES | SYNCHRONIZE;
			break;
		case FSCTL_DEVICE_ACCESS_FILE_ANY_ACCESS:
			oparms.desired_access = GENERIC_ALL;
			break;
		case FSCTL_DEVICE_ACCESS_FILE_READ_ACCESS:
			oparms.desired_access = GENERIC_READ;
			break;
		case FSCTL_DEVICE_ACCESS_FILE_WRITE_ACCESS:
			oparms.desired_access = GENERIC_WRITE;
			break;
		}
	} else if (qi.flags & PASSTHRU_SET_INFO) {
		oparms.desired_access = GENERIC_WRITE;
	} else {
		oparms.desired_access = FILE_READ_ATTRIBUTES | READ_CONTROL;
	}

	rc = SMB2_open_init(tcon, server,
			    &rqst[0], &oplock, &oparms, path);
	if (rc)
		goto free_output_buffer;
	smb2_set_next_command(tcon, &rqst[0]);

	/* Query */
	if (qi.flags & PASSTHRU_FSCTL) {
		/* Can eventually relax perm check since server enforces too */
		if (!capable(CAP_SYS_ADMIN)) {
			rc = -EPERM;
			goto free_open_req;
		}
		rqst[1].rq_iov = &vars->io_iov[0];
		rqst[1].rq_nvec = SMB2_IOCTL_IOV_SIZE;

		rc = SMB2_ioctl_init(tcon, server, &rqst[1], COMPOUND_FID, COMPOUND_FID,
				     qi.info_type, true, buffer, qi.output_buffer_length,
				     CIFSMaxBufSize - MAX_SMB2_CREATE_RESPONSE_SIZE -
				     MAX_SMB2_CLOSE_RESPONSE_SIZE);
		free_req1_func = SMB2_ioctl_free;
	} else if (qi.flags == PASSTHRU_SET_INFO) {
		/* Can eventually relax perm check since server enforces too */
		if (!capable(CAP_SYS_ADMIN)) {
			rc = -EPERM;
			goto free_open_req;
		}
		if (qi.output_buffer_length < 8) {
			rc = -EINVAL;
			goto free_open_req;
		}
		rqst[1].rq_iov = &vars->si_iov[0];
		rqst[1].rq_nvec = 1;

		/* MS-FSCC 2.4.13 FileEndOfFileInformation */
		size[0] = 8;
		data[0] = buffer;

		rc = SMB2_set_info_init(tcon, server, &rqst[1], COMPOUND_FID, COMPOUND_FID,
					current->tgid, FILE_END_OF_FILE_INFORMATION,
					SMB2_O_INFO_FILE, 0, data, size);
		free_req1_func = SMB2_set_info_free;
	} else if (qi.flags == PASSTHRU_QUERY_INFO) {
		rqst[1].rq_iov = &vars->qi_iov[0];
		rqst[1].rq_nvec = 1;

		rc = SMB2_query_info_init(tcon, server,
				  &rqst[1], COMPOUND_FID,
				  COMPOUND_FID, qi.file_info_class,
				  qi.info_type, qi.additional_information,
				  qi.input_buffer_length,
				  qi.output_buffer_length, buffer);
		free_req1_func = SMB2_query_info_free;
	} else { /* unknown flags */
		cifs_tcon_dbg(VFS, "Invalid passthru query flags: 0x%x\n",
			      qi.flags);
		rc = -EINVAL;
	}

	if (rc)
		goto free_open_req;
	smb2_set_next_command(tcon, &rqst[1]);
	smb2_set_related(&rqst[1]);

	/* Close */
	rqst[2].rq_iov = &vars->close_iov[0];
	rqst[2].rq_nvec = 1;

	rc = SMB2_close_init(tcon, server,
			     &rqst[2], COMPOUND_FID, COMPOUND_FID, false);
	if (rc)
		goto free_req_1;
	smb2_set_related(&rqst[2]);

	rc = compound_send_recv(xid, ses, server,
				flags, 3, rqst,
				resp_buftype, rsp_iov);
	if (rc)
		goto out;

	/* No need to bump num_remote_opens since handle immediately closed */
	if (qi.flags & PASSTHRU_FSCTL) {
		pqi = (struct smb_query_info __user *)arg;
		io_rsp = (struct smb2_ioctl_rsp *)rsp_iov[1].iov_base;
		if (le32_to_cpu(io_rsp->OutputCount) < qi.input_buffer_length)
			qi.input_buffer_length = le32_to_cpu(io_rsp->OutputCount);
		if (qi.input_buffer_length > 0 &&
		    le32_to_cpu(io_rsp->OutputOffset) + qi.input_buffer_length
		    > rsp_iov[1].iov_len) {
			rc = -EFAULT;
			goto out;
		}

		if (copy_to_user(&pqi->input_buffer_length,
				 &qi.input_buffer_length,
				 sizeof(qi.input_buffer_length))) {
			rc = -EFAULT;
			goto out;
		}

		if (copy_to_user((void __user *)pqi + sizeof(struct smb_query_info),
				 (const void *)io_rsp + le32_to_cpu(io_rsp->OutputOffset),
				 qi.input_buffer_length))
			rc = -EFAULT;
	} else {
		pqi = (struct smb_query_info __user *)arg;
		qi_rsp = (struct smb2_query_info_rsp *)rsp_iov[1].iov_base;
		if (le32_to_cpu(qi_rsp->OutputBufferLength) < qi.input_buffer_length)
			qi.input_buffer_length = le32_to_cpu(qi_rsp->OutputBufferLength);
		if (copy_to_user(&pqi->input_buffer_length,
				 &qi.input_buffer_length,
				 sizeof(qi.input_buffer_length))) {
			rc = -EFAULT;
			goto out;
		}

		if (copy_to_user(pqi + 1, qi_rsp->Buffer,
				 qi.input_buffer_length))
			rc = -EFAULT;
	}

out:
	free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base);
	free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base);
	free_rsp_buf(resp_buftype[2], rsp_iov[2].iov_base);
	SMB2_close_free(&rqst[2]);
free_req_1:
	free_req1_func(&rqst[1]);
free_open_req:
	SMB2_open_free(&rqst[0]);
free_output_buffer:
	kfree(buffer);
free_vars:
	kfree(vars);
	return rc;
}


parse_reparse_posix(struct reparse_posix_data *symlink_buf,
		      u32 plen, char **target_path,
		      struct cifs_sb_info *cifs_sb)
{
	unsigned int len;

	/* See MS-FSCC 2.1.2.6 for the 'NFS' style reparse tags */
	len = le16_to_cpu(symlink_buf->ReparseDataLength);

	if (le64_to_cpu(symlink_buf->InodeType) != NFS_SPECFILE_LNK) {
		cifs_dbg(VFS, "%lld not a supported symlink type\n",
			le64_to_cpu(symlink_buf->InodeType));
		return -EOPNOTSUPP;
	}

	*target_path = cifs_strndup_from_utf16(
				symlink_buf->PathBuffer,
				len, true, cifs_sb->local_nls);
	if (!(*target_path))
		return -ENOMEM;

	convert_delimiter(*target_path, '/');
	cifs_dbg(FYI, "%s: target path: %s\n", __func__, *target_path);

	return 0;
}


ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
	struct ccp_aes_engine *aes = &cmd->u.aes;
	struct ccp_dm_workarea key, ctx, final_wa, tag;
	struct ccp_data src, dst;
	struct ccp_data aad;
	struct ccp_op op;
	unsigned int dm_offset;
	unsigned int authsize;
	unsigned int jobid;
	unsigned int ilen;
	bool in_place = true; /* Default value */
	__be64 *final;
	int ret;

	struct scatterlist *p_inp, sg_inp[2];
	struct scatterlist *p_tag, sg_tag[2];
	struct scatterlist *p_outp, sg_outp[2];
	struct scatterlist *p_aad;

	if (!aes->iv)
		return -EINVAL;

	if (!((aes->key_len == AES_KEYSIZE_128) ||
		(aes->key_len == AES_KEYSIZE_192) ||
		(aes->key_len == AES_KEYSIZE_256)))
		return -EINVAL;

	if (!aes->key) /* Gotta have a key SGL */
		return -EINVAL;

	/* Zero defaults to 16 bytes, the maximum size */
	authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;
	switch (authsize) {
	case 16:
	case 15:
	case 14:
	case 13:
	case 12:
	case 8:
	case 4:
		break;
	default:
		return -EINVAL;
	}

	/* First, decompose the source buffer into AAD & PT,
	 * and the destination buffer into AAD, CT & tag, or
	 * the input into CT & tag.
	 * It is expected that the input and output SGs will
	 * be valid, even if the AAD and input lengths are 0.
	 */
	p_aad = aes->src;
	p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);
	p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);
	if (aes->action == CCP_AES_ACTION_ENCRYPT) {
		ilen = aes->src_len;
		p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);
	} else {
		/* Input length for decryption includes tag */
		ilen = aes->src_len - authsize;
		p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);
	}

	jobid = CCP_NEW_JOBID(cmd_q->ccp);

	memset(&op, 0, sizeof(op));
	op.cmd_q = cmd_q;
	op.jobid = jobid;
	op.sb_key = cmd_q->sb_key; /* Pre-allocated */
	op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
	op.init = 1;
	op.u.aes.type = aes->type;

	/* Copy the key to the LSB */
	ret = ccp_init_dm_workarea(&key, cmd_q,
				   CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
				   DMA_TO_DEVICE);
	if (ret)
		return ret;

	dm_offset = CCP_SB_BYTES - aes->key_len;
	ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
	if (ret)
		goto e_key;
	ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
			     CCP_PASSTHRU_BYTESWAP_256BIT);
	if (ret) {
		cmd->engine_error = cmd_q->cmd_error;
		goto e_key;
	}

	/* Copy the context (IV) to the LSB.
	 * There is an assumption here that the IV is 96 bits in length, plus
	 * a nonce of 32 bits. If no IV is present, use a zeroed buffer.
	 */
	ret = ccp_init_dm_workarea(&ctx, cmd_q,
				   CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
				   DMA_BIDIRECTIONAL);
	if (ret)
		goto e_key;

	dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
	ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
	if (ret)
		goto e_ctx;

	ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
			     CCP_PASSTHRU_BYTESWAP_256BIT);
	if (ret) {
		cmd->engine_error = cmd_q->cmd_error;
		goto e_ctx;
	}

	op.init = 1;
	if (aes->aad_len > 0) {
		/* Step 1: Run a GHASH over the Additional Authenticated Data */
		ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,
				    AES_BLOCK_SIZE,
				    DMA_TO_DEVICE);
		if (ret)
			goto e_ctx;

		op.u.aes.mode = CCP_AES_MODE_GHASH;
		op.u.aes.action = CCP_AES_GHASHAAD;

		while (aad.sg_wa.bytes_left) {
			ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);

			ret = cmd_q->ccp->vdata->perform->aes(&op);
			if (ret) {
				cmd->engine_error = cmd_q->cmd_error;
				goto e_aad;
			}

			ccp_process_data(&aad, NULL, &op);
			op.init = 0;
		}
	}

	op.u.aes.mode = CCP_AES_MODE_GCTR;
	op.u.aes.action = aes->action;

	if (ilen > 0) {
		/* Step 2: Run a GCTR over the plaintext */
		in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;

		ret = ccp_init_data(&src, cmd_q, p_inp, ilen,
				    AES_BLOCK_SIZE,
				    in_place ? DMA_BIDIRECTIONAL
					     : DMA_TO_DEVICE);
		if (ret)
			goto e_aad;

		if (in_place) {
			dst = src;
		} else {
			ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,
					    AES_BLOCK_SIZE, DMA_FROM_DEVICE);
			if (ret)
				goto e_src;
		}

		op.soc = 0;
		op.eom = 0;
		op.init = 1;
		while (src.sg_wa.bytes_left) {
			ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
			if (!src.sg_wa.bytes_left) {
				unsigned int nbytes = ilen % AES_BLOCK_SIZE;

				if (nbytes) {
					op.eom = 1;
					op.u.aes.size = (nbytes * 8) - 1;
				}
			}

			ret = cmd_q->ccp->vdata->perform->aes(&op);
			if (ret) {
				cmd->engine_error = cmd_q->cmd_error;
				goto e_dst;
			}

			ccp_process_data(&src, &dst, &op);
			op.init = 0;
		}
	}

	/* Step 3: Update the IV portion of the context with the original IV */
	ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
			       CCP_PASSTHRU_BYTESWAP_256BIT);
	if (ret) {
		cmd->engine_error = cmd_q->cmd_error;
		goto e_dst;
	}

	ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
	if (ret)
		goto e_dst;

	ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
			     CCP_PASSTHRU_BYTESWAP_256BIT);
	if (ret) {
		cmd->engine_error = cmd_q->cmd_error;
		goto e_dst;
	}

	/* Step 4: Concatenate the lengths of the AAD and source, and
	 * hash that 16 byte buffer.
	 */
	ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,
				   DMA_BIDIRECTIONAL);
	if (ret)
		goto e_dst;
	final = (__be64 *)final_wa.address;
	final[0] = cpu_to_be64(aes->aad_len * 8);
	final[1] = cpu_to_be64(ilen * 8);

	memset(&op, 0, sizeof(op));
	op.cmd_q = cmd_q;
	op.jobid = jobid;
	op.sb_key = cmd_q->sb_key; /* Pre-allocated */
	op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
	op.init = 1;
	op.u.aes.type = aes->type;
	op.u.aes.mode = CCP_AES_MODE_GHASH;
	op.u.aes.action = CCP_AES_GHASHFINAL;
	op.src.type = CCP_MEMTYPE_SYSTEM;
	op.src.u.dma.address = final_wa.dma.address;
	op.src.u.dma.length = AES_BLOCK_SIZE;
	op.dst.type = CCP_MEMTYPE_SYSTEM;
	op.dst.u.dma.address = final_wa.dma.address;
	op.dst.u.dma.length = AES_BLOCK_SIZE;
	op.eom = 1;
	op.u.aes.size = 0;
	ret = cmd_q->ccp->vdata->perform->aes(&op);
	if (ret)
		goto e_final_wa;

	if (aes->action == CCP_AES_ACTION_ENCRYPT) {
		/* Put the ciphered tag after the ciphertext. */
		ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize);
	} else {
		/* Does this ciphered tag match the input? */
		ret = ccp_init_dm_workarea(&tag, cmd_q, authsize,
					   DMA_BIDIRECTIONAL);
		if (ret)
			goto e_final_wa;
		ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize);
		if (ret) {
			ccp_dm_free(&tag);
			goto e_final_wa;
		}

		ret = crypto_memneq(tag.address, final_wa.address,
				    authsize) ? -EBADMSG : 0;
		ccp_dm_free(&tag);
	}

e_final_wa:
	ccp_dm_free(&final_wa);

e_dst:
	if (ilen > 0 && !in_place)
		ccp_free_data(&dst, cmd_q);

e_src:
	if (ilen > 0)
		ccp_free_data(&src, cmd_q);

e_aad:
	if (aes->aad_len)
		ccp_free_data(&aad, cmd_q);

e_ctx:
	ccp_dm_free(&ctx);

e_key:
	ccp_dm_free(&key);

	return ret;
}

crypto: ccp - fix resource leaks in ccp_run_aes_gcm_cmd()

There are three bugs in this code:

1) If we ccp_init_data() fails for &src then we need to free aad.
   Use goto e_aad instead of goto e_ctx.
2) The label to free the &final_wa was named incorrectly as "e_tag" but
   it should have been "e_final_wa".  One error path leaked &final_wa.
3) The &tag was leaked on one error path.  In that case, I added a free
   before the goto because the resource was local to that block.

Fixes: 36cf515b9bbe ("crypto: ccp - Enable support for AES GCM on v5 CCPs")
Reported-by: "minihanshen(沈明航)" <minihanshen@tencent.com>
Signed-off-by: Dan Carpenter <dan.carpenter@oracle.com>
Reviewed-by: John Allen <john.allen@amd.com>
Tested-by: John Allen <john.allen@amd.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>


static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
			    struct scatterlist *sg, unsigned int sg_offset,
			    unsigned int len)
{
	WARN_ON(!wa->address);

	scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
				 1);
}


ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
	struct ccp_ecc_engine *ecc = &cmd->u.ecc;

	ecc->ecc_result = 0;

	if (!ecc->mod ||
	    (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
		return -EINVAL;

	switch (ecc->function) {
	case CCP_ECC_FUNCTION_MMUL_384BIT:
	case CCP_ECC_FUNCTION_MADD_384BIT:
	case CCP_ECC_FUNCTION_MINV_384BIT:
		return ccp_run_ecc_mm_cmd(cmd_q, cmd);

	case CCP_ECC_FUNCTION_PADD_384BIT:
	case CCP_ECC_FUNCTION_PMUL_384BIT:
	case CCP_ECC_FUNCTION_PDBL_384BIT:
		return ccp_run_ecc_pm_cmd(cmd_q, cmd);

	default:
		return -EINVAL;
	}
}


ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
	struct ccp_passthru_engine *pt = &cmd->u.passthru;
	struct ccp_dm_workarea mask;
	struct ccp_data src, dst;
	struct ccp_op op;
	bool in_place = false;
	unsigned int i;
	int ret = 0;

	if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
		return -EINVAL;

	if (!pt->src || !pt->dst)
		return -EINVAL;

	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
		if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
			return -EINVAL;
		if (!pt->mask)
			return -EINVAL;
	}

	BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);

	memset(&op, 0, sizeof(op));
	op.cmd_q = cmd_q;
	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);

	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
		/* Load the mask */
		op.sb_key = cmd_q->sb_key;

		ret = ccp_init_dm_workarea(&mask, cmd_q,
					   CCP_PASSTHRU_SB_COUNT *
					   CCP_SB_BYTES,
					   DMA_TO_DEVICE);
		if (ret)
			return ret;

		ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
		if (ret)
			goto e_mask;
		ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
				     CCP_PASSTHRU_BYTESWAP_NOOP);
		if (ret) {
			cmd->engine_error = cmd_q->cmd_error;
			goto e_mask;
		}
	}

	/* Prepare the input and output data workareas. For in-place
	 * operations we need to set the dma direction to BIDIRECTIONAL
	 * and copy the src workarea to the dst workarea.
	 */
	if (sg_virt(pt->src) == sg_virt(pt->dst))
		in_place = true;

	ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
			    CCP_PASSTHRU_MASKSIZE,
			    in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
	if (ret)
		goto e_mask;

	if (in_place) {
		dst = src;
	} else {
		ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
				    CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
		if (ret)
			goto e_src;
	}

	/* Send data to the CCP Passthru engine
	 *   Because the CCP engine works on a single source and destination
	 *   dma address at a time, each entry in the source scatterlist
	 *   (after the dma_map_sg call) must be less than or equal to the
	 *   (remaining) length in the destination scatterlist entry and the
	 *   length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
	 */
	dst.sg_wa.sg_used = 0;
	for (i = 1; i <= src.sg_wa.dma_count; i++) {
		if (!dst.sg_wa.sg ||
		    (sg_dma_len(dst.sg_wa.sg) < sg_dma_len(src.sg_wa.sg))) {
			ret = -EINVAL;
			goto e_dst;
		}

		if (i == src.sg_wa.dma_count) {
			op.eom = 1;
			op.soc = 1;
		}

		op.src.type = CCP_MEMTYPE_SYSTEM;
		op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
		op.src.u.dma.offset = 0;
		op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);

		op.dst.type = CCP_MEMTYPE_SYSTEM;
		op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
		op.dst.u.dma.offset = dst.sg_wa.sg_used;
		op.dst.u.dma.length = op.src.u.dma.length;

		ret = cmd_q->ccp->vdata->perform->passthru(&op);
		if (ret) {
			cmd->engine_error = cmd_q->cmd_error;
			goto e_dst;
		}

		dst.sg_wa.sg_used += sg_dma_len(src.sg_wa.sg);
		if (dst.sg_wa.sg_used == sg_dma_len(dst.sg_wa.sg)) {
			dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
			dst.sg_wa.sg_used = 0;
		}
		src.sg_wa.sg = sg_next(src.sg_wa.sg);
	}

e_dst:
	if (!in_place)
		ccp_free_data(&dst, cmd_q);

e_src:
	ccp_free_data(&src, cmd_q);

e_mask:
	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
		ccp_dm_free(&mask);

	return ret;
}


static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
{
	return ccp_queue_buf(data, 1);
}


static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
				   unsigned int wa_offset,
				   struct scatterlist *sg,
				   unsigned int sg_offset,
				   unsigned int len)
{
	u8 *p, *q;
	int	rc;

	rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
	if (rc)
		return rc;

	p = wa->address + wa_offset;
	q = p + len - 1;
	while (p < q) {
		*p = *p ^ *q;
		*q = *p ^ *q;
		*p = *p ^ *q;
		p++;
		q--;
	}
	return 0;
}


static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
	struct ccp_ecc_engine *ecc = &cmd->u.ecc;
	struct ccp_dm_workarea src, dst;
	struct ccp_op op;
	int ret;
	u8 *save;

	if (!ecc->u.mm.operand_1 ||
	    (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
		return -EINVAL;

	if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
		if (!ecc->u.mm.operand_2 ||
		    (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
			return -EINVAL;

	if (!ecc->u.mm.result ||
	    (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
		return -EINVAL;

	memset(&op, 0, sizeof(op));
	op.cmd_q = cmd_q;
	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);

	/* Concatenate the modulus and the operands. Both the modulus and
	 * the operands must be in little endian format.  Since the input
	 * is in big endian format it must be converted and placed in a
	 * fixed length buffer.
	 */
	ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
				   DMA_TO_DEVICE);
	if (ret)
		return ret;

	/* Save the workarea address since it is updated in order to perform
	 * the concatenation
	 */
	save = src.address;

	/* Copy the ECC modulus */
	ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
	if (ret)
		goto e_src;
	src.address += CCP_ECC_OPERAND_SIZE;

	/* Copy the first operand */
	ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,
				      ecc->u.mm.operand_1_len);
	if (ret)
		goto e_src;
	src.address += CCP_ECC_OPERAND_SIZE;

	if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
		/* Copy the second operand */
		ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,
					      ecc->u.mm.operand_2_len);
		if (ret)
			goto e_src;
		src.address += CCP_ECC_OPERAND_SIZE;
	}

	/* Restore the workarea address */
	src.address = save;

	/* Prepare the output area for the operation */
	ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
				   DMA_FROM_DEVICE);
	if (ret)
		goto e_src;

	op.soc = 1;
	op.src.u.dma.address = src.dma.address;
	op.src.u.dma.offset = 0;
	op.src.u.dma.length = src.length;
	op.dst.u.dma.address = dst.dma.address;
	op.dst.u.dma.offset = 0;
	op.dst.u.dma.length = dst.length;

	op.u.ecc.function = cmd->u.ecc.function;

	ret = cmd_q->ccp->vdata->perform->ecc(&op);
	if (ret) {
		cmd->engine_error = cmd_q->cmd_error;
		goto e_dst;
	}

	ecc->ecc_result = le16_to_cpup(
		(const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
	if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
		ret = -EIO;
		goto e_dst;
	}

	/* Save the ECC result */
	ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,
				CCP_ECC_MODULUS_BYTES);

e_dst:
	ccp_dm_free(&dst);

e_src:
	ccp_dm_free(&src);

	return ret;
}


ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
				      struct ccp_cmd *cmd)
{
	struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
	struct ccp_dm_workarea mask;
	struct ccp_op op;
	int ret;

	if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
		return -EINVAL;

	if (!pt->src_dma || !pt->dst_dma)
		return -EINVAL;

	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
		if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
			return -EINVAL;
		if (!pt->mask)
			return -EINVAL;
	}

	BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);

	memset(&op, 0, sizeof(op));
	op.cmd_q = cmd_q;
	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);

	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
		/* Load the mask */
		op.sb_key = cmd_q->sb_key;

		mask.length = pt->mask_len;
		mask.dma.address = pt->mask;
		mask.dma.length = pt->mask_len;

		ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
				     CCP_PASSTHRU_BYTESWAP_NOOP);
		if (ret) {
			cmd->engine_error = cmd_q->cmd_error;
			return ret;
		}
	}

	/* Send data to the CCP Passthru engine */
	op.eom = 1;
	op.soc = 1;

	op.src.type = CCP_MEMTYPE_SYSTEM;
	op.src.u.dma.address = pt->src_dma;
	op.src.u.dma.offset = 0;
	op.src.u.dma.length = pt->src_len;

	op.dst.type = CCP_MEMTYPE_SYSTEM;
	op.dst.u.dma.address = pt->dst_dma;
	op.dst.u.dma.offset = 0;
	op.dst.u.dma.length = pt->src_len;

	ret = cmd_q->ccp->vdata->perform->passthru(&op);
	if (ret)
		cmd->engine_error = cmd_q->cmd_error;

	return ret;
}


static u32 ccp_gen_jobid(struct ccp_device *ccp)
{
	return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
}


ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
	struct ccp_aes_engine *aes = &cmd->u.aes;
	struct ccp_dm_workarea key, ctx;
	struct ccp_data src, dst;
	struct ccp_op op;
	unsigned int dm_offset;
	bool in_place = false;
	int ret;

	if (!((aes->key_len == AES_KEYSIZE_128) ||
	      (aes->key_len == AES_KEYSIZE_192) ||
	      (aes->key_len == AES_KEYSIZE_256)))
		return -EINVAL;

	if (((aes->mode == CCP_AES_MODE_ECB) ||
	     (aes->mode == CCP_AES_MODE_CBC)) &&
	    (aes->src_len & (AES_BLOCK_SIZE - 1)))
		return -EINVAL;

	if (!aes->key || !aes->src || !aes->dst)
		return -EINVAL;

	if (aes->mode != CCP_AES_MODE_ECB) {
		if (aes->iv_len != AES_BLOCK_SIZE)
			return -EINVAL;

		if (!aes->iv)
			return -EINVAL;
	}

	BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
	BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);

	ret = -EIO;
	memset(&op, 0, sizeof(op));
	op.cmd_q = cmd_q;
	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
	op.sb_key = cmd_q->sb_key;
	op.sb_ctx = cmd_q->sb_ctx;
	op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
	op.u.aes.type = aes->type;
	op.u.aes.mode = aes->mode;
	op.u.aes.action = aes->action;

	/* All supported key sizes fit in a single (32-byte) SB entry
	 * and must be in little endian format. Use the 256-bit byte
	 * swap passthru option to convert from big endian to little
	 * endian.
	 */
	ret = ccp_init_dm_workarea(&key, cmd_q,
				   CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
				   DMA_TO_DEVICE);
	if (ret)
		return ret;

	dm_offset = CCP_SB_BYTES - aes->key_len;
	ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
	if (ret)
		goto e_key;
	ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
			     CCP_PASSTHRU_BYTESWAP_256BIT);
	if (ret) {
		cmd->engine_error = cmd_q->cmd_error;
		goto e_key;
	}

	/* The AES context fits in a single (32-byte) SB entry and
	 * must be in little endian format. Use the 256-bit byte swap
	 * passthru option to convert from big endian to little endian.
	 */
	ret = ccp_init_dm_workarea(&ctx, cmd_q,
				   CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
				   DMA_BIDIRECTIONAL);
	if (ret)
		goto e_key;

	if (aes->mode != CCP_AES_MODE_ECB) {
		/* Load the AES context - convert to LE */
		dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
		ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
		if (ret)
			goto e_ctx;
		ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
				     CCP_PASSTHRU_BYTESWAP_256BIT);
		if (ret) {
			cmd->engine_error = cmd_q->cmd_error;
			goto e_ctx;
		}
	}
	switch (aes->mode) {
	case CCP_AES_MODE_CFB: /* CFB128 only */
	case CCP_AES_MODE_CTR:
		op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
		break;
	default:
		op.u.aes.size = 0;
	}

	/* Prepare the input and output data workareas. For in-place
	 * operations we need to set the dma direction to BIDIRECTIONAL
	 * and copy the src workarea to the dst workarea.
	 */
	if (sg_virt(aes->src) == sg_virt(aes->dst))
		in_place = true;

	ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
			    AES_BLOCK_SIZE,
			    in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
	if (ret)
		goto e_ctx;

	if (in_place) {
		dst = src;
	} else {
		ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
				    AES_BLOCK_SIZE, DMA_FROM_DEVICE);
		if (ret)
			goto e_src;
	}

	/* Send data to the CCP AES engine */
	while (src.sg_wa.bytes_left) {
		ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
		if (!src.sg_wa.bytes_left) {
			op.eom = 1;

			/* Since we don't retrieve the AES context in ECB
			 * mode we have to wait for the operation to complete
			 * on the last piece of data
			 */
			if (aes->mode == CCP_AES_MODE_ECB)
				op.soc = 1;
		}

		ret = cmd_q->ccp->vdata->perform->aes(&op);
		if (ret) {
			cmd->engine_error = cmd_q->cmd_error;
			goto e_dst;
		}

		ccp_process_data(&src, &dst, &op);
	}

	if (aes->mode != CCP_AES_MODE_ECB) {
		/* Retrieve the AES context - convert from LE to BE using
		 * 32-byte (256-bit) byteswapping
		 */
		ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
				       CCP_PASSTHRU_BYTESWAP_256BIT);
		if (ret) {
			cmd->engine_error = cmd_q->cmd_error;
			goto e_dst;
		}

		/* ...but we only need AES_BLOCK_SIZE bytes */
		dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
		ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
	}

e_dst:
	if (!in_place)
		ccp_free_data(&dst, cmd_q);

e_src:
	ccp_free_data(&src, cmd_q);

e_ctx:
	ccp_dm_free(&ctx);

e_key:
	ccp_dm_free(&key);

	return ret;
}


static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
				struct scatterlist *sg, u64 len,
				enum dma_data_direction dma_dir)
{
	memset(wa, 0, sizeof(*wa));

	wa->sg = sg;
	if (!sg)
		return 0;

	wa->nents = sg_nents_for_len(sg, len);
	if (wa->nents < 0)
		return wa->nents;

	wa->bytes_left = len;
	wa->sg_used = 0;

	if (len == 0)
		return 0;

	if (dma_dir == DMA_NONE)
		return 0;

	wa->dma_sg = sg;
	wa->dma_sg_head = sg;
	wa->dma_dev = dev;
	wa->dma_dir = dma_dir;
	wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
	if (!wa->dma_count)
		return -ENOMEM;

	return 0;
}


static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
			       struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
			       u32 byte_swap, bool from)
{
	struct ccp_op op;

	memset(&op, 0, sizeof(op));

	op.cmd_q = cmd_q;
	op.jobid = jobid;
	op.eom = 1;

	if (from) {
		op.soc = 1;
		op.src.type = CCP_MEMTYPE_SB;
		op.src.u.sb = sb;
		op.dst.type = CCP_MEMTYPE_SYSTEM;
		op.dst.u.dma.address = wa->dma.address;
		op.dst.u.dma.length = wa->length;
	} else {
		op.src.type = CCP_MEMTYPE_SYSTEM;
		op.src.u.dma.address = wa->dma.address;
		op.src.u.dma.length = wa->length;
		op.dst.type = CCP_MEMTYPE_SB;
		op.dst.u.sb = sb;
	}

	op.u.passthru.byte_swap = byte_swap;

	return cmd_q->ccp->vdata->perform->passthru(&op);
}


static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
{
	unsigned int nbytes = min_t(u64, len, wa->bytes_left);
	unsigned int sg_combined_len = 0;

	if (!wa->sg)
		return;

	wa->sg_used += nbytes;
	wa->bytes_left -= nbytes;
	if (wa->sg_used == sg_dma_len(wa->dma_sg)) {
		/* Advance to the next DMA scatterlist entry */
		wa->dma_sg = sg_next(wa->dma_sg);

		/* In the case that the DMA mapped scatterlist has entries
		 * that have been merged, the non-DMA mapped scatterlist
		 * must be advanced multiple times for each merged entry.
		 * This ensures that the current non-DMA mapped entry
		 * corresponds to the current DMA mapped entry.
		 */
		do {
			sg_combined_len += wa->sg->length;
			wa->sg = sg_next(wa->sg);
		} while (wa->sg_used > sg_combined_len);

		wa->sg_used = 0;
	}
}


static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
{
	ccp_dm_free(&data->dm_wa);
	ccp_sg_free(&data->sg_wa);
}


ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
	struct ccp_xts_aes_engine *xts = &cmd->u.xts;
	struct ccp_dm_workarea key, ctx;
	struct ccp_data src, dst;
	struct ccp_op op;
	unsigned int unit_size, dm_offset;
	bool in_place = false;
	unsigned int sb_count;
	enum ccp_aes_type aestype;
	int ret;

	switch (xts->unit_size) {
	case CCP_XTS_AES_UNIT_SIZE_16:
		unit_size = 16;
		break;
	case CCP_XTS_AES_UNIT_SIZE_512:
		unit_size = 512;
		break;
	case CCP_XTS_AES_UNIT_SIZE_1024:
		unit_size = 1024;
		break;
	case CCP_XTS_AES_UNIT_SIZE_2048:
		unit_size = 2048;
		break;
	case CCP_XTS_AES_UNIT_SIZE_4096:
		unit_size = 4096;
		break;

	default:
		return -EINVAL;
	}

	if (xts->key_len == AES_KEYSIZE_128)
		aestype = CCP_AES_TYPE_128;
	else if (xts->key_len == AES_KEYSIZE_256)
		aestype = CCP_AES_TYPE_256;
	else
		return -EINVAL;

	if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
		return -EINVAL;

	if (xts->iv_len != AES_BLOCK_SIZE)
		return -EINVAL;

	if (!xts->key || !xts->iv || !xts->src || !xts->dst)
		return -EINVAL;

	BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
	BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);

	ret = -EIO;
	memset(&op, 0, sizeof(op));
	op.cmd_q = cmd_q;
	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
	op.sb_key = cmd_q->sb_key;
	op.sb_ctx = cmd_q->sb_ctx;
	op.init = 1;
	op.u.xts.type = aestype;
	op.u.xts.action = xts->action;
	op.u.xts.unit_size = xts->unit_size;

	/* A version 3 device only supports 128-bit keys, which fits into a
	 * single SB entry. A version 5 device uses a 512-bit vector, so two
	 * SB entries.
	 */
	if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
		sb_count = CCP_XTS_AES_KEY_SB_COUNT;
	else
		sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
	ret = ccp_init_dm_workarea(&key, cmd_q,
				   sb_count * CCP_SB_BYTES,
				   DMA_TO_DEVICE);
	if (ret)
		return ret;

	if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
		/* All supported key sizes must be in little endian format.
		 * Use the 256-bit byte swap passthru option to convert from
		 * big endian to little endian.
		 */
		dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
		ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
		if (ret)
			goto e_key;
		ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
		if (ret)
			goto e_key;
	} else {
		/* Version 5 CCPs use a 512-bit space for the key: each portion
		 * occupies 256 bits, or one entire slot, and is zero-padded.
		 */
		unsigned int pad;

		dm_offset = CCP_SB_BYTES;
		pad = dm_offset - xts->key_len;
		ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
		if (ret)
			goto e_key;
		ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key,
				      xts->key_len, xts->key_len);
		if (ret)
			goto e_key;
	}
	ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
			     CCP_PASSTHRU_BYTESWAP_256BIT);
	if (ret) {
		cmd->engine_error = cmd_q->cmd_error;
		goto e_key;
	}

	/* The AES context fits in a single (32-byte) SB entry and
	 * for XTS is already in little endian format so no byte swapping
	 * is needed.
	 */
	ret = ccp_init_dm_workarea(&ctx, cmd_q,
				   CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
				   DMA_BIDIRECTIONAL);
	if (ret)
		goto e_key;

	ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
	if (ret)
		goto e_ctx;
	ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
			     CCP_PASSTHRU_BYTESWAP_NOOP);
	if (ret) {
		cmd->engine_error = cmd_q->cmd_error;
		goto e_ctx;
	}

	/* Prepare the input and output data workareas. For in-place
	 * operations we need to set the dma direction to BIDIRECTIONAL
	 * and copy the src workarea to the dst workarea.
	 */
	if (sg_virt(xts->src) == sg_virt(xts->dst))
		in_place = true;

	ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
			    unit_size,
			    in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
	if (ret)
		goto e_ctx;

	if (in_place) {
		dst = src;
	} else {
		ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
				    unit_size, DMA_FROM_DEVICE);
		if (ret)
			goto e_src;
	}

	/* Send data to the CCP AES engine */
	while (src.sg_wa.bytes_left) {
		ccp_prepare_data(&src, &dst, &op, unit_size, true);
		if (!src.sg_wa.bytes_left)
			op.eom = 1;

		ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
		if (ret) {
			cmd->engine_error = cmd_q->cmd_error;
			goto e_dst;
		}

		ccp_process_data(&src, &dst, &op);
	}

	/* Retrieve the AES context - convert from LE to BE using
	 * 32-byte (256-bit) byteswapping
	 */
	ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
			       CCP_PASSTHRU_BYTESWAP_256BIT);
	if (ret) {
		cmd->engine_error = cmd_q->cmd_error;
		goto e_dst;
	}

	/* ...but we only need AES_BLOCK_SIZE bytes */
	dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
	ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);

e_dst:
	if (!in_place)
		ccp_free_data(&dst, cmd_q);

e_src:
	ccp_free_data(&src, cmd_q);

e_ctx:
	ccp_dm_free(&ctx);

e_key:
	ccp_dm_free(&key);

	return ret;
}


static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
				    unsigned int wa_offset,
				    struct scatterlist *sg,
				    unsigned int sg_offset,
				    unsigned int len)
{
	u8 *p, *q;

	p = wa->address + wa_offset;
	q = p + len - 1;
	while (p < q) {
		*p = *p ^ *q;
		*q = *p ^ *q;
		*p = *p ^ *q;
		p++;
		q--;
	}

	ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
}


ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
	struct ccp_sha_engine *sha = &cmd->u.sha;
	struct ccp_dm_workarea ctx;
	struct ccp_data src;
	struct ccp_op op;
	unsigned int ioffset, ooffset;
	unsigned int digest_size;
	int sb_count;
	const void *init;
	u64 block_size;
	int ctx_size;
	int ret;

	switch (sha->type) {
	case CCP_SHA_TYPE_1:
		if (sha->ctx_len < SHA1_DIGEST_SIZE)
			return -EINVAL;
		block_size = SHA1_BLOCK_SIZE;
		break;
	case CCP_SHA_TYPE_224:
		if (sha->ctx_len < SHA224_DIGEST_SIZE)
			return -EINVAL;
		block_size = SHA224_BLOCK_SIZE;
		break;
	case CCP_SHA_TYPE_256:
		if (sha->ctx_len < SHA256_DIGEST_SIZE)
			return -EINVAL;
		block_size = SHA256_BLOCK_SIZE;
		break;
	case CCP_SHA_TYPE_384:
		if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
		    || sha->ctx_len < SHA384_DIGEST_SIZE)
			return -EINVAL;
		block_size = SHA384_BLOCK_SIZE;
		break;
	case CCP_SHA_TYPE_512:
		if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
		    || sha->ctx_len < SHA512_DIGEST_SIZE)
			return -EINVAL;
		block_size = SHA512_BLOCK_SIZE;
		break;
	default:
		return -EINVAL;
	}

	if (!sha->ctx)
		return -EINVAL;

	if (!sha->final && (sha->src_len & (block_size - 1)))
		return -EINVAL;

	/* The version 3 device can't handle zero-length input */
	if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {

		if (!sha->src_len) {
			unsigned int digest_len;
			const u8 *sha_zero;

			/* Not final, just return */
			if (!sha->final)
				return 0;

			/* CCP can't do a zero length sha operation so the
			 * caller must buffer the data.
			 */
			if (sha->msg_bits)
				return -EINVAL;

			/* The CCP cannot perform zero-length sha operations
			 * so the caller is required to buffer data for the
			 * final operation. However, a sha operation for a
			 * message with a total length of zero is valid so
			 * known values are required to supply the result.
			 */
			switch (sha->type) {
			case CCP_SHA_TYPE_1:
				sha_zero = sha1_zero_message_hash;
				digest_len = SHA1_DIGEST_SIZE;
				break;
			case CCP_SHA_TYPE_224:
				sha_zero = sha224_zero_message_hash;
				digest_len = SHA224_DIGEST_SIZE;
				break;
			case CCP_SHA_TYPE_256:
				sha_zero = sha256_zero_message_hash;
				digest_len = SHA256_DIGEST_SIZE;
				break;
			default:
				return -EINVAL;
			}

			scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
						 digest_len, 1);

			return 0;
		}
	}

	/* Set variables used throughout */
	switch (sha->type) {
	case CCP_SHA_TYPE_1:
		digest_size = SHA1_DIGEST_SIZE;
		init = (void *) ccp_sha1_init;
		ctx_size = SHA1_DIGEST_SIZE;
		sb_count = 1;
		if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
			ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
		else
			ooffset = ioffset = 0;
		break;
	case CCP_SHA_TYPE_224:
		digest_size = SHA224_DIGEST_SIZE;
		init = (void *) ccp_sha224_init;
		ctx_size = SHA256_DIGEST_SIZE;
		sb_count = 1;
		ioffset = 0;
		if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
			ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
		else
			ooffset = 0;
		break;
	case CCP_SHA_TYPE_256:
		digest_size = SHA256_DIGEST_SIZE;
		init = (void *) ccp_sha256_init;
		ctx_size = SHA256_DIGEST_SIZE;
		sb_count = 1;
		ooffset = ioffset = 0;
		break;
	case CCP_SHA_TYPE_384:
		digest_size = SHA384_DIGEST_SIZE;
		init = (void *) ccp_sha384_init;
		ctx_size = SHA512_DIGEST_SIZE;
		sb_count = 2;
		ioffset = 0;
		ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
		break;
	case CCP_SHA_TYPE_512:
		digest_size = SHA512_DIGEST_SIZE;
		init = (void *) ccp_sha512_init;
		ctx_size = SHA512_DIGEST_SIZE;
		sb_count = 2;
		ooffset = ioffset = 0;
		break;
	default:
		ret = -EINVAL;
		goto e_data;
	}

	/* For zero-length plaintext the src pointer is ignored;
	 * otherwise both parts must be valid
	 */
	if (sha->src_len && !sha->src)
		return -EINVAL;

	memset(&op, 0, sizeof(op));
	op.cmd_q = cmd_q;
	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
	op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
	op.u.sha.type = sha->type;
	op.u.sha.msg_bits = sha->msg_bits;

	/* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
	 * SHA384/512 require 2 adjacent SB slots, with the right half in the
	 * first slot, and the left half in the second. Each portion must then
	 * be in little endian format: use the 256-bit byte swap option.
	 */
	ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
				   DMA_BIDIRECTIONAL);
	if (ret)
		return ret;
	if (sha->first) {
		switch (sha->type) {
		case CCP_SHA_TYPE_1:
		case CCP_SHA_TYPE_224:
		case CCP_SHA_TYPE_256:
			memcpy(ctx.address + ioffset, init, ctx_size);
			break;
		case CCP_SHA_TYPE_384:
		case CCP_SHA_TYPE_512:
			memcpy(ctx.address + ctx_size / 2, init,
			       ctx_size / 2);
			memcpy(ctx.address, init + ctx_size / 2,
			       ctx_size / 2);
			break;
		default:
			ret = -EINVAL;
			goto e_ctx;
		}
	} else {
		/* Restore the context */
		ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
				      sb_count * CCP_SB_BYTES);
		if (ret)
			goto e_ctx;
	}

	ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
			     CCP_PASSTHRU_BYTESWAP_256BIT);
	if (ret) {
		cmd->engine_error = cmd_q->cmd_error;
		goto e_ctx;
	}

	if (sha->src) {
		/* Send data to the CCP SHA engine; block_size is set above */
		ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
				    block_size, DMA_TO_DEVICE);
		if (ret)
			goto e_ctx;

		while (src.sg_wa.bytes_left) {
			ccp_prepare_data(&src, NULL, &op, block_size, false);
			if (sha->final && !src.sg_wa.bytes_left)
				op.eom = 1;

			ret = cmd_q->ccp->vdata->perform->sha(&op);
			if (ret) {
				cmd->engine_error = cmd_q->cmd_error;
				goto e_data;
			}

			ccp_process_data(&src, NULL, &op);
		}
	} else {
		op.eom = 1;
		ret = cmd_q->ccp->vdata->perform->sha(&op);
		if (ret) {
			cmd->engine_error = cmd_q->cmd_error;
			goto e_data;
		}
	}

	/* Retrieve the SHA context - convert from LE to BE using
	 * 32-byte (256-bit) byteswapping to BE
	 */
	ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
			       CCP_PASSTHRU_BYTESWAP_256BIT);
	if (ret) {
		cmd->engine_error = cmd_q->cmd_error;
		goto e_data;
	}

	if (sha->final) {
		/* Finishing up, so get the digest */
		switch (sha->type) {
		case CCP_SHA_TYPE_1:
		case CCP_SHA_TYPE_224:
		case CCP_SHA_TYPE_256:
			ccp_get_dm_area(&ctx, ooffset,
					sha->ctx, 0,
					digest_size);
			break;
		case CCP_SHA_TYPE_384:
		case CCP_SHA_TYPE_512:
			ccp_get_dm_area(&ctx, 0,
					sha->ctx, LSB_ITEM_SIZE - ooffset,
					LSB_ITEM_SIZE);
			ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset,
					sha->ctx, 0,
					LSB_ITEM_SIZE - ooffset);
			break;
		default:
			ret = -EINVAL;
			goto e_data;
		}
	} else {
		/* Stash the context */
		ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
				sb_count * CCP_SB_BYTES);
	}

	if (sha->final && sha->opad) {
		/* HMAC operation, recursively perform final SHA */
		struct ccp_cmd hmac_cmd;
		struct scatterlist sg;
		u8 *hmac_buf;

		if (sha->opad_len != block_size) {
			ret = -EINVAL;
			goto e_data;
		}

		hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
		if (!hmac_buf) {
			ret = -ENOMEM;
			goto e_data;
		}
		sg_init_one(&sg, hmac_buf, block_size + digest_size);

		scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
		switch (sha->type) {
		case CCP_SHA_TYPE_1:
		case CCP_SHA_TYPE_224:
		case CCP_SHA_TYPE_256:
			memcpy(hmac_buf + block_size,
			       ctx.address + ooffset,
			       digest_size);
			break;
		case CCP_SHA_TYPE_384:
		case CCP_SHA_TYPE_512:
			memcpy(hmac_buf + block_size,
			       ctx.address + LSB_ITEM_SIZE + ooffset,
			       LSB_ITEM_SIZE);
			memcpy(hmac_buf + block_size +
			       (LSB_ITEM_SIZE - ooffset),
			       ctx.address,
			       LSB_ITEM_SIZE);
			break;
		default:
			kfree(hmac_buf);
			ret = -EINVAL;
			goto e_data;
		}

		memset(&hmac_cmd, 0, sizeof(hmac_cmd));
		hmac_cmd.engine = CCP_ENGINE_SHA;
		hmac_cmd.u.sha.type = sha->type;
		hmac_cmd.u.sha.ctx = sha->ctx;
		hmac_cmd.u.sha.ctx_len = sha->ctx_len;
		hmac_cmd.u.sha.src = &sg;
		hmac_cmd.u.sha.src_len = block_size + digest_size;
		hmac_cmd.u.sha.opad = NULL;
		hmac_cmd.u.sha.opad_len = 0;
		hmac_cmd.u.sha.first = 1;
		hmac_cmd.u.sha.final = 1;
		hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;

		ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
		if (ret)
			cmd->engine_error = hmac_cmd.engine_error;

		kfree(hmac_buf);
	}

e_data:
	if (sha->src)
		ccp_free_data(&src, cmd_q);

e_ctx:
	ccp_dm_free(&ctx);

	return ret;
}


static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
			  struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
			  u32 byte_swap)
{
	return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
}


static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
{
	return ccp_queue_buf(data, 0);
}


ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
	struct ccp_aes_engine *aes = &cmd->u.aes;
	struct ccp_dm_workarea key, ctx;
	struct ccp_data src;
	struct ccp_op op;
	unsigned int dm_offset;
	int ret;

	if (!((aes->key_len == AES_KEYSIZE_128) ||
	      (aes->key_len == AES_KEYSIZE_192) ||
	      (aes->key_len == AES_KEYSIZE_256)))
		return -EINVAL;

	if (aes->src_len & (AES_BLOCK_SIZE - 1))
		return -EINVAL;

	if (aes->iv_len != AES_BLOCK_SIZE)
		return -EINVAL;

	if (!aes->key || !aes->iv || !aes->src)
		return -EINVAL;

	if (aes->cmac_final) {
		if (aes->cmac_key_len != AES_BLOCK_SIZE)
			return -EINVAL;

		if (!aes->cmac_key)
			return -EINVAL;
	}

	BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
	BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);

	ret = -EIO;
	memset(&op, 0, sizeof(op));
	op.cmd_q = cmd_q;
	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
	op.sb_key = cmd_q->sb_key;
	op.sb_ctx = cmd_q->sb_ctx;
	op.init = 1;
	op.u.aes.type = aes->type;
	op.u.aes.mode = aes->mode;
	op.u.aes.action = aes->action;

	/* All supported key sizes fit in a single (32-byte) SB entry
	 * and must be in little endian format. Use the 256-bit byte
	 * swap passthru option to convert from big endian to little
	 * endian.
	 */
	ret = ccp_init_dm_workarea(&key, cmd_q,
				   CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
				   DMA_TO_DEVICE);
	if (ret)
		return ret;

	dm_offset = CCP_SB_BYTES - aes->key_len;
	ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
	if (ret)
		goto e_key;
	ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
			     CCP_PASSTHRU_BYTESWAP_256BIT);
	if (ret) {
		cmd->engine_error = cmd_q->cmd_error;
		goto e_key;
	}

	/* The AES context fits in a single (32-byte) SB entry and
	 * must be in little endian format. Use the 256-bit byte swap
	 * passthru option to convert from big endian to little endian.
	 */
	ret = ccp_init_dm_workarea(&ctx, cmd_q,
				   CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
				   DMA_BIDIRECTIONAL);
	if (ret)
		goto e_key;

	dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
	ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
	if (ret)
		goto e_ctx;
	ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
			     CCP_PASSTHRU_BYTESWAP_256BIT);
	if (ret) {
		cmd->engine_error = cmd_q->cmd_error;
		goto e_ctx;
	}

	/* Send data to the CCP AES engine */
	ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
			    AES_BLOCK_SIZE, DMA_TO_DEVICE);
	if (ret)
		goto e_ctx;

	while (src.sg_wa.bytes_left) {
		ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
		if (aes->cmac_final && !src.sg_wa.bytes_left) {
			op.eom = 1;

			/* Push the K1/K2 key to the CCP now */
			ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
					       op.sb_ctx,
					       CCP_PASSTHRU_BYTESWAP_256BIT);
			if (ret) {
				cmd->engine_error = cmd_q->cmd_error;
				goto e_src;
			}

			ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
					      aes->cmac_key_len);
			if (ret)
				goto e_src;
			ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
					     CCP_PASSTHRU_BYTESWAP_256BIT);
			if (ret) {
				cmd->engine_error = cmd_q->cmd_error;
				goto e_src;
			}
		}

		ret = cmd_q->ccp->vdata->perform->aes(&op);
		if (ret) {
			cmd->engine_error = cmd_q->cmd_error;
			goto e_src;
		}

		ccp_process_data(&src, NULL, &op);
	}

	/* Retrieve the AES context - convert from LE to BE using
	 * 32-byte (256-bit) byteswapping
	 */
	ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
			       CCP_PASSTHRU_BYTESWAP_256BIT);
	if (ret) {
		cmd->engine_error = cmd_q->cmd_error;
		goto e_src;
	}

	/* ...but we only need AES_BLOCK_SIZE bytes */
	dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
	ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);

e_src:
	ccp_free_data(&src, cmd_q);

e_ctx:
	ccp_dm_free(&ctx);

e_key:
	ccp_dm_free(&key);

	return ret;
}