FateWithShellmiaoComment/Fate/python/fate_client/flow_client/flow_cli/commands/data.py

#
#  Copyright 2019 The FATE Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
import click
import os
import sys
from flow_client.flow_cli.utils import cli_args
from flow_client.flow_cli.utils.cli_utils import preprocess, access_server, check_abs_path, prettify
from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
import json


@click.group(short_help="Data Operations")
@click.pass_context
def data(ctx):
    """
    \b
    Provides numbers of data operational commands, including upload, download and etc.
    For more details, please check out the help text.
    """
    pass


@data.command("upload", short_help="Upload Table Command")
@cli_args.CONF_PATH
@click.option('--verbose', is_flag=True, default=False,
              help="If specified, verbose mode will be turn on. "
                   "Users can have feedback on upload task in progress. (default: False)")
@click.option('--drop', is_flag=True, default=False,
              help="If specified, data of old version would be replaced by the current version. "
                   "Otherwise, current upload task would be rejected. (default: False)")
@click.pass_context
def upload(ctx, **kwargs):
    """
    \b
    - DESCRIPTION:
        Upload Data Table.

    \b
    - Usage:
        flow data upload -c fateflow/examples/upload/upload_guest.json
        flow data upload -c fateflow/examples/upload/upload_host.json --verbose --drop
    """
    kwargs['drop'] = 1 if kwargs['drop'] else 0
    kwargs['verbose'] = int(kwargs['verbose'])
    config_data, dsl_data = preprocess(**kwargs)
    if config_data.get('use_local_data', 1):
        file_name = check_abs_path(config_data.get('file'))
        if os.path.exists(file_name):
            with open(file_name, 'rb') as fp:
                data = MultipartEncoder(
                    fields={'file': (os.path.basename(file_name), fp, 'application/octet-stream')}
                )
                tag = [0]

                def read_callback(monitor):
                    if config_data.get('verbose') == 1:
                        sys.stdout.write("\r UPLOADING:{0}{1}".format(
                            "|" * (monitor.bytes_read * 100 // monitor.len), '%.2f%%' % (monitor.bytes_read * 100 // monitor.len)))
                        sys.stdout.flush()
                        if monitor.bytes_read / monitor.len == 1:
                            tag[0] += 1
                            if tag[0] == 2:
                                sys.stdout.write('\n')

                data = MultipartEncoderMonitor(data, read_callback)
                access_server('post', ctx, 'data/upload', json_data=None, data=data,
                              params=json.dumps(config_data), headers={'Content-Type': data.content_type})
        else:
            prettify(
                {
                    "retcode": 100,
                    "retmsg": "The file is obtained from the fate flow client machine, but it does not exist, "
                              "please check the path: {}".format(file_name)
                }
            )
    else:
        access_server('post', ctx, 'data/upload', config_data)


@data.command("download", short_help="Download Table Command")
@cli_args.CONF_PATH
@click.pass_context
def download(ctx, **kwargs):
    """
    \b
    - DESCRIPTION:
        Download Data Table.

    \b
    - Usage:
        flow data download -c fateflow/examples/download/download_table.json
    """
    config_data, dsl_data = preprocess(**kwargs)
    access_server('post', ctx, "data/download", config_data)


@data.command("writer", short_help="write Table Command")
@cli_args.CONF_PATH
@click.pass_context
def writer(ctx, **kwargs):
    """
    \b
    - DESCRIPTION:
        Download Data Table.

    \b
    - Usage:
        flow data download -c fateflow/examples/writer/external_storage.json
    """
    config_data, dsl_data = preprocess(**kwargs)
    access_server('post', ctx, "data/writer", config_data)


@data.command("upload-history", short_help="Upload History Command")
@cli_args.LIMIT
@cli_args.JOBID
@click.pass_context
def upload_history(ctx, **kwargs):
    """
    \b
    - DESCRIPTION:
        Query Upload Table History.

    \b
    - USAGE:
        flow data upload-history -l 20
        flow data upload-history --job-id $JOB_ID
    """
    config_data, dsl_data = preprocess(**kwargs)
    access_server('post', ctx, "data/upload/history", config_data)


# @data.command(short_help="")
@click.pass_context
def download_history(ctx):
    """

    """
    pass