FateWithShellmiaoComment/Fate/python/fate_client/flow_client/flow_cli/commands/component.py

#
#  Copyright 2019 The FATE Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
import os
import json
from datetime import datetime

import click
from contextlib import closing

from flow_client.flow_cli.utils import cli_args
from flow_client.flow_cli.utils.cli_utils import (prettify, preprocess, download_from_request,
                                                  access_server, check_abs_path)


@click.group(short_help="Component Operations")
@click.pass_context
def component(ctx):
    """
    \b
    Provides numbers of component operational commands, including metrics, parameters and etc.
    For more details, please check out the help text.
    """
    pass


@component.command("list", short_help="List Components Command")
@cli_args.JOBID_REQUIRED
@click.pass_context
def list(ctx, **kwargs):
    """
    \b
    - DESCRIPTION:
        List components of a specified job.

    \b
    - USAGE:
        flow component list -j $JOB_ID
    """
    config_data, dsl_data = preprocess(**kwargs)
    access_server('post', ctx, 'tracking/component/list', config_data)


@component.command("metrics", short_help="Component Metrics Command")
@cli_args.JOBID_REQUIRED
@cli_args.ROLE_REQUIRED
@cli_args.PARTYID_REQUIRED
@cli_args.COMPONENT_NAME_REQUIRED
@click.pass_context
def metrics(ctx, **kwargs):
    """
    \b
    - DESCRIPTION:
        Query the List of Metrics.

    \b
    - USAGE:
        flow component metrics -j $JOB_ID -r host -p 10000 -cpn hetero_feature_binning_0
    """
    config_data, dsl_data = preprocess(**kwargs)
    access_server('post', ctx, 'tracking/component/metrics', config_data)


@component.command("metric-all", short_help="Component Metric All Command")
@cli_args.JOBID_REQUIRED
@cli_args.ROLE_REQUIRED
@cli_args.PARTYID_REQUIRED
@cli_args.COMPONENT_NAME_REQUIRED
@click.pass_context
def metric_all(ctx, **kwargs):
    """
    \b
    - DESCRIPTION:
        Query All Metric Data.

    \b
    - USAGE:
        flow component metric-all -j $JOB_ID -r host -p 10000 -cpn hetero_feature_binning_0
    """
    config_data, dsl_data = preprocess(**kwargs)
    access_server('post', ctx, 'tracking/component/metric/all', config_data)


@component.command("metric-delete", short_help="Delete Metric Command")
@click.option('-d', '--date', type=click.STRING,
              help="An 8-digit valid date, format like 'YYYYMMDD'")
@cli_args.JOBID
@click.pass_context
def metric_delete(ctx, **kwargs):
    """
    \b
    - DESCRIPTION:
        Delete specified metric.
        If you input both two optional arguments, the 'date' argument will be detected in priority,
        while the job id will be ignored.

    \b
    - USAGE:
        flow component metric-delete -d 20200101
        flow component metric-delete -j $JOB_ID
    """
    config_data, dsl_data = preprocess(**kwargs)
    if config_data.get('date'):
        config_data['model'] = config_data.pop('date')
    access_server('post', ctx, 'tracking/component/metric/delete', config_data)


@component.command("parameters", short_help="Component Parameters Command")
@cli_args.JOBID_REQUIRED
@cli_args.ROLE_REQUIRED
@cli_args.PARTYID_REQUIRED
@cli_args.COMPONENT_NAME_REQUIRED
@click.pass_context
def parameters(ctx, **kwargs):
    """
    \b
    - DESCRIPTION:
        Query the parameters of a specified component.

    \b
    - USAGE:
        flow component parameters -j $JOB_ID -r host -p 10000 -cpn hetero_feature_binning_0
    """
    config_data, dsl_data = preprocess(**kwargs)
    access_server('post', ctx, 'tracking/component/parameters', config_data)


@component.command("output-data", short_help="Component Output Data Command")
@cli_args.JOBID_REQUIRED
@cli_args.ROLE_REQUIRED
@cli_args.PARTYID_REQUIRED
@cli_args.COMPONENT_NAME_REQUIRED
@cli_args.OUTPUT_PATH_REQUIRED
@click.option('-l', '--limit', type=click.INT, default=-1,
              help='limit count, defaults is -1 (download all output data)')
@click.pass_context
def output_data(ctx, **kwargs):
    """
    \b
    - DESCRIPTION:
        Download the Output Data of A Specified Component.

    \b
    - USAGE:
        flow component output-data -j $JOB_ID -r host -p 10000 -cpn hetero_feature_binning_0 --output-path ./examples/
    """
    config_data, dsl_data = preprocess(**kwargs)
    tar_file_name = 'job_{}_{}_{}_{}_output_data.tar.gz'.format(config_data['job_id'],
                                                                config_data['component_name'],
                                                                config_data['role'],
                                                                config_data['party_id'])
    extract_dir = os.path.join(config_data['output_path'], tar_file_name.replace('.tar.gz', ''))
    with closing(access_server('get', ctx, 'tracking/component/output/data/download',
                               config_data, False, stream=True)) as response:
        if response.status_code == 200:
            try:
                download_from_request(http_response=response, tar_file_name=tar_file_name, extract_dir=extract_dir)
                res = {
                    'retcode': 0,
                    'directory': os.path.abspath(extract_dir),
                    'retmsg': 'Download successfully, please check {} directory'.format(
                        os.path.abspath(extract_dir))}
            except BaseException:
                res = {'retcode': 100,
                       'retmsg': 'Download failed, please check if the parameters are correct.'}
        else:
            try:
                res = response.json()
            except Exception:
                res = {'retcode': 100,
                       'retmsg': 'Download failed, for more details please check logs/fate_flow/fate_flow_stat.log.'}
    prettify(res)


@component.command("output-model", short_help="Component Output Model Command")
@cli_args.JOBID_REQUIRED
@cli_args.ROLE_REQUIRED
@cli_args.PARTYID_REQUIRED
@cli_args.COMPONENT_NAME_REQUIRED
@click.pass_context
def output_model(ctx, **kwargs):
    """
    \b
    - DESCRIPTION:
        Query the Model of A Speicied Component.

    \b
    - USAGE:
        flow component output-model -j $JOB_ID -r host -p 10000 -cpn hetero_feature_binning_0
    """
    config_data, dsl_data = preprocess(**kwargs)
    access_server('post', ctx, 'tracking/component/output/model', config_data)


@component.command("output-data-table", short_help="Component Output Data Table Command")
@cli_args.JOBID_REQUIRED
@cli_args.ROLE_REQUIRED
@cli_args.PARTYID_REQUIRED
@cli_args.COMPONENT_NAME_REQUIRED
@click.pass_context
def output_data_table(ctx, **kwargs):
    """
    \b
    - DESCRIPTION:
        View Table Name and Namespace.

    \b
    - USAGE:
        flow component output-data-table -j $JOB_ID -r host -p 10000 -cpn hetero_feature_binning_0
    """
    config_data, dsl_data = preprocess(**kwargs)
    access_server('post', ctx, 'tracking/component/output/data/table', config_data)


@component.command("get-summary", short_help="Download Component Summary Command")
@cli_args.JOBID_REQUIRED
@cli_args.ROLE_REQUIRED
@cli_args.PARTYID_REQUIRED
@cli_args.COMPONENT_NAME_REQUIRED
@cli_args.OUTPUT_PATH
@click.pass_context
def get_summary(ctx, **kwargs):
    """
    \b
    - DESCRIPTION:
        Download summary of a specified component and save it as a json file.

    \b
    - USAGE:
        flow component get-summary -j $JOB_ID -r host -p 10000 -cpn hetero_feature_binning_0
        flow component get-summary -j $JOB_ID -r host -p 10000 -cpn hetero_feature_binning_0 -o ./examples/
    """
    config_data, dsl_data = preprocess(**kwargs)
    if config_data.get("output_path"):
        if not os.path.isdir(config_data.get("output_path")):
            res = {
                "retcode": 100,
                "retmsg": "Please input a valid directory path."
            }
        else:
            config_data["filename"] = "summary_{}_{}.json".format(config_data['component_name'],
                                                                  datetime.now().strftime('%Y%m%d%H%M%S'))
            config_data["output_path"] = os.path.join(check_abs_path(
                config_data["output_path"]), config_data["filename"])
            with closing(access_server("post", ctx, "tracking/component/summary/download",
                                       config_data, False, stream=True)) as response:
                if response.status_code == 200:
                    with open(config_data["output_path"], "wb") as fout:
                        for chunk in response.iter_content(1024):
                            if chunk:
                                fout.write(chunk)
                    res = {
                        "retcode": 0,
                        "retmsg": "The summary of component <{}> has been stored successfully. "
                                  "File path is: {}.".format(config_data["component_name"],
                                                             config_data["output_path"])
                    }
                else:
                    try:
                        res = response.json()
                    except Exception:
                        res = {"retcode": 100,
                               "retmsg": "Download component summary failed, "
                                         "for more details, please check logs/fate_flow/fate_flow_stat.log."}
        prettify(res)
    else:
        access_server("post", ctx, "tracking/component/summary/download", config_data)


@component.command('hetero-model-merge', short_help='Merge Hetero Model Command')
@cli_args.MODEL_ID_REQUIRED
@cli_args.MODEL_VERSION_REQUIRED
@cli_args.GUEST_PARTYID_REQUIRED
@cli_args.HOST_PARTYIDS_REQUIRED
@cli_args.COMPONENT_NAME_REQUIRED
@click.option('--model-type', type=click.STRING, required=True)
@click.option('--output-format', type=click.STRING, required=True)
@click.option('--target-name', type=click.STRING)
@click.option('--host-rename/--no-host-rename', is_flag=True, default=None)
@click.option('--include-guest-coef/--no-include-guest-coef', is_flag=True, default=None)
@cli_args.OUTPUT_PATH_REQUIRED
@click.pass_context
def hetero_model_merge(ctx, **kwargs):
    """
    \b
    - DESCRIPTION:
        Merge a hetero model.

    \b
    - USAGE:
        flow component hetero-model-merge --model-id guest-9999#host-9998#model --model-version 202208241838502253290 --guest-party-id 9999 --host-party-ids 9998,9997 --component-name hetero_secure_boost_0 --model-type secureboost --output-format pmml --target-name y --no-host-rename --no-include-guest-coef --output-path model.xml
    """
    config_data, dsl_data = preprocess(**kwargs)
    config_data['host_party_ids'] = config_data['host_party_ids'].split(',')

    response = access_server('post', ctx, 'component/hetero/merge', config_data, False)

    if not response.json().get('data'):
        prettify(response)
        return

    with open(config_data['output_path'], 'w', encoding='utf-8') as f:
        f.write(response.json()['data'])


@component.command('woe-array-extract', short_help='Extract WOE Array Command')
@cli_args.MODEL_ID_REQUIRED
@cli_args.MODEL_VERSION_REQUIRED
@cli_args.ROLE_REQUIRED
@cli_args.PARTYID_REQUIRED
@cli_args.COMPONENT_NAME_REQUIRED
@cli_args.OUTPUT_PATH_REQUIRED
@click.pass_context
def woe_array_extract(ctx, **kwargs):
    """
    \b
    - DESCRIPTION:
        Extract WOE array from a guest model.

    \b
    - USAGE:
        flow component woe-array-extract --model-id guest-9999#host-10000#model --model-version 202211142055541649630 --role guest --party-id 9999 --component-name hetero_feature_binning_0 --output-path woe_array.json
    """
    config_data, dsl_data = preprocess(**kwargs)

    response = access_server('post', ctx, 'component/woe_array/extract', config_data, False)

    if not response.json().get('data'):
        prettify(response)
        return

    with open(config_data['output_path'], 'w', encoding='utf-8') as f:
        json.dump(response.json()['data'], f)


@component.command('woe-array-merge', short_help='Merge WOE Array Command')
@cli_args.MODEL_ID_REQUIRED
@cli_args.MODEL_VERSION_REQUIRED
@cli_args.ROLE_REQUIRED
@cli_args.PARTYID_REQUIRED
@cli_args.COMPONENT_NAME_REQUIRED
@cli_args.INPUT_PATH_REQUIRED
@click.pass_context
def woe_array_merge(ctx, **kwargs):
    """
    \b
    - DESCRIPTION:
        Merge WOE array into a host model.

    \b
    - USAGE:
        flow component woe-array-merge --model-id guest-9999#host-10000#model --model-version 202211142055541649630 --role host --party-id 10000 --component-name hetero_feature_binning_0 --input-path woe_array.json
    """
    config_data, dsl_data = preprocess(**kwargs)

    with open(config_data.pop('input_path'), 'r', encoding='utf-8') as f:
        config_data['woe_array'] = json.load(f)

    access_server('post', ctx, 'component/woe_array/merge', config_data)