# Copyright (C) 2019-2024 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.

import argparse
import pathlib
import json
from tqdm import tqdm

import numpy as np

from pymilvus import Collection, connections, utility, DataType
from pymilvus.bulk_writer import LocalBulkWriter, BulkFileType


class MilvusEncoder(json.JSONEncoder):
    def default(self, o):
        if isinstance(o, np.ndarray):
            return o.tolist()
        if isinstance(o, np.float32) or isinstance(o, np.float16):
            return float(o)
        return json.JSONEncoder.default(self, o)


def dump_func(args):
    return dump_collection(args.db_file, args.collection, args.path)


def bfloat16_to_float32(byte_data):
    bfloat16_array = np.frombuffer(byte_data, dtype=np.uint16)
    bfloat16_as_uint16 = bfloat16_array.view(np.uint16)
    float32_as_uint32 = (bfloat16_as_uint16.astype(np.uint32)) << 16
    float32_array = float32_as_uint32.view(np.float32)
    return float32_array


def binary_to_int_list(packed_bytes):
    byte_array = np.frombuffer(packed_bytes, dtype=np.uint8)
    return np.unpackbits(byte_array)


def dump_collection(db_file, collection_name, path):
    if not pathlib.Path(db_file).is_file():
        raise RuntimeError('db_file: %s not exists' % db_file)

    if not pathlib.Path(path).parent.is_dir():
        raise RuntimeError('dump path(%s)\'s parent dir not exists: %s not exists' % path)

    connections.connect("default", uri=db_file)
    if not utility.has_collection(collection_name):
        raise RuntimeError("Collection: %s not exists" % collection_name)
    
    collection = Collection(collection_name)
    total_rows = collection.query("", output_fields=["count(*)"])[0]["count(*)"]
    is_auto = collection.primary_field.auto_id
    pk_name = collection.primary_field.name
    bfloat16_fields = [field.name for field in collection.schema.fields if field.dtype == DataType.BFLOAT16_VECTOR]
    bin_fields = [field.name for field in collection.schema.fields if field.dtype == DataType.BINARY_VECTOR]

    writer = LocalBulkWriter(
            schema=collection.schema,
            local_path=path,
            segment_size=512*1024*1024,
            file_type=BulkFileType.JSON
    )

    it = collection.query_iterator(output_fields=['*'])
    progress_bar = tqdm(total=total_rows, desc=f"Dump collection {collection_name}'s data")
    while True:
        rows = it.next()
        if not rows:
            it.close()
            break
        if is_auto:
            for row in rows:
                del row[pk_name]
        if bfloat16_fields:
            for row in rows:
                for name in bfloat16_fields:
                    if name in row:
                        row[name] = bfloat16_to_float32(row[name])

        if bin_fields:
            for row in rows:
                for name in bin_fields:
                    if name in row:
                        row[name] = binary_to_int_list(row[name][0])

        rows = json.loads(json.dumps(rows, cls=MilvusEncoder))
        for row in rows:
            writer.append_row(row)
        progress_bar.update(len(rows))
    writer.commit()
    print("Dump collection %s success" % collection_name)


def main():
    parser = argparse.ArgumentParser(prog='milvus-lite')
    subparsers = parser.add_subparsers(description='milvus-lite command line tool.')

    dump_cmd = subparsers.add_parser('dump', help='milvus-lite dump cmd')
    dump_cmd.add_argument('-d', '--db-file', type=str, help='milvus lite db file')
    dump_cmd.add_argument('-c', '--collection', type=str, help='collection that need to be dumped')
    dump_cmd.add_argument('-p', '--path', type=str, help='dump file storage path')
    dump_cmd.set_defaults(func=dump_func)

    args = parser.parse_args()
    if hasattr(args, 'func'):
        args.func(args)
    else:
        parser.print_help()


if __name__ == '__main__':
    main()