多进程拷贝文件

简单版

[root@Da scripts]# cat multi_copyfile.py 
from multiprocessing import Pool
import shutil
import os
src_dir = '/tmp/test'
dst_dir = '/root/test'
todo_copy_file_list = []
todo_copy_dir_list = []
def get_all_file_abs_path(path):
    for current_path, current_path_dir_list, current_path_file_list in os.walk(path):
        todo_copy_dir_list.extend(map(lambda a: '{}/{}'.format(current_path, a), current_path_dir_list))
        todo_copy_file_list.extend(map(lambda a: '{}/{}'.format(current_path, a), current_path_file_list))


def create_dir(dir):
    print('Create dir: {}'.format(dir))
    if not os.path.exists(dst_dir):
        os.mkdir(dst_dir)
    os.mkdir(dir)

def copy_file(src_path, dst_path):
    shutil.copyfile(src_path, dst_path)

if __name__ == '__main__':
    get_all_file_abs_path(src_dir)
    pool = Pool(4)

    for dir in todo_copy_dir_list:
        backup_dir_path = dir.replace(src_dir, dst_dir)
        pool.apply_async(create_dir, (backup_dir_path, ))

    for file in todo_copy_file_list:
        src_path, dst_path = file, file.replace(src_dir, dst_dir)
        pool.apply_async(copy_file, (src_path, dst_path, ))

    pool.close()
    pool.join()

todo

• 实时监控拷贝新文件