Project

General

Profile

Revision 830:7fd5b60ffc5a

ID7fd5b60ffc5a
Parent 829:0e26f41390ad
Child 831:663001078b4f

Added by Maarten Sneep about 2 years ago

Add filter to remove duplicate entries in the input (not file based but selected on the observed day)

View differences:

src/concatenate_aggregate.py
42 42
                        help='The netCDF files extracted by PyCAMA')
43 43

  
44 44
    args = parser.parse_args()
45
    
46
    args.input_files = uniq_days(args.input_files)
47
    
45 48
    if args.action == "concatenate":
46 49
        concatenate_pycama(args)
47 50
    elif args.action == "aggregate":
48 51
        aggregate_pycama(args)
49 52

  
53
def uniq_days(file_list):
54
    output = {}
55
    for src in file_list:
56
        steps_in_file = extract_time_step_count(src)
57
        for idx in range(steps_in_file):
58
            t, t_start, t_end = extract_time(fname, idx)
59
            t_str = "{0:%Y-%m-%d}".format(t)
60
            if t_str in output:
61
                output[t_str].append(src)
62
            else:
63
                output[t_str] = [src]
64
    
65
    file_list = []
66
    for val in output.values():
67
        if len(val) == 1:
68
            file_list.append(val[0])
69
        else:
70
            file_list.append(sorted(val, key=os.path.basename)[-1])
71
    
72
    return sorted(list(set(file_list)))
73
    
74
    
50 75
def aggregate_pycama(args):
51 76
    logger = setup_logging(production_logger=True, loglevel=args.log, errlevel="error")
52 77
    start_time = datetime.datetime.utcnow()

Also available in: Unified diff