Project

General

Profile

Revision 834:d989df597b80

IDd989df597b80
Parent 833:354ddca04053
Child 835:d623d705abd3

Added by Maarten Sneep about 2 years ago

Use the exclude parameter when ingesting existing files

View differences:

src/concatenate_aggregate.py
174 174
                    for idx in range(step_count):
175 175
                        if i == 0 and idx == 0:
176 176
                            continue
177
                        o = C(filename=src, time_index=idx)
177
                        o = C(filename=src, time_index=idx, exclude=args.exclude)
178 178
                        if o is None or not o.success or d is None or not d.success:
179 179
                            continue
180 180
                        d += o
......
359 359

  
360 360
            for C in cls_list:
361 361
                for idx in range(steps_in_file):
362
                    d = C(filename=src, time_index=idx)
362
                    d = C(filename=src, time_index=idx, exclude=args.exclude)
363 363
                    if d is None or not d.success:
364 364
                        continue
365 365
                    d.time_index_in_output = i+idx
src/pycama/AlongTrackPlot.py
230 230
    #
231 231
    #  This method reads the date from file to restore itself for plotting.
232 232
    #  The exected format is as written by the pycama.AlongTrackPlot.AlongTrackPlot.dump() method.
233
    def ingest(self, fname, time_step):
233
    def ingest(self, fname, time_step, exclude=None):
234 234
        self.time_index_in_output = time_step
235 235
        with h5py.File(fname, mode='r') as ref:
236 236
            try:
......
241 241
            self.variables = {k.decode('utf-8'): {} for k in ref['variable_names'][:]}
242 242

  
243 243
            for key in self.variable_names:
244
                if exclude is not None and key in exclude:
245
                    continue
244 246
                for value in ('mean', 'min', 'max', 'standard_deviation',
245 247
                              'q01', 'q05', 'q10', 'q25', 'q75', 'q90', 'q95', 'q99',
246 248
                              'inter_quartile_range', 'median', 'mode', 'count'):
src/pycama/AnalysisAndPlot.py
58 58
    #
59 59
    #  @Note Either `reader_data` or both `filename` and `time_index` must be supplied.
60 60
    #
61
    def __init__(self, reader_data=None, filename=None, time_index=None, **kwargs):
61
    def __init__(self, reader_data=None, filename=None, time_index=None, exclude=None, **kwargs):
62 62
        ## logging.Logger instance for messaging
63 63
        self.logger = logging.getLogger('PyCAMA')
64 64
        self.product = None
......
94 94
        else:
95 95
            self.time_index_in_output = None
96 96
            if filename is not None and time_index is not None:
97
                self.success = self.ingest(filename, time_index)
97
                self.success = self.ingest(filename, time_index, exclude=exclude)
98 98
                self.product, self.processing_mode = read_product_and_mode_from_file(filename)
99 99
                if not self.success:
100 100
                    self.logger.warning("Ingestion not successful for class %s", self.__class__.__name__)
......
165 165
    #
166 166
    #  Expected to be overridden in subclasses.
167 167
    #
168
    def ingest(self, fname, time_index):
168
    def ingest(self, fname, time_index, exclude=None):
169 169
        raise NotImplementedError("Method must be overridden in a subclass.")
170 170

  
171 171
    ## Write processed data to output netcdf file.
src/pycama/EventPlot.py
117 117
    #
118 118
    #  This method reads the date from file to restore itself for plotting.
119 119
    #  The exected format is as written by the pycama.EventPlot.EventPlot.dump() method.
120
    def ingest(self, fname, time_index):
120
    def ingest(self, fname, time_index, exclude=None):
121 121
        self.time_index_in_output = time_index
122 122
        with h5py.File(fname, 'r') as ref:
123 123
            try:
src/pycama/HistogramPlot.py
252 252
    #
253 253
    #  @param fname NetCDF file with input data.
254 254
    #  @param time_index Time slice to read.
255
    def ingest(self, fname, time_index):
255
    def ingest(self, fname, time_index, exclude=None):
256 256
        self.time_index_in_output = time_index
257 257
        with h5py.File(fname, 'r') as ref:
258 258
            try:
......
263 263
                return False
264 264

  
265 265
            vlist = [k.decode("ASCII") for k in ref['variable_names'][:]]
266
            if exclude is not None:
267
                vlist = [v for v in vlist if v not in exclude]
268
                
266 269
            self.input_variables = Namespace(variables={})
267 270

  
268 271
            for k in vlist:
src/pycama/IrradiancePlot.py
208 208
    #
209 209
    #  @param fname NetCDF file with input data.
210 210
    #  @param time_index Time slice to read.
211
    def ingest(self, fname, time_step):
211
    def ingest(self, fname, time_step, exclude=None):
212 212
        self.time_index_in_output = time_step
213 213
        with h5py.File(fname, mode='r') as ref:
214 214
            try:
......
228 228
                if vgrp.attrs['time_step'] == time_step:
229 229
                    self.variables[key] = {}
230 230
                    for name, vv in [(kkey,vval) for (kkey, vval) in vgrp.items() if isinstance(vval, h5py.Dataset)]:
231
                        if exclude is not None and name in exclude:
232
                            continue
233
                            
231 234
                        self.variables[key][name] = {}
232 235
                        self.variables[key][name]['data'] = vv[:]
233 236
                        self.variables[key][name]['dimensions'] = [d.keys()[0] for d in vv.dims]
src/pycama/OutlinePlot.py
167 167
    #
168 168
    #  @param fname NetCDF file with input data.
169 169
    #  @param time_index Time slice to read.
170
    def ingest(self, fname, time_index):
170
    def ingest(self, fname, time_index, exclude=None):
171 171
        self.time_index_in_output = time_index
172 172
        with h5py.File(fname, 'r') as ref:
173 173
            try:
src/pycama/ScatterPlot.py
345 345
    #
346 346
    #  @param fname NetCDF file with input data.
347 347
    #  @param time_index Time slice to read.
348
    def ingest(self, fname, time_index):
348
    def ingest(self, fname, time_index, exclude=None):
349 349
        self.time_index_in_output = time_index
350 350

  
351 351
        with netCDF4.Dataset(fname, 'r') as ref:
......
359 359
            self.pair_list = []
360 360
            self.index_list = []
361 361
            variable_names = grp.variables['variable_names_scatter'][:]
362
            if exclude is not None:
363
                variable_names = [v for v in variable_names if v not in exclude]
364
                
362 365
            self.logger.debug("{0}.ingest(): variable names [{1}]".format(self.__class__.__name__, ", ".join(variable_names)))
363 366
            for i1, n1 in enumerate(variable_names[:-1]):
364 367
                for i, n2 in enumerate(variable_names[i1+1:]):
src/pycama/WorldPlot.py
232 232
    #
233 233
    #  @param fname NetCDF file with input data.
234 234
    #  @param time_index Time slice to read.
235
    def ingest(self, fname, time_index):
235
    def ingest(self, fname, time_index, exclude=None):
236 236
        self.logger.debug("{0}.ingest(): reading {1}".format(self.__class__.__name__, fname))
237 237
        self.time_index_in_output = time_index
238 238
        with h5py.File(fname, 'r') as ref:
......
256 256
                if k not in grp.keys():
257 257
                    self.logger.debug("Variable name {0} requested but not available".format(k))
258 258
                    continue
259
                if exclude is not None and k in exclude:
260
                    self.logger.debug("Variable name {0} is excluded".format(k))
261
                    continue
262
                    
259 263
                self.variables_meta[k] = {}
260 264
                self.variables_meta[k]['noscanline'] = False
261 265
                var = grp[k]

Also available in: Unified diff