aqacf-ml-main/firealarm_ml/combine_fire_points.py

#!/usr/bin/env python
import os
import click
from pathlib import Path
import csv
import numpy as np
from tqdm import tqdm
from datetime import datetime as dt


FIRE_EXT = '.txt'


def transform_point(point):
    point = { k.strip(): v for k, v in point.items() }

    #timestr = point['Time'].strip()
    #if '-' in timestr:
    #    timestr = '0000'

    #dtime = dt.strptime(
    #    point['YearDay'].strip() + timestr,
    #    '%Y%j%H%M'
    #)

    return {
        'latitude': float(point['Lat'].strip()),
        'longitude': float(point['Lon'].strip()),
        'satellite': point['Satellite'].strip(),
        'method': point['Method'].strip(),
        'ecosystem': int(point['Ecosystem'].strip()),
        'fire_radiative_power': float(point['FRP'].strip()),
        #'time': dtime,
    }


def read_points(shapefile):

    basename = os.path.splitext(os.path.basename(shapefile))[0]

    date = dt.strptime(basename, 'hms_fire%Y%m%d')

    with open(shapefile, 'r') as f:
        reader = csv.DictReader(f)
        points = list(map(transform_point, reader))

    return date, points


@click.command()
@click.argument('firedir', type=click.Path(
    path_type=Path, exists=True
))
@click.argument('outputfile', type=click.Path(
    path_type=Path
))
def main(firedir, outputfile):

    fire_files = []
    for d, dirs, files in os.walk(firedir):
        for f in files:
            if f.endswith(FIRE_EXT):
                fire_files.append(os.path.join(d, f))

    dates = []
    all_points = []

    for sm in tqdm(sorted(fire_files), 'Reading fire points'):
        date, points = read_points(sm)
        for i in range(len(points)):
            dates.append(date)
        all_points += points

    get = lambda key, dtype: \
        np.array([p[key] for p in all_points], dtype=dtype)

    dates = np.array(dates, dtype='datetime64[D]')

    result = {
        'dates': dates,
        'latitudes': get('latitude', float),
        'longitudes': get('longitude', float),
        'satellites': get('satellite', str),
        'method': get('method', str),
        'ecosystem': get('ecosystem', int),
        'fire_radiative_power': get('fire_radiative_power', float),
    }

    np.savez_compressed(outputfile, **result)


if __name__ == '__main__':
    main()