preprocessor.py 6.81 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
#!/usr/bin/env python3

# Copyright (C) 2020-2021:
#   Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see http://www.gnu.org/licenses/.

"""
20
21
TODO: Update description

22
Returns an exposure model along with the asset geometries as either a csv or database file.
Danijel Schorlemmer's avatar
Applied    
Danijel Schorlemmer committed
23
Please use either `import` to import exposure data into the database or `export` to export to
24
an OpenQuake-compatible exposure CSV file.
25
26
27

Program Inputs:
---------------
28
29
30
31
32
33
34
`-d`, `--exposure-database`: str, required
Defines the path to an exposure model database file.
When using import module, it points to a file where the exposure database file is to be stored.
When using export module, it points the an existing exposure database file from which a csv
exposure file with header
`id,lon,lat,taxonomy,number,structural,night,occupancy,admin_name,admin_ID, tile_id,
tile_geometry, building_id ,building_geometry` is exported.
35

36
37
38
39
40
41
42
43
44
45
`-g`, `--building-geometries`: str, optional
Defines the path to the file that maps the building ID (`origin-id` for a building asset
in the exposure model) from which the asset is originated to the building geometry and the tile
ID that the building centroid is located in.
Only required if there are building assets in the input exposure model; with format as below:
`origin_id;geometry;tile_id`

`-i`, `--import-search-pattern`: str, optional
Search pattern for the input exposure model files (e.g. '*.csv').
It should follow the format below:
46
`id,lon,lat,taxonomy,number,structural,night,occupancy,admin_name,admin_ID,origin_id`
47
Only required when using import module.
48

49
50
`-o`, `--export-filepath`: str, optional
Defines the filepath to the output CSV exposure model file (OpenQuake compatible) with format
51
52
53
as below:
`id,lon,lat,taxonomy,number,structural,night,occupancy,admin_name,admin_ID, tile_id,
tile_geometry, building_id ,building_geometry`
54
Only required when using export module.
55

56
57
`-s`, `--spatialite-extension`: str, optional
Defines the file path of the spatialite extension. (default: mod_spatialite)
58
59

`-w` overwrite existing result file: optional
60
To overwrite an existing exposure model file.
61
62
63
64
65

In order to use this program, please note that:
The spacing of the grid is 10 arc-seconds. The tile ID starts from North-West corner of the
world, moves East by row, and finishes at the South-East corner of the world. First tile ID
is 0, last tile ID is 8,398,079,999 (total number of cells is 8,398,080,000).
66
67


68
69
70
71
72
73
74
75
76
77
Execution examples:
---------------
Create a database exposure model file:
python3 preprocessor.py import -d ../example_preprocessor/augmented_exposure.db
-i ../example_preprocessor/data/exposure_*.csv
-g ../example_preprocessor/data/building_to_cellid_map.csv

Create a csv exposure model file:
python3 preprocessor.py export -d ../example_preprocessor/augmented_exposure.db
-o ../example_preprocessor/augmented_exposure.csv
78

79
80
"""

81
import logging
82
83
84
import sys
import os
import argparse
85
import sqlite3
86
from database import ExposureDatabase
87

88
89
90
91
92
# Add a logger printing error, warning, info and debug messages to the screen
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
logger.addHandler(logging.StreamHandler(sys.stdout))

93
94

def main():
95
    """
96
    Returns an exposure model along with the asset geometries as either a csv or database file.
97
    """
98
    parser = argparse.ArgumentParser(
99
        description="This program imports exposure data into an exposure database "
100
        + "(Spatialite) from OpenQuake-style exposure data and OBM building data."
101
    )
102
103
104
105
106
107
108
    help_str = "'import' to import exposure data into the database, "
    help_str += "'export' to export to an OpenQuake-compatible exposure CSV file"
    parser.add_argument(
        "command",
        type=str,
        help=help_str,
    )
109
110
111
112
113
114
115
    parser.add_argument(
        "-d",
        "--exposure-database",
        required=True,
        type=str,
        help="File path to the exposure model database file (Required)",
    )
116
117
    parser.add_argument(
        "-i",
118
        "--import-search-pattern",
119
        required=False,
120
        type=str,
121
        help="Search pattern for the input exposure model files for importing (e.g. '*.csv')",
122
123
124
    )
    parser.add_argument(
        "-o",
125
        "--export-filepath",
126
        required=False,
127
        type=str,
128
        help="File path to the exposure model CSV file",
129
130
131
    )
    parser.add_argument(
        "-g",
132
        "--building-geometries",
133
134
        required=False,
        type=str,
135
136
        help="File path of the CSV file with building geometries (in EPSG:3857) "
        + "following the format: `building_id;geometry;cell_id`",
137
138
139
140
141
142
    )
    parser.add_argument(
        "-w",
        "--overwrite",
        required=False,
        action="store_true",
143
        help="Overwrite the existing exposure database file",
144
    )
145
146
147
148
149
150
151
152
    parser.add_argument(
        "-s",
        "--spatialite-extension",
        required=False,
        type=str,
        default="mod_spatialite",
        help="File path of the spatialite extension (default: mod_spatialite)",
    )
153
154
155

    args = parser.parse_args()

156
157
    command = args.command
    exposure_database = args.exposure_database
158
    export_filepath = args.export_filepath
159
    import_search_pattern = args.import_search_pattern
160
    building_geometries_filepath = args.building_geometries
161
    overwrite_exposure_database = args.overwrite
162
    spatialite_filepath = args.spatialite_extension
163

164
    if command == "import":
165
        if os.path.exists(exposure_database):
166
            if overwrite_exposure_database:
167
                os.remove(exposure_database)
168
169
170
            else:
                logger.warning("Database exists. Exiting ...")
                exit()
171
        db = ExposureDatabase(exposure_database, spatialite_filepath)
172
173
174
175
176
        try:
            db.create_connection_and_cursor()
        except sqlite3.OperationalError:
            logger.warning("Spatialite extension cannot be loaded. Exiting ...")
            exit()
177
        db.import_exposure(import_search_pattern, building_geometries_filepath)
178
    elif command == "export":
179
        db = ExposureDatabase(exposure_database, spatialite_filepath)
180
181
182
183
184
        try:
            db.create_connection_and_cursor(init_spatial_metadata=False)
        except sqlite3.OperationalError:
            logger.warning("Spatialite extension cannot be loaded. Exiting ...")
            exit()
185
        db.export_exposure(export_filepath)
186
187
188


main()