Commit 2c60a4f5 authored by Tara Evaz Zadeh's avatar Tara Evaz Zadeh
Browse files

Edited funcs to avoid opening and closing files repeatedly in for loops

parent 1c3ecd7b
Pipeline #18683 passed with stage
in 1 minute and 19 seconds
#!/usr/bin/env python3
# Copyright (c) 2020:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
# Copyright (c) 2020-2021:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
......@@ -33,39 +33,35 @@ def AddRespectiveCellIdToOBMExposureFiles(exposureFile_path, OSMSource_path, Res
exposureSource = pd.read_csv(exposureFile_path)
originids = exposureSource.origin_id
def append_list_as_row(file_name, list_of_elem):
with open(file_name, "a+", newline="") as write_obj:
csv_writer = csv.writer(write_obj)
csv_writer.writerow(list_of_elem)
with open(Result_path, "a+", newline="") as write_obj:
csv_writer = csv.writer(write_obj)
title = [
"id",
"lon",
"lat",
"taxonomy",
"number",
"structural",
"night",
"occupancy",
"admin_name",
"admin_ID",
"origin_id",
"respectiveCellId",
]
csv_writer.writerow(title)
OSMIdToCellId = {}
for MappingItem in OSMSource:
if MappingItem[0] in OSMIdToCellId:
# Ignore the additional geometry to keep everything unambiguous
pass
OSMIdToCellId[MappingItem[0]] = MappingItem[2]
title = [
"id",
"lon",
"lat",
"taxonomy",
"number",
"structural",
"night",
"occupancy",
"admin_name",
"admin_ID",
"origin_id",
"respectiveCellId",
]
append_list_as_row(Result_path, title)
OSMIdToCellId = {}
for MappingItem in OSMSource:
if MappingItem[0] in OSMIdToCellId:
# Ignore the additional geometry to keep everything unambiguous
pass
OSMIdToCellId[MappingItem[0]] = MappingItem[2]
for asset in range(exposureSource.shape[0]):
exposureArr = list(exposureSource.loc[asset])
origin_id = originids[asset]
respectiveCellId = OSMIdToCellId[origin_id]
exposureArr.append(respectiveCellId)
append_list_as_row(Result_path, exposureArr)
for asset_from_model in range(exposureSource.shape[0]):
asset_for_calculation = list(exposureSource.loc[asset_from_model])
origin_id = originids[asset_from_model]
respectiveCellId = OSMIdToCellId[origin_id]
asset_for_calculation.append(respectiveCellId)
csv_writer.writerow(asset_for_calculation)
print("Execution time of the script", (datetime.datetime.now() - startTime))
#!/usr/bin/env python3
# Copyright (c) 2020:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
# Copyright (c) 2020-2021:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
......@@ -19,11 +19,11 @@
import numpy as np
import pandas as pd
import csv
import losslib
import os
import argparse
from damage_calculator_tile_version import damageCalculator_TileVersion
import csv
def get_exposure_per_tile(
......@@ -46,84 +46,77 @@ def get_exposure_per_tile(
ground_motion_field, lons_whole_area, lats_whole_area, interpolation_method
)
all_cell_ids = np.loadtxt(cellIdSource_filepath, dtype="str", skiprows=1)
def write_asset_to_file(file_name, list_of_elem):
"""This function writes a list (list_of_elem) to a file (file_name)"""
# Open the file to write the lists to. a+ gives the possibility of reading and
# writing if file exists and creating it if it doesn't.
with open(file_name, "a+", newline="") as write_obj:
csv_writer = csv.writer(write_obj)
csv_writer.writerow(list_of_elem)
if exposure_type == "building":
title = [
"geometry",
"",
"origin_id",
"",
"RespectiveCellid",
"",
"asset_id",
"",
"lon",
"",
"lat",
"",
"taxonomy",
"",
"gm_value",
"",
"PoEs",
"",
"PoOs",
"",
"tot_num_buildings",
"",
"structural_no_damage",
"",
"structural_slight",
"",
"structural_moderate",
"",
"structural_extensive",
"",
"structural_complete",
]
tile_id = "respectiveCellId"
else:
title = [
"geometry",
"",
"origin_id",
"",
"asset_id",
"",
"lon",
"",
"lat",
"",
"taxonomy",
"",
"gm_value",
"",
"PoEs",
"",
"PoOs",
"",
"tot_num_buildings",
"",
"structural_no_damage",
"",
"structural_slight",
"",
"structural_moderate",
"",
"structural_extensive",
"",
"structural_complete",
]
tile_id = "origin_id"
write_asset_to_file(result_filepath, title)
with open(result_filepath, "a+", newline="") as write_obj:
csv_writer = csv.writer(write_obj)
if exposure_type == "building":
title = [
"geometry",
"",
"origin_id",
"",
"RespectiveCellid",
"",
"asset_id",
"",
"lon",
"",
"lat",
"",
"taxonomy",
"",
"gm_value",
"",
"PoEs",
"",
"PoOs",
"",
"num_buildings",
"",
"structural_no_damage",
"",
"structural_slight",
"",
"structural_moderate",
"",
"structural_extensive",
"",
"structural_complete",
]
tile_id = "respectiveCellId"
else:
title = [
"geometry",
"",
"origin_id",
"",
"asset_id",
"",
"lon",
"",
"lat",
"",
"taxonomy",
"",
"gm_value",
"",
"PoEs",
"",
"PoOs",
"",
"num_buildings",
"",
"structural_no_damage",
"",
"structural_slight",
"",
"structural_moderate",
"",
"structural_extensive",
"",
"structural_complete",
]
tile_id = "origin_id"
csv_writer.writerow(title)
for i in range(all_cell_ids.shape[0]):
cell_id = all_cell_ids[i]
......
#!/usr/bin/env python3
# Copyright (c) 2020:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
# Copyright (c) 2020-2021:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
......@@ -16,7 +16,6 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see http://www.gnu.org/licenses/.
import numpy as np
import csv
import datetime
......@@ -24,10 +23,10 @@ import losslib
def damageCalculator_TileVersion(
fullGroundMotionField,
Result_path,
fragilityFileDir,
exposures,
full_ground_motion_field,
result_filepath,
fragility_pathname,
exposure,
taxonomy_conversion_path,
shakemap_path,
geometry_source_path,
......@@ -36,12 +35,12 @@ def damageCalculator_TileVersion(
):
"""
Returns a file "Result_path" including damage results of a scenario earthquake using the
Returns a file "result_filepath" including damage results of a scenario earthquake using the
"losslib" functions.
Input:
------
- fullGroundMotionField: (Numpy nd-array of shape(m,n); m records with long,
- full_ground_motion_field: (Numpy nd-array of shape(m,n); m records with long,
lat, and n-2 gm-types)
contains locations and the ground-motion values of each location
in the format of lon, lat, gmValueofType1, gmValueofType2, ...,
......@@ -54,7 +53,7 @@ def damageCalculator_TileVersion(
As an example:
[23.6875,38.3402777778,'PGA','SA(0.3)','SA(0.6)','SA(1.0)']
Example extract:
>>> fullGroundMotionField
>>> full_ground_motion_field
array([[
[23.687 , 38.340, 0.179, 0.362 , 0.252,
0.152],
......@@ -66,22 +65,22 @@ def damageCalculator_TileVersion(
0.149]
]])
- Result_path: (str)
- result_filepath: (str)
Result file address.
Example extract:
>>> Result_path
>>> result_filepath
"/home/TileCalculations/DamageResult.csv"
- fragilityFileDir: (str)
- fragility_pathname: (str)
Address to the directory including all the fragility functions.
Example extract:
>>> fragilityFileDir
>>> fragility_pathname
"/home/TileCalculations/Fragilities"
- exposures: (pandas.DataFrame(series))
- exposure: (pandas.DataFrame(series))
informaion of the assets of the region of interest under hazard.
Example extract:
>>> exposures
>>> exposure
id lon lat ... admin_name admin_ID origin_id
0 GDE_Ind_0 23.687500 38.340278 ... Oropos GR_3514913 cell_2410244527
1 GDE_Ind_1 23.687500 38.340278 ... Oropos GR_3514913 cell_2410244527
......@@ -120,11 +119,11 @@ def damageCalculator_TileVersion(
Output:
-------
- Result_path: (arrays written to file)
- result_filepath: (arrays written to file)
file containing the damage elements for each asset of the exposure.
Example extract of the result file:
geometry,,origin_id,,asset_id,,lon,,lat,,taxonomy,,gmfValue,,PoEs,,PoOs,,
tot_num_buildings,,structural_No-damage,,structural_Slight,,structural_Moderate,,
num_buildings,,structural_No-damage,,structural_Slight,,structural_Moderate,,
structural_Extensive,,structural_Complete
"POLYGON ((23.68611111111113 38.3388888888889, 23.6888888888889 38.3388888888889,
23.6888888888889 38.34166666666667, 23.68611111111113 38.34166666666667,
......@@ -149,119 +148,116 @@ def damageCalculator_TileVersion(
# Skip the header
next(geometry_source, None)
# Read each column of the input "exposure"
taxonomies = exposures.taxonomy
tot_num_buildings_row = exposures.number
lons = exposures.lon
lats = exposures.lat
assetids = exposures.id
originids = exposures.origin_id
# Prepare Result File
def append_list_as_row(file_name, list_of_elem):
with open(file_name, "a+", newline="") as write_obj:
csv_writer = csv.writer(write_obj)
csv_writer.writerow(list_of_elem)
# Read each column of the exposure model
taxonomies = exposure.taxonomy
tot_num_buildings = exposure.number
lons = exposure.lon
lats = exposure.lat
asset_ids = exposure.id
origin_ids = exposure.origin_id
# Begin Computation
# Define a dictionary with keys as the ground-motion type and value as the column
# number of the ground-motion type in the shakemap file.
gmDict = {"PGA": 2, "SA(0.3)": 3, "SA(0.6)": 4, "SA(1.0)": 5, "SA(1)": 5}
# Calling the function "Taxonomy_to_Fragility" to get a dictionary with keys as the
# taxonomy and the values as both the fragility function name (excluding ".csv" part)
# and column of ground-motion_type in ground-motion_field file.
# taxonomy and the values as both the fragility function name (excluding the ".csv" part)
# and the column number of the respective ground-motion type in `ground-motion-field` file.
taxonomyToFragilityMap = losslib.Taxonomy_to_Fragility(
gmDict, taxonomyToFragilitySource, fragilityFileDir
gmDict, taxonomyToFragilitySource, fragility_pathname
)
# Calling the function "origin_id_to_geometry" to get a dictionary with keys as the
# origin_id and the value as the respective polygon.
origin_id_to_geometry_map = losslib.origin_id_to_geometry(geometry_source, exposureType)
# Define number of columns that contain the data in the fragiliy function files.
# Define number of columns that contain the data in the fragility function files.
cls = range(1, 101)
# Just a trick to have multiple commas between each result element, since we do
# not want a single comma as the delimiter due to having ploygons as a result element.
# not want a single comma as the delimiter due to having geometries as a result element.
a = [0, 2, 4, 6, 8]
# Looping through each line of the exposure file to do the computations line by line.
for asset in range(exposures.shape[0]):
taxonomy = taxonomies.iloc[asset]
fragilityFileName = taxonomyToFragilityMap[taxonomy][0] + ".csv"
tot_num_buildings = tot_num_buildings_row.iloc[asset]
lon = lons.iloc[asset]
lat = lats.iloc[asset]
asset_id = assetids.iloc[asset]
origin_id = originids.iloc[asset]
# Since for the OBM exposure files we also need to know the cell-id that the
# building is located in, referred as "RespectiveCellid" and the polygon of
# the buiding.
if exposureType == "OBM":
[geometry, RespectiveCellid] = origin_id_to_geometry_map[origin_id]
else:
geometry = origin_id_to_geometry_map[origin_id]
# Looping through each line of the exposure file to run the computations asset by asset.
with open(result_filepath, "a+", newline="") as write_obj:
csv_writer = csv.writer(write_obj)
for asset in range(exposure.shape[0]):
taxonomy = taxonomies.iloc[asset]
fragilityFileName = taxonomyToFragilityMap[taxonomy][0] + ".csv"
num_buildings = tot_num_buildings.iloc[asset]
lon = lons.iloc[asset]
lat = lats.iloc[asset]
asset_id = asset_ids.iloc[asset]
origin_id = origin_ids.iloc[asset]
# In case of `exposureType == "OBM"`, not only the geometry, but also the
# respective cell ID (in which the asset is located), is extracted
# using `origin_id_to_geometry_map`
if exposureType == "OBM":
[geometry, respective_cell_id] = origin_id_to_geometry_map[origin_id]
else:
geometry = origin_id_to_geometry_map[origin_id]
# Read fragility functions as numpy arrays.
fragility_function = np.loadtxt(
fragilityFileDir + "/" + fragilityFileName, delimiter=",", usecols=cls
)
# Achieve the ground-motion value from the fullGroundMotionField. Please note
# that the "fullGroundMotionField" contains many identical lines (since so many
# assets have same locations (same location leads to same ground-motion value))
# because we want it to have same number of lines and same orders as the exposure file,
# so that in this line we read the same line number as the exposure file.
gm_value = fullGroundMotionField[asset, taxonomyToFragilityMap[taxonomy][1]]
# Achieve Probabilities of exceedance and occurance.
[PoEs, PoOs] = losslib.get_PoEs(fragility_function, gm_value)
# Read fragility functions as numpy arrays.
fragility_function = np.loadtxt(
fragility_pathname + "/" + fragilityFileName, delimiter=",", usecols=cls
)
# Computing the ground-motion values from `full_ground_motion_field`. Please
# note that `full_ground_motion_field` contains many identical lines (because
# many of the assets have same locations and same location leads to the same
# ground-motion value). This way, `full_ground_motion_field` has the same number
# and order of assets as in the exposure model for simpler processing.
gm_value = full_ground_motion_field[asset, taxonomyToFragilityMap[taxonomy][1]]
# Computing probabilities of exceedance and occurrence.
[PoEs, PoOs] = losslib.get_PoEs(fragility_function, gm_value)
# Compute damage by assets
dmg_by_asset = [i * tot_num_buildings for i in PoOs]
for h in a:
dmg_by_asset.insert(h, "")
# Append results
if exposureType == "OBM":
arr0 = [
geometry,
"",
origin_id,
"",
"cell_" + RespectiveCellid,
"",
asset_id,
"",
lon,
"",
lat,
"",
taxonomy,
"",
gm_value,
"",
PoEs,
"",
PoOs,
"",
tot_num_buildings,
]
else:
arr0 = [
geometry,
"",
origin_id,
"",
asset_id,
"",
lon,
"",
lat,
"",
taxonomy,
"",
gm_value,
"",
PoEs,
"",
PoOs,
"",
tot_num_buildings,
]
arr0.extend(dmg_by_asset)
append_list_as_row(Result_path, arr0)
# Compute damage by assets
dmg_by_asset = [i * num_buildings for i in PoOs]
for h in a:
dmg_by_asset.insert(h, "")
# Append results
if exposureType == "OBM":
asset = [
geometry,
"",
origin_id,
"",
"cell_" + respective_cell_id,
"",
asset_id,
"",
lon,
"",
lat,
"",
taxonomy,
"",
gm_value,
"",
PoEs,
"",
PoOs,
"",
num_buildings,
]
else:
asset = [
geometry,
"",
origin_id,
"",
asset_id,
"",
lon,
"",
lat,
"",
taxonomy,
"",
gm_value,
"",
PoEs,
"",
PoOs,
"",
num_buildings,
]
asset.extend(dmg_by_asset)
csv_writer.writerow(asset)
print("Execution time of the script", (datetime.datetime.now() - startTime))
#!/usr/bin/env python3
# Copyright (c) 2020:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
# Copyright (c) 2020-2021:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment