Commit 8f855784 authored by shinde's avatar shinde
Browse files

Cleaned database.py code

parent a56db02b
Pipeline #21066 failed with stage
in 1 minute and 15 seconds
......@@ -148,17 +148,6 @@ class JapanDatabase(Database):
self.connection.execute(sql_statement)
logger.debug("Table Dwelling_Numbers created")
# Create Building_Numbers table
sql_statement = "CREATE TABLE Building_Numbers "
sql_statement += "(id INTEGER PRIMARY KEY,"
sql_statement += "district_id INTEGER,"
sql_statement += "building_type INTEGER,"
sql_statement += "construction_material INTEGER,"
sql_statement += "number_of_stories INTEGER,"
sql_statement += "number_of_buildings INTEGER);"
self.connection.execute(sql_statement)
logger.debug("Table Building_Numbers created")
def insert_district(self, admin_id, admin_name, geom):
"""
Inserts the ID, name, and geometry of a district to the District table.
......@@ -219,6 +208,7 @@ class JapanDatabase(Database):
Return:
id and description
"""
description = type_of_building
sql_statement = "INSERT or IGNORE into Building_Type " # `Ignore` ignores duplicate and stores unique value
sql_statement += "(description) "
......@@ -238,16 +228,6 @@ class JapanDatabase(Database):
Return:
id and description
"""
# read number of dwellings csv
# with open(construction_material_filepath) as csv_file:
# csv_reader = csv.reader(csv_file, delimiter=',') # Will modify to xslx in read_Number_of_Dwellings table
# next(csv_reader) # Skip first line
# for row in csv_reader:
# description = row[10]
# sql_statement = "INSERT OR IGNORE into Construction_Material "
# sql_statement += "(description) "
# sql_statement += "VALUES ('%s')" % description
# self.cursor.execute(sql_statement)
description = construction_material
sql_statement = "INSERT or IGNORE into Construction_Material " # `Ignore:stores unique value
......@@ -273,63 +253,6 @@ class JapanDatabase(Database):
sql_statement += "VALUES ('%s')" % description
self.cursor.execute(sql_statement)
# Read values into Dwelling_Type table
def read_dwelling_type(self, dwelling_type):
"""
Inserts unique values of types of dwellings in the Dwelling_Type table from
the Household Numbers file.
Args:
dwelling_type(str):
(Unique values of types of dwellings)
Return:
id and description
"""
description = dwelling_type
sql_statement = "INSERT OR IGNORE into Dwelling_Type " # `Ignore` ignores duplicate and stores unique value
sql_statement += "(description) "
sql_statement += "VALUES ('%s')" % description
self.cursor.execute(sql_statement)
# Read values into Dwelling_Size table
def read_dwelling_type(self, dwelling_size):
"""
Inserts unique values of types of sizes of dwellings in the Dwelling_Size table from
the Dwelling_Size file.
Args:
dwelling_size(str):
(Unique values of types of sizes of dwellings)
Return:
id and description
"""
description = dwelling_size
sql_statement = "INSERT OR IGNORE into Dwelling_Size " # `Ignore` ignores duplicate and stores unique value
sql_statement += "(description) "
sql_statement += "VALUES ('%s')" % description
self.cursor.execute(sql_statement)
# Read values into Tenure_Type table
def read_dwelling_type(self, tenure_type):
"""
Inserts unique values of types of tenures of dwellings in the Tenure_Type table from
the Dwelling_Size file.
Args:
tenure_type(str):
(Unique values of types of tenures of dwellings)
Return:
id and description
"""
description = tenure_type
sql_statement = "INSERT OR IGNORE into Tenure_Type " # `Ignore` ignores duplicate and stores unique value
sql_statement += "(description) "
sql_statement += "VALUES ('%s')" % description
self.cursor.execute(sql_statement)
def read_dwelling_numbers(self, dwelling_numbers_filepath):
"""
Inserts ids from District table, Building_Type table, Construction_Material table, Number_of_Stories table
......@@ -343,23 +266,23 @@ class JapanDatabase(Database):
Return:
id, district_id, building_type, construction_material, number_of_stories, number_of_dwellings
"""
# create an empty list
number_of_dwellings = []
# read original number of buildings file (xlsx)
stat_xslx = pd.read_excel(dwelling_numbers_filepath, header=10, usecols=[4, 5, 7, 9, 10, 11, 12, 13, 14])
for index, row in stat_xslx.iterrows():
if row['Identification Area Code'] != 1 and row['Identification Area Code'] != 'a':
# Read admin_id
admin_id = int((row['Area classification'].split('_'))[0])
# Read admin_id
admin_id = int((row['Area classification'].split('_'))[0])
# Identify district_id based on admin_id from the District table
sql_statement = "SELECT id FROM District "
sql_statement += "WHERE admin_id = %d" % admin_id
logger.info(sql_statement)
self.cursor.execute(sql_statement)
# Assign district_id
district_id = self.cursor.fetchall()[0][0]
logger.info(district_id)
# Identify district_id based on admin_id from the District table
sql_statement = "SELECT id FROM District "
sql_statement += "WHERE admin_id = %d" % admin_id
logger.info(sql_statement)
self.cursor.execute(sql_statement)
# Assign district_id
district_id = self.cursor.fetchall()
logger.info(district_id)
if district_id != []:
district_id = district_id[0][0]
# type_of_building in stat_xslx
type_of_building = row['Type of building']
......@@ -533,147 +456,9 @@ class JapanDatabase(Database):
number_of_stories, int(number_of_dwellings))
self.cursor.execute(sql_statement)
logger.info(sql_statement)
self.connection.commit()
else:
pass
self.connection.commit()
return self
# Read values into Building_Numbers table
def read_building_numbers(self, building_numbers_filepath):
"""
Inserts ids from District table, Building_Type table, Construction_Material table, Number_of_Stories table
corresponding to their description values in the number of buildings data file. Also, inserts number of
buildings for each Building_Numbers id.
Args:
building_numbers_filepath(str):
(File path to the original file of number_of_buildings)
Return:
id, district_id, buildings_type, construction_material, number_of_stories, number_of_buildings
"""
# read original number of buildings file (xlsx)
stat_xslx = pd.read_excel(building_numbers_filepath, header=10, usecols=[4, 5, 7, 9, 10, 11, 12])
for index, row in stat_xslx.iterrows():
# Read admin_id
admin_id = ((row['Area classification'].split('_'))[0])
# Identify district_id based on admin_id from the District table
sql_statement = "SELECT id FROM District "
sql_statement += "WHERE admin_id = %s" % admin_id
self.cursor.execute(sql_statement)
logger.info(sql_statement)
# Assign district_id
district_id = self.cursor.fetchall() # directly go the row instead of asking for it to go through the entire code.
if district_id != []:
district_id = district_id[0][0]
logger.info(district_id)
continue
# type_of_building
type_of_building = row['Type of building']
# Identify building_type
sql_statement = "SELECT id FROM Building_Type "
sql_statement += "WHERE description = '%s'" % type_of_building
self.cursor.execute(sql_statement)
logger.info(sql_statement)
# Assign building_type
building_type = self.cursor.fetchall()[0][0]
logger.info(building_type)
# number_of_stories
number_of_stories = row['Stories of building']
# Identify number_of_stories
sql_statement = "SELECT id FROM Number_of_Stories "
sql_statement += "WHERE description = '%s'" % number_of_stories
self.cursor.execute(sql_statement)
logger.info(sql_statement)
# Assign number_of_stories
number_of_stories = self.cursor.fetchall()[0][0]
logger.info(number_of_stories)
# Restructuring the table
# Add construction_material = 'total'
construction_material = 'total'
# Select ids from Construction_Material table for construction_material = 'total
sql_statement = "SELECT id FROM Construction_Material "
sql_statement += "WHERE description = '%s'" % construction_material
self.cursor.execute(sql_statement)
logger.info(sql_statement)
# Assign construction_material
construction_material = self.cursor.fetchall()[0][0]
logger.info(construction_material)
# number_of_buildings for construction_material = 'total'
number_of_buildings = str(row[' ']).replace('-', '0')
# Insert district_id, building_type, construction_material = 'total', number_of_stories,
# number_of_buildings for 'total' construction_material only into Building_Numbers table
sql_statement = "INSERT INTO Building_Numbers (district_id, building_type, construction_material, "
sql_statement += "number_of_stories, number_of_buildings) "
sql_statement += "VALUES (%d,%d,%d,%d,%d)" % (district_id, building_type, construction_material,
number_of_stories, int(number_of_buildings))
self.cursor.execute(sql_statement)
logger.info(sql_statement)
# Add construction_material = 'wooden'
construction_material = 'wooden'
# Select ids from Construction_Material table for construction_material = 'wooden'
sql_statement = "SELECT id FROM Construction_Material "
sql_statement += "WHERE description = '%s'" % construction_material
self.cursor.execute(sql_statement)
logger.info(sql_statement)
# Assign construction_material
construction_material = self.cursor.fetchall()[0][0]
logger.info(construction_material)
# number_of_buildings for construction_material = 'wooden'
number_of_buildings = str(row['Unnamed: 11']).replace('-', '0')
# Insert district_id, building_type, construction_material = 'total', number_of_stories,
# number_of_buildings for 'total' construction_material only into Building_Numbers table
sql_statement = "INSERT INTO Building_Numbers (district_id, building_type, construction_material, "
sql_statement += "number_of_stories, number_of_buildings) "
sql_statement += "VALUES (%d,%d,%d,%d,%d)" % (district_id, building_type, construction_material,
number_of_stories, int(number_of_buildings))
self.cursor.execute(sql_statement)
logger.info(sql_statement)
# Add construction_material = 'non_wooden'
construction_material = 'non_wooden'
# Select ids from Construction_Material table for construction_material = 'non_wooden'
sql_statement = "SELECT id FROM Construction_Material "
sql_statement += "WHERE description = '%s'" % construction_material
self.cursor.execute(sql_statement)
logger.info(sql_statement)
# Assign construction_material
construction_material = self.cursor.fetchall()[0][0]
logger.info(construction_material)
# number_of_buildings for construction_material = 'non_wooden'
number_of_buildings = str(row['Unnamed: 12']).replace('-', '0')
# Insert district_id, building_type, construction_material = 'total', number_of_stories,
# number_of_buildings for 'total' construction_material only into Building_Numbers table
sql_statement = "INSERT INTO Building_Numbers (district_id, building_type, construction_material, "
sql_statement += "number_of_stories, number_of_buildings) "
sql_statement += "VALUES (%d,%d,%d,%d,%d)" % (district_id, building_type, construction_material,
number_of_stories, int(number_of_buildings))
self.cursor.execute(sql_statement)
logger.info(sql_statement)
else:
pass
self.connection.commit()
......@@ -20,7 +20,7 @@
import logging
import sys
import sqlite3
from exposurejapan_database import JapanDatabase # pylint: disable=E0611,E0401
from database import JapanDatabase # pylint: disable=E0611,E0401
# Add a logger printing error, warning, info and debug messages to the screen
......@@ -43,8 +43,8 @@ def main():
exit()
db.create_tables()
db.read_districts_and_boundaries('/home/shinde/Nextcloud/GFZ_Team_Danijel/Japan/exposure_Model/gitlab/future_files/data/estat_bound_municipal.gpkg')
db.read_construction_material('/home/shinde/Nextcloud/GFZ_Team_Danijel/Japan/exposure_Model/gitlab/future_files/data/e008_3e.csv')
db.read_building_numbers('/home/shinde/Nextcloud/GFZ_Team_Danijel/Japan/exposure_Model/gitlab/future_files/data/e039_3e.xlsx')
db.read_dwelling_numbers('/home/shinde/Nextcloud/GFZ_Team_Danijel/Japan/exposure_Model/gitlab/future_files/data/e008_3e.xlsx')
logger.info("Dwelling Numbers file read")
# Leave the program
sys.exit()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment