Commit 435b56ec authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Some code improvements.

Former-commit-id: ee0b0225
Former-commit-id: 7d262cd7
parent eb6ff977
......@@ -12,10 +12,12 @@ import warnings
from datetime import datetime
from typing import Union # noqa F401 # flake8 issue
from six import PY3
import numpy as np
import pandas as pd
from pandas.io.sql import pandasSQL_builder, SQLTable
import psycopg2
import shapely
from shapely.wkb import loads as wkb_loads
from geoalchemy2.types import Geometry as GEOMETRY
from geopandas import GeoDataFrame, GeoSeries
from shapely.geometry import Polygon, box, MultiPolygon
......@@ -495,7 +497,7 @@ def get_overlapping_MGRS_tiles(conn_params, scene_ID=None, tgt_corners_lonlat=No
GDF = GeoDataFrame(records, columns=['grid100k', 'grid1mil', 'wkb_hex'])
GDF['shapelyPoly_LonLat'] = list(GDF['wkb_hex'].map(lambda wkb_hex: shapely.wkb.loads(wkb_hex, hex=True)))
GDF['shapelyPoly_LonLat'] = list(GDF['wkb_hex'].map(lambda wkb_hex: wkb_loads(wkb_hex, hex=True)))
GDF['granuleid'] = GDF['grid1mil'].str.cat(GDF['grid100k'])
return GDF[['granuleid', 'shapelyPoly_LonLat']]
......@@ -522,7 +524,7 @@ def get_overlapping_MGRS_tiles2(conn_params, scene_ID=None, tgt_corners_lonlat=N
GDF = GeoDataFrame(records, columns=['granuleid', 'wkb_hex'])
GDF['shapelyPoly_LonLat'] = list(GDF['wkb_hex'].map(lambda wkb_hex: shapely.wkb.loads(wkb_hex, hex=True)))
GDF['shapelyPoly_LonLat'] = list(GDF['wkb_hex'].map(lambda wkb_hex: wkb_loads(wkb_hex, hex=True)))
return GDF[['granuleid', 'shapelyPoly_LonLat']]
......@@ -905,7 +907,7 @@ class GMS_JOB(object):
gdf_recs['sceneid'] = list(gdf_recs.sceneid.map(lambda sceneid: int(sceneid)))
# wkb_hex bounds to shapely polygons
gdf_recs['polygons'] = list(gdf_recs.geom.map(lambda wkb_hex: shapely.wkb.loads(wkb_hex, hex=True)))
gdf_recs['polygons'] = list(gdf_recs.geom.map(lambda wkb_hex: wkb_loads(wkb_hex, hex=True)))
all_gdf_recs.append(gdf_recs)
all_gdf_miss.append(gdf_miss)
......@@ -950,7 +952,7 @@ class GMS_JOB(object):
self.exists_in_db = True
[setattr(self, attrName, res[0][i]) for i, attrName in enumerate(self.jobs_table_columns)]
self.bounds = shapely.wkb.loads(self.bounds, hex=True)
self.bounds = wkb_loads(self.bounds, hex=True)
# fill self.dataframe
records = get_info_from_postgreSQLdb(self.conn, 'scenes', ['satelliteid', 'sensorid', 'filename',
......@@ -968,7 +970,7 @@ class GMS_JOB(object):
id_senName_dict = dict(zip(all_senIDs, all_senNames))
gdf.insert(0, 'satellite', list(gdf.satelliteid.map(lambda satID: id_satName_dict[satID])))
gdf.insert(1, 'sensor', list(gdf.sensorid.map(lambda senID: id_senName_dict[senID])))
gdf['polygons'] = list(gdf.geom.map(lambda wkb_hex: shapely.wkb.loads(wkb_hex, hex=True)))
gdf['polygons'] = list(gdf.geom.map(lambda wkb_hex: wkb_loads(wkb_hex, hex=True)))
self.dataframe = gdf[['satellite', 'sensor', 'filename', 'sceneid', 'acquisitiondate', 'geom', 'polygons']]
......@@ -1010,7 +1012,7 @@ class GMS_JOB(object):
gdf_recs['sceneid'] = list(gdf_recs.sceneid.map(lambda sceneid: int(sceneid)))
# wkb_hex bounds to shapely polygons
gdf_recs['polygons'] = list(gdf_recs.geom.map(lambda wkb_hex: shapely.wkb.loads(wkb_hex, hex=True)))
gdf_recs['polygons'] = list(gdf_recs.geom.map(lambda wkb_hex: wkb_loads(wkb_hex, hex=True)))
return gdf_recs, gdf_miss
......@@ -1209,7 +1211,7 @@ def add_missing_filenames_in_pgSQLdb(conn_params): # FIXME
def pdDataFrame_to_sql_k(engine, frame, name, if_exists='fail', index=True,
index_label=None, schema=None, chunksize=None, dtype=None, **kwargs):
# type: (any,pd.DataFrame,str,str,bool,str,str,int,dict,dict) -> None
# type: (any,pd.DataFrame,str,str,bool,str,str,int,dict,any) -> None
"""Extends the standard function pandas.io.SQLDatabase.to_sql() with 'kwargs' which allows to set the primary key
of the target table for example. This is usually not possible with the standard to_sql() function.
......@@ -1224,17 +1226,17 @@ def pdDataFrame_to_sql_k(engine, frame, name, if_exists='fail', index=True,
If None, all rows will be written at once.
:param dtype: <dict> a dictionary of column names and corresponding postgreSQL types
The types should be a SQLAlchemy or GeoSQLAlchemy2 type,
:param kwargs: {'keys',...}
:param kwargs: keyword arguments to be passed to SQLTable
"""
pandas_sql = pd.io.sql.pandasSQL_builder(engine, schema=None, flavor=None)
pandas_sql = pandasSQL_builder(engine, schema=None, flavor=None)
if dtype is not None:
for col, my_type in dtype.items():
if not isinstance(to_instance(my_type), TypeEngine):
raise ValueError('The type of %s is not a SQLAlchemy type ' % col)
table = pd.io.sql.SQLTable(name, pandas_sql, frame=frame, index=index, if_exists=if_exists, index_label=index_label,
schema=schema, dtype=dtype, **kwargs)
table = SQLTable(name, pandas_sql, frame=frame, index=index, if_exists=if_exists, index_label=index_label,
schema=schema, dtype=dtype, **kwargs)
table.create()
table.insert(chunksize)
......@@ -1393,16 +1395,11 @@ def SQL_DB_to_csv():
connection = sqlite3.connect(CFG.job.path_database)
cursor = connection.cursor()
cursor.execute("SELECT * FROM processed_data")
if sys.version_info[0] < 3: # Python 2.x.x
with open(os.path.join(os.path.dirname(CFG.job.path_database), 'data_DB.csv'), 'wb') as csvfile:
csvwriter = csv.writer(csvfile)
csvwriter.writerow([i[0] for i in cursor.description])
csvwriter.writerows(cursor)
else:
with open(os.path.join(os.path.dirname(CFG.job.path_database), 'data_DB.csv'), 'w') as csvfile:
csvwriter = csv.writer(csvfile)
csvwriter.writerow([i[0] for i in cursor.description])
csvwriter.writerows(cursor)
with open(os.path.join(os.path.dirname(CFG.job.path_database), 'data_DB.csv'), 'w' if PY3 else 'wb') as csvfile:
csvwriter = csv.writer(csvfile)
csvwriter.writerow([i[0] for i in cursor.description])
csvwriter.writerows(cursor)
def postgreSQL_table_to_csv(conn_db, path_csv, tablename):
......
......@@ -24,3 +24,4 @@ geoalchemy2
sqlalchemy
psycopg2
# fmask # not pip installable
six
......@@ -13,7 +13,7 @@ with open('HISTORY.rst') as history_file:
requirements = [
'matplotlib', 'numpy', 'scikit-learn', 'scipy', 'gdal', 'pyproj', 'shapely', 'ephem', 'pyorbital', 'dill', 'pytz',
'pandas', 'numba', 'spectral>=0.16', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy',
'psycopg2', 'py_tools_ds', 'geoarray', 'arosics'
'psycopg2', 'py_tools_ds', 'geoarray', 'arosics', 'six'
# spectral<0.16 has some problems with writing signed integer 8bit data
# fmask # conda install -c conda-forge python-fmask
# 'pyhdf', # conda install --yes -c conda-forge pyhdf
......
......@@ -18,7 +18,7 @@ RUN /bin/bash -i -c "source /root/anaconda3/bin/activate ; \
conda install --yes -c conda-forge 'icu=58.*' lxml ; \
pip install pandas geopandas dicttoxml jsmin cerberus pyprind pint iso8601 tqdm mpld3 sphinx-argparse dill pytz \
spectral>0.16 psycopg2 pyorbital pyinstrument geoalchemy2 sqlalchemy py_tools_ds geoarray arosics \
flake8 pycodestyle pylint pydocstyle nose nose2 nose-htmloutput coverage rednose" # must include all the requirements needed to build the docs!
flake8 pycodestyle pylint pydocstyle nose nose2 nose-htmloutput coverage rednose six" # must include all the requirements needed to build the docs!
# copy some needed stuff to /root
#COPY *.pkl /root/ # EXAMPLE
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment