Commit 435b56ec authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Some code improvements.

Former-commit-id: ee0b0225
Former-commit-id: 7d262cd7
parent eb6ff977
...@@ -12,10 +12,12 @@ import warnings ...@@ -12,10 +12,12 @@ import warnings
from datetime import datetime from datetime import datetime
from typing import Union # noqa F401 # flake8 issue from typing import Union # noqa F401 # flake8 issue
from six import PY3
import numpy as np import numpy as np
import pandas as pd import pandas as pd
from pandas.io.sql import pandasSQL_builder, SQLTable
import psycopg2 import psycopg2
import shapely from shapely.wkb import loads as wkb_loads
from geoalchemy2.types import Geometry as GEOMETRY from geoalchemy2.types import Geometry as GEOMETRY
from geopandas import GeoDataFrame, GeoSeries from geopandas import GeoDataFrame, GeoSeries
from shapely.geometry import Polygon, box, MultiPolygon from shapely.geometry import Polygon, box, MultiPolygon
...@@ -495,7 +497,7 @@ def get_overlapping_MGRS_tiles(conn_params, scene_ID=None, tgt_corners_lonlat=No ...@@ -495,7 +497,7 @@ def get_overlapping_MGRS_tiles(conn_params, scene_ID=None, tgt_corners_lonlat=No
GDF = GeoDataFrame(records, columns=['grid100k', 'grid1mil', 'wkb_hex']) GDF = GeoDataFrame(records, columns=['grid100k', 'grid1mil', 'wkb_hex'])
GDF['shapelyPoly_LonLat'] = list(GDF['wkb_hex'].map(lambda wkb_hex: shapely.wkb.loads(wkb_hex, hex=True))) GDF['shapelyPoly_LonLat'] = list(GDF['wkb_hex'].map(lambda wkb_hex: wkb_loads(wkb_hex, hex=True)))
GDF['granuleid'] = GDF['grid1mil'].str.cat(GDF['grid100k']) GDF['granuleid'] = GDF['grid1mil'].str.cat(GDF['grid100k'])
return GDF[['granuleid', 'shapelyPoly_LonLat']] return GDF[['granuleid', 'shapelyPoly_LonLat']]
...@@ -522,7 +524,7 @@ def get_overlapping_MGRS_tiles2(conn_params, scene_ID=None, tgt_corners_lonlat=N ...@@ -522,7 +524,7 @@ def get_overlapping_MGRS_tiles2(conn_params, scene_ID=None, tgt_corners_lonlat=N
GDF = GeoDataFrame(records, columns=['granuleid', 'wkb_hex']) GDF = GeoDataFrame(records, columns=['granuleid', 'wkb_hex'])
GDF['shapelyPoly_LonLat'] = list(GDF['wkb_hex'].map(lambda wkb_hex: shapely.wkb.loads(wkb_hex, hex=True))) GDF['shapelyPoly_LonLat'] = list(GDF['wkb_hex'].map(lambda wkb_hex: wkb_loads(wkb_hex, hex=True)))
return GDF[['granuleid', 'shapelyPoly_LonLat']] return GDF[['granuleid', 'shapelyPoly_LonLat']]
...@@ -905,7 +907,7 @@ class GMS_JOB(object): ...@@ -905,7 +907,7 @@ class GMS_JOB(object):
gdf_recs['sceneid'] = list(gdf_recs.sceneid.map(lambda sceneid: int(sceneid))) gdf_recs['sceneid'] = list(gdf_recs.sceneid.map(lambda sceneid: int(sceneid)))
# wkb_hex bounds to shapely polygons # wkb_hex bounds to shapely polygons
gdf_recs['polygons'] = list(gdf_recs.geom.map(lambda wkb_hex: shapely.wkb.loads(wkb_hex, hex=True))) gdf_recs['polygons'] = list(gdf_recs.geom.map(lambda wkb_hex: wkb_loads(wkb_hex, hex=True)))
all_gdf_recs.append(gdf_recs) all_gdf_recs.append(gdf_recs)
all_gdf_miss.append(gdf_miss) all_gdf_miss.append(gdf_miss)
...@@ -950,7 +952,7 @@ class GMS_JOB(object): ...@@ -950,7 +952,7 @@ class GMS_JOB(object):
self.exists_in_db = True self.exists_in_db = True
[setattr(self, attrName, res[0][i]) for i, attrName in enumerate(self.jobs_table_columns)] [setattr(self, attrName, res[0][i]) for i, attrName in enumerate(self.jobs_table_columns)]
self.bounds = shapely.wkb.loads(self.bounds, hex=True) self.bounds = wkb_loads(self.bounds, hex=True)
# fill self.dataframe # fill self.dataframe
records = get_info_from_postgreSQLdb(self.conn, 'scenes', ['satelliteid', 'sensorid', 'filename', records = get_info_from_postgreSQLdb(self.conn, 'scenes', ['satelliteid', 'sensorid', 'filename',
...@@ -968,7 +970,7 @@ class GMS_JOB(object): ...@@ -968,7 +970,7 @@ class GMS_JOB(object):
id_senName_dict = dict(zip(all_senIDs, all_senNames)) id_senName_dict = dict(zip(all_senIDs, all_senNames))
gdf.insert(0, 'satellite', list(gdf.satelliteid.map(lambda satID: id_satName_dict[satID]))) gdf.insert(0, 'satellite', list(gdf.satelliteid.map(lambda satID: id_satName_dict[satID])))
gdf.insert(1, 'sensor', list(gdf.sensorid.map(lambda senID: id_senName_dict[senID]))) gdf.insert(1, 'sensor', list(gdf.sensorid.map(lambda senID: id_senName_dict[senID])))
gdf['polygons'] = list(gdf.geom.map(lambda wkb_hex: shapely.wkb.loads(wkb_hex, hex=True))) gdf['polygons'] = list(gdf.geom.map(lambda wkb_hex: wkb_loads(wkb_hex, hex=True)))
self.dataframe = gdf[['satellite', 'sensor', 'filename', 'sceneid', 'acquisitiondate', 'geom', 'polygons']] self.dataframe = gdf[['satellite', 'sensor', 'filename', 'sceneid', 'acquisitiondate', 'geom', 'polygons']]
...@@ -1010,7 +1012,7 @@ class GMS_JOB(object): ...@@ -1010,7 +1012,7 @@ class GMS_JOB(object):
gdf_recs['sceneid'] = list(gdf_recs.sceneid.map(lambda sceneid: int(sceneid))) gdf_recs['sceneid'] = list(gdf_recs.sceneid.map(lambda sceneid: int(sceneid)))
# wkb_hex bounds to shapely polygons # wkb_hex bounds to shapely polygons
gdf_recs['polygons'] = list(gdf_recs.geom.map(lambda wkb_hex: shapely.wkb.loads(wkb_hex, hex=True))) gdf_recs['polygons'] = list(gdf_recs.geom.map(lambda wkb_hex: wkb_loads(wkb_hex, hex=True)))
return gdf_recs, gdf_miss return gdf_recs, gdf_miss
...@@ -1209,7 +1211,7 @@ def add_missing_filenames_in_pgSQLdb(conn_params): # FIXME ...@@ -1209,7 +1211,7 @@ def add_missing_filenames_in_pgSQLdb(conn_params): # FIXME
def pdDataFrame_to_sql_k(engine, frame, name, if_exists='fail', index=True, def pdDataFrame_to_sql_k(engine, frame, name, if_exists='fail', index=True,
index_label=None, schema=None, chunksize=None, dtype=None, **kwargs): index_label=None, schema=None, chunksize=None, dtype=None, **kwargs):
# type: (any,pd.DataFrame,str,str,bool,str,str,int,dict,dict) -> None # type: (any,pd.DataFrame,str,str,bool,str,str,int,dict,any) -> None
"""Extends the standard function pandas.io.SQLDatabase.to_sql() with 'kwargs' which allows to set the primary key """Extends the standard function pandas.io.SQLDatabase.to_sql() with 'kwargs' which allows to set the primary key
of the target table for example. This is usually not possible with the standard to_sql() function. of the target table for example. This is usually not possible with the standard to_sql() function.
...@@ -1224,17 +1226,17 @@ def pdDataFrame_to_sql_k(engine, frame, name, if_exists='fail', index=True, ...@@ -1224,17 +1226,17 @@ def pdDataFrame_to_sql_k(engine, frame, name, if_exists='fail', index=True,
If None, all rows will be written at once. If None, all rows will be written at once.
:param dtype: <dict> a dictionary of column names and corresponding postgreSQL types :param dtype: <dict> a dictionary of column names and corresponding postgreSQL types
The types should be a SQLAlchemy or GeoSQLAlchemy2 type, The types should be a SQLAlchemy or GeoSQLAlchemy2 type,
:param kwargs: {'keys',...} :param kwargs: keyword arguments to be passed to SQLTable
""" """
pandas_sql = pd.io.sql.pandasSQL_builder(engine, schema=None, flavor=None) pandas_sql = pandasSQL_builder(engine, schema=None, flavor=None)
if dtype is not None: if dtype is not None:
for col, my_type in dtype.items(): for col, my_type in dtype.items():
if not isinstance(to_instance(my_type), TypeEngine): if not isinstance(to_instance(my_type), TypeEngine):
raise ValueError('The type of %s is not a SQLAlchemy type ' % col) raise ValueError('The type of %s is not a SQLAlchemy type ' % col)
table = pd.io.sql.SQLTable(name, pandas_sql, frame=frame, index=index, if_exists=if_exists, index_label=index_label, table = SQLTable(name, pandas_sql, frame=frame, index=index, if_exists=if_exists, index_label=index_label,
schema=schema, dtype=dtype, **kwargs) schema=schema, dtype=dtype, **kwargs)
table.create() table.create()
table.insert(chunksize) table.insert(chunksize)
...@@ -1393,16 +1395,11 @@ def SQL_DB_to_csv(): ...@@ -1393,16 +1395,11 @@ def SQL_DB_to_csv():
connection = sqlite3.connect(CFG.job.path_database) connection = sqlite3.connect(CFG.job.path_database)
cursor = connection.cursor() cursor = connection.cursor()
cursor.execute("SELECT * FROM processed_data") cursor.execute("SELECT * FROM processed_data")
if sys.version_info[0] < 3: # Python 2.x.x
with open(os.path.join(os.path.dirname(CFG.job.path_database), 'data_DB.csv'), 'wb') as csvfile: with open(os.path.join(os.path.dirname(CFG.job.path_database), 'data_DB.csv'), 'w' if PY3 else 'wb') as csvfile:
csvwriter = csv.writer(csvfile) csvwriter = csv.writer(csvfile)
csvwriter.writerow([i[0] for i in cursor.description]) csvwriter.writerow([i[0] for i in cursor.description])
csvwriter.writerows(cursor) csvwriter.writerows(cursor)
else:
with open(os.path.join(os.path.dirname(CFG.job.path_database), 'data_DB.csv'), 'w') as csvfile:
csvwriter = csv.writer(csvfile)
csvwriter.writerow([i[0] for i in cursor.description])
csvwriter.writerows(cursor)
def postgreSQL_table_to_csv(conn_db, path_csv, tablename): def postgreSQL_table_to_csv(conn_db, path_csv, tablename):
......
...@@ -24,3 +24,4 @@ geoalchemy2 ...@@ -24,3 +24,4 @@ geoalchemy2
sqlalchemy sqlalchemy
psycopg2 psycopg2
# fmask # not pip installable # fmask # not pip installable
six
...@@ -13,7 +13,7 @@ with open('HISTORY.rst') as history_file: ...@@ -13,7 +13,7 @@ with open('HISTORY.rst') as history_file:
requirements = [ requirements = [
'matplotlib', 'numpy', 'scikit-learn', 'scipy', 'gdal', 'pyproj', 'shapely', 'ephem', 'pyorbital', 'dill', 'pytz', 'matplotlib', 'numpy', 'scikit-learn', 'scipy', 'gdal', 'pyproj', 'shapely', 'ephem', 'pyorbital', 'dill', 'pytz',
'pandas', 'numba', 'spectral>=0.16', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy', 'pandas', 'numba', 'spectral>=0.16', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy',
'psycopg2', 'py_tools_ds', 'geoarray', 'arosics' 'psycopg2', 'py_tools_ds', 'geoarray', 'arosics', 'six'
# spectral<0.16 has some problems with writing signed integer 8bit data # spectral<0.16 has some problems with writing signed integer 8bit data
# fmask # conda install -c conda-forge python-fmask # fmask # conda install -c conda-forge python-fmask
# 'pyhdf', # conda install --yes -c conda-forge pyhdf # 'pyhdf', # conda install --yes -c conda-forge pyhdf
......
...@@ -18,7 +18,7 @@ RUN /bin/bash -i -c "source /root/anaconda3/bin/activate ; \ ...@@ -18,7 +18,7 @@ RUN /bin/bash -i -c "source /root/anaconda3/bin/activate ; \
conda install --yes -c conda-forge 'icu=58.*' lxml ; \ conda install --yes -c conda-forge 'icu=58.*' lxml ; \
pip install pandas geopandas dicttoxml jsmin cerberus pyprind pint iso8601 tqdm mpld3 sphinx-argparse dill pytz \ pip install pandas geopandas dicttoxml jsmin cerberus pyprind pint iso8601 tqdm mpld3 sphinx-argparse dill pytz \
spectral>0.16 psycopg2 pyorbital pyinstrument geoalchemy2 sqlalchemy py_tools_ds geoarray arosics \ spectral>0.16 psycopg2 pyorbital pyinstrument geoalchemy2 sqlalchemy py_tools_ds geoarray arosics \
flake8 pycodestyle pylint pydocstyle nose nose2 nose-htmloutput coverage rednose" # must include all the requirements needed to build the docs! flake8 pycodestyle pylint pydocstyle nose nose2 nose-htmloutput coverage rednose six" # must include all the requirements needed to build the docs!
# copy some needed stuff to /root # copy some needed stuff to /root
#COPY *.pkl /root/ # EXAMPLE #COPY *.pkl /root/ # EXAMPLE
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment