Commit 9540dd06 authored by Daniel Eggert's avatar Daniel Eggert
Browse files

refactored credentials into properties file

parent 30f22c84
......@@ -36,6 +36,7 @@ public class MetadatacrawlerModule {
private static final String INSERT_QUERY = "INSERT INTO scenes (entityid, datasetid, acquisitiondate, starttime, endtime, browseurl, dataaccessurl, downloadurl, metadataurl, modifieddate, orderurl, summary, bounds, satelliteid, sensorid, subsystemid, proc_level, cloudcover) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'METADATA', ?);";
@SuppressWarnings("unused")
public static void main(String[] args) throws Exception {
Map<MetadataCrawler, List<DatasetMetadata>> crawlerTasks = new HashMap<>();
......
......@@ -54,8 +54,8 @@ public class SyncDatabases {
connectToDestinationDB();
short l8Dataset = 104;
short l7Dataset = 112;
// short l8Dataset = 104;
// short l7Dataset = 112;
short s2Dataset = 249;
syncScenesTableForDataset(s2Dataset);
}
......@@ -110,12 +110,11 @@ public class SyncDatabases {
con.commit();
}
}
con.commit();
}
@SuppressWarnings("unused")
private static void syncScenesSentinel2Table() throws Exception {
// get the sceneids from the source db
......
......@@ -24,14 +24,14 @@ import de.potsdam.gfz.gms.metadatacrawler.model.SceneMetadata;
*/
public class GeoMultiSensMetadataCrawler extends MetadataCrawler {
private static final String DB_COLUMN_LABEL_DATASET_ID = "id";
private static final String DB_COLUMN_LABEL_DATASET_NAME = "name";
private static final String DB_COLUMN_LABEL_DATASET_DESC = "description";
private static final String DB_COLUMN_LABEL_DATASET_STARTDATE = "startdate";
private static final String DB_COLUMN_LABEL_DATASET_ENDDATE = "enddate";
private static final String DB_COLUMN_LABEL_DATASET_BOUNDS = "bounds";
private static final String DB_COLUMN_LABEL_DATASET_IMAGE_TYPE = "image_type";
private static final String DB_COLUMN_LABEL_DATASET_TOTALSCENES = "totalscenes";
private static final String DB_COLUMN_LABEL_DATASET_ID = "id";
private static final String DB_COLUMN_LABEL_DATASET_NAME = "name";
private static final String DB_COLUMN_LABEL_DATASET_DESC = "description";
private static final String DB_COLUMN_LABEL_DATASET_STARTDATE = "startdate";
private static final String DB_COLUMN_LABEL_DATASET_ENDDATE = "enddate";
private static final String DB_COLUMN_LABEL_DATASET_BOUNDS = "bounds";
private static final String DB_COLUMN_LABEL_DATASET_IMAGE_TYPE = "image_type";
private static final String DB_COLUMN_LABEL_DATASET_TOTALSCENES = "totalscenes";
private static final String DB_COLUMN_LABEL_SCENE_ID = "id";
private static final String DB_COLUMN_LABEL_SCENE_ENTITYID = "entityid";
......@@ -56,6 +56,16 @@ public class GeoMultiSensMetadataCrawler extends MetadataCrawler {
@SuppressWarnings("unused")
private static final String DB_COLUMN_LABEL_SCENE_PROC_LEVEL = "proc_level";
/*
* (non-Javadoc)
*
* @see de.potsdam.gfz.gms.metadatacrawler.crawler.MetadataCrawler#getProviderName()
*/
@Override
protected String getProviderName() {
return "gms";
}
/*
* (non-Javadoc)
*
......
......@@ -3,7 +3,11 @@
*/
package de.potsdam.gfz.gms.metadatacrawler.crawler;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.BlockingQueue;
import de.potsdam.gfz.gms.database.shared.Timestamp;
......@@ -17,9 +21,16 @@ import de.potsdam.gfz.gms.metadatacrawler.model.Sentinel2SceneMetadata;
*/
public abstract class MetadataCrawler {
private boolean running = false;
private boolean canceled = false;
private CrawlTask crawlTask = null;
private static final String FILENAME_CREDENTIALS_PROPTERTIES = "credentials.properties";
private static final String PROPS_KEY_USERNAME = ".username";
private static final String PROPS_KEY_PASSWORD = ".password";
private static final String PROPS_DEFAULT_VALUE_USERNAME = "gms_downloader";
private static final String PROPS_DEFAULT_VALUE_PASSWORD = "gms_downloader";
private boolean running = false;
private boolean canceled = false;
private CrawlTask crawlTask = null;
private class CrawlTask implements Runnable {
......@@ -227,4 +238,55 @@ public abstract class MetadataCrawler {
*/
protected abstract void crawlScenes(DatasetMetadata dataset, Timestamp start, Timestamp end, BlockingQueue<SceneMetadata> resultQueue);
protected abstract String getProviderName();
/**
* @return
*/
private static Properties loadCredentials() {
Properties props = new Properties();
try {
InputStream propsStream = MetadataCrawler.class.getResourceAsStream("/" + FILENAME_CREDENTIALS_PROPTERTIES);
if (propsStream == null) {
// not found on classpath - check root folder
File propsFile = new File(FILENAME_CREDENTIALS_PROPTERTIES);
if (!propsFile.exists()) {
System.err.println(FILENAME_CREDENTIALS_PROPTERTIES + " neither found on classpath nor on current directory. ");
return props;
}
propsStream = new FileInputStream(propsFile);
}
props.load(propsStream);
} catch (Throwable t) {
System.err.println("Error parsing Propertiesfile: " + FILENAME_CREDENTIALS_PROPTERTIES + "\t " + t.getMessage());
}
return props;
}
protected String[] getCredentials() {
Properties props = loadCredentials();
String[] credentials = new String[] { PROPS_DEFAULT_VALUE_USERNAME, PROPS_DEFAULT_VALUE_PASSWORD };
String provider = getProviderName();
String keyUsername = provider + PROPS_KEY_USERNAME;
String keyPassword = provider + PROPS_KEY_PASSWORD;
if (props.containsKey(keyUsername)) {
credentials[0] = props.getProperty(keyUsername, PROPS_DEFAULT_VALUE_USERNAME);
}
if (props.containsKey(keyPassword)) {
credentials[1] = props.getProperty(keyPassword, PROPS_DEFAULT_VALUE_PASSWORD);
}
return credentials;
}
}
......@@ -22,7 +22,6 @@ import de.potsdam.gfz.gms.database.shared.Timestamp;
import de.potsdam.gfz.gms.metadatacrawler.model.DatasetMetadata;
import de.potsdam.gfz.gms.metadatacrawler.model.SceneMetadata;
import de.potsdam.gfz.gms.metadatacrawler.model.Sentinel2SceneMetadata;
import de.potsdam.gfz.scihubapi.SciHubCredentials;
import de.potsdam.gfz.scihubapi.opensearch.SciHubSearch;
import de.potsdam.gfz.scihubapi.opensearch.SearchQuery;
import de.potsdam.gfz.scihubapi.opensearch.SearchResultEntry;
......@@ -35,7 +34,7 @@ public class ScientificDataHubCrawler extends MetadataCrawler {
// private static final Logger LOG = LoggerFactory.getLogger(ScientificDataHubCrawler.class);
private final SciHubSearch sciHubSearchApi = new SciHubSearch(SciHubCredentials.USERNAME, SciHubCredentials.PASSWORD);
private final SciHubSearch sciHubSearchApi;
// private static final String ENTRY_FIELD_NAME_ID = "id";
// private static final String ENTRY_FIELD_NAME_IDENTIFIER = "identifier";
......@@ -47,6 +46,24 @@ public class ScientificDataHubCrawler extends MetadataCrawler {
// private static final String user = "gmsdb";
// private static final String password = "gmsdb";
/**
*
*/
public ScientificDataHubCrawler() {
String[] credentials = getCredentials();
sciHubSearchApi = new SciHubSearch(credentials[0], credentials[1]);
}
/*
* (non-Javadoc)
*
* @see de.potsdam.gfz.gms.metadatacrawler.crawler.MetadataCrawler#getProviderName()
*/
@Override
protected String getProviderName() {
return "scihub";
}
@Override
protected void crawlScenes(DatasetMetadata dataset, Timestamp start, Timestamp end, BlockingQueue<SceneMetadata> resultQueue) {
while (sciHubSearchApi.isRunning()) {
......
......@@ -16,7 +16,6 @@ import org.postgis.Polygon;
import de.potsdam.gfz.gms.database.shared.Timestamp;
import de.potsdam.gfz.gms.metadatacrawler.model.DatasetMetadata;
import de.potsdam.gfz.gms.metadatacrawler.model.SceneMetadata;
import de.potsdam.gfz.usgsapi.UsgsCredentials;
import de.potsdam.gfz.usgsapi.json.ErrorCode;
import de.potsdam.gfz.usgsapi.json.UsgsApi;
import de.potsdam.gfz.usgsapi.json.datamodels.Bounds;
......@@ -38,12 +37,12 @@ import de.potsdam.gfz.usgsapi.json.response.SceneSearchResponse;
*/
public class UsgsCrawler extends MetadataCrawler {
private String apiKey = null;
private long apiKeyExpireTime = 0l;
private String apiKey = null;
private long apiKeyExpireTime = 0l;
private static final long APIKEY_EXPIRE_GRACE = 5 * 60 * 1000; // 5 min
private static final long APIKEY_EXPIRE_GRACE = 5 * 60 * 1000; // 5 min
private static final int MAX_RESULTS_PER_QUERY = 1000;
private static final int MAX_RESULTS_PER_QUERY = 1000;
/*
* (non-Javadoc)
......@@ -233,6 +232,16 @@ public class UsgsCrawler extends MetadataCrawler {
return true;
}
/*
* (non-Javadoc)
*
* @see de.potsdam.gfz.gms.metadatacrawler.crawler.MetadataCrawler#getProviderName()
*/
@Override
protected String getProviderName() {
return "usgs";
}
/**
* Returns <code>true</code> in we are still logged in or the login attempt was successful, <code>false</code> otherwise
*
......@@ -253,7 +262,9 @@ public class UsgsCrawler extends MetadataCrawler {
boolean error = false;
LoginResponse response = null;
try {
response = UsgsApi.sendRequest(new LoginRequest(UsgsCredentials.USERNAME, UsgsCredentials.PASSWORD));
String[] credentials = getCredentials();
response = UsgsApi.sendRequest(new LoginRequest(credentials[0], credentials[1]));
error = (response == null || response.hasError());
} catch (Exception e) {
System.err.println(e.getMessage());
......@@ -278,7 +289,7 @@ public class UsgsCrawler extends MetadataCrawler {
if (!ensureLogin()) {
// unable to login - return empty list
System.err.println("login error");
return new ArrayList<DatasetMetadata>(0);
return new ArrayList<>(0);
}
try {
......@@ -312,7 +323,7 @@ public class UsgsCrawler extends MetadataCrawler {
return datasets;
} catch (Exception e) {
e.printStackTrace();
return new ArrayList<DatasetMetadata>(0);
return new ArrayList<>(0);
}
}
......
# usgs credentials
usgs.username = eggert
usgs.password = |R8o.Yeop3#
# nasa credentials
nasa.username = gms_downloader
nasa.password = AJOucCitedAcce0
# esa/scihub credentials
scihub.username = gms_downloader
scihub.password = gms_downloader
\ No newline at end of file
......@@ -19,7 +19,6 @@ import org.postgis.PGgeometry;
import de.potsdam.gfz.gms.database.misc.DatabaseConfigConstants;
import de.potsdam.gfz.gms.database.shared.Timestamp;
import de.potsdam.gfz.gms.database.shared.Timestamp.TimeUnits;
import de.potsdam.gfz.scihubapi.SciHubCredentials;
import de.potsdam.gfz.scihubapi.opendata.SciHubOpenData;
/**
......@@ -64,7 +63,10 @@ public class InsertS2GranulesIntoSceneDB {
private static final String url = "jdbc:postgresql://" + host + "/geomultisens";
private static final String user = "gmsdb";
private static final String password = "gmsdb";
private static final SciHubOpenData sciHubData = new SciHubOpenData(SciHubCredentials.USERNAME, SciHubCredentials.PASSWORD);
private static final SciHubOpenData sciHubData = new SciHubOpenData("username", "password"); // TODO:
// set
// actual
// credentials
private static final Random RAND = new Random();
......
......@@ -7,7 +7,6 @@ import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import de.potsdam.gfz.gms.database.shared.Timestamp;
import de.potsdam.gfz.scihubapi.SciHubCredentials;
import de.potsdam.gfz.scihubapi.opensearch.SciHubSearch;
import de.potsdam.gfz.scihubapi.opensearch.SearchQuery;
import de.potsdam.gfz.scihubapi.opensearch.SearchResultEntry;
......@@ -16,12 +15,12 @@ public class ScihubApiTester {
public static void main(String[] args) throws Exception {
// init api
SciHubSearch scihubSearch = new SciHubSearch(SciHubCredentials.USERNAME, SciHubCredentials.PASSWORD);
SciHubSearch scihubSearch = new SciHubSearch("username", "password");
// define search query
SearchQuery query = new SearchQuery();
query.addParameter(SearchQuery.KEYWORD_BEGINPOSITION, "[ " + new Timestamp(2016, 9, 27, 6, 40).toISO8601String()
+ " TO " + new Timestamp(2016, 9, 27, 7,58).toISO8601String() + "]");
query.addParameter(SearchQuery.KEYWORD_BEGINPOSITION,
"[ " + new Timestamp(2016, 9, 27, 6, 40).toISO8601String() + " TO " + new Timestamp(2016, 9, 27, 7, 58).toISO8601String() + "]");
query.addParameter(SearchQuery.KEYWORD_PLATFORMNAME, "Sentinel-2");
// query.addParameter(SearchQuery.KEYWORD_FOOTPRINT, "\"Intersects(52.407703, 12.159007)\"");
......@@ -41,24 +40,24 @@ public class ScihubApiTester {
public int compare(SearchResultEntry o1, SearchResultEntry o2) {
Instant t1 = (Instant) o1.getFieldValue("beginposition");
Instant t2 = (Instant) o2.getFieldValue("beginposition");
if (t1.isBefore(t2) ) {
if (t1.isBefore(t2)) {
return -1;
} else if(t1.isAfter(t2)) {
} else if (t1.isAfter(t2)) {
return 1;
} else {
return 0;
}
}
});
for (SearchResultEntry entry : resultList) {
// printFields(entry);
System.out.println(entry.getFieldValue("id") + " | " + entry.getFieldValue("beginposition") + " | "
+ entry.getFieldValue("size"));
System.out.println(entry.getFieldValue("id") + " | " + entry.getFieldValue("beginposition") + " | " + entry.getFieldValue("size"));
}
}
@SuppressWarnings("unused")
private static void printFields(SearchResultEntry entry) {
for (String field : entry.getFields()) {
System.out.println(field + ":\t" + entry.getFieldValue(field));
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment