Commit b1734c90 authored by Daniel Eggert's avatar Daniel Eggert
Browse files

added verbose logging output

parent a7715017
......@@ -114,10 +114,17 @@ public class MetadatacrawlerModule {
SceneDatabase db = SceneDatabase.getInstance();
PreparedStatement pst = db.prepareCustomStatement(INSERT_QUERY);
System.out.println("\n********** Crawling metadata **********");
System.out.println("dataset: " + dataset.name);
System.out.println("crawler: " + crawler.getClass().getSimpleName());
System.out.println("query latest only: " + queryLatestOnly);
System.out.print("getting existing scenes for dataset: ");
Timestamp latestDateInDB = new Timestamp(Timestamp.MIN);
Set<String> dbScenes = getExistingScenes(dataset, latestDateInDB);
System.out.println(latestDateInDB.toISO8601String());
System.out.println("done (number of scenes in DB:" + dbScenes.size() + ")");
System.out.println("latest scene timestamp in DB: " + latestDateInDB.toISO8601String());
BlockingQueue<SceneMetadata> queue = new LinkedBlockingQueue<>();
......@@ -134,6 +141,9 @@ public class MetadatacrawlerModule {
start = new Timestamp(startOverride);
}
System.out.println("query date range: " + start.toISO8601String() + " TO " + end.toISO8601String());
System.out.println();
crawler.startCrawling(start, end, queue, dataset);
int count = 0;
......@@ -194,7 +204,7 @@ public class MetadatacrawlerModule {
pst.setDouble(++i, scene.cloudcover);
pst.executeUpdate();
if (++count % 100 == 0) {
if (++count % 1000 == 0) {
System.out.println("inserted " + count + " scenes into database");
db.commit();
}
......@@ -205,15 +215,20 @@ public class MetadatacrawlerModule {
// update dataset count
ResultSet rs = db.placeCustomQuery("SELECT count(id) from scenes where datasetid = " + dataset.id + ";");
int totalScenes = 0;
if (rs.next()) {
int totalScenes = rs.getInt(1);
totalScenes = rs.getInt(1);
db.prepareCustomStatement("UPDATE datasets SET totalscenes = " + totalScenes + " where id = " + dataset.id + ";").executeUpdate();
db.commit();
}
System.out.println("\ndataset: " + dataset.name);
System.out.println("crawler: " + crawler.getClass().getSimpleName());
System.out.println("total scenes in DB: " + totalScenes);
System.out.println("********** Crawling finished **********\n");
}
private static void printProvidedDatasets(MetadataCrawler crawler) {
public static void printProvidedDatasets(MetadataCrawler crawler) {
// get data from gms db
MetadataCrawler gmsCrawler = new GeoMultiSensMetadataCrawler();
// build dataset name map
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment