Commit 664a213c authored by Daniel Eggert's avatar Daniel Eggert
Browse files

moved all testing stuff to gms-testing

parent aff31122
/**
*
*/
package de.potsdam.gfz.gms.database;
import java.io.File;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import org.postgis.PGgeometry;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
/**
* @author Daniel Eggert (daniel.eggert@gfz-potsdam.de)
*
*/
public class CheckSentinel2Footprints {
private static final String host = "geoms.gfz-potsdam.de";
private static final String url = "jdbc:postgresql://" + host + "/geomultisens";
private static final String user = "gmsdb";
private static final String password = "gmsdb";
private static final File FOOTPRINTS_KML = new File("/home/eggi/Downloads/S2A_OPER_GIP_TILPAR_MPC__20151209T095117_V20150622T000000_21000101T000000_B00.kml");
/**
* @param args
* @throws Exception
*/
public static void main(String[] args) throws Exception {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
DocumentBuilder builder = factory.newDocumentBuilder();
Document doc = builder.parse(FOOTPRINTS_KML);
NodeList nl = doc.getElementsByTagName("Placemark");
// load the driver
Class.forName("org.postgresql.Driver");
// establish connection
Connection con = DriverManager.getConnection(url, user, password);
// register postgis geometry object
((org.postgresql.PGConnection) con).addDataType("geometry", PGgeometry.class);
((org.postgresql.PGConnection) con).addDataType("geography", PGgeometry.class);
PreparedStatement pst = con.prepareStatement("SELECT count(id) from footprints_sentinel2_granules where granuleid=?;");
System.out.println("num results: " + nl.getLength());
for (int i = 0; i < nl.getLength(); ++i) {
Element e = (Element) nl.item(i);
String code = ((Element) e.getElementsByTagName("name").item(0)).getTextContent();
pst.setString(1, code);
ResultSet rs = pst.executeQuery();
if (rs.next()) {
if (rs.getInt(1) == 0) {
System.out.println(code + " is missing in db");
}
}
}
}
}
/**
*
*/
package de.potsdam.gfz.gms.database;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
/**
* @author Daniel Eggert (daniel.eggert@gfz-potsdam.de)
*
*/
public class CleanDuplicateEntries {
/**
* @param args
* @throws Exception
*/
public static void main(String[] args) throws Exception {
SceneDatabase db = SceneDatabase.getInstance();
ResultSet rs = db.placeCustomQuery("select id, entityid from scenes where datasetid=112 order by id");
Set<String> scenes = new HashSet<>();
List<Long> toDelete = new ArrayList<>();
while (rs.next()) {
long id = rs.getLong(1);
String entityid = rs.getString(2);
if (scenes.contains(entityid)) {
toDelete.add(id);
} else {
scenes.add(entityid);
}
}
System.out.println("toDelete: " + toDelete.size());
if (!toDelete.isEmpty()) {
System.out.println(" sample id: " + toDelete.get(0));
}
PreparedStatement deleteStatement = db.prepareCustomStatement("delete from scenes where id=?");
for (long id : toDelete) {
deleteStatement.setLong(1, id);
if (deleteStatement.executeUpdate() != 1) {
System.err.println("deleted less or more than 1 row");
return;
}
db.commit();
}
System.out.println("done.");
}
}
package de.potsdam.gfz.gms.database;
import java.nio.ByteBuffer;
import java.util.Arrays;
import de.potsdam.gfz.gms.database.shared.Timestamp;
public class CloudCoverCheck {
@SuppressWarnings("unused")
public static void main(String[] args) throws Exception {
// landsat_tm(108), landsat_mss(107), landsat_8(104), landsat_7(112)
// short datasetids[] = new short[] { 108, 107, 104, 112 };
Timestamp start = new Timestamp(2009, 12, 31);
Timestamp end = new Timestamp(2015, 1, 1);
// Timestamp end = start.plus(20, Timestamp.TimeUnits.DAYS);
int seasonCode = 0;
byte minCloudcover = 0;
byte maxCloudcover = 100;
double[] polygon = new double[] { -14, 32, 50, 32, 50, 64, -14, 64, -14, 32 };
final ByteBuffer buffer = ByteBuffer.allocate(100 * 1000 * 1000);
final SceneDatabase db = SceneDatabase.getInstance();
short datasetid = 112;
buffer.clear();
db.getCompactSceneMetaDataListFromIndex(datasetid, start, end, seasonCode, minCloudcover, maxCloudcover, polygon, buffer);
buffer.flip();
/*
* Decode as follows: [short] datasetid [int] number of scenes list of scenes, with each scene: [int] sceneid [long] timestamp [byte] cloudcover
*/
short returnedDatasetid = buffer.getShort();
if (returnedDatasetid != datasetid) {
System.err.println("dataset ids dont match");
System.exit(1);
}
int numScenes = buffer.getInt();
System.out.println("processing dataset " + datasetid + " with " + numScenes);
int[] hist = new int[101];
byte[] vals = new byte[numScenes];
for (int i = 0; i < numScenes; ++i) {
final int id = buffer.getInt();
// skip timestamp and cloud cover
buffer.getLong();
byte indexCC = buffer.get();
vals[i] = indexCC;
++hist[indexCC];
}
Arrays.sort(vals);
System.out.println("median: " + vals[vals.length / 2]);
long sum = 0;
for (int i = 0; i < hist.length; ++i) {
sum += hist[i];
System.out.println(i + "\t" + hist[i] + "\t" + sum + " (" + ((byte) (100 * ((double) sum) / numScenes)) + ")");
}
}
}
/**
*
*/
package de.potsdam.gfz.gms.database;
import java.io.File;
import java.sql.PreparedStatement;
import java.util.Scanner;
/**
* @author Daniel Eggert (daniel.eggert@gfz-potsdam.de)
*
*/
public class CloudCoverNaNUpdate {
/**
* @param args
* @throws Exception
*/
public static void main(String[] args) throws Exception {
Scanner s = new Scanner(new File("failed.txt"));
SceneDatabase db = SceneDatabase.getInstance();
PreparedStatement pst = db.prepareCustomStatement("UPDATE scenes SET cloudcover = 'NaN' WHERE id=?;");
while (s.hasNext()) {
String[] lineSplit = s.nextLine().split("\\s");
if (lineSplit.length > 2 && lineSplit[2].equals("NaN")) {
// NaN value logged
long id = Long.parseLong(lineSplit[0]);
// update db
pst.setLong(1, id);
pst.executeUpdate();
db.commit();
}
}
s.close();
}
}
package de.potsdam.gfz.gms.database;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathFactory;
import org.w3c.dom.Document;
public class CloudCoverUpdate {
private static class Scene {
long id;
double cloudcover;
public Scene(long id, double cloudcover) {
super();
this.id = id;
this.cloudcover = cloudcover;
}
}
private static final Scene FINAL_SCENE = new Scene(-1, -1);
public static void main(String[] args) throws Exception {
// landsat_tm(108), landsat_mss(107), landsat_8(104), landsat_7(112)
short datasetids[] = new short[] { 104, 112 }; // , 108, 107, 112 };
// Timestamp start = Timestamp.MIN;
// Timestamp end = Timestamp.now();
// Timestamp end = start.plus(20, Timestamp.TimeUnits.DAYS);
// int seasonCode = 0;
// byte minCloudcover = 0;
// byte maxCloudcover = 100;
// double[] polygon = new double[] { -14, 32, 50, 32, 50, 64, -14, 64, -14, 32 };
// double[] polygon = new double[] { -180, -90, 180, -90, 180, 90, -180, 90, -180, -90 };
// final ByteBuffer buffer = ByteBuffer.allocate(200 * 1000 * 1000);
final SceneDatabase db = SceneDatabase.getInstance();
ExecutorService pool = Executors.newFixedThreadPool(20);
final BlockingQueue<Scene> sceneQueue = new LinkedBlockingQueue<>();
pool.execute(new Runnable() {
@Override
public void run() {
int count = 0;
PreparedStatement pst = db.prepareCustomStatement("UPDATE scenes SET cloudcover=? WHERE id=?;");
while (true) {
Scene s;
try {
s = sceneQueue.take();
if (s == FINAL_SCENE) {
return;
}
pst.setDouble(1, s.cloudcover);
pst.setLong(2, s.id);
// db.placeCustomQuery("UPDATE scenes SET cloudcover=" + s.cloudcover + " WHERE id=" + s.id + " returning 0;");
pst.executeUpdate();
db.commit();
if (++count % 100 == 0) {
System.out.println(count + " scenes updated");
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
});
BufferedWriter writer = new BufferedWriter(new FileWriter("failed.txt"));
PreparedStatement pst = db.prepareCustomStatement("SELECT id, metadataurl FROM scenes WHERE datasetid=? AND cloudcover<0;");
for (short datasetid : datasetids) {
// buffer.clear();
// db.getCompactSceneMetaDataListFromIndex(datasetid, start, end, seasonCode, minCloudcover, maxCloudcover, polygon, buffer);
// buffer.flip();
/*
* Decode as follows: [short] datasetid [int] number of scenes list of scenes, with each scene: [int] sceneid [long] timestamp [byte] cloudcover
*/
// short returnedDatasetid = buffer.getShort();
// if (returnedDatasetid != datasetid) {
// System.err.println("dataset ids dont match");
// System.exit(1);
// }
// int numScenes = buffer.getInt();
final Random rand = new Random();
System.out.println("processing dataset " + datasetid);
List<Future<?>> fList = new ArrayList<>();
pst.setShort(1, datasetid);
ResultSet rs = pst.executeQuery();
while (rs.next()) {
try {
final long id = rs.getLong(1);
final String urlStr = rs.getString(2);
Future<?> f = pool.submit(new Runnable() {
@Override
public void run() {
InputStream stream = null;
HttpURLConnection con = null;
try {
// random sleep
Thread.sleep(50 + rand.nextInt(150));
// establish connection
URL url = new URL(urlStr);
con = (HttpURLConnection) url.openConnection();
con.setConnectTimeout(5000);
int code = 0;
// check response code
try {
con.connect();
code = con.getResponseCode();
} catch (Exception e) {
}
if (code != 200) {
// connection error - backoff and retry
// random sleep
Thread.sleep(15000l + rand.nextInt(10000));
// re-establish connection
url = new URL(urlStr);
con = (HttpURLConnection) url.openConnection();
if (con.getResponseCode() != 200) {
// connection error - again - skip scene
System.err.println("Connection error: " + con.getResponseCode() + " - " + con.getResponseMessage());
writer.write(id + "\t" + url + "\t" + con.getResponseMessage() + "\n");
writer.flush();
con.disconnect();
return;
}
}
// connection successfully established - open stream
try {
stream = url.openStream();
} catch (Exception e) {
// stream error - backoff and retry
// random sleep
Thread.sleep(15000l + rand.nextInt(10000));
// re-establish connection
url = new URL(urlStr);
con = (HttpURLConnection) url.openConnection();
if (con.getResponseCode() != 200) {
System.err.println("Connection error after stream error: " + con.getResponseCode() + " - " + con.getResponseMessage());
writer.write(id + "\t" + url + "\t" + con.getResponseMessage() + "\n");
writer.flush();
if (stream != null) {
stream.close();
}
return;
}
try {
stream = url.openStream();
} catch (Exception ex) {
System.err.println("Stream error: " + con.getResponseCode() + " - " + con.getResponseMessage() + "\t" + ex.getMessage());
writer.write(id + "\t" + url + "\t" + con.getResponseMessage() + "\n");
writer.flush();
if (stream != null) {
stream.close();
}
con.disconnect();
return;
}
}
final DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
final XPath xpath = XPathFactory.newInstance().newXPath();
final XPathExpression expr = xpath.compile("//*[contains(@name,'Cloud Cover')]/*/text()");
Document doc = builder.parse(stream);
double cc = (Double) expr.evaluate(doc, XPathConstants.NUMBER);
if (cc < 0) {
cc = Double.NaN;
}
sceneQueue.add(new Scene(id, cc));
} catch (Exception e) {
e.printStackTrace();
}
if (stream != null) {
try {
stream.close();
} catch (IOException e) {
}
}
if (con != null) {
con.disconnect();
}
}
});
fList.add(f);
} catch (SQLException e) {
e.printStackTrace();
System.exit(1);
}
}
// wait for tasks to finish
for (Future<?> f : fList) {
f.get();
}
fList.clear();
}
sceneQueue.add(FINAL_SCENE);
// wait for pool to shut down
pool.shutdown();
pool.awaitTermination(1, TimeUnit.HOURS);
writer.close();
}
}
/**
*
*/
package de.potsdam.gfz.gms.database;
import java.util.Map;
import java.util.Map.Entry;
import org.junit.Test;
/**
* @author Daniel Eggert (daniel.eggert@gfz-potsdam.de)
*
*/
public class ConfigTest {
@Test
public void test() {
Map<String, String> conf = SceneDatabase.getInstance().getConfig();
for (Entry<String, String> e : conf.entrySet()) {
System.out.println(e.getKey() + ": " + e.getValue());
}
}
}
/**
*
*/
package de.potsdam.gfz.gms.database;
import java.util.Map.Entry;
import de.potsdam.gfz.gms.database.shared.Enums.proc_level;
import de.potsdam.gfz.gms.database.shared.SceneStateJobStats;
/**
* @author Daniel Eggert (daniel.eggert@gfz-potsdam.de)
*
*/
public class CustomStatementTest {
public static void main(String[] args) throws Exception {
SceneDatabase db = SceneDatabase.getInstance();
SceneStateJobStats<proc_level> stats = db.getProcLevelForJobScenes(26185619);
System.out.println("############# " + stats.jobid + " #############\n");
for (short datasetid : stats.getDatasets()) {
System.out.println("********* " + datasetid + " *********");
for (Entry<Long, proc_level> e : stats.getSceneStates(datasetid).entrySet()) {
System.out.println(e.getKey() + "\t" + e.getValue());
}
}
}
}
/**
*
*/
package de.potsdam.gfz.gms.database;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;