Commit f5acdcd2 authored by Daniel Eggert's avatar Daniel Eggert
Browse files

merged syncdatabases impl

parent 55b22a10
......@@ -7,7 +7,10 @@ import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import org.postgis.PGgeometry;
......@@ -21,17 +24,17 @@ import de.potsdam.gfz.gms.database.SceneDatabase;
public class SyncDatabases {
@SuppressWarnings("unused")
private static final String INSERT_SCENE_QUERY = "INSERT INTO scenes (entityid, datasetid, acquisitiondate, starttime, endtime, browseurl, dataaccessurl, downloadurl, metadataurl, modifieddate, orderurl, summary, bounds, satelliteid, sensorid, subsystemid, proc_level, cloudcover) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'METADATA', ?);";
private static final String INSERT_SCENE_QUERY = "INSERT INTO scenes (entityid, datasetid, acquisitiondate, starttime, endtime, browseurl, dataaccessurl, downloadurl, metadataurl, modifieddate, orderurl, summary, bounds, satelliteid, sensorid, subsystemid, proc_level, cloudcover) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'METADATA', ?);";
private static final String host = "141.20.27.70";
private static final String url = "jdbc:postgresql://" + host + "/geomultisens";
private static final String user = "gmsdb";
private static final String password = "gmsdb";
private static final String host = "141.20.27.70";
private static final String url = "jdbc:postgresql://" + host + "/geomultisens";
private static final String user = "gmsdb";
private static final String password = "gmsdb";
// source db
private static final SceneDatabase gfzDB = SceneDatabase.getInstance();
private static final SceneDatabase gfzDB = SceneDatabase.getInstance();
// destination db
private static Connection con = null;
private static Connection con = null;
private static void connectToDestinationDB() throws Exception {
// establish connection to second database (hu)
......@@ -54,46 +57,85 @@ public class SyncDatabases {
connectToDestinationDB();
<<<<<<< HEAD
=======
// short l8Dataset = 104;
// short l7Dataset = 112;
>>>>>>> 55b22a10b5881af249c0472c315f352776958996
short s2Dataset = 249;
syncScenesTableForDataset(s2Dataset);
short l8Dataset = 250;
short l7Dataset = 251;
syncScenesTableForDataset(l8Dataset);
syncScenesTableForDataset(l7Dataset);
// syncScenesSentinel2Table();
// syncScenesTableForDataset(s2Dataset);
}
private static void syncScenesTableForDataset(short datasetid) throws Exception {
String idsQueryString = "select id from scenes where datasetid=" + datasetid + ";";
// get ids from the source db
ResultSet rs = gfzDB.placeCustomQuery(idsQueryString);
Set<Long> ids = new HashSet<>();
Set<Long> gfzIds = new HashSet<>();
while (rs.next()) {
long id = rs.getLong(1);
ids.add(id);
gfzIds.add(rs.getLong(1));
}
rs.close();
System.out.println("gfz: " + gfzIds.size());
// get the ids from the destination db and remove them from the
// source idset
Set<Long> huIds = new HashSet<>();
rs = con.prepareStatement(idsQueryString).executeQuery();
int huCount = 0;
while (rs.next()) {
huIds.add(rs.getLong(1));
++huCount;
gfzIds.remove(rs.getLong(1));
}
System.out.println("gfz: " + ids.size());
System.out.println("hu: " + huIds.size());
ids.removeAll(huIds);
System.out.println("delta: " + ids.size());
System.out.println("hu: " + huCount);
con.setAutoCommit(false);
PreparedStatement insertPst = con.prepareStatement("INSERT INTO scenes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::proc_level);");
// PreparedStatement updateEntityIdPst = con.prepareStatement("update scenes set entityid=? where id=?;");
// int updatedEntityIdCount = 0;
// for (Entry<Long, String> e : huIds.entrySet()) {
// long id = e.getKey();
// String huEntityId = e.getValue();
//
// if (gfzIds.containsKey(id)) {
// // we already know this one, compare entityid
// String gfzEntityId = gfzIds.remove(id);
// if (!gfzEntityId.equals(huEntityId)) {
// // update entity id
// updateEntityIdPst.setString(1, gfzEntityId);
// updateEntityIdPst.setLong(2, id);
// updateEntityIdPst.executeUpdate();
//
// if (++updatedEntityIdCount % 1000 == 0) {
// System.out.println("updated entityids: " + updatedEntityIdCount);
// con.commit();
// }
// }
//
// }
// }
System.out.println("delta: " + gfzIds.size());
PreparedStatement insertPst = con.prepareStatement(
"INSERT INTO scenes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::proc_level);");
// get missing scenes from source db (gfz)
PreparedStatement selectPst = gfzDB.prepareCustomStatement("select * from scenes where datasetid=? and id = ANY (?);");
PreparedStatement selectPst = gfzDB
.prepareCustomStatement("select * from scenes where datasetid=? and id = ANY (?);");
selectPst.setShort(1, datasetid);
selectPst.setArray(2, con.createArrayOf("bigint", ids.toArray()));
selectPst.setArray(2, con.createArrayOf("bigint", gfzIds.toArray()));
int count = 0;
int numParam = 20;
......@@ -142,20 +184,23 @@ public class SyncDatabases {
ids.removeAll(huSceneids);
System.out.println("delta: " + ids.size());
PreparedStatement insertPst = con
.prepareStatement("INSERT INTO scenes_sentinel2 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);");
PreparedStatement insertPst = con.prepareStatement(
"INSERT INTO scenes_sentinel2 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);");
// get missing scenes from source db (gfz)
PreparedStatement selectPst = gfzDB.prepareCustomStatement("select * from scenes_sentinel2 where sceneid = ANY (?);");
PreparedStatement selectPst = gfzDB
.prepareCustomStatement("select * from scenes_sentinel2 where sceneid = ANY (?);");
selectPst.setArray(1, con.createArrayOf("bigint", ids.toArray()));
int count = 0;
// int param = 0;
String[] cols = new String[] { "", "sceneid", "title", "link", "link_alternative", "link_icon", "id", "summary", "processed", "cloudcoverpercentage", "filename", "format",
"identifier", "ingestiondate", "instrumentshortname", "sensoroperationalmode", "instrumentname", "footprint", "s2datatakeid", "platformidentifier", "orbitnumber",
"orbitdirection", "platformserialidentifier", "processingbaseline", "processinglevel", "producttype", "relativeorbitnumber", "platformname", "beginposition",
"endposition", "size" };
String[] cols = new String[] { "", "sceneid", "title", "link", "link_alternative", "link_icon", "id", "summary",
"processed", "cloudcoverpercentage", "filename", "format", "identifier", "ingestiondate",
"instrumentshortname", "sensoroperationalmode", "instrumentname", "footprint", "s2datatakeid",
"platformidentifier", "orbitnumber", "orbitdirection", "platformserialidentifier", "processingbaseline",
"processinglevel", "producttype", "relativeorbitnumber", "platformname", "beginposition", "endposition",
"size" };
ResultSet selectRS = selectPst.executeQuery();
while (selectRS.next()) {
......@@ -174,12 +219,14 @@ public class SyncDatabases {
// pst.setString(++param, s2.filename.replace(".SAFE", ".zip"));
// pst.setString(++param, s2.format);
// pst.setString(++param, s2.identifier);
// pst.setTimestamp(++param, java.sql.Timestamp.from(s2.ingestiondate));
// pst.setTimestamp(++param,
// java.sql.Timestamp.from(s2.ingestiondate));
// pst.setString(++param, s2.instrumentshortname);
// pst.setString(++param, s2.sensoroperationalmode);
// pst.setString(++param, s2.instrumentname);
// if (s2.footprint != null) {
// pst.setObject(++param, new PGgeometry(new Polygon(s2.footprint.toText())));
// pst.setObject(++param, new PGgeometry(new
// Polygon(s2.footprint.toText())));
// } else {
// System.err.println(s2.id + ": null footprint");
// pst.setNull(++param, Types.OTHER);
......@@ -194,8 +241,10 @@ public class SyncDatabases {
// pst.setString(++param, s2.producttype);
// pst.setInt(++param, s2.relativeorbitnumber);
// pst.setString(++param, s2.platformname);
// pst.setTimestamp(++param, java.sql.Timestamp.from(s2.beginposition));
// pst.setTimestamp(++param, java.sql.Timestamp.from(s2.endposition));
// pst.setTimestamp(++param,
// java.sql.Timestamp.from(s2.beginposition));
// pst.setTimestamp(++param,
// java.sql.Timestamp.from(s2.endposition));
// pst.setString(++param, s2.size);
insertPst.executeUpdate();
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment