Commit a36b8e9e authored by Daniel Eggert's avatar Daniel Eggert
Browse files

L8 bulk update

parent 80d2d5ce
package de.potsdam.gfz.gms.metadatacrawler;
import java.nio.file.Paths;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.HashSet;
import java.util.HashMap;
import java.util.Map;
import java.util.Scanner;
import java.util.Set;
import de.potsdam.gfz.gms.database.SceneDatabase;
public class LandsatCollection1CsvUpdate {
// browseAvailable,browseURL,sceneID,LANDSAT_PRODUCT_ID,sensor,acquisitionDate,dateUpdated,path,row,upperLeftCornerLatitude,upperLeftCornerLongitude,upperRightCornerLatitude,upperRightCornerLongitude,lowerLeftCornerLatitude,lowerLeftCornerLongitude,lowerRightCornerLatitude,lowerRightCornerLongitude,sceneCenterLatitude,sceneCenterLongitude,cloudCover,cloudCoverFull,dayOrNight,sunElevation,sunAzimuth,receivingStation,sceneStartTime,sceneStopTime,imageQuality1,DATA_TYPE_L1,cartURL,ROLL_ANGLE,GEOMETRIC_RMSE_MODEL,GEOMETRIC_RMSE_MODEL_X,GEOMETRIC_RMSE_MODEL_Y,FULL_PARTIAL_SCENE,NADIR_OFFNADIR,PROCESSING_SOFTWARE_VERSION,CPF_NAME,RLUT_FILE_NAME,BPF_NAME_OLI,BPF_NAME_TIRS,GROUND_CONTROL_POINTS_MODEL,GROUND_CONTROL_POINTS_VERSION,DATE_L1_GENERATED,TIRS_SSM_MODEL,COLLECTION_NUMBER,COLLECTION_CATEGORY,CLOUD_COVER_LAND
private static class Scene {
long id;
String entityId;
Scene(long id, String entityid) {
this.id = id;
this.entityId = entityid;
}
}
// browseAvailable,browseURL,sceneID,LANDSAT_PRODUCT_ID,sensor,acquisitionDate,dateUpdated,path,row,upperLeftCornerLatitude,upperLeftCornerLongitude,upperRightCornerLatitude,upperRightCornerLongitude,lowerLeftCornerLatitude,lowerLeftCornerLongitude,lowerRightCornerLatitude,lowerRightCornerLongitude,sceneCenterLatitude,sceneCenterLongitude,cloudCover,cloudCoverFull,dayOrNight,sunElevation,sunAzimuth,receivingStation,sceneStartTime,sceneStopTime,imageQuality1,DATA_TYPE_L1,cartURL,ROLL_ANGLE,GEOMETRIC_RMSE_MODEL,GEOMETRIC_RMSE_MODEL_X,GEOMETRIC_RMSE_MODEL_Y,FULL_PARTIAL_SCENE,NADIR_OFFNADIR,PROCESSING_SOFTWARE_VERSION,CPF_NAME,RLUT_FILE_NAME,BPF_NAME_OLI,BPF_NAME_TIRS,GROUND_CONTROL_POINTS_MODEL,GROUND_CONTROL_POINTS_VERSION,DATE_L1_GENERATED,TIRS_SSM_MODEL,COLLECTION_NUMBER,COLLECTION_CATEGORY,CLOUD_COVER_LAND
public static void main(String[] args) throws Throwable {
// get all entityids for L8C1 from database as well as from the csv file
SceneDatabase db = SceneDatabase.getInstance();
ResultSet rs = db.placeCustomQuery("select entityid from scenes where datasetid=250;");
PreparedStatement updatePst = db.prepareCustomStatement("update scenes set entityid=? where id=?;");
ResultSet rs = db.placeCustomQuery(
"select id, entityid from scenes where datasetid=250 and proc_level<'DOWNLOADED'::proc_level;");
Set<String> dbEntityIds = new HashSet<>();
Map<String, Scene> dbEntityIds = new HashMap<>();
while (rs.next()) {
String entityid = rs.getString(1);
dbEntityIds.add(entityid.substring(0, entityid.length() - 2));
long id = rs.getLong(1);
String entityid = rs.getString(2);
dbEntityIds.put(entityid.substring(0, entityid.length() - 2), new Scene(id, entityid));
}
System.out.println(dbEntityIds.size() + " scenes in db");
// Set<String> csvEntityIds = new HashSet<>();
int matchCount = 0;
int noMatchCount = 0;
Scanner s = new Scanner(Paths.get("/home/eggert/Downloads/LANDSAT_8_C1.csv"));
// skip header
......@@ -41,24 +50,31 @@ public class LandsatCollection1CsvUpdate {
String[] lineSplit = s.nextLine().split(",");
String entityid = lineSplit[2];
entityid = entityid.substring(0, entityid.length() - 2);
if (dbEntityIds.contains(entityid)) {
// match found
dbEntityIds.remove(entityid);
++matchCount;
} else {
// csvEntityIds.add(entityid);
++noMatchCount;
String mapKey = entityid.substring(0, entityid.length() - 2);
if (dbEntityIds.containsKey(mapKey)) {
// match found - check entire entityId
Scene scene = dbEntityIds.remove(mapKey);
if (!scene.entityId.equals(entityid)) {
// entityids differ in last two digits - update db
updatePst.setLong(2, scene.id);
updatePst.setString(1, entityid);
updatePst.executeUpdate();
// db.commit();
}
if (dbEntityIds.size() % 1000 == 0) {
db.commit();
System.out.println(dbEntityIds.size() + " left.");
}
}
}
s.close();
db.commit();
System.out.println(dbEntityIds.size() + " unmatched scenes in db");
System.out.println(noMatchCount + " unmatched scenes in csv");
System.out.println(matchCount + " matching scenes in db and csv");
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment