CSIP_NASS_LMOD.java [src/java/methods] Revision: default Date:
/*
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
package methods;
import csip.SessionLogger;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.net.URISyntaxException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.List;
import java.util.LinkedList;
import java.util.LinkedHashMap;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.FutureTask;
import lamps.CSIP_Const;
import lamps.utils.Partition;
import methods.objects.ERU;
import methods.objects.Nass_results;
import csip.utils.Numeric;
/**
*
* @author hokipka
*/
public class CSIP_NASS_LMOD {
/**
*
* @param outputDir
* @param hrus
* @param nass_data_result
* @throws IOException
* @throws URISyntaxException
*/
public static void NASS_data_check(File outputDir, List<ERU> hrus, HashMap<Integer, ArrayList<Nass_results>> nass_data_result, SessionLogger LOG) throws IOException, URISyntaxException {
//PrintWriter w = new PrintWriter(new FileWriter(new File(outputDir, CSIP_Const.NASS_dominant_result)));
//w_log.println("===> NASS_dominant_result");
LOG.info("===> NASS_dominant_result");
//w_log.flush();
List<String> records = new ArrayList<>();
records.add("# All results are based on NASS CropScape dominant crops");
records.add("Field-ID or HRU-ID, NASS CropScape Year, Dominant Vegetation or Crop, Confidence-Value(fraction) , Secondary Crop, Secondary Crop Confidence-Value(fraction), Irrigated, Irrigated Area %");
for (int i = 0; i < hrus.size(); i++) {
for (int j = 0; j < hrus.get(i).nass_list_data.size(); j++) {
String irrigated = "No";
double conf = hrus.get(i).nass_list_data.get(j).getcrop_confidence()[0];
if (hrus.get(i).irrigated) {
irrigated = "Yes";
}
double irri_area_portion = Math.round((hrus.get(i).irrigated_area * 100) / 100) > 100 ? 100 : Math.round(hrus.get(i).irrigated_area * 100) / 100;
String nd = "";
double confi2 = 0;
if (hrus.get(i).nass_list_data.get(j).getcrop_nass_names().length > 1) {
nd = hrus.get(i).nass_list_data.get(j).getcrop_nass_names()[1];
confi2 = hrus.get(i).nass_list_data.get(j).getcrop_confidence()[1];
}
if (Integer.parseInt(hrus.get(i).nass_list_data.get(j).getyear()) == 2017) {
records.add(hrus.get(i).ID + "," + hrus.get(i).nass_list_data.get(j).getyear() + "," + hrus.get(i).nass_list_data.get(j).getcrop_nass_names()[0] + "," + Double.toString(Numeric.round(conf, 3)) + "," + nd + "," + Double.toString(Numeric.round(confi2, 3)) + "," + irrigated + "," + String.valueOf(irri_area_portion));
} else {
records.add(hrus.get(i).ID + "," + hrus.get(i).nass_list_data.get(j).getyear() + "," + hrus.get(i).nass_list_data.get(j).getcrop_nass_names()[0] + "," + Double.toString(Numeric.round(conf, 3)) + "," + nd + "," + Double.toString(Numeric.round(confi2, 3)));
}
}
}
try (FileWriter writer = new FileWriter(new File(outputDir, CSIP_Const.AOI_NASS_dominant_result))) {
BufferedWriter buffWriter = new BufferedWriter(writer);
for (String record : records) {
buffWriter.write(record + "\r\n");
//writer.write(record);
}
//writer.flush();
buffWriter.close();
}
records.clear();
records.add("# All results are based on NASS CropScape crops");
records.add("Field-ID or HRU-ID, NASS CropScape Year, Vegetation or Crop, Confidence-Value(fraction)");
for (int a = 0; a < hrus.size(); a++) {
for (int j = 0; j < hrus.get(a).nass_list_data.size(); j++) {
for (int x = 0; x < hrus.get(a).nass_list_data.get(j).getcrop_nass_names().length; x++) {
records.add(hrus.get(a).ID + "," + hrus.get(a).nass_list_data.get(j).getyear() + "," + hrus.get(a).nass_list_data.get(j).getcrop_nass_names()[x] + "," + Double.toString(Numeric.round((hrus.get(a).nass_list_data.get(j).getcrop_confidence()[x]), 3)));
}
}
}
try (FileWriter writer = new FileWriter(new File(outputDir, CSIP_Const.AOI_LAMPS_CONF_result))) {
BufferedWriter buffWriter = new BufferedWriter(writer);
for (String record : records) {
buffWriter.write(record + "\r\n");
//writer.write(record);
}
//writer.flush();
buffWriter.close();
}
//w.close();
}
/**
*
* @param dataDir
* @param hrus
* @param delta
* @param LOG
* @throws IOException
* @throws URISyntaxException
* @throws SQLException
* @throws ClassNotFoundException
* @throws java.lang.InterruptedException
* @throws java.util.concurrent.ExecutionException
*/
public static void NASS_data_CropSeq(File dataDir, List<ERU> hrus, double delta, SessionLogger LOG) throws IOException, URISyntaxException, SQLException, ClassNotFoundException, InterruptedException, ExecutionException {
Map<String, String> dictonary_map = new HashMap<>();
Class.forName("org.h2.Driver");
try (Connection conn = DriverManager.getConnection("jdbc:h2:file:" + dataDir.getPath() + CSIP_Const.LocalDB, "sa", "")) {
String queryString = "select NASSNAME,ACRONYMNEW from \"PUBLIC\".dictionary";
try (Statement state = conn.createStatement()) {
ResultSet result = state.executeQuery(queryString);
while (result.next()) {
dictonary_map.put(result.getString("NASSNAME"), result.getString("ACRONYMNEW"));
}
state.close();
}
conn.close();
}
LOG.info(" done querying nass name acronym from DB");
hrus = ConcurrentConfidence(dictonary_map, hrus, delta, LOG);
}
/**
*
* @param dictonary_map
* @param hrus
* @param delta
* @param LOG
* @return
* @throws InterruptedException
* @throws ExecutionException
*/
public static List<ERU> ConcurrentConfidence(Map<String, String> dictonary_map, List<ERU> hrus, double delta, SessionLogger LOG) throws InterruptedException, ExecutionException {
final int threadNum = Partition.getThreadCount(hrus.size());;
final int first = 0;
final int last = hrus.size() - 1;
final List<ERU> fhrus = hrus;
final Map<String, String> fdictonary_map = dictonary_map;
final double fdelta = delta;
final SessionLogger fSL = LOG;
final int p = (last + 1) / threadNum;
int s = (last + 1) - p * threadNum;
List<ERU> allHRUs = new ArrayList();
// Prepare to execute and store the Futures
ExecutorService executor = Executors.newFixedThreadPool(threadNum);
List<FutureTask<List<ERU>>> taskList = new ArrayList<>();
for (int j = 0; j < threadNum; j++) {
final int ii = j;
FutureTask<List<ERU>> futureTask_1 = new FutureTask<>(new Callable<List<ERU>>() {
int inlast = (ii == threadNum - 1 ? last : p + (ii * p));
int infirst = (ii == 0 ? first : (p * ii) + 1);
@Override
public List<ERU> call() throws Exception {
return CSIP_NASS_LMOD.Confidence(infirst, inlast, fhrus, fdelta, fdictonary_map, ii, fSL);
}
});
taskList.add(futureTask_1);
executor.execute(futureTask_1);
}
// Wait until all results are available and combine them at the same time
for (int j = 0; j < threadNum; j++) {
FutureTask<List<ERU>> futureTask = taskList.get(j);
for (int i = 0; i < futureTask.get().size(); i++) {
allHRUs.add(futureTask.get().get(i));
}
}
executor.shutdown();
return allHRUs;
}
public static List<ERU> Confidence(int first, int last, List<ERU> hrus, double delta, Map<String, String> dictonary_map, int part, SessionLogger LOG) {
int count = 0;
List<ERU> allHRUs = new ArrayList();
int part_all = last - first;
long start_time = System.currentTimeMillis();
//w_log.println(" first : " + first + " last : " + last);
//w_log.flush();
for (int i = first; i <= last; i++) {
count += 1;
if (count == 1000) {
count = 0;
double progress = ((double) i - first) / ((double) part_all);
progress = (double) progress * 100.00;
//w_log.println(part + ".-" + ((int) progress) + "%...");
LOG.info(part + ".-" + ((int) progress) + "%...");
//w_log.flush();
}
//w_log.println(" HRU_ID: " + i);
//w_log.flush();
String hru_seq = "";
String hru_veg_seq = "";
List<String[]> list = new ArrayList();
int all_non_fieldcrop_count = 0;
int pot_weeds_count = 0;
int acc_crops_count = 0;
List<String> nodata_test = new ArrayList();
for (int j = 0; j < hrus.get(i).nass_list_data.size(); j++) {
//w_log.println(" [0]: " + hrus.get(i).nass_list_data.get(j).getcrop_nass_names()[0]);
//w_log.flush();
if (hrus.get(i).nass_list_data.get(j).getcrop_nass_names()[0].equals("Pasture/Grass")) {
hrus.get(i).nass_list_data.get(j).getcrop_nass_names()[0] = "Grass/Pasture";
}
if (hrus.get(i).nass_list_data.get(j).getcrop_nass_names()[0].equals("Forest")) {
hrus.get(i).nass_list_data.get(j).getcrop_nass_names()[0] = "Deciduous Forest";
}
hru_seq = hru_seq + dictonary_map.get(hrus.get(i).nass_list_data.get(j).getcrop_nass_names()[0]);
if (j < 1) {
hru_veg_seq = hrus.get(i).nass_list_data.get(j).getcrop_nass_names()[0];
} else {
hru_veg_seq = hru_veg_seq + ";" + hrus.get(i).nass_list_data.get(j).getcrop_nass_names()[0];
}
//if (Acc_years.contains(hrus.get(i).nass_list_data.get(j).getyear())) {
if (!hrus.get(i).nass_list_data.get(j).getcrop_nass_names()[0].contains("NODATA")) {
if (CSIP_Const.NonFieldCropClasses.contains(hrus.get(i).nass_list_data.get(j).getcrop_nass_names()[0])) {
if (CSIP_Const.potential_weeds.contains(hrus.get(i).nass_list_data.get(j).getcrop_nass_names()[0])) {
pot_weeds_count++;
} else {
all_non_fieldcrop_count++;
}
} else {
acc_crops_count++;
//System.out.println(" Field crop : " + hrus.get(i).nass_list_data.get(j).getcrop_nass_names()[0] + " year: " + j);
}
} else if (!nodata_test.contains(hrus.get(i).nass_list_data.get(j).getcrop_nass_names()[0])) {
nodata_test.add(hrus.get(i).nass_list_data.get(j).getcrop_nass_names()[0]);
all_non_fieldcrop_count++;
}
String[] year_crop_conf = new String[4];
year_crop_conf[0] = hrus.get(i).nass_list_data.get(j).getyear();
year_crop_conf[1] = "" + hrus.get(i).nass_list_data.get(j).getcrop()[0];
year_crop_conf[2] = "" + hrus.get(i).nass_list_data.get(j).getcrop_confidence()[0];
year_crop_conf[3] = "" + dictonary_map.get(hrus.get(i).nass_list_data.get(j).getcrop_nass_names()[0]);
list.add(year_crop_conf);
//}
}
if (all_non_fieldcrop_count == acc_crops_count && acc_crops_count > 0) {
double cheat = 1;
while (cheat > 0.51 || all_non_fieldcrop_count > 0) {
all_non_fieldcrop_count = all_non_fieldcrop_count - 1;
acc_crops_count = acc_crops_count + 1;
cheat = all_non_fieldcrop_count / acc_crops_count;
//non_field_crops_years / acc_crops_year_count < 0.51
}
//w_log.println(" equal crop/veg; acc_crop after cheat at HRU: " + hrus.get(i).ID + " crop: " + acc_crops_count + " non-crop: " + all_non_fieldcrop_count);
//w_log.flush();
}
hrus.get(i).non_field_crops_years = all_non_fieldcrop_count;
hrus.get(i).weeds_years = pot_weeds_count;
hrus.get(i).acc_crops_year_count = acc_crops_count;
//w_log.println(" HRU : " + i);
//w_log.println(" : " + hru_veg_seq);
//w_log.println(" : " + hru_seq);
// AbstractSuffixTree tree = new SimpleSuffixTree(hru_seq);
// w_log.println("Longest repeating substring "
// + tree.best.printResult() + " repetitions=" + tree.best.visits
// + " length=" + tree.best.stringDepth);
// CompactSuffixTree tree = new CompactSuffixTree(new SimpleSuffixTree(hru_seq));
// String properties = "rankdir=LR; node[shape=box fillcolor=gray95 style=filled]\n";
// System.out.println("digraph {\n" + properties + tree.root + "}");
//w_log.println(" non_field_crops_years : " + hrus.get(i).non_field_crops_years);
//w_log.println(" weeds_years : " + hrus.get(i).weeds_years);
//w_log.println(" acc_crops_year_count : " + hrus.get(i).acc_crops_year_count);
//w_log.flush();
Map<String, Double> map = new HashMap<>();
List<Double> conf = new ArrayList();
for (int j = 0; j < list.size(); j++) {
conf = new ArrayList();
if (Boolean.parseBoolean(list.get(j)[1])) {
conf.add(Double.parseDouble(list.get(j)[2]));
int hashid = Integer.parseInt(list.get(j)[0]);
String sum_hash_id = "" + hashid;
//double singlconf = (Double.parseDouble(list.get(j)[2]) * 0.1) / 100; special care
double singlconf = Double.parseDouble(list.get(j)[2]);
if (map.get(sum_hash_id) == null) {
map.put(sum_hash_id, singlconf);
//w_log.println(" sum_hash_id: " + sum_hash_id + " singlconf: " + singlconf);
//w_log.flush();
}
}
}
//w_log.println(" ");
//w_log.flush();
Map<String, Double> sorted = sortByValues(map);
double max_avg_conf = 0;
for (String key : sorted.keySet()) {
max_avg_conf = sorted.get(key);
//w_log.println(" max_avg_conf: " + max_avg_conf);
//w_log.flush();
break;
}
//w_log.println(" ");
//w_log.flush();
map = new HashMap<>();
Map<Double, String> seq_map = new HashMap<>();
Map<String, String> seq_acc = new HashMap<>();
List<String> character_combination = new ArrayList();
conf = new ArrayList();
List<Double> ac_list = new ArrayList();
for (int j = 0; j < list.size(); j++) {
conf = new ArrayList();
if (Boolean.parseBoolean(list.get(j)[1])) {
int co_years = 1;
String crop_char = list.get(j)[3];
String hru_seq_arr = crop_char;
conf.add(Double.parseDouble(list.get(j)[2]));
int hashid = Integer.parseInt(list.get(j)[0]);
String sum_hash_id = "" + hashid;
//double Rcs = (Double.parseDouble(list.get(j)[2]) * 0.1) / max_avg_conf; special care
double Rcs = (Double.parseDouble(list.get(j)[2])) / max_avg_conf;
//double Acs = Rcs + (CSIP_Const.weights.get(co_years) * CSIP_Const.Accuracy_Delta);
double Acs = Rcs + ((co_years - 1) * delta);
if (map.get(list.get(j)[0]) == null) {
if (ac_list.contains(Acs)) {
Acs = Acs + (0.000000001 * j);
}
ac_list.add(Acs);
if (!character_combination.contains(hru_seq_arr)) {
map.put(list.get(j)[0], Acs);
seq_map.put(Acs, hru_seq_arr);
seq_acc.put(hru_seq_arr, list.get(j)[0]);
//w_log.println(" Acs: " + Acs + " hru_seq_arr: " + hru_seq_arr + " list.get(j)[0]: " + list.get(j)[0]);
//w_log.flush();
character_combination.add(hru_seq_arr);
} else if (Acs > map.get(seq_acc.get(hru_seq_arr))) {
seq_map.remove(map.get(seq_acc.get(hru_seq_arr)));
map.remove(seq_acc.get(hru_seq_arr));
seq_acc.remove(hru_seq_arr);
map.put(list.get(j)[0], Acs);
seq_map.put(Acs, hru_seq_arr);
seq_acc.put(hru_seq_arr, list.get(j)[0]);
//w_log.println(" Acs: " + Acs + " hru_seq_arr: " + hru_seq_arr + " list.get(j)[0]: " + list.get(j)[0]);
//w_log.flush();
}
}
for (int k = j + 1; k < list.size(); k++) {
if (Boolean.parseBoolean(list.get(k)[1])) {
co_years++;
conf.add(Double.parseDouble(list.get(k)[2]));
double sum_conf = 0.0;
for (int l = 0; l < conf.size(); l++) {
sum_conf = sum_conf + conf.get(l);
}
double avg_confe = (sum_conf / conf.size());
sum_hash_id = sum_hash_id + "," + list.get(k)[0];
hru_seq_arr = hru_seq_arr + list.get(k)[3];
double Rc = avg_confe / max_avg_conf;
//double Ac = Rc + (CSIP_Const.weights.get(co_years) * CSIP_Const.Accuracy_Delta);
double Ac = Rc + ((co_years - 1) * delta);
if (map.get(sum_hash_id) == null) {
if (ac_list.contains(Ac)) {
Ac = Ac + (0.000000001 * k);
}
ac_list.add(Ac);
if (!character_combination.contains(hru_seq_arr)) {
map.put(sum_hash_id, Ac);
seq_map.put(Ac, hru_seq_arr);
seq_acc.put(hru_seq_arr, sum_hash_id);
character_combination.add(hru_seq_arr);
//w_log.println(" Ac: " + Ac + " hru_seq_arr: " + hru_seq_arr + " sum_hash_id: " + sum_hash_id);
//w_log.flush();
} else if (Ac > map.get(seq_acc.get(hru_seq_arr))) {
seq_map.remove(map.get(seq_acc.get(hru_seq_arr)));
map.remove(seq_acc.get(hru_seq_arr));
seq_acc.remove(hru_seq_arr);
map.put(sum_hash_id, Ac);
seq_map.put(Ac, hru_seq_arr);
seq_acc.put(hru_seq_arr, sum_hash_id);
//w_log.println(" Ac: " + Ac + " hru_seq_arr: " + hru_seq_arr + " sum_hash_id: " + sum_hash_id);
//w_log.flush();
}
}
} else {
sum_hash_id = "" + hashid;
hru_seq_arr = crop_char;
conf = new ArrayList();
co_years = 1;
break;
}
}
}
}
//w_log.println(" : " + hru_veg_seq);
//w_log.println(" : " + hru_seq);
for (String charcom : character_combination) {
List<Integer> inner_matches = new CSIP_NASS_LMOD_Matching().Match(charcom, hru_seq);
int ind = 0;
for (Integer inner_int : inner_matches) {
ind++;
}
//w_log.println(" char_com : " + charcom + " count : " + ind);
//w_log.flush();
}
Map<String, Double> sorted2 = sortByValues(map);
max_avg_conf = 0;
for (String key : sorted2.keySet()) {
max_avg_conf = sorted2.get(key);
//w_log.println(" max_avg_conf: " + max_avg_conf);
//w_log.flush();
break;
}
//w_log.println(" ");
//w_log.flush();
hrus.get(i).hru_seq_org = hru_seq;
hrus.get(i).hru_seq_work = hru_seq;
hrus.get(i).hru_veg_seq = hru_veg_seq;
hrus.get(i).seq_map = seq_map;
hrus.get(i).sorted_adjusted_conf = sorted2;
allHRUs.add(hrus.get(i));
}
long end_time = System.currentTimeMillis();
double difference = (end_time - start_time) / 1000;
//w_log.println(" Done first : " + first + " last : " + last + " time : " + difference + " s");
LOG.info(" Done first : " + first + " last : " + last + " time : " + difference + " s");
//w_log.flush();
return allHRUs;
}
/*
* Java method to sort Map in Java by value e.g. HashMap or Hashtable
* throw NullPointerException if Map contains null values
* It also sort values even if they are duplicates
*/
/**
*
* @param <K>
* @param <V>
* @param map
* @return
*/
public static <K extends Comparable, V extends Comparable> Map<K, V> sortByValues(Map<K, V> map) {
List<Map.Entry<K, V>> entries = new LinkedList<>(map.entrySet());
Collections.sort(entries, new Comparator<Map.Entry<K, V>>() {
@Override
public int compare(Entry<K, V> o1, Entry<K, V> o2) {
return o2.getValue().compareTo(o1.getValue());
}
});
//LinkedHashMap will keep the keys in the order they are inserted
//which is currently sorted on natural ordering
Map<K, V> sortedMap = new LinkedHashMap<>();
for (Map.Entry<K, V> entry : entries) {
sortedMap.put(entry.getKey(), entry.getValue());
}
return sortedMap;
}
}