Extract.java [src/java/util] Revision: default Date:
/*
* Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
* Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template
*/
package util;
import csip.utils.TextParser;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.DoubleSummaryStatistics;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
/**
*
* @author od
*/
public class Extract {
public static int[] extractCSVColumnAsInt(File file, String name) throws IOException {
TextParser p = new TextParser(file).autoClose(false);
String[] h = p.nextLine().split(",").asStringArray();
int idx = -1;
for (int i = 0; i < h.length; i++) {
if (h[i].equalsIgnoreCase(name)) {
idx = i;
break;
}
}
if (idx == -1)
throw new IllegalArgumentException("Not found: " + name);
// p.nextLine(1);
List<Integer> l = new ArrayList<>();
while (p.nextLineSkipEmpty().notEOF()) {
String[] row = p.split(",").asStringArray();
l.add(Integer.valueOf(row[idx]));
}
p.close();
return l.stream().mapToInt(Integer::intValue).toArray();
}
public static class Summary {
// Lists of annual values
List<Double> tot_contrib_area = new ArrayList<>();
List<Double> tot_precip_vol = new ArrayList<>();
List<Double> tot_irr_vol = new ArrayList<>();
List<Double> tot_water_discharge = new ArrayList<>();
List<Double> tot_hs_sloss = new ArrayList<>();
List<Double> tot_channel_sloss = new ArrayList<>();
List<Double> tot_sed_discharge = new ArrayList<>();
List<Double> sed_deliv_unit_area = new ArrayList<>();
List<Double> sed_deliv_ratio = new ArrayList<>();
List<Channel> channels;
List<Hillslope> hillslopes;
Summary(int noChannels, int noHillslopes) throws Exception {
channels = createList(Channel.class, noChannels);
hillslopes = createList(Hillslope.class, noHillslopes);
}
public DoubleSummaryStatistics getChannelSedYield(List<Integer> weIds) {
int years = tot_contrib_area.size();
List<Double> l = new ArrayList<>();
for (int y = 0; y < years; y++) {
double sum = 0.0;
for (Integer weId : weIds) {
Channel ch = channels.get(weId - 1);
sum += ch.getSed_yield().get(y);
}
l.add(sum);
}
return l.stream().mapToDouble(d -> d).summaryStatistics();
}
public DoubleSummaryStatistics getChannelSoilLoss(List<Integer> weIds) {
int years = tot_contrib_area.size();
List<Double> l = new ArrayList<>();
for (int y = 0; y < years; y++) {
double sum = 0.0;
for (int weId : weIds) {
Channel ch = channels.get(weId - 1);
sum += ch.getSoil_loss().get(y);
}
l.add(sum);
}
return l.stream().mapToDouble(d -> d).summaryStatistics();
}
public void collectChannelSoilLoss(List<Double> values, List<Integer> weIds) {
int years = tot_contrib_area.size();
for (int y = 0; y < years; y++) {
for (int weId : weIds) {
Channel ch = channels.get(weId - 1);
values.add(ch.getSoil_loss().get(y) / 1000);
}
}
}
public DoubleSummaryStatistics getHillslopeSoilLoss(List<Integer> weIds) {
int years = tot_contrib_area.size();
List<Double> l = new ArrayList<>();
for (int y = 0; y < years; y++) {
double sum = 0.0;
for (int weId : weIds) {
Hillslope hs = hillslopes.get(weId - 1);
sum += hs.getSoil_loss().get(y);
}
l.add(sum);
}
return l.stream().mapToDouble(d -> d).summaryStatistics();
}
public DoubleSummaryStatistics getHillslopeSedYield(List<Integer> weIds) {
int years = tot_contrib_area.size();
List<Double> l = new ArrayList<>();
for (int y = 0; y < years; y++) {
double sum = 0.0;
for (int weId : weIds) {
Hillslope hs = hillslopes.get(weId - 1);
sum += hs.getSed_yield().get(y);
}
l.add(sum);
}
return l.stream().mapToDouble(d -> d).summaryStatistics();
}
public List<Channel> getChannels() {
return channels;
}
public List<Hillslope> getHillslopes() {
return hillslopes;
}
void add(double tot_contrib_area,
double tot_precip_vol,
double tot_irr_vol,
double tot_water_discharge,
double tot_hs_sloss,
double tot_channel_sloss,
double tot_sed_discharge,
double sed_deliv_unit_area,
double sed_deliv_ratio) {
this.tot_contrib_area.add(tot_contrib_area);
this.tot_precip_vol.add(tot_precip_vol);
this.tot_irr_vol.add(tot_irr_vol);
this.tot_water_discharge.add(tot_water_discharge);
this.tot_hs_sloss.add(tot_hs_sloss);
this.tot_channel_sloss.add(tot_channel_sloss);
this.tot_sed_discharge.add(tot_sed_discharge);
this.sed_deliv_unit_area.add(sed_deliv_unit_area);
this.sed_deliv_ratio.add(sed_deliv_ratio);
}
public DoubleSummaryStatistics getTotContribAreaStats() {
return tot_contrib_area.stream().mapToDouble(d -> d).summaryStatistics();
}
public DoubleSummaryStatistics getTotPrecipVolStats() {
return tot_precip_vol.stream().mapToDouble(d -> d).summaryStatistics();
}
public DoubleSummaryStatistics getTotIrrVolStats() {
return tot_irr_vol.stream().mapToDouble(d -> d).summaryStatistics();
}
public DoubleSummaryStatistics getTotWaterDischargeStats() {
return tot_water_discharge.stream().mapToDouble(d -> d).summaryStatistics();
}
public DoubleSummaryStatistics getTotChannelSlossStats() {
return tot_channel_sloss.stream().mapToDouble(d -> d).summaryStatistics();
}
public DoubleSummaryStatistics getTotSedDischargeStats() {
return tot_sed_discharge.stream().mapToDouble(d -> d).summaryStatistics();
}
public DoubleSummaryStatistics getSedDelivUnitAreaStats() {
return sed_deliv_unit_area.stream().mapToDouble(d -> d).summaryStatistics();
}
public DoubleSummaryStatistics getSedDelivRatioStats() {
return sed_deliv_ratio.stream().mapToDouble(d -> d).summaryStatistics();
}
}
public static class Channel {
List<Double> disch_vol = new ArrayList<>();
List<Double> sed_yield = new ArrayList<>();
List<Double> soil_loss = new ArrayList<>();
List<Double> upland_charge = new ArrayList<>();
List<Double> ssurfflow_vol = new ArrayList<>();
void add(double disch_vol, double sed_yield, double soil_loss, double upland_charge, double ssurfflow_vol) {
this.disch_vol.add(disch_vol);
this.sed_yield.add(sed_yield);
this.soil_loss.add(soil_loss);
this.upland_charge.add(upland_charge);
this.ssurfflow_vol.add(ssurfflow_vol);
}
@Override
public String toString() {
return disch_vol + " "
+ sed_yield + " "
+ soil_loss + " "
+ upland_charge + " "
+ ssurfflow_vol;
}
public List<Double> getSed_yield() {
return sed_yield;
}
public List<Double> getSoil_loss() {
return soil_loss;
}
public DoubleSummaryStatistics getDischVolStats() {
return disch_vol.stream().mapToDouble(d -> d).summaryStatistics();
}
public DoubleSummaryStatistics getSedYieldStats() {
return sed_yield.stream().mapToDouble(d -> d).summaryStatistics();
}
public DoubleSummaryStatistics getSoilLossStats() {
return soil_loss.stream().mapToDouble(d -> d).summaryStatistics();
}
public DoubleSummaryStatistics getUplandChargeStats() {
return upland_charge.stream().mapToDouble(d -> d).summaryStatistics();
}
public DoubleSummaryStatistics getSsurfflowVolStats() {
return ssurfflow_vol.stream().mapToDouble(d -> d).summaryStatistics();
}
}
public static class Hillslope {
List<Double> runoff_vol = new ArrayList<>();
List<Double> subrunoff_vol = new ArrayList<>();
List<Double> soil_loss = new ArrayList<>();
List<Double> sed_dep = new ArrayList<>();
List<Double> sed_yield = new ArrayList<>();
void add(double runoff_vol, double subrunoff_vol, double soil_loss, double sed_dep, double sed_yield) {
this.runoff_vol.add(runoff_vol);
this.subrunoff_vol.add(subrunoff_vol);
this.soil_loss.add(soil_loss);
this.sed_dep.add(sed_dep);
this.sed_yield.add(sed_yield);
}
public List<Double> getSoil_loss() {
return soil_loss;
}
public List<Double> getSed_yield() {
return sed_yield;
}
public DoubleSummaryStatistics getRunoffVolStats() {
return runoff_vol.stream().mapToDouble(d -> d).summaryStatistics();
}
public DoubleSummaryStatistics getSubRunoffVolStats() {
return subrunoff_vol.stream().mapToDouble(d -> d).summaryStatistics();
}
public DoubleSummaryStatistics getSoilLossStats() {
return soil_loss.stream().mapToDouble(d -> d).summaryStatistics();
}
public DoubleSummaryStatistics getSedDepStats() {
return sed_dep.stream().mapToDouble(d -> d).summaryStatistics();
}
public DoubleSummaryStatistics getSedYieldStats() {
return sed_yield.stream().mapToDouble(d -> d).summaryStatistics();
}
}
// public static Summary getHillslopwStats(File sloss, int years) throws Exception {
//
//// col 5: hillslope spoil loss
//// double[] hs_sl = new double[years];
// for (int y = 1; y <= years; y++) {
//
// try (TextParser p = new TextParser(sloss)
// .autoClose(false)
// .toLineContainingAllOf(" ANNUAL SUMMARY FOR WATERSHED IN YEAR ", " " + y)
// .toLineStartingWith("Hillslopes ")
// .nextLine()) {
//
// while (p.nextLine().notEmpty()) {
// double[] row = p.split().rangeFrom(1).asDoubleArray();
//// System.out.println(Arrays.toString(row));
// ch.get((int) row[0] - 1).add(row[1], row[2], row[3], row[4], row[5]);
// }
//
// }
//
// TextParser tp = new TextParser(sloss).autoClose(false);
// tp.toLineContainingAllOf(" ANNUAL SUMMARY FOR WATERSHED IN YEAR ", " " + y);
//
// // col 3
// double d = tp.toLineContaining("Total irrigation volume in contributing area")
// .rightOf("=").leftOf("m^3").asDouble();
//
// // col 4
// double d1 = tp.toLineContaining("Total water discharge from outlet")
// .rightOf("=").leftOf("m^3").asDouble();
//
// // col 5: hillslope spoil loss
// double d2 = tp.toLineContaining("Total hillslope soil loss in contributing area")
// .rightOf("=").leftOf("tonnes").asDouble();
//
// System.out.println(d + " " + d1 + " " + d2);
// hs_sl[y - 1] = d;
// tp.close();
//
// }
//
// }
// map key: TD id -> WEPP id
//
public static Map<Integer, Integer> getChannelIdMap(File ouputSummary) throws Exception {
String s = FileUtils.readFileToString(ouputSummary, "utf-8");
JSONObject o = new JSONObject(s);
JSONArray ch = o.getJSONArray("Channels");
Map<Integer, Integer> m = new HashMap<>();
for (int i = 0; i < ch.length(); i++) {
JSONObject c = ch.getJSONObject(i);
m.put(c.getInt("id"), c.getInt("weppID"));
}
return m;
}
public static List<Integer> getWeppIdList(Map<Integer, Integer> idMap, List<Integer> tdIds, String name) {
List<Integer> weIds = new ArrayList<>();
for (Integer tdId : tdIds) {
Integer weId = idMap.get(tdId);
if (weId == null)
throw new RuntimeException(name + ": Taudem Id not found: " + tdId);
weIds.add(weId);
}
return weIds;
}
// TD Id -> WEPP Channel results
public static Map<Integer, JSONObject> getChannelResultsMap(File ouputSummary) throws Exception {
String s = FileUtils.readFileToString(ouputSummary, "utf-8");
JSONObject o = new JSONObject(s);
JSONArray ch = o.getJSONArray("Channels");
Map<Integer, JSONObject> m = new HashMap<>();
for (int i = 0; i < ch.length(); i++) {
JSONObject c = ch.getJSONObject(i);
m.put(c.getInt("id"), c);
}
return m;
}
public static Map<Integer, Integer> getHillslopeIdMap(JSONArray hs) throws Exception {
Map<Integer, Integer> m = new HashMap<>();
for (int i = 0; i < hs.length(); i++) {
JSONObject c = hs.getJSONObject(i);
m.put(c.getInt("id"), c.getInt("weppid"));
}
return m;
}
public static double getHSElementSum(JSONArray hs, String key) throws Exception {
return getHSElementStats(hs, key).getSum();
}
public static DoubleSummaryStatistics getHSElementStats(JSONArray hs, String key) throws Exception {
List<Double> l = new ArrayList<>();
for (int i = 0; i < hs.length(); i++) {
JSONObject h = hs.getJSONObject(i);
l.add(h.getDouble(key));
}
return l.stream().mapToDouble(d -> d).filter(v -> v > -999.0).summaryStatistics();
}
@Deprecated
public static Map<Integer, Integer> getHillslopeIdMap(File ouputSummary) throws Exception {
String s = FileUtils.readFileToString(ouputSummary, "utf-8");
JSONObject o = new JSONObject(s);
JSONArray ch = o.getJSONArray("Representative hillslope subcatchment summary");
Map<Integer, Integer> m = new HashMap<>();
for (int i = 0; i < ch.length(); i++) {
JSONObject c = ch.getJSONObject(i);
m.put(c.getInt("id"), c.getInt("weppid"));
}
return m;
}
public static Summary getChannelStats(File sloss, int years) throws Exception {
// Find number of channels:
int noChannels = 0;
try (TextParser p = new TextParser(sloss)
.autoClose(false)
.toLineContaining(" SOIL: ")) {
while (p.nextLine().notEmpty()) {
noChannels = p.split().intAt(1);
}
}
// Find number of channels:
int noHillslopes = new TextParser(sloss)
.autoClose(false)
.toLineStartingWith("Hillslope Elements:")
.rightOf("-")
.asInteger();
// System.out.println(noChannels + " " + noHillslopes);
Summary chd = new Summary(noChannels, noHillslopes);
List<Channel> ch = chd.getChannels();
for (int y = 1; y <= years; y++) {
try (TextParser p = new TextParser(sloss)
.autoClose(false)
.toLineContainingAllOf(" ANNUAL SUMMARY FOR WATERSHED IN YEAR ", " " + y)
.toLineStartingWith("Channels ")
.nextLine(4)) {
while (p.nextLine().notEmpty()) {
double[] row = p.split().rangeFrom(1).asDoubleArray();
if (row.length == 6)
// only channel row, skip impoundments
// System.out.println(Arrays.toString(row));
ch.get((int) row[0] - 1).add(row[1], row[2], row[3], row[4], row[5]);
}
}
}
// System.out.println(ch);
// System.out.println(ch.get(0).getSedYieldStats());
for (int y = 1; y <= years; y++) {
try (TextParser p = new TextParser(sloss)
.autoClose(false)
.toLineContainingAllOf(" ANNUAL SUMMARY FOR WATERSHED IN YEAR ", " " + y)
.toLineStartingWith("Delivery From Channel Outlet:")) {
double tc = p.toLineStartingWith("Total contributing area to outlet").rightOf("=").split().doubleAt(0);
double tp = p.toLineStartingWith("Total precipitation volume in contributing area").rightOf("=").split().doubleAt(0);
double ti = p.toLineStartingWith("Total irrigation volume in contributing area").rightOf("=").split().doubleAt(0);
double tw = p.toLineStartingWith("Total water discharge from outlet").rightOf("=").split().doubleAt(0);
double th = p.toLineStartingWith("Total hillslope soil loss in contributing area").rightOf("=").split().doubleAt(0);
double tch = p.toLineStartingWith("Total channel soil loss").rightOf("=").split().doubleAt(0);
double ts = p.toLineStartingWith("Total sediment discharge from outlet").rightOf("=").split().doubleAt(0);
double sdu = p.toLineStartingWith("Sed. delivery per unit area of watershed ").rightOf("=").split().doubleAt(0);
double sdr = p.toLineStartingWith("Sediment Delivery Ratio for Watershed").rightOf("=").asDouble();
chd.add(tc, tp, ti, tw, th, tch,
ts, sdu, sdr);
}
}
// System.out.println(chd.getTotContribAreaStats());
// System.out.println(chd.getTotChannelSlossStats());
// System.out.println(chd.getSedDelivRatioStats());
// System.out.println(chd.getTotSedDischargeStats());
List<Hillslope> hs = chd.getHillslopes();
for (int y = 1; y <= years; y++) {
try (TextParser p = new TextParser(sloss)
.autoClose(false)
.toLineContainingAllOf(" ANNUAL SUMMARY FOR WATERSHED IN YEAR ", " " + y)
.toLineStartingWith("Hillslopes ")
.nextLine()) {
while (p.nextLine().notEmpty()) {
double[] row = p.split().rangeFrom(1).asDoubleArray();
// System.out.println(Arrays.toString(row));
hs.get((int) row[0] - 1).add(row[1], row[2], row[3], row[4], row[5]);
}
}
}
return chd;
}
static <T> List<T> createList(Class<T> c, int len) throws Exception {
List<T> l = new ArrayList<>();
for (int i = 0; i < len; i++) {
l.add(c.newInstance());
}
return l;
}
public static void main(String[] args) throws Exception {
getChannelStats(new File("/od/projects/WEPPWS-Python/loss_pw0.txt"), 15);
}
}