guiLOADEST_Model.java [src/java/m/cfa/loadest] Revision: 21b1071edcaa5a300c84544b77f8ff777fe2d3b1 Date: Wed Jun 01 16:03:15 MDT 2022
package m.cfa.loadest;
import WaterData.WaterData;
import WaterData.WaterDataException;
import WaterData.WaterDataInterface;
import WaterData.WaterQualityInfo;
import m.cfa.DoubleArray;
import m.cfa.DoubleMath;
import csip.api.server.Executable;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import org.codehaus.jettison.json.JSONException;
/**
* Last Updated: 9-April-2019
* @author Tyler Wible & Tyler Dell
* @since 27-March-2013
*/
public class guiLOADEST_Model {
//Inputs
String directory = "C:/Projects/TylerWible_repos/NetBeans/data/CFA/LOADEST";
String database = "USGS";//"CDWR";//"STORET";//"CDSN";//"UserData";//
String orgId = "n/a";//"n/a";//"21COL001";//"CITYFTCO_WQX";//"n/a";//
String stationId = "06764880";//"CLAGRECO";//"000028";//"1EFF";//"n/a";//
String stationName = "South Platte River at Roscoe, Nebr.";//"Cache La Poudre Near Greeley";//"BIG THOMPSON R NEAR MOUTH";//"n/a";//"n/a";//
String wqTest = "00600 Total nitrogen, water, unfiltered, milligrams per liter -- mg/L";//"00625 Ammonia-nitrogen as N -- mg/L";//
int PTOPT = 1; // estimated values print option (0,1)
int SEOPT = 2; // standard error option (1-3)
int LDOPT = 0; // load option (0-3), if load option is 1 or 3 then need the number of user-defined seasons
int modelNumber = 11; // (0-11, 99) this picks which model to run if 0 then chooses best model from 1-9 based on error values
// if model number is 10, 11 user must specify
int PBMON = 5; // Begin month for periodic terms
int PEMON = 7; // End month for periodic terms
int ULFLAG = 4; // the desired output units (1-4)
String startDate = "";
String endDate = "";
boolean mergeDatasets = false;//true;//
String mergeMethod = "user";//"public";//"max";//"average";//"min";//
String userData = "";//"Date\tFlow\n1994-01-01\t37\n1994-01-02\t39\n1994-01-03\t40\n1994-01-04\t40\n1994-01-05\t40\n1994-01-06\t38\n1994-01-07\t44\n1994-01-08\t37\n1994-01-09\t37\n1994-01-10\t37\n1994-01-11\t35\n1994-01-12\t31\n1994-01-13\t31\n1994-01-14\t27\n1994-01-15\t27\n1994-01-16\t32\n1994-01-17\t25\n1994-01-18\t25\n1994-01-19\t29\n1994-01-20\t32\n1994-01-21\t31\n1994-01-22\t24\n1994-01-23\t29\n1994-01-24\t26\n1994-01-25\t31\n1994-01-26\t30\n1994-01-27\t30\n1994-01-28\t30\n1994-01-29\t24\n1994-01-30\t25\n1994-01-31\t24\n1994-02-01\t28\n1994-02-02\t26\n1994-02-03\t25\n1994-02-04\t26\n1994-02-05\t27\n1994-02-06\t26\n1994-02-07\t25\n1994-02-08\t27\n1994-02-09\t23\n1994-02-10\t22\n1994-02-11\t26\n1994-02-12\t29\n1994-02-13\t26\n1994-02-14\t26\n1994-02-15\t38\n1994-02-16\t40\n1994-02-17\t42\n1994-02-18\t41\n1994-02-19\t44\n1994-02-20\t44\n1994-02-21\t41\n1994-02-22\t34\n1994-02-23\t30\n1994-02-24\t34\n1994-02-25\t34\n1994-02-26\t36\n1994-02-27\t39\n1994-02-28\t54\n1994-03-01\t50\n1994-03-02\t49\n1994-03-03\t51\n1994-03-04\t51\n1994-03-05\t52\n1994-03-06\t54\n1994-03-07\t56\n1994-03-08\t51\n1994-03-09\t43\n1994-03-10\t54\n1994-03-11\t50\n1994-03-12\t46\n1994-03-13\t49\n1994-03-14\t51\n1994-03-15\t50\n1994-03-16\t48\n1994-03-17\t32\n1994-03-18\t56\n1994-03-19\t55\n1994-03-20\t59\n1994-03-21\t55\n1994-03-22\t43\n1994-03-23\t48\n1994-03-24\t46\n1994-03-25\t42\n1994-03-26\t43\n1994-03-27\t35\n1994-03-28\t28\n1994-03-29\t42\n1994-03-30\t38\n1994-03-31\t40\n1994-04-01\t41\n1994-04-02\t41\n1994-04-03\t39\n1994-04-04\t39\n1994-04-05\t41\n1994-04-06\t38\n1994-04-07\t36\n1994-04-08\t35\n1994-04-09\t37\n1994-04-10\t43\n1994-04-11\t41\n1994-04-12\t56\n1994-04-13\t56\n1994-04-14\t52\n1994-04-15\t56\n1994-04-16\t53\n1994-04-17\t56\n1994-04-18\t64\n1994-04-19\t69\n1994-04-20\t76\n1994-04-21\t87\n1994-04-22\t102\n1994-04-23\t130\n1994-04-24\t171\n1994-04-25\t244\n1994-04-26\t223\n1994-04-27\t176\n1994-04-28\t163\n1994-04-29\t158\n1994-04-30\t137\n1994-05-01\t138\n1994-05-02\t138\n1994-05-03\t136\n1994-05-04\t149\n1994-05-05\t149\n1994-05-06\t147\n1994-05-07\t188\n1994-05-08\t291\n1994-05-09\t401\n1994-05-10\t528\n1994-05-11\t476\n1994-05-12\t598\n1994-05-13\t752\n1994-05-14\t874\n1994-05-15\t1030\n1994-05-16\t1050\n1994-05-17\t1310\n1994-05-18\t1180\n1994-05-19\t1130\n1994-05-20\t1270\n1994-05-21\t1100\n1994-05-22\t1110\n1994-05-23\t1200\n1994-05-24\t1090\n1994-05-25\t1090\n1994-05-26\t1180\n1994-05-27\t1160\n1994-05-28\t1270\n1994-05-29\t1350\n1994-05-30\t1390\n1994-05-31\t1490\n1994-06-01\t1590\n1994-06-02\t1380\n1994-06-03\t1220\n1994-06-04\t1210\n1994-06-05\t1250\n1994-06-06\t1270\n1994-06-07\t1190\n1994-06-08\t1010\n1994-06-09\t1040\n1994-06-10\t953\n1994-06-11\t960\n1994-06-12\t916\n1994-06-13\t757\n1994-06-14\t692\n1994-06-15\t702\n1994-06-16\t722\n1994-06-17\t714\n1994-06-18\t700\n1994-06-19\t703\n1994-06-20\t767\n1994-06-21\t819\n1994-06-22\t764\n1994-06-23\t764\n1994-06-24\t686\n1994-06-25\t585\n1994-06-26\t499\n1994-06-27\t420\n1994-06-28\t385\n1994-06-29\t335\n1994-06-30\t278\n1994-07-01\t246\n1994-07-02\t274\n1994-07-03\t291\n1994-07-04\t312\n1994-07-05\t254\n1994-07-06\t175\n1994-07-07\t226\n1994-07-08\t297\n1994-07-09\t233\n1994-07-10\t155\n1994-07-11\t219\n1994-07-12\t254\n1994-07-13\t261\n1994-07-14\t268\n1994-07-15\t229\n1994-07-16\t185\n1994-07-17\t185\n1994-07-18\t267\n1994-07-19\t305\n1994-07-20\t303\n1994-07-21\t300\n1994-07-22\t276\n1994-07-23\t269\n1994-07-24\t327\n1994-07-25\t308\n1994-07-26\t296\n1994-07-27\t268\n1994-07-28\t248\n1994-07-29\t224\n1994-07-30\t215\n1994-07-31\t203\n1994-08-01\t237\n1994-08-02\t302\n1994-08-03\t350\n1994-08-04\t345\n1994-08-05\t346\n1994-08-06\t336\n1994-08-07\t315\n1994-08-08\t293\n1994-08-09\t289\n1994-08-10\t296\n1994-08-11\t319\n1994-08-12\t316\n1994-08-13\t305\n1994-08-14\t306\n1994-08-15\t291\n1994-08-16\t189\n1994-08-17\t160\n1994-08-18\t144\n1994-08-19\t154\n1994-08-20\t251\n1994-08-21\t264\n1994-08-22\t255\n1994-08-23\t245\n1994-08-24\t243\n1994-08-25\t227\n1994-08-26\t207\n1994-08-27\t147\n1994-08-28\t132\n1994-08-29\t122\n1994-08-30\t70\n1994-08-31\t68\n1994-09-01\t57\n1994-09-02\t61\n1994-09-03\t82\n1994-09-04\t102\n1994-09-05\t119\n1994-09-06\t119\n1994-09-07\t123\n1994-09-08\t108\n1994-09-09\t76\n1994-09-10\t48\n1994-09-11\t47\n1994-09-12\t45\n1994-09-13\t43\n1994-09-14\t47\n1994-09-15\t53\n1994-09-16\t46\n1994-09-17\t35\n1994-09-18\t35\n1994-09-19\t34\n1994-09-20\t35\n1994-09-21\t40\n1994-09-22\t72\n1994-09-23\t83\n1994-09-24\t65\n1994-09-25\t46\n1994-09-26\t39\n1994-09-27\t33\n1994-09-28\t26\n1994-09-29\t23\n1994-09-30\t22\n1994-10-01\t20\n1994-10-02\t27\n1994-10-03\t28\n1994-10-04\t33\n1994-10-05\t27\n1994-10-06\t23\n1994-10-07\t31\n1994-10-08\t28\n1994-10-09\t24\n1994-10-10\t21\n1994-10-11\t21\n1994-10-12\t21\n1994-10-13\t21\n1994-10-14\t18\n1994-10-15\t18\n1994-10-16\t18\n1994-10-17\t26\n1994-10-18\t27\n1994-10-19\t19\n1994-10-20\t12\n1994-10-21\t15\n1994-10-22\t13\n1994-10-23\t17\n1994-10-24\t21\n1994-10-25\t17\n1994-10-26\t18\n1994-10-27\t17\n1994-10-28\t18\n1994-10-29\t22\n1994-10-30\t24\n1994-10-31\t28\n1994-11-01\t21\n1994-11-02\t17\n1994-11-03\t22\n1994-11-04\t23\n1994-11-05\t13\n1994-11-06\t25\n1994-11-07\t30\n1994-11-08\t32\n1994-11-09\t25\n1994-11-10\t15\n1994-11-11\t18\n1994-11-12\t22\n1994-11-13\t28\n1994-11-14\t21\n1994-11-15\t6.8\n1994-11-16\t6.0\n1994-11-17\t15\n1994-11-18\t18\n1994-11-19\t23\n1994-11-20\t28\n1994-11-21\t34\n1994-11-22\t36\n1994-11-23\t21\n1994-11-24\t13\n1994-11-25\t23\n1994-11-26\t37\n1994-11-27\t29\n1994-11-28\t10\n1994-11-29\t11\n1994-11-30\t15\n1994-12-01\t28\n1994-12-02\t60\n1994-12-03\t62\n1994-12-04\t36\n1994-12-05\t22\n1994-12-06\t31\n1994-12-07\t23\n1994-12-08\t25\n1994-12-09\t12\n1994-12-10\t15\n1994-12-11\t24\n1994-12-12\t24\n1994-12-13\t31\n1994-12-14\t35\n1994-12-15\t34\n1994-12-16\t37\n1994-12-17\t32\n1994-12-18\t21\n1994-12-19\t21\n1994-12-20\t22\n1994-12-21\t22\n1994-12-22\t22\n1994-12-23\t19\n1994-12-24\t21\n1994-12-25\t22\n1994-12-26\t22\n1994-12-27\t23\n1994-12-28\t23\n1994-12-29\t22\n1994-12-30\t26\n1994-12-31\t22\n1995-01-01\t22"
//+ "$$Date\t00600\n1994-01-03\tnodata\n1994-01-13\t0.28\n1994-02-16\t0.29\n1994-03-07\t0.25\n1994-04-20\t0.25\n1994-05-03\t0.25\n1994-06-07\t0.25\n1994-06-14\t0.25\n1994-07-06\t0.25\n1994-08-04\t0.25\n1994-09-02\t0.28\n1994-10-11\t0.25\n1994-11-22\t0.33\n1994-12-13\t0.59";
//Global variable but not an input
String fileName = "";
//Outputs
String flowLen = "-1";
String wqLen = "-1";
String wqUnits = "?";
String start = "?";
String end = "?";
String dataSource = "?";
//Outputs
double dailymax = -1;
double dailymin = -1;
double dailyupperQuartile = -1;
double dailylowerQuartile = -1;
double dailymedian = -1;
double dailymean = -1;
double dailystandardDeviation = -1;
double monthlymax = -1;
double monthlymin = -1;
double monthlyupperQuartile = -1;
double monthlylowerQuartile = -1;
double monthlymedian = -1;
double monthlymean = -1;
double monthlystandardDeviation = -1;
double yearlymax = -1;
double yearlymin = -1;
double yearlyupperQuartile = -1;
double yearlylowerQuartile = -1;
double yearlymedian = -1;
double yearlymean = -1;
double yearlystandardDeviation = -1;
//Gets
// public File getOutputSummary() {
// return new File(directory, "echo.out");
// }
public File getOutputResult1() { return new File(directory, fileName + ".ind"); }
public File getOutputResult2() { return new File(directory, fileName + ".out"); }
public File getOutputResult3() { return new File(directory, fileName + ".res"); }
public File getDailyTimeseries(){ return new File(directory, "daily_graph.out"); }//for use with JSHighCharts
public File getDailyBoxplot(){ return new File(directory, "daily_boxplot.out"); }//for use with JSHighCharts
public File getMonthlyTimeseries(){ return new File(directory, "monthly_graph.out"); }//for use with JSHighCharts
public File getMonthlyBoxplot(){ return new File(directory, "monthly_boxplot.out"); }//for use with JSHighCharts
public File getYearlyTimeseries(){ return new File(directory, "yearly_graph.out"); }//for use with JSHighCharts
public File getYearlyBoxplot(){ return new File(directory, "yearly_boxplot.out"); }//for use with JSHighCharts
public String getFlowLen() { return flowLen; }
public String getWQLen() { return wqLen; }
public String getWQUnits() { return wqUnits; }
public String getStart(){ return start; }
public String getEnd(){ return end; }
public String getDataSource(){ return dataSource; }
public String getDailyMax(){ return String.valueOf(dailymax); }
public String getDailyMin(){ return String.valueOf(dailymin); }
public String getDailyUpperQuartile(){ return String.valueOf(dailyupperQuartile); }
public String getDailyLowerQuartile(){ return String.valueOf(dailylowerQuartile); }
public String getDailyMedian(){ return String.valueOf(dailymedian); }
public String getDailyMean(){ return String.valueOf(dailymean); }
public String getDailyStandardDeviation(){ return String.valueOf(dailystandardDeviation); }
public String getMonthlyMax(){ return String.valueOf(monthlymax); }
public String getMonthlyMin(){ return String.valueOf(monthlymin); }
public String getMonthlyUpperQuartile(){ return String.valueOf(monthlyupperQuartile); }
public String getMonthlyLowerQuartile(){ return String.valueOf(monthlylowerQuartile); }
public String getMonthlyMedian(){ return String.valueOf(monthlymedian); }
public String getMonthlyMean(){ return String.valueOf(monthlymean); }
public String getMonthlyStandardDeviation(){ return String.valueOf(monthlystandardDeviation); }
public String getYearlyMax(){ return String.valueOf(yearlymax); }
public String getYearlyMin(){ return String.valueOf(yearlymin); }
public String getYearlyUpperQuartile(){ return String.valueOf(yearlyupperQuartile); }
public String getYearlyLowerQuartile(){ return String.valueOf(yearlylowerQuartile); }
public String getYearlyMedian(){ return String.valueOf(yearlymedian); }
public String getYearlyMean(){ return String.valueOf(yearlymean); }
public String getYearlyStandardDeviation(){ return String.valueOf(yearlystandardDeviation); }
//Sets
public void setDirectory(String directory_str){ directory = directory_str; }
public void setDatabase(String database_str){ database = database_str; }
public void setOrganizationID(String orgId_str){ orgId = orgId_str; }
public void setStationName(String stationName_str){ stationName = stationName_str; }
public void setStationId(String stationId_str){ stationId = stationId_str; }
public void setWQtest(String wqTest_str){ wqTest = wqTest_str; }
public void setPTOPT(int PTOPT_int){ PTOPT = PTOPT_int; }
public void setSEOPT(int SEOPT_int){ SEOPT = SEOPT_int; }
public void setLDOPT(int LDOPT_int){ LDOPT = LDOPT_int; }
public void setmodelNumber(int modelNumber_int){ modelNumber = modelNumber_int; }
public void setPBMON(int PBMON_int){ PBMON = PBMON_int; }
public void setPEMON(int PEMON_int){ PEMON = PEMON_int; }
public void setULFLAG (int ULFLAG_int){ ULFLAG = ULFLAG_int; }
public void setStartDate(String startDate_str){ startDate = startDate_str; }
public void setEndDate(String endDate_str){ endDate = endDate_str; }
public void setMergeDatasets(boolean mergeDatasets_TF){ mergeDatasets = mergeDatasets_TF; }
public void setMergeMethod(String mergeMethod_str){ mergeMethod = mergeMethod_str; }
public void setUserData(String userData_str){ userData = userData_str; }
/**
* Control File Writer:
* This function will write the control file which is what LOADEST will look for to find the
* header, calibration, and estimation files.
* @throws IOException
*/
private void writeControl()throws IOException {
String path = directory + File.separator + "CONTROL.INP";
FileWriter writer = new FileWriter(path, false);
PrintWriter print_line = new PrintWriter(writer);
//Create the control file
print_line.printf("%s" + "\r\n", "######################################################################");
print_line.printf("%s" + "\r\n", "#");
print_line.printf("%s" + "\r\n", "# LOADEST Control File");
print_line.printf("%s" + "\r\n", "#");
print_line.printf("%s" + "\r\n", "#");
print_line.printf("%s" + "\r\n", "# line name of the:");
print_line.printf("%s" + "\r\n", "# ---- ------------");
print_line.printf("%s" + "\r\n", "# 1 header file");
print_line.printf("%s" + "\r\n", "# 2 calibration file");
print_line.printf("%s" + "\r\n", "# 3 estimation file");
print_line.printf("%s" + "\r\n", "#");
print_line.printf("%s" + "\r\n", "######################################################################");
print_line.printf("%s" + "\r\n", "HEADER.INP");
print_line.printf("%s" + "\r\n", "CALIB.INP");
print_line.printf("%s" + "\r\n", "EST.INP");
print_line.close();
writer.close();
System.out.println("Text File located at:\t" + path);
}
/**
* Header File Writer:
* This function will write the header file that is needed for LOADEST
* @throws IOException
*/
private void writeHeader()throws IOException {
String path = directory + File.separator + "HEADER.INP";
FileWriter writer = new FileWriter(path, false);
PrintWriter print_line = new PrintWriter(writer);
// Introduction for header file
print_line.printf("%s" + "\r\n", "######################################################################");
print_line.printf("%s" + "\r\n", "#");
print_line.printf("%s" + "\r\n", "# LOADEST Header File");
print_line.printf("%s" + "\r\n", "#");
print_line.printf("%s" + "\r\n", "# " + stationId + " " + stationName + " Database: " + database + ", Organization ID: " + orgId);
print_line.printf("%s" + "\r\n", "#");
print_line.printf("%s" + "\r\n", "######################################################################");
// Inputs 1-4 Title, estimated values print option, standard error option, and load option
print_line.printf("%s" + "\r\n",stationId + " " +stationName + " Database: " + database + ", Organization ID: " + orgId);
print_line.printf("%s" + "\r\n", PTOPT);
print_line.printf("%s" + "\r\n", SEOPT);
print_line.printf("%s" + "\r\n", LDOPT);
print_line.printf("%s" + "\r\n", "###########################################################");
// Inputs 5-6 are dependent on what the user defines for the load option
// 5 is the number of seasons desired by the user and 6 is the start and end dates for each season
// do not currenlty allow user to define seasons, only give option for yearly or monthly estimates
// i.e. LDOPT = 0 or 2, but if want to define seasons modify code below
// if (LDOPT == 1 || LDOPT == 3){
// print_line.printf("%s" + "\r\n", NSEAS);
// print_line.printf("%s" + "\r\n", "#");
// print_line.printf("%s" + "\r\n", "#SBEG SEND");
// int NSEAS = 1;
// for (int i=0; i < NSEAS; i++){
// print_line.printf("%s" + "\r\n",seasonBegin[i] + " " + seasonEnd[i] );
// }
// print_line.printf("%s" + "\r\n", "###########################################################");
// }
// Input 7 the model number
print_line.printf("%s" + "\r\n", modelNumber);
print_line.printf("%s" + "\r\n", "###########################################################");
// Inputs 8-11 are dependent on what the user defines for the model number.
//If model number is 10 or 11
if (modelNumber == 10 || modelNumber == 11){
print_line.printf("%s" + "\r\n", "#PBMN PEMN");
print_line.printf("%s" + "\r\n", PBMON + " " + PEMON); // input 8
print_line.printf("%s" + "\r\n", "###########################################################");
}
// Input 12, number of constituents = 1
print_line.printf("%s" + "\r\n", "1");
print_line.printf("%s" + "\r\n", "###########################################################");
// Input 13, is dependent on what the user defines for the number of constituents
if (fileName.length() > 43){
fileName = fileName.substring(0,43);
}
print_line.printf("%-45s%-5s%-5s" + "\r\n", fileName, "1", ULFLAG); // 1 denotes concentration in mg/L
print_line.printf("%s" + "\r\n", "###########################################################");
// //Listed below is inputs 12 and 13 for nconst > 1
// // must make sure to redefine cname, ucflag, and ulflag
// // Input 12, number of constituents
// print_line.printf("%s" + "\r\n", nconst);
// print_line.printf("%s" + "\r\n", "###########################################################");
//
// // Input 13, is dependent on what the user defines for the number of constituents
// for (int j=0; j < nconst; j++){
// if (cname[j].length() > 43){
// cname[j] = cname[j].substring(0,43);
// }
// print_line.printf("%-45s%-5s%-5s" + "\r\n", cname[j], ucflag[j], ulflag[j]);
//
// }
// print_line.printf("%s" + "\r\n", "###########################################################");
print_line.close();
writer.close();
System.out.println("Text File located at:\t" + path);
}
/**
* Calibration File Writer:
* This function will write the calibration file that is needed for LOADEST
* @param sortedData_combined the flow data, sortedData_combined[i][0] = date (yyyy-mm-dd), sortedData_combined[i][1] = value (cfs)
* @param WQdata_combined the water quality data, WQdata_combined[i][0] = date (yyyy-mm-dd), WQdata_combined[i][1] = value (units)
* @param units the units of the water quality data
* @throws IOException
*/
private void writeCalibration(String[][] sortedData_combined, String[][] WQdata_combined, String units) throws IOException {
//Convert data for Calibration file
ArrayList<String> CDATE = new ArrayList<>();
ArrayList<String> CTIME = new ArrayList<>();
ArrayList<String> CFLOW = new ArrayList<>();
ArrayList<String> CCONC = new ArrayList<>();
for(int i=0;i<WQdata_combined.length;i++){
for(int j=0; j<sortedData_combined.length;j++){
if(WQdata_combined[i][0].equalsIgnoreCase(sortedData_combined[j][0])){
//Keep the data with both a water quality test and a flow value
CDATE.add(WQdata_combined[i][0]);
CTIME.add("1200");
CFLOW.add(sortedData_combined[j][1]);
CCONC.add(WQdata_combined[i][1]); // need to make sure that CCONC is in mg/L
}
}
}
CCONC = convertToLOADESTunits(units,CCONC);
//Store result for analysis summary
wqLen = String.valueOf(CCONC.size());
String path = directory + File.separator + "CALIB.INP";
FileWriter writer = new FileWriter(path, false);
PrintWriter print_line = new PrintWriter(writer);
// Introduction for header file
print_line.printf("%s" + "\r\n", "##########################################################################################################");
print_line.printf("%s" + "\r\n", "#");
print_line.printf("%s" + "\r\n", "# LOADEST Calibration File");
print_line.printf("%s" + "\r\n", "#");
print_line.printf("%s" + "\r\n", "# " + stationId + " " + stationName + " Database: " + database + ", Organization ID: " + orgId);
print_line.printf("%s" + "\r\n", "#");
print_line.printf("%s" + "\r\n", "##########################################################################################################");
print_line.printf("%s" + "\r\n", "#");
print_line.printf("%s" + "\r\n", "#");
print_line.printf("%-12s%-10s%-10s%-10s" + "\r\n", "#CDATE","CTIME","CFLOW","CCONC");
print_line.printf("%s" + "\r\n", "##########################################################################################################");
print_line.printf("%s" + "\r\n", "#");
// Input the data
for(int i=0; i < CDATE.size(); i++){
// change the date into format needed for loadest
CDATE.set(i, changeDate(CDATE.get(i)));
print_line.printf("%-12s%-10s%-10s", CDATE.get(i), CTIME.get(i), CFLOW.get(i));
// add in concentrations
//Arbitrarily define the number of water quality parameters equal to 1, this can be modified later if need be
//for (int j=0; j < nconst; j++){// column counter if multiple concentrations are desired
// print_line.printf("%-10s",CCONC[i][j]);
//}
print_line.printf("%-10s",CCONC.get(i));
// print a new line
print_line.printf("\r\n");
}
print_line.close();
writer.close();
System.out.println("Text File located at:\t" + path);
}
/**
* Estimation File Writer:
* This function will write the estimation file needed for LOADEST
* @throws IOException
*/
private void writeEstimation(String[][] sortedData_combined) throws IOException {
//Convert data for Estimation file
String[] EDATE = new String[sortedData_combined.length];
String[] ETIME = new String[sortedData_combined.length];
String[] EFLOW = new String[sortedData_combined.length];
for(int i=0; i<sortedData_combined.length; i++){
EDATE[i] = sortedData_combined[i][0];
ETIME[i] = "1200";
EFLOW[i] = sortedData_combined[i][1];
}
// open the file writer and set path
String path = directory + File.separator + "EST.INP";
FileWriter writer = new FileWriter(path, false);
PrintWriter print_line = new PrintWriter(writer);
// Introduction for header file
print_line.printf("%s" + "\r\n", "######################################################################");
print_line.printf("%s" + "\r\n", "#");
print_line.printf("%s" + "\r\n", "# LOADEST Estimation File");
print_line.printf("%s" + "\r\n", "#");
print_line.printf("%s" + "\r\n", "# " + stationId + " " + stationName + " " + "Organization ID: " + orgId);
print_line.printf("%s" + "\r\n", "#");
print_line.printf("%s" + "\r\n", "######################################################################");
print_line.printf("%s" + "\r\n", "# Number of oberservations per day");
print_line.printf("%s" + "\r\n", "1");
print_line.printf("%s" + "\r\n", "#");
print_line.printf("%s" + "\r\n", "######################################################################");
print_line.printf("%s" + "\r\n", "#");
print_line.printf("%-12s%-10s%-10s" + "\r\n", "#EDATE", "ETIME", "EFLOW");
print_line.printf("%s" + "\r\n", "######################################################################");
// print the needed values
for (int i=0; i < EDATE.length; i++){
// change the date into format needed for loadest
EDATE[i] = changeDate(EDATE[i]);
print_line.printf("%-12s%-10s%-10s", EDATE[i], ETIME[i], EFLOW[i]);
// print a new line
print_line.printf("\r\n");
}
// Close the file writer
print_line.close();
writer.close();
System.out.println("Text File located at:\t" + path);
}
/**
* Date Formatter:
* This function will change the date into the format that LOADEST requires
* @param date - Old Format
* @return Date - New Format
*/
private String changeDate(String date){
String year = date.substring(0,4);
String month = date.substring(5,7);
String day = date.substring(8);
date = year + month + day;
return date;
}
/**
* Converts the provided water quality values into mg/l if possible, if not
* possible then throws and exception to choose a new water quality test
* @param units the current units of the water quality test values in CCONC
* @param CCONC an array containing the water quality test values
* @return CCONC multiplied by a conversion factor to convert it into mg/l
* @throws IOException
*/
private ArrayList<String> convertToLOADESTunits(String units, ArrayList<String> CCONC) throws IOException{
//Find the unit conversion for current water quality units
double conversion = 0;
if(units.equalsIgnoreCase("g/cm3") || units.equalsIgnoreCase("g/mL @ 20C")){
conversion = 1000;
}else if(units.equalsIgnoreCase("g/m3")){
conversion = 1000/Math.pow(100,3);
}else if(units.equalsIgnoreCase("mg/l") || units.equalsIgnoreCase("mg/l CaCO3") || units.equalsIgnoreCase("mg/l NH4") ||
units.equalsIgnoreCase("mg/l NO3") || units.equalsIgnoreCase("mg/l PO4") || units.equalsIgnoreCase("mg/l SiO2") ||
units.equalsIgnoreCase("mg/l as H") || units.equalsIgnoreCase("mg/l as N") || units.equalsIgnoreCase("mg/l as Na") ||
units.equalsIgnoreCase("mg/l as P") || units.equalsIgnoreCase("mg/l as S") || units.equalsIgnoreCase("mgC3H6O2/L")){
conversion = 1;
}else if(units.equalsIgnoreCase("mg/mL @25C")){
conversion = 1000;
}else if(units.equalsIgnoreCase("ng/l") || units.equalsIgnoreCase("pg/mL")){
conversion = 1/Math.pow(10,6);
}else if(units.equalsIgnoreCase("ng/m3") || units.equalsIgnoreCase("pg/l")){
conversion = 1/(Math.pow(10,6)*Math.pow(100,3));
}else if(units.equalsIgnoreCase("pg/m3")){
conversion = 1/(Math.pow(10,9)*Math.pow(100,3));
}else if(units.equalsIgnoreCase("ug/L 2,4-D") || units.equalsIgnoreCase("ug/L U3O8") || units.equalsIgnoreCase("ug/L as As") ||
units.equalsIgnoreCase("ug/L as Cl") || units.equalsIgnoreCase("ug/L as N") || units.equalsIgnoreCase("ug/L as P") ||
units.equalsIgnoreCase("ug/l") || units.equalsIgnoreCase("ugAtrazn/L")){
conversion = 1/1000;
}else if(units.equalsIgnoreCase("ug/m3")){
conversion = 1/(1000*Math.pow(100,3));
}else{
ArrayList<String> errorMessage = new ArrayList<>();
errorMessage.add("LOADEST is only able to operate on water quality units of mg/L. The currently selected water quality test is in units of " + units + ", please select a different water quality test that is in mg/L.");
writeError(errorMessage);
}
//Apply conversion to mg/L of current water quality data
for (int i = 0; i < CCONC.size(); i++){
double value = Double.parseDouble(CCONC.get(i));
CCONC.set(i, String.valueOf(value*conversion));
}
return CCONC;
}
/**
* This will open the file and grab the lines of data from it until it reaches the empty line
* @return returns the entire text file in textData
* @throws IOException
*/
private String[][] OpenLOADESTresultFile(String path) throws IOException {
FileReader fr = new FileReader(path);
BufferedReader textReader = new BufferedReader(fr);
int numberOfLines = readLines(path);
if(numberOfLines == 0){
ArrayList<String> errorMessage = new ArrayList<>();
errorMessage.add("LOADEST failed to return information for the provided nutrient data, the data is likely too sparce to perform an analysis on, please select a different water quality test, provide more data, or select a shorter analysis period.");
writeError(errorMessage);
}
String[] textData = new String[numberOfLines];
for (int i = 0; i < numberOfLines; i++){
textData[i]=textReader.readLine();
}
textReader.close();
fr.close();
// rctr is used to remove 8 lines of header from the file
int rctr = textData.length - 8;
String[][] dailyData = new String[rctr][2];
for (int i = 0; i < rctr; i++){
dailyData[i][0] = textData[(i+8)].substring(1,9);
dailyData[i][1] = textData[(i+8)].substring(28,38);
}
return dailyData;
}
/**
* This will determine how many lines contain total data. Used to in OpenFile
* @return
* @throws IOException
*/
private int readLines(String path) throws IOException{
FileReader file_to_read = new FileReader(path);
BufferedReader bf = new BufferedReader (file_to_read);
String aLine;
int numberOfLines = 0;
while ((aLine=bf.readLine()) != null){
numberOfLines++;
}
bf.close();
file_to_read.close();
return numberOfLines;
}
/**
* Writes out the error message, if any, for finding the file and then exits the program
* @param error string array to be written as each line of an error message
* @throws IOException
*/
public void writeError(ArrayList<String> error) throws IOException{
//Output data to text file
String errorContents = error.get(0);
for(int i=1; i<error.size(); i++){
errorContents = errorContents + "\n" + error.get(i);
}
throw new IOException("Error encountered. Please see the following message for details: \n" + errorContents);
}
/**
* Main statistics function calls other functions to calculate each statistic value then stores the results as global variables
* @param sortedData data on which statistical values are desired, sortedData[i][0] = date, soredData[i][1] = value
* @param timeStep a flag for which results the statistics will be stored to, either "daily", "monthly", or "yearly"
*/
private void CalculateStatistics(String[][] sortedData, String timeStep) {
//convert Data into proper format for calculations
ArrayList<Double> dataList = new ArrayList<>();
for(int i=0; i<sortedData.length; i++){
dataList.add(Double.parseDouble(sortedData[i][1]));
}
//Calculate statistics
double temp1 = DoubleMath.round(DoubleMath.max(dataList),3);//Call Max function
double temp2 = DoubleMath.round(DoubleMath.min(dataList),3);//Call Min function
double temp3 = DoubleMath.round(DoubleMath.Percentile_function(dataList,0.75),3);//Call Upper Quartile function
double temp4 = DoubleMath.round(DoubleMath.Percentile_function(dataList,0.25),3);//Call Lower Quartile function
double temp5 = DoubleMath.round(DoubleMath.median(dataList),3);//Call Median function
double temp6 = DoubleMath.round(DoubleMath.meanArithmetic(dataList),3);//Call Mean function
double temp7 = DoubleMath.round(DoubleMath.StandardDeviationSample(dataList),3);//Call standard deviation
if(timeStep.equalsIgnoreCase("daily")){
dailymax = temp1;
dailymin = temp2;
dailyupperQuartile = temp3;
dailylowerQuartile = temp4;
dailymedian = temp5;
dailymean = temp6;
dailystandardDeviation = temp7;
}else if(timeStep.equalsIgnoreCase("monthly")){
monthlymax = temp1;
monthlymin = temp2;
monthlyupperQuartile = temp3;
monthlylowerQuartile = temp4;
monthlymedian = temp5;
monthlymean = temp6;
monthlystandardDeviation = temp7;
}else if(timeStep.equalsIgnoreCase("yearly")){
yearlymax = temp1;
yearlymin = temp2;
yearlyupperQuartile = temp3;
yearlylowerQuartile = temp4;
yearlymedian = temp5;
yearlymean = temp6;
yearlystandardDeviation = temp7;
}
}
/**
* Primary LOADEST function
* It calls the subfunctions based on user selection/inputs.
* Calls STORET or USGS database queries and their respective subfunctions
* @param e the loadest.exe executable
* @throws IOException
* @throws InterruptedException
* @throws ParseException
* @throws org.codehaus.jettison.json.JSONException
*/
public void run(Executable e) throws IOException, InterruptedException, ParseException, JSONException, WaterDataException {
//If no date input, make it the maximum of available data
if(startDate == null || startDate.equalsIgnoreCase("")){
startDate = "1850-01-01";
}
if(endDate == null || endDate.equalsIgnoreCase("")){
// Pull current date for upper limit of data search
DateFormat desiredDateFormat = new SimpleDateFormat("yyyy-MM-dd");
Date currentDate = new Date();
endDate = desiredDateFormat.format(currentDate);
}
//Correct Inputs
String[] resultArray = WaterQualityInfo.getWqTestDataInfo(wqTest, database);
//String wqCode = resultArray[0];
String wqLabel = resultArray[1];
wqUnits = resultArray[2];
//double ldcConversion = Double.parseDouble(resultArray[3]);
//String ldcEndUnits = resultArray[4];
//Check if any flow and water quality data exists
String flowData_raw = "", wqData_raw = "";
if(userData.length() > 0){
String[] userDataList = userData.split("\\$\\$");
flowData_raw = userDataList[0];
wqData_raw = userDataList[1];
}
WaterDataInterface waterLib = WaterData.getNewWaterDataInterface(database, flowData_raw);
String[][] sortableData = waterLib.extractFlowData_formatted(directory, orgId, stationId, startDate, endDate);
WaterDataInterface waterLib2 = WaterData.getNewWaterDataInterface(database, wqData_raw);
String[][] WQdata = waterLib2.extractWaterQualityData_formatted(directory, orgId, stationId, startDate, endDate, wqTest);
dataSource = waterLib.getDataSourceCitation();
//Find the user uploaded data file and uses this if merged
String[][] sortableData_user = new String[0][0];
String[][] WQdata_user = new String[0][0];
if(mergeDatasets){
WaterDataInterface waterLibUser = WaterData.getNewWaterDataInterface("UserData", flowData_raw);
sortableData_user = waterLibUser.extractFlowData_formatted(directory, orgId, stationId, startDate, endDate);
WaterDataInterface waterLibUser2 = WaterData.getNewWaterDataInterface("UserData", wqData_raw);
WQdata_user = waterLibUser2.extractWaterQualityData_formatted(directory, orgId, stationId, startDate, endDate, wqTest);
}
//Sort the Data by date to remove duplicate date entries
String[][] sortedData = DoubleArray.removeDuplicateDates(sortableData);
String[][] sortedData_user = DoubleArray.removeDuplicateDates(sortableData_user);
//Merge the two datasets (if user data is empty nothing will be merged)
String[][] sortedData_combined = DoubleArray.mergeData(sortedData, sortedData_user, mergeMethod);
String[][] WQdata_combined = DoubleArray.mergeData(WQdata, WQdata_user, mergeMethod);
//Check if any data exists
ArrayList<String> errorMessage = new ArrayList<>();
if(sortedData_combined.length == 0){
if(sortedData.length == 0){
errorMessage.add("There is no available flow data in the " + database + " database for station '" + stationId + "' and the specified date range.");
if(database.equalsIgnoreCase("CDWR")){
errorMessage.add("The CDWR database is sensitive to the begin date used, try specifying a later begin date");
}
}
if(sortedData_user.length==0){
errorMessage.add("There is no uploaded flow data for station '" + stationId + "' and the specified date range.");
}
}
if(WQdata_combined.length == 0){
if(WQdata.length == 0){
errorMessage.add("There are no available '" + wqTest + "' water quality tests for station " + stationId+ " by: " +
database + " and the specified date range");
}
if(WQdata_user.length==0){
errorMessage.add("There are no uploaded '" + wqTest + "' water quality tests for station " + stationId+ " and the specified date range");
}
}
if(errorMessage.size() > 0){
writeError(errorMessage);
}
//Write control file
writeControl();
//Write Header File
String[] cname = wqLabel.split(",");
fileName = cname[0].trim();
fileName = fileName.replace(" ", "");//Remove spaces from the name
writeHeader();
// write the calibration variables based on the number of concentration values with a associated flow value
writeCalibration(sortedData_combined, WQdata_combined, wqUnits);
//Get estimation dates for LOADEST variables
writeEstimation(sortedData_combined);
//Call LOADEST exe
//Expected Input: "CONTROL.INP", "HEADER.INP", "CALIB.INP", and "EST.INP"
System.out.println("Calling LOADEST.exe");
e.exec();
//Expected Output: "echo.out" and fileName + ".ind", fileName + ".out", fileName + ".res"
if (!new File(directory, "echo.out").exists()) {
throw new FileNotFoundException("echo.out");
}
if (!new File(directory, fileName + ".ind").exists()) {
throw new FileNotFoundException(fileName + ".ind");
}
if (!new File(directory, fileName + ".out").exists()) {
throw new FileNotFoundException(fileName + ".out");
}
if (!new File(directory, fileName + ".res").exists()) {
throw new FileNotFoundException(fileName + ".res");
}
System.out.println("Finished LOADEST.exe");
// Daily result data
String[][] dailyData = OpenLOADESTresultFile(directory + File.separator + fileName + ".ind");
//convert results to daily,monthly,yearly
String[][] dailyresultData = DoubleArray.computeFlowMethod(dailyData, "daily", "total", true);
String[][] monthlyresultData = DoubleArray.computeFlowMethod(dailyData, "monthly", "total", true);
String[][] yearlyresultData = DoubleArray.computeFlowMethod(dailyData, "yearly", "total", true);
// Calculate Daily stats
CalculateStatistics(dailyresultData, "daily");
double dailyIQR = dailyupperQuartile - dailylowerQuartile;// Find IQR
ArrayList<Double> dailyoutliers = new ArrayList<>();// Get daily outliers
for (int i = 0; i < dailyresultData.length; i++){
double value = Double.parseDouble(dailyresultData[i][1]);
if(value < (dailylowerQuartile - 1.5 * dailyIQR) || value > (dailyupperQuartile + 1.5 * dailyIQR)){
dailyoutliers.add(value);
}
}
// Calculate Monthly stats
CalculateStatistics(monthlyresultData, "monthly");
double monthlyIQR = monthlyupperQuartile - monthlylowerQuartile;// Find IQR
ArrayList<Double> monthlyoutliers = new ArrayList<>();// Get monthly outliers
for (int i = 0; i < monthlyresultData.length; i++){
double value = Double.parseDouble(monthlyresultData[i][1]);
if(value < (monthlylowerQuartile - 1.5 * monthlyIQR) || value > (monthlyupperQuartile + 1.5 * monthlyIQR)){
monthlyoutliers.add(value);
}
}
// Calculate Yearly stats
CalculateStatistics(yearlyresultData, "yearly");
double yearlyIQR = yearlyupperQuartile - yearlylowerQuartile;// Find IQR
ArrayList<Double> yearlyoutliers = new ArrayList<>();// Get yearly outliers
for (int i = 0; i < yearlyresultData.length; i++){
double value = Double.parseDouble(yearlyresultData[i][1]);
if(value < (yearlylowerQuartile - 1.5 * yearlyIQR) || value > (yearlyupperQuartile + 1.5 * yearlyIQR)){
yearlyoutliers.add(value);
}
}
//Prep. boxplot data for output function
ArrayList<ArrayList<Double>> dailyBoxplotOutliers = new ArrayList<>();
ArrayList<ArrayList<Double>> monthlyBoxplotOutliers = new ArrayList<>();
ArrayList<ArrayList<Double>> yearlyBoxplotOutliers = new ArrayList<>();
dailyBoxplotOutliers.add(dailyoutliers);
monthlyBoxplotOutliers.add(monthlyoutliers);
yearlyBoxplotOutliers.add(yearlyoutliers);
double[][] dailyBoxplotData = {{dailymax},{dailyupperQuartile},{dailymedian},{dailylowerQuartile},{dailymin}};
double[][] monthlyBoxplotData = {{monthlymax},{monthlyupperQuartile},{monthlymedian},{monthlylowerQuartile},{monthlymin}};
double[][] yearlyBoxplotData = {{yearlymax},{yearlyupperQuartile},{yearlymedian},{yearlylowerQuartile},{yearlymin}};
//Put graph file writer call here
DoubleArray.writeTimeSeries(directory, dailyresultData, "daily", getDailyTimeseries().getName(), true);
DoubleArray.writeTimeSeries(directory, monthlyresultData, "monthly", getMonthlyTimeseries().getName(), true);
DoubleArray.writeTimeSeries(directory, yearlyresultData, "yearly", getYearlyTimeseries().getName(), true);
DoubleArray.writeBoxplot(directory, dailyBoxplotOutliers, dailyBoxplotData, getDailyBoxplot().getName());
DoubleArray.writeBoxplot(directory, monthlyBoxplotOutliers, monthlyBoxplotData, getMonthlyBoxplot().getName());
DoubleArray.writeBoxplot(directory, yearlyBoxplotOutliers, yearlyBoxplotData, getYearlyBoxplot().getName());
// Find what units that LOADEST has given the Load in
String graphUnits = "??";
if (ULFLAG == 1){
graphUnits = "kg";
}else if (ULFLAG ==2){
graphUnits = "mg";
}else if (ULFLAG == 3){
graphUnits = "lb";
}else{
graphUnits = "tons";
}
start = sortedData_combined[0][0];
end = sortedData_combined[sortedData_combined.length - 1][0];
flowLen = String.valueOf(sortedData_combined.length);
wqLen = String.valueOf(WQdata_combined.length);
wqUnits = graphUnits;
}
public static void main(String[] args) throws IOException, InterruptedException, Exception {
//Run model
guiLOADEST_Model loadest_Model = new guiLOADEST_Model();
loadest_Model.run(null);
}
}