guiLOADEST_Model.java [src/java/cfa] Revision: 785a73989e00201d9ce6064ee214199feab731c5 Date: Tue Aug 05 13:04:04 MDT 2014
package cfa;
import csip.Config;
import csip.utils.Binaries;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import oms3.util.ProcessComponent;
/**
* Last Updated: 5-August-2014
* @author Tyler Wible & Tyler Dell
* @since 27-March-2013
*/
public class guiLOADEST_Model {
//Inputs
String mainFolder = "C:/Projects/TylerWible/CodeDirectories/NetBeans/CSIP/data/CFA/LOADEST";
String database = "USGS";//"CDWR";//"STORET";//"UserData";//
String organizationName = "USGS";//"Co. Division of Water Resources";//"Colorado Dept. of Public Health & Environment";//
String stationID = "06741510";//"CLAGRECO";//"000028";//
String stationName = "BIG THOMPSON RIVER AT LOVELAND, CO.";//"Cache La Poudre Near Greeley";//"BIG THOMPSON R NEAR MOUTH";//
String wqTest = "00600 Total nitrogen, water, unfiltered, milligrams per liter, mg/L";
int PTOPT = 1; // estimated values print option (0,1)
int SEOPT = 2; // standard error option (1-3)
int LDOPT = 0; // load option (0-3), if load option is 1 or 3 then need the number of user-defined seasons
int modelNumber = 11; // (0-11, 99) this picks which model to run if 0 then chooses best model from 1-9 based on error values
// if model number is 10, 11 user must specify
int PBMON = 5; // Begin month for periodic terms
int PEMON = 7; // End month for periodic terms
int ULFLAG = 4; // the desired output units (1-4)
String beginDate = "";
String endDate = "";
String userData = "";//"Date\tFlow\n1994-01-01\t37\n1994-01-02\t39\n1994-01-03\t40\n1994-01-04\t40\n1994-01-05\t40\n1994-01-06\t38\n1994-01-07\t44\n1994-01-08\t37\n1994-01-09\t37\n1994-01-10\t37\n1994-01-11\t35\n1994-01-12\t31\n1994-01-13\t31\n1994-01-14\t27\n1994-01-15\t27\n1994-01-16\t32\n1994-01-17\t25\n1994-01-18\t25\n1994-01-19\t29\n1994-01-20\t32\n1994-01-21\t31\n1994-01-22\t24\n1994-01-23\t29\n1994-01-24\t26\n1994-01-25\t31\n1994-01-26\t30\n1994-01-27\t30\n1994-01-28\t30\n1994-01-29\t24\n1994-01-30\t25\n1994-01-31\t24\n1994-02-01\t28\n1994-02-02\t26\n1994-02-03\t25\n1994-02-04\t26\n1994-02-05\t27\n1994-02-06\t26\n1994-02-07\t25\n1994-02-08\t27\n1994-02-09\t23\n1994-02-10\t22\n1994-02-11\t26\n1994-02-12\t29\n1994-02-13\t26\n1994-02-14\t26\n1994-02-15\t38\n1994-02-16\t40\n1994-02-17\t42\n1994-02-18\t41\n1994-02-19\t44\n1994-02-20\t44\n1994-02-21\t41\n1994-02-22\t34\n1994-02-23\t30\n1994-02-24\t34\n1994-02-25\t34\n1994-02-26\t36\n1994-02-27\t39\n1994-02-28\t54\n1994-03-01\t50\n1994-03-02\t49\n1994-03-03\t51\n1994-03-04\t51\n1994-03-05\t52\n1994-03-06\t54\n1994-03-07\t56\n1994-03-08\t51\n1994-03-09\t43\n1994-03-10\t54\n1994-03-11\t50\n1994-03-12\t46\n1994-03-13\t49\n1994-03-14\t51\n1994-03-15\t50\n1994-03-16\t48\n1994-03-17\t32\n1994-03-18\t56\n1994-03-19\t55\n1994-03-20\t59\n1994-03-21\t55\n1994-03-22\t43\n1994-03-23\t48\n1994-03-24\t46\n1994-03-25\t42\n1994-03-26\t43\n1994-03-27\t35\n1994-03-28\t28\n1994-03-29\t42\n1994-03-30\t38\n1994-03-31\t40\n1994-04-01\t41\n1994-04-02\t41\n1994-04-03\t39\n1994-04-04\t39\n1994-04-05\t41\n1994-04-06\t38\n1994-04-07\t36\n1994-04-08\t35\n1994-04-09\t37\n1994-04-10\t43\n1994-04-11\t41\n1994-04-12\t56\n1994-04-13\t56\n1994-04-14\t52\n1994-04-15\t56\n1994-04-16\t53\n1994-04-17\t56\n1994-04-18\t64\n1994-04-19\t69\n1994-04-20\t76\n1994-04-21\t87\n1994-04-22\t102\n1994-04-23\t130\n1994-04-24\t171\n1994-04-25\t244\n1994-04-26\t223\n1994-04-27\t176\n1994-04-28\t163\n1994-04-29\t158\n1994-04-30\t137\n1994-05-01\t138\n1994-05-02\t138\n1994-05-03\t136\n1994-05-04\t149\n1994-05-05\t149\n1994-05-06\t147\n1994-05-07\t188\n1994-05-08\t291\n1994-05-09\t401\n1994-05-10\t528\n1994-05-11\t476\n1994-05-12\t598\n1994-05-13\t752\n1994-05-14\t874\n1994-05-15\t1030\n1994-05-16\t1050\n1994-05-17\t1310\n1994-05-18\t1180\n1994-05-19\t1130\n1994-05-20\t1270\n1994-05-21\t1100\n1994-05-22\t1110\n1994-05-23\t1200\n1994-05-24\t1090\n1994-05-25\t1090\n1994-05-26\t1180\n1994-05-27\t1160\n1994-05-28\t1270\n1994-05-29\t1350\n1994-05-30\t1390\n1994-05-31\t1490\n1994-06-01\t1590\n1994-06-02\t1380\n1994-06-03\t1220\n1994-06-04\t1210\n1994-06-05\t1250\n1994-06-06\t1270\n1994-06-07\t1190\n1994-06-08\t1010\n1994-06-09\t1040\n1994-06-10\t953\n1994-06-11\t960\n1994-06-12\t916\n1994-06-13\t757\n1994-06-14\t692\n1994-06-15\t702\n1994-06-16\t722\n1994-06-17\t714\n1994-06-18\t700\n1994-06-19\t703\n1994-06-20\t767\n1994-06-21\t819\n1994-06-22\t764\n1994-06-23\t764\n1994-06-24\t686\n1994-06-25\t585\n1994-06-26\t499\n1994-06-27\t420\n1994-06-28\t385\n1994-06-29\t335\n1994-06-30\t278\n1994-07-01\t246\n1994-07-02\t274\n1994-07-03\t291\n1994-07-04\t312\n1994-07-05\t254\n1994-07-06\t175\n1994-07-07\t226\n1994-07-08\t297\n1994-07-09\t233\n1994-07-10\t155\n1994-07-11\t219\n1994-07-12\t254\n1994-07-13\t261\n1994-07-14\t268\n1994-07-15\t229\n1994-07-16\t185\n1994-07-17\t185\n1994-07-18\t267\n1994-07-19\t305\n1994-07-20\t303\n1994-07-21\t300\n1994-07-22\t276\n1994-07-23\t269\n1994-07-24\t327\n1994-07-25\t308\n1994-07-26\t296\n1994-07-27\t268\n1994-07-28\t248\n1994-07-29\t224\n1994-07-30\t215\n1994-07-31\t203\n1994-08-01\t237\n1994-08-02\t302\n1994-08-03\t350\n1994-08-04\t345\n1994-08-05\t346\n1994-08-06\t336\n1994-08-07\t315\n1994-08-08\t293\n1994-08-09\t289\n1994-08-10\t296\n1994-08-11\t319\n1994-08-12\t316\n1994-08-13\t305\n1994-08-14\t306\n1994-08-15\t291\n1994-08-16\t189\n1994-08-17\t160\n1994-08-18\t144\n1994-08-19\t154\n1994-08-20\t251\n1994-08-21\t264\n1994-08-22\t255\n1994-08-23\t245\n1994-08-24\t243\n1994-08-25\t227\n1994-08-26\t207\n1994-08-27\t147\n1994-08-28\t132\n1994-08-29\t122\n1994-08-30\t70\n1994-08-31\t68\n1994-09-01\t57\n1994-09-02\t61\n1994-09-03\t82\n1994-09-04\t102\n1994-09-05\t119\n1994-09-06\t119\n1994-09-07\t123\n1994-09-08\t108\n1994-09-09\t76\n1994-09-10\t48\n1994-09-11\t47\n1994-09-12\t45\n1994-09-13\t43\n1994-09-14\t47\n1994-09-15\t53\n1994-09-16\t46\n1994-09-17\t35\n1994-09-18\t35\n1994-09-19\t34\n1994-09-20\t35\n1994-09-21\t40\n1994-09-22\t72\n1994-09-23\t83\n1994-09-24\t65\n1994-09-25\t46\n1994-09-26\t39\n1994-09-27\t33\n1994-09-28\t26\n1994-09-29\t23\n1994-09-30\t22\n1994-10-01\t20\n1994-10-02\t27\n1994-10-03\t28\n1994-10-04\t33\n1994-10-05\t27\n1994-10-06\t23\n1994-10-07\t31\n1994-10-08\t28\n1994-10-09\t24\n1994-10-10\t21\n1994-10-11\t21\n1994-10-12\t21\n1994-10-13\t21\n1994-10-14\t18\n1994-10-15\t18\n1994-10-16\t18\n1994-10-17\t26\n1994-10-18\t27\n1994-10-19\t19\n1994-10-20\t12\n1994-10-21\t15\n1994-10-22\t13\n1994-10-23\t17\n1994-10-24\t21\n1994-10-25\t17\n1994-10-26\t18\n1994-10-27\t17\n1994-10-28\t18\n1994-10-29\t22\n1994-10-30\t24\n1994-10-31\t28\n1994-11-01\t21\n1994-11-02\t17\n1994-11-03\t22\n1994-11-04\t23\n1994-11-05\t13\n1994-11-06\t25\n1994-11-07\t30\n1994-11-08\t32\n1994-11-09\t25\n1994-11-10\t15\n1994-11-11\t18\n1994-11-12\t22\n1994-11-13\t28\n1994-11-14\t21\n1994-11-15\t6.8\n1994-11-16\t6.0\n1994-11-17\t15\n1994-11-18\t18\n1994-11-19\t23\n1994-11-20\t28\n1994-11-21\t34\n1994-11-22\t36\n1994-11-23\t21\n1994-11-24\t13\n1994-11-25\t23\n1994-11-26\t37\n1994-11-27\t29\n1994-11-28\t10\n1994-11-29\t11\n1994-11-30\t15\n1994-12-01\t28\n1994-12-02\t60\n1994-12-03\t62\n1994-12-04\t36\n1994-12-05\t22\n1994-12-06\t31\n1994-12-07\t23\n1994-12-08\t25\n1994-12-09\t12\n1994-12-10\t15\n1994-12-11\t24\n1994-12-12\t24\n1994-12-13\t31\n1994-12-14\t35\n1994-12-15\t34\n1994-12-16\t37\n1994-12-17\t32\n1994-12-18\t21\n1994-12-19\t21\n1994-12-20\t22\n1994-12-21\t22\n1994-12-22\t22\n1994-12-23\t19\n1994-12-24\t21\n1994-12-25\t22\n1994-12-26\t22\n1994-12-27\t23\n1994-12-28\t23\n1994-12-29\t22\n1994-12-30\t26\n1994-12-31\t22\n1995-01-01\t22"
//+ "$$Date\t00600\n1994-01-03\tnodata\n1994-01-13\t0.28\n1994-02-16\t0.29\n1994-03-07\t0.25\n1994-04-20\t0.25\n1994-05-03\t0.25\n1994-06-07\t0.25\n1994-06-14\t0.25\n1994-07-06\t0.25\n1994-08-04\t0.25\n1994-09-02\t0.28\n1994-10-11\t0.25\n1994-11-22\t0.33\n1994-12-13\t0.59";
boolean mergeDatasets = false;//true;//
String mergeMethod = "user";//"public";//"max";//"average";//"min";//
//Global variable but not an input
String fileName = "";
//Outputs
String flowLen = "-1";
String wqLen = "-1";
String wqUnits = "?";
String start = "?";
String end = "?";
String dataSource = "?";
//Outputs
double dailymax = -1;
double dailymin = -1;
double dailyupperQuartile = -1;
double dailylowerQuartile = -1;
double dailymedian = -1;
double dailymean = -1;
double dailystandardDeviation = -1;
double monthlymax = -1;
double monthlymin = -1;
double monthlyupperQuartile = -1;
double monthlylowerQuartile = -1;
double monthlymedian = -1;
double monthlymean = -1;
double monthlystandardDeviation = -1;
double yearlymax = -1;
double yearlymin = -1;
double yearlyupperQuartile = -1;
double yearlylowerQuartile = -1;
double yearlymedian = -1;
double yearlymean = -1;
double yearlystandardDeviation = -1;
//Gets
// public File getOutputSummary() {
// return new File(mainFolder, "echo.out");
// }
public File getOutputResult1() {
return new File(mainFolder, fileName + ".ind");
}
public File getOutputResult2() {
return new File(mainFolder, fileName + ".out");
}
public File getOutputResult3() {
return new File(mainFolder, fileName + ".res");
}
public File getDailyTimeseries(){
return new File(mainFolder, "daily_graph.out");
}
public File getDailyBoxplot(){
return new File(mainFolder, "daily_boxplot.out");
}
public File getMonthlyTimeseries(){
return new File(mainFolder, "monthly_graph.out");
}
public File getMonthlyBoxplot(){
return new File(mainFolder, "monthly_boxplot.out");
}
public File getYearlyTimeseries(){
return new File(mainFolder, "yearly_graph.out");
}
public File getYearlyBoxplot(){
return new File(mainFolder, "yearly_boxplot.out");
}
public String getFlowLen() {
return flowLen;
}
public String getWQLen() {
return wqLen;
}
public String getWQUnits() {
return wqUnits;
}
public String getStart(){
return start;
}
public String getEnd(){
return end;
}
public String getDataSource(){
return dataSource;
}
public String getDailyMax(){
return String.valueOf(dailymax);
}
public String getDailyMin(){
return String.valueOf(dailymin);
}
public String getDailyUpperQuartile(){
return String.valueOf(dailyupperQuartile);
}
public String getDailyLowerQuartile(){
return String.valueOf(dailylowerQuartile);
}
public String getDailyMedian(){
return String.valueOf(dailymedian);
}
public String getDailyMean(){
return String.valueOf(dailymean);
}
public String getDailyStandardDeviation(){
return String.valueOf(dailystandardDeviation);
}
public String getMonthlyMax(){
return String.valueOf(monthlymax);
}
public String getMonthlyMin(){
return String.valueOf(monthlymin);
}
public String getMonthlyUpperQuartile(){
return String.valueOf(monthlyupperQuartile);
}
public String getMonthlyLowerQuartile(){
return String.valueOf(monthlylowerQuartile);
}
public String getMonthlyMedian(){
return String.valueOf(monthlymedian);
}
public String getMonthlyMean(){
return String.valueOf(monthlymean);
}
public String getMonthlyStandardDeviation(){
return String.valueOf(monthlystandardDeviation);
}
public String getYearlyMax(){
return String.valueOf(yearlymax);
}
public String getYearlyMin(){
return String.valueOf(yearlymin);
}
public String getYearlyUpperQuartile(){
return String.valueOf(yearlyupperQuartile);
}
public String getYearlyLowerQuartile(){
return String.valueOf(yearlylowerQuartile);
}
public String getYearlyMedian(){
return String.valueOf(yearlymedian);
}
public String getYearlyMean(){
return String.valueOf(yearlymean);
}
public String getYearlyStandardDeviation(){
return String.valueOf(yearlystandardDeviation);
}
//Sets
public void setMainFolder(String mainFolder) {
this.mainFolder = mainFolder;
}
public void setDatabase(String database) {
this.database = database;
}
public void setOrganizationName(String organizationName) {
this.organizationName = organizationName;
}
public void setStationName(String stationName) {
this.stationName = stationName;
}
public void setStationID(String stationID) {
this.stationID = stationID;
}
public void setWQtest(String wqTest) {
this.wqTest = wqTest;
}
public void setPTOPT(int PTOPT) {
this.PTOPT = PTOPT;
}
public void setSEOPT(int SEOPT) {
this.SEOPT = SEOPT;
}
public void setLDOPT(int LDOPT) {
this.LDOPT = LDOPT;
}
public void setmodelNumber(int modelNumber){
this.modelNumber = modelNumber;
}
public void setPBMON(int PBMON){
this.PBMON = PBMON;
}
public void setPEMON(int PEMON){
this.PEMON = PEMON;
}
public void setULFLAG (int ULFLAG){
this.ULFLAG = ULFLAG;
}
public void setBeginDate(String beginDate) {
this.beginDate = beginDate;
}
public void setEndDate(String endDate) {
this.endDate = endDate;
}
public void setUserData(String userData) {
this.userData = userData;
}
public void setMergeDatasets(boolean mergeDatasets) {
this.mergeDatasets = mergeDatasets;
}
public void setMergeMethod(String mergeMethod) {
this.mergeMethod = mergeMethod;
}
/**
* Control File Writer:
* This function will write the control file which is what LOADEST will look for to find the
* header, calibration, and estimation files.
* @throws IOException
*/
private void writeControl()throws IOException {
String path = mainFolder + File.separator + "CONTROL.INP";
FileWriter writer = new FileWriter(path, false);
PrintWriter print_line = new PrintWriter(writer);
//Create the control file
print_line.printf("%s" + "%n", "######################################################################");
print_line.printf("%s" + "%n", "#");
print_line.printf("%s" + "%n", "# LOADEST Control File");
print_line.printf("%s" + "%n", "#");
print_line.printf("%s" + "%n", "#");
print_line.printf("%s" + "%n", "# line name of the:");
print_line.printf("%s" + "%n", "# ---- ------------");
print_line.printf("%s" + "%n", "# 1 header file");
print_line.printf("%s" + "%n", "# 2 calibration file");
print_line.printf("%s" + "%n", "# 3 estimation file");
print_line.printf("%s" + "%n", "#");
print_line.printf("%s" + "%n", "######################################################################");
print_line.printf("%s" + "%n", "HEADER.INP");
print_line.printf("%s" + "%n", "CALIB.INP");
print_line.printf("%s" + "%n", "EST.INP");
print_line.close();
writer.close();
System.out.println("Text File located at:\t" + path);
}
/**
* Header File Writer:
* This function will write the header file that is needed for LOADEST
* @throws IOException
*/
private void writeHeader()throws IOException {
String path = mainFolder + File.separator + "HEADER.INP";
FileWriter writer = new FileWriter(path, false);
PrintWriter print_line = new PrintWriter(writer);
// Introduction for header file
print_line.printf("%s" + "%n", "######################################################################");
print_line.printf("%s" + "%n", "#");
print_line.printf("%s" + "%n", "# LOADEST Header File");
print_line.printf("%s" + "%n", "#");
print_line.printf("%s" + "%n", "# " + stationID + " " + stationName + " Database: " + database + ", Supervising Organization: " + organizationName);
print_line.printf("%s" + "%n", "#");
print_line.printf("%s" + "%n", "######################################################################");
// Inputs 1-4 Title, estimated values print option, standard error option, and load option
print_line.printf("%s" + "%n",stationID + " " +stationName + " Database: " + database + ", Supervising Organization: " + organizationName);
print_line.printf("%s" + "%n", PTOPT);
print_line.printf("%s" + "%n", SEOPT);
print_line.printf("%s" + "%n", LDOPT);
print_line.printf("%s" + "%n", "###########################################################");
// Inputs 5-6 are dependent on what the user defines for the load option
// 5 is the number of seasons desired by the user and 6 is the start and end dates for each season
// do not currenlty allow user to define seasons, only give option for yearly or monthly estimates
// i.e. LDOPT = 0 or 2, but if want to define seasons modify code below
// if (LDOPT == 1 || LDOPT == 3){
// print_line.printf("%s" + "%n", NSEAS);
// print_line.printf("%s" + "%n", "#");
// print_line.printf("%s" + "%n", "#SBEG SEND");
// int NSEAS = 1;
// for (int i=0; i < NSEAS; i++){
// print_line.printf("%s" + "%n",seasonBegin[i] + " " + seasonEnd[i] );
// }
// print_line.printf("%s" + "%n", "###########################################################");
// }
// Input 7 the model number
print_line.printf("%s" + "%n", modelNumber);
print_line.printf("%s" + "%n", "###########################################################");
// Inputs 8-11 are dependent on what the user defines for the model number.
//If model number is 10 or 11
if (modelNumber == 10 || modelNumber == 11){
print_line.printf("%s" + "%n", "#PBMN PEMN");
print_line.printf("%s" + "%n", PBMON + " " + PEMON); // input 8
print_line.printf("%s" + "%n", "###########################################################");
}
// Input 12, number of constituents = 1
print_line.printf("%s" + "%n", "1");
print_line.printf("%s" + "%n", "###########################################################");
// Input 13, is dependent on what the user defines for the number of constituents
if (fileName.length() > 43){
fileName = fileName.substring(0,43);
}
print_line.printf("%-45s%-5s%-5s" + "%n", fileName, "1", ULFLAG); // 1 denotes concentration in mg/L
print_line.printf("%s" + "%n", "###########################################################");
// //Listed below is inputs 12 and 13 for nconst > 1
// // must make sure to redefine cname, ucflag, and ulflag
// // Input 12, number of constituents
// print_line.printf("%s" + "%n", nconst);
// print_line.printf("%s" + "%n", "###########################################################");
//
// // Input 13, is dependent on what the user defines for the number of constituents
// for (int j=0; j < nconst; j++){
// if (cname[j].length() > 43){
// cname[j] = cname[j].substring(0,43);
// }
// print_line.printf("%-45s%-5s%-5s" + "%n", cname[j], ucflag[j], ulflag[j]);
//
// }
// print_line.printf("%s" + "%n", "###########################################################");
print_line.close();
writer.close();
System.out.println("Text File located at:\t" + path);
}
/**
* Calibration File Writer:
* This function will write the calibration file that is needed for LOADEST
* @param sortedData_combined the flow data, sortedData_combined[i][0] = date (yyyy-mm-dd), sortedData_combined[i][1] = value (cfs)
* @param WQdata_combined the water quality data, WQdata_combined[i][0] = date (yyyy-mm-dd), WQdata_combined[i][1] = value (units)
* @param units the units of the water quality data
* @throws IOException
*/
private void writeCalibration(String[][] sortedData_combined, String[][] WQdata_combined, String units) throws IOException {
//Convert data for Calibration file
ArrayList<String> CDATE = new ArrayList<String>();
ArrayList<String> CTIME = new ArrayList<String>();
ArrayList<String> CFLOW = new ArrayList<String>();
ArrayList<String> CCONC = new ArrayList<String>();
for(int i=0;i<WQdata_combined.length;i++){
for(int j=0; j<sortedData_combined.length;j++){
if(WQdata_combined[i][0].equalsIgnoreCase(sortedData_combined[j][0])){
//Keep the data with both a water quality test and a flow value
CDATE.add(WQdata_combined[i][0]);
CTIME.add("1200");
CFLOW.add(sortedData_combined[j][1]);
CCONC.add(WQdata_combined[i][1]); // need to make sure that CCONC is in mg/L
}
}
}
CCONC = convertToLOADESTunits(units,CCONC);
//Store result for analysis summary
this.wqLen = String.valueOf(CCONC.size());
String path = mainFolder + File.separator + "CALIB.INP";
FileWriter writer = new FileWriter(path, false);
PrintWriter print_line = new PrintWriter(writer);
// Introduction for header file
print_line.printf("%s" + "%n", "##########################################################################################################");
print_line.printf("%s" + "%n", "#");
print_line.printf("%s" + "%n", "# LOADEST Calibration File");
print_line.printf("%s" + "%n", "#");
print_line.printf("%s" + "%n", "# " + stationID + " " + stationName + " Database: " + database + ", Supervising Organization: " + organizationName );
print_line.printf("%s" + "%n", "#");
print_line.printf("%s" + "%n", "##########################################################################################################");
print_line.printf("%s" + "%n", "#");
print_line.printf("%s" + "%n", "#");
print_line.printf("%-12s%-10s%-10s%-10s" + "%n", "#CDATE","CTIME","CFLOW","CCONC");
print_line.printf("%s" + "%n", "##########################################################################################################");
print_line.printf("%s" + "%n", "#");
// Input the data
for(int i=0; i < CDATE.size(); i++){
// change the date into format needed for loadest
CDATE.set(i, changeDate(CDATE.get(i)));
print_line.printf("%-12s%-10s%-10s", CDATE.get(i), CTIME.get(i), CFLOW.get(i));
// add in concentrations
//Arbitrarily define the number of water quality parameters equal to 1, this can be modified later if need be
//for (int j=0; j < nconst; j++){// column counter if multiple concentrations are desired
// print_line.printf("%-10s",CCONC[i][j]);
//}
print_line.printf("%-10s",CCONC.get(i));
// print a new line
print_line.printf("%n");
}
print_line.close();
writer.close();
System.out.println("Text File located at:\t" + path);
}
/**
* Estimation File Writer:
* This function will write the estimation file needed for LOADEST
* @throws IOException
*/
private void writeEstimation(String[][] sortedData_combined) throws IOException {
//Convert data for Estimation file
String[] EDATE = new String[sortedData_combined.length];
String[] ETIME = new String[sortedData_combined.length];
String[] EFLOW = new String[sortedData_combined.length];
for(int i=0; i<sortedData_combined.length; i++){
EDATE[i] = sortedData_combined[i][0];
ETIME[i] = "1200";
EFLOW[i] = sortedData_combined[i][1];
}
// open the file writer and set path
String path = mainFolder + File.separator + "EST.INP";
FileWriter writer = new FileWriter(path, false);
PrintWriter print_line = new PrintWriter(writer);
// Introduction for header file
print_line.printf("%s" + "%n", "######################################################################");
print_line.printf("%s" + "%n", "#");
print_line.printf("%s" + "%n", "# LOADEST Estimation File");
print_line.printf("%s" + "%n", "#");
print_line.printf("%s" + "%n", "# " + stationID + " " + stationName + " " + "Agency: " + organizationName );
print_line.printf("%s" + "%n", "#");
print_line.printf("%s" + "%n", "######################################################################");
print_line.printf("%s" + "%n", "# Number of oberservations per day");
print_line.printf("%s" + "%n", "1");
print_line.printf("%s" + "%n", "#");
print_line.printf("%s" + "%n", "######################################################################");
print_line.printf("%s" + "%n", "#");
print_line.printf("%-12s%-10s%-10s" + "%n", "#EDATE", "ETIME", "EFLOW");
print_line.printf("%s" + "%n", "######################################################################");
// print the needed values
for (int i=0; i < EDATE.length; i++){
// change the date into format needed for loadest
EDATE[i] = changeDate(EDATE[i]);
print_line.printf("%-12s%-10s%-10s", EDATE[i], ETIME[i], EFLOW[i]);
// print a new line
print_line.printf("%n");
}
// Close the file writer
print_line.close();
writer.close();
System.out.println("Text File located at:\t" + path);
}
/**
* Date Formatter:
* This function will change the date into the format that LOADEST requires
* @param date - Old Format
* @return Date - New Format
*/
private String changeDate(String date) throws IOException {
String year = date.substring(0,4);
String month = date.substring(5,7);
String day = date.substring(8);
date = year + month + day;
return date;
}
/**
* Converts the provided water quality values into mg/l if possible, if not
* possible then throws and exception to choose a new water quality test
* @param units the current units of the water quality test values in CCONC
* @param CCONC an array containing the water quality test values
* @return CCONC multiplied by a conversion factor to convert it into mg/l
* @throws IOException
*/
private ArrayList<String> convertToLOADESTunits(String units, ArrayList<String> CCONC) throws IOException{
//Find the unit conversion for current water quality units
double conversion = 0;
if(units.equalsIgnoreCase("g/cm3") || units.equalsIgnoreCase("g/mL @ 20C")){
conversion = 1000;
}else if(units.equalsIgnoreCase("g/m3")){
conversion = 1000/Math.pow(100,3);
}else if(units.equalsIgnoreCase("mg/l") || units.equalsIgnoreCase("mg/l CaCO3") || units.equalsIgnoreCase("mg/l NH4") ||
units.equalsIgnoreCase("mg/l NO3") || units.equalsIgnoreCase("mg/l PO4") || units.equalsIgnoreCase("mg/l SiO2") ||
units.equalsIgnoreCase("mg/l as H") || units.equalsIgnoreCase("mg/l as N") || units.equalsIgnoreCase("mg/l as Na") ||
units.equalsIgnoreCase("mg/l as P") || units.equalsIgnoreCase("mg/l as S") || units.equalsIgnoreCase("mgC3H6O2/L")){
conversion = 1;
}else if(units.equalsIgnoreCase("mg/mL @25C")){
conversion = 1000;
}else if(units.equalsIgnoreCase("ng/l") || units.equalsIgnoreCase("pg/mL")){
conversion = 1/Math.pow(10,6);
}else if(units.equalsIgnoreCase("ng/m3") || units.equalsIgnoreCase("pg/l")){
conversion = 1/(Math.pow(10,6)*Math.pow(100,3));
}else if(units.equalsIgnoreCase("pg/m3")){
conversion = 1/(Math.pow(10,9)*Math.pow(100,3));
}else if(units.equalsIgnoreCase("ug/L 2,4-D") || units.equalsIgnoreCase("ug/L U3O8") || units.equalsIgnoreCase("ug/L as As") ||
units.equalsIgnoreCase("ug/L as Cl") || units.equalsIgnoreCase("ug/L as N") || units.equalsIgnoreCase("ug/L as P") ||
units.equalsIgnoreCase("ug/l") || units.equalsIgnoreCase("ugAtrazn/L")){
conversion = 1/1000;
}else if(units.equalsIgnoreCase("ug/m3")){
conversion = 1/(1000*Math.pow(100,3));
}else{
ArrayList<String> errorMessage = new ArrayList<String>();
errorMessage.add("LOADEST is only able to operate on water quality units of mg/L. The currently selected water quality test is in units of " + units + ", please select a different water quality test that is in mg/L.");
writeError(errorMessage);
}
//Apply conversion to mg/L of current water quality data
for (int i = 0; i < CCONC.size(); i++){
double value = Double.parseDouble(CCONC.get(i));
CCONC.set(i, String.valueOf(value*conversion));
}
return CCONC;
}
/**
* This will open the file and grab the lines of data from it until it reaches the empty line
* @return returns the entire text file in textData
* @throws IOException
*/
private String[][] OpenLOADESTresultFile(String path) throws IOException {
FileReader fr = new FileReader(path);
BufferedReader textReader = new BufferedReader(fr);
int numberOfLines = readLines(path);
if(numberOfLines == 0){
ArrayList<String> errorMessage = new ArrayList<String>();
errorMessage.add("LOADEST failed to return information for the provided nutrient data, the data is likely too sparce to perform an analysis on, please select a different water quality test, provide more data, or select a shorter analysis period.");
writeError(errorMessage);
}
String[] textData = new String[numberOfLines];
for (int i = 0; i < numberOfLines; i++){
textData[i]=textReader.readLine();
}
textReader.close();
fr.close();
// rctr is used to remove 8 lines of header from the file
int rctr = textData.length - 8;
String[][] dailyData = new String[rctr][2];
for (int i = 0; i < rctr; i++){
dailyData[i][0] = textData[(i+8)].substring(1,9);
dailyData[i][1] = textData[(i+8)].substring(28,38);
}
return dailyData;
}
/**
* This will determine how many lines contain total data. Used to in OpenFile
* @return
* @throws IOException
*/
private int readLines(String path) throws IOException{
FileReader file_to_read = new FileReader(path);
BufferedReader bf = new BufferedReader (file_to_read);
String aLine;
int numberOfLines = 0;
while ((aLine=bf.readLine()) != null){
numberOfLines++;
}
bf.close();
file_to_read.close();
return numberOfLines;
}
/**
* Writes out the error message, if any, for finding the file and then exits the program
* @param error string array to be written as each line of an error message
* @throws IOException
*/
public void writeError(ArrayList<String> error) throws IOException{
//Output data to text file
String errorContents = error.get(0);
for(int i=1; i<error.size(); i++){
errorContents = errorContents + "\n" + error.get(i);
}
throw new IOException("Error encountered. Please see the following message for details: \n" + errorContents);
}
/**
* Main statistics function calls other functions to calculate each statistic value then stores the results as global variables
* @param sortedData data on which statistical values are desired, sortedData[i][0] = date, soredData[i][1] = value
* @param timeStep a flag for which results the statistics will be stored to, either "daily", "monthly", or "yearly"
*/
private void CalculateStatistics(String[][] sortedData, String timeStep) {
//convert Data into proper format for calculations
ArrayList<Double> dataList = new ArrayList<Double>();
for(int i=0; i<sortedData.length; i++){
dataList.add(Double.parseDouble(sortedData[i][1]));
}
//Calculate statistics
DoubleMath doubleMath = new DoubleMath();
double temp1 = doubleMath.round(doubleMath.Min_Max(dataList, true),3);//Call Max function
double temp2 = doubleMath.round(doubleMath.Min_Max(dataList, false),3);//Call Min function
double temp3 = doubleMath.round(doubleMath.Percentile_function(dataList,0.75),3);//Call Upper Quartile function
double temp4 = doubleMath.round(doubleMath.Percentile_function(dataList,0.25),3);//Call Lower Quartile function
double temp5 = doubleMath.round(doubleMath.Median(dataList),3);//Call Median function
double temp6 = doubleMath.round(doubleMath.Average(dataList),3);//Call Mean function
double temp7 = doubleMath.round(doubleMath.StandardDeviationSample(dataList),3);//Call standard deviation
if(timeStep.equalsIgnoreCase("daily")){
dailymax = temp1;
dailymin = temp2;
dailyupperQuartile = temp3;
dailylowerQuartile = temp4;
dailymedian = temp5;
dailymean = temp6;
dailystandardDeviation = temp7;
}else if(timeStep.equalsIgnoreCase("monthly")){
monthlymax = temp1;
monthlymin = temp2;
monthlyupperQuartile = temp3;
monthlylowerQuartile = temp4;
monthlymedian = temp5;
monthlymean = temp6;
monthlystandardDeviation = temp7;
}else if(timeStep.equalsIgnoreCase("yearly")){
yearlymax = temp1;
yearlymin = temp2;
yearlyupperQuartile = temp3;
yearlylowerQuartile = temp4;
yearlymedian = temp5;
yearlymean = temp6;
yearlystandardDeviation = temp7;
}
}
/**
* Primary LOADEST function
* It calls the subfunctions based on user selection/inputs.
* Calls STORET or USGS database queries and their respective subfunctions
* @throws IOException
* @throws InterruptedException
*/
public void run() throws IOException, InterruptedException {
//Inputs
//assert args.length > 0;
//String mainFolder = args[0];
//String fileName = args[1];
//String organizationName = args[2];
//String stationID = args[3];
//String stationName = args[4];
//String wqTest = args[5];
//String beginDate = args[6];
//String endDate = args[7];
//String timeStep = args[8];
//String method = args[9];
//If no date input, make it the maximum of available data
if(beginDate == null || beginDate.equalsIgnoreCase("")){
beginDate = "1850-01-01";
}
if(endDate == null || endDate.equalsIgnoreCase("")){
// Pull current date for upper limit of data search
DateFormat desiredDateFormat = new SimpleDateFormat("yyyy-MM-dd");
Date currentDate = new Date();
endDate = desiredDateFormat.format(currentDate);
}
//Correct Inputs
int endIndex = wqTest.lastIndexOf(", ");//"00597 Dissolved nitrogen gas, water, unfiltered, milligrams per liter -- mg/l"
if(endIndex == -1){
endIndex = wqTest.lastIndexOf("--");
}
//Get Units and conversion for current WQ test
String WQlabel = wqTest.substring(11,endIndex);//cut off the "98335 " part before the test name and the units after the name
wqTest = wqTest.substring(0,5);//pull just the 5 digit USGS WQ code
USGS_Data usgs_Data = new USGS_Data();
String units = usgs_Data.getUSGSwqUnits(wqTest);
String graphUnits = "??";
//Check if any flow and water quality data exists
Data data = new Data();
Object[] returnArray1 = data.extractFlow_and_WQdata(mainFolder, database, organizationName, stationID, beginDate, endDate, userData, wqTest);
String[][] sortableData = (String[][]) returnArray1[0];
String[][] WQdata = (String[][]) returnArray1[1];
//Find the user uploaded data file and uses this if merged
String[][] sortableData_user = new String[0][0];
String[][] WQdata_user = new String[0][0];
if(mergeDatasets){
User_Data user_Data = new User_Data();
Object[] returnArray = user_Data.readUserFileLDC(userData, wqTest, beginDate, endDate);
sortableData_user = (String[][]) returnArray[0];
WQdata_user = (String[][]) returnArray[1];
}
//Sort the Data by date to remove duplicate date entries
DoubleArray doubleArray = new DoubleArray();
String[][] sortedData = doubleArray.removeDuplicateDates(sortableData);
String[][] sortedData_user = doubleArray.removeDuplicateDates(sortableData_user);
//Merge the two datasets (if user data is empty nothing will be merged)
String[][] sortedData_combined = doubleArray.mergeData(sortedData, sortedData_user, mergeMethod);
String[][] WQdata_combined = doubleArray.mergeData(WQdata, WQdata_user, mergeMethod);
//Check if any data exists
ArrayList<String> errorMessage = new ArrayList<String>();
if(sortedData_combined.length == 0){
if(sortedData.length == 0){
errorMessage.add("There is no available flow data in the " + database + " database for station '" + stationID + "' and the specified date range.");
if(database.equalsIgnoreCase("CDWR")){
errorMessage.add("The CDWR database is sensitive to the begin date used, try specifying a later begin date");
}
}
if(sortedData_user.length==0){
errorMessage.add("There is no uploaded flow data for station '" + stationID + "' and the specified date range.");
}
}
if(WQdata_combined.length == 0){
if(WQdata.length == 0){
errorMessage.add("There are no available '" + wqTest + "' water quality tests for station " + stationID+ " by: " +
database + " and the specified date range");
}
if(WQdata_user.length==0){
errorMessage.add("There are no uploaded '" + wqTest + "' water quality tests for station " + stationID+ " and the specified date range");
}
}
if(errorMessage.size() > 0){
writeError(errorMessage);
}
//Write control file
writeControl();
//Write Header File
String[] cname = WQlabel.split(",");
fileName = cname[0].trim();
fileName = fileName.replace(" ", "");//Remove spaces from the name
writeHeader();
// write the calibration variables based on the number of concentration values with a associated flow value
writeCalibration(sortedData_combined, WQdata_combined, units);
//Get estimation dates for LOADEST variables
writeEstimation(sortedData_combined);
//Call LOADEST exe
//Expected Input: "CONTROL.INP", "HEADER.INP", "CALIB.INP", and "EST.INP"
System.out.println("Calling LOADEST.exe");
String binDir = Config.getString("m.bin.dir", "/tmp/csip/bin");
File loadest = Binaries.unpackResource("/bin/win-x86/loadest.exe", new File(binDir));
ProcessComponent pc = new ProcessComponent();
if (File.pathSeparatorChar == ':') {
pc.exe = Config.getString("wine.path", "/usr/bin/wine");
pc.args = new String[]{loadest.toString()};
} else {
pc.exe = loadest.toString();
pc.args = new String[]{};
}
pc.working_dir = mainFolder;
pc.execute();
//Expected Output: "echo.out" and fileName + ".ind", fileName + ".out", fileName + ".res"
if (!new File(mainFolder, "echo.out").exists()) {
throw new FileNotFoundException("echo.out");
}
if (!new File(mainFolder, fileName + ".ind").exists()) {
throw new FileNotFoundException(fileName + ".ind");
}
if (!new File(mainFolder, fileName + ".out").exists()) {
throw new FileNotFoundException(fileName + ".out");
}
if (!new File(mainFolder, fileName + ".res").exists()) {
throw new FileNotFoundException(fileName + ".res");
}
System.out.println("Finished LOADEST.exe");
// Daily result data
String[][] dailyData = OpenLOADESTresultFile(mainFolder + File.separator + fileName + ".ind");
//convert results to daily,monthly,yearly
String[][] dailyresultData = doubleArray.computeFlowMethod(dailyData, "daily", "total", true);
String[][] monthlyresultData = doubleArray.computeFlowMethod(dailyData, "monthly", "total", true);
String[][] yearlyresultData = doubleArray.computeFlowMethod(dailyData, "yearly", "total", true);
// Calculate Daily stats
CalculateStatistics(dailyresultData, "daily");
double dailyIQR = dailyupperQuartile - dailylowerQuartile;// Find IQR
ArrayList<Double> dailyoutliers = new ArrayList<Double>();// Get daily outliers
for (int i = 0; i < dailyresultData.length; i++){
double value = Double.parseDouble(dailyresultData[i][1]);
if(value < (dailylowerQuartile - 1.5 * dailyIQR) || value > (dailyupperQuartile + 1.5 * dailyIQR)){
dailyoutliers.add(value);
}
}
// Calculate Monthly stats
CalculateStatistics(monthlyresultData, "monthly");
double monthlyIQR = monthlyupperQuartile - monthlylowerQuartile;// Find IQR
ArrayList<Double> monthlyoutliers = new ArrayList<Double>();// Get monthly outliers
for (int i = 0; i < monthlyresultData.length; i++){
double value = Double.parseDouble(monthlyresultData[i][1]);
if(value < (monthlylowerQuartile - 1.5 * monthlyIQR) || value > (monthlyupperQuartile + 1.5 * monthlyIQR)){
monthlyoutliers.add(value);
}
}
// Calculate Yearly stats
CalculateStatistics(yearlyresultData, "yearly");
double yearlyIQR = yearlyupperQuartile - yearlylowerQuartile;// Find IQR
ArrayList<Double> yearlyoutliers = new ArrayList<Double>();// Get yearly outliers
for (int i = 0; i < yearlyresultData.length; i++){
double value = Double.parseDouble(yearlyresultData[i][1]);
if(value < (yearlylowerQuartile - 1.5 * yearlyIQR) || value > (yearlyupperQuartile + 1.5 * yearlyIQR)){
yearlyoutliers.add(value);
}
}
//Prep. boxplot data for output function
ArrayList<ArrayList<Double>> dailyBoxplotOutliers = new ArrayList<ArrayList<Double>>();
ArrayList<ArrayList<Double>> monthlyBoxplotOutliers = new ArrayList<ArrayList<Double>>();
ArrayList<ArrayList<Double>> yearlyBoxplotOutliers = new ArrayList<ArrayList<Double>>();
dailyBoxplotOutliers.add(dailyoutliers);
monthlyBoxplotOutliers.add(monthlyoutliers);
yearlyBoxplotOutliers.add(yearlyoutliers);
double[][] dailyBoxplotData = {{dailymax},{dailyupperQuartile},{dailymedian},{dailylowerQuartile},{dailymin}};
double[][] monthlyBoxplotData = {{monthlymax},{monthlyupperQuartile},{monthlymedian},{monthlylowerQuartile},{monthlymin}};
double[][] yearlyBoxplotData = {{yearlymax},{yearlyupperQuartile},{yearlymedian},{yearlylowerQuartile},{yearlymin}};
//Put graph file writer call here
doubleArray.writeTimeSeries(mainFolder, dailyresultData, "daily", getDailyTimeseries().getName(), true);
doubleArray.writeTimeSeries(mainFolder, monthlyresultData, "monthly", getMonthlyTimeseries().getName(), true);
doubleArray.writeTimeSeries(mainFolder, yearlyresultData, "yearly", getYearlyTimeseries().getName(), true);
doubleArray.writeBoxplot(mainFolder, dailyBoxplotOutliers, dailyBoxplotData, getDailyBoxplot().getName());
doubleArray.writeBoxplot(mainFolder, monthlyBoxplotOutliers, monthlyBoxplotData, getMonthlyBoxplot().getName());
doubleArray.writeBoxplot(mainFolder, yearlyBoxplotOutliers, yearlyBoxplotData, getYearlyBoxplot().getName());
// Find what units that LOADEST has given the Load in
if (ULFLAG == 1){
graphUnits = "kg";
}else if (ULFLAG ==2){
graphUnits = "mg";
}else if (ULFLAG == 3){
graphUnits = "lb";
}else{
graphUnits = "tons";
}
//Get today's date for the source reference
Date currentDate = new Date();
SimpleDateFormat sourceDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm");
String today = sourceDateFormat.format(currentDate);
if(database.equalsIgnoreCase("USGS")){
this.dataSource = "Stream flow data and water quality test data courtesy of the U.S. Geological Survey, National Water Information System: Web Interface. http://waterdata.usgs.gov/nwis, accessed: " + today;
}else if(database.equalsIgnoreCase("STORET")){
this.dataSource = "Stream flow data and water quality test data courtesy of the U.S. Environmental Protection Agency, STORET. http://www.epa.gov/storet/index.html accessed: " + today;
}else if(database.equalsIgnoreCase("CDWR")){
this.dataSource = "Stream flow data and water quality test data courtesy of the Colorado Division of Water Resources, CDWR. http://www.dwr.state.co.us accessed: " + today;
}
this.start = sortedData[0][0];
this.end = sortedData[sortedData.length - 1][0];
this.flowLen = String.valueOf(sortedData.length);
this.wqUnits = graphUnits;
}
public static void main(String[] args) throws IOException, InterruptedException, Exception {
guiLOADEST_Model loadest_Model = new guiLOADEST_Model();
//Run model
loadest_Model.run();
}
}