guiLOADEST_Model.java [src/java/m/cfa/loadest] Revision: default  Date:
package m.cfa.loadest;

import WaterData.WaterData;
import WaterData.WaterDataException;
import WaterData.WaterDataInterface;
import WaterData.WaterQualityInfo;
import m.cfa.DoubleArray;
import m.cfa.DoubleMath;
import csip.api.server.Executable;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import org.codehaus.jettison.json.JSONException;

/**
* Last Updated: 9-April-2019
* @author Tyler Wible & Tyler Dell
* @since 27-March-2013
*/
public class guiLOADEST_Model {
    //Inputs
    String directory = "C:/Projects/TylerWible_repos/NetBeans/data/CFA/LOADEST";
    String database = "USGS";//"CDWR";//"STORET";//"CDSN";//"UserData";//
    String orgId = "n/a";//"n/a";//"21COL001";//"CITYFTCO_WQX";//"n/a";//
    String stationId = "06764880";//"CLAGRECO";//"000028";//"1EFF";//"n/a";//
    String stationName = "South Platte River at Roscoe, Nebr.";//"Cache La Poudre Near Greeley";//"BIG THOMPSON R NEAR MOUTH";//"n/a";//"n/a";//
    String wqTest = "00600 Total nitrogen, water, unfiltered, milligrams per liter -- mg/L";//"00625 Ammonia-nitrogen as N -- mg/L";//
    int PTOPT = 1; // estimated values print option (0,1)
    int SEOPT = 2; // standard error option (1-3)
    int LDOPT = 0; // load option (0-3), if load option is 1 or 3 then need the number of user-defined seasons
    int modelNumber = 11;  // (0-11, 99) this picks which model to run if 0 then chooses best model from 1-9 based on error values    
    // if model number is 10, 11 user must specify 
    int PBMON = 5; // Begin month for periodic terms
    int PEMON = 7; // End month for periodic terms
    int ULFLAG = 4; // the desired output units (1-4)
    String startDate = "";
    String endDate = "";
    boolean mergeDatasets = false;//true;// 
    String mergeMethod = "user";//"public";//"max";//"average";//"min";//
    String userData = "";//"Date\tFlow\n1994-01-01\t37\n1994-01-02\t39\n1994-01-03\t40\n1994-01-04\t40\n1994-01-05\t40\n1994-01-06\t38\n1994-01-07\t44\n1994-01-08\t37\n1994-01-09\t37\n1994-01-10\t37\n1994-01-11\t35\n1994-01-12\t31\n1994-01-13\t31\n1994-01-14\t27\n1994-01-15\t27\n1994-01-16\t32\n1994-01-17\t25\n1994-01-18\t25\n1994-01-19\t29\n1994-01-20\t32\n1994-01-21\t31\n1994-01-22\t24\n1994-01-23\t29\n1994-01-24\t26\n1994-01-25\t31\n1994-01-26\t30\n1994-01-27\t30\n1994-01-28\t30\n1994-01-29\t24\n1994-01-30\t25\n1994-01-31\t24\n1994-02-01\t28\n1994-02-02\t26\n1994-02-03\t25\n1994-02-04\t26\n1994-02-05\t27\n1994-02-06\t26\n1994-02-07\t25\n1994-02-08\t27\n1994-02-09\t23\n1994-02-10\t22\n1994-02-11\t26\n1994-02-12\t29\n1994-02-13\t26\n1994-02-14\t26\n1994-02-15\t38\n1994-02-16\t40\n1994-02-17\t42\n1994-02-18\t41\n1994-02-19\t44\n1994-02-20\t44\n1994-02-21\t41\n1994-02-22\t34\n1994-02-23\t30\n1994-02-24\t34\n1994-02-25\t34\n1994-02-26\t36\n1994-02-27\t39\n1994-02-28\t54\n1994-03-01\t50\n1994-03-02\t49\n1994-03-03\t51\n1994-03-04\t51\n1994-03-05\t52\n1994-03-06\t54\n1994-03-07\t56\n1994-03-08\t51\n1994-03-09\t43\n1994-03-10\t54\n1994-03-11\t50\n1994-03-12\t46\n1994-03-13\t49\n1994-03-14\t51\n1994-03-15\t50\n1994-03-16\t48\n1994-03-17\t32\n1994-03-18\t56\n1994-03-19\t55\n1994-03-20\t59\n1994-03-21\t55\n1994-03-22\t43\n1994-03-23\t48\n1994-03-24\t46\n1994-03-25\t42\n1994-03-26\t43\n1994-03-27\t35\n1994-03-28\t28\n1994-03-29\t42\n1994-03-30\t38\n1994-03-31\t40\n1994-04-01\t41\n1994-04-02\t41\n1994-04-03\t39\n1994-04-04\t39\n1994-04-05\t41\n1994-04-06\t38\n1994-04-07\t36\n1994-04-08\t35\n1994-04-09\t37\n1994-04-10\t43\n1994-04-11\t41\n1994-04-12\t56\n1994-04-13\t56\n1994-04-14\t52\n1994-04-15\t56\n1994-04-16\t53\n1994-04-17\t56\n1994-04-18\t64\n1994-04-19\t69\n1994-04-20\t76\n1994-04-21\t87\n1994-04-22\t102\n1994-04-23\t130\n1994-04-24\t171\n1994-04-25\t244\n1994-04-26\t223\n1994-04-27\t176\n1994-04-28\t163\n1994-04-29\t158\n1994-04-30\t137\n1994-05-01\t138\n1994-05-02\t138\n1994-05-03\t136\n1994-05-04\t149\n1994-05-05\t149\n1994-05-06\t147\n1994-05-07\t188\n1994-05-08\t291\n1994-05-09\t401\n1994-05-10\t528\n1994-05-11\t476\n1994-05-12\t598\n1994-05-13\t752\n1994-05-14\t874\n1994-05-15\t1030\n1994-05-16\t1050\n1994-05-17\t1310\n1994-05-18\t1180\n1994-05-19\t1130\n1994-05-20\t1270\n1994-05-21\t1100\n1994-05-22\t1110\n1994-05-23\t1200\n1994-05-24\t1090\n1994-05-25\t1090\n1994-05-26\t1180\n1994-05-27\t1160\n1994-05-28\t1270\n1994-05-29\t1350\n1994-05-30\t1390\n1994-05-31\t1490\n1994-06-01\t1590\n1994-06-02\t1380\n1994-06-03\t1220\n1994-06-04\t1210\n1994-06-05\t1250\n1994-06-06\t1270\n1994-06-07\t1190\n1994-06-08\t1010\n1994-06-09\t1040\n1994-06-10\t953\n1994-06-11\t960\n1994-06-12\t916\n1994-06-13\t757\n1994-06-14\t692\n1994-06-15\t702\n1994-06-16\t722\n1994-06-17\t714\n1994-06-18\t700\n1994-06-19\t703\n1994-06-20\t767\n1994-06-21\t819\n1994-06-22\t764\n1994-06-23\t764\n1994-06-24\t686\n1994-06-25\t585\n1994-06-26\t499\n1994-06-27\t420\n1994-06-28\t385\n1994-06-29\t335\n1994-06-30\t278\n1994-07-01\t246\n1994-07-02\t274\n1994-07-03\t291\n1994-07-04\t312\n1994-07-05\t254\n1994-07-06\t175\n1994-07-07\t226\n1994-07-08\t297\n1994-07-09\t233\n1994-07-10\t155\n1994-07-11\t219\n1994-07-12\t254\n1994-07-13\t261\n1994-07-14\t268\n1994-07-15\t229\n1994-07-16\t185\n1994-07-17\t185\n1994-07-18\t267\n1994-07-19\t305\n1994-07-20\t303\n1994-07-21\t300\n1994-07-22\t276\n1994-07-23\t269\n1994-07-24\t327\n1994-07-25\t308\n1994-07-26\t296\n1994-07-27\t268\n1994-07-28\t248\n1994-07-29\t224\n1994-07-30\t215\n1994-07-31\t203\n1994-08-01\t237\n1994-08-02\t302\n1994-08-03\t350\n1994-08-04\t345\n1994-08-05\t346\n1994-08-06\t336\n1994-08-07\t315\n1994-08-08\t293\n1994-08-09\t289\n1994-08-10\t296\n1994-08-11\t319\n1994-08-12\t316\n1994-08-13\t305\n1994-08-14\t306\n1994-08-15\t291\n1994-08-16\t189\n1994-08-17\t160\n1994-08-18\t144\n1994-08-19\t154\n1994-08-20\t251\n1994-08-21\t264\n1994-08-22\t255\n1994-08-23\t245\n1994-08-24\t243\n1994-08-25\t227\n1994-08-26\t207\n1994-08-27\t147\n1994-08-28\t132\n1994-08-29\t122\n1994-08-30\t70\n1994-08-31\t68\n1994-09-01\t57\n1994-09-02\t61\n1994-09-03\t82\n1994-09-04\t102\n1994-09-05\t119\n1994-09-06\t119\n1994-09-07\t123\n1994-09-08\t108\n1994-09-09\t76\n1994-09-10\t48\n1994-09-11\t47\n1994-09-12\t45\n1994-09-13\t43\n1994-09-14\t47\n1994-09-15\t53\n1994-09-16\t46\n1994-09-17\t35\n1994-09-18\t35\n1994-09-19\t34\n1994-09-20\t35\n1994-09-21\t40\n1994-09-22\t72\n1994-09-23\t83\n1994-09-24\t65\n1994-09-25\t46\n1994-09-26\t39\n1994-09-27\t33\n1994-09-28\t26\n1994-09-29\t23\n1994-09-30\t22\n1994-10-01\t20\n1994-10-02\t27\n1994-10-03\t28\n1994-10-04\t33\n1994-10-05\t27\n1994-10-06\t23\n1994-10-07\t31\n1994-10-08\t28\n1994-10-09\t24\n1994-10-10\t21\n1994-10-11\t21\n1994-10-12\t21\n1994-10-13\t21\n1994-10-14\t18\n1994-10-15\t18\n1994-10-16\t18\n1994-10-17\t26\n1994-10-18\t27\n1994-10-19\t19\n1994-10-20\t12\n1994-10-21\t15\n1994-10-22\t13\n1994-10-23\t17\n1994-10-24\t21\n1994-10-25\t17\n1994-10-26\t18\n1994-10-27\t17\n1994-10-28\t18\n1994-10-29\t22\n1994-10-30\t24\n1994-10-31\t28\n1994-11-01\t21\n1994-11-02\t17\n1994-11-03\t22\n1994-11-04\t23\n1994-11-05\t13\n1994-11-06\t25\n1994-11-07\t30\n1994-11-08\t32\n1994-11-09\t25\n1994-11-10\t15\n1994-11-11\t18\n1994-11-12\t22\n1994-11-13\t28\n1994-11-14\t21\n1994-11-15\t6.8\n1994-11-16\t6.0\n1994-11-17\t15\n1994-11-18\t18\n1994-11-19\t23\n1994-11-20\t28\n1994-11-21\t34\n1994-11-22\t36\n1994-11-23\t21\n1994-11-24\t13\n1994-11-25\t23\n1994-11-26\t37\n1994-11-27\t29\n1994-11-28\t10\n1994-11-29\t11\n1994-11-30\t15\n1994-12-01\t28\n1994-12-02\t60\n1994-12-03\t62\n1994-12-04\t36\n1994-12-05\t22\n1994-12-06\t31\n1994-12-07\t23\n1994-12-08\t25\n1994-12-09\t12\n1994-12-10\t15\n1994-12-11\t24\n1994-12-12\t24\n1994-12-13\t31\n1994-12-14\t35\n1994-12-15\t34\n1994-12-16\t37\n1994-12-17\t32\n1994-12-18\t21\n1994-12-19\t21\n1994-12-20\t22\n1994-12-21\t22\n1994-12-22\t22\n1994-12-23\t19\n1994-12-24\t21\n1994-12-25\t22\n1994-12-26\t22\n1994-12-27\t23\n1994-12-28\t23\n1994-12-29\t22\n1994-12-30\t26\n1994-12-31\t22\n1995-01-01\t22"
           //+ "$$Date\t00600\n1994-01-03\tnodata\n1994-01-13\t0.28\n1994-02-16\t0.29\n1994-03-07\t0.25\n1994-04-20\t0.25\n1994-05-03\t0.25\n1994-06-07\t0.25\n1994-06-14\t0.25\n1994-07-06\t0.25\n1994-08-04\t0.25\n1994-09-02\t0.28\n1994-10-11\t0.25\n1994-11-22\t0.33\n1994-12-13\t0.59";
    
    //Global variable but not an input
    String fileName = "";
 
    //Outputs
    String flowLen = "-1";
    String wqLen = "-1";
    String wqUnits = "?";
    String start = "?";
    String end = "?";
    String dataSource = "?";
    //Outputs
    double dailymax = -1;
    double dailymin = -1;
    double dailyupperQuartile = -1;
    double dailylowerQuartile = -1;
    double dailymedian = -1;
    double dailymean = -1;
    double dailystandardDeviation = -1;
    
    double monthlymax = -1;
    double monthlymin = -1;
    double monthlyupperQuartile = -1;
    double monthlylowerQuartile = -1;
    double monthlymedian = -1;
    double monthlymean = -1;
    double monthlystandardDeviation = -1;
    
    double yearlymax = -1;
    double yearlymin = -1;
    double yearlyupperQuartile = -1;
    double yearlylowerQuartile = -1;
    double yearlymedian = -1;
    double yearlymean = -1;
    double yearlystandardDeviation = -1;

    //Gets
//    public File getOutputSummary() {
//        return new File(directory, "echo.out");
//    }
    public File getOutputResult1() { return new File(directory, fileName + ".ind"); }
    public File getOutputResult2() { return new File(directory, fileName + ".out"); }
    public File getOutputResult3() { return new File(directory, fileName + ".res"); }
    public File getDailyTimeseries(){ return new File(directory, "daily_graph.out"); }//for use with JSHighCharts
    public File getDailyBoxplot(){ return new File(directory, "daily_boxplot.out"); }//for use with JSHighCharts
    public File getMonthlyTimeseries(){ return new File(directory, "monthly_graph.out"); }//for use with JSHighCharts
    public File getMonthlyBoxplot(){ return new File(directory, "monthly_boxplot.out"); }//for use with JSHighCharts
    public File getYearlyTimeseries(){ return new File(directory, "yearly_graph.out"); }//for use with JSHighCharts
    public File getYearlyBoxplot(){ return new File(directory, "yearly_boxplot.out"); }//for use with JSHighCharts
    public String getFlowLen() { return flowLen; }
    public String getWQLen() { return wqLen; }
    public String getWQUnits() { return wqUnits; }
    public String getStart(){ return start; }
    public String getEnd(){ return end; }
    public String getDataSource(){ return dataSource; }
    public String getDailyMax(){ return String.valueOf(dailymax); }
    public String getDailyMin(){ return String.valueOf(dailymin); }
    public String getDailyUpperQuartile(){ return String.valueOf(dailyupperQuartile); }
    public String getDailyLowerQuartile(){ return String.valueOf(dailylowerQuartile); }
    public String getDailyMedian(){ return String.valueOf(dailymedian); }
    public String getDailyMean(){ return String.valueOf(dailymean); }
    public String getDailyStandardDeviation(){ return String.valueOf(dailystandardDeviation); }
    public String getMonthlyMax(){ return String.valueOf(monthlymax); }
    public String getMonthlyMin(){ return String.valueOf(monthlymin); }
    public String getMonthlyUpperQuartile(){ return String.valueOf(monthlyupperQuartile); }
    public String getMonthlyLowerQuartile(){ return String.valueOf(monthlylowerQuartile); }
    public String getMonthlyMedian(){ return String.valueOf(monthlymedian); }
    public String getMonthlyMean(){ return String.valueOf(monthlymean); }
    public String getMonthlyStandardDeviation(){ return String.valueOf(monthlystandardDeviation); }
    public String getYearlyMax(){ return String.valueOf(yearlymax); }
    public String getYearlyMin(){ return String.valueOf(yearlymin); }
    public String getYearlyUpperQuartile(){ return String.valueOf(yearlyupperQuartile); }
    public String getYearlyLowerQuartile(){ return String.valueOf(yearlylowerQuartile); }
    public String getYearlyMedian(){ return String.valueOf(yearlymedian); }
    public String getYearlyMean(){ return String.valueOf(yearlymean); }
    public String getYearlyStandardDeviation(){ return String.valueOf(yearlystandardDeviation); }
    
    //Sets
    public void setDirectory(String directory_str){ directory = directory_str; }
    public void setDatabase(String database_str){ database = database_str; }
    public void setOrganizationID(String orgId_str){ orgId = orgId_str; }
    public void setStationName(String stationName_str){ stationName = stationName_str; }
    public void setStationId(String stationId_str){ stationId = stationId_str; }
    public void setWQtest(String wqTest_str){ wqTest = wqTest_str; }
    public void setPTOPT(int PTOPT_int){ PTOPT = PTOPT_int; }
    public void setSEOPT(int SEOPT_int){ SEOPT = SEOPT_int; }
    public void setLDOPT(int LDOPT_int){ LDOPT = LDOPT_int; }
    public void setmodelNumber(int modelNumber_int){ modelNumber = modelNumber_int; }
    public void setPBMON(int PBMON_int){ PBMON = PBMON_int; }
    public void setPEMON(int PEMON_int){ PEMON = PEMON_int; }
    public void setULFLAG (int ULFLAG_int){ ULFLAG = ULFLAG_int; }
    public void setStartDate(String startDate_str){ startDate = startDate_str; }
    public void setEndDate(String endDate_str){ endDate = endDate_str; }
    public void setMergeDatasets(boolean mergeDatasets_TF){ mergeDatasets = mergeDatasets_TF; }
    public void setMergeMethod(String mergeMethod_str){ mergeMethod = mergeMethod_str; }
    public void setUserData(String userData_str){ userData = userData_str; }
    /**
     * Control File Writer:
     * This function will write the control file which is what LOADEST will look for to find the 
     * header, calibration, and estimation files.
     * @throws IOException
     */
    private void writeControl()throws IOException {
        String path = directory + File.separator + "CONTROL.INP";
        FileWriter writer =  new FileWriter(path, false);
        PrintWriter print_line = new PrintWriter(writer);

        //Create the control file
        print_line.printf("%s" + "\r\n", "######################################################################");
        print_line.printf("%s" + "\r\n", "#");
        print_line.printf("%s" + "\r\n", "#  LOADEST Control File");
        print_line.printf("%s" + "\r\n", "#");
        print_line.printf("%s" + "\r\n", "#");
        print_line.printf("%s" + "\r\n", "# line        name of the:");
        print_line.printf("%s" + "\r\n", "# ----        ------------");
        print_line.printf("%s" + "\r\n", "#  1          header file");
        print_line.printf("%s" + "\r\n", "#  2          calibration file");
        print_line.printf("%s" + "\r\n", "#  3          estimation file");
        print_line.printf("%s" + "\r\n", "#");
        print_line.printf("%s" + "\r\n", "######################################################################");
        print_line.printf("%s" + "\r\n", "HEADER.INP");
        print_line.printf("%s" + "\r\n", "CALIB.INP");
        print_line.printf("%s" + "\r\n", "EST.INP");

        print_line.close();
        writer.close();
        System.out.println("Text File located at:\t" + path);
    }
    /**
     * Header File Writer:
     * This function will write the header file that is needed for LOADEST
     * @throws IOException
     */
    private void writeHeader()throws IOException {
        String path = directory + File.separator + "HEADER.INP";
        FileWriter writer =  new FileWriter(path, false);
        PrintWriter print_line = new PrintWriter(writer);

        // Introduction for header file
        print_line.printf("%s" + "\r\n", "######################################################################");
        print_line.printf("%s" + "\r\n", "#");
        print_line.printf("%s" + "\r\n", "#  LOADEST Header File");
        print_line.printf("%s" + "\r\n", "#");
        print_line.printf("%s" + "\r\n", "# " + stationId + " " + stationName + " Database: " + database + ", Organization ID: " + orgId);
        print_line.printf("%s" + "\r\n", "#");
        print_line.printf("%s" + "\r\n", "######################################################################");

        // Inputs 1-4 Title, estimated values print option, standard error option, and load option 
        print_line.printf("%s" + "\r\n",stationId + " " +stationName + " Database: " + database + ", Organization ID: " + orgId);
        print_line.printf("%s" + "\r\n", PTOPT);
        print_line.printf("%s" + "\r\n", SEOPT);
        print_line.printf("%s" + "\r\n", LDOPT);
        print_line.printf("%s" + "\r\n", "###########################################################");

        // Inputs 5-6 are dependent on what the user defines for the load option 
        // 5 is the number of seasons desired by the user and 6 is the start and end dates for each season
        // do not currenlty allow user to define seasons, only give option for yearly or monthly estimates
        // i.e. LDOPT = 0 or 2, but if want to define seasons modify code below
//        if (LDOPT == 1 || LDOPT == 3){  
//            print_line.printf("%s" + "\r\n", NSEAS);
//            print_line.printf("%s" + "\r\n", "#");
//            print_line.printf("%s" + "\r\n", "#SBEG      SEND");
//            int NSEAS = 1;
//            for (int i=0; i < NSEAS; i++){
//                print_line.printf("%s" + "\r\n",seasonBegin[i] + "       " + seasonEnd[i] );
//            }
//            print_line.printf("%s" + "\r\n", "###########################################################");
//        }

        // Input 7 the model number
        print_line.printf("%s" + "\r\n", modelNumber);
        print_line.printf("%s" + "\r\n", "###########################################################");
        // Inputs 8-11 are dependent on what the user defines for the model number.
        //If model number is 10 or 11
        if (modelNumber == 10 || modelNumber == 11){
            print_line.printf("%s" + "\r\n", "#PBMN  PEMN");
            print_line.printf("%s" + "\r\n", PBMON + "       " + PEMON); // input 8
            print_line.printf("%s" + "\r\n", "###########################################################");
        }

        // Input 12, number of constituents = 1
        print_line.printf("%s" + "\r\n", "1");
        print_line.printf("%s" + "\r\n", "###########################################################");

        // Input 13, is dependent on what the user defines for the number of constituents
        if (fileName.length() > 43){
            fileName = fileName.substring(0,43);
        }
        print_line.printf("%-45s%-5s%-5s" + "\r\n", fileName, "1", ULFLAG);  // 1 denotes concentration in mg/L 
        print_line.printf("%s" + "\r\n", "###########################################################");


//        //Listed below is inputs 12 and 13 for nconst > 1
//        // must make sure to redefine cname, ucflag, and ulflag
//        // Input 12, number of constituents
//        print_line.printf("%s" + "\r\n", nconst);
//        print_line.printf("%s" + "\r\n", "###########################################################");
//
//        // Input 13, is dependent on what the user defines for the number of constituents
//        for (int j=0; j < nconst; j++){
//            if (cname[j].length() > 43){
//                cname[j] = cname[j].substring(0,43);
//            }
//            print_line.printf("%-45s%-5s%-5s" + "\r\n", cname[j], ucflag[j], ulflag[j]); 
//
//        }
//        print_line.printf("%s" + "\r\n", "###########################################################");

        print_line.close();
        writer.close();
        System.out.println("Text File located at:\t" + path);
    }
    /**
     * Calibration File Writer:
     * This function will write the calibration file that is needed for LOADEST
     * @param  sortedData_combined  the flow data, sortedData_combined[i][0] = date (yyyy-mm-dd), sortedData_combined[i][1] = value (cfs)
     * @param  WQdata_combined  the water quality data, WQdata_combined[i][0] = date (yyyy-mm-dd), WQdata_combined[i][1] = value (units)
     * @param units  the units of the water quality data
     * @throws IOException
     */
    private void writeCalibration(String[][] sortedData_combined, String[][] WQdata_combined, String units) throws IOException {
        //Convert data for Calibration file
        ArrayList<String> CDATE = new ArrayList<>();
        ArrayList<String> CTIME = new ArrayList<>();
        ArrayList<String> CFLOW = new ArrayList<>();
        ArrayList<String> CCONC = new ArrayList<>();
        for(int i=0;i<WQdata_combined.length;i++){
            for(int j=0; j<sortedData_combined.length;j++){
                if(WQdata_combined[i][0].equalsIgnoreCase(sortedData_combined[j][0])){
                    //Keep the data with both a water quality test and a flow value
                    CDATE.add(WQdata_combined[i][0]);
                    CTIME.add("1200");
                    CFLOW.add(sortedData_combined[j][1]);
                    CCONC.add(WQdata_combined[i][1]); // need to make sure that CCONC is in mg/L
                }
            }
        }
        CCONC = convertToLOADESTunits(units,CCONC);
        //Store result for analysis summary
        wqLen = String.valueOf(CCONC.size());
        
        
        String path = directory + File.separator + "CALIB.INP";
        FileWriter writer =  new FileWriter(path, false);
        PrintWriter print_line = new PrintWriter(writer);
                          
        // Introduction for header file
        print_line.printf("%s" + "\r\n", "##########################################################################################################");
        print_line.printf("%s" + "\r\n", "#");
        print_line.printf("%s" + "\r\n", "#  LOADEST Calibration File");
        print_line.printf("%s" + "\r\n", "#");
        print_line.printf("%s" + "\r\n", "# " + stationId + " " + stationName + " Database: " + database + ", Organization ID: " + orgId);
        print_line.printf("%s" + "\r\n", "#");
        print_line.printf("%s" + "\r\n", "##########################################################################################################");
        print_line.printf("%s" + "\r\n", "#");
        print_line.printf("%s" + "\r\n", "#");
        print_line.printf("%-12s%-10s%-10s%-10s" + "\r\n", "#CDATE","CTIME","CFLOW","CCONC");
        print_line.printf("%s" + "\r\n", "##########################################################################################################");
        print_line.printf("%s" + "\r\n", "#");
        
        // Input the data
        for(int i=0; i < CDATE.size(); i++){
            // change the date into format needed for loadest
            CDATE.set(i, changeDate(CDATE.get(i)));
            
            print_line.printf("%-12s%-10s%-10s", CDATE.get(i), CTIME.get(i), CFLOW.get(i));
            // add in concentrations
            //Arbitrarily define the number of water quality parameters equal to 1, this can be modified later if need be
            //for (int j=0; j < nconst; j++){// column counter if multiple concentrations are desired
            //    print_line.printf("%-10s",CCONC[i][j]);
            //}
            print_line.printf("%-10s",CCONC.get(i));
            
            // print a new line
            print_line.printf("\r\n");
        }
        print_line.close();
        writer.close();
        System.out.println("Text File located at:\t" + path);
    }
    /**
     * Estimation File Writer:
     * This function will write the estimation file needed for LOADEST
     * @throws IOException
     */
    private void writeEstimation(String[][] sortedData_combined) throws IOException {
        //Convert data for Estimation file
        String[] EDATE = new String[sortedData_combined.length];
        String[] ETIME = new String[sortedData_combined.length];
        String[] EFLOW = new String[sortedData_combined.length];
        for(int i=0; i<sortedData_combined.length; i++){
            EDATE[i] = sortedData_combined[i][0];
            ETIME[i] = "1200";
            EFLOW[i] = sortedData_combined[i][1];
        }
        
        // open the file writer and set path
        String path = directory + File.separator + "EST.INP";
        FileWriter writer =  new FileWriter(path, false);
        PrintWriter print_line = new PrintWriter(writer);
        
        // Introduction for header file
        print_line.printf("%s" + "\r\n", "######################################################################");
        print_line.printf("%s" + "\r\n", "#");
        print_line.printf("%s" + "\r\n", "#  LOADEST Estimation File");
        print_line.printf("%s" + "\r\n", "#");
        print_line.printf("%s" + "\r\n", "# " + stationId + " " + stationName + " " + "Organization ID: " + orgId);
        print_line.printf("%s" + "\r\n", "#");
        print_line.printf("%s" + "\r\n", "######################################################################");
        print_line.printf("%s" + "\r\n", "# Number of oberservations per day");
        print_line.printf("%s" + "\r\n", "1");
        print_line.printf("%s" + "\r\n", "#");
        print_line.printf("%s" + "\r\n", "######################################################################");
        print_line.printf("%s" + "\r\n", "#");
        print_line.printf("%-12s%-10s%-10s" + "\r\n", "#EDATE", "ETIME", "EFLOW");
        print_line.printf("%s" + "\r\n", "######################################################################");
        
        // print the needed values
        for (int i=0; i < EDATE.length; i++){
            // change the date into format needed for loadest
            EDATE[i] = changeDate(EDATE[i]);
            
            print_line.printf("%-12s%-10s%-10s", EDATE[i], ETIME[i], EFLOW[i]);
            // print a new line
            print_line.printf("\r\n");
        }
        
        // Close the file writer 
        print_line.close();
        writer.close();
        System.out.println("Text File located at:\t" + path);
    }
    /**
     * Date Formatter: 
     * This function will change the date into the format that LOADEST requires
     * @param date - Old Format
     * @return Date - New Format
     */
    private String changeDate(String date){
        String year = date.substring(0,4); 
        String month = date.substring(5,7);
        String day = date.substring(8);
        
        date = year + month + day;
        return date;
    }
    /**
     * Converts the provided water quality values into mg/l if possible, if not 
     * possible then throws and exception to choose a new water quality test
     * @param units  the current units of the water quality test values in CCONC
     * @param CCONC  an array containing the water quality test values
     * @return  CCONC multiplied by a conversion factor to convert it into mg/l 
     * @throws IOException 
     */
    private ArrayList<String> convertToLOADESTunits(String units, ArrayList<String> CCONC) throws IOException{
        //Find the unit conversion for current water quality units
        double conversion = 0;
        if(units.equalsIgnoreCase("g/cm3") || units.equalsIgnoreCase("g/mL @ 20C")){
            conversion = 1000;
        }else if(units.equalsIgnoreCase("g/m3")){
            conversion = 1000/Math.pow(100,3);
        }else if(units.equalsIgnoreCase("mg/l") || units.equalsIgnoreCase("mg/l CaCO3") || units.equalsIgnoreCase("mg/l NH4") || 
                    units.equalsIgnoreCase("mg/l NO3") || units.equalsIgnoreCase("mg/l PO4") || units.equalsIgnoreCase("mg/l SiO2") || 
                    units.equalsIgnoreCase("mg/l as H") || units.equalsIgnoreCase("mg/l as N") || units.equalsIgnoreCase("mg/l as Na") || 
                    units.equalsIgnoreCase("mg/l as P") || units.equalsIgnoreCase("mg/l as S") || units.equalsIgnoreCase("mgC3H6O2/L")){
            conversion = 1;
        }else if(units.equalsIgnoreCase("mg/mL @25C")){
            conversion = 1000;
        }else if(units.equalsIgnoreCase("ng/l") || units.equalsIgnoreCase("pg/mL")){
            conversion = 1/Math.pow(10,6);
        }else if(units.equalsIgnoreCase("ng/m3") || units.equalsIgnoreCase("pg/l")){
            conversion = 1/(Math.pow(10,6)*Math.pow(100,3));
        }else if(units.equalsIgnoreCase("pg/m3")){
            conversion = 1/(Math.pow(10,9)*Math.pow(100,3));
        }else if(units.equalsIgnoreCase("ug/L 2,4-D") || units.equalsIgnoreCase("ug/L U3O8") || units.equalsIgnoreCase("ug/L as As") || 
                    units.equalsIgnoreCase("ug/L as Cl") || units.equalsIgnoreCase("ug/L as N") || units.equalsIgnoreCase("ug/L as P") || 
                    units.equalsIgnoreCase("ug/l") || units.equalsIgnoreCase("ugAtrazn/L")){
            conversion = 1/1000;
        }else if(units.equalsIgnoreCase("ug/m3")){
            conversion = 1/(1000*Math.pow(100,3));
        }else{
            ArrayList<String> errorMessage = new ArrayList<>();
            errorMessage.add("LOADEST is only able to operate on water quality units of mg/L. The currently selected water quality test is in units of " + units + ", please select a different water quality test that is in mg/L.");
            writeError(errorMessage);
        }
        
        //Apply conversion to mg/L of current water quality data
        for (int i = 0; i < CCONC.size(); i++){
            double value = Double.parseDouble(CCONC.get(i));
            CCONC.set(i, String.valueOf(value*conversion));
        }
        
        return CCONC;
    }
    private String[][] removeZeroFlowData(String[][] flowData){
        //Loop through flow data and only keey data greater than zero
        ArrayList<String> allDate = new ArrayList<>();
        ArrayList<String> allData = new ArrayList<>();
        for(int i=0; i<flowData.length; i++){
            double flowValue = Double.parseDouble(flowData[i][1]);
            if(flowValue > 0){
                allDate.add(flowData[i][0]);
                allData.add(flowData[i][1]);
            }
        }
        
        String[][] flowDataCleaned = new String[allData.size()][2];
        for(int i=0; i<allData.size(); i++){
            flowDataCleaned[i][0] = allDate.get(i);
            flowDataCleaned[i][1] = allData.get(i);
        }
        
        return flowDataCleaned;
    }
    /** 
     * This will open the file and grab the lines of data from it until it reaches the empty line
     * @return returns the entire text file in textData
     * @throws IOException
     */
    private String[][] OpenLOADESTresultFile(String path) throws IOException {
	
	FileReader fr = new FileReader(path);
	BufferedReader textReader = new BufferedReader(fr);
	int numberOfLines = readLines(path);
        if(numberOfLines == 0){
            ArrayList<String> errorMessage = new ArrayList<>();
            errorMessage.add("LOADEST failed to return information for the provided nutrient data, the data is likely too sparce to perform an analysis on, please select a different water quality test, provide more data, or select a shorter analysis period.");
            writeError(errorMessage);
        }
        
	String[] textData = new String[numberOfLines];
	for (int i = 0; i < numberOfLines; i++){
            textData[i]=textReader.readLine();
	}
	textReader.close();
        fr.close();
        
        // rctr is used to remove 8 lines of header from the file	
	int rctr = textData.length - 8;
	String[][] dailyData = new String[rctr][2];
	for (int i = 0; i < rctr; i++){
            dailyData[i][0] = textData[(i+8)].substring(1,9);
            dailyData[i][1] = textData[(i+8)].substring(28,38);
	}
        
	return dailyData;
    }
    /**
     * This will determine how many lines contain total data. Used to in OpenFile
     * @return
     * @throws IOException
     */
    private int readLines(String path) throws IOException{
	FileReader file_to_read = new FileReader(path);
	BufferedReader bf = new BufferedReader (file_to_read);
	String aLine;
	int numberOfLines = 0;
	while ((aLine=bf.readLine()) != null){
            numberOfLines++;
	}
	bf.close();
        file_to_read.close();
	
	return numberOfLines;
    }
    /**
     * Writes out the error message, if any, for finding the file and then exits the program
     * @param error  string array to be written as each line of an error message
     * @throws IOException
     */
    public void writeError(ArrayList<String> error) throws IOException{
        //Output data to text file
        String errorContents = error.get(0);
        for(int i=1; i<error.size(); i++){
            errorContents = errorContents + "\n" + error.get(i);
        }
        throw new IOException("Error encountered. Please see the following message for details: \n" + errorContents);
    }
    /**
     * Main statistics function calls other functions to calculate each statistic value then stores the results as global variables
     * @param sortedData  data on which statistical values are desired, sortedData[i][0] = date, soredData[i][1] = value
     * @param timeStep  a flag for which results the statistics will be stored to, either "daily", "monthly", or "yearly"
     */
    private void CalculateStatistics(String[][] sortedData, String timeStep) {
        //convert Data into proper format for calculations
        ArrayList<Double> dataList = new ArrayList<>();
        for(int i=0; i<sortedData.length; i++){
            dataList.add(Double.parseDouble(sortedData[i][1]));
        }
        
        //Calculate statistics
        double temp1 = DoubleMath.round(DoubleMath.max(dataList),3);//Call Max function
        double temp2 = DoubleMath.round(DoubleMath.min(dataList),3);//Call Min function
        double temp3 = DoubleMath.round(DoubleMath.Percentile_function(dataList,0.75),3);//Call Upper Quartile function
        double temp4 = DoubleMath.round(DoubleMath.Percentile_function(dataList,0.25),3);//Call Lower Quartile function
        double temp5 = DoubleMath.round(DoubleMath.median(dataList),3);//Call Median function
        double temp6 = DoubleMath.round(DoubleMath.meanArithmetic(dataList),3);//Call Mean function
        double temp7 = DoubleMath.round(DoubleMath.StandardDeviationSample(dataList),3);//Call standard deviation
        
        if(timeStep.equalsIgnoreCase("daily")){
            dailymax = temp1;
            dailymin = temp2;
            dailyupperQuartile = temp3;
            dailylowerQuartile = temp4;
            dailymedian = temp5;
            dailymean = temp6;
            dailystandardDeviation = temp7;
        }else if(timeStep.equalsIgnoreCase("monthly")){
            monthlymax = temp1;
            monthlymin = temp2;
            monthlyupperQuartile = temp3;
            monthlylowerQuartile = temp4;
            monthlymedian = temp5;
            monthlymean = temp6;
            monthlystandardDeviation = temp7;
        }else if(timeStep.equalsIgnoreCase("yearly")){
            yearlymax = temp1;
            yearlymin = temp2;
            yearlyupperQuartile = temp3;
            yearlylowerQuartile = temp4;
            yearlymedian = temp5;
            yearlymean = temp6;
            yearlystandardDeviation = temp7;
        }
    }
    /**
     * Primary LOADEST function
     * It calls the subfunctions based on user selection/inputs.
     * Calls STORET or USGS database queries and their respective subfunctions
     * @param e  the loadest.exe executable
     * @throws IOException 
     * @throws InterruptedException 
     * @throws ParseException 
     * @throws org.codehaus.jettison.json.JSONException 
     */
    public void run(Executable e) throws IOException, InterruptedException, ParseException, JSONException, WaterDataException {
        //If no date input, make it the maximum of available data
        if(startDate == null || startDate.equalsIgnoreCase("")){
            startDate = "1850-01-01";
        }
        if(endDate == null || endDate.equalsIgnoreCase("")){
            // Pull current date for upper limit of data search
            DateFormat desiredDateFormat = new SimpleDateFormat("yyyy-MM-dd");
            Date currentDate = new Date();
            endDate = desiredDateFormat.format(currentDate);
        }
        
        //Correct Inputs
        String[] resultArray = WaterQualityInfo.getWqTestDataInfo(wqTest, database);
        //String wqCode = resultArray[0];
        String wqLabel = resultArray[1];
        wqUnits = resultArray[2];
        //double ldcConversion = Double.parseDouble(resultArray[3]);
        //String ldcEndUnits = resultArray[4];
        
        //Check if any flow and water quality data exists
        String flowData_raw = "", wqData_raw = "";
        if(userData.length() > 0){
            String[] userDataList = userData.split("\\$\\$");
            flowData_raw = userDataList[0];
            wqData_raw = userDataList[1];
        }
        WaterDataInterface waterLib = WaterData.getNewWaterDataInterface(database, flowData_raw);
        String[][] sortableData = waterLib.extractFlowData_formatted(directory, orgId, stationId, startDate, endDate);
        WaterDataInterface waterLib2 = WaterData.getNewWaterDataInterface(database, wqData_raw);
        String[][] WQdata = waterLib2.extractWaterQualityData_formatted(directory, orgId, stationId, startDate, endDate, wqTest);
        dataSource = waterLib.getDataSourceCitation();
        
        //Find the user uploaded data file and uses this if merged
        String[][] sortableData_user = new String[0][0];
        String[][] WQdata_user = new String[0][0];
        if(mergeDatasets){
            WaterDataInterface waterLibUser = WaterData.getNewWaterDataInterface("UserData", flowData_raw);
            sortableData_user = waterLibUser.extractFlowData_formatted(directory, orgId, stationId, startDate, endDate);
            WaterDataInterface waterLibUser2 = WaterData.getNewWaterDataInterface("UserData", wqData_raw);
            WQdata_user = waterLibUser2.extractWaterQualityData_formatted(directory, orgId, stationId, startDate, endDate, wqTest);
        }
        
        //Sort the Data by date to remove duplicate date entries
        String[][] sortedData = DoubleArray.removeDuplicateDates(sortableData);
        String[][] sortedData_user = DoubleArray.removeDuplicateDates(sortableData_user);
        
        //Merge the two datasets (if user data is empty nothing will be merged)
        String[][] sortedData_combined = DoubleArray.mergeData(sortedData, sortedData_user, mergeMethod);
        String[][] WQdata_combined = DoubleArray.mergeData(WQdata, WQdata_user, mergeMethod);
        
        //remove '0' flow data (incompatible with the model)
        String[][] sortedData_combined_clean = removeZeroFlowData(sortedData_combined);
        
        //Check if any data exists
        ArrayList<String> errorMessage = new ArrayList<>();
        if(sortedData_combined_clean.length == 0){
            if(sortedData.length == 0){
                errorMessage.add("There is no available flow data in the " + database + " database for station '" + stationId + "' and the specified date range.");
                if(database.equalsIgnoreCase("CDWR")){
                    errorMessage.add("The CDWR database is sensitive to the begin date used, try specifying a later begin date");
                }
            }
            if(sortedData_user.length==0){
                errorMessage.add("There is no uploaded flow data for station '" + stationId + "' and the specified date range.");
            }
        }
        if(WQdata_combined.length == 0){
            if(WQdata.length == 0){
                errorMessage.add("There are no available '" + wqTest + "' water quality tests for station " + stationId+  " by: " + 
                    database + " and the specified date range");
            }
            if(WQdata_user.length==0){
                errorMessage.add("There are no uploaded '" + wqTest + "' water quality tests for station " + stationId+  " and the specified date range");
            }
        }
        if(errorMessage.size() > 0){
            writeError(errorMessage);
        }
        
        //Write control file
        writeControl();
        
        //Write Header File
        String[] cname = wqLabel.split(",");
        fileName = cname[0].trim();
        fileName = fileName.replace(" ", "");//Remove spaces from the name
        writeHeader();
        
        // write the calibration variables based on the number of concentration values with a associated flow value
        writeCalibration(sortedData_combined_clean, WQdata_combined, wqUnits);
        
        //Get estimation dates for LOADEST variables
        writeEstimation(sortedData_combined_clean);
        
        //Call LOADEST exe
        //Expected Input: "CONTROL.INP", "HEADER.INP", "CALIB.INP", and "EST.INP"
        System.out.println("Calling LOADEST.exe");
        e.exec();  
        
        //Expected Output: "echo.out" and fileName + ".ind", fileName + ".out", fileName + ".res"
        if (!new File(directory, "echo.out").exists()) {
            throw new FileNotFoundException("echo.out");
        }
        if (!new File(directory, fileName + ".ind").exists()) {
            throw new FileNotFoundException(fileName + ".ind");
        }
        if (!new File(directory, fileName + ".out").exists()) {
            throw new FileNotFoundException(fileName + ".out");
        }
        if (!new File(directory, fileName + ".res").exists()) {
            throw new FileNotFoundException(fileName + ".res");
        }
        System.out.println("Finished LOADEST.exe");

        // Daily result data
        String[][] dailyData = OpenLOADESTresultFile(directory + File.separator + fileName + ".ind");
       
        //convert results to daily,monthly,yearly
        String[][] dailyresultData = DoubleArray.computeFlowMethod(dailyData, "daily", "total", true);
        String[][] monthlyresultData = DoubleArray.computeFlowMethod(dailyData, "monthly", "total", true);
        String[][] yearlyresultData = DoubleArray.computeFlowMethod(dailyData, "yearly", "total", true);
        
        // Calculate Daily stats
        CalculateStatistics(dailyresultData, "daily");
    	double dailyIQR = dailyupperQuartile - dailylowerQuartile;// Find IQR
    	ArrayList<Double> dailyoutliers = new ArrayList<>();// Get daily outliers
    	for (int i = 0; i < dailyresultData.length; i++){
            double value = Double.parseDouble(dailyresultData[i][1]);
            if(value < (dailylowerQuartile - 1.5 * dailyIQR) || value > (dailyupperQuartile + 1.5 * dailyIQR)){		
                dailyoutliers.add(value);
            }
    	}
    	
    	// Calculate Monthly stats
        CalculateStatistics(monthlyresultData, "monthly");
    	double monthlyIQR = monthlyupperQuartile - monthlylowerQuartile;// Find IQR
        ArrayList<Double> monthlyoutliers = new ArrayList<>();// Get monthly outliers
    	for (int i = 0; i < monthlyresultData.length; i++){
            double value = Double.parseDouble(monthlyresultData[i][1]);
            if(value < (monthlylowerQuartile - 1.5 * monthlyIQR) || value > (monthlyupperQuartile + 1.5 * monthlyIQR)){		
                monthlyoutliers.add(value);
            }
    	}
    	
        // Calculate Yearly stats
        CalculateStatistics(yearlyresultData, "yearly");
    	double yearlyIQR = yearlyupperQuartile - yearlylowerQuartile;// Find IQR
        ArrayList<Double> yearlyoutliers = new ArrayList<>();// Get yearly outliers
    	for (int i = 0; i < yearlyresultData.length; i++){
            double value = Double.parseDouble(yearlyresultData[i][1]);
            if(value < (yearlylowerQuartile - 1.5 * yearlyIQR) || value > (yearlyupperQuartile + 1.5 * yearlyIQR)){		
                yearlyoutliers.add(value);
            }
    	}
        
        //Prep. boxplot data for output function
        ArrayList<ArrayList<Double>> dailyBoxplotOutliers = new ArrayList<>();
        ArrayList<ArrayList<Double>> monthlyBoxplotOutliers = new ArrayList<>();
        ArrayList<ArrayList<Double>> yearlyBoxplotOutliers = new ArrayList<>();
        dailyBoxplotOutliers.add(dailyoutliers);
        monthlyBoxplotOutliers.add(monthlyoutliers);
        yearlyBoxplotOutliers.add(yearlyoutliers);
        double[][] dailyBoxplotData = {{dailymax},{dailyupperQuartile},{dailymedian},{dailylowerQuartile},{dailymin}};
        double[][] monthlyBoxplotData = {{monthlymax},{monthlyupperQuartile},{monthlymedian},{monthlylowerQuartile},{monthlymin}};
        double[][] yearlyBoxplotData = {{yearlymax},{yearlyupperQuartile},{yearlymedian},{yearlylowerQuartile},{yearlymin}};
        
    	//Put graph file writer call here
    	DoubleArray.writeTimeSeries(directory, dailyresultData, "daily", getDailyTimeseries().getName(), true);
    	DoubleArray.writeTimeSeries(directory, monthlyresultData, "monthly", getMonthlyTimeseries().getName(), true);
    	DoubleArray.writeTimeSeries(directory, yearlyresultData, "yearly", getYearlyTimeseries().getName(), true);
        
    	DoubleArray.writeBoxplot(directory, dailyBoxplotOutliers, dailyBoxplotData, getDailyBoxplot().getName());
    	DoubleArray.writeBoxplot(directory, monthlyBoxplotOutliers, monthlyBoxplotData, getMonthlyBoxplot().getName());
    	DoubleArray.writeBoxplot(directory, yearlyBoxplotOutliers, yearlyBoxplotData, getYearlyBoxplot().getName());

        // Find what units that LOADEST has given the Load in
        String graphUnits = "??";
        if (ULFLAG == 1){
            graphUnits = "kg";
        }else if (ULFLAG ==2){
            graphUnits = "mg";
        }else if (ULFLAG == 3){
            graphUnits = "lb";
        }else{
            graphUnits = "tons";
        }
        start = sortedData_combined[0][0];
        end = sortedData_combined[sortedData_combined.length - 1][0];
        flowLen = String.valueOf(sortedData_combined.length);
        wqLen = String.valueOf(WQdata_combined.length);
        wqUnits = graphUnits;
    }
    public static void main(String[] args) throws IOException, InterruptedException, Exception {
        //Run model
        guiLOADEST_Model loadest_Model = new guiLOADEST_Model();
        loadest_Model.run(null);
    }
}