package cfa;

import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.SocketException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Iterator;

/**
* Last Updated: 3-September-2014
* @author Tyler Wible
* @since 21-June-2012
*/
public class USGS_Data {
    /**
     * Get the flow webpage and loop through and pull out the flow data for the current station
     * @param stationID  the USGS station ID for the current station
     * @param beginDate  the user specified begin date for the station (yyyy-mm-dd format)
     * @param endDate  the user specified end date for the station (yyyy-mm-dd format)
     * @return  a String[][] containing column1 = date(yyyy-mm-dd), column2 = flowValue
     * @throws IOException
     */
    public Object[] getUSGSflowData(String stationID, String beginDate, String endDate) throws IOException{
        //Get the webpage of data for the USGS flow station
        ArrayList<String> webpageAll = DownloadFlowWebpage(stationID, beginDate, endDate);
        
        //Pull out new arraylist of only the desired data from the arraylist to return as the web page result
        Iterator<String> iterate = webpageAll.iterator( );
        ArrayList<String> textData = new ArrayList<String>();

        while(iterate.hasNext()){
            String temp_pageData = (String) iterate.next();
            String[] f = temp_pageData.split("\t");

            if ((f.length >= 4) && ("USGS".equals(f[0]))) {
                boolean Ice = f[3].equalsIgnoreCase("Ice");
                boolean Ssn = f[3].equalsIgnoreCase("Ssn");
                boolean Dis = f[3].equalsIgnoreCase("Dis");
                boolean rat = f[3].equalsIgnoreCase("Rat");
                boolean eqp = f[3].equalsIgnoreCase("Eqp");
                boolean mnt = f[3].equalsIgnoreCase("Mnt");
                boolean other = f[3].equalsIgnoreCase("***");
                boolean blank = f[3].equalsIgnoreCase("");
                if (!Ice && !Ssn && !Dis && !rat && !eqp && !mnt && !other && !blank) {
                    //Pull out only the data needed to pass between sub-functions

                    //f[1] = StationID
                    //f[2] = Date
                    //f[3] = FlowValue
                    textData.add(f[1] + "\t" + f[2] + "\t" + f[3]);
                }
            }
        }

        //convert Array list into String[][] array (column1 = date, column2 = value)
        String[][] stringArray = new String[textData.size()][2];
        for(int i=0; i<stringArray.length; i++){
            String[] currentColumns = textData.get(i).split("\t");
            //currentColumns[0] = stationID
            //currentColumns[1] = date
            //currentColumns[2] = value
            stringArray[i][0] = currentColumns[1];
            stringArray[i][1] = currentColumns[2];
        }
        
        //Save analysis results
        String start = "-1";
        String end = "-1";
        if(stringArray.length > 0){
            start = stringArray[0][0];
            end = stringArray[stringArray.length - 1][0];
        }
        
        Object[] returnArray = {webpageAll, stringArray, start, end};
        return returnArray;
    }
    /**
     * Opens a web connection to USGS and returns the contents of a search for all flow data for the specific station and date range
     * @param stationID  the USGS station ID for the current station
     * @param beginDate  the user specified begin date for the station (yyyy-mm-dd format)
     * @param endDate  the user specified end date for the station (yyyy-mm-dd format)
     * @return an ArrayList<String> containing the results of the search for flow data using the above inputs
     * @throws IOException
     */
    public ArrayList<String> DownloadFlowWebpage(String stationID, String beginDate, String endDate) throws IOException {
        //Specify flow website from inputs
        String flowWebsite = "http://waterdata.usgs.gov/nwis/dv?cb_00060=on&format=rdb&begin_date=" + 
                beginDate + "&end_date=" + endDate + "&site_no=" + stationID + "&referred_module=sw";
        
        //Open the provided website
        URL webpage = new URL(flowWebsite);
        URLConnection yc = webpage.openConnection();
        BufferedReader in = new BufferedReader(new InputStreamReader(yc.getInputStream()));
        //Read out all of the webpage out into an ArrayList<String>
        String inputLine;
        ArrayList<String> pageData = new ArrayList<String>( );

        while((inputLine = in.readLine()) != null){
            pageData.add(inputLine);
        }
        in.close();

        return pageData;
    }
    /**
     * Get the flow webpage and loop through and pull out the flow data for the current station
     * @param stationID  the USGS station ID for the current station
     * @param beginDate  the user specified begin date for the station (yyyy-mm-dd format)
     * @param endDate  the user specified end date for the station (yyyy-mm-dd format)
     * @return  an ArrayList<String> containing the flow results tab deliminated (stationID \t date \t flowValue)
     * @throws IOException
     */
    public Object[] getUSGSPeakData(String stationID, String beginDate, String endDate) throws IOException{
        //Get peak flow data
        ArrayList<String> peakWebPage = DownloadPeakFlowWebpage(stationID);

        //Loop through and pull out the desired data
        ArrayList<String> textData = new ArrayList<String>();
        Iterator<String> iterate = peakWebPage.iterator( );

        while(iterate.hasNext()){
            String temp_pageData = (String) iterate.next();
            String[] f = temp_pageData.split("\t");

            if ((f.length >= 5) && ("USGS".equals(f[0]))) {
                boolean Ice = f[4].equalsIgnoreCase("Ice");
                boolean Ssn = f[4].equalsIgnoreCase("Ssn");
                boolean Dis = f[4].equalsIgnoreCase("Dis");
                boolean rat = f[4].equalsIgnoreCase("Rat");
                boolean eqp = f[4].equalsIgnoreCase("Eqp");
                boolean mnt = f[4].equalsIgnoreCase("Mnt");
                boolean other = f[4].equalsIgnoreCase("***");
                boolean blank = f[4].equalsIgnoreCase("");
                if (!Ice && !Ssn && !Dis && !rat && !eqp && !mnt && !other && !blank) {
                    //Keep only the rows which contain the desired values of "USGS StationNumber Date Time FlowValue"

                    //f[2] = date
                    //f[4] = peak flow (cfs)
                    textData.add(f[2] + "\t" + f[4]);
                }
            }
        }
        
        //Convert the array into a double array and remove data not within the date range provided
        double[][] doubleData = convertUSGSpeakData(textData, beginDate, endDate);
        
        //Save analysis results
        double start = -1;
        double end = -1;
        if(doubleData.length > 0){
            start = doubleData[0][0];
            end = doubleData[doubleData.length - 1][0];
        }
        
        Object[] returnArray = {peakWebPage, doubleData, start, end};
        return returnArray;
    }
        /**
     * Opens a web connection to USGS and returns the contents of a search for all peak flow data for the specific station
     * @param stationID  the USGS station ID for the current station
     * @return an ArrayList<String> containing the results of the search for flow data using the above inputs
     * @throws IOException
     */
    public ArrayList<String> DownloadPeakFlowWebpage(String stationID) throws IOException {
        //Specify flow website from inputs
        String peakWebsite = "http://nwis.waterdata.usgs.gov/nwis/peak?site_no=" + stationID + "&agency_cd=USGS&format=rdb";


        //Open the provided website
        URL webpage = new URL(peakWebsite);
        URLConnection yc = webpage.openConnection();
        BufferedReader in = new BufferedReader(new InputStreamReader(yc.getInputStream()));
        //Read out all of the webpage out into an ArrayList<String>
        String inputLine;
        ArrayList<String> pageData = new ArrayList<String>( );

        while((inputLine = in.readLine()) != null){
            pageData.add(inputLine);
        }

        return pageData;
    }
    /**
     * Converts the string array into a double array.  First it substrings the year from the 
     * first column (the date of the flows) and then only converts the year into a double.  
     * Then converts the flow values into doubles. It also only keeps data within the provided date range
     * @param stringData  The ArrayList<String> array containing dates (YYYY-mm-dd) in the first column 
     * and flow values (cfs) in the second column
     * @param beginDate  the user defined begin date
     * @param endDate  the user defined end date
     * @return returns a double[][] array the same size as the provided string array containing 
     * the first column of years and the second column of flow values
     */
    public double[][] convertUSGSpeakData(ArrayList<String> stringData, String beginDate, String endDate){
        String beginYear = beginDate.substring(0,4);
        String endYear = endDate.substring(0,4);
        int ctr = 0;

        for(int i=0; i<stringData.size(); i++){
            String[] f = stringData.get(i).split("\t");
            String year = f[0].substring(0,4);

            //Only keep flood data years within the user defined date range
            if(year.compareToIgnoreCase(beginYear) >=0 && year.compareToIgnoreCase(endYear) <= 0){
                ctr++;
            }
        }
        if(ctr == 1){
            System.out.println("There is " + ctr + " flood year in the current analysis");
        }else{
            System.out.println("There are " + ctr + " flood years in the current analysis");
        }

        //Initialize the return array
        double[][] doubleData = new double[ctr][2];
        ctr = 0;

        for(int i=0; i<stringData.size(); i++){
            String[] f = stringData.get(i).split("\t");
            String year = f[0].substring(0,4);

            //Only keep flood data years within the user defined date range
            if(year.compareToIgnoreCase(beginYear) >=0 && year.compareToIgnoreCase(endYear) <= 0){
                //convert the strings into doubles
                doubleData[ctr][0] = Double.valueOf(year);//year
                doubleData[ctr][1] = Double.valueOf(f[1]);//flow value
                ctr++;
            }
        }

        return doubleData;
    }
    /**
     * Get the 15 minute flow webpage and loop through and pull out the flow data for the current station
     * @param stationID  the USGS station ID for the current station
     * @param beginDate  the user specified begin date for the station (yyyy-mm-dd format)
     * @param endDate  the user specified end date for the station (yyyy-mm-dd format)
     * @return  a String[][] containing column1 = date(yyyy-mm-dd), column2 = flowValue
     * @throws IOException
     */
    public Object[] getUSGS15minFlowData(String stationID, String beginDate, String endDate) throws IOException{
        //Artificial limit due to the current status of USGS's Instantaneous Data Archive transition to NWIS
        if(beginDate.compareToIgnoreCase("2007-10-01") < 0){
            beginDate = "2007-10-01";
        }
        if(endDate.compareToIgnoreCase("2007-10-01") < 0){
            endDate = "2007-10-01";
        }
        
        //Get the webpage of data for the USGS flow station
        ArrayList<String> webpageAll = Download15minFlowWebpage(stationID, beginDate, endDate);
        
        //Pull out new arraylist of only the desired data from the arraylist to return as the web page result
        Iterator<String> iterate = webpageAll.iterator( );
        ArrayList<String> textData = new ArrayList<String>();

        while(iterate.hasNext()){
            String temp_pageData = (String) iterate.next();
            String[] f = temp_pageData.split("\t");

            if ((f.length >= 4) && ("USGS".equals(f[0]))) {
                boolean Ice = f[4].equalsIgnoreCase("Ice");
                boolean Ssn = f[4].equalsIgnoreCase("Ssn");
                boolean Dis = f[4].equalsIgnoreCase("Dis");
                boolean rat = f[4].equalsIgnoreCase("Rat");
                boolean eqp = f[4].equalsIgnoreCase("Eqp");
                boolean mnt = f[4].equalsIgnoreCase("Mnt");
                boolean other = f[4].equalsIgnoreCase("***");
                boolean blank = f[4].equalsIgnoreCase("");
                if (!Ice && !Ssn && !Dis && !rat && !eqp && !mnt && !other && !blank) {
                    //Pull out only the data needed to pass between sub-functions

                    //f[1] = StationID
                    //f[2] = Date
                    //f[3] = time location (ex. MDT = mountain daylight time)
                    //f[4] = FlowValue
                    textData.add(f[1] + "\t" + f[2] + "\t" + f[3] + "\t" + f[4]);
                }
            }
        }

        //convert Array list into String[][] array (column1 = date, column2 = value)
        String[][] stringArray = new String[textData.size()][2];
        for(int i=0; i<stringArray.length; i++){
            String[] currentColumns = textData.get(i).split("\t");
            //currentColumns[0] = stationID
            //currentColumns[1] = date (yyyy-MM-dd hh:mm)
            //currentColumns[2] = time location
            //currentColumns[3] = value
            stringArray[i][0] = currentColumns[1];
            stringArray[i][1] = currentColumns[3];
        }
        
        //Save analysis results
        String start = "-1";
        String end = "-1";
        if(stringArray.length > 0){
            start = stringArray[0][0];
            end = stringArray[stringArray.length - 1][0];
        }
        
        Object[] returnArray = {webpageAll, stringArray, start, end};
        return returnArray;
    }
    /**
     * Opens a web connection to USGS and returns the contents of a search for 15 minute flow data for the specific station and date range
     * @param stationID  the USGS station ID for the current station
     * @param beginDate  the user specified begin date for the station (yyyy-mm-dd format)
     * @param endDate  the user specified end date for the station (yyyy-mm-dd format)
     * @return an ArrayList<String> containing the results of the search for 15 minute flow data using the above inputs
     * @throws IOException
     */
    public ArrayList<String> Download15minFlowWebpage(String stationID, String beginDate, String endDate) throws IOException {
        //Specify flow website from inputs
        String flowWebsite = "http://nwis.waterdata.usgs.gov/nwis/uv?cb_00060=on&format=rdb&site_no=" + stationID + 
                             "&period=&begin_date=" + beginDate + "&end_date=" + endDate;
        //Instantaneous Data Archive (IDA) website (to be discontinued in 2015) = "http://ida.water.usgs.gov/ida/available_records.cfm?sn=" + stationID;
        
        //Open the provided website
        URL webpage = new URL(flowWebsite);
        URLConnection yc = webpage.openConnection();
        BufferedReader in = new BufferedReader(new InputStreamReader(yc.getInputStream()));
        //Read out all of the webpage out into an ArrayList<String>
        ArrayList<String> pageData = new ArrayList<String>( );
        boolean moreLines = true;
        while(moreLines){
            try{
                String inputLine = in.readLine();
                if(inputLine != null){
                    pageData.add(inputLine);
                }else{
                    moreLines = false;
                }
            }catch(SocketException e){
                //The webpage is angry for some reason so quit out
                moreLines = false;
            }
        }
        in.close();

        return pageData;
    }
    /**
     * Get the water quality webpage and loop through and pull out the water quality data for the current station
     * @param stationID  the USGS station ID for the current station
     * @return  a String[][] containing column1 = date(yyyy-mm-dd), column2 = flowValue
     * @throws IOException
     * @throws InterruptedException
     */
    public Object[] getUSGSwqData(String stationID) throws IOException, InterruptedException{

        //Get the webpage of data for the USGS flow station
        ArrayList<String> webpageAll = DownloadWQwebpage(stationID);
//      ArrayList<String> webpageAll = DownloadWQwebpage_HtmlUnit(stationID, wqTestCode);
	
        //Pull out new arraylist of only the desired data from the arraylist to return as the web page result
        Iterator<String> iterate = webpageAll.iterator( );
        ArrayList<String> textData = new ArrayList<String>();
        while(iterate.hasNext()){
            String temp_pageData = (String) iterate.next();
            String[] f = temp_pageData.split("\t");

            if ( (f.length >= 15) && (f[0].equals("USGS")) ) {
                String WQSample_code = f[12];
                String WQSample_result = f[14];
                boolean A = WQSample_code.equals("");
                boolean B = WQSample_result.equals("");
                if (!A && !B){
                    //Count only the rows which contain the desired values of "agency_cd	site_no	sample_dt...	
                    //	sample_tm	sample_end_dt	sample_end_tm	sample_start_time_datum_cd	tm_datum_rlbty_cd...	
                    //	coll_ent_cd	medium_cd	tu_id	body_part_id	parm_cd	remark_cd	result_va"

                    //Pull out only the data needed to pass between sub-functions
                    //f[1] = stationID
                    //f[2] = date
                    //f[12] = water quality test code
                    //f[14] = water quality test value
                    textData.add(f[1] + "\t" + f[2] + "\t" + f[12] + "\t" + f[14]);
                }
            }
        }

        //convert Array list into String[][] array (column1 = date, column2 = value)
        String[][] stringArray = new String[textData.size()][3];
        for(int i=0; i<textData.size(); i++){
            String[] currentColumns = textData.get(i).split("\t");
            //currentColumns[0] = stationID
            //currentColumns[1] = date
            //currentColumns[2] = water quality test code
            //currentColumns[3] = water quality test value

            stringArray[i][0] = currentColumns[1];//date
            stringArray[i][1] = currentColumns[2];//test code
            stringArray[i][2] = currentColumns[3];//value
        }
        
        //Save analysis results
        String start = "-1";
        String end = "-1";
        if(stringArray.length > 0){
            start = stringArray[0][0];
            end = stringArray[stringArray.length - 1][0];
        }
        
        Object[] returnArray = {webpageAll, stringArray, start, end};
        return returnArray;
    }
    /**
     * Opens a web connection to USGS and returns the contents of a search for all water quality data for the specific station
     * @param stationID  the USGS station ID for the current station
     * @return an ArrayList<String> containing the results of the search for water quality data using the above input
     * @throws IOException
     */
    public ArrayList<String> DownloadWQwebpage(String stationID) throws IOException, InterruptedException {
        //Specify flow website from inputs
//      String WQWebsite = "http://waterdata.usgs.gov/nwis/nwisman/?site_no=" + stationID + "&agency_cd=USGS";
        String WQWebsite = "http://nwis.waterdata.usgs.gov/usa/nwis/qwdata/?site_no=" + stationID + "&agency_cd=USGS&inventory_output=0&rdb_inventory_output=value&TZoutput=0&pm_cd_compare=Greater%20than&radio_parm_cds=all_parm_cds&qw_attributes=0&format=rdb&qw_sample_wide=0&rdb_qw_attributes=0&date_format=YYYY-MM-DD&rdb_compression=value&submitted_form=brief_list";		

        //Open the provided website
        URL webpage = new URL(WQWebsite);
        URLConnection yc = webpage.openConnection();
        BufferedReader in = new BufferedReader(new InputStreamReader(yc.getInputStream()));

        boolean webpageError = true;//assume the webpage has errored and try to reboot it
        int sleepCounter = 0;
        //Loop until the webpage exists
        while(webpageError){
            sleepCounter++;
            webpageError = getWQPage(in, "Network Issues");
            Thread.sleep(1000); // do nothing for 1000 milliseconds (1 second)
            in.close();
            yc = webpage.openConnection();
            in = new BufferedReader(new InputStreamReader(yc.getInputStream()));

            //If waiting more than 15 seconds (arbirary time amount, change as needed), the page likely didn't load 
            //or the station search didn't work so end the program and report this error
            if(sleepCounter > 15){
                return new ArrayList<String>();
            }
        }

        //Read out all of the webpage out into an ArrayList<String>
        String inputLine;
        ArrayList<String> pageData = new ArrayList<String>( );
        while((inputLine = in.readLine()) != null){
            pageData.add(inputLine);
        }
        return pageData;
    }
    /**
     * Checks if the webpage contains the search keyword of not, usually used to find a webpage error and return a false value
     * @param in  the BufferedReader for the webpage
     * @param keyword  the String keyword that is looked for in the webpage
     * @return  returns true if the webpage contains the keyword, false otherwise
     * @throws IOException
     */
    public boolean getWQPage(BufferedReader in, String keyword) throws IOException{
        String inputLine = "";

        boolean containsKeyword = false;
//        System.out.println("Current webpage: \n");
        while((inputLine = in.readLine()) != null) {
            if(inputLine.contains(keyword)){
//                System.out.println(inputLine);
                containsKeyword = true;
                break;
            }
        }
        return containsKeyword;
    }
    /**
     * Opens a web connection to USGS and returns the contents of a search for all water quality data for the specific station
     * @param stationID  the USGS station ID for the current station
     * @param wqTest  the 5 digit USGS water qualiyt (WQ) test code that the user has requested for download
     * @return  an ArrayList containing the USGS provided header for the text as well as all the WQ data for the provided wqTest code
     * @throws IOException
     * @throws InterruptedException
     */
    public ArrayList<String> getUSGSwqData_partial(String stationID, String wqTest) throws IOException, InterruptedException {

        //Specify flow website from inputs
//      String WQWebsite = "http://waterdata.usgs.gov/nwis/nwisman/?site_no=" + stationID + "&agency_cd=USGS";
        String WQWebsite = "http://nwis.waterdata.usgs.gov/usa/nwis/qwdata/?site_no=" + stationID + "&agency_cd=USGS&inventory_output=0&rdb_inventory_output=value&TZoutput=0&pm_cd_compare=Greater%20than&radio_parm_cds=all_parm_cds&qw_attributes=0&format=rdb&qw_sample_wide=0&rdb_qw_attributes=0&date_format=YYYY-MM-DD&rdb_compression=value&submitted_form=brief_list";		

        //Open the provided website
        URL webpage = new URL(WQWebsite);
        URLConnection yc = webpage.openConnection();
        BufferedReader in = new BufferedReader(new InputStreamReader(yc.getInputStream()));

        boolean webpageError = true;//assume the webpage has errored and try to reboot it
        int sleepCounter = 0;
        //Loop until the webpage exists
        while(webpageError){
            sleepCounter++;
            webpageError = getWQPage(in, "Network Issues");
            Thread.sleep(1000); // do nothing for 1000 milliseconds (1 second)
            in.close();
            yc = webpage.openConnection();
            in = new BufferedReader(new InputStreamReader(yc.getInputStream()));

            //If waiting more than 15 seconds (arbirary time amount, change as needed), the page likely didn't load 
            //or the station search didn't work so end the program and report this error
            if(sleepCounter > 15){
                return new ArrayList<String>();
            }
        }

        //Read out all of the webpage out into an ArrayList<String>
        String inputLine;
        ArrayList<String> pageData = new ArrayList<String>( );
        while((inputLine = in.readLine()) != null){
            //Deliminate the row based on tabs
            String[] f = inputLine.split("\t");
            if(!f[0].equalsIgnoreCase("USGS")){
                //If the current line does not start with "USGS" then it is part of the header that should be kept
                pageData.add(inputLine);
//                System.out.println(inputLine);

            }else{
                //If the current line does start with USGS, check if its row contains the proper WQ test and keep it if it matches
                if(f[12].equalsIgnoreCase(wqTest)){
                    //f[12] = water quality test code
                    pageData.add(inputLine);
//                    System.out.println(inputLine);
                }
            }
        }
        in.close();
        return pageData;
    }
    /**
     * Reduces all water quality data to just that of the requested parameter
     * @param allData  all water quality data for the earlier provided date range and station ID (column1 = date, column2 = wqTestcode, column3 = value)
     * @param wqTestCode  the requested water quality parameter
     * @param beginDate  the user defined begin date for data search
     * @param endDate  the user defined end date for data search
     * @return
     * @throws IOException 
     */
    public String[][] minimizeUSGSWQdata(String[][] allData, String wqTestCode, String beginDate, String endDate) throws IOException{
        int ctr = 0;
        for(int i=0; i<allData.length; i++){
            if(i != 0){//Ignore the first row containing the station name
                if(allData[i][1].equalsIgnoreCase(wqTestCode) &&
                  (allData[i][0].compareTo(beginDate) >= 0) && (allData[i][0].compareTo(endDate) <= 0)){
                    ctr++;
                }
            }
        }

        String[][] reducedData = new String[ctr][2];
        ctr=0;
        for(int i=0; i<allData.length; i++){
            if(i != 0){//Ignore the first row containing the station name
                if(allData[i][1].equalsIgnoreCase(wqTestCode) &&
                  (allData[i][0].compareTo(beginDate) >= 0) && (allData[i][0].compareTo(endDate) <= 0)){
                    reducedData[ctr][0] = allData[i][0];//date
                    reducedData[ctr][1] = allData[i][2];//WQ test result value

                    ctr++;
                }
            }
        }
        return reducedData;
    }
    /**
     * @param parameterCode  USGS parameter code for a specific water quality test.
     * @return a string with the type of units for the current test.
     */
     public String getUSGSwqUnits(String parameterCode) throws IOException{
         URL webpage = new URL("http://nwis.waterdata.usgs.gov/usa/nwis/pmcodes?radio_pm_search=param_group&pm_group=All+--+include+all+parameter+groups&pm_search=&casrn_search=&srsname_search=&format=rdb&show=parameter_group_nm&show=parameter_nm&show=casrn&show=srsname&show=parameter_units");
         URLConnection yc = webpage.openConnection();
         BufferedReader in = new BufferedReader(new InputStreamReader(yc.getInputStream()));
         String inputLine;
         int line_length = 0;
         String units = "0";
         //Find the units of the specified test
         while ((inputLine = in.readLine()) != null) {
             String[] f = inputLine.split("\t");
             line_length = f.length;
             if((line_length >= 6) && (f[0].length() == 5)){
                 if(f[0].equals(parameterCode)){
                     units = f[5];
                 }
             }
         }
         return units;
     }
     /**
     * @param units  the units of the current USGS water quality test.
     * @return a double with the correct conversion factor for the units.
     */
     public double getUSGSwqConversion(String units){
         double conversion = 0;
         if(units.equalsIgnoreCase("#/l")){
             conversion = (1000)*(java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("#/m3")){
             conversion = (java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("#/ml")){
             conversion = (1000)*(1000)*(java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("MPN/100 ml")){
             conversion = (1.0/100.0)*(1000)*(1000)*(java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("MPN/100L")){
             conversion = (1.0/100.0)*(1000)*(java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("cfu/100ml")){
             conversion = (1.0/100.0)*(1000)*(1000)*(java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("cfu/mL")){
             conversion = (1000)*(1000)*(java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("col/mL")){
             conversion = (1000)*(1000)*(java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("cysts/100L")){
             conversion = (1.0/100.0)*(1000)*(java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("cysts/10L")){
             conversion = (1.0/10.0)*(1000)*(java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("g/cm3") || units.equalsIgnoreCase("g/mL @ 20C")){
             conversion = (java.lang.Math.pow(10,-6))*(1/1)*(1000)*(1000)*(java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("g/m3")){
             conversion = (java.lang.Math.pow(10,-6))*(java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("mg/l") || units.equalsIgnoreCase("mg/l CaCO3") || units.equalsIgnoreCase("mg/l NH4") || 
                     units.equalsIgnoreCase("mg/l NO3") || units.equalsIgnoreCase("mg/l PO4") || units.equalsIgnoreCase("mg/l SiO2") || 
                     units.equalsIgnoreCase("mg/l as H") || units.equalsIgnoreCase("mg/l as N") || units.equalsIgnoreCase("mg/l as Na") || 
                     units.equalsIgnoreCase("mg/l as P") || units.equalsIgnoreCase("mg/l as S") || units.equalsIgnoreCase("mgC3H6O2/L")){
             conversion = (java.lang.Math.pow(10,-6))*(1000)*(java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("mg/mL @25C")){
             conversion = (java.lang.Math.pow(10,-6))*(1000)*(1000)*(java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("ml/l")){
             conversion = (1000)*(java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("ng/l") || units.equalsIgnoreCase("pg/mL")){
             conversion = (java.lang.Math.pow(10,-12))*(1000)*(java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("ng/m3") || units.equalsIgnoreCase("pg/l")){
             conversion = (java.lang.Math.pow(10,-12))*(java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("ocyst/100L")){
             conversion = (1.0/100.0)*(1000)*(java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("oocyst/10L")){
             conversion = (1.0/10.0)*(1000)*(java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("pfu/100L")){
             conversion = (1.0/100.0)*(1000)*(java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("pfu/100ml")){
             conversion = (1.0/100.0)*(1000)*(1000)*(java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("pg/m3")){
             conversion = (java.lang.Math.pow(10,-15))*(java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("ug/L 2,4-D") || units.equalsIgnoreCase("ug/L U3O8") || units.equalsIgnoreCase("ug/L as As") || 
                     units.equalsIgnoreCase("ug/L as Cl") || units.equalsIgnoreCase("ug/L as N") || units.equalsIgnoreCase("ug/L as P") || 
                     units.equalsIgnoreCase("ug/l") || units.equalsIgnoreCase("ugAtrazn/L")){
             conversion = (java.lang.Math.pow(10,-9))*(1000)*(java.lang.Math.pow(0.3048,3))*(86400);
         }else if(units.equalsIgnoreCase("ug/m3")){
             conversion = (java.lang.Math.pow(10,-9))*(java.lang.Math.pow(0.3048,3))*(86400);
         }
         return conversion;
     }
     /**
     * @param units  the units of the current USGS water quality test.
     * @return a string with the end result units of the conversion.
     */
     public String getUSGSwqEndUnits(String units){
         String endUnits = "No Units";
         if(units.equalsIgnoreCase("#/l") || units.equalsIgnoreCase("#/m3") || units.equalsIgnoreCase("#/ml")){
             endUnits = "#/day";
         }else if(units.equalsIgnoreCase("MPN/100 ml") || units.equalsIgnoreCase("MPN/100L")){
             endUnits = "MPN/day";
         }else if(units.equalsIgnoreCase("cfu/100ml") || units.equalsIgnoreCase("cfu/mL")){
             endUnits = "cfu/day";
         }else if(units.equalsIgnoreCase("col/mL")){
             endUnits = "col/day";
         }else if(units.equalsIgnoreCase("cysts/100L") || units.equalsIgnoreCase("cysts/10L")){
             endUnits = "cysts/day";//= cysts/100L*cfs
         }else if(units.equalsIgnoreCase("mg/l") || units.equalsIgnoreCase("mg/l CaCO3") || units.equalsIgnoreCase("mg/l NH4") || 
                 units.equalsIgnoreCase("mg/l NO3") || units.equalsIgnoreCase("mg/l PO4") || units.equalsIgnoreCase("mg/l SiO2") || 
                 units.equalsIgnoreCase("mg/l as H") || units.equalsIgnoreCase("mg/l as N") || units.equalsIgnoreCase("mg/l as Na") || 
                 units.equalsIgnoreCase("mg/l as P") || units.equalsIgnoreCase("mg/l as S") || units.equalsIgnoreCase("mgC3H6O2/L") || 
                 units.equalsIgnoreCase("g/cm3") || units.equalsIgnoreCase("g/mL @ 20C") || units.equalsIgnoreCase("g/m3") || 
                 units.equalsIgnoreCase("mg/mL @25C") || units.equalsIgnoreCase("ng/l") || units.equalsIgnoreCase("pg/mL") ||
                 units.equalsIgnoreCase("ng/m3") || units.equalsIgnoreCase("pg/l") || units.equalsIgnoreCase("pg/m3") || 
                 units.equalsIgnoreCase("ug/L 2,4-D") || units.equalsIgnoreCase("ug/L U3O8") || units.equalsIgnoreCase("ug/L as As") || 
                 units.equalsIgnoreCase("ug/L as Cl") || units.equalsIgnoreCase("ug/L as N") || units.equalsIgnoreCase("ug/L as P") || 
                 units.equalsIgnoreCase("ug/l") || units.equalsIgnoreCase("ugAtrazn/L") || units.equalsIgnoreCase("ug/m3")){
             endUnits = "kg/day";//= mg/l*cfs
         }else if(units.equalsIgnoreCase("ml/l")){
             endUnits = "ml/day";//= mg/l*cfs
         }else if(units.equalsIgnoreCase("pfu/100L") || units.equalsIgnoreCase("pfu/100ml")){
             endUnits = "pfu/day";
         }else if(units.equalsIgnoreCase("ocyst/100L")){
             endUnits = "ocyst/day";
         }else if(units.equalsIgnoreCase("oocyst/10L")){
             endUnits = "oocyst/day";
         }
         return endUnits;
     }
    /**
     * Extract the USGS water quality tests of 00061 (discharge in cfs) and 30209 (discharge in cms) and combine them
     * with the provided flow data set within the provided date range
     * @param flowData  existing stream flow data (column 1 =  dates, column 2 = flow values)
     * @param allWQdata  all existing stream water quality data (column 1 =  dates, column 2 = flow values)
     * @param beginDate  the start of the desired data date range
     * @param endDate  the end of the desired data date range
     * @return
     * @throws IOException 
     */
    public String[][] getUSGSwqFlowData(String[][] flowData, String[][] allWQdata, String beginDate, String endDate) throws IOException{
        DoubleArray doubleArray = new DoubleArray();
        
        //Extract USGS water quality code 00061 for dischage in cfs
        String[][] WQFlow1 = minimizeUSGSWQdata(allWQdata, "00061", beginDate, endDate);
        //Extract USGS water quality code 30209 for discharge test in m^3/s (cms)
        String[][] WQFlow2 = minimizeUSGSWQdata(allWQdata, "30209", beginDate, endDate);

        //Convert the m^3 to ft^3/s
        for(int i=0; i<WQFlow2.length; i++){
            WQFlow2[i][1] = Double.toString((Double.parseDouble(WQFlow2[i][1])*(3.2808399*3.2808399*3.2808399)));
        }

        //combine the WQ flows (cfs and the converted cms data) into a single variable to then combine with the Flowdata
        String[][] WQDataflows = doubleArray.mergeData(WQFlow1, WQFlow2, "public");//The "public" attribute keeps the first dataset in case of duplicates

        //Combine flow data and WQ flow data into a variable of dates and flow values to be sorted
        flowData = doubleArray.mergeData(flowData, WQDataflows, "public");//The "public" attribute keeps the first dataset in case of duplicates
        
        return flowData;
    }
    /**
     * Get the rating curve (stage-discharge relationship) data for use by the user
     * @param stationID  the USGS station ID for the current station
     * @return  a double[][] containing column1 = discharge(ft3/s), column2 = depth(ft)
     * @throws IOException
     */
    public Object[] getUSGSratingCurve(String stationID) throws IOException{
        //Get the webpage of data for the USGS flow station
        ArrayList<String> webpageAll = DownloadRatingCurveWebpage(stationID);
        
        //Pull out new arraylist of only the desired data from the arraylist to return as the web page result
        Iterator<String> iterate = webpageAll.iterator( );
        ArrayList<String> textData = new ArrayList<String>();

        while(iterate.hasNext()){
            String temp_pageData = (String) iterate.next();
            String[] f = temp_pageData.split("\t");
            try{
                String text1 = temp_pageData.substring(0,1);
                if((f.length >= 3) && (!"#".equals(text1))){
                    try{
                        double depth = Double.parseDouble(f[0]);
                        double shift = Double.parseDouble(f[1]);
                        double discharge = Double.parseDouble(f[2]);
                        textData.add(f[0] + "\t" + f[1] + "\t" + f[2]);
                    }catch(NumberFormatException e){
                        //Skip this entry and move on to the next line in the file
                    }
                }
                
            }catch(IndexOutOfBoundsException e){
                //Move on to the next line in the file
            }
            
        }

        //convert Array list into double[][] array (column1 = depth, column2 = discharge)
        double[][] ratingCurveData = new double[textData.size()][2];
        for(int i=0; i<ratingCurveData.length; i++){
            String[] currentColumns = textData.get(i).split("\t");
            //currentColumns[0] = depth
            //currentColumns[1] = shift
            //currentColumns[2] = discharge
            ratingCurveData[i][0] = Double.parseDouble(currentColumns[2]);
            ratingCurveData[i][1] = Double.parseDouble(currentColumns[0]);
        }
        
        Object[] returnArray = {webpageAll, ratingCurveData};
        return returnArray;
    }
    /**
     * Opens a web connection to USGS and returns the contents of a search for 15 minute flow data for the specific station and date range
     * @param stationID  the USGS station ID for the current station
     * @return an ArrayList<String> containing the results of the search for 15 minute flow data using the above inputs
     * @throws IOException
     */
    public ArrayList<String> DownloadRatingCurveWebpage(String stationID) throws IOException {
        //Specify rating curve website from inputs
        String ratingCurveWebsite = "http://nwis.waterdata.usgs.gov/nwisweb/data/ratings/exsa/USGS." + stationID + ".exsa.rdb";
        //"http://nwis.waterdata.usgs.gov/nwisweb/data/ratings/exsa/USGS.06741510.exsa.rdb"
        
        //Open the provided website
        URL webpage = new URL(ratingCurveWebsite);
        URLConnection yc = webpage.openConnection();
        ArrayList<String> pageData = new ArrayList<String>();
        try{
            BufferedReader in = new BufferedReader(new InputStreamReader(yc.getInputStream()));
            //Read out all of the webpage out into an ArrayList<String>
            String inputLine;

            while((inputLine = in.readLine()) != null){
                pageData.add(inputLine);
            }
            in.close();

            return pageData;
        }catch(FileNotFoundException e){
            //This station has no rating curve data, so return nothing
            return pageData;
        }
    }
}