Data.java [src/java/cfa] Revision: 742ac16335c3fadf896dcd0edfc0e78b7f588321  Date: Mon Jan 12 16:20:25 MST 2015
package cfa;
import java.io.IOException;
import java.util.ArrayList;

/**
* Last Updated: 12-January-2015
* @author Tyler Wible
* @since 25-January-2014
*/
public class Data {
    /**
     * Main Data extraction for daily flow data from the various databases that this tool can access
     * @param mainFolder  the output file location (used by STORET Data extraction)
     * @param database  the database from which to extract daily flow data (USGS, UserData, STORET, or CDWR)
     * @param organizationName  the organization which provided the data to the database (used by STORET Data extraction)
     * @param stationID  the station ID for which flow data is desired
     * @param beginDate  the begin date of desired flow data (yyyy-MM-dd)
     * @param endDate  the end date of desired flow data (yyyy-MM-dd)
     * @param userData  a concatenated string of User Data (tab-delimited) to extract flow data from (column1 = date, column2 = value)
     * @return  a String[][] of all the flow data available for the specified period (column1 = date yyyy-MM-dd format, column2 = value)
     * @throws Exception 
     */
    public String[][] extractFlowData(String mainFolder,
                                      String database,
                                      String organizationName,
                                      String stationID,
                                      String beginDate,
                                      String endDate,
                                      String userData) throws Exception{
        //Depending on the provided inputs, search for and return flow data
        String[][] flowData = new String[0][2];
        if(database.equalsIgnoreCase("USGS")){
            //Search for USGS flow data
            USGS_Data usgs_Data = new USGS_Data();
            Object[] returnArray = usgs_Data.getUSGSflowData(stationID, beginDate, endDate);
            //ArrayList<String> webpageAll = (ArrayList<String>) returnArray[0];
            flowData = (String[][]) returnArray[1];
            //String start = (String) returnArray[2];
            //String end = (String) returnArray[3];
            
            //If there is minimal flow data, extract discharge data from the water quality database
            if(flowData.length < 10){
                //Retrieve all WQ data from USGS website
                Object[] returnArray1 = usgs_Data.getUSGSwqData(stationID);
                //ArrayList<String> webpageAll = (ArrayList<String>) returnArray1[0];
                String[][] allWQdata = (String[][]) returnArray1[1];
                //String start = (String) returnArray1[2];
                //String end = (String) returnArray1[3];
                
                //Extract and combine USGS discharge water quality codes with the flow dataset
                flowData = usgs_Data.getUSGSwqFlowData(flowData, allWQdata, beginDate, endDate);
            }

        }else if(database.equalsIgnoreCase("UserData")){
            //Find the user uploaded data file and uses this for a timeseries graph
            User_Data user_Data = new User_Data();
            flowData = user_Data.readUserFile(userData, "flow", beginDate, endDate);

        }else if(database.equalsIgnoreCase("STORET")){
            //Search for STORET peak flow data
            STORET_Data storet_Data = new STORET_Data();
            String zip_location = storet_Data.downloadSTORET(mainFolder, organizationName, stationID, "flow", beginDate, endDate);

            //Unzip results file and extract all flow data
            flowData = storet_Data.Unzip_STORETDownloadFiles(zip_location, "flow", true);
            
        }else if(database.equalsIgnoreCase("CDWR")){
            //Search for CDWR flow data
            CDWR_Data cdwr_Data = new CDWR_Data();
            Object[] returnArray = cdwr_Data.getCDWRflowData(stationID, beginDate, endDate, "Daily");
            //ArrayList<String> webpageAll = (ArrayList<String>) returnArray[0];
            flowData = (String[][]) returnArray[1];
            //String start = (String) returnArray[2];
            //String end = (String) returnArray[3];
        }
        
        return flowData;
    }
    /**
     * Main Data extraction for water quality data from the various databases that this tool can access
     * @param mainFolder  the output file location (used by STORET Data extraction)
     * @param database  the database from which to extract water quality data (USGS, UserData, STORET, or CDWR)
     * @param organizationName  the organization which provided the data to the database (used by STORET Data extraction)
     * @param stationID  the station ID for which water quality data is desired
     * @param beginDate  the begin date of desired water quality data (yyyy-MM-dd)
     * @param endDate  the end date of desired water quality data (yyyy-MM-dd)
     * @param userData  a concatenated string of User Data (tab-delimited) to extract water quality data from (column1 = date, column2 = value)
     * @param wqTest  the water quality test desired (if USGS it is the 5-digit water quality code for their database)
     * @return  a String[][] of all the specified water quality data available for the specified period (column1 = date yyyy-MM-dd format, column2 = value)
     * @throws IOException 
     * @throws InterruptedException 
     */
    public Object[] extractWQdata(String mainFolder,
                                  String database,
                                  String organizationName,
                                  String stationID,
                                  String beginDate,
                                  String endDate,
                                  String userData,
                                  String wqTest) throws IOException, InterruptedException{
        String[][] WQdata = new String[0][2];
        String WQlabel = "??";
        String graphUnits = "??";
        if(database.equalsIgnoreCase("USGS")){            
            if(wqTest.length() > 5){
                //Pull only the code portion of the WQ test
                int endIndex = wqTest.lastIndexOf(", ");
                if(endIndex == -1){
                    endIndex = wqTest.lastIndexOf("--");
                }
                WQlabel = wqTest.substring(11,endIndex);//cut off the "98335      " part before the test name and the units after the name
                WQlabel = WQlabel.split(",")[0];
                wqTest = wqTest.substring(0,5);//pull just the 5 digit USGS WQ code
            }
            
            //Retrieve all WQ data from USGS website
            USGS_Data usgs_Data = new USGS_Data();
            Object[] returnArray1 = usgs_Data.getUSGSwqData(stationID);
            //ArrayList<String> webpageAll = (ArrayList<String>) returnArray1[0];
            String[][] allWQdata = (String[][]) returnArray1[1];
            //String start = (String) returnArray1[2];
            //String end = (String) returnArray1[3];
            
            //Extract USGS water quality code for current wqTest only
            WQdata = usgs_Data.minimizeUSGSWQdata(allWQdata, wqTest, beginDate, endDate);
            
            //Get Units and conversion for current WQ test
            graphUnits = usgs_Data.getUSGSwqUnits(wqTest);

        }else if(database.equalsIgnoreCase("UserData")){
            if(wqTest.length() > 5){
                //Pull only the code portion of the WQ test
                int endIndex = wqTest.lastIndexOf(", ");
                if(endIndex == -1){
                    endIndex = wqTest.lastIndexOf("--");
                }
                WQlabel = wqTest.substring(11,endIndex);//cut off the "98335      " part before the test name and the units after the name
                WQlabel = WQlabel.split(",")[0];
                wqTest = wqTest.substring(0,5);//pull just the 5 digit USGS WQ code
            }
            
            //Find the user uploaded data file and uses this for a timeseries graph
            User_Data user_Data = new User_Data();
            WQdata = user_Data.readUserFile(userData, wqTest, beginDate, endDate);
            
            //Use the header to get the WQ test name
            USGS_Data usgs_Data = new USGS_Data();
            graphUnits = usgs_Data.getUSGSwqUnits(wqTest);//Because user uploaded file headers are wqTest
            
        }else if(database.equalsIgnoreCase("STORET")){
            //Search for STORET flow data
            STORET_Data storet_Data = new STORET_Data();
            String zipLocation = storet_Data.downloadSTORET(mainFolder, organizationName, stationID, "flow", beginDate, endDate);
            
            //Unzip results file and extract all flow and WQ results
            WQdata = storet_Data.Unzip_STORETDownloadFiles(zipLocation, wqTest, true);
            WQlabel = wqTest;
            graphUnits = "mg/L";
            
        }else if(database.equalsIgnoreCase("CDWR")){
            ArrayList<String> errorMessage = new ArrayList<String>();
            errorMessage.add("There is no available water quality data available for the CDWR database. This feature is only available for stations in the USGS or STORET databases.");
            writeError(errorMessage);
        }
        
        Object[] returnArray = {WQdata, graphUnits, WQlabel};
        return returnArray;
    }
    /**
     * Main Data extraction for daily flow and water quality data from the various databases that this tool can access (used for LDC and LOADEST)
     * @param mainFolder  the output file location (used by STORET Data extraction)
     * @param database  the database from which to extract water quality data (USGS, UserData, STORET, or CDWR)
     * @param organizationName  the organization which provided the data to the database (used by STORET Data extraction)
     * @param stationID  the station ID for which flow and water quality data is desired
     * @param beginDate  the begin date of desired flow and water quality data (yyyy-MM-dd)
     * @param endDate  the end date of desired flow and water quality data (yyyy-MM-dd)
     * @param userData  a concatenated string of User Data (tab-delimited) to extract flow (column1 = date, column2 = value) 
     * and water quality data (column1 = date, column2 = value) with a "$$" delimiter between the two types of data
     * @param wqTest  the water quality test desired (if USGS it is the 5-digit water quality code for their database)
     * @return  a String[][] of all the specified water quality data available for the specified period (column1 = date yyyy-MM-dd format, column2 = value)
     * @throws IOException 
     * @throws InterruptedException 
     */
    public Object[] extractFlow_and_WQdata(String mainFolder,
                                           String database,
                                           String organizationName,
                                           String stationID,
                                           String beginDate,
                                           String endDate,
                                           String userData,
                                           String wqTest) throws IOException, InterruptedException{
        String[][] flowData = new String[0][2];
        String[][] WQdata = new String[0][2];
        if(database.equalsIgnoreCase("USGS")){
            //Search for USGS flow data
            USGS_Data usgs_Data = new USGS_Data();
            Object[] returnArray = usgs_Data.getUSGSflowData(stationID, beginDate, endDate);
            //ArrayList<String> webpageAll = (ArrayList<String>) returnArray[0];
            flowData = (String[][]) returnArray[1];
            //String start = (String) returnArray[2];
            //String end = (String) returnArray[3];
            
            //Retrieve all WQ data from USGS website
            Object[] returnArray1 = usgs_Data.getUSGSwqData(stationID);
            //ArrayList<String> webpageAll = (ArrayList<String>) returnArray1[0];
            String[][] allWQdata = (String[][]) returnArray1[1];
            //String start = (String) returnArray1[2];
            //String end = (String) returnArray1[3];
            
            if(wqTest.length() > 5){
                //Pull only the code portion of the WQ test
                int endIndex = wqTest.lastIndexOf(", ");
                if(endIndex == -1){
                    endIndex = wqTest.lastIndexOf("--");
                }
                String WQlabel = wqTest.substring(11,endIndex);//cut off the "98335      " part before the test name and the units after the name
                WQlabel = WQlabel.split(",")[0];
                wqTest = wqTest.substring(0,5);//pull just the 5 digit USGS WQ code
            }
            
            //Extract USGS water quality code for current wqTest only
            WQdata = usgs_Data.minimizeUSGSWQdata(allWQdata, wqTest, beginDate, endDate);
            //Extract and combine USGS discharge water quality codes with the flow dataset
            flowData = usgs_Data.getUSGSwqFlowData(flowData, allWQdata, beginDate, endDate);

        }else if(database.equalsIgnoreCase("UserData")){
            //Find the user uploaded data file and uses this for a timeseries graph
            User_Data user_Data = new User_Data();
            Object[] returnArray = user_Data.readUserFileLDC(userData, wqTest, beginDate, endDate);
            flowData = (String[][]) returnArray[0];
            WQdata = (String[][]) returnArray[1];
            
        }else if(database.equalsIgnoreCase("STORET")){
            //Search for STORET flow data
            STORET_Data storet_Data = new STORET_Data();
            String zipLocation = storet_Data.downloadSTORET(mainFolder, organizationName, stationID, "all", beginDate, endDate);

            //Unzip results file and extract all flow and WQ results
            flowData = storet_Data.Unzip_STORETDownloadFiles(zipLocation, "flow", false);
            WQdata = storet_Data.Unzip_STORETDownloadFiles(zipLocation, wqTest, true);
            
        }else if(database.equalsIgnoreCase("UserData")){
            ArrayList<String> errorMessage = new ArrayList<String>();
            errorMessage.add("There is no available water quality data available for the CDWR database. This feature is only available for stations in the USGS or STORET databases.");
            writeError(errorMessage);
        }
        
        Object[] returnArray = {flowData, WQdata};
        return returnArray;
    }
    /**
     * Main Data extraction for flood flow data from the various databases that this tool can access
     * @param mainFolder  the output file location (used by STORET Data extraction)
     * @param database  the database from which to extract daily flow data (USGS, UserData, STORET, or CDWR)
     * @param organizationName  the organization which provided the data to the database (used by STORET Data extraction)
     * @param stationID  the station ID for which flow data is desired
     * @param beginDate  the begin date of desired flow data (yyyy-MM-dd)
     * @param endDate  the end date of desired flow data (yyyy-MM-dd)
     * @param userData  a concatenated string of User Data (tab-delimited) to extract flow data from (column1 = date, column2 = value)
     * @return  a String[][] of all the flow data available for the specified period (column1 = date yyyy-MM-dd format, column2 = value)
     * @throws Exception 
     */
    public double[][] extractFloodData(String mainFolder,
                                       String database,
                                       String organizationName,
                                       String stationID,
                                       String beginDate,
                                       String endDate,
                                       String userData) throws IOException, Exception{
        DoubleArray doubleArray = new DoubleArray();
        
        double[][] peakFlowData = new double[0][0];
        if(database.equalsIgnoreCase("USGS")){
            //Search for USGS peak flow data
            USGS_Data usgs_Data = new USGS_Data();
            Object[] returnArray = usgs_Data.getUSGSPeakData(stationID, beginDate, endDate);
            //ArrayList<String> webpageAll = (ArrayList<String>) returnArray[0];
            peakFlowData = (double[][]) returnArray[1];
            //double start = (double) returnArray[2];
            //double end = (double) returnArray[3];

        }else if(database.equalsIgnoreCase("UserData")){
            //Find the user uploaded data file and uses this for a timeseries graph
            User_Data user_Data = new User_Data();
            String[][] flowData = user_Data.readUserFile(userData, "flow", beginDate, endDate);

            //Removed duplicate dates
            flowData = doubleArray.removeDuplicateDates(flowData);

            //Convert into an annual peak time series
            peakFlowData = doubleArray.convertSTORETpeakData(flowData);

        }else if(database.equalsIgnoreCase("STORET")){
            //Search for STORET peak flow data
            System.out.println("calling downloadSTORET");
            STORET_Data storet_Data = new STORET_Data();
            String zip_location = storet_Data.downloadSTORET(mainFolder, organizationName, stationID, "flow", beginDate, endDate);

            //Unzip results file and extract all flow data
            String[][] flowData = storet_Data.Unzip_STORETDownloadFiles(zip_location, "flow", true);
            
            //Removed duplicate dates
            flowData = doubleArray.removeDuplicateDates(flowData);

            //Convert into an annual peak time series
            peakFlowData = doubleArray.convertSTORETpeakData(flowData);
            
        }else if(database.equalsIgnoreCase("CDWR")){
            //Search for CDWR flow data
            CDWR_Data cdwr_Data = new CDWR_Data();
            Object[] returnArray = cdwr_Data.getCDWRflowData(stationID, beginDate, endDate, "Daily");
            //ArrayList<String> webpageAll = (ArrayList<String>) returnArray[0];
            String[][] flowData = (String[][]) returnArray[1];
            //double start = (double) returnArray[2];
            //double end = (double) returnArray[3];
            
            //Removed duplicate dates
            flowData = doubleArray.removeDuplicateDates(flowData);

            //Convert into an annual peak time series
            peakFlowData = doubleArray.convertSTORETpeakData(flowData);
        }
        
        return peakFlowData;
    }
    /**
     * Extracts 15 minute data from USGS and/or User data or hourly data from CDWR (as long as the date is formatted properly) returns an error otherwise
     * @param mainFolder  the output file location (used by STORET Data extraction)
     * @param database  the database from which to extract water quality data (USGS, UserData, STORET, or CDWR)
     * @param stationID  the station ID for which flow and water quality data is desired
     * @param beginDate  the begin date of desired flow and water quality data (yyyy-MM-dd)
     * @param endDate  the end date of desired flow and water quality data (yyyy-MM-dd)
     * @param userData  a concatenated string of User Data (tab-delimited) to extract flow (column1 = date, column2 = value)
     * @return  a String[][] of all the specified 15-minute flow data available for the specified period (column1 = date yyyy-MM-dd HH:mm format, column2 = value)
     * @throws IOException
     * @throws InterruptedException 
     * @throws Exception
     */
    public String[][] extractInstantaneousFlowData(String mainFolder,
                                                   String database,
                                                   String stationID,
                                                   String beginDate,
                                                   String endDate,
                                                   String userData) throws IOException, InterruptedException, Exception{
        //Depending on the provided inputs, search for and return flow data
        String[][] flowData = new String[0][2];
        if(database.equalsIgnoreCase("USGS")){
            //Search for USGS flow data
            USGS_Data usgs_Data = new USGS_Data();
            Object[] returnArray = usgs_Data.getUSGS15minFlowData(stationID, beginDate, endDate);
            //ArrayList<String> webpageAll = (ArrayList<String>) returnArray[0];
            flowData = (String[][]) returnArray[1];
            //String start = (String) returnArray[2];
            //String end = (String) returnArray[3];

        }else if(database.equalsIgnoreCase("UserData")){
            //Find the user uploaded data file and uses this for a timeseries graph
            User_Data user_Data = new User_Data();
            flowData = user_Data.read15minUserFile(userData, "flow", beginDate + " 00:00", endDate + " 23:00");

        }else if(database.equalsIgnoreCase("STORET")){
            ArrayList<String> errorMessage = new ArrayList<String>();
            errorMessage.add("There is no available 15-minute flow data available for the STORET database. This feature is only available for stations in the USGS NWIS database.");
            writeError(errorMessage);
            
        }else if(database.equalsIgnoreCase("CDWR")){
            //Search for CDWR flow data
            CDWR_Data cdwr_Data = new CDWR_Data();
            Object[] returnArray = cdwr_Data.getCDWRflowData(stationID, beginDate, endDate, "15-min");
            //ArrayList<String> webpageAll = (ArrayList<String>) returnArray[0];
            flowData = (String[][]) returnArray[1];
            //String start = (String) returnArray[2];
            //String end = (String) returnArray[3];
        }
        
        return flowData;
    }
    /**
     * Main Data extraction for stage-discharge relationship data from the various databases that this tool can access
     * @param database  the database from which to extract data (USGS, UserData, STORET, or CDWR)
     * @param stationID  the station ID for which data is desired
     * @return
     * @throws IOException 
     */
    public double[][] extractStageDischarge(String database, String stationID) throws IOException, Exception{
        //Depending on the provided inputs, search for and return flow data
        double[][] ratingCurve = new double[0][2];
        if(database.equalsIgnoreCase("USGS")){
            //Search for USGS flow data
            USGS_Data usgs_Data = new USGS_Data();
            Object[] returnArray = usgs_Data.getUSGSratingCurve(stationID);
            //ArrayList<String> webpageAll = (ArrayList<String>) returnArray[0];
            ratingCurve = (double[][]) returnArray[1];

        }else if(database.equalsIgnoreCase("STORET")){
            ArrayList<String> errorMessage = new ArrayList<String>();
            errorMessage.add("There is no rating curve data available for STORET stations.");
            writeError(errorMessage);
            
        }else if(database.equalsIgnoreCase("CDWR")){
            //Search for USGS flow data
            CDWR_Data cdwr_Data = new CDWR_Data();
            Object[] returnArray = cdwr_Data.getCDWRratingCurve(stationID);
            //ArrayList<String> webpageAll = (ArrayList<String>) returnArray[0];
            ratingCurve = (double[][]) returnArray[1];
        }
        
        return ratingCurve;
    }
    /**
     * Writes out the error message, if any, for finding the file and then exits the program
     * @param error  string array to be written as each line of an error message
     * @throws IOException
     */
    public void writeError(ArrayList<String> error) throws IOException{
        //Output data to text file
        String errorContents = error.get(0);
        for(int i=1; i<error.size(); i++){
            errorContents = errorContents + "\n" + error.get(i);
        }
        throw new IOException("Error encountered. Please see the following message for details: \n" + errorContents);
    }
}