NADPData.java [src/AirData] Revision: default  Date:
package AirData;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Iterator;

/**
* Last Updated: 20-April-2017
* @author Lana Millard, Tyler Wible
* @since 9-June-2015
*/
public class NADPData implements AirDataInterface {
    String database = "NADP";

    @Override
    public ArrayList<String> extractAnnualDepositionData_raw(String stationID, String beginDate, String endDate, String wqTest) throws AirDataException {
        //Specify annual website from inputs
        String beginYear = beginDate.substring(0,4);
        String endYear = endDate.substring(0,4);
        //Website Example: http://nadp.sws.uiuc.edu/nadpdata/seasRep.asp?action=seasRep.asp&site=AK03&allSites=False&wholestate=False&custom=False&startyr=1986&endyr=1986&DataType=mg%2FL&reportType=csv&Seas=105&useType=6&useDescription=Nutrient+levels+in+the+South+Platte+River+Basin&submit=Get+Data&userid=290598
        String annualUrl ="http://nadp.sws.uiuc.edu/nadpdata/seasRep.asp?action=seasRep.asp&site="+stationID+"&allSites=False&wholestate=False&custom=False&startyr=" + beginYear + "&endyr=" + endYear + "&DataType=mg%2FL&reportType=csv&Seas=105&useType=6&useDescription=Nutrient+levels+in+the+South+Platte+River+Basin&submit=Get+Data&userid=290598";
        
        //Fetch the annual data webpage for the current NADP station
        ArrayList<String> webpageAll = new ArrayList<>();
        try {
            webpageAll = downloadWebpage(annualUrl);
        } catch (IOException ex) {
            throw new AirDataException("The was an issue extracting " + database + " deposition data from the specified URl: " + annualUrl + "." + ex.getMessage());
        }
        return webpageAll;
    }

    @Override
    public String[][] extractAnnualDepositionData_formatted(String stationID, String beginDate, String endDate, String wqTest) throws AirDataException {
        //Fetch annual data
        ArrayList<String> webpageAll = extractAnnualDepositionData_raw(stationID, beginDate, endDate, wqTest);
        
        //Extract data from the result webpage
        Iterator<String> iterate = webpageAll.iterator( );
        ArrayList<String> stationData = new ArrayList<>();
        int dataIndex=0;
        while(iterate.hasNext()){
            String temp_pageData = iterate.next();
            String[] f = temp_pageData.split(",");
            if (f[0].contains("SiteID")) { 
                //Find the column containing the desired data
                for(int i=0; i<f.length; i++){
                    if(wqTest.equalsIgnoreCase("Ca") && f[i].contains("Ca")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("Mg") && f[i].contains("Mg")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("K") && f[i].contains("K")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("Na") && f[i].contains("Na")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("NH4") && f[i].contains("NH4")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("NO3") && f[i].contains("NO3")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("Cl") && f[i].contains("Cl")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("SO4") && f[i].contains("SO4")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("Field_pH") && f[i].contains("Fld pH")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("Field_Conductivity") && f[i].contains("Fld Cond")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("Lab_pH") && f[i].contains("\"pH\"")){       //LAB pH
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("Lab_Conductivity") && f[i].contains("\"Cond.\"")){   //LAB Conductivity
                        dataIndex = i;
                    }
		}
            }else if(f[0].equals("\"" + stationID + "\"") && !f[dataIndex].equalsIgnoreCase("") ){
                // Save correct data data from webpage onto an Arraylist
                if(dataIndex > 0 && f.length > dataIndex && !f[dataIndex].equalsIgnoreCase("")){        
                    double dataParam = Double.parseDouble(f[dataIndex]);
                    if (dataParam > 0) {
                        //Originally denoted as "--" in the HTML form, which means there 
                        //is missing data for that year, these values became negative in Comma Delimited form
                        //Do not record if this is the case

                        // Convert date format to yyyy-mm-dd
                        String date = f[2]+"-01-01";

                        stationData.add(date + "\t" + f[dataIndex]);
                    }
                }
            }
        }
        
        //Reformat data
        String[][] returnArray = new String[stationData.size()][2];
        for(int i=0; i<returnArray.length; i++){
            String[] currentColumns = stationData.get(i).split("\t");
            //currentColumns[0] = date
            //currentColumns[1] = value
            returnArray[i][0] = currentColumns[0];              
            returnArray[i][1] = currentColumns[1];
        }
        return returnArray;
    }

    @Override
    public ArrayList<String> extractMonthlyDepositionData_raw(String stationID, String beginDate, String endDate, String wqTest) throws AirDataException {
        //Specify monthly website from inputs
        String beginMonth = beginDate.substring(5,7);
        String beginYear = beginDate.substring(0,4);
        String endMonth = endDate.substring(5,7);
        String endYear = endDate.substring(0,4);
        //Website Example: http://nadp.sws.uiuc.edu/nadpdata/monthlyReport.asp?action=monthlyReport.asp&site=AK03&allSites=False&wholestate=False&custom=False&startyr=1980&endyr=2015&DataType=mg%2FL&reportType=csv&All=True&useType=6&useDescription=Nutrient+levels+in+the+South+Platte+River+Basin&submit=Get+Data&userid=290598
        String monthlyUrl = "http://nadp.sws.uiuc.edu/nadpdata/monthlyReport.asp?action=monthlyReport.asp&site=" +stationID+"&allSites=False&wholestate=False&custom=False&startmonth="+beginMonth+"&startyr="+beginYear+"&endmonth=" + endMonth + "&endyr=" + endYear + "&DataType=mg%2FL&reportType=csv&All=True&useType=6&useDescription=Nutrient+levels+in+the+South+Platte+River+Basin&submit=Get+Data&userid=290598";
        
        //Fetch the monthly data webpage for the current NADP station
        ArrayList<String> webpageAll = new ArrayList<>();
        try {
            webpageAll = downloadWebpage(monthlyUrl);
        } catch (IOException ex) {
            throw new AirDataException("The was an issue extracting " + database + " deposition data from the specified URl: " + monthlyUrl + "." + ex.getMessage());
        }
        return webpageAll;
    }

    @Override
    public String[][] extractMonthlyDepositionData_formatted(String stationID, String beginDate, String endDate, String wqTest) throws AirDataException {
        //Fetch monthly data
        ArrayList<String> webpageAll = extractMonthlyDepositionData_raw(stationID, beginDate, endDate, wqTest);
        
        //Extract data from the result webpage
        Iterator<String> iterate = webpageAll.iterator( );
        ArrayList<String> stationData = new ArrayList<>();
        int dataIndex=0;
        while(iterate.hasNext()){
            String temp_pageData = (String) iterate.next();
            String[] f = temp_pageData.split(",");
           
            // Specify column that contains datatype that the user desires
            if (f[0].contains("SiteID")) { 
                for(int i=0; i<f.length; i++){
                    if(wqTest.equalsIgnoreCase("Ca") && f[i].contains("Ca")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("Mg") && f[i].contains("Mg")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("K") && f[i].contains("K")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("Na") && f[i].contains("Na")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("NH4") && f[i].contains("NH4")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("NO3") && f[i].contains("NO3")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("Cl") && f[i].contains("Cl")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("SO4") && f[i].contains("SO4")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("Lab_pH") && f[i].contains("\"pH\"")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("Lab_Conductivity") && f[i].contains("\"Cond.\"")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("Field_pH") && f[i].contains("Fld pH")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("Field_Conductivity") && f[i].contains("Fld Cond")){
                        dataIndex = i;
                    }
                }
            }
            
            // Record data from webpage onto an Arraylist
            else if(f[0].equals("\"" + stationID + "\"") && !f[dataIndex].equalsIgnoreCase("") ){
                if(dataIndex > 0 && f.length > dataIndex && !f[dataIndex].equalsIgnoreCase("")){         
                    double dataParam = Double.parseDouble(f[dataIndex]);
                    if (dataParam > 0) {
                        //Originally denoted as "--" in the HTML form, which means there 
                        //is missing data for that year, these values became negative in Comma Delimited form
                        //Do not record if this is the case

                        // Convert date format to yyyy-mm-dd
                        String date = f[2] + "-" + getmonthMM(f[1].substring(1,4)) + "-01";
                        stationData.add(date + "\t" + f[dataIndex]); 
                    }
                }    
            }
        }
         
        //Reformat data
        String[][] returnArray = new String[stationData.size()][2];
        for(int i=0; i<returnArray.length; i++){
            String[] currentColumns = stationData.get(i).split("\t");
            //currentColumns[0] = date
            //currentColumns[1] = value
            returnArray[i][0] = currentColumns[0];
            returnArray[i][1] = currentColumns[1];
        }
        return returnArray;
    }

    @Override
    public ArrayList<String> extractWeeklyDepositionData_raw(String stationID, String beginDate, String endDate, String wqTest) throws AirDataException {
        //Specify monthly website from inputs
        String beginMonth = beginDate.substring(5,7);
        String beginYear = beginDate.substring(0,4);
        String endMonth = endDate.substring(5,7);
        String endYear = endDate.substring(0,4);
        //Website Example: http://nadp.sws.uiuc.edu/nadpdata/weeklyReport.asp?action=weeklyReport.asp&site=AK03&allSites=False&wholestate=False&custom=False&startmonth=01&startyr=1980&endmonth=12&endyr=2015&reportType=csv&useType=6&useDescription=Nutrient+levels+in+the+South+Platte+River+Basin&submit=Get+Data&userid=290598
        String weeklyUrl = "http://nadp.sws.uiuc.edu/nadpdata/weeklyReport.asp?action=weeklyReport.asp&site="+stationID+ "&allSites=False&wholestate=False&custom=False&startmonth="+beginMonth+"&startyr="+beginYear+"&endmonth=" + endMonth + "&endyr=" + endYear + "&reportType=csv&useType=6&useDescription=Nutrient+levels+in+the+South+Platte+River+Basin&submit=Get+Data&userid=290598";
        
        //Fetch the weekly data webpage for the current NADP station
        ArrayList<String> webpageAll = new ArrayList<>();
        try {
            webpageAll = downloadWebpage(weeklyUrl);
        } catch (IOException ex) {
            throw new AirDataException("The was an issue extracting " + database + " deposition data from the specified URl: " + weeklyUrl + "." + ex.getMessage());
        }
        return webpageAll;
    }

    @Override
    public String[][] extractWeeklyDepositionData_formatted(String stationID, String beginDate, String endDate, String wqTest) throws AirDataException {
        //Fetch weekly data
        ArrayList<String> webpageAll = extractWeeklyDepositionData_raw(stationID, beginDate, endDate, wqTest);
        
        //Extract data from the result webpage
        Iterator<String> iterate = webpageAll.iterator( );
        ArrayList<String> stationData = new ArrayList<>();
        int dataIndex=0;
        while(iterate.hasNext()){
            String temp_pageData = (String) iterate.next();
            String[] f = temp_pageData.split(",");
            
            // Specify column that contains datatype that the user desires
            if (f[0].contains("SiteID")) { 
                for(int i=0; i<f.length; i++){
                    if(wqTest.equalsIgnoreCase("Ca") && f[i].contains("Ca")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("Mg") && f[i].contains("Mg")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("K") && f[i].contains("K")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("Na") && f[i].contains("Na")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("NH4") && f[i].contains("NH4")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("NO3") && f[i].contains("NO3")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("Cl") && f[i].contains("Cl")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("SO4") && f[i].contains("SO4")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("Lab_pH") && f[i].contains("pH Lab")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("Lab_Conductivity") && f[i].contains("Lab Cond")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("Field_pH") && f[i].contains("pH Field")){
                        dataIndex = i;
                    }else if(wqTest.equalsIgnoreCase("Field_Conductivity") && f[i].contains("Field Cond")){
                        dataIndex = i;
                    }
                }
            }
            
            // Record data from webpage onto an Arraylist
            else if(f[0].equals("\"" + stationID + "\"") && !f[dataIndex].equalsIgnoreCase("") ){
                if(dataIndex > 0 && f.length > dataIndex && !f[dataIndex].equalsIgnoreCase("")){        
                    double dataParam = Double.parseDouble(f[dataIndex]);      
                    if (dataParam > 0) {
                        //Originally denoted as "--" in the HTML form, which means there 
                        //is missing data for that year, these values became negative in Comma Delimited form
                        //Do not record if this is the case
                        String detectionError = "";
                        // Concentrations below the detection limit
                        if (f[dataIndex-1].contains("<")){
                            detectionError = "<";    
                        }
                        // Convert date format to yyyy-mm-dd
                        String dateOn = convertDate(f[1]);
                        String dateOff= convertDate(f[2]);
                        stationData.add(dateOn+ "\t" + dateOff + "\t" + detectionError+f[dataIndex]);  
                    } 
                }
            }
        }
         
        //Reformat data
        String[][] returnArray = new String[stationData.size()][3];
        for(int i=0; i<returnArray.length; i++){
            String[] currentColumns = stationData.get(i).split("\t");
            //currentColumns[0] = date on
            //currentColumns[1]= date off
            //currentColumns[2] = value
            returnArray[i][0] = currentColumns[0];
            returnArray[i][1] = currentColumns[1];
            returnArray[i][2] = currentColumns[2];
        }
        return returnArray;
    }
    
    // This method is used to get the corresponding 2-digit number (String) for each 3-letter month abbreviation (String).
    private static String getmonthMM(String month) {
        String monthNum;
        switch (month) {
            case "Jan": monthNum = "01";
                break;
            case "Feb": monthNum = "02";
                break;
            case "Mar": monthNum = "03";
                break;
            case "Apr": monthNum = "04";
                break;
            case "May": monthNum = "05";
                break;
            case "Jun": monthNum = "06";
                break;
            case "Jul": monthNum = "07";
                break;
            case "Aug": monthNum = "08";
                break;
            case "Sep": monthNum = "09";
                break;
            case "Oct": monthNum = "10";
                break;
            case "Nov": monthNum = "11";
                break;
            default: monthNum = "12";
        }  
        return monthNum;
    }
    /**
     * This converts a 'mm/dd/yyyy' date formatted string to 'yyyy-mm-dd' date format
     * @param origDate  a date in 'mm/dd/yyyy/ format
     * @return  the 'yyyy-mm-dd' formatted version of the original input date
     */
    private static String convertDate(String origDate){
        int index1 = origDate.indexOf("/");
        int index2 = origDate.indexOf("/", index1 + 1);
        String month = origDate.substring(1,index1);
        String day = origDate.substring(index1 + 1, index2);
        String year = origDate.substring(index2 + 1,origDate.length() - 2);
        
        if(month.length() < 2){
            month = "0" + month;
        }
        if(day.length() < 2){
            day = "0" + day;
        }
        
        String newDate = year + "-" + month + "-" + day;
        return newDate;
    }
    /**
     * Opens a web connection to USGS and returns the contents of a REST/url search for the specified url
     * @param nadpUrl  a formatted url for requested NADP data
     * @return an ArrayList containing the results of the web service search for flow data using the above inputs
     * @throws IOException
     */
    private ArrayList<String> downloadWebpage(String nadpUrl) throws IOException{
        //Open the provided website
        URL webpage = new URL(nadpUrl);
        URLConnection yc = webpage.openConnection();
        BufferedReader in = new BufferedReader(new InputStreamReader(yc.getInputStream()));
            
        //Read out all of the webpage out into an ArrayList<String>
        String inputLine;
        ArrayList<String> pageData = new ArrayList<>();
        while((inputLine = in.readLine()) != null){
            inputLine = inputLine.replace("&quoi;", "\"");
            pageData.add(inputLine);
        }
        in.close();
        return pageData;
    }
}