WaterData_WqDataPortal.java [src/WaterData] Revision: default  Date:
package WaterData;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import utils.WebPageUtils;

/**
* Last Updated: 9-April-2019
* @author Tyler Wible
* @since 20-November-2018
* The Water Quality Data Portal covers STORET, USGS, and STEWARDS databases, more information available here:
* https://www.waterqualitydata.us/webservices_documentation/
*/
public class WaterData_WqDataPortal implements WaterDataInterface{
    String database = "STORET"; //This is currently only setup for STORET stations, so it needs expand to USGS and STEWARDS
    
    @Override
    public String getDataSourceCitation(){
        //Get today's date for the source reference
        DateFormat sourceDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm");
        String today = sourceDateFormat.format(new Date());
        
        //Cite EPA WQX/STORET's Water Quality Data Portal
        String dataSource = "Stream flow data and water quality test data retrieved from the U.S. Environmental Protection Agency, WQX/STORET, Water Quality Data Portal. https://www.waterqualitydata.us/, accessed: " + today;
        return dataSource;
    }

    @Override
    public ArrayList<String> extractFlowData_raw(String directory, String orgId, String stationId, String startDate, String endDate) throws WaterDataException {
        //Get flow website from inputs
        String wqTest = "00060 Flow -- cfs";
        ArrayList<String> webpageAll = extractWaterQualityData_raw(directory, orgId, stationId, startDate, endDate, wqTest);
        return webpageAll;
    }

    @Override
    public String[][] extractFlowData_formatted(String directory, String orgId, String stationId, String startDate, String endDate) throws WaterDataException{
        //Fetch flow data
        String wqTest = "00060 Flow -- cfs";
        String[][] returnArray = extractWaterQualityData_formatted(directory, orgId, stationId, startDate, endDate, wqTest);
        return returnArray;
    }

    @Override
    public ArrayList<String> extractWaterQualityData_raw(String directory, String orgId, String stationId, String startDate, String endDate, String wqTest) throws WaterDataException {
        //Fix dates to match Webservice-expected "MM-DD-YYYY" format
        String newStartDate = reformatDate(startDate);
        String newEndDate = reformatDate(endDate);
        
        //URL encode stationId and orgId to prevent issues
        String newStationId = reformatString(stationId);
        String newOrgId = reformatString(orgId);
        
        //Specify water quality website from inputs
        //String wqUrl = "https://www.waterqualitydata.us/data/Result/search?providers=STORET&organization=MNPCA_BIO&siteid=MNPCA_BIO-S010-342&mimeType=csv"
        String wqUrl = "https://www.waterqualitydata.us/data/Result/search?providers=STORET&" +
                "organization=" + newOrgId +
                "&siteid=" + newStationId +
                "&startDateLo=" + newStartDate +
                "&startDateHi=" + newEndDate;
        if(wqTest.isEmpty() || wqTest.equalsIgnoreCase("all")){
            wqUrl += "&mimeType=tsv";
        }else{
            try {
                String[] resultArray = WaterQualityInfo.getWqTestDataInfo(wqTest, database);
                //String wqCode = resultArray[0];
                String wqLabel = resultArray[1];
                //String units = resultArray[2];
                //double ldcConversion = Double.parseDouble(resultArray[3]);
                //String ldcEndUnits = resultArray[4];
                
                //URL encode wqLabel to prevent issues
                String newWqLabel = reformatString(wqLabel);
                wqUrl += "&characteristicName=" + newWqLabel + "&mimeType=tsv";
                //wqUrl += "&pCode=" + wqCode + "&mimeType=tsv";
            } catch (IOException ex) {
                throw new WaterDataException("The was an issue extracting " + database + " water quality parameters from the specified water quality test: " + wqTest + "." + ex.getMessage());
            }
        }
        
        //Fetch the water quality data webpage for the current USGS station
        ArrayList<String> webpageAll = new ArrayList<>();
        try {
            webpageAll = WebPageUtils.downloadWebpage(wqUrl);
        } catch (IOException ex) {
            throw new WaterDataException("The was an issue extracting " + database + " water quality data from the specified URl: " + wqUrl + "." + ex.getMessage());
        }
        return webpageAll;
    }
    
    @Override
    public String[][] extractWaterQualityData_formatted(String directory, String orgId, String stationId, String startDate, String endDate, String wqTest) throws WaterDataException {
        //Fetch water quality data
        ArrayList<String> webpageAll = extractWaterQualityData_raw(directory, orgId, stationId, startDate, endDate, wqTest);
        
         //Get current wq units
        int index1 = wqTest.indexOf("--");
        String desiredUnits = wqTest.substring(index1 + 2).trim();
        
        //Reformat data
        int size = webpageAll.size() - 1;//don't keep header in final size
        if (size < 0){
            size = 0;
        }
        String[][] returnArray = new String[size][2];
        for(int i=1; i<webpageAll.size(); i++){
            String[] currentColumns = webpageAll.get(i).split("\t");
            //currentColumns[6] = date
            //currentColumns[31] = parameter name (wqTest)
            //currentColumns[33] = value
            //currentColumns[34] = units
            //currentColumns[30] = result detection condition
            //currentColumns[58] = detection limit type
            //currentColumns[59] = detection limit value
            //currentColumns[60] = detection limit units
            double conversion = WaterQualityInfo.getWQconversion(desiredUnits, currentColumns[34]);
            double value = Double.parseDouble(currentColumns[33]) * conversion;
            returnArray[i-1][0] = currentColumns[6];
            returnArray[i-1][1] = String.valueOf(value);
        }
        return returnArray;
    }

    @Override
    public ArrayList<String> extractFloodData_raw(String directory, String orgId, String stationId, String startDate, String endDate) throws WaterDataException {
        throw new WaterDataException("The " + database + " database contains no flood discharge data.");
    }
    
    @Override
    public double[][] extractFloodData_formatted(String directory, String orgId, String stationId, String startDate, String endDate) throws WaterDataException {
        throw new WaterDataException("The " + database + " database contains no flood discharge data.");
    }

    @Override
    public ArrayList<String> extractInstantaneousFlowData_raw(String directory, String stationId, String startDate, String endDate) throws WaterDataException {
        throw new WaterDataException("The " + database + " database contains no instantaneous (15-minute) flow data.");
    }

    @Override
    public String[][] extractInstantaneousFlowData_formatted(String directory, String stationId, String startDate, String endDate) throws WaterDataException {
        throw new WaterDataException("The " + database + " database contains no instantaneous (15-minute) flow data.");
    }

    @Override
    public ArrayList<String> extractStageDischarge_raw(String stationId) throws WaterDataException {
        throw new WaterDataException("The " + database + " database contains no stage-discharge (rating curve) data.");
    }

    @Override
    public double[][] extractStageDischarge_formatted(String stationId) throws WaterDataException {
        throw new WaterDataException("The " + database + " database contains no stage-discharge (rating curve) data.");
    }
    /**
     * * Reformats dates from csip-lib-water expected "YYYY-MM-DD" format to Water Quality Data Portal expected "MM-DD-YYYY" format
     * @param originalDate a string date in "YYYY-MM-DD" format
     * @return  a string date in "MM-DD-YYYY" format
     */
    private String reformatDate(String originalDate){
        String year = originalDate.substring(0,4);
        String month = originalDate.substring(5,7);
        String day = originalDate.substring(8);
        String newDate = month + "-" + day + "-" + year;
        return newDate;
    }
    /**
     * URL encode input strings for Water Quality Data Portal so they work properly (i.e. no " " in station IDs)
     * @param originalString a string
     * @return  a url encoded string
     */
    private String reformatString(String originalString) throws WaterDataException{
        String newString = originalString;
        try {
            newString = URLEncoder.encode(newString, "UTF-8");
        } catch (UnsupportedEncodingException ex) {
            throw new WaterDataException("The was an issue extracting " + database + " water quality data for the specified input: '" + originalString + "'." + ex.getMessage());
        }
        return newString;
    }
}