WaterData_WqDataPortal_narrow.java [src/WaterData] Revision: default  Date:
package WaterData;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import utils.WebPageUtils;

/**
* Last Updated: 15-April-2019
* @author Tyler Wible
* @since 15-January-2019
* The Water Quality Data Portal covers STORET, USGS, and STEWARDS databases, more information available here:
* https://www.waterqualitydata.us/webservices_documentation/
*/
public class WaterData_WqDataPortal_narrow implements WaterDataInterface{
    String database = "STORET"; //This is currently only setup for STORET stations, so it needs expand to USGS and STEWARDS
    
    @Override
    public String getDataSourceCitation(){
        //Get today's date for the source reference
        DateFormat sourceDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm");
        String today = sourceDateFormat.format(new Date());
        
        //Cite EPA WQX/STORET's Water Quality Data Portal
        String dataSource = "Stream flow data and water quality test data retrieved from the U.S. Environmental Protection Agency, WQX/STORET, Water Quality Data Portal. https://www.waterqualitydata.us/, accessed: " + today;
        return dataSource;
    }

    @Override
    public ArrayList<String> extractFlowData_raw(String directory, String orgId, String stationId, String startDate, String endDate) throws WaterDataException {
        //Get flow website from inputs
        String wqTest = "00060 Flow -- cfs";
        ArrayList<String> webpageAll = extractWaterQualityData_raw(directory, orgId, stationId, startDate, endDate, wqTest);
        return webpageAll;
    }

    @Override
    public String[][] extractFlowData_formatted(String directory, String orgId, String stationId, String startDate, String endDate) throws WaterDataException{
        //Fetch flow data
        String wqTest = "00060 Flow -- cfs";
        String[][] returnArray = extractWaterQualityData_formatted(directory, orgId, stationId, startDate, endDate, wqTest);
        return returnArray;
    }

    @Override
    public ArrayList<String> extractWaterQualityData_raw(String directory, String orgId, String stationId, String startDate, String endDate, String wqTest) throws WaterDataException {
        //Fix dates to match Webservice-expected "MM-DD-YYYY" format
        String newStartDate = reformatDate(startDate);
        String newEndDate = reformatDate(endDate);
        
        //URL encode stationId and orgId to prevent issues
        String newStationId = reformatString(stationId);
        String newOrgId = reformatString(orgId);
        
        //Specify water quality website from inputs
        //String wqUrl = "https://www.waterqualitydata.us/data/Result/search?providers=STORET&dataProfile=narrowResult&organization=MNPCA_BIO&siteid=MNPCA_BIO-S010-342&mimeType=csv"
        String wqUrl = "https://www.waterqualitydata.us/data/Result/search?providers=STORET&dataProfile=narrowResult&" +
                "organization=" + newOrgId +
                "&siteid=" + newStationId +
                "&startDateLo=" + newStartDate +
                "&startDateHi=" + newEndDate;
        if(wqTest.isEmpty() || wqTest.equalsIgnoreCase("all")){
            wqUrl += "&mimeType=tsv";
        }else if(wqTest.equalsIgnoreCase("flow")){
            wqUrl += "&characteristicName=Flow&mimeType=tsv";
        }else{
            try {
                String[] resultArray = WaterQualityInfo.getWqTestDataInfo(wqTest, database);
                //String wqCode = resultArray[0];
                String wqLabel = resultArray[1];
                //String units = resultArray[2];
                //double ldcConversion = Double.parseDouble(resultArray[3]);
                //String ldcEndUnits = resultArray[4];
                
                //URL encode wqLabel to prevent issues
                String newWqLabel = reformatString(wqLabel);
                wqUrl += "&characteristicName=" + newWqLabel + "&mimeType=tsv";
            } catch (IOException ex) {
                throw new WaterDataException("The was an issue extracting " + database + " water quality parameters from the specified water quality test: " + wqTest + "." + ex.getMessage());
            }
        }
        
        //Fetch the water quality data webpage for the current USGS station
        ArrayList<String> webpageAll = new ArrayList<>();
        try {
            webpageAll = WebPageUtils.downloadWebpage(wqUrl);
        } catch (IOException ex) {
            throw new WaterDataException("The was an issue extracting " + database + " water quality data from the specified URl: " + wqUrl + "." + ex.getMessage());
        }
        
        //Check if this is a water quality parameter with 2 types (dissolved/filtered and total/unfiltered)
        ArrayList<String> webpageMin = new ArrayList<>();
        if(wqTest.isEmpty() || wqTest.equalsIgnoreCase("all")){
            //Save everything
            for(int i=0; i<webpageAll.size(); i++){
                webpageMin.add(webpageAll.get(i));
            }
        }else{
            //Determine if dissolved (filtered) or total (unfiltered) test data is requested
            boolean dissolvedTF = false;
            boolean totalTF = false;
//            try {
            String wqCode = wqTest.substring(0,5);//pull just the 5 digit USGS WQ code
//                String wqLabel = WaterQualityInfo.getUSGSwqLabel(wqCode);
//                if(wqLabel.contains(" filtered")){
//                    dissolvedTF = true;
//                }else if(wqLabel.contains(" unfiltered")){
//                    totalTF = true;
//                }
            if(wqTest.contains(" filtered")){
                dissolvedTF = true;
            }else if(wqTest.contains(" unfiltered")){
                totalTF = true;
            }
//            } catch (IOException ex) {
//                throw new WaterDataException("The was an issue extracting " + database + " water quality parameters from the specified water quality test: " + wqTest + "." + ex.getMessage());
//            }
            
            //Save only the requested type of data (dissolved/filtered or total/unfiltered)
            for(int i=0; i<webpageAll.size(); i++){
                String currentLine = webpageAll.get(i);
                if(i == 0){
                    webpageMin.add(currentLine);
                }else{
                    String[] currentColumns = webpageAll.get(i).split("\t");
                    //currentColumns[3] = date
                    //currentColumns[7] = result Identifier (look up value for getting detection limit info)
                    //currentColumns[9] = result detection condition
                    //currentColumns[11] = parameter name (wqTest)
                    //currentColumns[12] = parameter type (i.e. total, dissolved, total recoverable)
                    //currentColumns[13] = value
                    //currentColumns[14] = units
                    //currentColumns[15] = measure qualification code
                    //currentColumns[71] = ResultDetectionQuantitationLimitUrl

                    //Check for dissolved/total/none result flags
                    if(currentColumns[12].isEmpty()){
                        //It is a water quality parameter that has neither a dissolved or total 'result sample fraction'
                        webpageMin.add(currentLine);
                    }else if(!currentColumns[12].isEmpty() && currentColumns[12].equalsIgnoreCase("dissolved") && dissolvedTF){
                        //This is the dissolved version of the parameter and that is what was requested (based on USGS parameter code name)
                        webpageMin.add(currentLine);
                    }else if(!currentColumns[12].isEmpty() && currentColumns[12].equalsIgnoreCase("total") && totalTF){
                        //This is the total version of the parameter and that is what was requested (based on USGS parameter code name)
                        webpageMin.add(currentLine);
                    }
                }
            }
        }
        return webpageMin;
    }
    
    @Override
    public String[][] extractWaterQualityData_formatted(String directory, String orgId, String stationId, String startDate, String endDate, String wqTest) throws WaterDataException {
        //Fetch water quality data
        ArrayList<String> webpageAll = extractWaterQualityData_raw(directory, orgId, stationId, startDate, endDate, wqTest);
        
        //Get current wq units
        int index1 = wqTest.indexOf("--");
        String desiredUnits = wqTest.substring(index1 + 2).trim();
        
        //Only keep non-blank values for data arrays
        int ctr = 0;
        for(int i=1; i<webpageAll.size(); i++){
            String[] currentColumns = webpageAll.get(i).split("\t");
            if(!currentColumns[13].isEmpty()){
                ctr++;
            }
        }
        
        //Reformat data
        String[][] returnArray = new String[ctr][2];
        ctr = 0;
        for(int i=1; i<webpageAll.size(); i++){
            String[] currentColumns = webpageAll.get(i).split("\t");
            //currentColumns[3] = date
            //currentColumns[7] = result Identifier (look up value for getting detection limit info)
            //currentColumns[9] = result detection condition
            //currentColumns[11] = parameter name (wqTest)
            //currentColumns[12] = parameter type (i.e. total, dissolved, total recoverable)
            //currentColumns[13] = value
            //currentColumns[14] = units
            //currentColumns[15] = measure qualification code
            //currentColumns[71] = ResultDetectionQuantitationLimitUrl
            if(!currentColumns[13].isEmpty()){
                double conversion = WaterQualityInfo.getWQconversion(desiredUnits, currentColumns[14]);
                double value = Double.parseDouble(currentColumns[13]) * conversion;
                returnArray[ctr][0] = currentColumns[3];
                returnArray[ctr][1] = String.valueOf(value);
                ctr++;
            }
            
        }
        return returnArray;
    }

    @Override
    public ArrayList<String> extractFloodData_raw(String directory, String orgId, String stationId, String startDate, String endDate) throws WaterDataException {
        throw new WaterDataException("The " + database + " database contains no flood discharge data.");
    }
    
    @Override
    public double[][] extractFloodData_formatted(String directory, String orgId, String stationId, String startDate, String endDate) throws WaterDataException {
        throw new WaterDataException("The " + database + " database contains no flood discharge data.");
    }

    @Override
    public ArrayList<String> extractInstantaneousFlowData_raw(String directory, String stationId, String startDate, String endDate) throws WaterDataException {
        throw new WaterDataException("The " + database + " database contains no instantaneous (15-minute) flow data.");
    }

    @Override
    public String[][] extractInstantaneousFlowData_formatted(String directory, String stationId, String startDate, String endDate) throws WaterDataException {
        throw new WaterDataException("The " + database + " database contains no instantaneous (15-minute) flow data.");
    }

    @Override
    public ArrayList<String> extractStageDischarge_raw(String stationId) throws WaterDataException {
        throw new WaterDataException("The " + database + " database contains no stage-discharge (rating curve) data.");
    }

    @Override
    public double[][] extractStageDischarge_formatted(String stationId) throws WaterDataException {
        throw new WaterDataException("The " + database + " database contains no stage-discharge (rating curve) data.");
    }
    /**
     * Reformats dates from csip-lib-water expected "YYYY-MM-DD" format to Water Quality Data Portal expected "MM-DD-YYYY" format
     * @param originalDate a string date in "YYYY-MM-DD" format
     * @return  a string date in "MM-DD-YYYY" format
     */
    private String reformatDate(String originalDate){
        String year = originalDate.substring(0,4);
        String month = originalDate.substring(5,7);
        String day = originalDate.substring(8);
        String newDate = month + "-" + day + "-" + year;
        return newDate;
    }
    /**
     * URL encode input strings for Water Quality Data Portal so they work properly (i.e. no " " in station IDs)
     * @param originalString a string
     * @return  a url encoded string
     */
    private String reformatString(String originalString) throws WaterDataException{
        String newString = originalString;
        try {
            newString = URLEncoder.encode(newString, "UTF-8");
        } catch (UnsupportedEncodingException ex) {
            throw new WaterDataException("The was an issue extracting " + database + " water quality data for the specified input: '" + originalString + "'." + ex.getMessage());
        }
        return newString;
    }
}