WaterData_WqDataPortal_narrow.java [src/WaterData] Revision: default Date:
package WaterData;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import utils.WebPageUtils;
/**
* Last Updated: 15-April-2019
* @author Tyler Wible
* @since 15-January-2019
* The Water Quality Data Portal covers STORET, USGS, and STEWARDS databases, more information available here:
* https://www.waterqualitydata.us/webservices_documentation/
*/
public class WaterData_WqDataPortal_narrow implements WaterDataInterface{
String database = "STORET"; //This is currently only setup for STORET stations, so it needs expand to USGS and STEWARDS
@Override
public String getDataSourceCitation(){
//Get today's date for the source reference
DateFormat sourceDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm");
String today = sourceDateFormat.format(new Date());
//Cite EPA WQX/STORET's Water Quality Data Portal
String dataSource = "Stream flow data and water quality test data retrieved from the U.S. Environmental Protection Agency, WQX/STORET, Water Quality Data Portal. https://www.waterqualitydata.us/, accessed: " + today;
return dataSource;
}
@Override
public ArrayList<String> extractFlowData_raw(String directory, String orgId, String stationId, String startDate, String endDate) throws WaterDataException {
//Get flow website from inputs
String wqTest = "00060 Flow -- cfs";
ArrayList<String> webpageAll = extractWaterQualityData_raw(directory, orgId, stationId, startDate, endDate, wqTest);
return webpageAll;
}
@Override
public String[][] extractFlowData_formatted(String directory, String orgId, String stationId, String startDate, String endDate) throws WaterDataException{
//Fetch flow data
String wqTest = "00060 Flow -- cfs";
String[][] returnArray = extractWaterQualityData_formatted(directory, orgId, stationId, startDate, endDate, wqTest);
return returnArray;
}
@Override
public ArrayList<String> extractWaterQualityData_raw(String directory, String orgId, String stationId, String startDate, String endDate, String wqTest) throws WaterDataException {
//Fix dates to match Webservice-expected "MM-DD-YYYY" format
String newStartDate = reformatDate(startDate);
String newEndDate = reformatDate(endDate);
//URL encode stationId and orgId to prevent issues
String newStationId = reformatString(stationId);
String newOrgId = reformatString(orgId);
//Specify water quality website from inputs
//String wqUrl = "https://www.waterqualitydata.us/data/Result/search?providers=STORET&dataProfile=narrowResult&organization=MNPCA_BIO&siteid=MNPCA_BIO-S010-342&mimeType=csv"
String wqUrl = "https://www.waterqualitydata.us/data/Result/search?providers=STORET&dataProfile=narrowResult&" +
"organization=" + newOrgId +
"&siteid=" + newStationId +
"&startDateLo=" + newStartDate +
"&startDateHi=" + newEndDate;
if(wqTest.isEmpty() || wqTest.equalsIgnoreCase("all")){
wqUrl += "&mimeType=tsv";
}else if(wqTest.equalsIgnoreCase("flow")){
wqUrl += "&characteristicName=Flow&mimeType=tsv";
}else{
try {
String[] resultArray = WaterQualityInfo.getWqTestDataInfo(wqTest, database);
//String wqCode = resultArray[0];
String wqLabel = resultArray[1];
//String units = resultArray[2];
//double ldcConversion = Double.parseDouble(resultArray[3]);
//String ldcEndUnits = resultArray[4];
//URL encode wqLabel to prevent issues
String newWqLabel = reformatString(wqLabel);
wqUrl += "&characteristicName=" + newWqLabel + "&mimeType=tsv";
} catch (IOException ex) {
throw new WaterDataException("The was an issue extracting " + database + " water quality parameters from the specified water quality test: " + wqTest + "." + ex.getMessage());
}
}
//Fetch the water quality data webpage for the current USGS station
ArrayList<String> webpageAll = new ArrayList<>();
try {
webpageAll = WebPageUtils.downloadWebpage(wqUrl);
} catch (IOException ex) {
throw new WaterDataException("The was an issue extracting " + database + " water quality data from the specified URl: " + wqUrl + "." + ex.getMessage());
}
//Check if this is a water quality parameter with 2 types (dissolved/filtered and total/unfiltered)
ArrayList<String> webpageMin = new ArrayList<>();
if(wqTest.isEmpty() || wqTest.equalsIgnoreCase("all")){
//Save everything
for(int i=0; i<webpageAll.size(); i++){
webpageMin.add(webpageAll.get(i));
}
}else{
//Determine if dissolved (filtered) or total (unfiltered) test data is requested
boolean dissolvedTF = false;
boolean totalTF = false;
// try {
String wqCode = wqTest.substring(0,5);//pull just the 5 digit USGS WQ code
// String wqLabel = WaterQualityInfo.getUSGSwqLabel(wqCode);
// if(wqLabel.contains(" filtered")){
// dissolvedTF = true;
// }else if(wqLabel.contains(" unfiltered")){
// totalTF = true;
// }
if(wqTest.contains(" filtered")){
dissolvedTF = true;
}else if(wqTest.contains(" unfiltered")){
totalTF = true;
}
// } catch (IOException ex) {
// throw new WaterDataException("The was an issue extracting " + database + " water quality parameters from the specified water quality test: " + wqTest + "." + ex.getMessage());
// }
//Save only the requested type of data (dissolved/filtered or total/unfiltered)
for(int i=0; i<webpageAll.size(); i++){
String currentLine = webpageAll.get(i);
if(i == 0){
webpageMin.add(currentLine);
}else{
String[] currentColumns = webpageAll.get(i).split("\t");
//currentColumns[3] = date
//currentColumns[7] = result Identifier (look up value for getting detection limit info)
//currentColumns[9] = result detection condition
//currentColumns[11] = parameter name (wqTest)
//currentColumns[12] = parameter type (i.e. total, dissolved, total recoverable)
//currentColumns[13] = value
//currentColumns[14] = units
//currentColumns[15] = measure qualification code
//currentColumns[71] = ResultDetectionQuantitationLimitUrl
//Check for dissolved/total/none result flags
if(currentColumns[12].isEmpty()){
//It is a water quality parameter that has neither a dissolved or total 'result sample fraction'
webpageMin.add(currentLine);
}else if(!currentColumns[12].isEmpty() && currentColumns[12].equalsIgnoreCase("dissolved") && dissolvedTF){
//This is the dissolved version of the parameter and that is what was requested (based on USGS parameter code name)
webpageMin.add(currentLine);
}else if(!currentColumns[12].isEmpty() && currentColumns[12].equalsIgnoreCase("total") && totalTF){
//This is the total version of the parameter and that is what was requested (based on USGS parameter code name)
webpageMin.add(currentLine);
}
}
}
}
return webpageMin;
}
@Override
public String[][] extractWaterQualityData_formatted(String directory, String orgId, String stationId, String startDate, String endDate, String wqTest) throws WaterDataException {
//Fetch water quality data
ArrayList<String> webpageAll = extractWaterQualityData_raw(directory, orgId, stationId, startDate, endDate, wqTest);
//Get current wq units
int index1 = wqTest.indexOf("--");
String desiredUnits = wqTest.substring(index1 + 2).trim();
//Only keep non-blank values for data arrays
int ctr = 0;
for(int i=1; i<webpageAll.size(); i++){
String[] currentColumns = webpageAll.get(i).split("\t");
if(!currentColumns[13].isEmpty()){
ctr++;
}
}
//Reformat data
String[][] returnArray = new String[ctr][2];
ctr = 0;
for(int i=1; i<webpageAll.size(); i++){
String[] currentColumns = webpageAll.get(i).split("\t");
//currentColumns[3] = date
//currentColumns[7] = result Identifier (look up value for getting detection limit info)
//currentColumns[9] = result detection condition
//currentColumns[11] = parameter name (wqTest)
//currentColumns[12] = parameter type (i.e. total, dissolved, total recoverable)
//currentColumns[13] = value
//currentColumns[14] = units
//currentColumns[15] = measure qualification code
//currentColumns[71] = ResultDetectionQuantitationLimitUrl
if(!currentColumns[13].isEmpty()){
double conversion = WaterQualityInfo.getWQconversion(desiredUnits, currentColumns[14]);
double value = Double.parseDouble(currentColumns[13]) * conversion;
returnArray[ctr][0] = currentColumns[3];
returnArray[ctr][1] = String.valueOf(value);
ctr++;
}
}
return returnArray;
}
@Override
public ArrayList<String> extractFloodData_raw(String directory, String orgId, String stationId, String startDate, String endDate) throws WaterDataException {
throw new WaterDataException("The " + database + " database contains no flood discharge data.");
}
@Override
public double[][] extractFloodData_formatted(String directory, String orgId, String stationId, String startDate, String endDate) throws WaterDataException {
throw new WaterDataException("The " + database + " database contains no flood discharge data.");
}
@Override
public ArrayList<String> extractInstantaneousFlowData_raw(String directory, String stationId, String startDate, String endDate) throws WaterDataException {
throw new WaterDataException("The " + database + " database contains no instantaneous (15-minute) flow data.");
}
@Override
public String[][] extractInstantaneousFlowData_formatted(String directory, String stationId, String startDate, String endDate) throws WaterDataException {
throw new WaterDataException("The " + database + " database contains no instantaneous (15-minute) flow data.");
}
@Override
public ArrayList<String> extractStageDischarge_raw(String stationId) throws WaterDataException {
throw new WaterDataException("The " + database + " database contains no stage-discharge (rating curve) data.");
}
@Override
public double[][] extractStageDischarge_formatted(String stationId) throws WaterDataException {
throw new WaterDataException("The " + database + " database contains no stage-discharge (rating curve) data.");
}
/**
* Reformats dates from csip-lib-water expected "YYYY-MM-DD" format to Water Quality Data Portal expected "MM-DD-YYYY" format
* @param originalDate a string date in "YYYY-MM-DD" format
* @return a string date in "MM-DD-YYYY" format
*/
private String reformatDate(String originalDate){
String year = originalDate.substring(0,4);
String month = originalDate.substring(5,7);
String day = originalDate.substring(8);
String newDate = month + "-" + day + "-" + year;
return newDate;
}
/**
* URL encode input strings for Water Quality Data Portal so they work properly (i.e. no " " in station IDs)
* @param originalString a string
* @return a url encoded string
*/
private String reformatString(String originalString) throws WaterDataException{
String newString = originalString;
try {
newString = URLEncoder.encode(newString, "UTF-8");
} catch (UnsupportedEncodingException ex) {
throw new WaterDataException("The was an issue extracting " + database + " water quality data for the specified input: '" + originalString + "'." + ex.getMessage());
}
return newString;
}
}