WaterData_WqDataPortal.java [src/WaterData] Revision: default Date:
package WaterData;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import utils.WebPageUtils;
/**
* Last Updated: 9-April-2019
* @author Tyler Wible
* @since 20-November-2018
* The Water Quality Data Portal covers STORET, USGS, and STEWARDS databases, more information available here:
* https://www.waterqualitydata.us/webservices_documentation/
*/
public class WaterData_WqDataPortal implements WaterDataInterface{
String database = "STORET"; //This is currently only setup for STORET stations, so it needs expand to USGS and STEWARDS
@Override
public String getDataSourceCitation(){
//Get today's date for the source reference
DateFormat sourceDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm");
String today = sourceDateFormat.format(new Date());
//Cite EPA WQX/STORET's Water Quality Data Portal
String dataSource = "Stream flow data and water quality test data retrieved from the U.S. Environmental Protection Agency, WQX/STORET, Water Quality Data Portal. https://www.waterqualitydata.us/, accessed: " + today;
return dataSource;
}
@Override
public ArrayList<String> extractFlowData_raw(String directory, String orgId, String stationId, String startDate, String endDate) throws WaterDataException {
//Get flow website from inputs
String wqTest = "00060 Flow -- cfs";
ArrayList<String> webpageAll = extractWaterQualityData_raw(directory, orgId, stationId, startDate, endDate, wqTest);
return webpageAll;
}
@Override
public String[][] extractFlowData_formatted(String directory, String orgId, String stationId, String startDate, String endDate) throws WaterDataException{
//Fetch flow data
String wqTest = "00060 Flow -- cfs";
String[][] returnArray = extractWaterQualityData_formatted(directory, orgId, stationId, startDate, endDate, wqTest);
return returnArray;
}
@Override
public ArrayList<String> extractWaterQualityData_raw(String directory, String orgId, String stationId, String startDate, String endDate, String wqTest) throws WaterDataException {
//Fix dates to match Webservice-expected "MM-DD-YYYY" format
String newStartDate = reformatDate(startDate);
String newEndDate = reformatDate(endDate);
//URL encode stationId and orgId to prevent issues
String newStationId = reformatString(stationId);
String newOrgId = reformatString(orgId);
//Specify water quality website from inputs
//String wqUrl = "https://www.waterqualitydata.us/data/Result/search?providers=STORET&organization=MNPCA_BIO&siteid=MNPCA_BIO-S010-342&mimeType=csv"
String wqUrl = "https://www.waterqualitydata.us/data/Result/search?providers=STORET&" +
"organization=" + newOrgId +
"&siteid=" + newStationId +
"&startDateLo=" + newStartDate +
"&startDateHi=" + newEndDate;
if(wqTest.isEmpty() || wqTest.equalsIgnoreCase("all")){
wqUrl += "&mimeType=tsv";
}else{
try {
String[] resultArray = WaterQualityInfo.getWqTestDataInfo(wqTest, database);
//String wqCode = resultArray[0];
String wqLabel = resultArray[1];
//String units = resultArray[2];
//double ldcConversion = Double.parseDouble(resultArray[3]);
//String ldcEndUnits = resultArray[4];
//URL encode wqLabel to prevent issues
String newWqLabel = reformatString(wqLabel);
wqUrl += "&characteristicName=" + newWqLabel + "&mimeType=tsv";
//wqUrl += "&pCode=" + wqCode + "&mimeType=tsv";
} catch (IOException ex) {
throw new WaterDataException("The was an issue extracting " + database + " water quality parameters from the specified water quality test: " + wqTest + "." + ex.getMessage());
}
}
//Fetch the water quality data webpage for the current USGS station
ArrayList<String> webpageAll = new ArrayList<>();
try {
webpageAll = WebPageUtils.downloadWebpage(wqUrl);
} catch (IOException ex) {
throw new WaterDataException("The was an issue extracting " + database + " water quality data from the specified URl: " + wqUrl + "." + ex.getMessage());
}
return webpageAll;
}
@Override
public String[][] extractWaterQualityData_formatted(String directory, String orgId, String stationId, String startDate, String endDate, String wqTest) throws WaterDataException {
//Fetch water quality data
ArrayList<String> webpageAll = extractWaterQualityData_raw(directory, orgId, stationId, startDate, endDate, wqTest);
//Get current wq units
int index1 = wqTest.indexOf("--");
String desiredUnits = wqTest.substring(index1 + 2).trim();
//Reformat data
int size = webpageAll.size() - 1;//don't keep header in final size
if (size < 0){
size = 0;
}
String[][] returnArray = new String[size][2];
for(int i=1; i<webpageAll.size(); i++){
String[] currentColumns = webpageAll.get(i).split("\t");
//currentColumns[6] = date
//currentColumns[31] = parameter name (wqTest)
//currentColumns[33] = value
//currentColumns[34] = units
//currentColumns[30] = result detection condition
//currentColumns[58] = detection limit type
//currentColumns[59] = detection limit value
//currentColumns[60] = detection limit units
double conversion = WaterQualityInfo.getWQconversion(desiredUnits, currentColumns[34]);
double value = Double.parseDouble(currentColumns[33]) * conversion;
returnArray[i-1][0] = currentColumns[6];
returnArray[i-1][1] = String.valueOf(value);
}
return returnArray;
}
@Override
public ArrayList<String> extractFloodData_raw(String directory, String orgId, String stationId, String startDate, String endDate) throws WaterDataException {
throw new WaterDataException("The " + database + " database contains no flood discharge data.");
}
@Override
public double[][] extractFloodData_formatted(String directory, String orgId, String stationId, String startDate, String endDate) throws WaterDataException {
throw new WaterDataException("The " + database + " database contains no flood discharge data.");
}
@Override
public ArrayList<String> extractInstantaneousFlowData_raw(String directory, String stationId, String startDate, String endDate) throws WaterDataException {
throw new WaterDataException("The " + database + " database contains no instantaneous (15-minute) flow data.");
}
@Override
public String[][] extractInstantaneousFlowData_formatted(String directory, String stationId, String startDate, String endDate) throws WaterDataException {
throw new WaterDataException("The " + database + " database contains no instantaneous (15-minute) flow data.");
}
@Override
public ArrayList<String> extractStageDischarge_raw(String stationId) throws WaterDataException {
throw new WaterDataException("The " + database + " database contains no stage-discharge (rating curve) data.");
}
@Override
public double[][] extractStageDischarge_formatted(String stationId) throws WaterDataException {
throw new WaterDataException("The " + database + " database contains no stage-discharge (rating curve) data.");
}
/**
* * Reformats dates from csip-lib-water expected "YYYY-MM-DD" format to Water Quality Data Portal expected "MM-DD-YYYY" format
* @param originalDate a string date in "YYYY-MM-DD" format
* @return a string date in "MM-DD-YYYY" format
*/
private String reformatDate(String originalDate){
String year = originalDate.substring(0,4);
String month = originalDate.substring(5,7);
String day = originalDate.substring(8);
String newDate = month + "-" + day + "-" + year;
return newDate;
}
/**
* URL encode input strings for Water Quality Data Portal so they work properly (i.e. no " " in station IDs)
* @param originalString a string
* @return a url encoded string
*/
private String reformatString(String originalString) throws WaterDataException{
String newString = originalString;
try {
newString = URLEncoder.encode(newString, "UTF-8");
} catch (UnsupportedEncodingException ex) {
throw new WaterDataException("The was an issue extracting " + database + " water quality data for the specified input: '" + originalString + "'." + ex.getMessage());
}
return newString;
}
}