WaterData_CDSN.java [src/WaterData] Revision: default Date:
package WaterData;
import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import utils.WebPageUtils;
/**
* Last Updated: 9-April-2019
* @author Tyler Wible
* @since 19-April-2016
*/
public class WaterData_CDSN implements WaterDataInterface{
public String database = "CDSN";
@Override
public String getDataSourceCitation(){
//Get today's date for the source reference
DateFormat sourceDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm");
String today = sourceDateFormat.format(new Date());
//Cite CDSN
String dataSource = "Stream flow data and water quality test data retrieved from the Colorado Data Sharing Network (CDSN). http://www.coloradowaterdata.org/, accessed: " + today;
return dataSource;
}
@Override
public ArrayList<String> extractFlowData_raw(String directory, String orgId, String stationId, String startDate, String endDate) throws WaterDataException {
//Specify flow website from inputs
String flowUrl = createCdsnUrl(orgId, stationId, "flow", startDate, endDate);
//Fetch the flow data webpage for the current CDSN station
ArrayList<String> webpageAll = new ArrayList<>();
try {
ArrayList<String> webpageRaw = WebPageUtils.downloadWebpage_slowData(flowUrl);
webpageAll = parseCdsnJsonResult(webpageRaw, orgId, stationId, "flow");
} catch (IOException ex) {
throw new WaterDataException("The was an issue extracting " + database + " flow data from the specified URl: " + flowUrl + "." + ex.getMessage());
}
return webpageAll;
}
@Override
public String[][] extractFlowData_formatted(String directory, String orgId, String stationId, String startDate, String endDate) throws WaterDataException {
//Fetch flow data
ArrayList<String> stationData = extractFlowData_raw(directory, orgId, stationId, startDate, endDate);
//Reformat data
String[][] returnArray = new String[stationData.size() - 1][2];
for(int i=1; i<stationData.size(); i++){
String[] currentColumns = stationData.get(i).split("\t");
returnArray[i-1][0] = currentColumns[4]; //date
returnArray[i-1][1] = currentColumns[12]; //value
}
return returnArray;
}
@Override
public ArrayList<String> extractWaterQualityData_raw(String directory, String orgId, String stationId, String startDate, String endDate, String wqTest) throws WaterDataException {
//Specify water quality website from inputs
String wqUrl = createCdsnUrl(orgId, stationId, wqTest, startDate, endDate);
//Fetch the water quality data webpage for the current CDSN station
ArrayList<String> webpageAll = new ArrayList<>();
try {
ArrayList<String> webpageRaw = WebPageUtils.downloadWebpage_slowData(wqUrl);
webpageAll = parseCdsnJsonResult(webpageRaw, orgId, stationId, wqTest);
} catch (IOException ex) {
throw new WaterDataException("The was an issue extracting " + database + " water quality data from the specified URl: " + wqUrl + "." + ex.getMessage());
}
return webpageAll;
}
@Override
public String[][] extractWaterQualityData_formatted(String directory, String orgId, String stationId, String startDate, String endDate, String wqTest) throws WaterDataException {
//Fetch water quality data
ArrayList<String> stationData = extractWaterQualityData_raw(directory, orgId, stationId, startDate, endDate, wqTest);
//Reformat data
String[][] returnArray = new String[stationData.size() - 1][2];
for(int i=1; i<stationData.size(); i++){
String[] currentColumns = stationData.get(i).split("\t");
returnArray[i-1][0] = currentColumns[4]; //date
returnArray[i-1][1] = currentColumns[12]; //value
}
return returnArray;
}
@Override
public ArrayList<String> extractFloodData_raw(String directory, String orgId, String stationId, String startDate, String endDate) throws WaterDataException {
throw new WaterDataException("The " + database + " database contains no flood discharge data.");
}
@Override
public double[][] extractFloodData_formatted(String directory, String orgId, String stationId, String startDate, String endDate) throws WaterDataException {
throw new WaterDataException("The " + database + " database contains no flood discharge data.");
}
@Override
public ArrayList<String> extractInstantaneousFlowData_raw(String directory, String stationId, String startDate, String endDate) throws WaterDataException {
throw new WaterDataException("The " + database + " database contains no instantaneous (15-minute) flow data.");
}
@Override
public String[][] extractInstantaneousFlowData_formatted(String directory, String stationId, String startDate, String endDate) throws WaterDataException {
throw new WaterDataException("The " + database + " database contains no instantaneous (15-minute) flow data.");
}
@Override
public ArrayList<String> extractStageDischarge_raw(String stationId) throws WaterDataException {
throw new WaterDataException("The " + database + " database contains no stage-discharge (rating curve) data.");
}
@Override
public double[][] extractStageDischarge_formatted(String stationId) throws WaterDataException {
throw new WaterDataException("The " + database + " database contains no stage-discharge (rating curve) data.");
}
/**
* Creates a rest url for CDSN for retrieving data
* @param orgId The ID for the supervising organization for the station within STORET
* @param stationId station ID of the current station, used to search the STORET databes
* @param wqTest The word 'flow' or the formatted name for the water quality test being used, Format is: '00600 Total nitrogen, water, unfiltered, milligrams per liter -- mg/L' = '5-digit-USGS-water-quality-test-code test-name -- units'
* @param startDate the used defined begin date of search (yyyy-mm-dd)
* @param endDate the user defined end date of search (yyyy-mm-dd)
* @return a String of the web url for the data search
*/
private static String createCdsnUrl(String orgId, String stationId, String wqTest, String startDate, String endDate){
//If no date input, make it the maximum of available data
if(startDate == null || startDate.equalsIgnoreCase("")){
startDate = "1900-01-01";
}
if(endDate == null || endDate.equalsIgnoreCase("")){
// Pull current date for upper limit of data search
DateFormat desiredDateFormat = new SimpleDateFormat("yyyy-MM-dd");
Date currentDate = new Date();
endDate = desiredDateFormat.format(currentDate);
}
//Reformat dates for CDSN format only
String minDate = startDate.substring(5,7) + "-" + startDate.substring(8) + "-" + startDate.substring(0,4);
String maxDate = endDate.substring(5,7) + "-" + endDate.substring(8) + "-" + endDate.substring(0,4);
//Specify data search url from inputs
//https://awqms.goldsystems.com/api/ResultsVer1?ContentType=json&OrganizationIdentifiersCsv=CITYFTCO_WQX&MonitoringLocationIdentifiersCsv=1EFF&MinDate=01-01-1900&MaxDate=04-25-2016&Characteristic=Ammonia-nitrogen
String cdsnDataUrl = "https://awqms.goldsystems.com/api/ResultsVer1?ContentType=json" +
"&OrganizationIdentifiersCsv=" + orgId +
"&MonitoringLocationIdentifiersCsv=" + stationId +
"&MinDate=" + minDate +
"&MaxDate=" + maxDate;
if(!wqTest.isEmpty() && !wqTest.equalsIgnoreCase("all")){
String characteristic_name = wqTest;
if(wqTest.length() > 5){
//extract "Ammonia-nitrogen as N" from "00625 Ammonia-nitrogen as N -- mg/L"
characteristic_name = wqTest.substring(6, wqTest.indexOf("--") - 1);
}
cdsnDataUrl = cdsnDataUrl + "&Characteristic=" + characteristic_name;
}
//Possible search refinement key words to add to url string above
//ActivityType (lots)
//Characteristic (lots)
//ContentType (xml, json)
//CountyName
//Huc8
//Huc12
//MaxDate (mm/dd/yyyy or mm-dd-yyyy)
//MediaName (Air, Water, Biological, Soil, Sediment, Other, Tissue, Habitat)
//Media SubdivisionName (lots)
//MinDate (mm/dd/yyyy or mm-dd-yyyy)
//MonitoringLocationIdentifiersCsv ("id1, id2, id3")
//MonitoringLocationType (lots)
//OrganizationIdentifiersCsv ("id1, id2, id3")
//ProjectIdentifier (lots)
//StateCode (CO)
//WaterBodyName (lots)
//WatershedManagementUnit (lots)
return cdsnDataUrl;
}
/**
* Parses the CDSN JSON response and reformats it into a result list
* @param webpageRaw An ArrayList containing the raw JSON result from the CDSN search request
* @param orgId The ID for the supervising organization for the station within STORET
* @param stationId station ID of the current station, used to search the STORET databes
* @param wqTest The word 'flow' or the formatted name for the water quality test being used, Format is: '00600 Total nitrogen, water, unfiltered, milligrams per liter -- mg/L' = '5-digit-USGS-water-quality-test-code test-name -- units'
* @return an ArrayList containing the results of the web service search for flow data using the above inputs
* @throws IOException
*/
private ArrayList<String> parseCdsnJsonResult(ArrayList<String> webpageRaw, String orgId, String stationId, String wqTest) throws WaterDataException {
//Parse results
ArrayList<String> webpageAll = new ArrayList<>();
if(webpageRaw.size() > 0){
webpageAll.add("OrganizationIdentifier\tMonitoringLocationIdentifier\tMediaSubdivisionName\tActivityIdentifier\tStartDate\tStartTime\tMediaName\tActivityType\tCharacteristicName\tSampleFraction\tValueType\tStatus\tResultValue\tResultUnit\tAnalyticalMethodContext\tAnalyticalMethodIdentifier\tDetectionCondition\tQualifierCode");
//parse the json string into a JSONArray and the object(s) within it
try{
JSONArray cdsnResponse = new JSONArray(webpageRaw.get(0));
for(int i=0; i<cdsnResponse.length(); i++){
JSONObject currentActivity = cdsnResponse.getJSONObject(i);
String orgIdresult = currentActivity.getString("OrganizationIdentifier");
//String stationId = currentActivity.getString("MonitoringLocationIdentifier");
String mediaSubdivision = currentActivity.getString("MediaSubdivisionName");
String activityId = currentActivity.getString("ActivityIdentifier");
String date = currentActivity.getString("StartDate");
String time = currentActivity.getString("StartTime");
String mediaName = currentActivity.getString("MediaName");
String activityType = currentActivity.getString("ActivityType");
//Parse 'result' JSONArray within each 'activity' JSONObject
JSONArray resultArray = currentActivity.getJSONArray("Results");
for(int j=0; j<resultArray.length(); j++){
JSONObject currentResult = resultArray.getJSONObject(j);
String characteristic = currentResult.getString("CharacteristicName");
String fraction = currentResult.getString("SampleFraction");
String valueType = currentResult.getString("ValueType");
String status = currentResult.getString("Status");
String resultValue = currentResult.getString("ResultValue");
String units = currentResult.getString("ResultUnit");
String methodContext = currentResult.getString("AnalyticalMethodContext");
String methodIdentifier = currentResult.getString("AnalyticalMethodIdentifier");
String detectionCondition = currentResult.getString("DetectionCondition");
String QualifierCode = currentResult.getString("QualifierCode");
//Save results
webpageAll.add(orgIdresult + "\t"+stationId+"\t"+mediaSubdivision+"\t"+activityId+"\t"+
date+"\t"+time+"\t"+mediaName+"\t"+activityType+"\t"+characteristic+"\t"+
fraction+"\t"+valueType+"\t"+status+"\t"+resultValue+"\t"+units+"\t"+
methodContext+"\t"+methodIdentifier+"\t"+detectionCondition+"\t"+QualifierCode);
}
}
}catch(JSONException ex){
throw new WaterDataException("There was an issue parsing the data for the " + database + " database station: " + stationId + " by: " + orgId + " contains no flood discharge data." + ex.getMessage());
}
}else{
webpageAll.add("No " + wqTest + " data found for CDSN Station: " + stationId + " by: " + orgId);
}
return webpageAll;
}
}