NADPData.java [src/AirData] Revision: default Date:
package AirData;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Iterator;
/**
* Last Updated: 20-April-2017
* @author Lana Millard, Tyler Wible
* @since 9-June-2015
*/
public class NADPData implements AirDataInterface {
String database = "NADP";
@Override
public ArrayList<String> extractAnnualDepositionData_raw(String stationID, String beginDate, String endDate, String wqTest) throws AirDataException {
//Specify annual website from inputs
String beginYear = beginDate.substring(0,4);
String endYear = endDate.substring(0,4);
//Website Example: http://nadp.sws.uiuc.edu/nadpdata/seasRep.asp?action=seasRep.asp&site=AK03&allSites=False&wholestate=False&custom=False&startyr=1986&endyr=1986&DataType=mg%2FL&reportType=csv&Seas=105&useType=6&useDescription=Nutrient+levels+in+the+South+Platte+River+Basin&submit=Get+Data&userid=290598
String annualUrl ="http://nadp.sws.uiuc.edu/nadpdata/seasRep.asp?action=seasRep.asp&site="+stationID+"&allSites=False&wholestate=False&custom=False&startyr=" + beginYear + "&endyr=" + endYear + "&DataType=mg%2FL&reportType=csv&Seas=105&useType=6&useDescription=Nutrient+levels+in+the+South+Platte+River+Basin&submit=Get+Data&userid=290598";
//Fetch the annual data webpage for the current NADP station
ArrayList<String> webpageAll = new ArrayList<>();
try {
webpageAll = downloadWebpage(annualUrl);
} catch (IOException ex) {
throw new AirDataException("The was an issue extracting " + database + " deposition data from the specified URl: " + annualUrl + "." + ex.getMessage());
}
return webpageAll;
}
@Override
public String[][] extractAnnualDepositionData_formatted(String stationID, String beginDate, String endDate, String wqTest) throws AirDataException {
//Fetch annual data
ArrayList<String> webpageAll = extractAnnualDepositionData_raw(stationID, beginDate, endDate, wqTest);
//Extract data from the result webpage
Iterator<String> iterate = webpageAll.iterator( );
ArrayList<String> stationData = new ArrayList<>();
int dataIndex=0;
while(iterate.hasNext()){
String temp_pageData = iterate.next();
String[] f = temp_pageData.split(",");
if (f[0].contains("SiteID")) {
//Find the column containing the desired data
for(int i=0; i<f.length; i++){
if(wqTest.equalsIgnoreCase("Ca") && f[i].contains("Ca")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("Mg") && f[i].contains("Mg")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("K") && f[i].contains("K")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("Na") && f[i].contains("Na")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("NH4") && f[i].contains("NH4")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("NO3") && f[i].contains("NO3")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("Cl") && f[i].contains("Cl")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("SO4") && f[i].contains("SO4")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("Field_pH") && f[i].contains("Fld pH")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("Field_Conductivity") && f[i].contains("Fld Cond")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("Lab_pH") && f[i].contains("\"pH\"")){ //LAB pH
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("Lab_Conductivity") && f[i].contains("\"Cond.\"")){ //LAB Conductivity
dataIndex = i;
}
}
}else if(f[0].equals("\"" + stationID + "\"") && !f[dataIndex].equalsIgnoreCase("") ){
// Save correct data data from webpage onto an Arraylist
if(dataIndex > 0 && f.length > dataIndex && !f[dataIndex].equalsIgnoreCase("")){
double dataParam = Double.parseDouble(f[dataIndex]);
if (dataParam > 0) {
//Originally denoted as "--" in the HTML form, which means there
//is missing data for that year, these values became negative in Comma Delimited form
//Do not record if this is the case
// Convert date format to yyyy-mm-dd
String date = f[2]+"-01-01";
stationData.add(date + "\t" + f[dataIndex]);
}
}
}
}
//Reformat data
String[][] returnArray = new String[stationData.size()][2];
for(int i=0; i<returnArray.length; i++){
String[] currentColumns = stationData.get(i).split("\t");
//currentColumns[0] = date
//currentColumns[1] = value
returnArray[i][0] = currentColumns[0];
returnArray[i][1] = currentColumns[1];
}
return returnArray;
}
@Override
public ArrayList<String> extractMonthlyDepositionData_raw(String stationID, String beginDate, String endDate, String wqTest) throws AirDataException {
//Specify monthly website from inputs
String beginMonth = beginDate.substring(5,7);
String beginYear = beginDate.substring(0,4);
String endMonth = endDate.substring(5,7);
String endYear = endDate.substring(0,4);
//Website Example: http://nadp.sws.uiuc.edu/nadpdata/monthlyReport.asp?action=monthlyReport.asp&site=AK03&allSites=False&wholestate=False&custom=False&startyr=1980&endyr=2015&DataType=mg%2FL&reportType=csv&All=True&useType=6&useDescription=Nutrient+levels+in+the+South+Platte+River+Basin&submit=Get+Data&userid=290598
String monthlyUrl = "http://nadp.sws.uiuc.edu/nadpdata/monthlyReport.asp?action=monthlyReport.asp&site=" +stationID+"&allSites=False&wholestate=False&custom=False&startmonth="+beginMonth+"&startyr="+beginYear+"&endmonth=" + endMonth + "&endyr=" + endYear + "&DataType=mg%2FL&reportType=csv&All=True&useType=6&useDescription=Nutrient+levels+in+the+South+Platte+River+Basin&submit=Get+Data&userid=290598";
//Fetch the monthly data webpage for the current NADP station
ArrayList<String> webpageAll = new ArrayList<>();
try {
webpageAll = downloadWebpage(monthlyUrl);
} catch (IOException ex) {
throw new AirDataException("The was an issue extracting " + database + " deposition data from the specified URl: " + monthlyUrl + "." + ex.getMessage());
}
return webpageAll;
}
@Override
public String[][] extractMonthlyDepositionData_formatted(String stationID, String beginDate, String endDate, String wqTest) throws AirDataException {
//Fetch monthly data
ArrayList<String> webpageAll = extractMonthlyDepositionData_raw(stationID, beginDate, endDate, wqTest);
//Extract data from the result webpage
Iterator<String> iterate = webpageAll.iterator( );
ArrayList<String> stationData = new ArrayList<>();
int dataIndex=0;
while(iterate.hasNext()){
String temp_pageData = (String) iterate.next();
String[] f = temp_pageData.split(",");
// Specify column that contains datatype that the user desires
if (f[0].contains("SiteID")) {
for(int i=0; i<f.length; i++){
if(wqTest.equalsIgnoreCase("Ca") && f[i].contains("Ca")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("Mg") && f[i].contains("Mg")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("K") && f[i].contains("K")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("Na") && f[i].contains("Na")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("NH4") && f[i].contains("NH4")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("NO3") && f[i].contains("NO3")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("Cl") && f[i].contains("Cl")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("SO4") && f[i].contains("SO4")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("Lab_pH") && f[i].contains("\"pH\"")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("Lab_Conductivity") && f[i].contains("\"Cond.\"")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("Field_pH") && f[i].contains("Fld pH")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("Field_Conductivity") && f[i].contains("Fld Cond")){
dataIndex = i;
}
}
}
// Record data from webpage onto an Arraylist
else if(f[0].equals("\"" + stationID + "\"") && !f[dataIndex].equalsIgnoreCase("") ){
if(dataIndex > 0 && f.length > dataIndex && !f[dataIndex].equalsIgnoreCase("")){
double dataParam = Double.parseDouble(f[dataIndex]);
if (dataParam > 0) {
//Originally denoted as "--" in the HTML form, which means there
//is missing data for that year, these values became negative in Comma Delimited form
//Do not record if this is the case
// Convert date format to yyyy-mm-dd
String date = f[2] + "-" + getmonthMM(f[1].substring(1,4)) + "-01";
stationData.add(date + "\t" + f[dataIndex]);
}
}
}
}
//Reformat data
String[][] returnArray = new String[stationData.size()][2];
for(int i=0; i<returnArray.length; i++){
String[] currentColumns = stationData.get(i).split("\t");
//currentColumns[0] = date
//currentColumns[1] = value
returnArray[i][0] = currentColumns[0];
returnArray[i][1] = currentColumns[1];
}
return returnArray;
}
@Override
public ArrayList<String> extractWeeklyDepositionData_raw(String stationID, String beginDate, String endDate, String wqTest) throws AirDataException {
//Specify monthly website from inputs
String beginMonth = beginDate.substring(5,7);
String beginYear = beginDate.substring(0,4);
String endMonth = endDate.substring(5,7);
String endYear = endDate.substring(0,4);
//Website Example: http://nadp.sws.uiuc.edu/nadpdata/weeklyReport.asp?action=weeklyReport.asp&site=AK03&allSites=False&wholestate=False&custom=False&startmonth=01&startyr=1980&endmonth=12&endyr=2015&reportType=csv&useType=6&useDescription=Nutrient+levels+in+the+South+Platte+River+Basin&submit=Get+Data&userid=290598
String weeklyUrl = "http://nadp.sws.uiuc.edu/nadpdata/weeklyReport.asp?action=weeklyReport.asp&site="+stationID+ "&allSites=False&wholestate=False&custom=False&startmonth="+beginMonth+"&startyr="+beginYear+"&endmonth=" + endMonth + "&endyr=" + endYear + "&reportType=csv&useType=6&useDescription=Nutrient+levels+in+the+South+Platte+River+Basin&submit=Get+Data&userid=290598";
//Fetch the weekly data webpage for the current NADP station
ArrayList<String> webpageAll = new ArrayList<>();
try {
webpageAll = downloadWebpage(weeklyUrl);
} catch (IOException ex) {
throw new AirDataException("The was an issue extracting " + database + " deposition data from the specified URl: " + weeklyUrl + "." + ex.getMessage());
}
return webpageAll;
}
@Override
public String[][] extractWeeklyDepositionData_formatted(String stationID, String beginDate, String endDate, String wqTest) throws AirDataException {
//Fetch weekly data
ArrayList<String> webpageAll = extractWeeklyDepositionData_raw(stationID, beginDate, endDate, wqTest);
//Extract data from the result webpage
Iterator<String> iterate = webpageAll.iterator( );
ArrayList<String> stationData = new ArrayList<>();
int dataIndex=0;
while(iterate.hasNext()){
String temp_pageData = (String) iterate.next();
String[] f = temp_pageData.split(",");
// Specify column that contains datatype that the user desires
if (f[0].contains("SiteID")) {
for(int i=0; i<f.length; i++){
if(wqTest.equalsIgnoreCase("Ca") && f[i].contains("Ca")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("Mg") && f[i].contains("Mg")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("K") && f[i].contains("K")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("Na") && f[i].contains("Na")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("NH4") && f[i].contains("NH4")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("NO3") && f[i].contains("NO3")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("Cl") && f[i].contains("Cl")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("SO4") && f[i].contains("SO4")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("Lab_pH") && f[i].contains("pH Lab")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("Lab_Conductivity") && f[i].contains("Lab Cond")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("Field_pH") && f[i].contains("pH Field")){
dataIndex = i;
}else if(wqTest.equalsIgnoreCase("Field_Conductivity") && f[i].contains("Field Cond")){
dataIndex = i;
}
}
}
// Record data from webpage onto an Arraylist
else if(f[0].equals("\"" + stationID + "\"") && !f[dataIndex].equalsIgnoreCase("") ){
if(dataIndex > 0 && f.length > dataIndex && !f[dataIndex].equalsIgnoreCase("")){
double dataParam = Double.parseDouble(f[dataIndex]);
if (dataParam > 0) {
//Originally denoted as "--" in the HTML form, which means there
//is missing data for that year, these values became negative in Comma Delimited form
//Do not record if this is the case
String detectionError = "";
// Concentrations below the detection limit
if (f[dataIndex-1].contains("<")){
detectionError = "<";
}
// Convert date format to yyyy-mm-dd
String dateOn = convertDate(f[1]);
String dateOff= convertDate(f[2]);
stationData.add(dateOn+ "\t" + dateOff + "\t" + detectionError+f[dataIndex]);
}
}
}
}
//Reformat data
String[][] returnArray = new String[stationData.size()][3];
for(int i=0; i<returnArray.length; i++){
String[] currentColumns = stationData.get(i).split("\t");
//currentColumns[0] = date on
//currentColumns[1]= date off
//currentColumns[2] = value
returnArray[i][0] = currentColumns[0];
returnArray[i][1] = currentColumns[1];
returnArray[i][2] = currentColumns[2];
}
return returnArray;
}
// This method is used to get the corresponding 2-digit number (String) for each 3-letter month abbreviation (String).
private static String getmonthMM(String month) {
String monthNum;
switch (month) {
case "Jan": monthNum = "01";
break;
case "Feb": monthNum = "02";
break;
case "Mar": monthNum = "03";
break;
case "Apr": monthNum = "04";
break;
case "May": monthNum = "05";
break;
case "Jun": monthNum = "06";
break;
case "Jul": monthNum = "07";
break;
case "Aug": monthNum = "08";
break;
case "Sep": monthNum = "09";
break;
case "Oct": monthNum = "10";
break;
case "Nov": monthNum = "11";
break;
default: monthNum = "12";
}
return monthNum;
}
/**
* This converts a 'mm/dd/yyyy' date formatted string to 'yyyy-mm-dd' date format
* @param origDate a date in 'mm/dd/yyyy/ format
* @return the 'yyyy-mm-dd' formatted version of the original input date
*/
private static String convertDate(String origDate){
int index1 = origDate.indexOf("/");
int index2 = origDate.indexOf("/", index1 + 1);
String month = origDate.substring(1,index1);
String day = origDate.substring(index1 + 1, index2);
String year = origDate.substring(index2 + 1,origDate.length() - 2);
if(month.length() < 2){
month = "0" + month;
}
if(day.length() < 2){
day = "0" + day;
}
String newDate = year + "-" + month + "-" + day;
return newDate;
}
/**
* Opens a web connection to USGS and returns the contents of a REST/url search for the specified url
* @param nadpUrl a formatted url for requested NADP data
* @return an ArrayList containing the results of the web service search for flow data using the above inputs
* @throws IOException
*/
private ArrayList<String> downloadWebpage(String nadpUrl) throws IOException{
//Open the provided website
URL webpage = new URL(nadpUrl);
URLConnection yc = webpage.openConnection();
BufferedReader in = new BufferedReader(new InputStreamReader(yc.getInputStream()));
//Read out all of the webpage out into an ArrayList<String>
String inputLine;
ArrayList<String> pageData = new ArrayList<>();
while((inputLine = in.readLine()) != null){
inputLine = inputLine.replace("&quoi;", "\"");
pageData.add(inputLine);
}
in.close();
return pageData;
}
}