USGSwell_Data.java [src/java/groundwater] Revision: Date:
package groundwater;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Iterator;
/**
* Last Updated: 29-December-2016
* @author Tyler Wible
* @since 3-November-2014
*/
public class USGSwell_Data {
/**
* Get the flow webpage and loop through and pull out the flow data for the current station
* @param stationID the USGS station ID for the current station
* @param beginDate the user specified begin date for the station (yyyy-mm-dd format)
* @param endDate the user specified end date for the station (yyyy-mm-dd format)
* @return a String[][] containing column1 = date(yyyy-mm-dd), column2 = flowValue
* @throws IOException
*/
public static Object[] getUSGSwellHeightData(String stationID, String beginDate, String endDate) throws IOException{
//Get the webpage of data for the USGS well station
ArrayList<String> webpageAll = DownloadWellHeightWebpage(stationID, beginDate, endDate);
//Pull out new arraylist of only the desired data from the arraylist to return as the web page result
Iterator<String> iterate = webpageAll.iterator( );
ArrayList<String> textData = new ArrayList<String>();
while(iterate.hasNext()){
String temp_pageData = (String) iterate.next();
String[] f = temp_pageData.split("\t");
if ((f.length >= 8) && ("USGS".equals(f[0]))) {
boolean Ice = f[7].equalsIgnoreCase("Ice");
boolean Ssn = f[7].equalsIgnoreCase("Ssn");
boolean Dis = f[7].equalsIgnoreCase("Dis");
boolean rat = f[7].equalsIgnoreCase("Rat");
boolean eqp = f[7].equalsIgnoreCase("Eqp");
boolean other = f[7].equalsIgnoreCase("***");
boolean blank = f[7].equalsIgnoreCase("");
if (!Ice && !Ssn && !Dis && !rat && !eqp && !other && !blank) {
//Pull out only the data needed to pass between sub-functions
//f[1] = StationID
//f[2] = Date
//f[3] = FlowValue
textData.add(f[1] + "\t" + f[2] + "\t" + f[7]);
}
}
}
//convert Array list into String[][] array (column1 = date, column2 = value)
String[][] stringArray = new String[textData.size()][2];
for(int i=0; i<stringArray.length; i++){
String[] currentColumns = textData.get(i).split("\t");
//currentColumns[0] = stationID
//currentColumns[1] = date
//currentColumns[2] = value
stringArray[i][0] = currentColumns[1];
stringArray[i][1] = currentColumns[2];
}
//Save analysis results
String start = "-1";
String end = "-1";
if(stringArray.length > 0){
start = stringArray[0][0];
end = stringArray[stringArray.length - 1][0];
}
Object[] returnArray = {webpageAll, stringArray, start, end};
return returnArray;
}
/**
* Opens a web connection to USGS and returns the contents of a search for all well height data for the specific station and date range
* @param stationID the USGS station ID for the current station
* @param beginDate the user specified begin date for the station (yyyy-mm-dd format)
* @param endDate the user specified end date for the station (yyyy-mm-dd format)
* @return an ArrayList<String> containing the results of the search for well height data using the above inputs
* @throws IOException
*/
public static ArrayList<String> DownloadWellHeightWebpage(String stationID, String beginDate, String endDate) throws IOException {
//https://waterdata.usgs.gov/nwis/dv?cb_62611=on&format=rdb&site_no=404450103013501&referred_module=sw&period=&begin_date=1900-01-01&end_date=2015-08-20
String heightWebsite_URL = "https://waterdata.usgs.gov/nwis/dv?cb_62611=on&format=rdb&site_no=" +
stationID + "&referred_module=sw&period=&begin_date=" + beginDate + "&end_date=" + endDate;
URL webpage = new URL(heightWebsite_URL);
URLConnection yc = webpage.openConnection();
BufferedReader in = new BufferedReader(new InputStreamReader(yc.getInputStream()));
//Index variables to count how many lines are in the web page and pull out all of the web page data
String inputLine;
ArrayList<String> pageData = new ArrayList<String>( );
while((inputLine = in.readLine()) != null){
pageData.add(inputLine);
}
return pageData;
}
/**
* Get the flow webpage and loop through and pull out the flow data for the current station
* @param stationID the USGS station ID for the current station
* @param beginDate the user specified begin date for the station (yyyy-mm-dd format)
* @param endDate the user specified end date for the station (yyyy-mm-dd format)
* @return a String[][] containing column1 = date(yyyy-mm-dd), column2 = flowValue
* @throws IOException
*/
public static Object[] getUSGSwatertableDepthData(String stationID, String beginDate, String endDate) throws IOException{
//Get the webpage of data for the USGS well station
ArrayList<String> webpageAll = DownloadWatertableDepthWebpage(stationID, beginDate, endDate);
//Pull out new arraylist of only the desired data from the arraylist to return as the web page result
Iterator<String> iterate = webpageAll.iterator( );
ArrayList<String> textData = new ArrayList<String>();
while(iterate.hasNext()){
String temp_pageData = (String) iterate.next();
String[] f = temp_pageData.split("\t");
if ((f.length >= 4) && ("USGS".equals(f[0]))) {
boolean Ice = f[3].equalsIgnoreCase("Ice");
boolean Ssn = f[3].equalsIgnoreCase("Ssn");
boolean Dis = f[3].equalsIgnoreCase("Dis");
boolean rat = f[3].equalsIgnoreCase("Rat");
boolean eqp = f[3].equalsIgnoreCase("Eqp");
boolean other = f[3].equalsIgnoreCase("***");
boolean blank = f[3].equalsIgnoreCase("");
if (!Ice && !Ssn && !Dis && !rat && !eqp && !other && !blank) {
//Pull out only the data needed to pass between sub-functions
//f[1] = StationID
//f[2] = Date
//f[3] = FlowValue
textData.add(f[1] + "\t" + f[2] + "\t" + f[3]);
}
}
}
//convert Array list into String[][] array (column1 = date, column2 = value)
String[][] stringArray = new String[textData.size()][2];
for(int i=0; i<stringArray.length; i++){
String[] currentColumns = textData.get(i).split("\t");
//currentColumns[0] = stationID
//currentColumns[1] = date
//currentColumns[2] = value
stringArray[i][0] = currentColumns[1];
stringArray[i][1] = currentColumns[2];
}
//Save analysis results
String start = "-1";
String end = "-1";
if(stringArray.length > 0){
start = stringArray[0][0];
end = stringArray[stringArray.length - 1][0];
}
Object[] returnArray = {webpageAll, stringArray, start, end};
return returnArray;
}
/**
* Opens a web connection to USGS and returns the contents of a search for all well height data for the specific station and date range
* @param stationID the USGS station ID for the current station
* @param beginDate the user specified begin date for the station (yyyy-mm-dd format)
* @param endDate the user specified end date for the station (yyyy-mm-dd format)
* @return an ArrayList<String> containing the results of the search for well height data using the above inputs
* @throws IOException
*/
public static ArrayList<String> DownloadWatertableDepthWebpage(String stationID, String beginDate, String endDate) throws IOException {
//https://waterdata.usgs.gov/nwis/dv?cb_72019=on&format=rdb&site_no=404915077500801&referred_module=sw&period=&begin_date=1900-01-01&end_date=2015-08-20
String heightWebsite_URL = "https://waterdata.usgs.gov/nwis/dv?cb_72019=on&format=rdb&site_no=" +
stationID + "&referred_module=sw&period=&begin_date=" + beginDate + "&end_date=" + endDate;
URL webpage = new URL(heightWebsite_URL);
URLConnection yc = webpage.openConnection();
BufferedReader in = new BufferedReader(new InputStreamReader(yc.getInputStream()));
//Index variables to count how many lines are in the web page and pull out all of the web page data
String inputLine;
ArrayList<String> pageData = new ArrayList<String>( );
while((inputLine = in.readLine()) != null){
pageData.add(inputLine);
}
return pageData;
}
/**
* Get the water quality webpage and loop through and pull out the correct water quality test data for the current station
* @param stationID the USGS station ID for the current station
* @param beginDate the user specified begin date for the station (yyyy-mm-dd format)
* @param endDate the user specified end date for the station (yyyy-mm-dd format)
* @return a String[][] containing column1 = date(yyyy-mm-dd), column2 = flowValue
* @throws IOException
*/
public static Object[] getUSGSwellWQdata(String stationID, String wqTest, String beginDate, String endDate) throws IOException{
//Get the webpage of data for the USGS well station
ArrayList<String> webpageAll = DownloadWellWQwebpage(stationID);
//Pull out new arraylist of only the desired data from the arraylist to return as the web page result
Iterator<String> iterate = webpageAll.iterator( );
ArrayList<String> textData = new ArrayList<String>();
while(iterate.hasNext()){
String temp_pageData = (String) iterate.next();
String[] f = temp_pageData.split("\t");
if ( (f.length >= 15) && (f[0].equals("USGS")) ) {
String WQSample_code = f[12];
String WQSample_result = f[14];
boolean A = WQSample_code.equals("");
boolean B = WQSample_result.equals("");
if (!A && !B){
//Count only the rows which contain the desired values of "agency_cd ...
//site_no sample_dt sample_tm sample_end_dt sample_end_tm ...
//sample_start_time_datum_cd tm_datum_rlbty_cd coll_ent_cd ...
//medium_cd tu_id body_part_id parm_cd remark_cd result_va ...
//val_qual_tx meth_cd dqi_cd rpt_lev_va rpt_lev_cd lab_std_va anl_ent_cd"
//Pull out only the data needed to pass between sub-functions
//f[1] = stationID
//f[2] = date
//f[3] = time
//f[12] = water quality test code
//f[14] = water quality test value
textData.add(f[1] + "\t" + f[2] + "\t" + f[12] + "\t" + f[14]);
}
}
}
//convert Array list into String[][] array (column1 = date, column2 = value)
String[][] stringArray = new String[textData.size()][3];
for(int i=0; i<textData.size(); i++){
String[] currentColumns = textData.get(i).split("\t");
//currentColumns[0] = stationID
//currentColumns[1] = date
//currentColumns[2] = water quality test code
//currentColumns[3] = water quality test value
stringArray[i][0] = currentColumns[1];//date
stringArray[i][1] = currentColumns[2];//test code
stringArray[i][2] = currentColumns[3];//value
}
if(wqTest.length() > 5){
wqTest = wqTest.substring(0,5);//pull just the 5 digit USGS WQ code
}
//Extract USGS water quality code for current wqTest only
stringArray = minimizeUSGSWQdata(stringArray, wqTest, beginDate, endDate);
//Save analysis results
String start = "-1";
String end = "-1";
if(stringArray.length > 0){
start = stringArray[0][0];
end = stringArray[stringArray.length - 1][0];
}
Object[] returnArray = {webpageAll, stringArray, start, end};
return returnArray;
}
/**
* Opens a web connection to USGS and returns the contents of a search for all well height data for the specific station and date range
* @param stationID the USGS station ID for the current station
* @return an ArrayList<String> containing the results of the search for well height data using the above inputs
* @throws IOException
*/
public static ArrayList<String> DownloadWellWQwebpage(String stationID) throws IOException {
String WQWebsite_URL = "https://nwis.waterdata.usgs.gov/nwis/qwdata/?site_no=" + stationID + "&agency_cd=USGS&format=rdb";
URL webpage = new URL(WQWebsite_URL);
URLConnection yc = webpage.openConnection();
BufferedReader in = new BufferedReader(new InputStreamReader(yc.getInputStream()));
//Index variables to count how many lines are in the web page and pull out all of the web page data
String inputLine;
ArrayList<String> pageData = new ArrayList<String>( );
while((inputLine = in.readLine()) != null){
pageData.add(inputLine);
}
return pageData;
}
/**
* Reduces all water quality data to just that of the requested parameter
* @param allData all water quality data for the earlier provided date range and station ID (column1 = date, column2 = wqTestcode, column3 = value)
* @param wqTestCode the requested water quality parameter
* @param beginDate the user defined begin date for data search
* @param endDate the user defined end date for data search
* @return
* @throws IOException
*/
public static String[][] minimizeUSGSWQdata(String[][] allData, String wqTestCode, String beginDate, String endDate) throws IOException{
int ctr = 0;
for(int i=0; i<allData.length; i++){
if(i != 0){//Ignore the first row containing the station name
if(allData[i][1].equalsIgnoreCase(wqTestCode) &&
(allData[i][0].compareTo(beginDate) >= 0) && (allData[i][0].compareTo(endDate) <= 0)){
ctr++;
}
}
}
String[][] reducedData = new String[ctr][2];
ctr=0;
for(int i=0; i<allData.length; i++){
if(i != 0){//Ignore the first row containing the station name
if(allData[i][1].equalsIgnoreCase(wqTestCode) &&
(allData[i][0].compareTo(beginDate) >= 0) && (allData[i][0].compareTo(endDate) <= 0)){
reducedData[ctr][0] = allData[i][0];//date
reducedData[ctr][1] = allData[i][2];//WQ test result value
ctr++;
}
}
}
return reducedData;
}
/**
* @param parameterCode USGS parameter code for a specific water quality test.
* @return a string with the type of units for the current test.
*/
public static String getUSGSwqUnits(String parameterCode) throws IOException{
URL webpage = new URL("https://nwis.waterdata.usgs.gov/usa/nwis/pmcodes?radio_pm_search=param_group&pm_group=All+--+include+all+parameter+groups&pm_search=&casrn_search=&srsname_search=&format=rdb&show=parameter_group_nm&show=parameter_nm&show=casrn&show=srsname&show=parameter_units");
URLConnection yc = webpage.openConnection();
BufferedReader in = new BufferedReader(new InputStreamReader(yc.getInputStream()));
String inputLine;
int line_length = 0;
String units = "0";
//Find the units of the specified test
while ((inputLine = in.readLine()) != null) {
String[] f = inputLine.split("\t");
line_length = f.length;
if((line_length >= 6) && (f[0].length() == 5)){
if(f[0].equals(parameterCode)){
units = f[5];
}
}
}
return units;
}
}