STORET_Data.java [src/java/cfa] Revision: af6981c8e32f99e7dfe23929483fbaa0c6d867f3 Date: Wed Feb 05 09:43:06 MST 2014
package cfa;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.WebWindow;
import com.gargoylesoftware.htmlunit.WebWindowEvent;
import com.gargoylesoftware.htmlunit.WebWindowListener;
import com.gargoylesoftware.htmlunit.html.HtmlButtonInput;
import com.gargoylesoftware.htmlunit.html.HtmlForm;
import com.gargoylesoftware.htmlunit.html.HtmlInput;
import com.gargoylesoftware.htmlunit.html.HtmlOption;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.html.HtmlRadioButtonInput;
import com.gargoylesoftware.htmlunit.html.HtmlSelect;
/**
* Last Updated: 5-February-2014
* @author Tyler Wible
* @since 22-June-2012
*/
public class STORET_Data {
final LinkedList<WebWindow> windows = new LinkedList<WebWindow>();
/**
* Sub-function to convert month number into a month text
* @param month_String the month number (Ex. 01).
* @return month text (Ex. JAN).
*/
public String ConvertMonth (String month_String){
String month_number = "JAN";
if (month_String.equalsIgnoreCase("01")){
month_number = "JAN";
}else if (month_String.equalsIgnoreCase("02")){
month_number = "FEB";
}else if (month_String.equalsIgnoreCase("03")){
month_number = "MAR";
}else if (month_String.equalsIgnoreCase("04")){
month_number = "APR";
}else if (month_String.equalsIgnoreCase("05")){
month_number = "MAY";
}else if (month_String.equalsIgnoreCase("06")){
month_number = "JUN";
}else if (month_String.equalsIgnoreCase("07")){
month_number = "JUL";
}else if (month_String.equalsIgnoreCase("08")){
month_number = "AUG";
}else if (month_String.equalsIgnoreCase("09")){
month_number = "SEP";
}else if (month_String.equalsIgnoreCase("10")){
month_number = "OCT";
}else if (month_String.equalsIgnoreCase("11")){
month_number = "NOV";
}else if (month_String.equalsIgnoreCase("12")){
month_number = "DEC";
}
return month_number;
}
/**
* Sub-function to convert month number into a month text
* @param month_String the month number (Ex. 01).
* @return month text (Ex. JAN).
*/
public String ConvertDay (String day_String){
if (day_String.equalsIgnoreCase("01")){
day_String = "1";
}else if (day_String.equalsIgnoreCase("02")){
day_String = "2";
}else if (day_String.equalsIgnoreCase("03")){
day_String = "3";
}else if (day_String.equalsIgnoreCase("04")){
day_String = "4";
}else if (day_String.equalsIgnoreCase("05")){
day_String = "5";
}else if (day_String.equalsIgnoreCase("06")){
day_String = "6";
}else if (day_String.equalsIgnoreCase("07")){
day_String = "7";
}else if (day_String.equalsIgnoreCase("08")){
day_String = "8";
}else if (day_String.equalsIgnoreCase("09")){
day_String = "9";
}
return day_String;
}
/**
* Get list of information contained in listbox on the specified page with the specified attribute "name"
* @param select The HtmlSelect listbox
* @param listboxType Which listbox is being interacted with,
* the organization names, station names search
* @param name The current user selection which is being searched for in the listbox
*/
public void selectListboxData(HtmlSelect select, String listboxType, String name){
//Iterate through the options of the select box, find and select the organization specified
List<HtmlOption> selectOptions = select.getOptions();
Iterator<HtmlOption> listIterate = selectOptions.iterator();
int ctr = 0;
while(listIterate.hasNext()){
HtmlOption currentOption = listIterate.next();
String optionText1 = currentOption.getText();
System.out.println(optionText1);
if(ctr > 1){//Skip the first two lines
String optionText = currentOption.getText();
String optionTextClip = optionText;
if(!listboxType.equals("Characteristics")){//Only trim organizations and stations
int index1 = optionText.indexOf(" ");
if(index1 != -1){
optionTextClip = optionText.substring(index1);
optionTextClip = optionTextClip.trim();
}
}
System.out.println(optionTextClip);
//If the current option is the desired one, select it
if(optionTextClip.toLowerCase().contains(name.toLowerCase())){
//If the current select option is the organization name, then select it and break the loop
String org_index = currentOption.getValueAttribute();
select.setSelectedAttribute(org_index, true);
System.out.println(listboxType + " selected");
break;
}
}
ctr++;
}
}
/**
* Assembles an error message to be returned early if an error occurs and the page isn't working properly.
* @param error_number The error code to be displayed
* @return a string containing the error message and number
*/
public String ExitSystem(String error_number){
String message = "An unexpected error occured while downloading your requested data. Report '" + error_number + "' to the webmaster";
return message;
}
/**
* Writes out the error message, if any, for finding the file and then exits the program
* @param error string array to be written as each line of an error message
* @throws IOException
*/
public void writeError(String[] error) throws IOException{
//Output data to text file
String errorContents = error[0];
for(int i=1; i<error.length; i++){
errorContents = errorContents + "\n" + error[i];
}
throw new IOException("Error encountered. Please see the following message for details: \n" + errorContents);
}
/**
* Tries to access the url zip download and downloads the zip once it exists
* @param downloadUrl The URL for the ZIP data download page
* @param downloadfile The name of the downloaded ZIP file
* @param mainFolder the folder in which to download the file
* @return the zipfile location
*/
public String downloadzip(String downloadUrl, String downloadfile, String mainFolder){
System.out.println("File name: " + downloadfile);
System.out.println("Website: " + downloadUrl);
String zipfile_location = mainFolder + "/" + downloadfile;
try{//Get a connection to the URL
boolean exists = false;
URL url = new URL(downloadUrl);
url.openConnection();
InputStream reader = null;
//Loop until the website URL exists
int ctr=0;
while(!exists){
try{
reader = url.openStream();
System.out.println("Step 9ba: website download exists");
exists = true;
}catch(FileNotFoundException e){
try{
System.out.println("Try: " + ctr + " download website does not exist");
ctr++;
Thread.sleep(5000); // do nothing for 5000 milliseconds (5 second)
//Return an error if the webpage didn't respond given enough time
if(ctr > 180){
String message = "Error STORET FLDC0014. Waiting for the webpage (" + downloadUrl + ") timed out please try again";
return message;
}
}catch(InterruptedException f){
f.printStackTrace();
}
}
}
//Start a buffered file writer to output the result of the download file
FileOutputStream writer = new FileOutputStream(zipfile_location);
byte[] buffer = new byte[153600];
int bytesRead = 0;
//read the file 150KB at a time
while ((bytesRead = reader.read(buffer)) > 0){
writer.write(buffer, 0, bytesRead);
buffer = new byte[153600];
}
//Close the reader and writer
writer.close();
reader.close();
}catch(MalformedURLException e){
e.printStackTrace();
}catch (IOException e){
e.printStackTrace();
}
System.out.println("Step 9c: finished downloading file");
return zipfile_location;
}
/**
* Interfaces with the STORET download website and searches for the data for a specific station and
* read it out into a variable depending on what data was requested
* @param mainFolder file location to download the resulting STORET zip file into
* @param organizationName name of the supervizing agency for the current station, used to search the STORET database
* @param stationID station ID of the current station, used to search the STORET databes
* @param characteristicLong the full name of the characteristic being searched for
* @param beginDate the used defined begin date of search
* @param endDate the user defined end date of search
* @return a String containing the full path to the downloaded zip file (ex. "C:/tempFolder/myZipfile.zip")
* @throws IOException
*/
public String downloadSTORET(String mainFolder,
String organizationName,
String stationID,
String characteristicLong,
String beginDate,
String endDate) throws IOException{
String characteristicShort = characteristicLong;
//Artificial limit due to the properties of the STORET website
if(beginDate.compareToIgnoreCase("1900-01-01") < 0){
beginDate = "1900-01-01";
}
if(endDate.compareToIgnoreCase("1900-01-01") < 0){
endDate = "1900-01-01";
}
//Edit inputs to fit search criteria and to get extra info
String email_address = "eRAMSInfo@gmail.com";
String start_year = beginDate.substring(0,4);
String start_month = beginDate.substring(5,7);
start_month = ConvertMonth(start_month);
String start_day = beginDate.substring(8);
start_day = ConvertDay(start_day);
String end_year = endDate.substring(0,4);
String end_month = endDate.substring(5,7);
end_month = ConvertMonth(end_month);
String end_day = endDate.substring(8);
end_day = ConvertDay(end_day);
if(characteristicLong.length() > 20){
characteristicShort = characteristicLong.substring(0,20);
}
//old
//String mainUrl = "http://iaspub.epa.gov/storpubl/DW_resultcriteria_station";
//new
String mainUrl = "http://ofmpub.epa.gov/storpubl/dw_pages.resultcriteria";
//Create Webclient with specific properties for STORET webpage
WebClient webClient = new WebClient();
webClient.setThrowExceptionOnScriptError(false);
webClient.addWebWindowListener(new WebWindowListener(){
public void webWindowClosed(WebWindowEvent event){
}
public void webWindowContentChanged(WebWindowEvent event){
}
public void webWindowOpened(WebWindowEvent event){
windows.add(event.getWebWindow());
}
});
//Get STORET "Results by Station" webpage
HtmlPage mainPage = null;
try {
mainPage = webClient.getPage(mainUrl);
}catch (FailingHttpStatusCodeException e) {
e.printStackTrace();
}catch (MalformedURLException e) {
e.printStackTrace();
}catch (IOException e) {
e.printStackTrace();
}
if(mainPage == null){
return ExitSystem("Error Data_STORET_0001");
}
System.out.println("Step 1: on webpage");
//Selecct Search Type
List<?> buttonList = (List<?>) mainPage.getByXPath("//input[@value='Organization and Station']");
if(buttonList.size() == 1){
System.out.println(buttonList.size());
HtmlRadioButtonInput searchTypeRadioButton = (HtmlRadioButtonInput) buttonList.get(0);
searchTypeRadioButton.focus();
mainPage = searchTypeRadioButton.click();//onclick javascript function will check this radio button and uncheck the others
// searchTypeRadioButton.setChecked(true);
System.out.println(searchTypeRadioButton.getValueAttribute());
System.out.println(searchTypeRadioButton.getCheckedAttribute());
System.out.println("Step 2: selected searchtype");
}
//Select Organization ID
List<?> formList = (List<?>) mainPage.getForms();
if(formList.size() >= 2){
HtmlForm searchForm = (HtmlForm) formList.get(1);
HtmlSelect select = (HtmlSelect)searchForm.getSelectsByName("D_ORG_LIST").get(0);
select.focus();
selectListboxData(select,"Organizations",organizationName);
}else{
return ExitSystem("Error Data_STORET_0002");
}
System.out.println("Step 3: Try Selecting a station");
//Input Station ID
HtmlInput queryInput2 = mainPage.getElementByName("as_station");
queryInput2.setValueAttribute(stationID);
//Search Organization for current Station ID (push "Search Stations" button and get pop-up window)
List<?> popupList = (List<?>) mainPage.getByXPath("//input[@value='Search Stations']");
if (popupList.size() == 1){
try{
HtmlButtonInput popup = (HtmlButtonInput) popupList.get(0);
popup.focus();
HtmlPage popupPage_sta = popup.click();
//Select Station ID
List<?> formList2 = (List<?>) popupPage_sta.getForms();
HtmlForm searchForm2 = (HtmlForm) formList2.get(0);
HtmlSelect select2 = (HtmlSelect)searchForm2.getSelectsByName("d_station_list").get(0);
selectListboxData(select2,"Stations",stationID);
//Get update main page by pushing "Select" button on pop-up
List<?> list = (List<?>) popupPage_sta.getByXPath("//input[@value='Select']");
if (list.size() == 1){
try{
HtmlButtonInput select_button = (HtmlButtonInput) list.get(0);
select_button.focus();
mainPage = select_button.click();
}catch(Exception error){
System.err.println(error.toString());
}
}else{
return ExitSystem("Error Data_STORET_0004");
}
}catch(Exception error){
error.printStackTrace();
return ExitSystem("Error Data_STORET_0005");
}
}else{
return ExitSystem("Error Data_STORET_0003");
}
System.out.println("Step 4: selected station from popup");
//Select Date Range
//Select Start year
HtmlForm searchForm3 = (HtmlForm) formList.get(1);
HtmlSelect select3 = (HtmlSelect)searchForm3.getSelectsByName("FROM_YYYY1").get(0);
mainPage = (HtmlPage) select3.setSelectedAttribute(start_year, true);
//Select Start month
HtmlSelect select4 = (HtmlSelect)searchForm3.getSelectsByName("FROM_MON1").get(0);
mainPage = (HtmlPage) select4.setSelectedAttribute(start_month, true);
//Select Start day
HtmlSelect select5 = (HtmlSelect)searchForm3.getSelectsByName("FROM_DD1").get(0);
mainPage = (HtmlPage) select5.setSelectedAttribute(start_day, true);
//Select End year
HtmlSelect select6 = (HtmlSelect)searchForm3.getSelectsByName("END_YYYY1").get(0);
mainPage = (HtmlPage) select6.setSelectedAttribute(end_year, true);
//Select End month
HtmlSelect select7 = (HtmlSelect)searchForm3.getSelectsByName("END_MON1").get(0);
mainPage = (HtmlPage) select7.setSelectedAttribute(end_month, true);
//Select End day
HtmlSelect select8 = (HtmlSelect)searchForm3.getSelectsByName("END_DD1").get(0);
mainPage = (HtmlPage) select8.setSelectedAttribute(end_day, true);
System.out.println("Step 5: selected dates");
//Input Characteristic
if(!characteristicShort.equalsIgnoreCase("flow") && !characteristicShort.equalsIgnoreCase("all")){
//If the search characteristic is not flow or all then search for the current characteristic
HtmlInput queryInput9 = mainPage.getElementByName("as_char");
queryInput9.setValueAttribute(characteristicShort);
//Search Characteristic for current characteristic (push "Search" button and get pop-up window)
List<?> popupList9 = (List<?>) mainPage.getByXPath("//input[@value='Search']");
try{
//Select first value in characteristic "search by" list box which should be "Characteristic Name"
HtmlForm searchForm9 = (HtmlForm) formList.get(1);
HtmlSelect select9 = (HtmlSelect)searchForm9.getSelectsByName("as_char_alias_type").get(0);
select9.setSelectedAttribute("0", true);
//Push "Search" button and get pop-up window
HtmlButtonInput popup_cha= (HtmlButtonInput) popupList9.get(1);
popup_cha.focus();
HtmlPage popupPage_cha = popup_cha.click();
//Select Characteristic
List<?> formList10 = (List<?>) popupPage_cha.getForms();
HtmlForm searchForm10 = (HtmlForm) formList10.get(0);
HtmlSelect select10 = (HtmlSelect)searchForm10.getSelectsByName("as_char").get(0);
selectListboxData(select10,"Characteristics",characteristicLong);
//Get update main page by pushing "Select" button on pop-up
List<?> list = (List<?>) popupPage_cha.getByXPath("//input[@value='Select']");
if (list.size() == 1){
try{
HtmlButtonInput select_button = (HtmlButtonInput) list.get(0);
select_button.focus();
mainPage = select_button.click();
}catch(Exception error){
error.printStackTrace();
return ExitSystem("Error Data_STORET_0008");
}
}else{
return ExitSystem("Error Data_STORET_0007");
}
}catch(Exception error){
error.printStackTrace();
return ExitSystem("Error Data_STORET_0006");
}
System.out.println("Step 6a: selected a characteristic");
}
//Input flow characteristic
if(!characteristicShort.equalsIgnoreCase("all")){
//If the search characteristic is not all then search for flow data,
// not performing this operation defaults to all data for a STORET station,
//which is used by the "data download" version of this model
HtmlInput queryInput11 = mainPage.getElementByName("as_char");
queryInput11.setValueAttribute("Flow");
//Search Characteristic for current characteristic (push "Search" button and get pop-up window)
List<?> popupList11 = (List<?>) mainPage.getByXPath("//input[@value='Search']");
try{
//Select first value in characteristic "search by" list box which should be "Characteristic Name"
HtmlForm searchForm11 = (HtmlForm) formList.get(1);
HtmlSelect select11 = (HtmlSelect)searchForm11.getSelectsByName("as_char_alias_type").get(0);
select11.setSelectedAttribute("0", true);
//Push "Search" button and get pop-up window
HtmlButtonInput popup_cha= (HtmlButtonInput) popupList11.get(1);
popup_cha.focus();
HtmlPage popupPage_cha = popup_cha.click();
//Select Characteristic
List<?> formList12 = (List<?>) popupPage_cha.getForms();
HtmlForm searchForm12 = (HtmlForm) formList12.get(0);
HtmlSelect select12 = (HtmlSelect)searchForm12.getSelectsByName("as_char").get(0);
selectListboxData(select12,"Flow Characteristic","Flow");
//Get update main page by pushing "Select" button on pop-up
List<?> list = (List<?>) popupPage_cha.getByXPath("//input[@value='Select']");
if (list.size() == 1){
try{
HtmlButtonInput select_button = (HtmlButtonInput) list.get(0);
select_button.focus();
mainPage = select_button.click();
}catch(Exception error){
error.printStackTrace();
return ExitSystem("Error Data_STORET_0008");
}
}else{
return ExitSystem("Error Data_STORET_0007");
}
}catch(Exception error){
error.printStackTrace();
return ExitSystem("Error Data_STORET_0006");
}
System.out.println("Step 6b: selected flow characteristic");
}
//FileWriter write = new FileWriter(mainFolder + "/page1.txt", false);
//PrintWriter print_line = new PrintWriter(write);
//print_line.printf("%s" + "%n", mainPage.asXml());
//print_line.close();
//Select "Continue" Button
List<?> buttonList2 = (List<?>) mainPage.getByXPath("//input[@value='Continue']");
HtmlPage resultPage = null;
try{
if(buttonList2.size() == 1){
HtmlButtonInput select11 = (HtmlButtonInput) buttonList2.get(0);
select11.focus();
resultPage = select11.click();
}else{
return ExitSystem("Error Data_STORET_0009");
}
}catch(IOException error){
error.printStackTrace();
return ExitSystem("Error Data_STORET_0010");
}
System.out.println("Step 7: submitted request, now entering final inputs");
String xmlSource_result = "";
xmlSource_result = resultPage.asXml();
//Exit if there are no results or too many to download "Immediately"
if(xmlSource_result.contains("No Results match") || xmlSource_result.contains("The number of Results that match your search criteria has exceeded")){
String message = "Error: There is no available data for station '" + stationID + "' and the specified date range";
return message;
}
//Enter email address
// HtmlSelect queryInput12 = resultPage.getElementByName("v_userprofile");
// queryInput12.setSelectedAttribute("8", true);//Select value=8 which is the public user
HtmlInput queryInput13 = resultPage.getElementByName("v_email");
queryInput13.setValueAttribute(email_address);
HtmlInput queryInput14 = resultPage.getElementByName("v_prefix");
queryInput14.setValueAttribute("tmd");
//write = new FileWriter(mainFolder + "/page2.txt", false);
//print_line = new PrintWriter(write);
//print_line.printf("%s" + "%n", resultPage.asXml());
//print_line.close();
//Select "Immediate" Batch Processing button
List<?> buttonList3 = (List<?>) resultPage.getByXPath("//input[@value='Immediate']");
HtmlPage BatchPage = null;
try{
if(buttonList3.size() == 3){
HtmlButtonInput select14 = (HtmlButtonInput) buttonList3.get(1);
select14.focus();
BatchPage = select14.click();
}else{
return ExitSystem("Error v0011");
}
}catch(IOException error){
error.printStackTrace();
return ExitSystem("Error Data_STORET_0012");
}
System.out.println("Step 8: submitted data extraction, waiting on webpage to return");
//write = new FileWriter(mainFolder + "/page3.txt", false);
//print_line = new PrintWriter(write);
//print_line.printf("%s" + "%n", BatchPage.asXml());
//print_line.close();
//Find url to download webpage
String xmlSource_batch = BatchPage.asXml();
int urlInt_begin = xmlSource_batch.indexOf("http://www.epa.gov/storpubl/modern/downloads/tmd");
int urlInt_end = xmlSource_batch.indexOf("zip", urlInt_begin);
if(urlInt_begin == -1 || urlInt_end == -1){
return ExitSystem("Error Data_STORET_0013");
}
String downloadUrl = "";
downloadUrl = xmlSource_batch.substring(urlInt_begin,urlInt_end + 3);
String zipfile = downloadUrl.substring(downloadUrl.indexOf("tmd"));
//Download STORET data
System.out.println("Step 9a: Downloading Zip file");
String zipfile_location = downloadzip(downloadUrl, zipfile, mainFolder);
//Close web interaction
webClient.closeAllWindows();
//Check if there is an error downloading the file
if(zipfile_location.contains("Error")){
String[] errorMessage = {"There is a problem extracting the data from STORET for station '" + stationID + "' by: " +
organizationName, zipfile_location};
writeError(errorMessage);
}
//Return filename/location
System.out.println("Step 10: request complete");
return zipfile_location;
}
/**
* Unzips the downloaded zip-file
* @param zip_location The location on the drive where the downloaded STORET zip file was saved
*/
public void UnZip_file(String zip_location){
try {
File mainDir = new File(zip_location.substring(0,zip_location.indexOf("tmd")));
ZipFile zipFile = new ZipFile(zip_location);
Enumeration<?> enumeration = zipFile.entries();
File new_file = null;
FileOutputStream fout_strm = null;
//While there are multiple files inside the zip file, keep unzipping until all have been unzipped.
while (enumeration.hasMoreElements()) {
ZipEntry zipEntry = (ZipEntry) enumeration.nextElement();
InputStream in_strm = zipFile.getInputStream(zipEntry);
byte[] buffer_bytes = new byte[1024];
int bytesRead = 0;
new_file = new File(mainDir.getAbsolutePath() + File.separator + zipEntry.getName());
if(zipEntry.isDirectory()){
new_file.mkdirs();
continue;
}else{
new_file.getParentFile().mkdirs();
new_file.createNewFile();
}
fout_strm = new FileOutputStream(new_file);
while((bytesRead = in_strm.read(buffer_bytes)) != -1){
fout_strm.write(buffer_bytes, 0, bytesRead);
}
if(fout_strm != null){
fout_strm.close();
}
in_strm.close();
}
zipFile.close();
}catch(IOException e){
e.printStackTrace();
}
}
/**
* Opens file and pulls out water quality data for LDC
* @param partialPath the folder path to the file (ex. "C:/temp/myFolder/")
* @param fileName the name of the file (ex "myFile.txt")
* @param Characteristic The characteristic currently searched for in the text file, either flow or the user-defined characteristic
* @return a String[][] containing column1 = date(yyyy-mm-dd), column2 = flowValue
*/
public String[][] OpenWQFile(String partialPath, String fileName, String Characteristic) throws IOException {
//Open file
FileReader fr = new FileReader(partialPath + fileName);
BufferedReader textReader = new BufferedReader(fr);
ArrayList<String> textData = new ArrayList<String>();
String currentLine = "";
int ctr = 0;
double temp = 0;
while((currentLine = textReader.readLine()) !=null){
String f[] = currentLine.split("\t");
//Depending on current characteristic
if((f.length >= 25) && (ctr != 0)){
//Ignore effluent data
if(f[1].equalsIgnoreCase("EFFLUENT")){
continue;
}
//Ignore Non-detect data
if(f[23].equalsIgnoreCase("*Non-detect")){
continue;
}
if(f[24].indexOf(" ") != -1){
f[24]= f[24].substring(0,f[24].indexOf(" "));
}
if(Characteristic.equals("Flow")){
//Correct units of flow values
if(f[24].equalsIgnoreCase("cm3/sec")){//convert from cm3/sec to cfs
temp = Double.parseDouble(f[23])*(1/0.0283168466);
f[24] = "cfs";
f[23] = String.valueOf(temp);
}
if(f[18].equalsIgnoreCase("Flow") && f[24].equalsIgnoreCase("cfs")){
//Pull out only the data needed to pass between sub-functions
//f[1] = Station name
//f[11] = date
//f[23] = flow value
textData.add(f[1] +"\t"+ f[11] + "\t" + f[23]);
}
}else if (Characteristic.equals("Escherichia coli") || Characteristic.equals("Fecal Coliform") ||
Characteristic.equals("Total Coliform")){
if(f[18].equalsIgnoreCase(Characteristic) && f[24].equalsIgnoreCase("#/100ml")){
//Pull out only the data needed to pass between sub-functions
//f[1] = Station name
//f[11] = date
//f[23] = WQ result value (in #/100ml)
textData.add(f[1] +"\t"+ f[11] + "\t" + f[23]);
}
}else{
//Correct units of WQ tests
if(f[24].equalsIgnoreCase("ug/l")){
temp = Double.parseDouble(f[23])*(1/1000);
f[23] = String.valueOf(temp);
f[24] = "mg/l";
}else if(f[24].equalsIgnoreCase("g/l")){
temp = Double.parseDouble(f[23])*(1000/1);
f[23] = String.valueOf(temp);
f[24] = "mg/l";
}
if(f[18].equalsIgnoreCase(Characteristic) && f[24].equalsIgnoreCase("mg/l")){
//Pull out only the data needed to pass between sub-functions
//f[1] = Station name
//f[11] = date
//f[23] = WQ result value (in mg/L)
textData.add(f[1] +"\t"+ f[11] + "\t" + f[23]);
}
}
}
ctr++;
}
textReader.close();
fr.close();
//convert Array list into String[][] array (column1 = date, column2 = value)
String[][] stringArray = new String[textData.size()][2];
for(int i=0; i<stringArray.length; i++){
String[] currentColumns = textData.get(i).split("\t");
//currentLine[0] = stationID
//currentLine[1] = date
//currentLine[2] = value
//If STORET date includes time (yyyy-mm-dd hh:mm:ss) substring out only the date
if(currentColumns[1].length() > 10){
currentColumns[1] = currentColumns[1].substring(0,10);
}
stringArray[i][0] = currentColumns[1];
stringArray[i][1] = currentColumns[2];
}
return stringArray;
}
/**
* Opens the file and returns everything
* @param partialPath the folder path to the file (ex. "C:/temp/myFolder/")
* @param fileName the name of the file (ex "myFile.txt")
* @return an ArrayList<String> containing all of the rows in the file
* @throws IOException
*/
public ArrayList<String> OpenAllFile(String partialPath, String fileName) throws IOException {
//Open file
FileReader fr = new FileReader(partialPath + fileName);
BufferedReader textReader = new BufferedReader(fr);
ArrayList<String> textData = new ArrayList<String>();
String currentLine = "";
while((currentLine = textReader.readLine()) !=null){
textData.add(currentLine);
}
textReader.close();
fr.close();
return textData;
}
/**
* Unzips the downloaded STORET zip file and extracts the contents into an ArrayList<String> for download purposes
* @param zipLocation The location on the drive where the extracted text file from the downloaded STORET zip file was saved
* @return an ArrayList<String> containing the downloaded datas
* @throws IOException
*/
public ArrayList<String> Unzip_STORETDownloadFilesAll(String zipLocation) throws IOException{
//Create file names of other unzipped files
String partialPath = zipLocation.substring(0,zipLocation.indexOf("tmd"));
String data_location = zipLocation.substring(zipLocation.indexOf("tmd"),zipLocation.indexOf(".zip"));
String data_location2 = "Data_" + data_location + "_Metadata.txt";
data_location = "Data_" + data_location + "_RegResults.txt";
System.out.println("Unzipping STORET result files");
UnZip_file(zipLocation);
System.out.println("Completed unzipping STORET result files");
ArrayList<String> results = new ArrayList<String>();
System.out.println("Reading STORET result file for WQ characteristic");
//Extract "reg_results" file and extract data WQ from this combined file
results = OpenAllFile(partialPath, data_location);
//Delete text files and zip file
System.out.println("deleting excess files");
File zip = new File(zipLocation);
File text1 = new File(partialPath + data_location);
File text2 = new File(partialPath + data_location2);
boolean zipTF = zip.delete();
boolean text1TF = text1.delete();
boolean text2TF = text2.delete();
if(!zipTF || !text1TF || !text2TF){
System.out.println("One of the downloaded water quality files was not deleted properly");
}
return results;
}
/**
* Unzips the downloaded STORET zip file and extracts the contents into the
* extpected list for LDC/FDC based on the provided characteristic
* @param zipLocation The location on the drive where the extracted text
* file from the downloaded STORET zip file was saved
* @param characteristic The characteristic currently searched for in the
* text file, either flow or the user-defined characteristic
* @param deleteTrue a boolean that if true deletes the downloaded files,
* otherwise keeps them (this is to delete the files for the CFA models
* except LDC which has 1 download for flow and WQ and needs to keep the
* files until the second dataset is read out)
* @return a String[][] containing column1 = date(yyyy-mm-dd), column2 = value
* @throws IOException
*/
public String[][] Unzip_STORETDownloadFiles(String zipLocation, String characteristic, boolean deleteTrue) throws IOException {
//Create file names of other unzipped files
String partialPath = zipLocation.substring(0,zipLocation.indexOf("tmd"));
String data_location = zipLocation.substring(zipLocation.indexOf("tmd"),zipLocation.indexOf(".zip"));
String data_location2 = "Data_" + data_location + "_Metadata.txt";
String data_location3 = "Data_" + data_location + "_ActivityGroups.txt";
data_location = "Data_" + data_location + "_RegResults.txt";
System.out.println("Unzipping STORET result files");
UnZip_file(zipLocation);
System.out.println("Completed unzipping STORET result files");
String[][] results = new String[0][2];
if(!characteristic.equalsIgnoreCase("Flow")){
//If the characteristic is not flow, then unzip the file but do not delete it
System.out.println("Reading STORET result file for WQ characteristic");
//Extract "reg_results" file and extract data WQ from this combined file
results = OpenWQFile(partialPath, data_location, characteristic);
}else{
//If the characteristic is flow, then unzip the file and delete it once finished
System.out.println("Reading STORET result file for Flow characteristic");
//Extract "reg_results" file and extract flow data from this combined file
results = OpenWQFile(partialPath, data_location, "Flow");
}
if(deleteTrue){
//Delete text files and zip file
System.out.println("deleting excess files");
File zip = new File(zipLocation);
File text1 = new File(partialPath + data_location);
File text2 = new File(partialPath + data_location2);
File text3 = new File(partialPath + data_location3);
boolean zipTF = zip.delete();
boolean text1TF = text1.delete();
boolean text2TF = text2.delete();
boolean text3TF = text3.delete();
if(!zipTF || !text1TF || !text2TF || !text3TF){
System.out.println("One of the downloaded water quality files was not deleted properly");
}
}
return results;
}
}